1#!/usr/bin/env bash 2 3function xtrace_disable() { 4 if [ "$XTRACE_DISABLED" != "yes" ]; then 5 PREV_BASH_OPTS="$-" 6 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 7 XTRACE_DISABLED="yes" 8 fi 9 set +x 10 elif [ -z $XTRACE_NESTING_LEVEL ]; then 11 XTRACE_NESTING_LEVEL=1 12 else 13 XTRACE_NESTING_LEVEL=$((++XTRACE_NESTING_LEVEL)) 14 fi 15} 16 17xtrace_disable 18set -e 19shopt -s expand_aliases 20 21source "$rootdir/test/common/applications.sh" 22source "$rootdir/scripts/common.sh" 23if [[ -e $rootdir/test/common/build_config.sh ]]; then 24 source "$rootdir/test/common/build_config.sh" 25elif [[ -e $rootdir/mk/config.mk ]]; then 26 build_config=$(< "$rootdir/mk/config.mk") 27 source <(echo "${build_config//\?=/=}") 28else 29 source "$rootdir/CONFIG" 30fi 31 32# Dummy function to be called after restoring xtrace just so that it appears in the 33# xtrace log. This way we can consistently track when xtrace is enabled/disabled. 34function xtrace_enable() { 35 # We have to do something inside a function in bash, and calling any command 36 # (even `:`) will produce an xtrace entry, so we just define another function. 37 function xtrace_dummy() { :; } 38} 39 40# Keep it as alias to avoid xtrace_enable backtrace always pointing to xtrace_restore. 41# xtrace_enable will appear as called directly from the user script, from the same line 42# that "called" xtrace_restore. 43alias xtrace_restore='if [ -z $XTRACE_NESTING_LEVEL ]; then 44 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 45 XTRACE_DISABLED="no"; PREV_BASH_OPTS=""; set -x; xtrace_enable; 46 fi 47else 48 XTRACE_NESTING_LEVEL=$((--XTRACE_NESTING_LEVEL)); 49 if [ $XTRACE_NESTING_LEVEL -eq "0" ]; then 50 unset XTRACE_NESTING_LEVEL 51 fi 52fi' 53 54: ${RUN_NIGHTLY:=0} 55export RUN_NIGHTLY 56 57# Set defaults for missing test config options 58: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 59export SPDK_AUTOTEST_DEBUG_APPS 60: ${SPDK_RUN_VALGRIND=0} 61export SPDK_RUN_VALGRIND 62: ${SPDK_RUN_FUNCTIONAL_TEST=0} 63export SPDK_RUN_FUNCTIONAL_TEST 64: ${SPDK_TEST_UNITTEST=0} 65export SPDK_TEST_UNITTEST 66: ${SPDK_TEST_AUTOBUILD=0} 67export SPDK_TEST_AUTOBUILD 68: ${SPDK_TEST_RELEASE_BUILD=0} 69export SPDK_TEST_RELEASE_BUILD 70: ${SPDK_TEST_ISAL=0} 71export SPDK_TEST_ISAL 72: ${SPDK_TEST_ISCSI=0} 73export SPDK_TEST_ISCSI 74: ${SPDK_TEST_ISCSI_INITIATOR=0} 75export SPDK_TEST_ISCSI_INITIATOR 76: ${SPDK_TEST_NVME=0} 77export SPDK_TEST_NVME 78: ${SPDK_TEST_NVME_CLI=0} 79export SPDK_TEST_NVME_CLI 80: ${SPDK_TEST_NVME_CUSE=0} 81export SPDK_TEST_NVME_CUSE 82: ${SPDK_TEST_NVMF=0} 83export SPDK_TEST_NVMF 84: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 85export SPDK_TEST_NVMF_TRANSPORT 86: ${SPDK_TEST_RBD=0} 87export SPDK_TEST_RBD 88: ${SPDK_TEST_VHOST=0} 89export SPDK_TEST_VHOST 90: ${SPDK_TEST_BLOCKDEV=0} 91export SPDK_TEST_BLOCKDEV 92: ${SPDK_TEST_IOAT=0} 93export SPDK_TEST_IOAT 94: ${SPDK_TEST_BLOBFS=0} 95export SPDK_TEST_BLOBFS 96: ${SPDK_TEST_VHOST_INIT=0} 97export SPDK_TEST_VHOST_INIT 98: ${SPDK_TEST_PMDK=0} 99export SPDK_TEST_PMDK 100: ${SPDK_TEST_LVOL=0} 101export SPDK_TEST_LVOL 102: ${SPDK_TEST_JSON=0} 103export SPDK_TEST_JSON 104: ${SPDK_TEST_REDUCE=0} 105export SPDK_TEST_REDUCE 106: ${SPDK_RUN_ASAN=0} 107export SPDK_RUN_ASAN 108: ${SPDK_RUN_UBSAN=0} 109export SPDK_RUN_UBSAN 110: ${SPDK_RUN_EXTERNAL_DPDK=""} 111export SPDK_RUN_EXTERNAL_DPDK 112: ${SPDK_RUN_NON_ROOT=0} 113export SPDK_RUN_NON_ROOT 114: ${SPDK_TEST_CRYPTO=0} 115export SPDK_TEST_CRYPTO 116: ${SPDK_TEST_FTL=0} 117export SPDK_TEST_FTL 118: ${SPDK_TEST_OCF=0} 119export SPDK_TEST_OCF 120: ${SPDK_TEST_FTL_EXTENDED=0} 121export SPDK_TEST_FTL_EXTENDED 122: ${SPDK_TEST_VMD=0} 123export SPDK_TEST_VMD 124: ${SPDK_TEST_OPAL=0} 125export SPDK_TEST_OPAL 126: ${SPDK_TEST_NATIVE_DPDK} 127export SPDK_TEST_NATIVE_DPDK 128: ${SPDK_AUTOTEST_X=true} 129export SPDK_AUTOTEST_X 130: ${SPDK_TEST_RAID5=0} 131export SPDK_TEST_RAID5 132: ${SPDK_TEST_URING=0} 133export SPDK_TEST_URING 134: ${SPDK_TEST_USE_IGB_UIO:=0} 135export SPDK_TEST_USE_IGB_UIO 136 137export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib" 138export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR 139 140# Tell setup.sh to wait for block devices upon each reset 141export PCI_BLOCK_SYNC_ON_RESET=yes 142 143# Export PYTHONPATH with addition of RPC framework. New scripts can be created 144# specific use cases for tests. 145export PYTHONPATH=$PYTHONPATH:$rootdir/scripts 146 147# Don't create Python .pyc files. When running with sudo these will be 148# created with root ownership and can cause problems when cleaning the repository. 149export PYTHONDONTWRITEBYTECODE=1 150 151# Export flag to skip the known bug that exists in librados 152# Bug is reported on ceph bug tracker with number 24078 153export ASAN_OPTIONS=new_delete_type_mismatch=0 154export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1' 155 156# Export LeakSanitizer option to use suppression file in order to prevent false positives 157# and known leaks in external executables or libraries from showing up. 158asan_suppression_file="/var/tmp/asan_suppression_file" 159sudo rm -rf "$asan_suppression_file" 160cat << EOL >> "$asan_suppression_file" 161# ASAN has some bugs around thread_local variables. We have a destructor in place 162# to free the thread contexts, but ASAN complains about the leak before those 163# destructors have a chance to run. So suppress this one specific leak using 164# LSAN_OPTIONS. 165leak:spdk_fs_alloc_thread_ctx 166 167# Suppress known leaks in fio project 168leak:$CONFIG_FIO_SOURCE_DIR/parse.c 169leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 170leak:$CONFIG_FIO_SOURCE_DIR/init.c 171leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 172leak:fio_memalign 173leak:spdk_fio_io_u_init 174 175# Suppress leaks in libiscsi 176leak:libiscsi.so 177EOL 178 179# Suppress leaks in libfuse3 180echo "leak:libfuse3.so" >> "$asan_suppression_file" 181 182export LSAN_OPTIONS=suppressions="$asan_suppression_file" 183 184export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 185 186if [ -z "$DEPENDENCY_DIR" ]; then 187 export DEPENDENCY_DIR=/home/sys_sgsw 188else 189 export DEPENDENCY_DIR 190fi 191 192# Export location of where all the SPDK binaries are 193export SPDK_BIN_DIR="$rootdir/build/bin" 194export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 195 196# pass our valgrind desire on to unittest.sh 197if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 198 export valgrind='' 199fi 200 201if [ "$(uname -s)" = "Linux" ]; then 202 MAKE="make" 203 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 204 # Override the default HUGEMEM in scripts/setup.sh to allocate 8GB in hugepages. 205 export HUGEMEM=8192 206 if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then 207 export DRIVER_OVERRIDE=$rootdir/dpdk/build-tmp/kernel/linux/igb_uio/igb_uio.ko 208 # Building kernel modules requires root privileges 209 MAKE="sudo $MAKE" 210 fi 211elif [ "$(uname -s)" = "FreeBSD" ]; then 212 MAKE="gmake" 213 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 214 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 215 export HUGEMEM=2048 216else 217 echo "Unknown OS \"$(uname -s)\"" 218 exit 1 219fi 220 221if [ -z "$output_dir" ]; then 222 mkdir -p "$rootdir/../output" 223 export output_dir="$rootdir/../output" 224fi 225 226TEST_MODE= 227for i in "$@"; do 228 case "$i" in 229 --iso) 230 TEST_MODE=iso 231 ;; 232 --transport=*) 233 TEST_TRANSPORT="${i#*=}" 234 ;; 235 --sock=*) 236 TEST_SOCK="${i#*=}" 237 ;; 238 esac 239done 240 241# start rpc.py coprocess if it's not started yet 242if [[ -z $RPC_PIPE_PID ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 243 coproc RPC_PIPE { "$rootdir/scripts/rpc.py" --server; } 244 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 245 # all descriptors will automatically close together with this bash 246 # process, this will make rpc.py stop reading and exit gracefully 247fi 248 249function set_test_storage() { 250 [[ -v testdir ]] || return 0 251 252 local requested_size=$1 # bytes 253 local mount target_dir 254 255 local -A mounts fss sizes avails uses 256 local source fs size avail mount use 257 258 local storage_fallback storage_candidates 259 local storage_fallback_purge 260 261 shopt -s nullglob 262 storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 263 shopt -u nullglob 264 265 if ((${#storage_fallback_purge[@]} > 0)); then 266 printf '* Purging old temporary test storage (%s)\n' \ 267 "${storage_fallback_purge[*]}" >&2 268 sudo rm -rf "${storage_fallback_purge[@]}" 269 fi 270 271 storage_fallback=$(mktemp -udt spdk.XXXXXX) 272 storage_candidates=( 273 "$testdir" 274 "$storage_fallback/tests/${testdir##*/}" 275 "$storage_fallback" 276 ) 277 278 if [[ -n $ADD_TEST_STORAGE ]]; then 279 # List of dirs|mounts separated by whitespaces 280 storage_candidates+=($ADD_TEST_STORAGE) 281 fi 282 283 if [[ -n $DEDICATED_TEST_STORAGE ]]; then 284 # Single, dedicated dir|mount 285 storage_candidates=("$DEDICATED_TEST_STORAGE") 286 fi 287 288 mkdir -p "${storage_candidates[@]}" 289 290 # add some headroom - 64M 291 requested_size=$((requested_size + (64 << 20))) 292 293 while read -r source fs size use avail _ mount; do 294 mounts["$mount"]=$source fss["$mount"]=$fs 295 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 296 uses["$mount"]=$((use * 1024)) 297 done < <(df -T | grep -v Filesystem) 298 299 printf '* Looking for test storage...\n' >&2 300 301 local target_space new_size 302 for target_dir in "${storage_candidates[@]}"; do 303 # FreeBSD's df is lacking the --output arg 304 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 305 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 306 307 target_space=${avails["$mount"]} 308 if ((target_space == 0 || target_space < requested_size)); then 309 continue 310 fi 311 if ((target_space >= requested_size)); then 312 # For in-memory fs, and / make sure our requested size won't fill most of the space. 313 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 314 new_size=$((uses["$mount"] + requested_size)) 315 if ((new_size * 100 / sizes["$mount"] > 95)); then 316 continue 317 fi 318 fi 319 fi 320 export SPDK_TEST_STORAGE=$target_dir 321 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 322 return 0 323 done 324 printf '* Test storage is not available\n' 325 return 1 326} 327 328function get_config_params() { 329 xtrace_disable 330 config_params='--enable-debug --enable-werror' 331 332 # for options with dependencies but no test flag, set them here 333 if [ -f /usr/include/infiniband/verbs.h ]; then 334 config_params+=' --with-rdma' 335 fi 336 337 if [ $(uname -s) == "FreeBSD" ]; then 338 intel="hw.model: Intel" 339 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 340 else 341 intel="GenuineIntel" 342 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 343 fi 344 if [[ "$cpu_vendor" != *"$intel"* ]]; then 345 config_params+=" --without-idxd" 346 else 347 config_params+=" --with-idxd" 348 fi 349 350 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 351 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 352 fi 353 354 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 355 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 356 fi 357 358 if [ -d /usr/include/iscsi ]; then 359 libiscsi_version=$(grep LIBISCSI_API_VERSION /usr/include/iscsi/iscsi.h | head -1 | awk '{print $3}' | awk -F '(' '{print $2}' | awk -F ')' '{print $1}') 360 if [ $libiscsi_version -ge 20150621 ]; then 361 config_params+=' --with-iscsi-initiator' 362 fi 363 fi 364 365 if [ $SPDK_TEST_UNITTEST -eq 0 ]; then 366 config_params+=' --disable-unit-tests' 367 fi 368 369 if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then 370 config_params+=' --with-nvme-cuse' 371 fi 372 373 # for options with both dependencies and a test flag, set them here 374 if [ -f /usr/include/libpmemblk.h ] && [ $SPDK_TEST_PMDK -eq 1 ]; then 375 config_params+=' --with-pmdk' 376 fi 377 378 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_REDUCE -eq 1 ]; then 379 if ge "$(nasm --version | awk '{print $3}')" 2.14 && [[ $SPDK_TEST_ISAL -eq 1 ]]; then 380 config_params+=' --with-reduce' 381 fi 382 fi 383 384 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 385 config_params+=' --with-rbd' 386 fi 387 388 # for options with no required dependencies, just test flags, set them here 389 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 390 config_params+=' --with-crypto' 391 fi 392 393 if [ $SPDK_TEST_OCF -eq 1 ]; then 394 config_params+=" --with-ocf" 395 fi 396 397 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 398 config_params+=' --enable-ubsan' 399 fi 400 401 if [ $SPDK_RUN_ASAN -eq 1 ]; then 402 config_params+=' --enable-asan' 403 fi 404 405 if [ "$(uname -s)" = "Linux" ]; then 406 config_params+=' --enable-coverage' 407 fi 408 409 if [ $SPDK_TEST_ISAL -eq 0 ]; then 410 config_params+=' --without-isal' 411 fi 412 413 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 414 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 415 config_params+=' --with-fuse' 416 fi 417 fi 418 419 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 420 config_params+=' --with-raid5' 421 fi 422 423 # Check whether liburing library header exists 424 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 425 config_params+=' --with-uring' 426 fi 427 428 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 429 config_params+=" --with-dpdk=$SPDK_RUN_EXTERNAL_DPDK" 430 fi 431 432 if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then 433 config_params+=" --with-igb-uio-driver" 434 fi 435 echo "$config_params" 436 xtrace_restore 437} 438 439function rpc_cmd() { 440 xtrace_disable 441 local rsp rc 442 443 echo "$@" >&$RPC_PIPE_INPUT 444 while read -t 5 -ru $RPC_PIPE_OUTPUT rsp; do 445 if [[ $rsp == "**STATUS="* ]]; then 446 break 447 fi 448 echo "$rsp" 449 done 450 451 rc=${rsp#*=} 452 xtrace_restore 453 [[ $rc == 0 ]] 454} 455 456function rpc_cmd_simple_data_json() { 457 458 local elems="$1[@]" elem 459 local -gA jq_out=() 460 local jq val 461 462 local lvs=( 463 "uuid" 464 "name" 465 "base_bdev" 466 "total_data_clusters" 467 "free_clusters" 468 "block_size" 469 "cluster_size" 470 ) 471 472 local bdev=( 473 "name" 474 "aliases[0]" 475 "block_size" 476 "num_blocks" 477 "uuid" 478 "product_name" 479 ) 480 481 [[ -v $elems ]] || return 1 482 483 for elem in "${!elems}"; do 484 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 485 done 486 jq+=',"\n"' 487 488 shift 489 while read -r elem val; do 490 jq_out["$elem"]=$val 491 done < <(rpc_cmd "$@" | jq -jr "$jq") 492 ((${#jq_out[@]} > 0)) || return 1 493} 494 495function NOT() { 496 local es=0 497 498 "$@" || es=$? 499 500 # Logic looks like so: 501 # - return false if command exit successfully 502 # - return false if command exit after receiving a core signal (FIXME: or any signal?) 503 # - return true if command exit with an error 504 505 # This naively assumes that the process doesn't exit with > 128 on its own. 506 if ((es > 128)); then 507 es=$((es & ~128)) 508 case "$es" in 509 3) ;& # SIGQUIT 510 4) ;& # SIGILL 511 6) ;& # SIGABRT 512 8) ;& # SIGFPE 513 9) ;& # SIGKILL 514 11) es=0 ;; # SIGSEGV 515 *) es=1 ;; 516 esac 517 elif [[ -n $EXIT_STATUS ]] && ((es != EXIT_STATUS)); then 518 es=0 519 fi 520 521 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 522 ((!es == 0)) 523} 524 525function timing() { 526 direction="$1" 527 testname="$2" 528 529 now=$(date +%s) 530 531 if [ "$direction" = "enter" ]; then 532 export timing_stack="${timing_stack};${now}" 533 export test_stack="${test_stack};${testname}" 534 else 535 touch "$output_dir/timing.txt" 536 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 537 538 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 539 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 540 541 elapsed=$((now - start_time - child_time)) 542 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 543 544 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 545 fi 546} 547 548function timing_enter() { 549 xtrace_disable 550 timing "enter" "$1" 551 xtrace_restore 552} 553 554function timing_exit() { 555 xtrace_disable 556 timing "exit" "$1" 557 xtrace_restore 558} 559 560function timing_finish() { 561 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 562 if [ -x "$flamegraph" ]; then 563 "$flamegraph" \ 564 --title 'Build Timing' \ 565 --nametype 'Step:' \ 566 --countname seconds \ 567 $output_dir/timing.txt \ 568 > $output_dir/timing.svg 569 fi 570} 571 572function create_test_list() { 573 xtrace_disable 574 # First search all scripts in main SPDK directory. 575 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 576 # Follow up with search in test directory recursively. 577 completion+=$(grep -rshI --include="*.sh" --exclude="autotest_common.sh" -e "run_test " $rootdir/test) 578 printf "%s" "$completion" | grep -v "#" \ 579 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 580 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 581 xtrace_restore 582} 583 584function gdb_attach() { 585 gdb -q --batch \ 586 -ex 'handle SIGHUP nostop pass' \ 587 -ex 'handle SIGQUIT nostop pass' \ 588 -ex 'handle SIGPIPE nostop pass' \ 589 -ex 'handle SIGALRM nostop pass' \ 590 -ex 'handle SIGTERM nostop pass' \ 591 -ex 'handle SIGUSR1 nostop pass' \ 592 -ex 'handle SIGUSR2 nostop pass' \ 593 -ex 'handle SIGCHLD nostop pass' \ 594 -ex 'set print thread-events off' \ 595 -ex 'cont' \ 596 -ex 'thread apply all bt' \ 597 -ex 'quit' \ 598 --tty=/dev/stdout \ 599 -p $1 600} 601 602function process_core() { 603 # Note that this always was racy as we can't really sync with the kernel 604 # to see if there's any core queued up for writing. We could check if 605 # collector is running and wait for it explicitly, but it doesn't seem 606 # to be worth the effort. So assume that if we are being called via 607 # trap, as in, when some error has occurred, wait up to 5s for any 608 # potential cores. If we are called just for cleanup at the very end, 609 # don't wait since all the tests ended successfully, hence having any 610 # critical cores lying around is unlikely. 611 local es=$? 612 ((es != 0)) && sleep 5s 613 614 local coredumps core 615 616 shopt -s nullglob 617 coredumps=("$output_dir/coredumps/"*.bt.txt) 618 shopt -u nullglob 619 620 ((${#coredumps[@]} > 0)) || return 0 621 chmod -R a+r "$output_dir/coredumps" 622 623 for core in "${coredumps[@]}"; do 624 cat <<- BT 625 ##### CORE BT ${core##*/} ##### 626 627 $(<"$core") 628 629 -- 630 BT 631 done 632 return 1 633} 634 635function process_shm() { 636 type=$1 637 id=$2 638 if [ "$type" = "--pid" ]; then 639 id="pid${id}" 640 elif [ "$type" = "--id" ]; then 641 id="${id}" 642 else 643 echo "Please specify to search for pid or shared memory id." 644 return 1 645 fi 646 647 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 648 649 if [[ -z $shm_files ]]; then 650 echo "SHM File for specified PID or shared memory id: ${id} not found!" 651 return 1 652 fi 653 for n in $shm_files; do 654 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 655 done 656 return 0 657} 658 659function waitforlisten() { 660 # $1 = process pid 661 if [ -z "$1" ]; then 662 exit 1 663 fi 664 665 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 666 667 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 668 # turn off trace for this loop 669 xtrace_disable 670 local ret=0 671 local i 672 for ((i = 40; i != 0; i--)); do 673 # if the process is no longer running, then exit the script 674 # since it means the application crashed 675 if ! kill -s 0 $1; then 676 echo "ERROR: process (pid: $1) is no longer running" 677 ret=1 678 break 679 fi 680 681 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 682 break 683 fi 684 685 sleep 0.5 686 done 687 688 xtrace_restore 689 if ((i == 0)); then 690 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 691 ret=1 692 fi 693 return $ret 694} 695 696function waitfornbd() { 697 local nbd_name=$1 698 local i 699 700 for ((i = 1; i <= 20; i++)); do 701 if grep -q -w $nbd_name /proc/partitions; then 702 break 703 else 704 sleep 0.1 705 fi 706 done 707 708 # The nbd device is now recognized as a block device, but there can be 709 # a small delay before we can start I/O to that block device. So loop 710 # here trying to read the first block of the nbd block device to a temp 711 # file. Note that dd returns success when reading an empty file, so we 712 # need to check the size of the output file instead. 713 for ((i = 1; i <= 20; i++)); do 714 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 715 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 716 rm -f "$SPDK_TEST_STORAGE/nbdtest" 717 if [ "$size" != "0" ]; then 718 return 0 719 else 720 sleep 0.1 721 fi 722 done 723 724 return 1 725} 726 727function waitforbdev() { 728 local bdev_name=$1 729 local i 730 731 for ((i = 1; i <= 20; i++)); do 732 if $rpc_py bdev_get_bdevs | jq -r '.[] .name' | grep -qw $bdev_name; then 733 return 0 734 fi 735 736 if $rpc_py bdev_get_bdevs | jq -r '.[] .aliases' | grep -qw $bdev_name; then 737 return 0 738 fi 739 740 sleep 0.1 741 done 742 743 return 1 744} 745 746function make_filesystem() { 747 local fstype=$1 748 local dev_name=$2 749 local i=0 750 local force 751 752 if [ $fstype = ext4 ]; then 753 force=-F 754 else 755 force=-f 756 fi 757 758 while ! mkfs.${fstype} $force ${dev_name}; do 759 if [ $i -ge 15 ]; then 760 return 1 761 fi 762 i=$((i + 1)) 763 sleep 1 764 done 765 766 return 0 767} 768 769function killprocess() { 770 # $1 = process pid 771 if [ -z "$1" ]; then 772 exit 1 773 fi 774 775 if kill -0 $1; then 776 if [ $(uname) = Linux ]; then 777 process_name=$(ps --no-headers -o comm= $1) 778 else 779 process_name=$(ps -c -o command $1 | tail -1) 780 fi 781 if [ "$process_name" = "sudo" ]; then 782 # kill the child process, which is the actual app 783 # (assume $1 has just one child) 784 local child 785 child="$(pgrep -P $1)" 786 echo "killing process with pid $child" 787 kill $child 788 else 789 echo "killing process with pid $1" 790 kill $1 791 fi 792 793 # wait for the process regardless if its the dummy sudo one 794 # or the actual app - it should terminate anyway 795 wait $1 796 else 797 # the process is not there anymore 798 echo "Process with pid $1 is not found" 799 exit 1 800 fi 801} 802 803function iscsicleanup() { 804 echo "Cleaning up iSCSI connection" 805 iscsiadm -m node --logout || true 806 iscsiadm -m node -o delete || true 807 rm -rf /var/lib/iscsi/nodes/* 808} 809 810function stop_iscsi_service() { 811 if cat /etc/*-release | grep Ubuntu; then 812 service open-iscsi stop 813 else 814 service iscsid stop 815 fi 816} 817 818function start_iscsi_service() { 819 if cat /etc/*-release | grep Ubuntu; then 820 service open-iscsi start 821 else 822 service iscsid start 823 fi 824} 825 826function rbd_setup() { 827 # $1 = monitor ip address 828 # $2 = name of the namespace 829 if [ -z "$1" ]; then 830 echo "No monitor IP address provided for ceph" 831 exit 1 832 fi 833 if [ -n "$2" ]; then 834 if ip netns list | grep "$2"; then 835 NS_CMD="ip netns exec $2" 836 else 837 echo "No namespace $2 exists" 838 exit 1 839 fi 840 fi 841 842 if hash ceph; then 843 export PG_NUM=128 844 export RBD_POOL=rbd 845 export RBD_NAME=foo 846 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 847 $NS_CMD $rootdir/scripts/ceph/start.sh $1 848 849 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 850 $NS_CMD rbd create $RBD_NAME --size 1000 851 fi 852} 853 854function rbd_cleanup() { 855 if hash ceph; then 856 $rootdir/scripts/ceph/stop.sh || true 857 rm -f /var/tmp/ceph_raw.img 858 fi 859} 860 861function _start_stub() { 862 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 863 # but ASLR can still be unreliable in some cases. 864 # We will reenable it again after multi-process testing is complete in kill_stub(). 865 # Save current setting so it can be restored upon calling kill_stub(). 866 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 867 echo 0 > /proc/sys/kernel/randomize_va_space 868 $rootdir/test/app/stub/stub $1 & 869 stubpid=$! 870 echo Waiting for stub to ready for secondary processes... 871 while ! [ -e /var/run/spdk_stub0 ]; do 872 # If stub dies while we wait, bail 873 [[ -e /proc/$stubpid ]] || return 1 874 sleep 1s 875 done 876 echo done. 877} 878 879function start_stub() { 880 if ! _start_stub "$@"; then 881 echo "stub failed" >&2 882 return 1 883 fi 884} 885 886function kill_stub() { 887 if [[ -e /proc/$stubpid ]]; then 888 kill $1 $stubpid 889 wait $stubpid 890 fi 2> /dev/null || : 891 rm -f /var/run/spdk_stub0 892 # Re-enable ASLR now that we are done with multi-process testing 893 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 894 # randomizing and is the default on all recent Linux kernels 895 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 896} 897 898function run_test() { 899 if [ $# -le 1 ]; then 900 echo "Not enough parameters" 901 echo "usage: run_test test_name test_script [script_params]" 902 exit 1 903 fi 904 905 xtrace_disable 906 local test_name="$1" 907 shift 908 909 if [ -n "$test_domain" ]; then 910 export test_domain="${test_domain}.${test_name}" 911 else 912 export test_domain="$test_name" 913 fi 914 915 timing_enter $test_name 916 echo "************************************" 917 echo "START TEST $test_name" 918 echo "************************************" 919 xtrace_restore 920 time "$@" 921 xtrace_disable 922 echo "************************************" 923 echo "END TEST $test_name" 924 echo "************************************" 925 timing_exit $test_name 926 927 export test_domain=${test_domain%"$test_name"} 928 if [ -n "$test_domain" ]; then 929 export test_domain=${test_domain%?} 930 fi 931 932 if [ -z "$test_domain" ]; then 933 echo "top_level $test_name" >> $output_dir/test_completions.txt 934 else 935 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 936 fi 937 xtrace_restore 938} 939 940function skip_run_test_with_warning() { 941 echo "WARNING: $1" 942 echo "Test run may fail if run with autorun.sh" 943 echo "Please check your $rootdir/test/common/skipped_tests.txt" 944} 945 946function print_backtrace() { 947 # if errexit is not enabled, don't print a backtrace 948 [[ "$-" =~ e ]] || return 0 949 950 local args=("${BASH_ARGV[@]}") 951 952 xtrace_disable 953 echo "========== Backtrace start: ==========" 954 echo "" 955 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do 956 local func="${FUNCNAME[$i]}" 957 local line_nr="${BASH_LINENO[$((i - 1))]}" 958 local src="${BASH_SOURCE[$i]}" 959 local bt="" cmdline=() 960 961 if [[ -f $src ]]; then 962 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 963 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 964 fi 965 966 # If extdebug set the BASH_ARGC[i], try to fetch all the args 967 if ((BASH_ARGC[i] > 0)); then 968 # Use argc as index to reverse the stack 969 local argc=${BASH_ARGC[i]} arg 970 for arg in "${args[@]::BASH_ARGC[i]}"; do 971 cmdline[argc--]="[\"$arg\"]" 972 done 973 args=("${args[@]:BASH_ARGC[i]}") 974 fi 975 976 echo "in $src:$line_nr -> $func($( 977 IFS="," 978 printf '%s\n' "${cmdline[*]:-[]}" 979 ))" 980 echo " ..." 981 echo "${bt:-backtrace unavailable}" 982 echo " ..." 983 done 984 echo "" 985 echo "========== Backtrace end ==========" 986 xtrace_restore 987 return 0 988} 989 990function waitforserial() { 991 local i=0 992 local nvme_device_counter=1 993 if [[ -n "$2" ]]; then 994 nvme_device_counter=$2 995 fi 996 997 while [ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]; do 998 [ $i -lt 15 ] || break 999 i=$((i + 1)) 1000 echo "Waiting for devices" 1001 sleep 1 1002 done 1003 1004 if [[ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]]; then 1005 return 1 1006 fi 1007 1008 return 0 1009} 1010 1011function waitforserial_disconnect() { 1012 local i=0 1013 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1014 [ $i -lt 15 ] || break 1015 i=$((i + 1)) 1016 echo "Waiting for disconnect devices" 1017 sleep 1 1018 done 1019 1020 if lsblk -l -o NAME | grep -q -w $1; then 1021 return 1 1022 fi 1023 1024 return 0 1025} 1026 1027function waitforblk() { 1028 local i=0 1029 while ! lsblk -l -o NAME | grep -q -w $1; do 1030 [ $i -lt 15 ] || break 1031 i=$((i + 1)) 1032 sleep 1 1033 done 1034 1035 if ! lsblk -l -o NAME | grep -q -w $1; then 1036 return 1 1037 fi 1038 1039 return 0 1040} 1041 1042function waitforblk_disconnect() { 1043 local i=0 1044 while lsblk -l -o NAME | grep -q -w $1; do 1045 [ $i -lt 15 ] || break 1046 i=$((i + 1)) 1047 sleep 1 1048 done 1049 1050 if lsblk -l -o NAME | grep -q -w $1; then 1051 return 1 1052 fi 1053 1054 return 0 1055} 1056 1057function waitforfile() { 1058 local i=0 1059 while [ ! -e $1 ]; do 1060 [ $i -lt 200 ] || break 1061 i=$((i + 1)) 1062 sleep 0.1 1063 done 1064 1065 if [ ! -e $1 ]; then 1066 return 1 1067 fi 1068 1069 return 0 1070} 1071 1072function fio_config_gen() { 1073 local config_file=$1 1074 local workload=$2 1075 local bdev_type=$3 1076 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1077 1078 if [ -e "$config_file" ]; then 1079 echo "Configuration File Already Exists!: $config_file" 1080 return 1 1081 fi 1082 1083 if [ -z "$workload" ]; then 1084 workload=randrw 1085 fi 1086 1087 touch $1 1088 1089 cat > $1 << EOL 1090[global] 1091thread=1 1092group_reporting=1 1093direct=1 1094norandommap=1 1095percentile_list=50:99:99.9:99.99:99.999 1096time_based=1 1097ramp_time=0 1098EOL 1099 1100 if [ "$workload" == "verify" ]; then 1101 cat <<- EOL >> $config_file 1102 verify=sha1 1103 verify_backlog=1024 1104 rw=randwrite 1105 EOL 1106 1107 # To avoid potential data race issue due to the AIO device 1108 # flush mechanism, add the flag to serialize the writes. 1109 # This is to fix the intermittent IO failure issue of #935 1110 if [ "$bdev_type" == "AIO" ]; then 1111 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1112 echo "serialize_overlap=1" >> $config_file 1113 fi 1114 fi 1115 elif [ "$workload" == "trim" ]; then 1116 echo "rw=trimwrite" >> $config_file 1117 else 1118 echo "rw=$workload" >> $config_file 1119 fi 1120} 1121 1122function fio_bdev() { 1123 # Setup fio binary cmd line 1124 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1125 local bdev_plugin="$rootdir/build/fio/spdk_bdev" 1126 1127 # Preload AddressSanitizer library to fio if fio_plugin was compiled with it 1128 local asan_lib 1129 asan_lib=$(ldd $bdev_plugin | grep libasan | awk '{print $3}') 1130 1131 LD_PRELOAD="$asan_lib $bdev_plugin" "$fio_dir"/fio "$@" 1132} 1133 1134function fio_nvme() { 1135 # Setup fio binary cmd line 1136 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1137 local nvme_plugin="$rootdir/build/fio/spdk_nvme" 1138 1139 # Preload AddressSanitizer library to fio if fio_plugin was compiled with it 1140 asan_lib=$(ldd $nvme_plugin | grep libasan | awk '{print $3}') 1141 1142 LD_PRELOAD="$asan_lib $nvme_plugin" "$fio_dir"/fio "$@" 1143} 1144 1145function get_lvs_free_mb() { 1146 local lvs_uuid=$1 1147 local lvs_info 1148 local fc 1149 local cs 1150 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1151 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1152 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1153 1154 # Change to MB's 1155 free_mb=$((fc * cs / 1024 / 1024)) 1156 echo "$free_mb" 1157} 1158 1159function get_bdev_size() { 1160 local bdev_name=$1 1161 local bdev_info 1162 local bs 1163 local nb 1164 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1165 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1166 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1167 1168 # Change to MB's 1169 bdev_size=$((bs * nb / 1024 / 1024)) 1170 echo "$bdev_size" 1171} 1172 1173function autotest_cleanup() { 1174 $rootdir/scripts/setup.sh reset 1175 $rootdir/scripts/setup.sh cleanup 1176 if [ $(uname -s) = "Linux" ]; then 1177 if grep -q '#define SPDK_CONFIG_IGB_UIO_DRIVER 1' $rootdir/include/spdk/config.h; then 1178 [[ -e /sys/module/igb_uio ]] && rmmod igb_uio 1179 else 1180 modprobe -r uio_pci_generic 1181 fi 1182 fi 1183 rm -rf "$asan_suppression_file" 1184 if [[ -n $old_core_pattern ]]; then 1185 echo "$old_core_pattern" > /proc/sys/kernel/core_pattern 1186 fi 1187 if [[ -e /proc/$udevadm_pid/status ]]; then 1188 kill "$udevadm_pid" || : 1189 fi 1190 revert_soft_roce 1191} 1192 1193function freebsd_update_contigmem_mod() { 1194 if [ $(uname) = FreeBSD ]; then 1195 kldunload contigmem.ko || true 1196 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 1197 echo "Warning: SPDK only works on FreeBSD with patches that only exist in SPDK's dpdk submodule" 1198 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/ 1199 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/ 1200 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/ 1201 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/ 1202 else 1203 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1204 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1205 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1206 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1207 fi 1208 fi 1209} 1210 1211function get_nvme_name_from_bdf() { 1212 blkname=() 1213 1214 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1215 if [ -z "$nvme_devs" ]; then 1216 return 1217 fi 1218 for dev in $nvme_devs; do 1219 link_name=$(readlink /sys/block/$dev/device/device) || true 1220 if [ -z "$link_name" ]; then 1221 link_name=$(readlink /sys/block/$dev/device) 1222 fi 1223 bdf=$(basename "$link_name") 1224 if [ "$bdf" = "$1" ]; then 1225 blkname+=($dev) 1226 fi 1227 done 1228 1229 printf '%s\n' "${blkname[@]}" 1230} 1231 1232function get_nvme_ctrlr_from_bdf() { 1233 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1234 if [[ -z "$bdf_sysfs_path" ]]; then 1235 return 1236 fi 1237 1238 printf '%s\n' "$(basename $bdf_sysfs_path)" 1239} 1240 1241# Get BDF addresses of all NVMe drives currently attached to 1242# uio-pci-generic or vfio-pci 1243function get_nvme_bdfs() { 1244 xtrace_disable 1245 bdfs=$(jq -r .config[].params.traddr <<< $($rootdir/scripts/gen_nvme.sh)) 1246 if [[ -z $bdfs ]]; then 1247 echo "No devices to test on!" 1248 exit 1 1249 fi 1250 echo "$bdfs" 1251 xtrace_restore 1252} 1253 1254# Same as function above, but just get the first disks BDF address 1255function get_first_nvme_bdf() { 1256 head -1 <<< "$(get_nvme_bdfs)" 1257} 1258 1259function nvme_namespace_revert() { 1260 $rootdir/scripts/setup.sh 1261 sleep 1 1262 bdfs=$(get_nvme_bdfs) 1263 1264 $rootdir/scripts/setup.sh reset 1265 1266 for bdf in $bdfs; do 1267 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1268 if [[ -z "$nvme_ctrlr" ]]; then 1269 continue 1270 fi 1271 1272 # Check Optional Admin Command Support for Namespace Management 1273 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1274 oacs_ns_manage=$((oacs & 0x8)) 1275 1276 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1277 # This assumes every NVMe controller contains single namespace, 1278 # encompassing Total NVM Capacity and formatted as 512 block size. 1279 # 512 block size is needed for test/vhost/vhost_boot.sh to 1280 # succesfully run. 1281 1282 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1283 if [[ "$unvmcap" -eq 0 ]]; then 1284 # All available space already used 1285 continue 1286 fi 1287 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1288 blksize=512 1289 1290 size=$((tnvmcap / blksize)) 1291 1292 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c 0 || true 1293 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1294 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1295 nvme attach-ns ${nvme_ctrlr} -n 1 -c 0 1296 nvme reset ${nvme_ctrlr} 1297 waitforfile "${nvme_ctrlr}n1" 1298 fi 1299 done 1300} 1301 1302# Get BDFs based on device ID, such as 0x0a54 1303function get_nvme_bdfs_by_id() { 1304 local bdfs=() 1305 1306 for bdf in $(get_nvme_bdfs); do 1307 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1308 if [[ "$device" == "$1" ]]; then 1309 bdfs+=($bdf) 1310 fi 1311 done 1312 1313 printf '%s\n' "${bdfs[@]}" 1314} 1315 1316function opal_revert_cleanup() { 1317 # The OPAL CI tests is only used for P4510 devices. 1318 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1319 if [[ -z ${bdfs[0]} ]]; then 1320 return 0 1321 fi 1322 1323 $SPDK_BIN_DIR/spdk_tgt & 1324 spdk_tgt_pid=$! 1325 waitforlisten $spdk_tgt_pid 1326 1327 bdf_id=0 1328 for bdf in "${bdfs[@]}"; do 1329 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme"${bdf_id} -t "pcie" -a ${bdf} 1330 # Ignore if this fails. 1331 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b "nvme"${bdf_id} -p test || true 1332 ((++bdf_id)) 1333 done 1334 1335 killprocess $spdk_tgt_pid 1336} 1337 1338function pap() { 1339 while read -r file; do 1340 cat <<- FILE 1341 --- $file --- 1342 $(<"$file") 1343 --- $file --- 1344 FILE 1345 rm -f "$file" 1346 done < <(find "$@" -type f | sort -u) 1347} 1348 1349# Define temp storage for all the tests. Look for 2GB at minimum 1350set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1351 1352set -o errtrace 1353shopt -s extdebug 1354trap "trap - ERR; print_backtrace >&2" ERR 1355 1356PS4=' \t \$ ' 1357if $SPDK_AUTOTEST_X; then 1358 # explicitly enable xtraces, overriding any tracking information. 1359 unset XTRACE_DISABLED 1360 unset XTRACE_NESTING_LEVEL 1361 set -x 1362 xtrace_enable 1363else 1364 xtrace_restore 1365fi 1366
