1#!/usr/bin/env bash 2 3function xtrace_disable() { 4 if [ "$XTRACE_DISABLED" != "yes" ]; then 5 PREV_BASH_OPTS="$-" 6 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 7 XTRACE_DISABLED="yes" 8 fi 9 set +x 10 elif [ -z $XTRACE_NESTING_LEVEL ]; then 11 XTRACE_NESTING_LEVEL=1 12 else 13 XTRACE_NESTING_LEVEL=$((++XTRACE_NESTING_LEVEL)) 14 fi 15} 16 17xtrace_disable 18set -e 19shopt -s expand_aliases 20 21source "$rootdir/test/common/applications.sh" 22if [[ -e $rootdir/test/common/build_config.sh ]]; then 23 source "$rootdir/test/common/build_config.sh" 24elif [[ -e $rootdir/mk/config.mk ]]; then 25 build_config=$(< "$rootdir/mk/config.mk") 26 source <(echo "${build_config//\?=/=}") 27else 28 source "$rootdir/CONFIG" 29fi 30 31# Dummy function to be called after restoring xtrace just so that it appears in the 32# xtrace log. This way we can consistently track when xtrace is enabled/disabled. 33function xtrace_enable() { 34 # We have to do something inside a function in bash, and calling any command 35 # (even `:`) will produce an xtrace entry, so we just define another function. 36 function xtrace_dummy() { :; } 37} 38 39# Keep it as alias to avoid xtrace_enable backtrace always pointing to xtrace_restore. 40# xtrace_enable will appear as called directly from the user script, from the same line 41# that "called" xtrace_restore. 42alias xtrace_restore='if [ -z $XTRACE_NESTING_LEVEL ]; then 43 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 44 XTRACE_DISABLED="no"; PREV_BASH_OPTS=""; set -x; xtrace_enable; 45 fi 46else 47 XTRACE_NESTING_LEVEL=$((--XTRACE_NESTING_LEVEL)); 48 if [ $XTRACE_NESTING_LEVEL -eq "0" ]; then 49 unset XTRACE_NESTING_LEVEL 50 fi 51fi' 52 53: ${RUN_NIGHTLY:=0} 54export RUN_NIGHTLY 55 56# Set defaults for missing test config options 57: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 58export SPDK_AUTOTEST_DEBUG_APPS 59: ${SPDK_RUN_VALGRIND=0} 60export SPDK_RUN_VALGRIND 61: ${SPDK_RUN_FUNCTIONAL_TEST=0} 62export SPDK_RUN_FUNCTIONAL_TEST 63: ${SPDK_TEST_UNITTEST=0} 64export SPDK_TEST_UNITTEST 65: ${SPDK_TEST_AUTOBUILD=0} 66export SPDK_TEST_AUTOBUILD 67: ${SPDK_TEST_ISAL=0} 68export SPDK_TEST_ISAL 69: ${SPDK_TEST_ISCSI=0} 70export SPDK_TEST_ISCSI 71: ${SPDK_TEST_ISCSI_INITIATOR=0} 72export SPDK_TEST_ISCSI_INITIATOR 73: ${SPDK_TEST_NVME=0} 74export SPDK_TEST_NVME 75: ${SPDK_TEST_NVME_CLI=0} 76export SPDK_TEST_NVME_CLI 77: ${SPDK_TEST_NVME_CUSE=0} 78export SPDK_TEST_NVME_CUSE 79: ${SPDK_TEST_NVMF=0} 80export SPDK_TEST_NVMF 81: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 82export SPDK_TEST_NVMF_TRANSPORT 83: ${SPDK_TEST_RBD=0} 84export SPDK_TEST_RBD 85: ${SPDK_TEST_VHOST=0} 86export SPDK_TEST_VHOST 87: ${SPDK_TEST_BLOCKDEV=0} 88export SPDK_TEST_BLOCKDEV 89: ${SPDK_TEST_IOAT=0} 90export SPDK_TEST_IOAT 91: ${SPDK_TEST_BLOBFS=0} 92export SPDK_TEST_BLOBFS 93: ${SPDK_TEST_VHOST_INIT=0} 94export SPDK_TEST_VHOST_INIT 95: ${SPDK_TEST_PMDK=0} 96export SPDK_TEST_PMDK 97: ${SPDK_TEST_LVOL=0} 98export SPDK_TEST_LVOL 99: ${SPDK_TEST_JSON=0} 100export SPDK_TEST_JSON 101: ${SPDK_TEST_REDUCE=0} 102export SPDK_TEST_REDUCE 103: ${SPDK_RUN_ASAN=0} 104export SPDK_RUN_ASAN 105: ${SPDK_RUN_UBSAN=0} 106export SPDK_RUN_UBSAN 107: ${SPDK_RUN_EXTERNAL_DPDK=""} 108export SPDK_RUN_EXTERNAL_DPDK 109: ${SPDK_RUN_NON_ROOT=0} 110export SPDK_RUN_NON_ROOT 111: ${SPDK_TEST_CRYPTO=0} 112export SPDK_TEST_CRYPTO 113: ${SPDK_TEST_FTL=0} 114export SPDK_TEST_FTL 115: ${SPDK_TEST_OCF=0} 116export SPDK_TEST_OCF 117: ${SPDK_TEST_FTL_EXTENDED=0} 118export SPDK_TEST_FTL_EXTENDED 119: ${SPDK_TEST_VMD=0} 120export SPDK_TEST_VMD 121: ${SPDK_TEST_OPAL=0} 122export SPDK_TEST_OPAL 123: ${SPDK_TEST_NATIVE_DPDK} 124export SPDK_TEST_NATIVE_DPDK 125: ${SPDK_AUTOTEST_X=true} 126export SPDK_AUTOTEST_X 127: ${SPDK_TEST_RAID5=0} 128export SPDK_TEST_RAID5 129: ${SPDK_TEST_URING=0} 130export SPDK_TEST_URING 131: ${SPDK_TEST_USE_IGB_UIO:=0} 132export SPDK_TEST_USE_IGB_UIO 133 134export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib" 135export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR 136 137# Tell setup.sh to wait for block devices upon each reset 138export PCI_BLOCK_SYNC_ON_RESET=yes 139 140# Export PYTHONPATH with addition of RPC framework. New scripts can be created 141# specific use cases for tests. 142export PYTHONPATH=$PYTHONPATH:$rootdir/scripts 143 144# Don't create Python .pyc files. When running with sudo these will be 145# created with root ownership and can cause problems when cleaning the repository. 146export PYTHONDONTWRITEBYTECODE=1 147 148# Export flag to skip the known bug that exists in librados 149# Bug is reported on ceph bug tracker with number 24078 150export ASAN_OPTIONS=new_delete_type_mismatch=0 151export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1' 152 153# Export LeakSanitizer option to use suppression file in order to prevent false positives 154# and known leaks in external executables or libraries from showing up. 155asan_suppression_file="/var/tmp/asan_suppression_file" 156sudo rm -rf "$asan_suppression_file" 157cat << EOL >> "$asan_suppression_file" 158# ASAN has some bugs around thread_local variables. We have a destructor in place 159# to free the thread contexts, but ASAN complains about the leak before those 160# destructors have a chance to run. So suppress this one specific leak using 161# LSAN_OPTIONS. 162leak:spdk_fs_alloc_thread_ctx 163 164# Suppress known leaks in fio project 165leak:$CONFIG_FIO_SOURCE_DIR/parse.c 166leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 167leak:$CONFIG_FIO_SOURCE_DIR/init.c 168leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 169leak:fio_memalign 170leak:spdk_fio_io_u_init 171 172# Suppress leaks in libiscsi 173leak:libiscsi.so 174EOL 175 176# Suppress leaks in libfuse3 177echo "leak:libfuse3.so" >> "$asan_suppression_file" 178 179export LSAN_OPTIONS=suppressions="$asan_suppression_file" 180 181export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 182 183if [ -z "$DEPENDENCY_DIR" ]; then 184 export DEPENDENCY_DIR=/home/sys_sgsw 185else 186 export DEPENDENCY_DIR 187fi 188 189# Export location of where all the SPDK binaries are 190export SPDK_BIN_DIR="$rootdir/build/bin" 191export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 192 193# pass our valgrind desire on to unittest.sh 194if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 195 export valgrind='' 196fi 197 198if [ "$(uname -s)" = "Linux" ]; then 199 MAKE="make" 200 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 201 # Override the default HUGEMEM in scripts/setup.sh to allocate 8GB in hugepages. 202 export HUGEMEM=8192 203 if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then 204 export DRIVER_OVERRIDE=$rootdir/dpdk/build-tmp/kernel/linux/igb_uio/igb_uio.ko 205 # Building kernel modules requires root privileges 206 MAKE="sudo $MAKE" 207 fi 208elif [ "$(uname -s)" = "FreeBSD" ]; then 209 MAKE="gmake" 210 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 211 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 212 export HUGEMEM=2048 213else 214 echo "Unknown OS \"$(uname -s)\"" 215 exit 1 216fi 217 218if [ -z "$output_dir" ]; then 219 mkdir -p "$rootdir/../output" 220 export output_dir="$rootdir/../output" 221fi 222 223TEST_MODE= 224for i in "$@"; do 225 case "$i" in 226 --iso) 227 TEST_MODE=iso 228 ;; 229 --transport=*) 230 TEST_TRANSPORT="${i#*=}" 231 ;; 232 --sock=*) 233 TEST_SOCK="${i#*=}" 234 ;; 235 esac 236done 237 238# start rpc.py coprocess if it's not started yet 239if [[ -z $RPC_PIPE_PID ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 240 coproc RPC_PIPE { "$rootdir/scripts/rpc.py" --server; } 241 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 242 # all descriptors will automatically close together with this bash 243 # process, this will make rpc.py stop reading and exit gracefully 244fi 245 246function set_test_storage() { 247 [[ -v testdir ]] || return 0 248 249 local requested_size=$1 # bytes 250 local mount target_dir 251 252 local -A mounts fss sizes avails uses 253 local source fs size avail mount use 254 255 local storage_fallback storage_candidates 256 local storage_fallback_purge 257 258 shopt -s nullglob 259 storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 260 shopt -u nullglob 261 262 if ((${#storage_fallback_purge[@]} > 0)); then 263 printf '* Purging old temporary test storage (%s)\n' \ 264 "${storage_fallback_purge[*]}" >&2 265 sudo rm -rf "${storage_fallback_purge[@]}" 266 fi 267 268 storage_fallback=$(mktemp -udt spdk.XXXXXX) 269 storage_candidates=( 270 "$testdir" 271 "$storage_fallback/tests/${testdir##*/}" 272 "$storage_fallback" 273 ) 274 275 if [[ -n $ADD_TEST_STORAGE ]]; then 276 # List of dirs|mounts separated by whitespaces 277 storage_candidates+=($ADD_TEST_STORAGE) 278 fi 279 280 if [[ -n $DEDICATED_TEST_STORAGE ]]; then 281 # Single, dedicated dir|mount 282 storage_candidates=("$DEDICATED_TEST_STORAGE") 283 fi 284 285 mkdir -p "${storage_candidates[@]}" 286 287 # add some headroom - 64M 288 requested_size=$((requested_size + (64 << 20))) 289 290 while read -r source fs size use avail _ mount; do 291 mounts["$mount"]=$source fss["$mount"]=$fs 292 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 293 uses["$mount"]=$((use * 1024)) 294 done < <(df -T | grep -v Filesystem) 295 296 printf '* Looking for test storage...\n' >&2 297 298 local target_space new_size 299 for target_dir in "${storage_candidates[@]}"; do 300 # FreeBSD's df is lacking the --output arg 301 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 302 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 303 304 target_space=${avails["$mount"]} 305 if ((target_space == 0 || target_space < requested_size)); then 306 continue 307 fi 308 if ((target_space >= requested_size)); then 309 # For in-memory fs, and / make sure our requested size won't fill most of the space. 310 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 311 new_size=$((uses["$mount"] + requested_size)) 312 if ((new_size * 100 / sizes["$mount"] > 95)); then 313 continue 314 fi 315 fi 316 fi 317 export SPDK_TEST_STORAGE=$target_dir 318 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 319 return 0 320 done 321 printf '* Test storage is not available\n' 322 return 1 323} 324 325function get_config_params() { 326 xtrace_disable 327 config_params='--enable-debug --enable-werror' 328 329 # for options with dependencies but no test flag, set them here 330 if [ -f /usr/include/infiniband/verbs.h ]; then 331 config_params+=' --with-rdma' 332 fi 333 334 if [ $(uname -s) == "FreeBSD" ]; then 335 intel="hw.model: Intel" 336 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 337 else 338 intel="GenuineIntel" 339 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 340 fi 341 if [[ "$cpu_vendor" != *"$intel"* ]]; then 342 config_params+=" --without-idxd" 343 else 344 config_params+=" --with-idxd" 345 fi 346 347 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 348 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 349 fi 350 351 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 352 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 353 fi 354 355 if [ -d /usr/include/iscsi ]; then 356 libiscsi_version=$(grep LIBISCSI_API_VERSION /usr/include/iscsi/iscsi.h | head -1 | awk '{print $3}' | awk -F '(' '{print $2}' | awk -F ')' '{print $1}') 357 if [ $libiscsi_version -ge 20150621 ]; then 358 config_params+=' --with-iscsi-initiator' 359 fi 360 fi 361 362 if [ $SPDK_TEST_UNITTEST -eq 0 ]; then 363 config_params+=' --disable-unit-tests' 364 fi 365 366 if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then 367 config_params+=' --with-nvme-cuse' 368 fi 369 370 # for options with both dependencies and a test flag, set them here 371 if [ -f /usr/include/libpmemblk.h ] && [ $SPDK_TEST_PMDK -eq 1 ]; then 372 config_params+=' --with-pmdk' 373 fi 374 375 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_REDUCE -eq 1 ]; then 376 if [ $SPDK_TEST_ISAL -eq 1 ]; then 377 config_params+=' --with-reduce' 378 fi 379 fi 380 381 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 382 config_params+=' --with-rbd' 383 fi 384 385 # for options with no required dependencies, just test flags, set them here 386 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 387 config_params+=' --with-crypto' 388 fi 389 390 if [ $SPDK_TEST_OCF -eq 1 ]; then 391 config_params+=" --with-ocf" 392 fi 393 394 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 395 config_params+=' --enable-ubsan' 396 fi 397 398 if [ $SPDK_RUN_ASAN -eq 1 ]; then 399 config_params+=' --enable-asan' 400 fi 401 402 if [ "$(uname -s)" = "Linux" ]; then 403 config_params+=' --enable-coverage' 404 fi 405 406 if [ $SPDK_TEST_ISAL -eq 0 ]; then 407 config_params+=' --without-isal' 408 fi 409 410 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 411 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 412 config_params+=' --with-fuse' 413 fi 414 fi 415 416 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 417 config_params+=' --with-raid5' 418 fi 419 420 # Check whether liburing library header exists 421 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 422 config_params+=' --with-uring' 423 fi 424 425 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 426 config_params+=" --with-dpdk=$SPDK_RUN_EXTERNAL_DPDK" 427 fi 428 429 if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then 430 config_params+=" --with-igb-uio-driver" 431 fi 432 echo "$config_params" 433 xtrace_restore 434} 435 436function rpc_cmd() { 437 xtrace_disable 438 local rsp rc 439 440 echo "$@" >&$RPC_PIPE_INPUT 441 while read -t 5 -ru $RPC_PIPE_OUTPUT rsp; do 442 if [[ $rsp == "**STATUS="* ]]; then 443 break 444 fi 445 echo "$rsp" 446 done 447 448 rc=${rsp#*=} 449 xtrace_restore 450 [[ $rc == 0 ]] 451} 452 453function rpc_cmd_simple_data_json() { 454 455 local elems="$1[@]" elem 456 local -gA jq_out=() 457 local jq val 458 459 local lvs=( 460 "uuid" 461 "name" 462 "base_bdev" 463 "total_data_clusters" 464 "free_clusters" 465 "block_size" 466 "cluster_size" 467 ) 468 469 local bdev=( 470 "name" 471 "aliases[0]" 472 "block_size" 473 "num_blocks" 474 "uuid" 475 "product_name" 476 ) 477 478 [[ -v $elems ]] || return 1 479 480 for elem in "${!elems}"; do 481 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 482 done 483 jq+=',"\n"' 484 485 shift 486 while read -r elem val; do 487 jq_out["$elem"]=$val 488 done < <(rpc_cmd "$@" | jq -jr "$jq") 489 ((${#jq_out[@]} > 0)) || return 1 490} 491 492function NOT() { 493 local es=0 494 495 "$@" || es=$? 496 497 # Logic looks like so: 498 # - return false if command exit successfully 499 # - return false if command exit after receiving a core signal (FIXME: or any signal?) 500 # - return true if command exit with an error 501 502 # This naively assumes that the process doesn't exit with > 128 on its own. 503 if ((es > 128)); then 504 es=$((es & ~128)) 505 case "$es" in 506 3) ;& # SIGQUIT 507 4) ;& # SIGILL 508 6) ;& # SIGABRT 509 8) ;& # SIGFPE 510 9) ;& # SIGKILL 511 11) es=0 ;; # SIGSEGV 512 *) es=1 ;; 513 esac 514 elif [[ -n $EXIT_STATUS ]] && ((es != EXIT_STATUS)); then 515 es=0 516 fi 517 518 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 519 ((!es == 0)) 520} 521 522function timing() { 523 direction="$1" 524 testname="$2" 525 526 now=$(date +%s) 527 528 if [ "$direction" = "enter" ]; then 529 export timing_stack="${timing_stack};${now}" 530 export test_stack="${test_stack};${testname}" 531 else 532 touch "$output_dir/timing.txt" 533 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 534 535 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 536 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 537 538 elapsed=$((now - start_time - child_time)) 539 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 540 541 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 542 fi 543} 544 545function timing_enter() { 546 xtrace_disable 547 timing "enter" "$1" 548 xtrace_restore 549} 550 551function timing_exit() { 552 xtrace_disable 553 timing "exit" "$1" 554 xtrace_restore 555} 556 557function timing_finish() { 558 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 559 if [ -x "$flamegraph" ]; then 560 "$flamegraph" \ 561 --title 'Build Timing' \ 562 --nametype 'Step:' \ 563 --countname seconds \ 564 $output_dir/timing.txt \ 565 > $output_dir/timing.svg 566 fi 567} 568 569function create_test_list() { 570 xtrace_disable 571 # First search all scripts in main SPDK directory. 572 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 573 # Follow up with search in test directory recursively. 574 completion+=$(grep -rshI --include="*.sh" --exclude="autotest_common.sh" -e "run_test " $rootdir/test) 575 printf "%s" "$completion" | grep -v "#" \ 576 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 577 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 578 xtrace_restore 579} 580 581function gdb_attach() { 582 gdb -q --batch \ 583 -ex 'handle SIGHUP nostop pass' \ 584 -ex 'handle SIGQUIT nostop pass' \ 585 -ex 'handle SIGPIPE nostop pass' \ 586 -ex 'handle SIGALRM nostop pass' \ 587 -ex 'handle SIGTERM nostop pass' \ 588 -ex 'handle SIGUSR1 nostop pass' \ 589 -ex 'handle SIGUSR2 nostop pass' \ 590 -ex 'handle SIGCHLD nostop pass' \ 591 -ex 'set print thread-events off' \ 592 -ex 'cont' \ 593 -ex 'thread apply all bt' \ 594 -ex 'quit' \ 595 --tty=/dev/stdout \ 596 -p $1 597} 598 599function process_core() { 600 ret=0 601 while IFS= read -r -d '' core; do 602 exe=$(eu-readelf -n "$core" | grep psargs | sed "s/.*psargs: \([^ \'\" ]*\).*/\1/") 603 if [[ ! -f "$exe" ]]; then 604 exe=$(eu-readelf -n "$core" | grep -oP -m1 "$exe.+") 605 fi 606 echo "exe for $core is $exe" 607 if [[ -n "$exe" ]]; then 608 if hash gdb &> /dev/null; then 609 gdb -batch -ex "thread apply all bt full" $exe $core 610 fi 611 cp $exe $output_dir 612 fi 613 mv $core $output_dir 614 chmod a+r $output_dir/$core 615 ret=1 616 done < <(find . -type f \( -name 'core.[0-9]*' -o -name 'core' -o -name '*.core' \) -print0) 617 return $ret 618} 619 620function process_shm() { 621 type=$1 622 id=$2 623 if [ "$type" = "--pid" ]; then 624 id="pid${id}" 625 elif [ "$type" = "--id" ]; then 626 id="${id}" 627 else 628 echo "Please specify to search for pid or shared memory id." 629 return 1 630 fi 631 632 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 633 634 if [[ -z $shm_files ]]; then 635 echo "SHM File for specified PID or shared memory id: ${id} not found!" 636 return 1 637 fi 638 for n in $shm_files; do 639 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 640 done 641 return 0 642} 643 644function waitforlisten() { 645 # $1 = process pid 646 if [ -z "$1" ]; then 647 exit 1 648 fi 649 650 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 651 652 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 653 # turn off trace for this loop 654 xtrace_disable 655 local ret=0 656 local i 657 for ((i = 40; i != 0; i--)); do 658 # if the process is no longer running, then exit the script 659 # since it means the application crashed 660 if ! kill -s 0 $1; then 661 echo "ERROR: process (pid: $1) is no longer running" 662 ret=1 663 break 664 fi 665 666 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 667 break 668 fi 669 670 sleep 0.5 671 done 672 673 xtrace_restore 674 if ((i == 0)); then 675 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 676 ret=1 677 fi 678 return $ret 679} 680 681function waitfornbd() { 682 local nbd_name=$1 683 local i 684 685 for ((i = 1; i <= 20; i++)); do 686 if grep -q -w $nbd_name /proc/partitions; then 687 break 688 else 689 sleep 0.1 690 fi 691 done 692 693 # The nbd device is now recognized as a block device, but there can be 694 # a small delay before we can start I/O to that block device. So loop 695 # here trying to read the first block of the nbd block device to a temp 696 # file. Note that dd returns success when reading an empty file, so we 697 # need to check the size of the output file instead. 698 for ((i = 1; i <= 20; i++)); do 699 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 700 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 701 rm -f "$SPDK_TEST_STORAGE/nbdtest" 702 if [ "$size" != "0" ]; then 703 return 0 704 else 705 sleep 0.1 706 fi 707 done 708 709 return 1 710} 711 712function waitforbdev() { 713 local bdev_name=$1 714 local i 715 716 for ((i = 1; i <= 20; i++)); do 717 if $rpc_py bdev_get_bdevs | jq -r '.[] .name' | grep -qw $bdev_name; then 718 return 0 719 fi 720 721 if $rpc_py bdev_get_bdevs | jq -r '.[] .aliases' | grep -qw $bdev_name; then 722 return 0 723 fi 724 725 sleep 0.1 726 done 727 728 return 1 729} 730 731function make_filesystem() { 732 local fstype=$1 733 local dev_name=$2 734 local i=0 735 local force 736 737 if [ $fstype = ext4 ]; then 738 force=-F 739 else 740 force=-f 741 fi 742 743 while ! mkfs.${fstype} $force ${dev_name}; do 744 if [ $i -ge 15 ]; then 745 return 1 746 fi 747 i=$((i + 1)) 748 sleep 1 749 done 750 751 return 0 752} 753 754function killprocess() { 755 # $1 = process pid 756 if [ -z "$1" ]; then 757 exit 1 758 fi 759 760 if kill -0 $1; then 761 if [ $(uname) = Linux ]; then 762 process_name=$(ps --no-headers -o comm= $1) 763 else 764 process_name=$(ps -c -o command $1 | tail -1) 765 fi 766 if [ "$process_name" = "sudo" ]; then 767 # kill the child process, which is the actual app 768 # (assume $1 has just one child) 769 local child 770 child="$(pgrep -P $1)" 771 echo "killing process with pid $child" 772 kill $child 773 else 774 echo "killing process with pid $1" 775 kill $1 776 fi 777 778 # wait for the process regardless if its the dummy sudo one 779 # or the actual app - it should terminate anyway 780 wait $1 781 else 782 # the process is not there anymore 783 echo "Process with pid $1 is not found" 784 exit 1 785 fi 786} 787 788function iscsicleanup() { 789 echo "Cleaning up iSCSI connection" 790 iscsiadm -m node --logout || true 791 iscsiadm -m node -o delete || true 792 rm -rf /var/lib/iscsi/nodes/* 793} 794 795function stop_iscsi_service() { 796 if cat /etc/*-release | grep Ubuntu; then 797 service open-iscsi stop 798 else 799 service iscsid stop 800 fi 801} 802 803function start_iscsi_service() { 804 if cat /etc/*-release | grep Ubuntu; then 805 service open-iscsi start 806 else 807 service iscsid start 808 fi 809} 810 811function rbd_setup() { 812 # $1 = monitor ip address 813 # $2 = name of the namespace 814 if [ -z "$1" ]; then 815 echo "No monitor IP address provided for ceph" 816 exit 1 817 fi 818 if [ -n "$2" ]; then 819 if ip netns list | grep "$2"; then 820 NS_CMD="ip netns exec $2" 821 else 822 echo "No namespace $2 exists" 823 exit 1 824 fi 825 fi 826 827 if hash ceph; then 828 export PG_NUM=128 829 export RBD_POOL=rbd 830 export RBD_NAME=foo 831 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 832 $NS_CMD $rootdir/scripts/ceph/start.sh $1 833 834 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 835 $NS_CMD rbd create $RBD_NAME --size 1000 836 fi 837} 838 839function rbd_cleanup() { 840 if hash ceph; then 841 $rootdir/scripts/ceph/stop.sh || true 842 rm -f /var/tmp/ceph_raw.img 843 fi 844} 845 846function nvme_cli_build() { 847 if [[ -z "${DEPENDENCY_DIR}" ]]; then 848 echo DEPENDENCY_DIR not defined! 849 exit 1 850 fi 851 852 spdk_nvme_cli="${DEPENDENCY_DIR}/nvme-cli" 853 854 if [[ ! -d $spdk_nvme_cli ]]; then 855 echo "nvme-cli repository not found at $spdk_nvme_cli; skipping tests." 856 exit 1 857 fi 858 859 if ! grep -q "DEF_VER=v1.6" $spdk_nvme_cli/NVME-VERSION-GEN; then 860 echo "SPDK supports only \"spdk/nvme-cli\" project on \"spdk-1.6\" branch." 861 exit 1 862 fi 863 864 # Build against the version of SPDK under test 865 pushd $spdk_nvme_cli 866 867 # Remove and recreate git index in case it became corrupted 868 if ! git clean -dfx; then 869 rm -f .git/index 870 git clean -dfx 871 git reset --hard 872 fi 873 874 rm -f "$spdk_nvme_cli/spdk" 875 ln -sf "$rootdir" "$spdk_nvme_cli/spdk" 876 877 make -j$(nproc) LDFLAGS="$(make -s -C $spdk_nvme_cli/spdk ldflags)" 878 popd 879} 880 881function _start_stub() { 882 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 883 # but ASLR can still be unreliable in some cases. 884 # We will reenable it again after multi-process testing is complete in kill_stub(). 885 # Save current setting so it can be restored upon calling kill_stub(). 886 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 887 echo 0 > /proc/sys/kernel/randomize_va_space 888 $rootdir/test/app/stub/stub $1 & 889 stubpid=$! 890 echo Waiting for stub to ready for secondary processes... 891 while ! [ -e /var/run/spdk_stub0 ]; do 892 # If stub dies while we wait, bail 893 [[ -e /proc/$stubpid ]] || return 1 894 sleep 1s 895 done 896 echo done. 897} 898 899function start_stub() { 900 if ! _start_stub "$@"; then 901 echo "stub failed" >&2 902 return 1 903 fi 904} 905 906function kill_stub() { 907 if [[ -e /proc/$stubpid ]]; then 908 kill $1 $stubpid 909 wait $stubpid 910 fi 2> /dev/null || : 911 rm -f /var/run/spdk_stub0 912 # Re-enable ASLR now that we are done with multi-process testing 913 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 914 # randomizing and is the default on all recent Linux kernels 915 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 916} 917 918function run_test() { 919 if [ $# -le 1 ]; then 920 echo "Not enough parameters" 921 echo "usage: run_test test_name test_script [script_params]" 922 exit 1 923 fi 924 925 xtrace_disable 926 local test_name="$1" 927 shift 928 929 if [ -n "$test_domain" ]; then 930 export test_domain="${test_domain}.${test_name}" 931 else 932 export test_domain="$test_name" 933 fi 934 935 timing_enter $test_name 936 echo "************************************" 937 echo "START TEST $test_name" 938 echo "************************************" 939 xtrace_restore 940 time "$@" 941 xtrace_disable 942 echo "************************************" 943 echo "END TEST $test_name" 944 echo "************************************" 945 timing_exit $test_name 946 947 export test_domain=${test_domain%"$test_name"} 948 if [ -n "$test_domain" ]; then 949 export test_domain=${test_domain%?} 950 fi 951 952 if [ -z "$test_domain" ]; then 953 echo "top_level $test_name" >> $output_dir/test_completions.txt 954 else 955 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 956 fi 957 xtrace_restore 958} 959 960function skip_run_test_with_warning() { 961 echo "WARNING: $1" 962 echo "Test run may fail if run with autorun.sh" 963 echo "Please check your $rootdir/test/common/skipped_tests.txt" 964} 965 966function print_backtrace() { 967 # if errexit is not enabled, don't print a backtrace 968 [[ "$-" =~ e ]] || return 0 969 970 local args=("${BASH_ARGV[@]}") 971 972 xtrace_disable 973 echo "========== Backtrace start: ==========" 974 echo "" 975 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do 976 local func="${FUNCNAME[$i]}" 977 local line_nr="${BASH_LINENO[$((i - 1))]}" 978 local src="${BASH_SOURCE[$i]}" 979 local bt="" cmdline=() 980 981 if [[ -f $src ]]; then 982 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 983 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 984 fi 985 986 # If extdebug set the BASH_ARGC[i], try to fetch all the args 987 if ((BASH_ARGC[i] > 0)); then 988 # Use argc as index to reverse the stack 989 local argc=${BASH_ARGC[i]} arg 990 for arg in "${args[@]::BASH_ARGC[i]}"; do 991 cmdline[argc--]="[\"$arg\"]" 992 done 993 args=("${args[@]:BASH_ARGC[i]}") 994 fi 995 996 echo "in $src:$line_nr -> $func($( 997 IFS="," 998 printf '%s\n' "${cmdline[*]:-[]}" 999 ))" 1000 echo " ..." 1001 echo "${bt:-backtrace unavailable}" 1002 echo " ..." 1003 done 1004 echo "" 1005 echo "========== Backtrace end ==========" 1006 xtrace_restore 1007 return 0 1008} 1009 1010function waitforserial() { 1011 local i=0 1012 local nvme_device_counter=1 1013 if [[ -n "$2" ]]; then 1014 nvme_device_counter=$2 1015 fi 1016 1017 while [ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]; do 1018 [ $i -lt 15 ] || break 1019 i=$((i + 1)) 1020 echo "Waiting for devices" 1021 sleep 1 1022 done 1023 1024 if [[ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]]; then 1025 return 1 1026 fi 1027 1028 return 0 1029} 1030 1031function waitforserial_disconnect() { 1032 local i=0 1033 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1034 [ $i -lt 15 ] || break 1035 i=$((i + 1)) 1036 echo "Waiting for disconnect devices" 1037 sleep 1 1038 done 1039 1040 if lsblk -l -o NAME | grep -q -w $1; then 1041 return 1 1042 fi 1043 1044 return 0 1045} 1046 1047function waitforblk() { 1048 local i=0 1049 while ! lsblk -l -o NAME | grep -q -w $1; do 1050 [ $i -lt 15 ] || break 1051 i=$((i + 1)) 1052 sleep 1 1053 done 1054 1055 if ! lsblk -l -o NAME | grep -q -w $1; then 1056 return 1 1057 fi 1058 1059 return 0 1060} 1061 1062function waitforblk_disconnect() { 1063 local i=0 1064 while lsblk -l -o NAME | grep -q -w $1; do 1065 [ $i -lt 15 ] || break 1066 i=$((i + 1)) 1067 sleep 1 1068 done 1069 1070 if lsblk -l -o NAME | grep -q -w $1; then 1071 return 1 1072 fi 1073 1074 return 0 1075} 1076 1077function waitforfile() { 1078 local i=0 1079 while [ ! -e $1 ]; do 1080 [ $i -lt 200 ] || break 1081 i=$((i + 1)) 1082 sleep 0.1 1083 done 1084 1085 if [ ! -e $1 ]; then 1086 return 1 1087 fi 1088 1089 return 0 1090} 1091 1092function fio_config_gen() { 1093 local config_file=$1 1094 local workload=$2 1095 local bdev_type=$3 1096 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1097 1098 if [ -e "$config_file" ]; then 1099 echo "Configuration File Already Exists!: $config_file" 1100 return 1 1101 fi 1102 1103 if [ -z "$workload" ]; then 1104 workload=randrw 1105 fi 1106 1107 touch $1 1108 1109 cat > $1 << EOL 1110[global] 1111thread=1 1112group_reporting=1 1113direct=1 1114norandommap=1 1115percentile_list=50:99:99.9:99.99:99.999 1116time_based=1 1117ramp_time=0 1118EOL 1119 1120 if [ "$workload" == "verify" ]; then 1121 cat <<- EOL >> $config_file 1122 verify=sha1 1123 verify_backlog=1024 1124 rw=randwrite 1125 EOL 1126 1127 # To avoid potential data race issue due to the AIO device 1128 # flush mechanism, add the flag to serialize the writes. 1129 # This is to fix the intermittent IO failure issue of #935 1130 if [ "$bdev_type" == "AIO" ]; then 1131 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1132 echo "serialize_overlap=1" >> $config_file 1133 fi 1134 fi 1135 elif [ "$workload" == "trim" ]; then 1136 echo "rw=trimwrite" >> $config_file 1137 else 1138 echo "rw=$workload" >> $config_file 1139 fi 1140} 1141 1142function fio_bdev() { 1143 # Setup fio binary cmd line 1144 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1145 local bdev_plugin="$rootdir/build/fio/spdk_bdev" 1146 1147 # Preload AddressSanitizer library to fio if fio_plugin was compiled with it 1148 local asan_lib 1149 asan_lib=$(ldd $bdev_plugin | grep libasan | awk '{print $3}') 1150 1151 LD_PRELOAD="$asan_lib $bdev_plugin" "$fio_dir"/fio "$@" 1152} 1153 1154function fio_nvme() { 1155 # Setup fio binary cmd line 1156 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1157 local nvme_plugin="$rootdir/build/fio/spdk_nvme" 1158 1159 # Preload AddressSanitizer library to fio if fio_plugin was compiled with it 1160 asan_lib=$(ldd $nvme_plugin | grep libasan | awk '{print $3}') 1161 1162 LD_PRELOAD="$asan_lib $nvme_plugin" "$fio_dir"/fio "$@" 1163} 1164 1165function get_lvs_free_mb() { 1166 local lvs_uuid=$1 1167 local lvs_info 1168 local fc 1169 local cs 1170 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1171 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1172 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1173 1174 # Change to MB's 1175 free_mb=$((fc * cs / 1024 / 1024)) 1176 echo "$free_mb" 1177} 1178 1179function get_bdev_size() { 1180 local bdev_name=$1 1181 local bdev_info 1182 local bs 1183 local nb 1184 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1185 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1186 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1187 1188 # Change to MB's 1189 bdev_size=$((bs * nb / 1024 / 1024)) 1190 echo "$bdev_size" 1191} 1192 1193function autotest_cleanup() { 1194 $rootdir/scripts/setup.sh reset 1195 $rootdir/scripts/setup.sh cleanup 1196 if [ $(uname -s) = "Linux" ]; then 1197 if grep -q '#define SPDK_CONFIG_IGB_UIO_DRIVER 1' $rootdir/include/spdk/config.h; then 1198 [[ -e /sys/module/igb_uio ]] && rmmod igb_uio 1199 else 1200 modprobe -r uio_pci_generic 1201 fi 1202 fi 1203 rm -rf "$asan_suppression_file" 1204 if [[ -n $old_core_pattern ]]; then 1205 echo "$old_core_pattern" > /proc/sys/kernel/core_pattern 1206 fi 1207 if [[ -e /proc/$udevadm_pid/status ]]; then 1208 kill "$udevadm_pid" || : 1209 fi 1210 revert_soft_roce 1211} 1212 1213function freebsd_update_contigmem_mod() { 1214 if [ $(uname) = FreeBSD ]; then 1215 kldunload contigmem.ko || true 1216 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 1217 echo "Warning: SPDK only works on FreeBSD with patches that only exist in SPDK's dpdk submodule" 1218 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/ 1219 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/ 1220 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/ 1221 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/ 1222 else 1223 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1224 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1225 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1226 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1227 fi 1228 fi 1229} 1230 1231function get_nvme_name_from_bdf() { 1232 blkname=() 1233 1234 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1235 if [ -z "$nvme_devs" ]; then 1236 return 1237 fi 1238 for dev in $nvme_devs; do 1239 link_name=$(readlink /sys/block/$dev/device/device) || true 1240 if [ -z "$link_name" ]; then 1241 link_name=$(readlink /sys/block/$dev/device) 1242 fi 1243 bdf=$(basename "$link_name") 1244 if [ "$bdf" = "$1" ]; then 1245 blkname+=($dev) 1246 fi 1247 done 1248 1249 printf '%s\n' "${blkname[@]}" 1250} 1251 1252function get_nvme_ctrlr_from_bdf() { 1253 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1254 if [[ -z "$bdf_sysfs_path" ]]; then 1255 return 1256 fi 1257 1258 printf '%s\n' "$(basename $bdf_sysfs_path)" 1259} 1260 1261# Get BDF addresses of all NVMe drives currently attached to 1262# uio-pci-generic or vfio-pci 1263function get_nvme_bdfs() { 1264 xtrace_disable 1265 bdfs=$(jq -r .config[].params.traddr <<< $($rootdir/scripts/gen_nvme.sh --json)) 1266 if [[ -z $bdfs ]]; then 1267 echo "No devices to test on!" 1268 exit 1 1269 fi 1270 echo "$bdfs" 1271 xtrace_restore 1272} 1273 1274# Same as function above, but just get the first disks BDF address 1275function get_first_nvme_bdf() { 1276 head -1 <<< "$(get_nvme_bdfs)" 1277} 1278 1279function nvme_namespace_revert() { 1280 $rootdir/scripts/setup.sh 1281 sleep 1 1282 bdfs=$(get_nvme_bdfs) 1283 1284 $rootdir/scripts/setup.sh reset 1285 1286 for bdf in $bdfs; do 1287 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1288 if [[ -z "$nvme_ctrlr" ]]; then 1289 continue 1290 fi 1291 1292 # Check Optional Admin Command Support for Namespace Management 1293 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1294 oacs_ns_manage=$((oacs & 0x8)) 1295 1296 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1297 # This assumes every NVMe controller contains single namespace, 1298 # encompassing Total NVM Capacity and formatted as 512 block size. 1299 # 512 block size is needed for test/vhost/vhost_boot.sh to 1300 # succesfully run. 1301 1302 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1303 if [[ "$unvmcap" -eq 0 ]]; then 1304 # All available space already used 1305 continue 1306 fi 1307 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1308 blksize=512 1309 1310 size=$((tnvmcap / blksize)) 1311 1312 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c 0 || true 1313 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1314 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1315 nvme attach-ns ${nvme_ctrlr} -n 1 -c 0 1316 nvme reset ${nvme_ctrlr} 1317 waitforfile "${nvme_ctrlr}n1" 1318 fi 1319 done 1320} 1321 1322# Get BDFs based on device ID, such as 0x0a54 1323function get_nvme_bdfs_by_id() { 1324 local bdfs=() 1325 1326 for bdf in $(get_nvme_bdfs); do 1327 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1328 if [[ "$device" == "$1" ]]; then 1329 bdfs+=($bdf) 1330 fi 1331 done 1332 1333 printf '%s\n' "${bdfs[@]}" 1334} 1335 1336function opal_revert_cleanup() { 1337 # The OPAL CI tests is only used for P4510 devices. 1338 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1339 if [[ -z ${bdfs[0]} ]]; then 1340 return 0 1341 fi 1342 1343 $SPDK_BIN_DIR/spdk_tgt & 1344 spdk_tgt_pid=$! 1345 waitforlisten $spdk_tgt_pid 1346 1347 for bdf in "${bdfs[@]}"; do 1348 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme0" -t "pcie" -a ${bdf} 1349 # Ignore if this fails. 1350 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b nvme0 -p test || true 1351 done 1352 1353 killprocess $spdk_tgt_pid 1354} 1355 1356function pap() { 1357 while read -r file; do 1358 cat <<- FILE 1359 --- $file --- 1360 $(<"$file") 1361 --- $file --- 1362 FILE 1363 rm -f "$file" 1364 done < <(find "$@" -type f | sort -u) 1365} 1366 1367# Define temp storage for all the tests. Look for 2GB at minimum 1368set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1369 1370set -o errtrace 1371shopt -s extdebug 1372trap "trap - ERR; print_backtrace >&2" ERR 1373 1374PS4=' \t \$ ' 1375if $SPDK_AUTOTEST_X; then 1376 # explicitly enable xtraces, overriding any tracking information. 1377 unset XTRACE_DISABLED 1378 unset XTRACE_NESTING_LEVEL 1379 set -x 1380 xtrace_enable 1381else 1382 xtrace_restore 1383fi 1384
