1#!/usr/bin/env bash 2 3function xtrace_fd() { 4 if [[ -n $BASH_XTRACEFD && -e /proc/self/fd/$BASH_XTRACEFD ]]; then 5 # Close it first to make sure it's sane 6 exec {BASH_XTRACEFD}>&- 7 fi 8 exec {BASH_XTRACEFD}>&2 9 10 set -x 11} 12 13function xtrace_disable() { 14 if [ "$XTRACE_DISABLED" != "yes" ]; then 15 PREV_BASH_OPTS="$-" 16 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 17 XTRACE_DISABLED="yes" 18 fi 19 set +x 20 elif [ -z $XTRACE_NESTING_LEVEL ]; then 21 XTRACE_NESTING_LEVEL=1 22 else 23 XTRACE_NESTING_LEVEL=$((++XTRACE_NESTING_LEVEL)) 24 fi 25} 26 27xtrace_disable 28set -e 29shopt -s expand_aliases 30 31if [[ -e $rootdir/test/common/build_config.sh ]]; then 32 source "$rootdir/test/common/build_config.sh" 33elif [[ -e $rootdir/mk/config.mk ]]; then 34 build_config=$(< "$rootdir/mk/config.mk") 35 source <(echo "${build_config//\?=/=}") 36else 37 source "$rootdir/CONFIG" 38fi 39 40# Source scripts after the config so that the definitions are available. 41source "$rootdir/test/common/applications.sh" 42source "$rootdir/scripts/common.sh" 43 44# Dummy function to be called after restoring xtrace just so that it appears in the 45# xtrace log. This way we can consistently track when xtrace is enabled/disabled. 46function xtrace_enable() { 47 # We have to do something inside a function in bash, and calling any command 48 # (even `:`) will produce an xtrace entry, so we just define another function. 49 function xtrace_dummy() { :; } 50} 51 52# Keep it as alias to avoid xtrace_enable backtrace always pointing to xtrace_restore. 53# xtrace_enable will appear as called directly from the user script, from the same line 54# that "called" xtrace_restore. 55alias xtrace_restore='if [ -z $XTRACE_NESTING_LEVEL ]; then 56 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 57 XTRACE_DISABLED="no"; PREV_BASH_OPTS=""; set -x; xtrace_enable; 58 fi 59else 60 XTRACE_NESTING_LEVEL=$((--XTRACE_NESTING_LEVEL)); 61 if [ $XTRACE_NESTING_LEVEL -eq "0" ]; then 62 unset XTRACE_NESTING_LEVEL 63 fi 64fi' 65 66: ${RUN_NIGHTLY:=0} 67export RUN_NIGHTLY 68 69# Set defaults for missing test config options 70: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 71export SPDK_AUTOTEST_DEBUG_APPS 72: ${SPDK_RUN_VALGRIND=0} 73export SPDK_RUN_VALGRIND 74: ${SPDK_RUN_FUNCTIONAL_TEST=0} 75export SPDK_RUN_FUNCTIONAL_TEST 76: ${SPDK_TEST_UNITTEST=0} 77export SPDK_TEST_UNITTEST 78: ${SPDK_TEST_AUTOBUILD=0} 79export SPDK_TEST_AUTOBUILD 80: ${SPDK_TEST_RELEASE_BUILD=0} 81export SPDK_TEST_RELEASE_BUILD 82: ${SPDK_TEST_ISAL=0} 83export SPDK_TEST_ISAL 84: ${SPDK_TEST_ISCSI=0} 85export SPDK_TEST_ISCSI 86: ${SPDK_TEST_ISCSI_INITIATOR=0} 87export SPDK_TEST_ISCSI_INITIATOR 88: ${SPDK_TEST_NVME=0} 89export SPDK_TEST_NVME 90: ${SPDK_TEST_NVME_PMR=0} 91export SPDK_TEST_NVME_PMR 92: ${SPDK_TEST_NVME_SCC=0} 93export SPDK_TEST_NVME_SCC 94: ${SPDK_TEST_NVME_BP=0} 95export SPDK_TEST_NVME_BP 96: ${SPDK_TEST_NVME_CLI=0} 97export SPDK_TEST_NVME_CLI 98: ${SPDK_TEST_NVME_CUSE=0} 99export SPDK_TEST_NVME_CUSE 100: ${SPDK_TEST_NVMF=0} 101export SPDK_TEST_NVMF 102: ${SPDK_TEST_VFIOUSER=0} 103export SPDK_TEST_VFIOUSER 104: ${SPDK_TEST_VFIOUSER_QEMU=0} 105export SPDK_TEST_VFIOUSER_QEMU 106: ${SPDK_TEST_FUZZER=0} 107export SPDK_TEST_FUZZER 108: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 109export SPDK_TEST_NVMF_TRANSPORT 110: ${SPDK_TEST_RBD=0} 111export SPDK_TEST_RBD 112: ${SPDK_TEST_VHOST=0} 113export SPDK_TEST_VHOST 114: ${SPDK_TEST_BLOCKDEV=0} 115export SPDK_TEST_BLOCKDEV 116: ${SPDK_TEST_IOAT=0} 117export SPDK_TEST_IOAT 118: ${SPDK_TEST_BLOBFS=0} 119export SPDK_TEST_BLOBFS 120: ${SPDK_TEST_VHOST_INIT=0} 121export SPDK_TEST_VHOST_INIT 122: ${SPDK_TEST_PMDK=0} 123export SPDK_TEST_PMDK 124: ${SPDK_TEST_LVOL=0} 125export SPDK_TEST_LVOL 126: ${SPDK_TEST_REDUCE=0} 127export SPDK_TEST_REDUCE 128: ${SPDK_RUN_ASAN=0} 129export SPDK_RUN_ASAN 130: ${SPDK_RUN_UBSAN=0} 131export SPDK_RUN_UBSAN 132: ${SPDK_RUN_EXTERNAL_DPDK=""} 133export SPDK_RUN_EXTERNAL_DPDK 134: ${SPDK_RUN_NON_ROOT=0} 135export SPDK_RUN_NON_ROOT 136: ${SPDK_TEST_CRYPTO=0} 137export SPDK_TEST_CRYPTO 138: ${SPDK_TEST_FTL=0} 139export SPDK_TEST_FTL 140: ${SPDK_TEST_OCF=0} 141export SPDK_TEST_OCF 142: ${SPDK_TEST_VMD=0} 143export SPDK_TEST_VMD 144: ${SPDK_TEST_OPAL=0} 145export SPDK_TEST_OPAL 146: ${SPDK_TEST_NATIVE_DPDK} 147export SPDK_TEST_NATIVE_DPDK 148: ${SPDK_AUTOTEST_X=true} 149export SPDK_AUTOTEST_X 150: ${SPDK_TEST_RAID5=0} 151export SPDK_TEST_RAID5 152: ${SPDK_TEST_URING=0} 153export SPDK_TEST_URING 154: ${SPDK_TEST_USDT=0} 155export SPDK_TEST_USDT 156: ${SPDK_TEST_USE_IGB_UIO:=0} 157export SPDK_TEST_USE_IGB_UIO 158: ${SPDK_TEST_SCHEDULER:=0} 159export SPDK_TEST_SCHEDULER 160: ${SPDK_TEST_SCANBUILD:=0} 161export SPDK_TEST_SCANBUILD 162: ${SPDK_TEST_NVMF_NICS:=} 163export SPDK_TEST_NVMF_NICS 164 165# always test with SPDK shared objects. 166export SPDK_LIB_DIR="$rootdir/build/lib" 167export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib" 168export VFIO_LIB_DIR="$rootdir/build/libvfio-user/usr/local/lib" 169export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR:$VFIO_LIB_DIR 170 171# Tell setup.sh to wait for block devices upon each reset 172export PCI_BLOCK_SYNC_ON_RESET=yes 173 174# Export PYTHONPATH with addition of RPC framework. New scripts can be created 175# specific use cases for tests. 176export PYTHONPATH=$PYTHONPATH:$rootdir/python 177 178# Don't create Python .pyc files. When running with sudo these will be 179# created with root ownership and can cause problems when cleaning the repository. 180export PYTHONDONTWRITEBYTECODE=1 181 182# Export flag to skip the known bug that exists in librados 183# Bug is reported on ceph bug tracker with number 24078 184export ASAN_OPTIONS=new_delete_type_mismatch=0:disable_coredump=0 185export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1:disable_coredump=0' 186 187# Export LeakSanitizer option to use suppression file in order to prevent false positives 188# and known leaks in external executables or libraries from showing up. 189asan_suppression_file="/var/tmp/asan_suppression_file" 190rm -rf "$asan_suppression_file" 2> /dev/null || sudo rm -rf "$asan_suppression_file" 191cat << EOL >> "$asan_suppression_file" 192# ASAN has some bugs around thread_local variables. We have a destructor in place 193# to free the thread contexts, but ASAN complains about the leak before those 194# destructors have a chance to run. So suppress this one specific leak using 195# LSAN_OPTIONS. 196leak:spdk_fs_alloc_thread_ctx 197 198# Suppress known leaks in fio project 199leak:$CONFIG_FIO_SOURCE_DIR/parse.c 200leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 201leak:$CONFIG_FIO_SOURCE_DIR/init.c 202leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 203leak:fio_memalign 204leak:spdk_fio_io_u_init 205# Suppress leaks in gperftools-libs from fio 206leak:libtcmalloc_minimal.so 207 208# Suppress leaks in libiscsi 209leak:libiscsi.so 210EOL 211 212# Suppress leaks in libfuse3 213echo "leak:libfuse3.so" >> "$asan_suppression_file" 214 215export LSAN_OPTIONS=suppressions="$asan_suppression_file" 216 217export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 218 219if [ -z "$DEPENDENCY_DIR" ]; then 220 export DEPENDENCY_DIR=$HOME/spdk_dependencies 221else 222 export DEPENDENCY_DIR 223fi 224 225# Export location of where all the SPDK binaries are 226export SPDK_BIN_DIR="$rootdir/build/bin" 227export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 228 229# pass our valgrind desire on to unittest.sh 230if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 231 export valgrind='' 232fi 233 234if [ "$(uname -s)" = "Linux" ]; then 235 HUGEMEM=${HUGEMEM:-4096} 236 export CLEAR_HUGE=yes 237 if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_REDUCE -eq 1 ]]; then 238 # Make sure that memory is distributed across all NUMA nodes - by default, all goes to 239 # node0, but if QAT devices are attached to a different node, all of their VFs will end 240 # up under that node too and memory needs to be available there for the tests. 241 export HUGE_EVEN_ALLOC=yes 242 fi 243 244 MAKE="make" 245 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 246 if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then 247 export DRIVER_OVERRIDE=igb_uio 248 # Building kernel modules requires root privileges 249 MAKE="sudo $MAKE" 250 fi 251elif [ "$(uname -s)" = "FreeBSD" ]; then 252 MAKE="gmake" 253 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 254 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 255 HUGEMEM=${HUGEMEM:-2048} 256elif [ "$(uname -s)" = "Windows" ]; then 257 MAKE="make" 258 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 259 # Keep the default 2GB for Windows. 260 HUGEMEM=${HUGEMEM:-2048} 261else 262 echo "Unknown OS \"$(uname -s)\"" 263 exit 1 264fi 265 266export HUGEMEM=$HUGEMEM 267 268if [ -z "$output_dir" ]; then 269 mkdir -p "$rootdir/../output" 270 export output_dir="$rootdir/../output" 271fi 272 273TEST_MODE= 274for i in "$@"; do 275 case "$i" in 276 --iso) 277 TEST_MODE=iso 278 ;; 279 --transport=*) 280 TEST_TRANSPORT="${i#*=}" 281 ;; 282 --sock=*) 283 TEST_SOCK="${i#*=}" 284 ;; 285 esac 286done 287 288# start rpc.py coprocess if it's not started yet 289if [[ -z $RPC_PIPE_PID ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 290 # Include list to all known plugins we use in the tests 291 PYTHONPATH+=":$rootdir/test/rpc_plugins" 292 coproc RPC_PIPE { PYTHONPATH="$PYTHONPATH" "$rootdir/scripts/rpc.py" --server; } 293 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 294 # all descriptors will automatically close together with this bash 295 # process, this will make rpc.py stop reading and exit gracefully 296fi 297 298function set_test_storage() { 299 [[ -v testdir ]] || return 0 300 301 local requested_size=$1 # bytes 302 local mount target_dir 303 304 local -A mounts fss sizes avails uses 305 local source fs size avail mount use 306 307 local storage_fallback storage_candidates 308 309 storage_fallback=$(mktemp -udt spdk.XXXXXX) 310 storage_candidates=( 311 "$testdir" 312 "$storage_fallback/tests/${testdir##*/}" 313 "$storage_fallback" 314 ) 315 316 if [[ -n $ADD_TEST_STORAGE ]]; then 317 # List of dirs|mounts separated by whitespaces 318 storage_candidates+=($ADD_TEST_STORAGE) 319 fi 320 321 if [[ -n $DEDICATED_TEST_STORAGE ]]; then 322 # Single, dedicated dir|mount 323 storage_candidates=("$DEDICATED_TEST_STORAGE") 324 fi 325 326 mkdir -p "${storage_candidates[@]}" 327 328 # add some headroom - 64M 329 requested_size=$((requested_size + (64 << 20))) 330 331 while read -r source fs size use avail _ mount; do 332 mounts["$mount"]=$source fss["$mount"]=$fs 333 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 334 uses["$mount"]=$((use * 1024)) 335 done < <(df -T | grep -v Filesystem) 336 337 printf '* Looking for test storage...\n' >&2 338 339 local target_space new_size 340 for target_dir in "${storage_candidates[@]}"; do 341 # FreeBSD's df is lacking the --output arg 342 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 343 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 344 345 target_space=${avails["$mount"]} 346 if ((target_space == 0 || target_space < requested_size)); then 347 continue 348 fi 349 if ((target_space >= requested_size)); then 350 # For in-memory fs, and / make sure our requested size won't fill most of the space. 351 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 352 new_size=$((uses["$mount"] + requested_size)) 353 if ((new_size * 100 / sizes["$mount"] > 95)); then 354 continue 355 fi 356 fi 357 fi 358 export SPDK_TEST_STORAGE=$target_dir 359 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 360 return 0 361 done 362 printf '* Test storage is not available\n' 363 return 1 364} 365 366function get_config_params() { 367 xtrace_disable 368 config_params='--enable-debug --enable-werror' 369 370 # for options with dependencies but no test flag, set them here 371 if [ -f /usr/include/infiniband/verbs.h ]; then 372 config_params+=' --with-rdma' 373 fi 374 375 if [ $SPDK_TEST_USDT -eq 1 ]; then 376 config_params+=" --with-usdt" 377 fi 378 379 if [ $(uname -s) == "FreeBSD" ]; then 380 intel="hw.model: Intel" 381 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 382 else 383 intel="GenuineIntel" 384 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 385 fi 386 if [[ "$cpu_vendor" != *"$intel"* ]]; then 387 config_params+=" --without-idxd" 388 else 389 config_params+=" --with-idxd" 390 fi 391 392 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 393 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 394 fi 395 396 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 397 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 398 fi 399 400 if [ -d /usr/include/iscsi ]; then 401 [[ $(< /usr/include/iscsi/iscsi.h) =~ "define LIBISCSI_API_VERSION ("([0-9]+)")" ]] \ 402 && libiscsi_version=${BASH_REMATCH[1]} 403 if ((libiscsi_version >= 20150621)); then 404 config_params+=' --with-iscsi-initiator' 405 fi 406 fi 407 408 if [[ $SPDK_TEST_UNITTEST -eq 0 && \ 409 $SPDK_TEST_SCANBUILD -eq 0 && \ 410 $SPDK_TEST_AUTOBUILD -eq 0 ]]; then 411 config_params+=' --disable-unit-tests' 412 fi 413 414 if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then 415 config_params+=' --with-nvme-cuse' 416 fi 417 418 # for options with both dependencies and a test flag, set them here 419 if [ -f /usr/include/libpmemblk.h ] && [ $SPDK_TEST_PMDK -eq 1 ]; then 420 config_params+=' --with-pmdk' 421 fi 422 423 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_REDUCE -eq 1 ]; then 424 if ge "$(nasm --version | awk '{print $3}')" 2.14 && [[ $SPDK_TEST_ISAL -eq 1 ]]; then 425 config_params+=' --with-reduce' 426 fi 427 fi 428 429 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 430 config_params+=' --with-rbd' 431 fi 432 433 # for options with no required dependencies, just test flags, set them here 434 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 435 config_params+=' --with-crypto' 436 fi 437 438 if [ $SPDK_TEST_OCF -eq 1 ]; then 439 config_params+=" --with-ocf" 440 fi 441 442 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 443 config_params+=' --enable-ubsan' 444 fi 445 446 if [ $SPDK_RUN_ASAN -eq 1 ]; then 447 config_params+=' --enable-asan' 448 fi 449 450 if [ "$(uname -s)" = "Linux" ]; then 451 config_params+=' --enable-coverage' 452 fi 453 454 if [ $SPDK_TEST_ISAL -eq 0 ]; then 455 config_params+=' --without-isal' 456 fi 457 458 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 459 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 460 config_params+=' --with-fuse' 461 fi 462 fi 463 464 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 465 config_params+=' --with-raid5f' 466 fi 467 468 if [ $SPDK_TEST_VFIOUSER -eq 1 ] || [ $SPDK_TEST_VFIOUSER_QEMU -eq 1 ]; then 469 config_params+=' --with-vfio-user' 470 fi 471 472 # Check whether liburing library header exists 473 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 474 config_params+=' --with-uring' 475 fi 476 477 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 478 config_params+=" --with-dpdk=$SPDK_RUN_EXTERNAL_DPDK" 479 fi 480 481 echo "$config_params" 482 xtrace_restore 483} 484 485function rpc_cmd() { 486 xtrace_disable 487 local rsp rc=1 488 local stdin cmd cmds_number=0 status_number=0 status 489 490 if (($#)); then 491 cmds_number=1 492 echo "$@" >&$RPC_PIPE_INPUT 493 elif [[ ! -t 0 ]]; then 494 mapfile -t stdin <&0 495 cmds_number=${#stdin[@]} 496 printf '%s\n' "${stdin[@]}" >&$RPC_PIPE_INPUT 497 else 498 return 0 499 fi 500 501 while read -t 5 -ru $RPC_PIPE_OUTPUT rsp; do 502 if [[ $rsp == "**STATUS="* ]]; then 503 status[${rsp#*=}]=$rsp 504 if ((++status_number == cmds_number)); then 505 break 506 fi 507 continue 508 fi 509 echo "$rsp" 510 done 511 512 rc=${!status[*]} 513 xtrace_restore 514 [[ $rc == 0 ]] 515} 516 517function rpc_cmd_simple_data_json() { 518 519 local elems="$1[@]" elem 520 local -gA jq_out=() 521 local jq val 522 523 local lvs=( 524 "uuid" 525 "name" 526 "base_bdev" 527 "total_data_clusters" 528 "free_clusters" 529 "block_size" 530 "cluster_size" 531 ) 532 533 local bdev=( 534 "name" 535 "aliases[0]" 536 "block_size" 537 "num_blocks" 538 "uuid" 539 "product_name" 540 ) 541 542 [[ -v $elems ]] || return 1 543 544 for elem in "${!elems}"; do 545 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 546 done 547 jq+=',"\n"' 548 549 shift 550 while read -r elem val; do 551 jq_out["$elem"]=$val 552 done < <(rpc_cmd "$@" | jq -jr "$jq") 553 ((${#jq_out[@]} > 0)) || return 1 554} 555 556function NOT() { 557 local es=0 558 559 "$@" || es=$? 560 561 # Logic looks like so: 562 # - return false if command exit successfully 563 # - return false if command exit after receiving a core signal (FIXME: or any signal?) 564 # - return true if command exit with an error 565 566 # This naively assumes that the process doesn't exit with > 128 on its own. 567 if ((es > 128)); then 568 es=$((es & ~128)) 569 case "$es" in 570 3) ;& # SIGQUIT 571 4) ;& # SIGILL 572 6) ;& # SIGABRT 573 8) ;& # SIGFPE 574 9) ;& # SIGKILL 575 11) es=0 ;; # SIGSEGV 576 *) es=1 ;; 577 esac 578 elif [[ -n $EXIT_STATUS ]] && ((es != EXIT_STATUS)); then 579 es=0 580 fi 581 582 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 583 ((!es == 0)) 584} 585 586function timing() { 587 direction="$1" 588 testname="$2" 589 590 now=$(date +%s) 591 592 if [ "$direction" = "enter" ]; then 593 export timing_stack="${timing_stack};${now}" 594 export test_stack="${test_stack};${testname}" 595 else 596 touch "$output_dir/timing.txt" 597 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 598 599 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 600 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 601 602 elapsed=$((now - start_time - child_time)) 603 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 604 605 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 606 fi 607} 608 609function timing_enter() { 610 xtrace_disable 611 timing "enter" "$1" 612 xtrace_restore 613} 614 615function timing_exit() { 616 xtrace_disable 617 timing "exit" "$1" 618 xtrace_restore 619} 620 621function timing_finish() { 622 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 623 if [ -x "$flamegraph" ]; then 624 "$flamegraph" \ 625 --title 'Build Timing' \ 626 --nametype 'Step:' \ 627 --countname seconds \ 628 $output_dir/timing.txt \ 629 > $output_dir/timing.svg 630 fi 631} 632 633function create_test_list() { 634 xtrace_disable 635 # First search all scripts in main SPDK directory. 636 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 637 # Follow up with search in test directory recursively. 638 completion+=$(grep -rshI --include="*.sh" --exclude="*autotest_common.sh" -e "run_test " $rootdir/test) 639 printf "%s" "$completion" | grep -v "#" \ 640 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 641 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 642 xtrace_restore 643} 644 645function gdb_attach() { 646 gdb -q --batch \ 647 -ex 'handle SIGHUP nostop pass' \ 648 -ex 'handle SIGQUIT nostop pass' \ 649 -ex 'handle SIGPIPE nostop pass' \ 650 -ex 'handle SIGALRM nostop pass' \ 651 -ex 'handle SIGTERM nostop pass' \ 652 -ex 'handle SIGUSR1 nostop pass' \ 653 -ex 'handle SIGUSR2 nostop pass' \ 654 -ex 'handle SIGCHLD nostop pass' \ 655 -ex 'set print thread-events off' \ 656 -ex 'cont' \ 657 -ex 'thread apply all bt' \ 658 -ex 'quit' \ 659 --tty=/dev/stdout \ 660 -p $1 661} 662 663function process_core() { 664 # Note that this always was racy as we can't really sync with the kernel 665 # to see if there's any core queued up for writing. We could check if 666 # collector is running and wait for it explicitly, but it doesn't seem 667 # to be worth the effort. So assume that if we are being called via 668 # trap, as in, when some error has occurred, wait up to 10s for any 669 # potential cores. If we are called just for cleanup at the very end, 670 # don't wait since all the tests ended successfully, hence having any 671 # critical cores lying around is unlikely. 672 ((autotest_es != 0)) && sleep 10 673 674 local coredumps core 675 676 shopt -s nullglob 677 coredumps=("$output_dir/coredumps/"*.bt.txt) 678 shopt -u nullglob 679 680 ((${#coredumps[@]} > 0)) || return 0 681 chmod -R a+r "$output_dir/coredumps" 682 683 for core in "${coredumps[@]}"; do 684 cat <<- BT 685 ##### CORE BT ${core##*/} ##### 686 687 $(<"$core") 688 689 -- 690 BT 691 done 692 return 1 693} 694 695function process_shm() { 696 type=$1 697 id=$2 698 if [ "$type" = "--pid" ]; then 699 id="pid${id}" 700 fi 701 702 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 703 704 if [[ -z $shm_files ]]; then 705 echo "SHM File for specified PID or shared memory id: ${id} not found!" 706 return 1 707 fi 708 for n in $shm_files; do 709 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 710 done 711 return 0 712} 713 714function waitforlisten() { 715 # $1 = process pid 716 if [ -z "$1" ]; then 717 exit 1 718 fi 719 720 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 721 722 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 723 # turn off trace for this loop 724 xtrace_disable 725 local ret=0 726 local i 727 for ((i = 100; i != 0; i--)); do 728 # if the process is no longer running, then exit the script 729 # since it means the application crashed 730 if ! kill -s 0 $1; then 731 echo "ERROR: process (pid: $1) is no longer running" 732 ret=1 733 break 734 fi 735 736 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 737 break 738 fi 739 740 sleep 0.5 741 done 742 743 xtrace_restore 744 if ((i == 0)); then 745 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 746 ret=1 747 fi 748 return $ret 749} 750 751function waitfornbd() { 752 local nbd_name=$1 753 local i 754 755 for ((i = 1; i <= 20; i++)); do 756 if grep -q -w $nbd_name /proc/partitions; then 757 break 758 else 759 sleep 0.1 760 fi 761 done 762 763 # The nbd device is now recognized as a block device, but there can be 764 # a small delay before we can start I/O to that block device. So loop 765 # here trying to read the first block of the nbd block device to a temp 766 # file. Note that dd returns success when reading an empty file, so we 767 # need to check the size of the output file instead. 768 for ((i = 1; i <= 20; i++)); do 769 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 770 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 771 rm -f "$SPDK_TEST_STORAGE/nbdtest" 772 if [ "$size" != "0" ]; then 773 return 0 774 else 775 sleep 0.1 776 fi 777 done 778 779 return 1 780} 781 782function waitforbdev() { 783 local bdev_name=$1 784 local bdev_timeout=$2 785 local i 786 [[ -z $bdev_timeout ]] && bdev_timeout=2000 # ms 787 788 $rpc_py bdev_wait_for_examine 789 790 if $rpc_py bdev_get_bdevs -b $bdev_name -t $bdev_timeout; then 791 return 0 792 fi 793 794 return 1 795} 796 797function make_filesystem() { 798 local fstype=$1 799 local dev_name=$2 800 local i=0 801 local force 802 803 if [ $fstype = ext4 ]; then 804 force=-F 805 else 806 force=-f 807 fi 808 809 while ! mkfs.${fstype} $force ${dev_name}; do 810 if [ $i -ge 15 ]; then 811 return 1 812 fi 813 i=$((i + 1)) 814 sleep 1 815 done 816 817 return 0 818} 819 820function killprocess() { 821 # $1 = process pid 822 if [ -z "$1" ]; then 823 return 1 824 fi 825 826 if kill -0 $1; then 827 if [ $(uname) = Linux ]; then 828 process_name=$(ps --no-headers -o comm= $1) 829 else 830 process_name=$(ps -c -o command $1 | tail -1) 831 fi 832 if [ "$process_name" = "sudo" ]; then 833 # kill the child process, which is the actual app 834 # (assume $1 has just one child) 835 local child 836 child="$(pgrep -P $1)" 837 echo "killing process with pid $child" 838 kill $child 839 else 840 echo "killing process with pid $1" 841 kill $1 842 fi 843 844 # wait for the process regardless if its the dummy sudo one 845 # or the actual app - it should terminate anyway 846 wait $1 847 else 848 # the process is not there anymore 849 echo "Process with pid $1 is not found" 850 fi 851} 852 853function iscsicleanup() { 854 echo "Cleaning up iSCSI connection" 855 iscsiadm -m node --logout || true 856 iscsiadm -m node -o delete || true 857 rm -rf /var/lib/iscsi/nodes/* 858} 859 860function stop_iscsi_service() { 861 if cat /etc/*-release | grep Ubuntu; then 862 service open-iscsi stop 863 else 864 service iscsid stop 865 fi 866} 867 868function start_iscsi_service() { 869 if cat /etc/*-release | grep Ubuntu; then 870 service open-iscsi start 871 else 872 service iscsid start 873 fi 874} 875 876function rbd_setup() { 877 # $1 = monitor ip address 878 # $2 = name of the namespace 879 if [ -z "$1" ]; then 880 echo "No monitor IP address provided for ceph" 881 exit 1 882 fi 883 if [ -n "$2" ]; then 884 if ip netns list | grep "$2"; then 885 NS_CMD="ip netns exec $2" 886 else 887 echo "No namespace $2 exists" 888 exit 1 889 fi 890 fi 891 892 if hash ceph; then 893 export PG_NUM=128 894 export RBD_POOL=rbd 895 export RBD_NAME=foo 896 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 897 $NS_CMD $rootdir/scripts/ceph/start.sh $1 898 899 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 900 $NS_CMD rbd create $RBD_NAME --size 1000 901 fi 902} 903 904function rbd_cleanup() { 905 if hash ceph; then 906 $rootdir/scripts/ceph/stop.sh || true 907 rm -f /var/tmp/ceph_raw.img 908 fi 909} 910 911function _start_stub() { 912 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 913 # but ASLR can still be unreliable in some cases. 914 # We will reenable it again after multi-process testing is complete in kill_stub(). 915 # Save current setting so it can be restored upon calling kill_stub(). 916 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 917 echo 0 > /proc/sys/kernel/randomize_va_space 918 $rootdir/test/app/stub/stub $1 & 919 stubpid=$! 920 echo Waiting for stub to ready for secondary processes... 921 while ! [ -e /var/run/spdk_stub0 ]; do 922 # If stub dies while we wait, bail 923 [[ -e /proc/$stubpid ]] || return 1 924 sleep 1s 925 done 926 echo done. 927} 928 929function start_stub() { 930 if ! _start_stub "$@"; then 931 echo "stub failed" >&2 932 return 1 933 fi 934} 935 936function kill_stub() { 937 if [[ -e /proc/$stubpid ]]; then 938 kill $1 $stubpid 939 wait $stubpid 940 fi 2> /dev/null || : 941 rm -f /var/run/spdk_stub0 942 # Re-enable ASLR now that we are done with multi-process testing 943 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 944 # randomizing and is the default on all recent Linux kernels 945 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 946} 947 948function run_test() { 949 if [ $# -le 1 ]; then 950 echo "Not enough parameters" 951 echo "usage: run_test test_name test_script [script_params]" 952 exit 1 953 fi 954 955 xtrace_disable 956 local test_name="$1" 957 shift 958 959 if [ -n "$test_domain" ]; then 960 export test_domain="${test_domain}.${test_name}" 961 else 962 export test_domain="$test_name" 963 fi 964 965 timing_enter $test_name 966 echo "************************************" 967 echo "START TEST $test_name" 968 echo "************************************" 969 xtrace_restore 970 time "$@" 971 xtrace_disable 972 echo "************************************" 973 echo "END TEST $test_name" 974 echo "************************************" 975 timing_exit $test_name 976 977 export test_domain=${test_domain%"$test_name"} 978 if [ -n "$test_domain" ]; then 979 export test_domain=${test_domain%?} 980 fi 981 982 if [ -z "$test_domain" ]; then 983 echo "top_level $test_name" >> $output_dir/test_completions.txt 984 else 985 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 986 fi 987 xtrace_restore 988} 989 990function skip_run_test_with_warning() { 991 echo "WARNING: $1" 992 echo "Test run may fail if run with autorun.sh" 993 echo "Please check your $rootdir/test/common/skipped_tests.txt" 994} 995 996function print_backtrace() { 997 # if errexit is not enabled, don't print a backtrace 998 [[ "$-" =~ e ]] || return 0 999 1000 local args=("${BASH_ARGV[@]}") 1001 1002 xtrace_disable 1003 # Reset IFS in case we were called from an environment where it was modified 1004 IFS=" "$'\t'$'\n' 1005 echo "========== Backtrace start: ==========" 1006 echo "" 1007 for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do 1008 local func="${FUNCNAME[$i]}" 1009 local line_nr="${BASH_LINENO[$((i - 1))]}" 1010 local src="${BASH_SOURCE[$i]}" 1011 local bt="" cmdline=() 1012 1013 if [[ -f $src ]]; then 1014 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 1015 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 1016 fi 1017 1018 # If extdebug set the BASH_ARGC[i], try to fetch all the args 1019 if ((BASH_ARGC[i] > 0)); then 1020 # Use argc as index to reverse the stack 1021 local argc=${BASH_ARGC[i]} arg 1022 for arg in "${args[@]::BASH_ARGC[i]}"; do 1023 cmdline[argc--]="[\"$arg\"]" 1024 done 1025 args=("${args[@]:BASH_ARGC[i]}") 1026 fi 1027 1028 echo "in $src:$line_nr -> $func($( 1029 IFS="," 1030 printf '%s\n' "${cmdline[*]:-[]}" 1031 ))" 1032 echo " ..." 1033 echo "${bt:-backtrace unavailable}" 1034 echo " ..." 1035 done 1036 echo "" 1037 echo "========== Backtrace end ==========" 1038 xtrace_restore 1039 return 0 1040} 1041 1042function waitforserial() { 1043 local i=0 1044 local nvme_device_counter=1 nvme_devices=0 1045 if [[ -n "$2" ]]; then 1046 nvme_device_counter=$2 1047 fi 1048 1049 # Wait initially for min 4s to make sure all devices are ready for use. It seems 1050 # that we may be racing with a kernel where in some cases immediate disconnect may 1051 # leave dangling subsystem with no-op block devices which can't be used nor removed 1052 # (unless kernel is rebooted) and which start to negatively affect all the tests. 1053 sleep 4 1054 while ((i++ <= 15)); do 1055 nvme_devices=$(lsblk -l -o NAME,SERIAL | grep -c "$1") 1056 ((nvme_devices == nvme_device_counter)) && return 0 1057 if ((nvme_devices > nvme_device_counter)); then 1058 echo "$nvme_device_counter device(s) expected, found $nvme_devices" >&2 1059 fi 1060 echo "Waiting for devices" 1061 sleep 1 1062 done 1063 return 1 1064} 1065 1066function waitforserial_disconnect() { 1067 local i=0 1068 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1069 [ $i -lt 15 ] || break 1070 i=$((i + 1)) 1071 echo "Waiting for disconnect devices" 1072 sleep 1 1073 done 1074 1075 if lsblk -l -o NAME,SERIAL | grep -q -w $1; then 1076 return 1 1077 fi 1078 1079 return 0 1080} 1081 1082function waitforblk() { 1083 local i=0 1084 while ! lsblk -l -o NAME | grep -q -w $1; do 1085 [ $i -lt 15 ] || break 1086 i=$((i + 1)) 1087 sleep 1 1088 done 1089 1090 if ! lsblk -l -o NAME | grep -q -w $1; then 1091 return 1 1092 fi 1093 1094 return 0 1095} 1096 1097function waitforblk_disconnect() { 1098 local i=0 1099 while lsblk -l -o NAME | grep -q -w $1; do 1100 [ $i -lt 15 ] || break 1101 i=$((i + 1)) 1102 sleep 1 1103 done 1104 1105 if lsblk -l -o NAME | grep -q -w $1; then 1106 return 1 1107 fi 1108 1109 return 0 1110} 1111 1112function waitforfile() { 1113 local i=0 1114 while [ ! -e $1 ]; do 1115 [ $i -lt 200 ] || break 1116 i=$((i + 1)) 1117 sleep 0.1 1118 done 1119 1120 if [ ! -e $1 ]; then 1121 return 1 1122 fi 1123 1124 return 0 1125} 1126 1127function fio_config_gen() { 1128 local config_file=$1 1129 local workload=$2 1130 local bdev_type=$3 1131 local env_context=$4 1132 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1133 1134 if [ -e "$config_file" ]; then 1135 echo "Configuration File Already Exists!: $config_file" 1136 return 1 1137 fi 1138 1139 if [ -z "$workload" ]; then 1140 workload=randrw 1141 fi 1142 1143 if [ -n "$env_context" ]; then 1144 env_context="env_context=$env_context" 1145 fi 1146 1147 touch $1 1148 1149 cat > $1 << EOL 1150[global] 1151thread=1 1152$env_context 1153group_reporting=1 1154direct=1 1155norandommap=1 1156percentile_list=50:99:99.9:99.99:99.999 1157time_based=1 1158ramp_time=0 1159EOL 1160 1161 if [ "$workload" == "verify" ]; then 1162 cat <<- EOL >> $config_file 1163 verify=sha1 1164 verify_backlog=1024 1165 rw=randwrite 1166 EOL 1167 1168 # To avoid potential data race issue due to the AIO device 1169 # flush mechanism, add the flag to serialize the writes. 1170 # This is to fix the intermittent IO failure issue of #935 1171 if [ "$bdev_type" == "AIO" ]; then 1172 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1173 echo "serialize_overlap=1" >> $config_file 1174 fi 1175 fi 1176 elif [ "$workload" == "trim" ]; then 1177 echo "rw=trimwrite" >> $config_file 1178 else 1179 echo "rw=$workload" >> $config_file 1180 fi 1181} 1182 1183function fio_plugin() { 1184 # Setup fio binary cmd line 1185 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1186 # gcc and clang uses different sanitizer libraries 1187 local sanitizers=(libasan libclang_rt.asan) 1188 local plugin=$1 1189 shift 1190 1191 local asan_lib= 1192 for sanitizer in "${sanitizers[@]}"; do 1193 asan_lib=$(ldd $plugin | grep $sanitizer | awk '{print $3}') 1194 if [[ -n "$asan_lib" ]]; then 1195 break 1196 fi 1197 done 1198 1199 # Preload the sanitizer library to fio if fio_plugin was compiled with it 1200 LD_PRELOAD="$asan_lib $plugin" "$fio_dir"/fio "$@" 1201} 1202 1203function fio_bdev() { 1204 fio_plugin "$rootdir/build/fio/spdk_bdev" "$@" 1205} 1206 1207function fio_nvme() { 1208 fio_plugin "$rootdir/build/fio/spdk_nvme" "$@" 1209} 1210 1211function get_lvs_free_mb() { 1212 local lvs_uuid=$1 1213 local lvs_info 1214 local fc 1215 local cs 1216 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1217 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1218 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1219 1220 # Change to MB's 1221 free_mb=$((fc * cs / 1024 / 1024)) 1222 echo "$free_mb" 1223} 1224 1225function get_bdev_size() { 1226 local bdev_name=$1 1227 local bdev_info 1228 local bs 1229 local nb 1230 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1231 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1232 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1233 1234 # Change to MB's 1235 bdev_size=$((bs * nb / 1024 / 1024)) 1236 echo "$bdev_size" 1237} 1238 1239function autotest_cleanup() { 1240 local autotest_es=$? 1241 xtrace_disable 1242 1243 # catch any stray core files and kill all remaining SPDK processes. Update 1244 # autotest_es in case autotest reported success but cores and/or processes 1245 # were left behind regardless. 1246 1247 process_core || autotest_es=1 1248 reap_spdk_processes || autotest_es=1 1249 1250 $rootdir/scripts/setup.sh reset 1251 $rootdir/scripts/setup.sh cleanup 1252 if [ $(uname -s) = "Linux" ]; then 1253 if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then 1254 [[ -e /sys/module/igb_uio ]] && rmmod igb_uio 1255 else 1256 modprobe -r uio_pci_generic 1257 fi 1258 fi 1259 rm -rf "$asan_suppression_file" 1260 if [[ -n $old_core_pattern ]]; then 1261 echo "$old_core_pattern" > /proc/sys/kernel/core_pattern 1262 fi 1263 if [[ -e /proc/$udevadm_pid/status ]]; then 1264 kill "$udevadm_pid" || : 1265 fi 1266 1267 shopt -s nullglob 1268 local storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 1269 shopt -u nullglob 1270 1271 if ((${#storage_fallback_purge[@]} > 0)); then 1272 rm -rf "${storage_fallback_purge[@]}" 1273 fi 1274 1275 xtrace_restore 1276 return $autotest_es 1277} 1278 1279function freebsd_update_contigmem_mod() { 1280 if [ $(uname) = FreeBSD ]; then 1281 kldunload contigmem.ko || true 1282 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 1283 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/ 1284 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/ 1285 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/ 1286 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/ 1287 else 1288 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1289 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1290 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1291 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1292 fi 1293 fi 1294} 1295 1296function get_nvme_name_from_bdf() { 1297 blkname=() 1298 1299 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1300 if [ -z "$nvme_devs" ]; then 1301 return 1302 fi 1303 for dev in $nvme_devs; do 1304 link_name=$(readlink /sys/block/$dev/device/device) || true 1305 if [ -z "$link_name" ]; then 1306 link_name=$(readlink /sys/block/$dev/device) 1307 fi 1308 bdf=$(basename "$link_name") 1309 if [ "$bdf" = "$1" ]; then 1310 blkname+=($dev) 1311 fi 1312 done 1313 1314 printf '%s\n' "${blkname[@]}" 1315} 1316 1317function get_nvme_ctrlr_from_bdf() { 1318 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1319 if [[ -z "$bdf_sysfs_path" ]]; then 1320 return 1321 fi 1322 1323 printf '%s\n' "$(basename $bdf_sysfs_path)" 1324} 1325 1326# Get BDF addresses of all NVMe drives currently attached to 1327# uio-pci-generic or vfio-pci 1328function get_nvme_bdfs() { 1329 xtrace_disable 1330 bdfs=$(jq -r .config[].params.traddr <<< $($rootdir/scripts/gen_nvme.sh)) 1331 if [[ -z $bdfs ]]; then 1332 echo "No devices to test on!" 1333 exit 1 1334 fi 1335 echo "$bdfs" 1336 xtrace_restore 1337} 1338 1339# Same as function above, but just get the first disks BDF address 1340function get_first_nvme_bdf() { 1341 head -1 <<< "$(get_nvme_bdfs)" 1342} 1343 1344function nvme_namespace_revert() { 1345 $rootdir/scripts/setup.sh 1346 sleep 1 1347 bdfs=$(get_nvme_bdfs) 1348 1349 $rootdir/scripts/setup.sh reset 1350 1351 for bdf in $bdfs; do 1352 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1353 if [[ -z "$nvme_ctrlr" ]]; then 1354 continue 1355 fi 1356 1357 # Check Optional Admin Command Support for Namespace Management 1358 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1359 oacs_ns_manage=$((oacs & 0x8)) 1360 1361 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1362 # This assumes every NVMe controller contains single namespace, 1363 # encompassing Total NVM Capacity and formatted as 512 block size. 1364 # 512 block size is needed for test/vhost/vhost_boot.sh to 1365 # successfully run. 1366 1367 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1368 if [[ "$unvmcap" -eq 0 ]]; then 1369 # All available space already used 1370 continue 1371 fi 1372 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1373 blksize=512 1374 1375 size=$((tnvmcap / blksize)) 1376 1377 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c 0 || true 1378 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1379 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1380 nvme attach-ns ${nvme_ctrlr} -n 1 -c 0 1381 nvme reset ${nvme_ctrlr} 1382 waitforfile "${nvme_ctrlr}n1" 1383 fi 1384 done 1385} 1386 1387# Get BDFs based on device ID, such as 0x0a54 1388function get_nvme_bdfs_by_id() { 1389 local bdfs=() 1390 1391 for bdf in $(get_nvme_bdfs); do 1392 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1393 if [[ "$device" == "$1" ]]; then 1394 bdfs+=($bdf) 1395 fi 1396 done 1397 1398 printf '%s\n' "${bdfs[@]}" 1399} 1400 1401function opal_revert_cleanup() { 1402 # The OPAL CI tests is only used for P4510 devices. 1403 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1404 if [[ -z ${bdfs[0]} ]]; then 1405 return 0 1406 fi 1407 1408 $SPDK_BIN_DIR/spdk_tgt & 1409 spdk_tgt_pid=$! 1410 waitforlisten $spdk_tgt_pid 1411 1412 bdf_id=0 1413 for bdf in "${bdfs[@]}"; do 1414 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme"${bdf_id} -t "pcie" -a ${bdf} 1415 # Ignore if this fails. 1416 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b "nvme"${bdf_id} -p test || true 1417 ((++bdf_id)) 1418 done 1419 1420 killprocess $spdk_tgt_pid 1421} 1422 1423function pap() { 1424 while read -r file; do 1425 cat <<- FILE 1426 --- $file --- 1427 $(<"$file") 1428 --- $file --- 1429 FILE 1430 rm -f "$file" 1431 done < <(find "$@" -type f | sort -u) 1432} 1433 1434function get_proc_paths() { 1435 local procs proc 1436 if [[ $(uname -s) == Linux ]]; then 1437 for proc in /proc/[0-9]*; do 1438 [[ -e $proc/exe ]] || continue 1439 procs[${proc##*/}]=$(readlink -f "$proc/exe") 1440 done 1441 elif [[ $(uname -s) == FreeBSD ]]; then 1442 while read -r proc _ _ path; do 1443 [[ -e $path ]] || continue 1444 procs[proc]=$path 1445 done < <(procstat -ab) 1446 fi 1447 1448 for proc in "${!procs[@]}"; do 1449 echo "$proc" "${procs[proc]}" 1450 done 1451} 1452 1453is_exec_file() { [[ -f $1 && $(file "$1") =~ ELF.+executable ]]; } 1454 1455function reap_spdk_processes() { 1456 local bins bin 1457 local misc_bins 1458 1459 while read -r bin; do 1460 is_exec_file "$bin" && misc_bins+=("$bin") 1461 done < <(find "$rootdir"/test/{app,env,event} -type f) 1462 1463 mapfile -t bins < <(readlink -f "$SPDK_BIN_DIR/"* "$SPDK_EXAMPLE_DIR/"* "${misc_bins[@]}") 1464 1465 local spdk_pid spdk_pids path 1466 while read -r spdk_pid path; do 1467 if [[ ${bins[*]/$path/} != "${bins[*]}" ]]; then 1468 echo "$path is still up ($spdk_pid), killing" 1469 spdk_pids[spdk_pid]=$path 1470 fi 1471 done < <(get_proc_paths) 1472 1473 ((${#spdk_pids[@]} > 0)) || return 0 1474 1475 kill -SIGTERM "${!spdk_pids[@]}" 2> /dev/null || : 1476 # Wait a bit and then use the stick 1477 sleep 2 1478 kill -SIGKILL "${!spdk_pids[@]}" 2> /dev/null || : 1479 1480 return 1 1481} 1482 1483function is_block_zoned() { 1484 local device=$1 1485 1486 [[ -e /sys/block/$device/queue/zoned ]] || return 1 1487 [[ $(< "/sys/block/$device/queue/zoned") != none ]] 1488} 1489 1490function get_zoned_devs() { 1491 local -gA zoned_devs=() 1492 local nvme bdf 1493 1494 for nvme in /sys/block/nvme*; do 1495 if is_block_zoned "${nvme##*/}"; then 1496 zoned_devs["${nvme##*/}"]=$(< "$nvme/device/address") 1497 fi 1498 done 1499} 1500 1501# Define temp storage for all the tests. Look for 2GB at minimum 1502set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1503 1504set -o errtrace 1505shopt -s extdebug 1506trap "trap - ERR; print_backtrace >&2" ERR 1507 1508PS4=' \t -- ${BASH_SOURCE#${BASH_SOURCE%/*/*}/}@${LINENO} -- \$ ' 1509if $SPDK_AUTOTEST_X; then 1510 # explicitly enable xtraces, overriding any tracking information. 1511 unset XTRACE_DISABLED 1512 unset XTRACE_NESTING_LEVEL 1513 xtrace_fd 1514 xtrace_enable 1515else 1516 xtrace_restore 1517fi 1518
