1#!/usr/bin/env bash 2 3rpc_py=rpc_cmd 4 5function xtrace_fd() { 6 if [[ -n $BASH_XTRACEFD && -e /proc/self/fd/$BASH_XTRACEFD ]]; then 7 # Close it first to make sure it's sane 8 exec {BASH_XTRACEFD}>&- 9 fi 10 exec {BASH_XTRACEFD}>&2 11 12 set -x 13} 14 15function xtrace_disable() { 16 if [ "$XTRACE_DISABLED" != "yes" ]; then 17 PREV_BASH_OPTS="$-" 18 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 19 XTRACE_DISABLED="yes" 20 fi 21 set +x 22 elif [ -z $XTRACE_NESTING_LEVEL ]; then 23 XTRACE_NESTING_LEVEL=1 24 else 25 XTRACE_NESTING_LEVEL=$((++XTRACE_NESTING_LEVEL)) 26 fi 27} 28 29xtrace_disable 30set -e 31shopt -s expand_aliases 32 33if [[ -e $rootdir/test/common/build_config.sh ]]; then 34 source "$rootdir/test/common/build_config.sh" 35elif [[ -e $rootdir/mk/config.mk ]]; then 36 build_config=$(< "$rootdir/mk/config.mk") 37 source <(echo "${build_config//\?=/=}") 38else 39 source "$rootdir/CONFIG" 40fi 41 42# Source scripts after the config so that the definitions are available. 43source "$rootdir/test/common/applications.sh" 44source "$rootdir/scripts/common.sh" 45 46# Dummy function to be called after restoring xtrace just so that it appears in the 47# xtrace log. This way we can consistently track when xtrace is enabled/disabled. 48function xtrace_enable() { 49 # We have to do something inside a function in bash, and calling any command 50 # (even `:`) will produce an xtrace entry, so we just define another function. 51 function xtrace_dummy() { :; } 52} 53 54# Keep it as alias to avoid xtrace_enable backtrace always pointing to xtrace_restore. 55# xtrace_enable will appear as called directly from the user script, from the same line 56# that "called" xtrace_restore. 57alias xtrace_restore='if [ -z $XTRACE_NESTING_LEVEL ]; then 58 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 59 XTRACE_DISABLED="no"; PREV_BASH_OPTS=""; set -x; xtrace_enable; 60 fi 61else 62 XTRACE_NESTING_LEVEL=$((--XTRACE_NESTING_LEVEL)); 63 if [ $XTRACE_NESTING_LEVEL -eq "0" ]; then 64 unset XTRACE_NESTING_LEVEL 65 fi 66fi' 67 68: ${RUN_NIGHTLY:=0} 69export RUN_NIGHTLY 70 71# Set defaults for missing test config options 72: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 73export SPDK_AUTOTEST_DEBUG_APPS 74: ${SPDK_RUN_VALGRIND=0} 75export SPDK_RUN_VALGRIND 76: ${SPDK_RUN_FUNCTIONAL_TEST=0} 77export SPDK_RUN_FUNCTIONAL_TEST 78: ${SPDK_TEST_UNITTEST=0} 79export SPDK_TEST_UNITTEST 80: ${SPDK_TEST_AUTOBUILD=0} 81export SPDK_TEST_AUTOBUILD 82: ${SPDK_TEST_RELEASE_BUILD=0} 83export SPDK_TEST_RELEASE_BUILD 84: ${SPDK_TEST_ISAL=0} 85export SPDK_TEST_ISAL 86: ${SPDK_TEST_ISCSI=0} 87export SPDK_TEST_ISCSI 88: ${SPDK_TEST_ISCSI_INITIATOR=0} 89export SPDK_TEST_ISCSI_INITIATOR 90: ${SPDK_TEST_NVME=0} 91export SPDK_TEST_NVME 92: ${SPDK_TEST_NVME_PMR=0} 93export SPDK_TEST_NVME_PMR 94: ${SPDK_TEST_NVME_SCC=0} 95export SPDK_TEST_NVME_SCC 96: ${SPDK_TEST_NVME_BP=0} 97export SPDK_TEST_NVME_BP 98: ${SPDK_TEST_NVME_CLI=0} 99export SPDK_TEST_NVME_CLI 100: ${SPDK_TEST_NVME_CUSE=0} 101export SPDK_TEST_NVME_CUSE 102: ${SPDK_TEST_NVMF=0} 103export SPDK_TEST_NVMF 104: ${SPDK_TEST_VFIOUSER=0} 105export SPDK_TEST_VFIOUSER 106: ${SPDK_TEST_VFIOUSER_QEMU=0} 107export SPDK_TEST_VFIOUSER_QEMU 108: ${SPDK_TEST_FUZZER=0} 109export SPDK_TEST_FUZZER 110: ${SPDK_TEST_FUZZER_SHORT=0} 111export SPDK_TEST_FUZZER_SHORT 112: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 113export SPDK_TEST_NVMF_TRANSPORT 114: ${SPDK_TEST_RBD=0} 115export SPDK_TEST_RBD 116: ${SPDK_TEST_VHOST=0} 117export SPDK_TEST_VHOST 118: ${SPDK_TEST_BLOCKDEV=0} 119export SPDK_TEST_BLOCKDEV 120: ${SPDK_TEST_IOAT=0} 121export SPDK_TEST_IOAT 122: ${SPDK_TEST_BLOBFS=0} 123export SPDK_TEST_BLOBFS 124: ${SPDK_TEST_VHOST_INIT=0} 125export SPDK_TEST_VHOST_INIT 126: ${SPDK_TEST_PMDK=0} 127export SPDK_TEST_PMDK 128: ${SPDK_TEST_LVOL=0} 129export SPDK_TEST_LVOL 130: ${SPDK_TEST_REDUCE=0} 131export SPDK_TEST_REDUCE 132: ${SPDK_RUN_ASAN=0} 133export SPDK_RUN_ASAN 134: ${SPDK_RUN_UBSAN=0} 135export SPDK_RUN_UBSAN 136: ${SPDK_RUN_EXTERNAL_DPDK=""} 137export SPDK_RUN_EXTERNAL_DPDK 138: ${SPDK_RUN_NON_ROOT=0} 139export SPDK_RUN_NON_ROOT 140: ${SPDK_TEST_CRYPTO=0} 141export SPDK_TEST_CRYPTO 142: ${SPDK_TEST_FTL=0} 143export SPDK_TEST_FTL 144: ${SPDK_TEST_OCF=0} 145export SPDK_TEST_OCF 146: ${SPDK_TEST_VMD=0} 147export SPDK_TEST_VMD 148: ${SPDK_TEST_OPAL=0} 149export SPDK_TEST_OPAL 150: ${SPDK_TEST_NATIVE_DPDK} 151export SPDK_TEST_NATIVE_DPDK 152: ${SPDK_AUTOTEST_X=true} 153export SPDK_AUTOTEST_X 154: ${SPDK_TEST_RAID5=0} 155export SPDK_TEST_RAID5 156: ${SPDK_TEST_URING=0} 157export SPDK_TEST_URING 158: ${SPDK_TEST_USDT=0} 159export SPDK_TEST_USDT 160: ${SPDK_TEST_USE_IGB_UIO:=0} 161export SPDK_TEST_USE_IGB_UIO 162: ${SPDK_TEST_SCHEDULER:=0} 163export SPDK_TEST_SCHEDULER 164: ${SPDK_TEST_SCANBUILD:=0} 165export SPDK_TEST_SCANBUILD 166: ${SPDK_TEST_NVMF_NICS:=} 167export SPDK_TEST_NVMF_NICS 168: ${SPDK_TEST_SMA=0} 169export SPDK_TEST_SMA 170: ${SPDK_TEST_DAOS=0} 171export SPDK_TEST_DAOS 172: ${SPDK_TEST_XNVME:=0} 173export SPDK_TEST_XNVME 174 175# always test with SPDK shared objects. 176export SPDK_LIB_DIR="$rootdir/build/lib" 177export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib" 178export VFIO_LIB_DIR="$rootdir/build/libvfio-user/usr/local/lib" 179export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR:$VFIO_LIB_DIR 180 181# Tell setup.sh to wait for block devices upon each reset 182export PCI_BLOCK_SYNC_ON_RESET=yes 183 184# Export PYTHONPATH with addition of RPC framework. New scripts can be created 185# specific use cases for tests. 186export PYTHONPATH=$PYTHONPATH:$rootdir/python 187 188# Don't create Python .pyc files. When running with sudo these will be 189# created with root ownership and can cause problems when cleaning the repository. 190export PYTHONDONTWRITEBYTECODE=1 191 192# Export flag to skip the known bug that exists in librados 193# Bug is reported on ceph bug tracker with number 24078 194export ASAN_OPTIONS=new_delete_type_mismatch=0:disable_coredump=0 195export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1:disable_coredump=0' 196 197# Export LeakSanitizer option to use suppression file in order to prevent false positives 198# and known leaks in external executables or libraries from showing up. 199asan_suppression_file="/var/tmp/asan_suppression_file" 200rm -rf "$asan_suppression_file" 2> /dev/null || sudo rm -rf "$asan_suppression_file" 201cat << EOL >> "$asan_suppression_file" 202# ASAN has some bugs around thread_local variables. We have a destructor in place 203# to free the thread contexts, but ASAN complains about the leak before those 204# destructors have a chance to run. So suppress this one specific leak using 205# LSAN_OPTIONS. 206leak:spdk_fs_alloc_thread_ctx 207 208# Suppress known leaks in fio project 209leak:$CONFIG_FIO_SOURCE_DIR/parse.c 210leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 211leak:$CONFIG_FIO_SOURCE_DIR/init.c 212leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 213leak:fio_memalign 214leak:spdk_fio_io_u_init 215# Suppress leaks in gperftools-libs from fio 216leak:libtcmalloc_minimal.so 217 218# Suppress leaks in libiscsi 219leak:libiscsi.so 220EOL 221 222# Suppress leaks in libfuse3 223echo "leak:libfuse3.so" >> "$asan_suppression_file" 224 225export LSAN_OPTIONS=suppressions="$asan_suppression_file" 226 227export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 228 229if [ -z "$DEPENDENCY_DIR" ]; then 230 export DEPENDENCY_DIR=$HOME/spdk_dependencies 231else 232 export DEPENDENCY_DIR 233fi 234 235# Export location of where all the SPDK binaries are 236export SPDK_BIN_DIR="$rootdir/build/bin" 237export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 238 239# for vhost, vfio-user tests 240export QEMU_BIN=${QEMU_BIN:-} 241export VFIO_QEMU_BIN=${VFIO_QEMU_BIN:-} 242 243export AR_TOOL=$rootdir/scripts/ar-xnvme-fixer 244 245# pass our valgrind desire on to unittest.sh 246if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 247 export valgrind='' 248fi 249 250if [ "$(uname -s)" = "Linux" ]; then 251 HUGEMEM=${HUGEMEM:-4096} 252 export CLEAR_HUGE=yes 253 if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_REDUCE -eq 1 ]]; then 254 # Make sure that memory is distributed across all NUMA nodes - by default, all goes to 255 # node0, but if QAT devices are attached to a different node, all of their VFs will end 256 # up under that node too and memory needs to be available there for the tests. 257 export HUGE_EVEN_ALLOC=yes 258 fi 259 260 MAKE="make" 261 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 262 if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then 263 export DRIVER_OVERRIDE=igb_uio 264 # Building kernel modules requires root privileges 265 MAKE="sudo $MAKE" 266 fi 267elif [ "$(uname -s)" = "FreeBSD" ]; then 268 MAKE="gmake" 269 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 270 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 271 HUGEMEM=${HUGEMEM:-2048} 272elif [ "$(uname -s)" = "Windows" ]; then 273 MAKE="make" 274 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 275 # Keep the default 2GB for Windows. 276 HUGEMEM=${HUGEMEM:-2048} 277else 278 echo "Unknown OS \"$(uname -s)\"" 279 exit 1 280fi 281 282export HUGEMEM=$HUGEMEM 283 284if [ -z "$output_dir" ]; then 285 mkdir -p "$rootdir/../output" 286 export output_dir="$rootdir/../output" 287fi 288 289TEST_MODE= 290for i in "$@"; do 291 case "$i" in 292 --iso) 293 TEST_MODE=iso 294 ;; 295 --transport=*) 296 TEST_TRANSPORT="${i#*=}" 297 ;; 298 --sock=*) 299 TEST_SOCK="${i#*=}" 300 ;; 301 esac 302done 303 304# start rpc.py coprocess if it's not started yet 305if [[ -z $RPC_PIPE_PID ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 306 # Include list to all known plugins we use in the tests 307 PYTHONPATH+=":$rootdir/test/rpc_plugins" 308 coproc RPC_PIPE { PYTHONPATH="$PYTHONPATH" "$rootdir/scripts/rpc.py" --server; } 309 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 310 # all descriptors will automatically close together with this bash 311 # process, this will make rpc.py stop reading and exit gracefully 312fi 313 314function set_test_storage() { 315 [[ -v testdir ]] || return 0 316 317 local requested_size=$1 # bytes 318 local mount target_dir 319 320 local -A mounts fss sizes avails uses 321 local source fs size avail mount use 322 323 local storage_fallback storage_candidates 324 325 storage_fallback=$(mktemp -udt spdk.XXXXXX) 326 storage_candidates=( 327 "$testdir" 328 "$storage_fallback/tests/${testdir##*/}" 329 "$storage_fallback" 330 ) 331 332 if [[ -n $ADD_TEST_STORAGE ]]; then 333 # List of dirs|mounts separated by whitespaces 334 storage_candidates+=($ADD_TEST_STORAGE) 335 fi 336 337 if [[ -n $DEDICATED_TEST_STORAGE ]]; then 338 # Single, dedicated dir|mount 339 storage_candidates=("$DEDICATED_TEST_STORAGE") 340 fi 341 342 mkdir -p "${storage_candidates[@]}" 343 344 # add some headroom - 64M 345 requested_size=$((requested_size + (64 << 20))) 346 347 while read -r source fs size use avail _ mount; do 348 mounts["$mount"]=$source fss["$mount"]=$fs 349 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 350 uses["$mount"]=$((use * 1024)) 351 done < <(df -T | grep -v Filesystem) 352 353 printf '* Looking for test storage...\n' >&2 354 355 local target_space new_size 356 for target_dir in "${storage_candidates[@]}"; do 357 # FreeBSD's df is lacking the --output arg 358 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 359 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 360 361 target_space=${avails["$mount"]} 362 if ((target_space == 0 || target_space < requested_size)); then 363 continue 364 fi 365 if ((target_space >= requested_size)); then 366 # For in-memory fs, and / make sure our requested size won't fill most of the space. 367 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 368 new_size=$((uses["$mount"] + requested_size)) 369 if ((new_size * 100 / sizes["$mount"] > 95)); then 370 continue 371 fi 372 fi 373 fi 374 export SPDK_TEST_STORAGE=$target_dir 375 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 376 return 0 377 done 378 printf '* Test storage is not available\n' 379 return 1 380} 381 382function get_config_params() { 383 xtrace_disable 384 config_params='--enable-debug --enable-werror' 385 386 # for options with dependencies but no test flag, set them here 387 if [ -f /usr/include/infiniband/verbs.h ]; then 388 config_params+=' --with-rdma' 389 fi 390 391 if [ $SPDK_TEST_USDT -eq 1 ]; then 392 config_params+=" --with-usdt" 393 fi 394 395 if [ $(uname -s) == "FreeBSD" ]; then 396 intel="hw.model: Intel" 397 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 398 else 399 intel="GenuineIntel" 400 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 401 fi 402 if [[ "$cpu_vendor" != *"$intel"* ]]; then 403 config_params+=" --without-idxd" 404 else 405 config_params+=" --with-idxd" 406 fi 407 408 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 409 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 410 fi 411 412 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 413 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 414 fi 415 416 if [ -d /usr/include/iscsi ]; then 417 [[ $(< /usr/include/iscsi/iscsi.h) =~ "define LIBISCSI_API_VERSION ("([0-9]+)")" ]] \ 418 && libiscsi_version=${BASH_REMATCH[1]} 419 if ((libiscsi_version >= 20150621)); then 420 config_params+=' --with-iscsi-initiator' 421 fi 422 fi 423 424 if [[ $SPDK_TEST_UNITTEST -eq 0 && \ 425 $SPDK_TEST_SCANBUILD -eq 0 && \ 426 $SPDK_TEST_AUTOBUILD -eq 0 ]]; then 427 config_params+=' --disable-unit-tests' 428 fi 429 430 if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then 431 config_params+=' --with-nvme-cuse' 432 fi 433 434 # for options with both dependencies and a test flag, set them here 435 if [ -f /usr/include/libpmemblk.h ] && [ $SPDK_TEST_PMDK -eq 1 ]; then 436 config_params+=' --with-pmdk' 437 fi 438 439 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_REDUCE -eq 1 ]; then 440 if ge "$(nasm --version | awk '{print $3}')" 2.14 && [[ $SPDK_TEST_ISAL -eq 1 ]]; then 441 config_params+=' --with-reduce' 442 fi 443 fi 444 445 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 446 config_params+=' --with-rbd' 447 fi 448 449 # for options with no required dependencies, just test flags, set them here 450 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 451 config_params+=' --with-crypto' 452 fi 453 454 if [ $SPDK_TEST_OCF -eq 1 ]; then 455 config_params+=" --with-ocf" 456 fi 457 458 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 459 config_params+=' --enable-ubsan' 460 fi 461 462 if [ $SPDK_RUN_ASAN -eq 1 ]; then 463 config_params+=' --enable-asan' 464 fi 465 466 if [ "$(uname -s)" = "Linux" ]; then 467 config_params+=' --enable-coverage' 468 fi 469 470 if [ $SPDK_TEST_ISAL -eq 0 ]; then 471 config_params+=' --without-isal' 472 fi 473 474 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 475 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 476 config_params+=' --with-fuse' 477 fi 478 fi 479 480 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 481 config_params+=' --with-raid5f' 482 fi 483 484 if [ $SPDK_TEST_VFIOUSER -eq 1 ] || [ $SPDK_TEST_VFIOUSER_QEMU -eq 1 ]; then 485 config_params+=' --with-vfio-user' 486 fi 487 488 # Check whether liburing library header exists 489 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 490 config_params+=' --with-uring' 491 fi 492 493 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 494 config_params+=" --with-dpdk=$SPDK_RUN_EXTERNAL_DPDK" 495 fi 496 497 if [[ $SPDK_TEST_SMA -eq 1 ]]; then 498 config_params+=' --with-sma' 499 fi 500 501 if [ -f /usr/include/daos.h ] && [ $SPDK_TEST_DAOS -eq 1 ]; then 502 config_params+=' --with-daos' 503 fi 504 505 # Make the xnvme module available for the tests 506 if [[ $SPDK_TEST_XNVME -eq 1 ]]; then 507 config_params+=' --with-xnvme' 508 fi 509 510 echo "$config_params" 511 xtrace_restore 512} 513 514function rpc_cmd() { 515 xtrace_disable 516 local rsp rc=1 517 local stdin cmd cmds_number=0 status_number=0 status 518 519 if (($#)); then 520 cmds_number=1 521 echo "$@" >&$RPC_PIPE_INPUT 522 elif [[ ! -t 0 ]]; then 523 mapfile -t stdin <&0 524 cmds_number=${#stdin[@]} 525 printf '%s\n' "${stdin[@]}" >&$RPC_PIPE_INPUT 526 else 527 return 0 528 fi 529 530 while read -t 5 -ru $RPC_PIPE_OUTPUT rsp; do 531 if [[ $rsp == "**STATUS="* ]]; then 532 status[${rsp#*=}]=$rsp 533 if ((++status_number == cmds_number)); then 534 break 535 fi 536 continue 537 fi 538 echo "$rsp" 539 done 540 541 rc=${!status[*]} 542 xtrace_restore 543 [[ $rc == 0 ]] 544} 545 546function rpc_cmd_simple_data_json() { 547 548 local elems="$1[@]" elem 549 local -gA jq_out=() 550 local jq val 551 552 local lvs=( 553 "uuid" 554 "name" 555 "base_bdev" 556 "total_data_clusters" 557 "free_clusters" 558 "block_size" 559 "cluster_size" 560 ) 561 562 local bdev=( 563 "name" 564 "aliases[0]" 565 "block_size" 566 "num_blocks" 567 "uuid" 568 "product_name" 569 ) 570 571 [[ -v $elems ]] || return 1 572 573 for elem in "${!elems}"; do 574 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 575 done 576 jq+=',"\n"' 577 578 shift 579 while read -r elem val; do 580 jq_out["$elem"]=$val 581 done < <(rpc_cmd "$@" | jq -jr "$jq") 582 ((${#jq_out[@]} > 0)) || return 1 583} 584 585function NOT() { 586 local es=0 587 588 "$@" || es=$? 589 590 # Logic looks like so: 591 # - return false if command exit successfully 592 # - return false if command exit after receiving a core signal (FIXME: or any signal?) 593 # - return true if command exit with an error 594 595 # This naively assumes that the process doesn't exit with > 128 on its own. 596 if ((es > 128)); then 597 es=$((es & ~128)) 598 case "$es" in 599 3) ;& # SIGQUIT 600 4) ;& # SIGILL 601 6) ;& # SIGABRT 602 8) ;& # SIGFPE 603 9) ;& # SIGKILL 604 11) es=0 ;; # SIGSEGV 605 *) es=1 ;; 606 esac 607 elif [[ -n $EXIT_STATUS ]] && ((es != EXIT_STATUS)); then 608 es=0 609 fi 610 611 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 612 ((!es == 0)) 613} 614 615function timing() { 616 direction="$1" 617 testname="$2" 618 619 now=$(date +%s) 620 621 if [ "$direction" = "enter" ]; then 622 export timing_stack="${timing_stack};${now}" 623 export test_stack="${test_stack};${testname}" 624 else 625 touch "$output_dir/timing.txt" 626 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 627 628 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 629 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 630 631 elapsed=$((now - start_time - child_time)) 632 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 633 634 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 635 fi 636} 637 638function timing_enter() { 639 xtrace_disable 640 timing "enter" "$1" 641 xtrace_restore 642} 643 644function timing_exit() { 645 xtrace_disable 646 timing "exit" "$1" 647 xtrace_restore 648} 649 650function timing_finish() { 651 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 652 if [ -x "$flamegraph" ]; then 653 "$flamegraph" \ 654 --title 'Build Timing' \ 655 --nametype 'Step:' \ 656 --countname seconds \ 657 $output_dir/timing.txt \ 658 > $output_dir/timing.svg 659 fi 660} 661 662function create_test_list() { 663 xtrace_disable 664 # First search all scripts in main SPDK directory. 665 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 666 # Follow up with search in test directory recursively. 667 completion+=$(grep -rshI --include="*.sh" --exclude="*autotest_common.sh" -e "run_test " $rootdir/test) 668 printf "%s" "$completion" | grep -v "#" \ 669 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 670 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 671 xtrace_restore 672} 673 674function gdb_attach() { 675 gdb -q --batch \ 676 -ex 'handle SIGHUP nostop pass' \ 677 -ex 'handle SIGQUIT nostop pass' \ 678 -ex 'handle SIGPIPE nostop pass' \ 679 -ex 'handle SIGALRM nostop pass' \ 680 -ex 'handle SIGTERM nostop pass' \ 681 -ex 'handle SIGUSR1 nostop pass' \ 682 -ex 'handle SIGUSR2 nostop pass' \ 683 -ex 'handle SIGCHLD nostop pass' \ 684 -ex 'set print thread-events off' \ 685 -ex 'cont' \ 686 -ex 'thread apply all bt' \ 687 -ex 'quit' \ 688 --tty=/dev/stdout \ 689 -p $1 690} 691 692function process_core() { 693 # Note that this always was racy as we can't really sync with the kernel 694 # to see if there's any core queued up for writing. We could check if 695 # collector is running and wait for it explicitly, but it doesn't seem 696 # to be worth the effort. So assume that if we are being called via 697 # trap, as in, when some error has occurred, wait up to 10s for any 698 # potential cores. If we are called just for cleanup at the very end, 699 # don't wait since all the tests ended successfully, hence having any 700 # critical cores lying around is unlikely. 701 ((autotest_es != 0)) && sleep 10 702 703 local coredumps core 704 705 shopt -s nullglob 706 coredumps=("$output_dir/coredumps/"*.bt.txt) 707 shopt -u nullglob 708 709 ((${#coredumps[@]} > 0)) || return 0 710 chmod -R a+r "$output_dir/coredumps" 711 712 for core in "${coredumps[@]}"; do 713 cat <<- BT 714 ##### CORE BT ${core##*/} ##### 715 716 $(<"$core") 717 718 -- 719 BT 720 done 721 return 1 722} 723 724function process_shm() { 725 type=$1 726 id=$2 727 if [ "$type" = "--pid" ]; then 728 id="pid${id}" 729 fi 730 731 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 732 733 if [[ -z $shm_files ]]; then 734 echo "SHM File for specified PID or shared memory id: ${id} not found!" 735 return 1 736 fi 737 for n in $shm_files; do 738 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 739 done 740 return 0 741} 742 743function waitforlisten() { 744 # $1 = process pid 745 if [ -z "$1" ]; then 746 exit 1 747 fi 748 749 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 750 751 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 752 # turn off trace for this loop 753 xtrace_disable 754 local ret=0 755 local i 756 for ((i = 100; i != 0; i--)); do 757 # if the process is no longer running, then exit the script 758 # since it means the application crashed 759 if ! kill -s 0 $1; then 760 echo "ERROR: process (pid: $1) is no longer running" 761 ret=1 762 break 763 fi 764 765 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 766 break 767 fi 768 769 sleep 0.5 770 done 771 772 xtrace_restore 773 if ((i == 0)); then 774 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 775 ret=1 776 fi 777 return $ret 778} 779 780function waitfornbd() { 781 local nbd_name=$1 782 local i 783 784 for ((i = 1; i <= 20; i++)); do 785 if grep -q -w $nbd_name /proc/partitions; then 786 break 787 else 788 sleep 0.1 789 fi 790 done 791 792 # The nbd device is now recognized as a block device, but there can be 793 # a small delay before we can start I/O to that block device. So loop 794 # here trying to read the first block of the nbd block device to a temp 795 # file. Note that dd returns success when reading an empty file, so we 796 # need to check the size of the output file instead. 797 for ((i = 1; i <= 20; i++)); do 798 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 799 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 800 rm -f "$SPDK_TEST_STORAGE/nbdtest" 801 if [ "$size" != "0" ]; then 802 return 0 803 else 804 sleep 0.1 805 fi 806 done 807 808 return 1 809} 810 811function waitforbdev() { 812 local bdev_name=$1 813 local bdev_timeout=$2 814 local i 815 [[ -z $bdev_timeout ]] && bdev_timeout=2000 # ms 816 817 $rpc_py bdev_wait_for_examine 818 819 if $rpc_py bdev_get_bdevs -b $bdev_name -t $bdev_timeout; then 820 return 0 821 fi 822 823 return 1 824} 825 826function make_filesystem() { 827 local fstype=$1 828 local dev_name=$2 829 local i=0 830 local force 831 832 if [ $fstype = ext4 ]; then 833 force=-F 834 else 835 force=-f 836 fi 837 838 while ! mkfs.${fstype} $force ${dev_name}; do 839 if [ $i -ge 15 ]; then 840 return 1 841 fi 842 i=$((i + 1)) 843 sleep 1 844 done 845 846 return 0 847} 848 849function killprocess() { 850 # $1 = process pid 851 if [ -z "$1" ]; then 852 return 1 853 fi 854 855 if kill -0 $1; then 856 if [ $(uname) = Linux ]; then 857 process_name=$(ps --no-headers -o comm= $1) 858 else 859 process_name=$(ps -c -o command $1 | tail -1) 860 fi 861 if [ "$process_name" = "sudo" ]; then 862 # kill the child process, which is the actual app 863 # (assume $1 has just one child) 864 local child 865 child="$(pgrep -P $1)" 866 echo "killing process with pid $child" 867 kill $child 868 else 869 echo "killing process with pid $1" 870 kill $1 871 fi 872 873 # wait for the process regardless if its the dummy sudo one 874 # or the actual app - it should terminate anyway 875 wait $1 876 else 877 # the process is not there anymore 878 echo "Process with pid $1 is not found" 879 fi 880} 881 882function iscsicleanup() { 883 echo "Cleaning up iSCSI connection" 884 iscsiadm -m node --logout || true 885 iscsiadm -m node -o delete || true 886 rm -rf /var/lib/iscsi/nodes/* 887} 888 889function stop_iscsi_service() { 890 if cat /etc/*-release | grep Ubuntu; then 891 service open-iscsi stop 892 else 893 service iscsid stop 894 fi 895} 896 897function start_iscsi_service() { 898 if cat /etc/*-release | grep Ubuntu; then 899 service open-iscsi start 900 else 901 service iscsid start 902 fi 903} 904 905function rbd_setup() { 906 # $1 = monitor ip address 907 # $2 = name of the namespace 908 if [ -z "$1" ]; then 909 echo "No monitor IP address provided for ceph" 910 exit 1 911 fi 912 if [ -n "$2" ]; then 913 if ip netns list | grep "$2"; then 914 NS_CMD="ip netns exec $2" 915 else 916 echo "No namespace $2 exists" 917 exit 1 918 fi 919 fi 920 921 if hash ceph; then 922 export PG_NUM=128 923 export RBD_POOL=rbd 924 export RBD_NAME=foo 925 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 926 $NS_CMD $rootdir/scripts/ceph/start.sh $1 927 928 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 929 $NS_CMD rbd create $RBD_NAME --size 1000 930 fi 931} 932 933function rbd_cleanup() { 934 if hash ceph; then 935 $rootdir/scripts/ceph/stop.sh || true 936 rm -f /var/tmp/ceph_raw.img 937 fi 938} 939 940function daos_setup() { 941 # $1 = pool name 942 # $2 = cont name 943 if [ -z "$1" ]; then 944 echo "No pool name provided" 945 exit 1 946 fi 947 if [ -z "$2" ]; then 948 echo "No cont name provided" 949 exit 1 950 fi 951 952 dmg pool create --size=10G $1 || true 953 daos container create --type=POSIX --label=$2 $1 || true 954} 955 956function daos_cleanup() { 957 local pool=${1:-testpool} 958 local cont=${2:-testcont} 959 960 daos container destroy -f $pool $cont || true 961 sudo dmg pool destroy -f $pool || true 962} 963 964function _start_stub() { 965 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 966 # but ASLR can still be unreliable in some cases. 967 # We will reenable it again after multi-process testing is complete in kill_stub(). 968 # Save current setting so it can be restored upon calling kill_stub(). 969 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 970 echo 0 > /proc/sys/kernel/randomize_va_space 971 $rootdir/test/app/stub/stub $1 & 972 stubpid=$! 973 echo Waiting for stub to ready for secondary processes... 974 while ! [ -e /var/run/spdk_stub0 ]; do 975 # If stub dies while we wait, bail 976 [[ -e /proc/$stubpid ]] || return 1 977 sleep 1s 978 done 979 echo done. 980} 981 982function start_stub() { 983 if ! _start_stub "$@"; then 984 echo "stub failed" >&2 985 return 1 986 fi 987} 988 989function kill_stub() { 990 if [[ -e /proc/$stubpid ]]; then 991 kill $1 $stubpid 992 wait $stubpid 993 fi 2> /dev/null || : 994 rm -f /var/run/spdk_stub0 995 # Re-enable ASLR now that we are done with multi-process testing 996 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 997 # randomizing and is the default on all recent Linux kernels 998 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 999} 1000 1001function run_test() { 1002 if [ $# -le 1 ]; then 1003 echo "Not enough parameters" 1004 echo "usage: run_test test_name test_script [script_params]" 1005 exit 1 1006 fi 1007 1008 xtrace_disable 1009 local test_name="$1" 1010 shift 1011 1012 if [ -n "$test_domain" ]; then 1013 export test_domain="${test_domain}.${test_name}" 1014 else 1015 export test_domain="$test_name" 1016 fi 1017 1018 timing_enter $test_name 1019 echo "************************************" 1020 echo "START TEST $test_name" 1021 echo "************************************" 1022 xtrace_restore 1023 time "$@" 1024 xtrace_disable 1025 echo "************************************" 1026 echo "END TEST $test_name" 1027 echo "************************************" 1028 timing_exit $test_name 1029 1030 export test_domain=${test_domain%"$test_name"} 1031 if [ -n "$test_domain" ]; then 1032 export test_domain=${test_domain%?} 1033 fi 1034 1035 if [ -z "$test_domain" ]; then 1036 echo "top_level $test_name" >> $output_dir/test_completions.txt 1037 else 1038 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 1039 fi 1040 xtrace_restore 1041} 1042 1043function skip_run_test_with_warning() { 1044 echo "WARNING: $1" 1045 echo "Test run may fail if run with autorun.sh" 1046 echo "Please check your $rootdir/test/common/skipped_tests.txt" 1047} 1048 1049function print_backtrace() { 1050 # if errexit is not enabled, don't print a backtrace 1051 [[ "$-" =~ e ]] || return 0 1052 1053 local args=("${BASH_ARGV[@]}") 1054 1055 xtrace_disable 1056 # Reset IFS in case we were called from an environment where it was modified 1057 IFS=" "$'\t'$'\n' 1058 echo "========== Backtrace start: ==========" 1059 echo "" 1060 for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do 1061 local func="${FUNCNAME[$i]}" 1062 local line_nr="${BASH_LINENO[$((i - 1))]}" 1063 local src="${BASH_SOURCE[$i]}" 1064 local bt="" cmdline=() 1065 1066 if [[ -f $src ]]; then 1067 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 1068 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 1069 fi 1070 1071 # If extdebug set the BASH_ARGC[i], try to fetch all the args 1072 if ((BASH_ARGC[i] > 0)); then 1073 # Use argc as index to reverse the stack 1074 local argc=${BASH_ARGC[i]} arg 1075 for arg in "${args[@]::BASH_ARGC[i]}"; do 1076 cmdline[argc--]="[\"$arg\"]" 1077 done 1078 args=("${args[@]:BASH_ARGC[i]}") 1079 fi 1080 1081 echo "in $src:$line_nr -> $func($( 1082 IFS="," 1083 printf '%s\n' "${cmdline[*]:-[]}" 1084 ))" 1085 echo " ..." 1086 echo "${bt:-backtrace unavailable}" 1087 echo " ..." 1088 done 1089 echo "" 1090 echo "========== Backtrace end ==========" 1091 xtrace_restore 1092 return 0 1093} 1094 1095function waitforserial() { 1096 local i=0 1097 local nvme_device_counter=1 nvme_devices=0 1098 if [[ -n "$2" ]]; then 1099 nvme_device_counter=$2 1100 fi 1101 1102 # Wait initially for min 4s to make sure all devices are ready for use. It seems 1103 # that we may be racing with a kernel where in some cases immediate disconnect may 1104 # leave dangling subsystem with no-op block devices which can't be used nor removed 1105 # (unless kernel is rebooted) and which start to negatively affect all the tests. 1106 sleep 4 1107 while ((i++ <= 15)); do 1108 nvme_devices=$(lsblk -l -o NAME,SERIAL | grep -c "$1") 1109 ((nvme_devices == nvme_device_counter)) && return 0 1110 if ((nvme_devices > nvme_device_counter)); then 1111 echo "$nvme_device_counter device(s) expected, found $nvme_devices" >&2 1112 fi 1113 echo "Waiting for devices" 1114 sleep 1 1115 done 1116 return 1 1117} 1118 1119function waitforserial_disconnect() { 1120 local i=0 1121 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1122 [ $i -lt 15 ] || break 1123 i=$((i + 1)) 1124 echo "Waiting for disconnect devices" 1125 sleep 1 1126 done 1127 1128 if lsblk -l -o NAME,SERIAL | grep -q -w $1; then 1129 return 1 1130 fi 1131 1132 return 0 1133} 1134 1135function waitforblk() { 1136 local i=0 1137 while ! lsblk -l -o NAME | grep -q -w $1; do 1138 [ $i -lt 15 ] || break 1139 i=$((i + 1)) 1140 sleep 1 1141 done 1142 1143 if ! lsblk -l -o NAME | grep -q -w $1; then 1144 return 1 1145 fi 1146 1147 return 0 1148} 1149 1150function waitforblk_disconnect() { 1151 local i=0 1152 while lsblk -l -o NAME | grep -q -w $1; do 1153 [ $i -lt 15 ] || break 1154 i=$((i + 1)) 1155 sleep 1 1156 done 1157 1158 if lsblk -l -o NAME | grep -q -w $1; then 1159 return 1 1160 fi 1161 1162 return 0 1163} 1164 1165function waitforfile() { 1166 local i=0 1167 while [ ! -e $1 ]; do 1168 [ $i -lt 200 ] || break 1169 i=$((i + 1)) 1170 sleep 0.1 1171 done 1172 1173 if [ ! -e $1 ]; then 1174 return 1 1175 fi 1176 1177 return 0 1178} 1179 1180function fio_config_gen() { 1181 local config_file=$1 1182 local workload=$2 1183 local bdev_type=$3 1184 local env_context=$4 1185 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1186 1187 if [ -e "$config_file" ]; then 1188 echo "Configuration File Already Exists!: $config_file" 1189 return 1 1190 fi 1191 1192 if [ -z "$workload" ]; then 1193 workload=randrw 1194 fi 1195 1196 if [ -n "$env_context" ]; then 1197 env_context="env_context=$env_context" 1198 fi 1199 1200 touch $1 1201 1202 cat > $1 << EOL 1203[global] 1204thread=1 1205$env_context 1206group_reporting=1 1207direct=1 1208norandommap=1 1209percentile_list=50:99:99.9:99.99:99.999 1210time_based=1 1211ramp_time=0 1212EOL 1213 1214 if [ "$workload" == "verify" ]; then 1215 cat <<- EOL >> $config_file 1216 verify=sha1 1217 verify_backlog=1024 1218 rw=randwrite 1219 EOL 1220 1221 # To avoid potential data race issue due to the AIO device 1222 # flush mechanism, add the flag to serialize the writes. 1223 # This is to fix the intermittent IO failure issue of #935 1224 if [ "$bdev_type" == "AIO" ]; then 1225 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1226 echo "serialize_overlap=1" >> $config_file 1227 fi 1228 fi 1229 elif [ "$workload" == "trim" ]; then 1230 echo "rw=trimwrite" >> $config_file 1231 else 1232 echo "rw=$workload" >> $config_file 1233 fi 1234} 1235 1236function fio_plugin() { 1237 # Setup fio binary cmd line 1238 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1239 # gcc and clang uses different sanitizer libraries 1240 local sanitizers=(libasan libclang_rt.asan) 1241 local plugin=$1 1242 shift 1243 1244 local asan_lib= 1245 for sanitizer in "${sanitizers[@]}"; do 1246 asan_lib=$(ldd $plugin | grep $sanitizer | awk '{print $3}') 1247 if [[ -n "$asan_lib" ]]; then 1248 break 1249 fi 1250 done 1251 1252 # Preload the sanitizer library to fio if fio_plugin was compiled with it 1253 LD_PRELOAD="$asan_lib $plugin" "$fio_dir"/fio "$@" 1254} 1255 1256function fio_bdev() { 1257 fio_plugin "$rootdir/build/fio/spdk_bdev" "$@" 1258} 1259 1260function fio_nvme() { 1261 fio_plugin "$rootdir/build/fio/spdk_nvme" "$@" 1262} 1263 1264function get_lvs_free_mb() { 1265 local lvs_uuid=$1 1266 local lvs_info 1267 local fc 1268 local cs 1269 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1270 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1271 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1272 1273 # Change to MB's 1274 free_mb=$((fc * cs / 1024 / 1024)) 1275 echo "$free_mb" 1276} 1277 1278function get_bdev_size() { 1279 local bdev_name=$1 1280 local bdev_info 1281 local bs 1282 local nb 1283 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1284 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1285 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1286 1287 # Change to MB's 1288 bdev_size=$((bs * nb / 1024 / 1024)) 1289 echo "$bdev_size" 1290} 1291 1292function autotest_cleanup() { 1293 local autotest_es=$? 1294 xtrace_disable 1295 1296 # catch any stray core files and kill all remaining SPDK processes. Update 1297 # autotest_es in case autotest reported success but cores and/or processes 1298 # were left behind regardless. 1299 1300 process_core || autotest_es=1 1301 reap_spdk_processes || autotest_es=1 1302 1303 $rootdir/scripts/setup.sh reset 1304 $rootdir/scripts/setup.sh cleanup 1305 if [ $(uname -s) = "Linux" ]; then 1306 if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then 1307 [[ -e /sys/module/igb_uio ]] && rmmod igb_uio 1308 else 1309 modprobe -r uio_pci_generic 1310 fi 1311 fi 1312 rm -rf "$asan_suppression_file" 1313 if [[ -n $old_core_pattern ]]; then 1314 echo "$old_core_pattern" > /proc/sys/kernel/core_pattern 1315 fi 1316 if [[ -e /proc/$udevadm_pid/status ]]; then 1317 kill "$udevadm_pid" || : 1318 fi 1319 1320 shopt -s nullglob 1321 local storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 1322 shopt -u nullglob 1323 1324 if ((${#storage_fallback_purge[@]} > 0)); then 1325 rm -rf "${storage_fallback_purge[@]}" 1326 fi 1327 1328 if ((autotest_es)); then 1329 if [[ $(uname) == FreeBSD ]]; then 1330 ps aux 1331 elif [[ $(uname) == Linux ]]; then 1332 # Get more detailed view 1333 grep . /proc/[0-9]*/status 1334 # Dump some extra info into kernel log 1335 echo "######## Autotest Cleanup Dump ########" > /dev/kmsg 1336 # Show cpus backtraces 1337 echo l > /proc/sysrq-trigger 1338 # Show mem usage 1339 echo m > /proc/sysrq-trigger 1340 # show task states 1341 echo t > /proc/sysrq-trigger 1342 # show blocked tasks 1343 echo w > /proc/sysrq-trigger 1344 1345 fi > "$output_dir/proc_list.txt" 2>&1 || : 1346 fi 1347 xtrace_restore 1348 return $autotest_es 1349} 1350 1351function freebsd_update_contigmem_mod() { 1352 if [ $(uname) = FreeBSD ]; then 1353 kldunload contigmem.ko || true 1354 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 1355 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/ 1356 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/ 1357 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/ 1358 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/ 1359 else 1360 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1361 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1362 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1363 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1364 fi 1365 fi 1366} 1367 1368function get_nvme_name_from_bdf() { 1369 blkname=() 1370 1371 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1372 if [ -z "$nvme_devs" ]; then 1373 return 1374 fi 1375 for dev in $nvme_devs; do 1376 link_name=$(readlink /sys/block/$dev/device/device) || true 1377 if [ -z "$link_name" ]; then 1378 link_name=$(readlink /sys/block/$dev/device) 1379 fi 1380 bdf=$(basename "$link_name") 1381 if [ "$bdf" = "$1" ]; then 1382 blkname+=($dev) 1383 fi 1384 done 1385 1386 printf '%s\n' "${blkname[@]}" 1387} 1388 1389function get_nvme_ctrlr_from_bdf() { 1390 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1391 if [[ -z "$bdf_sysfs_path" ]]; then 1392 return 1393 fi 1394 1395 printf '%s\n' "$(basename $bdf_sysfs_path)" 1396} 1397 1398# Get BDF addresses of all NVMe drives currently attached to 1399# uio-pci-generic or vfio-pci 1400function get_nvme_bdfs() { 1401 xtrace_disable 1402 bdfs=$(jq -r .config[].params.traddr <<< $($rootdir/scripts/gen_nvme.sh)) 1403 if [[ -z $bdfs ]]; then 1404 echo "No devices to test on!" 1405 exit 1 1406 fi 1407 echo "$bdfs" 1408 xtrace_restore 1409} 1410 1411# Same as function above, but just get the first disks BDF address 1412function get_first_nvme_bdf() { 1413 head -1 <<< "$(get_nvme_bdfs)" 1414} 1415 1416function nvme_namespace_revert() { 1417 $rootdir/scripts/setup.sh 1418 sleep 1 1419 bdfs=$(get_nvme_bdfs) 1420 1421 $rootdir/scripts/setup.sh reset 1422 1423 for bdf in $bdfs; do 1424 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1425 if [[ -z "$nvme_ctrlr" ]]; then 1426 continue 1427 fi 1428 1429 # Check Optional Admin Command Support for Namespace Management 1430 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1431 oacs_ns_manage=$((oacs & 0x8)) 1432 1433 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1434 # This assumes every NVMe controller contains single namespace, 1435 # encompassing Total NVM Capacity and formatted as 512 block size. 1436 # 512 block size is needed for test/vhost/vhost_boot.sh to 1437 # successfully run. 1438 1439 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1440 if [[ "$unvmcap" -eq 0 ]]; then 1441 # All available space already used 1442 continue 1443 fi 1444 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1445 blksize=512 1446 1447 size=$((tnvmcap / blksize)) 1448 1449 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c 0 || true 1450 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1451 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1452 nvme attach-ns ${nvme_ctrlr} -n 1 -c 0 1453 nvme reset ${nvme_ctrlr} 1454 waitforfile "${nvme_ctrlr}n1" 1455 fi 1456 done 1457} 1458 1459# Get BDFs based on device ID, such as 0x0a54 1460function get_nvme_bdfs_by_id() { 1461 local bdfs=() 1462 1463 for bdf in $(get_nvme_bdfs); do 1464 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1465 if [[ "$device" == "$1" ]]; then 1466 bdfs+=($bdf) 1467 fi 1468 done 1469 1470 printf '%s\n' "${bdfs[@]}" 1471} 1472 1473function opal_revert_cleanup() { 1474 # The OPAL CI tests is only used for P4510 devices. 1475 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1476 if [[ -z ${bdfs[0]} ]]; then 1477 return 0 1478 fi 1479 1480 $SPDK_BIN_DIR/spdk_tgt & 1481 spdk_tgt_pid=$! 1482 waitforlisten $spdk_tgt_pid 1483 1484 bdf_id=0 1485 for bdf in "${bdfs[@]}"; do 1486 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme"${bdf_id} -t "pcie" -a ${bdf} 1487 # Ignore if this fails. 1488 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b "nvme"${bdf_id} -p test || true 1489 ((++bdf_id)) 1490 done 1491 1492 killprocess $spdk_tgt_pid 1493} 1494 1495function pap() { 1496 while read -r file; do 1497 cat <<- FILE 1498 --- $file --- 1499 $(<"$file") 1500 --- $file --- 1501 FILE 1502 rm -f "$file" 1503 done < <(find "$@" -type f | sort -u) 1504} 1505 1506function get_proc_paths() { 1507 local procs proc 1508 if [[ $(uname -s) == Linux ]]; then 1509 for proc in /proc/[0-9]*; do 1510 [[ -e $proc/exe ]] || continue 1511 procs[${proc##*/}]=$(readlink -f "$proc/exe") 1512 done 1513 elif [[ $(uname -s) == FreeBSD ]]; then 1514 while read -r proc _ _ path; do 1515 [[ -e $path ]] || continue 1516 procs[proc]=$path 1517 done < <(procstat -ab) 1518 fi 1519 1520 for proc in "${!procs[@]}"; do 1521 echo "$proc" "${procs[proc]}" 1522 done 1523} 1524 1525is_exec_file() { [[ -f $1 && $(file "$1") =~ ELF.+executable ]]; } 1526 1527function reap_spdk_processes() { 1528 local bins bin 1529 local misc_bins 1530 1531 while read -r bin; do 1532 is_exec_file "$bin" && misc_bins+=("$bin") 1533 done < <(find "$rootdir"/test/{app,env,event} -type f) 1534 1535 mapfile -t bins < <(readlink -f "$SPDK_BIN_DIR/"* "$SPDK_EXAMPLE_DIR/"* "${misc_bins[@]}") 1536 1537 local spdk_pid spdk_pids path 1538 while read -r spdk_pid path; do 1539 if [[ ${bins[*]/$path/} != "${bins[*]}" ]]; then 1540 echo "$path is still up ($spdk_pid), killing" 1541 spdk_pids[spdk_pid]=$path 1542 fi 1543 done < <(get_proc_paths) 1544 1545 ((${#spdk_pids[@]} > 0)) || return 0 1546 1547 kill -SIGTERM "${!spdk_pids[@]}" 2> /dev/null || : 1548 # Wait a bit and then use the stick 1549 sleep 2 1550 kill -SIGKILL "${!spdk_pids[@]}" 2> /dev/null || : 1551 1552 return 1 1553} 1554 1555function is_block_zoned() { 1556 local device=$1 1557 1558 [[ -e /sys/block/$device/queue/zoned ]] || return 1 1559 [[ $(< "/sys/block/$device/queue/zoned") != none ]] 1560} 1561 1562function get_zoned_devs() { 1563 local -gA zoned_devs=() 1564 local nvme bdf 1565 1566 for nvme in /sys/block/nvme*; do 1567 if is_block_zoned "${nvme##*/}"; then 1568 zoned_devs["${nvme##*/}"]=$(< "$nvme/device/address") 1569 fi 1570 done 1571} 1572 1573# Define temp storage for all the tests. Look for 2GB at minimum 1574set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1575 1576set -o errtrace 1577shopt -s extdebug 1578trap "trap - ERR; print_backtrace >&2" ERR 1579 1580PS4=' \t -- ${BASH_SOURCE#${BASH_SOURCE%/*/*}/}@${LINENO} -- \$ ' 1581if $SPDK_AUTOTEST_X; then 1582 # explicitly enable xtraces, overriding any tracking information. 1583 unset XTRACE_DISABLED 1584 unset XTRACE_NESTING_LEVEL 1585 xtrace_fd 1586 xtrace_enable 1587else 1588 xtrace_restore 1589fi 1590
