1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2015 Intel Corporation 4# All rights reserved. 5# 6rpc_py=rpc_cmd 7 8function xtrace_fd() { 9 if [[ -n $BASH_XTRACEFD && -e /proc/self/fd/$BASH_XTRACEFD ]]; then 10 # Close it first to make sure it's sane 11 exec {BASH_XTRACEFD}>&- 12 fi 13 exec {BASH_XTRACEFD}>&2 14 15 set -x 16} 17 18function xtrace_disable() { 19 if [ "$XTRACE_DISABLED" != "yes" ]; then 20 PREV_BASH_OPTS="$-" 21 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 22 XTRACE_DISABLED="yes" 23 fi 24 set +x 25 elif [ -z $XTRACE_NESTING_LEVEL ]; then 26 XTRACE_NESTING_LEVEL=1 27 else 28 XTRACE_NESTING_LEVEL=$((++XTRACE_NESTING_LEVEL)) 29 fi 30} 31 32xtrace_disable 33set -e 34shopt -s expand_aliases 35 36if [[ -e $rootdir/test/common/build_config.sh ]]; then 37 source "$rootdir/test/common/build_config.sh" 38elif [[ -e $rootdir/mk/config.mk ]]; then 39 build_config=$(< "$rootdir/mk/config.mk") 40 source <(echo "${build_config//\?=/=}") 41else 42 source "$rootdir/CONFIG" 43fi 44 45# Source scripts after the config so that the definitions are available. 46source "$rootdir/test/common/applications.sh" 47source "$rootdir/scripts/common.sh" 48 49# Dummy function to be called after restoring xtrace just so that it appears in the 50# xtrace log. This way we can consistently track when xtrace is enabled/disabled. 51function xtrace_enable() { 52 # We have to do something inside a function in bash, and calling any command 53 # (even `:`) will produce an xtrace entry, so we just define another function. 54 function xtrace_dummy() { :; } 55} 56 57# Keep it as alias to avoid xtrace_enable backtrace always pointing to xtrace_restore. 58# xtrace_enable will appear as called directly from the user script, from the same line 59# that "called" xtrace_restore. 60alias xtrace_restore='if [ -z $XTRACE_NESTING_LEVEL ]; then 61 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 62 XTRACE_DISABLED="no"; PREV_BASH_OPTS=""; set -x; xtrace_enable; 63 fi 64else 65 XTRACE_NESTING_LEVEL=$((--XTRACE_NESTING_LEVEL)); 66 if [ $XTRACE_NESTING_LEVEL -eq "0" ]; then 67 unset XTRACE_NESTING_LEVEL 68 fi 69fi' 70 71: ${RUN_NIGHTLY:=0} 72export RUN_NIGHTLY 73 74# Set defaults for missing test config options 75: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 76export SPDK_AUTOTEST_DEBUG_APPS 77: ${SPDK_RUN_VALGRIND=0} 78export SPDK_RUN_VALGRIND 79: ${SPDK_RUN_FUNCTIONAL_TEST=0} 80export SPDK_RUN_FUNCTIONAL_TEST 81: ${SPDK_TEST_UNITTEST=0} 82export SPDK_TEST_UNITTEST 83: ${SPDK_TEST_AUTOBUILD=""} 84export SPDK_TEST_AUTOBUILD 85: ${SPDK_TEST_RELEASE_BUILD=0} 86export SPDK_TEST_RELEASE_BUILD 87: ${SPDK_TEST_ISAL=0} 88export SPDK_TEST_ISAL 89: ${SPDK_TEST_ISCSI=0} 90export SPDK_TEST_ISCSI 91: ${SPDK_TEST_ISCSI_INITIATOR=0} 92export SPDK_TEST_ISCSI_INITIATOR 93: ${SPDK_TEST_NVME=0} 94export SPDK_TEST_NVME 95: ${SPDK_TEST_NVME_PMR=0} 96export SPDK_TEST_NVME_PMR 97: ${SPDK_TEST_NVME_SCC=0} 98export SPDK_TEST_NVME_SCC 99: ${SPDK_TEST_NVME_BP=0} 100export SPDK_TEST_NVME_BP 101: ${SPDK_TEST_NVME_CLI=0} 102export SPDK_TEST_NVME_CLI 103: ${SPDK_TEST_NVME_CUSE=0} 104export SPDK_TEST_NVME_CUSE 105: ${SPDK_TEST_NVMF=0} 106export SPDK_TEST_NVMF 107: ${SPDK_TEST_VFIOUSER=0} 108export SPDK_TEST_VFIOUSER 109: ${SPDK_TEST_VFIOUSER_QEMU=0} 110export SPDK_TEST_VFIOUSER_QEMU 111: ${SPDK_TEST_FUZZER=0} 112export SPDK_TEST_FUZZER 113: ${SPDK_TEST_FUZZER_SHORT=0} 114export SPDK_TEST_FUZZER_SHORT 115: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 116export SPDK_TEST_NVMF_TRANSPORT 117: ${SPDK_TEST_RBD=0} 118export SPDK_TEST_RBD 119: ${SPDK_TEST_VHOST=0} 120export SPDK_TEST_VHOST 121: ${SPDK_TEST_BLOCKDEV=0} 122export SPDK_TEST_BLOCKDEV 123: ${SPDK_TEST_IOAT=0} 124export SPDK_TEST_IOAT 125: ${SPDK_TEST_BLOBFS=0} 126export SPDK_TEST_BLOBFS 127: ${SPDK_TEST_VHOST_INIT=0} 128export SPDK_TEST_VHOST_INIT 129: ${SPDK_TEST_PMDK=0} 130export SPDK_TEST_PMDK 131: ${SPDK_TEST_LVOL=0} 132export SPDK_TEST_LVOL 133: ${SPDK_TEST_VBDEV_COMPRESS=0} 134export SPDK_TEST_VBDEV_COMPRESS 135: ${SPDK_RUN_ASAN=0} 136export SPDK_RUN_ASAN 137: ${SPDK_RUN_UBSAN=0} 138export SPDK_RUN_UBSAN 139: ${SPDK_RUN_EXTERNAL_DPDK=""} 140export SPDK_RUN_EXTERNAL_DPDK 141: ${SPDK_RUN_NON_ROOT=0} 142export SPDK_RUN_NON_ROOT 143: ${SPDK_TEST_CRYPTO=0} 144export SPDK_TEST_CRYPTO 145: ${SPDK_TEST_FTL=0} 146export SPDK_TEST_FTL 147: ${SPDK_TEST_OCF=0} 148export SPDK_TEST_OCF 149: ${SPDK_TEST_VMD=0} 150export SPDK_TEST_VMD 151: ${SPDK_TEST_OPAL=0} 152export SPDK_TEST_OPAL 153: ${SPDK_TEST_NATIVE_DPDK} 154export SPDK_TEST_NATIVE_DPDK 155: ${SPDK_AUTOTEST_X=true} 156export SPDK_AUTOTEST_X 157: ${SPDK_TEST_RAID5=0} 158export SPDK_TEST_RAID5 159: ${SPDK_TEST_URING=0} 160export SPDK_TEST_URING 161: ${SPDK_TEST_USDT=0} 162export SPDK_TEST_USDT 163: ${SPDK_TEST_USE_IGB_UIO:=0} 164export SPDK_TEST_USE_IGB_UIO 165: ${SPDK_TEST_SCHEDULER:=0} 166export SPDK_TEST_SCHEDULER 167: ${SPDK_TEST_SCANBUILD:=0} 168export SPDK_TEST_SCANBUILD 169: ${SPDK_TEST_NVMF_NICS:=} 170export SPDK_TEST_NVMF_NICS 171: ${SPDK_TEST_SMA=0} 172export SPDK_TEST_SMA 173: ${SPDK_TEST_DAOS=0} 174export SPDK_TEST_DAOS 175: ${SPDK_TEST_XNVME:=0} 176export SPDK_TEST_XNVME 177# Comma-separated list of fuzzer targets matching test/fuzz/llvm/$target 178: ${SPDK_TEST_FUZZER_TARGET:=} 179export SPDK_TEST_FUZZER_TARGET 180 181# always test with SPDK shared objects. 182export SPDK_LIB_DIR="$rootdir/build/lib" 183export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib" 184export VFIO_LIB_DIR="$rootdir/build/libvfio-user/usr/local/lib" 185export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR:$VFIO_LIB_DIR 186 187# Tell setup.sh to wait for block devices upon each reset 188export PCI_BLOCK_SYNC_ON_RESET=yes 189 190# Export PYTHONPATH with addition of RPC framework. New scripts can be created 191# specific use cases for tests. 192export PYTHONPATH=$PYTHONPATH:$rootdir/python 193 194# Don't create Python .pyc files. When running with sudo these will be 195# created with root ownership and can cause problems when cleaning the repository. 196export PYTHONDONTWRITEBYTECODE=1 197 198# Export flag to skip the known bug that exists in librados 199# Bug is reported on ceph bug tracker with number 24078 200export ASAN_OPTIONS=new_delete_type_mismatch=0:disable_coredump=0 201export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1:disable_coredump=0' 202 203# Export LeakSanitizer option to use suppression file in order to prevent false positives 204# and known leaks in external executables or libraries from showing up. 205asan_suppression_file="/var/tmp/asan_suppression_file" 206rm -rf "$asan_suppression_file" 2> /dev/null || sudo rm -rf "$asan_suppression_file" 207cat << EOL >> "$asan_suppression_file" 208# ASAN has some bugs around thread_local variables. We have a destructor in place 209# to free the thread contexts, but ASAN complains about the leak before those 210# destructors have a chance to run. So suppress this one specific leak using 211# LSAN_OPTIONS. 212leak:spdk_fs_alloc_thread_ctx 213 214# Suppress known leaks in fio project 215leak:$CONFIG_FIO_SOURCE_DIR/parse.c 216leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 217leak:$CONFIG_FIO_SOURCE_DIR/init.c 218leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 219leak:fio_memalign 220leak:spdk_fio_io_u_init 221# Suppress leaks in gperftools-libs from fio 222leak:libtcmalloc_minimal.so 223 224# Suppress leaks in libiscsi 225leak:libiscsi.so 226EOL 227 228# Suppress leaks in libfuse3 229echo "leak:libfuse3.so" >> "$asan_suppression_file" 230 231export LSAN_OPTIONS=suppressions="$asan_suppression_file" 232 233export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 234 235if [ -z "$DEPENDENCY_DIR" ]; then 236 export DEPENDENCY_DIR=$HOME/spdk_dependencies 237else 238 export DEPENDENCY_DIR 239fi 240 241# Export location of where all the SPDK binaries are 242export SPDK_BIN_DIR="$rootdir/build/bin" 243export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 244 245# for vhost, vfio-user tests 246export QEMU_BIN=${QEMU_BIN:-} 247export VFIO_QEMU_BIN=${VFIO_QEMU_BIN:-} 248 249export AR_TOOL=$rootdir/scripts/ar-xnvme-fixer 250 251# pass our valgrind desire on to unittest.sh 252if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 253 export valgrind='' 254else 255 # unset all DEBUGINFOD_* vars that may affect our valgrind instance 256 unset -v "${!DEBUGINFOD_@}" 257fi 258 259if [ "$(uname -s)" = "Linux" ]; then 260 HUGEMEM=${HUGEMEM:-4096} 261 export CLEAR_HUGE=yes 262 if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]]; then 263 # Make sure that memory is distributed across all NUMA nodes - by default, all goes to 264 # node0, but if QAT devices are attached to a different node, all of their VFs will end 265 # up under that node too and memory needs to be available there for the tests. 266 export HUGE_EVEN_ALLOC=yes 267 fi 268 269 MAKE="make" 270 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 271elif [ "$(uname -s)" = "FreeBSD" ]; then 272 MAKE="gmake" 273 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 274 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 275 HUGEMEM=${HUGEMEM:-2048} 276elif [ "$(uname -s)" = "Windows" ]; then 277 MAKE="make" 278 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 279 # Keep the default 2GB for Windows. 280 HUGEMEM=${HUGEMEM:-2048} 281else 282 echo "Unknown OS \"$(uname -s)\"" 283 exit 1 284fi 285 286export HUGEMEM=$HUGEMEM 287 288if [ -z "$output_dir" ]; then 289 mkdir -p "$rootdir/../output" 290 export output_dir="$rootdir/../output" 291fi 292 293TEST_MODE= 294for i in "$@"; do 295 case "$i" in 296 --iso) 297 TEST_MODE=iso 298 ;; 299 --transport=*) 300 TEST_TRANSPORT="${i#*=}" 301 ;; 302 --sock=*) 303 TEST_SOCK="${i#*=}" 304 ;; 305 esac 306done 307 308# start rpc.py coprocess if it's not started yet 309if [[ -z $RPC_PIPE_PID ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 310 # Include list to all known plugins we use in the tests 311 PYTHONPATH+=":$rootdir/test/rpc_plugins" 312 coproc RPC_PIPE { PYTHONPATH="$PYTHONPATH" "$rootdir/scripts/rpc.py" --server; } 313 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 314 # all descriptors will automatically close together with this bash 315 # process, this will make rpc.py stop reading and exit gracefully 316fi 317 318function set_test_storage() { 319 [[ -v testdir ]] || return 0 320 321 local requested_size=$1 # bytes 322 local mount target_dir 323 324 local -A mounts fss sizes avails uses 325 local source fs size avail mount use 326 327 local storage_fallback storage_candidates 328 329 storage_fallback=$(mktemp -udt spdk.XXXXXX) 330 storage_candidates=( 331 "$testdir" 332 "$storage_fallback/tests/${testdir##*/}" 333 "$storage_fallback" 334 ) 335 336 if [[ -n $ADD_TEST_STORAGE ]]; then 337 # List of dirs|mounts separated by whitespaces 338 storage_candidates+=($ADD_TEST_STORAGE) 339 fi 340 341 if [[ -n $DEDICATED_TEST_STORAGE ]]; then 342 # Single, dedicated dir|mount 343 storage_candidates=("$DEDICATED_TEST_STORAGE") 344 fi 345 346 mkdir -p "${storage_candidates[@]}" 347 348 # add some headroom - 64M 349 requested_size=$((requested_size + (64 << 20))) 350 351 while read -r source fs size use avail _ mount; do 352 mounts["$mount"]=$source fss["$mount"]=$fs 353 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 354 uses["$mount"]=$((use * 1024)) 355 done < <(df -T | grep -v Filesystem) 356 357 printf '* Looking for test storage...\n' >&2 358 359 local target_space new_size 360 for target_dir in "${storage_candidates[@]}"; do 361 # FreeBSD's df is lacking the --output arg 362 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 363 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 364 365 target_space=${avails["$mount"]} 366 if ((target_space == 0 || target_space < requested_size)); then 367 continue 368 fi 369 if ((target_space >= requested_size)); then 370 # For in-memory fs, and / make sure our requested size won't fill most of the space. 371 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 372 new_size=$((uses["$mount"] + requested_size)) 373 if ((new_size * 100 / sizes["$mount"] > 95)); then 374 continue 375 fi 376 fi 377 fi 378 export SPDK_TEST_STORAGE=$target_dir 379 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 380 return 0 381 done 382 printf '* Test storage is not available\n' 383 return 1 384} 385 386function get_config_params() { 387 xtrace_disable 388 config_params='--enable-debug --enable-werror' 389 390 # for options with dependencies but no test flag, set them here 391 if [ -f /usr/include/infiniband/verbs.h ]; then 392 config_params+=' --with-rdma' 393 fi 394 395 if [ $SPDK_TEST_USDT -eq 1 ]; then 396 config_params+=" --with-usdt" 397 fi 398 399 if [ $(uname -s) == "FreeBSD" ]; then 400 intel="hw.model: Intel" 401 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 402 else 403 intel="GenuineIntel" 404 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 405 fi 406 if [[ "$cpu_vendor" != *"$intel"* ]]; then 407 config_params+=" --without-idxd" 408 else 409 config_params+=" --with-idxd" 410 fi 411 412 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 413 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 414 fi 415 416 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 417 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 418 fi 419 420 if [ -d /usr/include/iscsi ]; then 421 [[ $(< /usr/include/iscsi/iscsi.h) =~ "define LIBISCSI_API_VERSION ("([0-9]+)")" ]] \ 422 && libiscsi_version=${BASH_REMATCH[1]} 423 if ((libiscsi_version >= 20150621)); then 424 config_params+=' --with-iscsi-initiator' 425 fi 426 fi 427 428 if [[ $SPDK_TEST_UNITTEST -eq 0 && \ 429 $SPDK_TEST_SCANBUILD -eq 0 && -z \ 430 $SPDK_TEST_AUTOBUILD ]]; then 431 config_params+=' --disable-unit-tests' 432 fi 433 434 if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then 435 config_params+=' --with-nvme-cuse' 436 fi 437 438 # for options with both dependencies and a test flag, set them here 439 if [ -f /usr/include/libpmemblk.h ] && [ $SPDK_TEST_PMDK -eq 1 ]; then 440 config_params+=' --with-pmdk' 441 fi 442 443 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]; then 444 if ge "$(nasm --version | awk '{print $3}')" 2.14 && [[ $SPDK_TEST_ISAL -eq 1 ]]; then 445 config_params+=' --with-vbdev-compress' 446 fi 447 fi 448 449 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 450 config_params+=' --with-rbd' 451 fi 452 453 # for options with no required dependencies, just test flags, set them here 454 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 455 config_params+=' --with-crypto' 456 fi 457 458 if [ $SPDK_TEST_OCF -eq 1 ]; then 459 config_params+=" --with-ocf" 460 fi 461 462 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 463 config_params+=' --enable-ubsan' 464 fi 465 466 if [ $SPDK_RUN_ASAN -eq 1 ]; then 467 config_params+=' --enable-asan' 468 fi 469 470 if [ "$(uname -s)" = "Linux" ]; then 471 config_params+=' --enable-coverage' 472 fi 473 474 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 475 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 476 config_params+=' --with-fuse' 477 fi 478 fi 479 480 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 481 config_params+=' --with-raid5f' 482 fi 483 484 if [ $SPDK_TEST_VFIOUSER -eq 1 ] || [ $SPDK_TEST_VFIOUSER_QEMU -eq 1 ]; then 485 config_params+=' --with-vfio-user' 486 fi 487 488 # Check whether liburing library header exists 489 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 490 config_params+=' --with-uring' 491 fi 492 493 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 494 config_params+=" --with-dpdk=$SPDK_RUN_EXTERNAL_DPDK" 495 fi 496 497 if [[ $SPDK_TEST_SMA -eq 1 ]]; then 498 config_params+=' --with-sma' 499 config_params+=' --with-crypto' 500 fi 501 502 if [ -f /usr/include/daos.h ] && [ $SPDK_TEST_DAOS -eq 1 ]; then 503 config_params+=' --with-daos' 504 fi 505 506 # Make the xnvme module available for the tests 507 if [[ $SPDK_TEST_XNVME -eq 1 ]]; then 508 config_params+=' --with-xnvme' 509 fi 510 511 if [[ $SPDK_TEST_FUZZER -eq 1 ]]; then 512 config_params+=" $(get_fuzzer_target_config)" 513 fi 514 515 echo "$config_params" 516 xtrace_restore 517} 518 519function get_fuzzer_target_config() { 520 local -A fuzzer_targets_to_config=() 521 local config target 522 523 fuzzer_targets_to_config["vfio"]="--with-vfio-user" 524 for target in $(get_fuzzer_targets); do 525 [[ -n ${fuzzer_targets_to_config["$target"]} ]] || continue 526 config+=("${fuzzer_targets_to_config["$target"]}") 527 done 528 529 if ((${#config[@]} > 0)); then 530 echo "${config[*]}" 531 fi 532} 533 534function get_fuzzer_targets() { 535 local fuzzers=() 536 537 if [[ -n $SPDK_TEST_FUZZER_TARGET ]]; then 538 IFS="," read -ra fuzzers <<< "$SPDK_TEST_FUZZER_TARGET" 539 else 540 fuzzers=("$rootdir/test/fuzz/llvm/"*) 541 fuzzers=("${fuzzers[@]##*/}") 542 fi 543 544 echo "${fuzzers[*]}" 545} 546 547function rpc_cmd() { 548 xtrace_disable 549 local rsp rc=1 550 local stdin cmd cmds_number=0 status_number=0 status 551 552 if (($#)); then 553 cmds_number=1 554 echo "$@" >&$RPC_PIPE_INPUT 555 elif [[ ! -t 0 ]]; then 556 mapfile -t stdin <&0 557 cmds_number=${#stdin[@]} 558 printf '%s\n' "${stdin[@]}" >&$RPC_PIPE_INPUT 559 else 560 return 0 561 fi 562 563 while read -t 15 -ru $RPC_PIPE_OUTPUT rsp; do 564 if [[ $rsp == "**STATUS="* ]]; then 565 status[${rsp#*=}]=$rsp 566 if ((++status_number == cmds_number)); then 567 break 568 fi 569 continue 570 fi 571 echo "$rsp" 572 done 573 574 rc=${!status[*]} 575 xtrace_restore 576 [[ $rc == 0 ]] 577} 578 579function rpc_cmd_simple_data_json() { 580 581 local elems="$1[@]" elem 582 local -gA jq_out=() 583 local jq val 584 585 local lvs=( 586 "uuid" 587 "name" 588 "base_bdev" 589 "total_data_clusters" 590 "free_clusters" 591 "block_size" 592 "cluster_size" 593 ) 594 595 local bdev=( 596 "name" 597 "aliases[0]" 598 "block_size" 599 "num_blocks" 600 "uuid" 601 "product_name" 602 ) 603 604 [[ -v $elems ]] || return 1 605 606 for elem in "${!elems}"; do 607 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 608 done 609 jq+=',"\n"' 610 611 shift 612 while read -r elem val; do 613 jq_out["$elem"]=$val 614 done < <(rpc_cmd "$@" | jq -jr "$jq") 615 ((${#jq_out[@]} > 0)) || return 1 616} 617 618function NOT() { 619 local es=0 620 621 "$@" || es=$? 622 623 # Logic looks like so: 624 # - return false if command exit successfully 625 # - return false if command exit after receiving a core signal (FIXME: or any signal?) 626 # - return true if command exit with an error 627 628 # This naively assumes that the process doesn't exit with > 128 on its own. 629 if ((es > 128)); then 630 es=$((es & ~128)) 631 case "$es" in 632 3) ;& # SIGQUIT 633 4) ;& # SIGILL 634 6) ;& # SIGABRT 635 8) ;& # SIGFPE 636 9) ;& # SIGKILL 637 11) es=0 ;; # SIGSEGV 638 *) es=1 ;; 639 esac 640 elif [[ -n $EXIT_STATUS ]] && ((es != EXIT_STATUS)); then 641 es=0 642 fi 643 644 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 645 ((!es == 0)) 646} 647 648function timing() { 649 direction="$1" 650 testname="$2" 651 652 now=$(date +%s) 653 654 if [ "$direction" = "enter" ]; then 655 export timing_stack="${timing_stack};${now}" 656 export test_stack="${test_stack};${testname}" 657 else 658 touch "$output_dir/timing.txt" 659 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 660 661 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 662 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 663 664 elapsed=$((now - start_time - child_time)) 665 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 666 667 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 668 fi 669} 670 671function timing_enter() { 672 xtrace_disable 673 timing "enter" "$1" 674 xtrace_restore 675} 676 677function timing_exit() { 678 xtrace_disable 679 timing "exit" "$1" 680 xtrace_restore 681} 682 683function timing_finish() { 684 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 685 if [ -x "$flamegraph" ]; then 686 "$flamegraph" \ 687 --title 'Build Timing' \ 688 --nametype 'Step:' \ 689 --countname seconds \ 690 $output_dir/timing.txt \ 691 > $output_dir/timing.svg 692 fi 693} 694 695function create_test_list() { 696 xtrace_disable 697 # First search all scripts in main SPDK directory. 698 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 699 # Follow up with search in test directory recursively. 700 completion+=$'\n'$(grep -rshI --include="*.sh" --exclude="*autotest_common.sh" -e "run_test " $rootdir/test) 701 printf "%s" "$completion" | grep -v "#" \ 702 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 703 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 704 xtrace_restore 705} 706 707function gdb_attach() { 708 gdb -q --batch \ 709 -ex 'handle SIGHUP nostop pass' \ 710 -ex 'handle SIGQUIT nostop pass' \ 711 -ex 'handle SIGPIPE nostop pass' \ 712 -ex 'handle SIGALRM nostop pass' \ 713 -ex 'handle SIGTERM nostop pass' \ 714 -ex 'handle SIGUSR1 nostop pass' \ 715 -ex 'handle SIGUSR2 nostop pass' \ 716 -ex 'handle SIGCHLD nostop pass' \ 717 -ex 'set print thread-events off' \ 718 -ex 'cont' \ 719 -ex 'thread apply all bt' \ 720 -ex 'quit' \ 721 --tty=/dev/stdout \ 722 -p $1 723} 724 725function process_core() { 726 # Note that this always was racy as we can't really sync with the kernel 727 # to see if there's any core queued up for writing. We could check if 728 # collector is running and wait for it explicitly, but it doesn't seem 729 # to be worth the effort. So assume that if we are being called via 730 # trap, as in, when some error has occurred, wait up to 10s for any 731 # potential cores. If we are called just for cleanup at the very end, 732 # don't wait since all the tests ended successfully, hence having any 733 # critical cores lying around is unlikely. 734 ((autotest_es != 0)) && sleep 10 735 736 local coredumps core 737 738 shopt -s nullglob 739 coredumps=("$output_dir/coredumps/"*.bt.txt) 740 shopt -u nullglob 741 742 ((${#coredumps[@]} > 0)) || return 0 743 chmod -R a+r "$output_dir/coredumps" 744 745 for core in "${coredumps[@]}"; do 746 cat <<- BT 747 ##### CORE BT ${core##*/} ##### 748 749 $(<"$core") 750 751 -- 752 BT 753 done 754 return 1 755} 756 757function process_shm() { 758 type=$1 759 id=$2 760 if [ "$type" = "--pid" ]; then 761 id="pid${id}" 762 fi 763 764 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 765 766 if [[ -z $shm_files ]]; then 767 echo "SHM File for specified PID or shared memory id: ${id} not found!" 768 return 1 769 fi 770 for n in $shm_files; do 771 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 772 done 773 return 0 774} 775 776# Parameters: 777# $1 - process pid 778# $2 - rpc address (optional) 779# $3 - max retries (optional) 780function waitforlisten() { 781 if [ -z "$1" ]; then 782 exit 1 783 fi 784 785 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 786 local max_retries=${3:-100} 787 788 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 789 # turn off trace for this loop 790 xtrace_disable 791 local ret=0 792 local i 793 for ((i = max_retries; i != 0; i--)); do 794 # if the process is no longer running, then exit the script 795 # since it means the application crashed 796 if ! kill -s 0 $1; then 797 echo "ERROR: process (pid: $1) is no longer running" 798 ret=1 799 break 800 fi 801 802 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 803 break 804 fi 805 806 sleep 0.5 807 done 808 809 xtrace_restore 810 if ((i == 0)); then 811 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 812 ret=1 813 fi 814 return $ret 815} 816 817function waitfornbd() { 818 local nbd_name=$1 819 local i 820 821 for ((i = 1; i <= 20; i++)); do 822 if grep -q -w $nbd_name /proc/partitions; then 823 break 824 else 825 sleep 0.1 826 fi 827 done 828 829 # The nbd device is now recognized as a block device, but there can be 830 # a small delay before we can start I/O to that block device. So loop 831 # here trying to read the first block of the nbd block device to a temp 832 # file. Note that dd returns success when reading an empty file, so we 833 # need to check the size of the output file instead. 834 for ((i = 1; i <= 20; i++)); do 835 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 836 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 837 rm -f "$SPDK_TEST_STORAGE/nbdtest" 838 if [ "$size" != "0" ]; then 839 return 0 840 else 841 sleep 0.1 842 fi 843 done 844 845 return 1 846} 847 848function waitforbdev() { 849 local bdev_name=$1 850 local bdev_timeout=$2 851 local i 852 [[ -z $bdev_timeout ]] && bdev_timeout=2000 # ms 853 854 $rpc_py bdev_wait_for_examine 855 856 if $rpc_py bdev_get_bdevs -b $bdev_name -t $bdev_timeout; then 857 return 0 858 fi 859 860 return 1 861} 862 863function make_filesystem() { 864 local fstype=$1 865 local dev_name=$2 866 local i=0 867 local force 868 869 if [ $fstype = ext4 ]; then 870 force=-F 871 else 872 force=-f 873 fi 874 875 while ! mkfs.${fstype} $force ${dev_name}; do 876 if [ $i -ge 15 ]; then 877 return 1 878 fi 879 i=$((i + 1)) 880 sleep 1 881 done 882 883 return 0 884} 885 886function killprocess() { 887 # $1 = process pid 888 if [ -z "$1" ]; then 889 return 1 890 fi 891 892 if kill -0 $1; then 893 if [ $(uname) = Linux ]; then 894 process_name=$(ps --no-headers -o comm= $1) 895 else 896 process_name=$(ps -c -o command $1 | tail -1) 897 fi 898 if [ "$process_name" = "sudo" ]; then 899 # kill the child process, which is the actual app 900 # (assume $1 has just one child) 901 local child 902 child="$(pgrep -P $1)" 903 echo "killing process with pid $child" 904 kill $child 905 else 906 echo "killing process with pid $1" 907 kill $1 908 fi 909 910 # wait for the process regardless if its the dummy sudo one 911 # or the actual app - it should terminate anyway 912 wait $1 913 else 914 # the process is not there anymore 915 echo "Process with pid $1 is not found" 916 fi 917} 918 919function iscsicleanup() { 920 echo "Cleaning up iSCSI connection" 921 iscsiadm -m node --logout || true 922 iscsiadm -m node -o delete || true 923 rm -rf /var/lib/iscsi/nodes/* 924} 925 926function stop_iscsi_service() { 927 if cat /etc/*-release | grep Ubuntu; then 928 service open-iscsi stop 929 else 930 service iscsid stop 931 fi 932} 933 934function start_iscsi_service() { 935 if cat /etc/*-release | grep Ubuntu; then 936 service open-iscsi start 937 else 938 service iscsid start 939 fi 940} 941 942function rbd_setup() { 943 # $1 = monitor ip address 944 # $2 = name of the namespace 945 if [ -z "$1" ]; then 946 echo "No monitor IP address provided for ceph" 947 exit 1 948 fi 949 if [ -n "$2" ]; then 950 if ip netns list | grep "$2"; then 951 NS_CMD="ip netns exec $2" 952 else 953 echo "No namespace $2 exists" 954 exit 1 955 fi 956 fi 957 958 if hash ceph; then 959 export PG_NUM=128 960 export RBD_POOL=rbd 961 export RBD_NAME=foo 962 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 963 $NS_CMD $rootdir/scripts/ceph/start.sh $1 964 965 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 966 $NS_CMD rbd create $RBD_NAME --size 1000 967 fi 968} 969 970function rbd_cleanup() { 971 if hash ceph; then 972 $rootdir/scripts/ceph/stop.sh || true 973 rm -f /var/tmp/ceph_raw.img 974 fi 975} 976 977function daos_setup() { 978 # $1 = pool name 979 # $2 = cont name 980 if [ -z "$1" ]; then 981 echo "No pool name provided" 982 exit 1 983 fi 984 if [ -z "$2" ]; then 985 echo "No cont name provided" 986 exit 1 987 fi 988 989 dmg pool create --size=10G $1 || true 990 daos container create --type=POSIX --label=$2 $1 || true 991} 992 993function daos_cleanup() { 994 local pool=${1:-testpool} 995 local cont=${2:-testcont} 996 997 daos container destroy -f $pool $cont || true 998 sudo dmg pool destroy -f $pool || true 999} 1000 1001function _start_stub() { 1002 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 1003 # but ASLR can still be unreliable in some cases. 1004 # We will re-enable it again after multi-process testing is complete in kill_stub(). 1005 # Save current setting so it can be restored upon calling kill_stub(). 1006 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 1007 echo 0 > /proc/sys/kernel/randomize_va_space 1008 $rootdir/test/app/stub/stub $1 & 1009 stubpid=$! 1010 echo Waiting for stub to ready for secondary processes... 1011 while ! [ -e /var/run/spdk_stub0 ]; do 1012 # If stub dies while we wait, bail 1013 [[ -e /proc/$stubpid ]] || return 1 1014 sleep 1s 1015 done 1016 echo done. 1017} 1018 1019function start_stub() { 1020 if ! _start_stub "$@"; then 1021 echo "stub failed" >&2 1022 return 1 1023 fi 1024} 1025 1026function kill_stub() { 1027 if [[ -e /proc/$stubpid ]]; then 1028 kill $1 $stubpid 1029 wait $stubpid 1030 fi 2> /dev/null || : 1031 rm -f /var/run/spdk_stub0 1032 # Re-enable ASLR now that we are done with multi-process testing 1033 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 1034 # randomizing and is the default on all recent Linux kernels 1035 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 1036} 1037 1038function run_test() { 1039 if [ $# -le 1 ]; then 1040 echo "Not enough parameters" 1041 echo "usage: run_test test_name test_script [script_params]" 1042 exit 1 1043 fi 1044 1045 xtrace_disable 1046 local test_name="$1" 1047 shift 1048 1049 if [ -n "$test_domain" ]; then 1050 export test_domain="${test_domain}.${test_name}" 1051 else 1052 export test_domain="$test_name" 1053 fi 1054 1055 timing_enter $test_name 1056 echo "************************************" 1057 echo "START TEST $test_name" 1058 echo "************************************" 1059 xtrace_restore 1060 time "$@" 1061 xtrace_disable 1062 echo "************************************" 1063 echo "END TEST $test_name" 1064 echo "************************************" 1065 timing_exit $test_name 1066 1067 export test_domain=${test_domain%"$test_name"} 1068 if [ -n "$test_domain" ]; then 1069 export test_domain=${test_domain%?} 1070 fi 1071 1072 if [ -z "$test_domain" ]; then 1073 echo "top_level $test_name" >> $output_dir/test_completions.txt 1074 else 1075 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 1076 fi 1077 xtrace_restore 1078} 1079 1080function skip_run_test_with_warning() { 1081 echo "WARNING: $1" 1082 echo "Test run may fail if run with autorun.sh" 1083 echo "Please check your $rootdir/test/common/skipped_tests.txt" 1084} 1085 1086function print_backtrace() { 1087 # if errexit is not enabled, don't print a backtrace 1088 [[ "$-" =~ e ]] || return 0 1089 1090 local args=("${BASH_ARGV[@]}") 1091 1092 xtrace_disable 1093 # Reset IFS in case we were called from an environment where it was modified 1094 IFS=" "$'\t'$'\n' 1095 echo "========== Backtrace start: ==========" 1096 echo "" 1097 for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do 1098 local func="${FUNCNAME[$i]}" 1099 local line_nr="${BASH_LINENO[$((i - 1))]}" 1100 local src="${BASH_SOURCE[$i]}" 1101 local bt="" cmdline=() 1102 1103 if [[ -f $src ]]; then 1104 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 1105 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 1106 fi 1107 1108 # If extdebug set the BASH_ARGC[i], try to fetch all the args 1109 if ((BASH_ARGC[i] > 0)); then 1110 # Use argc as index to reverse the stack 1111 local argc=${BASH_ARGC[i]} arg 1112 for arg in "${args[@]::BASH_ARGC[i]}"; do 1113 cmdline[argc--]="[\"$arg\"]" 1114 done 1115 args=("${args[@]:BASH_ARGC[i]}") 1116 fi 1117 1118 echo "in $src:$line_nr -> $func($( 1119 IFS="," 1120 printf '%s\n' "${cmdline[*]:-[]}" 1121 ))" 1122 echo " ..." 1123 echo "${bt:-backtrace unavailable}" 1124 echo " ..." 1125 done 1126 echo "" 1127 echo "========== Backtrace end ==========" 1128 xtrace_restore 1129 return 0 1130} 1131 1132function waitforserial() { 1133 local i=0 1134 local nvme_device_counter=1 nvme_devices=0 1135 if [[ -n "$2" ]]; then 1136 nvme_device_counter=$2 1137 fi 1138 1139 # Wait initially for min 4s to make sure all devices are ready for use. It seems 1140 # that we may be racing with a kernel where in some cases immediate disconnect may 1141 # leave dangling subsystem with no-op block devices which can't be used nor removed 1142 # (unless kernel is rebooted) and which start to negatively affect all the tests. 1143 sleep 4 1144 while ((i++ <= 15)); do 1145 nvme_devices=$(lsblk -l -o NAME,SERIAL | grep -c "$1") 1146 ((nvme_devices == nvme_device_counter)) && return 0 1147 if ((nvme_devices > nvme_device_counter)); then 1148 echo "$nvme_device_counter device(s) expected, found $nvme_devices" >&2 1149 fi 1150 echo "Waiting for devices" 1151 sleep 1 1152 done 1153 return 1 1154} 1155 1156function waitforserial_disconnect() { 1157 local i=0 1158 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1159 [ $i -lt 15 ] || break 1160 i=$((i + 1)) 1161 echo "Waiting for disconnect devices" 1162 sleep 1 1163 done 1164 1165 if lsblk -l -o NAME,SERIAL | grep -q -w $1; then 1166 return 1 1167 fi 1168 1169 return 0 1170} 1171 1172function waitforblk() { 1173 local i=0 1174 while ! lsblk -l -o NAME | grep -q -w $1; do 1175 [ $i -lt 15 ] || break 1176 i=$((i + 1)) 1177 sleep 1 1178 done 1179 1180 if ! lsblk -l -o NAME | grep -q -w $1; then 1181 return 1 1182 fi 1183 1184 return 0 1185} 1186 1187function waitforblk_disconnect() { 1188 local i=0 1189 while lsblk -l -o NAME | grep -q -w $1; do 1190 [ $i -lt 15 ] || break 1191 i=$((i + 1)) 1192 sleep 1 1193 done 1194 1195 if lsblk -l -o NAME | grep -q -w $1; then 1196 return 1 1197 fi 1198 1199 return 0 1200} 1201 1202function waitforfile() { 1203 local i=0 1204 while [ ! -e $1 ]; do 1205 [ $i -lt 200 ] || break 1206 i=$((i + 1)) 1207 sleep 0.1 1208 done 1209 1210 if [ ! -e $1 ]; then 1211 return 1 1212 fi 1213 1214 return 0 1215} 1216 1217function fio_config_gen() { 1218 local config_file=$1 1219 local workload=$2 1220 local bdev_type=$3 1221 local env_context=$4 1222 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1223 1224 if [ -e "$config_file" ]; then 1225 echo "Configuration File Already Exists!: $config_file" 1226 return 1 1227 fi 1228 1229 if [ -z "$workload" ]; then 1230 workload=randrw 1231 fi 1232 1233 if [ -n "$env_context" ]; then 1234 env_context="env_context=$env_context" 1235 fi 1236 1237 touch $1 1238 1239 cat > $1 << EOL 1240[global] 1241thread=1 1242$env_context 1243group_reporting=1 1244direct=1 1245norandommap=1 1246percentile_list=50:99:99.9:99.99:99.999 1247time_based=1 1248ramp_time=0 1249EOL 1250 1251 if [ "$workload" == "verify" ]; then 1252 cat <<- EOL >> $config_file 1253 verify=sha1 1254 verify_backlog=1024 1255 rw=randwrite 1256 EOL 1257 1258 # To avoid potential data race issue due to the AIO device 1259 # flush mechanism, add the flag to serialize the writes. 1260 # This is to fix the intermittent IO failure issue of #935 1261 if [ "$bdev_type" == "AIO" ]; then 1262 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1263 echo "serialize_overlap=1" >> $config_file 1264 fi 1265 fi 1266 elif [ "$workload" == "trim" ]; then 1267 echo "rw=trimwrite" >> $config_file 1268 else 1269 echo "rw=$workload" >> $config_file 1270 fi 1271} 1272 1273function fio_plugin() { 1274 # Setup fio binary cmd line 1275 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1276 # gcc and clang uses different sanitizer libraries 1277 local sanitizers=(libasan libclang_rt.asan) 1278 local plugin=$1 1279 shift 1280 1281 local asan_lib= 1282 for sanitizer in "${sanitizers[@]}"; do 1283 asan_lib=$(ldd $plugin | grep $sanitizer | awk '{print $3}') 1284 if [[ -n "$asan_lib" ]]; then 1285 break 1286 fi 1287 done 1288 1289 # Preload the sanitizer library to fio if fio_plugin was compiled with it 1290 LD_PRELOAD="$asan_lib $plugin" "$fio_dir"/fio "$@" 1291} 1292 1293function fio_bdev() { 1294 fio_plugin "$rootdir/build/fio/spdk_bdev" "$@" 1295} 1296 1297function fio_nvme() { 1298 fio_plugin "$rootdir/build/fio/spdk_nvme" "$@" 1299} 1300 1301function get_lvs_free_mb() { 1302 local lvs_uuid=$1 1303 local lvs_info 1304 local fc 1305 local cs 1306 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1307 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1308 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1309 1310 # Change to MB's 1311 free_mb=$((fc * cs / 1024 / 1024)) 1312 echo "$free_mb" 1313} 1314 1315function get_bdev_size() { 1316 local bdev_name=$1 1317 local bdev_info 1318 local bs 1319 local nb 1320 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1321 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1322 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1323 1324 # Change to MB's 1325 bdev_size=$((bs * nb / 1024 / 1024)) 1326 echo "$bdev_size" 1327} 1328 1329function autotest_cleanup() { 1330 local autotest_es=$? 1331 xtrace_disable 1332 1333 # catch any stray core files and kill all remaining SPDK processes. Update 1334 # autotest_es in case autotest reported success but cores and/or processes 1335 # were left behind regardless. 1336 1337 process_core || autotest_es=1 1338 reap_spdk_processes || autotest_es=1 1339 1340 $rootdir/scripts/setup.sh reset 1341 $rootdir/scripts/setup.sh cleanup 1342 if [ $(uname -s) = "Linux" ]; then 1343 modprobe -r uio_pci_generic 1344 fi 1345 rm -rf "$asan_suppression_file" 1346 if [[ -n $old_core_pattern ]]; then 1347 echo "$old_core_pattern" > /proc/sys/kernel/core_pattern 1348 fi 1349 if [[ -e /proc/$udevadm_pid/status ]]; then 1350 kill "$udevadm_pid" || : 1351 fi 1352 1353 shopt -s nullglob 1354 local storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 1355 shopt -u nullglob 1356 1357 if ((${#storage_fallback_purge[@]} > 0)); then 1358 rm -rf "${storage_fallback_purge[@]}" 1359 fi 1360 1361 if ((autotest_es)); then 1362 if [[ $(uname) == FreeBSD ]]; then 1363 ps aux 1364 elif [[ $(uname) == Linux ]]; then 1365 # Get more detailed view 1366 grep . /proc/[0-9]*/status 1367 # Dump some extra info into kernel log 1368 echo "######## Autotest Cleanup Dump ########" > /dev/kmsg 1369 # Show cpus backtraces 1370 echo l > /proc/sysrq-trigger 1371 # Show mem usage 1372 echo m > /proc/sysrq-trigger 1373 # show task states 1374 echo t > /proc/sysrq-trigger 1375 # show blocked tasks 1376 echo w > /proc/sysrq-trigger 1377 1378 fi > "$output_dir/proc_list.txt" 2>&1 || : 1379 fi 1380 xtrace_restore 1381 return $autotest_es 1382} 1383 1384function freebsd_update_contigmem_mod() { 1385 if [ $(uname) = FreeBSD ]; then 1386 kldunload contigmem.ko || true 1387 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 1388 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/ 1389 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/ 1390 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/ 1391 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/ 1392 else 1393 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1394 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1395 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1396 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1397 fi 1398 fi 1399} 1400 1401function get_nvme_name_from_bdf() { 1402 blkname=() 1403 1404 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1405 if [ -z "$nvme_devs" ]; then 1406 return 1407 fi 1408 for dev in $nvme_devs; do 1409 link_name=$(readlink /sys/block/$dev/device/device) || true 1410 if [ -z "$link_name" ]; then 1411 link_name=$(readlink /sys/block/$dev/device) 1412 fi 1413 bdf=$(basename "$link_name") 1414 if [ "$bdf" = "$1" ]; then 1415 blkname+=($dev) 1416 fi 1417 done 1418 1419 printf '%s\n' "${blkname[@]}" 1420} 1421 1422function get_nvme_ctrlr_from_bdf() { 1423 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1424 if [[ -z "$bdf_sysfs_path" ]]; then 1425 return 1426 fi 1427 1428 printf '%s\n' "$(basename $bdf_sysfs_path)" 1429} 1430 1431# Get BDF addresses of all NVMe drives currently attached to 1432# uio-pci-generic or vfio-pci 1433function get_nvme_bdfs() { 1434 xtrace_disable 1435 bdfs=$(jq -r .config[].params.traddr <<< $($rootdir/scripts/gen_nvme.sh)) 1436 if [[ -z $bdfs ]]; then 1437 echo "No devices to test on!" 1438 exit 1 1439 fi 1440 echo "$bdfs" 1441 xtrace_restore 1442} 1443 1444# Same as function above, but just get the first disks BDF address 1445function get_first_nvme_bdf() { 1446 head -1 <<< "$(get_nvme_bdfs)" 1447} 1448 1449function nvme_namespace_revert() { 1450 $rootdir/scripts/setup.sh 1451 sleep 1 1452 bdfs=$(get_nvme_bdfs) 1453 1454 $rootdir/scripts/setup.sh reset 1455 1456 for bdf in $bdfs; do 1457 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1458 if [[ -z "$nvme_ctrlr" ]]; then 1459 continue 1460 fi 1461 1462 # Check Optional Admin Command Support for Namespace Management 1463 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1464 oacs_ns_manage=$((oacs & 0x8)) 1465 1466 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1467 # This assumes every NVMe controller contains single namespace, 1468 # encompassing Total NVM Capacity and formatted as 512 block size. 1469 # 512 block size is needed for test/vhost/vhost_boot.sh to 1470 # successfully run. 1471 1472 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1473 if [[ "$unvmcap" -eq 0 ]]; then 1474 # All available space already used 1475 continue 1476 fi 1477 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1478 blksize=512 1479 1480 size=$((tnvmcap / blksize)) 1481 1482 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c 0 || true 1483 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1484 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1485 nvme attach-ns ${nvme_ctrlr} -n 1 -c 0 1486 nvme reset ${nvme_ctrlr} 1487 waitforfile "${nvme_ctrlr}n1" 1488 fi 1489 done 1490} 1491 1492# Get BDFs based on device ID, such as 0x0a54 1493function get_nvme_bdfs_by_id() { 1494 local bdfs=() 1495 1496 for bdf in $(get_nvme_bdfs); do 1497 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1498 if [[ "$device" == "$1" ]]; then 1499 bdfs+=($bdf) 1500 fi 1501 done 1502 1503 printf '%s\n' "${bdfs[@]}" 1504} 1505 1506function opal_revert_cleanup() { 1507 # The OPAL CI tests is only used for P4510 devices. 1508 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1509 if [[ -z ${bdfs[0]} ]]; then 1510 return 0 1511 fi 1512 1513 $SPDK_BIN_DIR/spdk_tgt & 1514 spdk_tgt_pid=$! 1515 waitforlisten $spdk_tgt_pid 1516 1517 bdf_id=0 1518 for bdf in "${bdfs[@]}"; do 1519 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme"${bdf_id} -t "pcie" -a ${bdf} 1520 # Ignore if this fails. 1521 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b "nvme"${bdf_id} -p test || true 1522 ((++bdf_id)) 1523 done 1524 1525 killprocess $spdk_tgt_pid 1526} 1527 1528function pap() { 1529 while read -r file; do 1530 cat <<- FILE 1531 --- $file --- 1532 $(<"$file") 1533 --- $file --- 1534 FILE 1535 rm -f "$file" 1536 done < <(find "$@" -type f | sort -u) 1537} 1538 1539function get_proc_paths() { 1540 local procs proc 1541 if [[ $(uname -s) == Linux ]]; then 1542 for proc in /proc/[0-9]*; do 1543 [[ -e $proc/exe ]] || continue 1544 procs[${proc##*/}]=$(readlink -f "$proc/exe") 1545 done 1546 elif [[ $(uname -s) == FreeBSD ]]; then 1547 while read -r proc _ _ path; do 1548 [[ -e $path ]] || continue 1549 procs[proc]=$path 1550 done < <(procstat -ab) 1551 fi 1552 1553 for proc in "${!procs[@]}"; do 1554 echo "$proc" "${procs[proc]}" 1555 done 1556} 1557 1558is_exec_file() { [[ -f $1 && $(file "$1") =~ ELF.+executable ]]; } 1559 1560function reap_spdk_processes() { 1561 local bins bin 1562 local misc_bins 1563 1564 while read -r bin; do 1565 is_exec_file "$bin" && misc_bins+=("$bin") 1566 done < <(find "$rootdir"/test/{app,env,event} -type f) 1567 1568 mapfile -t bins < <(readlink -f "$SPDK_BIN_DIR/"* "$SPDK_EXAMPLE_DIR/"* "${misc_bins[@]}") 1569 1570 local spdk_pid spdk_pids path 1571 while read -r spdk_pid path; do 1572 if [[ ${bins[*]/$path/} != "${bins[*]}" ]]; then 1573 echo "$path is still up ($spdk_pid), killing" 1574 spdk_pids[spdk_pid]=$path 1575 fi 1576 done < <(get_proc_paths) 1577 1578 ((${#spdk_pids[@]} > 0)) || return 0 1579 1580 kill -SIGTERM "${!spdk_pids[@]}" 2> /dev/null || : 1581 # Wait a bit and then use the stick 1582 sleep 2 1583 kill -SIGKILL "${!spdk_pids[@]}" 2> /dev/null || : 1584 1585 return 1 1586} 1587 1588function is_block_zoned() { 1589 local device=$1 1590 1591 [[ -e /sys/block/$device/queue/zoned ]] || return 1 1592 [[ $(< "/sys/block/$device/queue/zoned") != none ]] 1593} 1594 1595function get_zoned_devs() { 1596 local -gA zoned_devs=() 1597 local nvme bdf 1598 1599 for nvme in /sys/block/nvme*; do 1600 if is_block_zoned "${nvme##*/}"; then 1601 zoned_devs["${nvme##*/}"]=$(< "$nvme/device/address") 1602 fi 1603 done 1604} 1605 1606# Define temp storage for all the tests. Look for 2GB at minimum 1607set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1608 1609set -o errtrace 1610shopt -s extdebug 1611trap "trap - ERR; print_backtrace >&2" ERR 1612 1613PS4=' \t -- ${BASH_SOURCE#${BASH_SOURCE%/*/*}/}@${LINENO} -- \$ ' 1614if $SPDK_AUTOTEST_X; then 1615 # explicitly enable xtraces, overriding any tracking information. 1616 unset XTRACE_DISABLED 1617 unset XTRACE_NESTING_LEVEL 1618 xtrace_fd 1619 xtrace_enable 1620else 1621 xtrace_restore 1622fi 1623
