1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2015 Intel Corporation 4# All rights reserved. 5# 6rpc_py=rpc_cmd 7 8function xtrace_fd() { 9 if [[ -n $BASH_XTRACEFD && -e /proc/self/fd/$BASH_XTRACEFD ]]; then 10 # Close it first to make sure it's sane 11 exec {BASH_XTRACEFD}>&- 12 fi 13 exec {BASH_XTRACEFD}>&2 14 15 set -x 16} 17 18function xtrace_disable() { 19 if [ "$XTRACE_DISABLED" != "yes" ]; then 20 PREV_BASH_OPTS="$-" 21 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 22 XTRACE_DISABLED="yes" 23 fi 24 set +x 25 elif [ -z $XTRACE_NESTING_LEVEL ]; then 26 XTRACE_NESTING_LEVEL=1 27 else 28 XTRACE_NESTING_LEVEL=$((++XTRACE_NESTING_LEVEL)) 29 fi 30} 31 32xtrace_disable 33set -e 34shopt -s expand_aliases 35 36if [[ -e $rootdir/test/common/build_config.sh ]]; then 37 source "$rootdir/test/common/build_config.sh" 38elif [[ -e $rootdir/mk/config.mk ]]; then 39 build_config=$(< "$rootdir/mk/config.mk") 40 source <(echo "${build_config//\?=/=}") 41else 42 source "$rootdir/CONFIG" 43fi 44 45# Source scripts after the config so that the definitions are available. 46source "$rootdir/test/common/applications.sh" 47source "$rootdir/scripts/common.sh" 48 49# Dummy function to be called after restoring xtrace just so that it appears in the 50# xtrace log. This way we can consistently track when xtrace is enabled/disabled. 51function xtrace_enable() { 52 # We have to do something inside a function in bash, and calling any command 53 # (even `:`) will produce an xtrace entry, so we just define another function. 54 function xtrace_dummy() { :; } 55} 56 57# Keep it as alias to avoid xtrace_enable backtrace always pointing to xtrace_restore. 58# xtrace_enable will appear as called directly from the user script, from the same line 59# that "called" xtrace_restore. 60alias xtrace_restore='if [ -z $XTRACE_NESTING_LEVEL ]; then 61 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 62 XTRACE_DISABLED="no"; PREV_BASH_OPTS=""; set -x; xtrace_enable; 63 fi 64else 65 XTRACE_NESTING_LEVEL=$((--XTRACE_NESTING_LEVEL)); 66 if [ $XTRACE_NESTING_LEVEL -eq "0" ]; then 67 unset XTRACE_NESTING_LEVEL 68 fi 69fi' 70 71: ${RUN_NIGHTLY:=0} 72export RUN_NIGHTLY 73 74# Set defaults for missing test config options 75: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 76export SPDK_AUTOTEST_DEBUG_APPS 77: ${SPDK_RUN_VALGRIND=0} 78export SPDK_RUN_VALGRIND 79: ${SPDK_RUN_FUNCTIONAL_TEST=0} 80export SPDK_RUN_FUNCTIONAL_TEST 81: ${SPDK_TEST_UNITTEST=0} 82export SPDK_TEST_UNITTEST 83: ${SPDK_TEST_AUTOBUILD=""} 84export SPDK_TEST_AUTOBUILD 85: ${SPDK_TEST_RELEASE_BUILD=0} 86export SPDK_TEST_RELEASE_BUILD 87: ${SPDK_TEST_ISAL=0} 88export SPDK_TEST_ISAL 89: ${SPDK_TEST_ISCSI=0} 90export SPDK_TEST_ISCSI 91: ${SPDK_TEST_ISCSI_INITIATOR=0} 92export SPDK_TEST_ISCSI_INITIATOR 93: ${SPDK_TEST_NVME=0} 94export SPDK_TEST_NVME 95: ${SPDK_TEST_NVME_PMR=0} 96export SPDK_TEST_NVME_PMR 97: ${SPDK_TEST_NVME_SCC=0} 98export SPDK_TEST_NVME_SCC 99: ${SPDK_TEST_NVME_BP=0} 100export SPDK_TEST_NVME_BP 101: ${SPDK_TEST_NVME_CLI=0} 102export SPDK_TEST_NVME_CLI 103: ${SPDK_TEST_NVME_CUSE=0} 104export SPDK_TEST_NVME_CUSE 105: ${SPDK_TEST_NVMF=0} 106export SPDK_TEST_NVMF 107: ${SPDK_TEST_VFIOUSER=0} 108export SPDK_TEST_VFIOUSER 109: ${SPDK_TEST_VFIOUSER_QEMU=0} 110export SPDK_TEST_VFIOUSER_QEMU 111: ${SPDK_TEST_FUZZER=0} 112export SPDK_TEST_FUZZER 113: ${SPDK_TEST_FUZZER_SHORT=0} 114export SPDK_TEST_FUZZER_SHORT 115: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 116export SPDK_TEST_NVMF_TRANSPORT 117: ${SPDK_TEST_RBD=0} 118export SPDK_TEST_RBD 119: ${SPDK_TEST_VHOST=0} 120export SPDK_TEST_VHOST 121: ${SPDK_TEST_BLOCKDEV=0} 122export SPDK_TEST_BLOCKDEV 123: ${SPDK_TEST_IOAT=0} 124export SPDK_TEST_IOAT 125: ${SPDK_TEST_BLOBFS=0} 126export SPDK_TEST_BLOBFS 127: ${SPDK_TEST_VHOST_INIT=0} 128export SPDK_TEST_VHOST_INIT 129: ${SPDK_TEST_PMDK=0} 130export SPDK_TEST_PMDK 131: ${SPDK_TEST_LVOL=0} 132export SPDK_TEST_LVOL 133: ${SPDK_TEST_VBDEV_COMPRESS=0} 134export SPDK_TEST_VBDEV_COMPRESS 135: ${SPDK_RUN_ASAN=0} 136export SPDK_RUN_ASAN 137: ${SPDK_RUN_UBSAN=0} 138export SPDK_RUN_UBSAN 139: ${SPDK_RUN_EXTERNAL_DPDK=""} 140export SPDK_RUN_EXTERNAL_DPDK 141: ${SPDK_RUN_NON_ROOT=0} 142export SPDK_RUN_NON_ROOT 143: ${SPDK_TEST_CRYPTO=0} 144export SPDK_TEST_CRYPTO 145: ${SPDK_TEST_FTL=0} 146export SPDK_TEST_FTL 147: ${SPDK_TEST_OCF=0} 148export SPDK_TEST_OCF 149: ${SPDK_TEST_VMD=0} 150export SPDK_TEST_VMD 151: ${SPDK_TEST_OPAL=0} 152export SPDK_TEST_OPAL 153: ${SPDK_TEST_NATIVE_DPDK} 154export SPDK_TEST_NATIVE_DPDK 155: ${SPDK_AUTOTEST_X=true} 156export SPDK_AUTOTEST_X 157: ${SPDK_TEST_RAID5=0} 158export SPDK_TEST_RAID5 159: ${SPDK_TEST_URING=0} 160export SPDK_TEST_URING 161: ${SPDK_TEST_USDT=0} 162export SPDK_TEST_USDT 163: ${SPDK_TEST_USE_IGB_UIO:=0} 164export SPDK_TEST_USE_IGB_UIO 165: ${SPDK_TEST_SCHEDULER:=0} 166export SPDK_TEST_SCHEDULER 167: ${SPDK_TEST_SCANBUILD:=0} 168export SPDK_TEST_SCANBUILD 169: ${SPDK_TEST_NVMF_NICS:=} 170export SPDK_TEST_NVMF_NICS 171: ${SPDK_TEST_SMA=0} 172export SPDK_TEST_SMA 173: ${SPDK_TEST_DAOS=0} 174export SPDK_TEST_DAOS 175: ${SPDK_TEST_XNVME:=0} 176export SPDK_TEST_XNVME 177# Comma-separated list of fuzzer targets matching test/fuzz/llvm/$target 178: ${SPDK_TEST_FUZZER_TARGET:=} 179export SPDK_TEST_FUZZER_TARGET 180: ${SPDK_TEST_NVMF_MDNS=0} 181export SPDK_TEST_NVMF_MDNS 182 183# always test with SPDK shared objects. 184export SPDK_LIB_DIR="$rootdir/build/lib" 185export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib" 186export VFIO_LIB_DIR="$rootdir/build/libvfio-user/usr/local/lib" 187export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR:$VFIO_LIB_DIR 188 189# Tell setup.sh to wait for block devices upon each reset 190export PCI_BLOCK_SYNC_ON_RESET=yes 191 192# Export PYTHONPATH with addition of RPC framework. New scripts can be created 193# specific use cases for tests. 194export PYTHONPATH=$PYTHONPATH:$rootdir/python 195 196# Don't create Python .pyc files. When running with sudo these will be 197# created with root ownership and can cause problems when cleaning the repository. 198export PYTHONDONTWRITEBYTECODE=1 199 200# Export flag to skip the known bug that exists in librados 201# Bug is reported on ceph bug tracker with number 24078 202export ASAN_OPTIONS=new_delete_type_mismatch=0:disable_coredump=0 203export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1:disable_coredump=0' 204 205# Export LeakSanitizer option to use suppression file in order to prevent false positives 206# and known leaks in external executables or libraries from showing up. 207asan_suppression_file="/var/tmp/asan_suppression_file" 208rm -rf "$asan_suppression_file" 2> /dev/null || sudo rm -rf "$asan_suppression_file" 209cat << EOL >> "$asan_suppression_file" 210# ASAN has some bugs around thread_local variables. We have a destructor in place 211# to free the thread contexts, but ASAN complains about the leak before those 212# destructors have a chance to run. So suppress this one specific leak using 213# LSAN_OPTIONS. 214leak:spdk_fs_alloc_thread_ctx 215 216# Suppress known leaks in fio project 217leak:$CONFIG_FIO_SOURCE_DIR/parse.c 218leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 219leak:$CONFIG_FIO_SOURCE_DIR/init.c 220leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 221leak:fio_memalign 222leak:spdk_fio_io_u_init 223# Suppress leaks in gperftools-libs from fio 224leak:libtcmalloc_minimal.so 225 226# Suppress leaks in libiscsi 227leak:libiscsi.so 228EOL 229 230# Suppress leaks in libfuse3 231echo "leak:libfuse3.so" >> "$asan_suppression_file" 232 233export LSAN_OPTIONS=suppressions="$asan_suppression_file" 234 235export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 236 237if [ -z "$DEPENDENCY_DIR" ]; then 238 export DEPENDENCY_DIR=$HOME/spdk_dependencies 239else 240 export DEPENDENCY_DIR 241fi 242 243# Export location of where all the SPDK binaries are 244export SPDK_BIN_DIR="$rootdir/build/bin" 245export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 246 247# for vhost, vfio-user tests 248export QEMU_BIN=${QEMU_BIN:-} 249export VFIO_QEMU_BIN=${VFIO_QEMU_BIN:-} 250 251export AR_TOOL=$rootdir/scripts/ar-xnvme-fixer 252 253# pass our valgrind desire on to unittest.sh 254if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 255 export valgrind='' 256else 257 # unset all DEBUGINFOD_* vars that may affect our valgrind instance 258 unset -v "${!DEBUGINFOD_@}" 259fi 260 261if [ "$(uname -s)" = "Linux" ]; then 262 HUGEMEM=${HUGEMEM:-4096} 263 export CLEAR_HUGE=yes 264 if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]]; then 265 # Make sure that memory is distributed across all NUMA nodes - by default, all goes to 266 # node0, but if QAT devices are attached to a different node, all of their VFs will end 267 # up under that node too and memory needs to be available there for the tests. 268 export HUGE_EVEN_ALLOC=yes 269 fi 270 271 MAKE="make" 272 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 273elif [ "$(uname -s)" = "FreeBSD" ]; then 274 MAKE="gmake" 275 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 276 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 277 HUGEMEM=${HUGEMEM:-2048} 278elif [ "$(uname -s)" = "Windows" ]; then 279 MAKE="make" 280 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 281 # Keep the default 2GB for Windows. 282 HUGEMEM=${HUGEMEM:-2048} 283else 284 echo "Unknown OS \"$(uname -s)\"" 285 exit 1 286fi 287 288export HUGEMEM=$HUGEMEM 289 290if [ -z "$output_dir" ]; then 291 mkdir -p "$rootdir/../output" 292 export output_dir="$rootdir/../output" 293fi 294 295TEST_MODE= 296for i in "$@"; do 297 case "$i" in 298 --iso) 299 TEST_MODE=iso 300 ;; 301 --transport=*) 302 TEST_TRANSPORT="${i#*=}" 303 ;; 304 --sock=*) 305 TEST_SOCK="${i#*=}" 306 ;; 307 esac 308done 309 310# start rpc.py coprocess if it's not started yet 311if [[ -z $RPC_PIPE_PID ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 312 # Include list to all known plugins we use in the tests 313 PYTHONPATH+=":$rootdir/test/rpc_plugins" 314 coproc RPC_PIPE { PYTHONPATH="$PYTHONPATH" "$rootdir/scripts/rpc.py" --server; } 315 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 316 # all descriptors will automatically close together with this bash 317 # process, this will make rpc.py stop reading and exit gracefully 318fi 319 320function set_test_storage() { 321 [[ -v testdir ]] || return 0 322 323 local requested_size=$1 # bytes 324 local mount target_dir 325 326 local -A mounts fss sizes avails uses 327 local source fs size avail mount use 328 329 local storage_fallback storage_candidates 330 331 storage_fallback=$(mktemp -udt spdk.XXXXXX) 332 storage_candidates=( 333 "$testdir" 334 "$storage_fallback/tests/${testdir##*/}" 335 "$storage_fallback" 336 ) 337 338 if [[ -n $ADD_TEST_STORAGE ]]; then 339 # List of dirs|mounts separated by whitespaces 340 storage_candidates+=($ADD_TEST_STORAGE) 341 fi 342 343 if [[ -n $DEDICATED_TEST_STORAGE ]]; then 344 # Single, dedicated dir|mount 345 storage_candidates=("$DEDICATED_TEST_STORAGE") 346 fi 347 348 mkdir -p "${storage_candidates[@]}" 349 350 # add some headroom - 64M 351 requested_size=$((requested_size + (64 << 20))) 352 353 while read -r source fs size use avail _ mount; do 354 mounts["$mount"]=$source fss["$mount"]=$fs 355 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 356 uses["$mount"]=$((use * 1024)) 357 done < <(df -T | grep -v Filesystem) 358 359 printf '* Looking for test storage...\n' >&2 360 361 local target_space new_size 362 for target_dir in "${storage_candidates[@]}"; do 363 # FreeBSD's df is lacking the --output arg 364 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 365 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 366 367 target_space=${avails["$mount"]} 368 if ((target_space == 0 || target_space < requested_size)); then 369 continue 370 fi 371 if ((target_space >= requested_size)); then 372 # For in-memory fs, and / make sure our requested size won't fill most of the space. 373 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 374 new_size=$((uses["$mount"] + requested_size)) 375 if ((new_size * 100 / sizes["$mount"] > 95)); then 376 continue 377 fi 378 fi 379 fi 380 export SPDK_TEST_STORAGE=$target_dir 381 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 382 return 0 383 done 384 printf '* Test storage is not available\n' 385 return 1 386} 387 388function get_config_params() { 389 xtrace_disable 390 config_params='--enable-debug --enable-werror' 391 392 # for options with dependencies but no test flag, set them here 393 if [ -f /usr/include/infiniband/verbs.h ]; then 394 config_params+=' --with-rdma' 395 fi 396 397 if [ $SPDK_TEST_USDT -eq 1 ]; then 398 config_params+=" --with-usdt" 399 fi 400 401 if [ $(uname -s) == "FreeBSD" ]; then 402 intel="hw.model: Intel" 403 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 404 else 405 intel="GenuineIntel" 406 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 407 fi 408 if [[ "$cpu_vendor" != *"$intel"* ]]; then 409 config_params+=" --without-idxd" 410 else 411 config_params+=" --with-idxd" 412 fi 413 414 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 415 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 416 fi 417 418 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 419 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 420 fi 421 422 if [ -d /usr/include/iscsi ]; then 423 [[ $(< /usr/include/iscsi/iscsi.h) =~ "define LIBISCSI_API_VERSION ("([0-9]+)")" ]] \ 424 && libiscsi_version=${BASH_REMATCH[1]} 425 if ((libiscsi_version >= 20150621)); then 426 config_params+=' --with-iscsi-initiator' 427 fi 428 fi 429 430 if [[ $SPDK_TEST_UNITTEST -eq 0 && \ 431 $SPDK_TEST_SCANBUILD -eq 0 && -z \ 432 $SPDK_TEST_AUTOBUILD ]]; then 433 config_params+=' --disable-unit-tests' 434 fi 435 436 if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then 437 config_params+=' --with-nvme-cuse' 438 fi 439 440 # for options with both dependencies and a test flag, set them here 441 if [ -f /usr/include/libpmemblk.h ] && [ $SPDK_TEST_PMDK -eq 1 ]; then 442 config_params+=' --with-pmdk' 443 fi 444 445 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]; then 446 if ge "$(nasm --version | awk '{print $3}')" 2.14 && [[ $SPDK_TEST_ISAL -eq 1 ]]; then 447 config_params+=' --with-vbdev-compress --with-dpdk-compressdev' 448 fi 449 fi 450 451 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 452 config_params+=' --with-rbd' 453 fi 454 455 # for options with no required dependencies, just test flags, set them here 456 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 457 config_params+=' --with-crypto' 458 fi 459 460 if [ $SPDK_TEST_OCF -eq 1 ]; then 461 config_params+=" --with-ocf" 462 fi 463 464 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 465 config_params+=' --enable-ubsan' 466 fi 467 468 if [ $SPDK_RUN_ASAN -eq 1 ]; then 469 config_params+=' --enable-asan' 470 fi 471 472 if [ "$(uname -s)" = "Linux" ]; then 473 config_params+=' --enable-coverage' 474 fi 475 476 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 477 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 478 config_params+=' --with-fuse' 479 fi 480 fi 481 482 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 483 config_params+=' --with-raid5f' 484 fi 485 486 if [ $SPDK_TEST_VFIOUSER -eq 1 ] || [ $SPDK_TEST_VFIOUSER_QEMU -eq 1 ] || [ $SPDK_TEST_SMA -eq 1 ]; then 487 config_params+=' --with-vfio-user' 488 fi 489 490 # Check whether liburing library header exists 491 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 492 config_params+=' --with-uring' 493 fi 494 495 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 496 config_params+=" --with-dpdk=$SPDK_RUN_EXTERNAL_DPDK" 497 fi 498 499 if [[ $SPDK_TEST_SMA -eq 1 ]]; then 500 config_params+=' --with-sma' 501 config_params+=' --with-crypto' 502 fi 503 504 if [ -f /usr/include/daos.h ] && [ $SPDK_TEST_DAOS -eq 1 ]; then 505 config_params+=' --with-daos' 506 fi 507 508 # Make the xnvme module available for the tests 509 if [[ $SPDK_TEST_XNVME -eq 1 ]]; then 510 config_params+=' --with-xnvme' 511 fi 512 513 if [[ $SPDK_TEST_FUZZER -eq 1 ]]; then 514 config_params+=" $(get_fuzzer_target_config)" 515 fi 516 517 if [[ $SPDK_TEST_NVMF_MDNS -eq 1 ]]; then 518 config_params+=' --with-avahi' 519 fi 520 521 echo "$config_params" 522 xtrace_restore 523} 524 525function get_fuzzer_target_config() { 526 local -A fuzzer_targets_to_config=() 527 local config target 528 529 fuzzer_targets_to_config["vfio"]="--with-vfio-user" 530 for target in $(get_fuzzer_targets); do 531 [[ -n ${fuzzer_targets_to_config["$target"]} ]] || continue 532 config+=("${fuzzer_targets_to_config["$target"]}") 533 done 534 535 if ((${#config[@]} > 0)); then 536 echo "${config[*]}" 537 fi 538} 539 540function get_fuzzer_targets() { 541 local fuzzers=() 542 543 if [[ -n $SPDK_TEST_FUZZER_TARGET ]]; then 544 IFS="," read -ra fuzzers <<< "$SPDK_TEST_FUZZER_TARGET" 545 else 546 fuzzers=("$rootdir/test/fuzz/llvm/"*) 547 fuzzers=("${fuzzers[@]##*/}") 548 fi 549 550 echo "${fuzzers[*]}" 551} 552 553function rpc_cmd() { 554 xtrace_disable 555 local rsp rc=1 556 local stdin cmd cmds_number=0 status_number=0 status 557 558 if (($#)); then 559 cmds_number=1 560 echo "$@" >&$RPC_PIPE_INPUT 561 elif [[ ! -t 0 ]]; then 562 mapfile -t stdin <&0 563 cmds_number=${#stdin[@]} 564 printf '%s\n' "${stdin[@]}" >&$RPC_PIPE_INPUT 565 else 566 return 0 567 fi 568 569 while read -t 15 -ru $RPC_PIPE_OUTPUT rsp; do 570 if [[ $rsp == "**STATUS="* ]]; then 571 status[${rsp#*=}]=$rsp 572 if ((++status_number == cmds_number)); then 573 break 574 fi 575 continue 576 fi 577 echo "$rsp" 578 done 579 580 rc=${!status[*]} 581 xtrace_restore 582 [[ $rc == 0 ]] 583} 584 585function rpc_cmd_simple_data_json() { 586 587 local elems="$1[@]" elem 588 local -gA jq_out=() 589 local jq val 590 591 local lvs=( 592 "uuid" 593 "name" 594 "base_bdev" 595 "total_data_clusters" 596 "free_clusters" 597 "block_size" 598 "cluster_size" 599 ) 600 601 local bdev=( 602 "name" 603 "aliases[0]" 604 "block_size" 605 "num_blocks" 606 "uuid" 607 "product_name" 608 ) 609 610 [[ -v $elems ]] || return 1 611 612 for elem in "${!elems}"; do 613 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 614 done 615 jq+=',"\n"' 616 617 shift 618 while read -r elem val; do 619 jq_out["$elem"]=$val 620 done < <(rpc_cmd "$@" | jq -jr "$jq") 621 ((${#jq_out[@]} > 0)) || return 1 622} 623 624function NOT() { 625 local es=0 626 627 "$@" || es=$? 628 629 # Logic looks like so: 630 # - return false if command exit successfully 631 # - return false if command exit after receiving a core signal (FIXME: or any signal?) 632 # - return true if command exit with an error 633 634 # This naively assumes that the process doesn't exit with > 128 on its own. 635 if ((es > 128)); then 636 es=$((es & ~128)) 637 case "$es" in 638 3) ;& # SIGQUIT 639 4) ;& # SIGILL 640 6) ;& # SIGABRT 641 8) ;& # SIGFPE 642 9) ;& # SIGKILL 643 11) es=0 ;; # SIGSEGV 644 *) es=1 ;; 645 esac 646 elif [[ -n $EXIT_STATUS ]] && ((es != EXIT_STATUS)); then 647 es=0 648 fi 649 650 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 651 ((!es == 0)) 652} 653 654function timing() { 655 direction="$1" 656 testname="$2" 657 658 now=$(date +%s) 659 660 if [ "$direction" = "enter" ]; then 661 export timing_stack="${timing_stack};${now}" 662 export test_stack="${test_stack};${testname}" 663 else 664 touch "$output_dir/timing.txt" 665 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 666 667 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 668 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 669 670 elapsed=$((now - start_time - child_time)) 671 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 672 673 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 674 fi 675} 676 677function timing_enter() { 678 xtrace_disable 679 timing "enter" "$1" 680 xtrace_restore 681} 682 683function timing_exit() { 684 xtrace_disable 685 timing "exit" "$1" 686 xtrace_restore 687} 688 689function timing_finish() { 690 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 691 if [ -x "$flamegraph" ]; then 692 "$flamegraph" \ 693 --title 'Build Timing' \ 694 --nametype 'Step:' \ 695 --countname seconds \ 696 $output_dir/timing.txt \ 697 > $output_dir/timing.svg 698 fi 699} 700 701function create_test_list() { 702 xtrace_disable 703 # First search all scripts in main SPDK directory. 704 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 705 # Follow up with search in test directory recursively. 706 completion+=$'\n'$(grep -rshI --include="*.sh" --exclude="*autotest_common.sh" -e "run_test " $rootdir/test) 707 printf "%s" "$completion" | grep -v "#" \ 708 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 709 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 710 xtrace_restore 711} 712 713function gdb_attach() { 714 gdb -q --batch \ 715 -ex 'handle SIGHUP nostop pass' \ 716 -ex 'handle SIGQUIT nostop pass' \ 717 -ex 'handle SIGPIPE nostop pass' \ 718 -ex 'handle SIGALRM nostop pass' \ 719 -ex 'handle SIGTERM nostop pass' \ 720 -ex 'handle SIGUSR1 nostop pass' \ 721 -ex 'handle SIGUSR2 nostop pass' \ 722 -ex 'handle SIGCHLD nostop pass' \ 723 -ex 'set print thread-events off' \ 724 -ex 'cont' \ 725 -ex 'thread apply all bt' \ 726 -ex 'quit' \ 727 --tty=/dev/stdout \ 728 -p $1 729} 730 731function process_core() { 732 # Note that this always was racy as we can't really sync with the kernel 733 # to see if there's any core queued up for writing. We could check if 734 # collector is running and wait for it explicitly, but it doesn't seem 735 # to be worth the effort. So assume that if we are being called via 736 # trap, as in, when some error has occurred, wait up to 10s for any 737 # potential cores. If we are called just for cleanup at the very end, 738 # don't wait since all the tests ended successfully, hence having any 739 # critical cores lying around is unlikely. 740 ((autotest_es != 0)) && sleep 10 741 742 local coredumps core 743 744 shopt -s nullglob 745 coredumps=("$output_dir/coredumps/"*.bt.txt) 746 shopt -u nullglob 747 748 ((${#coredumps[@]} > 0)) || return 0 749 chmod -R a+r "$output_dir/coredumps" 750 751 for core in "${coredumps[@]}"; do 752 cat <<- BT 753 ##### CORE BT ${core##*/} ##### 754 755 $(<"$core") 756 757 -- 758 BT 759 done 760 return 1 761} 762 763function process_shm() { 764 type=$1 765 id=$2 766 if [ "$type" = "--pid" ]; then 767 id="pid${id}" 768 fi 769 770 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 771 772 if [[ -z $shm_files ]]; then 773 echo "SHM File for specified PID or shared memory id: ${id} not found!" 774 return 1 775 fi 776 for n in $shm_files; do 777 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 778 done 779 return 0 780} 781 782# Parameters: 783# $1 - process pid 784# $2 - rpc address (optional) 785# $3 - max retries (optional) 786function waitforlisten() { 787 if [ -z "$1" ]; then 788 exit 1 789 fi 790 791 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 792 local max_retries=${3:-100} 793 794 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 795 # turn off trace for this loop 796 xtrace_disable 797 local ret=0 798 local i 799 for ((i = max_retries; i != 0; i--)); do 800 # if the process is no longer running, then exit the script 801 # since it means the application crashed 802 if ! kill -s 0 $1; then 803 echo "ERROR: process (pid: $1) is no longer running" 804 ret=1 805 break 806 fi 807 808 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 809 break 810 fi 811 812 sleep 0.5 813 done 814 815 xtrace_restore 816 if ((i == 0)); then 817 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 818 ret=1 819 fi 820 return $ret 821} 822 823function waitfornbd() { 824 local nbd_name=$1 825 local i 826 827 for ((i = 1; i <= 20; i++)); do 828 if grep -q -w $nbd_name /proc/partitions; then 829 break 830 else 831 sleep 0.1 832 fi 833 done 834 835 # The nbd device is now recognized as a block device, but there can be 836 # a small delay before we can start I/O to that block device. So loop 837 # here trying to read the first block of the nbd block device to a temp 838 # file. Note that dd returns success when reading an empty file, so we 839 # need to check the size of the output file instead. 840 for ((i = 1; i <= 20; i++)); do 841 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 842 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 843 rm -f "$SPDK_TEST_STORAGE/nbdtest" 844 if [ "$size" != "0" ]; then 845 return 0 846 else 847 sleep 0.1 848 fi 849 done 850 851 return 1 852} 853 854function waitforbdev() { 855 local bdev_name=$1 856 local bdev_timeout=$2 857 local i 858 [[ -z $bdev_timeout ]] && bdev_timeout=2000 # ms 859 860 $rpc_py bdev_wait_for_examine 861 862 if $rpc_py bdev_get_bdevs -b $bdev_name -t $bdev_timeout; then 863 return 0 864 fi 865 866 return 1 867} 868 869function make_filesystem() { 870 local fstype=$1 871 local dev_name=$2 872 local i=0 873 local force 874 875 if [ $fstype = ext4 ]; then 876 force=-F 877 else 878 force=-f 879 fi 880 881 while ! mkfs.${fstype} $force ${dev_name}; do 882 if [ $i -ge 15 ]; then 883 return 1 884 fi 885 i=$((i + 1)) 886 sleep 1 887 done 888 889 return 0 890} 891 892function killprocess() { 893 # $1 = process pid 894 if [ -z "$1" ]; then 895 return 1 896 fi 897 898 if kill -0 $1; then 899 if [ $(uname) = Linux ]; then 900 process_name=$(ps --no-headers -o comm= $1) 901 else 902 process_name=$(ps -c -o command $1 | tail -1) 903 fi 904 if [ "$process_name" = "sudo" ]; then 905 # kill the child process, which is the actual app 906 # (assume $1 has just one child) 907 local child 908 child="$(pgrep -P $1)" 909 echo "killing process with pid $child" 910 kill $child 911 else 912 echo "killing process with pid $1" 913 kill $1 914 fi 915 916 # wait for the process regardless if its the dummy sudo one 917 # or the actual app - it should terminate anyway 918 wait $1 919 else 920 # the process is not there anymore 921 echo "Process with pid $1 is not found" 922 fi 923} 924 925function iscsicleanup() { 926 echo "Cleaning up iSCSI connection" 927 iscsiadm -m node --logout || true 928 iscsiadm -m node -o delete || true 929 rm -rf /var/lib/iscsi/nodes/* 930} 931 932function stop_iscsi_service() { 933 if cat /etc/*-release | grep Ubuntu; then 934 service open-iscsi stop 935 else 936 service iscsid stop 937 fi 938} 939 940function start_iscsi_service() { 941 if cat /etc/*-release | grep Ubuntu; then 942 service open-iscsi start 943 else 944 service iscsid start 945 fi 946} 947 948function rbd_setup() { 949 # $1 = monitor ip address 950 # $2 = name of the namespace 951 if [ -z "$1" ]; then 952 echo "No monitor IP address provided for ceph" 953 exit 1 954 fi 955 if [ -n "$2" ]; then 956 if ip netns list | grep "$2"; then 957 NS_CMD="ip netns exec $2" 958 else 959 echo "No namespace $2 exists" 960 exit 1 961 fi 962 fi 963 964 if hash ceph; then 965 export PG_NUM=128 966 export RBD_POOL=rbd 967 export RBD_NAME=foo 968 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 969 $NS_CMD $rootdir/scripts/ceph/start.sh $1 970 971 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 972 $NS_CMD rbd create $RBD_NAME --size 1000 973 fi 974} 975 976function rbd_cleanup() { 977 if hash ceph; then 978 $rootdir/scripts/ceph/stop.sh || true 979 rm -f /var/tmp/ceph_raw.img 980 fi 981} 982 983function daos_setup() { 984 # $1 = pool name 985 # $2 = cont name 986 if [ -z "$1" ]; then 987 echo "No pool name provided" 988 exit 1 989 fi 990 if [ -z "$2" ]; then 991 echo "No cont name provided" 992 exit 1 993 fi 994 995 dmg pool create --size=10G $1 || true 996 daos container create --type=POSIX --label=$2 $1 || true 997} 998 999function daos_cleanup() { 1000 local pool=${1:-testpool} 1001 local cont=${2:-testcont} 1002 1003 daos container destroy -f $pool $cont || true 1004 sudo dmg pool destroy -f $pool || true 1005} 1006 1007function _start_stub() { 1008 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 1009 # but ASLR can still be unreliable in some cases. 1010 # We will re-enable it again after multi-process testing is complete in kill_stub(). 1011 # Save current setting so it can be restored upon calling kill_stub(). 1012 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 1013 echo 0 > /proc/sys/kernel/randomize_va_space 1014 $rootdir/test/app/stub/stub $1 & 1015 stubpid=$! 1016 echo Waiting for stub to ready for secondary processes... 1017 while ! [ -e /var/run/spdk_stub0 ]; do 1018 # If stub dies while we wait, bail 1019 [[ -e /proc/$stubpid ]] || return 1 1020 sleep 1s 1021 done 1022 echo done. 1023} 1024 1025function start_stub() { 1026 if ! _start_stub "$@"; then 1027 echo "stub failed" >&2 1028 return 1 1029 fi 1030} 1031 1032function kill_stub() { 1033 if [[ -e /proc/$stubpid ]]; then 1034 kill $1 $stubpid 1035 wait $stubpid 1036 fi 2> /dev/null || : 1037 rm -f /var/run/spdk_stub0 1038 # Re-enable ASLR now that we are done with multi-process testing 1039 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 1040 # randomizing and is the default on all recent Linux kernels 1041 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 1042} 1043 1044function run_test() { 1045 if [ $# -le 1 ]; then 1046 echo "Not enough parameters" 1047 echo "usage: run_test test_name test_script [script_params]" 1048 exit 1 1049 fi 1050 1051 xtrace_disable 1052 local test_name="$1" 1053 shift 1054 1055 if [ -n "$test_domain" ]; then 1056 export test_domain="${test_domain}.${test_name}" 1057 else 1058 export test_domain="$test_name" 1059 fi 1060 1061 timing_enter $test_name 1062 echo "************************************" 1063 echo "START TEST $test_name" 1064 echo "************************************" 1065 xtrace_restore 1066 time "$@" 1067 xtrace_disable 1068 echo "************************************" 1069 echo "END TEST $test_name" 1070 echo "************************************" 1071 timing_exit $test_name 1072 1073 export test_domain=${test_domain%"$test_name"} 1074 if [ -n "$test_domain" ]; then 1075 export test_domain=${test_domain%?} 1076 fi 1077 1078 if [ -z "$test_domain" ]; then 1079 echo "top_level $test_name" >> $output_dir/test_completions.txt 1080 else 1081 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 1082 fi 1083 xtrace_restore 1084} 1085 1086function skip_run_test_with_warning() { 1087 echo "WARNING: $1" 1088 echo "Test run may fail if run with autorun.sh" 1089 echo "Please check your $rootdir/test/common/skipped_tests.txt" 1090} 1091 1092function print_backtrace() { 1093 # if errexit is not enabled, don't print a backtrace 1094 [[ "$-" =~ e ]] || return 0 1095 1096 local args=("${BASH_ARGV[@]}") 1097 1098 xtrace_disable 1099 # Reset IFS in case we were called from an environment where it was modified 1100 IFS=" "$'\t'$'\n' 1101 echo "========== Backtrace start: ==========" 1102 echo "" 1103 for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do 1104 local func="${FUNCNAME[$i]}" 1105 local line_nr="${BASH_LINENO[$((i - 1))]}" 1106 local src="${BASH_SOURCE[$i]}" 1107 local bt="" cmdline=() 1108 1109 if [[ -f $src ]]; then 1110 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 1111 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 1112 fi 1113 1114 # If extdebug set the BASH_ARGC[i], try to fetch all the args 1115 if ((BASH_ARGC[i] > 0)); then 1116 # Use argc as index to reverse the stack 1117 local argc=${BASH_ARGC[i]} arg 1118 for arg in "${args[@]::BASH_ARGC[i]}"; do 1119 cmdline[argc--]="[\"$arg\"]" 1120 done 1121 args=("${args[@]:BASH_ARGC[i]}") 1122 fi 1123 1124 echo "in $src:$line_nr -> $func($( 1125 IFS="," 1126 printf '%s\n' "${cmdline[*]:-[]}" 1127 ))" 1128 echo " ..." 1129 echo "${bt:-backtrace unavailable}" 1130 echo " ..." 1131 done 1132 echo "" 1133 echo "========== Backtrace end ==========" 1134 xtrace_restore 1135 return 0 1136} 1137 1138function waitforserial() { 1139 local i=0 1140 local nvme_device_counter=1 nvme_devices=0 1141 if [[ -n "$2" ]]; then 1142 nvme_device_counter=$2 1143 fi 1144 1145 # Wait initially for min 2s to make sure all devices are ready for use. 1146 sleep 2 1147 while ((i++ <= 15)); do 1148 nvme_devices=$(lsblk -l -o NAME,SERIAL | grep -c "$1") 1149 ((nvme_devices == nvme_device_counter)) && return 0 1150 if ((nvme_devices > nvme_device_counter)); then 1151 echo "$nvme_device_counter device(s) expected, found $nvme_devices" >&2 1152 fi 1153 echo "Waiting for devices" 1154 sleep 1 1155 done 1156 return 1 1157} 1158 1159function waitforserial_disconnect() { 1160 local i=0 1161 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1162 [ $i -lt 15 ] || break 1163 i=$((i + 1)) 1164 echo "Waiting for disconnect devices" 1165 sleep 1 1166 done 1167 1168 if lsblk -l -o NAME,SERIAL | grep -q -w $1; then 1169 return 1 1170 fi 1171 1172 return 0 1173} 1174 1175function waitforblk() { 1176 local i=0 1177 while ! lsblk -l -o NAME | grep -q -w $1; do 1178 [ $i -lt 15 ] || break 1179 i=$((i + 1)) 1180 sleep 1 1181 done 1182 1183 if ! lsblk -l -o NAME | grep -q -w $1; then 1184 return 1 1185 fi 1186 1187 return 0 1188} 1189 1190function waitforblk_disconnect() { 1191 local i=0 1192 while lsblk -l -o NAME | grep -q -w $1; do 1193 [ $i -lt 15 ] || break 1194 i=$((i + 1)) 1195 sleep 1 1196 done 1197 1198 if lsblk -l -o NAME | grep -q -w $1; then 1199 return 1 1200 fi 1201 1202 return 0 1203} 1204 1205function waitforfile() { 1206 local i=0 1207 while [ ! -e $1 ]; do 1208 [ $i -lt 200 ] || break 1209 i=$((i + 1)) 1210 sleep 0.1 1211 done 1212 1213 if [ ! -e $1 ]; then 1214 return 1 1215 fi 1216 1217 return 0 1218} 1219 1220function fio_config_gen() { 1221 local config_file=$1 1222 local workload=$2 1223 local bdev_type=$3 1224 local env_context=$4 1225 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1226 1227 if [ -e "$config_file" ]; then 1228 echo "Configuration File Already Exists!: $config_file" 1229 return 1 1230 fi 1231 1232 if [ -z "$workload" ]; then 1233 workload=randrw 1234 fi 1235 1236 if [ -n "$env_context" ]; then 1237 env_context="env_context=$env_context" 1238 fi 1239 1240 touch $1 1241 1242 cat > $1 << EOL 1243[global] 1244thread=1 1245$env_context 1246group_reporting=1 1247direct=1 1248norandommap=1 1249percentile_list=50:99:99.9:99.99:99.999 1250time_based=1 1251ramp_time=0 1252EOL 1253 1254 if [ "$workload" == "verify" ]; then 1255 cat <<- EOL >> $config_file 1256 verify=sha1 1257 verify_backlog=1024 1258 rw=randwrite 1259 EOL 1260 1261 # To avoid potential data race issue due to the AIO device 1262 # flush mechanism, add the flag to serialize the writes. 1263 # This is to fix the intermittent IO failure issue of #935 1264 if [ "$bdev_type" == "AIO" ]; then 1265 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1266 echo "serialize_overlap=1" >> $config_file 1267 fi 1268 fi 1269 elif [ "$workload" == "trim" ]; then 1270 echo "rw=trimwrite" >> $config_file 1271 else 1272 echo "rw=$workload" >> $config_file 1273 fi 1274} 1275 1276function fio_plugin() { 1277 # Setup fio binary cmd line 1278 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1279 # gcc and clang uses different sanitizer libraries 1280 local sanitizers=(libasan libclang_rt.asan) 1281 local plugin=$1 1282 shift 1283 1284 local asan_lib= 1285 for sanitizer in "${sanitizers[@]}"; do 1286 asan_lib=$(ldd $plugin | grep $sanitizer | awk '{print $3}') 1287 if [[ -n "$asan_lib" ]]; then 1288 break 1289 fi 1290 done 1291 1292 # Preload the sanitizer library to fio if fio_plugin was compiled with it 1293 LD_PRELOAD="$asan_lib $plugin" "$fio_dir"/fio "$@" 1294} 1295 1296function fio_bdev() { 1297 fio_plugin "$rootdir/build/fio/spdk_bdev" "$@" 1298} 1299 1300function fio_nvme() { 1301 fio_plugin "$rootdir/build/fio/spdk_nvme" "$@" 1302} 1303 1304function get_lvs_free_mb() { 1305 local lvs_uuid=$1 1306 local lvs_info 1307 local fc 1308 local cs 1309 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1310 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1311 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1312 1313 # Change to MB's 1314 free_mb=$((fc * cs / 1024 / 1024)) 1315 echo "$free_mb" 1316} 1317 1318function get_bdev_size() { 1319 local bdev_name=$1 1320 local bdev_info 1321 local bs 1322 local nb 1323 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1324 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1325 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1326 1327 # Change to MB's 1328 bdev_size=$((bs * nb / 1024 / 1024)) 1329 echo "$bdev_size" 1330} 1331 1332function autotest_cleanup() { 1333 local autotest_es=$? 1334 xtrace_disable 1335 1336 # catch any stray core files and kill all remaining SPDK processes. Update 1337 # autotest_es in case autotest reported success but cores and/or processes 1338 # were left behind regardless. 1339 1340 process_core || autotest_es=1 1341 reap_spdk_processes || autotest_es=1 1342 1343 $rootdir/scripts/setup.sh reset 1344 $rootdir/scripts/setup.sh cleanup 1345 if [ $(uname -s) = "Linux" ]; then 1346 modprobe -r uio_pci_generic 1347 fi 1348 rm -rf "$asan_suppression_file" 1349 if [[ -n $old_core_pattern ]]; then 1350 echo "$old_core_pattern" > /proc/sys/kernel/core_pattern 1351 fi 1352 if [[ -e /proc/$udevadm_pid/status ]]; then 1353 kill "$udevadm_pid" || : 1354 fi 1355 1356 shopt -s nullglob 1357 local storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 1358 shopt -u nullglob 1359 1360 if ((${#storage_fallback_purge[@]} > 0)); then 1361 rm -rf "${storage_fallback_purge[@]}" 1362 fi 1363 1364 if ((autotest_es)); then 1365 if [[ $(uname) == FreeBSD ]]; then 1366 ps aux 1367 elif [[ $(uname) == Linux ]]; then 1368 # Get more detailed view 1369 grep . /proc/[0-9]*/status 1370 # Dump some extra info into kernel log 1371 echo "######## Autotest Cleanup Dump ########" > /dev/kmsg 1372 # Show cpus backtraces 1373 echo l > /proc/sysrq-trigger 1374 # Show mem usage 1375 echo m > /proc/sysrq-trigger 1376 # show task states 1377 echo t > /proc/sysrq-trigger 1378 # show blocked tasks 1379 echo w > /proc/sysrq-trigger 1380 1381 fi > "$output_dir/proc_list.txt" 2>&1 || : 1382 fi 1383 xtrace_restore 1384 return $autotest_es 1385} 1386 1387function freebsd_update_contigmem_mod() { 1388 if [ $(uname) = FreeBSD ]; then 1389 kldunload contigmem.ko || true 1390 if [ -n "$SPDK_RUN_EXTERNAL_DPDK" ]; then 1391 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/ 1392 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/ 1393 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/ 1394 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/ 1395 else 1396 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1397 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1398 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1399 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1400 fi 1401 fi 1402} 1403 1404function get_nvme_name_from_bdf() { 1405 blkname=() 1406 1407 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1408 if [ -z "$nvme_devs" ]; then 1409 return 1410 fi 1411 for dev in $nvme_devs; do 1412 link_name=$(readlink /sys/block/$dev/device/device) || true 1413 if [ -z "$link_name" ]; then 1414 link_name=$(readlink /sys/block/$dev/device) 1415 fi 1416 bdf=$(basename "$link_name") 1417 if [ "$bdf" = "$1" ]; then 1418 blkname+=($dev) 1419 fi 1420 done 1421 1422 printf '%s\n' "${blkname[@]}" 1423} 1424 1425function get_nvme_ctrlr_from_bdf() { 1426 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1427 if [[ -z "$bdf_sysfs_path" ]]; then 1428 return 1429 fi 1430 1431 printf '%s\n' "$(basename $bdf_sysfs_path)" 1432} 1433 1434# Get BDF addresses of all NVMe drives currently attached to 1435# uio-pci-generic or vfio-pci 1436function get_nvme_bdfs() { 1437 xtrace_disable 1438 bdfs=$(jq -r .config[].params.traddr <<< $($rootdir/scripts/gen_nvme.sh)) 1439 if [[ -z $bdfs ]]; then 1440 echo "No devices to test on!" 1441 exit 1 1442 fi 1443 echo "$bdfs" 1444 xtrace_restore 1445} 1446 1447# Same as function above, but just get the first disks BDF address 1448function get_first_nvme_bdf() { 1449 head -1 <<< "$(get_nvme_bdfs)" 1450} 1451 1452function nvme_namespace_revert() { 1453 $rootdir/scripts/setup.sh 1454 sleep 1 1455 bdfs=$(get_nvme_bdfs) 1456 1457 $rootdir/scripts/setup.sh reset 1458 1459 for bdf in $bdfs; do 1460 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1461 if [[ -z "$nvme_ctrlr" ]]; then 1462 continue 1463 fi 1464 1465 # Check Optional Admin Command Support for Namespace Management 1466 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1467 oacs_ns_manage=$((oacs & 0x8)) 1468 1469 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1470 # This assumes every NVMe controller contains single namespace, 1471 # encompassing Total NVM Capacity and formatted as 512 block size. 1472 # 512 block size is needed for test/vhost/vhost_boot.sh to 1473 # successfully run. 1474 1475 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1476 if [[ "$unvmcap" -eq 0 ]]; then 1477 # All available space already used 1478 continue 1479 fi 1480 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1481 blksize=512 1482 1483 size=$((tnvmcap / blksize)) 1484 1485 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c 0 || true 1486 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1487 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1488 nvme attach-ns ${nvme_ctrlr} -n 1 -c 0 1489 nvme reset ${nvme_ctrlr} 1490 waitforfile "${nvme_ctrlr}n1" 1491 fi 1492 done 1493} 1494 1495# Get BDFs based on device ID, such as 0x0a54 1496function get_nvme_bdfs_by_id() { 1497 local bdfs=() 1498 1499 for bdf in $(get_nvme_bdfs); do 1500 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1501 if [[ "$device" == "$1" ]]; then 1502 bdfs+=($bdf) 1503 fi 1504 done 1505 1506 printf '%s\n' "${bdfs[@]}" 1507} 1508 1509function opal_revert_cleanup() { 1510 # The OPAL CI tests is only used for P4510 devices. 1511 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1512 if [[ -z ${bdfs[0]} ]]; then 1513 return 0 1514 fi 1515 1516 $SPDK_BIN_DIR/spdk_tgt & 1517 spdk_tgt_pid=$! 1518 waitforlisten $spdk_tgt_pid 1519 1520 bdf_id=0 1521 for bdf in "${bdfs[@]}"; do 1522 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme"${bdf_id} -t "pcie" -a ${bdf} 1523 # Ignore if this fails. 1524 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b "nvme"${bdf_id} -p test || true 1525 ((++bdf_id)) 1526 done 1527 1528 killprocess $spdk_tgt_pid 1529} 1530 1531function pap() { 1532 while read -r file; do 1533 cat <<- FILE 1534 --- $file --- 1535 $(<"$file") 1536 --- $file --- 1537 FILE 1538 rm -f "$file" 1539 done < <(find "$@" -type f | sort -u) 1540} 1541 1542function get_proc_paths() { 1543 local procs proc 1544 if [[ $(uname -s) == Linux ]]; then 1545 for proc in /proc/[0-9]*; do 1546 [[ -e $proc/exe ]] || continue 1547 procs[${proc##*/}]=$(readlink -f "$proc/exe") 1548 done 1549 elif [[ $(uname -s) == FreeBSD ]]; then 1550 while read -r proc _ _ path; do 1551 [[ -e $path ]] || continue 1552 procs[proc]=$path 1553 done < <(procstat -ab) 1554 fi 1555 1556 for proc in "${!procs[@]}"; do 1557 echo "$proc" "${procs[proc]}" 1558 done 1559} 1560 1561is_exec_file() { [[ -f $1 && $(file "$1") =~ ELF.+executable ]]; } 1562 1563function reap_spdk_processes() { 1564 local bins bin 1565 local misc_bins 1566 1567 while read -r bin; do 1568 is_exec_file "$bin" && misc_bins+=("$bin") 1569 done < <(find "$rootdir"/test/{app,env,event} -type f) 1570 1571 mapfile -t bins < <(readlink -f "$SPDK_BIN_DIR/"* "$SPDK_EXAMPLE_DIR/"* "${misc_bins[@]}") 1572 1573 local spdk_pid spdk_pids path 1574 while read -r spdk_pid path; do 1575 if [[ ${bins[*]/$path/} != "${bins[*]}" ]]; then 1576 echo "$path is still up ($spdk_pid), killing" 1577 spdk_pids[spdk_pid]=$path 1578 fi 1579 done < <(get_proc_paths) 1580 1581 ((${#spdk_pids[@]} > 0)) || return 0 1582 1583 kill -SIGTERM "${!spdk_pids[@]}" 2> /dev/null || : 1584 # Wait a bit and then use the stick 1585 sleep 2 1586 kill -SIGKILL "${!spdk_pids[@]}" 2> /dev/null || : 1587 1588 return 1 1589} 1590 1591function is_block_zoned() { 1592 local device=$1 1593 1594 [[ -e /sys/block/$device/queue/zoned ]] || return 1 1595 [[ $(< "/sys/block/$device/queue/zoned") != none ]] 1596} 1597 1598function get_zoned_devs() { 1599 local -gA zoned_devs=() 1600 local nvme bdf 1601 1602 for nvme in /sys/block/nvme*; do 1603 if is_block_zoned "${nvme##*/}"; then 1604 zoned_devs["${nvme##*/}"]=$(< "$nvme/device/address") 1605 fi 1606 done 1607} 1608 1609# Define temp storage for all the tests. Look for 2GB at minimum 1610set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1611 1612set -o errtrace 1613shopt -s extdebug 1614trap "trap - ERR; print_backtrace >&2" ERR 1615 1616PS4=' \t -- ${BASH_SOURCE#${BASH_SOURCE%/*/*}/}@${LINENO} -- \$ ' 1617if $SPDK_AUTOTEST_X; then 1618 # explicitly enable xtraces, overriding any tracking information. 1619 unset XTRACE_DISABLED 1620 unset XTRACE_NESTING_LEVEL 1621 xtrace_fd 1622 xtrace_enable 1623else 1624 xtrace_restore 1625fi 1626
