1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2015 Intel Corporation 4# All rights reserved. 5# Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 6# 7rpc_py=rpc_cmd 8 9function xtrace_disable() { 10 set +x 11 X_STACK+=("${FUNCNAME[*]}") # push 12} 13 14function xtrace_restore() { 15 # unset'ing foo[-1] under older Bash (4.2 -> Centos7) won't work, hence the dance 16 unset -v "X_STACK[${#X_STACK[@]} - 1 < 0 ? 0 : ${#X_STACK[@]} - 1]" # pop 17 if ((${#X_STACK[@]} == 0)); then 18 set -x 19 fi 20} 21 22function xtrace_disable_per_cmd() { eval "$* ${BASH_XTRACEFD}> /dev/null"; } 23 24function xtrace_fd() { 25 if [[ -n ${BASH_XTRACEFD:-} && -e /proc/self/fd/$BASH_XTRACEFD ]]; then 26 # Close it first to make sure it's sane 27 exec {BASH_XTRACEFD}>&- 28 fi 29 exec {BASH_XTRACEFD}>&2 30 31 xtrace_restore 32} 33 34set -e 35shopt -s nullglob 36shopt -s extglob 37shopt -s inherit_errexit 38 39if [ -z "${output_dir:-}" ]; then 40 mkdir -p "$rootdir/../output" 41 export output_dir="$rootdir/../output" 42fi 43 44if [[ -e $rootdir/test/common/build_config.sh ]]; then 45 source "$rootdir/test/common/build_config.sh" 46elif [[ -e $rootdir/mk/config.mk ]]; then 47 build_config=$(< "$rootdir/mk/config.mk") 48 source <(echo "${build_config//\?=/=}") 49else 50 source "$rootdir/CONFIG" 51fi 52 53# Source scripts after the config so that the definitions are available. 54source "$rootdir/test/common/applications.sh" 55source "$rootdir/scripts/common.sh" 56source "$rootdir/scripts/perf/pm/common" 57 58: ${RUN_NIGHTLY:=0} 59export RUN_NIGHTLY 60 61# Set defaults for missing test config options 62: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 63export SPDK_AUTOTEST_DEBUG_APPS 64: ${SPDK_RUN_VALGRIND=0} 65export SPDK_RUN_VALGRIND 66: ${SPDK_RUN_FUNCTIONAL_TEST=0} 67export SPDK_RUN_FUNCTIONAL_TEST 68: ${SPDK_TEST_UNITTEST=0} 69export SPDK_TEST_UNITTEST 70: ${SPDK_TEST_AUTOBUILD=""} 71export SPDK_TEST_AUTOBUILD 72: ${SPDK_TEST_RELEASE_BUILD=0} 73export SPDK_TEST_RELEASE_BUILD 74: ${SPDK_TEST_ISAL=0} 75export SPDK_TEST_ISAL 76: ${SPDK_TEST_ISCSI=0} 77export SPDK_TEST_ISCSI 78: ${SPDK_TEST_ISCSI_INITIATOR=0} 79export SPDK_TEST_ISCSI_INITIATOR 80: ${SPDK_TEST_NVME=0} 81export SPDK_TEST_NVME 82: ${SPDK_TEST_NVME_PMR=0} 83export SPDK_TEST_NVME_PMR 84: ${SPDK_TEST_NVME_BP=0} 85export SPDK_TEST_NVME_BP 86: ${SPDK_TEST_NVME_CLI=0} 87export SPDK_TEST_NVME_CLI 88: ${SPDK_TEST_NVME_CUSE=0} 89export SPDK_TEST_NVME_CUSE 90: ${SPDK_TEST_NVME_FDP=0} 91export SPDK_TEST_NVME_FDP 92: ${SPDK_TEST_NVMF=0} 93export SPDK_TEST_NVMF 94: ${SPDK_TEST_VFIOUSER=0} 95export SPDK_TEST_VFIOUSER 96: ${SPDK_TEST_VFIOUSER_QEMU=0} 97export SPDK_TEST_VFIOUSER_QEMU 98: ${SPDK_TEST_FUZZER=0} 99export SPDK_TEST_FUZZER 100: ${SPDK_TEST_FUZZER_SHORT=0} 101export SPDK_TEST_FUZZER_SHORT 102: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 103export SPDK_TEST_NVMF_TRANSPORT 104: ${SPDK_TEST_RBD=0} 105export SPDK_TEST_RBD 106: ${SPDK_TEST_VHOST=0} 107export SPDK_TEST_VHOST 108: ${SPDK_TEST_BLOCKDEV=0} 109export SPDK_TEST_BLOCKDEV 110: ${SPDK_TEST_RAID=0} 111export SPDK_TEST_RAID 112: ${SPDK_TEST_IOAT=0} 113export SPDK_TEST_IOAT 114: ${SPDK_TEST_BLOBFS=0} 115export SPDK_TEST_BLOBFS 116: ${SPDK_TEST_VHOST_INIT=0} 117export SPDK_TEST_VHOST_INIT 118: ${SPDK_TEST_LVOL=0} 119export SPDK_TEST_LVOL 120: ${SPDK_TEST_VBDEV_COMPRESS=0} 121export SPDK_TEST_VBDEV_COMPRESS 122: ${SPDK_RUN_ASAN=0} 123export SPDK_RUN_ASAN 124: ${SPDK_RUN_UBSAN=0} 125export SPDK_RUN_UBSAN 126: ${SPDK_RUN_EXTERNAL_DPDK=""} 127export SPDK_RUN_EXTERNAL_DPDK 128: ${SPDK_RUN_NON_ROOT=0} 129export SPDK_RUN_NON_ROOT 130: ${SPDK_TEST_CRYPTO=0} 131export SPDK_TEST_CRYPTO 132: ${SPDK_TEST_FTL=0} 133export SPDK_TEST_FTL 134: ${SPDK_TEST_OCF=0} 135export SPDK_TEST_OCF 136: ${SPDK_TEST_VMD=0} 137export SPDK_TEST_VMD 138: ${SPDK_TEST_OPAL=0} 139export SPDK_TEST_OPAL 140: ${SPDK_TEST_NATIVE_DPDK} 141export SPDK_TEST_NATIVE_DPDK 142: ${SPDK_AUTOTEST_X=true} 143export SPDK_AUTOTEST_X 144: ${SPDK_TEST_URING=0} 145export SPDK_TEST_URING 146: ${SPDK_TEST_USDT=0} 147export SPDK_TEST_USDT 148: ${SPDK_TEST_USE_IGB_UIO:=0} 149export SPDK_TEST_USE_IGB_UIO 150: ${SPDK_TEST_SCHEDULER:=0} 151export SPDK_TEST_SCHEDULER 152: ${SPDK_TEST_SCANBUILD:=0} 153export SPDK_TEST_SCANBUILD 154: ${SPDK_TEST_NVMF_NICS:=} 155export SPDK_TEST_NVMF_NICS 156: ${SPDK_TEST_SMA=0} 157export SPDK_TEST_SMA 158: ${SPDK_TEST_DAOS=0} 159export SPDK_TEST_DAOS 160: ${SPDK_TEST_XNVME:=0} 161export SPDK_TEST_XNVME 162: ${SPDK_TEST_ACCEL:=0} 163export SPDK_TEST_ACCEL 164: ${SPDK_TEST_ACCEL_DSA=0} 165export SPDK_TEST_ACCEL_DSA 166: ${SPDK_TEST_ACCEL_IAA=0} 167export SPDK_TEST_ACCEL_IAA 168# Comma-separated list of fuzzer targets matching test/fuzz/llvm/$target 169: ${SPDK_TEST_FUZZER_TARGET:=} 170export SPDK_TEST_FUZZER_TARGET 171: ${SPDK_TEST_NVMF_MDNS=0} 172export SPDK_TEST_NVMF_MDNS 173: ${SPDK_JSONRPC_GO_CLIENT=0} 174export SPDK_JSONRPC_GO_CLIENT 175: ${SPDK_TEST_SETUP=0} 176export SPDK_TEST_SETUP 177 178# always test with SPDK shared objects. 179export SPDK_LIB_DIR="$rootdir/build/lib" 180export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib" 181export VFIO_LIB_DIR="$rootdir/build/libvfio-user/usr/local/lib" 182export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR:$VFIO_LIB_DIR 183 184# Tell setup.sh to wait for block devices upon each reset 185export PCI_BLOCK_SYNC_ON_RESET=yes 186 187# Export PYTHONPATH with addition of RPC framework. New scripts can be created 188# specific use cases for tests. 189export PYTHONPATH=$PYTHONPATH:$rootdir/python 190 191# Don't create Python .pyc files. When running with sudo these will be 192# created with root ownership and can cause problems when cleaning the repository. 193export PYTHONDONTWRITEBYTECODE=1 194 195# Export new_delete_type_mismatch to skip the known bug that exists in librados 196# https://tracker.ceph.com/issues/24078 197export ASAN_OPTIONS=new_delete_type_mismatch=0:disable_coredump=0:abort_on_error=1:use_sigaltstack=0 198export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1:disable_coredump=0:exitcode=134' 199 200# Export LeakSanitizer option to use suppression file in order to prevent false positives 201# and known leaks in external executables or libraries from showing up. 202asan_suppression_file="/var/tmp/asan_suppression_file" 203rm -rf "$asan_suppression_file" 2> /dev/null || sudo rm -rf "$asan_suppression_file" 204cat << EOL >> "$asan_suppression_file" 205# ASAN has some bugs around thread_local variables. We have a destructor in place 206# to free the thread contexts, but ASAN complains about the leak before those 207# destructors have a chance to run. So suppress this one specific leak using 208# LSAN_OPTIONS. 209leak:spdk_fs_alloc_thread_ctx 210 211# Suppress known leaks in fio project 212leak:$CONFIG_FIO_SOURCE_DIR/parse.c 213leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 214leak:$CONFIG_FIO_SOURCE_DIR/init.c 215leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 216leak:fio_memalign 217leak:spdk_fio_io_u_init 218# Suppress leaks in gperftools-libs from fio 219leak:libtcmalloc_minimal.so 220 221# Suppress leaks in libiscsi 222leak:libiscsi.so 223 224# Suppress leaks in libcrypto 225# Below is caused by openssl 3.0.8 leaks 226leak:libcrypto.so 227 228# Suppress leaks in accel-config 229# Versions with unresolved leaks: 230# v3.4.6.4 [Fedora 37] 231leak:add_wq 232leak:add_group 233# v3.5.2 [Fedora 38] 234leak:accfg_get_param_str 235# v4.0 [Fedora 39] 236leak:__scandir64_tail 237EOL 238 239# Suppress leaks in libfuse3 240echo "leak:libfuse3.so" >> "$asan_suppression_file" 241 242export LSAN_OPTIONS=suppressions="$asan_suppression_file" 243 244export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 245 246if [ -z "${DEPENDENCY_DIR:-}" ]; then 247 export DEPENDENCY_DIR=/var/spdk/dependencies 248else 249 export DEPENDENCY_DIR 250fi 251 252# Export location of where all the SPDK binaries are 253export SPDK_BIN_DIR="$rootdir/build/bin" 254export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 255 256# for vhost, vfio-user tests 257export QEMU_BIN=${QEMU_BIN:-} 258export VFIO_QEMU_BIN=${VFIO_QEMU_BIN:-} 259 260export AR_TOOL=$rootdir/scripts/ar-xnvme-fixer 261 262# For testing nvmes which are attached to some sort of a fanout switch in the CI pool 263export UNBIND_ENTIRE_IOMMU_GROUP=${UNBIND_ENTIRE_IOMMU_GROUP:-no} 264 265_LCOV_MAIN=0 266_LCOV_LLVM=1 267_LCOV=$LCOV_MAIN 268[[ $CC == *clang* || $SPDK_TEST_FUZZER -eq 1 ]] && _LCOV=$_LCOV_LLVM 269 270_lcov_opt[_LCOV_LLVM]="--gcov-tool $rootdir/test/fuzz/llvm/llvm-gcov.sh" 271_lcov_opt[_LCOV_MAIN]="" 272 273lcov_opt=${_lcov_opt[_LCOV]} 274 275# pass our valgrind desire on to unittest.sh 276if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 277 export valgrind='' 278else 279 # unset all DEBUGINFOD_* vars that may affect our valgrind instance 280 unset -v "${!DEBUGINFOD_@}" 281fi 282 283if [ "$(uname -s)" = "Linux" ]; then 284 HUGEMEM=${HUGEMEM:-4096} 285 export CLEAR_HUGE=yes 286 287 MAKE="make" 288 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 289elif [ "$(uname -s)" = "FreeBSD" ]; then 290 MAKE="gmake" 291 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 292 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 293 HUGEMEM=${HUGEMEM:-2048} 294elif [ "$(uname -s)" = "Windows" ]; then 295 MAKE="make" 296 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 297 # Keep the default 2GB for Windows. 298 HUGEMEM=${HUGEMEM:-2048} 299else 300 echo "Unknown OS \"$(uname -s)\"" 301 exit 1 302fi 303 304export HUGEMEM=$HUGEMEM 305 306NO_HUGE=() 307TEST_MODE= 308for i in "$@"; do 309 case "$i" in 310 --iso) 311 TEST_MODE=iso 312 ;; 313 --transport=*) 314 TEST_TRANSPORT="${i#*=}" 315 ;; 316 --sock=*) 317 TEST_SOCK="${i#*=}" 318 ;; 319 --no-hugepages) 320 NO_HUGE=(--no-huge -s 1024) 321 ;; 322 --interrupt-mode) 323 TEST_INTERRUPT_MODE=1 324 ;; 325 esac 326done 327 328# start rpc.py coprocess if it's not started yet 329if [[ -z ${RPC_PIPE_PID:-} ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 330 # Include list to all known plugins we use in the tests 331 PYTHONPATH+=":$rootdir/test/rpc_plugins" 332 coproc RPC_PIPE { PYTHONPATH="$PYTHONPATH" "$rootdir/scripts/rpc.py" --server; } 333 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 334 # all descriptors will automatically close together with this bash 335 # process, this will make rpc.py stop reading and exit gracefully 336fi 337 338function set_test_storage() { 339 [[ -v testdir ]] || return 0 340 341 local requested_size=$1 # bytes 342 local mount target_dir 343 344 local -A mounts fss sizes avails uses 345 local source fs size avail mount use 346 347 local storage_fallback storage_candidates 348 349 storage_fallback=$(mktemp -udt spdk.XXXXXX) 350 storage_candidates=( 351 "$testdir" 352 "$storage_fallback/tests/${testdir##*/}" 353 "$storage_fallback" 354 ) 355 356 if [[ -n ${ADD_TEST_STORAGE:-} ]]; then 357 # List of dirs|mounts separated by whitespaces 358 storage_candidates+=($ADD_TEST_STORAGE) 359 fi 360 361 if [[ -n ${DEDICATED_TEST_STORAGE:-} ]]; then 362 # Single, dedicated dir|mount 363 storage_candidates=("$DEDICATED_TEST_STORAGE") 364 fi 365 366 mkdir -p "${storage_candidates[@]}" 367 368 # add some headroom - 64M 369 requested_size=$((requested_size + (64 << 20))) 370 371 while read -r source fs size use avail _ mount; do 372 mounts["$mount"]=$source fss["$mount"]=$fs 373 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 374 uses["$mount"]=$((use * 1024)) 375 done < <(df -T | grep -v Filesystem) 376 377 printf '* Looking for test storage...\n' >&2 378 379 local target_space new_size 380 for target_dir in "${storage_candidates[@]}"; do 381 # FreeBSD's df is lacking the --output arg 382 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 383 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 384 385 target_space=${avails["$mount"]} 386 if ((target_space == 0 || target_space < requested_size)); then 387 continue 388 fi 389 if ((target_space >= requested_size)); then 390 # For in-memory fs, and / make sure our requested size won't fill most of the space. 391 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 392 new_size=$((uses["$mount"] + requested_size)) 393 if ((new_size * 100 / sizes["$mount"] > 95)); then 394 continue 395 fi 396 fi 397 fi 398 export SPDK_TEST_STORAGE=$target_dir 399 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 400 return 0 401 done 402 printf '* Test storage is not available\n' 403 return 1 404} 405 406function get_config_params() { 407 xtrace_disable 408 config_params='--enable-debug --enable-werror' 409 410 # for options with dependencies but no test flag, set them here 411 if [ -f /usr/include/infiniband/verbs.h ]; then 412 config_params+=' --with-rdma' 413 fi 414 415 if [ $SPDK_TEST_USDT -eq 1 ]; then 416 config_params+=" --with-usdt" 417 fi 418 419 case "$(uname -s)" in 420 FreeBSD) [[ $(sysctl -n hw.model) == Intel* ]] ;; 421 Linux) [[ $(< /proc/cpuinfo) == *GenuineIntel* ]] ;; 422 *) false ;; 423 esac && config_params+=" --with-idxd" || config_params+=" --without-idxd" 424 425 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 426 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 427 fi 428 429 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 430 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 431 fi 432 433 if [ -d /usr/include/iscsi ]; then 434 [[ $(< /usr/include/iscsi/iscsi.h) =~ "define LIBISCSI_API_VERSION ("([0-9]+)")" ]] \ 435 && libiscsi_version=${BASH_REMATCH[1]} 436 if ((libiscsi_version >= 20150621)); then 437 config_params+=' --with-iscsi-initiator' 438 fi 439 fi 440 441 if [[ $SPDK_TEST_UNITTEST -eq 0 && 442 $SPDK_TEST_SCANBUILD -eq 0 && -z ${SPDK_TEST_AUTOBUILD:-} ]]; then 443 config_params+=' --disable-unit-tests' 444 fi 445 446 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]; then 447 if ge "$(nasm --version | awk '{print $3}')" 2.14 && [[ $SPDK_TEST_ISAL -eq 1 ]]; then 448 config_params+=' --with-vbdev-compress --with-dpdk-compressdev' 449 fi 450 fi 451 452 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 453 config_params+=' --with-rbd' 454 fi 455 456 # for options with no required dependencies, just test flags, set them here 457 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 458 config_params+=' --with-crypto' 459 fi 460 461 if [ $SPDK_TEST_OCF -eq 1 ]; then 462 config_params+=" --with-ocf" 463 fi 464 465 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 466 config_params+=' --enable-ubsan' 467 fi 468 469 if [ $SPDK_RUN_ASAN -eq 1 ]; then 470 config_params+=' --enable-asan' 471 fi 472 473 config_params+=' --enable-coverage' 474 475 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 476 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 477 config_params+=' --with-fuse' 478 fi 479 fi 480 481 if [[ -f /usr/include/liburing/io_uring.h && -f /usr/include/linux/ublk_cmd.h ]]; then 482 config_params+=' --with-ublk' 483 fi 484 485 if [ $SPDK_TEST_RAID -eq 1 ]; then 486 config_params+=' --with-raid5f' 487 fi 488 489 if [ $SPDK_TEST_VFIOUSER -eq 1 ] || [ $SPDK_TEST_VFIOUSER_QEMU -eq 1 ] || [ $SPDK_TEST_SMA -eq 1 ]; then 490 config_params+=' --with-vfio-user' 491 fi 492 493 # Check whether liburing library header exists 494 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 495 config_params+=' --with-uring' 496 fi 497 498 if [ -n "${SPDK_RUN_EXTERNAL_DPDK:-}" ]; then 499 config_params+=" --with-dpdk=$SPDK_RUN_EXTERNAL_DPDK" 500 fi 501 502 if [[ $SPDK_TEST_SMA -eq 1 ]]; then 503 config_params+=' --with-sma' 504 config_params+=' --with-crypto' 505 fi 506 507 if [ -f /usr/include/daos.h ] && [ $SPDK_TEST_DAOS -eq 1 ]; then 508 config_params+=' --with-daos' 509 fi 510 511 # Make the xnvme module available for the tests 512 if [[ $SPDK_TEST_XNVME -eq 1 ]]; then 513 config_params+=' --with-xnvme' 514 fi 515 516 if [[ $SPDK_TEST_FUZZER -eq 1 ]]; then 517 config_params+=" $(get_fuzzer_target_config)" 518 fi 519 520 if [[ $SPDK_TEST_NVMF_MDNS -eq 1 ]]; then 521 config_params+=' --with-avahi' 522 fi 523 524 if [[ $SPDK_JSONRPC_GO_CLIENT -eq 1 ]]; then 525 config_params+=' --with-golang' 526 fi 527 528 echo "$config_params" 529 xtrace_restore 530} 531 532function get_fuzzer_target_config() { 533 local -A fuzzer_targets_to_config=() 534 local config target 535 536 fuzzer_targets_to_config["vfio"]="--with-vfio-user" 537 for target in $(get_fuzzer_targets); do 538 [[ -n ${fuzzer_targets_to_config["$target"]:-} ]] || continue 539 config+=("${fuzzer_targets_to_config["$target"]}") 540 done 541 542 if ((${#config[@]} > 0)); then 543 echo "${config[*]}" 544 fi 545} 546 547function get_fuzzer_targets() { 548 local fuzzers=() 549 550 if [[ -n ${SPDK_TEST_FUZZER_TARGET:-} ]]; then 551 IFS="," read -ra fuzzers <<< "$SPDK_TEST_FUZZER_TARGET" 552 else 553 fuzzers=("$rootdir/test/fuzz/llvm/"*) 554 fuzzers=("${fuzzers[@]##*/}") 555 fi 556 557 echo "${fuzzers[*]}" 558} 559 560function rpc_cmd() { 561 xtrace_disable 562 local rsp rc=1 563 local stdin cmd cmds_number=0 status_number=0 status 564 565 if (($#)); then 566 cmds_number=1 567 echo "$@" >&$RPC_PIPE_INPUT 568 elif [[ ! -t 0 ]]; then 569 mapfile -t stdin <&0 570 cmds_number=${#stdin[@]} 571 printf '%s\n' "${stdin[@]}" >&$RPC_PIPE_INPUT 572 else 573 return 0 574 fi 575 576 while read -t "${RPC_PIPE_TIMEOUT:-15}" -ru $RPC_PIPE_OUTPUT rsp; do 577 if [[ $rsp == "**STATUS="* ]]; then 578 status[${rsp#*=}]=$rsp 579 if ((++status_number == cmds_number)); then 580 break 581 fi 582 continue 583 fi 584 echo "$rsp" 585 done 586 587 rc=${!status[*]} 588 xtrace_restore 589 [[ $rc == 0 ]] 590} 591 592function rpc_cmd_simple_data_json() { 593 594 local elems="$1[@]" elem 595 local -gA jq_out=() 596 local jq val 597 598 local lvs=( 599 "uuid" 600 "name" 601 "base_bdev" 602 "total_data_clusters" 603 "free_clusters" 604 "block_size" 605 "cluster_size" 606 ) 607 608 local bdev=( 609 "name" 610 "aliases[0]" 611 "block_size" 612 "num_blocks" 613 "uuid" 614 "product_name" 615 "supported_io_types.read" 616 "supported_io_types.write" 617 "driver_specific.lvol.clone" 618 "driver_specific.lvol.base_snapshot" 619 "driver_specific.lvol.esnap_clone" 620 "driver_specific.lvol.external_snapshot_name" 621 ) 622 623 [[ -v $elems ]] || return 1 624 625 for elem in "${!elems}"; do 626 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 627 done 628 jq+=',"\n"' 629 630 shift 631 while read -r elem val; do 632 jq_out["$elem"]=$val 633 done < <(rpc_cmd "$@" | jq -jr "$jq") 634 ((${#jq_out[@]} > 0)) || return 1 635} 636 637function valid_exec_arg() { 638 local arg=$1 639 # First argument must be the executable so do some basic sanity checks first. For bash, this 640 # covers two basic cases where es == 126 || es == 127 so catch them early on and fail hard 641 # if needed. 642 case "$(type -t "$arg")" in 643 builtin | function) ;; 644 file) arg=$(type -P "$arg") && [[ -x $arg ]] ;; 645 *) return 1 ;; 646 esac 647} 648 649function NOT() { 650 local es=0 651 652 valid_exec_arg "$@" || return 1 653 "$@" || es=$? 654 655 # Logic looks like so: 656 # - return false if command exit successfully 657 # - return false if command exit after receiving a core signal (FIXME: or any signal?) 658 # - return true if command exit with an error 659 660 # This naively assumes that the process doesn't exit with > 128 on its own. 661 if ((es > 128)); then 662 es=$((es & ~128)) 663 case "$es" in 664 3) ;& # SIGQUIT 665 4) ;& # SIGILL 666 6) ;& # SIGABRT 667 8) ;& # SIGFPE 668 9) ;& # SIGKILL 669 11) es=0 ;; # SIGSEGV 670 *) es=1 ;; 671 esac 672 elif [[ -n ${EXIT_STATUS:-} ]] && ((es != EXIT_STATUS)); then 673 es=0 674 fi 675 676 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 677 ((!es == 0)) 678} 679 680function timing() { 681 direction="$1" 682 testname="$2" 683 684 now=$(date +%s) 685 686 if [ "$direction" = "enter" ]; then 687 export timing_stack="${timing_stack:-};${now}" 688 export test_stack="${test_stack:-};${testname}" 689 else 690 touch "$output_dir/timing.txt" 691 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 692 693 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 694 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 695 696 elapsed=$((now - start_time - child_time)) 697 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 698 699 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 700 fi 701} 702 703function timing_cmd() ( 704 # The use-case here is this: ts=$(timing_cmd echo bar). Since stdout is always redirected 705 # to a pipe handling the $(), lookup the stdin's device and determine if it's sane to send 706 # cmd's output to it. If not, just null it. 707 local cmd_es=$? 708 709 [[ -t 0 ]] && exec {cmd_out}>&0 || exec {cmd_out}> /dev/null 710 711 local time=0 TIMEFORMAT=%2R # seconds 712 713 # We redirect cmd's std{out,err} to a separate fd dup'ed to stdin's device (or /dev/null) to 714 # catch only output from the time builtin - output from the actual cmd would be still visible, 715 # but $() will return just the time's data, hence making it possible to just do: 716 # time_of_super_verbose_cmd=$(timing_cmd super_verbose_cmd) 717 time=$({ time "$@" >&"$cmd_out" 2>&1; } 2>&1) || cmd_es=$? 718 echo "$time" 719 720 return "$cmd_es" 721) 722 723function timing_enter() { 724 xtrace_disable 725 timing "enter" "$1" 726 xtrace_restore 727} 728 729function timing_exit() { 730 xtrace_disable 731 timing "exit" "$1" 732 xtrace_restore 733} 734 735function timing_finish() { 736 [[ -e $output_dir/timing.txt ]] || return 0 737 738 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 739 [[ -x "$flamegraph" ]] || return 1 740 741 "$flamegraph" \ 742 --title 'Build Timing' \ 743 --nametype 'Step:' \ 744 --countname seconds \ 745 "$output_dir/timing.txt" \ 746 > "$output_dir/timing.svg" 747} 748 749function create_test_list() { 750 xtrace_disable 751 # First search all scripts in main SPDK directory. 752 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 753 # Follow up with search in test directory recursively. 754 completion+=$'\n'$(grep -rshI --include="*.sh" --exclude="*autotest_common.sh" -e "run_test " $rootdir/test) 755 printf "%s" "$completion" | grep -v "#" \ 756 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 757 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 758 xtrace_restore 759} 760 761function gdb_attach() { 762 gdb -q --batch \ 763 -ex 'handle SIGHUP nostop pass' \ 764 -ex 'handle SIGQUIT nostop pass' \ 765 -ex 'handle SIGPIPE nostop pass' \ 766 -ex 'handle SIGALRM nostop pass' \ 767 -ex 'handle SIGTERM nostop pass' \ 768 -ex 'handle SIGUSR1 nostop pass' \ 769 -ex 'handle SIGUSR2 nostop pass' \ 770 -ex 'handle SIGCHLD nostop pass' \ 771 -ex 'set print thread-events off' \ 772 -ex 'cont' \ 773 -ex 'thread apply all bt' \ 774 -ex 'quit' \ 775 --tty=/dev/stdout \ 776 -p $1 777} 778 779function process_core() { 780 # Note that this always was racy as we can't really sync with the kernel 781 # to see if there's any core queued up for writing. We could check if 782 # collector is running and wait for it explicitly, but it doesn't seem 783 # to be worth the effort. So assume that if we are being called via 784 # trap, as in, when some error has occurred, wait up to 10s for any 785 # potential cores. If we are called just for cleanup at the very end, 786 # don't wait since all the tests ended successfully, hence having any 787 # critical cores lying around is unlikely. 788 ((autotest_es != 0)) && sleep 10 789 790 local coredumps core 791 792 coredumps=("$output_dir/coredumps/"*.bt.txt) 793 794 ((${#coredumps[@]} > 0)) || return 0 795 chmod -R a+r "$output_dir/coredumps" 796 797 for core in "${coredumps[@]}"; do 798 cat <<- BT 799 ##### CORE BT ${core##*/} ##### 800 801 $(< "$core") 802 803 -- 804 BT 805 done 806 return 1 807} 808 809function process_shm() { 810 type=$1 811 id=$2 812 if [ "$type" = "--pid" ]; then 813 id="pid${id}" 814 fi 815 816 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 817 818 if [[ -z ${shm_files:-} ]]; then 819 echo "SHM File for specified PID or shared memory id: ${id} not found!" 820 return 1 821 fi 822 for n in $shm_files; do 823 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 824 done 825 return 0 826} 827 828# Parameters: 829# $1 - process pid 830# $2 - rpc address (optional) 831# $3 - max retries (optional) 832function waitforlisten() { 833 if [ -z "${1:-}" ]; then 834 exit 1 835 fi 836 837 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 838 local max_retries=${3:-100} 839 840 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 841 # turn off trace for this loop 842 xtrace_disable 843 local ret=0 844 local i 845 for ((i = max_retries; i != 0; i--)); do 846 # if the process is no longer running, then exit the script 847 # since it means the application crashed 848 if ! kill -s 0 $1; then 849 echo "ERROR: process (pid: $1) is no longer running" 850 ret=1 851 break 852 fi 853 854 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 855 break 856 fi 857 858 sleep 0.5 859 done 860 861 xtrace_restore 862 if ((i == 0)); then 863 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 864 ret=1 865 fi 866 return $ret 867} 868 869function waitfornbd() { 870 local nbd_name=$1 871 local i 872 873 for ((i = 1; i <= 20; i++)); do 874 if grep -q -w $nbd_name /proc/partitions; then 875 break 876 else 877 sleep 0.1 878 fi 879 done 880 881 # The nbd device is now recognized as a block device, but there can be 882 # a small delay before we can start I/O to that block device. So loop 883 # here trying to read the first block of the nbd block device to a temp 884 # file. Note that dd returns success when reading an empty file, so we 885 # need to check the size of the output file instead. 886 for ((i = 1; i <= 20; i++)); do 887 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 888 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 889 rm -f "$SPDK_TEST_STORAGE/nbdtest" 890 if [ "$size" != "0" ]; then 891 return 0 892 else 893 sleep 0.1 894 fi 895 done 896 897 return 1 898} 899 900function waitforbdev() { 901 local bdev_name=$1 902 local bdev_timeout=$2 903 local i 904 [[ -z ${bdev_timeout:-} ]] && bdev_timeout=2000 # ms 905 906 $rpc_py bdev_wait_for_examine 907 908 if $rpc_py bdev_get_bdevs -b $bdev_name -t $bdev_timeout; then 909 return 0 910 fi 911 912 return 1 913} 914 915function waitforcondition() { 916 local cond=$1 917 local max=${2:-10} 918 while ((max--)); do 919 if eval $cond; then 920 return 0 921 fi 922 sleep 1 923 done 924 return 1 925} 926 927function make_filesystem() { 928 local fstype=$1 929 local dev_name=$2 930 local i=0 931 local force 932 933 if [ $fstype = ext4 ]; then 934 force=-F 935 else 936 force=-f 937 fi 938 939 while ! mkfs.${fstype} $force ${dev_name}; do 940 if [ $i -ge 15 ]; then 941 return 1 942 fi 943 i=$((i + 1)) 944 sleep 1 945 done 946 947 return 0 948} 949 950function killprocess() { 951 # $1 = process pid 952 if [ -z "${1:-}" ]; then 953 return 1 954 fi 955 956 if kill -0 $1; then 957 if [ $(uname) = Linux ]; then 958 process_name=$(ps --no-headers -o comm= $1) 959 else 960 process_name=$(ps -c -o command $1 | tail -1) 961 fi 962 if [ "$process_name" = "sudo" ]; then 963 # kill the child process, which is the actual app 964 # (assume $1 has just one child) 965 local child 966 child="$(pgrep -P $1)" 967 echo "killing process with pid $child" 968 kill $child 969 else 970 echo "killing process with pid $1" 971 kill $1 972 fi 973 974 # wait for the process regardless if its the dummy sudo one 975 # or the actual app - it should terminate anyway 976 wait $1 977 else 978 # the process is not there anymore 979 echo "Process with pid $1 is not found" 980 fi 981} 982 983function iscsicleanup() { 984 echo "Cleaning up iSCSI connection" 985 iscsiadm -m node --logout || true 986 iscsiadm -m node -o delete || true 987 rm -rf /var/lib/iscsi/nodes/* 988} 989 990function stop_iscsi_service() { 991 if cat /etc/*-release | grep Ubuntu; then 992 service open-iscsi stop 993 else 994 service iscsid stop 995 fi 996} 997 998function start_iscsi_service() { 999 if cat /etc/*-release | grep Ubuntu; then 1000 service open-iscsi start 1001 else 1002 service iscsid start 1003 fi 1004} 1005 1006function rbd_setup() { 1007 # $1 = monitor ip address 1008 # $2 = name of the namespace 1009 if [ -z "${1:-}" ]; then 1010 echo "No monitor IP address provided for ceph" 1011 exit 1 1012 fi 1013 if [ -n "${2:-}" ]; then 1014 if ip netns list | grep "$2"; then 1015 NS_CMD="ip netns exec $2" 1016 else 1017 echo "No namespace $2 exists" 1018 exit 1 1019 fi 1020 fi 1021 1022 if hash ceph; then 1023 export PG_NUM=128 1024 export RBD_POOL=rbd 1025 export RBD_NAME=foo 1026 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 1027 $NS_CMD $rootdir/scripts/ceph/start.sh $1 1028 1029 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 1030 $NS_CMD rbd create $RBD_NAME --size 1000 1031 fi 1032} 1033 1034function rbd_cleanup() { 1035 if hash ceph; then 1036 $rootdir/scripts/ceph/stop.sh || true 1037 rm -f /var/tmp/ceph_raw.img 1038 fi 1039} 1040 1041function daos_setup() { 1042 # $1 = pool name 1043 # $2 = cont name 1044 if [ -z "${1:-}" ]; then 1045 echo "No pool name provided" 1046 exit 1 1047 fi 1048 if [ -z "${2:-}" ]; then 1049 echo "No cont name provided" 1050 exit 1 1051 fi 1052 1053 dmg pool create --size=10G $1 || true 1054 daos container create --type=POSIX --label=$2 $1 || true 1055} 1056 1057function daos_cleanup() { 1058 local pool=${1:-testpool} 1059 local cont=${2:-testcont} 1060 1061 daos container destroy -f $pool $cont || true 1062 sudo dmg pool destroy -f $pool || true 1063} 1064 1065function _start_stub() { 1066 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 1067 # but ASLR can still be unreliable in some cases. 1068 # We will re-enable it again after multi-process testing is complete in kill_stub(). 1069 # Save current setting so it can be restored upon calling kill_stub(). 1070 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 1071 echo 0 > /proc/sys/kernel/randomize_va_space 1072 $rootdir/test/app/stub/stub $1 & 1073 stubpid=$! 1074 echo Waiting for stub to ready for secondary processes... 1075 while ! [ -e /var/run/spdk_stub0 ]; do 1076 # If stub dies while we wait, bail 1077 [[ -e /proc/$stubpid ]] || return 1 1078 sleep 1s 1079 done 1080 echo done. 1081} 1082 1083function start_stub() { 1084 if ! _start_stub "$@"; then 1085 echo "stub failed" >&2 1086 return 1 1087 fi 1088} 1089 1090function kill_stub() { 1091 if [[ -e /proc/$stubpid ]]; then 1092 kill $1 $stubpid 1093 wait $stubpid 1094 fi 2> /dev/null || : 1095 rm -f /var/run/spdk_stub0 1096 # Re-enable ASLR now that we are done with multi-process testing 1097 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 1098 # randomizing and is the default on all recent Linux kernels 1099 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 1100} 1101 1102function run_test() { 1103 if [ $# -le 1 ]; then 1104 echo "Not enough parameters" 1105 echo "usage: run_test test_name test_script [script_params]" 1106 exit 1 1107 fi 1108 1109 xtrace_disable 1110 local test_name="$1" pid 1111 shift 1112 1113 if [ -n "${test_domain:-}" ]; then 1114 export test_domain="${test_domain}.${test_name}" 1115 else 1116 export test_domain="$test_name" 1117 fi 1118 1119 # Signal our daemons to update the test tag 1120 update_tag_monitor_resources "$test_domain" 1121 1122 timing_enter $test_name 1123 echo "************************************" 1124 echo "START TEST $test_name" 1125 echo "************************************" 1126 xtrace_restore 1127 time "$@" 1128 xtrace_disable 1129 echo "************************************" 1130 echo "END TEST $test_name" 1131 echo "************************************" 1132 timing_exit $test_name 1133 1134 export test_domain=${test_domain%"$test_name"} 1135 if [ -n "$test_domain" ]; then 1136 export test_domain=${test_domain%?} 1137 fi 1138 1139 if [ -z "${test_domain:-}" ]; then 1140 echo "top_level $test_name" >> $output_dir/test_completions.txt 1141 else 1142 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 1143 fi 1144 xtrace_restore 1145} 1146 1147function skip_run_test_with_warning() { 1148 echo "WARNING: $1" 1149 echo "Test run may fail if run with autorun.sh" 1150 echo "Please check your $rootdir/test/common/skipped_tests.txt" 1151} 1152 1153function print_backtrace() { 1154 # if errexit is not enabled, don't print a backtrace 1155 [[ "$-" =~ e ]] || return 0 1156 1157 local args=("${BASH_ARGV[@]}") 1158 1159 xtrace_disable 1160 # Reset IFS in case we were called from an environment where it was modified 1161 IFS=" "$'\t'$'\n' 1162 echo "========== Backtrace start: ==========" 1163 echo "" 1164 for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do 1165 local func="${FUNCNAME[$i]}" 1166 local line_nr="${BASH_LINENO[$((i - 1))]}" 1167 local src="${BASH_SOURCE[$i]}" 1168 local bt="" cmdline=() 1169 1170 if [[ -f $src ]]; then 1171 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 1172 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 1173 fi 1174 1175 # If extdebug set the BASH_ARGC[i], try to fetch all the args 1176 if ((BASH_ARGC[i] > 0)); then 1177 # Use argc as index to reverse the stack 1178 local argc=${BASH_ARGC[i]} arg 1179 for arg in "${args[@]::BASH_ARGC[i]}"; do 1180 cmdline[argc--]="[\"$arg\"]" 1181 done 1182 args=("${args[@]:BASH_ARGC[i]}") 1183 fi 1184 1185 echo "in $src:$line_nr -> $func($( 1186 IFS="," 1187 printf '%s\n' "${cmdline[*]:-[]}" 1188 ))" 1189 echo " ..." 1190 echo "${bt:-backtrace unavailable}" 1191 echo " ..." 1192 done 1193 echo "" 1194 echo "========== Backtrace end ==========" 1195 xtrace_restore 1196 return 0 1197} 1198 1199function waitforserial() { 1200 local i=0 1201 local nvme_device_counter=1 nvme_devices=0 1202 if [[ -n "${2:-}" ]]; then 1203 nvme_device_counter=$2 1204 fi 1205 1206 # Wait initially for min 2s to make sure all devices are ready for use. 1207 sleep 2 1208 while ((i++ <= 15)); do 1209 nvme_devices=$(lsblk -l -o NAME,SERIAL | grep -c "$1") 1210 ((nvme_devices == nvme_device_counter)) && return 0 1211 if ((nvme_devices > nvme_device_counter)); then 1212 echo "$nvme_device_counter device(s) expected, found $nvme_devices" >&2 1213 fi 1214 echo "Waiting for devices" 1215 sleep 1 1216 done 1217 return 1 1218} 1219 1220function waitforserial_disconnect() { 1221 local i=0 1222 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1223 [ $i -lt 15 ] || break 1224 i=$((i + 1)) 1225 echo "Waiting for disconnect devices" 1226 sleep 1 1227 done 1228 1229 if lsblk -l -o NAME,SERIAL | grep -q -w $1; then 1230 return 1 1231 fi 1232 1233 return 0 1234} 1235 1236function waitforblk() { 1237 local i=0 1238 while ! lsblk -l -o NAME | grep -q -w $1; do 1239 [ $i -lt 15 ] || break 1240 i=$((i + 1)) 1241 sleep 1 1242 done 1243 1244 if ! lsblk -l -o NAME | grep -q -w $1; then 1245 return 1 1246 fi 1247 1248 return 0 1249} 1250 1251function waitforblk_disconnect() { 1252 local i=0 1253 while lsblk -l -o NAME | grep -q -w $1; do 1254 [ $i -lt 15 ] || break 1255 i=$((i + 1)) 1256 sleep 1 1257 done 1258 1259 if lsblk -l -o NAME | grep -q -w $1; then 1260 return 1 1261 fi 1262 1263 return 0 1264} 1265 1266function waitforfile() { 1267 local i=0 1268 while [ ! -e $1 ]; do 1269 [ $i -lt 200 ] || break 1270 i=$((i + 1)) 1271 sleep 0.1 1272 done 1273 1274 if [ ! -e $1 ]; then 1275 return 1 1276 fi 1277 1278 return 0 1279} 1280 1281function fio_config_gen() { 1282 local config_file=$1 1283 local workload=$2 1284 local bdev_type=$3 1285 local env_context=$4 1286 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1287 1288 if [ -e "$config_file" ]; then 1289 echo "Configuration File Already Exists!: $config_file" 1290 return 1 1291 fi 1292 1293 if [ -z "${workload:-}" ]; then 1294 workload=randrw 1295 fi 1296 1297 if [ -n "$env_context" ]; then 1298 env_context="env_context=$env_context" 1299 fi 1300 1301 touch $1 1302 1303 cat > $1 << EOL 1304[global] 1305thread=1 1306$env_context 1307group_reporting=1 1308direct=1 1309norandommap=1 1310percentile_list=50:99:99.9:99.99:99.999 1311time_based=1 1312ramp_time=0 1313EOL 1314 1315 if [ "$workload" == "verify" ]; then 1316 cat <<- EOL >> $config_file 1317 verify=sha1 1318 verify_backlog=1024 1319 rw=randwrite 1320 EOL 1321 1322 # To avoid potential data race issue due to the AIO device 1323 # flush mechanism, add the flag to serialize the writes. 1324 # This is to fix the intermittent IO failure issue of #935 1325 if [ "$bdev_type" == "AIO" ]; then 1326 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1327 echo "serialize_overlap=1" >> $config_file 1328 fi 1329 fi 1330 elif [ "$workload" == "trim" ]; then 1331 echo "rw=trimwrite" >> $config_file 1332 else 1333 echo "rw=$workload" >> $config_file 1334 fi 1335} 1336 1337function fio_plugin() { 1338 # Setup fio binary cmd line 1339 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1340 # gcc and clang uses different sanitizer libraries 1341 local sanitizers=(libasan libclang_rt.asan) 1342 local plugin=$1 1343 shift 1344 1345 local asan_lib= 1346 for sanitizer in "${sanitizers[@]}"; do 1347 asan_lib=$(ldd $plugin | grep $sanitizer | awk '{print $3}') 1348 if [[ -n "${asan_lib:-}" ]]; then 1349 break 1350 fi 1351 done 1352 1353 # Preload the sanitizer library to fio if fio_plugin was compiled with it 1354 LD_PRELOAD="$asan_lib $plugin" "$fio_dir"/fio "$@" 1355} 1356 1357function fio_bdev() { 1358 fio_plugin "$rootdir/build/fio/spdk_bdev" "$@" 1359} 1360 1361function fio_nvme() { 1362 fio_plugin "$rootdir/build/fio/spdk_nvme" "$@" 1363} 1364 1365function get_lvs_free_mb() { 1366 local lvs_uuid=$1 1367 local lvs_info 1368 local fc 1369 local cs 1370 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1371 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1372 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1373 1374 # Change to MB's 1375 free_mb=$((fc * cs / 1024 / 1024)) 1376 echo "$free_mb" 1377} 1378 1379function get_bdev_size() { 1380 local bdev_name=$1 1381 local bdev_info 1382 local bs 1383 local nb 1384 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1385 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1386 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1387 1388 # Change to MB's 1389 bdev_size=$((bs * nb / 1024 / 1024)) 1390 echo "$bdev_size" 1391} 1392 1393function autotest_cleanup() { 1394 local autotest_es=$? 1395 xtrace_disable 1396 1397 # Slurp at_app_exit() so we can kill all lingering vhost and qemu processes 1398 # in one swing. We do this in a subshell as vhost/common.sh is too eager to 1399 # do some extra work which we don't care about in this context. 1400 # shellcheck source=/dev/null 1401 vhost_reap() (source "$rootdir/test/vhost/common.sh" &> /dev/null || return 0 && at_app_exit) 1402 1403 # catch any stray core files and kill all remaining SPDK processes. Update 1404 # autotest_es in case autotest reported success but cores and/or processes 1405 # were left behind regardless. 1406 1407 process_core || autotest_es=1 1408 reap_spdk_processes || autotest_es=1 1409 vhost_reap || autotest_es=1 1410 1411 $rootdir/scripts/setup.sh reset 1412 $rootdir/scripts/setup.sh cleanup 1413 if [ $(uname -s) = "Linux" ]; then 1414 modprobe -r uio_pci_generic 1415 fi 1416 rm -rf "$asan_suppression_file" 1417 if [[ -n ${old_core_pattern:-} ]]; then 1418 echo "$old_core_pattern" > /proc/sys/kernel/core_pattern 1419 fi 1420 if [[ -e /proc/$udevadm_pid/status ]]; then 1421 kill "$udevadm_pid" || : 1422 fi 1423 1424 local storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 1425 1426 if ((${#storage_fallback_purge[@]} > 0)); then 1427 rm -rf "${storage_fallback_purge[@]}" 1428 fi 1429 1430 if ((autotest_es)); then 1431 if [[ $(uname) == FreeBSD ]]; then 1432 ps aux 1433 elif [[ $(uname) == Linux ]]; then 1434 # Get more detailed view 1435 grep . /proc/[0-9]*/status 1436 # Dump some extra info into kernel log 1437 echo "######## Autotest Cleanup Dump ########" > /dev/kmsg 1438 # Show cpus backtraces 1439 echo l > /proc/sysrq-trigger 1440 # Show mem usage 1441 echo m > /proc/sysrq-trigger 1442 # show blocked tasks 1443 echo w > /proc/sysrq-trigger 1444 1445 fi > "$output_dir/proc_list.txt" 2>&1 || : 1446 fi 1447 1448 stop_monitor_resources 1449 1450 xtrace_restore 1451 return $autotest_es 1452} 1453 1454function freebsd_update_contigmem_mod() { 1455 if [ $(uname) = FreeBSD ]; then 1456 kldunload contigmem.ko || true 1457 if [ -n "${SPDK_RUN_EXTERNAL_DPDK:-}" ]; then 1458 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/ 1459 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/ 1460 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/ 1461 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/ 1462 else 1463 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1464 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1465 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1466 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1467 fi 1468 fi 1469} 1470 1471function freebsd_set_maxsock_buf() { 1472 # FreeBSD needs 4MB maxsockbuf size to pass socket unit tests. 1473 # Otherwise tests fail due to ENOBUFS when trying to do setsockopt(SO_RCVBUF|SO_SNDBUF). 1474 # See https://github.com/spdk/spdk/issues/2943 1475 if [[ $(uname) = FreeBSD ]] && (($(sysctl -n kern.ipc.maxsockbuf) < 4194304)); then 1476 sysctl kern.ipc.maxsockbuf=4194304 1477 fi 1478} 1479 1480function get_nvme_name_from_bdf() { 1481 get_block_dev_from_nvme "$@" 1482} 1483 1484function get_nvme_ctrlr_from_bdf() { 1485 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1486 if [[ -z "${bdf_sysfs_path:-}" ]]; then 1487 return 1488 fi 1489 1490 printf '%s\n' "$(basename $bdf_sysfs_path)" 1491} 1492 1493# Get BDF addresses of all NVMe drives currently attached to 1494# uio-pci-generic or vfio-pci 1495function get_nvme_bdfs() { 1496 local bdfs=() 1497 bdfs=($("$rootdir/scripts/gen_nvme.sh" | jq -r '.config[].params.traddr')) 1498 if ((${#bdfs[@]} == 0)); then 1499 echo "No bdevs found" >&2 1500 return 1 1501 fi 1502 printf '%s\n' "${bdfs[@]}" 1503} 1504 1505# Same as function above, but just get the first disks BDF address 1506function get_first_nvme_bdf() { 1507 local bdfs=() 1508 bdfs=($(get_nvme_bdfs)) 1509 1510 echo "${bdfs[0]}" 1511} 1512 1513function nvme_namespace_revert() { 1514 $rootdir/scripts/setup.sh 1515 sleep 1 1516 local bdfs=() 1517 # If there are no nvme bdfs, just return immediately 1518 bdfs=($(get_nvme_bdfs)) || return 0 1519 1520 $rootdir/scripts/setup.sh reset 1521 1522 for bdf in "${bdfs[@]}"; do 1523 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1524 if [[ -z "${nvme_ctrlr:-}" ]]; then 1525 continue 1526 fi 1527 1528 # Check Optional Admin Command Support for Namespace Management 1529 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1530 oacs_ns_manage=$((oacs & 0x8)) 1531 1532 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1533 # This assumes every NVMe controller contains single namespace, 1534 # encompassing Total NVM Capacity and formatted as 512 block size. 1535 # 512 block size is needed for test/vhost/vhost_boot.sh to 1536 # successfully run. 1537 1538 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1539 if [[ "$unvmcap" -eq 0 ]]; then 1540 # All available space already used 1541 continue 1542 fi 1543 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1544 cntlid=$(nvme id-ctrl ${nvme_ctrlr} | grep cntlid | cut -d: -f2) 1545 blksize=512 1546 1547 size=$((tnvmcap / blksize)) 1548 1549 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c $cntlid || true 1550 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1551 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1552 nvme attach-ns ${nvme_ctrlr} -n 1 -c $cntlid 1553 nvme reset ${nvme_ctrlr} 1554 waitforfile "${nvme_ctrlr}n1" 1555 fi 1556 done 1557} 1558 1559# Get BDFs based on device ID, such as 0x0a54 1560function get_nvme_bdfs_by_id() { 1561 local bdfs=() _bdfs=() 1562 _bdfs=($(get_nvme_bdfs)) || return 0 1563 for bdf in "${_bdfs[@]}"; do 1564 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1565 if [[ "$device" == "$1" ]]; then 1566 bdfs+=($bdf) 1567 fi 1568 done 1569 1570 ((${#bdfs[@]} > 0)) || return 0 1571 printf '%s\n' "${bdfs[@]}" 1572} 1573 1574function opal_revert_cleanup() { 1575 # The OPAL CI tests is only used for P4510 devices. 1576 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1577 if [[ -z ${bdfs[0]:-} ]]; then 1578 return 0 1579 fi 1580 1581 $SPDK_BIN_DIR/spdk_tgt & 1582 spdk_tgt_pid=$! 1583 waitforlisten $spdk_tgt_pid 1584 1585 bdf_id=0 1586 for bdf in "${bdfs[@]}"; do 1587 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme"${bdf_id} -t "pcie" -a ${bdf} 1588 # Ignore if this fails. 1589 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b "nvme"${bdf_id} -p test || true 1590 ((++bdf_id)) 1591 done 1592 1593 killprocess $spdk_tgt_pid 1594} 1595 1596function pap() { 1597 while read -r file; do 1598 cat <<- FILE 1599 --- $file --- 1600 $(< "$file") 1601 --- $file --- 1602 FILE 1603 rm -f "$file" 1604 done < <(find "$@" -type f | sort -u) 1605} 1606 1607function get_proc_paths() { 1608 case "$(uname -s)" in 1609 Linux) # ps -e -opid,exe <- not supported under {centos7,rocky8}'s procps-ng 1610 local pid exe 1611 for pid in /proc/[0-9]*; do 1612 exe=$(readlink "$pid/exe") || continue 1613 exe=${exe/ (deleted)/} 1614 echo "${pid##*/} $exe" 1615 done 1616 ;; 1617 FreeeBSD) procstat -ab | awk '{print $1, $4}' ;; 1618 esac 1619} 1620 1621exec_files() { file "$@" | awk -F: '/ELF.+executable/{print $1}'; } 1622 1623function reap_spdk_processes() { 1624 local bins test_bins procs 1625 local spdk_procs spdk_pids 1626 1627 mapfile -t test_bins < <(find "$rootdir"/test/{app,env,event,nvme} -type f) 1628 mapfile -t bins < <( 1629 exec_files "${test_bins[@]}" 1630 readlink -f "$SPDK_BIN_DIR/"* "$SPDK_EXAMPLE_DIR/"* 1631 ) 1632 ((${#bins[@]} > 0)) || return 0 1633 1634 mapfile -t spdk_procs < <(get_proc_paths | grep -E "$( 1635 IFS="|" 1636 echo "${bins[*]#$rootdir/}" 1637 )" || true) 1638 ((${#spdk_procs[@]} > 0)) || return 0 1639 1640 printf '%s is still up, killing\n' "${spdk_procs[@]}" >&2 1641 mapfile -t spdk_pids < <(printf '%s\n' "${spdk_procs[@]}" | awk '{print $1}') 1642 1643 kill -SIGKILL "${spdk_pids[@]}" 2> /dev/null || : 1644 return 1 1645} 1646 1647function is_block_zoned() { 1648 local device=$1 1649 1650 [[ -e /sys/block/$device/queue/zoned ]] || return 1 1651 [[ $(< "/sys/block/$device/queue/zoned") != none ]] 1652} 1653 1654function get_zoned_devs() { 1655 local -gA zoned_devs=() 1656 local nvme bdf 1657 1658 for nvme in /sys/block/nvme*; do 1659 if is_block_zoned "${nvme##*/}"; then 1660 zoned_devs["${nvme##*/}"]=$(< "$nvme/device/address") 1661 fi 1662 done 1663} 1664 1665function is_pid_child() { 1666 local pid=$1 _pid 1667 1668 while read -r _pid; do 1669 ((pid == _pid)) && return 0 1670 done < <(jobs -pr) 1671 1672 return 1 1673} 1674 1675# Define temp storage for all the tests. Look for 2GB at minimum 1676set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1677 1678set -o errtrace 1679shopt -s extdebug 1680trap "trap - ERR; print_backtrace >&2" ERR 1681 1682PS4=' \t ${test_domain:-} -- ${BASH_SOURCE#${BASH_SOURCE%/*/*}/}@${LINENO} -- \$ ' 1683if $SPDK_AUTOTEST_X; then 1684 # explicitly enable xtraces, overriding any tracking information. 1685 xtrace_fd 1686else 1687 xtrace_disable 1688fi 1689 1690if [[ $CONFIG_COVERAGE == y ]]; then 1691 if lt "$(lcov --version | awk '{print $NF}')" 2; then 1692 lcov_rc_opt="--rc lcov_branch_coverage=1 --rc lcov_function_coverage=1" 1693 else 1694 lcov_rc_opt="--rc branch_coverage=1 --rc function_coverage=1" 1695 fi 1696 export LCOV_OPTS=" 1697 $lcov_rc_opt 1698 --rc genhtml_branch_coverage=1 1699 --rc genhtml_function_coverage=1 1700 --rc genhtml_legend=1 1701 --rc geninfo_all_blocks=1 1702 --rc geninfo_unexecuted_blocks=1 1703 $lcov_opt 1704 " 1705 export LCOV="lcov $LCOV_OPTS" 1706fi 1707
