1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2015 Intel Corporation 4# All rights reserved. 5# Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 6# 7rpc_py=rpc_cmd 8 9function xtrace_disable() { 10 set +x 11 X_STACK+=("${FUNCNAME[*]}") # push 12} 13 14function xtrace_restore() { 15 # unset'ing foo[-1] under older Bash (4.2 -> Centos7) won't work, hence the dance 16 unset -v "X_STACK[${#X_STACK[@]} - 1 < 0 ? 0 : ${#X_STACK[@]} - 1]" # pop 17 if ((${#X_STACK[@]} == 0)); then 18 set -x 19 fi 20} 21 22function xtrace_disable_per_cmd() { eval "$* ${BASH_XTRACEFD}> /dev/null"; } 23 24function xtrace_fd() { 25 if [[ -n ${BASH_XTRACEFD:-} && -e /proc/self/fd/$BASH_XTRACEFD ]]; then 26 # Close it first to make sure it's sane 27 exec {BASH_XTRACEFD}>&- 28 fi 29 exec {BASH_XTRACEFD}>&2 30 31 xtrace_restore 32} 33 34set -e 35shopt -s nullglob 36shopt -s extglob 37 38if [[ -e $rootdir/test/common/build_config.sh ]]; then 39 source "$rootdir/test/common/build_config.sh" 40elif [[ -e $rootdir/mk/config.mk ]]; then 41 build_config=$(< "$rootdir/mk/config.mk") 42 source <(echo "${build_config//\?=/=}") 43else 44 source "$rootdir/CONFIG" 45fi 46 47# Source scripts after the config so that the definitions are available. 48source "$rootdir/test/common/applications.sh" 49source "$rootdir/scripts/common.sh" 50 51: ${RUN_NIGHTLY:=0} 52export RUN_NIGHTLY 53 54# Set defaults for missing test config options 55: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 56export SPDK_AUTOTEST_DEBUG_APPS 57: ${SPDK_RUN_VALGRIND=0} 58export SPDK_RUN_VALGRIND 59: ${SPDK_RUN_FUNCTIONAL_TEST=0} 60export SPDK_RUN_FUNCTIONAL_TEST 61: ${SPDK_TEST_UNITTEST=0} 62export SPDK_TEST_UNITTEST 63: ${SPDK_TEST_AUTOBUILD=""} 64export SPDK_TEST_AUTOBUILD 65: ${SPDK_TEST_RELEASE_BUILD=0} 66export SPDK_TEST_RELEASE_BUILD 67: ${SPDK_TEST_ISAL=0} 68export SPDK_TEST_ISAL 69: ${SPDK_TEST_ISCSI=0} 70export SPDK_TEST_ISCSI 71: ${SPDK_TEST_ISCSI_INITIATOR=0} 72export SPDK_TEST_ISCSI_INITIATOR 73: ${SPDK_TEST_NVME=0} 74export SPDK_TEST_NVME 75: ${SPDK_TEST_NVME_PMR=0} 76export SPDK_TEST_NVME_PMR 77: ${SPDK_TEST_NVME_BP=0} 78export SPDK_TEST_NVME_BP 79: ${SPDK_TEST_NVME_CLI=0} 80export SPDK_TEST_NVME_CLI 81: ${SPDK_TEST_NVME_CUSE=0} 82export SPDK_TEST_NVME_CUSE 83: ${SPDK_TEST_NVME_FDP=0} 84export SPDK_TEST_NVME_FDP 85: ${SPDK_TEST_NVMF=0} 86export SPDK_TEST_NVMF 87: ${SPDK_TEST_VFIOUSER=0} 88export SPDK_TEST_VFIOUSER 89: ${SPDK_TEST_VFIOUSER_QEMU=0} 90export SPDK_TEST_VFIOUSER_QEMU 91: ${SPDK_TEST_FUZZER=0} 92export SPDK_TEST_FUZZER 93: ${SPDK_TEST_FUZZER_SHORT=0} 94export SPDK_TEST_FUZZER_SHORT 95: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 96export SPDK_TEST_NVMF_TRANSPORT 97: ${SPDK_TEST_RBD=0} 98export SPDK_TEST_RBD 99: ${SPDK_TEST_VHOST=0} 100export SPDK_TEST_VHOST 101: ${SPDK_TEST_BLOCKDEV=0} 102export SPDK_TEST_BLOCKDEV 103: ${SPDK_TEST_IOAT=0} 104export SPDK_TEST_IOAT 105: ${SPDK_TEST_BLOBFS=0} 106export SPDK_TEST_BLOBFS 107: ${SPDK_TEST_VHOST_INIT=0} 108export SPDK_TEST_VHOST_INIT 109: ${SPDK_TEST_LVOL=0} 110export SPDK_TEST_LVOL 111: ${SPDK_TEST_VBDEV_COMPRESS=0} 112export SPDK_TEST_VBDEV_COMPRESS 113: ${SPDK_RUN_ASAN=0} 114export SPDK_RUN_ASAN 115: ${SPDK_RUN_UBSAN=0} 116export SPDK_RUN_UBSAN 117: ${SPDK_RUN_EXTERNAL_DPDK=""} 118export SPDK_RUN_EXTERNAL_DPDK 119: ${SPDK_RUN_NON_ROOT=0} 120export SPDK_RUN_NON_ROOT 121: ${SPDK_TEST_CRYPTO=0} 122export SPDK_TEST_CRYPTO 123: ${SPDK_TEST_FTL=0} 124export SPDK_TEST_FTL 125: ${SPDK_TEST_OCF=0} 126export SPDK_TEST_OCF 127: ${SPDK_TEST_VMD=0} 128export SPDK_TEST_VMD 129: ${SPDK_TEST_OPAL=0} 130export SPDK_TEST_OPAL 131: ${SPDK_TEST_NATIVE_DPDK} 132export SPDK_TEST_NATIVE_DPDK 133: ${SPDK_AUTOTEST_X=true} 134export SPDK_AUTOTEST_X 135: ${SPDK_TEST_RAID5=0} 136export SPDK_TEST_RAID5 137: ${SPDK_TEST_URING=0} 138export SPDK_TEST_URING 139: ${SPDK_TEST_USDT=0} 140export SPDK_TEST_USDT 141: ${SPDK_TEST_USE_IGB_UIO:=0} 142export SPDK_TEST_USE_IGB_UIO 143: ${SPDK_TEST_SCHEDULER:=0} 144export SPDK_TEST_SCHEDULER 145: ${SPDK_TEST_SCANBUILD:=0} 146export SPDK_TEST_SCANBUILD 147: ${SPDK_TEST_NVMF_NICS:=} 148export SPDK_TEST_NVMF_NICS 149: ${SPDK_TEST_SMA=0} 150export SPDK_TEST_SMA 151: ${SPDK_TEST_DAOS=0} 152export SPDK_TEST_DAOS 153: ${SPDK_TEST_XNVME:=0} 154export SPDK_TEST_XNVME 155: ${SPDK_TEST_ACCEL_DSA=0} 156export SPDK_TEST_ACCEL_DSA 157: ${SPDK_TEST_ACCEL_IAA=0} 158export SPDK_TEST_ACCEL_IAA 159: ${SPDK_TEST_ACCEL_IOAT=0} 160export SPDK_TEST_ACCEL_IOAT 161# Comma-separated list of fuzzer targets matching test/fuzz/llvm/$target 162: ${SPDK_TEST_FUZZER_TARGET:=} 163export SPDK_TEST_FUZZER_TARGET 164: ${SPDK_TEST_NVMF_MDNS=0} 165export SPDK_TEST_NVMF_MDNS 166: ${SPDK_JSONRPC_GO_CLIENT=0} 167export SPDK_JSONRPC_GO_CLIENT 168 169# always test with SPDK shared objects. 170export SPDK_LIB_DIR="$rootdir/build/lib" 171export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib" 172export VFIO_LIB_DIR="$rootdir/build/libvfio-user/usr/local/lib" 173export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR:$VFIO_LIB_DIR 174 175# Tell setup.sh to wait for block devices upon each reset 176export PCI_BLOCK_SYNC_ON_RESET=yes 177 178# Export PYTHONPATH with addition of RPC framework. New scripts can be created 179# specific use cases for tests. 180export PYTHONPATH=$PYTHONPATH:$rootdir/python 181 182# Don't create Python .pyc files. When running with sudo these will be 183# created with root ownership and can cause problems when cleaning the repository. 184export PYTHONDONTWRITEBYTECODE=1 185 186# Export new_delete_type_mismatch to skip the known bug that exists in librados 187# https://tracker.ceph.com/issues/24078 188export ASAN_OPTIONS=new_delete_type_mismatch=0:disable_coredump=0:abort_on_error=1:use_sigaltstack=0 189export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1:disable_coredump=0:exitcode=134' 190 191# Export LeakSanitizer option to use suppression file in order to prevent false positives 192# and known leaks in external executables or libraries from showing up. 193asan_suppression_file="/var/tmp/asan_suppression_file" 194rm -rf "$asan_suppression_file" 2> /dev/null || sudo rm -rf "$asan_suppression_file" 195cat << EOL >> "$asan_suppression_file" 196# ASAN has some bugs around thread_local variables. We have a destructor in place 197# to free the thread contexts, but ASAN complains about the leak before those 198# destructors have a chance to run. So suppress this one specific leak using 199# LSAN_OPTIONS. 200leak:spdk_fs_alloc_thread_ctx 201 202# Suppress known leaks in fio project 203leak:$CONFIG_FIO_SOURCE_DIR/parse.c 204leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 205leak:$CONFIG_FIO_SOURCE_DIR/init.c 206leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 207leak:fio_memalign 208leak:spdk_fio_io_u_init 209# Suppress leaks in gperftools-libs from fio 210leak:libtcmalloc_minimal.so 211 212# Suppress leaks in libiscsi 213leak:libiscsi.so 214 215# Suppress leaks in libcrypto 216# Below is caused by openssl 3.0.8 leaks 217leak:libcrypto.so 218EOL 219 220# Suppress leaks in libfuse3 221echo "leak:libfuse3.so" >> "$asan_suppression_file" 222 223export LSAN_OPTIONS=suppressions="$asan_suppression_file" 224 225export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 226 227if [ -z "${DEPENDENCY_DIR:-}" ]; then 228 export DEPENDENCY_DIR=/var/spdk/dependencies 229else 230 export DEPENDENCY_DIR 231fi 232 233# Export location of where all the SPDK binaries are 234export SPDK_BIN_DIR="$rootdir/build/bin" 235export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 236 237# for vhost, vfio-user tests 238export QEMU_BIN=${QEMU_BIN:-} 239export VFIO_QEMU_BIN=${VFIO_QEMU_BIN:-} 240 241export AR_TOOL=$rootdir/scripts/ar-xnvme-fixer 242 243# For testing nvmes which are attached to some sort of a fanout switch in the CI pool 244export UNBIND_ENTIRE_IOMMU_GROUP=${UNBIND_ENTIRE_IOMMU_GROUP:-no} 245 246# pass our valgrind desire on to unittest.sh 247if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 248 export valgrind='' 249else 250 # unset all DEBUGINFOD_* vars that may affect our valgrind instance 251 unset -v "${!DEBUGINFOD_@}" 252fi 253 254if [ "$(uname -s)" = "Linux" ]; then 255 HUGEMEM=${HUGEMEM:-4096} 256 export CLEAR_HUGE=yes 257 if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]]; then 258 # Make sure that memory is distributed across all NUMA nodes - by default, all goes to 259 # node0, but if QAT devices are attached to a different node, all of their VFs will end 260 # up under that node too and memory needs to be available there for the tests. 261 export HUGE_EVEN_ALLOC=yes 262 fi 263 264 MAKE="make" 265 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 266elif [ "$(uname -s)" = "FreeBSD" ]; then 267 MAKE="gmake" 268 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 269 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 270 HUGEMEM=${HUGEMEM:-2048} 271elif [ "$(uname -s)" = "Windows" ]; then 272 MAKE="make" 273 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 274 # Keep the default 2GB for Windows. 275 HUGEMEM=${HUGEMEM:-2048} 276else 277 echo "Unknown OS \"$(uname -s)\"" 278 exit 1 279fi 280 281export HUGEMEM=$HUGEMEM 282 283if [ -z "${output_dir:-}" ]; then 284 mkdir -p "$rootdir/../output" 285 export output_dir="$rootdir/../output" 286fi 287 288NO_HUGE=() 289TEST_MODE= 290for i in "$@"; do 291 case "$i" in 292 --iso) 293 TEST_MODE=iso 294 ;; 295 --transport=*) 296 TEST_TRANSPORT="${i#*=}" 297 ;; 298 --sock=*) 299 TEST_SOCK="${i#*=}" 300 ;; 301 --no-hugepages) 302 NO_HUGE=(--no-huge -s 1024) 303 ;; 304 esac 305done 306 307# start rpc.py coprocess if it's not started yet 308if [[ -z ${RPC_PIPE_PID:-} ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 309 # Include list to all known plugins we use in the tests 310 PYTHONPATH+=":$rootdir/test/rpc_plugins" 311 coproc RPC_PIPE { PYTHONPATH="$PYTHONPATH" "$rootdir/scripts/rpc.py" --server; } 312 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 313 # all descriptors will automatically close together with this bash 314 # process, this will make rpc.py stop reading and exit gracefully 315fi 316 317function set_test_storage() { 318 [[ -v testdir ]] || return 0 319 320 local requested_size=$1 # bytes 321 local mount target_dir 322 323 local -A mounts fss sizes avails uses 324 local source fs size avail mount use 325 326 local storage_fallback storage_candidates 327 328 storage_fallback=$(mktemp -udt spdk.XXXXXX) 329 storage_candidates=( 330 "$testdir" 331 "$storage_fallback/tests/${testdir##*/}" 332 "$storage_fallback" 333 ) 334 335 if [[ -n ${ADD_TEST_STORAGE:-} ]]; then 336 # List of dirs|mounts separated by whitespaces 337 storage_candidates+=($ADD_TEST_STORAGE) 338 fi 339 340 if [[ -n ${DEDICATED_TEST_STORAGE:-} ]]; then 341 # Single, dedicated dir|mount 342 storage_candidates=("$DEDICATED_TEST_STORAGE") 343 fi 344 345 mkdir -p "${storage_candidates[@]}" 346 347 # add some headroom - 64M 348 requested_size=$((requested_size + (64 << 20))) 349 350 while read -r source fs size use avail _ mount; do 351 mounts["$mount"]=$source fss["$mount"]=$fs 352 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 353 uses["$mount"]=$((use * 1024)) 354 done < <(df -T | grep -v Filesystem) 355 356 printf '* Looking for test storage...\n' >&2 357 358 local target_space new_size 359 for target_dir in "${storage_candidates[@]}"; do 360 # FreeBSD's df is lacking the --output arg 361 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 362 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 363 364 target_space=${avails["$mount"]} 365 if ((target_space == 0 || target_space < requested_size)); then 366 continue 367 fi 368 if ((target_space >= requested_size)); then 369 # For in-memory fs, and / make sure our requested size won't fill most of the space. 370 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 371 new_size=$((uses["$mount"] + requested_size)) 372 if ((new_size * 100 / sizes["$mount"] > 95)); then 373 continue 374 fi 375 fi 376 fi 377 export SPDK_TEST_STORAGE=$target_dir 378 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 379 return 0 380 done 381 printf '* Test storage is not available\n' 382 return 1 383} 384 385function get_config_params() { 386 xtrace_disable 387 config_params='--enable-debug --enable-werror' 388 389 # for options with dependencies but no test flag, set them here 390 if [ -f /usr/include/infiniband/verbs.h ]; then 391 config_params+=' --with-rdma' 392 fi 393 394 if [ $SPDK_TEST_USDT -eq 1 ]; then 395 config_params+=" --with-usdt" 396 fi 397 398 if [ $(uname -s) == "FreeBSD" ]; then 399 intel="hw.model: Intel" 400 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 401 else 402 intel="GenuineIntel" 403 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 404 fi 405 if [[ "$cpu_vendor" != *"$intel"* ]]; then 406 config_params+=" --without-idxd" 407 else 408 config_params+=" --with-idxd" 409 fi 410 411 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 412 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 413 fi 414 415 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 416 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 417 fi 418 419 if [ -d /usr/include/iscsi ]; then 420 [[ $(< /usr/include/iscsi/iscsi.h) =~ "define LIBISCSI_API_VERSION ("([0-9]+)")" ]] \ 421 && libiscsi_version=${BASH_REMATCH[1]} 422 if ((libiscsi_version >= 20150621)); then 423 config_params+=' --with-iscsi-initiator' 424 fi 425 fi 426 427 if [[ $SPDK_TEST_UNITTEST -eq 0 && \ 428 $SPDK_TEST_SCANBUILD -eq 0 && -z \ 429 ${SPDK_TEST_AUTOBUILD:-} ]]; then 430 config_params+=' --disable-unit-tests' 431 fi 432 433 if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then 434 config_params+=' --with-nvme-cuse' 435 fi 436 437 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]; then 438 if ge "$(nasm --version | awk '{print $3}')" 2.14 && [[ $SPDK_TEST_ISAL -eq 1 ]]; then 439 config_params+=' --with-vbdev-compress --with-dpdk-compressdev' 440 fi 441 fi 442 443 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 444 config_params+=' --with-rbd' 445 fi 446 447 # for options with no required dependencies, just test flags, set them here 448 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 449 config_params+=' --with-crypto' 450 fi 451 452 if [ $SPDK_TEST_OCF -eq 1 ]; then 453 config_params+=" --with-ocf" 454 fi 455 456 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 457 config_params+=' --enable-ubsan' 458 fi 459 460 if [ $SPDK_RUN_ASAN -eq 1 ]; then 461 config_params+=' --enable-asan' 462 fi 463 464 if [ "$(uname -s)" = "Linux" ]; then 465 config_params+=' --enable-coverage' 466 fi 467 468 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 469 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 470 config_params+=' --with-fuse' 471 fi 472 fi 473 474 if [[ -f /usr/include/liburing/io_uring.h && -f /usr/include/linux/ublk_cmd.h ]]; then 475 config_params+=' --with-ublk' 476 fi 477 478 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 479 config_params+=' --with-raid5f' 480 fi 481 482 if [ $SPDK_TEST_VFIOUSER -eq 1 ] || [ $SPDK_TEST_VFIOUSER_QEMU -eq 1 ] || [ $SPDK_TEST_SMA -eq 1 ]; then 483 config_params+=' --with-vfio-user' 484 fi 485 486 # Check whether liburing library header exists 487 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 488 config_params+=' --with-uring' 489 fi 490 491 if [ -n "${SPDK_RUN_EXTERNAL_DPDK:-}" ]; then 492 config_params+=" --with-dpdk=$SPDK_RUN_EXTERNAL_DPDK" 493 fi 494 495 if [[ $SPDK_TEST_SMA -eq 1 ]]; then 496 config_params+=' --with-sma' 497 config_params+=' --with-crypto' 498 fi 499 500 if [ -f /usr/include/daos.h ] && [ $SPDK_TEST_DAOS -eq 1 ]; then 501 config_params+=' --with-daos' 502 fi 503 504 # Make the xnvme module available for the tests 505 if [[ $SPDK_TEST_XNVME -eq 1 ]]; then 506 config_params+=' --with-xnvme' 507 fi 508 509 if [[ $SPDK_TEST_FUZZER -eq 1 ]]; then 510 config_params+=" $(get_fuzzer_target_config)" 511 fi 512 513 if [[ $SPDK_TEST_NVMF_MDNS -eq 1 ]]; then 514 config_params+=' --with-avahi' 515 fi 516 517 if [[ $SPDK_JSONRPC_GO_CLIENT -eq 1 ]]; then 518 config_params+=' --with-golang' 519 fi 520 521 echo "$config_params" 522 xtrace_restore 523} 524 525function get_fuzzer_target_config() { 526 local -A fuzzer_targets_to_config=() 527 local config target 528 529 fuzzer_targets_to_config["vfio"]="--with-vfio-user" 530 for target in $(get_fuzzer_targets); do 531 [[ -n ${fuzzer_targets_to_config["$target"]:-} ]] || continue 532 config+=("${fuzzer_targets_to_config["$target"]}") 533 done 534 535 if ((${#config[@]} > 0)); then 536 echo "${config[*]}" 537 fi 538} 539 540function get_fuzzer_targets() { 541 local fuzzers=() 542 543 if [[ -n ${SPDK_TEST_FUZZER_TARGET:-} ]]; then 544 IFS="," read -ra fuzzers <<< "$SPDK_TEST_FUZZER_TARGET" 545 else 546 fuzzers=("$rootdir/test/fuzz/llvm/"*) 547 fuzzers=("${fuzzers[@]##*/}") 548 fi 549 550 echo "${fuzzers[*]}" 551} 552 553function rpc_cmd() { 554 xtrace_disable 555 local rsp rc=1 556 local stdin cmd cmds_number=0 status_number=0 status 557 558 if (($#)); then 559 cmds_number=1 560 echo "$@" >&$RPC_PIPE_INPUT 561 elif [[ ! -t 0 ]]; then 562 mapfile -t stdin <&0 563 cmds_number=${#stdin[@]} 564 printf '%s\n' "${stdin[@]}" >&$RPC_PIPE_INPUT 565 else 566 return 0 567 fi 568 569 while read -t 15 -ru $RPC_PIPE_OUTPUT rsp; do 570 if [[ $rsp == "**STATUS="* ]]; then 571 status[${rsp#*=}]=$rsp 572 if ((++status_number == cmds_number)); then 573 break 574 fi 575 continue 576 fi 577 echo "$rsp" 578 done 579 580 rc=${!status[*]} 581 xtrace_restore 582 [[ $rc == 0 ]] 583} 584 585function rpc_cmd_simple_data_json() { 586 587 local elems="$1[@]" elem 588 local -gA jq_out=() 589 local jq val 590 591 local lvs=( 592 "uuid" 593 "name" 594 "base_bdev" 595 "total_data_clusters" 596 "free_clusters" 597 "block_size" 598 "cluster_size" 599 ) 600 601 local bdev=( 602 "name" 603 "aliases[0]" 604 "block_size" 605 "num_blocks" 606 "uuid" 607 "product_name" 608 "supported_io_types.read" 609 "supported_io_types.write" 610 "driver_specific.lvol.clone" 611 "driver_specific.lvol.base_snapshot" 612 "driver_specific.lvol.esnap_clone" 613 "driver_specific.lvol.external_snapshot_name" 614 ) 615 616 [[ -v $elems ]] || return 1 617 618 for elem in "${!elems}"; do 619 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 620 done 621 jq+=',"\n"' 622 623 shift 624 while read -r elem val; do 625 jq_out["$elem"]=$val 626 done < <(rpc_cmd "$@" | jq -jr "$jq") 627 ((${#jq_out[@]} > 0)) || return 1 628} 629 630function valid_exec_arg() { 631 local arg=$1 632 # First argument must be the executable so do some basic sanity checks first. For bash, this 633 # covers two basic cases where es == 126 || es == 127 so catch them early on and fail hard 634 # if needed. 635 case "$(type -t "$arg")" in 636 builtin | function) ;; 637 file) arg=$(type -P "$arg") && [[ -x $arg ]] ;; 638 *) return 1 ;; 639 esac 640} 641 642function NOT() { 643 local es=0 644 645 valid_exec_arg "$@" || return 1 646 "$@" || es=$? 647 648 # Logic looks like so: 649 # - return false if command exit successfully 650 # - return false if command exit after receiving a core signal (FIXME: or any signal?) 651 # - return true if command exit with an error 652 653 # This naively assumes that the process doesn't exit with > 128 on its own. 654 if ((es > 128)); then 655 es=$((es & ~128)) 656 case "$es" in 657 3) ;& # SIGQUIT 658 4) ;& # SIGILL 659 6) ;& # SIGABRT 660 8) ;& # SIGFPE 661 9) ;& # SIGKILL 662 11) es=0 ;; # SIGSEGV 663 *) es=1 ;; 664 esac 665 elif [[ -n ${EXIT_STATUS:-} ]] && ((es != EXIT_STATUS)); then 666 es=0 667 fi 668 669 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 670 ((!es == 0)) 671} 672 673function timing() { 674 direction="$1" 675 testname="$2" 676 677 now=$(date +%s) 678 679 if [ "$direction" = "enter" ]; then 680 export timing_stack="${timing_stack:-};${now}" 681 export test_stack="${test_stack:-};${testname}" 682 else 683 touch "$output_dir/timing.txt" 684 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 685 686 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 687 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 688 689 elapsed=$((now - start_time - child_time)) 690 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 691 692 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 693 fi 694} 695 696function timing_cmd() ( 697 # The use-case here is this: ts=$(timing_cmd echo bar). Since stdout is always redirected 698 # to a pipe handling the $(), lookup the stdin's device and determine if it's sane to send 699 # cmd's output to it. If not, just null it. 700 701 [[ -t 0 ]] && exec {cmd_out}>&0 || exec {cmd_out}> /dev/null 702 703 local time=0 TIMEFORMAT=%2R # seconds 704 705 # We redirect cmd's std{out,err} to a separate fd dup'ed to stdin's device (or /dev/null) to 706 # catch only output from the time builtin - output from the actual cmd would be still visible, 707 # but $() will return just the time's data, hence making it possible to just do: 708 # time_of_super_verbose_cmd=$(timing_cmd super_verbose_cmd) 709 time=$({ time "$@" >&"$cmd_out" 2>&1; } 2>&1) 710 711 echo "$time" 712) 713 714function timing_enter() { 715 xtrace_disable 716 timing "enter" "$1" 717 xtrace_restore 718} 719 720function timing_exit() { 721 xtrace_disable 722 timing "exit" "$1" 723 xtrace_restore 724} 725 726function timing_finish() { 727 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 728 if [ -x "$flamegraph" ]; then 729 "$flamegraph" \ 730 --title 'Build Timing' \ 731 --nametype 'Step:' \ 732 --countname seconds \ 733 $output_dir/timing.txt \ 734 > $output_dir/timing.svg 735 fi 736} 737 738function create_test_list() { 739 xtrace_disable 740 # First search all scripts in main SPDK directory. 741 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 742 # Follow up with search in test directory recursively. 743 completion+=$'\n'$(grep -rshI --include="*.sh" --exclude="*autotest_common.sh" -e "run_test " $rootdir/test) 744 printf "%s" "$completion" | grep -v "#" \ 745 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 746 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 747 xtrace_restore 748} 749 750function gdb_attach() { 751 gdb -q --batch \ 752 -ex 'handle SIGHUP nostop pass' \ 753 -ex 'handle SIGQUIT nostop pass' \ 754 -ex 'handle SIGPIPE nostop pass' \ 755 -ex 'handle SIGALRM nostop pass' \ 756 -ex 'handle SIGTERM nostop pass' \ 757 -ex 'handle SIGUSR1 nostop pass' \ 758 -ex 'handle SIGUSR2 nostop pass' \ 759 -ex 'handle SIGCHLD nostop pass' \ 760 -ex 'set print thread-events off' \ 761 -ex 'cont' \ 762 -ex 'thread apply all bt' \ 763 -ex 'quit' \ 764 --tty=/dev/stdout \ 765 -p $1 766} 767 768function process_core() { 769 # Note that this always was racy as we can't really sync with the kernel 770 # to see if there's any core queued up for writing. We could check if 771 # collector is running and wait for it explicitly, but it doesn't seem 772 # to be worth the effort. So assume that if we are being called via 773 # trap, as in, when some error has occurred, wait up to 10s for any 774 # potential cores. If we are called just for cleanup at the very end, 775 # don't wait since all the tests ended successfully, hence having any 776 # critical cores lying around is unlikely. 777 ((autotest_es != 0)) && sleep 10 778 779 local coredumps core 780 781 coredumps=("$output_dir/coredumps/"*.bt.txt) 782 783 ((${#coredumps[@]} > 0)) || return 0 784 chmod -R a+r "$output_dir/coredumps" 785 786 for core in "${coredumps[@]}"; do 787 cat <<- BT 788 ##### CORE BT ${core##*/} ##### 789 790 $(<"$core") 791 792 -- 793 BT 794 done 795 return 1 796} 797 798function process_shm() { 799 type=$1 800 id=$2 801 if [ "$type" = "--pid" ]; then 802 id="pid${id}" 803 fi 804 805 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 806 807 if [[ -z ${shm_files:-} ]]; then 808 echo "SHM File for specified PID or shared memory id: ${id} not found!" 809 return 1 810 fi 811 for n in $shm_files; do 812 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 813 done 814 return 0 815} 816 817# Parameters: 818# $1 - process pid 819# $2 - rpc address (optional) 820# $3 - max retries (optional) 821function waitforlisten() { 822 if [ -z "${1:-}" ]; then 823 exit 1 824 fi 825 826 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 827 local max_retries=${3:-100} 828 829 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 830 # turn off trace for this loop 831 xtrace_disable 832 local ret=0 833 local i 834 for ((i = max_retries; i != 0; i--)); do 835 # if the process is no longer running, then exit the script 836 # since it means the application crashed 837 if ! kill -s 0 $1; then 838 echo "ERROR: process (pid: $1) is no longer running" 839 ret=1 840 break 841 fi 842 843 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 844 break 845 fi 846 847 sleep 0.5 848 done 849 850 xtrace_restore 851 if ((i == 0)); then 852 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 853 ret=1 854 fi 855 return $ret 856} 857 858function waitfornbd() { 859 local nbd_name=$1 860 local i 861 862 for ((i = 1; i <= 20; i++)); do 863 if grep -q -w $nbd_name /proc/partitions; then 864 break 865 else 866 sleep 0.1 867 fi 868 done 869 870 # The nbd device is now recognized as a block device, but there can be 871 # a small delay before we can start I/O to that block device. So loop 872 # here trying to read the first block of the nbd block device to a temp 873 # file. Note that dd returns success when reading an empty file, so we 874 # need to check the size of the output file instead. 875 for ((i = 1; i <= 20; i++)); do 876 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 877 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 878 rm -f "$SPDK_TEST_STORAGE/nbdtest" 879 if [ "$size" != "0" ]; then 880 return 0 881 else 882 sleep 0.1 883 fi 884 done 885 886 return 1 887} 888 889function waitforbdev() { 890 local bdev_name=$1 891 local bdev_timeout=$2 892 local i 893 [[ -z ${bdev_timeout:-} ]] && bdev_timeout=2000 # ms 894 895 $rpc_py bdev_wait_for_examine 896 897 if $rpc_py bdev_get_bdevs -b $bdev_name -t $bdev_timeout; then 898 return 0 899 fi 900 901 return 1 902} 903 904function make_filesystem() { 905 local fstype=$1 906 local dev_name=$2 907 local i=0 908 local force 909 910 if [ $fstype = ext4 ]; then 911 force=-F 912 else 913 force=-f 914 fi 915 916 while ! mkfs.${fstype} $force ${dev_name}; do 917 if [ $i -ge 15 ]; then 918 return 1 919 fi 920 i=$((i + 1)) 921 sleep 1 922 done 923 924 return 0 925} 926 927function killprocess() { 928 # $1 = process pid 929 if [ -z "${1:-}" ]; then 930 return 1 931 fi 932 933 if kill -0 $1; then 934 if [ $(uname) = Linux ]; then 935 process_name=$(ps --no-headers -o comm= $1) 936 else 937 process_name=$(ps -c -o command $1 | tail -1) 938 fi 939 if [ "$process_name" = "sudo" ]; then 940 # kill the child process, which is the actual app 941 # (assume $1 has just one child) 942 local child 943 child="$(pgrep -P $1)" 944 echo "killing process with pid $child" 945 kill $child 946 else 947 echo "killing process with pid $1" 948 kill $1 949 fi 950 951 # wait for the process regardless if its the dummy sudo one 952 # or the actual app - it should terminate anyway 953 wait $1 954 else 955 # the process is not there anymore 956 echo "Process with pid $1 is not found" 957 fi 958} 959 960function iscsicleanup() { 961 echo "Cleaning up iSCSI connection" 962 iscsiadm -m node --logout || true 963 iscsiadm -m node -o delete || true 964 rm -rf /var/lib/iscsi/nodes/* 965} 966 967function stop_iscsi_service() { 968 if cat /etc/*-release | grep Ubuntu; then 969 service open-iscsi stop 970 else 971 service iscsid stop 972 fi 973} 974 975function start_iscsi_service() { 976 if cat /etc/*-release | grep Ubuntu; then 977 service open-iscsi start 978 else 979 service iscsid start 980 fi 981} 982 983function rbd_setup() { 984 # $1 = monitor ip address 985 # $2 = name of the namespace 986 if [ -z "${1:-}" ]; then 987 echo "No monitor IP address provided for ceph" 988 exit 1 989 fi 990 if [ -n "${2:-}" ]; then 991 if ip netns list | grep "$2"; then 992 NS_CMD="ip netns exec $2" 993 else 994 echo "No namespace $2 exists" 995 exit 1 996 fi 997 fi 998 999 if hash ceph; then 1000 export PG_NUM=128 1001 export RBD_POOL=rbd 1002 export RBD_NAME=foo 1003 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 1004 $NS_CMD $rootdir/scripts/ceph/start.sh $1 1005 1006 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 1007 $NS_CMD rbd create $RBD_NAME --size 1000 1008 fi 1009} 1010 1011function rbd_cleanup() { 1012 if hash ceph; then 1013 $rootdir/scripts/ceph/stop.sh || true 1014 rm -f /var/tmp/ceph_raw.img 1015 fi 1016} 1017 1018function daos_setup() { 1019 # $1 = pool name 1020 # $2 = cont name 1021 if [ -z "${1:-}" ]; then 1022 echo "No pool name provided" 1023 exit 1 1024 fi 1025 if [ -z "${2:-}" ]; then 1026 echo "No cont name provided" 1027 exit 1 1028 fi 1029 1030 dmg pool create --size=10G $1 || true 1031 daos container create --type=POSIX --label=$2 $1 || true 1032} 1033 1034function daos_cleanup() { 1035 local pool=${1:-testpool} 1036 local cont=${2:-testcont} 1037 1038 daos container destroy -f $pool $cont || true 1039 sudo dmg pool destroy -f $pool || true 1040} 1041 1042function _start_stub() { 1043 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 1044 # but ASLR can still be unreliable in some cases. 1045 # We will re-enable it again after multi-process testing is complete in kill_stub(). 1046 # Save current setting so it can be restored upon calling kill_stub(). 1047 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 1048 echo 0 > /proc/sys/kernel/randomize_va_space 1049 $rootdir/test/app/stub/stub $1 & 1050 stubpid=$! 1051 echo Waiting for stub to ready for secondary processes... 1052 while ! [ -e /var/run/spdk_stub0 ]; do 1053 # If stub dies while we wait, bail 1054 [[ -e /proc/$stubpid ]] || return 1 1055 sleep 1s 1056 done 1057 echo done. 1058} 1059 1060function start_stub() { 1061 if ! _start_stub "$@"; then 1062 echo "stub failed" >&2 1063 return 1 1064 fi 1065} 1066 1067function kill_stub() { 1068 if [[ -e /proc/$stubpid ]]; then 1069 kill $1 $stubpid 1070 wait $stubpid 1071 fi 2> /dev/null || : 1072 rm -f /var/run/spdk_stub0 1073 # Re-enable ASLR now that we are done with multi-process testing 1074 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 1075 # randomizing and is the default on all recent Linux kernels 1076 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 1077} 1078 1079function run_test() { 1080 if [ $# -le 1 ]; then 1081 echo "Not enough parameters" 1082 echo "usage: run_test test_name test_script [script_params]" 1083 exit 1 1084 fi 1085 1086 xtrace_disable 1087 local test_name="$1" 1088 shift 1089 1090 if [ -n "${test_domain:-}" ]; then 1091 export test_domain="${test_domain}.${test_name}" 1092 else 1093 export test_domain="$test_name" 1094 fi 1095 1096 timing_enter $test_name 1097 echo "************************************" 1098 echo "START TEST $test_name" 1099 echo "************************************" 1100 xtrace_restore 1101 time "$@" 1102 xtrace_disable 1103 echo "************************************" 1104 echo "END TEST $test_name" 1105 echo "************************************" 1106 timing_exit $test_name 1107 1108 export test_domain=${test_domain%"$test_name"} 1109 if [ -n "$test_domain" ]; then 1110 export test_domain=${test_domain%?} 1111 fi 1112 1113 if [ -z "${test_domain:-}" ]; then 1114 echo "top_level $test_name" >> $output_dir/test_completions.txt 1115 else 1116 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 1117 fi 1118 xtrace_restore 1119} 1120 1121function skip_run_test_with_warning() { 1122 echo "WARNING: $1" 1123 echo "Test run may fail if run with autorun.sh" 1124 echo "Please check your $rootdir/test/common/skipped_tests.txt" 1125} 1126 1127function print_backtrace() { 1128 # if errexit is not enabled, don't print a backtrace 1129 [[ "$-" =~ e ]] || return 0 1130 1131 local args=("${BASH_ARGV[@]}") 1132 1133 xtrace_disable 1134 # Reset IFS in case we were called from an environment where it was modified 1135 IFS=" "$'\t'$'\n' 1136 echo "========== Backtrace start: ==========" 1137 echo "" 1138 for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do 1139 local func="${FUNCNAME[$i]}" 1140 local line_nr="${BASH_LINENO[$((i - 1))]}" 1141 local src="${BASH_SOURCE[$i]}" 1142 local bt="" cmdline=() 1143 1144 if [[ -f $src ]]; then 1145 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 1146 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 1147 fi 1148 1149 # If extdebug set the BASH_ARGC[i], try to fetch all the args 1150 if ((BASH_ARGC[i] > 0)); then 1151 # Use argc as index to reverse the stack 1152 local argc=${BASH_ARGC[i]} arg 1153 for arg in "${args[@]::BASH_ARGC[i]}"; do 1154 cmdline[argc--]="[\"$arg\"]" 1155 done 1156 args=("${args[@]:BASH_ARGC[i]}") 1157 fi 1158 1159 echo "in $src:$line_nr -> $func($( 1160 IFS="," 1161 printf '%s\n' "${cmdline[*]:-[]}" 1162 ))" 1163 echo " ..." 1164 echo "${bt:-backtrace unavailable}" 1165 echo " ..." 1166 done 1167 echo "" 1168 echo "========== Backtrace end ==========" 1169 xtrace_restore 1170 return 0 1171} 1172 1173function waitforserial() { 1174 local i=0 1175 local nvme_device_counter=1 nvme_devices=0 1176 if [[ -n "${2:-}" ]]; then 1177 nvme_device_counter=$2 1178 fi 1179 1180 # Wait initially for min 2s to make sure all devices are ready for use. 1181 sleep 2 1182 while ((i++ <= 15)); do 1183 nvme_devices=$(lsblk -l -o NAME,SERIAL | grep -c "$1") 1184 ((nvme_devices == nvme_device_counter)) && return 0 1185 if ((nvme_devices > nvme_device_counter)); then 1186 echo "$nvme_device_counter device(s) expected, found $nvme_devices" >&2 1187 fi 1188 echo "Waiting for devices" 1189 sleep 1 1190 done 1191 return 1 1192} 1193 1194function waitforserial_disconnect() { 1195 local i=0 1196 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1197 [ $i -lt 15 ] || break 1198 i=$((i + 1)) 1199 echo "Waiting for disconnect devices" 1200 sleep 1 1201 done 1202 1203 if lsblk -l -o NAME,SERIAL | grep -q -w $1; then 1204 return 1 1205 fi 1206 1207 return 0 1208} 1209 1210function waitforblk() { 1211 local i=0 1212 while ! lsblk -l -o NAME | grep -q -w $1; do 1213 [ $i -lt 15 ] || break 1214 i=$((i + 1)) 1215 sleep 1 1216 done 1217 1218 if ! lsblk -l -o NAME | grep -q -w $1; then 1219 return 1 1220 fi 1221 1222 return 0 1223} 1224 1225function waitforblk_disconnect() { 1226 local i=0 1227 while lsblk -l -o NAME | grep -q -w $1; do 1228 [ $i -lt 15 ] || break 1229 i=$((i + 1)) 1230 sleep 1 1231 done 1232 1233 if lsblk -l -o NAME | grep -q -w $1; then 1234 return 1 1235 fi 1236 1237 return 0 1238} 1239 1240function waitforfile() { 1241 local i=0 1242 while [ ! -e $1 ]; do 1243 [ $i -lt 200 ] || break 1244 i=$((i + 1)) 1245 sleep 0.1 1246 done 1247 1248 if [ ! -e $1 ]; then 1249 return 1 1250 fi 1251 1252 return 0 1253} 1254 1255function fio_config_gen() { 1256 local config_file=$1 1257 local workload=$2 1258 local bdev_type=$3 1259 local env_context=$4 1260 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1261 1262 if [ -e "$config_file" ]; then 1263 echo "Configuration File Already Exists!: $config_file" 1264 return 1 1265 fi 1266 1267 if [ -z "${workload:-}" ]; then 1268 workload=randrw 1269 fi 1270 1271 if [ -n "$env_context" ]; then 1272 env_context="env_context=$env_context" 1273 fi 1274 1275 touch $1 1276 1277 cat > $1 << EOL 1278[global] 1279thread=1 1280$env_context 1281group_reporting=1 1282direct=1 1283norandommap=1 1284percentile_list=50:99:99.9:99.99:99.999 1285time_based=1 1286ramp_time=0 1287EOL 1288 1289 if [ "$workload" == "verify" ]; then 1290 cat <<- EOL >> $config_file 1291 verify=sha1 1292 verify_backlog=1024 1293 rw=randwrite 1294 EOL 1295 1296 # To avoid potential data race issue due to the AIO device 1297 # flush mechanism, add the flag to serialize the writes. 1298 # This is to fix the intermittent IO failure issue of #935 1299 if [ "$bdev_type" == "AIO" ]; then 1300 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1301 echo "serialize_overlap=1" >> $config_file 1302 fi 1303 fi 1304 elif [ "$workload" == "trim" ]; then 1305 echo "rw=trimwrite" >> $config_file 1306 else 1307 echo "rw=$workload" >> $config_file 1308 fi 1309} 1310 1311function fio_plugin() { 1312 # Setup fio binary cmd line 1313 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1314 # gcc and clang uses different sanitizer libraries 1315 local sanitizers=(libasan libclang_rt.asan) 1316 local plugin=$1 1317 shift 1318 1319 local asan_lib= 1320 for sanitizer in "${sanitizers[@]}"; do 1321 asan_lib=$(ldd $plugin | grep $sanitizer | awk '{print $3}') 1322 if [[ -n "${asan_lib:-}" ]]; then 1323 break 1324 fi 1325 done 1326 1327 # Preload the sanitizer library to fio if fio_plugin was compiled with it 1328 LD_PRELOAD="$asan_lib $plugin" "$fio_dir"/fio "$@" 1329} 1330 1331function fio_bdev() { 1332 fio_plugin "$rootdir/build/fio/spdk_bdev" "$@" 1333} 1334 1335function fio_nvme() { 1336 fio_plugin "$rootdir/build/fio/spdk_nvme" "$@" 1337} 1338 1339function get_lvs_free_mb() { 1340 local lvs_uuid=$1 1341 local lvs_info 1342 local fc 1343 local cs 1344 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1345 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1346 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1347 1348 # Change to MB's 1349 free_mb=$((fc * cs / 1024 / 1024)) 1350 echo "$free_mb" 1351} 1352 1353function get_bdev_size() { 1354 local bdev_name=$1 1355 local bdev_info 1356 local bs 1357 local nb 1358 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1359 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1360 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1361 1362 # Change to MB's 1363 bdev_size=$((bs * nb / 1024 / 1024)) 1364 echo "$bdev_size" 1365} 1366 1367function autotest_cleanup() { 1368 local autotest_es=$? 1369 xtrace_disable 1370 1371 # Slurp at_app_exit() so we can kill all lingering vhost and qemu processes 1372 # in one swing. We do this in a subshell as vhost/common.sh is too eager to 1373 # do some extra work which we don't care about in this context. 1374 # shellcheck source=/dev/null 1375 vhost_reap() (source "$rootdir/test/vhost/common.sh" &> /dev/null || return 0 && at_app_exit) 1376 1377 # catch any stray core files and kill all remaining SPDK processes. Update 1378 # autotest_es in case autotest reported success but cores and/or processes 1379 # were left behind regardless. 1380 1381 process_core || autotest_es=1 1382 reap_spdk_processes || autotest_es=1 1383 vhost_reap || autotest_es=1 1384 1385 $rootdir/scripts/setup.sh reset 1386 $rootdir/scripts/setup.sh cleanup 1387 if [ $(uname -s) = "Linux" ]; then 1388 modprobe -r uio_pci_generic 1389 fi 1390 rm -rf "$asan_suppression_file" 1391 if [[ -n ${old_core_pattern:-} ]]; then 1392 echo "$old_core_pattern" > /proc/sys/kernel/core_pattern 1393 fi 1394 if [[ -e /proc/$udevadm_pid/status ]]; then 1395 kill "$udevadm_pid" || : 1396 fi 1397 1398 local storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 1399 1400 if ((${#storage_fallback_purge[@]} > 0)); then 1401 rm -rf "${storage_fallback_purge[@]}" 1402 fi 1403 1404 if ((autotest_es)); then 1405 if [[ $(uname) == FreeBSD ]]; then 1406 ps aux 1407 elif [[ $(uname) == Linux ]]; then 1408 # Get more detailed view 1409 grep . /proc/[0-9]*/status 1410 # Dump some extra info into kernel log 1411 echo "######## Autotest Cleanup Dump ########" > /dev/kmsg 1412 # Show cpus backtraces 1413 echo l > /proc/sysrq-trigger 1414 # Show mem usage 1415 echo m > /proc/sysrq-trigger 1416 # show task states 1417 echo t > /proc/sysrq-trigger 1418 # show blocked tasks 1419 echo w > /proc/sysrq-trigger 1420 1421 fi > "$output_dir/proc_list.txt" 2>&1 || : 1422 fi 1423 xtrace_restore 1424 return $autotest_es 1425} 1426 1427function freebsd_update_contigmem_mod() { 1428 if [ $(uname) = FreeBSD ]; then 1429 kldunload contigmem.ko || true 1430 if [ -n "${SPDK_RUN_EXTERNAL_DPDK:-}" ]; then 1431 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/ 1432 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/ 1433 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/ 1434 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/ 1435 else 1436 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1437 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1438 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1439 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1440 fi 1441 fi 1442} 1443 1444function freebsd_set_maxsock_buf() { 1445 # FreeBSD needs 4MB maxsockbuf size to pass socket unit tests. 1446 # Otherwise tests fail due to ENOBUFS when trying to do setsockopt(SO_RCVBUF|SO_SNDBUF). 1447 # See https://github.com/spdk/spdk/issues/2943 1448 if [[ $(uname) = FreeBSD ]] && (($(sysctl -n kern.ipc.maxsockbuf) < 4194304)); then 1449 sysctl kern.ipc.maxsockbuf=4194304 1450 fi 1451} 1452 1453function get_nvme_name_from_bdf() { 1454 blkname=() 1455 1456 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1457 if [ -z "${nvme_devs:-}" ]; then 1458 return 1459 fi 1460 for dev in $nvme_devs; do 1461 link_name=$(readlink /sys/block/$dev/device/device) || true 1462 if [ -z "${link_name:-}" ]; then 1463 link_name=$(readlink /sys/block/$dev/device) 1464 fi 1465 bdf=$(basename "$link_name") 1466 if [ "$bdf" = "$1" ]; then 1467 blkname+=($dev) 1468 fi 1469 done 1470 1471 printf '%s\n' "${blkname[@]}" 1472} 1473 1474function get_nvme_ctrlr_from_bdf() { 1475 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1476 if [[ -z "${bdf_sysfs_path:-}" ]]; then 1477 return 1478 fi 1479 1480 printf '%s\n' "$(basename $bdf_sysfs_path)" 1481} 1482 1483# Get BDF addresses of all NVMe drives currently attached to 1484# uio-pci-generic or vfio-pci 1485function get_nvme_bdfs() { 1486 local bdfs=() 1487 bdfs=($("$rootdir/scripts/gen_nvme.sh" | jq -r '.config[].params.traddr')) 1488 if ((${#bdfs[@]} == 0)); then 1489 echo "No bdevs found" >&2 1490 return 1 1491 fi 1492 printf '%s\n' "${bdfs[@]}" 1493} 1494 1495# Same as function above, but just get the first disks BDF address 1496function get_first_nvme_bdf() { 1497 local bdfs=() 1498 bdfs=($(get_nvme_bdfs)) 1499 1500 echo "${bdfs[0]}" 1501} 1502 1503function nvme_namespace_revert() { 1504 $rootdir/scripts/setup.sh 1505 sleep 1 1506 local bdfs=() 1507 bdfs=($(get_nvme_bdfs)) 1508 1509 $rootdir/scripts/setup.sh reset 1510 1511 for bdf in "${bdfs[@]}"; do 1512 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1513 if [[ -z "${nvme_ctrlr:-}" ]]; then 1514 continue 1515 fi 1516 1517 # Check Optional Admin Command Support for Namespace Management 1518 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1519 oacs_ns_manage=$((oacs & 0x8)) 1520 1521 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1522 # This assumes every NVMe controller contains single namespace, 1523 # encompassing Total NVM Capacity and formatted as 512 block size. 1524 # 512 block size is needed for test/vhost/vhost_boot.sh to 1525 # successfully run. 1526 1527 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1528 if [[ "$unvmcap" -eq 0 ]]; then 1529 # All available space already used 1530 continue 1531 fi 1532 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1533 cntlid=$(nvme id-ctrl ${nvme_ctrlr} | grep cntlid | cut -d: -f2) 1534 blksize=512 1535 1536 size=$((tnvmcap / blksize)) 1537 1538 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c $cntlid || true 1539 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1540 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1541 nvme attach-ns ${nvme_ctrlr} -n 1 -c $cntlid 1542 nvme reset ${nvme_ctrlr} 1543 waitforfile "${nvme_ctrlr}n1" 1544 fi 1545 done 1546} 1547 1548# Get BDFs based on device ID, such as 0x0a54 1549function get_nvme_bdfs_by_id() { 1550 local bdfs=() 1551 1552 for bdf in $(get_nvme_bdfs); do 1553 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1554 if [[ "$device" == "$1" ]]; then 1555 bdfs+=($bdf) 1556 fi 1557 done 1558 1559 printf '%s\n' "${bdfs[@]}" 1560} 1561 1562function opal_revert_cleanup() { 1563 # The OPAL CI tests is only used for P4510 devices. 1564 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1565 if [[ -z ${bdfs[0]:-} ]]; then 1566 return 0 1567 fi 1568 1569 $SPDK_BIN_DIR/spdk_tgt & 1570 spdk_tgt_pid=$! 1571 waitforlisten $spdk_tgt_pid 1572 1573 bdf_id=0 1574 for bdf in "${bdfs[@]}"; do 1575 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme"${bdf_id} -t "pcie" -a ${bdf} 1576 # Ignore if this fails. 1577 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b "nvme"${bdf_id} -p test || true 1578 ((++bdf_id)) 1579 done 1580 1581 killprocess $spdk_tgt_pid 1582} 1583 1584function pap() { 1585 while read -r file; do 1586 cat <<- FILE 1587 --- $file --- 1588 $(<"$file") 1589 --- $file --- 1590 FILE 1591 rm -f "$file" 1592 done < <(find "$@" -type f | sort -u) 1593} 1594 1595function get_proc_paths() { 1596 case "$(uname -s)" in 1597 Linux) # ps -e -opid,exe <- not supported under {centos7,rocky8}'s procps-ng 1598 local pid exe 1599 for pid in /proc/[0-9]*; do 1600 exe=$(readlink "$pid/exe") || continue 1601 exe=${exe/ (deleted)/} 1602 echo "${pid##*/} $exe" 1603 done 1604 ;; 1605 FreeeBSD) procstat -ab | awk '{print $1, $4}' ;; 1606 esac 1607} 1608 1609exec_files() { file "$@" | awk -F: '/ELF.+executable/{print $1}'; } 1610 1611function reap_spdk_processes() { 1612 local bins test_bins procs 1613 local spdk_procs spdk_pids 1614 1615 mapfile -t test_bins < <(find "$rootdir"/test/{app,env,event} -type f) 1616 mapfile -t bins < <( 1617 exec_files "${test_bins[@]}" 1618 readlink -f "$SPDK_BIN_DIR/"* "$SPDK_EXAMPLE_DIR/"* 1619 ) 1620 1621 mapfile -t spdk_procs < <(get_proc_paths | grep -E "$( 1622 IFS="|" 1623 echo "${bins[*]#$rootdir/}" 1624 )" || true) 1625 ((${#spdk_procs[@]} > 0)) || return 0 1626 1627 printf '%s is still up, killing\n' "${spdk_procs[@]}" >&2 1628 mapfile -t spdk_pids < <(printf '%s\n' "${spdk_procs[@]}" | awk '{print $1}') 1629 1630 kill -SIGKILL "${spdk_pids[@]}" 2> /dev/null || : 1631 return 1 1632} 1633 1634function is_block_zoned() { 1635 local device=$1 1636 1637 [[ -e /sys/block/$device/queue/zoned ]] || return 1 1638 [[ $(< "/sys/block/$device/queue/zoned") != none ]] 1639} 1640 1641function get_zoned_devs() { 1642 local -gA zoned_devs=() 1643 local nvme bdf 1644 1645 for nvme in /sys/block/nvme*; do 1646 if is_block_zoned "${nvme##*/}"; then 1647 zoned_devs["${nvme##*/}"]=$(< "$nvme/device/address") 1648 fi 1649 done 1650} 1651 1652# Define temp storage for all the tests. Look for 2GB at minimum 1653set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1654 1655set -o errtrace 1656shopt -s extdebug 1657trap "trap - ERR; print_backtrace >&2" ERR 1658 1659PS4=' \t -- ${BASH_SOURCE#${BASH_SOURCE%/*/*}/}@${LINENO} -- \$ ' 1660if $SPDK_AUTOTEST_X; then 1661 # explicitly enable xtraces, overriding any tracking information. 1662 xtrace_fd 1663else 1664 xtrace_disable 1665fi 1666
