1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2015 Intel Corporation 4# All rights reserved. 5# Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 6# 7rpc_py=rpc_cmd 8 9function xtrace_disable() { 10 set +x 11 X_STACK+=("${FUNCNAME[*]}") # push 12} 13 14function xtrace_restore() { 15 # unset'ing foo[-1] under older Bash (4.2 -> Centos7) won't work, hence the dance 16 unset -v "X_STACK[${#X_STACK[@]} - 1 < 0 ? 0 : ${#X_STACK[@]} - 1]" # pop 17 if ((${#X_STACK[@]} == 0)); then 18 set -x 19 fi 20} 21 22function xtrace_disable_per_cmd() { eval "$* ${BASH_XTRACEFD}> /dev/null"; } 23 24function xtrace_fd() { 25 if [[ -n ${BASH_XTRACEFD:-} && -e /proc/self/fd/$BASH_XTRACEFD ]]; then 26 # Close it first to make sure it's sane 27 exec {BASH_XTRACEFD}>&- 28 fi 29 exec {BASH_XTRACEFD}>&2 30 31 xtrace_restore 32} 33 34set -e 35shopt -s nullglob 36shopt -s extglob 37 38if [[ -e $rootdir/test/common/build_config.sh ]]; then 39 source "$rootdir/test/common/build_config.sh" 40elif [[ -e $rootdir/mk/config.mk ]]; then 41 build_config=$(< "$rootdir/mk/config.mk") 42 source <(echo "${build_config//\?=/=}") 43else 44 source "$rootdir/CONFIG" 45fi 46 47# Source scripts after the config so that the definitions are available. 48source "$rootdir/test/common/applications.sh" 49source "$rootdir/scripts/common.sh" 50source "$rootdir/scripts/perf/pm/common" 51 52: ${RUN_NIGHTLY:=0} 53export RUN_NIGHTLY 54 55# Set defaults for missing test config options 56: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 57export SPDK_AUTOTEST_DEBUG_APPS 58: ${SPDK_RUN_VALGRIND=0} 59export SPDK_RUN_VALGRIND 60: ${SPDK_RUN_FUNCTIONAL_TEST=0} 61export SPDK_RUN_FUNCTIONAL_TEST 62: ${SPDK_TEST_UNITTEST=0} 63export SPDK_TEST_UNITTEST 64: ${SPDK_TEST_AUTOBUILD=""} 65export SPDK_TEST_AUTOBUILD 66: ${SPDK_TEST_RELEASE_BUILD=0} 67export SPDK_TEST_RELEASE_BUILD 68: ${SPDK_TEST_ISAL=0} 69export SPDK_TEST_ISAL 70: ${SPDK_TEST_ISCSI=0} 71export SPDK_TEST_ISCSI 72: ${SPDK_TEST_ISCSI_INITIATOR=0} 73export SPDK_TEST_ISCSI_INITIATOR 74: ${SPDK_TEST_NVME=0} 75export SPDK_TEST_NVME 76: ${SPDK_TEST_NVME_PMR=0} 77export SPDK_TEST_NVME_PMR 78: ${SPDK_TEST_NVME_BP=0} 79export SPDK_TEST_NVME_BP 80: ${SPDK_TEST_NVME_CLI=0} 81export SPDK_TEST_NVME_CLI 82: ${SPDK_TEST_NVME_CUSE=0} 83export SPDK_TEST_NVME_CUSE 84: ${SPDK_TEST_NVME_FDP=0} 85export SPDK_TEST_NVME_FDP 86: ${SPDK_TEST_NVMF=0} 87export SPDK_TEST_NVMF 88: ${SPDK_TEST_VFIOUSER=0} 89export SPDK_TEST_VFIOUSER 90: ${SPDK_TEST_VFIOUSER_QEMU=0} 91export SPDK_TEST_VFIOUSER_QEMU 92: ${SPDK_TEST_FUZZER=0} 93export SPDK_TEST_FUZZER 94: ${SPDK_TEST_FUZZER_SHORT=0} 95export SPDK_TEST_FUZZER_SHORT 96: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 97export SPDK_TEST_NVMF_TRANSPORT 98: ${SPDK_TEST_RBD=0} 99export SPDK_TEST_RBD 100: ${SPDK_TEST_VHOST=0} 101export SPDK_TEST_VHOST 102: ${SPDK_TEST_BLOCKDEV=0} 103export SPDK_TEST_BLOCKDEV 104: ${SPDK_TEST_IOAT=0} 105export SPDK_TEST_IOAT 106: ${SPDK_TEST_BLOBFS=0} 107export SPDK_TEST_BLOBFS 108: ${SPDK_TEST_VHOST_INIT=0} 109export SPDK_TEST_VHOST_INIT 110: ${SPDK_TEST_LVOL=0} 111export SPDK_TEST_LVOL 112: ${SPDK_TEST_VBDEV_COMPRESS=0} 113export SPDK_TEST_VBDEV_COMPRESS 114: ${SPDK_RUN_ASAN=0} 115export SPDK_RUN_ASAN 116: ${SPDK_RUN_UBSAN=0} 117export SPDK_RUN_UBSAN 118: ${SPDK_RUN_EXTERNAL_DPDK=""} 119export SPDK_RUN_EXTERNAL_DPDK 120: ${SPDK_RUN_NON_ROOT=0} 121export SPDK_RUN_NON_ROOT 122: ${SPDK_TEST_CRYPTO=0} 123export SPDK_TEST_CRYPTO 124: ${SPDK_TEST_FTL=0} 125export SPDK_TEST_FTL 126: ${SPDK_TEST_OCF=0} 127export SPDK_TEST_OCF 128: ${SPDK_TEST_VMD=0} 129export SPDK_TEST_VMD 130: ${SPDK_TEST_OPAL=0} 131export SPDK_TEST_OPAL 132: ${SPDK_TEST_NATIVE_DPDK} 133export SPDK_TEST_NATIVE_DPDK 134: ${SPDK_AUTOTEST_X=true} 135export SPDK_AUTOTEST_X 136: ${SPDK_TEST_RAID5=0} 137export SPDK_TEST_RAID5 138: ${SPDK_TEST_URING=0} 139export SPDK_TEST_URING 140: ${SPDK_TEST_USDT=0} 141export SPDK_TEST_USDT 142: ${SPDK_TEST_USE_IGB_UIO:=0} 143export SPDK_TEST_USE_IGB_UIO 144: ${SPDK_TEST_SCHEDULER:=0} 145export SPDK_TEST_SCHEDULER 146: ${SPDK_TEST_SCANBUILD:=0} 147export SPDK_TEST_SCANBUILD 148: ${SPDK_TEST_NVMF_NICS:=} 149export SPDK_TEST_NVMF_NICS 150: ${SPDK_TEST_SMA=0} 151export SPDK_TEST_SMA 152: ${SPDK_TEST_DAOS=0} 153export SPDK_TEST_DAOS 154: ${SPDK_TEST_XNVME:=0} 155export SPDK_TEST_XNVME 156: ${SPDK_TEST_ACCEL_DSA=0} 157export SPDK_TEST_ACCEL_DSA 158: ${SPDK_TEST_ACCEL_IAA=0} 159export SPDK_TEST_ACCEL_IAA 160: ${SPDK_TEST_ACCEL_IOAT=0} 161export SPDK_TEST_ACCEL_IOAT 162# Comma-separated list of fuzzer targets matching test/fuzz/llvm/$target 163: ${SPDK_TEST_FUZZER_TARGET:=} 164export SPDK_TEST_FUZZER_TARGET 165: ${SPDK_TEST_NVMF_MDNS=0} 166export SPDK_TEST_NVMF_MDNS 167: ${SPDK_JSONRPC_GO_CLIENT=0} 168export SPDK_JSONRPC_GO_CLIENT 169 170# always test with SPDK shared objects. 171export SPDK_LIB_DIR="$rootdir/build/lib" 172export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib" 173export VFIO_LIB_DIR="$rootdir/build/libvfio-user/usr/local/lib" 174export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR:$VFIO_LIB_DIR 175 176# Tell setup.sh to wait for block devices upon each reset 177export PCI_BLOCK_SYNC_ON_RESET=yes 178 179# Export PYTHONPATH with addition of RPC framework. New scripts can be created 180# specific use cases for tests. 181export PYTHONPATH=$PYTHONPATH:$rootdir/python 182 183# Don't create Python .pyc files. When running with sudo these will be 184# created with root ownership and can cause problems when cleaning the repository. 185export PYTHONDONTWRITEBYTECODE=1 186 187# Export new_delete_type_mismatch to skip the known bug that exists in librados 188# https://tracker.ceph.com/issues/24078 189export ASAN_OPTIONS=new_delete_type_mismatch=0:disable_coredump=0:abort_on_error=1:use_sigaltstack=0 190export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1:disable_coredump=0:exitcode=134' 191 192# Export LeakSanitizer option to use suppression file in order to prevent false positives 193# and known leaks in external executables or libraries from showing up. 194asan_suppression_file="/var/tmp/asan_suppression_file" 195rm -rf "$asan_suppression_file" 2> /dev/null || sudo rm -rf "$asan_suppression_file" 196cat << EOL >> "$asan_suppression_file" 197# ASAN has some bugs around thread_local variables. We have a destructor in place 198# to free the thread contexts, but ASAN complains about the leak before those 199# destructors have a chance to run. So suppress this one specific leak using 200# LSAN_OPTIONS. 201leak:spdk_fs_alloc_thread_ctx 202 203# Suppress known leaks in fio project 204leak:$CONFIG_FIO_SOURCE_DIR/parse.c 205leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 206leak:$CONFIG_FIO_SOURCE_DIR/init.c 207leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 208leak:fio_memalign 209leak:spdk_fio_io_u_init 210# Suppress leaks in gperftools-libs from fio 211leak:libtcmalloc_minimal.so 212 213# Suppress leaks in libiscsi 214leak:libiscsi.so 215 216# Suppress leaks in libcrypto 217# Below is caused by openssl 3.0.8 leaks 218leak:libcrypto.so 219EOL 220 221# Suppress leaks in libfuse3 222echo "leak:libfuse3.so" >> "$asan_suppression_file" 223 224export LSAN_OPTIONS=suppressions="$asan_suppression_file" 225 226export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 227 228if [ -z "${DEPENDENCY_DIR:-}" ]; then 229 export DEPENDENCY_DIR=/var/spdk/dependencies 230else 231 export DEPENDENCY_DIR 232fi 233 234# Export location of where all the SPDK binaries are 235export SPDK_BIN_DIR="$rootdir/build/bin" 236export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 237 238# for vhost, vfio-user tests 239export QEMU_BIN=${QEMU_BIN:-} 240export VFIO_QEMU_BIN=${VFIO_QEMU_BIN:-} 241 242export AR_TOOL=$rootdir/scripts/ar-xnvme-fixer 243 244# For testing nvmes which are attached to some sort of a fanout switch in the CI pool 245export UNBIND_ENTIRE_IOMMU_GROUP=${UNBIND_ENTIRE_IOMMU_GROUP:-no} 246 247# pass our valgrind desire on to unittest.sh 248if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 249 export valgrind='' 250else 251 # unset all DEBUGINFOD_* vars that may affect our valgrind instance 252 unset -v "${!DEBUGINFOD_@}" 253fi 254 255if [ "$(uname -s)" = "Linux" ]; then 256 HUGEMEM=${HUGEMEM:-4096} 257 export CLEAR_HUGE=yes 258 if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]]; then 259 # Make sure that memory is distributed across all NUMA nodes - by default, all goes to 260 # node0, but if QAT devices are attached to a different node, all of their VFs will end 261 # up under that node too and memory needs to be available there for the tests. 262 export HUGE_EVEN_ALLOC=yes 263 fi 264 265 MAKE="make" 266 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 267elif [ "$(uname -s)" = "FreeBSD" ]; then 268 MAKE="gmake" 269 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 270 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 271 HUGEMEM=${HUGEMEM:-2048} 272elif [ "$(uname -s)" = "Windows" ]; then 273 MAKE="make" 274 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 275 # Keep the default 2GB for Windows. 276 HUGEMEM=${HUGEMEM:-2048} 277else 278 echo "Unknown OS \"$(uname -s)\"" 279 exit 1 280fi 281 282export HUGEMEM=$HUGEMEM 283 284if [ -z "${output_dir:-}" ]; then 285 mkdir -p "$rootdir/../output" 286 export output_dir="$rootdir/../output" 287fi 288 289NO_HUGE=() 290TEST_MODE= 291for i in "$@"; do 292 case "$i" in 293 --iso) 294 TEST_MODE=iso 295 ;; 296 --transport=*) 297 TEST_TRANSPORT="${i#*=}" 298 ;; 299 --sock=*) 300 TEST_SOCK="${i#*=}" 301 ;; 302 --no-hugepages) 303 NO_HUGE=(--no-huge -s 1024) 304 ;; 305 esac 306done 307 308# start rpc.py coprocess if it's not started yet 309if [[ -z ${RPC_PIPE_PID:-} ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 310 # Include list to all known plugins we use in the tests 311 PYTHONPATH+=":$rootdir/test/rpc_plugins" 312 coproc RPC_PIPE { PYTHONPATH="$PYTHONPATH" "$rootdir/scripts/rpc.py" --server; } 313 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 314 # all descriptors will automatically close together with this bash 315 # process, this will make rpc.py stop reading and exit gracefully 316fi 317 318function set_test_storage() { 319 [[ -v testdir ]] || return 0 320 321 local requested_size=$1 # bytes 322 local mount target_dir 323 324 local -A mounts fss sizes avails uses 325 local source fs size avail mount use 326 327 local storage_fallback storage_candidates 328 329 storage_fallback=$(mktemp -udt spdk.XXXXXX) 330 storage_candidates=( 331 "$testdir" 332 "$storage_fallback/tests/${testdir##*/}" 333 "$storage_fallback" 334 ) 335 336 if [[ -n ${ADD_TEST_STORAGE:-} ]]; then 337 # List of dirs|mounts separated by whitespaces 338 storage_candidates+=($ADD_TEST_STORAGE) 339 fi 340 341 if [[ -n ${DEDICATED_TEST_STORAGE:-} ]]; then 342 # Single, dedicated dir|mount 343 storage_candidates=("$DEDICATED_TEST_STORAGE") 344 fi 345 346 mkdir -p "${storage_candidates[@]}" 347 348 # add some headroom - 64M 349 requested_size=$((requested_size + (64 << 20))) 350 351 while read -r source fs size use avail _ mount; do 352 mounts["$mount"]=$source fss["$mount"]=$fs 353 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 354 uses["$mount"]=$((use * 1024)) 355 done < <(df -T | grep -v Filesystem) 356 357 printf '* Looking for test storage...\n' >&2 358 359 local target_space new_size 360 for target_dir in "${storage_candidates[@]}"; do 361 # FreeBSD's df is lacking the --output arg 362 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 363 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 364 365 target_space=${avails["$mount"]} 366 if ((target_space == 0 || target_space < requested_size)); then 367 continue 368 fi 369 if ((target_space >= requested_size)); then 370 # For in-memory fs, and / make sure our requested size won't fill most of the space. 371 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 372 new_size=$((uses["$mount"] + requested_size)) 373 if ((new_size * 100 / sizes["$mount"] > 95)); then 374 continue 375 fi 376 fi 377 fi 378 export SPDK_TEST_STORAGE=$target_dir 379 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 380 return 0 381 done 382 printf '* Test storage is not available\n' 383 return 1 384} 385 386function get_config_params() { 387 xtrace_disable 388 config_params='--enable-debug --enable-werror' 389 390 # for options with dependencies but no test flag, set them here 391 if [ -f /usr/include/infiniband/verbs.h ]; then 392 config_params+=' --with-rdma' 393 fi 394 395 if [ $SPDK_TEST_USDT -eq 1 ]; then 396 config_params+=" --with-usdt" 397 fi 398 399 if [ $(uname -s) == "FreeBSD" ]; then 400 intel="hw.model: Intel" 401 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 402 else 403 intel="GenuineIntel" 404 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 405 fi 406 if [[ "$cpu_vendor" != *"$intel"* ]]; then 407 config_params+=" --without-idxd" 408 else 409 config_params+=" --with-idxd" 410 fi 411 412 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 413 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 414 fi 415 416 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 417 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 418 fi 419 420 if [ -d /usr/include/iscsi ]; then 421 [[ $(< /usr/include/iscsi/iscsi.h) =~ "define LIBISCSI_API_VERSION ("([0-9]+)")" ]] \ 422 && libiscsi_version=${BASH_REMATCH[1]} 423 if ((libiscsi_version >= 20150621)); then 424 config_params+=' --with-iscsi-initiator' 425 fi 426 fi 427 428 if [[ $SPDK_TEST_UNITTEST -eq 0 && \ 429 $SPDK_TEST_SCANBUILD -eq 0 && -z \ 430 ${SPDK_TEST_AUTOBUILD:-} ]]; then 431 config_params+=' --disable-unit-tests' 432 fi 433 434 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]; then 435 if ge "$(nasm --version | awk '{print $3}')" 2.14 && [[ $SPDK_TEST_ISAL -eq 1 ]]; then 436 config_params+=' --with-vbdev-compress --with-dpdk-compressdev' 437 fi 438 fi 439 440 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 441 config_params+=' --with-rbd' 442 fi 443 444 # for options with no required dependencies, just test flags, set them here 445 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 446 config_params+=' --with-crypto' 447 fi 448 449 if [ $SPDK_TEST_OCF -eq 1 ]; then 450 config_params+=" --with-ocf" 451 fi 452 453 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 454 config_params+=' --enable-ubsan' 455 fi 456 457 if [ $SPDK_RUN_ASAN -eq 1 ]; then 458 config_params+=' --enable-asan' 459 fi 460 461 if [ "$(uname -s)" = "Linux" ]; then 462 config_params+=' --enable-coverage' 463 fi 464 465 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 466 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 467 config_params+=' --with-fuse' 468 fi 469 fi 470 471 if [[ -f /usr/include/liburing/io_uring.h && -f /usr/include/linux/ublk_cmd.h ]]; then 472 config_params+=' --with-ublk' 473 fi 474 475 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 476 config_params+=' --with-raid5f' 477 fi 478 479 if [ $SPDK_TEST_VFIOUSER -eq 1 ] || [ $SPDK_TEST_VFIOUSER_QEMU -eq 1 ] || [ $SPDK_TEST_SMA -eq 1 ]; then 480 config_params+=' --with-vfio-user' 481 fi 482 483 # Check whether liburing library header exists 484 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 485 config_params+=' --with-uring' 486 fi 487 488 if [ -n "${SPDK_RUN_EXTERNAL_DPDK:-}" ]; then 489 config_params+=" --with-dpdk=$SPDK_RUN_EXTERNAL_DPDK" 490 fi 491 492 if [[ $SPDK_TEST_SMA -eq 1 ]]; then 493 config_params+=' --with-sma' 494 config_params+=' --with-crypto' 495 fi 496 497 if [ -f /usr/include/daos.h ] && [ $SPDK_TEST_DAOS -eq 1 ]; then 498 config_params+=' --with-daos' 499 fi 500 501 # Make the xnvme module available for the tests 502 if [[ $SPDK_TEST_XNVME -eq 1 ]]; then 503 config_params+=' --with-xnvme' 504 fi 505 506 if [[ $SPDK_TEST_FUZZER -eq 1 ]]; then 507 config_params+=" $(get_fuzzer_target_config)" 508 fi 509 510 if [[ $SPDK_TEST_NVMF_MDNS -eq 1 ]]; then 511 config_params+=' --with-avahi' 512 fi 513 514 if [[ $SPDK_JSONRPC_GO_CLIENT -eq 1 ]]; then 515 config_params+=' --with-golang' 516 fi 517 518 echo "$config_params" 519 xtrace_restore 520} 521 522function get_fuzzer_target_config() { 523 local -A fuzzer_targets_to_config=() 524 local config target 525 526 fuzzer_targets_to_config["vfio"]="--with-vfio-user" 527 for target in $(get_fuzzer_targets); do 528 [[ -n ${fuzzer_targets_to_config["$target"]:-} ]] || continue 529 config+=("${fuzzer_targets_to_config["$target"]}") 530 done 531 532 if ((${#config[@]} > 0)); then 533 echo "${config[*]}" 534 fi 535} 536 537function get_fuzzer_targets() { 538 local fuzzers=() 539 540 if [[ -n ${SPDK_TEST_FUZZER_TARGET:-} ]]; then 541 IFS="," read -ra fuzzers <<< "$SPDK_TEST_FUZZER_TARGET" 542 else 543 fuzzers=("$rootdir/test/fuzz/llvm/"*) 544 fuzzers=("${fuzzers[@]##*/}") 545 fi 546 547 echo "${fuzzers[*]}" 548} 549 550function rpc_cmd() { 551 xtrace_disable 552 local rsp rc=1 553 local stdin cmd cmds_number=0 status_number=0 status 554 555 if (($#)); then 556 cmds_number=1 557 echo "$@" >&$RPC_PIPE_INPUT 558 elif [[ ! -t 0 ]]; then 559 mapfile -t stdin <&0 560 cmds_number=${#stdin[@]} 561 printf '%s\n' "${stdin[@]}" >&$RPC_PIPE_INPUT 562 else 563 return 0 564 fi 565 566 while read -t 15 -ru $RPC_PIPE_OUTPUT rsp; do 567 if [[ $rsp == "**STATUS="* ]]; then 568 status[${rsp#*=}]=$rsp 569 if ((++status_number == cmds_number)); then 570 break 571 fi 572 continue 573 fi 574 echo "$rsp" 575 done 576 577 rc=${!status[*]} 578 xtrace_restore 579 [[ $rc == 0 ]] 580} 581 582function rpc_cmd_simple_data_json() { 583 584 local elems="$1[@]" elem 585 local -gA jq_out=() 586 local jq val 587 588 local lvs=( 589 "uuid" 590 "name" 591 "base_bdev" 592 "total_data_clusters" 593 "free_clusters" 594 "block_size" 595 "cluster_size" 596 ) 597 598 local bdev=( 599 "name" 600 "aliases[0]" 601 "block_size" 602 "num_blocks" 603 "uuid" 604 "product_name" 605 "supported_io_types.read" 606 "supported_io_types.write" 607 "driver_specific.lvol.clone" 608 "driver_specific.lvol.base_snapshot" 609 "driver_specific.lvol.esnap_clone" 610 "driver_specific.lvol.external_snapshot_name" 611 ) 612 613 [[ -v $elems ]] || return 1 614 615 for elem in "${!elems}"; do 616 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 617 done 618 jq+=',"\n"' 619 620 shift 621 while read -r elem val; do 622 jq_out["$elem"]=$val 623 done < <(rpc_cmd "$@" | jq -jr "$jq") 624 ((${#jq_out[@]} > 0)) || return 1 625} 626 627function valid_exec_arg() { 628 local arg=$1 629 # First argument must be the executable so do some basic sanity checks first. For bash, this 630 # covers two basic cases where es == 126 || es == 127 so catch them early on and fail hard 631 # if needed. 632 case "$(type -t "$arg")" in 633 builtin | function) ;; 634 file) arg=$(type -P "$arg") && [[ -x $arg ]] ;; 635 *) return 1 ;; 636 esac 637} 638 639function NOT() { 640 local es=0 641 642 valid_exec_arg "$@" || return 1 643 "$@" || es=$? 644 645 # Logic looks like so: 646 # - return false if command exit successfully 647 # - return false if command exit after receiving a core signal (FIXME: or any signal?) 648 # - return true if command exit with an error 649 650 # This naively assumes that the process doesn't exit with > 128 on its own. 651 if ((es > 128)); then 652 es=$((es & ~128)) 653 case "$es" in 654 3) ;& # SIGQUIT 655 4) ;& # SIGILL 656 6) ;& # SIGABRT 657 8) ;& # SIGFPE 658 9) ;& # SIGKILL 659 11) es=0 ;; # SIGSEGV 660 *) es=1 ;; 661 esac 662 elif [[ -n ${EXIT_STATUS:-} ]] && ((es != EXIT_STATUS)); then 663 es=0 664 fi 665 666 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 667 ((!es == 0)) 668} 669 670function timing() { 671 direction="$1" 672 testname="$2" 673 674 now=$(date +%s) 675 676 if [ "$direction" = "enter" ]; then 677 export timing_stack="${timing_stack:-};${now}" 678 export test_stack="${test_stack:-};${testname}" 679 else 680 touch "$output_dir/timing.txt" 681 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 682 683 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 684 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 685 686 elapsed=$((now - start_time - child_time)) 687 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 688 689 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 690 fi 691} 692 693function timing_cmd() ( 694 # The use-case here is this: ts=$(timing_cmd echo bar). Since stdout is always redirected 695 # to a pipe handling the $(), lookup the stdin's device and determine if it's sane to send 696 # cmd's output to it. If not, just null it. 697 698 [[ -t 0 ]] && exec {cmd_out}>&0 || exec {cmd_out}> /dev/null 699 700 local time=0 TIMEFORMAT=%2R # seconds 701 702 # We redirect cmd's std{out,err} to a separate fd dup'ed to stdin's device (or /dev/null) to 703 # catch only output from the time builtin - output from the actual cmd would be still visible, 704 # but $() will return just the time's data, hence making it possible to just do: 705 # time_of_super_verbose_cmd=$(timing_cmd super_verbose_cmd) 706 time=$({ time "$@" >&"$cmd_out" 2>&1; } 2>&1) 707 708 echo "$time" 709) 710 711function timing_enter() { 712 xtrace_disable 713 timing "enter" "$1" 714 xtrace_restore 715} 716 717function timing_exit() { 718 xtrace_disable 719 timing "exit" "$1" 720 xtrace_restore 721} 722 723function timing_finish() { 724 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 725 if [ -x "$flamegraph" ]; then 726 "$flamegraph" \ 727 --title 'Build Timing' \ 728 --nametype 'Step:' \ 729 --countname seconds \ 730 $output_dir/timing.txt \ 731 > $output_dir/timing.svg 732 fi 733} 734 735function create_test_list() { 736 xtrace_disable 737 # First search all scripts in main SPDK directory. 738 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 739 # Follow up with search in test directory recursively. 740 completion+=$'\n'$(grep -rshI --include="*.sh" --exclude="*autotest_common.sh" -e "run_test " $rootdir/test) 741 printf "%s" "$completion" | grep -v "#" \ 742 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 743 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 744 xtrace_restore 745} 746 747function gdb_attach() { 748 gdb -q --batch \ 749 -ex 'handle SIGHUP nostop pass' \ 750 -ex 'handle SIGQUIT nostop pass' \ 751 -ex 'handle SIGPIPE nostop pass' \ 752 -ex 'handle SIGALRM nostop pass' \ 753 -ex 'handle SIGTERM nostop pass' \ 754 -ex 'handle SIGUSR1 nostop pass' \ 755 -ex 'handle SIGUSR2 nostop pass' \ 756 -ex 'handle SIGCHLD nostop pass' \ 757 -ex 'set print thread-events off' \ 758 -ex 'cont' \ 759 -ex 'thread apply all bt' \ 760 -ex 'quit' \ 761 --tty=/dev/stdout \ 762 -p $1 763} 764 765function process_core() { 766 # Note that this always was racy as we can't really sync with the kernel 767 # to see if there's any core queued up for writing. We could check if 768 # collector is running and wait for it explicitly, but it doesn't seem 769 # to be worth the effort. So assume that if we are being called via 770 # trap, as in, when some error has occurred, wait up to 10s for any 771 # potential cores. If we are called just for cleanup at the very end, 772 # don't wait since all the tests ended successfully, hence having any 773 # critical cores lying around is unlikely. 774 ((autotest_es != 0)) && sleep 10 775 776 local coredumps core 777 778 coredumps=("$output_dir/coredumps/"*.bt.txt) 779 780 ((${#coredumps[@]} > 0)) || return 0 781 chmod -R a+r "$output_dir/coredumps" 782 783 for core in "${coredumps[@]}"; do 784 cat <<- BT 785 ##### CORE BT ${core##*/} ##### 786 787 $(<"$core") 788 789 -- 790 BT 791 done 792 return 1 793} 794 795function process_shm() { 796 type=$1 797 id=$2 798 if [ "$type" = "--pid" ]; then 799 id="pid${id}" 800 fi 801 802 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 803 804 if [[ -z ${shm_files:-} ]]; then 805 echo "SHM File for specified PID or shared memory id: ${id} not found!" 806 return 1 807 fi 808 for n in $shm_files; do 809 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 810 done 811 return 0 812} 813 814# Parameters: 815# $1 - process pid 816# $2 - rpc address (optional) 817# $3 - max retries (optional) 818function waitforlisten() { 819 if [ -z "${1:-}" ]; then 820 exit 1 821 fi 822 823 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 824 local max_retries=${3:-100} 825 826 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 827 # turn off trace for this loop 828 xtrace_disable 829 local ret=0 830 local i 831 for ((i = max_retries; i != 0; i--)); do 832 # if the process is no longer running, then exit the script 833 # since it means the application crashed 834 if ! kill -s 0 $1; then 835 echo "ERROR: process (pid: $1) is no longer running" 836 ret=1 837 break 838 fi 839 840 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 841 break 842 fi 843 844 sleep 0.5 845 done 846 847 xtrace_restore 848 if ((i == 0)); then 849 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 850 ret=1 851 fi 852 return $ret 853} 854 855function waitfornbd() { 856 local nbd_name=$1 857 local i 858 859 for ((i = 1; i <= 20; i++)); do 860 if grep -q -w $nbd_name /proc/partitions; then 861 break 862 else 863 sleep 0.1 864 fi 865 done 866 867 # The nbd device is now recognized as a block device, but there can be 868 # a small delay before we can start I/O to that block device. So loop 869 # here trying to read the first block of the nbd block device to a temp 870 # file. Note that dd returns success when reading an empty file, so we 871 # need to check the size of the output file instead. 872 for ((i = 1; i <= 20; i++)); do 873 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 874 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 875 rm -f "$SPDK_TEST_STORAGE/nbdtest" 876 if [ "$size" != "0" ]; then 877 return 0 878 else 879 sleep 0.1 880 fi 881 done 882 883 return 1 884} 885 886function waitforbdev() { 887 local bdev_name=$1 888 local bdev_timeout=$2 889 local i 890 [[ -z ${bdev_timeout:-} ]] && bdev_timeout=2000 # ms 891 892 $rpc_py bdev_wait_for_examine 893 894 if $rpc_py bdev_get_bdevs -b $bdev_name -t $bdev_timeout; then 895 return 0 896 fi 897 898 return 1 899} 900 901function make_filesystem() { 902 local fstype=$1 903 local dev_name=$2 904 local i=0 905 local force 906 907 if [ $fstype = ext4 ]; then 908 force=-F 909 else 910 force=-f 911 fi 912 913 while ! mkfs.${fstype} $force ${dev_name}; do 914 if [ $i -ge 15 ]; then 915 return 1 916 fi 917 i=$((i + 1)) 918 sleep 1 919 done 920 921 return 0 922} 923 924function killprocess() { 925 # $1 = process pid 926 if [ -z "${1:-}" ]; then 927 return 1 928 fi 929 930 if kill -0 $1; then 931 if [ $(uname) = Linux ]; then 932 process_name=$(ps --no-headers -o comm= $1) 933 else 934 process_name=$(ps -c -o command $1 | tail -1) 935 fi 936 if [ "$process_name" = "sudo" ]; then 937 # kill the child process, which is the actual app 938 # (assume $1 has just one child) 939 local child 940 child="$(pgrep -P $1)" 941 echo "killing process with pid $child" 942 kill $child 943 else 944 echo "killing process with pid $1" 945 kill $1 946 fi 947 948 # wait for the process regardless if its the dummy sudo one 949 # or the actual app - it should terminate anyway 950 wait $1 951 else 952 # the process is not there anymore 953 echo "Process with pid $1 is not found" 954 fi 955} 956 957function iscsicleanup() { 958 echo "Cleaning up iSCSI connection" 959 iscsiadm -m node --logout || true 960 iscsiadm -m node -o delete || true 961 rm -rf /var/lib/iscsi/nodes/* 962} 963 964function stop_iscsi_service() { 965 if cat /etc/*-release | grep Ubuntu; then 966 service open-iscsi stop 967 else 968 service iscsid stop 969 fi 970} 971 972function start_iscsi_service() { 973 if cat /etc/*-release | grep Ubuntu; then 974 service open-iscsi start 975 else 976 service iscsid start 977 fi 978} 979 980function rbd_setup() { 981 # $1 = monitor ip address 982 # $2 = name of the namespace 983 if [ -z "${1:-}" ]; then 984 echo "No monitor IP address provided for ceph" 985 exit 1 986 fi 987 if [ -n "${2:-}" ]; then 988 if ip netns list | grep "$2"; then 989 NS_CMD="ip netns exec $2" 990 else 991 echo "No namespace $2 exists" 992 exit 1 993 fi 994 fi 995 996 if hash ceph; then 997 export PG_NUM=128 998 export RBD_POOL=rbd 999 export RBD_NAME=foo 1000 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 1001 $NS_CMD $rootdir/scripts/ceph/start.sh $1 1002 1003 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 1004 $NS_CMD rbd create $RBD_NAME --size 1000 1005 fi 1006} 1007 1008function rbd_cleanup() { 1009 if hash ceph; then 1010 $rootdir/scripts/ceph/stop.sh || true 1011 rm -f /var/tmp/ceph_raw.img 1012 fi 1013} 1014 1015function daos_setup() { 1016 # $1 = pool name 1017 # $2 = cont name 1018 if [ -z "${1:-}" ]; then 1019 echo "No pool name provided" 1020 exit 1 1021 fi 1022 if [ -z "${2:-}" ]; then 1023 echo "No cont name provided" 1024 exit 1 1025 fi 1026 1027 dmg pool create --size=10G $1 || true 1028 daos container create --type=POSIX --label=$2 $1 || true 1029} 1030 1031function daos_cleanup() { 1032 local pool=${1:-testpool} 1033 local cont=${2:-testcont} 1034 1035 daos container destroy -f $pool $cont || true 1036 sudo dmg pool destroy -f $pool || true 1037} 1038 1039function _start_stub() { 1040 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 1041 # but ASLR can still be unreliable in some cases. 1042 # We will re-enable it again after multi-process testing is complete in kill_stub(). 1043 # Save current setting so it can be restored upon calling kill_stub(). 1044 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 1045 echo 0 > /proc/sys/kernel/randomize_va_space 1046 $rootdir/test/app/stub/stub $1 & 1047 stubpid=$! 1048 echo Waiting for stub to ready for secondary processes... 1049 while ! [ -e /var/run/spdk_stub0 ]; do 1050 # If stub dies while we wait, bail 1051 [[ -e /proc/$stubpid ]] || return 1 1052 sleep 1s 1053 done 1054 echo done. 1055} 1056 1057function start_stub() { 1058 if ! _start_stub "$@"; then 1059 echo "stub failed" >&2 1060 return 1 1061 fi 1062} 1063 1064function kill_stub() { 1065 if [[ -e /proc/$stubpid ]]; then 1066 kill $1 $stubpid 1067 wait $stubpid 1068 fi 2> /dev/null || : 1069 rm -f /var/run/spdk_stub0 1070 # Re-enable ASLR now that we are done with multi-process testing 1071 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 1072 # randomizing and is the default on all recent Linux kernels 1073 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 1074} 1075 1076function run_test() { 1077 if [ $# -le 1 ]; then 1078 echo "Not enough parameters" 1079 echo "usage: run_test test_name test_script [script_params]" 1080 exit 1 1081 fi 1082 1083 xtrace_disable 1084 local test_name="$1" pid 1085 shift 1086 1087 if [ -n "${test_domain:-}" ]; then 1088 export test_domain="${test_domain}.${test_name}" 1089 else 1090 export test_domain="$test_name" 1091 fi 1092 1093 # Signal our daemons to update the test tag 1094 echo "$test_domain" > "$TEST_TAG_FILE" 1095 for pid in "$output_dir/collect-"@(cpu-load|cpu-temp|vmstat).pid; do 1096 kill -USR1 "$(< "$pid")" || true 1097 done 1098 1099 timing_enter $test_name 1100 echo "************************************" 1101 echo "START TEST $test_name" 1102 echo "************************************" 1103 xtrace_restore 1104 time "$@" 1105 xtrace_disable 1106 echo "************************************" 1107 echo "END TEST $test_name" 1108 echo "************************************" 1109 timing_exit $test_name 1110 1111 export test_domain=${test_domain%"$test_name"} 1112 if [ -n "$test_domain" ]; then 1113 export test_domain=${test_domain%?} 1114 fi 1115 1116 if [ -z "${test_domain:-}" ]; then 1117 echo "top_level $test_name" >> $output_dir/test_completions.txt 1118 else 1119 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 1120 fi 1121 xtrace_restore 1122} 1123 1124function skip_run_test_with_warning() { 1125 echo "WARNING: $1" 1126 echo "Test run may fail if run with autorun.sh" 1127 echo "Please check your $rootdir/test/common/skipped_tests.txt" 1128} 1129 1130function print_backtrace() { 1131 # if errexit is not enabled, don't print a backtrace 1132 [[ "$-" =~ e ]] || return 0 1133 1134 local args=("${BASH_ARGV[@]}") 1135 1136 xtrace_disable 1137 # Reset IFS in case we were called from an environment where it was modified 1138 IFS=" "$'\t'$'\n' 1139 echo "========== Backtrace start: ==========" 1140 echo "" 1141 for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do 1142 local func="${FUNCNAME[$i]}" 1143 local line_nr="${BASH_LINENO[$((i - 1))]}" 1144 local src="${BASH_SOURCE[$i]}" 1145 local bt="" cmdline=() 1146 1147 if [[ -f $src ]]; then 1148 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 1149 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 1150 fi 1151 1152 # If extdebug set the BASH_ARGC[i], try to fetch all the args 1153 if ((BASH_ARGC[i] > 0)); then 1154 # Use argc as index to reverse the stack 1155 local argc=${BASH_ARGC[i]} arg 1156 for arg in "${args[@]::BASH_ARGC[i]}"; do 1157 cmdline[argc--]="[\"$arg\"]" 1158 done 1159 args=("${args[@]:BASH_ARGC[i]}") 1160 fi 1161 1162 echo "in $src:$line_nr -> $func($( 1163 IFS="," 1164 printf '%s\n' "${cmdline[*]:-[]}" 1165 ))" 1166 echo " ..." 1167 echo "${bt:-backtrace unavailable}" 1168 echo " ..." 1169 done 1170 echo "" 1171 echo "========== Backtrace end ==========" 1172 xtrace_restore 1173 return 0 1174} 1175 1176function waitforserial() { 1177 local i=0 1178 local nvme_device_counter=1 nvme_devices=0 1179 if [[ -n "${2:-}" ]]; then 1180 nvme_device_counter=$2 1181 fi 1182 1183 # Wait initially for min 2s to make sure all devices are ready for use. 1184 sleep 2 1185 while ((i++ <= 15)); do 1186 nvme_devices=$(lsblk -l -o NAME,SERIAL | grep -c "$1") 1187 ((nvme_devices == nvme_device_counter)) && return 0 1188 if ((nvme_devices > nvme_device_counter)); then 1189 echo "$nvme_device_counter device(s) expected, found $nvme_devices" >&2 1190 fi 1191 echo "Waiting for devices" 1192 sleep 1 1193 done 1194 return 1 1195} 1196 1197function waitforserial_disconnect() { 1198 local i=0 1199 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1200 [ $i -lt 15 ] || break 1201 i=$((i + 1)) 1202 echo "Waiting for disconnect devices" 1203 sleep 1 1204 done 1205 1206 if lsblk -l -o NAME,SERIAL | grep -q -w $1; then 1207 return 1 1208 fi 1209 1210 return 0 1211} 1212 1213function waitforblk() { 1214 local i=0 1215 while ! lsblk -l -o NAME | grep -q -w $1; do 1216 [ $i -lt 15 ] || break 1217 i=$((i + 1)) 1218 sleep 1 1219 done 1220 1221 if ! lsblk -l -o NAME | grep -q -w $1; then 1222 return 1 1223 fi 1224 1225 return 0 1226} 1227 1228function waitforblk_disconnect() { 1229 local i=0 1230 while lsblk -l -o NAME | grep -q -w $1; do 1231 [ $i -lt 15 ] || break 1232 i=$((i + 1)) 1233 sleep 1 1234 done 1235 1236 if lsblk -l -o NAME | grep -q -w $1; then 1237 return 1 1238 fi 1239 1240 return 0 1241} 1242 1243function waitforfile() { 1244 local i=0 1245 while [ ! -e $1 ]; do 1246 [ $i -lt 200 ] || break 1247 i=$((i + 1)) 1248 sleep 0.1 1249 done 1250 1251 if [ ! -e $1 ]; then 1252 return 1 1253 fi 1254 1255 return 0 1256} 1257 1258function fio_config_gen() { 1259 local config_file=$1 1260 local workload=$2 1261 local bdev_type=$3 1262 local env_context=$4 1263 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1264 1265 if [ -e "$config_file" ]; then 1266 echo "Configuration File Already Exists!: $config_file" 1267 return 1 1268 fi 1269 1270 if [ -z "${workload:-}" ]; then 1271 workload=randrw 1272 fi 1273 1274 if [ -n "$env_context" ]; then 1275 env_context="env_context=$env_context" 1276 fi 1277 1278 touch $1 1279 1280 cat > $1 << EOL 1281[global] 1282thread=1 1283$env_context 1284group_reporting=1 1285direct=1 1286norandommap=1 1287percentile_list=50:99:99.9:99.99:99.999 1288time_based=1 1289ramp_time=0 1290EOL 1291 1292 if [ "$workload" == "verify" ]; then 1293 cat <<- EOL >> $config_file 1294 verify=sha1 1295 verify_backlog=1024 1296 rw=randwrite 1297 EOL 1298 1299 # To avoid potential data race issue due to the AIO device 1300 # flush mechanism, add the flag to serialize the writes. 1301 # This is to fix the intermittent IO failure issue of #935 1302 if [ "$bdev_type" == "AIO" ]; then 1303 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1304 echo "serialize_overlap=1" >> $config_file 1305 fi 1306 fi 1307 elif [ "$workload" == "trim" ]; then 1308 echo "rw=trimwrite" >> $config_file 1309 else 1310 echo "rw=$workload" >> $config_file 1311 fi 1312} 1313 1314function fio_plugin() { 1315 # Setup fio binary cmd line 1316 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1317 # gcc and clang uses different sanitizer libraries 1318 local sanitizers=(libasan libclang_rt.asan) 1319 local plugin=$1 1320 shift 1321 1322 local asan_lib= 1323 for sanitizer in "${sanitizers[@]}"; do 1324 asan_lib=$(ldd $plugin | grep $sanitizer | awk '{print $3}') 1325 if [[ -n "${asan_lib:-}" ]]; then 1326 break 1327 fi 1328 done 1329 1330 # Preload the sanitizer library to fio if fio_plugin was compiled with it 1331 LD_PRELOAD="$asan_lib $plugin" "$fio_dir"/fio "$@" 1332} 1333 1334function fio_bdev() { 1335 fio_plugin "$rootdir/build/fio/spdk_bdev" "$@" 1336} 1337 1338function fio_nvme() { 1339 fio_plugin "$rootdir/build/fio/spdk_nvme" "$@" 1340} 1341 1342function get_lvs_free_mb() { 1343 local lvs_uuid=$1 1344 local lvs_info 1345 local fc 1346 local cs 1347 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1348 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1349 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1350 1351 # Change to MB's 1352 free_mb=$((fc * cs / 1024 / 1024)) 1353 echo "$free_mb" 1354} 1355 1356function get_bdev_size() { 1357 local bdev_name=$1 1358 local bdev_info 1359 local bs 1360 local nb 1361 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1362 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1363 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1364 1365 # Change to MB's 1366 bdev_size=$((bs * nb / 1024 / 1024)) 1367 echo "$bdev_size" 1368} 1369 1370function autotest_cleanup() { 1371 local autotest_es=$? 1372 xtrace_disable 1373 1374 # Slurp at_app_exit() so we can kill all lingering vhost and qemu processes 1375 # in one swing. We do this in a subshell as vhost/common.sh is too eager to 1376 # do some extra work which we don't care about in this context. 1377 # shellcheck source=/dev/null 1378 vhost_reap() (source "$rootdir/test/vhost/common.sh" &> /dev/null || return 0 && at_app_exit) 1379 1380 # catch any stray core files and kill all remaining SPDK processes. Update 1381 # autotest_es in case autotest reported success but cores and/or processes 1382 # were left behind regardless. 1383 1384 process_core || autotest_es=1 1385 reap_spdk_processes || autotest_es=1 1386 vhost_reap || autotest_es=1 1387 1388 $rootdir/scripts/setup.sh reset 1389 $rootdir/scripts/setup.sh cleanup 1390 if [ $(uname -s) = "Linux" ]; then 1391 modprobe -r uio_pci_generic 1392 fi 1393 rm -rf "$asan_suppression_file" 1394 if [[ -n ${old_core_pattern:-} ]]; then 1395 echo "$old_core_pattern" > /proc/sys/kernel/core_pattern 1396 fi 1397 if [[ -e /proc/$udevadm_pid/status ]]; then 1398 kill "$udevadm_pid" || : 1399 fi 1400 1401 local storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 1402 1403 if ((${#storage_fallback_purge[@]} > 0)); then 1404 rm -rf "${storage_fallback_purge[@]}" 1405 fi 1406 1407 if ((autotest_es)); then 1408 if [[ $(uname) == FreeBSD ]]; then 1409 ps aux 1410 elif [[ $(uname) == Linux ]]; then 1411 # Get more detailed view 1412 grep . /proc/[0-9]*/status 1413 # Dump some extra info into kernel log 1414 echo "######## Autotest Cleanup Dump ########" > /dev/kmsg 1415 # Show cpus backtraces 1416 echo l > /proc/sysrq-trigger 1417 # Show mem usage 1418 echo m > /proc/sysrq-trigger 1419 # show task states 1420 echo t > /proc/sysrq-trigger 1421 # show blocked tasks 1422 echo w > /proc/sysrq-trigger 1423 1424 fi > "$output_dir/proc_list.txt" 2>&1 || : 1425 fi 1426 1427 local pid 1428 for pid in "$output_dir/"*.pid; do 1429 killprocess "$(< "$pid")" 1430 done || : 1431 1432 rm -f "$output_dir/"*.pid 1433 rm -f "$TEST_TAG_FILE" 1434 1435 xtrace_restore 1436 return $autotest_es 1437} 1438 1439function freebsd_update_contigmem_mod() { 1440 if [ $(uname) = FreeBSD ]; then 1441 kldunload contigmem.ko || true 1442 if [ -n "${SPDK_RUN_EXTERNAL_DPDK:-}" ]; then 1443 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/ 1444 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/ 1445 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/ 1446 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/ 1447 else 1448 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1449 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1450 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1451 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1452 fi 1453 fi 1454} 1455 1456function freebsd_set_maxsock_buf() { 1457 # FreeBSD needs 4MB maxsockbuf size to pass socket unit tests. 1458 # Otherwise tests fail due to ENOBUFS when trying to do setsockopt(SO_RCVBUF|SO_SNDBUF). 1459 # See https://github.com/spdk/spdk/issues/2943 1460 if [[ $(uname) = FreeBSD ]] && (($(sysctl -n kern.ipc.maxsockbuf) < 4194304)); then 1461 sysctl kern.ipc.maxsockbuf=4194304 1462 fi 1463} 1464 1465function get_nvme_name_from_bdf() { 1466 blkname=() 1467 1468 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1469 if [ -z "${nvme_devs:-}" ]; then 1470 return 1471 fi 1472 for dev in $nvme_devs; do 1473 link_name=$(readlink /sys/block/$dev/device/device) || true 1474 if [ -z "${link_name:-}" ]; then 1475 link_name=$(readlink /sys/block/$dev/device) 1476 fi 1477 bdf=$(basename "$link_name") 1478 if [ "$bdf" = "$1" ]; then 1479 blkname+=($dev) 1480 fi 1481 done 1482 1483 printf '%s\n' "${blkname[@]}" 1484} 1485 1486function get_nvme_ctrlr_from_bdf() { 1487 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1488 if [[ -z "${bdf_sysfs_path:-}" ]]; then 1489 return 1490 fi 1491 1492 printf '%s\n' "$(basename $bdf_sysfs_path)" 1493} 1494 1495# Get BDF addresses of all NVMe drives currently attached to 1496# uio-pci-generic or vfio-pci 1497function get_nvme_bdfs() { 1498 local bdfs=() 1499 bdfs=($("$rootdir/scripts/gen_nvme.sh" | jq -r '.config[].params.traddr')) 1500 if ((${#bdfs[@]} == 0)); then 1501 echo "No bdevs found" >&2 1502 return 1 1503 fi 1504 printf '%s\n' "${bdfs[@]}" 1505} 1506 1507# Same as function above, but just get the first disks BDF address 1508function get_first_nvme_bdf() { 1509 local bdfs=() 1510 bdfs=($(get_nvme_bdfs)) 1511 1512 echo "${bdfs[0]}" 1513} 1514 1515function nvme_namespace_revert() { 1516 $rootdir/scripts/setup.sh 1517 sleep 1 1518 local bdfs=() 1519 bdfs=($(get_nvme_bdfs)) 1520 1521 $rootdir/scripts/setup.sh reset 1522 1523 for bdf in "${bdfs[@]}"; do 1524 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1525 if [[ -z "${nvme_ctrlr:-}" ]]; then 1526 continue 1527 fi 1528 1529 # Check Optional Admin Command Support for Namespace Management 1530 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1531 oacs_ns_manage=$((oacs & 0x8)) 1532 1533 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1534 # This assumes every NVMe controller contains single namespace, 1535 # encompassing Total NVM Capacity and formatted as 512 block size. 1536 # 512 block size is needed for test/vhost/vhost_boot.sh to 1537 # successfully run. 1538 1539 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1540 if [[ "$unvmcap" -eq 0 ]]; then 1541 # All available space already used 1542 continue 1543 fi 1544 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1545 cntlid=$(nvme id-ctrl ${nvme_ctrlr} | grep cntlid | cut -d: -f2) 1546 blksize=512 1547 1548 size=$((tnvmcap / blksize)) 1549 1550 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c $cntlid || true 1551 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1552 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1553 nvme attach-ns ${nvme_ctrlr} -n 1 -c $cntlid 1554 nvme reset ${nvme_ctrlr} 1555 waitforfile "${nvme_ctrlr}n1" 1556 fi 1557 done 1558} 1559 1560# Get BDFs based on device ID, such as 0x0a54 1561function get_nvme_bdfs_by_id() { 1562 local bdfs=() 1563 1564 for bdf in $(get_nvme_bdfs); do 1565 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1566 if [[ "$device" == "$1" ]]; then 1567 bdfs+=($bdf) 1568 fi 1569 done 1570 1571 printf '%s\n' "${bdfs[@]}" 1572} 1573 1574function opal_revert_cleanup() { 1575 # The OPAL CI tests is only used for P4510 devices. 1576 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1577 if [[ -z ${bdfs[0]:-} ]]; then 1578 return 0 1579 fi 1580 1581 $SPDK_BIN_DIR/spdk_tgt & 1582 spdk_tgt_pid=$! 1583 waitforlisten $spdk_tgt_pid 1584 1585 bdf_id=0 1586 for bdf in "${bdfs[@]}"; do 1587 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme"${bdf_id} -t "pcie" -a ${bdf} 1588 # Ignore if this fails. 1589 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b "nvme"${bdf_id} -p test || true 1590 ((++bdf_id)) 1591 done 1592 1593 killprocess $spdk_tgt_pid 1594} 1595 1596function pap() { 1597 while read -r file; do 1598 cat <<- FILE 1599 --- $file --- 1600 $(<"$file") 1601 --- $file --- 1602 FILE 1603 rm -f "$file" 1604 done < <(find "$@" -type f | sort -u) 1605} 1606 1607function get_proc_paths() { 1608 case "$(uname -s)" in 1609 Linux) # ps -e -opid,exe <- not supported under {centos7,rocky8}'s procps-ng 1610 local pid exe 1611 for pid in /proc/[0-9]*; do 1612 exe=$(readlink "$pid/exe") || continue 1613 exe=${exe/ (deleted)/} 1614 echo "${pid##*/} $exe" 1615 done 1616 ;; 1617 FreeeBSD) procstat -ab | awk '{print $1, $4}' ;; 1618 esac 1619} 1620 1621exec_files() { file "$@" | awk -F: '/ELF.+executable/{print $1}'; } 1622 1623function reap_spdk_processes() { 1624 local bins test_bins procs 1625 local spdk_procs spdk_pids 1626 1627 mapfile -t test_bins < <(find "$rootdir"/test/{app,env,event} -type f) 1628 mapfile -t bins < <( 1629 exec_files "${test_bins[@]}" 1630 readlink -f "$SPDK_BIN_DIR/"* "$SPDK_EXAMPLE_DIR/"* 1631 ) 1632 1633 mapfile -t spdk_procs < <(get_proc_paths | grep -E "$( 1634 IFS="|" 1635 echo "${bins[*]#$rootdir/}" 1636 )" || true) 1637 ((${#spdk_procs[@]} > 0)) || return 0 1638 1639 printf '%s is still up, killing\n' "${spdk_procs[@]}" >&2 1640 mapfile -t spdk_pids < <(printf '%s\n' "${spdk_procs[@]}" | awk '{print $1}') 1641 1642 kill -SIGKILL "${spdk_pids[@]}" 2> /dev/null || : 1643 return 1 1644} 1645 1646function is_block_zoned() { 1647 local device=$1 1648 1649 [[ -e /sys/block/$device/queue/zoned ]] || return 1 1650 [[ $(< "/sys/block/$device/queue/zoned") != none ]] 1651} 1652 1653function get_zoned_devs() { 1654 local -gA zoned_devs=() 1655 local nvme bdf 1656 1657 for nvme in /sys/block/nvme*; do 1658 if is_block_zoned "${nvme##*/}"; then 1659 zoned_devs["${nvme##*/}"]=$(< "$nvme/device/address") 1660 fi 1661 done 1662} 1663 1664# Define temp storage for all the tests. Look for 2GB at minimum 1665set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1666 1667set -o errtrace 1668shopt -s extdebug 1669trap "trap - ERR; print_backtrace >&2" ERR 1670 1671PS4=' \t -- ${BASH_SOURCE#${BASH_SOURCE%/*/*}/}@${LINENO} -- \$ ' 1672if $SPDK_AUTOTEST_X; then 1673 # explicitly enable xtraces, overriding any tracking information. 1674 xtrace_fd 1675else 1676 xtrace_disable 1677fi 1678
