1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2015 Intel Corporation 4# All rights reserved. 5# Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 6# 7rpc_py=rpc_cmd 8 9function xtrace_disable() { 10 set +x 11 X_STACK+=("${FUNCNAME[*]}") # push 12} 13 14function xtrace_restore() { 15 # unset'ing foo[-1] under older Bash (4.2 -> Centos7) won't work, hence the dance 16 unset -v "X_STACK[${#X_STACK[@]} - 1 < 0 ? 0 : ${#X_STACK[@]} - 1]" # pop 17 if ((${#X_STACK[@]} == 0)); then 18 set -x 19 fi 20} 21 22function xtrace_disable_per_cmd() { eval "$* ${BASH_XTRACEFD}> /dev/null"; } 23 24function xtrace_fd() { 25 if [[ -n ${BASH_XTRACEFD:-} && -e /proc/self/fd/$BASH_XTRACEFD ]]; then 26 # Close it first to make sure it's sane 27 exec {BASH_XTRACEFD}>&- 28 fi 29 exec {BASH_XTRACEFD}>&2 30 31 xtrace_restore 32} 33 34set -e 35shopt -s nullglob 36shopt -s extglob 37 38if [[ -e $rootdir/test/common/build_config.sh ]]; then 39 source "$rootdir/test/common/build_config.sh" 40elif [[ -e $rootdir/mk/config.mk ]]; then 41 build_config=$(< "$rootdir/mk/config.mk") 42 source <(echo "${build_config//\?=/=}") 43else 44 source "$rootdir/CONFIG" 45fi 46 47# Source scripts after the config so that the definitions are available. 48source "$rootdir/test/common/applications.sh" 49source "$rootdir/scripts/common.sh" 50source "$rootdir/scripts/perf/pm/common" 51 52: ${RUN_NIGHTLY:=0} 53export RUN_NIGHTLY 54 55# Set defaults for missing test config options 56: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 57export SPDK_AUTOTEST_DEBUG_APPS 58: ${SPDK_RUN_VALGRIND=0} 59export SPDK_RUN_VALGRIND 60: ${SPDK_RUN_FUNCTIONAL_TEST=0} 61export SPDK_RUN_FUNCTIONAL_TEST 62: ${SPDK_TEST_UNITTEST=0} 63export SPDK_TEST_UNITTEST 64: ${SPDK_TEST_AUTOBUILD=""} 65export SPDK_TEST_AUTOBUILD 66: ${SPDK_TEST_RELEASE_BUILD=0} 67export SPDK_TEST_RELEASE_BUILD 68: ${SPDK_TEST_ISAL=0} 69export SPDK_TEST_ISAL 70: ${SPDK_TEST_ISCSI=0} 71export SPDK_TEST_ISCSI 72: ${SPDK_TEST_ISCSI_INITIATOR=0} 73export SPDK_TEST_ISCSI_INITIATOR 74: ${SPDK_TEST_NVME=0} 75export SPDK_TEST_NVME 76: ${SPDK_TEST_NVME_PMR=0} 77export SPDK_TEST_NVME_PMR 78: ${SPDK_TEST_NVME_BP=0} 79export SPDK_TEST_NVME_BP 80: ${SPDK_TEST_NVME_CLI=0} 81export SPDK_TEST_NVME_CLI 82: ${SPDK_TEST_NVME_CUSE=0} 83export SPDK_TEST_NVME_CUSE 84: ${SPDK_TEST_NVME_FDP=0} 85export SPDK_TEST_NVME_FDP 86: ${SPDK_TEST_NVMF=0} 87export SPDK_TEST_NVMF 88: ${SPDK_TEST_VFIOUSER=0} 89export SPDK_TEST_VFIOUSER 90: ${SPDK_TEST_VFIOUSER_QEMU=0} 91export SPDK_TEST_VFIOUSER_QEMU 92: ${SPDK_TEST_FUZZER=0} 93export SPDK_TEST_FUZZER 94: ${SPDK_TEST_FUZZER_SHORT=0} 95export SPDK_TEST_FUZZER_SHORT 96: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 97export SPDK_TEST_NVMF_TRANSPORT 98: ${SPDK_TEST_RBD=0} 99export SPDK_TEST_RBD 100: ${SPDK_TEST_VHOST=0} 101export SPDK_TEST_VHOST 102: ${SPDK_TEST_BLOCKDEV=0} 103export SPDK_TEST_BLOCKDEV 104: ${SPDK_TEST_IOAT=0} 105export SPDK_TEST_IOAT 106: ${SPDK_TEST_BLOBFS=0} 107export SPDK_TEST_BLOBFS 108: ${SPDK_TEST_VHOST_INIT=0} 109export SPDK_TEST_VHOST_INIT 110: ${SPDK_TEST_LVOL=0} 111export SPDK_TEST_LVOL 112: ${SPDK_TEST_VBDEV_COMPRESS=0} 113export SPDK_TEST_VBDEV_COMPRESS 114: ${SPDK_RUN_ASAN=0} 115export SPDK_RUN_ASAN 116: ${SPDK_RUN_UBSAN=0} 117export SPDK_RUN_UBSAN 118: ${SPDK_RUN_EXTERNAL_DPDK=""} 119export SPDK_RUN_EXTERNAL_DPDK 120: ${SPDK_RUN_NON_ROOT=0} 121export SPDK_RUN_NON_ROOT 122: ${SPDK_TEST_CRYPTO=0} 123export SPDK_TEST_CRYPTO 124: ${SPDK_TEST_FTL=0} 125export SPDK_TEST_FTL 126: ${SPDK_TEST_OCF=0} 127export SPDK_TEST_OCF 128: ${SPDK_TEST_VMD=0} 129export SPDK_TEST_VMD 130: ${SPDK_TEST_OPAL=0} 131export SPDK_TEST_OPAL 132: ${SPDK_TEST_NATIVE_DPDK} 133export SPDK_TEST_NATIVE_DPDK 134: ${SPDK_AUTOTEST_X=true} 135export SPDK_AUTOTEST_X 136: ${SPDK_TEST_RAID5=0} 137export SPDK_TEST_RAID5 138: ${SPDK_TEST_URING=0} 139export SPDK_TEST_URING 140: ${SPDK_TEST_USDT=0} 141export SPDK_TEST_USDT 142: ${SPDK_TEST_USE_IGB_UIO:=0} 143export SPDK_TEST_USE_IGB_UIO 144: ${SPDK_TEST_SCHEDULER:=0} 145export SPDK_TEST_SCHEDULER 146: ${SPDK_TEST_SCANBUILD:=0} 147export SPDK_TEST_SCANBUILD 148: ${SPDK_TEST_NVMF_NICS:=} 149export SPDK_TEST_NVMF_NICS 150: ${SPDK_TEST_SMA=0} 151export SPDK_TEST_SMA 152: ${SPDK_TEST_DAOS=0} 153export SPDK_TEST_DAOS 154: ${SPDK_TEST_XNVME:=0} 155export SPDK_TEST_XNVME 156: ${SPDK_TEST_ACCEL_DSA=0} 157export SPDK_TEST_ACCEL_DSA 158: ${SPDK_TEST_ACCEL_IAA=0} 159export SPDK_TEST_ACCEL_IAA 160# Comma-separated list of fuzzer targets matching test/fuzz/llvm/$target 161: ${SPDK_TEST_FUZZER_TARGET:=} 162export SPDK_TEST_FUZZER_TARGET 163: ${SPDK_TEST_NVMF_MDNS=0} 164export SPDK_TEST_NVMF_MDNS 165: ${SPDK_JSONRPC_GO_CLIENT=0} 166export SPDK_JSONRPC_GO_CLIENT 167 168# always test with SPDK shared objects. 169export SPDK_LIB_DIR="$rootdir/build/lib" 170export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib" 171export VFIO_LIB_DIR="$rootdir/build/libvfio-user/usr/local/lib" 172export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR:$VFIO_LIB_DIR 173 174# Tell setup.sh to wait for block devices upon each reset 175export PCI_BLOCK_SYNC_ON_RESET=yes 176 177# Export PYTHONPATH with addition of RPC framework. New scripts can be created 178# specific use cases for tests. 179export PYTHONPATH=$PYTHONPATH:$rootdir/python 180 181# Don't create Python .pyc files. When running with sudo these will be 182# created with root ownership and can cause problems when cleaning the repository. 183export PYTHONDONTWRITEBYTECODE=1 184 185# Export new_delete_type_mismatch to skip the known bug that exists in librados 186# https://tracker.ceph.com/issues/24078 187export ASAN_OPTIONS=new_delete_type_mismatch=0:disable_coredump=0:abort_on_error=1:use_sigaltstack=0 188export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1:disable_coredump=0:exitcode=134' 189 190# Export LeakSanitizer option to use suppression file in order to prevent false positives 191# and known leaks in external executables or libraries from showing up. 192asan_suppression_file="/var/tmp/asan_suppression_file" 193rm -rf "$asan_suppression_file" 2> /dev/null || sudo rm -rf "$asan_suppression_file" 194cat << EOL >> "$asan_suppression_file" 195# ASAN has some bugs around thread_local variables. We have a destructor in place 196# to free the thread contexts, but ASAN complains about the leak before those 197# destructors have a chance to run. So suppress this one specific leak using 198# LSAN_OPTIONS. 199leak:spdk_fs_alloc_thread_ctx 200 201# Suppress known leaks in fio project 202leak:$CONFIG_FIO_SOURCE_DIR/parse.c 203leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 204leak:$CONFIG_FIO_SOURCE_DIR/init.c 205leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 206leak:fio_memalign 207leak:spdk_fio_io_u_init 208# Suppress leaks in gperftools-libs from fio 209leak:libtcmalloc_minimal.so 210 211# Suppress leaks in libiscsi 212leak:libiscsi.so 213 214# Suppress leaks in libcrypto 215# Below is caused by openssl 3.0.8 leaks 216leak:libcrypto.so 217EOL 218 219# Suppress leaks in libfuse3 220echo "leak:libfuse3.so" >> "$asan_suppression_file" 221 222export LSAN_OPTIONS=suppressions="$asan_suppression_file" 223 224export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 225 226if [ -z "${DEPENDENCY_DIR:-}" ]; then 227 export DEPENDENCY_DIR=/var/spdk/dependencies 228else 229 export DEPENDENCY_DIR 230fi 231 232# Export location of where all the SPDK binaries are 233export SPDK_BIN_DIR="$rootdir/build/bin" 234export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 235 236# for vhost, vfio-user tests 237export QEMU_BIN=${QEMU_BIN:-} 238export VFIO_QEMU_BIN=${VFIO_QEMU_BIN:-} 239 240export AR_TOOL=$rootdir/scripts/ar-xnvme-fixer 241 242# For testing nvmes which are attached to some sort of a fanout switch in the CI pool 243export UNBIND_ENTIRE_IOMMU_GROUP=${UNBIND_ENTIRE_IOMMU_GROUP:-no} 244 245# pass our valgrind desire on to unittest.sh 246if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 247 export valgrind='' 248else 249 # unset all DEBUGINFOD_* vars that may affect our valgrind instance 250 unset -v "${!DEBUGINFOD_@}" 251fi 252 253if [ "$(uname -s)" = "Linux" ]; then 254 HUGEMEM=${HUGEMEM:-4096} 255 export CLEAR_HUGE=yes 256 if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]]; then 257 # Make sure that memory is distributed across all NUMA nodes - by default, all goes to 258 # node0, but if QAT devices are attached to a different node, all of their VFs will end 259 # up under that node too and memory needs to be available there for the tests. 260 export HUGE_EVEN_ALLOC=yes 261 fi 262 263 MAKE="make" 264 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 265elif [ "$(uname -s)" = "FreeBSD" ]; then 266 MAKE="gmake" 267 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 268 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 269 HUGEMEM=${HUGEMEM:-2048} 270elif [ "$(uname -s)" = "Windows" ]; then 271 MAKE="make" 272 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 273 # Keep the default 2GB for Windows. 274 HUGEMEM=${HUGEMEM:-2048} 275else 276 echo "Unknown OS \"$(uname -s)\"" 277 exit 1 278fi 279 280export HUGEMEM=$HUGEMEM 281 282if [ -z "${output_dir:-}" ]; then 283 mkdir -p "$rootdir/../output" 284 export output_dir="$rootdir/../output" 285fi 286 287NO_HUGE=() 288TEST_MODE= 289for i in "$@"; do 290 case "$i" in 291 --iso) 292 TEST_MODE=iso 293 ;; 294 --transport=*) 295 TEST_TRANSPORT="${i#*=}" 296 ;; 297 --sock=*) 298 TEST_SOCK="${i#*=}" 299 ;; 300 --no-hugepages) 301 NO_HUGE=(--no-huge -s 1024) 302 ;; 303 esac 304done 305 306# start rpc.py coprocess if it's not started yet 307if [[ -z ${RPC_PIPE_PID:-} ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 308 # Include list to all known plugins we use in the tests 309 PYTHONPATH+=":$rootdir/test/rpc_plugins" 310 coproc RPC_PIPE { PYTHONPATH="$PYTHONPATH" "$rootdir/scripts/rpc.py" --server; } 311 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 312 # all descriptors will automatically close together with this bash 313 # process, this will make rpc.py stop reading and exit gracefully 314fi 315 316function set_test_storage() { 317 [[ -v testdir ]] || return 0 318 319 local requested_size=$1 # bytes 320 local mount target_dir 321 322 local -A mounts fss sizes avails uses 323 local source fs size avail mount use 324 325 local storage_fallback storage_candidates 326 327 storage_fallback=$(mktemp -udt spdk.XXXXXX) 328 storage_candidates=( 329 "$testdir" 330 "$storage_fallback/tests/${testdir##*/}" 331 "$storage_fallback" 332 ) 333 334 if [[ -n ${ADD_TEST_STORAGE:-} ]]; then 335 # List of dirs|mounts separated by whitespaces 336 storage_candidates+=($ADD_TEST_STORAGE) 337 fi 338 339 if [[ -n ${DEDICATED_TEST_STORAGE:-} ]]; then 340 # Single, dedicated dir|mount 341 storage_candidates=("$DEDICATED_TEST_STORAGE") 342 fi 343 344 mkdir -p "${storage_candidates[@]}" 345 346 # add some headroom - 64M 347 requested_size=$((requested_size + (64 << 20))) 348 349 while read -r source fs size use avail _ mount; do 350 mounts["$mount"]=$source fss["$mount"]=$fs 351 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 352 uses["$mount"]=$((use * 1024)) 353 done < <(df -T | grep -v Filesystem) 354 355 printf '* Looking for test storage...\n' >&2 356 357 local target_space new_size 358 for target_dir in "${storage_candidates[@]}"; do 359 # FreeBSD's df is lacking the --output arg 360 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 361 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 362 363 target_space=${avails["$mount"]} 364 if ((target_space == 0 || target_space < requested_size)); then 365 continue 366 fi 367 if ((target_space >= requested_size)); then 368 # For in-memory fs, and / make sure our requested size won't fill most of the space. 369 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 370 new_size=$((uses["$mount"] + requested_size)) 371 if ((new_size * 100 / sizes["$mount"] > 95)); then 372 continue 373 fi 374 fi 375 fi 376 export SPDK_TEST_STORAGE=$target_dir 377 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 378 return 0 379 done 380 printf '* Test storage is not available\n' 381 return 1 382} 383 384function get_config_params() { 385 xtrace_disable 386 config_params='--enable-debug --enable-werror' 387 388 # for options with dependencies but no test flag, set them here 389 if [ -f /usr/include/infiniband/verbs.h ]; then 390 config_params+=' --with-rdma' 391 fi 392 393 if [ $SPDK_TEST_USDT -eq 1 ]; then 394 config_params+=" --with-usdt" 395 fi 396 397 if [ $(uname -s) == "FreeBSD" ]; then 398 intel="hw.model: Intel" 399 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 400 else 401 intel="GenuineIntel" 402 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 403 fi 404 if [[ "$cpu_vendor" != *"$intel"* ]]; then 405 config_params+=" --without-idxd" 406 else 407 config_params+=" --with-idxd" 408 fi 409 410 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 411 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 412 fi 413 414 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 415 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 416 fi 417 418 if [ -d /usr/include/iscsi ]; then 419 [[ $(< /usr/include/iscsi/iscsi.h) =~ "define LIBISCSI_API_VERSION ("([0-9]+)")" ]] \ 420 && libiscsi_version=${BASH_REMATCH[1]} 421 if ((libiscsi_version >= 20150621)); then 422 config_params+=' --with-iscsi-initiator' 423 fi 424 fi 425 426 if [[ $SPDK_TEST_UNITTEST -eq 0 && \ 427 $SPDK_TEST_SCANBUILD -eq 0 && -z \ 428 ${SPDK_TEST_AUTOBUILD:-} ]]; then 429 config_params+=' --disable-unit-tests' 430 fi 431 432 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]; then 433 if ge "$(nasm --version | awk '{print $3}')" 2.14 && [[ $SPDK_TEST_ISAL -eq 1 ]]; then 434 config_params+=' --with-vbdev-compress --with-dpdk-compressdev' 435 fi 436 fi 437 438 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 439 config_params+=' --with-rbd' 440 fi 441 442 # for options with no required dependencies, just test flags, set them here 443 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 444 config_params+=' --with-crypto' 445 fi 446 447 if [ $SPDK_TEST_OCF -eq 1 ]; then 448 config_params+=" --with-ocf" 449 fi 450 451 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 452 config_params+=' --enable-ubsan' 453 fi 454 455 if [ $SPDK_RUN_ASAN -eq 1 ]; then 456 config_params+=' --enable-asan' 457 fi 458 459 if [ "$(uname -s)" = "Linux" ]; then 460 config_params+=' --enable-coverage' 461 fi 462 463 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 464 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 465 config_params+=' --with-fuse' 466 fi 467 fi 468 469 if [[ -f /usr/include/liburing/io_uring.h && -f /usr/include/linux/ublk_cmd.h ]]; then 470 config_params+=' --with-ublk' 471 fi 472 473 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 474 config_params+=' --with-raid5f' 475 fi 476 477 if [ $SPDK_TEST_VFIOUSER -eq 1 ] || [ $SPDK_TEST_VFIOUSER_QEMU -eq 1 ] || [ $SPDK_TEST_SMA -eq 1 ]; then 478 config_params+=' --with-vfio-user' 479 fi 480 481 # Check whether liburing library header exists 482 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 483 config_params+=' --with-uring' 484 fi 485 486 if [ -n "${SPDK_RUN_EXTERNAL_DPDK:-}" ]; then 487 config_params+=" --with-dpdk=$SPDK_RUN_EXTERNAL_DPDK" 488 fi 489 490 if [[ $SPDK_TEST_SMA -eq 1 ]]; then 491 config_params+=' --with-sma' 492 config_params+=' --with-crypto' 493 fi 494 495 if [ -f /usr/include/daos.h ] && [ $SPDK_TEST_DAOS -eq 1 ]; then 496 config_params+=' --with-daos' 497 fi 498 499 # Make the xnvme module available for the tests 500 if [[ $SPDK_TEST_XNVME -eq 1 ]]; then 501 config_params+=' --with-xnvme' 502 fi 503 504 if [[ $SPDK_TEST_FUZZER -eq 1 ]]; then 505 config_params+=" $(get_fuzzer_target_config)" 506 fi 507 508 if [[ $SPDK_TEST_NVMF_MDNS -eq 1 ]]; then 509 config_params+=' --with-avahi' 510 fi 511 512 if [[ $SPDK_JSONRPC_GO_CLIENT -eq 1 ]]; then 513 config_params+=' --with-golang' 514 fi 515 516 echo "$config_params" 517 xtrace_restore 518} 519 520function get_fuzzer_target_config() { 521 local -A fuzzer_targets_to_config=() 522 local config target 523 524 fuzzer_targets_to_config["vfio"]="--with-vfio-user" 525 for target in $(get_fuzzer_targets); do 526 [[ -n ${fuzzer_targets_to_config["$target"]:-} ]] || continue 527 config+=("${fuzzer_targets_to_config["$target"]}") 528 done 529 530 if ((${#config[@]} > 0)); then 531 echo "${config[*]}" 532 fi 533} 534 535function get_fuzzer_targets() { 536 local fuzzers=() 537 538 if [[ -n ${SPDK_TEST_FUZZER_TARGET:-} ]]; then 539 IFS="," read -ra fuzzers <<< "$SPDK_TEST_FUZZER_TARGET" 540 else 541 fuzzers=("$rootdir/test/fuzz/llvm/"*) 542 fuzzers=("${fuzzers[@]##*/}") 543 fi 544 545 echo "${fuzzers[*]}" 546} 547 548function rpc_cmd() { 549 xtrace_disable 550 local rsp rc=1 551 local stdin cmd cmds_number=0 status_number=0 status 552 553 if (($#)); then 554 cmds_number=1 555 echo "$@" >&$RPC_PIPE_INPUT 556 elif [[ ! -t 0 ]]; then 557 mapfile -t stdin <&0 558 cmds_number=${#stdin[@]} 559 printf '%s\n' "${stdin[@]}" >&$RPC_PIPE_INPUT 560 else 561 return 0 562 fi 563 564 while read -t 15 -ru $RPC_PIPE_OUTPUT rsp; do 565 if [[ $rsp == "**STATUS="* ]]; then 566 status[${rsp#*=}]=$rsp 567 if ((++status_number == cmds_number)); then 568 break 569 fi 570 continue 571 fi 572 echo "$rsp" 573 done 574 575 rc=${!status[*]} 576 xtrace_restore 577 [[ $rc == 0 ]] 578} 579 580function rpc_cmd_simple_data_json() { 581 582 local elems="$1[@]" elem 583 local -gA jq_out=() 584 local jq val 585 586 local lvs=( 587 "uuid" 588 "name" 589 "base_bdev" 590 "total_data_clusters" 591 "free_clusters" 592 "block_size" 593 "cluster_size" 594 ) 595 596 local bdev=( 597 "name" 598 "aliases[0]" 599 "block_size" 600 "num_blocks" 601 "uuid" 602 "product_name" 603 "supported_io_types.read" 604 "supported_io_types.write" 605 "driver_specific.lvol.clone" 606 "driver_specific.lvol.base_snapshot" 607 "driver_specific.lvol.esnap_clone" 608 "driver_specific.lvol.external_snapshot_name" 609 ) 610 611 [[ -v $elems ]] || return 1 612 613 for elem in "${!elems}"; do 614 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 615 done 616 jq+=',"\n"' 617 618 shift 619 while read -r elem val; do 620 jq_out["$elem"]=$val 621 done < <(rpc_cmd "$@" | jq -jr "$jq") 622 ((${#jq_out[@]} > 0)) || return 1 623} 624 625function valid_exec_arg() { 626 local arg=$1 627 # First argument must be the executable so do some basic sanity checks first. For bash, this 628 # covers two basic cases where es == 126 || es == 127 so catch them early on and fail hard 629 # if needed. 630 case "$(type -t "$arg")" in 631 builtin | function) ;; 632 file) arg=$(type -P "$arg") && [[ -x $arg ]] ;; 633 *) return 1 ;; 634 esac 635} 636 637function NOT() { 638 local es=0 639 640 valid_exec_arg "$@" || return 1 641 "$@" || es=$? 642 643 # Logic looks like so: 644 # - return false if command exit successfully 645 # - return false if command exit after receiving a core signal (FIXME: or any signal?) 646 # - return true if command exit with an error 647 648 # This naively assumes that the process doesn't exit with > 128 on its own. 649 if ((es > 128)); then 650 es=$((es & ~128)) 651 case "$es" in 652 3) ;& # SIGQUIT 653 4) ;& # SIGILL 654 6) ;& # SIGABRT 655 8) ;& # SIGFPE 656 9) ;& # SIGKILL 657 11) es=0 ;; # SIGSEGV 658 *) es=1 ;; 659 esac 660 elif [[ -n ${EXIT_STATUS:-} ]] && ((es != EXIT_STATUS)); then 661 es=0 662 fi 663 664 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 665 ((!es == 0)) 666} 667 668function timing() { 669 direction="$1" 670 testname="$2" 671 672 now=$(date +%s) 673 674 if [ "$direction" = "enter" ]; then 675 export timing_stack="${timing_stack:-};${now}" 676 export test_stack="${test_stack:-};${testname}" 677 else 678 touch "$output_dir/timing.txt" 679 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 680 681 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 682 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 683 684 elapsed=$((now - start_time - child_time)) 685 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 686 687 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 688 fi 689} 690 691function timing_cmd() ( 692 # The use-case here is this: ts=$(timing_cmd echo bar). Since stdout is always redirected 693 # to a pipe handling the $(), lookup the stdin's device and determine if it's sane to send 694 # cmd's output to it. If not, just null it. 695 696 [[ -t 0 ]] && exec {cmd_out}>&0 || exec {cmd_out}> /dev/null 697 698 local time=0 TIMEFORMAT=%2R # seconds 699 700 # We redirect cmd's std{out,err} to a separate fd dup'ed to stdin's device (or /dev/null) to 701 # catch only output from the time builtin - output from the actual cmd would be still visible, 702 # but $() will return just the time's data, hence making it possible to just do: 703 # time_of_super_verbose_cmd=$(timing_cmd super_verbose_cmd) 704 time=$({ time "$@" >&"$cmd_out" 2>&1; } 2>&1) 705 706 echo "$time" 707) 708 709function timing_enter() { 710 xtrace_disable 711 timing "enter" "$1" 712 xtrace_restore 713} 714 715function timing_exit() { 716 xtrace_disable 717 timing "exit" "$1" 718 xtrace_restore 719} 720 721function timing_finish() { 722 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 723 if [ -x "$flamegraph" ]; then 724 "$flamegraph" \ 725 --title 'Build Timing' \ 726 --nametype 'Step:' \ 727 --countname seconds \ 728 $output_dir/timing.txt \ 729 > $output_dir/timing.svg 730 fi 731} 732 733function create_test_list() { 734 xtrace_disable 735 # First search all scripts in main SPDK directory. 736 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 737 # Follow up with search in test directory recursively. 738 completion+=$'\n'$(grep -rshI --include="*.sh" --exclude="*autotest_common.sh" -e "run_test " $rootdir/test) 739 printf "%s" "$completion" | grep -v "#" \ 740 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 741 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 742 xtrace_restore 743} 744 745function gdb_attach() { 746 gdb -q --batch \ 747 -ex 'handle SIGHUP nostop pass' \ 748 -ex 'handle SIGQUIT nostop pass' \ 749 -ex 'handle SIGPIPE nostop pass' \ 750 -ex 'handle SIGALRM nostop pass' \ 751 -ex 'handle SIGTERM nostop pass' \ 752 -ex 'handle SIGUSR1 nostop pass' \ 753 -ex 'handle SIGUSR2 nostop pass' \ 754 -ex 'handle SIGCHLD nostop pass' \ 755 -ex 'set print thread-events off' \ 756 -ex 'cont' \ 757 -ex 'thread apply all bt' \ 758 -ex 'quit' \ 759 --tty=/dev/stdout \ 760 -p $1 761} 762 763function process_core() { 764 # Note that this always was racy as we can't really sync with the kernel 765 # to see if there's any core queued up for writing. We could check if 766 # collector is running and wait for it explicitly, but it doesn't seem 767 # to be worth the effort. So assume that if we are being called via 768 # trap, as in, when some error has occurred, wait up to 10s for any 769 # potential cores. If we are called just for cleanup at the very end, 770 # don't wait since all the tests ended successfully, hence having any 771 # critical cores lying around is unlikely. 772 ((autotest_es != 0)) && sleep 10 773 774 local coredumps core 775 776 coredumps=("$output_dir/coredumps/"*.bt.txt) 777 778 ((${#coredumps[@]} > 0)) || return 0 779 chmod -R a+r "$output_dir/coredumps" 780 781 for core in "${coredumps[@]}"; do 782 cat <<- BT 783 ##### CORE BT ${core##*/} ##### 784 785 $(<"$core") 786 787 -- 788 BT 789 done 790 return 1 791} 792 793function process_shm() { 794 type=$1 795 id=$2 796 if [ "$type" = "--pid" ]; then 797 id="pid${id}" 798 fi 799 800 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 801 802 if [[ -z ${shm_files:-} ]]; then 803 echo "SHM File for specified PID or shared memory id: ${id} not found!" 804 return 1 805 fi 806 for n in $shm_files; do 807 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 808 done 809 return 0 810} 811 812# Parameters: 813# $1 - process pid 814# $2 - rpc address (optional) 815# $3 - max retries (optional) 816function waitforlisten() { 817 if [ -z "${1:-}" ]; then 818 exit 1 819 fi 820 821 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 822 local max_retries=${3:-100} 823 824 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 825 # turn off trace for this loop 826 xtrace_disable 827 local ret=0 828 local i 829 for ((i = max_retries; i != 0; i--)); do 830 # if the process is no longer running, then exit the script 831 # since it means the application crashed 832 if ! kill -s 0 $1; then 833 echo "ERROR: process (pid: $1) is no longer running" 834 ret=1 835 break 836 fi 837 838 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 839 break 840 fi 841 842 sleep 0.5 843 done 844 845 xtrace_restore 846 if ((i == 0)); then 847 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 848 ret=1 849 fi 850 return $ret 851} 852 853function waitfornbd() { 854 local nbd_name=$1 855 local i 856 857 for ((i = 1; i <= 20; i++)); do 858 if grep -q -w $nbd_name /proc/partitions; then 859 break 860 else 861 sleep 0.1 862 fi 863 done 864 865 # The nbd device is now recognized as a block device, but there can be 866 # a small delay before we can start I/O to that block device. So loop 867 # here trying to read the first block of the nbd block device to a temp 868 # file. Note that dd returns success when reading an empty file, so we 869 # need to check the size of the output file instead. 870 for ((i = 1; i <= 20; i++)); do 871 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 872 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 873 rm -f "$SPDK_TEST_STORAGE/nbdtest" 874 if [ "$size" != "0" ]; then 875 return 0 876 else 877 sleep 0.1 878 fi 879 done 880 881 return 1 882} 883 884function waitforbdev() { 885 local bdev_name=$1 886 local bdev_timeout=$2 887 local i 888 [[ -z ${bdev_timeout:-} ]] && bdev_timeout=2000 # ms 889 890 $rpc_py bdev_wait_for_examine 891 892 if $rpc_py bdev_get_bdevs -b $bdev_name -t $bdev_timeout; then 893 return 0 894 fi 895 896 return 1 897} 898 899function make_filesystem() { 900 local fstype=$1 901 local dev_name=$2 902 local i=0 903 local force 904 905 if [ $fstype = ext4 ]; then 906 force=-F 907 else 908 force=-f 909 fi 910 911 while ! mkfs.${fstype} $force ${dev_name}; do 912 if [ $i -ge 15 ]; then 913 return 1 914 fi 915 i=$((i + 1)) 916 sleep 1 917 done 918 919 return 0 920} 921 922function killprocess() { 923 # $1 = process pid 924 if [ -z "${1:-}" ]; then 925 return 1 926 fi 927 928 if kill -0 $1; then 929 if [ $(uname) = Linux ]; then 930 process_name=$(ps --no-headers -o comm= $1) 931 else 932 process_name=$(ps -c -o command $1 | tail -1) 933 fi 934 if [ "$process_name" = "sudo" ]; then 935 # kill the child process, which is the actual app 936 # (assume $1 has just one child) 937 local child 938 child="$(pgrep -P $1)" 939 echo "killing process with pid $child" 940 kill $child 941 else 942 echo "killing process with pid $1" 943 kill $1 944 fi 945 946 # wait for the process regardless if its the dummy sudo one 947 # or the actual app - it should terminate anyway 948 wait $1 949 else 950 # the process is not there anymore 951 echo "Process with pid $1 is not found" 952 fi 953} 954 955function iscsicleanup() { 956 echo "Cleaning up iSCSI connection" 957 iscsiadm -m node --logout || true 958 iscsiadm -m node -o delete || true 959 rm -rf /var/lib/iscsi/nodes/* 960} 961 962function stop_iscsi_service() { 963 if cat /etc/*-release | grep Ubuntu; then 964 service open-iscsi stop 965 else 966 service iscsid stop 967 fi 968} 969 970function start_iscsi_service() { 971 if cat /etc/*-release | grep Ubuntu; then 972 service open-iscsi start 973 else 974 service iscsid start 975 fi 976} 977 978function rbd_setup() { 979 # $1 = monitor ip address 980 # $2 = name of the namespace 981 if [ -z "${1:-}" ]; then 982 echo "No monitor IP address provided for ceph" 983 exit 1 984 fi 985 if [ -n "${2:-}" ]; then 986 if ip netns list | grep "$2"; then 987 NS_CMD="ip netns exec $2" 988 else 989 echo "No namespace $2 exists" 990 exit 1 991 fi 992 fi 993 994 if hash ceph; then 995 export PG_NUM=128 996 export RBD_POOL=rbd 997 export RBD_NAME=foo 998 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 999 $NS_CMD $rootdir/scripts/ceph/start.sh $1 1000 1001 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 1002 $NS_CMD rbd create $RBD_NAME --size 1000 1003 fi 1004} 1005 1006function rbd_cleanup() { 1007 if hash ceph; then 1008 $rootdir/scripts/ceph/stop.sh || true 1009 rm -f /var/tmp/ceph_raw.img 1010 fi 1011} 1012 1013function daos_setup() { 1014 # $1 = pool name 1015 # $2 = cont name 1016 if [ -z "${1:-}" ]; then 1017 echo "No pool name provided" 1018 exit 1 1019 fi 1020 if [ -z "${2:-}" ]; then 1021 echo "No cont name provided" 1022 exit 1 1023 fi 1024 1025 dmg pool create --size=10G $1 || true 1026 daos container create --type=POSIX --label=$2 $1 || true 1027} 1028 1029function daos_cleanup() { 1030 local pool=${1:-testpool} 1031 local cont=${2:-testcont} 1032 1033 daos container destroy -f $pool $cont || true 1034 sudo dmg pool destroy -f $pool || true 1035} 1036 1037function _start_stub() { 1038 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 1039 # but ASLR can still be unreliable in some cases. 1040 # We will re-enable it again after multi-process testing is complete in kill_stub(). 1041 # Save current setting so it can be restored upon calling kill_stub(). 1042 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 1043 echo 0 > /proc/sys/kernel/randomize_va_space 1044 $rootdir/test/app/stub/stub $1 & 1045 stubpid=$! 1046 echo Waiting for stub to ready for secondary processes... 1047 while ! [ -e /var/run/spdk_stub0 ]; do 1048 # If stub dies while we wait, bail 1049 [[ -e /proc/$stubpid ]] || return 1 1050 sleep 1s 1051 done 1052 echo done. 1053} 1054 1055function start_stub() { 1056 if ! _start_stub "$@"; then 1057 echo "stub failed" >&2 1058 return 1 1059 fi 1060} 1061 1062function kill_stub() { 1063 if [[ -e /proc/$stubpid ]]; then 1064 kill $1 $stubpid 1065 wait $stubpid 1066 fi 2> /dev/null || : 1067 rm -f /var/run/spdk_stub0 1068 # Re-enable ASLR now that we are done with multi-process testing 1069 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 1070 # randomizing and is the default on all recent Linux kernels 1071 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 1072} 1073 1074function run_test() { 1075 if [ $# -le 1 ]; then 1076 echo "Not enough parameters" 1077 echo "usage: run_test test_name test_script [script_params]" 1078 exit 1 1079 fi 1080 1081 xtrace_disable 1082 local test_name="$1" pid 1083 shift 1084 1085 if [ -n "${test_domain:-}" ]; then 1086 export test_domain="${test_domain}.${test_name}" 1087 else 1088 export test_domain="$test_name" 1089 fi 1090 1091 # Signal our daemons to update the test tag 1092 echo "$test_domain" > "$TEST_TAG_FILE" 1093 for pid in "$output_dir/collect-"@(cpu-load|cpu-temp|vmstat).pid; do 1094 kill -USR1 "$(< "$pid")" || true 1095 done 1096 1097 timing_enter $test_name 1098 echo "************************************" 1099 echo "START TEST $test_name" 1100 echo "************************************" 1101 xtrace_restore 1102 time "$@" 1103 xtrace_disable 1104 echo "************************************" 1105 echo "END TEST $test_name" 1106 echo "************************************" 1107 timing_exit $test_name 1108 1109 export test_domain=${test_domain%"$test_name"} 1110 if [ -n "$test_domain" ]; then 1111 export test_domain=${test_domain%?} 1112 fi 1113 1114 if [ -z "${test_domain:-}" ]; then 1115 echo "top_level $test_name" >> $output_dir/test_completions.txt 1116 else 1117 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 1118 fi 1119 xtrace_restore 1120} 1121 1122function skip_run_test_with_warning() { 1123 echo "WARNING: $1" 1124 echo "Test run may fail if run with autorun.sh" 1125 echo "Please check your $rootdir/test/common/skipped_tests.txt" 1126} 1127 1128function print_backtrace() { 1129 # if errexit is not enabled, don't print a backtrace 1130 [[ "$-" =~ e ]] || return 0 1131 1132 local args=("${BASH_ARGV[@]}") 1133 1134 xtrace_disable 1135 # Reset IFS in case we were called from an environment where it was modified 1136 IFS=" "$'\t'$'\n' 1137 echo "========== Backtrace start: ==========" 1138 echo "" 1139 for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do 1140 local func="${FUNCNAME[$i]}" 1141 local line_nr="${BASH_LINENO[$((i - 1))]}" 1142 local src="${BASH_SOURCE[$i]}" 1143 local bt="" cmdline=() 1144 1145 if [[ -f $src ]]; then 1146 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 1147 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 1148 fi 1149 1150 # If extdebug set the BASH_ARGC[i], try to fetch all the args 1151 if ((BASH_ARGC[i] > 0)); then 1152 # Use argc as index to reverse the stack 1153 local argc=${BASH_ARGC[i]} arg 1154 for arg in "${args[@]::BASH_ARGC[i]}"; do 1155 cmdline[argc--]="[\"$arg\"]" 1156 done 1157 args=("${args[@]:BASH_ARGC[i]}") 1158 fi 1159 1160 echo "in $src:$line_nr -> $func($( 1161 IFS="," 1162 printf '%s\n' "${cmdline[*]:-[]}" 1163 ))" 1164 echo " ..." 1165 echo "${bt:-backtrace unavailable}" 1166 echo " ..." 1167 done 1168 echo "" 1169 echo "========== Backtrace end ==========" 1170 xtrace_restore 1171 return 0 1172} 1173 1174function waitforserial() { 1175 local i=0 1176 local nvme_device_counter=1 nvme_devices=0 1177 if [[ -n "${2:-}" ]]; then 1178 nvme_device_counter=$2 1179 fi 1180 1181 # Wait initially for min 2s to make sure all devices are ready for use. 1182 sleep 2 1183 while ((i++ <= 15)); do 1184 nvme_devices=$(lsblk -l -o NAME,SERIAL | grep -c "$1") 1185 ((nvme_devices == nvme_device_counter)) && return 0 1186 if ((nvme_devices > nvme_device_counter)); then 1187 echo "$nvme_device_counter device(s) expected, found $nvme_devices" >&2 1188 fi 1189 echo "Waiting for devices" 1190 sleep 1 1191 done 1192 return 1 1193} 1194 1195function waitforserial_disconnect() { 1196 local i=0 1197 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1198 [ $i -lt 15 ] || break 1199 i=$((i + 1)) 1200 echo "Waiting for disconnect devices" 1201 sleep 1 1202 done 1203 1204 if lsblk -l -o NAME,SERIAL | grep -q -w $1; then 1205 return 1 1206 fi 1207 1208 return 0 1209} 1210 1211function waitforblk() { 1212 local i=0 1213 while ! lsblk -l -o NAME | grep -q -w $1; do 1214 [ $i -lt 15 ] || break 1215 i=$((i + 1)) 1216 sleep 1 1217 done 1218 1219 if ! lsblk -l -o NAME | grep -q -w $1; then 1220 return 1 1221 fi 1222 1223 return 0 1224} 1225 1226function waitforblk_disconnect() { 1227 local i=0 1228 while lsblk -l -o NAME | grep -q -w $1; do 1229 [ $i -lt 15 ] || break 1230 i=$((i + 1)) 1231 sleep 1 1232 done 1233 1234 if lsblk -l -o NAME | grep -q -w $1; then 1235 return 1 1236 fi 1237 1238 return 0 1239} 1240 1241function waitforfile() { 1242 local i=0 1243 while [ ! -e $1 ]; do 1244 [ $i -lt 200 ] || break 1245 i=$((i + 1)) 1246 sleep 0.1 1247 done 1248 1249 if [ ! -e $1 ]; then 1250 return 1 1251 fi 1252 1253 return 0 1254} 1255 1256function fio_config_gen() { 1257 local config_file=$1 1258 local workload=$2 1259 local bdev_type=$3 1260 local env_context=$4 1261 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1262 1263 if [ -e "$config_file" ]; then 1264 echo "Configuration File Already Exists!: $config_file" 1265 return 1 1266 fi 1267 1268 if [ -z "${workload:-}" ]; then 1269 workload=randrw 1270 fi 1271 1272 if [ -n "$env_context" ]; then 1273 env_context="env_context=$env_context" 1274 fi 1275 1276 touch $1 1277 1278 cat > $1 << EOL 1279[global] 1280thread=1 1281$env_context 1282group_reporting=1 1283direct=1 1284norandommap=1 1285percentile_list=50:99:99.9:99.99:99.999 1286time_based=1 1287ramp_time=0 1288EOL 1289 1290 if [ "$workload" == "verify" ]; then 1291 cat <<- EOL >> $config_file 1292 verify=sha1 1293 verify_backlog=1024 1294 rw=randwrite 1295 EOL 1296 1297 # To avoid potential data race issue due to the AIO device 1298 # flush mechanism, add the flag to serialize the writes. 1299 # This is to fix the intermittent IO failure issue of #935 1300 if [ "$bdev_type" == "AIO" ]; then 1301 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1302 echo "serialize_overlap=1" >> $config_file 1303 fi 1304 fi 1305 elif [ "$workload" == "trim" ]; then 1306 echo "rw=trimwrite" >> $config_file 1307 else 1308 echo "rw=$workload" >> $config_file 1309 fi 1310} 1311 1312function fio_plugin() { 1313 # Setup fio binary cmd line 1314 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1315 # gcc and clang uses different sanitizer libraries 1316 local sanitizers=(libasan libclang_rt.asan) 1317 local plugin=$1 1318 shift 1319 1320 local asan_lib= 1321 for sanitizer in "${sanitizers[@]}"; do 1322 asan_lib=$(ldd $plugin | grep $sanitizer | awk '{print $3}') 1323 if [[ -n "${asan_lib:-}" ]]; then 1324 break 1325 fi 1326 done 1327 1328 # Preload the sanitizer library to fio if fio_plugin was compiled with it 1329 LD_PRELOAD="$asan_lib $plugin" "$fio_dir"/fio "$@" 1330} 1331 1332function fio_bdev() { 1333 fio_plugin "$rootdir/build/fio/spdk_bdev" "$@" 1334} 1335 1336function fio_nvme() { 1337 fio_plugin "$rootdir/build/fio/spdk_nvme" "$@" 1338} 1339 1340function get_lvs_free_mb() { 1341 local lvs_uuid=$1 1342 local lvs_info 1343 local fc 1344 local cs 1345 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1346 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1347 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1348 1349 # Change to MB's 1350 free_mb=$((fc * cs / 1024 / 1024)) 1351 echo "$free_mb" 1352} 1353 1354function get_bdev_size() { 1355 local bdev_name=$1 1356 local bdev_info 1357 local bs 1358 local nb 1359 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1360 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1361 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1362 1363 # Change to MB's 1364 bdev_size=$((bs * nb / 1024 / 1024)) 1365 echo "$bdev_size" 1366} 1367 1368function autotest_cleanup() { 1369 local autotest_es=$? 1370 xtrace_disable 1371 1372 # Slurp at_app_exit() so we can kill all lingering vhost and qemu processes 1373 # in one swing. We do this in a subshell as vhost/common.sh is too eager to 1374 # do some extra work which we don't care about in this context. 1375 # shellcheck source=/dev/null 1376 vhost_reap() (source "$rootdir/test/vhost/common.sh" &> /dev/null || return 0 && at_app_exit) 1377 1378 # catch any stray core files and kill all remaining SPDK processes. Update 1379 # autotest_es in case autotest reported success but cores and/or processes 1380 # were left behind regardless. 1381 1382 process_core || autotest_es=1 1383 reap_spdk_processes || autotest_es=1 1384 vhost_reap || autotest_es=1 1385 1386 $rootdir/scripts/setup.sh reset 1387 $rootdir/scripts/setup.sh cleanup 1388 if [ $(uname -s) = "Linux" ]; then 1389 modprobe -r uio_pci_generic 1390 fi 1391 rm -rf "$asan_suppression_file" 1392 if [[ -n ${old_core_pattern:-} ]]; then 1393 echo "$old_core_pattern" > /proc/sys/kernel/core_pattern 1394 fi 1395 if [[ -e /proc/$udevadm_pid/status ]]; then 1396 kill "$udevadm_pid" || : 1397 fi 1398 1399 local storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 1400 1401 if ((${#storage_fallback_purge[@]} > 0)); then 1402 rm -rf "${storage_fallback_purge[@]}" 1403 fi 1404 1405 if ((autotest_es)); then 1406 if [[ $(uname) == FreeBSD ]]; then 1407 ps aux 1408 elif [[ $(uname) == Linux ]]; then 1409 # Get more detailed view 1410 grep . /proc/[0-9]*/status 1411 # Dump some extra info into kernel log 1412 echo "######## Autotest Cleanup Dump ########" > /dev/kmsg 1413 # Show cpus backtraces 1414 echo l > /proc/sysrq-trigger 1415 # Show mem usage 1416 echo m > /proc/sysrq-trigger 1417 # show task states 1418 echo t > /proc/sysrq-trigger 1419 # show blocked tasks 1420 echo w > /proc/sysrq-trigger 1421 1422 fi > "$output_dir/proc_list.txt" 2>&1 || : 1423 fi 1424 1425 local pid 1426 for pid in "$output_dir/"*.pid; do 1427 killprocess "$(< "$pid")" 1428 done || : 1429 1430 rm -f "$output_dir/"*.pid 1431 rm -f "$TEST_TAG_FILE" 1432 1433 xtrace_restore 1434 return $autotest_es 1435} 1436 1437function freebsd_update_contigmem_mod() { 1438 if [ $(uname) = FreeBSD ]; then 1439 kldunload contigmem.ko || true 1440 if [ -n "${SPDK_RUN_EXTERNAL_DPDK:-}" ]; then 1441 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/ 1442 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/ 1443 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/ 1444 cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/ 1445 else 1446 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1447 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1448 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1449 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1450 fi 1451 fi 1452} 1453 1454function freebsd_set_maxsock_buf() { 1455 # FreeBSD needs 4MB maxsockbuf size to pass socket unit tests. 1456 # Otherwise tests fail due to ENOBUFS when trying to do setsockopt(SO_RCVBUF|SO_SNDBUF). 1457 # See https://github.com/spdk/spdk/issues/2943 1458 if [[ $(uname) = FreeBSD ]] && (($(sysctl -n kern.ipc.maxsockbuf) < 4194304)); then 1459 sysctl kern.ipc.maxsockbuf=4194304 1460 fi 1461} 1462 1463function get_nvme_name_from_bdf() { 1464 blkname=() 1465 1466 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1467 if [ -z "${nvme_devs:-}" ]; then 1468 return 1469 fi 1470 for dev in $nvme_devs; do 1471 link_name=$(readlink /sys/block/$dev/device/device) || true 1472 if [ -z "${link_name:-}" ]; then 1473 link_name=$(readlink /sys/block/$dev/device) 1474 fi 1475 bdf=$(basename "$link_name") 1476 if [ "$bdf" = "$1" ]; then 1477 blkname+=($dev) 1478 fi 1479 done 1480 1481 printf '%s\n' "${blkname[@]}" 1482} 1483 1484function get_nvme_ctrlr_from_bdf() { 1485 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1486 if [[ -z "${bdf_sysfs_path:-}" ]]; then 1487 return 1488 fi 1489 1490 printf '%s\n' "$(basename $bdf_sysfs_path)" 1491} 1492 1493# Get BDF addresses of all NVMe drives currently attached to 1494# uio-pci-generic or vfio-pci 1495function get_nvme_bdfs() { 1496 local bdfs=() 1497 bdfs=($("$rootdir/scripts/gen_nvme.sh" | jq -r '.config[].params.traddr')) 1498 if ((${#bdfs[@]} == 0)); then 1499 echo "No bdevs found" >&2 1500 return 1 1501 fi 1502 printf '%s\n' "${bdfs[@]}" 1503} 1504 1505# Same as function above, but just get the first disks BDF address 1506function get_first_nvme_bdf() { 1507 local bdfs=() 1508 bdfs=($(get_nvme_bdfs)) 1509 1510 echo "${bdfs[0]}" 1511} 1512 1513function nvme_namespace_revert() { 1514 $rootdir/scripts/setup.sh 1515 sleep 1 1516 local bdfs=() 1517 bdfs=($(get_nvme_bdfs)) 1518 1519 $rootdir/scripts/setup.sh reset 1520 1521 for bdf in "${bdfs[@]}"; do 1522 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1523 if [[ -z "${nvme_ctrlr:-}" ]]; then 1524 continue 1525 fi 1526 1527 # Check Optional Admin Command Support for Namespace Management 1528 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1529 oacs_ns_manage=$((oacs & 0x8)) 1530 1531 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1532 # This assumes every NVMe controller contains single namespace, 1533 # encompassing Total NVM Capacity and formatted as 512 block size. 1534 # 512 block size is needed for test/vhost/vhost_boot.sh to 1535 # successfully run. 1536 1537 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1538 if [[ "$unvmcap" -eq 0 ]]; then 1539 # All available space already used 1540 continue 1541 fi 1542 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1543 cntlid=$(nvme id-ctrl ${nvme_ctrlr} | grep cntlid | cut -d: -f2) 1544 blksize=512 1545 1546 size=$((tnvmcap / blksize)) 1547 1548 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c $cntlid || true 1549 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1550 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1551 nvme attach-ns ${nvme_ctrlr} -n 1 -c $cntlid 1552 nvme reset ${nvme_ctrlr} 1553 waitforfile "${nvme_ctrlr}n1" 1554 fi 1555 done 1556} 1557 1558# Get BDFs based on device ID, such as 0x0a54 1559function get_nvme_bdfs_by_id() { 1560 local bdfs=() 1561 1562 for bdf in $(get_nvme_bdfs); do 1563 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1564 if [[ "$device" == "$1" ]]; then 1565 bdfs+=($bdf) 1566 fi 1567 done 1568 1569 printf '%s\n' "${bdfs[@]}" 1570} 1571 1572function opal_revert_cleanup() { 1573 # The OPAL CI tests is only used for P4510 devices. 1574 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1575 if [[ -z ${bdfs[0]:-} ]]; then 1576 return 0 1577 fi 1578 1579 $SPDK_BIN_DIR/spdk_tgt & 1580 spdk_tgt_pid=$! 1581 waitforlisten $spdk_tgt_pid 1582 1583 bdf_id=0 1584 for bdf in "${bdfs[@]}"; do 1585 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme"${bdf_id} -t "pcie" -a ${bdf} 1586 # Ignore if this fails. 1587 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b "nvme"${bdf_id} -p test || true 1588 ((++bdf_id)) 1589 done 1590 1591 killprocess $spdk_tgt_pid 1592} 1593 1594function pap() { 1595 while read -r file; do 1596 cat <<- FILE 1597 --- $file --- 1598 $(<"$file") 1599 --- $file --- 1600 FILE 1601 rm -f "$file" 1602 done < <(find "$@" -type f | sort -u) 1603} 1604 1605function get_proc_paths() { 1606 case "$(uname -s)" in 1607 Linux) # ps -e -opid,exe <- not supported under {centos7,rocky8}'s procps-ng 1608 local pid exe 1609 for pid in /proc/[0-9]*; do 1610 exe=$(readlink "$pid/exe") || continue 1611 exe=${exe/ (deleted)/} 1612 echo "${pid##*/} $exe" 1613 done 1614 ;; 1615 FreeeBSD) procstat -ab | awk '{print $1, $4}' ;; 1616 esac 1617} 1618 1619exec_files() { file "$@" | awk -F: '/ELF.+executable/{print $1}'; } 1620 1621function reap_spdk_processes() { 1622 local bins test_bins procs 1623 local spdk_procs spdk_pids 1624 1625 mapfile -t test_bins < <(find "$rootdir"/test/{app,env,event} -type f) 1626 mapfile -t bins < <( 1627 exec_files "${test_bins[@]}" 1628 readlink -f "$SPDK_BIN_DIR/"* "$SPDK_EXAMPLE_DIR/"* 1629 ) 1630 1631 mapfile -t spdk_procs < <(get_proc_paths | grep -E "$( 1632 IFS="|" 1633 echo "${bins[*]#$rootdir/}" 1634 )" || true) 1635 ((${#spdk_procs[@]} > 0)) || return 0 1636 1637 printf '%s is still up, killing\n' "${spdk_procs[@]}" >&2 1638 mapfile -t spdk_pids < <(printf '%s\n' "${spdk_procs[@]}" | awk '{print $1}') 1639 1640 kill -SIGKILL "${spdk_pids[@]}" 2> /dev/null || : 1641 return 1 1642} 1643 1644function is_block_zoned() { 1645 local device=$1 1646 1647 [[ -e /sys/block/$device/queue/zoned ]] || return 1 1648 [[ $(< "/sys/block/$device/queue/zoned") != none ]] 1649} 1650 1651function get_zoned_devs() { 1652 local -gA zoned_devs=() 1653 local nvme bdf 1654 1655 for nvme in /sys/block/nvme*; do 1656 if is_block_zoned "${nvme##*/}"; then 1657 zoned_devs["${nvme##*/}"]=$(< "$nvme/device/address") 1658 fi 1659 done 1660} 1661 1662# Define temp storage for all the tests. Look for 2GB at minimum 1663set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1664 1665set -o errtrace 1666shopt -s extdebug 1667trap "trap - ERR; print_backtrace >&2" ERR 1668 1669PS4=' \t -- ${BASH_SOURCE#${BASH_SOURCE%/*/*}/}@${LINENO} -- \$ ' 1670if $SPDK_AUTOTEST_X; then 1671 # explicitly enable xtraces, overriding any tracking information. 1672 xtrace_fd 1673else 1674 xtrace_disable 1675fi 1676
