1#!/usr/bin/env bash 2 3function xtrace_disable() { 4 if [ "$XTRACE_DISABLED" != "yes" ]; then 5 PREV_BASH_OPTS="$-" 6 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 7 XTRACE_DISABLED="yes" 8 fi 9 set +x 10 elif [ -z $XTRACE_NESTING_LEVEL ]; then 11 XTRACE_NESTING_LEVEL=1 12 else 13 XTRACE_NESTING_LEVEL=$((++XTRACE_NESTING_LEVEL)) 14 fi 15} 16 17xtrace_disable 18set -e 19shopt -s expand_aliases 20 21source "$rootdir/test/common/applications.sh" 22if [[ -e $rootdir/test/common/build_config.sh ]]; then 23 source "$rootdir/test/common/build_config.sh" 24elif [[ -e $rootdir/mk/config.mk ]]; then 25 build_config=$(< "$rootdir/mk/config.mk") 26 source <(echo "${build_config//\?=/=}") 27else 28 source "$rootdir/CONFIG" 29fi 30 31# Dummy function to be called after restoring xtrace just so that it appears in the 32# xtrace log. This way we can consistently track when xtrace is enabled/disabled. 33function xtrace_enable() { 34 # We have to do something inside a function in bash, and calling any command 35 # (even `:`) will produce an xtrace entry, so we just define another function. 36 function xtrace_dummy() { :; } 37} 38 39# Keep it as alias to avoid xtrace_enable backtrace always pointing to xtrace_restore. 40# xtrace_enable will appear as called directly from the user script, from the same line 41# that "called" xtrace_restore. 42alias xtrace_restore='if [ -z $XTRACE_NESTING_LEVEL ]; then 43 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 44 XTRACE_DISABLED="no"; PREV_BASH_OPTS=""; set -x; xtrace_enable; 45 fi 46else 47 XTRACE_NESTING_LEVEL=$((--XTRACE_NESTING_LEVEL)); 48 if [ $XTRACE_NESTING_LEVEL -eq "0" ]; then 49 unset XTRACE_NESTING_LEVEL 50 fi 51fi' 52 53: ${RUN_NIGHTLY:=0} 54export RUN_NIGHTLY 55 56# Set defaults for missing test config options 57: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 58export SPDK_AUTOTEST_DEBUG_APPS 59: ${SPDK_RUN_VALGRIND=0} 60export SPDK_RUN_VALGRIND 61: ${SPDK_RUN_FUNCTIONAL_TEST=0} 62export SPDK_RUN_FUNCTIONAL_TEST 63: ${SPDK_TEST_UNITTEST=0} 64export SPDK_TEST_UNITTEST 65: ${SPDK_TEST_AUTOBUILD=0} 66export SPDK_TEST_AUTOBUILD 67: ${SPDK_TEST_ISAL=0} 68export SPDK_TEST_ISAL 69: ${SPDK_TEST_ISCSI=0} 70export SPDK_TEST_ISCSI 71: ${SPDK_TEST_ISCSI_INITIATOR=0} 72export SPDK_TEST_ISCSI_INITIATOR 73: ${SPDK_TEST_NVME=0} 74export SPDK_TEST_NVME 75: ${SPDK_TEST_NVME_CLI=0} 76export SPDK_TEST_NVME_CLI 77: ${SPDK_TEST_NVME_CUSE=0} 78export SPDK_TEST_NVME_CUSE 79: ${SPDK_TEST_NVMF=0} 80export SPDK_TEST_NVMF 81: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 82export SPDK_TEST_NVMF_TRANSPORT 83: ${SPDK_TEST_RBD=0} 84export SPDK_TEST_RBD 85: ${SPDK_TEST_VHOST=0} 86export SPDK_TEST_VHOST 87: ${SPDK_TEST_BLOCKDEV=0} 88export SPDK_TEST_BLOCKDEV 89: ${SPDK_TEST_IOAT=0} 90export SPDK_TEST_IOAT 91: ${SPDK_TEST_BLOBFS=0} 92export SPDK_TEST_BLOBFS 93: ${SPDK_TEST_VHOST_INIT=0} 94export SPDK_TEST_VHOST_INIT 95: ${SPDK_TEST_PMDK=0} 96export SPDK_TEST_PMDK 97: ${SPDK_TEST_LVOL=0} 98export SPDK_TEST_LVOL 99: ${SPDK_TEST_JSON=0} 100export SPDK_TEST_JSON 101: ${SPDK_TEST_REDUCE=0} 102export SPDK_TEST_REDUCE 103: ${SPDK_TEST_VPP=0} 104export SPDK_TEST_VPP 105: ${SPDK_RUN_ASAN=0} 106export SPDK_RUN_ASAN 107: ${SPDK_RUN_UBSAN=0} 108export SPDK_RUN_UBSAN 109: ${SPDK_RUN_INSTALLED_DPDK=0} 110export SPDK_RUN_INSTALLED_DPDK 111: ${SPDK_RUN_NON_ROOT=0} 112export SPDK_RUN_NON_ROOT 113: ${SPDK_TEST_CRYPTO=0} 114export SPDK_TEST_CRYPTO 115: ${SPDK_TEST_FTL=0} 116export SPDK_TEST_FTL 117: ${SPDK_TEST_OCF=0} 118export SPDK_TEST_OCF 119: ${SPDK_TEST_FTL_EXTENDED=0} 120export SPDK_TEST_FTL_EXTENDED 121: ${SPDK_TEST_VMD=0} 122export SPDK_TEST_VMD 123: ${SPDK_TEST_OPAL=0} 124export SPDK_TEST_OPAL 125: ${SPDK_AUTOTEST_X=true} 126export SPDK_AUTOTEST_X 127: ${SPDK_TEST_RAID5=0} 128export SPDK_TEST_RAID5 129: ${SPDK_TEST_URING=0} 130export SPDK_TEST_URING 131 132# Export PYTHONPATH with addition of RPC framework. New scripts can be created 133# specific use cases for tests. 134export PYTHONPATH=$PYTHONPATH:$rootdir/scripts 135 136# Don't create Python .pyc files. When running with sudo these will be 137# created with root ownership and can cause problems when cleaning the repository. 138export PYTHONDONTWRITEBYTECODE=1 139 140# Export flag to skip the known bug that exists in librados 141# Bug is reported on ceph bug tracker with number 24078 142export ASAN_OPTIONS=new_delete_type_mismatch=0 143export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1' 144 145# Export LeakSanitizer option to use suppression file in order to prevent false positives 146# and known leaks in external executables or libraries from showing up. 147asan_suppression_file="/var/tmp/asan_suppression_file" 148sudo rm -rf "$asan_suppression_file" 149cat << EOL >> "$asan_suppression_file" 150# ASAN has some bugs around thread_local variables. We have a destructor in place 151# to free the thread contexts, but ASAN complains about the leak before those 152# destructors have a chance to run. So suppress this one specific leak using 153# LSAN_OPTIONS. 154leak:spdk_fs_alloc_thread_ctx 155 156# Suppress known leaks in fio project 157leak:$CONFIG_FIO_SOURCE_DIR/parse.c 158leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 159leak:$CONFIG_FIO_SOURCE_DIR/init.c 160leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 161leak:fio_memalign 162leak:spdk_fio_io_u_init 163 164# Suppress leaks in libiscsi 165leak:libiscsi.so 166EOL 167 168# Suppress leaks in libfuse3 169echo "leak:libfuse3.so" >> "$asan_suppression_file" 170 171export LSAN_OPTIONS=suppressions="$asan_suppression_file" 172 173export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 174 175if [ -z "$DEPENDENCY_DIR" ]; then 176 export DEPENDENCY_DIR=/home/sys_sgsw 177else 178 export DEPENDENCY_DIR 179fi 180 181# Export location of where all the SPDK binaries are 182export SPDK_BIN_DIR="$rootdir/build/bin" 183export SPDK_EXAMPLE_DIR="$rootdir/build/examples" 184 185# pass our valgrind desire on to unittest.sh 186if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 187 export valgrind='' 188fi 189 190if [ "$(uname -s)" = "Linux" ]; then 191 MAKE="make" 192 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 193 DPDK_LINUX_DIR=/usr/share/dpdk/x86_64-default-linuxapp-gcc 194 if [ -d $DPDK_LINUX_DIR ] && [ $SPDK_RUN_INSTALLED_DPDK -eq 1 ]; then 195 WITH_DPDK_DIR=$DPDK_LINUX_DIR 196 fi 197 # Override the default HUGEMEM in scripts/setup.sh to allocate 8GB in hugepages. 198 export HUGEMEM=8192 199elif [ "$(uname -s)" = "FreeBSD" ]; then 200 MAKE="gmake" 201 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 202 DPDK_FREEBSD_DIR=/usr/local/share/dpdk/x86_64-native-bsdapp-clang 203 if [ -d $DPDK_FREEBSD_DIR ] && [ $SPDK_RUN_INSTALLED_DPDK -eq 1 ]; then 204 WITH_DPDK_DIR=$DPDK_FREEBSD_DIR 205 fi 206 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 207 export HUGEMEM=2048 208else 209 echo "Unknown OS \"$(uname -s)\"" 210 exit 1 211fi 212 213if [ -z "$output_dir" ]; then 214 if [ -z "$rootdir" ] || [ ! -d "$rootdir/../output" ]; then 215 output_dir=. 216 else 217 output_dir=$rootdir/../output 218 fi 219 export output_dir 220fi 221 222TEST_MODE= 223for i in "$@"; do 224 case "$i" in 225 --iso) 226 TEST_MODE=iso 227 ;; 228 --transport=*) 229 TEST_TRANSPORT="${i#*=}" 230 ;; 231 --sock=*) 232 TEST_SOCK="${i#*=}" 233 ;; 234 esac 235done 236 237# start rpc.py coprocess if it's not started yet 238if [[ -z $RPC_PIPE_PID ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 239 coproc RPC_PIPE { "$rootdir/scripts/rpc.py" --server; } 240 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 241 # all descriptors will automatically close together with this bash 242 # process, this will make rpc.py stop reading and exit gracefully 243fi 244 245if [ $SPDK_TEST_VPP -eq 1 ]; then 246 VPP_PATH="/usr/local/src/vpp-19.04/build-root/install-vpp_debug-native/vpp/" 247 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${VPP_PATH}/lib/ 248 export PATH=${PATH}:${VPP_PATH}/bin/ 249fi 250 251function set_test_storage() { 252 [[ -v testdir ]] || return 0 253 254 local requested_size=$1 # bytes 255 local mount target_dir 256 257 local -A mounts fss sizes avails uses 258 local source fs size avail mount use 259 260 local storage_fallback storage_candidates 261 local storage_fallback_purge 262 263 storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) 264 265 if ((${#storage_fallback_purge[@]} > 0)); then 266 printf '* Purging old temporary test storage (%s)\n' \ 267 "${storage_fallback_purge[*]}" >&2 268 rm -rf "${storage_fallback_purge[@]}" 269 fi 270 271 storage_fallback=$(mktemp -udt spdk.XXXXXX) 272 storage_candidates=( 273 "$testdir" 274 "$storage_fallback/tests/${testdir##*/}" 275 "$storage_fallback" 276 ) 277 278 if [[ -n $ADD_TEST_STORAGE ]]; then 279 # List of dirs|mounts separated by whitespaces 280 storage_candidates+=($ADD_TEST_STORAGE) 281 fi 282 283 if [[ -n $DEDICATED_TEST_STORAGE ]]; then 284 # Single, dedicated dir|mount 285 storage_candidates=("$DEDICATED_TEST_STORAGE") 286 fi 287 288 mkdir -p "${storage_candidates[@]}" 289 290 # add some headroom - 64M 291 requested_size=$((requested_size + (64 << 20))) 292 293 while read -r source fs size use avail _ mount; do 294 mounts["$mount"]=$source fss["$mount"]=$fs 295 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 296 uses["$mount"]=$((use * 1024)) 297 done < <(df -T | grep -v Filesystem) 298 299 printf '* Looking for test storage...\n' >&2 300 301 local target_space new_size 302 for target_dir in "${storage_candidates[@]}"; do 303 # FreeBSD's df is lacking the --output arg 304 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 305 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 306 307 target_space=${avails["$mount"]} 308 if ((target_space == 0 || target_space < requested_size)); then 309 continue 310 fi 311 if ((target_space >= requested_size)); then 312 # For in-memory fs, and / make sure our requested size won't fill most of the space. 313 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 314 new_size=$((uses["$mount"] + requested_size)) 315 if ((new_size * 100 / sizes["$mount"] > 95)); then 316 continue 317 fi 318 fi 319 fi 320 export SPDK_TEST_STORAGE=$target_dir 321 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 322 return 0 323 done 324 printf '* Test storage is not available\n' 325 return 1 326} 327 328function get_config_params() { 329 xtrace_disable 330 config_params='--enable-debug --enable-werror' 331 332 # for options with dependencies but no test flag, set them here 333 if [ -f /usr/include/infiniband/verbs.h ]; then 334 config_params+=' --with-rdma' 335 fi 336 337 if [ $(uname -s) == "FreeBSD" ]; then 338 intel="hw.model: Intel" 339 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 340 else 341 intel="GenuineIntel" 342 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 343 fi 344 if [[ "$cpu_vendor" != *"$intel"* ]]; then 345 config_params+=" --without-idxd" 346 else 347 config_params+=" --with-idxd" 348 fi 349 350 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 351 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 352 fi 353 354 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 355 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 356 fi 357 358 if [ -d /usr/include/iscsi ]; then 359 libiscsi_version=$(grep LIBISCSI_API_VERSION /usr/include/iscsi/iscsi.h | head -1 | awk '{print $3}' | awk -F '(' '{print $2}' | awk -F ')' '{print $1}') 360 if [ $libiscsi_version -ge 20150621 ]; then 361 config_params+=' --with-iscsi-initiator' 362 fi 363 fi 364 365 if [ $SPDK_TEST_UNITTEST -eq 0 ]; then 366 config_params+=' --disable-unit-tests' 367 fi 368 369 if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then 370 config_params+=' --with-nvme-cuse' 371 fi 372 373 # for options with both dependencies and a test flag, set them here 374 if [ -f /usr/include/libpmemblk.h ] && [ $SPDK_TEST_PMDK -eq 1 ]; then 375 config_params+=' --with-pmdk' 376 fi 377 378 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_REDUCE -eq 1 ]; then 379 if [ $SPDK_TEST_ISAL -eq 1 ]; then 380 config_params+=' --with-reduce' 381 fi 382 fi 383 384 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 385 config_params+=' --with-rbd' 386 fi 387 388 if [ $SPDK_TEST_VPP -eq 1 ]; then 389 config_params+=" --with-vpp=${VPP_PATH}" 390 fi 391 392 # for options with no required dependencies, just test flags, set them here 393 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 394 config_params+=' --with-crypto' 395 fi 396 397 if [ $SPDK_TEST_OCF -eq 1 ]; then 398 config_params+=" --with-ocf" 399 fi 400 401 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 402 config_params+=' --enable-ubsan' 403 fi 404 405 if [ $SPDK_RUN_ASAN -eq 1 ]; then 406 config_params+=' --enable-asan' 407 fi 408 409 if [ "$(uname -s)" = "Linux" ]; then 410 config_params+=' --enable-coverage' 411 fi 412 413 if [ $SPDK_TEST_ISAL -eq 0 ]; then 414 config_params+=' --without-isal' 415 fi 416 417 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 418 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 419 config_params+=' --with-fuse' 420 fi 421 fi 422 423 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 424 config_params+=' --with-raid5' 425 fi 426 427 # Check whether liburing library header exists 428 if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then 429 config_params+=' --with-uring' 430 fi 431 432 # By default, --with-dpdk is not set meaning the SPDK build will use the DPDK submodule. 433 # If a DPDK installation is found in a well-known location though, WITH_DPDK_DIR will be 434 # set which will override the default and use that DPDK installation instead. 435 if [ -n "$WITH_DPDK_DIR" ]; then 436 config_params+=" --with-dpdk=$WITH_DPDK_DIR" 437 fi 438 439 echo "$config_params" 440 xtrace_restore 441} 442 443function rpc_cmd() { 444 xtrace_disable 445 local rsp rc 446 447 echo "$@" >&$RPC_PIPE_INPUT 448 while read -t 5 -ru $RPC_PIPE_OUTPUT rsp; do 449 if [[ $rsp == "**STATUS="* ]]; then 450 break 451 fi 452 echo "$rsp" 453 done 454 455 rc=${rsp#*=} 456 xtrace_restore 457 [[ $rc == 0 ]] 458} 459 460function rpc_cmd_simple_data_json() { 461 462 local elems="$1[@]" elem 463 local -gA jq_out=() 464 local jq val 465 466 local lvs=( 467 "uuid" 468 "name" 469 "base_bdev" 470 "total_data_clusters" 471 "free_clusters" 472 "block_size" 473 "cluster_size" 474 ) 475 476 local bdev=( 477 "name" 478 "aliases[0]" 479 "block_size" 480 "num_blocks" 481 "uuid" 482 "product_name" 483 ) 484 485 [[ -v $elems ]] || return 1 486 487 for elem in "${!elems}"; do 488 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 489 done 490 jq+=',"\n"' 491 492 shift 493 while read -r elem val; do 494 jq_out["$elem"]=$val 495 done < <(rpc_cmd "$@" | jq -jr "$jq") 496 ((${#jq_out[@]} > 0)) || return 1 497} 498 499# invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 500function NOT() { 501 if "$@"; then 502 return 1 503 fi 504} 505 506function timing() { 507 direction="$1" 508 testname="$2" 509 510 now=$(date +%s) 511 512 if [ "$direction" = "enter" ]; then 513 export timing_stack="${timing_stack};${now}" 514 export test_stack="${test_stack};${testname}" 515 else 516 touch "$output_dir/timing.txt" 517 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 518 519 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 520 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 521 522 elapsed=$((now - start_time - child_time)) 523 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 524 525 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 526 fi 527} 528 529function timing_enter() { 530 xtrace_disable 531 timing "enter" "$1" 532 xtrace_restore 533} 534 535function timing_exit() { 536 xtrace_disable 537 timing "exit" "$1" 538 xtrace_restore 539} 540 541function timing_finish() { 542 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 543 if [ -x "$flamegraph" ]; then 544 "$flamegraph" \ 545 --title 'Build Timing' \ 546 --nametype 'Step:' \ 547 --countname seconds \ 548 $output_dir/timing.txt \ 549 > $output_dir/timing.svg 550 fi 551} 552 553function create_test_list() { 554 xtrace_disable 555 # First search all scripts in main SPDK directory. 556 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 557 # Follow up with search in test directory recursively. 558 completion+=$(grep -rshI --include="*.sh" --exclude="autotest_common.sh" -e "run_test " $rootdir/test) 559 printf "%s" "$completion" | grep -v "#" \ 560 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 561 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 562 xtrace_restore 563} 564 565function gdb_attach() { 566 gdb -q --batch \ 567 -ex 'handle SIGHUP nostop pass' \ 568 -ex 'handle SIGQUIT nostop pass' \ 569 -ex 'handle SIGPIPE nostop pass' \ 570 -ex 'handle SIGALRM nostop pass' \ 571 -ex 'handle SIGTERM nostop pass' \ 572 -ex 'handle SIGUSR1 nostop pass' \ 573 -ex 'handle SIGUSR2 nostop pass' \ 574 -ex 'handle SIGCHLD nostop pass' \ 575 -ex 'set print thread-events off' \ 576 -ex 'cont' \ 577 -ex 'thread apply all bt' \ 578 -ex 'quit' \ 579 --tty=/dev/stdout \ 580 -p $1 581} 582 583function process_core() { 584 ret=0 585 while IFS= read -r -d '' core; do 586 exe=$(eu-readelf -n "$core" | grep psargs | sed "s/.*psargs: \([^ \'\" ]*\).*/\1/") 587 if [[ ! -f "$exe" ]]; then 588 exe=$(eu-readelf -n "$core" | grep -oP -m1 "$exe.+") 589 fi 590 echo "exe for $core is $exe" 591 if [[ -n "$exe" ]]; then 592 if hash gdb &> /dev/null; then 593 gdb -batch -ex "thread apply all bt full" $exe $core 594 fi 595 cp $exe $output_dir 596 fi 597 mv $core $output_dir 598 chmod a+r $output_dir/$core 599 ret=1 600 done < <(find . -type f \( -name 'core\.?[0-9]*' -o -name '*.core' \) -print0) 601 return $ret 602} 603 604function process_shm() { 605 type=$1 606 id=$2 607 if [ "$type" = "--pid" ]; then 608 id="pid${id}" 609 elif [ "$type" = "--id" ]; then 610 id="${id}" 611 else 612 echo "Please specify to search for pid or shared memory id." 613 return 1 614 fi 615 616 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 617 618 if [[ -z $shm_files ]]; then 619 echo "SHM File for specified PID or shared memory id: ${id} not found!" 620 return 1 621 fi 622 for n in $shm_files; do 623 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 624 done 625 return 0 626} 627 628function waitforlisten() { 629 # $1 = process pid 630 if [ -z "$1" ]; then 631 exit 1 632 fi 633 634 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 635 636 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 637 # turn off trace for this loop 638 xtrace_disable 639 local ret=0 640 local i 641 for ((i = 40; i != 0; i--)); do 642 # if the process is no longer running, then exit the script 643 # since it means the application crashed 644 if ! kill -s 0 $1; then 645 echo "ERROR: process (pid: $1) is no longer running" 646 ret=1 647 break 648 fi 649 650 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 651 break 652 fi 653 654 sleep 0.5 655 done 656 657 xtrace_restore 658 if ((i == 0)); then 659 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 660 ret=1 661 fi 662 return $ret 663} 664 665function waitfornbd() { 666 local nbd_name=$1 667 local i 668 669 for ((i = 1; i <= 20; i++)); do 670 if grep -q -w $nbd_name /proc/partitions; then 671 break 672 else 673 sleep 0.1 674 fi 675 done 676 677 # The nbd device is now recognized as a block device, but there can be 678 # a small delay before we can start I/O to that block device. So loop 679 # here trying to read the first block of the nbd block device to a temp 680 # file. Note that dd returns success when reading an empty file, so we 681 # need to check the size of the output file instead. 682 for ((i = 1; i <= 20; i++)); do 683 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 684 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 685 rm -f "$SPDK_TEST_STORAGE/nbdtest" 686 if [ "$size" != "0" ]; then 687 return 0 688 else 689 sleep 0.1 690 fi 691 done 692 693 return 1 694} 695 696function waitforbdev() { 697 local bdev_name=$1 698 local i 699 700 for ((i = 1; i <= 20; i++)); do 701 if $rpc_py bdev_get_bdevs | jq -r '.[] .name' | grep -qw $bdev_name; then 702 return 0 703 fi 704 705 if $rpc_py bdev_get_bdevs | jq -r '.[] .aliases' | grep -qw $bdev_name; then 706 return 0 707 fi 708 709 sleep 0.1 710 done 711 712 return 1 713} 714 715function make_filesystem() { 716 local fstype=$1 717 local dev_name=$2 718 local i=0 719 local force 720 721 if [ $fstype = ext4 ]; then 722 force=-F 723 else 724 force=-f 725 fi 726 727 while ! mkfs.${fstype} $force ${dev_name}; do 728 if [ $i -ge 15 ]; then 729 return 1 730 fi 731 i=$((i + 1)) 732 sleep 1 733 done 734 735 return 0 736} 737 738function killprocess() { 739 # $1 = process pid 740 if [ -z "$1" ]; then 741 exit 1 742 fi 743 744 if kill -0 $1; then 745 if [ $(uname) = Linux ]; then 746 process_name=$(ps --no-headers -o comm= $1) 747 else 748 process_name=$(ps -c -o command $1 | tail -1) 749 fi 750 if [ "$process_name" = "sudo" ]; then 751 # kill the child process, which is the actual app 752 # (assume $1 has just one child) 753 local child 754 child="$(pgrep -P $1)" 755 echo "killing process with pid $child" 756 kill $child 757 else 758 echo "killing process with pid $1" 759 kill $1 760 fi 761 762 # wait for the process regardless if its the dummy sudo one 763 # or the actual app - it should terminate anyway 764 wait $1 765 else 766 # the process is not there anymore 767 echo "Process with pid $1 is not found" 768 exit 1 769 fi 770} 771 772function iscsicleanup() { 773 echo "Cleaning up iSCSI connection" 774 iscsiadm -m node --logout || true 775 iscsiadm -m node -o delete || true 776 rm -rf /var/lib/iscsi/nodes/* 777} 778 779function stop_iscsi_service() { 780 if cat /etc/*-release | grep Ubuntu; then 781 service open-iscsi stop 782 else 783 service iscsid stop 784 fi 785} 786 787function start_iscsi_service() { 788 if cat /etc/*-release | grep Ubuntu; then 789 service open-iscsi start 790 else 791 service iscsid start 792 fi 793} 794 795function rbd_setup() { 796 # $1 = monitor ip address 797 # $2 = name of the namespace 798 if [ -z "$1" ]; then 799 echo "No monitor IP address provided for ceph" 800 exit 1 801 fi 802 if [ -n "$2" ]; then 803 if ip netns list | grep "$2"; then 804 NS_CMD="ip netns exec $2" 805 else 806 echo "No namespace $2 exists" 807 exit 1 808 fi 809 fi 810 811 if hash ceph; then 812 export PG_NUM=128 813 export RBD_POOL=rbd 814 export RBD_NAME=foo 815 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 816 $NS_CMD $rootdir/scripts/ceph/start.sh $1 817 818 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 819 $NS_CMD rbd create $RBD_NAME --size 1000 820 fi 821} 822 823function rbd_cleanup() { 824 if hash ceph; then 825 $rootdir/scripts/ceph/stop.sh || true 826 rm -f /var/tmp/ceph_raw.img 827 fi 828} 829 830function nvme_cli_build() { 831 if [[ -z "${DEPENDENCY_DIR}" ]]; then 832 echo DEPENDENCY_DIR not defined! 833 exit 1 834 fi 835 836 spdk_nvme_cli="${DEPENDENCY_DIR}/nvme-cli" 837 838 if [[ ! -d $spdk_nvme_cli ]]; then 839 echo "nvme-cli repository not found at $spdk_nvme_cli; skipping tests." 840 exit 1 841 fi 842 843 if ! grep -q "DEF_VER=v1.6" $spdk_nvme_cli/NVME-VERSION-GEN; then 844 echo "SPDK supports only \"spdk/nvme-cli\" project on \"spdk-1.6\" branch." 845 exit 1 846 fi 847 848 # Build against the version of SPDK under test 849 pushd $spdk_nvme_cli 850 851 # Remove and recreate git index in case it became corrupted 852 if ! git clean -dfx; then 853 rm -f .git/index 854 git clean -dfx 855 git reset --hard 856 fi 857 858 rm -f "$spdk_nvme_cli/spdk" 859 ln -sf "$rootdir" "$spdk_nvme_cli/spdk" 860 861 make -j$(nproc) LDFLAGS="$(make -s -C $spdk_nvme_cli/spdk ldflags)" 862 popd 863} 864 865function _start_stub() { 866 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 867 # but ASLR can still be unreliable in some cases. 868 # We will reenable it again after multi-process testing is complete in kill_stub(). 869 # Save current setting so it can be restored upon calling kill_stub(). 870 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 871 echo 0 > /proc/sys/kernel/randomize_va_space 872 $rootdir/test/app/stub/stub $1 & 873 stubpid=$! 874 echo Waiting for stub to ready for secondary processes... 875 while ! [ -e /var/run/spdk_stub0 ]; do 876 # If stub dies while we wait, bail 877 [[ -e /proc/$stubpid ]] || return 1 878 sleep 1s 879 done 880 echo done. 881} 882 883function start_stub() { 884 if ! _start_stub "$@"; then 885 echo "stub failed" >&2 886 return 1 887 fi 888} 889 890function kill_stub() { 891 if [[ -e /proc/$stubpid ]]; then 892 kill $1 $stubpid 893 wait $stubpid 894 fi 2> /dev/null || : 895 rm -f /var/run/spdk_stub0 896 # Re-enable ASLR now that we are done with multi-process testing 897 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 898 # randomizing and is the default on all recent Linux kernels 899 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 900} 901 902function run_test() { 903 if [ $# -le 1 ]; then 904 echo "Not enough parameters" 905 echo "usage: run_test test_name test_script [script_params]" 906 exit 1 907 fi 908 909 xtrace_disable 910 local test_name="$1" 911 shift 912 913 if [ -n "$test_domain" ]; then 914 export test_domain="${test_domain}.${test_name}" 915 else 916 export test_domain="$test_name" 917 fi 918 919 timing_enter $test_name 920 echo "************************************" 921 echo "START TEST $test_name" 922 echo "************************************" 923 xtrace_restore 924 time "$@" 925 xtrace_disable 926 echo "************************************" 927 echo "END TEST $test_name" 928 echo "************************************" 929 timing_exit $test_name 930 931 export test_domain=${test_domain%"$test_name"} 932 if [ -n "$test_domain" ]; then 933 export test_domain=${test_domain%?} 934 fi 935 936 if [ -z "$test_domain" ]; then 937 echo "top_level $test_name" >> $output_dir/test_completions.txt 938 else 939 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 940 fi 941 xtrace_restore 942} 943 944function skip_run_test_with_warning() { 945 echo "WARNING: $1" 946 echo "Test run may fail if run with autorun.sh" 947 echo "Please check your $rootdir/test/common/skipped_tests.txt" 948} 949 950function print_backtrace() { 951 # if errexit is not enabled, don't print a backtrace 952 [[ "$-" =~ e ]] || return 0 953 954 local args=("${BASH_ARGV[@]}") 955 956 xtrace_disable 957 echo "========== Backtrace start: ==========" 958 echo "" 959 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do 960 local func="${FUNCNAME[$i]}" 961 local line_nr="${BASH_LINENO[$((i - 1))]}" 962 local src="${BASH_SOURCE[$i]}" 963 local bt="" cmdline=() 964 965 if [[ -f $src ]]; then 966 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 967 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 968 fi 969 970 # If extdebug set the BASH_ARGC[i], try to fetch all the args 971 if ((BASH_ARGC[i] > 0)); then 972 # Use argc as index to reverse the stack 973 local argc=${BASH_ARGC[i]} arg 974 for arg in "${args[@]::BASH_ARGC[i]}"; do 975 cmdline[argc--]="[\"$arg\"]" 976 done 977 args=("${args[@]:BASH_ARGC[i]}") 978 fi 979 980 echo "in $src:$line_nr -> $func($( 981 IFS="," 982 printf '%s\n' "${cmdline[*]:-[]}" 983 ))" 984 echo " ..." 985 echo "${bt:-backtrace unavailable}" 986 echo " ..." 987 done 988 echo "" 989 echo "========== Backtrace end ==========" 990 xtrace_restore 991 return 0 992} 993 994function waitforserial() { 995 local i=0 996 local nvme_device_counter=1 997 if [[ -n "$2" ]]; then 998 nvme_device_counter=$2 999 fi 1000 1001 while [ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]; do 1002 [ $i -lt 15 ] || break 1003 i=$((i + 1)) 1004 echo "Waiting for devices" 1005 sleep 1 1006 done 1007 1008 if [[ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]]; then 1009 return 1 1010 fi 1011 1012 return 0 1013} 1014 1015function waitforserial_disconnect() { 1016 local i=0 1017 while lsblk -o NAME,SERIAL | grep -q -w $1; do 1018 [ $i -lt 15 ] || break 1019 i=$((i + 1)) 1020 echo "Waiting for disconnect devices" 1021 sleep 1 1022 done 1023 1024 if lsblk -l -o NAME | grep -q -w $1; then 1025 return 1 1026 fi 1027 1028 return 0 1029} 1030 1031function waitforblk() { 1032 local i=0 1033 while ! lsblk -l -o NAME | grep -q -w $1; do 1034 [ $i -lt 15 ] || break 1035 i=$((i + 1)) 1036 sleep 1 1037 done 1038 1039 if ! lsblk -l -o NAME | grep -q -w $1; then 1040 return 1 1041 fi 1042 1043 return 0 1044} 1045 1046function waitforblk_disconnect() { 1047 local i=0 1048 while lsblk -l -o NAME | grep -q -w $1; do 1049 [ $i -lt 15 ] || break 1050 i=$((i + 1)) 1051 sleep 1 1052 done 1053 1054 if lsblk -l -o NAME | grep -q -w $1; then 1055 return 1 1056 fi 1057 1058 return 0 1059} 1060 1061function waitforfile() { 1062 local i=0 1063 while [ ! -e $1 ]; do 1064 [ $i -lt 200 ] || break 1065 i=$((i + 1)) 1066 sleep 0.1 1067 done 1068 1069 if [ ! -e $1 ]; then 1070 return 1 1071 fi 1072 1073 return 0 1074} 1075 1076function fio_config_gen() { 1077 local config_file=$1 1078 local workload=$2 1079 local bdev_type=$3 1080 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1081 1082 if [ -e "$config_file" ]; then 1083 echo "Configuration File Already Exists!: $config_file" 1084 return 1 1085 fi 1086 1087 if [ -z "$workload" ]; then 1088 workload=randrw 1089 fi 1090 1091 touch $1 1092 1093 cat > $1 << EOL 1094[global] 1095thread=1 1096group_reporting=1 1097direct=1 1098norandommap=1 1099percentile_list=50:99:99.9:99.99:99.999 1100time_based=1 1101ramp_time=0 1102EOL 1103 1104 if [ "$workload" == "verify" ]; then 1105 cat <<- EOL >> $config_file 1106 verify=sha1 1107 verify_backlog=1024 1108 rw=randwrite 1109 EOL 1110 1111 # To avoid potential data race issue due to the AIO device 1112 # flush mechanism, add the flag to serialize the writes. 1113 # This is to fix the intermittent IO failure issue of #935 1114 if [ "$bdev_type" == "AIO" ]; then 1115 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1116 echo "serialize_overlap=1" >> $config_file 1117 fi 1118 fi 1119 elif [ "$workload" == "trim" ]; then 1120 echo "rw=trimwrite" >> $config_file 1121 else 1122 echo "rw=$workload" >> $config_file 1123 fi 1124} 1125 1126function fio_bdev() { 1127 # Setup fio binary cmd line 1128 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1129 local bdev_plugin="$rootdir/build/fio/spdk_bdev" 1130 1131 # Preload AddressSanitizer library to fio if fio_plugin was compiled with it 1132 local asan_lib 1133 asan_lib=$(ldd $bdev_plugin | grep libasan | awk '{print $3}') 1134 1135 LD_PRELOAD="$asan_lib $bdev_plugin" "$fio_dir"/fio "$@" 1136} 1137 1138function fio_nvme() { 1139 # Setup fio binary cmd line 1140 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1141 local nvme_plugin="$rootdir/build/fio/spdk_nvme" 1142 1143 # Preload AddressSanitizer library to fio if fio_plugin was compiled with it 1144 asan_lib=$(ldd $nvme_plugin | grep libasan | awk '{print $3}') 1145 1146 LD_PRELOAD="$asan_lib $nvme_plugin" "$fio_dir"/fio "$@" 1147} 1148 1149function get_lvs_free_mb() { 1150 local lvs_uuid=$1 1151 local lvs_info 1152 local fc 1153 local cs 1154 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1155 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1156 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1157 1158 # Change to MB's 1159 free_mb=$((fc * cs / 1024 / 1024)) 1160 echo "$free_mb" 1161} 1162 1163function get_bdev_size() { 1164 local bdev_name=$1 1165 local bdev_info 1166 local bs 1167 local nb 1168 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1169 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1170 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1171 1172 # Change to MB's 1173 bdev_size=$((bs * nb / 1024 / 1024)) 1174 echo "$bdev_size" 1175} 1176 1177function autotest_cleanup() { 1178 $rootdir/scripts/setup.sh reset 1179 $rootdir/scripts/setup.sh cleanup 1180 if [ $(uname -s) = "Linux" ]; then 1181 if grep -q '#define SPDK_CONFIG_IGB_UIO_DRIVER 1' $rootdir/include/spdk/config.h; then 1182 [[ -e /sys/module/igb_uio ]] && rmmod igb_uio 1183 else 1184 modprobe -r uio_pci_generic 1185 fi 1186 fi 1187 rm -rf "$asan_suppression_file" 1188} 1189 1190function freebsd_update_contigmem_mod() { 1191 if [ $(uname) = FreeBSD ]; then 1192 kldunload contigmem.ko || true 1193 if [ -n "$WITH_DPDK_DIR" ]; then 1194 echo "Warning: SPDK only works on FreeBSD with patches that only exist in SPDK's dpdk submodule" 1195 cp -f "$WITH_DPDK_DIR/kmod/contigmem.ko" /boot/modules/ 1196 cp -f "$WITH_DPDK_DIR/kmod/contigmem.ko" /boot/kernel/ 1197 cp -f "$WITH_DPDK_DIR/kmod/nic_uio.ko" /boot/modules/ 1198 cp -f "$WITH_DPDK_DIR/kmod/nic_uio.ko" /boot/kernel/ 1199 else 1200 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1201 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1202 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1203 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1204 fi 1205 fi 1206} 1207 1208function get_nvme_name_from_bdf() { 1209 blkname=() 1210 1211 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1212 if [ -z "$nvme_devs" ]; then 1213 return 1214 fi 1215 for dev in $nvme_devs; do 1216 link_name=$(readlink /sys/block/$dev/device/device) || true 1217 if [ -z "$link_name" ]; then 1218 link_name=$(readlink /sys/block/$dev/device) 1219 fi 1220 bdf=$(basename "$link_name") 1221 if [ "$bdf" = "$1" ]; then 1222 blkname+=($dev) 1223 fi 1224 done 1225 1226 printf '%s\n' "${blkname[@]}" 1227} 1228 1229function get_nvme_ctrlr_from_bdf() { 1230 bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") 1231 if [[ -z "$bdf_sysfs_path" ]]; then 1232 return 1233 fi 1234 1235 printf '%s\n' "$(basename $bdf_sysfs_path)" 1236} 1237 1238# Get BDF addresses of all NVMe drives currently attached to 1239# uio-pci-generic or vfio-pci 1240function get_nvme_bdfs() { 1241 xtrace_disable 1242 bdfs=$(jq -r .config[].params.traddr <<< $($rootdir/scripts/gen_nvme.sh --json)) 1243 if [[ -z $bdfs ]]; then 1244 echo "No devices to test on!" 1245 exit 1 1246 fi 1247 echo "$bdfs" 1248 xtrace_restore 1249} 1250 1251# Same as function above, but just get the first disks BDF address 1252function get_first_nvme_bdf() { 1253 head -1 <<< "$(get_nvme_bdfs)" 1254} 1255 1256function nvme_namespace_revert() { 1257 $rootdir/scripts/setup.sh 1258 sleep 1 1259 bdfs=$(get_nvme_bdfs) 1260 1261 $rootdir/scripts/setup.sh reset 1262 sleep 1 1263 1264 for bdf in $bdfs; do 1265 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) 1266 if [[ -z "$nvme_ctrlr" ]]; then 1267 continue 1268 fi 1269 1270 # Check Optional Admin Command Support for Namespace Management 1271 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) 1272 oacs_ns_manage=$((oacs & 0x8)) 1273 1274 if [[ "$oacs_ns_manage" -ne 0 ]]; then 1275 # This assumes every NVMe controller contains single namespace, 1276 # encompassing Total NVM Capacity and formatted as 512 block size. 1277 # 512 block size is needed for test/vhost/vhost_boot.sh to 1278 # succesfully run. 1279 1280 unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) 1281 if [[ "$unvmcap" -eq 0 ]]; then 1282 # All available space already used 1283 continue 1284 fi 1285 tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) 1286 blksize=512 1287 1288 size=$((tnvmcap / blksize)) 1289 1290 nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c 0 || true 1291 nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true 1292 nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} 1293 nvme attach-ns ${nvme_ctrlr} -n 1 -c 0 1294 nvme reset ${nvme_ctrlr} 1295 waitforblk "${nvme_ctrlr}n1" 1296 fi 1297 done 1298} 1299 1300# Get BDFs based on device ID, such as 0x0a54 1301function get_nvme_bdfs_by_id() { 1302 local bdfs=() 1303 1304 for bdf in $(get_nvme_bdfs); do 1305 device=$(cat /sys/bus/pci/devices/$bdf/device) || true 1306 if [[ "$device" == "$1" ]]; then 1307 bdfs+=($bdf) 1308 fi 1309 done 1310 1311 printf '%s\n' "${bdfs[@]}" 1312} 1313 1314function opal_revert_cleanup() { 1315 # The OPAL CI tests is only used for P4510 devices. 1316 mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) 1317 if [[ -z ${bdfs[0]} ]]; then 1318 return 0 1319 fi 1320 1321 $SPDK_BIN_DIR/spdk_tgt & 1322 spdk_tgt_pid=$! 1323 waitforlisten $spdk_tgt_pid 1324 1325 for bdf in "${bdfs[@]}"; do 1326 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme0" -t "pcie" -a ${bdf} 1327 # Ignore if this fails. 1328 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b nvme0 -p test || true 1329 done 1330 1331 killprocess $spdk_tgt_pid 1332} 1333 1334# Define temp storage for all the tests. Look for 2GB at minimum 1335set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1336 1337set -o errtrace 1338shopt -s extdebug 1339trap "trap - ERR; print_backtrace >&2" ERR 1340 1341PS4=' \t \$ ' 1342if $SPDK_AUTOTEST_X; then 1343 # explicitly enable xtraces, overriding any tracking information. 1344 unset XTRACE_DISABLED 1345 unset XTRACE_NESTING_LEVEL 1346 set -x 1347 xtrace_enable 1348else 1349 xtrace_restore 1350fi 1351
