1#!/usr/bin/env bash 2 3function xtrace_disable() { 4 if [ "$XTRACE_DISABLED" != "yes" ]; then 5 PREV_BASH_OPTS="$-" 6 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 7 XTRACE_DISABLED="yes" 8 fi 9 set +x 10 elif [ -z $XTRACE_NESTING_LEVEL ]; then 11 XTRACE_NESTING_LEVEL=1 12 else 13 XTRACE_NESTING_LEVEL=$((++XTRACE_NESTING_LEVEL)) 14 fi 15} 16 17xtrace_disable 18set -e 19shopt -s expand_aliases 20 21source "$rootdir/test/common/applications.sh" 22if [[ -e $rootdir/test/common/build_config.sh ]]; then 23 source "$rootdir/test/common/build_config.sh" 24elif [[ -e $rootdir/mk/config.mk ]]; then 25 build_config=$(< "$rootdir/mk/config.mk") 26 source <(echo "${build_config//\?=/=}") 27else 28 source "$rootdir/CONFIG" 29fi 30 31# Dummy function to be called after restoring xtrace just so that it appears in the 32# xtrace log. This way we can consistently track when xtrace is enabled/disabled. 33function xtrace_enable() { 34 # We have to do something inside a function in bash, and calling any command 35 # (even `:`) will produce an xtrace entry, so we just define another function. 36 function xtrace_dummy() { :; } 37} 38 39# Keep it as alias to avoid xtrace_enable backtrace always pointing to xtrace_restore. 40# xtrace_enable will appear as called directly from the user script, from the same line 41# that "called" xtrace_restore. 42alias xtrace_restore='if [ -z $XTRACE_NESTING_LEVEL ]; then 43 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then 44 XTRACE_DISABLED="no"; PREV_BASH_OPTS=""; set -x; xtrace_enable; 45 fi 46else 47 XTRACE_NESTING_LEVEL=$((--XTRACE_NESTING_LEVEL)); 48 if [ $XTRACE_NESTING_LEVEL -eq "0" ]; then 49 unset XTRACE_NESTING_LEVEL 50 fi 51fi' 52 53: ${RUN_NIGHTLY:=0} 54export RUN_NIGHTLY 55 56# Set defaults for missing test config options 57: ${SPDK_AUTOTEST_DEBUG_APPS:=0} 58export SPDK_AUTOTEST_DEBUG_APPS 59: ${SPDK_RUN_VALGRIND=0} 60export SPDK_RUN_VALGRIND 61: ${SPDK_RUN_FUNCTIONAL_TEST=0} 62export SPDK_RUN_FUNCTIONAL_TEST 63: ${SPDK_TEST_UNITTEST=0} 64export SPDK_TEST_UNITTEST 65: ${SPDK_TEST_AUTOBUILD=0} 66export SPDK_TEST_AUTOBUILD 67: ${SPDK_TEST_ISAL=0} 68export SPDK_TEST_ISAL 69: ${SPDK_TEST_ISCSI=0} 70export SPDK_TEST_ISCSI 71: ${SPDK_TEST_ISCSI_INITIATOR=0} 72export SPDK_TEST_ISCSI_INITIATOR 73: ${SPDK_TEST_NVME=0} 74export SPDK_TEST_NVME 75: ${SPDK_TEST_NVME_CLI=0} 76export SPDK_TEST_NVME_CLI 77: ${SPDK_TEST_NVME_CUSE=0} 78export SPDK_TEST_NVME_CUSE 79: ${SPDK_TEST_NVMF=0} 80export SPDK_TEST_NVMF 81: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} 82export SPDK_TEST_NVMF_TRANSPORT 83: ${SPDK_TEST_RBD=0} 84export SPDK_TEST_RBD 85: ${SPDK_TEST_VHOST=0} 86export SPDK_TEST_VHOST 87: ${SPDK_TEST_BLOCKDEV=0} 88export SPDK_TEST_BLOCKDEV 89: ${SPDK_TEST_IOAT=0} 90export SPDK_TEST_IOAT 91: ${SPDK_TEST_BLOBFS=0} 92export SPDK_TEST_BLOBFS 93: ${SPDK_TEST_VHOST_INIT=0} 94export SPDK_TEST_VHOST_INIT 95: ${SPDK_TEST_PMDK=0} 96export SPDK_TEST_PMDK 97: ${SPDK_TEST_LVOL=0} 98export SPDK_TEST_LVOL 99: ${SPDK_TEST_JSON=0} 100export SPDK_TEST_JSON 101: ${SPDK_TEST_REDUCE=0} 102export SPDK_TEST_REDUCE 103: ${SPDK_TEST_VPP=0} 104export SPDK_TEST_VPP 105: ${SPDK_RUN_ASAN=0} 106export SPDK_RUN_ASAN 107: ${SPDK_RUN_UBSAN=0} 108export SPDK_RUN_UBSAN 109: ${SPDK_RUN_INSTALLED_DPDK=0} 110export SPDK_RUN_INSTALLED_DPDK 111: ${SPDK_RUN_NON_ROOT=0} 112export SPDK_RUN_NON_ROOT 113: ${SPDK_TEST_CRYPTO=0} 114export SPDK_TEST_CRYPTO 115: ${SPDK_TEST_FTL=0} 116export SPDK_TEST_FTL 117: ${SPDK_TEST_OCF=0} 118export SPDK_TEST_OCF 119: ${SPDK_TEST_FTL_EXTENDED=0} 120export SPDK_TEST_FTL_EXTENDED 121: ${SPDK_TEST_VMD=0} 122export SPDK_TEST_VMD 123: ${SPDK_TEST_OPAL=0} 124export SPDK_TEST_OPAL 125: ${SPDK_AUTOTEST_X=true} 126export SPDK_AUTOTEST_X 127: ${SPDK_TEST_RAID5=0} 128export SPDK_TEST_RAID5 129 130# Export PYTHONPATH with addition of RPC framework. New scripts can be created 131# specific use cases for tests. 132export PYTHONPATH=$PYTHONPATH:$rootdir/scripts 133 134# Don't create Python .pyc files. When running with sudo these will be 135# created with root ownership and can cause problems when cleaning the repository. 136export PYTHONDONTWRITEBYTECODE=1 137 138# Export flag to skip the known bug that exists in librados 139# Bug is reported on ceph bug tracker with number 24078 140export ASAN_OPTIONS=new_delete_type_mismatch=0 141export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1' 142 143# Export LeakSanitizer option to use suppression file in order to prevent false positives 144# and known leaks in external executables or libraries from showing up. 145asan_suppression_file="/var/tmp/asan_suppression_file" 146sudo rm -rf "$asan_suppression_file" 147cat << EOL >> "$asan_suppression_file" 148# ASAN has some bugs around thread_local variables. We have a destructor in place 149# to free the thread contexts, but ASAN complains about the leak before those 150# destructors have a chance to run. So suppress this one specific leak using 151# LSAN_OPTIONS. 152leak:spdk_fs_alloc_thread_ctx 153 154# Suppress known leaks in fio project 155leak:$CONFIG_FIO_SOURCE_DIR/parse.c 156leak:$CONFIG_FIO_SOURCE_DIR/iolog.c 157leak:$CONFIG_FIO_SOURCE_DIR/init.c 158leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c 159leak:fio_memalign 160leak:spdk_fio_io_u_init 161 162# Suppress leaks in libiscsi 163leak:libiscsi.so 164EOL 165 166# Suppress leaks in libfuse3 167echo "leak:libfuse3.so" >> "$asan_suppression_file" 168 169export LSAN_OPTIONS=suppressions="$asan_suppression_file" 170 171export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" 172 173if [ -z "$DEPENDENCY_DIR" ]; then 174 export DEPENDENCY_DIR=/home/sys_sgsw 175else 176 export DEPENDENCY_DIR 177fi 178 179# pass our valgrind desire on to unittest.sh 180if [ $SPDK_RUN_VALGRIND -eq 0 ]; then 181 export valgrind='' 182fi 183 184if [ "$(uname -s)" = "Linux" ]; then 185 MAKE="make" 186 MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} 187 DPDK_LINUX_DIR=/usr/share/dpdk/x86_64-default-linuxapp-gcc 188 if [ -d $DPDK_LINUX_DIR ] && [ $SPDK_RUN_INSTALLED_DPDK -eq 1 ]; then 189 WITH_DPDK_DIR=$DPDK_LINUX_DIR 190 fi 191 # Override the default HUGEMEM in scripts/setup.sh to allocate 8GB in hugepages. 192 export HUGEMEM=8192 193elif [ "$(uname -s)" = "FreeBSD" ]; then 194 MAKE="gmake" 195 MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} 196 DPDK_FREEBSD_DIR=/usr/local/share/dpdk/x86_64-native-bsdapp-clang 197 if [ -d $DPDK_FREEBSD_DIR ] && [ $SPDK_RUN_INSTALLED_DPDK -eq 1 ]; then 198 WITH_DPDK_DIR=$DPDK_FREEBSD_DIR 199 fi 200 # FreeBSD runs a much more limited set of tests, so keep the default 2GB. 201 export HUGEMEM=2048 202else 203 echo "Unknown OS \"$(uname -s)\"" 204 exit 1 205fi 206 207if [ -z "$output_dir" ]; then 208 if [ -z "$rootdir" ] || [ ! -d "$rootdir/../output" ]; then 209 output_dir=. 210 else 211 output_dir=$rootdir/../output 212 fi 213 export output_dir 214fi 215 216TEST_MODE= 217for i in "$@"; do 218 case "$i" in 219 --iso) 220 TEST_MODE=iso 221 ;; 222 --transport=*) 223 TEST_TRANSPORT="${i#*=}" 224 ;; 225 --sock=*) 226 TEST_SOCK="${i#*=}" 227 ;; 228 esac 229done 230 231# start rpc.py coprocess if it's not started yet 232if [[ -z $RPC_PIPE_PID ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then 233 coproc RPC_PIPE { "$rootdir/scripts/rpc.py" --server; } 234 exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} 235 # all descriptors will automatically close together with this bash 236 # process, this will make rpc.py stop reading and exit gracefully 237fi 238 239if [ $SPDK_TEST_VPP -eq 1 ]; then 240 VPP_PATH="/usr/local/src/vpp-19.04/build-root/install-vpp_debug-native/vpp/" 241 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${VPP_PATH}/lib/ 242 export PATH=${PATH}:${VPP_PATH}/bin/ 243fi 244 245function set_test_storage() { 246 [[ -v testdir ]] || return 0 247 248 local requested_size=$1 # bytes 249 local mount target_dir 250 251 local -A mounts fss sizes avails uses 252 local source fs size avail mount use 253 254 local storage_fallback storage_candidates 255 256 storage_fallback=/tmp/spdk 257 storage_candidates=( 258 "$testdir" 259 "$storage_fallback/tests/${testdir##*/}" 260 "$storage_fallback" 261 ) 262 263 if [[ -n $ADD_TEST_STORAGE ]]; then 264 # List of dirs|mounts separated by whitespaces 265 storage_candidates+=($ADD_TEST_STORAGE) 266 fi 267 268 if [[ -n $DEDICATED_TEST_STORAGE ]]; then 269 # Single, dedicated dir|mount 270 storage_candidates=("$DEDICATED_TEST_STORAGE") 271 fi 272 273 mkdir -p "${storage_candidates[@]}" 274 275 # add some headroom - 64M 276 requested_size=$((requested_size + (64 << 20))) 277 278 while read -r source fs size use avail _ mount; do 279 mounts["$mount"]=$source fss["$mount"]=$fs 280 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) 281 uses["$mount"]=$((use * 1024)) 282 done < <(df -T | grep -v Filesystem) 283 284 printf '* Looking for test storage...\n' >&2 285 286 local target_space new_size 287 for target_dir in "${storage_candidates[@]}"; do 288 # FreeBSD's df is lacking the --output arg 289 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") 290 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') 291 292 target_space=${avails["$mount"]} 293 if ((target_space == 0 || target_space < requested_size)); then 294 continue 295 fi 296 if ((target_space >= requested_size)); then 297 # For in-memory fs, and / make sure our requested size won't fill most of the space. 298 if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then 299 new_size=$((uses["$mount"] + requested_size)) 300 if ((new_size * 100 / sizes["$mount"] > 95)); then 301 continue 302 fi 303 fi 304 fi 305 export SPDK_TEST_STORAGE=$target_dir 306 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 307 return 0 308 done 309 printf '* Test storage is not available\n' 310 return 1 311} 312 313function get_config_params() { 314 xtrace_disable 315 config_params='--enable-debug --enable-werror' 316 317 # for options with dependencies but no test flag, set them here 318 if [ -f /usr/include/infiniband/verbs.h ]; then 319 config_params+=' --with-rdma' 320 fi 321 322 if [ $(uname -s) == "FreeBSD" ]; then 323 intel="hw.model: Intel" 324 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) 325 else 326 intel="GenuineIntel" 327 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) 328 fi 329 if [[ "$cpu_vendor" != *"$intel"* ]]; then 330 config_params+=" --without-idxd" 331 else 332 config_params+=" --with-idxd" 333 fi 334 335 if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then 336 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" 337 fi 338 339 if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then 340 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' 341 fi 342 343 if [ -d /usr/include/iscsi ]; then 344 libiscsi_version=$(grep LIBISCSI_API_VERSION /usr/include/iscsi/iscsi.h | head -1 | awk '{print $3}' | awk -F '(' '{print $2}' | awk -F ')' '{print $1}') 345 if [ $libiscsi_version -ge 20150621 ]; then 346 config_params+=' --with-iscsi-initiator' 347 fi 348 fi 349 350 if [ $SPDK_TEST_UNITTEST -eq 0 ]; then 351 config_params+=' --disable-unit-tests' 352 fi 353 354 if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then 355 config_params+=' --with-nvme-cuse' 356 fi 357 358 # for options with both dependencies and a test flag, set them here 359 if [ -f /usr/include/libpmemblk.h ] && [ $SPDK_TEST_PMDK -eq 1 ]; then 360 config_params+=' --with-pmdk' 361 fi 362 363 if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_REDUCE -eq 1 ]; then 364 if [ $SPDK_TEST_ISAL -eq 1 ]; then 365 config_params+=' --with-reduce' 366 fi 367 fi 368 369 if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then 370 config_params+=' --with-rbd' 371 fi 372 373 if [ $SPDK_TEST_VPP -eq 1 ]; then 374 config_params+=" --with-vpp=${VPP_PATH}" 375 fi 376 377 # for options with no required dependencies, just test flags, set them here 378 if [ $SPDK_TEST_CRYPTO -eq 1 ]; then 379 config_params+=' --with-crypto' 380 fi 381 382 if [ $SPDK_TEST_OCF -eq 1 ]; then 383 config_params+=" --with-ocf" 384 fi 385 386 if [ $SPDK_RUN_UBSAN -eq 1 ]; then 387 config_params+=' --enable-ubsan' 388 fi 389 390 if [ $SPDK_RUN_ASAN -eq 1 ]; then 391 config_params+=' --enable-asan' 392 fi 393 394 if [ "$(uname -s)" = "Linux" ]; then 395 config_params+=' --enable-coverage' 396 fi 397 398 if [ $SPDK_TEST_ISAL -eq 0 ]; then 399 config_params+=' --without-isal' 400 fi 401 402 if [ $SPDK_TEST_BLOBFS -eq 1 ]; then 403 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then 404 config_params+=' --with-fuse' 405 fi 406 fi 407 408 if [ $SPDK_TEST_RAID5 -eq 1 ]; then 409 config_params+=' --with-raid5' 410 fi 411 412 # By default, --with-dpdk is not set meaning the SPDK build will use the DPDK submodule. 413 # If a DPDK installation is found in a well-known location though, WITH_DPDK_DIR will be 414 # set which will override the default and use that DPDK installation instead. 415 if [ -n "$WITH_DPDK_DIR" ]; then 416 config_params+=" --with-dpdk=$WITH_DPDK_DIR" 417 fi 418 419 echo "$config_params" 420 xtrace_restore 421} 422 423function rpc_cmd() { 424 xtrace_disable 425 local rsp rc 426 427 echo "$@" >&$RPC_PIPE_INPUT 428 while read -t 5 -ru $RPC_PIPE_OUTPUT rsp; do 429 if [[ $rsp == "**STATUS="* ]]; then 430 break 431 fi 432 echo "$rsp" 433 done 434 435 rc=${rsp#*=} 436 xtrace_restore 437 [[ $rc == 0 ]] 438} 439 440function rpc_cmd_simple_data_json() { 441 442 local elems="$1[@]" elem 443 local -gA jq_out=() 444 local jq val 445 446 local lvs=( 447 "uuid" 448 "name" 449 "base_bdev" 450 "total_data_clusters" 451 "free_clusters" 452 "block_size" 453 "cluster_size" 454 ) 455 456 local bdev=( 457 "name" 458 "aliases[0]" 459 "block_size" 460 "num_blocks" 461 "uuid" 462 "product_name" 463 ) 464 465 [[ -v $elems ]] || return 1 466 467 for elem in "${!elems}"; do 468 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" 469 done 470 jq+=',"\n"' 471 472 shift 473 while read -r elem val; do 474 jq_out["$elem"]=$val 475 done < <(rpc_cmd "$@" | jq -jr "$jq") 476 ((${#jq_out[@]} > 0)) || return 1 477} 478 479# invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) 480function NOT() { 481 if "$@"; then 482 return 1 483 fi 484} 485 486function timing() { 487 direction="$1" 488 testname="$2" 489 490 now=$(date +%s) 491 492 if [ "$direction" = "enter" ]; then 493 export timing_stack="${timing_stack};${now}" 494 export test_stack="${test_stack};${testname}" 495 else 496 touch "$output_dir/timing.txt" 497 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') 498 499 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') 500 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') 501 502 elapsed=$((now - start_time - child_time)) 503 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt 504 505 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') 506 fi 507} 508 509function timing_enter() { 510 xtrace_disable 511 timing "enter" "$1" 512 xtrace_restore 513} 514 515function timing_exit() { 516 xtrace_disable 517 timing "exit" "$1" 518 xtrace_restore 519} 520 521function timing_finish() { 522 flamegraph='/usr/local/FlameGraph/flamegraph.pl' 523 if [ -x "$flamegraph" ]; then 524 "$flamegraph" \ 525 --title 'Build Timing' \ 526 --nametype 'Step:' \ 527 --countname seconds \ 528 $output_dir/timing.txt \ 529 > $output_dir/timing.svg 530 fi 531} 532 533function create_test_list() { 534 xtrace_disable 535 # First search all scripts in main SPDK directory. 536 completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) 537 # Follow up with search in test directory recursively. 538 completion+=$(grep -rshI --include="*.sh" --exclude="autotest_common.sh" -e "run_test " $rootdir/test) 539 printf "%s" "$completion" | grep -v "#" \ 540 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ 541 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true 542 xtrace_restore 543} 544 545function gdb_attach() { 546 gdb -q --batch \ 547 -ex 'handle SIGHUP nostop pass' \ 548 -ex 'handle SIGQUIT nostop pass' \ 549 -ex 'handle SIGPIPE nostop pass' \ 550 -ex 'handle SIGALRM nostop pass' \ 551 -ex 'handle SIGTERM nostop pass' \ 552 -ex 'handle SIGUSR1 nostop pass' \ 553 -ex 'handle SIGUSR2 nostop pass' \ 554 -ex 'handle SIGCHLD nostop pass' \ 555 -ex 'set print thread-events off' \ 556 -ex 'cont' \ 557 -ex 'thread apply all bt' \ 558 -ex 'quit' \ 559 --tty=/dev/stdout \ 560 -p $1 561} 562 563function process_core() { 564 ret=0 565 while IFS= read -r -d '' core; do 566 exe=$(eu-readelf -n "$core" | grep psargs | sed "s/.*psargs: \([^ \'\" ]*\).*/\1/") 567 if [[ ! -f "$exe" ]]; then 568 exe=$(eu-readelf -n "$core" | grep -oP -m1 "$exe.+") 569 fi 570 echo "exe for $core is $exe" 571 if [[ -n "$exe" ]]; then 572 if hash gdb &> /dev/null; then 573 gdb -batch -ex "thread apply all bt full" $exe $core 574 fi 575 cp $exe $output_dir 576 fi 577 mv $core $output_dir 578 chmod a+r $output_dir/$core 579 ret=1 580 done < <(find . -type f \( -name 'core\.?[0-9]*' -o -name '*.core' \) -print0) 581 return $ret 582} 583 584function process_shm() { 585 type=$1 586 id=$2 587 if [ "$type" = "--pid" ]; then 588 id="pid${id}" 589 elif [ "$type" = "--id" ]; then 590 id="${id}" 591 else 592 echo "Please specify to search for pid or shared memory id." 593 return 1 594 fi 595 596 shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") 597 598 if [[ -z $shm_files ]]; then 599 echo "SHM File for specified PID or shared memory id: ${id} not found!" 600 return 1 601 fi 602 for n in $shm_files; do 603 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} 604 done 605 return 0 606} 607 608function waitforlisten() { 609 # $1 = process pid 610 if [ -z "$1" ]; then 611 exit 1 612 fi 613 614 local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" 615 616 echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." 617 # turn off trace for this loop 618 xtrace_disable 619 local ret=0 620 local i 621 for ((i = 40; i != 0; i--)); do 622 # if the process is no longer running, then exit the script 623 # since it means the application crashed 624 if ! kill -s 0 $1; then 625 echo "ERROR: process (pid: $1) is no longer running" 626 ret=1 627 break 628 fi 629 630 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then 631 break 632 fi 633 634 sleep 0.5 635 done 636 637 xtrace_restore 638 if ((i == 0)); then 639 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" 640 ret=1 641 fi 642 return $ret 643} 644 645function waitfornbd() { 646 local nbd_name=$1 647 local i 648 649 for ((i = 1; i <= 20; i++)); do 650 if grep -q -w $nbd_name /proc/partitions; then 651 break 652 else 653 sleep 0.1 654 fi 655 done 656 657 # The nbd device is now recognized as a block device, but there can be 658 # a small delay before we can start I/O to that block device. So loop 659 # here trying to read the first block of the nbd block device to a temp 660 # file. Note that dd returns success when reading an empty file, so we 661 # need to check the size of the output file instead. 662 for ((i = 1; i <= 20; i++)); do 663 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct 664 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") 665 rm -f "$SPDK_TEST_STORAGE/nbdtest" 666 if [ "$size" != "0" ]; then 667 return 0 668 else 669 sleep 0.1 670 fi 671 done 672 673 return 1 674} 675 676function waitforbdev() { 677 local bdev_name=$1 678 local i 679 680 for ((i = 1; i <= 20; i++)); do 681 if $rpc_py bdev_get_bdevs | jq -r '.[] .name' | grep -qw $bdev_name; then 682 return 0 683 fi 684 685 if $rpc_py bdev_get_bdevs | jq -r '.[] .aliases' | grep -qw $bdev_name; then 686 return 0 687 fi 688 689 sleep 0.1 690 done 691 692 return 1 693} 694 695function make_filesystem() { 696 local fstype=$1 697 local dev_name=$2 698 local i=0 699 local force 700 701 if [ $fstype = ext4 ]; then 702 force=-F 703 else 704 force=-f 705 fi 706 707 while ! mkfs.${fstype} $force ${dev_name}; do 708 if [ $i -ge 15 ]; then 709 return 1 710 fi 711 i=$((i + 1)) 712 sleep 1 713 done 714 715 return 0 716} 717 718function killprocess() { 719 # $1 = process pid 720 if [ -z "$1" ]; then 721 exit 1 722 fi 723 724 if kill -0 $1; then 725 if [ $(uname) = Linux ]; then 726 process_name=$(ps --no-headers -o comm= $1) 727 else 728 process_name=$(ps -c -o command $1 | tail -1) 729 fi 730 if [ "$process_name" = "sudo" ]; then 731 # kill the child process, which is the actual app 732 # (assume $1 has just one child) 733 local child 734 child="$(pgrep -P $1)" 735 echo "killing process with pid $child" 736 kill $child 737 else 738 echo "killing process with pid $1" 739 kill $1 740 fi 741 742 # wait for the process regardless if its the dummy sudo one 743 # or the actual app - it should terminate anyway 744 wait $1 745 else 746 # the process is not there anymore 747 echo "Process with pid $1 is not found" 748 exit 1 749 fi 750} 751 752function iscsicleanup() { 753 echo "Cleaning up iSCSI connection" 754 iscsiadm -m node --logout || true 755 iscsiadm -m node -o delete || true 756 rm -rf /var/lib/iscsi/nodes/* 757} 758 759function stop_iscsi_service() { 760 if cat /etc/*-release | grep Ubuntu; then 761 service open-iscsi stop 762 else 763 service iscsid stop 764 fi 765} 766 767function start_iscsi_service() { 768 if cat /etc/*-release | grep Ubuntu; then 769 service open-iscsi start 770 else 771 service iscsid start 772 fi 773} 774 775function rbd_setup() { 776 # $1 = monitor ip address 777 # $2 = name of the namespace 778 if [ -z "$1" ]; then 779 echo "No monitor IP address provided for ceph" 780 exit 1 781 fi 782 if [ -n "$2" ]; then 783 if ip netns list | grep "$2"; then 784 NS_CMD="ip netns exec $2" 785 else 786 echo "No namespace $2 exists" 787 exit 1 788 fi 789 fi 790 791 if hash ceph; then 792 export PG_NUM=128 793 export RBD_POOL=rbd 794 export RBD_NAME=foo 795 $NS_CMD $rootdir/scripts/ceph/stop.sh || true 796 $NS_CMD $rootdir/scripts/ceph/start.sh $1 797 798 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true 799 $NS_CMD rbd create $RBD_NAME --size 1000 800 fi 801} 802 803function rbd_cleanup() { 804 if hash ceph; then 805 $rootdir/scripts/ceph/stop.sh || true 806 rm -f /var/tmp/ceph_raw.img 807 fi 808} 809 810function _start_stub() { 811 # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, 812 # but ASLR can still be unreliable in some cases. 813 # We will reenable it again after multi-process testing is complete in kill_stub(). 814 # Save current setting so it can be restored upon calling kill_stub(). 815 _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) 816 echo 0 > /proc/sys/kernel/randomize_va_space 817 $rootdir/test/app/stub/stub $1 & 818 stubpid=$! 819 echo Waiting for stub to ready for secondary processes... 820 while ! [ -e /var/run/spdk_stub0 ]; do 821 # If stub dies while we wait, bail 822 [[ -e /proc/$stubpid ]] || return 1 823 sleep 1s 824 done 825 echo done. 826} 827 828function start_stub() { 829 if ! _start_stub "$@"; then 830 echo "stub failed" >&2 831 return 1 832 fi 833} 834 835function kill_stub() { 836 if [[ -e /proc/$stubpid ]]; then 837 kill $1 $stubpid 838 wait $stubpid 839 fi 2> /dev/null || : 840 rm -f /var/run/spdk_stub0 841 # Re-enable ASLR now that we are done with multi-process testing 842 # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment 843 # randomizing and is the default on all recent Linux kernels 844 echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space 845} 846 847function run_test() { 848 if [ $# -le 1 ]; then 849 echo "Not enough parameters" 850 echo "usage: run_test test_name test_script [script_params]" 851 exit 1 852 fi 853 854 xtrace_disable 855 local test_name="$1" 856 shift 857 858 if [ -n "$test_domain" ]; then 859 export test_domain="${test_domain}.${test_name}" 860 else 861 export test_domain="$test_name" 862 fi 863 864 timing_enter $test_name 865 echo "************************************" 866 echo "START TEST $test_name" 867 echo "************************************" 868 xtrace_restore 869 time "$@" 870 xtrace_disable 871 echo "************************************" 872 echo "END TEST $test_name" 873 echo "************************************" 874 timing_exit $test_name 875 876 export test_domain=${test_domain%"$test_name"} 877 if [ -n "$test_domain" ]; then 878 export test_domain=${test_domain%?} 879 fi 880 881 if [ -z "$test_domain" ]; then 882 echo "top_level $test_name" >> $output_dir/test_completions.txt 883 else 884 echo "$test_domain $test_name" >> $output_dir/test_completions.txt 885 fi 886 xtrace_restore 887} 888 889function skip_run_test_with_warning() { 890 echo "WARNING: $1" 891 echo "Test run may fail if run with autorun.sh" 892 echo "Please check your $rootdir/test/common/skipped_tests.txt" 893} 894 895function print_backtrace() { 896 # if errexit is not enabled, don't print a backtrace 897 [[ "$-" =~ e ]] || return 0 898 899 local args=("${BASH_ARGV[@]}") 900 901 xtrace_disable 902 echo "========== Backtrace start: ==========" 903 echo "" 904 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do 905 local func="${FUNCNAME[$i]}" 906 local line_nr="${BASH_LINENO[$((i - 1))]}" 907 local src="${BASH_SOURCE[$i]}" 908 local bt="" cmdline=() 909 910 if [[ -f $src ]]; then 911 bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ 912 | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") 913 fi 914 915 # If extdebug set the BASH_ARGC[i], try to fetch all the args 916 if ((BASH_ARGC[i] > 0)); then 917 # Use argc as index to reverse the stack 918 local argc=${BASH_ARGC[i]} arg 919 for arg in "${args[@]::BASH_ARGC[i]}"; do 920 cmdline[argc--]="[\"$arg\"]" 921 done 922 args=("${args[@]:BASH_ARGC[i]}") 923 fi 924 925 echo "in $src:$line_nr -> $func($( 926 IFS="," 927 printf '%s\n' "${cmdline[*]:-[]}" 928 ))" 929 echo " ..." 930 echo "${bt:-backtrace unavailable}" 931 echo " ..." 932 done 933 echo "" 934 echo "========== Backtrace end ==========" 935 xtrace_restore 936 return 0 937} 938 939function waitforserial() { 940 local i=0 941 local nvme_device_counter=1 942 if [[ -n "$2" ]]; then 943 nvme_device_counter=$2 944 fi 945 946 while [ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]; do 947 [ $i -lt 15 ] || break 948 i=$((i + 1)) 949 echo "Waiting for devices" 950 sleep 1 951 done 952 953 if [[ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]]; then 954 return 1 955 fi 956 957 return 0 958} 959 960function waitforserial_disconnect() { 961 local i=0 962 while lsblk -o NAME,SERIAL | grep -q -w $1; do 963 [ $i -lt 15 ] || break 964 i=$((i + 1)) 965 echo "Waiting for disconnect devices" 966 sleep 1 967 done 968 969 if lsblk -l -o NAME | grep -q -w $1; then 970 return 1 971 fi 972 973 return 0 974} 975 976function waitforblk() { 977 local i=0 978 while ! lsblk -l -o NAME | grep -q -w $1; do 979 [ $i -lt 15 ] || break 980 i=$((i + 1)) 981 sleep 1 982 done 983 984 if ! lsblk -l -o NAME | grep -q -w $1; then 985 return 1 986 fi 987 988 return 0 989} 990 991function waitforblk_disconnect() { 992 local i=0 993 while lsblk -l -o NAME | grep -q -w $1; do 994 [ $i -lt 15 ] || break 995 i=$((i + 1)) 996 sleep 1 997 done 998 999 if lsblk -l -o NAME | grep -q -w $1; then 1000 return 1 1001 fi 1002 1003 return 0 1004} 1005 1006function waitforfile() { 1007 local i=0 1008 while [ ! -e $1 ]; do 1009 [ $i -lt 200 ] || break 1010 i=$((i + 1)) 1011 sleep 0.1 1012 done 1013 1014 if [ ! -e $1 ]; then 1015 return 1 1016 fi 1017 1018 return 0 1019} 1020 1021function fio_config_gen() { 1022 local config_file=$1 1023 local workload=$2 1024 local bdev_type=$3 1025 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1026 1027 if [ -e "$config_file" ]; then 1028 echo "Configuration File Already Exists!: $config_file" 1029 return 1 1030 fi 1031 1032 if [ -z "$workload" ]; then 1033 workload=randrw 1034 fi 1035 1036 touch $1 1037 1038 cat > $1 << EOL 1039[global] 1040thread=1 1041group_reporting=1 1042direct=1 1043norandommap=1 1044percentile_list=50:99:99.9:99.99:99.999 1045time_based=1 1046ramp_time=0 1047EOL 1048 1049 if [ "$workload" == "verify" ]; then 1050 cat <<- EOL >> $config_file 1051 verify=sha1 1052 verify_backlog=1024 1053 rw=randwrite 1054 EOL 1055 1056 # To avoid potential data race issue due to the AIO device 1057 # flush mechanism, add the flag to serialize the writes. 1058 # This is to fix the intermittent IO failure issue of #935 1059 if [ "$bdev_type" == "AIO" ]; then 1060 if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then 1061 echo "serialize_overlap=1" >> $config_file 1062 fi 1063 fi 1064 elif [ "$workload" == "trim" ]; then 1065 echo "rw=trimwrite" >> $config_file 1066 else 1067 echo "rw=$workload" >> $config_file 1068 fi 1069} 1070 1071function fio_bdev() { 1072 # Setup fio binary cmd line 1073 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1074 local bdev_plugin="$rootdir/examples/bdev/fio_plugin/fio_plugin" 1075 1076 # Preload AddressSanitizer library to fio if fio_plugin was compiled with it 1077 local asan_lib 1078 asan_lib=$(ldd $bdev_plugin | grep libasan | awk '{print $3}') 1079 1080 LD_PRELOAD="$asan_lib $bdev_plugin" "$fio_dir"/fio "$@" 1081} 1082 1083function fio_nvme() { 1084 # Setup fio binary cmd line 1085 local fio_dir=$CONFIG_FIO_SOURCE_DIR 1086 local nvme_plugin="$rootdir/examples/nvme/fio_plugin/fio_plugin" 1087 1088 # Preload AddressSanitizer library to fio if fio_plugin was compiled with it 1089 asan_lib=$(ldd $nvme_plugin | grep libasan | awk '{print $3}') 1090 1091 LD_PRELOAD="$asan_lib $nvme_plugin" "$fio_dir"/fio "$@" 1092} 1093 1094function get_lvs_free_mb() { 1095 local lvs_uuid=$1 1096 local lvs_info 1097 local fc 1098 local cs 1099 lvs_info=$($rpc_py bdev_lvol_get_lvstores) 1100 fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") 1101 cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") 1102 1103 # Change to MB's 1104 free_mb=$((fc * cs / 1024 / 1024)) 1105 echo "$free_mb" 1106} 1107 1108function get_bdev_size() { 1109 local bdev_name=$1 1110 local bdev_info 1111 local bs 1112 local nb 1113 bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) 1114 bs=$(jq ".[] .block_size" <<< "$bdev_info") 1115 nb=$(jq ".[] .num_blocks" <<< "$bdev_info") 1116 1117 # Change to MB's 1118 bdev_size=$((bs * nb / 1024 / 1024)) 1119 echo "$bdev_size" 1120} 1121 1122function autotest_cleanup() { 1123 $rootdir/scripts/setup.sh reset 1124 $rootdir/scripts/setup.sh cleanup 1125 if [ $(uname -s) = "Linux" ]; then 1126 if grep -q '#define SPDK_CONFIG_IGB_UIO_DRIVER 1' $rootdir/include/spdk/config.h; then 1127 [[ -e /sys/module/igb_uio ]] && rmmod igb_uio 1128 else 1129 modprobe -r uio_pci_generic 1130 fi 1131 fi 1132 rm -rf "$asan_suppression_file" 1133} 1134 1135function freebsd_update_contigmem_mod() { 1136 if [ $(uname) = FreeBSD ]; then 1137 kldunload contigmem.ko || true 1138 if [ -n "$WITH_DPDK_DIR" ]; then 1139 echo "Warning: SPDK only works on FreeBSD with patches that only exist in SPDK's dpdk submodule" 1140 cp -f "$WITH_DPDK_DIR/kmod/contigmem.ko" /boot/modules/ 1141 cp -f "$WITH_DPDK_DIR/kmod/contigmem.ko" /boot/kernel/ 1142 cp -f "$WITH_DPDK_DIR/kmod/nic_uio.ko" /boot/modules/ 1143 cp -f "$WITH_DPDK_DIR/kmod/nic_uio.ko" /boot/kernel/ 1144 else 1145 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ 1146 cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ 1147 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ 1148 cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ 1149 fi 1150 fi 1151} 1152 1153function get_nvme_name_from_bdf() { 1154 blkname=() 1155 1156 nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true 1157 if [ -z "$nvme_devs" ]; then 1158 return 1159 fi 1160 for dev in $nvme_devs; do 1161 link_name=$(readlink /sys/block/$dev/device/device) || true 1162 if [ -z "$link_name" ]; then 1163 link_name=$(readlink /sys/block/$dev/device) 1164 fi 1165 bdf=$(basename "$link_name") 1166 if [ "$bdf" = "$1" ]; then 1167 blkname+=($dev) 1168 fi 1169 done 1170 1171 printf '%s\n' "${blkname[@]}" 1172} 1173 1174function opal_revert_cleanup() { 1175 $rootdir/app/spdk_tgt/spdk_tgt & 1176 spdk_tgt_pid=$! 1177 waitforlisten $spdk_tgt_pid 1178 1179 # OPAL test only runs on the first NVMe device 1180 # So we just revert the first one here 1181 bdf=$($rootdir/scripts/gen_nvme.sh --json | jq -r '.config[].params | select(.name=="Nvme0").traddr') 1182 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme0" -t "pcie" -a $bdf 1183 # Ignore if this fails. 1184 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b nvme0 -p test || true 1185 1186 killprocess $spdk_tgt_pid 1187} 1188 1189# Get BDF addresses of all NVMe drives currently attached to 1190# uio-pci-generic or vfio-pci 1191function get_nvme_bdfs() { 1192 xtrace_disable 1193 jq -r .config[].params.traddr <<< $(scripts/gen_nvme.sh --json) 1194 xtrace_restore 1195} 1196 1197# Same as function above, but just get the first disks BDF address 1198function get_first_nvme_bdf() { 1199 head -1 <<< "$(get_nvme_bdfs)" 1200} 1201 1202# Define temp storage for all the tests. Look for 2GB at minimum 1203set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" 1204 1205set -o errtrace 1206shopt -s extdebug 1207trap "trap - ERR; print_backtrace >&2" ERR 1208 1209PS4=' \t \$ ' 1210if $SPDK_AUTOTEST_X; then 1211 # explicitly enable xtraces, overriding any tracking information. 1212 unset XTRACE_DISABLED 1213 unset XTRACE_NESTING_LEVEL 1214 set -x 1215 xtrace_enable 1216else 1217 xtrace_restore 1218fi 1219
