1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2017 Intel Corporation 4# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. 5# All rights reserved. 6# 7testdir=$(readlink -f $(dirname $0)) 8rootdir=$(readlink -f $testdir/../../..) 9source $rootdir/test/common/autotest_common.sh 10source $rootdir/test/nvmf/common.sh 11 12MALLOC_BDEV_SIZE=64 13MALLOC_BLOCK_SIZE=512 14 15function starttarget() { 16 nvmftestinit 17 18 # Start the target 19 nvmfappstart -m 0x1E 20 21 $rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 22 23 num_subsystems=({1..10}) 24 25 timing_enter create_subsystems 26 # Create subsystems 27 rm -rf $testdir/rpcs.txt 28 for i in "${num_subsystems[@]}"; do 29 cat <<- EOL >> $testdir/rpcs.txt 30 bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc$i 31 nvmf_create_subsystem nqn.2016-06.io.spdk:cnode$i -a -s SPDK$i 32 nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode$i Malloc$i 33 nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode$i -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT 34 EOL 35 done 36 $rpc_py < $testdir/rpcs.txt 37 timing_exit create_subsystems 38 39} 40 41function stoptarget() { 42 rm -f ./local-job0-0-verify.state 43 rm -rf $testdir/bdevperf.conf 44 rm -rf $testdir/rpcs.txt 45 46 nvmftestfini 47} 48 49function waitforio() { 50 # $1 = RPC socket 51 if [ -z "$1" ]; then 52 exit 1 53 fi 54 # $2 = bdev name 55 if [ -z "$2" ]; then 56 exit 1 57 fi 58 local ret=1 59 local i 60 for ((i = 10; i != 0; i--)); do 61 read_io_count=$($rpc_py -s $1 bdev_get_iostat -b $2 | jq -r '.bdevs[0].num_read_ops') 62 # A few I/O will happen during initial examine. So wait until at least 100 I/O 63 # have completed to know that bdevperf is really generating the I/O. 64 if [ $read_io_count -ge 100 ]; then 65 ret=0 66 break 67 fi 68 sleep 0.25 69 done 70 return $ret 71} 72 73# Test 1: Kill the initiator unexpectedly with no I/O outstanding 74function nvmf_shutdown_tc1() { 75 starttarget 76 77 # Run bdev_svc, which connects but does not issue I/O 78 $rootdir/test/app/bdev_svc/bdev_svc -m 0x1 -i 1 -r /var/tmp/bdevperf.sock --json <(gen_nvmf_target_json "${num_subsystems[@]}") & 79 perfpid=$! 80 waitforlisten $perfpid /var/tmp/bdevperf.sock 81 $rpc_py -s /var/tmp/bdevperf.sock framework_wait_init 82 83 # Kill bdev_svc 84 kill -9 $perfpid || true 85 rm -f /var/run/spdk_bdev1 86 87 # Verify the target stays up 88 sleep 1 89 kill -0 $nvmfpid 90 91 # Connect with bdevperf and confirm it works 92 $rootdir/build/examples/bdevperf --json <(gen_nvmf_target_json "${num_subsystems[@]}") \ 93 -q 64 -o 65536 -w verify -t 1 94 95 stoptarget 96} 97 98# Test 2: Kill initiator unexpectedly with I/O outstanding 99function nvmf_shutdown_tc2() { 100 starttarget 101 102 # Run bdevperf 103 $rootdir/build/examples/bdevperf -r /var/tmp/bdevperf.sock --json <(gen_nvmf_target_json "${num_subsystems[@]}") -q 64 -o 65536 -w verify -t 10 & 104 perfpid=$! 105 waitforlisten $perfpid /var/tmp/bdevperf.sock 106 $rpc_py -s /var/tmp/bdevperf.sock framework_wait_init 107 108 waitforio /var/tmp/bdevperf.sock Nvme1n1 109 110 # Kill bdevperf half way through 111 killprocess $perfpid 112 113 # Verify the target stays up 114 sleep 1 115 kill -0 $nvmfpid 116 117 stoptarget 118} 119 120# Test 3: Kill the target unexpectedly with I/O outstanding 121function nvmf_shutdown_tc3() { 122 starttarget 123 124 # Run bdevperf 125 $rootdir/build/examples/bdevperf -r /var/tmp/bdevperf.sock --json <(gen_nvmf_target_json "${num_subsystems[@]}") -q 64 -o 65536 -w verify -t 10 & 126 perfpid=$! 127 waitforlisten $perfpid /var/tmp/bdevperf.sock 128 $rpc_py -s /var/tmp/bdevperf.sock framework_wait_init 129 130 # Expand the trap to clean up bdevperf if something goes wrong 131 trap 'process_shm --id $NVMF_APP_SHM_ID; kill -9 $perfpid || true; nvmftestfini; exit 1' SIGINT SIGTERM EXIT 132 133 waitforio /var/tmp/bdevperf.sock Nvme1n1 134 135 # Kill the target half way through 136 killprocess $nvmfpid 137 sleep 1 138 NOT wait $perfpid 139 140 stoptarget 141} 142 143# Test 4: Kill the target unexpectedly with I/O outstanding, highly fragmented payload 144function nvmf_shutdown_tc4() { 145 starttarget 146 147 # Run nvme_perf with highly fragmented payload 148 $rootdir/build/bin/spdk_nvme_perf -q 128 -o 45056 -O 4096 -w randwrite -t 20 -r "trtype:$TEST_TRANSPORT adrfam:IPV4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT" -P 4 & 149 perfpid=$! 150 sleep 5 151 # Expand the trap to clean up bdevperf if something goes wrong 152 trap 'process_shm --id $NVMF_APP_SHM_ID; kill -9 $perfpid || true; nvmftestfini; exit 1' SIGINT SIGTERM EXIT 153 154 # Kill the target half way through 155 killprocess $nvmfpid 156 sleep 1 157 # Due to IOs are completed with errors, perf exits with bad status 158 NOT wait $perfpid 159 stoptarget 160} 161 162run_test "nvmf_shutdown_tc1" nvmf_shutdown_tc1 163run_test "nvmf_shutdown_tc2" nvmf_shutdown_tc2 164run_test "nvmf_shutdown_tc3" nvmf_shutdown_tc3 165# Temporarily disable on e810 due to issue #3523 166if ! [[ "$SPDK_TEST_NVMF_NICS" == "e810" && "$TEST_TRANSPORT" == "rdma" ]]; then 167 run_test "nvmf_shutdown_tc4" nvmf_shutdown_tc4 168fi 169 170trap - SIGINT SIGTERM EXIT 171