1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2021 Intel Corporation 4# All rights reserved. 5# 6 7testdir=$(readlink -f $(dirname $0)) 8rootdir=$(readlink -f $testdir/../../..) 9source $rootdir/test/setup/common.sh 10source $rootdir/test/common/autotest_common.sh 11source $rootdir/test/nvmf/common.sh 12 13tgt_core_mask='0x3' 14bdevperf_core_mask='0x4' 15bdevperf_rpc_sock=/var/tmp/bdevperf.sock 16bdevperf_rpc_pid=-1 17 18nvmftestinit 19 20function get_subsystem_nqn() { 21 echo nqn.2016-06.io.spdk:system_$1 22} 23 24function create_subsystem_and_connect_on_netdev() { 25 local -a dev_name 26 27 dev_name=$1 28 malloc_name=$dev_name 29 nqn=$(get_subsystem_nqn "$dev_name") 30 ip=$(get_ip_address "$dev_name") 31 serial=SPDK000$dev_name 32 33 MALLOC_BDEV_SIZE=128 34 MALLOC_BLOCK_SIZE=512 35 36 $rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b $malloc_name 37 $rpc_py nvmf_create_subsystem $nqn -a -s $serial 38 $rpc_py nvmf_subsystem_add_ns $nqn $malloc_name 39 $rpc_py nvmf_subsystem_add_listener $nqn -t $TEST_TRANSPORT -a $ip -s $NVMF_PORT 40 41 return 0 42} 43 44function create_subsystem_and_connect() { 45 local -gA netdev_nvme_dict 46 netdev_nvme_dict=() 47 48 $rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 "$@" 49 for net_dev in $(get_rdma_if_list); do 50 netdev_nvme_dict[$net_dev]="$(create_subsystem_and_connect_on_netdev $net_dev)" 51 done 52 53 return 0 54} 55 56function rescan_pci() { 57 echo 1 > /sys/bus/pci/rescan 58} 59 60function get_pci_dir() { 61 dev_name=$1 62 readlink -f /sys/bus/pci/devices/*/net/${dev_name}/device 63} 64 65function remove_one_nic() { 66 dev_name=$1 67 echo 1 > $(get_pci_dir $dev_name)/remove 68} 69 70function get_rdma_device_name() { 71 dev_name=$1 72 ls $(get_pci_dir $dev_name)/infiniband 73} 74 75function check_rdma_dev_exists_in_nvmf_tgt() { 76 local rdma_dev_name=$1 77 $rpc_py nvmf_get_stats | jq -r '.poll_groups[0].transports[].devices[].name' | grep "$rdma_dev_name" 78 return $? 79} 80 81function get_rdma_dev_count_in_nvmf_tgt() { 82 local rdma_dev_name=$1 83 $rpc_py nvmf_get_stats | jq -r '.poll_groups[0].transports[].devices | length' 84} 85 86function generate_io_traffic_with_bdevperf() { 87 local dev_names=("$@") 88 89 mkdir -p $testdir 90 $rootdir/build/examples/bdevperf -m $bdevperf_core_mask -z -r $bdevperf_rpc_sock -q 128 -o 4096 -w verify -t 90 &> $testdir/try.txt & 91 bdevperf_pid=$! 92 93 trap 'process_shm --id $NVMF_APP_SHM_ID; cat $testdir/try.txt; rm -f $testdir/try.txt; kill -9 $bdevperf_pid; nvmftestfini; exit 1' SIGINT SIGTERM EXIT 94 waitforlisten $bdevperf_pid $bdevperf_rpc_sock 95 96 # Create a controller and set multipath behavior 97 # bdev_retry_count is set to -1 means infinite reconnects 98 $rpc_py -s $bdevperf_rpc_sock bdev_nvme_set_options -r -1 99 100 for dev_name in "${dev_names[@]}"; do 101 nqn=$(get_subsystem_nqn $dev_name) 102 tgt_ip=$(get_ip_address "$dev_name") 103 104 # -l -1 ctrlr_loss_timeout_sec -1 means infinite reconnects 105 # -o 1 reconnect_delay_sec time to delay a reconnect retry is limited to 1 sec 106 $rpc_py -s $bdevperf_rpc_sock bdev_nvme_attach_controller -b Nvme_$dev_name -t $TEST_TRANSPORT -a $tgt_ip -s $NVMF_PORT -f ipv4 -n $nqn -l -1 -o 1 107 done 108 109 $rootdir/examples/bdev/bdevperf/bdevperf.py -t 120 -s $bdevperf_rpc_sock perform_tests & 110 bdevperf_rpc_pid=$! 111 112 sleep 5 113} 114 115function stop_bdevperf() { 116 wait $bdevperf_rpc_pid 117 118 # NOTE: rdma-core <= v43.0 has memleak bug (fixed in commit 7720071f). 119 killprocess $bdevperf_pid || true 120 bdevperf_pid= 121 122 cat $testdir/try.txt 123 124 trap - SIGINT SIGTERM EXIT 125 rm -f $testdir/try.txt 126} 127 128function test_remove_and_rescan() { 129 nvmfappstart -m "$tgt_core_mask" 130 131 create_subsystem_and_connect "$@" 132 133 generate_io_traffic_with_bdevperf "${!netdev_nvme_dict[@]}" 134 135 for net_dev in "${!netdev_nvme_dict[@]}"; do 136 nvme_dev=${netdev_nvme_dict[$net_dev]} 137 rdma_dev_name=$(get_rdma_device_name $net_dev) 138 origin_ip=$(get_ip_address "$net_dev") 139 pci_dir=$(get_pci_dir $net_dev) 140 141 if ! check_rdma_dev_exists_in_nvmf_tgt "$rdma_dev_name"; then 142 echo "Device $rdma_dev_name is not registered in tgt". 143 exit 1 144 fi 145 146 remove_one_nic $net_dev 147 148 for i in $(seq 1 10); do 149 if ! check_rdma_dev_exists_in_nvmf_tgt "$rdma_dev_name"; then 150 break 151 fi 152 if [[ $i == 10 ]]; then 153 # failed to remove this device 154 exit 1 155 fi 156 sleep 1 157 done 158 159 ib_count_after_remove=$(get_rdma_dev_count_in_nvmf_tgt) 160 161 rescan_pci 162 163 for i in $(seq 1 10); do 164 new_net_dev=$(ls ${pci_dir}/net || echo) 165 if [[ -z $new_net_dev ]]; then 166 sleep 1 167 elif [[ $new_net_dev != "$net_dev" ]]; then 168 echo "Device name changed after rescan, try rename." 169 ip link set $new_net_dev down && ip link set $new_net_dev name $net_dev 170 sleep 1 171 else 172 break 173 fi 174 done 175 176 if [[ -z $new_net_dev ]]; then 177 exit 1 178 fi 179 180 ip link set $net_dev up 181 if [[ -z $(get_ip_address "$net_dev") ]]; then 182 ip addr add $origin_ip/24 dev $net_dev 183 fi 184 185 # if rdma device name is renamed, nvmf_get_stats may return an obsoleted name. 186 # so we check ib device count here instead of the device name. 187 for i in $(seq 1 10); do 188 ib_count=$(get_rdma_dev_count_in_nvmf_tgt) 189 if ((ib_count > ib_count_after_remove)); then 190 break 191 fi 192 193 if [[ $i == 10 ]]; then 194 # failed to rescan this device 195 exit 1 196 fi 197 sleep 2 198 done 199 done 200 201 stop_bdevperf 202 203 # NOTE: rdma-core <= v43.0 has memleak bug (fixed in commit 7720071f). 204 killprocess $nvmfpid || true 205 nvmfpid= 206 207 return 0 208} 209 210function check_env_for_test_bonding_slaves() { 211 # only test with dual-port CX4/CX5. 212 213 local -gA port_nic_map 214 local -g target_nics 215 216 # gather dev with same bus-device. 217 for bdf in "${mlx[@]}"; do 218 pci_net_devs=("/sys/bus/pci/devices/$bdf/net/"*) 219 pci_net_devs=("${pci_net_devs[@]##*/}") 220 221 bd=$(echo ${bdf} | cut -d '.' -f 1) 222 223 port_nic_map[$bd]="${pci_net_devs[*]} ${port_nic_map[$bd]}" 224 done 225 226 for x in "${port_nic_map[@]}"; do 227 ports=($x) 228 if ((${#ports[@]} >= 2)); then 229 target_nics=(${ports[@]}) 230 return 0 231 fi 232 done 233 234 return 1 235} 236 237BOND_NAME="bond_nvmf" 238BOND_IP="10.11.11.26" 239BOND_MASK="24" 240 241function clean_bond_device() { 242 if ip link | grep $BOND_NAME; then 243 ip link del $BOND_NAME 244 fi 245 for net_dev in "${target_nics[@]}"; do 246 ip link set $net_dev up 247 done 248} 249 250function test_bonding_slaves_on_nics() { 251 nic1=$1 252 nic2=$2 253 254 clean_bond_device 255 ip link add $BOND_NAME type bond mode 1 256 ip link set $nic1 down && sudo ip link set $nic1 master $BOND_NAME 257 ip link set $nic2 down && sudo ip link set $nic2 master $BOND_NAME 258 ip link set $BOND_NAME up 259 ip addr add ${BOND_IP}/${BOND_MASK} dev $BOND_NAME 260 261 # check slaves here 262 slaves=($(cat /sys/class/net/${BOND_NAME}/bonding/slaves)) 263 if ((${#slaves[@]} != 2)); then 264 exit 1 265 fi 266 267 # wait ib driver activated on bond device 268 sleep 5 269 270 nvmfappstart -m "$tgt_core_mask" 271 $rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 272 273 create_subsystem_and_connect_on_netdev $BOND_NAME 274 275 ib_count=$(get_rdma_dev_count_in_nvmf_tgt) 276 echo "IB Count: " $ib_count 277 278 generate_io_traffic_with_bdevperf $BOND_NAME 279 280 sleep 2 281 echo -$nic1 | sudo tee /sys/class/net/${BOND_NAME}/bonding/slaves 282 sleep 10 283 echo +$nic1 | sudo tee /sys/class/net/${BOND_NAME}/bonding/slaves 284 285 ib_count2=$ib_count 286 for i in $(seq 1 10); do 287 ib_count2=$(get_rdma_dev_count_in_nvmf_tgt) 288 if ((ib_count2 == ib_count)); then 289 break 290 fi 291 sleep 2 292 done 293 if ((ib_count2 != ib_count)); then 294 exit 1 295 fi 296 297 stop_bdevperf 298 299 # NOTE: rdma-core <= v43.0 has memleak bug (fixed in commit 7720071f). 300 killprocess $nvmfpid || true 301 nvmfpid= 302 return 0 303} 304 305function test_bond_slaves() { 306 check_env_for_test_bonding_slaves 307 if [[ -z "$target_nics" ]]; then 308 echo "No available nic ports to run this test." 309 exit 0 310 fi 311 test_bonding_slaves_on_nics "${target_nics[@]}" 312} 313 314run_test "nvmf_device_removal_pci_remove_no_srq" test_remove_and_rescan --no-srq 315run_test "nvmf_device_removal_pci_remove" test_remove_and_rescan 316# bond slaves case needs lag_master & vport_manager are enabled by mlxconfig 317# and not work on CI machine currently. 318# run_test "nvmf_device_removal_bond_slaves" test_bond_slaves 319 320nvmftestfini 321clean_bond_device 322