1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2021 Intel Corporation 4# All rights reserved. 5# 6 7testdir=$(readlink -f $(dirname $0)) 8rootdir=$(readlink -f $testdir/../../..) 9source $rootdir/test/setup/common.sh 10source $rootdir/test/common/autotest_common.sh 11source $rootdir/test/nvmf/common.sh 12 13tgt_core_mask='0x3' 14bdevperf_core_mask='0x4' 15bdevperf_rpc_sock=/var/tmp/bdevperf.sock 16bdevperf_rpc_pid=-1 17 18nvmftestinit 19 20function get_subsystem_nqn() { 21 echo nqn.2016-06.io.spdk:system_$1 22} 23 24function create_subsystem_and_connect_on_netdev() { 25 local -a dev_name 26 27 dev_name=$1 28 malloc_name=$dev_name 29 nqn=$(get_subsystem_nqn "$dev_name") 30 ip=$(get_ip_address "$dev_name") 31 serial=SPDK000$dev_name 32 33 MALLOC_BDEV_SIZE=128 34 MALLOC_BLOCK_SIZE=512 35 36 $rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b $malloc_name 37 $rpc_py nvmf_create_subsystem $nqn -a -s $serial 38 $rpc_py nvmf_subsystem_add_ns $nqn $malloc_name 39 $rpc_py nvmf_subsystem_add_listener $nqn -t $TEST_TRANSPORT -a $ip -s $NVMF_PORT 40 41 return 0 42} 43 44function create_subsystem_and_connect() { 45 local -gA netdev_nvme_dict 46 netdev_nvme_dict=() 47 48 $rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 "$@" 49 for net_dev in $(get_rdma_if_list); do 50 netdev_nvme_dict[$net_dev]="$(create_subsystem_and_connect_on_netdev $net_dev)" 51 done 52 53 return 0 54} 55 56function rescan_pci() { 57 echo 1 > /sys/bus/pci/rescan 58} 59 60function get_pci_dir() { 61 dev_name=$1 62 readlink -f /sys/bus/pci/devices/*/net/${dev_name}/device 63} 64 65function remove_one_nic() { 66 dev_name=$1 67 echo 1 > $(get_pci_dir $dev_name)/remove 68} 69 70function get_rdma_device_name() { 71 dev_name=$1 72 ls $(get_pci_dir $dev_name)/infiniband 73} 74 75function check_rdma_dev_exists_in_nvmf_tgt() { 76 local rdma_dev_name=$1 77 $rpc_py nvmf_get_stats | jq -r '.poll_groups[0].transports[].devices[].name' | grep "$rdma_dev_name" 78 return $? 79} 80 81function get_rdma_dev_count_in_nvmf_tgt() { 82 local rdma_dev_name=$1 83 $rpc_py nvmf_get_stats | jq -r '.poll_groups[0].transports[].devices | length' 84} 85 86function generate_io_traffic_with_bdevperf() { 87 local dev_names=("$@") 88 89 mkdir -p $testdir 90 $rootdir/build/examples/bdevperf -m $bdevperf_core_mask -z -r $bdevperf_rpc_sock -q 128 -o 4096 -w verify -t 90 &> $testdir/try.txt & 91 bdevperf_pid=$! 92 93 trap 'process_shm --id $NVMF_APP_SHM_ID; cat $testdir/try.txt; rm -f $testdir/try.txt; kill -9 $bdevperf_pid; nvmftestfini; exit 1' SIGINT SIGTERM EXIT 94 waitforlisten $bdevperf_pid $bdevperf_rpc_sock 95 96 # Create a controller and set multipath behavior 97 # bdev_retry_count is set to -1 means infinite reconnects 98 $rpc_py -s $bdevperf_rpc_sock bdev_nvme_set_options -r -1 99 100 for dev_name in "${dev_names[@]}"; do 101 nqn=$(get_subsystem_nqn $dev_name) 102 tgt_ip=$(get_ip_address "$dev_name") 103 104 # -l -1 ctrlr_loss_timeout_sec -1 means infinite reconnects 105 # -o 1 reconnect_delay_sec time to delay a reconnect retry is limited to 1 sec 106 $rpc_py -s $bdevperf_rpc_sock bdev_nvme_attach_controller -b Nvme_$dev_name -t $TEST_TRANSPORT -a $tgt_ip -s $NVMF_PORT -f ipv4 -n $nqn -l -1 -o 1 107 done 108 109 $rootdir/examples/bdev/bdevperf/bdevperf.py -t 120 -s $bdevperf_rpc_sock perform_tests & 110 bdevperf_rpc_pid=$! 111 112 sleep 5 113} 114 115function stop_bdevperf() { 116 wait $bdevperf_rpc_pid 117 118 killprocess $bdevperf_pid 119 bdevperf_pid= 120 121 cat $testdir/try.txt 122 123 trap - SIGINT SIGTERM EXIT 124 rm -f $testdir/try.txt 125} 126 127function test_remove_and_rescan() { 128 nvmfappstart -m "$tgt_core_mask" 129 130 create_subsystem_and_connect "$@" 131 132 generate_io_traffic_with_bdevperf "${!netdev_nvme_dict[@]}" 133 134 for net_dev in "${!netdev_nvme_dict[@]}"; do 135 nvme_dev=${netdev_nvme_dict[$net_dev]} 136 rdma_dev_name=$(get_rdma_device_name $net_dev) 137 origin_ip=$(get_ip_address "$net_dev") 138 pci_dir=$(get_pci_dir $net_dev) 139 140 if ! check_rdma_dev_exists_in_nvmf_tgt "$rdma_dev_name"; then 141 echo "Device $rdma_dev_name is not registered in tgt". 142 exit 1 143 fi 144 145 remove_one_nic $net_dev 146 147 for i in $(seq 1 10); do 148 if ! check_rdma_dev_exists_in_nvmf_tgt "$rdma_dev_name"; then 149 break 150 fi 151 if [[ $i == 10 ]]; then 152 # failed to remove this device 153 exit 1 154 fi 155 sleep 1 156 done 157 158 ib_count_after_remove=$(get_rdma_dev_count_in_nvmf_tgt) 159 160 rescan_pci 161 162 for i in $(seq 1 10); do 163 new_net_dev=$(ls ${pci_dir}/net || echo) 164 if [[ -z $new_net_dev ]]; then 165 sleep 1 166 elif [[ $new_net_dev != "$net_dev" ]]; then 167 echo "Device name changed after rescan, try rename." 168 ip link set $new_net_dev down && ip link set $new_net_dev name $net_dev 169 sleep 1 170 else 171 break 172 fi 173 done 174 175 if [[ -z $new_net_dev ]]; then 176 exit 1 177 fi 178 179 ip link set $net_dev up 180 if [[ -z $(get_ip_address "$net_dev") ]]; then 181 ip addr add $origin_ip/24 dev $net_dev 182 fi 183 184 # if rdma device name is renamed, nvmf_get_stats may return an obsoleted name. 185 # so we check ib device count here instead of the device name. 186 for i in $(seq 1 10); do 187 ib_count=$(get_rdma_dev_count_in_nvmf_tgt) 188 if ((ib_count > ib_count_after_remove)); then 189 break 190 fi 191 192 if [[ $i == 10 ]]; then 193 # failed to rescan this device 194 exit 1 195 fi 196 sleep 2 197 done 198 done 199 200 stop_bdevperf 201 202 killprocess $nvmfpid 203 nvmfpid= 204 205 return 0 206} 207 208function check_env_for_test_bonding_slaves() { 209 # only test with dual-port CX4/CX5. 210 211 local -gA port_nic_map 212 local -g target_nics 213 214 # gather dev with same bus-device. 215 for bdf in "${mlx[@]}"; do 216 pci_net_devs=("/sys/bus/pci/devices/$bdf/net/"*) 217 pci_net_devs=("${pci_net_devs[@]##*/}") 218 219 bd=$(echo ${bdf} | cut -d '.' -f 1) 220 221 port_nic_map[$bd]="${pci_net_devs[*]} ${port_nic_map[$bd]}" 222 done 223 224 for x in "${port_nic_map[@]}"; do 225 ports=($x) 226 if ((${#ports[@]} >= 2)); then 227 target_nics=(${ports[@]}) 228 return 0 229 fi 230 done 231 232 return 1 233} 234 235BOND_NAME="bond_nvmf" 236BOND_IP="10.11.11.26" 237BOND_MASK="24" 238 239function clean_bond_device() { 240 if ip link | grep $BOND_NAME; then 241 ip link del $BOND_NAME 242 fi 243 for net_dev in "${target_nics[@]}"; do 244 ip link set $net_dev up 245 done 246} 247 248function test_bonding_slaves_on_nics() { 249 nic1=$1 250 nic2=$2 251 252 clean_bond_device 253 ip link add $BOND_NAME type bond mode 1 254 ip link set $nic1 down && sudo ip link set $nic1 master $BOND_NAME 255 ip link set $nic2 down && sudo ip link set $nic2 master $BOND_NAME 256 ip link set $BOND_NAME up 257 ip addr add ${BOND_IP}/${BOND_MASK} dev $BOND_NAME 258 259 # check slaves here 260 slaves=($(cat /sys/class/net/${BOND_NAME}/bonding/slaves)) 261 if ((${#slaves[@]} != 2)); then 262 exit 1 263 fi 264 265 # wait ib driver activated on bond device 266 sleep 5 267 268 nvmfappstart -m "$tgt_core_mask" 269 $rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 270 271 create_subsystem_and_connect_on_netdev $BOND_NAME 272 273 ib_count=$(get_rdma_dev_count_in_nvmf_tgt) 274 echo "IB Count: " $ib_count 275 276 generate_io_traffic_with_bdevperf $BOND_NAME 277 278 sleep 2 279 echo -$nic1 | sudo tee /sys/class/net/${BOND_NAME}/bonding/slaves 280 sleep 10 281 echo +$nic1 | sudo tee /sys/class/net/${BOND_NAME}/bonding/slaves 282 283 ib_count2=$ib_count 284 for i in $(seq 1 10); do 285 ib_count2=$(get_rdma_dev_count_in_nvmf_tgt) 286 if ((ib_count2 == ib_count)); then 287 break 288 fi 289 sleep 2 290 done 291 if ((ib_count2 != ib_count)); then 292 exit 1 293 fi 294 295 stop_bdevperf 296 297 killprocess $nvmfpid 298 nvmfpid= 299 return 0 300} 301 302function test_bond_slaves() { 303 check_env_for_test_bonding_slaves 304 if [[ -z "$target_nics" ]]; then 305 echo "No available nic ports to run this test." 306 exit 0 307 fi 308 test_bonding_slaves_on_nics "${target_nics[@]}" 309} 310 311run_test "nvmf_device_removal_pci_remove_no_srq" test_remove_and_rescan --no-srq 312run_test "nvmf_device_removal_pci_remove" test_remove_and_rescan 313# bond slaves case needs lag_master & vport_manager are enabled by mlxconfig 314# and not work on CI machine currently. 315# run_test "nvmf_device_removal_bond_slaves" test_bond_slaves 316 317nvmftestfini 318clean_bond_device 319