1set -xe 2 3# Vhost SCSI hotremove tests 4# 5# # Objective 6# The purpose of these tests is to verify that SPDK vhost remains stable during 7# hot-remove operations performed on SCSI controllers devices. 8# Hot-remove is a scenario where a NVMe device is removed when already in use. 9# Tests consist of 4 test cases. 10# 11# # Test cases description 12# 1. FIO I/O traffic is run during hot-remove operations. 13# By default FIO uses default_integrity*.job config files located in 14# test/vhost/hotplug/fio_jobs directory. 15# 2. FIO mode of operation is random write (randwrite) with verification enabled 16# which results in also performing read operations. 17 18function prepare_fio_cmd_tc1() { 19 print_test_fio_header 20 21 run_fio="$fio_bin --eta=never " 22 for vm_num in $1; do 23 cp $fio_job $tmp_detach_job 24 vm_dir=$VM_DIR/$vm_num 25 vm_check_scsi_location $vm_num 26 for disk in $SCSI_DISK; do 27 echo "[nvme-host$disk]" >> $tmp_detach_job 28 echo "filename=/dev/$disk" >> $tmp_detach_job 29 echo "size=100%" >> $tmp_detach_job 30 done 31 vm_scp "$vm_num" $tmp_detach_job 127.0.0.1:/root/default_integrity_2discs.job 32 run_fio+="--client=127.0.0.1,$(vm_fio_socket $vm_num) --remote-config /root/default_integrity_2discs.job " 33 rm $tmp_detach_job 34 done 35} 36 37# Vhost SCSI hot-remove test cases. 38 39# Test Case 1 40function scsi_hotremove_tc1() { 41 echo "Scsi hotremove test case 1" 42 traddr="" 43 get_traddr "Nvme0" 44 # 1. Run the command to hot remove NVMe disk. 45 delete_nvme "Nvme0" 46 # 2. If vhost had crashed then tests would stop running 47 sleep 1 48 add_nvme "HotInNvme0" "$traddr" 49} 50 51# Test Case 2 52function scsi_hotremove_tc2() { 53 echo "Scsi hotremove test case 2" 54 # 1. Attach split NVMe bdevs to scsi controller. 55 $rpc_py add_vhost_scsi_lun naa.Nvme0n1p0.0 0 HotInNvme0n1p0 56 $rpc_py add_vhost_scsi_lun naa.Nvme0n1p1.0 0 Mallocp0 57 $rpc_py add_vhost_scsi_lun naa.Nvme0n1p2.1 0 HotInNvme0n1p1 58 $rpc_py add_vhost_scsi_lun naa.Nvme0n1p3.1 0 Mallocp1 59 60 # 2. Run two VMs, attached to scsi controllers. 61 vms_setup 62 vm_run_with_arg 0 1 63 vms_prepare "0 1" 64 65 vm_check_scsi_location "0" 66 local disks="$SCSI_DISK" 67 68 traddr="" 69 get_traddr "Nvme0" 70 prepare_fio_cmd_tc1 "0 1" 71 # 3. Run FIO I/O traffic with verification enabled on on both NVMe disks in VM. 72 $run_fio & 73 local last_pid=$! 74 sleep 3 75 # 4. Run the command to hot remove NVMe disk. 76 delete_nvme "HotInNvme0" 77 78 # 5. Check that fio job run on hot-remove device stopped on VM. 79 # Expected: Fio should return error message and return code != 0. 80 wait_for_finish $last_pid || retcode=$? 81 check_fio_retcode "Scsi hotremove test case 2: Iteration 1." 1 $retcode 82 83 # 6. Check if removed devices are gone from VM. 84 vm_check_scsi_location "0" 85 local new_disks="$SCSI_DISK" 86 check_disks "$disks" "$new_disks" 87 # 7. Reboot both VMs. 88 reboot_all_and_prepare "0 1" 89 # 8. Run FIO I/O traffic with verification enabled on on both VMs. 90 local retcode=0 91 $run_fio & 92 wait_for_finish $! || retcode=$? 93 # 9. Check that fio job run on hot-remove device stopped on both VMs. 94 # Expected: Fio should return error message and return code != 0. 95 check_fio_retcode "Scsi hotremove test case 2: Iteration 2." 1 $retcode 96 vm_shutdown_all 97 add_nvme "HotInNvme1" "$traddr" 98 sleep 1 99} 100 101# Test Case 3 102function scsi_hotremove_tc3() { 103 echo "Scsi hotremove test case 3" 104 # 1. Attach added NVMe bdev to scsi controller. 105 $rpc_py add_vhost_scsi_lun naa.Nvme0n1p0.0 0 HotInNvme1n1p0 106 # 2. Run two VM, attached to scsi controllers. 107 vm_run_with_arg 0 1 108 vms_prepare "0 1" 109 vm_check_scsi_location "0" 110 local disks="$SCSI_DISK" 111 traddr="" 112 get_traddr "Nvme0" 113 # 3. Run FIO I/O traffic with verification enabled on on both NVMe disks in VMs. 114 prepare_fio_cmd_tc1 "0" 115 $run_fio & 116 local last_pid=$! 117 sleep 3 118 # 4. Run the command to hot remove NVMe disk. 119 delete_nvme "HotInNvme1" 120 # 5. Check that fio job run on hot-remove device stopped on first VM. 121 # Expected: Fio should return error message and return code != 0. 122 wait_for_finish $last_pid || retcode=$? 123 check_fio_retcode "Scsi hotremove test case 3: Iteration 1." 1 $retcode 124 # 6. Check if removed devices are gone from lsblk. 125 vm_check_scsi_location "0" 126 local new_disks="$SCSI_DISK" 127 check_disks "$disks" "$new_disks" 128 # 7. Reboot both VMs. 129 reboot_all_and_prepare "0 1" 130 # 8. Run FIO I/O traffic with verification enabled on on both VMs. 131 local retcode=0 132 $run_fio & 133 wait_for_finish $! || retcode=$? 134 # 9. Check that fio job run on hot-remove device stopped on both VMs. 135 # Expected: Fio should return error message and return code != 0. 136 check_fio_retcode "Scsi hotremove test case 3: Iteration 2." 1 $retcode 137 vm_shutdown_all 138 add_nvme "HotInNvme2" "$traddr" 139 sleep 1 140} 141 142# Test Case 4 143function scsi_hotremove_tc4() { 144 echo "Scsi hotremove test case 4" 145 # 1. Attach NVMe bdevs to scsi controllers. 146 $rpc_py add_vhost_scsi_lun naa.Nvme0n1p0.0 0 HotInNvme2n1p0 147 $rpc_py add_vhost_scsi_lun naa.Nvme0n1p2.1 0 HotInNvme2n1p1 148 # 2. Run two VMs, attach to scsi controller. 149 vm_run_with_arg 0 1 150 vms_prepare "0 1" 151 152 # 3. Run FIO I/O traffic with verification enabled on first VM. 153 vm_check_scsi_location "0" 154 local disks_vm0="$SCSI_DISK" 155 # 4. Run FIO I/O traffic with verification enabled on second VM. 156 prepare_fio_cmd_tc1 "0" 157 $run_fio & 158 last_pid_vm0=$! 159 160 vm_check_scsi_location "1" 161 local disks_vm1="$SCSI_DISK" 162 prepare_fio_cmd_tc1 "1" 163 $run_fio & 164 local last_pid_vm1=$! 165 prepare_fio_cmd_tc1 "0 1" 166 sleep 3 167 # 5. Run the command to hot remove NVMe disk. 168 traddr="" 169 get_traddr "Nvme0" 170 delete_nvme "HotInNvme2" 171 # 6. Check that fio job run on hot-removed devices stopped. 172 # Expected: Fio should return error message and return code != 0. 173 local retcode_vm0=0 174 wait_for_finish $last_pid_vm0 || retcode_vm0=$? 175 local retcode_vm1=0 176 wait_for_finish $last_pid_vm1 || retcode_vm1=$? 177 check_fio_retcode "Scsi hotremove test case 4: Iteration 1." 1 $retcode_vm0 178 check_fio_retcode "Scsi hotremove test case 4: Iteration 2." 1 $retcode_vm1 179 180 # 7. Check if removed devices are gone from lsblk. 181 vm_check_scsi_location "0" 182 local new_disks_vm0="$SCSI_DISK" 183 check_disks "$disks_vm0" "$new_disks_vm0" 184 vm_check_scsi_location "1" 185 local new_disks_vm1="$SCSI_DISK" 186 check_disks "$disks_vm1" "$new_disks_vm1" 187 188 # 8. Reboot both VMs. 189 reboot_all_and_prepare "0 1" 190 # 9. Run FIO I/O traffic with verification enabled on on not-removed NVMe disk. 191 local retcode=0 192 $run_fio & 193 wait_for_finish $! || retcode=$? 194 # 10. Check that fio job run on hot-removed device stopped. 195 # Expect: Fio should return error message and return code != 0. 196 check_fio_retcode "Scsi hotremove test case 4: Iteration 3." 1 $retcode 197 prepare_fio_cmd_tc1 "0 1" 198 # 11. Run FIO I/O traffic with verification enabled on on not-removed NVMe disk. 199 local retcode=0 200 $run_fio & 201 wait_for_finish $! || retcode=$? 202 # 12. Check finished status FIO. Write and read in the not-removed. 203 # NVMe disk should be successful. 204 # Expected: Fio should return return code == 0. 205 check_fio_retcode "Scsi hotremove test case 4: Iteration 4." 0 $retcode 206 vm_shutdown_all 207 add_nvme "HotInNvme3" "$traddr" 208 sleep 1 209 $rpc_py remove_vhost_scsi_target naa.Nvme0n1p1.0 0 210 $rpc_py remove_vhost_scsi_target naa.Nvme0n1p3.1 0 211} 212 213function pre_scsi_hotremove_test_case() { 214 $rpc_py construct_vhost_scsi_controller naa.Nvme0n1p0.0 215 $rpc_py construct_vhost_scsi_controller naa.Nvme0n1p1.0 216 $rpc_py construct_vhost_scsi_controller naa.Nvme0n1p2.1 217 $rpc_py construct_vhost_scsi_controller naa.Nvme0n1p3.1 218} 219 220function post_scsi_hotremove_test_case() { 221 $rpc_py remove_vhost_controller naa.Nvme0n1p0.0 222 $rpc_py remove_vhost_controller naa.Nvme0n1p1.0 223 $rpc_py remove_vhost_controller naa.Nvme0n1p2.1 224 $rpc_py remove_vhost_controller naa.Nvme0n1p3.1 225} 226 227pre_scsi_hotremove_test_case 228scsi_hotremove_tc1 229scsi_hotremove_tc2 230scsi_hotremove_tc3 231scsi_hotremove_tc4 232post_scsi_hotremove_test_case 233