1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright (C) 2017 Intel Corporation 3# All rights reserved. 4# 5 6set -xe 7 8# Vhost SCSI hotremove tests 9# 10# # Objective 11# The purpose of these tests is to verify that SPDK vhost remains stable during 12# hot-remove operations performed on SCSI controllers devices. 13# Hot-remove is a scenario where a NVMe device is removed when already in use. 14# Tests consist of 4 test cases. 15# 16# # Test cases description 17# 1. FIO I/O traffic is run during hot-remove operations. 18# By default FIO uses default_integrity*.job config files located in 19# test/vhost/hotplug/fio_jobs directory. 20# 2. FIO mode of operation is random write (randwrite) with verification enabled 21# which results in also performing read operations. 22 23function prepare_fio_cmd_tc1() { 24 print_test_fio_header 25 26 run_fio="$fio_bin --eta=never " 27 for vm_num in $1; do 28 cp $fio_job $tmp_detach_job 29 vm_check_scsi_location $vm_num 30 for disk in $SCSI_DISK; do 31 cat <<- EOL >> $tmp_detach_job 32 [nvme-host$disk] 33 filename=/dev/$disk 34 size=100% 35 EOL 36 done 37 vm_scp "$vm_num" $tmp_detach_job 127.0.0.1:/root/default_integrity_2discs.job 38 run_fio+="--client=127.0.0.1,$(vm_fio_socket $vm_num) --remote-config /root/default_integrity_2discs.job " 39 rm $tmp_detach_job 40 done 41} 42 43# Vhost SCSI hot-remove test cases. 44 45# Test Case 1 46function scsi_hotremove_tc1() { 47 echo "Scsi hotremove test case 1" 48 traddr="" 49 get_traddr "Nvme0" 50 # 1. Run the command to hot remove NVMe disk. 51 delete_nvme "Nvme0" 52 # 2. If vhost had crashed then tests would stop running 53 sleep 10 54 add_nvme "HotInNvme0" "$traddr" 55} 56 57# Test Case 2 58function scsi_hotremove_tc2() { 59 echo "Scsi hotremove test case 2" 60 # 1. Attach split NVMe bdevs to scsi controller. 61 $rpc_py vhost_scsi_controller_add_target naa.Nvme0n1p0.0 0 HotInNvme0n1p0 62 $rpc_py vhost_scsi_controller_add_target naa.Nvme0n1p1.0 0 Mallocp0 63 $rpc_py vhost_scsi_controller_add_target naa.Nvme0n1p2.1 0 HotInNvme0n1p1 64 $rpc_py vhost_scsi_controller_add_target naa.Nvme0n1p3.1 0 Mallocp1 65 66 # 2. Run two VMs, attached to scsi controllers. 67 vms_setup 68 vm_run_with_arg 0 1 69 vms_prepare "0 1" 70 71 vm_check_scsi_location "0" 72 local disks="$SCSI_DISK" 73 74 traddr="" 75 get_traddr "Nvme0" 76 prepare_fio_cmd_tc1 "0 1" 77 # 3. Run FIO I/O traffic with verification enabled on on both NVMe disks in VM. 78 $run_fio & 79 local last_pid=$! 80 sleep 3 81 # 4. Run the command to hot remove NVMe disk. 82 delete_nvme "HotInNvme0" 83 84 # 5. Check that fio job run on hot-remove device stopped on VM. 85 # Expected: Fio should return error message and return code != 0. 86 wait_for_finish $last_pid || retcode=$? 87 check_fio_retcode "Scsi hotremove test case 2: Iteration 1." 1 $retcode 88 89 # 6. Check if removed devices are gone from VM. 90 vm_check_scsi_location "0" 91 local new_disks="$SCSI_DISK" 92 check_disks "$disks" "$new_disks" 93 # 7. Reboot both VMs. 94 reboot_all_and_prepare "0 1" 95 # 8. Run FIO I/O traffic with verification enabled on on both VMs. 96 local retcode=0 97 $run_fio & 98 wait_for_finish $! || retcode=$? 99 # 9. Check that fio job run on hot-remove device stopped on both VMs. 100 # Expected: Fio should return error message and return code != 0. 101 check_fio_retcode "Scsi hotremove test case 2: Iteration 2." 1 $retcode 102 vm_shutdown_all 103 add_nvme "HotInNvme1" "$traddr" 104 sleep 1 105} 106 107# Test Case 3 108function scsi_hotremove_tc3() { 109 echo "Scsi hotremove test case 3" 110 # 1. Attach added NVMe bdev to scsi controller. 111 $rpc_py vhost_scsi_controller_add_target naa.Nvme0n1p0.0 0 HotInNvme1n1p0 112 # 2. Run two VM, attached to scsi controllers. 113 vm_run_with_arg 0 1 114 vms_prepare "0 1" 115 vm_check_scsi_location "0" 116 local disks="$SCSI_DISK" 117 traddr="" 118 get_traddr "Nvme0" 119 # 3. Run FIO I/O traffic with verification enabled on on both NVMe disks in VMs. 120 prepare_fio_cmd_tc1 "0" 121 $run_fio & 122 local last_pid=$! 123 sleep 3 124 # 4. Run the command to hot remove NVMe disk. 125 delete_nvme "HotInNvme1" 126 # 5. Check that fio job run on hot-remove device stopped on first VM. 127 # Expected: Fio should return error message and return code != 0. 128 wait_for_finish $last_pid || retcode=$? 129 check_fio_retcode "Scsi hotremove test case 3: Iteration 1." 1 $retcode 130 # 6. Check if removed devices are gone from lsblk. 131 vm_check_scsi_location "0" 132 local new_disks="$SCSI_DISK" 133 check_disks "$disks" "$new_disks" 134 # 7. Reboot both VMs. 135 reboot_all_and_prepare "0 1" 136 # 8. Run FIO I/O traffic with verification enabled on on both VMs. 137 local retcode=0 138 $run_fio & 139 wait_for_finish $! || retcode=$? 140 # 9. Check that fio job run on hot-remove device stopped on both VMs. 141 # Expected: Fio should return error message and return code != 0. 142 check_fio_retcode "Scsi hotremove test case 3: Iteration 2." 1 $retcode 143 vm_shutdown_all 144 add_nvme "HotInNvme2" "$traddr" 145 sleep 1 146} 147 148# Test Case 4 149function scsi_hotremove_tc4() { 150 echo "Scsi hotremove test case 4" 151 # 1. Attach NVMe bdevs to scsi controllers. 152 $rpc_py vhost_scsi_controller_add_target naa.Nvme0n1p0.0 0 HotInNvme2n1p0 153 $rpc_py vhost_scsi_controller_add_target naa.Nvme0n1p2.1 0 HotInNvme2n1p1 154 # 2. Run two VMs, attach to scsi controller. 155 vm_run_with_arg 0 1 156 vms_prepare "0 1" 157 158 # 3. Run FIO I/O traffic with verification enabled on first VM. 159 vm_check_scsi_location "0" 160 local disks_vm0="$SCSI_DISK" 161 # 4. Run FIO I/O traffic with verification enabled on second VM. 162 prepare_fio_cmd_tc1 "0" 163 $run_fio & 164 last_pid_vm0=$! 165 166 vm_check_scsi_location "1" 167 local disks_vm1="$SCSI_DISK" 168 prepare_fio_cmd_tc1 "1" 169 $run_fio & 170 local last_pid_vm1=$! 171 prepare_fio_cmd_tc1 "0 1" 172 sleep 3 173 # 5. Run the command to hot remove NVMe disk. 174 traddr="" 175 get_traddr "Nvme0" 176 delete_nvme "HotInNvme2" 177 # 6. Check that fio job run on hot-removed devices stopped. 178 # Expected: Fio should return error message and return code != 0. 179 local retcode_vm0=0 180 wait_for_finish $last_pid_vm0 || retcode_vm0=$? 181 local retcode_vm1=0 182 wait_for_finish $last_pid_vm1 || retcode_vm1=$? 183 check_fio_retcode "Scsi hotremove test case 4: Iteration 1." 1 $retcode_vm0 184 check_fio_retcode "Scsi hotremove test case 4: Iteration 2." 1 $retcode_vm1 185 186 # 7. Check if removed devices are gone from lsblk. 187 vm_check_scsi_location "0" 188 local new_disks_vm0="$SCSI_DISK" 189 check_disks "$disks_vm0" "$new_disks_vm0" 190 vm_check_scsi_location "1" 191 local new_disks_vm1="$SCSI_DISK" 192 check_disks "$disks_vm1" "$new_disks_vm1" 193 194 # 8. Reboot both VMs. 195 reboot_all_and_prepare "0 1" 196 # 9. Run FIO I/O traffic with verification enabled on on not-removed NVMe disk. 197 local retcode=0 198 $run_fio & 199 wait_for_finish $! || retcode=$? 200 # 10. Check that fio job run on hot-removed device stopped. 201 # Expect: Fio should return error message and return code != 0. 202 check_fio_retcode "Scsi hotremove test case 4: Iteration 3." 1 $retcode 203 prepare_fio_cmd_tc1 "0 1" 204 # 11. Run FIO I/O traffic with verification enabled on on not-removed NVMe disk. 205 local retcode=0 206 $run_fio & 207 wait_for_finish $! || retcode=$? 208 # 12. Check finished status FIO. Write and read in the not-removed. 209 # NVMe disk should be successful. 210 # Expected: Fio should return return code == 0. 211 check_fio_retcode "Scsi hotremove test case 4: Iteration 4." 0 $retcode 212 vm_shutdown_all 213 add_nvme "HotInNvme3" "$traddr" 214 sleep 1 215 $rpc_py vhost_scsi_controller_remove_target naa.Nvme0n1p1.0 0 216 $rpc_py vhost_scsi_controller_remove_target naa.Nvme0n1p3.1 0 217} 218 219function pre_scsi_hotremove_test_case() { 220 $rpc_py vhost_create_scsi_controller naa.Nvme0n1p0.0 221 $rpc_py vhost_create_scsi_controller naa.Nvme0n1p1.0 222 $rpc_py vhost_create_scsi_controller naa.Nvme0n1p2.1 223 $rpc_py vhost_create_scsi_controller naa.Nvme0n1p3.1 224} 225 226function post_scsi_hotremove_test_case() { 227 $rpc_py vhost_delete_controller naa.Nvme0n1p0.0 228 $rpc_py vhost_delete_controller naa.Nvme0n1p1.0 229 $rpc_py vhost_delete_controller naa.Nvme0n1p2.1 230 $rpc_py vhost_delete_controller naa.Nvme0n1p3.1 231} 232 233pre_scsi_hotremove_test_case 234scsi_hotremove_tc1 235scsi_hotremove_tc2 236scsi_hotremove_tc3 237scsi_hotremove_tc4 238post_scsi_hotremove_test_case 239