1#!/usr/bin/env bash 2 3testdir=$(readlink -f "$(dirname "$0")") 4rootdir=$(readlink -f "$testdir/../..") 5 6source "$rootdir/test/common/autotest_common.sh" 7source "$rootdir/test/vfio_user/common.sh" 8source "$testdir/common.sh" 9 10function create_device() { 11 local pfid=${1:-1} 12 local vfid=${2:-0} 13 14 "$rootdir/scripts/sma-client.py" <<- CREATE 15 { 16 "method": "CreateDevice", 17 "params": { 18 "nvme": { 19 "physical_id": "$pfid", 20 "virtual_id": "$vfid" 21 } 22 } 23 } 24 CREATE 25} 26 27function delete_device() { 28 "$rootdir/scripts/sma-client.py" <<- DELETE 29 { 30 "method": "DeleteDevice", 31 "params": { 32 "handle": "$1" 33 } 34 } 35 DELETE 36} 37 38function attach_volume() { 39 "$rootdir/scripts/sma-client.py" <<- ATTACH 40 { 41 "method": "AttachVolume", 42 "params": { 43 "device_handle": "$1", 44 "volume": { 45 "volume_id": "$(uuid2base64 $2)" 46 } 47 } 48 } 49 ATTACH 50} 51 52function detach_volume() { 53 "$rootdir/scripts/sma-client.py" <<- DETACH 54 { 55 "method": "DetachVolume", 56 "params": { 57 "device_handle": "$1", 58 "volume_id": "$(uuid2base64 $2)" 59 } 60 } 61 DETACH 62} 63 64function vm_count_nvme() { 65 vm_exec $1 "grep -l SPDK /sys/class/nvme/*/model" | wc -l 66} 67 68function vm_check_subsys_volume() { 69 local vm_id=$1 70 local nqn=$2 71 local uuid=$3 72 73 nvme="$(vm_exec $vm_id "grep -l $nqn /sys/class/nvme/*/subsysnqn" | awk -F/ '{print $5}')" 74 if [[ -z "$nvme" ]]; then 75 error "FAILED no NVMe on vm=$vm_id with nqn=$nqn" 76 return 1 77 fi 78 79 tmpuuid="$(vm_exec $vm_id "grep -l $uuid /sys/class/nvme/$nvme/nvme*/uuid")" 80 if [[ -z "$tmpuuid" ]]; then 81 return 1 82 fi 83} 84 85function vm_check_subsys_nqn() { 86 sleep 1 87 nqn=$(vm_exec $1 "grep -l $2 /sys/class/nvme/*/subsysnqn") 88 if [[ -z "$nqn" ]]; then 89 error "FAILED no NVMe on vm=$1 with nqn=$2" 90 return 1 91 fi 92} 93 94function cleanup() { 95 vm_kill_all 96 killprocess $tgtpid 97 killprocess $smapid 98 if [ -e "${VFO_ROOT_PATH}" ]; then rm -rf "${VFO_ROOT_PATH}"; fi 99} 100 101trap "cleanup; exit 1" SIGINT SIGTERM EXIT 102 103# SSH VM Password 104VM_PASSWORD=root 105vm_no=0 106 107VFO_ROOT_PATH="/tmp/sma/vfio-user/qemu" 108 109if [ -e "${VFO_ROOT_PATH}" ]; then rm -rf "${VFO_ROOT_PATH}"; fi 110mkdir -p "${VFO_ROOT_PATH}" 111 112# Cleanup old VM: 113used_vms=$vm_no 114vm_kill_all 115 116vm_setup --os="$VM_IMAGE" --disk-type=virtio --force=$vm_no --qemu-args="-qmp tcp:localhost:10005,server,nowait -device pci-bridge,chassis_nr=1,id=pci.spdk.0" 117 118# Run pre-configured VM and wait for them to start 119vm_run $vm_no 120vm_wait_for_boot 300 $vm_no 121 122# Start SPDK 123$rootdir/build/bin/spdk_tgt & 124tgtpid=$! 125waitforlisten $tgtpid 126 127# Prepare the target 128rpc_cmd bdev_null_create null0 100 4096 129rpc_cmd bdev_null_create null1 100 4096 130 131# Start SMA server 132$rootdir/scripts/sma.py -c <( 133 cat <<- EOF 134 devices: 135 - name: 'vfiouser' 136 params: 137 buses: 138 - name: 'pci.spdk.0' 139 count: 32 140 qmp_addr: 127.0.0.1 141 qmp_port: 10005 142 crypto: 143 name: 'bdev_crypto' 144 params: 145 driver: 'crypto_aesni_mb' 146 EOF 147) & 148smapid=$! 149 150# Wait until the SMA starts listening 151sma_waitforlisten 152 153# Make sure a TCP transport has been created 154rpc_cmd nvmf_get_transports --trtype VFIOUSER 155 156# Make sure no nvme subsystems are present 157[[ $(vm_exec ${vm_no} nvme list-subsys -o json | jq -r '.Subsystems | length') -eq 0 ]] 158 159# Create a couple of devices and verify them via RPC and SSH 160device0=$(create_device 0 0 | jq -r '.handle') 161rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 162vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-0 163 164# Check that there are two subsystems (1 created above + discovery) 165[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 2 ]] 166 167device1=$(create_device 1 0 | jq -r '.handle') 168rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 169rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 170[[ "$device0" != "$device1" ]] 171vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-1 172 173# Check that there are three subsystems (2 created above + discovery) 174[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 3 ]] 175 176# Verify the method is idempotent and sending the same gRPCs won't create new 177# devices and will return the same IDs 178tmp0=$(create_device 0 0 | jq -r '.handle') 179tmp1=$(create_device 1 0 | jq -r '.handle') 180 181[[ $(vm_count_nvme ${vm_no}) -eq 2 ]] 182 183[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 3 ]] 184[[ "$tmp0" == "$device0" ]] 185[[ "$tmp1" == "$device1" ]] 186 187# Now remove them verifying via RPC 188delete_device "$device0" 189NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 190rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 191[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 2 ]] 192[[ $(vm_count_nvme ${vm_no}) -eq 1 ]] 193 194delete_device "$device1" 195NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 196NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 197[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 1 ]] 198[[ $(vm_count_nvme ${vm_no}) -eq 0 ]] 199 200# Finally check that removing a non-existing device is also sucessful 201delete_device "$device0" 202delete_device "$device1" 203 204# Check volume attach/detach 205device0=$(create_device 0 0 | jq -r '.handle') 206device1=$(create_device 1 0 | jq -r '.handle') 207uuid0=$(rpc_cmd bdev_get_bdevs -b null0 | jq -r '.[].uuid') 208uuid1=$(rpc_cmd bdev_get_bdevs -b null1 | jq -r '.[].uuid') 209 210# Attach the first volume to a first subsystem 211attach_volume "$device0" "$uuid0" 212[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 213[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 0 ]] 214[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 215vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 216 217attach_volume "$device1" "$uuid1" 218[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 219[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] 220[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 221[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] 222vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 223 224# Attach the same device again and see that it won't fail 225attach_volume "$device0" "$uuid0" 226attach_volume "$device1" "$uuid1" 227[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 228[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] 229[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 230[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] 231vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 232NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid1 233vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 234NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid0 235 236# Cross detach volumes and verify they not fail and have not been removed from the subsystems 237detach_volume "$device0" "$uuid1" 238detach_volume "$device1" "$uuid0" 239[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 240[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] 241[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 242[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] 243vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 244NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid1 245vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 246NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid0 247 248# Detach volumes and verify they have been removed from the subsystems 249detach_volume "$device0" "$uuid0" 250detach_volume "$device1" "$uuid1" 251[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 0 ]] 252[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 0 ]] 253NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 254NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 255 256# Detach volumes once again and verify they will not fail 257detach_volume "$device0" "$uuid0" 258detach_volume "$device1" "$uuid1" 259detach_volume "$device0" "$uuid1" 260detach_volume "$device1" "$uuid0" 261 262delete_device "$device0" 263delete_device "$device1" 264 265key0=1234567890abcdef1234567890abcdef 266device0=$(create_device 0 0 | jq -r '.handle') 267uuid0=$(rpc_cmd bdev_get_bdevs -b null0 | jq -r '.[].uuid') 268 269# Now check vfio-user attach with bdev crypto 270"$rootdir/scripts/sma-client.py" <<- ATTACH 271 { 272 "method": "AttachVolume", 273 "params": { 274 "device_handle": "$device0", 275 "volume": { 276 "volume_id": "$(uuid2base64 $uuid0)", 277 "crypto": { 278 "cipher": "$(get_cipher AES_CBC)", 279 "key": "$(format_key $key0)" 280 } 281 } 282 } 283 } 284ATTACH 285 286# Make sure that the namespace exposed in the subsystem is a crypto bdev 287ns_bdev=$(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].name') 288[[ $(rpc_cmd bdev_get_bdevs -b "$ns_bdev" | jq -r '.[0].product_name') == "crypto" ]] 289crypto_bdev=$(rpc_cmd bdev_get_bdevs -b "$ns_bdev" | jq -r '.[] | select(.product_name == "crypto")') 290[[ $(rpc_cmd bdev_get_bdevs | jq -r '[.[] | select(.product_name == "crypto")] | length') -eq 1 ]] 291 292[[ $(jq -r '.driver_specific.crypto.key' <<< "$crypto_bdev") == "$key0" ]] 293 294detach_volume "$device0" "$uuid0" 295delete_device "$device0" 296[[ $(rpc_cmd bdev_get_bdevs | jq -r '.[] | select(.product_name == "crypto")' | jq -r length) -eq 0 ]] 297 298cleanup 299trap - SIGINT SIGTERM EXIT 300