1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2022 Intel Corporation 4# All rights reserved. 5# 6testdir=$(readlink -f "$(dirname "$0")") 7rootdir=$(readlink -f "$testdir/../..") 8 9source "$rootdir/test/common/autotest_common.sh" 10source "$rootdir/test/vfio_user/common.sh" 11source "$testdir/common.sh" 12 13function create_device() { 14 local pfid=${1:-1} 15 local vfid=${2:-0} 16 17 "$rootdir/scripts/sma-client.py" <<- CREATE 18 { 19 "method": "CreateDevice", 20 "params": { 21 "nvme": { 22 "physical_id": "$pfid", 23 "virtual_id": "$vfid" 24 } 25 } 26 } 27 CREATE 28} 29 30function delete_device() { 31 "$rootdir/scripts/sma-client.py" <<- DELETE 32 { 33 "method": "DeleteDevice", 34 "params": { 35 "handle": "$1" 36 } 37 } 38 DELETE 39} 40 41function attach_volume() { 42 "$rootdir/scripts/sma-client.py" <<- ATTACH 43 { 44 "method": "AttachVolume", 45 "params": { 46 "device_handle": "$1", 47 "volume": { 48 "volume_id": "$(uuid2base64 $2)" 49 } 50 } 51 } 52 ATTACH 53} 54 55function detach_volume() { 56 "$rootdir/scripts/sma-client.py" <<- DETACH 57 { 58 "method": "DetachVolume", 59 "params": { 60 "device_handle": "$1", 61 "volume_id": "$(uuid2base64 $2)" 62 } 63 } 64 DETACH 65} 66 67function vm_count_nvme() { 68 vm_exec $1 "grep -l SPDK /sys/class/nvme/*/model" | wc -l 69} 70 71function vm_check_subsys_volume() { 72 local vm_id=$1 73 local nqn=$2 74 local uuid=$3 75 76 nvme="$(vm_exec $vm_id "grep -l $nqn /sys/class/nvme/*/subsysnqn" | awk -F/ '{print $5}')" 77 if [[ -z "$nvme" ]]; then 78 error "FAILED no NVMe on vm=$vm_id with nqn=$nqn" 79 return 1 80 fi 81 82 tmpuuid="$(vm_exec $vm_id "grep -l $uuid /sys/class/nvme/$nvme/nvme*/uuid")" 83 if [[ -z "$tmpuuid" ]]; then 84 return 1 85 fi 86} 87 88function vm_check_subsys_nqn() { 89 sleep 1 90 nqn=$(vm_exec $1 "grep -l $2 /sys/class/nvme/*/subsysnqn") 91 if [[ -z "$nqn" ]]; then 92 error "FAILED no NVMe on vm=$1 with nqn=$2" 93 return 1 94 fi 95} 96 97function cleanup() { 98 vm_kill_all 99 killprocess $tgtpid 100 killprocess $smapid 101 if [ -e "${VFO_ROOT_PATH}" ]; then rm -rf "${VFO_ROOT_PATH}"; fi 102} 103 104trap "cleanup; exit 1" SIGINT SIGTERM EXIT 105 106# SSH VM Password 107VM_PASSWORD=root 108vm_no=0 109 110VFO_ROOT_PATH="/tmp/sma/vfio-user/qemu" 111 112if [ -e "${VFO_ROOT_PATH}" ]; then rm -rf "${VFO_ROOT_PATH}"; fi 113mkdir -p "${VFO_ROOT_PATH}" 114 115# Cleanup old VM: 116used_vms=$vm_no 117vm_kill_all 118 119vm_setup --os="$VM_IMAGE" --disk-type=virtio --force=$vm_no --qemu-args="-qmp tcp:localhost:10005,server,nowait \ 120-device pci-bridge,chassis_nr=1,id=pci.spdk.0 \ 121-device pci-bridge,chassis_nr=2,id=pci.spdk.1" 122 123# Run pre-configured VM and wait for them to start 124vm_run $vm_no 125vm_wait_for_boot 300 $vm_no 126 127# Start SPDK 128$rootdir/build/bin/spdk_tgt & 129tgtpid=$! 130waitforlisten $tgtpid 131 132# Prepare the target 133rpc_cmd bdev_null_create null0 100 4096 134rpc_cmd bdev_null_create null1 100 4096 135 136# Start SMA server 137$rootdir/scripts/sma.py -c <( 138 cat <<- EOF 139 devices: 140 - name: 'vfiouser' 141 params: 142 buses: 143 - name: 'pci.spdk.0' 144 count: 32 145 - name: 'pci.spdk.1' 146 count: 32 147 qmp_addr: 127.0.0.1 148 qmp_port: 10005 149 crypto: 150 name: 'bdev_crypto' 151 params: 152 driver: 'crypto_aesni_mb' 153 EOF 154) & 155smapid=$! 156 157# Wait until the SMA starts listening 158sma_waitforlisten 159 160# Make sure a TCP transport has been created 161rpc_cmd nvmf_get_transports --trtype VFIOUSER 162 163# Make sure no nvme subsystems are present 164[[ $(vm_exec ${vm_no} nvme list-subsys -o json | jq -r '.Subsystems | length') -eq 0 ]] 165 166# Create a couple of devices and verify them via RPC and SSH 167device0=$(create_device 0 0 | jq -r '.handle') 168rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 169vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-0 170 171# Check that there are two subsystems (1 created above + discovery) 172[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 2 ]] 173 174device1=$(create_device 1 0 | jq -r '.handle') 175rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 176rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 177[[ "$device0" != "$device1" ]] 178vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-1 179 180# Check that there are three subsystems (2 created above + discovery) 181[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 3 ]] 182 183# Verify the method is idempotent and sending the same gRPCs won't create new 184# devices and will return the same IDs 185tmp0=$(create_device 0 0 | jq -r '.handle') 186tmp1=$(create_device 1 0 | jq -r '.handle') 187 188[[ $(vm_count_nvme ${vm_no}) -eq 2 ]] 189 190[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 3 ]] 191[[ "$tmp0" == "$device0" ]] 192[[ "$tmp1" == "$device1" ]] 193 194# Now remove them verifying via RPC 195delete_device "$device0" 196NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 197rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 198[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 2 ]] 199[[ $(vm_count_nvme ${vm_no}) -eq 1 ]] 200 201delete_device "$device1" 202NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 203NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 204[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 1 ]] 205[[ $(vm_count_nvme ${vm_no}) -eq 0 ]] 206 207# Finally check that removing a non-existing device is also successful 208delete_device "$device0" 209delete_device "$device1" 210 211# Check volume attach/detach 212device0=$(create_device 0 0 | jq -r '.handle') 213device1=$(create_device 1 0 | jq -r '.handle') 214uuid0=$(rpc_cmd bdev_get_bdevs -b null0 | jq -r '.[].uuid') 215uuid1=$(rpc_cmd bdev_get_bdevs -b null1 | jq -r '.[].uuid') 216 217# Attach the first volume to a first subsystem 218attach_volume "$device0" "$uuid0" 219[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 220[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 0 ]] 221[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 222vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 223 224attach_volume "$device1" "$uuid1" 225[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 226[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] 227[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 228[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] 229vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 230 231# Attach the same device again and see that it won't fail 232attach_volume "$device0" "$uuid0" 233attach_volume "$device1" "$uuid1" 234[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 235[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] 236[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 237[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] 238vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 239NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid1 240vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 241NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid0 242 243# Cross detach volumes and verify they not fail and have not been removed from the subsystems 244detach_volume "$device0" "$uuid1" 245detach_volume "$device1" "$uuid0" 246[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 247[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] 248[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 249[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] 250vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 251NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid1 252vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 253NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid0 254 255# Detach volumes and verify they have been removed from the subsystems 256detach_volume "$device0" "$uuid0" 257detach_volume "$device1" "$uuid1" 258[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 0 ]] 259[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 0 ]] 260NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 261NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 262 263# Detach volumes once again and verify they will not fail 264detach_volume "$device0" "$uuid0" 265detach_volume "$device1" "$uuid1" 266detach_volume "$device0" "$uuid1" 267detach_volume "$device1" "$uuid0" 268 269delete_device "$device0" 270delete_device "$device1" 271 272# Create device with allocation on second bus 273device3=$(create_device 42 0 | jq -r '.handle') 274vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-42 275 276# Verify that device can be found on second bus and properly deleted 277delete_device "$device3" 278NOT vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-42 279 280key0=1234567890abcdef1234567890abcdef 281device0=$(create_device 0 0 | jq -r '.handle') 282uuid0=$(rpc_cmd bdev_get_bdevs -b null0 | jq -r '.[].uuid') 283 284# Now check vfio-user attach with bdev crypto 285"$rootdir/scripts/sma-client.py" <<- ATTACH 286 { 287 "method": "AttachVolume", 288 "params": { 289 "device_handle": "$device0", 290 "volume": { 291 "volume_id": "$(uuid2base64 $uuid0)", 292 "crypto": { 293 "cipher": "$(get_cipher AES_CBC)", 294 "key": "$(format_key $key0)" 295 } 296 } 297 } 298 } 299ATTACH 300 301# Make sure that the namespace exposed in the subsystem is a crypto bdev 302ns_bdev=$(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].name') 303[[ $(rpc_cmd bdev_get_bdevs -b "$ns_bdev" | jq -r '.[0].product_name') == "crypto" ]] 304crypto_bdev=$(rpc_cmd bdev_get_bdevs -b "$ns_bdev" | jq -r '.[] | select(.product_name == "crypto")') 305[[ $(rpc_cmd bdev_get_bdevs | jq -r '[.[] | select(.product_name == "crypto")] | length') -eq 1 ]] 306 307[[ $(jq -r '.driver_specific.crypto.key' <<< "$crypto_bdev") == "$key0" ]] 308 309detach_volume "$device0" "$uuid0" 310delete_device "$device0" 311[[ $(rpc_cmd bdev_get_bdevs | jq -r '.[] | select(.product_name == "crypto")' | jq -r length) -eq 0 ]] 312 313# Test qos 314device_vfio_user=1 315device0=$(create_device 0 0 | jq -r '.handle') 316attach_volume "$device0" "$uuid0" 317 318# First check the capabilities 319diff <(get_qos_caps $device_vfio_user | jq --sort-keys) <( 320 jq --sort-keys <<- CAPS 321 { 322 "max_volume_caps": { 323 "rw_iops": true, 324 "rd_bandwidth": true, 325 "wr_bandwidth": true, 326 "rw_bandwidth": true 327 } 328 } 329 CAPS 330) 331 332"$rootdir/scripts/sma-client.py" <<- EOF 333 { 334 "method": "SetQos", 335 "params": { 336 "device_handle": "$device0", 337 "volume_id": "$(uuid2base64 $uuid0)", 338 "maximum": { 339 "rd_iops": 0, 340 "wr_iops": 0, 341 "rw_iops": 3, 342 "rd_bandwidth": 4, 343 "wr_bandwidth": 5, 344 "rw_bandwidth": 6 345 } 346 } 347 } 348EOF 349 350# Make sure that limits were changed 351diff <(rpc_cmd bdev_get_bdevs -b null0 | jq --sort-keys '.[].assigned_rate_limits') <( 352 jq --sort-keys <<- EOF 353 { 354 "rw_ios_per_sec": 3000, 355 "rw_mbytes_per_sec": 6, 356 "r_mbytes_per_sec": 4, 357 "w_mbytes_per_sec": 5 358 } 359 EOF 360) 361 362detach_volume "$device0" "$uuid0" 363delete_device "$device0" 364 365cleanup 366trap - SIGINT SIGTERM EXIT 367