1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2022 Intel Corporation 4# All rights reserved. 5# 6testdir=$(readlink -f "$(dirname "$0")") 7rootdir=$(readlink -f "$testdir/../..") 8 9source "$rootdir/test/common/autotest_common.sh" 10source "$rootdir/test/vfio_user/common.sh" 11source "$testdir/common.sh" 12 13function create_device() { 14 local pfid=${1:-1} 15 local vfid=${2:-0} 16 17 "$rootdir/scripts/sma-client.py" <<- CREATE 18 { 19 "method": "CreateDevice", 20 "params": { 21 "nvme": { 22 "physical_id": "$pfid", 23 "virtual_id": "$vfid" 24 } 25 } 26 } 27 CREATE 28} 29 30function delete_device() { 31 "$rootdir/scripts/sma-client.py" <<- DELETE 32 { 33 "method": "DeleteDevice", 34 "params": { 35 "handle": "$1" 36 } 37 } 38 DELETE 39} 40 41function attach_volume() { 42 "$rootdir/scripts/sma-client.py" <<- ATTACH 43 { 44 "method": "AttachVolume", 45 "params": { 46 "device_handle": "$1", 47 "volume": { 48 "volume_id": "$(uuid2base64 $2)" 49 } 50 } 51 } 52 ATTACH 53} 54 55function detach_volume() { 56 "$rootdir/scripts/sma-client.py" <<- DETACH 57 { 58 "method": "DetachVolume", 59 "params": { 60 "device_handle": "$1", 61 "volume_id": "$(uuid2base64 $2)" 62 } 63 } 64 DETACH 65} 66 67function vm_count_nvme() { 68 vm_exec $1 "grep -sl SPDK /sys/class/nvme/*/model || true" | wc -l 69} 70 71function vm_check_subsys_volume() { 72 local vm_id=$1 73 local nqn=$2 74 local uuid=$3 75 76 nvme="$(vm_exec $vm_id "grep -l $nqn /sys/class/nvme/*/subsysnqn" | awk -F/ '{print $5}')" 77 if [[ -z "$nvme" ]]; then 78 error "FAILED no NVMe on vm=$vm_id with nqn=$nqn" 79 return 1 80 fi 81 82 tmpuuid="$(vm_exec $vm_id "grep -l $uuid /sys/class/nvme/$nvme/nvme*/uuid")" 83 if [[ -z "$tmpuuid" ]]; then 84 return 1 85 fi 86} 87 88function vm_check_subsys_nqn() { 89 sleep 1 90 nqn=$(vm_exec $1 "grep -l $2 /sys/class/nvme/*/subsysnqn") 91 if [[ -z "$nqn" ]]; then 92 error "FAILED no NVMe on vm=$1 with nqn=$2" 93 return 1 94 fi 95} 96 97function cleanup() { 98 vm_kill_all 99 killprocess $tgtpid 100 killprocess $smapid 101 if [ -e "${VFO_ROOT_PATH}" ]; then rm -rf "${VFO_ROOT_PATH}"; fi 102} 103 104trap "cleanup; exit 1" SIGINT SIGTERM EXIT 105 106# SSH VM Password 107VM_PASSWORD=root 108vm_no=0 109 110VFO_ROOT_PATH="/tmp/sma/vfio-user/qemu" 111 112if [ -e "${VFO_ROOT_PATH}" ]; then rm -rf "${VFO_ROOT_PATH}"; fi 113mkdir -p "${VFO_ROOT_PATH}" 114 115# Cleanup old VM: 116used_vms=$vm_no 117vm_kill_all 118 119vm_setup --os="$VM_IMAGE" --disk-type=virtio --force=$vm_no --qemu-args="-qmp tcp:localhost:10005,server,nowait \ 120-device pci-bridge,chassis_nr=1,id=pci.spdk.0 \ 121-device pci-bridge,chassis_nr=2,id=pci.spdk.1" 122 123# Run pre-configured VM and wait for them to start 124vm_run $vm_no 125vm_wait_for_boot 300 $vm_no 126 127# Start SPDK 128$rootdir/build/bin/spdk_tgt --wait-for-rpc & 129tgtpid=$! 130waitforlisten $tgtpid 131 132# Configure accel crypto module & operations 133rpc_cmd dpdk_cryptodev_scan_accel_module 134rpc_cmd dpdk_cryptodev_set_driver -d crypto_aesni_mb 135rpc_cmd accel_assign_opc -o encrypt -m dpdk_cryptodev 136rpc_cmd accel_assign_opc -o decrypt -m dpdk_cryptodev 137rpc_cmd framework_start_init 138 139# Prepare the target 140rpc_cmd bdev_null_create null0 100 4096 141rpc_cmd bdev_null_create null1 100 4096 142 143# Start SMA server 144$rootdir/scripts/sma.py -c <( 145 cat <<- EOF 146 devices: 147 - name: 'vfiouser' 148 params: 149 buses: 150 - name: 'pci.spdk.0' 151 count: 32 152 - name: 'pci.spdk.1' 153 count: 32 154 qmp_addr: 127.0.0.1 155 qmp_port: 10005 156 crypto: 157 name: 'bdev_crypto' 158 EOF 159) & 160smapid=$! 161 162# Wait until the SMA starts listening 163sma_waitforlisten 164 165# Make sure a TCP transport has been created 166rpc_cmd nvmf_get_transports --trtype VFIOUSER 167 168# Make sure no nvme subsystems are present 169vm_exec ${vm_no} '[[ ! -e /sys/class/nvme-subsystem/nvme-subsys0 ]]' 170 171# Create a couple of devices and verify them via RPC and SSH 172device0=$(create_device 0 0 | jq -r '.handle') 173rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 174vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-0 175 176# Check that there are two subsystems (1 created above + discovery) 177[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 2 ]] 178 179device1=$(create_device 1 0 | jq -r '.handle') 180rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 181rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 182[[ "$device0" != "$device1" ]] 183vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-1 184 185# Check that there are three subsystems (2 created above + discovery) 186[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 3 ]] 187 188# Verify the method is idempotent and sending the same gRPCs won't create new 189# devices and will return the same IDs 190tmp0=$(create_device 0 0 | jq -r '.handle') 191tmp1=$(create_device 1 0 | jq -r '.handle') 192 193[[ $(vm_count_nvme ${vm_no}) -eq 2 ]] 194 195[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 3 ]] 196[[ "$tmp0" == "$device0" ]] 197[[ "$tmp1" == "$device1" ]] 198 199# Now remove them verifying via RPC 200delete_device "$device0" 201NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 202rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 203[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 2 ]] 204[[ $(vm_count_nvme ${vm_no}) -eq 1 ]] 205 206delete_device "$device1" 207NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 208NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 209[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 1 ]] 210[[ $(vm_count_nvme ${vm_no}) -eq 0 ]] 211 212# Finally check that removing a non-existing device is also successful 213delete_device "$device0" 214delete_device "$device1" 215 216# Check volume attach/detach 217device0=$(create_device 0 0 | jq -r '.handle') 218device1=$(create_device 1 0 | jq -r '.handle') 219uuid0=$(rpc_cmd bdev_get_bdevs -b null0 | jq -r '.[].uuid') 220uuid1=$(rpc_cmd bdev_get_bdevs -b null1 | jq -r '.[].uuid') 221 222# Attach the first volume to a first subsystem 223attach_volume "$device0" "$uuid0" 224[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 225[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 0 ]] 226[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 227vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 228 229attach_volume "$device1" "$uuid1" 230[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 231[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] 232[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 233[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] 234vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 235 236# Attach the same device again and see that it won't fail 237attach_volume "$device0" "$uuid0" 238attach_volume "$device1" "$uuid1" 239[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 240[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] 241[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 242[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] 243vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 244NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid1 245vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 246NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid0 247 248# Cross detach volumes and verify they not fail and have not been removed from the subsystems 249detach_volume "$device0" "$uuid1" 250detach_volume "$device1" "$uuid0" 251[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] 252[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] 253[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] 254[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] 255vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 256NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid1 257vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 258NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid0 259 260# Detach volumes and verify they have been removed from the subsystems 261detach_volume "$device0" "$uuid0" 262detach_volume "$device1" "$uuid1" 263[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 0 ]] 264[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 0 ]] 265NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 266NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 267 268# Detach volumes once again and verify they will not fail 269detach_volume "$device0" "$uuid0" 270detach_volume "$device1" "$uuid1" 271detach_volume "$device0" "$uuid1" 272detach_volume "$device1" "$uuid0" 273 274delete_device "$device0" 275delete_device "$device1" 276 277# Create device with allocation on second bus 278device3=$(create_device 42 0 | jq -r '.handle') 279vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-42 280 281# Verify that device can be found on second bus and properly deleted 282delete_device "$device3" 283NOT vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-42 284 285key0=1234567890abcdef1234567890abcdef 286device0=$(create_device 0 0 | jq -r '.handle') 287uuid0=$(rpc_cmd bdev_get_bdevs -b null0 | jq -r '.[].uuid') 288 289# Now check vfio-user attach with bdev crypto 290"$rootdir/scripts/sma-client.py" <<- ATTACH 291 { 292 "method": "AttachVolume", 293 "params": { 294 "device_handle": "$device0", 295 "volume": { 296 "volume_id": "$(uuid2base64 $uuid0)", 297 "crypto": { 298 "cipher": "$(get_cipher AES_CBC)", 299 "key": "$(format_key $key0)" 300 } 301 } 302 } 303 } 304ATTACH 305 306# Make sure that the namespace exposed in the subsystem is a crypto bdev 307ns_bdev=$(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].name') 308[[ $(rpc_cmd bdev_get_bdevs -b "$ns_bdev" | jq -r '.[0].product_name') == "crypto" ]] 309crypto_bdev=$(rpc_cmd bdev_get_bdevs -b "$ns_bdev" | jq -r '.[] | select(.product_name == "crypto")') 310[[ $(rpc_cmd bdev_get_bdevs | jq -r '[.[] | select(.product_name == "crypto")] | length') -eq 1 ]] 311 312key_name=$(jq -r '.driver_specific.crypto.key_name' <<< "$crypto_bdev") 313key_obj=$(rpc_cmd accel_crypto_keys_get -k $key_name) 314[[ $(jq -r '.[0].key' <<< "$key_obj") == "$key0" ]] 315[[ $(jq -r '.[0].cipher' <<< "$key_obj") == "AES_CBC" ]] 316 317detach_volume "$device0" "$uuid0" 318delete_device "$device0" 319[[ $(rpc_cmd bdev_get_bdevs | jq -r '.[] | select(.product_name == "crypto")' | jq -r length) -eq 0 ]] 320 321# Test qos 322device_vfio_user=1 323device0=$(create_device 0 0 | jq -r '.handle') 324attach_volume "$device0" "$uuid0" 325 326# First check the capabilities 327diff <(get_qos_caps $device_vfio_user | jq --sort-keys) <( 328 jq --sort-keys <<- CAPS 329 { 330 "max_volume_caps": { 331 "rw_iops": true, 332 "rd_bandwidth": true, 333 "wr_bandwidth": true, 334 "rw_bandwidth": true 335 } 336 } 337 CAPS 338) 339 340"$rootdir/scripts/sma-client.py" <<- EOF 341 { 342 "method": "SetQos", 343 "params": { 344 "device_handle": "$device0", 345 "volume_id": "$(uuid2base64 $uuid0)", 346 "maximum": { 347 "rd_iops": 0, 348 "wr_iops": 0, 349 "rw_iops": 3, 350 "rd_bandwidth": 4, 351 "wr_bandwidth": 5, 352 "rw_bandwidth": 6 353 } 354 } 355 } 356EOF 357 358# Make sure that limits were changed 359diff <(rpc_cmd bdev_get_bdevs -b null0 | jq --sort-keys '.[].assigned_rate_limits') <( 360 jq --sort-keys <<- EOF 361 { 362 "rw_ios_per_sec": 3000, 363 "rw_mbytes_per_sec": 6, 364 "r_mbytes_per_sec": 4, 365 "w_mbytes_per_sec": 5 366 } 367 EOF 368) 369 370detach_volume "$device0" "$uuid0" 371delete_device "$device0" 372 373cleanup 374trap - SIGINT SIGTERM EXIT 375