1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2022 Intel Corporation 4# All rights reserved. 5# 6testdir=$(readlink -f "$(dirname "$0")") 7rootdir=$(readlink -f "$testdir/../..") 8 9source "$rootdir/test/common/autotest_common.sh" 10source "$testdir/common.sh" 11 12sma_py="$rootdir/scripts/sma-client.py" 13rpc_py="$rootdir/scripts/rpc.py" 14 15t1sock='/var/tmp/spdk.sock1' 16t2sock='/var/tmp/spdk.sock2' 17invalid_port=8008 18t1dscport=8009 19t2dscport1=8010 20t2dscport2=8011 21t1nqn='nqn.2016-06.io.spdk:node1' 22t2nqn='nqn.2016-06.io.spdk:node2' 23hostnqn='nqn.2016-06.io.spdk:host0' 24cleanup_period=1 25 26function cleanup() { 27 killprocess $smapid 28 killprocess $tgtpid 29 killprocess $t1pid 30 killprocess $t2pid 31} 32 33function format_endpoints() { 34 local eps=("$@") 35 for ((i = 0; i < ${#eps[@]}; i++)); do 36 cat <<- EOF 37 { 38 "trtype": "tcp", 39 "traddr": "127.0.0.1", 40 "trsvcid": "${eps[i]}" 41 } 42 EOF 43 if ! ((i + 1 == ${#@})); then 44 echo , 45 fi 46 done 47} 48 49function format_volume() { 50 local volume_id=$1 51 shift 52 53 cat <<- EOF 54 "volume": { 55 "volume_id": "$(uuid2base64 $volume_id)", 56 "nvmf": { 57 "hostnqn": "$hostnqn", 58 "discovery": { 59 "discovery_endpoints": [ 60 $(format_endpoints "$@") 61 ] 62 } 63 } 64 } 65 EOF 66} 67 68function create_device() { 69 local nqn=$1 70 local volume_id=$2 71 local volume= 72 73 shift 74 if [[ -n "$volume_id" ]]; then 75 volume="$(format_volume "$@")," 76 fi 77 78 $sma_py <<- EOF 79 { 80 "method": "CreateDevice", 81 "params": { 82 $volume 83 "nvmf_tcp": { 84 "subnqn": "$nqn", 85 "adrfam": "ipv4", 86 "traddr": "127.0.0.1", 87 "trsvcid": "4419" 88 } 89 } 90 } 91 EOF 92} 93 94function delete_device() { 95 $sma_py <<- EOF 96 { 97 "method": "DeleteDevice", 98 "params": { 99 "handle": "$1" 100 } 101 } 102 EOF 103} 104 105function attach_volume() { 106 local device_id=$1 107 108 shift 109 $sma_py <<- EOF 110 { 111 "method": "AttachVolume", 112 "params": { 113 $(format_volume "$@"), 114 "device_handle": "$device_id" 115 } 116 } 117 EOF 118} 119 120function detach_volume() { 121 $sma_py <<- EOF 122 { 123 "method": "DetachVolume", 124 "params": { 125 "device_handle": "$1", 126 "volume_id": "$(uuid2base64 $2)" 127 } 128 } 129 EOF 130} 131 132trap "cleanup; exit 1" SIGINT SIGTERM EXIT 133 134# Start two remote targets 135$rootdir/build/bin/spdk_tgt -r $t1sock -m 0x1 & 136t1pid=$! 137$rootdir/build/bin/spdk_tgt -r $t2sock -m 0x2 & 138t2pid=$! 139 140# One target that the SMA will configure 141$rootdir/build/bin/spdk_tgt -m 0x4 & 142tgtpid=$! 143 144# And finally the SMA itself 145$rootdir/scripts/sma.py -c <( 146 cat <<- EOF 147 discovery_timeout: 5 148 volume_cleanup_period: $cleanup_period 149 devices: 150 - name: 'nvmf_tcp' 151 EOF 152) & 153smapid=$! 154 155waitforlisten $tgtpid 156waitforlisten $t1pid "$t1sock" 157waitforlisten $t2pid "$t2sock" 158 159# Prepare the targets. The first one has a single subsystem with a single volume and a single 160# discovery listener. The second one also has a single subsystem, but has two volumes attached to 161# it and has two discovery listeners. 162t1uuid=$(uuidgen) 163t2uuid=$(uuidgen) 164t2uuid2=$(uuidgen) 165 166$rpc_py -s $t1sock <<- EOF 167 nvmf_create_transport -t tcp 168 bdev_null_create null0 128 4096 -u $t1uuid 169 nvmf_create_subsystem $t1nqn -s SPDK00000000000001 -d SPDK_Controller1 170 nvmf_subsystem_add_host $t1nqn $hostnqn 171 nvmf_subsystem_add_ns $t1nqn $t1uuid 172 nvmf_subsystem_add_listener $t1nqn -t tcp -a 127.0.0.1 -s 4420 173 nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t1dscport 174EOF 175 176$rpc_py -s $t2sock <<- EOF 177 nvmf_create_transport -t tcp 178 bdev_null_create null0 128 4096 -u $t2uuid 179 bdev_null_create null1 128 4096 -u $t2uuid2 180 nvmf_create_subsystem $t2nqn -s SPDK00000000000001 -d SPDK_Controller1 181 nvmf_subsystem_add_host $t2nqn $hostnqn 182 nvmf_subsystem_add_ns $t2nqn $t2uuid 183 nvmf_subsystem_add_ns $t2nqn $t2uuid2 184 nvmf_subsystem_add_listener $t2nqn -t tcp -a 127.0.0.1 -s 4421 185 nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t2dscport1 186 nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t2dscport2 187EOF 188 189# Wait until the SMA starts listening 190sma_waitforlisten 191 192localnqn='nqn.2016-06.io.spdk:local0' 193 194# Create a device 195device_id=$(create_device $localnqn | jq -r '.handle') 196 197# Check that it's been created 198$rpc_py nvmf_get_subsystems $localnqn 199 200# Attach a volume specifying both targets 201attach_volume $device_id $t1uuid $t1dscport $t2dscport1 202 203# Check that a connection has been made to discovery services on both targets 204[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]] 205 206$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 207$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1 208 209# Check that the volume was attached to the device 210[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 211[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]] 212 213# Attach the other volume, this time specify only single target 214attach_volume $device_id $t2uuid $t2dscport1 215 216# Check that both volumes are attached to the device 217[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]] 218[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 2 ]] 219$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t1uuid 220$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid 221 222# Detach the first volume 223detach_volume $device_id $t1uuid 224 225# Check that there's a connection to a single target now (because we've only specified a single 226# target when connecting the other volume). 227[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 228$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1 229# Check that the volume was actually removed 230[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 231[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t2uuid" ]] 232 233# Detach the other volume 234detach_volume $device_id $t2uuid 235 236# And verify it's gone too 237[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 238[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]] 239 240# Check that specifying an invalid volume UUID results in an error 241NOT attach_volume $device_id $(uuidgen) $t1dscport 242[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 243[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]] 244 245# Attach them again, this time both volumes specify both targets 246volumes=($t1uuid $t2uuid) 247for volume_id in "${volumes[@]}"; do 248 attach_volume $device_id $volume_id $t1dscport $t2dscport1 249done 250 251[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]] 252$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 253$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1 254$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t1uuid 255$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid 256 257# Detach one and see that both targets are still connected 258detach_volume $device_id $t1uuid 259 260[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]] 261$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 262$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1 263 264# Try to delete the device and verify that it fails if it has volumes attached to it 265NOT delete_device $device_id 266 267[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]] 268$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 269$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1 270 271# After detaching the second volume, it should be possible to delete the device 272detach_volume $device_id $t2uuid 273delete_device $device_id 274 275[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 276NOT $rpc_py nvmf_get_subsystems $localnqn 277 278# Create a device and attach a volume immediately 279device_id=$(create_device $localnqn $t1uuid $t1dscport | jq -r '.handle') 280 281# Verify that there's a connection to the target and the volume is attached to the device 282[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 283$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 284[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 285[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]] 286 287# Make sure it's also possible to detach it 288detach_volume $device_id $t1uuid 289 290[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 291[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]] 292 293# Check that discovery referrals work correctly 294attach_volume $device_id $t2uuid $t2dscport1 $t2dscport2 295 296# Check that only a single connection has been made 297[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 298[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 299[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t2uuid" ]] 300 301# Add the other volume from the same target/subsystem, but use a single discovery endpoint 302attach_volume $device_id $t2uuid2 $t2dscport2 303 304# Check that the volume was attached to the subsystem, but no extra connection has been made 305[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 306[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 2 ]] 307$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid 308$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid2 309 310# Reset the device 311detach_volume $device_id $t1uuid 312detach_volume $device_id $t2uuid 313detach_volume $device_id $t2uuid2 314delete_device $device_id 315[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 316 317device_id=$(create_device $localnqn | jq -r '.handle') 318 319# Check subsystem NQN verification, start with a valid one 320$sma_py <<- EOF 321 { 322 "method": "AttachVolume", 323 "params": { 324 "volume": { 325 "volume_id": "$(uuid2base64 $t1uuid)", 326 "nvmf": { 327 "hostnqn": "$hostnqn", 328 "subnqn": "$t1nqn", 329 "discovery": { 330 "discovery_endpoints": [ 331 { 332 "trtype": "tcp", 333 "traddr": "127.0.0.1", 334 "trsvcid": "$t1dscport" 335 } 336 ] 337 } 338 } 339 }, 340 "device_handle": "$device_id" 341 } 342 } 343EOF 344 345[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 346$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 347[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 348[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]] 349 350# Then check incorrect subnqn 351NOT $sma_py <<- EOF 352 { 353 "method": "AttachVolume", 354 "params": { 355 "volume": { 356 "volume_id": "$(uuid2base64 $t2uuid)", 357 "nvmf": { 358 "hostnqn": "$hostnqn", 359 "subnqn": "${t2nqn}-invalid", 360 "discovery": { 361 "discovery_endpoints": [ 362 { 363 "trtype": "tcp", 364 "traddr": "127.0.0.1", 365 "trsvcid": "$t2dscport1" 366 } 367 ] 368 } 369 } 370 }, 371 "device_handle": "$device_id" 372 } 373 } 374EOF 375 376# Verify the volume hasn't been attached 377[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 378$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 379[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 380[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]] 381 382# Check incorrect hostnqn 383NOT $sma_py <<- EOF 384 { 385 "method": "AttachVolume", 386 "params": { 387 "volume": { 388 "volume_id": "$(uuid2base64 $t2uuid)", 389 "nvmf": { 390 "hostnqn": "${hostnqn}-invalid", 391 "discovery": { 392 "discovery_endpoints": [ 393 { 394 "trtype": "tcp", 395 "traddr": "127.0.0.1", 396 "trsvcid": "$t2dscport1" 397 } 398 ] 399 } 400 } 401 }, 402 "device_handle": "$device_id" 403 } 404 } 405EOF 406 407# Verify that the volume wasn't attached 408[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 409$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 410[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 411[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]] 412 413# Check that the the attach will fail if there's nobody listening on the discovery endpoint 414NOT attach_volume $device_id $(uuidgen) $invalid_port 415[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 416$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 417 418# Make sure that the discovery service is stopped if a volume is disconnected outside of SMA (e.g. 419# by removing it from the target) 420$rpc_py -s $t1sock nvmf_subsystem_remove_ns $t1nqn 1 421# Give SMA some time to be notified about the change 422sleep $((cleanup_period + 1)) 423[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 424$rpc_py -s $t1sock nvmf_subsystem_add_ns $t1nqn $t1uuid 425 426# Do the same, but this time attach two volumes and check that the discovery service is only 427# stopped once both volumes are disconnected 428attach_volume $device_id $t2uuid $t2dscport1 429attach_volume $device_id $t2uuid2 $t2dscport1 430[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 2 ]] 431[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 432$rpc_py -s $t2sock nvmf_subsystem_remove_ns $t2nqn 2 433# Give SMA some time to be notified about the change 434sleep $((cleanup_period + 1)) 435# One of the volumes should be gone, but the discovery service should still be running 436[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 437[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 438$rpc_py -s $t2sock nvmf_subsystem_remove_ns $t2nqn 1 439# Give SMA some time to be notified about the change 440sleep $((cleanup_period + 1)) 441# Now that both are gone, the discovery service should be stopped too 442[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]] 443[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 444$rpc_py -s $t2sock nvmf_subsystem_add_ns $t2nqn $t2uuid 445$rpc_py -s $t2sock nvmf_subsystem_add_ns $t2nqn $t2uuid2 446 447delete_device $device_id 448 449cleanup 450trap - SIGINT SIGTERM EXIT 451