1#!/usr/bin/env bash 2 3testdir=$(readlink -f "$(dirname "$0")") 4rootdir=$(readlink -f "$testdir/../..") 5 6source "$rootdir/test/common/autotest_common.sh" 7source "$testdir/common.sh" 8 9sma_py="$rootdir/scripts/sma-client.py" 10rpc_py="$rootdir/scripts/rpc.py" 11 12t1sock='/var/tmp/spdk.sock1' 13t2sock='/var/tmp/spdk.sock2' 14invalid_port=8008 15t1dscport=8009 16t2dscport1=8010 17t2dscport2=8011 18t1nqn='nqn.2016-06.io.spdk:node1' 19t2nqn='nqn.2016-06.io.spdk:node2' 20hostnqn='nqn.2016-06.io.spdk:host0' 21cleanup_period=1 22 23function cleanup() { 24 killprocess $smapid 25 killprocess $tgtpid 26 killprocess $t1pid 27 killprocess $t2pid 28} 29 30function format_endpoints() { 31 local eps=("$@") 32 for ((i = 0; i < ${#eps[@]}; i++)); do 33 cat <<- EOF 34 { 35 "trtype": "tcp", 36 "traddr": "127.0.0.1", 37 "trsvcid": "${eps[i]}" 38 } 39 EOF 40 if ! ((i + 1 == ${#@})); then 41 echo , 42 fi 43 done 44} 45 46function format_volume() { 47 local volume_id=$1 48 shift 49 50 cat <<- EOF 51 "volume": { 52 "volume_id": "$(uuid2base64 $volume_id)", 53 "nvmf": { 54 "hostnqn": "$hostnqn", 55 "discovery": { 56 "discovery_endpoints": [ 57 $(format_endpoints "$@") 58 ] 59 } 60 } 61 } 62 EOF 63} 64 65function create_device() { 66 local nqn=$1 67 local volume_id=$2 68 local volume= 69 70 shift 71 if [[ -n "$volume_id" ]]; then 72 volume="$(format_volume "$@")," 73 fi 74 75 $sma_py <<- EOF 76 { 77 "method": "CreateDevice", 78 "params": { 79 $volume 80 "nvmf_tcp": { 81 "subnqn": "$nqn", 82 "adrfam": "ipv4", 83 "traddr": "127.0.0.1", 84 "trsvcid": "4419" 85 } 86 } 87 } 88 EOF 89} 90 91function delete_device() { 92 $sma_py <<- EOF 93 { 94 "method": "DeleteDevice", 95 "params": { 96 "handle": "$1" 97 } 98 } 99 EOF 100} 101 102function attach_volume() { 103 local device_id=$1 104 105 shift 106 $sma_py <<- EOF 107 { 108 "method": "AttachVolume", 109 "params": { 110 $(format_volume "$@"), 111 "device_handle": "$device_id" 112 } 113 } 114 EOF 115} 116 117function detach_volume() { 118 $sma_py <<- EOF 119 { 120 "method": "DetachVolume", 121 "params": { 122 "device_handle": "$1", 123 "volume_id": "$(uuid2base64 $2)" 124 } 125 } 126 EOF 127} 128 129trap "cleanup; exit 1" SIGINT SIGTERM EXIT 130 131# Start two remote targets 132$rootdir/build/bin/spdk_tgt -r $t1sock -m 0x1 & 133t1pid=$! 134$rootdir/build/bin/spdk_tgt -r $t2sock -m 0x2 & 135t2pid=$! 136 137# One target that the SMA will configure 138$rootdir/build/bin/spdk_tgt -m 0x4 & 139tgtpid=$! 140 141# And finally the SMA itself 142$rootdir/scripts/sma.py -c <( 143 cat <<- EOF 144 discovery_timeout: 5 145 volume_cleanup_period: $cleanup_period 146 devices: 147 - name: 'nvmf_tcp' 148 EOF 149) & 150smapid=$! 151 152waitforlisten $t1pid 153waitforlisten $t2pid 154 155# Prepare the targets. The first one has a single subsystem with a single volume and a single 156# discovery listener. The second one also has a single subsystem, but has two volumes attached to 157# it and has two discovery listeners. 158t1uuid=$(uuidgen) 159t2uuid=$(uuidgen) 160t2uuid2=$(uuidgen) 161 162$rpc_py -s $t1sock <<- EOF 163 nvmf_create_transport -t tcp 164 bdev_null_create null0 128 4096 -u $t1uuid 165 nvmf_create_subsystem $t1nqn -s SPDK00000000000001 -d SPDK_Controller1 166 nvmf_subsystem_add_host $t1nqn $hostnqn 167 nvmf_subsystem_add_ns $t1nqn $t1uuid 168 nvmf_subsystem_add_listener $t1nqn -t tcp -a 127.0.0.1 -s 4420 169 nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t1dscport 170EOF 171 172$rpc_py -s $t2sock <<- EOF 173 nvmf_create_transport -t tcp 174 bdev_null_create null0 128 4096 -u $t2uuid 175 bdev_null_create null1 128 4096 -u $t2uuid2 176 nvmf_create_subsystem $t2nqn -s SPDK00000000000001 -d SPDK_Controller1 177 nvmf_subsystem_add_host $t2nqn $hostnqn 178 nvmf_subsystem_add_ns $t2nqn $t2uuid 179 nvmf_subsystem_add_ns $t2nqn $t2uuid2 180 nvmf_subsystem_add_listener $t2nqn -t tcp -a 127.0.0.1 -s 4421 181 nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t2dscport1 182 nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t2dscport2 183EOF 184 185# Wait until the SMA starts listening 186sma_waitforlisten 187 188localnqn='nqn.2016-06.io.spdk:local0' 189 190# Create a device 191device_id=$(create_device $localnqn | jq -r '.handle') 192 193# Check that it's been created 194$rpc_py nvmf_get_subsystems $localnqn 195 196# Attach a volume specifying both targets 197attach_volume $device_id $t1uuid $t1dscport $t2dscport1 198 199# Check that a connection has been made to discovery services on both targets 200[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]] 201 202$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 203$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1 204 205# Check that the volume was attached to the device 206[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 207[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]] 208 209# Attach the other volume, this time specify only single target 210attach_volume $device_id $t2uuid $t2dscport1 211 212# Check that both volumes are attached to the device 213[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]] 214[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 2 ]] 215$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t1uuid 216$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid 217 218# Detach the first volume 219detach_volume $device_id $t1uuid 220 221# Check that there's a connection to a single target now (because we've only specified a single 222# target when connecting the other volume). 223[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 224$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1 225# Check that the volume was actually removed 226[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 227[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t2uuid" ]] 228 229# Detach the other volume 230detach_volume $device_id $t2uuid 231 232# And verify it's gone too 233[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 234[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]] 235 236# Check that specifying an invalid volume UUID results in an error 237NOT attach_volume $device_id $(uuidgen) $t1dscport 238[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 239[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]] 240 241# Attach them again, this time both volumes specify both targets 242volumes=($t1uuid $t2uuid) 243for volume_id in "${volumes[@]}"; do 244 attach_volume $device_id $volume_id $t1dscport $t2dscport1 245done 246 247[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]] 248$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 249$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1 250$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t1uuid 251$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid 252 253# Detach one and see that both targets are still connected 254detach_volume $device_id $t1uuid 255 256[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]] 257$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 258$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1 259 260# Delete the device and verify that this also causes the volumes to be disconnected 261delete_device $device_id 262 263[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 264NOT $rpc_py nvmf_get_subsystems $localnqn 265 266# Create a device and attach a volume immediately 267device_id=$(create_device $localnqn $t1uuid $t1dscport | jq -r '.handle') 268 269# Verify that there's a connection to the target and the volume is attached to the device 270[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 271$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 272[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 273[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]] 274 275# Make sure it's also possible to detach it 276detach_volume $device_id $t1uuid 277 278[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 279[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]] 280 281# Check that discovery referrals work correctly 282attach_volume $device_id $t2uuid $t2dscport1 $t2dscport2 283 284# Check that only a single connection has been made 285[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 286[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 287[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t2uuid" ]] 288 289# Add the other volume from the same target/subsystem, but use a single discovery endpoint 290attach_volume $device_id $t2uuid2 $t2dscport2 291 292# Check that the volume was attached to the subsystem, but no extra connection has been made 293[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 294[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 2 ]] 295$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid 296$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid2 297 298# Reset the device 299delete_device $device_id 300[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 301 302device_id=$(create_device $localnqn | jq -r '.handle') 303 304# Check subsystem NQN verification, start with a valid one 305$sma_py <<- EOF 306 { 307 "method": "AttachVolume", 308 "params": { 309 "volume": { 310 "volume_id": "$(uuid2base64 $t1uuid)", 311 "nvmf": { 312 "hostnqn": "$hostnqn", 313 "subnqn": "$t1nqn", 314 "discovery": { 315 "discovery_endpoints": [ 316 { 317 "trtype": "tcp", 318 "traddr": "127.0.0.1", 319 "trsvcid": "$t1dscport" 320 } 321 ] 322 } 323 } 324 }, 325 "device_handle": "$device_id" 326 } 327 } 328EOF 329 330[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 331$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 332[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 333[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]] 334 335# Then check incorrect subnqn 336NOT $sma_py <<- EOF 337 { 338 "method": "AttachVolume", 339 "params": { 340 "volume": { 341 "volume_id": "$(uuid2base64 $t2uuid)", 342 "nvmf": { 343 "hostnqn": "$hostnqn", 344 "subnqn": "${t2nqn}-invalid", 345 "discovery": { 346 "discovery_endpoints": [ 347 { 348 "trtype": "tcp", 349 "traddr": "127.0.0.1", 350 "trsvcid": "$t2dscport1" 351 } 352 ] 353 } 354 } 355 }, 356 "device_handle": "$device_id" 357 } 358 } 359EOF 360 361# Verify the volume hasn't been attached 362[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 363$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 364[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 365[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]] 366 367# Check incorrect hostnqn 368NOT $sma_py <<- EOF 369 { 370 "method": "AttachVolume", 371 "params": { 372 "volume": { 373 "volume_id": "$(uuid2base64 $t2uuid)", 374 "nvmf": { 375 "hostnqn": "${hostnqn}-invalid", 376 "discovery": { 377 "discovery_endpoints": [ 378 { 379 "trtype": "tcp", 380 "traddr": "127.0.0.1", 381 "trsvcid": "$t2dscport1" 382 } 383 ] 384 } 385 } 386 }, 387 "device_handle": "$device_id" 388 } 389 } 390EOF 391 392# Verify that the volume wasn't attached 393[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 394$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 395[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 396[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]] 397 398# Check that the the attach will fail if there's nobody listening on the discovery endpoint 399NOT attach_volume $device_id $(uuidgen) $invalid_port 400[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 401$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport 402 403# Make sure that the discovery service is stopped if a volume is disconnected outside of SMA (e.g. 404# by removing it from the target) 405$rpc_py -s $t1sock nvmf_subsystem_remove_ns $t1nqn 1 406# Give SMA some time to be notified about the change 407sleep $((cleanup_period + 1)) 408[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 409$rpc_py -s $t1sock nvmf_subsystem_add_ns $t1nqn $t1uuid 410 411# Do the same, but this time attach two volumes and check that the discovery service is only 412# stopped once both volumes are disconnected 413attach_volume $device_id $t2uuid $t2dscport1 414attach_volume $device_id $t2uuid2 $t2dscport1 415[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 2 ]] 416[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 417$rpc_py -s $t2sock nvmf_subsystem_remove_ns $t2nqn 2 418# Give SMA some time to be notified about the change 419sleep $((cleanup_period + 1)) 420# One of the volumes should be gone, but the discovery service should still be running 421[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]] 422[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]] 423$rpc_py -s $t2sock nvmf_subsystem_remove_ns $t2nqn 1 424# Give SMA some time to be notified about the change 425sleep $((cleanup_period + 1)) 426# Now that both are gone, the discovery service should be stopped too 427[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]] 428[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]] 429$rpc_py -s $t2sock nvmf_subsystem_add_ns $t2nqn $t2uuid 430$rpc_py -s $t2sock nvmf_subsystem_add_ns $t2nqn $t2uuid2 431 432delete_device $device_id 433 434cleanup 435trap - SIGINT SIGTERM EXIT 436