1#!/usr/bin/env bash 2 3rootdir=$(readlink -f $(dirname $0)/../..) 4source "$rootdir/test/common/autotest_common.sh" 5source "$rootdir/test/nvmf/common.sh" 6 7if [[ $SPDK_TEST_ISCSI -eq 1 ]]; then 8 source "$rootdir/test/iscsi_tgt/common.sh" 9fi 10 11if [[ $SPDK_TEST_VHOST -ne 1 && $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 12 SPDK_TEST_VHOST=1 13 echo "WARNING: Virtio initiator JSON_config test requires vhost target." 14 echo " Setting SPDK_TEST_VHOST=1 for duration of current script." 15fi 16 17if (( SPDK_TEST_BLOCKDEV + \ 18 SPDK_TEST_ISCSI + 19 SPDK_TEST_NVMF + 20 SPDK_TEST_VHOST + 21 SPDK_TEST_VHOST_INIT + 22 SPDK_TEST_PMDK + 23 SPDK_TEST_RBD == 0 )); then 24 echo "WARNING: No tests are enabled so not running JSON configuration tests" 25 exit 0 26fi 27 28declare -A app_pid=([target]= [initiator]=) 29declare -A app_socket=([target]='/var/tmp/spdk_tgt.sock' [initiator]='/var/tmp/spdk_initiator.sock') 30declare -A app_params=([target]='-m 0x1 -s 1024' [initiator]='-m 0x2 -g -u -s 1024') 31declare -A configs_path=([target]="$rootdir/spdk_tgt_config.json" [initiator]="$rootdir/spdk_initiator_config.json") 32 33function tgt_rpc() { 34 $rootdir/scripts/rpc.py -s "${app_socket[target]}" "$@" 35} 36 37function initiator_rpc() { 38 $rootdir/scripts/rpc.py -s "${app_socket[initiator]}" "$@" 39} 40 41RE_UUID="[[:alnum:]-]+" 42last_event_id=0 43 44function tgt_check_notification_types() { 45 timing_enter $FUNCNAME 46 47 local ret=0 48 local enabled_types=" 49 bdev_register 50 bdev_unregister 51 " 52 53 get_types=$(tgt_rpc get_notification_types | jq -r '.[]') 54 if [ "$(echo $enabled_types)" != "$(echo $get_types)" ]; then 55 echo "ERROR: expected types:" $enabled_types ", but got:" $get_types 56 ret=1 57 fi 58 59 timing_exit $FUNCNAME 60 return $ret 61} 62 63function tgt_check_notifications() { 64 local event_line event ev_type ev_ctx 65 local rc="" 66 67 while read event_line; do 68 # remove ID 69 event="${event_line%:*}" 70 71 ev_type=${event%:*} 72 ev_ctx=${event#*:} 73 74 ex_ev_type=${1%%:*} 75 ex_ev_ctx=${1#*:} 76 77 last_event_id=${event_line##*:} 78 79 # set rc=false in case of failure so all errors can be printed 80 if (( $# == 0 )); then 81 echo "ERROR: got extra event: $event_line" 82 rc=false 83 continue 84 elif ! echo "$ev_type" | grep -E -q "^${ex_ev_type}\$" || ! echo "$ev_ctx" | grep -E -q "^${ex_ev_ctx}\$"; then 85 echo "ERROR: expected event '$1' but got '$event' (whole event line: $event_line)" 86 rc=false 87 fi 88 89 shift 90 done < <(tgt_rpc get_notifications -i ${last_event_id} | jq -r '.[] | "\(.type):\(.ctx):\(.id)"') 91 92 $rc 93 94 if (( $# != 0 )); then 95 echo "ERROR: missing events:" 96 echo "$@" 97 return 1 98 fi 99} 100 101# $1 - target / initiator 102# $2..$n app parameters 103function json_config_test_start_app() { 104 local app=$1 105 shift 106 107 [[ -n "${#app_socket[$app]}" ]] # Check app type 108 [[ -z "${app_pid[$app]}" ]] # Assert if app is not running 109 110 local app_extra_params="" 111 if [[ $SPDK_TEST_VHOST -eq 1 || $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 112 # If PWD is nfs/sshfs we can't create UNIX sockets there. Always use safe location instead. 113 app_extra_params='-S /var/tmp' 114 fi 115 116 $rootdir/app/spdk_tgt/spdk_tgt ${app_params[$app]} ${app_extra_params} -r ${app_socket[$app]} "$@" & 117 app_pid[$app]=$! 118 119 echo "Waiting for $app to run..." 120 waitforlisten ${app_pid[$app]} ${app_socket[$app]} 121 echo "" 122} 123 124# $1 - target / initiator 125function json_config_test_shutdown_app() { 126 local app=$1 127 128 # Check app type && assert app was started 129 [[ -n "${#app_socket[$app]}" ]] 130 [[ -n "${app_pid[$app]}" ]] 131 132 # kill_instance RPC will trigger ASAN 133 kill -SIGINT ${app_pid[$app]} 134 135 for (( i=0; i<30; i++ )); do 136 if ! kill -0 ${app_pid[$app]} 2>/dev/null; then 137 app_pid[$app]= 138 break 139 fi 140 sleep 0.5 141 done 142 143 if [[ -n "${app_pid[$app]}" ]]; then 144 echo "SPDK $app shutdown timeout" 145 return 1 146 fi 147 148 echo "SPDK $app shutdown done" 149} 150 151function create_bdev_subsystem_config() { 152 timing_enter $FUNCNAME 153 154 local expected_notifications=() 155 156 if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then 157 local lvol_store_base_bdev=Nvme0n1 158 if ! tgt_rpc get_bdevs --name ${lvol_store_base_bdev} >/dev/null; then 159 if [[ $(uname -s) = Linux ]]; then 160 lvol_store_base_bdev=aio_disk 161 echo "WARNING: No NVMe drive found. Using '$lvol_store_base_bdev' instead." 162 else 163 echo "ERROR: No NVMe drive found and bdev_aio is not supported on $(uname -s)." 164 timing_exit $FUNCNAME 165 return 1 166 fi 167 fi 168 169 tgt_rpc construct_split_vbdev $lvol_store_base_bdev 2 170 tgt_rpc construct_split_vbdev Malloc0 3 171 tgt_rpc bdev_malloc_create 8 4096 --name Malloc3 172 tgt_rpc construct_passthru_bdev -b Malloc3 -p PTBdevFromMalloc3 173 174 tgt_rpc construct_null_bdev Null0 32 512 175 176 tgt_rpc bdev_malloc_create 32 512 --name Malloc0 177 tgt_rpc bdev_malloc_create 16 4096 --name Malloc1 178 179 expected_notifications+=( 180 bdev_register:${lvol_store_base_bdev} 181 bdev_register:${lvol_store_base_bdev}p0 182 bdev_register:${lvol_store_base_bdev}p1 183 bdev_register:Malloc3 184 bdev_register:PTBdevFromMalloc3 185 bdev_register:Null0 186 bdev_register:Malloc0p0 187 bdev_register:Malloc0p1 188 bdev_register:Malloc0p2 189 bdev_register:Malloc0 190 bdev_register:Malloc1 191 ) 192 193 if [[ $(uname -s) = Linux ]]; then 194 # This AIO bdev must be large enough to be used as LVOL store 195 dd if=/dev/zero of=/tmp/sample_aio bs=1024 count=102400 196 tgt_rpc bdev_aio_create /tmp/sample_aio aio_disk 1024 197 expected_notifications+=( bdev_register:aio_disk ) 198 fi 199 200 # For LVOLs use split to check for proper order of initialization. 201 # If LVOLs cofniguration will be reordered (eg moved before splits or AIO/NVMe) 202 # it should fail loading JSON config from file. 203 tgt_rpc construct_lvol_store -c 1048576 ${lvol_store_base_bdev}p0 lvs_test 204 tgt_rpc construct_lvol_bdev -l lvs_test lvol0 32 205 tgt_rpc construct_lvol_bdev -l lvs_test -t lvol1 32 206 tgt_rpc snapshot_lvol_bdev lvs_test/lvol0 snapshot0 207 tgt_rpc bdev_lvol_clone lvs_test/snapshot0 clone0 208 209 expected_notifications+=( 210 "bdev_register:$RE_UUID" 211 "bdev_register:$RE_UUID" 212 "bdev_register:$RE_UUID" 213 "bdev_register:$RE_UUID" 214 ) 215 fi 216 217 if [[ $SPDK_TEST_CRYPTO -eq 1 ]]; then 218 tgt_rpc bdev_malloc_create 8 1024 --name MallocForCryptoBdev 219 if [[ $(lspci -d:37c8 | wc -l) -eq 0 ]]; then 220 local crypto_dirver=crypto_aesni_mb 221 else 222 local crypto_dirver=crypto_qat 223 fi 224 225 tgt_rpc bdev_crypto_create MallocForCryptoBdev CryptoMallocBdev $crypto_dirver 0123456789123456 226 expected_notifications+=( 227 bdev_register:MallocForCryptoBdev 228 bdev_register:CryptoMallocBdev 229 ) 230 fi 231 232 if [[ $SPDK_TEST_PMDK -eq 1 ]]; then 233 pmem_pool_file=$(mktemp /tmp/pool_file1.XXXXX) 234 rm -f $pmem_pool_file 235 tgt_rpc create_pmem_pool $pmem_pool_file 128 4096 236 tgt_rpc construct_pmem_bdev -n pmem1 $pmem_pool_file 237 expected_notifications+=( bdev_register:pmem1 ) 238 fi 239 240 if [[ $SPDK_TEST_RBD -eq 1 ]]; then 241 rbd_setup 127.0.0.1 242 tgt_rpc construct_rbd_bdev $RBD_POOL $RBD_NAME 4096 243 expected_notifications+=( bdev_register:Ceph0 ) 244 fi 245 246 tgt_check_notifications "${expected_notifications[@]}" 247 248 timing_exit $FUNCNAME 249} 250 251function cleanup_bdev_subsystem_config() { 252 timing_enter $FUNCNAME 253 254 if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then 255 tgt_rpc destroy_lvol_bdev lvs_test/clone0 256 tgt_rpc destroy_lvol_bdev lvs_test/lvol0 257 tgt_rpc destroy_lvol_bdev lvs_test/snapshot0 258 tgt_rpc destroy_lvol_store -l lvs_test 259 fi 260 261 if [[ $(uname -s) = Linux ]]; then 262 rm -f /tmp/sample_aio 263 fi 264 265 if [[ $SPDK_TEST_PMDK -eq 1 && -n "$pmem_pool_file" && -f "$pmem_pool_file" ]]; then 266 tgt_rpc delete_pmem_bdev pmem1 267 tgt_rpc delete_pmem_pool $pmem_pool_file 268 rm -f $pmem_pool_file 269 fi 270 271 if [[ $SPDK_TEST_RBD -eq 1 ]]; then 272 rbd_cleanup 273 fi 274 275 timing_exit $FUNCNAME 276} 277 278function create_vhost_subsystem_config() { 279 timing_enter $FUNCNAME 280 281 tgt_rpc bdev_malloc_create 64 1024 --name MallocForVhost0 282 tgt_rpc construct_split_vbdev MallocForVhost0 8 283 284 tgt_rpc construct_vhost_scsi_controller VhostScsiCtrlr0 285 tgt_rpc add_vhost_scsi_lun VhostScsiCtrlr0 0 MallocForVhost0p3 286 tgt_rpc add_vhost_scsi_lun VhostScsiCtrlr0 -1 MallocForVhost0p4 287 tgt_rpc set_vhost_controller_coalescing VhostScsiCtrlr0 1 100 288 289 tgt_rpc construct_vhost_blk_controller VhostBlkCtrlr0 MallocForVhost0p5 290 291# FIXME: enable after vhost-nvme is properly implemented against the latest rte_vhost (DPDK 19.05+) 292# tgt_rpc construct_vhost_nvme_controller VhostNvmeCtrlr0 16 293# tgt_rpc add_vhost_nvme_ns VhostNvmeCtrlr0 MallocForVhost0p6 294 295 timing_exit $FUNCNAME 296} 297 298function create_iscsi_subsystem_config() { 299 timing_enter $FUNCNAME 300 tgt_rpc bdev_malloc_create 64 1024 --name MallocForIscsi0 301 tgt_rpc add_portal_group $PORTAL_TAG 127.0.0.1:$ISCSI_PORT 302 tgt_rpc add_initiator_group $INITIATOR_TAG $INITIATOR_NAME $NETMASK 303 tgt_rpc construct_target_node Target3 Target3_alias 'MallocForIscsi0:0' $PORTAL_TAG:$INITIATOR_TAG 64 -d 304 timing_exit $FUNCNAME 305} 306 307function create_nvmf_subsystem_config() { 308 timing_enter $FUNCNAME 309 310 RDMA_IP_LIST=$(get_available_rdma_ips) 311 NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1) 312 if [[ -z $NVMF_FIRST_TARGET_IP ]]; then 313 echo "Error: no NIC for nvmf test" 314 return 1 315 fi 316 317 tgt_rpc bdev_malloc_create 8 512 --name MallocForNvmf0 318 tgt_rpc bdev_malloc_create 4 1024 --name MallocForNvmf1 319 320 tgt_rpc nvmf_create_transport -t RDMA -u 8192 -c 0 321 tgt_rpc nvmf_subsystem_create nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 322 tgt_rpc nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 MallocForNvmf0 323 tgt_rpc nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 MallocForNvmf1 324 tgt_rpc nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t RDMA -a $NVMF_FIRST_TARGET_IP -s "$NVMF_PORT" 325 326 timing_exit $FUNCNAME 327} 328 329function create_virtio_initiator_config() { 330 timing_enter $FUNCNAME 331 initiator_rpc construct_virtio_dev -t user -a /var/tmp/VhostScsiCtrlr0 -d scsi VirtioScsiCtrlr0 332 initiator_rpc construct_virtio_dev -t user -a /var/tmp/VhostBlkCtrlr0 -d blk VirtioBlk0 333 # TODO: initiator_rpc construct_virtio_dev -t user -a /var/tmp/VhostNvmeCtrlr0 -d nvme VirtioNvme0 334 timing_exit $FUNCNAME 335} 336 337 338function json_config_test_init() 339{ 340 timing_enter $FUNCNAME 341 timing_enter json_config_setup_target 342 343 json_config_test_start_app target --wait-for-rpc 344 345 #TODO: global subsystem params 346 347 # Load nvme configuration. The load_config will issue start_subsystem_init automatically 348 ( 349 echo '{"subsystems": ['; 350 $rootdir/scripts/gen_nvme.sh --json 351 echo ']}' 352 ) | tgt_rpc load_config 353 354 tgt_check_notification_types 355 356 if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then 357 create_bdev_subsystem_config 358 fi 359 360 if [[ $SPDK_TEST_VHOST -eq 1 ]]; then 361 create_vhost_subsystem_config 362 fi 363 364 if [[ $SPDK_TEST_ISCSI -eq 1 ]]; then 365 create_iscsi_subsystem_config 366 fi 367 368 if [[ $SPDK_TEST_NVMF -eq 1 ]]; then 369 create_nvmf_subsystem_config 370 fi 371 timing_exit json_config_setup_target 372 373 if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 374 json_config_test_start_app initiator 375 create_virtio_initiator_config 376 fi 377 378 tgt_rpc bdev_malloc_create 8 512 --name MallocBdevForConfigChangeCheck 379 380 timing_exit $FUNCNAME 381} 382 383function json_config_test_fini() { 384 timing_enter $FUNCNAME 385 local ret=0 386 387 if [[ -n "${app_pid[initiator]}" ]]; then 388 if ! json_config_test_shutdown_app initiator; then 389 kill -9 ${app_pid[initiator]} 390 app_pid[initiator]= 391 ret=1 392 fi 393 fi 394 395 if [[ -n "${app_pid[target]}" ]]; then 396 397 # Remove any artifacts we created (files, lvol etc) 398 cleanup_bdev_subsystem_config 399 400 # SPDK_TEST_NVMF: Should we clear something? 401 402 if ! json_config_test_shutdown_app target; then 403 kill -9 ${app_pid[target]} 404 app_pid[target]= 405 ret=1 406 fi 407 fi 408 409 rm -f "${configs_path[@]}" 410 timing_exit $FUNCNAME 411 return $ret 412} 413 414function json_config_clear() { 415 [[ -n "${#app_socket[$1]}" ]] # Check app type 416 $rootdir/test/json_config/clear_config.py -s ${app_socket[$1]} clear_config 417 418 # Check if config is clean. 419 # Global params can't be cleared so need to filter them out. 420 local config_filter="$rootdir/test/json_config/config_filter.py" 421 422 # RPC's used to cleanup configuration (e.g. to delete split and nvme bdevs) 423 # complete immediately and they don't wait for the unregister callback. 424 # It causes that configuration may not be fully cleaned at this moment and 425 # we should to wait a while. (See github issue #789) 426 count=100 427 while [ $count -gt 0 ] ; do 428 $rootdir/scripts/rpc.py -s "${app_socket[$1]}" save_config | $config_filter -method delete_global_parameters | $config_filter -method check_empty && break 429 count=$(( $count -1 )) 430 sleep 0.1 431 done 432 433 if [ $count -eq 0 ] ; then 434 return 1 435 fi 436} 437 438on_error_exit() { 439 set -x 440 set +e 441 print_backtrace 442 trap - ERR 443 echo "Error on $1 - $2" 444 json_config_test_fini 445 exit 1 446} 447 448trap 'on_error_exit "${FUNCNAME}" "${LINENO}"' ERR 449echo "INFO: JSON configuration test init" 450json_config_test_init 451 452tgt_rpc save_config > ${configs_path[target]} 453 454echo "INFO: shutting down applications..." 455if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 456 initiator_rpc save_config > ${configs_path[initiator]} 457 json_config_clear initiator 458 json_config_test_shutdown_app initiator 459fi 460 461json_config_clear target 462json_config_test_shutdown_app target 463 464echo "INFO: relaunching applications..." 465json_config_test_start_app target --json ${configs_path[target]} 466if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 467 json_config_test_start_app initiator --json ${configs_path[initiator]} 468fi 469 470echo "INFO: Checking if target configuration is the same..." 471$rootdir/test/json_config/json_diff.sh <(tgt_rpc save_config) "${configs_path[target]}" 472if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 473 echo "INFO: Checking if virtio initiator configuration is the same..." 474 $rootdir/test/json_config/json_diff.sh <(initiator_rpc save_config) "${configs_path[initiator]}" 475fi 476 477echo "INFO: changing configuration and checking if this can be detected..." 478# Self test to check if configuration diff can be detected. 479tgt_rpc delete_malloc_bdev MallocBdevForConfigChangeCheck 480if $rootdir/test/json_config/json_diff.sh <(tgt_rpc save_config) "${configs_path[target]}" >/dev/null; then 481 echo "ERROR: intentional configuration difference not detected!" 482 false 483else 484 echo "INFO: configuration change detected." 485fi 486 487json_config_test_fini 488 489echo "INFO: Success" 490