1#!/usr/bin/env bash 2 3rootdir=$(readlink -f $(dirname $0)/../..) 4source "$rootdir/test/common/autotest_common.sh" 5source "$rootdir/test/nvmf/common.sh" 6 7if [[ $SPDK_TEST_ISCSI -eq 1 ]]; then 8 source "$rootdir/test/iscsi_tgt/common.sh" 9fi 10 11if [[ $SPDK_TEST_VHOST -ne 1 && $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 12 SPDK_TEST_VHOST=1 13 echo "WARNING: Virtio initiator JSON_config test requires vhost target." 14 echo " Setting SPDK_TEST_VHOST=1 for duration of current script." 15fi 16 17if ((SPDK_TEST_BLOCKDEV + \ 18 SPDK_TEST_ISCSI + \ 19 SPDK_TEST_NVMF + \ 20 SPDK_TEST_VHOST + \ 21 SPDK_TEST_VHOST_INIT + \ 22 SPDK_TEST_PMDK + \ 23 SPDK_TEST_RBD == 0)); then 24 echo "WARNING: No tests are enabled so not running JSON configuration tests" 25 exit 0 26fi 27 28declare -A app_pid=([target]="" [initiator]="") 29declare -A app_socket=([target]='/var/tmp/spdk_tgt.sock' [initiator]='/var/tmp/spdk_initiator.sock') 30declare -A app_params=([target]='-m 0x1 -s 1024' [initiator]='-m 0x2 -g -u -s 1024') 31declare -A configs_path=([target]="$rootdir/spdk_tgt_config.json" [initiator]="$rootdir/spdk_initiator_config.json") 32 33function tgt_rpc() { 34 $rootdir/scripts/rpc.py -s "${app_socket[target]}" "$@" 35} 36 37function initiator_rpc() { 38 $rootdir/scripts/rpc.py -s "${app_socket[initiator]}" "$@" 39} 40 41RE_UUID="[[:alnum:]-]+" 42last_event_id=0 43 44function tgt_check_notification_types() { 45 timing_enter "${FUNCNAME[0]}" 46 47 local ret=0 48 local enabled_types=("bdev_register" "bdev_unregister") 49 50 local get_types=($(tgt_rpc notify_get_types | jq -r '.[]')) 51 if [[ ${enabled_types[*]} != "${get_types[*]}" ]]; then 52 echo "ERROR: expected types: ${enabled_types[*]}, but got: ${get_types[*]}" 53 ret=1 54 fi 55 56 timing_exit "${FUNCNAME[0]}" 57 return $ret 58} 59 60function tgt_check_notifications() { 61 local event_line event ev_type ev_ctx 62 local rc="" 63 64 while read -r event_line; do 65 # remove ID 66 event="${event_line%:*}" 67 68 ev_type=${event%:*} 69 ev_ctx=${event#*:} 70 71 ex_ev_type=${1%%:*} 72 ex_ev_ctx=${1#*:} 73 74 last_event_id=${event_line##*:} 75 76 # set rc=false in case of failure so all errors can be printed 77 if (($# == 0)); then 78 echo "ERROR: got extra event: $event_line" 79 rc=false 80 continue 81 elif ! echo "$ev_type" | grep -E -q "^${ex_ev_type}\$" || ! echo "$ev_ctx" | grep -E -q "^${ex_ev_ctx}\$"; then 82 echo "ERROR: expected event '$1' but got '$event' (whole event line: $event_line)" 83 rc=false 84 fi 85 86 shift 87 done < <(tgt_rpc notify_get_notifications -i ${last_event_id} | jq -r '.[] | "\(.type):\(.ctx):\(.id)"') 88 89 $rc 90 91 if (($# != 0)); then 92 echo "ERROR: missing events:" 93 echo "$@" 94 return 1 95 fi 96} 97 98# $1 - target / initiator 99# $2..$n app parameters 100function json_config_test_start_app() { 101 local app=$1 102 shift 103 104 [[ -n "${#app_socket[$app]}" ]] # Check app type 105 [[ -z "${app_pid[$app]}" ]] # Assert if app is not running 106 107 local app_extra_params="" 108 if [[ $SPDK_TEST_VHOST -eq 1 || $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 109 # If PWD is nfs/sshfs we can't create UNIX sockets there. Always use safe location instead. 110 app_extra_params='-S /var/tmp' 111 fi 112 113 $SPDK_BIN_DIR/spdk_tgt ${app_params[$app]} ${app_extra_params} -r ${app_socket[$app]} "$@" & 114 app_pid[$app]=$! 115 116 echo "Waiting for $app to run..." 117 waitforlisten ${app_pid[$app]} ${app_socket[$app]} 118 echo "" 119} 120 121# $1 - target / initiator 122function json_config_test_shutdown_app() { 123 local app=$1 124 125 # Check app type && assert app was started 126 [[ -n "${#app_socket[$app]}" ]] 127 [[ -n "${app_pid[$app]}" ]] 128 129 # spdk_kill_instance RPC will trigger ASAN 130 kill -SIGINT ${app_pid[$app]} 131 132 for ((i = 0; i < 30; i++)); do 133 if ! kill -0 ${app_pid[$app]} 2> /dev/null; then 134 app_pid[$app]= 135 break 136 fi 137 sleep 0.5 138 done 139 140 if [[ -n "${app_pid[$app]}" ]]; then 141 echo "SPDK $app shutdown timeout" 142 return 1 143 fi 144 145 echo "SPDK $app shutdown done" 146} 147 148function create_bdev_subsystem_config() { 149 timing_enter "${FUNCNAME[0]}" 150 151 local expected_notifications=() 152 153 if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then 154 local lvol_store_base_bdev=Nvme0n1 155 if ! tgt_rpc get_bdevs --name ${lvol_store_base_bdev} > /dev/null; then 156 if [[ $(uname -s) = Linux ]]; then 157 lvol_store_base_bdev=aio_disk 158 echo "WARNING: No NVMe drive found. Using '$lvol_store_base_bdev' instead." 159 else 160 echo "ERROR: No NVMe drive found and bdev_aio is not supported on $(uname -s)." 161 timing_exit "${FUNCNAME[0]}" 162 return 1 163 fi 164 fi 165 166 tgt_rpc bdev_split_create $lvol_store_base_bdev 2 167 tgt_rpc bdev_split_create Malloc0 3 168 tgt_rpc bdev_malloc_create 8 4096 --name Malloc3 169 tgt_rpc bdev_passthru_create -b Malloc3 -p PTBdevFromMalloc3 170 171 tgt_rpc bdev_null_create Null0 32 512 172 173 tgt_rpc bdev_malloc_create 32 512 --name Malloc0 174 tgt_rpc bdev_malloc_create 16 4096 --name Malloc1 175 176 expected_notifications+=( 177 bdev_register:${lvol_store_base_bdev} 178 bdev_register:${lvol_store_base_bdev}p0 179 bdev_register:${lvol_store_base_bdev}p1 180 bdev_register:Malloc3 181 bdev_register:PTBdevFromMalloc3 182 bdev_register:Null0 183 bdev_register:Malloc0p0 184 bdev_register:Malloc0p1 185 bdev_register:Malloc0p2 186 bdev_register:Malloc0 187 bdev_register:Malloc1 188 ) 189 190 if [[ $(uname -s) = Linux ]]; then 191 # This AIO bdev must be large enough to be used as LVOL store 192 dd if=/dev/zero of="$SPDK_TEST_STORAGE/sample_aio" bs=1024 count=102400 193 tgt_rpc bdev_aio_create "$SPDK_TEST_STORAGE/sample_aio" aio_disk 1024 194 expected_notifications+=(bdev_register:aio_disk) 195 fi 196 197 # For LVOLs use split to check for proper order of initialization. 198 # If LVOLs cofniguration will be reordered (eg moved before splits or AIO/NVMe) 199 # it should fail loading JSON config from file. 200 tgt_rpc bdev_lvol_create_lvstore -c 1048576 ${lvol_store_base_bdev}p0 lvs_test 201 tgt_rpc bdev_lvol_create -l lvs_test lvol0 32 202 tgt_rpc bdev_lvol_create -l lvs_test -t lvol1 32 203 tgt_rpc bdev_lvol_snapshot lvs_test/lvol0 snapshot0 204 tgt_rpc bdev_lvol_clone lvs_test/snapshot0 clone0 205 206 expected_notifications+=( 207 "bdev_register:$RE_UUID" 208 "bdev_register:$RE_UUID" 209 "bdev_register:$RE_UUID" 210 "bdev_register:$RE_UUID" 211 ) 212 fi 213 214 if [[ $SPDK_TEST_CRYPTO -eq 1 ]]; then 215 tgt_rpc bdev_malloc_create 8 1024 --name MallocForCryptoBdev 216 if [[ $(lspci -d:37c8 | wc -l) -eq 0 ]]; then 217 local crypto_dirver=crypto_aesni_mb 218 else 219 local crypto_dirver=crypto_qat 220 fi 221 222 tgt_rpc bdev_crypto_create MallocForCryptoBdev CryptoMallocBdev $crypto_dirver 0123456789123456 223 expected_notifications+=( 224 bdev_register:MallocForCryptoBdev 225 bdev_register:CryptoMallocBdev 226 ) 227 fi 228 229 if [[ $SPDK_TEST_PMDK -eq 1 ]]; then 230 pmem_pool_file=$(mktemp /tmp/pool_file1.XXXXX) 231 rm -f $pmem_pool_file 232 tgt_rpc create_pmem_pool $pmem_pool_file 128 4096 233 tgt_rpc bdev_pmem_create -n pmem1 $pmem_pool_file 234 expected_notifications+=(bdev_register:pmem1) 235 fi 236 237 if [[ $SPDK_TEST_RBD -eq 1 ]]; then 238 rbd_setup 127.0.0.1 239 tgt_rpc bdev_rbd_create $RBD_POOL $RBD_NAME 4096 240 expected_notifications+=(bdev_register:Ceph0) 241 fi 242 243 tgt_check_notifications "${expected_notifications[@]}" 244 245 timing_exit "${FUNCNAME[0]}" 246} 247 248function cleanup_bdev_subsystem_config() { 249 timing_enter "${FUNCNAME[0]}" 250 251 if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then 252 tgt_rpc bdev_lvol_delete lvs_test/clone0 253 tgt_rpc bdev_lvol_delete lvs_test/lvol0 254 tgt_rpc bdev_lvol_delete lvs_test/snapshot0 255 tgt_rpc bdev_lvol_delete_lvstore -l lvs_test 256 fi 257 258 if [[ $(uname -s) = Linux ]]; then 259 rm -f "$SPDK_TEST_STORAGE/sample_aio" 260 fi 261 262 if [[ $SPDK_TEST_PMDK -eq 1 && -n "$pmem_pool_file" && -f "$pmem_pool_file" ]]; then 263 tgt_rpc bdev_pmem_delete pmem1 264 tgt_rpc bdev_pmem_delete_pool $pmem_pool_file 265 rm -f $pmem_pool_file 266 fi 267 268 if [[ $SPDK_TEST_RBD -eq 1 ]]; then 269 rbd_cleanup 270 fi 271 272 timing_exit "${FUNCNAME[0]}" 273} 274 275function create_vhost_subsystem_config() { 276 timing_enter "${FUNCNAME[0]}" 277 278 tgt_rpc bdev_malloc_create 64 1024 --name MallocForVhost0 279 tgt_rpc bdev_split_create MallocForVhost0 8 280 281 tgt_rpc vhost_create_scsi_controller VhostScsiCtrlr0 282 tgt_rpc vhost_scsi_controller_add_target VhostScsiCtrlr0 0 MallocForVhost0p3 283 tgt_rpc vhost_scsi_controller_add_target VhostScsiCtrlr0 -1 MallocForVhost0p4 284 tgt_rpc vhost_controller_set_coalescing VhostScsiCtrlr0 1 100 285 286 tgt_rpc vhost_create_blk_controller VhostBlkCtrlr0 MallocForVhost0p5 287 288 # FIXME: enable after vhost-nvme is properly implemented against the latest rte_vhost (DPDK 19.05+) 289 # tgt_rpc vhost_create_nvme_controller VhostNvmeCtrlr0 16 290 # tgt_rpc vhost_nvme_controller_add_ns VhostNvmeCtrlr0 MallocForVhost0p6 291 292 timing_exit "${FUNCNAME[0]}" 293} 294 295function create_iscsi_subsystem_config() { 296 timing_enter "${FUNCNAME[0]}" 297 tgt_rpc bdev_malloc_create 64 1024 --name MallocForIscsi0 298 tgt_rpc iscsi_create_portal_group $PORTAL_TAG 127.0.0.1:$ISCSI_PORT 299 tgt_rpc iscsi_create_initiator_group $INITIATOR_TAG $INITIATOR_NAME $NETMASK 300 tgt_rpc iscsi_create_target_node Target3 Target3_alias 'MallocForIscsi0:0' $PORTAL_TAG:$INITIATOR_TAG 64 -d 301 timing_exit "${FUNCNAME[0]}" 302} 303 304function create_nvmf_subsystem_config() { 305 timing_enter "${FUNCNAME[0]}" 306 307 RDMA_IP_LIST=$(get_available_rdma_ips) 308 NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1) 309 if [[ -z $NVMF_FIRST_TARGET_IP ]]; then 310 echo "Error: no NIC for nvmf test" 311 return 1 312 fi 313 314 tgt_rpc bdev_malloc_create 8 512 --name MallocForNvmf0 315 tgt_rpc bdev_malloc_create 4 1024 --name MallocForNvmf1 316 317 tgt_rpc nvmf_create_transport -t RDMA -u 8192 -c 0 318 tgt_rpc nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 319 tgt_rpc nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 MallocForNvmf0 320 tgt_rpc nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 MallocForNvmf1 321 tgt_rpc nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t RDMA -a $NVMF_FIRST_TARGET_IP -s "$NVMF_PORT" 322 323 timing_exit "${FUNCNAME[0]}" 324} 325 326function create_virtio_initiator_config() { 327 timing_enter "${FUNCNAME[0]}" 328 initiator_rpc bdev_virtio_attach_controller -t user -a /var/tmp/VhostScsiCtrlr0 -d scsi VirtioScsiCtrlr0 329 # FIXME: Specifying --vq-count is workaround issue #1583. 330 initiator_rpc bdev_virtio_attach_controller -t user -a /var/tmp/VhostBlkCtrlr0 -d blk --vq-count 2 VirtioBlk0 331 # TODO: initiator_rpc bdev_virtio_attach_controller -t user -a /var/tmp/VhostNvmeCtrlr0 -d nvme VirtioNvme0 332 timing_exit "${FUNCNAME[0]}" 333} 334 335function json_config_test_init() { 336 timing_enter "${FUNCNAME[0]}" 337 timing_enter json_config_setup_target 338 339 json_config_test_start_app target --wait-for-rpc 340 341 #TODO: global subsystem params 342 343 # Load nvme configuration. The load_config will issue framework_start_init automatically 344 ( 345 echo '{"subsystems": [' 346 $rootdir/scripts/gen_nvme.sh --json | jq -r "del(.config[] | select(.params.name!=\"Nvme0\"))" 347 echo ']}' 348 ) | tgt_rpc load_config 349 350 tgt_check_notification_types 351 352 if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then 353 create_bdev_subsystem_config 354 fi 355 356 if [[ $SPDK_TEST_VHOST -eq 1 ]]; then 357 create_vhost_subsystem_config 358 fi 359 360 if [[ $SPDK_TEST_ISCSI -eq 1 ]]; then 361 create_iscsi_subsystem_config 362 fi 363 364 if [[ $SPDK_TEST_NVMF -eq 1 ]]; then 365 create_nvmf_subsystem_config 366 fi 367 timing_exit json_config_setup_target 368 369 if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 370 json_config_test_start_app initiator 371 create_virtio_initiator_config 372 fi 373 374 tgt_rpc bdev_malloc_create 8 512 --name MallocBdevForConfigChangeCheck 375 376 timing_exit "${FUNCNAME[0]}" 377} 378 379function json_config_test_fini() { 380 timing_enter "${FUNCNAME[0]}" 381 local ret=0 382 383 if [[ -n "${app_pid[initiator]}" ]]; then 384 killprocess ${app_pid[initiator]} 385 fi 386 387 if [[ -n "${app_pid[target]}" ]]; then 388 389 # Remove any artifacts we created (files, lvol etc) 390 cleanup_bdev_subsystem_config 391 392 # SPDK_TEST_NVMF: Should we clear something? 393 killprocess ${app_pid[target]} 394 fi 395 396 rm -f "${configs_path[@]}" 397 timing_exit "${FUNCNAME[0]}" 398 return $ret 399} 400 401function json_config_clear() { 402 [[ -n "${#app_socket[$1]}" ]] # Check app type 403 $rootdir/test/json_config/clear_config.py -s ${app_socket[$1]} clear_config 404 405 # Check if config is clean. 406 # Global params can't be cleared so need to filter them out. 407 local config_filter="$rootdir/test/json_config/config_filter.py" 408 409 # RPC's used to cleanup configuration (e.g. to delete split and nvme bdevs) 410 # complete immediately and they don't wait for the unregister callback. 411 # It causes that configuration may not be fully cleaned at this moment and 412 # we should to wait a while. (See github issue #789) 413 count=100 414 while [ $count -gt 0 ]; do 415 $rootdir/scripts/rpc.py -s "${app_socket[$1]}" save_config | $config_filter -method delete_global_parameters | $config_filter -method check_empty && break 416 count=$((count - 1)) 417 sleep 0.1 418 done 419 420 if [ $count -eq 0 ]; then 421 return 1 422 fi 423} 424 425on_error_exit() { 426 set -x 427 set +e 428 print_backtrace 429 trap - ERR 430 echo "Error on $1 - $2" 431 json_config_test_fini 432 exit 1 433} 434 435trap 'on_error_exit "${FUNCNAME}" "${LINENO}"' ERR 436echo "INFO: JSON configuration test init" 437json_config_test_init 438 439tgt_rpc save_config > ${configs_path[target]} 440 441echo "INFO: shutting down applications..." 442if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 443 initiator_rpc save_config > ${configs_path[initiator]} 444 json_config_clear initiator 445 json_config_test_shutdown_app initiator 446fi 447 448json_config_clear target 449json_config_test_shutdown_app target 450 451echo "INFO: relaunching applications..." 452json_config_test_start_app target --json ${configs_path[target]} 453if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 454 json_config_test_start_app initiator --json ${configs_path[initiator]} 455fi 456 457echo "INFO: Checking if target configuration is the same..." 458$rootdir/test/json_config/json_diff.sh <(tgt_rpc save_config) "${configs_path[target]}" 459if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 460 echo "INFO: Checking if virtio initiator configuration is the same..." 461 $rootdir/test/json_config/json_diff.sh <(initiator_rpc save_config) "${configs_path[initiator]}" 462fi 463 464echo "INFO: changing configuration and checking if this can be detected..." 465# Self test to check if configuration diff can be detected. 466tgt_rpc bdev_malloc_delete MallocBdevForConfigChangeCheck 467if $rootdir/test/json_config/json_diff.sh <(tgt_rpc save_config) "${configs_path[target]}" > /dev/null; then 468 echo "ERROR: intentional configuration difference not detected!" 469 false 470else 471 echo "INFO: configuration change detected." 472fi 473 474json_config_test_fini 475 476echo "INFO: Success" 477