1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2018 Intel Corporation 4# All rights reserved. 5# 6rootdir=$(readlink -f $(dirname $0)/../..) 7source "$rootdir/test/common/autotest_common.sh" 8source "$rootdir/test/nvmf/common.sh" 9 10if [[ $SPDK_TEST_ISCSI -eq 1 ]]; then 11 source "$rootdir/test/iscsi_tgt/common.sh" 12fi 13 14if [[ $SPDK_TEST_VHOST -ne 1 && $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 15 SPDK_TEST_VHOST=1 16 echo "WARNING: Virtio initiator JSON_config test requires vhost target." 17 echo " Setting SPDK_TEST_VHOST=1 for duration of current script." 18fi 19 20if ((SPDK_TEST_BLOCKDEV + \ 21 SPDK_TEST_ISCSI + \ 22 SPDK_TEST_NVMF + \ 23 SPDK_TEST_VHOST + \ 24 SPDK_TEST_VHOST_INIT + \ 25 SPDK_TEST_RBD == 0)); then 26 echo "WARNING: No tests are enabled so not running JSON configuration tests" 27 exit 0 28fi 29 30declare -A app_pid=([target]="" [initiator]="") 31declare -A app_socket=([target]='/var/tmp/spdk_tgt.sock' [initiator]='/var/tmp/spdk_initiator.sock') 32declare -A app_params=([target]='-m 0x1 -s 1024' [initiator]='-m 0x2 -g -u -s 1024') 33declare -A configs_path=([target]="$rootdir/spdk_tgt_config.json" [initiator]="$rootdir/spdk_initiator_config.json") 34 35function tgt_rpc() { 36 $rootdir/scripts/rpc.py -s "${app_socket[target]}" "$@" 37} 38 39function initiator_rpc() { 40 $rootdir/scripts/rpc.py -s "${app_socket[initiator]}" "$@" 41} 42 43last_event_id=0 44 45function tgt_check_notification_types() { 46 timing_enter "${FUNCNAME[0]}" 47 48 local ret=0 49 local enabled_types=("bdev_register" "bdev_unregister") 50 51 local get_types=($(tgt_rpc notify_get_types | jq -r '.[]')) 52 if [[ ${enabled_types[*]} != "${get_types[*]}" ]]; then 53 echo "ERROR: expected types: ${enabled_types[*]}, but got: ${get_types[*]}" 54 ret=1 55 fi 56 57 timing_exit "${FUNCNAME[0]}" 58 return $ret 59} 60 61get_notifications() { 62 local ev_type ev_ctx event_id 63 64 while IFS=":" read -r ev_type ev_ctx event_id; do 65 echo "$ev_type:$ev_ctx" 66 done < <(tgt_rpc notify_get_notifications -i "$last_event_id" | jq -r '.[] | "\(.type):\(.ctx):\(.id)"') 67} 68 69function tgt_check_notifications() { 70 local events_to_check 71 local recorded_events 72 73 # Seems like notifications don't necessarily have to come in order, so make sure they are sorted 74 events_to_check=($(printf '%s\n' "$@" | sort)) 75 recorded_events=($(get_notifications | sort)) 76 77 if [[ ${events_to_check[*]} != "${recorded_events[*]}" ]]; then 78 cat <<- ERROR 79 Expected events did not match. 80 81 Expected: 82 $(printf ' %s\n' "${events_to_check[@]}") 83 Recorded: 84 $(printf ' %s\n' "${recorded_events[@]}") 85 ERROR 86 return 1 87 fi 88 89 cat <<- INFO 90 Expected events matched: 91 $(printf ' %s\n' "${recorded_events[@]}") 92 INFO 93} 94 95# $1 - target / initiator 96# $2..$n app parameters 97function json_config_test_start_app() { 98 local app=$1 99 shift 100 101 [[ -n "${#app_socket[$app]}" ]] # Check app type 102 [[ -z "${app_pid[$app]}" ]] # Assert if app is not running 103 104 local app_extra_params="" 105 if [[ $SPDK_TEST_VHOST -eq 1 || $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 106 # If PWD is nfs/sshfs we can't create UNIX sockets there. Always use safe location instead. 107 app_extra_params='-S /var/tmp' 108 fi 109 110 $SPDK_BIN_DIR/spdk_tgt ${app_params[$app]} ${app_extra_params} -r ${app_socket[$app]} "$@" & 111 app_pid[$app]=$! 112 113 echo "Waiting for $app to run..." 114 waitforlisten ${app_pid[$app]} ${app_socket[$app]} 115 echo "" 116} 117 118# $1 - target / initiator 119function json_config_test_shutdown_app() { 120 local app=$1 121 122 # Check app type && assert app was started 123 [[ -n "${#app_socket[$app]}" ]] 124 [[ -n "${app_pid[$app]}" ]] 125 126 # spdk_kill_instance RPC will trigger ASAN 127 kill -SIGINT ${app_pid[$app]} 128 129 for ((i = 0; i < 30; i++)); do 130 if ! kill -0 ${app_pid[$app]} 2> /dev/null; then 131 app_pid[$app]= 132 break 133 fi 134 sleep 0.5 135 done 136 137 if [[ -n "${app_pid[$app]}" ]]; then 138 echo "SPDK $app shutdown timeout" 139 return 1 140 fi 141 142 echo "SPDK $app shutdown done" 143} 144 145function create_accel_config() { 146 timing_enter "${FUNCNAME[0]}" 147 148 if [[ $SPDK_TEST_CRYPTO -eq 1 ]]; then 149 tgt_rpc dpdk_cryptodev_scan_accel_module 150 tgt_rpc accel_assign_opc -o encrypt -m dpdk_cryptodev 151 tgt_rpc accel_assign_opc -o decrypt -m dpdk_cryptodev 152 fi 153 154 timing_exit "${FUNCNAME[0]}" 155} 156 157function create_bdev_subsystem_config() { 158 timing_enter "${FUNCNAME[0]}" 159 160 local expected_notifications=() 161 162 # Consider multiple nvme devices loaded into the subsystem prior running 163 # the tests. 164 expected_notifications+=($(get_notifications)) 165 166 if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then 167 local lvol_store_base_bdev=Nvme0n1 168 169 tgt_rpc bdev_split_create $lvol_store_base_bdev 2 170 tgt_rpc bdev_split_create Malloc0 3 171 tgt_rpc bdev_malloc_create 8 4096 --name Malloc3 172 tgt_rpc bdev_passthru_create -b Malloc3 -p PTBdevFromMalloc3 173 174 tgt_rpc bdev_null_create Null0 32 512 175 176 tgt_rpc bdev_malloc_create 32 512 --name Malloc0 177 tgt_rpc bdev_malloc_create 16 4096 --name Malloc1 178 179 expected_notifications+=( 180 bdev_register:${lvol_store_base_bdev}p1 181 bdev_register:${lvol_store_base_bdev}p0 182 bdev_register:Malloc3 183 bdev_register:PTBdevFromMalloc3 184 bdev_register:Null0 185 bdev_register:Malloc0 186 bdev_register:Malloc0p2 187 bdev_register:Malloc0p1 188 bdev_register:Malloc0p0 189 bdev_register:Malloc1 190 ) 191 192 # This AIO bdev must be large enough to be used as LVOL store 193 dd if=/dev/zero of="$SPDK_TEST_STORAGE/sample_aio" bs=1024 count=102400 194 tgt_rpc bdev_aio_create "$SPDK_TEST_STORAGE/sample_aio" aio_disk 1024 195 expected_notifications+=(bdev_register:aio_disk) 196 197 # For LVOLs use split to check for proper order of initialization. 198 # If LVOLs configuration will be reordered (eg moved before splits or AIO/NVMe) 199 # it should fail loading JSON config from file. 200 tgt_rpc bdev_lvol_create_lvstore -c 1048576 ${lvol_store_base_bdev}p0 lvs_test 201 202 expected_notifications+=( 203 "bdev_register:$(tgt_rpc bdev_lvol_create -l lvs_test lvol0 32)" 204 "bdev_register:$(tgt_rpc bdev_lvol_create -l lvs_test -t lvol1 32)" 205 "bdev_register:$(tgt_rpc bdev_lvol_snapshot lvs_test/lvol0 snapshot0)" 206 "bdev_register:$(tgt_rpc bdev_lvol_clone lvs_test/snapshot0 clone0)" 207 ) 208 fi 209 210 if [[ $SPDK_TEST_CRYPTO -eq 1 ]]; then 211 tgt_rpc bdev_malloc_create 8 1024 --name MallocForCryptoBdev 212 if [[ $(lspci -d:37c8 | wc -l) -eq 0 ]]; then 213 local crypto_driver=crypto_aesni_mb 214 else 215 local crypto_driver=crypto_qat 216 fi 217 218 tgt_rpc bdev_crypto_create MallocForCryptoBdev CryptoMallocBdev -p $crypto_driver -k 01234567891234560123456789123456 219 expected_notifications+=( 220 bdev_register:MallocForCryptoBdev 221 bdev_register:CryptoMallocBdev 222 ) 223 fi 224 225 if [[ $SPDK_TEST_RBD -eq 1 ]]; then 226 rbd_setup 127.0.0.1 227 tgt_rpc bdev_rbd_create $RBD_POOL $RBD_NAME 4096 228 expected_notifications+=(bdev_register:Ceph0) 229 fi 230 231 tgt_check_notifications "${expected_notifications[@]}" 232 233 timing_exit "${FUNCNAME[0]}" 234} 235 236function cleanup_bdev_subsystem_config() { 237 timing_enter "${FUNCNAME[0]}" 238 239 if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then 240 tgt_rpc bdev_lvol_delete lvs_test/clone0 241 tgt_rpc bdev_lvol_delete lvs_test/lvol0 242 tgt_rpc bdev_lvol_delete lvs_test/snapshot0 243 tgt_rpc bdev_lvol_delete_lvstore -l lvs_test 244 fi 245 246 if [[ $(uname -s) = Linux ]]; then 247 rm -f "$SPDK_TEST_STORAGE/sample_aio" 248 fi 249 250 if [[ $SPDK_TEST_RBD -eq 1 ]]; then 251 rbd_cleanup 252 fi 253 254 timing_exit "${FUNCNAME[0]}" 255} 256 257function create_vhost_subsystem_config() { 258 timing_enter "${FUNCNAME[0]}" 259 260 tgt_rpc bdev_malloc_create 64 1024 --name MallocForVhost0 261 tgt_rpc bdev_split_create MallocForVhost0 8 262 263 tgt_rpc vhost_create_scsi_controller VhostScsiCtrlr0 264 tgt_rpc vhost_scsi_controller_add_target VhostScsiCtrlr0 0 MallocForVhost0p3 265 tgt_rpc vhost_scsi_controller_add_target VhostScsiCtrlr0 -1 MallocForVhost0p4 266 tgt_rpc vhost_controller_set_coalescing VhostScsiCtrlr0 1 100 267 268 tgt_rpc vhost_create_blk_controller VhostBlkCtrlr0 MallocForVhost0p5 269 270 timing_exit "${FUNCNAME[0]}" 271} 272 273function create_iscsi_subsystem_config() { 274 timing_enter "${FUNCNAME[0]}" 275 tgt_rpc bdev_malloc_create 64 1024 --name MallocForIscsi0 276 tgt_rpc iscsi_create_portal_group $PORTAL_TAG 127.0.0.1:$ISCSI_PORT 277 tgt_rpc iscsi_create_initiator_group $INITIATOR_TAG $INITIATOR_NAME $NETMASK 278 tgt_rpc iscsi_create_target_node Target3 Target3_alias 'MallocForIscsi0:0' $PORTAL_TAG:$INITIATOR_TAG 64 -d 279 timing_exit "${FUNCNAME[0]}" 280} 281 282function create_nvmf_subsystem_config() { 283 timing_enter "${FUNCNAME[0]}" 284 285 NVMF_FIRST_TARGET_IP="127.0.0.1" 286 if [[ $SPDK_TEST_NVMF_TRANSPORT == "rdma" ]]; then 287 TEST_TRANSPORT=$SPDK_TEST_NVMF_TRANSPORT nvmftestinit 288 fi 289 290 if [[ -z $NVMF_FIRST_TARGET_IP ]]; then 291 echo "Error: no NIC for nvmf test" 292 return 1 293 fi 294 295 tgt_rpc bdev_malloc_create 8 512 --name MallocForNvmf0 296 tgt_rpc bdev_malloc_create 4 1024 --name MallocForNvmf1 297 298 tgt_rpc nvmf_create_transport -t $SPDK_TEST_NVMF_TRANSPORT -u 8192 -c 0 299 tgt_rpc nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 300 tgt_rpc nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 MallocForNvmf0 301 tgt_rpc nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 MallocForNvmf1 302 tgt_rpc nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $SPDK_TEST_NVMF_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s "$NVMF_PORT" 303 304 timing_exit "${FUNCNAME[0]}" 305} 306 307function create_virtio_initiator_config() { 308 timing_enter "${FUNCNAME[0]}" 309 initiator_rpc bdev_virtio_attach_controller -t user -a /var/tmp/VhostScsiCtrlr0 -d scsi VirtioScsiCtrlr0 310 initiator_rpc bdev_virtio_attach_controller -t user -a /var/tmp/VhostBlkCtrlr0 -d blk VirtioBlk0 311 timing_exit "${FUNCNAME[0]}" 312} 313 314function json_config_test_init() { 315 timing_enter "${FUNCNAME[0]}" 316 timing_enter json_config_setup_target 317 318 json_config_test_start_app target --wait-for-rpc 319 320 #TODO: global subsystem params 321 322 create_accel_config 323 324 # Load nvme configuration. The load_config will issue framework_start_init automatically 325 ( 326 $rootdir/scripts/gen_nvme.sh --json-with-subsystems 327 ) | tgt_rpc load_config 328 329 tgt_check_notification_types 330 331 if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then 332 create_bdev_subsystem_config 333 fi 334 335 if [[ $SPDK_TEST_VHOST -eq 1 ]]; then 336 create_vhost_subsystem_config 337 fi 338 339 if [[ $SPDK_TEST_ISCSI -eq 1 ]]; then 340 create_iscsi_subsystem_config 341 fi 342 343 if [[ $SPDK_TEST_NVMF -eq 1 ]]; then 344 create_nvmf_subsystem_config 345 fi 346 timing_exit json_config_setup_target 347 348 if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 349 json_config_test_start_app initiator 350 create_virtio_initiator_config 351 fi 352 353 tgt_rpc bdev_malloc_create 8 512 --name MallocBdevForConfigChangeCheck 354 355 timing_exit "${FUNCNAME[0]}" 356} 357 358function json_config_test_fini() { 359 timing_enter "${FUNCNAME[0]}" 360 local ret=0 361 362 if [[ -n "${app_pid[initiator]}" ]]; then 363 if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 364 initiator_rpc bdev_virtio_detach_controller VirtioScsiCtrlr0 || : 365 initiator_rpc bdev_virtio_detach_controller VirtioBlk0 || : 366 fi 367 killprocess ${app_pid[initiator]} 368 fi 369 370 if [[ -n "${app_pid[target]}" ]]; then 371 372 # Remove any artifacts we created (files, lvol etc) 373 cleanup_bdev_subsystem_config 374 375 # SPDK_TEST_NVMF: Should we clear something? 376 killprocess ${app_pid[target]} 377 fi 378 379 rm -f "${configs_path[@]}" 380 timing_exit "${FUNCNAME[0]}" 381 return $ret 382} 383 384function json_config_clear() { 385 [[ -n "${#app_socket[$1]}" ]] # Check app type 386 $rootdir/test/json_config/clear_config.py -s ${app_socket[$1]} clear_config 387 388 # Check if config is clean. 389 # Global params can't be cleared so need to filter them out. 390 local config_filter="$rootdir/test/json_config/config_filter.py" 391 392 # RPC's used to cleanup configuration (e.g. to delete split and nvme bdevs) 393 # complete immediately and they don't wait for the unregister callback. 394 # It causes that configuration may not be fully cleaned at this moment and 395 # we should to wait a while. (See github issue #789) 396 count=100 397 while [ $count -gt 0 ]; do 398 $rootdir/scripts/rpc.py -s "${app_socket[$1]}" save_config | $config_filter -method delete_global_parameters | $config_filter -method check_empty && break 399 count=$((count - 1)) 400 sleep 0.1 401 done 402 403 if [ $count -eq 0 ]; then 404 return 1 405 fi 406} 407 408on_error_exit() { 409 set -x 410 set +e 411 print_backtrace 412 trap - ERR 413 echo "Error on $1 - $2" 414 json_config_test_fini 415 exit 1 416} 417 418trap 'on_error_exit "${FUNCNAME}" "${LINENO}"' ERR 419echo "INFO: JSON configuration test init" 420json_config_test_init 421 422tgt_rpc save_config > ${configs_path[target]} 423 424echo "INFO: shutting down applications..." 425if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 426 initiator_rpc save_config > ${configs_path[initiator]} 427 json_config_clear initiator 428 json_config_test_shutdown_app initiator 429fi 430 431json_config_clear target 432json_config_test_shutdown_app target 433 434echo "INFO: relaunching applications..." 435json_config_test_start_app target --json ${configs_path[target]} 436if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 437 json_config_test_start_app initiator --json ${configs_path[initiator]} 438fi 439 440echo "INFO: Checking if target configuration is the same..." 441$rootdir/test/json_config/json_diff.sh <(tgt_rpc save_config) "${configs_path[target]}" 442if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then 443 echo "INFO: Checking if virtio initiator configuration is the same..." 444 $rootdir/test/json_config/json_diff.sh <(initiator_rpc save_config) "${configs_path[initiator]}" 445fi 446 447echo "INFO: changing configuration and checking if this can be detected..." 448# Self test to check if configuration diff can be detected. 449tgt_rpc bdev_malloc_delete MallocBdevForConfigChangeCheck 450if $rootdir/test/json_config/json_diff.sh <(tgt_rpc save_config) "${configs_path[target]}" > /dev/null; then 451 echo "ERROR: intentional configuration difference not detected!" 452 false 453else 454 echo "INFO: configuration change detected." 455fi 456 457json_config_test_fini 458 459echo "INFO: Success" 460