1# SPDX-License-Identifier: BSD-3-Clause 2# Copyright (C) 2016 Intel Corporation 3# Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES 4# All rights reserved. 5# 6 7[[ $(uname -s) == FreeBSD ]] && return 0 8 9NVMF_PORT=4420 10NVMF_SECOND_PORT=4421 11NVMF_THIRD_PORT=4422 12NVMF_IP_PREFIX="192.168.100" 13NVMF_IP_LEAST_ADDR=8 14NVMF_TCP_IP_ADDRESS="127.0.0.1" 15NVMF_TRANSPORT_OPTS="" 16NVMF_SERIAL=SPDK00000000000001 17NVME_HOSTNQN=$(nvme gen-hostnqn) 18NVME_HOSTID=${NVME_HOSTNQN##*:} 19NVME_HOST=("--hostnqn=$NVME_HOSTNQN" "--hostid=$NVME_HOSTID") 20NVME_CONNECT="nvme connect" 21NET_TYPE=${NET_TYPE:-phy-fallback} 22 23function build_nvmf_app_args() { 24 if [ $SPDK_RUN_NON_ROOT -eq 1 ]; then 25 # We assume that test script is started from sudo 26 NVMF_APP=(sudo -E -u $SUDO_USER "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" "${NVMF_APP[@]}") 27 fi 28 NVMF_APP+=(-i "$NVMF_APP_SHM_ID" -e 0xFFFF) 29 30 if [ -n "$SPDK_HUGE_DIR" ]; then 31 NVMF_APP+=(--huge-dir "$SPDK_HUGE_DIR") 32 elif [ $SPDK_RUN_NON_ROOT -eq 1 ]; then 33 echo "In non-root test mode you have to set SPDK_HUGE_DIR variable." >&2 34 echo "For example:" >&2 35 echo "sudo mkdir /mnt/spdk_hugetlbfs" >&2 36 echo "sudo chown ${SUDO_USER}: /mnt/spdk_hugetlbfs" >&2 37 echo "export SPDK_HUGE_DIR=/mnt/spdk_hugetlbfs" >&2 38 return 1 39 fi 40} 41 42source "$rootdir/scripts/common.sh" 43 44: ${NVMF_APP_SHM_ID="0"} 45export NVMF_APP_SHM_ID 46build_nvmf_app_args 47 48have_pci_nics=0 49 50function rxe_cfg() { 51 "$rootdir/scripts/rxe_cfg_small.sh" "$@" 52} 53 54function load_ib_rdma_modules() { 55 if [ $(uname) != Linux ]; then 56 return 0 57 fi 58 59 modprobe ib_cm 60 modprobe ib_core 61 modprobe ib_umad 62 modprobe ib_uverbs 63 modprobe iw_cm 64 modprobe rdma_cm 65 modprobe rdma_ucm 66} 67 68function allocate_nic_ips() { 69 ((count = NVMF_IP_LEAST_ADDR)) 70 for nic_name in $(get_rdma_if_list); do 71 ip="$(get_ip_address $nic_name)" 72 if [[ -z $ip ]]; then 73 ip addr add $NVMF_IP_PREFIX.$count/24 dev $nic_name 74 ip link set $nic_name up 75 ((count = count + 1)) 76 fi 77 # dump configuration for debug log 78 ip addr show $nic_name 79 done 80} 81 82function get_available_rdma_ips() { 83 for nic_name in $(get_rdma_if_list); do 84 get_ip_address $nic_name 85 done 86} 87 88function get_rdma_if_list() { 89 local net_dev rxe_net_dev rxe_net_devs 90 91 mapfile -t rxe_net_devs < <(rxe_cfg rxe-net) 92 93 if ((${#net_devs[@]} == 0)); then 94 return 1 95 fi 96 97 # Pick only these devices which were found during gather_supported_nvmf_pci_devs() run 98 for net_dev in "${net_devs[@]}"; do 99 for rxe_net_dev in "${rxe_net_devs[@]}"; do 100 if [[ $net_dev == "$rxe_net_dev" ]]; then 101 echo "$net_dev" 102 continue 2 103 fi 104 done 105 done 106} 107 108function get_ip_address() { 109 interface=$1 110 ip -o -4 addr show $interface | awk '{print $4}' | cut -d"/" -f1 111} 112 113function nvmfcleanup() { 114 sync 115 116 if [ "$TEST_TRANSPORT" == "tcp" ] || [ "$TEST_TRANSPORT" == "rdma" ]; then 117 set +e 118 for i in {1..20}; do 119 modprobe -v -r nvme-$TEST_TRANSPORT 120 if modprobe -v -r nvme-fabrics; then 121 set -e 122 return 0 123 fi 124 sleep 1 125 done 126 set -e 127 128 # So far unable to remove the kernel modules. Try 129 # one more time and let it fail. 130 # Allow the transport module to fail for now. See Jim's comment 131 # about the nvme-tcp module below. 132 modprobe -v -r nvme-$TEST_TRANSPORT || true 133 modprobe -v -r nvme-fabrics 134 fi 135} 136 137function nvmf_veth_init() { 138 NVMF_INITIATOR_IP=10.0.0.1 139 NVMF_FIRST_TARGET_IP=10.0.0.2 140 NVMF_SECOND_TARGET_IP=10.0.0.3 141 NVMF_BRIDGE="nvmf_br" 142 NVMF_INITIATOR_INTERFACE="nvmf_init_if" 143 NVMF_INITIATOR_BRIDGE="nvmf_init_br" 144 NVMF_TARGET_NAMESPACE="nvmf_tgt_ns_spdk" 145 NVMF_TARGET_NS_CMD=(ip netns exec "$NVMF_TARGET_NAMESPACE") 146 NVMF_TARGET_INTERFACE="nvmf_tgt_if" 147 NVMF_TARGET_INTERFACE2="nvmf_tgt_if2" 148 NVMF_TARGET_BRIDGE="nvmf_tgt_br" 149 NVMF_TARGET_BRIDGE2="nvmf_tgt_br2" 150 151 ip link set $NVMF_INITIATOR_BRIDGE nomaster || true 152 ip link set $NVMF_TARGET_BRIDGE nomaster || true 153 ip link set $NVMF_TARGET_BRIDGE2 nomaster || true 154 ip link set $NVMF_INITIATOR_BRIDGE down || true 155 ip link set $NVMF_TARGET_BRIDGE down || true 156 ip link set $NVMF_TARGET_BRIDGE2 down || true 157 ip link delete $NVMF_BRIDGE type bridge || true 158 ip link delete $NVMF_INITIATOR_INTERFACE || true 159 "${NVMF_TARGET_NS_CMD[@]}" ip link delete $NVMF_TARGET_INTERFACE || true 160 "${NVMF_TARGET_NS_CMD[@]}" ip link delete $NVMF_TARGET_INTERFACE2 || true 161 162 # Create network namespace 163 ip netns add $NVMF_TARGET_NAMESPACE 164 165 # Create veth (Virtual ethernet) interface pairs 166 ip link add $NVMF_INITIATOR_INTERFACE type veth peer name $NVMF_INITIATOR_BRIDGE 167 ip link add $NVMF_TARGET_INTERFACE type veth peer name $NVMF_TARGET_BRIDGE 168 ip link add $NVMF_TARGET_INTERFACE2 type veth peer name $NVMF_TARGET_BRIDGE2 169 170 # Associate veth interface pairs with network namespace 171 ip link set $NVMF_TARGET_INTERFACE netns $NVMF_TARGET_NAMESPACE 172 ip link set $NVMF_TARGET_INTERFACE2 netns $NVMF_TARGET_NAMESPACE 173 174 # Allocate IP addresses 175 ip addr add $NVMF_INITIATOR_IP/24 dev $NVMF_INITIATOR_INTERFACE 176 "${NVMF_TARGET_NS_CMD[@]}" ip addr add $NVMF_FIRST_TARGET_IP/24 dev $NVMF_TARGET_INTERFACE 177 "${NVMF_TARGET_NS_CMD[@]}" ip addr add $NVMF_SECOND_TARGET_IP/24 dev $NVMF_TARGET_INTERFACE2 178 179 # Link up veth interfaces 180 ip link set $NVMF_INITIATOR_INTERFACE up 181 ip link set $NVMF_INITIATOR_BRIDGE up 182 ip link set $NVMF_TARGET_BRIDGE up 183 ip link set $NVMF_TARGET_BRIDGE2 up 184 "${NVMF_TARGET_NS_CMD[@]}" ip link set $NVMF_TARGET_INTERFACE up 185 "${NVMF_TARGET_NS_CMD[@]}" ip link set $NVMF_TARGET_INTERFACE2 up 186 "${NVMF_TARGET_NS_CMD[@]}" ip link set lo up 187 188 # Create a bridge 189 ip link add $NVMF_BRIDGE type bridge 190 ip link set $NVMF_BRIDGE up 191 192 # Add veth interfaces to the bridge 193 ip link set $NVMF_INITIATOR_BRIDGE master $NVMF_BRIDGE 194 ip link set $NVMF_TARGET_BRIDGE master $NVMF_BRIDGE 195 ip link set $NVMF_TARGET_BRIDGE2 master $NVMF_BRIDGE 196 197 # Accept connections from veth interface 198 iptables -I INPUT 1 -i $NVMF_INITIATOR_INTERFACE -p tcp --dport $NVMF_PORT -j ACCEPT 199 iptables -A FORWARD -i $NVMF_BRIDGE -o $NVMF_BRIDGE -j ACCEPT 200 201 # Verify connectivity 202 ping -c 1 $NVMF_FIRST_TARGET_IP 203 ping -c 1 $NVMF_SECOND_TARGET_IP 204 "${NVMF_TARGET_NS_CMD[@]}" ping -c 1 $NVMF_INITIATOR_IP 205 206 NVMF_APP=("${NVMF_TARGET_NS_CMD[@]}" "${NVMF_APP[@]}") 207} 208 209function nvmf_veth_fini() { 210 # Cleanup bridge, veth interfaces, and network namespace 211 # Note: removing one veth removes the pair 212 ip link set $NVMF_INITIATOR_BRIDGE nomaster 213 ip link set $NVMF_TARGET_BRIDGE nomaster 214 ip link set $NVMF_TARGET_BRIDGE2 nomaster 215 ip link set $NVMF_INITIATOR_BRIDGE down 216 ip link set $NVMF_TARGET_BRIDGE down 217 ip link set $NVMF_TARGET_BRIDGE2 down 218 ip link delete $NVMF_BRIDGE type bridge 219 ip link delete $NVMF_INITIATOR_INTERFACE 220 "${NVMF_TARGET_NS_CMD[@]}" ip link delete $NVMF_TARGET_INTERFACE 221 "${NVMF_TARGET_NS_CMD[@]}" ip link delete $NVMF_TARGET_INTERFACE2 222 remove_spdk_ns 223} 224 225function nvmf_tcp_init() { 226 NVMF_INITIATOR_IP=10.0.0.1 227 NVMF_FIRST_TARGET_IP=10.0.0.2 228 TCP_INTERFACE_LIST=("${net_devs[@]}") 229 230 # We need two net devs at minimum 231 ((${#TCP_INTERFACE_LIST[@]} > 1)) 232 233 NVMF_TARGET_INTERFACE=${TCP_INTERFACE_LIST[0]} 234 NVMF_INITIATOR_INTERFACE=${TCP_INTERFACE_LIST[1]} 235 236 # Skip case nvmf_multipath in nvmf_tcp_init(), it will be covered by nvmf_veth_init(). 237 NVMF_SECOND_TARGET_IP="" 238 239 NVMF_TARGET_NAMESPACE="${NVMF_TARGET_INTERFACE}_ns_spdk" 240 NVMF_TARGET_NS_CMD=(ip netns exec "$NVMF_TARGET_NAMESPACE") 241 ip -4 addr flush $NVMF_TARGET_INTERFACE || true 242 ip -4 addr flush $NVMF_INITIATOR_INTERFACE || true 243 244 # Create network namespace 245 ip netns add $NVMF_TARGET_NAMESPACE 246 247 # Associate phy interface pairs with network namespace 248 ip link set $NVMF_TARGET_INTERFACE netns $NVMF_TARGET_NAMESPACE 249 250 # Allocate IP addresses 251 ip addr add $NVMF_INITIATOR_IP/24 dev $NVMF_INITIATOR_INTERFACE 252 "${NVMF_TARGET_NS_CMD[@]}" ip addr add $NVMF_FIRST_TARGET_IP/24 dev $NVMF_TARGET_INTERFACE 253 254 # Link up phy interfaces 255 ip link set $NVMF_INITIATOR_INTERFACE up 256 257 "${NVMF_TARGET_NS_CMD[@]}" ip link set $NVMF_TARGET_INTERFACE up 258 "${NVMF_TARGET_NS_CMD[@]}" ip link set lo up 259 260 # Accept connections from phy interface 261 iptables -I INPUT 1 -i $NVMF_INITIATOR_INTERFACE -p tcp --dport $NVMF_PORT -j ACCEPT 262 263 # Verify connectivity 264 ping -c 1 $NVMF_FIRST_TARGET_IP 265 "${NVMF_TARGET_NS_CMD[@]}" ping -c 1 $NVMF_INITIATOR_IP 266 267 NVMF_APP=("${NVMF_TARGET_NS_CMD[@]}" "${NVMF_APP[@]}") 268} 269 270function nvmf_tcp_fini() { 271 if [[ "$NVMF_TARGET_NAMESPACE" == "nvmf_tgt_ns" ]]; then 272 nvmf_veth_fini 273 return 0 274 fi 275 remove_spdk_ns 276 ip -4 addr flush $NVMF_INITIATOR_INTERFACE || : 277} 278 279function gather_supported_nvmf_pci_devs() { 280 # Go through the entire pci bus and gather all ethernet controllers we support for the nvmf tests. 281 # Focus on the hardware that's currently being tested by the CI. 282 xtrace_disable 283 cache_pci_bus_sysfs 284 xtrace_restore 285 286 local intel=0x8086 mellanox=0x15b3 pci 287 288 local -a pci_devs=() 289 local -a pci_net_devs=() 290 local -A pci_drivers=() 291 292 local -ga net_devs=() 293 local -ga e810=() 294 local -ga x722=() 295 local -ga mlx=() 296 297 # E810-XXV 298 e810+=(${pci_bus_cache["$intel:0x1592"]}) 299 e810+=(${pci_bus_cache["$intel:0x159b"]}) 300 # X722 10G 301 x722+=(${pci_bus_cache["$intel:0x37d2"]}) 302 # BlueField 3 303 mlx+=(${pci_bus_cache["$mellanox:0xa2dc"]}) 304 # ConnectX-7 305 mlx+=(${pci_bus_cache["$mellanox:0x1021"]}) 306 # BlueField 2 307 mlx+=(${pci_bus_cache["$mellanox:0xa2d6"]}) 308 # ConnectX-6 Dx 309 mlx+=(${pci_bus_cache["$mellanox:0x101d"]}) 310 # ConnectX-5 311 mlx+=(${pci_bus_cache["$mellanox:0x1017"]}) 312 mlx+=(${pci_bus_cache["$mellanox:0x1019"]}) 313 # ConnectX-4 314 mlx+=(${pci_bus_cache["$mellanox:0x1015"]}) 315 mlx+=(${pci_bus_cache["$mellanox:0x1013"]}) 316 317 pci_devs+=("${e810[@]}") 318 if [[ $TEST_TRANSPORT == rdma ]]; then 319 pci_devs+=("${x722[@]}") 320 pci_devs+=("${mlx[@]}") 321 fi 322 323 # Try to respect what CI wants to test and override pci_devs[] 324 if [[ $SPDK_TEST_NVMF_NICS == mlx5 ]]; then 325 pci_devs=("${mlx[@]}") 326 elif [[ $SPDK_TEST_NVMF_NICS == e810 ]]; then 327 pci_devs=("${e810[@]}") 328 elif [[ $SPDK_TEST_NVMF_NICS == x722 ]]; then 329 pci_devs=("${x722[@]}") 330 fi 331 332 if ((${#pci_devs[@]} == 0)); then 333 return 1 334 fi 335 336 # Load proper kernel modules if necessary 337 for pci in "${pci_devs[@]}"; do 338 echo "Found $pci (${pci_ids_vendor["$pci"]} - ${pci_ids_device["$pci"]})" 339 if [[ ${pci_mod_resolved["$pci"]} == unknown ]]; then 340 echo "Unresolved modalias for $pci (${pci_mod_driver["$pci"]}). Driver not installed|builtin?" 341 continue 342 fi 343 if [[ ${pci_bus_driver["$pci"]} == unbound ]]; then 344 echo "$pci not bound, needs ${pci_mod_resolved["$pci"]}" 345 pci_drivers["${pci_mod_resolved["$pci"]}"]=1 346 fi 347 if [[ ${pci_ids_device["$pci"]} == "0x1017" ]] \ 348 || [[ ${pci_ids_device["$pci"]} == "0x1019" ]] \ 349 || [[ $TEST_TRANSPORT == rdma ]]; then 350 # Reduce maximum number of queues when connecting with 351 # ConnectX-5 NICs. When using host systems with nproc > 64 352 # connecting with default options (where default equals to 353 # number of host online CPUs) creating all IO queues 354 # takes too much time and results in keep-alive timeout. 355 # See: 356 # https://github.com/spdk/spdk/issues/2772 357 # 0x1017 - MT27800 Family ConnectX-5 358 # 0x1019 - MT28800 Family ConnectX-5 Ex 359 NVME_CONNECT="nvme connect -i 15" 360 fi 361 done 362 363 if ((${#pci_drivers[@]} > 0)); then 364 echo "Loading kernel modules: ${!pci_drivers[*]}" 365 modprobe -a "${!pci_drivers[@]}" 366 fi 367 368 # E810 cards also need irdma driver to be around. 369 if [[ $SPDK_TEST_NVMF_NICS == e810 && $TEST_TRANSPORT == rdma ]]; then 370 if [[ -e /sys/module/irdma/parameters/roce_ena ]]; then 371 # Our tests don't play well with iWARP protocol. Make sure we use RoCEv2 instead. 372 (($(< /sys/module/irdma/parameters/roce_ena) != 1)) && modprobe -r irdma 373 fi 374 modinfo irdma && modprobe irdma roce_ena=1 375 fi > /dev/null 376 377 # All devices detected, kernel modules loaded. Now look under net class to see if there 378 # are any net devices bound to the controllers. 379 for pci in "${pci_devs[@]}"; do 380 pci_net_devs=("/sys/bus/pci/devices/$pci/net/"*) 381 if ((${#pci_net_devs[@]} == 0)); then 382 echo "No net devices associated with $pci" 383 continue 384 fi 385 pci_net_devs=("${pci_net_devs[@]##*/}") 386 echo "Found net devices under $pci: ${pci_net_devs[*]}" 387 net_devs+=("${pci_net_devs[@]}") 388 done 389 390 if ((${#net_devs[@]} == 0)); then 391 return 1 392 fi 393} 394 395prepare_net_devs() { 396 local -g is_hw=no 397 398 remove_spdk_ns 399 400 [[ $NET_TYPE != virt ]] && gather_supported_nvmf_pci_devs && is_hw=yes 401 402 if [[ $is_hw == yes ]]; then 403 if [[ $TEST_TRANSPORT == tcp ]]; then 404 nvmf_tcp_init 405 elif [[ $TEST_TRANSPORT == rdma ]]; then 406 rdma_device_init 407 fi 408 return 0 409 elif [[ $NET_TYPE == phy ]]; then 410 echo "ERROR: No supported devices were found, cannot run the $TEST_TRANSPORT test" 411 return 1 412 elif [[ $NET_TYPE == phy-fallback ]]; then 413 echo "WARNING: No supported devices were found, fallback requested for $TEST_TRANSPORT test" 414 fi 415 416 # NET_TYPE == virt or phy-fallback 417 if [[ $TEST_TRANSPORT == tcp ]]; then 418 nvmf_veth_init 419 return 0 420 fi 421 422 echo "ERROR: virt and fallback setup is not supported for $TEST_TRANSPORT" 423 return 1 424} 425 426function nvmftestinit() { 427 if [ -z $TEST_TRANSPORT ]; then 428 echo "transport not specified - use --transport= to specify" 429 return 1 430 fi 431 432 trap 'nvmftestfini' SIGINT SIGTERM EXIT 433 434 prepare_net_devs 435 436 if [ "$TEST_MODE" == "iso" ]; then 437 $rootdir/scripts/setup.sh 438 fi 439 440 NVMF_TRANSPORT_OPTS="-t $TEST_TRANSPORT" 441 if [[ "$TEST_TRANSPORT" == "rdma" ]]; then 442 RDMA_IP_LIST=$(get_available_rdma_ips) 443 NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1) 444 NVMF_SECOND_TARGET_IP=$(echo "$RDMA_IP_LIST" | tail -n +2 | head -n 1) 445 if [ -z $NVMF_FIRST_TARGET_IP ]; then 446 echo "no RDMA NIC for nvmf test" 447 exit 1 448 fi 449 NVMF_TRANSPORT_OPTS="$NVMF_TRANSPORT_OPTS --num-shared-buffers 1024" 450 elif [[ "$TEST_TRANSPORT" == "tcp" ]]; then 451 NVMF_TRANSPORT_OPTS="$NVMF_TRANSPORT_OPTS -o" 452 fi 453 454 if [ "$TEST_TRANSPORT" == "tcp" ] || [ "$TEST_TRANSPORT" == "rdma" ]; then 455 # currently we run the host/perf test for TCP even on systems without kernel nvme-tcp 456 # support; that's fine since the host/perf test uses the SPDK initiator 457 # maybe later we will enforce modprobe to succeed once we have systems in the test pool 458 # with nvme-tcp kernel support - but until then let this pass so we can still run the 459 # host/perf test with the tcp transport 460 modprobe nvme-$TEST_TRANSPORT || true 461 fi 462} 463 464function nvmfappstart() { 465 timing_enter start_nvmf_tgt 466 "${NVMF_APP[@]}" "$@" & 467 nvmfpid=$! 468 waitforlisten $nvmfpid 469 timing_exit start_nvmf_tgt 470 trap 'process_shm --id $NVMF_APP_SHM_ID || :; nvmftestfini' SIGINT SIGTERM EXIT 471} 472 473function nvmftestfini() { 474 nvmfcleanup || : 475 if [ -n "$nvmfpid" ]; then 476 killprocess $nvmfpid 477 fi 478 if [ "$TEST_MODE" == "iso" ]; then 479 $rootdir/scripts/setup.sh reset 480 fi 481 if [[ "$TEST_TRANSPORT" == "tcp" ]]; then 482 nvmf_tcp_fini 483 fi 484} 485 486function rdma_device_init() { 487 load_ib_rdma_modules 488 allocate_nic_ips 489} 490 491function nvme_connect() { 492 local init_count 493 init_count=$(nvme list | wc -l) 494 495 if ! nvme connect "$@"; then return $?; fi 496 497 for i in $(seq 1 10); do 498 if [ $(nvme list | wc -l) -gt $init_count ]; then 499 return 0 500 else 501 sleep 1s 502 fi 503 done 504 return 1 505} 506 507function get_nvme_devs() { 508 local dev _ 509 510 while read -r dev _; do 511 if [[ $dev == /dev/nvme* ]]; then 512 echo "$dev" 513 fi 514 done < <(nvme list) 515} 516 517function gen_nvmf_target_json() { 518 local subsystem config=() 519 520 for subsystem in "${@:-1}"; do 521 config+=( 522 "$( 523 cat <<- EOF 524 { 525 "params": { 526 "name": "Nvme$subsystem", 527 "trtype": "$TEST_TRANSPORT", 528 "traddr": "$NVMF_FIRST_TARGET_IP", 529 "adrfam": "ipv4", 530 "trsvcid": "$NVMF_PORT", 531 "subnqn": "nqn.2016-06.io.spdk:cnode$subsystem", 532 "hostnqn": "nqn.2016-06.io.spdk:host$subsystem", 533 "hdgst": ${hdgst:-false}, 534 "ddgst": ${ddgst:-false} 535 }, 536 "method": "bdev_nvme_attach_controller" 537 } 538 EOF 539 )" 540 ) 541 done 542 jq . <<- JSON 543 { 544 "subsystems": [ 545 { 546 "subsystem": "bdev", 547 "config": [ 548 { 549 "method": "bdev_nvme_set_options", 550 "params": { 551 "action_on_timeout": "none", 552 "timeout_us": 0, 553 "retry_count": 4, 554 "arbitration_burst": 0, 555 "low_priority_weight": 0, 556 "medium_priority_weight": 0, 557 "high_priority_weight": 0, 558 "nvme_adminq_poll_period_us": 10000, 559 "keep_alive_timeout_ms" : 10000, 560 "nvme_ioq_poll_period_us": 0, 561 "io_queue_requests": 0, 562 "delay_cmd_submit": true 563 } 564 }, 565 $( 566 IFS="," 567 printf '%s\n' "${config[*]}" 568 ), 569 { 570 "method": "bdev_wait_for_examine" 571 } 572 ] 573 } 574 ] 575 } 576 JSON 577} 578 579function remove_spdk_ns() { 580 local ns 581 while read -r ns _; do 582 [[ $ns == *_spdk ]] || continue 583 ip netns delete "$ns" 584 done < <(ip netns list) 585 # Let it settle 586 sleep 1 587} 588 589configure_kernel_target() { 590 # Keep it global in scope for easier cleanup 591 kernel_name=${1:-kernel_target} 592 nvmet=/sys/kernel/config/nvmet 593 kernel_subsystem=$nvmet/subsystems/$kernel_name 594 kernel_namespace=$kernel_subsystem/namespaces/1 595 kernel_port=$nvmet/ports/1 596 597 local block nvme 598 599 if [[ ! -e /sys/module/nvmet ]]; then 600 modprobe nvmet 601 fi 602 603 [[ -e $nvmet ]] 604 605 "$rootdir/scripts/setup.sh" reset 606 607 # Find nvme with an active ns device 608 for block in /sys/block/nvme*; do 609 [[ -e $block ]] || continue 610 block_in_use "${block##*/}" || nvme="/dev/${block##*/}" 611 done 612 613 [[ -b $nvme ]] 614 615 mkdir "$kernel_subsystem" 616 mkdir "$kernel_namespace" 617 mkdir "$kernel_port" 618 619 # It allows only %llx value and for some reason kernel swaps the byte order 620 # so setting the serial is not very useful here 621 # "$kernel_subsystem/attr_serial" 622 echo "SPDK-$kernel_name" > "$kernel_subsystem/attr_model" 623 624 echo 1 > "$kernel_subsystem/attr_allow_any_host" 625 echo "$nvme" > "$kernel_namespace/device_path" 626 echo 1 > "$kernel_namespace/enable" 627 628 # By default use initiator ip which was set by nvmftestinit(). This is the 629 # interface which resides in the main net namespace and which is visible 630 # to nvmet. 631 632 echo "$NVMF_INITIATOR_IP" > "$kernel_port/addr_traddr" 633 echo "$TEST_TRANSPORT" > "$kernel_port/addr_trtype" 634 echo "$NVMF_PORT" > "$kernel_port/addr_trsvcid" 635 echo ipv4 > "$kernel_port/addr_adrfam" 636 637 # Enable the listener by linking the port to previously created subsystem 638 ln -s "$kernel_subsystem" "$kernel_port/subsystems/" 639 640 # Check if target is available 641 nvme discover "${NVME_HOST[@]}" -a "$NVMF_INITIATOR_IP" -t "$TEST_TRANSPORT" -s "$NVMF_PORT" 642} 643 644clean_kernel_target() { 645 [[ -e $kernel_subsystem ]] || return 0 646 647 echo 0 > "$kernel_namespace/enable" 648 649 rm -f "$kernel_port/subsystems/$kernel_name" 650 rmdir "$kernel_namespace" 651 rmdir "$kernel_port" 652 rmdir "$kernel_subsystem" 653 654 modules=(/sys/module/nvmet/holders/*) 655 656 modprobe -r "${modules[@]##*/}" nvmet 657} 658