1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2022 Intel Corporation 4# All rights reserved. 5# 6testdir=$(readlink -f $(dirname $0)) 7rootdir=$(readlink -f $testdir/../..) 8source $rootdir/scripts/common.sh 9source $rootdir/test/common/autotest_common.sh 10 11# Pci bus hotplug 12# Helper function to remove/attach cotrollers 13debug_remove_attach_helper() { 14 local helper_time=0 15 16 helper_time=$(timing_cmd remove_attach_helper "$@") 17 printf 'remove_attach_helper took %ss to complete (handling %u nvme drive(s))' \ 18 "$helper_time" "$nvme_count" >&2 19} 20 21remove_attach_helper() { 22 local hotplug_events=$1 23 local hotplug_wait=$2 24 local use_bdev=${3:-false} 25 local dev bdfs 26 27 # We need to make sure we wait long enough for hotplug to initialize the devices 28 # and start IO - if we start removing devices before that happens we will end up 29 # stepping on hotplug's toes forcing it to fail to report proper count of given 30 # events. 31 sleep "$hotplug_wait" 32 33 while ((hotplug_events--)); do 34 for dev in "${nvmes[@]}"; do 35 echo 1 > "/sys/bus/pci/devices/$dev/remove" 36 done 37 38 if "$use_bdev"; then 39 # Since we removed all the devices, when the sleep settles, we expect to find no bdevs 40 sleep "$hotplug_wait" && (($(rpc_cmd bdev_get_bdevs | jq 'length') == 0)) 41 fi || return 1 42 43 # Avoid setup.sh as it does some extra work which is not relevant for this test. 44 echo 1 > "/sys/bus/pci/rescan" 45 46 for dev in "${nvmes[@]}"; do 47 echo "${pci_bus_driver["$dev"]}" > "/sys/bus/pci/devices/$dev/driver_override" 48 echo "$dev" > "/sys/bus/pci/devices/$dev/driver/unbind" 49 echo "$dev" > "/sys/bus/pci/drivers_probe" 50 echo "" > "/sys/bus/pci/devices/$dev/driver_override" 51 done 52 53 # Wait now for hotplug to reattach to the devices 54 sleep "$((hotplug_wait * nvme_count))" 55 56 if "$use_bdev"; then 57 # See if we get all the bdevs back in one bulk 58 bdfs=($(rpc_cmd bdev_get_bdevs | jq -r '.[].driver_specific.nvme[].pci_address' | sort)) 59 [[ ${bdfs[*]} == "${nvmes[*]}" ]] 60 fi 61 done 62} 63 64run_hotplug() { 65 trap 'killprocess $hotplug_pid; exit 1' SIGINT SIGTERM EXIT 66 67 "$SPDK_EXAMPLE_DIR/hotplug" \ 68 -i 0 \ 69 -t 0 \ 70 -n $((hotplug_events * nvme_count)) \ 71 -r $((hotplug_events * nvme_count)) \ 72 -l warning & 73 hotplug_pid=$! 74 75 debug_remove_attach_helper "$hotplug_events" "$hotplug_wait" false 76 77 # Wait in case hotplug app is lagging behind 78 # and kill it, if it hung. 79 sleep $hotplug_wait 80 81 if ! kill -0 "$hotplug_pid"; then 82 # hotplug already finished, check for the error code. 83 wait "$hotplug_pid" 84 else 85 echo "Killing hotplug application" 86 killprocess $hotplug_pid 87 return 1 88 fi 89 90 trap - SIGINT SIGTERM EXIT 91} 92 93# SPDK target hotplug 94tgt_run_hotplug() { 95 local dev 96 97 $SPDK_BIN_DIR/spdk_tgt & 98 spdk_tgt_pid=$! 99 100 trap 'killprocess ${spdk_tgt_pid}; echo 1 > /sys/bus/pci/rescan; exit 1' SIGINT SIGTERM EXIT 101 waitforlisten $spdk_tgt_pid 102 103 for dev in "${!nvmes[@]}"; do 104 rpc_cmd bdev_nvme_attach_controller -b "Nvme0$dev" -t PCIe -a "${nvmes[dev]}" 105 waitforbdev "Nvme0${dev}n1" "$hotplug_wait" 106 done 107 108 rpc_cmd bdev_nvme_set_hotplug -e 109 110 debug_remove_attach_helper "$hotplug_events" "$hotplug_wait" true 111 # Verify reregistering hotplug poller 112 rpc_cmd bdev_nvme_set_hotplug -d 113 rpc_cmd bdev_nvme_set_hotplug -e 114 115 debug_remove_attach_helper "$hotplug_events" "$hotplug_wait" true 116 117 trap - SIGINT SIGTERM EXIT 118 killprocess $spdk_tgt_pid 119} 120 121# Preparation 122"$rootdir/scripts/setup.sh" 123 124hotplug_wait=6 125hotplug_events=3 126nvmes=($(nvme_in_userspace)) 127nvme_count=${#nvmes[@]} 128 129xtrace_disable 130cache_pci_bus 131xtrace_restore 132 133# Run pci bus hotplug test 134run_hotplug 135 136# Run SPDK target based hotplug 137tgt_run_hotplug 138