1#!/usr/bin/env bash 2 3testdir=$(readlink -f $(dirname $0)) 4rootdir=$(readlink -f $testdir/../..) 5source $rootdir/test/common/autotest_common.sh 6source $testdir/nbd_common.sh 7 8rpc_py="$rootdir/scripts/rpc.py" 9conf_file="$testdir/bdev.json" 10# Make sure the configuration is clean 11: > "$conf_file" 12 13function cleanup() { 14 rm -f "$SPDK_TEST_STORAGE/aiofile" 15 rm -f "$SPDK_TEST_STORAGE/spdk-pmem-pool" 16 rm -f "$conf_file" 17 18 if [[ $test_type == rbd ]]; then 19 rbd_cleanup 20 fi 21} 22 23function start_spdk_tgt() { 24 "$SPDK_BIN_DIR/spdk_tgt" & 25 spdk_tgt_pid=$! 26 trap 'killprocess "$spdk_tgt_pid"; exit 1' SIGINT SIGTERM EXIT 27 waitforlisten "$spdk_tgt_pid" 28} 29 30function setup_bdev_conf() { 31 "$rpc_py" <<- RPC 32 bdev_split_create Malloc1 2 33 bdev_split_create -s 4 Malloc2 8 34 bdev_malloc_create -b Malloc0 32 512 35 bdev_malloc_create -b Malloc1 32 512 36 bdev_malloc_create -b Malloc2 32 512 37 bdev_malloc_create -b Malloc3 32 512 38 bdev_malloc_create -b Malloc4 32 512 39 bdev_malloc_create -b Malloc5 32 512 40 bdev_passthru_create -p TestPT -b Malloc3 41 bdev_raid_create -n raid0 -z 64 -r 0 -b "Malloc4 Malloc5" 42 RPC 43 $rpc_py bdev_set_qos_limit --rw_mbytes_per_sec 100 Malloc3 44 $rpc_py bdev_set_qos_limit --rw_ios_per_sec 20000 Malloc0 45 if [[ $(uname -s) != "FreeBSD" ]]; then 46 dd if=/dev/zero of="$SPDK_TEST_STORAGE/aiofile" bs=2048 count=5000 47 "$rpc_py" bdev_aio_create "$SPDK_TEST_STORAGE/aiofile" AIO0 2048 48 fi 49} 50 51function setup_nvme_conf() { 52 "$rootdir/scripts/gen_nvme.sh" | "$rpc_py" load_subsystem_config 53} 54 55function setup_gpt_conf() { 56 if [[ $(uname -s) = Linux ]] && hash sgdisk; then 57 $rootdir/scripts/setup.sh reset 58 # Get nvme devices by following drivers' links towards nvme class 59 local nvme_devs=(/sys/bus/pci/drivers/nvme/*/nvme/nvme*/nvme*n*) nvme_dev 60 gpt_nvme="" 61 # Pick first device which doesn't have any valid partition table 62 for nvme_dev in "${nvme_devs[@]}"; do 63 dev=/dev/${nvme_dev##*/} 64 if ! pt=$(parted "$dev" -ms print 2>&1); then 65 [[ $pt == *"$dev: unrecognised disk label"* ]] || continue 66 gpt_nvme=$dev 67 break 68 fi 69 done 70 if [[ -n $gpt_nvme ]]; then 71 # Create gpt partition table 72 parted -s "$gpt_nvme" mklabel gpt mkpart first '0%' '50%' mkpart second '50%' '100%' 73 # change the GUID to SPDK GUID value 74 # FIXME: Hardcode this in some common place, this value should not be changed much 75 IFS="()" read -r _ SPDK_GPT_GUID _ < <(grep SPDK_GPT_PART_TYPE_GUID module/bdev/gpt/gpt.h) 76 SPDK_GPT_GUID=${SPDK_GPT_GUID//, /-} SPDK_GPT_GUID=${SPDK_GPT_GUID//0x/} 77 sgdisk -t "1:$SPDK_GPT_GUID" "$gpt_nvme" 78 sgdisk -t "2:$SPDK_GPT_GUID" "$gpt_nvme" 79 "$rootdir/scripts/setup.sh" 80 "$rpc_py" bdev_get_bdevs 81 setup_nvme_conf 82 else 83 printf 'Did not find any nvme block devices to work with, aborting the test\n' >&2 84 "$rootdir/scripts/setup.sh" 85 return 1 86 fi 87 else 88 # Not supported platform or missing tooling, nothing to be done, simply exit the test 89 # in a graceful manner. 90 trap - SIGINT SIGTERM EXIT 91 killprocess "$spdk_tgt_pid" 92 cleanup 93 exit 0 94 fi 95} 96 97function setup_crypto_aesni_conf() { 98 # Malloc0 and Malloc1 use AESNI 99 "$rpc_py" <<- RPC 100 bdev_malloc_create -b Malloc0 16 512 101 bdev_malloc_create -b Malloc1 16 512 102 bdev_crypto_create Malloc0 crypto_ram crypto_aesni_mb 0123456789123456 103 bdev_crypto_create Malloc1 crypto_ram2 crypto_aesni_mb 9012345678912345 104 RPC 105} 106 107function setup_crypto_qat_conf() { 108 # Malloc0 will use QAT AES_CBC 109 # Malloc1 will use QAT AES_XTS 110 "$rpc_py" <<- RPC 111 bdev_malloc_create -b Malloc0 16 512 112 bdev_malloc_create -b Malloc1 16 512 113 bdev_crypto_create Malloc0 crypto_ram crypto_qat 0123456789123456 114 bdev_crypto_create -c AES_XTS -k2 0123456789123456 Malloc1 crypto_ram3 crypto_qat 0123456789123456 115 RPC 116 "$rpc_py" bdev_get_bdevs -b Malloc1 117} 118 119function setup_pmem_conf() { 120 if hash pmempool; then 121 rm -f "$SPDK_TEST_STORAGE/spdk-pmem-pool" 122 pmempool create blk --size=32M 512 "$SPDK_TEST_STORAGE/spdk-pmem-pool" 123 "$rpc_py" bdev_pmem_create -n Pmem0 "$SPDK_TEST_STORAGE/spdk-pmem-pool" 124 else 125 return 1 126 fi 127} 128 129function setup_rbd_conf() { 130 timing_enter rbd_setup 131 rbd_setup 127.0.0.1 132 timing_exit rbd_setup 133 134 "$rpc_py" bdev_rbd_create -b Ceph0 rbd foo 512 135} 136 137function bdev_bounds() { 138 $testdir/bdevio/bdevio -w -s $PRE_RESERVED_MEM --json "$conf_file" & 139 bdevio_pid=$! 140 trap 'killprocess $bdevio_pid; exit 1' SIGINT SIGTERM EXIT 141 echo "Process bdevio pid: $bdevio_pid" 142 waitforlisten $bdevio_pid 143 $testdir/bdevio/tests.py perform_tests 144 killprocess $bdevio_pid 145 trap - SIGINT SIGTERM EXIT 146} 147 148function nbd_function_test() { 149 if [ $(uname -s) = Linux ] && modprobe -n nbd; then 150 local rpc_server=/var/tmp/spdk-nbd.sock 151 local conf=$1 152 local nbd_all=($(ls /dev/nbd* | grep -v p)) 153 local bdev_all=($bdevs_name) 154 local nbd_num=${#bdevs_all[@]} 155 if [ ${#nbd_all[@]} -le $nbd_num ]; then 156 nbd_num=${#nbd_all[@]} 157 fi 158 local nbd_list=(${nbd_all[@]:0:$nbd_num}) 159 local bdev_list=(${bdev_all[@]:0:$nbd_num}) 160 161 if [ ! -e $conf ]; then 162 return 1 163 fi 164 165 modprobe nbd 166 $rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 --json "$conf" & 167 nbd_pid=$! 168 trap 'killprocess $nbd_pid; exit 1' SIGINT SIGTERM EXIT 169 echo "Process nbd pid: $nbd_pid" 170 waitforlisten $nbd_pid $rpc_server 171 172 nbd_rpc_start_stop_verify $rpc_server "${bdev_list[*]}" 173 nbd_rpc_data_verify $rpc_server "${bdev_list[*]}" "${nbd_list[*]}" 174 175 killprocess $nbd_pid 176 trap - SIGINT SIGTERM EXIT 177 fi 178 179 return 0 180} 181 182function fio_test_suite() { 183 # Generate the fio config file given the list of all unclaimed bdevs 184 fio_config_gen $testdir/bdev.fio verify AIO 185 for b in $(echo $bdevs | jq -r '.name'); do 186 echo "[job_$b]" >> $testdir/bdev.fio 187 echo "filename=$b" >> $testdir/bdev.fio 188 done 189 190 local fio_params="--ioengine=spdk_bdev --iodepth=8 --bs=4k --runtime=10 $testdir/bdev.fio --spdk_json_conf=$conf_file" 191 192 run_test "bdev_fio_rw_verify" fio_bdev $fio_params --spdk_mem=$PRE_RESERVED_MEM \ 193 --output=$output_dir/blockdev_fio_verify.txt 194 rm -f ./*.state 195 rm -f $testdir/bdev.fio 196 197 # Generate the fio config file given the list of all unclaimed bdevs that support unmap 198 fio_config_gen $testdir/bdev.fio trim 199 if [ "$(echo $bdevs | jq -r 'select(.supported_io_types.unmap == true) | .name')" != "" ]; then 200 for b in $(echo $bdevs | jq -r 'select(.supported_io_types.unmap == true) | .name'); do 201 echo "[job_$b]" >> $testdir/bdev.fio 202 echo "filename=$b" >> $testdir/bdev.fio 203 done 204 else 205 rm -f $testdir/bdev.fio 206 return 0 207 fi 208 209 run_test "bdev_fio_trim" fio_bdev $fio_params --output=$output_dir/blockdev_trim.txt 210 rm -f ./*.state 211 rm -f $testdir/bdev.fio 212} 213 214function get_io_result() { 215 local limit_type=$1 216 local qos_dev=$2 217 local iostat_result 218 iostat_result=$($rootdir/scripts/iostat.py -d -i 1 -t $QOS_RUN_TIME | grep $qos_dev | tail -1) 219 if [ $limit_type = IOPS ]; then 220 iostat_result=$(awk '{print $2}' <<< $iostat_result) 221 elif [ $limit_type = BANDWIDTH ]; then 222 iostat_result=$(awk '{print $6}' <<< $iostat_result) 223 fi 224 225 echo ${iostat_result/.*/} 226} 227 228function run_qos_test() { 229 local qos_limit=$1 230 local qos_result=0 231 232 qos_result=$(get_io_result $2 $3) 233 if [ $2 = BANDWIDTH ]; then 234 qos_limit=$((qos_limit * 1024)) 235 fi 236 lower_limit=$((qos_limit * 9 / 10)) 237 upper_limit=$((qos_limit * 11 / 10)) 238 239 # QoS realization is related with bytes transfered. It currently has some variation. 240 if [ $qos_result -lt $lower_limit ] || [ $qos_result -gt $upper_limit ]; then 241 echo "Failed to limit the io read rate of NULL bdev by qos" 242 $rpc_py bdev_malloc_delete $QOS_DEV_1 243 $rpc_py bdev_null_delete $QOS_DEV_2 244 killprocess $QOS_PID 245 exit 1 246 fi 247} 248 249function qos_function_test() { 250 local qos_lower_iops_limit=1000 251 local qos_lower_bw_limit=2 252 local io_result=0 253 local iops_limit=0 254 local bw_limit=0 255 256 io_result=$(get_io_result IOPS $QOS_DEV_1) 257 # Set the IOPS limit as one quarter of the measured performance without QoS 258 iops_limit=$(((io_result / 4) / qos_lower_iops_limit * qos_lower_iops_limit)) 259 if [ $iops_limit -gt $qos_lower_iops_limit ]; then 260 261 # Run bdevperf with IOPS rate limit on bdev 1 262 $rpc_py bdev_set_qos_limit --rw_ios_per_sec $iops_limit $QOS_DEV_1 263 run_test "bdev_qos_iops" run_qos_test $iops_limit IOPS $QOS_DEV_1 264 265 # Run bdevperf with bandwidth rate limit on bdev 2 266 # Set the bandwidth limit as 1/10 of the measure performance without QoS 267 bw_limit=$(get_io_result BANDWIDTH $QOS_DEV_2) 268 bw_limit=$((bw_limit / 1024 / 10)) 269 if [ $bw_limit -lt $qos_lower_bw_limit ]; then 270 bw_limit=$qos_lower_bw_limit 271 fi 272 $rpc_py bdev_set_qos_limit --rw_mbytes_per_sec $bw_limit $QOS_DEV_2 273 run_test "bdev_qos_bw" run_qos_test $bw_limit BANDWIDTH $QOS_DEV_2 274 275 # Run bdevperf with additional read only bandwidth rate limit on bdev 1 276 $rpc_py bdev_set_qos_limit --r_mbytes_per_sec $qos_lower_bw_limit $QOS_DEV_1 277 run_test "bdev_qos_ro_bw" run_qos_test $qos_lower_bw_limit BANDWIDTH $QOS_DEV_1 278 else 279 echo "Actual IOPS without limiting is too low - exit testing" 280 fi 281} 282 283function qos_test_suite() { 284 # Run bdevperf with QoS disabled first 285 "$testdir/bdevperf/bdevperf" -z -m 0x2 -q 256 -o 4096 -w randread -t 60 & 286 QOS_PID=$! 287 echo "Process qos testing pid: $QOS_PID" 288 trap 'killprocess $QOS_PID; exit 1' SIGINT SIGTERM EXIT 289 waitforlisten $QOS_PID 290 291 $rpc_py bdev_malloc_create -b $QOS_DEV_1 128 512 292 waitforbdev $QOS_DEV_1 293 $rpc_py bdev_null_create $QOS_DEV_2 128 512 294 waitforbdev $QOS_DEV_2 295 296 $rootdir/test/bdev/bdevperf/bdevperf.py perform_tests & 297 qos_function_test 298 299 $rpc_py bdev_malloc_delete $QOS_DEV_1 300 $rpc_py bdev_null_delete $QOS_DEV_2 301 killprocess $QOS_PID 302 trap - SIGINT SIGTERM EXIT 303} 304 305# Inital bdev creation and configuration 306#----------------------------------------------------- 307QOS_DEV_1="Malloc_0" 308QOS_DEV_2="Null_1" 309QOS_RUN_TIME=5 310 311if [ $(uname -s) = Linux ]; then 312 # Test dynamic memory management. All hugepages will be reserved at runtime 313 PRE_RESERVED_MEM=0 314else 315 # Dynamic memory management is not supported on BSD 316 PRE_RESERVED_MEM=2048 317fi 318 319test_type=${1:-bdev} 320start_spdk_tgt 321case "$test_type" in 322 bdev) 323 setup_bdev_conf 324 ;; 325 nvme) 326 setup_nvme_conf 327 ;; 328 gpt) 329 setup_gpt_conf 330 ;; 331 crypto_aesni) 332 setup_crypto_aesni_conf 333 ;; 334 crypto_qat) 335 setup_crypto_qat_conf 336 ;; 337 pmem) 338 setup_pmem_conf 339 ;; 340 rbd) 341 setup_rbd_conf 342 ;; 343 *) 344 echo "invalid test name" 345 exit 1 346 ;; 347esac 348 349"$rpc_py" bdev_wait_for_examine 350 351# Generate json config and use it throughout all the tests 352cat <<- CONF > "$conf_file" 353 {"subsystems":[ 354 $("$rpc_py" save_subsystem_config -n bdev) 355 ]} 356CONF 357 358bdevs=$("$rpc_py" bdev_get_bdevs | jq -r '.[] | select(.claimed == false)') 359bdevs_name=$(echo $bdevs | jq -r '.name') 360bdev_list=($bdevs_name) 361hello_world_bdev=${bdev_list[0]} 362trap - SIGINT SIGTERM EXIT 363killprocess "$spdk_tgt_pid" 364# End bdev configuration 365#----------------------------------------------------- 366 367run_test "bdev_hello_world" $SPDK_EXAMPLE_DIR/hello_bdev --json "$conf_file" -b "$hello_world_bdev" 368run_test "bdev_bounds" bdev_bounds 369run_test "bdev_nbd" nbd_function_test $conf_file "$bdevs_name" 370if [[ $CONFIG_FIO_PLUGIN == y ]]; then 371 if [ "$test_type" = "nvme" ] || [ "$test_type" = "gpt" ]; then 372 # TODO: once we get real multi-ns drives, re-enable this test for NVMe. 373 echo "skipping fio tests on NVMe due to multi-ns failures." 374 else 375 run_test "bdev_fio" fio_test_suite 376 fi 377else 378 echo "FIO not available" 379 exit 1 380fi 381 382run_test "bdev_verify" $testdir/bdevperf/bdevperf --json "$conf_file" -q 128 -o 4096 -w verify -t 5 -C -m 0x3 383run_test "bdev_write_zeroes" $testdir/bdevperf/bdevperf --json "$conf_file" -q 128 -o 4096 -w write_zeroes -t 1 384 385if [[ $test_type == bdev ]]; then 386 run_test "bdev_qos" qos_test_suite 387fi 388 389# Temporarily disabled - infinite loop 390# if [ $RUN_NIGHTLY -eq 1 ]; then 391# run_test "bdev_reset" $testdir/bdevperf/bdevperf --json "$conf_file" -q 16 -w reset -o 4096 -t 60 392# fi 393 394# Bdev and configuration cleanup below this line 395#----------------------------------------------------- 396if [ "$test_type" = "gpt" ]; then 397 "$rootdir/scripts/setup.sh" reset 398 if [[ -b $gpt_nvme ]]; then 399 dd if=/dev/zero of="$gpt_nvme" bs=4096 count=8 oflag=direct 400 fi 401fi 402 403cleanup 404