1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2019 Intel Corporation 4# All rights reserved. 5# 6testdir=$(readlink -f $(dirname $0)) 7rootdir=$(readlink -f $testdir/../..) 8rpc_server=/var/tmp/spdk-raid.sock 9tmp_file=$SPDK_TEST_STORAGE/raidrandtest 10 11source $rootdir/test/common/autotest_common.sh 12source $testdir/nbd_common.sh 13 14rpc_py="$rootdir/scripts/rpc.py -s $rpc_server" 15 16function raid_unmap_data_verify() { 17 if hash blkdiscard; then 18 local nbd=$1 19 local rpc_server=$2 20 local blksize 21 blksize=$(lsblk -o LOG-SEC $nbd | grep -v LOG-SEC | cut -d ' ' -f 5) 22 local rw_blk_num=4096 23 local rw_len=$((blksize * rw_blk_num)) 24 local unmap_blk_offs=(0 1028 321) 25 local unmap_blk_nums=(128 2035 456) 26 local unmap_off 27 local unmap_len 28 29 # data write 30 dd if=/dev/urandom of=$tmp_file bs=$blksize count=$rw_blk_num 31 dd if=$tmp_file of=$nbd bs=$blksize count=$rw_blk_num oflag=direct 32 blockdev --flushbufs $nbd 33 34 # confirm random data is written correctly in raid0 device 35 cmp -b -n $rw_len $tmp_file $nbd 36 37 for ((i = 0; i < ${#unmap_blk_offs[@]}; i++)); do 38 unmap_off=$((blksize * ${unmap_blk_offs[$i]})) 39 unmap_len=$((blksize * ${unmap_blk_nums[$i]})) 40 41 # data unmap on tmp_file 42 dd if=/dev/zero of=$tmp_file bs=$blksize seek=${unmap_blk_offs[$i]} count=${unmap_blk_nums[$i]} conv=notrunc 43 44 # data unmap on raid bdev 45 blkdiscard -o $unmap_off -l $unmap_len $nbd 46 blockdev --flushbufs $nbd 47 48 # data verify after unmap 49 cmp -b -n $rw_len $tmp_file $nbd 50 done 51 fi 52 53 return 0 54} 55 56function on_error_exit() { 57 if [ -n "$raid_pid" ]; then 58 killprocess $raid_pid 59 fi 60 61 rm -f $tmp_file 62 print_backtrace 63 exit 1 64} 65 66function configure_raid_bdev() { 67 local raid_level=$1 68 rm -rf $testdir/rpcs.txt 69 70 cat <<- EOL >> $testdir/rpcs.txt 71 bdev_malloc_create 32 $base_blocklen $base_malloc_params -b Base_1 72 bdev_malloc_create 32 $base_blocklen $base_malloc_params -b Base_2 73 bdev_raid_create -z 64 -r $raid_level -b "Base_1 Base_2" -n raid 74 EOL 75 $rpc_py < $testdir/rpcs.txt 76 77 rm -rf $testdir/rpcs.txt 78} 79 80function raid_function_test() { 81 local raid_level=$1 82 local nbd=/dev/nbd0 83 local raid_bdev 84 85 $rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -L bdev_raid & 86 raid_pid=$! 87 echo "Process raid pid: $raid_pid" 88 waitforlisten $raid_pid $rpc_server 89 90 configure_raid_bdev $raid_level 91 raid_bdev=$($rpc_py bdev_raid_get_bdevs online | jq -r '.[0]["name"] | select(.)') 92 if [ $raid_bdev = "" ]; then 93 echo "No raid0 device in SPDK app" 94 return 1 95 fi 96 97 nbd_start_disks $rpc_server $raid_bdev $nbd 98 count=$(nbd_get_count $rpc_server) 99 if [ $count -ne 1 ]; then 100 return 1 101 fi 102 103 raid_unmap_data_verify $nbd $rpc_server 104 105 nbd_stop_disks $rpc_server $nbd 106 count=$(nbd_get_count $rpc_server) 107 if [ $count -ne 0 ]; then 108 return 1 109 fi 110 111 killprocess $raid_pid 112 113 return 0 114} 115 116function verify_raid_bdev_state() { 117 local raid_bdev_name=$1 118 local expected_state=$2 119 local raid_level=$3 120 local strip_size=$4 121 local num_base_bdevs_operational=$5 122 local raid_bdev_info 123 local num_base_bdevs 124 local num_base_bdevs_discovered 125 local tmp 126 127 raid_bdev_info=$($rpc_py bdev_raid_get_bdevs all | jq -r ".[] | select(.name == \"$raid_bdev_name\")") 128 129 xtrace_disable 130 if [ -z "$raid_bdev_info" ]; then 131 echo "No raid device \"$raid_bdev_name\" in SPDK app" 132 return 1 133 fi 134 135 raid_bdev_info=$($rpc_py bdev_raid_get_bdevs $expected_state | jq -r ".[] | select(.name == \"$raid_bdev_name\")") 136 if [ -z "$raid_bdev_info" ]; then 137 echo "$raid_bdev_name is not in $expected_state state" 138 return 1 139 fi 140 141 tmp=$(echo $raid_bdev_info | jq -r '.state') 142 if [ "$tmp" != $expected_state ]; then 143 echo "incorrect state: $tmp, expected: $expected_state" 144 return 1 145 fi 146 147 tmp=$(echo $raid_bdev_info | jq -r '.raid_level') 148 if [ "$tmp" != $raid_level ]; then 149 echo "incorrect level: $tmp, expected: $raid_level" 150 return 1 151 fi 152 153 tmp=$(echo $raid_bdev_info | jq -r '.strip_size_kb') 154 if [ "$tmp" != $strip_size ]; then 155 echo "incorrect strip size: $tmp, expected: $strip_size" 156 return 1 157 fi 158 159 num_base_bdevs=$(echo $raid_bdev_info | jq -r '[.base_bdevs_list[]] | length') 160 tmp=$(echo $raid_bdev_info | jq -r '.num_base_bdevs') 161 if [ "$num_base_bdevs" != "$tmp" ]; then 162 echo "incorrect num_base_bdevs: $tmp, expected: $num_base_bdevs" 163 return 1 164 fi 165 166 num_base_bdevs_discovered=$(echo $raid_bdev_info | jq -r '[.base_bdevs_list[] | select(.is_configured)] | length') 167 tmp=$(echo $raid_bdev_info | jq -r '.num_base_bdevs_discovered') 168 if [ "$num_base_bdevs_discovered" != "$tmp" ]; then 169 echo "incorrect num_base_bdevs_discovered: $tmp, expected: $num_base_bdevs_discovered" 170 return 1 171 fi 172 173 tmp=$(echo $raid_bdev_info | jq -r '.num_base_bdevs_operational') 174 if [ "$num_base_bdevs_operational" != "$tmp" ]; then 175 echo "incorrect num_base_bdevs_operational $tmp, expected: $num_base_bdevs_operational" 176 return 1 177 fi 178 179 xtrace_restore 180} 181 182function verify_raid_bdev_process() { 183 local raid_bdev_name=$1 184 local process_type=$2 185 local target=$3 186 local raid_bdev_info 187 188 raid_bdev_info=$($rpc_py bdev_raid_get_bdevs all | jq -r ".[] | select(.name == \"$raid_bdev_name\")") 189 190 [[ $(jq -r '.process.type // "none"' <<< "$raid_bdev_info") == "$process_type" ]] 191 [[ $(jq -r '.process.target // "none"' <<< "$raid_bdev_info") == "$target" ]] 192} 193 194function verify_raid_bdev_properties() { 195 local raid_bdev_name=$1 196 local raid_bdev_info 197 local base_bdev_info 198 local base_bdev_names 199 local name 200 201 raid_bdev_info=$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq '.[]') 202 base_bdev_names=$(jq -r '.driver_specific.raid.base_bdevs_list[] | select(.is_configured == true).name' <<< "$raid_bdev_info") 203 204 for name in $base_bdev_names; do 205 base_bdev_info=$($rpc_py bdev_get_bdevs -b $name | jq '.[]') 206 [[ $(jq '.block_size' <<< "$raid_bdev_info") == $(jq '.block_size' <<< "$base_bdev_info") ]] 207 [[ $(jq '.md_size' <<< "$raid_bdev_info") == $(jq '.md_size' <<< "$base_bdev_info") ]] 208 [[ $(jq '.md_interleave' <<< "$raid_bdev_info") == $(jq '.md_interleave' <<< "$base_bdev_info") ]] 209 [[ $(jq '.dif_type' <<< "$raid_bdev_info") == $(jq '.dif_type' <<< "$base_bdev_info") ]] 210 done 211} 212 213function has_redundancy() { 214 case $1 in 215 "raid1" | "raid5f") return 0 ;; 216 *) return 1 ;; 217 esac 218} 219 220function raid_state_function_test() { 221 local raid_level=$1 222 local num_base_bdevs=$2 223 local superblock=$3 224 local raid_bdev 225 local base_bdevs=($(for ((i = 1; i <= num_base_bdevs; i++)); do echo BaseBdev$i; done)) 226 local raid_bdev_name="Existed_Raid" 227 local strip_size 228 local strip_size_create_arg 229 local superblock_create_arg 230 231 if [ $raid_level != "raid1" ]; then 232 strip_size=64 233 strip_size_create_arg="-z $strip_size" 234 else 235 strip_size=0 236 fi 237 238 if [ $superblock = true ]; then 239 superblock_create_arg="-s" 240 else 241 superblock_create_arg="" 242 fi 243 244 $rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -L bdev_raid & 245 raid_pid=$! 246 echo "Process raid pid: $raid_pid" 247 waitforlisten $raid_pid $rpc_server 248 249 # Step1: create a RAID bdev with no base bdevs 250 # Expect state: CONFIGURING 251 $rpc_py bdev_raid_create $strip_size_create_arg $superblock_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name 252 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 253 $rpc_py bdev_raid_delete $raid_bdev_name 254 255 # Step2: create one base bdev and add to the RAID bdev 256 # Expect state: CONFIGURING 257 $rpc_py bdev_raid_create $strip_size_create_arg $superblock_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name 258 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b ${base_bdevs[0]} 259 waitforbdev ${base_bdevs[0]} 260 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 261 $rpc_py bdev_raid_delete $raid_bdev_name 262 263 # Step3: create remaining base bdevs and add to the RAID bdev 264 # Expect state: ONLINE 265 $rpc_py bdev_raid_create $strip_size_create_arg $superblock_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name 266 for ((i = 1; i < num_base_bdevs; i++)); do 267 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 268 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b ${base_bdevs[$i]} 269 waitforbdev ${base_bdevs[$i]} 270 done 271 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs 272 verify_raid_bdev_properties $raid_bdev_name 273 274 # Step4: delete one base bdev from the RAID bdev 275 $rpc_py bdev_malloc_delete ${base_bdevs[0]} 276 local expected_state 277 if ! has_redundancy $raid_level; then 278 expected_state="offline" 279 else 280 expected_state="online" 281 fi 282 verify_raid_bdev_state $raid_bdev_name $expected_state $raid_level $strip_size $((num_base_bdevs - 1)) 283 284 # Step5: delete remaining base bdevs from the RAID bdev 285 # Expect state: removed from system 286 for ((i = 1; i < num_base_bdevs; i++)); do 287 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[0]["name"]') 288 if [ "$raid_bdev" != $raid_bdev_name ]; then 289 echo "$raid_bdev_name removed before all base bdevs were deleted" 290 return 1 291 fi 292 $rpc_py bdev_malloc_delete ${base_bdevs[$i]} 293 done 294 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[0]["name"] | select(.)') 295 if [ -n "$raid_bdev" ]; then 296 echo "$raid_bdev_name is not removed" 297 return 1 298 fi 299 300 if [ $num_base_bdevs -gt 2 ]; then 301 # Test removing and re-adding base bdevs when in CONFIGURING state 302 for ((i = 1; i < num_base_bdevs; i++)); do 303 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b ${base_bdevs[$i]} 304 waitforbdev ${base_bdevs[$i]} 305 done 306 $rpc_py bdev_raid_create $strip_size_create_arg $superblock_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name 307 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 308 309 $rpc_py bdev_raid_remove_base_bdev ${base_bdevs[1]} 310 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 311 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[1].is_configured') == "false" ]] 312 313 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b ${base_bdevs[0]} 314 waitforbdev ${base_bdevs[0]} 315 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 316 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[0].is_configured') == "true" ]] 317 318 $rpc_py bdev_raid_remove_base_bdev ${base_bdevs[2]} 319 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 320 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[2].is_configured') == "false" ]] 321 322 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name ${base_bdevs[2]} 323 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 324 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[2].is_configured') == "true" ]] 325 326 $rpc_py bdev_malloc_delete ${base_bdevs[0]} 327 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 328 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[0].is_configured') == "false" ]] 329 330 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name ${base_bdevs[1]} 331 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 332 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[1].is_configured') == "true" ]] 333 334 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b NewBaseBdev -u "$($rpc_py bdev_raid_get_bdevs all | jq -r '.[0].base_bdevs_list[0].uuid')" 335 waitforbdev NewBaseBdev 336 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs 337 verify_raid_bdev_properties $raid_bdev_name 338 339 $rpc_py bdev_raid_delete $raid_bdev_name 340 fi 341 342 killprocess $raid_pid 343 344 return 0 345} 346 347function raid0_resize_test() { 348 local blksize=$base_blocklen 349 local bdev_size_mb=32 350 local new_bdev_size_mb=$((bdev_size_mb * 2)) 351 local blkcnt 352 local raid_size_mb 353 local new_raid_size_mb 354 355 $rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -L bdev_raid & 356 raid_pid=$! 357 echo "Process raid pid: $raid_pid" 358 waitforlisten $raid_pid $rpc_server 359 360 $rpc_py bdev_null_create Base_1 $bdev_size_mb $blksize 361 $rpc_py bdev_null_create Base_2 $bdev_size_mb $blksize 362 363 $rpc_py bdev_raid_create -z 64 -r 0 -b "Base_1 Base_2" -n Raid 364 365 # Resize Base_1 first. 366 $rpc_py bdev_null_resize Base_1 $new_bdev_size_mb 367 368 # The size of Raid should not be changed. 369 blkcnt=$($rpc_py bdev_get_bdevs -b Raid | jq '.[].num_blocks') 370 raid_size_mb=$((blkcnt * blksize / 1048576)) 371 if [ $raid_size_mb != $((bdev_size_mb * 2)) ]; then 372 echo "resize failed" 373 return 1 374 fi 375 376 # Resize Base_2 next. 377 $rpc_py bdev_null_resize Base_2 $new_bdev_size_mb 378 379 # The size of Raid should be updated to the expected value. 380 blkcnt=$($rpc_py bdev_get_bdevs -b Raid | jq '.[].num_blocks') 381 raid_size_mb=$((blkcnt * blksize / 1048576)) 382 if [ $raid_size_mb != $((new_bdev_size_mb * 2)) ]; then 383 echo "resize failed" 384 return 1 385 fi 386 387 killprocess $raid_pid 388 389 return 0 390} 391 392function raid_superblock_test() { 393 local raid_level=$1 394 local num_base_bdevs=$2 395 local base_bdevs_malloc=() 396 local base_bdevs_pt=() 397 local base_bdevs_pt_uuid=() 398 local raid_bdev_name="raid_bdev1" 399 local strip_size 400 local strip_size_create_arg 401 local raid_bdev_uuid 402 local raid_bdev 403 404 if [ $raid_level != "raid1" ]; then 405 strip_size=64 406 strip_size_create_arg="-z $strip_size" 407 else 408 strip_size=0 409 fi 410 411 "$rootdir/test/app/bdev_svc/bdev_svc" -r $rpc_server -L bdev_raid & 412 raid_pid=$! 413 waitforlisten $raid_pid $rpc_server 414 415 # Create base bdevs 416 for ((i = 1; i <= num_base_bdevs; i++)); do 417 local bdev_malloc="malloc$i" 418 local bdev_pt="pt$i" 419 local bdev_pt_uuid="00000000-0000-0000-0000-00000000000$i" 420 421 base_bdevs_malloc+=($bdev_malloc) 422 base_bdevs_pt+=($bdev_pt) 423 base_bdevs_pt_uuid+=($bdev_pt_uuid) 424 425 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b $bdev_malloc 426 $rpc_py bdev_passthru_create -b $bdev_malloc -p $bdev_pt -u $bdev_pt_uuid 427 done 428 429 # Create RAID bdev with superblock 430 $rpc_py bdev_raid_create $strip_size_create_arg -r $raid_level -b "${base_bdevs_pt[*]}" -n $raid_bdev_name -s 431 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs 432 verify_raid_bdev_properties $raid_bdev_name 433 434 # Get RAID bdev's UUID 435 raid_bdev_uuid=$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[] | .uuid') 436 if [ -z "$raid_bdev_uuid" ]; then 437 return 1 438 fi 439 440 # Stop the RAID bdev 441 $rpc_py bdev_raid_delete $raid_bdev_name 442 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]') 443 if [ -n "$raid_bdev" ]; then 444 return 1 445 fi 446 447 # Delete the passthru bdevs 448 for i in "${base_bdevs_pt[@]}"; do 449 $rpc_py bdev_passthru_delete $i 450 done 451 if [ "$($rpc_py bdev_get_bdevs | jq -r '[.[] | select(.product_name == "passthru")] | any')" == "true" ]; then 452 return 1 453 fi 454 455 # Try to create new RAID bdev from malloc bdevs 456 # Should fail due to superblock still present on base bdevs 457 NOT $rpc_py bdev_raid_create $strip_size_create_arg -r $raid_level -b "${base_bdevs_malloc[*]}" -n $raid_bdev_name 458 459 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]') 460 if [ -n "$raid_bdev" ]; then 461 return 1 462 fi 463 464 # Re-add first base bdev 465 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[0]} -p ${base_bdevs_pt[0]} -u ${base_bdevs_pt_uuid[0]} 466 467 # Check if the RAID bdev was assembled from superblock 468 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 469 470 if [ $num_base_bdevs -gt 2 ]; then 471 # Re-add the second base bdev and remove it again 472 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[1]} -p ${base_bdevs_pt[1]} -u ${base_bdevs_pt_uuid[1]} 473 $rpc_py bdev_passthru_delete ${base_bdevs_pt[1]} 474 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 475 fi 476 477 # Re-add remaining base bdevs 478 for ((i = 1; i < num_base_bdevs; i++)); do 479 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]} 480 done 481 482 # Check if the RAID bdev is in online state 483 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs 484 verify_raid_bdev_properties $raid_bdev_name 485 486 # Check if the RAID bdev has the same UUID as when first created 487 if [ "$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[] | .uuid')" != "$raid_bdev_uuid" ]; then 488 return 1 489 fi 490 491 if has_redundancy $raid_level; then 492 # Delete one base bdev 493 $rpc_py bdev_passthru_delete ${base_bdevs_pt[0]} 494 495 # Check if the RAID bdev is in online state (degraded) 496 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 497 498 # Stop the RAID bdev 499 $rpc_py bdev_raid_delete $raid_bdev_name 500 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]') 501 if [ -n "$raid_bdev" ]; then 502 return 1 503 fi 504 505 # Delete remaining base bdevs 506 for ((i = 1; i < num_base_bdevs; i++)); do 507 $rpc_py bdev_passthru_delete ${base_bdevs_pt[$i]} 508 done 509 510 # Re-add base bdevs from the second up to (not including) the last one 511 for ((i = 1; i < num_base_bdevs - 1; i++)); do 512 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]} 513 514 # Check if the RAID bdev is in configuring state 515 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $((num_base_bdevs - 1)) 516 done 517 518 # Re-add the last base bdev 519 i=$((num_base_bdevs - 1)) 520 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]} 521 522 # Check if the RAID bdev is in online state (degraded) 523 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 524 525 if [ $num_base_bdevs -gt 2 ]; then 526 # Stop the RAID bdev 527 $rpc_py bdev_raid_delete $raid_bdev_name 528 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]') 529 if [ -n "$raid_bdev" ]; then 530 return 1 531 fi 532 533 # Re-add first base bdev 534 # This is the "failed" device and contains the "old" version of the superblock 535 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[0]} -p ${base_bdevs_pt[0]} -u ${base_bdevs_pt_uuid[0]} 536 537 # Check if the RAID bdev is in configuring state 538 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 539 540 # Delete remaining base bdevs 541 for ((i = 1; i < num_base_bdevs; i++)); do 542 $rpc_py bdev_passthru_delete ${base_bdevs_pt[$i]} 543 done 544 545 # Re-add the last base bdev 546 i=$((num_base_bdevs - 1)) 547 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]} 548 549 # Check if the RAID bdev is in configuring state 550 # This should use the newer superblock version and have n-1 online base bdevs 551 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $((num_base_bdevs - 1)) 552 553 # Re-add remaining base bdevs 554 for ((i = 1; i < num_base_bdevs - 1; i++)); do 555 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]} 556 done 557 558 # Check if the RAID bdev is in online state (degraded) 559 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 560 fi 561 562 # Check if the RAID bdev has the same UUID as when first created 563 if [ "$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[] | .uuid')" != "$raid_bdev_uuid" ]; then 564 return 1 565 fi 566 fi 567 568 killprocess $raid_pid 569 570 return 0 571} 572 573function raid_rebuild_test() { 574 local raid_level=$1 575 local num_base_bdevs=$2 576 local superblock=$3 577 local background_io=$4 578 local verify=$5 579 local base_bdevs=($(for ((i = 1; i <= num_base_bdevs; i++)); do echo BaseBdev$i; done)) 580 local raid_bdev_name="raid_bdev1" 581 local strip_size 582 local create_arg 583 local raid_bdev_size 584 local data_offset 585 586 if [ $raid_level != "raid1" ]; then 587 if [ $background_io = true ]; then 588 echo "skipping rebuild test with io for level $raid_level" 589 return 1 590 fi 591 strip_size=64 592 create_arg+=" -z $strip_size" 593 else 594 strip_size=0 595 fi 596 597 if [ $superblock = true ]; then 598 create_arg+=" -s" 599 fi 600 601 "$rootdir/build/examples/bdevperf" -r $rpc_server -T $raid_bdev_name -t 60 -w randrw -M 50 -o 3M -q 2 -U -z -L bdev_raid & 602 raid_pid=$! 603 waitforlisten $raid_pid $rpc_server 604 605 # Create base bdevs 606 for bdev in "${base_bdevs[@]}"; do 607 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b ${bdev}_malloc 608 $rpc_py bdev_passthru_create -b ${bdev}_malloc -p $bdev 609 done 610 611 # Create spare bdev 612 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b "spare_malloc" 613 $rpc_py bdev_delay_create -b "spare_malloc" -d "spare_delay" -r 0 -t 0 -w 100000 -n 100000 614 $rpc_py bdev_passthru_create -b "spare_delay" -p "spare" 615 616 # Create RAID bdev 617 $rpc_py bdev_raid_create $create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name 618 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs 619 620 # Get RAID bdev's size 621 raid_bdev_size=$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[].num_blocks') 622 623 # Get base bdev's data offset 624 data_offset=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[].base_bdevs_list[0].data_offset') 625 626 if [ $background_io = true ]; then 627 # Start user I/O 628 "$rootdir/examples/bdev/bdevperf/bdevperf.py" -s $rpc_server perform_tests & 629 elif [ $verify = true ]; then 630 local write_unit_size 631 632 # Write random data to the RAID bdev 633 nbd_start_disks $rpc_server $raid_bdev_name /dev/nbd0 634 if [ $raid_level = "raid5f" ]; then 635 write_unit_size=$((strip_size * 2 * (num_base_bdevs - 1))) 636 echo $((base_blocklen * write_unit_size / 1024)) > /sys/block/nbd0/queue/max_sectors_kb 637 else 638 write_unit_size=1 639 fi 640 dd if=/dev/urandom of=/dev/nbd0 bs=$((base_blocklen * write_unit_size)) count=$((raid_bdev_size / write_unit_size)) oflag=direct 641 nbd_stop_disks $rpc_server /dev/nbd0 642 fi 643 644 # Remove one base bdev 645 $rpc_py bdev_raid_remove_base_bdev ${base_bdevs[0]} 646 647 # Check if the RAID bdev is in online state (degraded) 648 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 649 650 # Add bdev for rebuild 651 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name "spare" 652 sleep 1 653 654 # Check if rebuild started 655 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 656 657 # Remove the rebuild target bdev 658 $rpc_py bdev_raid_remove_base_bdev "spare" 659 660 # Check if the RAID bdev is in online state (degraded) 661 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 662 663 # Check if rebuild was stopped 664 verify_raid_bdev_process $raid_bdev_name "none" "none" 665 666 # Again, start the rebuild 667 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name "spare" 668 sleep 1 669 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 670 671 if [ $superblock = true ] && [ $with_io = false ]; then 672 # Stop the RAID bdev 673 $rpc_py bdev_raid_delete $raid_bdev_name 674 [[ $($rpc_py bdev_raid_get_bdevs all | jq 'length') == 0 ]] 675 676 # Remove the passthru base bdevs, then re-add them to assemble the raid bdev again 677 for ((i = 0; i < num_base_bdevs; i++)); do 678 $rpc_py bdev_passthru_delete ${base_bdevs[$i]} 679 done 680 for ((i = 0; i < num_base_bdevs; i++)); do 681 $rpc_py bdev_passthru_create -b ${base_bdevs[$i]}_malloc -p ${base_bdevs[$i]} 682 done 683 684 # Check if the RAID bdev is in online state (degraded) 685 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 686 687 # Check if rebuild is not started 688 verify_raid_bdev_process $raid_bdev_name "none" "none" 689 690 # Again, start the rebuild 691 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name "spare" 692 sleep 1 693 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 694 fi 695 696 local num_base_bdevs_operational=$num_base_bdevs 697 698 if [ $raid_level = "raid1" ] && [ $num_base_bdevs -gt 2 ]; then 699 # Remove one more base bdev (not rebuild target) 700 $rpc_py bdev_raid_remove_base_bdev ${base_bdevs[1]} 701 702 # Ignore this bdev later when comparing data 703 base_bdevs[1]="" 704 ((num_base_bdevs_operational--)) 705 706 # Check if rebuild is still running 707 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 708 fi 709 710 # Wait for rebuild to finish 711 local timeout=$((SECONDS + 30)) 712 while ((SECONDS < timeout)); do 713 if ! verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" > /dev/null; then 714 break 715 fi 716 sleep 1 717 done 718 719 # Check if rebuild is not running and the RAID bdev has the correct number of operational devices 720 verify_raid_bdev_process $raid_bdev_name "none" "none" 721 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs_operational 722 723 # Stop the RAID bdev 724 $rpc_py bdev_raid_delete $raid_bdev_name 725 [[ $($rpc_py bdev_raid_get_bdevs all | jq 'length') == 0 ]] 726 727 if [ $verify = true ]; then 728 if [ $background_io = true ]; then 729 # Compare data on the rebuilt and other base bdevs 730 nbd_start_disks $rpc_server "spare" "/dev/nbd0" 731 for bdev in "${base_bdevs[@]:1}"; do 732 if [ -z "$bdev" ]; then 733 continue 734 fi 735 nbd_start_disks $rpc_server $bdev "/dev/nbd1" 736 cmp -i $((data_offset * base_blocklen)) /dev/nbd0 /dev/nbd1 737 nbd_stop_disks $rpc_server "/dev/nbd1" 738 done 739 nbd_stop_disks $rpc_server "/dev/nbd0" 740 else 741 # Compare data on the removed and rebuilt base bdevs 742 nbd_start_disks $rpc_server "${base_bdevs[0]} spare" "/dev/nbd0 /dev/nbd1" 743 cmp -i $((data_offset * base_blocklen)) /dev/nbd0 /dev/nbd1 744 nbd_stop_disks $rpc_server "/dev/nbd0 /dev/nbd1" 745 fi 746 fi 747 748 if [ $superblock = true ]; then 749 # Remove the passthru base bdevs, then re-add them to assemble the raid bdev again 750 for bdev in "${base_bdevs[@]}"; do 751 if [ -z "$bdev" ]; then 752 continue 753 fi 754 $rpc_py bdev_passthru_delete $bdev 755 $rpc_py bdev_passthru_create -b ${bdev}_malloc -p $bdev 756 done 757 $rpc_py bdev_passthru_delete "spare" 758 $rpc_py bdev_passthru_create -b "spare_delay" -p "spare" 759 760 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs_operational 761 verify_raid_bdev_process $raid_bdev_name "none" "none" 762 [[ $($rpc_py bdev_raid_get_bdevs all | jq -r '.[].base_bdevs_list[0].name') == "spare" ]] 763 764 # Remove and re-add a base bdev - rebuild should start automatically 765 $rpc_py bdev_raid_remove_base_bdev "spare" 766 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs_operational - 1)) 767 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name "spare" 768 sleep 1 769 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 770 771 # Same as above but re-add through examine 772 $rpc_py bdev_passthru_delete "spare" 773 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs_operational - 1)) 774 $rpc_py bdev_passthru_create -b "spare_delay" -p "spare" 775 sleep 1 776 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 777 778 # Stop the rebuild 779 $rpc_py bdev_passthru_delete "spare" 780 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs_operational - 1)) 781 verify_raid_bdev_process $raid_bdev_name "none" "none" 782 783 # Re-adding a base bdev that was replaced (no longer is a member of the array) should not be allowed 784 $rpc_py bdev_passthru_delete ${base_bdevs[0]} 785 $rpc_py bdev_passthru_create -b ${base_bdevs[0]}_malloc -p ${base_bdevs[0]} 786 sleep 1 787 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs_operational - 1)) 788 verify_raid_bdev_process $raid_bdev_name "none" "none" 789 NOT $rpc_py bdev_raid_add_base_bdev $raid_bdev_name ${base_bdevs[0]} 790 sleep 1 791 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs_operational - 1)) 792 verify_raid_bdev_process $raid_bdev_name "none" "none" 793 fi 794 795 killprocess $raid_pid 796 797 return 0 798} 799 800trap 'on_error_exit;' ERR 801 802base_blocklen=512 803 804if [ $(uname -s) = Linux ] && modprobe -n nbd; then 805 has_nbd=true 806 modprobe nbd 807 run_test "raid_function_test_raid0" raid_function_test raid0 808 run_test "raid_function_test_concat" raid_function_test concat 809fi 810 811run_test "raid0_resize_test" raid0_resize_test 812 813for n in {2..4}; do 814 for level in raid0 concat raid1; do 815 run_test "raid_state_function_test" raid_state_function_test $level $n false 816 run_test "raid_state_function_test_sb" raid_state_function_test $level $n true 817 run_test "raid_superblock_test" raid_superblock_test $level $n 818 done 819done 820 821if [ "$has_nbd" = true ]; then 822 for n in 2 4; do 823 run_test "raid_rebuild_test" raid_rebuild_test raid1 $n false false true 824 run_test "raid_rebuild_test_sb" raid_rebuild_test raid1 $n true false true 825 run_test "raid_rebuild_test_io" raid_rebuild_test raid1 $n false true true 826 run_test "raid_rebuild_test_sb_io" raid_rebuild_test raid1 $n true true true 827 done 828fi 829 830if [ "$CONFIG_RAID5F" == y ]; then 831 for n in {3..4}; do 832 run_test "raid5f_state_function_test" raid_state_function_test raid5f $n false 833 run_test "raid5f_state_function_test_sb" raid_state_function_test raid5f $n true 834 run_test "raid5f_superblock_test" raid_superblock_test raid5f $n 835 if [ "$has_nbd" = true ]; then 836 run_test "raid5f_rebuild_test" raid_rebuild_test raid5f $n false false true 837 run_test "raid5f_rebuild_test_sb" raid_rebuild_test raid5f $n true false true 838 fi 839 done 840fi 841 842base_blocklen=4096 843 844run_test "raid_state_function_test_sb_4k" raid_state_function_test raid1 2 true 845run_test "raid_superblock_test_4k" raid_superblock_test raid1 2 846if [ "$has_nbd" = true ]; then 847 run_test "raid_rebuild_test_sb_4k" raid_rebuild_test raid1 2 true false true 848fi 849 850base_malloc_params="-m 32" 851run_test "raid_state_function_test_sb_md_separate" raid_state_function_test raid1 2 true 852run_test "raid_superblock_test_md_separate" raid_superblock_test raid1 2 853if [ "$has_nbd" = true ]; then 854 run_test "raid_rebuild_test_sb_md_separate" raid_rebuild_test raid1 2 true false true 855fi 856 857base_malloc_params="-m 32 -i" 858run_test "raid_state_function_test_sb_md_interleaved" raid_state_function_test raid1 2 true 859run_test "raid_superblock_test_md_interleaved" raid_superblock_test raid1 2 860run_test "raid_rebuild_test_sb_md_interleaved" raid_rebuild_test raid1 2 true false false 861 862rm -f $tmp_file 863