1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2019 Intel Corporation 4# All rights reserved. 5# 6testdir=$(readlink -f $(dirname $0)) 7rootdir=$(readlink -f $testdir/../..) 8rpc_server=/var/tmp/spdk-raid.sock 9tmp_file=$SPDK_TEST_STORAGE/raidrandtest 10 11source $rootdir/test/common/autotest_common.sh 12source $testdir/nbd_common.sh 13 14rpc_py="$rootdir/scripts/rpc.py -s $rpc_server" 15 16function raid_unmap_data_verify() { 17 if hash blkdiscard; then 18 local nbd=$1 19 local rpc_server=$2 20 local blksize 21 blksize=$(lsblk -o LOG-SEC $nbd | grep -v LOG-SEC | cut -d ' ' -f 5) 22 local rw_blk_num=4096 23 local rw_len=$((blksize * rw_blk_num)) 24 local unmap_blk_offs=(0 1028 321) 25 local unmap_blk_nums=(128 2035 456) 26 local unmap_off 27 local unmap_len 28 29 # data write 30 dd if=/dev/urandom of=$tmp_file bs=$blksize count=$rw_blk_num 31 dd if=$tmp_file of=$nbd bs=$blksize count=$rw_blk_num oflag=direct 32 blockdev --flushbufs $nbd 33 34 # confirm random data is written correctly in raid0 device 35 cmp -b -n $rw_len $tmp_file $nbd 36 37 for ((i = 0; i < ${#unmap_blk_offs[@]}; i++)); do 38 unmap_off=$((blksize * ${unmap_blk_offs[$i]})) 39 unmap_len=$((blksize * ${unmap_blk_nums[$i]})) 40 41 # data unmap on tmp_file 42 dd if=/dev/zero of=$tmp_file bs=$blksize seek=${unmap_blk_offs[$i]} count=${unmap_blk_nums[$i]} conv=notrunc 43 44 # data unmap on raid bdev 45 blkdiscard -o $unmap_off -l $unmap_len $nbd 46 blockdev --flushbufs $nbd 47 48 # data verify after unmap 49 cmp -b -n $rw_len $tmp_file $nbd 50 done 51 fi 52 53 return 0 54} 55 56function on_error_exit() { 57 if [ -n "$raid_pid" ]; then 58 killprocess $raid_pid 59 fi 60 61 rm -f $tmp_file 62 print_backtrace 63 exit 1 64} 65 66function configure_raid_bdev() { 67 local raid_level=$1 68 rm -rf $testdir/rpcs.txt 69 70 cat <<- EOL >> $testdir/rpcs.txt 71 bdev_malloc_create 32 $base_blocklen $base_malloc_params -b Base_1 72 bdev_malloc_create 32 $base_blocklen $base_malloc_params -b Base_2 73 bdev_raid_create -z 64 -r $raid_level -b "Base_1 Base_2" -n raid 74 EOL 75 $rpc_py < $testdir/rpcs.txt 76 77 rm -rf $testdir/rpcs.txt 78} 79 80function raid_function_test() { 81 local raid_level=$1 82 local nbd=/dev/nbd0 83 local raid_bdev 84 85 $rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -L bdev_raid & 86 raid_pid=$! 87 echo "Process raid pid: $raid_pid" 88 waitforlisten $raid_pid $rpc_server 89 90 configure_raid_bdev $raid_level 91 raid_bdev=$($rpc_py bdev_raid_get_bdevs online | jq -r '.[0]["name"] | select(.)') 92 if [ $raid_bdev = "" ]; then 93 echo "No raid0 device in SPDK app" 94 return 1 95 fi 96 97 nbd_start_disks $rpc_server $raid_bdev $nbd 98 count=$(nbd_get_count $rpc_server) 99 if [ $count -ne 1 ]; then 100 return 1 101 fi 102 103 raid_unmap_data_verify $nbd $rpc_server 104 105 nbd_stop_disks $rpc_server $nbd 106 count=$(nbd_get_count $rpc_server) 107 if [ $count -ne 0 ]; then 108 return 1 109 fi 110 111 killprocess $raid_pid 112 113 return 0 114} 115 116function verify_raid_bdev_state() { 117 local raid_bdev_name=$1 118 local expected_state=$2 119 local raid_level=$3 120 local strip_size=$4 121 local num_base_bdevs_operational=$5 122 local raid_bdev_info 123 local num_base_bdevs 124 local num_base_bdevs_discovered 125 local tmp 126 127 raid_bdev_info=$($rpc_py bdev_raid_get_bdevs all | jq -r ".[] | select(.name == \"$raid_bdev_name\")") 128 129 xtrace_disable 130 if [ -z "$raid_bdev_info" ]; then 131 echo "No raid device \"$raid_bdev_name\" in SPDK app" 132 return 1 133 fi 134 135 raid_bdev_info=$($rpc_py bdev_raid_get_bdevs $expected_state | jq -r ".[] | select(.name == \"$raid_bdev_name\")") 136 if [ -z "$raid_bdev_info" ]; then 137 echo "$raid_bdev_name is not in $expected_state state" 138 return 1 139 fi 140 141 tmp=$(echo $raid_bdev_info | jq -r '.state') 142 if [ "$tmp" != $expected_state ]; then 143 echo "incorrect state: $tmp, expected: $expected_state" 144 return 1 145 fi 146 147 tmp=$(echo $raid_bdev_info | jq -r '.raid_level') 148 if [ "$tmp" != $raid_level ]; then 149 echo "incorrect level: $tmp, expected: $raid_level" 150 return 1 151 fi 152 153 tmp=$(echo $raid_bdev_info | jq -r '.strip_size_kb') 154 if [ "$tmp" != $strip_size ]; then 155 echo "incorrect strip size: $tmp, expected: $strip_size" 156 return 1 157 fi 158 159 num_base_bdevs=$(echo $raid_bdev_info | jq -r '[.base_bdevs_list[]] | length') 160 tmp=$(echo $raid_bdev_info | jq -r '.num_base_bdevs') 161 if [ "$num_base_bdevs" != "$tmp" ]; then 162 echo "incorrect num_base_bdevs: $tmp, expected: $num_base_bdevs" 163 return 1 164 fi 165 166 num_base_bdevs_discovered=$(echo $raid_bdev_info | jq -r '[.base_bdevs_list[] | select(.is_configured)] | length') 167 tmp=$(echo $raid_bdev_info | jq -r '.num_base_bdevs_discovered') 168 if [ "$num_base_bdevs_discovered" != "$tmp" ]; then 169 echo "incorrect num_base_bdevs_discovered: $tmp, expected: $num_base_bdevs_discovered" 170 return 1 171 fi 172 173 tmp=$(echo $raid_bdev_info | jq -r '.num_base_bdevs_operational') 174 if [ "$num_base_bdevs_operational" != "$tmp" ]; then 175 echo "incorrect num_base_bdevs_operational $tmp, expected: $num_base_bdevs_operational" 176 return 1 177 fi 178 179 xtrace_restore 180} 181 182function verify_raid_bdev_process() { 183 local raid_bdev_name=$1 184 local process_type=$2 185 local target=$3 186 local raid_bdev_info 187 188 raid_bdev_info=$($rpc_py bdev_raid_get_bdevs all | jq -r ".[] | select(.name == \"$raid_bdev_name\")") 189 190 [[ $(jq -r '.process.type // "none"' <<< "$raid_bdev_info") == "$process_type" ]] 191 [[ $(jq -r '.process.target // "none"' <<< "$raid_bdev_info") == "$target" ]] 192} 193 194function verify_raid_bdev_properties() { 195 local raid_bdev_name=$1 196 local raid_bdev_info 197 local base_bdev_info 198 local base_bdev_names 199 local name 200 201 raid_bdev_info=$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq '.[]') 202 base_bdev_names=$(jq -r '.driver_specific.raid.base_bdevs_list[] | select(.is_configured == true).name' <<< "$raid_bdev_info") 203 204 for name in $base_bdev_names; do 205 base_bdev_info=$($rpc_py bdev_get_bdevs -b $name | jq '.[]') 206 [[ $(jq '.block_size' <<< "$raid_bdev_info") == $(jq '.block_size' <<< "$base_bdev_info") ]] 207 [[ $(jq '.md_size' <<< "$raid_bdev_info") == $(jq '.md_size' <<< "$base_bdev_info") ]] 208 [[ $(jq '.md_interleave' <<< "$raid_bdev_info") == $(jq '.md_interleave' <<< "$base_bdev_info") ]] 209 [[ $(jq '.dif_type' <<< "$raid_bdev_info") == $(jq '.dif_type' <<< "$base_bdev_info") ]] 210 done 211} 212 213function has_redundancy() { 214 case $1 in 215 "raid1" | "raid5f") return 0 ;; 216 *) return 1 ;; 217 esac 218} 219 220function raid_state_function_test() { 221 local raid_level=$1 222 local num_base_bdevs=$2 223 local superblock=$3 224 local raid_bdev 225 local base_bdevs=($(for ((i = 1; i <= num_base_bdevs; i++)); do echo BaseBdev$i; done)) 226 local raid_bdev_name="Existed_Raid" 227 local strip_size 228 local strip_size_create_arg 229 local superblock_create_arg 230 231 if [ $raid_level != "raid1" ]; then 232 strip_size=64 233 strip_size_create_arg="-z $strip_size" 234 else 235 strip_size=0 236 fi 237 238 if [ $superblock = true ]; then 239 superblock_create_arg="-s" 240 else 241 superblock_create_arg="" 242 fi 243 244 $rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -L bdev_raid & 245 raid_pid=$! 246 echo "Process raid pid: $raid_pid" 247 waitforlisten $raid_pid $rpc_server 248 249 # Step1: create a RAID bdev with no base bdevs 250 # Expect state: CONFIGURING 251 $rpc_py bdev_raid_create $strip_size_create_arg $superblock_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name 252 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 253 $rpc_py bdev_raid_delete $raid_bdev_name 254 255 # Step2: create one base bdev and add to the RAID bdev 256 # Expect state: CONFIGURING 257 $rpc_py bdev_raid_create $strip_size_create_arg $superblock_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name 258 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b ${base_bdevs[0]} 259 waitforbdev ${base_bdevs[0]} 260 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 261 $rpc_py bdev_raid_delete $raid_bdev_name 262 263 # Step3: create remaining base bdevs and add to the RAID bdev 264 # Expect state: ONLINE 265 $rpc_py bdev_raid_create $strip_size_create_arg $superblock_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name 266 for ((i = 1; i < num_base_bdevs; i++)); do 267 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 268 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b ${base_bdevs[$i]} 269 waitforbdev ${base_bdevs[$i]} 270 done 271 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs 272 verify_raid_bdev_properties $raid_bdev_name 273 274 # Step4: delete one base bdev from the RAID bdev 275 $rpc_py bdev_malloc_delete ${base_bdevs[0]} 276 local expected_state 277 if ! has_redundancy $raid_level; then 278 expected_state="offline" 279 else 280 expected_state="online" 281 fi 282 verify_raid_bdev_state $raid_bdev_name $expected_state $raid_level $strip_size $((num_base_bdevs - 1)) 283 284 # Step5: delete remaining base bdevs from the RAID bdev 285 # Expect state: removed from system 286 for ((i = 1; i < num_base_bdevs; i++)); do 287 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[0]["name"]') 288 if [ "$raid_bdev" != $raid_bdev_name ]; then 289 echo "$raid_bdev_name removed before all base bdevs were deleted" 290 return 1 291 fi 292 $rpc_py bdev_malloc_delete ${base_bdevs[$i]} 293 done 294 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[0]["name"] | select(.)') 295 if [ -n "$raid_bdev" ]; then 296 echo "$raid_bdev_name is not removed" 297 return 1 298 fi 299 300 if [ $num_base_bdevs -gt 2 ]; then 301 # Test removing and re-adding base bdevs when in CONFIGURING state 302 for ((i = 1; i < num_base_bdevs; i++)); do 303 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b ${base_bdevs[$i]} 304 waitforbdev ${base_bdevs[$i]} 305 done 306 $rpc_py bdev_raid_create $strip_size_create_arg $superblock_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name 307 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 308 309 $rpc_py bdev_raid_remove_base_bdev ${base_bdevs[1]} 310 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 311 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[1].is_configured') == "false" ]] 312 313 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b ${base_bdevs[0]} 314 waitforbdev ${base_bdevs[0]} 315 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 316 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[0].is_configured') == "true" ]] 317 318 $rpc_py bdev_raid_remove_base_bdev ${base_bdevs[2]} 319 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 320 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[2].is_configured') == "false" ]] 321 322 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name ${base_bdevs[2]} 323 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 324 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[2].is_configured') == "true" ]] 325 326 $rpc_py bdev_malloc_delete ${base_bdevs[0]} 327 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 328 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[0].is_configured') == "false" ]] 329 330 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name ${base_bdevs[1]} 331 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 332 [[ $($rpc_py bdev_raid_get_bdevs all | jq '.[0].base_bdevs_list[1].is_configured') == "true" ]] 333 334 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b NewBaseBdev -u "$($rpc_py bdev_raid_get_bdevs all | jq -r '.[0].base_bdevs_list[0].uuid')" 335 waitforbdev NewBaseBdev 336 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs 337 verify_raid_bdev_properties $raid_bdev_name 338 339 $rpc_py bdev_raid_delete $raid_bdev_name 340 fi 341 342 killprocess $raid_pid 343 344 return 0 345} 346 347function raid0_resize_test() { 348 local blksize=$base_blocklen 349 local bdev_size_mb=32 350 local new_bdev_size_mb=$((bdev_size_mb * 2)) 351 local blkcnt 352 local raid_size_mb 353 local new_raid_size_mb 354 355 $rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -L bdev_raid & 356 raid_pid=$! 357 echo "Process raid pid: $raid_pid" 358 waitforlisten $raid_pid $rpc_server 359 360 $rpc_py bdev_null_create Base_1 $bdev_size_mb $blksize 361 $rpc_py bdev_null_create Base_2 $bdev_size_mb $blksize 362 363 $rpc_py bdev_raid_create -z 64 -r 0 -b "Base_1 Base_2" -n Raid 364 365 # Resize Base_1 first. 366 $rpc_py bdev_null_resize Base_1 $new_bdev_size_mb 367 368 # The size of Raid should not be changed. 369 blkcnt=$($rpc_py bdev_get_bdevs -b Raid | jq '.[].num_blocks') 370 raid_size_mb=$((blkcnt * blksize / 1048576)) 371 if [ $raid_size_mb != $((bdev_size_mb * 2)) ]; then 372 echo "resize failed" 373 return 1 374 fi 375 376 # Resize Base_2 next. 377 $rpc_py bdev_null_resize Base_2 $new_bdev_size_mb 378 379 # The size of Raid should be updated to the expected value. 380 blkcnt=$($rpc_py bdev_get_bdevs -b Raid | jq '.[].num_blocks') 381 raid_size_mb=$((blkcnt * blksize / 1048576)) 382 if [ $raid_size_mb != $((new_bdev_size_mb * 2)) ]; then 383 echo "resize failed" 384 return 1 385 fi 386 387 killprocess $raid_pid 388 389 return 0 390} 391 392function raid_superblock_test() { 393 local raid_level=$1 394 local num_base_bdevs=$2 395 local base_bdevs_malloc=() 396 local base_bdevs_pt=() 397 local base_bdevs_pt_uuid=() 398 local raid_bdev_name="raid_bdev1" 399 local strip_size 400 local strip_size_create_arg 401 local raid_bdev_uuid 402 local raid_bdev 403 404 if [ $raid_level != "raid1" ]; then 405 strip_size=64 406 strip_size_create_arg="-z $strip_size" 407 else 408 strip_size=0 409 fi 410 411 "$rootdir/test/app/bdev_svc/bdev_svc" -r $rpc_server -L bdev_raid & 412 raid_pid=$! 413 waitforlisten $raid_pid $rpc_server 414 415 # Create base bdevs 416 for ((i = 1; i <= num_base_bdevs; i++)); do 417 local bdev_malloc="malloc$i" 418 local bdev_pt="pt$i" 419 local bdev_pt_uuid="00000000-0000-0000-0000-00000000000$i" 420 421 base_bdevs_malloc+=($bdev_malloc) 422 base_bdevs_pt+=($bdev_pt) 423 base_bdevs_pt_uuid+=($bdev_pt_uuid) 424 425 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b $bdev_malloc 426 $rpc_py bdev_passthru_create -b $bdev_malloc -p $bdev_pt -u $bdev_pt_uuid 427 done 428 429 # Create RAID bdev with superblock 430 $rpc_py bdev_raid_create $strip_size_create_arg -r $raid_level -b "${base_bdevs_pt[*]}" -n $raid_bdev_name -s 431 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs 432 verify_raid_bdev_properties $raid_bdev_name 433 434 # Get RAID bdev's UUID 435 raid_bdev_uuid=$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[] | .uuid') 436 if [ -z "$raid_bdev_uuid" ]; then 437 return 1 438 fi 439 440 # Stop the RAID bdev 441 $rpc_py bdev_raid_delete $raid_bdev_name 442 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]') 443 if [ -n "$raid_bdev" ]; then 444 return 1 445 fi 446 447 # Delete the passthru bdevs 448 for i in "${base_bdevs_pt[@]}"; do 449 $rpc_py bdev_passthru_delete $i 450 done 451 if [ "$($rpc_py bdev_get_bdevs | jq -r '[.[] | select(.product_name == "passthru")] | any')" == "true" ]; then 452 return 1 453 fi 454 455 # Try to create new RAID bdev from malloc bdevs 456 # Should fail due to superblock still present on base bdevs 457 NOT $rpc_py bdev_raid_create $strip_size_create_arg -r $raid_level -b "${base_bdevs_malloc[*]}" -n $raid_bdev_name 458 459 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]') 460 if [ -n "$raid_bdev" ]; then 461 return 1 462 fi 463 464 # Re-add first base bdev 465 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[0]} -p ${base_bdevs_pt[0]} -u ${base_bdevs_pt_uuid[0]} 466 467 # Check if the RAID bdev was assembled from superblock 468 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 469 470 if [ $num_base_bdevs -gt 2 ]; then 471 # Re-add the second base bdev and remove it again 472 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[1]} -p ${base_bdevs_pt[1]} -u ${base_bdevs_pt_uuid[1]} 473 $rpc_py bdev_passthru_delete ${base_bdevs_pt[1]} 474 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs 475 fi 476 477 # Re-add remaining base bdevs 478 for ((i = 1; i < num_base_bdevs; i++)); do 479 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]} 480 done 481 482 # Check if the RAID bdev is in online state 483 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs 484 verify_raid_bdev_properties $raid_bdev_name 485 486 # Check if the RAID bdev has the same UUID as when first created 487 if [ "$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[] | .uuid')" != "$raid_bdev_uuid" ]; then 488 return 1 489 fi 490 491 if has_redundancy $raid_level; then 492 # Delete one base bdev 493 $rpc_py bdev_passthru_delete ${base_bdevs_pt[0]} 494 495 # Check if the RAID bdev is in online state (degraded) 496 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 497 498 # Stop the RAID bdev 499 $rpc_py bdev_raid_delete $raid_bdev_name 500 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]') 501 if [ -n "$raid_bdev" ]; then 502 return 1 503 fi 504 505 # Delete remaining base bdevs 506 for ((i = 1; i < num_base_bdevs; i++)); do 507 $rpc_py bdev_passthru_delete ${base_bdevs_pt[$i]} 508 done 509 510 # Re-add base bdevs from the second up to (not including) the last one 511 for ((i = 1; i < num_base_bdevs - 1; i++)); do 512 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]} 513 514 # Check if the RAID bdev is in configuring state 515 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $((num_base_bdevs - 1)) 516 done 517 518 # Re-add the last base bdev 519 i=$((num_base_bdevs - 1)) 520 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]} 521 522 # Check if the RAID bdev is in online state (degraded) 523 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 524 525 # Stop the RAID bdev 526 $rpc_py bdev_raid_delete $raid_bdev_name 527 raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]') 528 if [ -n "$raid_bdev" ]; then 529 return 1 530 fi 531 532 if [ $num_base_bdevs -gt 2 ]; then 533 # Delete the last base bdev 534 i=$((num_base_bdevs - 1)) 535 $rpc_py bdev_passthru_delete ${base_bdevs_pt[$i]} 536 fi 537 538 # Re-add first base bdev 539 # This is the "failed" device and contains the "old" version of the superblock 540 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[0]} -p ${base_bdevs_pt[0]} -u ${base_bdevs_pt_uuid[0]} 541 542 if [ $num_base_bdevs -gt 2 ]; then 543 # Check if the RAID bdev is in configuring state 544 # This should use the newer superblock version and have n-1 online base bdevs 545 verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $((num_base_bdevs - 1)) 546 [[ $($rpc_py bdev_raid_get_bdevs configuring | jq -r '.[].base_bdevs_list[0].is_configured') == "false" ]] 547 548 # Re-add the last base bdev 549 $rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]} 550 fi 551 552 # Check if the RAID bdev is in online state (degraded) 553 # This should use the newer superblock version and have n-1 online base bdevs 554 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 555 [[ $($rpc_py bdev_raid_get_bdevs online | jq -r '.[].base_bdevs_list[0].is_configured') == "false" ]] 556 557 # Check if the RAID bdev has the same UUID as when first created 558 if [ "$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[] | .uuid')" != "$raid_bdev_uuid" ]; then 559 return 1 560 fi 561 fi 562 563 killprocess $raid_pid 564 565 return 0 566} 567 568function raid_rebuild_test() { 569 local raid_level=$1 570 local num_base_bdevs=$2 571 local superblock=$3 572 local background_io=$4 573 local verify=$5 574 local base_bdevs=($(for ((i = 1; i <= num_base_bdevs; i++)); do echo BaseBdev$i; done)) 575 local raid_bdev_name="raid_bdev1" 576 local strip_size 577 local create_arg 578 local raid_bdev_size 579 local data_offset 580 581 if [ $raid_level != "raid1" ]; then 582 if [ $background_io = true ]; then 583 echo "skipping rebuild test with io for level $raid_level" 584 return 1 585 fi 586 strip_size=64 587 create_arg+=" -z $strip_size" 588 else 589 strip_size=0 590 fi 591 592 if [ $superblock = true ]; then 593 create_arg+=" -s" 594 fi 595 596 "$rootdir/build/examples/bdevperf" -r $rpc_server -T $raid_bdev_name -t 60 -w randrw -M 50 -o 3M -q 2 -U -z -L bdev_raid & 597 raid_pid=$! 598 waitforlisten $raid_pid $rpc_server 599 600 # Create base bdevs 601 for bdev in "${base_bdevs[@]}"; do 602 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b ${bdev}_malloc 603 $rpc_py bdev_passthru_create -b ${bdev}_malloc -p $bdev 604 done 605 606 # Create spare bdev 607 $rpc_py bdev_malloc_create 32 $base_blocklen $base_malloc_params -b "spare_malloc" 608 $rpc_py bdev_delay_create -b "spare_malloc" -d "spare_delay" -r 0 -t 0 -w 100000 -n 100000 609 $rpc_py bdev_passthru_create -b "spare_delay" -p "spare" 610 611 # Create RAID bdev 612 $rpc_py bdev_raid_create $create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name 613 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs 614 615 # Get RAID bdev's size 616 raid_bdev_size=$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[].num_blocks') 617 618 # Get base bdev's data offset 619 data_offset=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[].base_bdevs_list[0].data_offset') 620 621 if [ $background_io = true ]; then 622 # Start user I/O 623 "$rootdir/examples/bdev/bdevperf/bdevperf.py" -s $rpc_server perform_tests & 624 elif [ $verify = true ]; then 625 local write_unit_size 626 627 # Write random data to the RAID bdev 628 nbd_start_disks $rpc_server $raid_bdev_name /dev/nbd0 629 if [ $raid_level = "raid5f" ]; then 630 write_unit_size=$((strip_size * 2 * (num_base_bdevs - 1))) 631 echo $((base_blocklen * write_unit_size / 1024)) > /sys/block/nbd0/queue/max_sectors_kb 632 else 633 write_unit_size=1 634 fi 635 dd if=/dev/urandom of=/dev/nbd0 bs=$((base_blocklen * write_unit_size)) count=$((raid_bdev_size / write_unit_size)) oflag=direct 636 nbd_stop_disks $rpc_server /dev/nbd0 637 fi 638 639 # Remove one base bdev 640 $rpc_py bdev_raid_remove_base_bdev ${base_bdevs[0]} 641 642 # Check if the RAID bdev is in online state (degraded) 643 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 644 645 # Add bdev for rebuild 646 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name "spare" 647 sleep 1 648 649 # Check if rebuild started 650 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 651 652 # Remove the rebuild target bdev 653 $rpc_py bdev_raid_remove_base_bdev "spare" 654 655 # Check if the RAID bdev is in online state (degraded) 656 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 657 658 # Check if rebuild was stopped 659 verify_raid_bdev_process $raid_bdev_name "none" "none" 660 661 # Again, start the rebuild 662 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name "spare" 663 sleep 1 664 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 665 666 if [ $superblock = true ] && [ $with_io = false ]; then 667 # Stop the RAID bdev 668 $rpc_py bdev_raid_delete $raid_bdev_name 669 [[ $($rpc_py bdev_raid_get_bdevs all | jq 'length') == 0 ]] 670 671 # Remove the passthru base bdevs, then re-add them to assemble the raid bdev again 672 for ((i = 0; i < num_base_bdevs; i++)); do 673 $rpc_py bdev_passthru_delete ${base_bdevs[$i]} 674 done 675 for ((i = 0; i < num_base_bdevs; i++)); do 676 $rpc_py bdev_passthru_create -b ${base_bdevs[$i]}_malloc -p ${base_bdevs[$i]} 677 done 678 679 # Check if the RAID bdev is in online state (degraded) 680 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)) 681 682 # Check if rebuild is not started 683 verify_raid_bdev_process $raid_bdev_name "none" "none" 684 685 # Again, start the rebuild 686 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name "spare" 687 sleep 1 688 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 689 fi 690 691 local num_base_bdevs_operational=$num_base_bdevs 692 693 if [ $raid_level = "raid1" ] && [ $num_base_bdevs -gt 2 ]; then 694 # Remove one more base bdev (not rebuild target) 695 $rpc_py bdev_raid_remove_base_bdev ${base_bdevs[1]} 696 697 # Ignore this bdev later when comparing data 698 base_bdevs[1]="" 699 ((num_base_bdevs_operational--)) 700 701 # Check if rebuild is still running 702 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 703 fi 704 705 # Wait for rebuild to finish 706 local timeout=$((SECONDS + 30)) 707 while ((SECONDS < timeout)); do 708 if ! verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" > /dev/null; then 709 break 710 fi 711 sleep 1 712 done 713 714 # Check if rebuild is not running and the RAID bdev has the correct number of operational devices 715 verify_raid_bdev_process $raid_bdev_name "none" "none" 716 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs_operational 717 718 # Stop the RAID bdev 719 $rpc_py bdev_raid_delete $raid_bdev_name 720 [[ $($rpc_py bdev_raid_get_bdevs all | jq 'length') == 0 ]] 721 722 if [ $verify = true ]; then 723 if [ $background_io = true ]; then 724 # Compare data on the rebuilt and other base bdevs 725 nbd_start_disks $rpc_server "spare" "/dev/nbd0" 726 for bdev in "${base_bdevs[@]:1}"; do 727 if [ -z "$bdev" ]; then 728 continue 729 fi 730 nbd_start_disks $rpc_server $bdev "/dev/nbd1" 731 cmp -i $((data_offset * base_blocklen)) /dev/nbd0 /dev/nbd1 732 nbd_stop_disks $rpc_server "/dev/nbd1" 733 done 734 nbd_stop_disks $rpc_server "/dev/nbd0" 735 else 736 # Compare data on the removed and rebuilt base bdevs 737 nbd_start_disks $rpc_server "${base_bdevs[0]} spare" "/dev/nbd0 /dev/nbd1" 738 cmp -i $((data_offset * base_blocklen)) /dev/nbd0 /dev/nbd1 739 nbd_stop_disks $rpc_server "/dev/nbd0 /dev/nbd1" 740 fi 741 fi 742 743 if [ $superblock = true ]; then 744 # Remove then re-add a base bdev to assemble the raid bdev again 745 $rpc_py bdev_passthru_delete "spare" 746 $rpc_py bdev_passthru_create -b "spare_delay" -p "spare" 747 748 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs_operational 749 verify_raid_bdev_process $raid_bdev_name "none" "none" 750 [[ $($rpc_py bdev_raid_get_bdevs all | jq -r '.[].base_bdevs_list[0].name') == "spare" ]] 751 752 # Remove and re-add a base bdev - rebuild should start automatically 753 $rpc_py bdev_raid_remove_base_bdev "spare" 754 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs_operational - 1)) 755 $rpc_py bdev_raid_add_base_bdev $raid_bdev_name "spare" 756 sleep 1 757 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 758 759 # Same as above but re-add through examine 760 $rpc_py bdev_passthru_delete "spare" 761 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs_operational - 1)) 762 $rpc_py bdev_passthru_create -b "spare_delay" -p "spare" 763 sleep 1 764 verify_raid_bdev_process $raid_bdev_name "rebuild" "spare" 765 766 # Stop the rebuild 767 $rpc_py bdev_passthru_delete "spare" 768 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs_operational - 1)) 769 verify_raid_bdev_process $raid_bdev_name "none" "none" 770 771 # Re-adding a base bdev that was replaced (no longer is a member of the array) should not be allowed 772 $rpc_py bdev_passthru_delete ${base_bdevs[0]} 773 $rpc_py bdev_passthru_create -b ${base_bdevs[0]}_malloc -p ${base_bdevs[0]} 774 sleep 1 775 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs_operational - 1)) 776 verify_raid_bdev_process $raid_bdev_name "none" "none" 777 NOT $rpc_py bdev_raid_add_base_bdev $raid_bdev_name ${base_bdevs[0]} 778 sleep 1 779 verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs_operational - 1)) 780 verify_raid_bdev_process $raid_bdev_name "none" "none" 781 fi 782 783 killprocess $raid_pid 784 785 return 0 786} 787 788trap 'on_error_exit;' ERR 789 790base_blocklen=512 791 792if [ $(uname -s) = Linux ] && modprobe -n nbd; then 793 has_nbd=true 794 modprobe nbd 795 run_test "raid_function_test_raid0" raid_function_test raid0 796 run_test "raid_function_test_concat" raid_function_test concat 797fi 798 799run_test "raid0_resize_test" raid0_resize_test 800 801for n in {2..4}; do 802 for level in raid0 concat raid1; do 803 run_test "raid_state_function_test" raid_state_function_test $level $n false 804 run_test "raid_state_function_test_sb" raid_state_function_test $level $n true 805 run_test "raid_superblock_test" raid_superblock_test $level $n 806 done 807done 808 809if [ "$has_nbd" = true ]; then 810 for n in 2 4; do 811 run_test "raid_rebuild_test" raid_rebuild_test raid1 $n false false true 812 run_test "raid_rebuild_test_sb" raid_rebuild_test raid1 $n true false true 813 run_test "raid_rebuild_test_io" raid_rebuild_test raid1 $n false true true 814 run_test "raid_rebuild_test_sb_io" raid_rebuild_test raid1 $n true true true 815 done 816fi 817 818if [ "$CONFIG_RAID5F" == y ]; then 819 for n in {3..4}; do 820 run_test "raid5f_state_function_test" raid_state_function_test raid5f $n false 821 run_test "raid5f_state_function_test_sb" raid_state_function_test raid5f $n true 822 run_test "raid5f_superblock_test" raid_superblock_test raid5f $n 823 if [ "$has_nbd" = true ]; then 824 run_test "raid5f_rebuild_test" raid_rebuild_test raid5f $n false false true 825 run_test "raid5f_rebuild_test_sb" raid_rebuild_test raid5f $n true false true 826 fi 827 done 828fi 829 830base_blocklen=4096 831 832run_test "raid_state_function_test_sb_4k" raid_state_function_test raid1 2 true 833run_test "raid_superblock_test_4k" raid_superblock_test raid1 2 834if [ "$has_nbd" = true ]; then 835 run_test "raid_rebuild_test_sb_4k" raid_rebuild_test raid1 2 true false true 836fi 837 838base_malloc_params="-m 32" 839run_test "raid_state_function_test_sb_md_separate" raid_state_function_test raid1 2 true 840run_test "raid_superblock_test_md_separate" raid_superblock_test raid1 2 841if [ "$has_nbd" = true ]; then 842 run_test "raid_rebuild_test_sb_md_separate" raid_rebuild_test raid1 2 true false true 843fi 844 845base_malloc_params="-m 32 -i" 846run_test "raid_state_function_test_sb_md_interleaved" raid_state_function_test raid1 2 true 847run_test "raid_superblock_test_md_interleaved" raid_superblock_test raid1 2 848run_test "raid_rebuild_test_sb_md_interleaved" raid_rebuild_test raid1 2 true false false 849 850rm -f $tmp_file 851