1#!/usr/bin/env bash 2 3testdir=$(readlink -f $(dirname $0)) 4rootdir=$(readlink -f $testdir/../..) 5source $rootdir/test/common/autotest_common.sh 6source $testdir/nbd_common.sh 7 8rpc_py=rpc_cmd 9conf_file="$testdir/bdev.json" 10# Make sure the configuration is clean 11: > "$conf_file" 12 13function cleanup() { 14 rm -f "$SPDK_TEST_STORAGE/aiofile" 15 rm -f "$SPDK_TEST_STORAGE/spdk-pmem-pool" 16 rm -f "$conf_file" 17 18 if [[ $test_type == rbd ]]; then 19 rbd_cleanup 20 fi 21 22 if [[ "$test_type" = "gpt" ]]; then 23 "$rootdir/scripts/setup.sh" reset 24 if [[ -b $gpt_nvme ]]; then 25 wipefs --all "$gpt_nvme" 26 fi 27 fi 28} 29 30function start_spdk_tgt() { 31 "$SPDK_BIN_DIR/spdk_tgt" "$env_ctx" & 32 spdk_tgt_pid=$! 33 trap 'killprocess "$spdk_tgt_pid"; exit 1' SIGINT SIGTERM EXIT 34 waitforlisten "$spdk_tgt_pid" 35} 36 37function setup_bdev_conf() { 38 "$rpc_py" <<- RPC 39 bdev_split_create Malloc1 2 40 bdev_split_create -s 4 Malloc2 8 41 bdev_malloc_create -b Malloc0 32 512 42 bdev_malloc_create -b Malloc1 32 512 43 bdev_malloc_create -b Malloc2 32 512 44 bdev_malloc_create -b Malloc3 32 512 45 bdev_malloc_create -b Malloc4 32 512 46 bdev_malloc_create -b Malloc5 32 512 47 bdev_malloc_create -b Malloc6 32 512 48 bdev_malloc_create -b Malloc7 32 512 49 bdev_passthru_create -p TestPT -b Malloc3 50 bdev_raid_create -n raid0 -z 64 -r 0 -b "Malloc4 Malloc5" 51 bdev_raid_create -n concat0 -z 64 -r concat -b "Malloc6 Malloc7" 52 bdev_set_qos_limit --rw_mbytes_per_sec 100 Malloc3 53 bdev_set_qos_limit --rw_ios_per_sec 20000 Malloc0 54 RPC 55 if [[ $(uname -s) != "FreeBSD" ]]; then 56 dd if=/dev/zero of="$SPDK_TEST_STORAGE/aiofile" bs=2048 count=5000 57 "$rpc_py" bdev_aio_create "$SPDK_TEST_STORAGE/aiofile" AIO0 2048 58 fi 59} 60 61function setup_nvme_conf() { 62 local json 63 mapfile -t json < <("$rootdir/scripts/gen_nvme.sh") 64 "$rpc_py" load_subsystem_config -j "'${json[*]}'" 65} 66 67function setup_gpt_conf() { 68 $rootdir/scripts/setup.sh reset 69 get_zoned_devs 70 # Get nvme devices by following drivers' links towards nvme class 71 local nvme_devs=(/sys/bus/pci/drivers/nvme/*/nvme/nvme*/nvme*n*) nvme_dev 72 gpt_nvme="" 73 # Pick first device which doesn't have any valid partition table 74 for nvme_dev in "${nvme_devs[@]}"; do 75 [[ -z ${zoned_devs["${nvme_dev##*/}"]} ]] || continue 76 dev=/dev/${nvme_dev##*/} 77 if ! pt=$(parted "$dev" -ms print 2>&1); then 78 [[ $pt == *"$dev: unrecognised disk label"* ]] || continue 79 gpt_nvme=$dev 80 break 81 fi 82 done 83 if [[ -n $gpt_nvme ]]; then 84 # Create gpt partition table 85 parted -s "$gpt_nvme" mklabel gpt mkpart SPDK_TEST_first '0%' '50%' mkpart SPDK_TEST_second '50%' '100%' 86 # change the GUID to SPDK GUID value 87 SPDK_GPT_GUID=$(get_spdk_gpt) 88 sgdisk -t "1:$SPDK_GPT_GUID" "$gpt_nvme" 89 sgdisk -t "2:$SPDK_GPT_GUID" "$gpt_nvme" 90 "$rootdir/scripts/setup.sh" 91 "$rpc_py" bdev_get_bdevs 92 setup_nvme_conf 93 else 94 printf 'Did not find any nvme block devices to work with, aborting the test\n' >&2 95 "$rootdir/scripts/setup.sh" 96 return 1 97 fi 98} 99 100function setup_crypto_aesni_conf() { 101 # Malloc0 and Malloc1 use AESNI 102 "$rpc_py" <<- RPC 103 bdev_malloc_create -b Malloc0 16 512 104 bdev_malloc_create -b Malloc1 16 512 105 bdev_crypto_create Malloc0 crypto_ram crypto_aesni_mb 01234567891234560123456789123456 106 bdev_crypto_create Malloc1 crypto_ram2 crypto_aesni_mb 90123456789123459012345678912345 107 RPC 108} 109 110function setup_crypto_qat_conf() { 111 # Malloc0 will use QAT AES_CBC 112 # Malloc1 will use QAT AES_XTS 113 "$rpc_py" <<- RPC 114 bdev_malloc_create -b Malloc0 16 512 115 bdev_malloc_create -b Malloc1 16 512 116 bdev_crypto_create Malloc0 crypto_ram crypto_qat 01234567891234560123456789123456 117 bdev_crypto_create -c AES_XTS -k2 01234567891234560123456789123456 Malloc1 crypto_ram3 crypto_qat 01234567891234560123456789123456 118 bdev_get_bdevs -b Malloc1 119 RPC 120} 121 122function setup_crypto_mlx5_conf() { 123 local key=$1 124 local block_key 125 local tweak_key 126 if [ ${#key} == 96 ]; then 127 # 96 bytes is 64 + 32 - AES_XTS_256 in hexlified format 128 # Copy first 64 chars into the 'key'. This gives 32 in the 129 # binary or 256 bit. 130 block_key=${key:0:64} 131 # Copy the the rest of the key and pass it as the 'key2'. 132 tweak_key=${key:64:32} 133 elif [ ${#key} == 160 ]; then 134 # 160 bytes is 128 + 32 - AES_XTS_512 in hexlified format 135 # Copy first 128 chars into the 'key'. This gives 64 in the 136 # binary or 512 bit. 137 block_key=${key:0:128} 138 # Copy the the rest of the key and pass it as the 'key2'. 139 tweak_key=${key:128:32} 140 else 141 echo "ERROR: Invalid DEK size for MLX5 crypto setup: ${#key}" 142 echo "ERROR: Supported key sizes for MLX5: 96 bytes (AES_XTS_256) and 160 bytes (AES_XTS_512)." 143 return 1 144 fi 145 146 # Malloc0 will use MLX5 AES_XTS 147 "$rpc_py" <<- RPC 148 bdev_malloc_create -b Malloc0 16 512 149 bdev_crypto_create -c AES_XTS -k2 $tweak_key Malloc0 crypto_ram4 mlx5_pci $block_key 150 bdev_get_bdevs -b Malloc0 151 RPC 152} 153 154function setup_pmem_conf() { 155 if hash pmempool; then 156 rm -f "$SPDK_TEST_STORAGE/spdk-pmem-pool" 157 pmempool create blk --size=32M 512 "$SPDK_TEST_STORAGE/spdk-pmem-pool" 158 "$rpc_py" bdev_pmem_create -n Pmem0 "$SPDK_TEST_STORAGE/spdk-pmem-pool" 159 else 160 return 1 161 fi 162} 163 164function setup_rbd_conf() { 165 timing_enter rbd_setup 166 rbd_setup 127.0.0.1 167 timing_exit rbd_setup 168 169 "$rpc_py" bdev_rbd_create -b Ceph0 rbd foo 512 170} 171 172function bdev_bounds() { 173 $testdir/bdevio/bdevio -w -s $PRE_RESERVED_MEM --json "$conf_file" "$env_ctx" & 174 bdevio_pid=$! 175 trap 'cleanup; killprocess $bdevio_pid; exit 1' SIGINT SIGTERM EXIT 176 echo "Process bdevio pid: $bdevio_pid" 177 waitforlisten $bdevio_pid 178 $testdir/bdevio/tests.py perform_tests 179 killprocess $bdevio_pid 180 trap - SIGINT SIGTERM EXIT 181} 182 183function nbd_function_test() { 184 if [ $(uname -s) = Linux ] && modprobe -n nbd; then 185 local rpc_server=/var/tmp/spdk-nbd.sock 186 local conf=$1 187 local nbd_all=($(ls /dev/nbd* | grep -v p)) 188 local bdev_all=($bdevs_name) 189 local nbd_num=${#bdev_all[@]} 190 if ((nbd_num < 1)); then 191 # There should be at least one bdev and one valid nbd device 192 return 1 193 fi 194 if [ ${#nbd_all[@]} -le $nbd_num ]; then 195 nbd_num=${#nbd_all[@]} 196 fi 197 local nbd_list=(${nbd_all[@]:0:$nbd_num}) 198 local bdev_list=(${bdev_all[@]:0:$nbd_num}) 199 200 if [ ! -e $conf ]; then 201 return 1 202 fi 203 204 modprobe nbd 205 $rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 --json "$conf" "$env_ctx" & 206 nbd_pid=$! 207 trap 'cleanup; killprocess $nbd_pid; exit 1' SIGINT SIGTERM EXIT 208 echo "Process nbd pid: $nbd_pid" 209 waitforlisten $nbd_pid $rpc_server 210 211 nbd_rpc_start_stop_verify $rpc_server "${bdev_list[*]}" 212 nbd_rpc_data_verify $rpc_server "${bdev_list[*]}" "${nbd_list[*]}" 213 214 killprocess $nbd_pid 215 trap - SIGINT SIGTERM EXIT 216 fi 217 218 return 0 219} 220 221function fio_test_suite() { 222 local env_context 223 224 # Make sure that state files and anything else produced by fio test will 225 # stay at the testdir. 226 pushd $testdir 227 trap 'rm -f ./*.state; popd; exit 1' SIGINT SIGTERM EXIT 228 229 # Generate the fio config file given the list of all unclaimed bdevs 230 env_context=$(echo "$env_ctx" | sed 's/--env-context=//') 231 fio_config_gen $testdir/bdev.fio verify AIO "$env_context" 232 for b in $(echo $bdevs | jq -r '.name'); do 233 echo "[job_$b]" >> $testdir/bdev.fio 234 echo "filename=$b" >> $testdir/bdev.fio 235 done 236 237 local fio_params="--ioengine=spdk_bdev --iodepth=8 --bs=4k --runtime=10 $testdir/bdev.fio \ 238 --verify_state_save=0 --spdk_json_conf=$conf_file" 239 240 run_test "bdev_fio_rw_verify" fio_bdev $fio_params --spdk_mem=$PRE_RESERVED_MEM --aux-path=$output_dir 241 rm -f ./*.state 242 rm -f $testdir/bdev.fio 243 244 # Generate the fio config file given the list of all unclaimed bdevs that support unmap 245 fio_config_gen $testdir/bdev.fio trim "" "$env_context" 246 if [ "$(echo $bdevs | jq -r 'select(.supported_io_types.unmap == true) | .name')" != "" ]; then 247 for b in $(echo $bdevs | jq -r 'select(.supported_io_types.unmap == true) | .name'); do 248 echo "[job_$b]" >> $testdir/bdev.fio 249 echo "filename=$b" >> $testdir/bdev.fio 250 done 251 else 252 rm -f $testdir/bdev.fio 253 popd 254 trap - SIGINT SIGTERM EXIT 255 return 0 256 fi 257 258 run_test "bdev_fio_trim" fio_bdev $fio_params --verify_state_save=0 --aux-path=$output_dir 259 rm -f ./*.state 260 rm -f $testdir/bdev.fio 261 popd 262 trap - SIGINT SIGTERM EXIT 263} 264 265function get_io_result() { 266 local limit_type=$1 267 local qos_dev=$2 268 local iostat_result 269 iostat_result=$($rootdir/scripts/iostat.py -d -i 1 -t $QOS_RUN_TIME | grep $qos_dev | tail -1) 270 if [ $limit_type = IOPS ]; then 271 iostat_result=$(awk '{print $2}' <<< $iostat_result) 272 elif [ $limit_type = BANDWIDTH ]; then 273 iostat_result=$(awk '{print $6}' <<< $iostat_result) 274 fi 275 276 echo ${iostat_result/.*/} 277} 278 279function run_qos_test() { 280 local qos_limit=$1 281 local qos_result=0 282 283 qos_result=$(get_io_result $2 $3) 284 if [ $2 = BANDWIDTH ]; then 285 qos_limit=$((qos_limit * 1024)) 286 fi 287 lower_limit=$((qos_limit * 9 / 10)) 288 upper_limit=$((qos_limit * 11 / 10)) 289 290 # QoS realization is related with bytes transferred. It currently has some variation. 291 if [ $qos_result -lt $lower_limit ] || [ $qos_result -gt $upper_limit ]; then 292 echo "Failed to limit the io read rate of NULL bdev by qos" 293 $rpc_py bdev_malloc_delete $QOS_DEV_1 294 $rpc_py bdev_null_delete $QOS_DEV_2 295 killprocess $QOS_PID 296 exit 1 297 fi 298} 299 300function qos_function_test() { 301 local qos_lower_iops_limit=1000 302 local qos_lower_bw_limit=2 303 local io_result=0 304 local iops_limit=0 305 local bw_limit=0 306 307 io_result=$(get_io_result IOPS $QOS_DEV_1) 308 # Set the IOPS limit as one quarter of the measured performance without QoS 309 iops_limit=$(((io_result / 4) / qos_lower_iops_limit * qos_lower_iops_limit)) 310 if [ $iops_limit -gt $qos_lower_iops_limit ]; then 311 312 # Run bdevperf with IOPS rate limit on bdev 1 313 $rpc_py bdev_set_qos_limit --rw_ios_per_sec $iops_limit $QOS_DEV_1 314 run_test "bdev_qos_iops" run_qos_test $iops_limit IOPS $QOS_DEV_1 315 316 # Run bdevperf with bandwidth rate limit on bdev 2 317 # Set the bandwidth limit as 1/10 of the measure performance without QoS 318 bw_limit=$(get_io_result BANDWIDTH $QOS_DEV_2) 319 bw_limit=$((bw_limit / 1024 / 10)) 320 if [ $bw_limit -lt $qos_lower_bw_limit ]; then 321 bw_limit=$qos_lower_bw_limit 322 fi 323 $rpc_py bdev_set_qos_limit --rw_mbytes_per_sec $bw_limit $QOS_DEV_2 324 run_test "bdev_qos_bw" run_qos_test $bw_limit BANDWIDTH $QOS_DEV_2 325 326 # Run bdevperf with additional read only bandwidth rate limit on bdev 1 327 $rpc_py bdev_set_qos_limit --r_mbytes_per_sec $qos_lower_bw_limit $QOS_DEV_1 328 run_test "bdev_qos_ro_bw" run_qos_test $qos_lower_bw_limit BANDWIDTH $QOS_DEV_1 329 else 330 echo "Actual IOPS without limiting is too low - exit testing" 331 fi 332} 333 334function qos_test_suite() { 335 # Run bdevperf with QoS disabled first 336 "$testdir/bdevperf/bdevperf" -z -m 0x2 -q 256 -o 4096 -w randread -t 60 "$env_ctx" & 337 QOS_PID=$! 338 echo "Process qos testing pid: $QOS_PID" 339 trap 'cleanup; killprocess $QOS_PID; exit 1' SIGINT SIGTERM EXIT 340 waitforlisten $QOS_PID 341 342 $rpc_py bdev_malloc_create -b $QOS_DEV_1 128 512 343 waitforbdev $QOS_DEV_1 344 $rpc_py bdev_null_create $QOS_DEV_2 128 512 345 waitforbdev $QOS_DEV_2 346 347 $rootdir/test/bdev/bdevperf/bdevperf.py perform_tests & 348 qos_function_test 349 350 $rpc_py bdev_malloc_delete $QOS_DEV_1 351 $rpc_py bdev_null_delete $QOS_DEV_2 352 killprocess $QOS_PID 353 trap - SIGINT SIGTERM EXIT 354} 355 356function error_test_suite() { 357 DEV_1="Dev_1" 358 DEV_2="Dev_2" 359 ERR_DEV="EE_Dev_1" 360 361 # Run bdevperf with 1 normal bdev and 1 error bdev, also continue on error 362 "$testdir/bdevperf/bdevperf" -z -m 0x2 -q 16 -o 4096 -w randread -t 5 -f "$env_ctx" & 363 ERR_PID=$! 364 echo "Process error testing pid: $ERR_PID" 365 waitforlisten $ERR_PID 366 367 $rpc_py bdev_malloc_create -b $DEV_1 128 512 368 waitforbdev $DEV_1 369 $rpc_py bdev_error_create $DEV_1 370 $rpc_py bdev_malloc_create -b $DEV_2 128 512 371 waitforbdev $DEV_2 372 $rpc_py bdev_error_inject_error $ERR_DEV 'all' 'failure' -n 5 373 374 $rootdir/test/bdev/bdevperf/bdevperf.py -t 1 perform_tests & 375 sleep 1 376 377 # Bdevperf is expected to be there as the continue on error is set 378 if kill -0 $ERR_PID; then 379 echo "Process is existed as continue on error is set. Pid: $ERR_PID" 380 else 381 echo "Process exited unexpectedly. Pid: $ERR_PID" 382 exit 1 383 fi 384 385 # Delete the error devices 386 $rpc_py bdev_error_delete $ERR_DEV 387 $rpc_py bdev_malloc_delete $DEV_1 388 sleep 5 389 # Expected to exit normally 390 killprocess $ERR_PID 391 392 # Run bdevperf with 1 normal bdev and 1 error bdev, and exit on error 393 "$testdir/bdevperf/bdevperf" -z -m 0x2 -q 16 -o 4096 -w randread -t 5 "$env_ctx" & 394 ERR_PID=$! 395 echo "Process error testing pid: $ERR_PID" 396 waitforlisten $ERR_PID 397 398 $rpc_py bdev_malloc_create -b $DEV_1 128 512 399 waitforbdev $DEV_1 400 $rpc_py bdev_error_create $DEV_1 401 $rpc_py bdev_malloc_create -b $DEV_2 128 512 402 waitforbdev $DEV_2 403 $rpc_py bdev_error_inject_error $ERR_DEV 'all' 'failure' -n 5 404 405 $rootdir/test/bdev/bdevperf/bdevperf.py -t 1 perform_tests & 406 NOT wait $ERR_PID 407} 408 409function qd_sampling_function_test() { 410 local bdev_name=$1 411 local sampling_period=10 412 local iostats 413 414 $rpc_py bdev_set_qd_sampling_period $bdev_name $sampling_period 415 416 iostats=$($rpc_py bdev_get_iostat -b $bdev_name) 417 418 qd_sampling_period=$(jq -r '.bdevs[0].queue_depth_polling_period' <<< "$iostats") 419 420 if [ $qd_sampling_period == null ] || [ $qd_sampling_period -ne $sampling_period ]; then 421 echo "Qeueue depth polling period is not right" 422 $rpc_py bdev_malloc_delete $QD_DEV 423 killprocess $QD_PID 424 exit 1 425 fi 426} 427 428function qd_sampling_test_suite() { 429 QD_DEV="Malloc_QD" 430 431 "$testdir/bdevperf/bdevperf" -z -m 0x3 -q 256 -o 4096 -w randread -t 5 -C "$env_ctx" & 432 QD_PID=$! 433 echo "Process bdev QD sampling period testing pid: $QD_PID" 434 trap 'cleanup; killprocess $QD_PID; exit 1' SIGINT SIGTERM EXIT 435 waitforlisten $QD_PID 436 437 $rpc_py bdev_malloc_create -b $QD_DEV 128 512 438 waitforbdev $QD_DEV 439 440 $rootdir/test/bdev/bdevperf/bdevperf.py perform_tests & 441 sleep 2 442 qd_sampling_function_test $QD_DEV 443 444 $rpc_py bdev_malloc_delete $QD_DEV 445 killprocess $QD_PID 446 trap - SIGINT SIGTERM EXIT 447} 448 449# Inital bdev creation and configuration 450#----------------------------------------------------- 451QOS_DEV_1="Malloc_0" 452QOS_DEV_2="Null_1" 453QOS_RUN_TIME=5 454 455if [ $(uname -s) = Linux ]; then 456 # Test dynamic memory management. All hugepages will be reserved at runtime 457 PRE_RESERVED_MEM=0 458else 459 # Dynamic memory management is not supported on BSD 460 PRE_RESERVED_MEM=2048 461fi 462 463test_type=${1:-bdev} 464crypto_device=$2 465wcs_file=$3 466dek=$4 467env_ctx="" 468if [ -n "$crypto_device" ] && [ -n "$wcs_file" ]; then 469 # We need full path here since fio perf test does 'pushd' to the test dir 470 # and crypto login of fio plugin test can fail. 471 wcs_file=$(readlink -f $wcs_file) 472 if [ -f $wcs_file ]; then 473 env_ctx="--env-context=--allow=$crypto_device,class=crypto,wcs_file=$wcs_file" 474 else 475 echo "ERROR: Credentials file $3 is not found!" 476 exit 1 477 fi 478fi 479start_spdk_tgt 480case "$test_type" in 481 bdev) 482 setup_bdev_conf 483 ;; 484 nvme) 485 setup_nvme_conf 486 ;; 487 gpt) 488 setup_gpt_conf 489 ;; 490 crypto_aesni) 491 setup_crypto_aesni_conf 492 ;; 493 crypto_qat) 494 setup_crypto_qat_conf 495 ;; 496 crypto_mlx5) 497 setup_crypto_mlx5_conf $dek 498 ;; 499 pmem) 500 setup_pmem_conf 501 ;; 502 rbd) 503 setup_rbd_conf 504 ;; 505 *) 506 echo "invalid test name" 507 exit 1 508 ;; 509esac 510 511"$rpc_py" bdev_wait_for_examine 512 513# Generate json config and use it throughout all the tests 514cat <<- CONF > "$conf_file" 515 {"subsystems":[ 516 $("$rpc_py" save_subsystem_config -n bdev) 517 ]} 518CONF 519 520bdevs=$("$rpc_py" bdev_get_bdevs | jq -r '.[] | select(.claimed == false)') 521bdevs_name=$(echo $bdevs | jq -r '.name') 522bdev_list=($bdevs_name) 523hello_world_bdev=${bdev_list[0]} 524trap - SIGINT SIGTERM EXIT 525killprocess "$spdk_tgt_pid" 526# End bdev configuration 527#----------------------------------------------------- 528 529trap "cleanup" SIGINT SIGTERM EXIT 530 531run_test "bdev_hello_world" $SPDK_EXAMPLE_DIR/hello_bdev --json "$conf_file" -b "$hello_world_bdev" "$env_ctx" 532run_test "bdev_bounds" bdev_bounds "$env_ctx" 533run_test "bdev_nbd" nbd_function_test $conf_file "$bdevs_name" "$env_ctx" 534if [[ $CONFIG_FIO_PLUGIN == y ]]; then 535 if [ "$test_type" = "nvme" ] || [ "$test_type" = "gpt" ]; then 536 # TODO: once we get real multi-ns drives, re-enable this test for NVMe. 537 echo "skipping fio tests on NVMe due to multi-ns failures." 538 else 539 run_test "bdev_fio" fio_test_suite "$env_ctx" 540 fi 541else 542 echo "FIO not available" 543 exit 1 544fi 545 546trap "cleanup" SIGINT SIGTERM EXIT 547 548run_test "bdev_verify" $testdir/bdevperf/bdevperf --json "$conf_file" -q 128 -o 4096 -w verify -t 5 -C -m 0x3 "$env_ctx" 549run_test "bdev_write_zeroes" $testdir/bdevperf/bdevperf --json "$conf_file" -q 128 -o 4096 -w write_zeroes -t 1 "$env_ctx" 550 551if [[ $test_type == bdev ]]; then 552 run_test "bdev_qos" qos_test_suite "$env_ctx" 553 run_test "bdev_qd_sampling" qd_sampling_test_suite "$env_ctx" 554 run_test "bdev_error" error_test_suite "$env_ctx" 555fi 556 557# Temporarily disabled - infinite loop 558# if [ $RUN_NIGHTLY -eq 1 ]; then 559# run_test "bdev_reset" $testdir/bdevperf/bdevperf --json "$conf_file" -q 16 -w reset -o 4096 -t 60 "$env_ctx" 560# fi 561 562# Bdev and configuration cleanup below this line 563#----------------------------------------------------- 564 565trap - SIGINT SIGTERM EXIT 566cleanup 567