1: ${SPDK_VHOST_VERBOSE=false} 2: ${VHOST_DIR="$HOME/vhost_test"} 3: ${QEMU_BIN:="qemu-system-x86_64"} 4: ${QEMU_IMG_BIN="qemu-img"} 5 6TEST_DIR=$(readlink -f $rootdir/..) 7VM_DIR=$VHOST_DIR/vms 8TARGET_DIR=$VHOST_DIR/vhost 9VM_PASSWORD="root" 10 11VM_IMAGE=${VM_IMAGE:-"$DEPENDENCY_DIR/spdk_test_image.qcow2"} 12DEFAULT_FIO_BIN=${DEFAULT_FIO_BIN:-"$DEPENDENCY_DIR/fio"} 13FIO_BIN=${FIO_BIN:-"$DEFAULT_FIO_BIN"} 14 15WORKDIR=$(readlink -f "$(dirname "$0")") 16 17if ! hash $QEMU_IMG_BIN $QEMU_BIN; then 18 echo 'ERROR: QEMU is not installed on this system. Unable to run vhost tests.' >&2 19 exit 1 20fi 21 22mkdir -p $VHOST_DIR 23mkdir -p $VM_DIR 24mkdir -p $TARGET_DIR 25 26# 27# Source config describing QEMU and VHOST cores and NUMA 28# 29source $rootdir/test/vhost/common/autotest.config 30 31function vhosttestinit() { 32 if [ "$TEST_MODE" == "iso" ]; then 33 $rootdir/scripts/setup.sh 34 fi 35 36 if [[ -e $VM_IMAGE.gz && ! -e $VM_IMAGE ]]; then 37 gzip -dc "$VM_IMAGE.gz" > "$VM_IMAGE" 38 fi 39 40 # Look for the VM image 41 if [[ ! -f $VM_IMAGE ]]; then 42 [[ $1 != "--no_vm" ]] || return 0 43 echo "$VM_IMAGE is missing" >&2 44 return 1 45 fi 46} 47 48function vhosttestfini() { 49 if [ "$TEST_MODE" == "iso" ]; then 50 $rootdir/scripts/setup.sh reset 51 fi 52} 53 54function message() { 55 local verbose_out 56 if ! $SPDK_VHOST_VERBOSE; then 57 verbose_out="" 58 elif [[ ${FUNCNAME[2]} == "source" ]]; then 59 verbose_out=" (file $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[1]})" 60 else 61 verbose_out=" (function ${FUNCNAME[2]}:${BASH_LINENO[1]})" 62 fi 63 64 local msg_type="$1" 65 shift 66 echo -e "${msg_type}${verbose_out}: $*" 67} 68 69function fail() { 70 echo "===========" >&2 71 message "FAIL" "$@" >&2 72 echo "===========" >&2 73 exit 1 74} 75 76function error() { 77 echo "===========" >&2 78 message "ERROR" "$@" >&2 79 echo "===========" >&2 80 # Don't 'return 1' since the stack trace will be incomplete (why?) missing upper command. 81 false 82} 83 84function warning() { 85 message "WARN" "$@" >&2 86} 87 88function notice() { 89 message "INFO" "$@" 90} 91 92function check_qemu_packedring_support() { 93 qemu_version=$($QEMU_BIN -version | grep -Po "(?<=version )\d+.\d+.\d+") 94 if [[ "$qemu_version" < "4.2.0" ]]; then 95 error "This qemu binary does not support packed ring" 96 fi 97} 98 99function get_vhost_dir() { 100 local vhost_name="$1" 101 102 if [[ -z "$vhost_name" ]]; then 103 error "vhost name must be provided to get_vhost_dir" 104 return 1 105 fi 106 107 echo "$TARGET_DIR/${vhost_name}" 108} 109 110function vhost_run() { 111 local OPTIND 112 local vhost_name 113 local run_gen_nvme=true 114 local vhost_bin="vhost" 115 116 while getopts "n:a:b:g" optchar; do 117 case "$optchar" in 118 n) vhost_name="$OPTARG" ;; 119 a) vhost_args="$OPTARG" ;; 120 b) vhost_bin="$OPTARG" ;; 121 g) 122 run_gen_nvme=false 123 notice "Skipping gen_nvme.sh NVMe bdev configuration" 124 ;; 125 *) 126 error "Unknown param $optchar" 127 return 1 128 ;; 129 esac 130 done 131 132 if [[ -z "$vhost_name" ]]; then 133 error "vhost name must be provided to vhost_run" 134 return 1 135 fi 136 137 local vhost_dir 138 vhost_dir="$(get_vhost_dir $vhost_name)" 139 local vhost_app="$SPDK_BIN_DIR/$vhost_bin" 140 local vhost_log_file="$vhost_dir/vhost.log" 141 local vhost_pid_file="$vhost_dir/vhost.pid" 142 local vhost_socket="$vhost_dir/usvhost" 143 notice "starting vhost app in background" 144 [[ -r "$vhost_pid_file" ]] && vhost_kill $vhost_name 145 [[ -d $vhost_dir ]] && rm -f $vhost_dir/* 146 mkdir -p $vhost_dir 147 148 if [[ ! -x $vhost_app ]]; then 149 error "application not found: $vhost_app" 150 return 1 151 fi 152 153 local cmd="$vhost_app -r $vhost_dir/rpc.sock $vhost_args" 154 if [[ "$vhost_bin" =~ vhost ]]; then 155 cmd+=" -S $vhost_dir" 156 fi 157 158 notice "Loging to: $vhost_log_file" 159 notice "Socket: $vhost_socket" 160 notice "Command: $cmd" 161 162 timing_enter vhost_start 163 164 $cmd & 165 vhost_pid=$! 166 echo $vhost_pid > $vhost_pid_file 167 168 notice "waiting for app to run..." 169 waitforlisten "$vhost_pid" "$vhost_dir/rpc.sock" 170 #do not generate nvmes if pci access is disabled 171 if [[ "$cmd" != *"--no-pci"* ]] && [[ "$cmd" != *"-u"* ]] && $run_gen_nvme; then 172 $rootdir/scripts/gen_nvme.sh | $rootdir/scripts/rpc.py -s $vhost_dir/rpc.sock load_subsystem_config 173 fi 174 175 notice "vhost started - pid=$vhost_pid" 176 177 timing_exit vhost_start 178} 179 180function vhost_kill() { 181 local rc=0 182 local vhost_name="$1" 183 184 if [[ -z "$vhost_name" ]]; then 185 error "Must provide vhost name to vhost_kill" 186 return 0 187 fi 188 189 local vhost_dir 190 vhost_dir="$(get_vhost_dir $vhost_name)" 191 local vhost_pid_file="$vhost_dir/vhost.pid" 192 193 if [[ ! -r $vhost_pid_file ]]; then 194 warning "no vhost pid file found" 195 return 0 196 fi 197 198 timing_enter vhost_kill 199 local vhost_pid 200 vhost_pid="$(cat $vhost_pid_file)" 201 notice "killing vhost (PID $vhost_pid) app" 202 203 if kill -INT $vhost_pid > /dev/null; then 204 notice "sent SIGINT to vhost app - waiting 60 seconds to exit" 205 for ((i = 0; i < 60; i++)); do 206 if kill -0 $vhost_pid; then 207 echo "." 208 sleep 1 209 else 210 break 211 fi 212 done 213 if kill -0 $vhost_pid; then 214 error "ERROR: vhost was NOT killed - sending SIGABRT" 215 kill -ABRT $vhost_pid 216 rm $vhost_pid_file 217 rc=1 218 else 219 while kill -0 $vhost_pid; do 220 echo "." 221 done 222 fi 223 elif kill -0 $vhost_pid; then 224 error "vhost NOT killed - you need to kill it manually" 225 rc=1 226 else 227 notice "vhost was not running" 228 fi 229 230 timing_exit vhost_kill 231 if [[ $rc == 0 ]]; then 232 rm $vhost_pid_file 233 fi 234 235 rm -rf "$vhost_dir" 236 237 return $rc 238} 239 240function vhost_rpc() { 241 local vhost_name="$1" 242 243 if [[ -z "$vhost_name" ]]; then 244 error "vhost name must be provided to vhost_rpc" 245 return 1 246 fi 247 shift 248 249 $rootdir/scripts/rpc.py -s $(get_vhost_dir $vhost_name)/rpc.sock "$@" 250} 251 252### 253# Mgmt functions 254### 255 256function assert_number() { 257 [[ "$1" =~ [0-9]+ ]] && return 0 258 259 error "Invalid or missing parameter: need number but got '$1'" 260 return 1 261} 262 263# Run command on vm with given password 264# First argument - vm number 265# Second argument - ssh password for vm 266# 267function vm_sshpass() { 268 vm_num_is_valid $1 || return 1 269 270 local ssh_cmd 271 ssh_cmd="sshpass -p $2 ssh \ 272 -o UserKnownHostsFile=/dev/null \ 273 -o StrictHostKeyChecking=no \ 274 -o User=root \ 275 -p $(vm_ssh_socket $1) $VM_SSH_OPTIONS 127.0.0.1" 276 277 shift 2 278 $ssh_cmd "$@" 279} 280 281# Helper to validate VM number 282# param $1 VM number 283# 284function vm_num_is_valid() { 285 [[ "$1" =~ ^[0-9]+$ ]] && return 0 286 287 error "Invalid or missing parameter: vm number '$1'" 288 return 1 289} 290 291# Print network socket for given VM number 292# param $1 virtual machine number 293# 294function vm_ssh_socket() { 295 vm_num_is_valid $1 || return 1 296 local vm_dir="$VM_DIR/$1" 297 298 cat $vm_dir/ssh_socket 299} 300 301function vm_fio_socket() { 302 vm_num_is_valid $1 || return 1 303 local vm_dir="$VM_DIR/$1" 304 305 cat $vm_dir/fio_socket 306} 307 308# Execute command on given VM 309# param $1 virtual machine number 310# 311function vm_exec() { 312 vm_num_is_valid $1 || return 1 313 314 local vm_num="$1" 315 shift 316 317 sshpass -p "$VM_PASSWORD" ssh \ 318 -o UserKnownHostsFile=/dev/null \ 319 -o StrictHostKeyChecking=no \ 320 -o User=root \ 321 -p $(vm_ssh_socket $vm_num) $VM_SSH_OPTIONS 127.0.0.1 \ 322 "$@" 323} 324 325# Execute scp command on given VM 326# param $1 virtual machine number 327# 328function vm_scp() { 329 vm_num_is_valid $1 || return 1 330 331 local vm_num="$1" 332 shift 333 334 sshpass -p "$VM_PASSWORD" scp \ 335 -o UserKnownHostsFile=/dev/null \ 336 -o StrictHostKeyChecking=no \ 337 -o User=root \ 338 -P $(vm_ssh_socket $vm_num) $VM_SSH_OPTIONS \ 339 "$@" 340} 341 342# check if specified VM is running 343# param $1 VM num 344function vm_is_running() { 345 vm_num_is_valid $1 || return 1 346 local vm_dir="$VM_DIR/$1" 347 348 if [[ ! -r $vm_dir/qemu.pid ]]; then 349 return 1 350 fi 351 352 local vm_pid 353 vm_pid="$(cat $vm_dir/qemu.pid)" 354 355 if /bin/kill -0 $vm_pid; then 356 return 0 357 else 358 if [[ $EUID -ne 0 ]]; then 359 warning "not root - assuming VM running since can't be checked" 360 return 0 361 fi 362 363 # not running - remove pid file 364 rm $vm_dir/qemu.pid 365 return 1 366 fi 367} 368 369# check if specified VM is running 370# param $1 VM num 371function vm_os_booted() { 372 vm_num_is_valid $1 || return 1 373 local vm_dir="$VM_DIR/$1" 374 375 if [[ ! -r $vm_dir/qemu.pid ]]; then 376 error "VM $1 is not running" 377 return 1 378 fi 379 380 if ! VM_SSH_OPTIONS="-o ControlMaster=no" vm_exec $1 "true" 2> /dev/null; then 381 # Shutdown existing master. Ignore errors as it might not exist. 382 VM_SSH_OPTIONS="-O exit" vm_exec $1 "true" 2> /dev/null 383 return 1 384 fi 385 386 return 0 387} 388 389# Shutdown given VM 390# param $1 virtual machine number 391# return non-zero in case of error. 392function vm_shutdown() { 393 vm_num_is_valid $1 || return 1 394 local vm_dir="$VM_DIR/$1" 395 if [[ ! -d "$vm_dir" ]]; then 396 error "VM$1 ($vm_dir) not exist - setup it first" 397 return 1 398 fi 399 400 if ! vm_is_running $1; then 401 notice "VM$1 ($vm_dir) is not running" 402 return 0 403 fi 404 405 # Temporarily disabling exit flag for next ssh command, since it will 406 # "fail" due to shutdown 407 notice "Shutting down virtual machine $vm_dir" 408 set +e 409 vm_exec $1 "nohup sh -c 'shutdown -h -P now'" || true 410 notice "VM$1 is shutting down - wait a while to complete" 411 set -e 412} 413 414# Kill given VM 415# param $1 virtual machine number 416# 417function vm_kill() { 418 vm_num_is_valid $1 || return 1 419 local vm_dir="$VM_DIR/$1" 420 421 if [[ ! -r $vm_dir/qemu.pid ]]; then 422 return 0 423 fi 424 425 local vm_pid 426 vm_pid="$(cat $vm_dir/qemu.pid)" 427 428 notice "Killing virtual machine $vm_dir (pid=$vm_pid)" 429 # First kill should fail, second one must fail 430 if /bin/kill $vm_pid; then 431 notice "process $vm_pid killed" 432 rm $vm_dir/qemu.pid 433 rm -rf $vm_dir 434 elif vm_is_running $1; then 435 error "Process $vm_pid NOT killed" 436 return 1 437 fi 438} 439 440# List all VM numbers in VM_DIR 441# 442function vm_list_all() { 443 local vms 444 vms="$( 445 shopt -s nullglob 446 echo $VM_DIR/[0-9]* 447 )" 448 if [[ -n "$vms" ]]; then 449 basename --multiple $vms 450 fi 451} 452 453# Kills all VM in $VM_DIR 454# 455function vm_kill_all() { 456 local vm 457 for vm in $(vm_list_all); do 458 vm_kill $vm 459 done 460 461 rm -rf $VM_DIR 462} 463 464# Shutdown all VM in $VM_DIR 465# 466function vm_shutdown_all() { 467 # XXX: temporarily disable to debug shutdown issue 468 # xtrace_disable 469 470 local vms 471 vms=$(vm_list_all) 472 local vm 473 474 for vm in $vms; do 475 vm_shutdown $vm 476 done 477 478 notice "Waiting for VMs to shutdown..." 479 local timeo=30 480 while [[ $timeo -gt 0 ]]; do 481 local all_vms_down=1 482 for vm in $vms; do 483 if vm_is_running $vm; then 484 all_vms_down=0 485 break 486 fi 487 done 488 489 if [[ $all_vms_down == 1 ]]; then 490 notice "All VMs successfully shut down" 491 xtrace_restore 492 return 0 493 fi 494 495 ((timeo -= 1)) 496 sleep 1 497 done 498 499 rm -rf $VM_DIR 500 501 xtrace_restore 502} 503 504function vm_setup() { 505 xtrace_disable 506 local OPTIND optchar vm_num 507 508 local os="" 509 local os_mode="" 510 local qemu_args=() 511 local disk_type_g=NOT_DEFINED 512 local read_only="false" 513 # List created of a strings separated with a ":" 514 local disks=() 515 local raw_cache="" 516 local vm_incoming="" 517 local vm_migrate_to="" 518 local force_vm="" 519 local guest_memory=1024 520 local queue_number="" 521 local vhost_dir 522 local packed=false 523 vhost_dir="$(get_vhost_dir 0)" 524 while getopts ':-:' optchar; do 525 case "$optchar" in 526 -) 527 case "$OPTARG" in 528 os=*) os="${OPTARG#*=}" ;; 529 os-mode=*) os_mode="${OPTARG#*=}" ;; 530 qemu-args=*) qemu_args+=("${OPTARG#*=}") ;; 531 disk-type=*) disk_type_g="${OPTARG#*=}" ;; 532 read-only=*) read_only="${OPTARG#*=}" ;; 533 disks=*) IFS=":" read -ra disks <<< "${OPTARG#*=}" ;; 534 raw-cache=*) raw_cache=",cache${OPTARG#*=}" ;; 535 force=*) force_vm=${OPTARG#*=} ;; 536 memory=*) guest_memory=${OPTARG#*=} ;; 537 queue_num=*) queue_number=${OPTARG#*=} ;; 538 incoming=*) vm_incoming="${OPTARG#*=}" ;; 539 migrate-to=*) vm_migrate_to="${OPTARG#*=}" ;; 540 vhost-name=*) vhost_dir="$(get_vhost_dir ${OPTARG#*=})" ;; 541 spdk-boot=*) local boot_from="${OPTARG#*=}" ;; 542 packed) packed=true ;; 543 *) 544 error "unknown argument $OPTARG" 545 return 1 546 ;; 547 esac 548 ;; 549 *) 550 error "vm_create Unknown param $OPTARG" 551 return 1 552 ;; 553 esac 554 done 555 556 # Find next directory we can use 557 if [[ -n $force_vm ]]; then 558 vm_num=$force_vm 559 560 vm_num_is_valid $vm_num || return 1 561 local vm_dir="$VM_DIR/$vm_num" 562 [[ -d $vm_dir ]] && warning "removing existing VM in '$vm_dir'" 563 else 564 local vm_dir="" 565 566 set +x 567 for ((i = 0; i <= 256; i++)); do 568 local vm_dir="$VM_DIR/$i" 569 [[ ! -d $vm_dir ]] && break 570 done 571 xtrace_restore 572 573 vm_num=$i 574 fi 575 576 if [[ $vm_num -eq 256 ]]; then 577 error "no free VM found. do some cleanup (256 VMs created, are you insane?)" 578 return 1 579 fi 580 581 if [[ -n "$vm_migrate_to" && -n "$vm_incoming" ]]; then 582 error "'--incoming' and '--migrate-to' cannot be used together" 583 return 1 584 elif [[ -n "$vm_incoming" ]]; then 585 if [[ -n "$os_mode" || -n "$os" ]]; then 586 error "'--incoming' can't be used together with '--os' nor '--os-mode'" 587 return 1 588 fi 589 590 os_mode="original" 591 os="$VM_DIR/$vm_incoming/os.qcow2" 592 elif [[ -n "$vm_migrate_to" ]]; then 593 [[ "$os_mode" != "backing" ]] && warning "Using 'backing' mode for OS since '--migrate-to' is used" 594 os_mode=backing 595 fi 596 597 notice "Creating new VM in $vm_dir" 598 mkdir -p $vm_dir 599 600 if [[ "$os_mode" == "backing" ]]; then 601 notice "Creating backing file for OS image file: $os" 602 if ! $QEMU_IMG_BIN create -f qcow2 -b $os $vm_dir/os.qcow2 -F qcow2; then 603 error "Failed to create OS backing file in '$vm_dir/os.qcow2' using '$os'" 604 return 1 605 fi 606 607 local os=$vm_dir/os.qcow2 608 elif [[ "$os_mode" == "original" ]]; then 609 warning "Using original OS image file: $os" 610 elif [[ "$os_mode" != "snapshot" ]]; then 611 if [[ -z "$os_mode" ]]; then 612 notice "No '--os-mode' parameter provided - using 'snapshot'" 613 os_mode="snapshot" 614 else 615 error "Invalid '--os-mode=$os_mode'" 616 return 1 617 fi 618 fi 619 620 local qemu_mask_param="VM_${vm_num}_qemu_mask" 621 local qemu_numa_node_param="VM_${vm_num}_qemu_numa_node" 622 623 if [[ -z "${!qemu_mask_param}" ]] || [[ -z "${!qemu_numa_node_param}" ]]; then 624 error "Parameters ${qemu_mask_param} or ${qemu_numa_node_param} not found in autotest.config file" 625 return 1 626 fi 627 628 local task_mask=${!qemu_mask_param} 629 630 notice "TASK MASK: $task_mask" 631 local cmd=(taskset -a -c "$task_mask" "$QEMU_BIN") 632 local vm_socket_offset=$((10000 + 100 * vm_num)) 633 634 local ssh_socket=$((vm_socket_offset + 0)) 635 local fio_socket=$((vm_socket_offset + 1)) 636 local monitor_port=$((vm_socket_offset + 2)) 637 local migration_port=$((vm_socket_offset + 3)) 638 local gdbserver_socket=$((vm_socket_offset + 4)) 639 local vnc_socket=$((100 + vm_num)) 640 local qemu_pid_file="$vm_dir/qemu.pid" 641 local cpu_num=0 642 643 set +x 644 # cpu list for taskset can be comma separated or range 645 # or both at the same time, so first split on commas 646 cpu_list=$(echo $task_mask | tr "," "\n") 647 queue_number=0 648 for c in $cpu_list; do 649 # if range is detected - count how many cpus 650 if [[ $c =~ [0-9]+-[0-9]+ ]]; then 651 val=$((c - 1)) 652 val=${val#-} 653 else 654 val=1 655 fi 656 cpu_num=$((cpu_num + val)) 657 queue_number=$((queue_number + val)) 658 done 659 660 if [ -z $queue_number ]; then 661 queue_number=$cpu_num 662 fi 663 664 xtrace_restore 665 666 local node_num=${!qemu_numa_node_param} 667 local boot_disk_present=false 668 notice "NUMA NODE: $node_num" 669 cmd+=(-m "$guest_memory" --enable-kvm -cpu host -smp "$cpu_num" -vga std -vnc ":$vnc_socket" -daemonize) 670 cmd+=(-object "memory-backend-file,id=mem,size=${guest_memory}M,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes=$node_num,policy=bind") 671 [[ $os_mode == snapshot ]] && cmd+=(-snapshot) 672 [[ -n "$vm_incoming" ]] && cmd+=(-incoming "tcp:0:$migration_port") 673 cmd+=(-monitor "telnet:127.0.0.1:$monitor_port,server,nowait") 674 cmd+=(-numa "node,memdev=mem") 675 cmd+=(-pidfile "$qemu_pid_file") 676 cmd+=(-serial "file:$vm_dir/serial.log") 677 cmd+=(-D "$vm_dir/qemu.log") 678 cmd+=(-chardev "file,path=$vm_dir/seabios.log,id=seabios" -device "isa-debugcon,iobase=0x402,chardev=seabios") 679 cmd+=(-net "user,hostfwd=tcp::$ssh_socket-:22,hostfwd=tcp::$fio_socket-:8765") 680 cmd+=(-net nic) 681 if [[ -z "$boot_from" ]]; then 682 cmd+=(-drive "file=$os,if=none,id=os_disk") 683 cmd+=(-device "ide-hd,drive=os_disk,bootindex=0") 684 fi 685 686 if ((${#disks[@]} == 0)) && [[ $disk_type_g == virtio* ]]; then 687 disks=("default_virtio.img") 688 elif ((${#disks[@]} == 0)); then 689 error "No disks defined, aborting" 690 return 1 691 fi 692 693 for disk in "${disks[@]}"; do 694 # Each disk can define its type in a form of a disk_name,type. The remaining parts 695 # of the string are dropped. 696 IFS="," read -r disk disk_type _ <<< "$disk" 697 [[ -z $disk_type ]] && disk_type=$disk_type_g 698 699 case $disk_type in 700 virtio) 701 local raw_name="RAWSCSI" 702 local raw_disk=$vm_dir/test.img 703 704 # Create disk file if it not exist or it is smaller than 1G 705 if [[ -f $disk && $(stat --printf="%s" $disk) -ge $((1024 * 1024 * 1024)) ]]; then 706 raw_disk=$disk 707 notice "Using existing image $raw_disk" 708 else 709 notice "Creating Virtio disc $raw_disk" 710 dd if=/dev/zero of=$raw_disk bs=1024k count=1024 711 fi 712 713 cmd+=(-device "virtio-scsi-pci,num_queues=$queue_number") 714 cmd+=(-device "scsi-hd,drive=hd$i,vendor=$raw_name") 715 cmd+=(-drive "if=none,id=hd$i,file=$raw_disk,format=raw$raw_cache") 716 ;; 717 spdk_vhost_scsi) 718 notice "using socket $vhost_dir/naa.$disk.$vm_num" 719 cmd+=(-chardev "socket,id=char_$disk,path=$vhost_dir/naa.$disk.$vm_num") 720 cmd+=(-device "vhost-user-scsi-pci,id=scsi_$disk,num_queues=$queue_number,chardev=char_$disk") 721 if [[ "$disk" == "$boot_from" ]]; then 722 cmd[-1]+=,bootindex=0 723 boot_disk_present=true 724 fi 725 ;; 726 spdk_vhost_blk) 727 notice "using socket $vhost_dir/naa.$disk.$vm_num" 728 cmd+=(-chardev "socket,id=char_$disk,path=$vhost_dir/naa.$disk.$vm_num") 729 cmd+=(-device "vhost-user-blk-pci,num-queues=$queue_number,chardev=char_$disk") 730 if [[ "$disk" == "$boot_from" ]]; then 731 cmd[-1]+=,bootindex=0 732 boot_disk_present=true 733 fi 734 735 if $packed; then 736 check_qemu_packedring_support 737 notice "Enabling packed ring support for VM $vm_num, controller $vhost_dir/naa.$disk.$vm_num" 738 cmd[-1]+=,packed=on 739 fi 740 ;; 741 kernel_vhost) 742 if [[ -z $disk ]]; then 743 error "need WWN for $disk_type" 744 return 1 745 elif [[ ! $disk =~ ^[[:alpha:]]{3}[.][[:xdigit:]]+$ ]]; then 746 error "$disk_type - disk(wnn)=$disk does not look like WNN number" 747 return 1 748 fi 749 notice "Using kernel vhost disk wwn=$disk" 750 cmd+=(-device "vhost-scsi-pci,wwpn=$disk,num_queues=$queue_number") 751 ;; 752 vfio_user) 753 notice "using socket $VM_DIR/$vm_num/domain/muser$disk/$disk/cntrl" 754 cmd+=(-device "vfio-user-pci,x-msg-timeout=5000,socket=$VM_DIR/$vm_num/muser/domain/muser$disk/$disk/cntrl") 755 if [[ "$disk" == "$boot_from" ]]; then 756 cmd[-1]+=",bootindex=0" 757 boot_disk_present=true 758 fi 759 ;; 760 *) 761 error "unknown mode '$disk_type', use: virtio, spdk_vhost_scsi, spdk_vhost_blk, kernel_vhost or vfio_user" 762 return 1 763 ;; 764 esac 765 done 766 767 if [[ -n $boot_from ]] && [[ $boot_disk_present == false ]]; then 768 error "Boot from $boot_from is selected but device is not present" 769 return 1 770 fi 771 772 ((${#qemu_args[@]})) && cmd+=("${qemu_args[@]}") 773 notice "Saving to $vm_dir/run.sh" 774 cat <<- RUN > "$vm_dir/run.sh" 775 #!/bin/bash 776 rootdir=$rootdir 777 source "\$rootdir/test/scheduler/common.sh" 778 qemu_log () { 779 echo "=== qemu.log ===" 780 [[ -s $vm_dir/qemu.log ]] && cat $vm_dir/qemu.log 781 echo "=== qemu.log ===" 782 } 783 784 if [[ \$EUID -ne 0 ]]; then 785 echo "Go away user come back as root" 786 exit 1 787 fi 788 789 trap "qemu_log" EXIT 790 791 qemu_cmd=($(printf '%s\n' "${cmd[@]}")) 792 chmod +r $vm_dir/* 793 echo "Running VM in $vm_dir" 794 rm -f $qemu_pid_file 795 cgroup=\$(get_cgroup \$$) 796 set_cgroup_attr_top_bottom \$$ cgroup.subtree_control "+cpuset" 797 create_cgroup \$cgroup/qemu.$vm_num 798 set_cgroup_attr "\$cgroup/qemu.$vm_num" cpuset.mems "$node_num" 799 set_cgroup_attr "\$cgroup/qemu.$vm_num" cpuset.cpus "$task_mask" 800 "\${qemu_cmd[@]}" 801 802 echo "Waiting for QEMU pid file" 803 sleep 1 804 [[ ! -f $qemu_pid_file ]] && sleep 1 805 [[ ! -f $qemu_pid_file ]] && echo "ERROR: no qemu pid file found" && exit 1 806 set_cgroup_attr "\$cgroup/qemu.$vm_num" cgroup.threads \$(< "$qemu_pid_file") 807 exit 0 808 # EOF 809 RUN 810 chmod +x $vm_dir/run.sh 811 812 # Save generated sockets redirection 813 echo $ssh_socket > $vm_dir/ssh_socket 814 echo $fio_socket > $vm_dir/fio_socket 815 echo $monitor_port > $vm_dir/monitor_port 816 817 rm -f $vm_dir/migration_port 818 [[ -z $vm_incoming ]] || echo $migration_port > $vm_dir/migration_port 819 820 echo $gdbserver_socket > $vm_dir/gdbserver_socket 821 echo $vnc_socket >> $vm_dir/vnc_socket 822 823 [[ -z $vm_incoming ]] || ln -fs $VM_DIR/$vm_incoming $vm_dir/vm_incoming 824 [[ -z $vm_migrate_to ]] || ln -fs $VM_DIR/$vm_migrate_to $vm_dir/vm_migrate_to 825} 826 827function vm_run() { 828 local OPTIND optchar vm 829 local run_all=false 830 local vms_to_run="" 831 832 while getopts 'a-:' optchar; do 833 case "$optchar" in 834 a) run_all=true ;; 835 *) 836 error "Unknown param $OPTARG" 837 return 1 838 ;; 839 esac 840 done 841 842 if $run_all; then 843 vms_to_run="$(vm_list_all)" 844 else 845 shift $((OPTIND - 1)) 846 for vm in "$@"; do 847 vm_num_is_valid $1 || return 1 848 if [[ ! -x $VM_DIR/$vm/run.sh ]]; then 849 error "VM$vm not defined - setup it first" 850 return 1 851 fi 852 vms_to_run+=" $vm" 853 done 854 fi 855 856 for vm in $vms_to_run; do 857 if vm_is_running $vm; then 858 warning "VM$vm ($VM_DIR/$vm) already running" 859 continue 860 fi 861 862 notice "running $VM_DIR/$vm/run.sh" 863 if ! $VM_DIR/$vm/run.sh; then 864 error "FAILED to run vm $vm" 865 return 1 866 fi 867 done 868} 869 870function vm_print_logs() { 871 vm_num=$1 872 warning "================" 873 warning "QEMU LOG:" 874 if [[ -r $VM_DIR/$vm_num/qemu.log ]]; then 875 cat $VM_DIR/$vm_num/qemu.log 876 else 877 warning "LOG qemu.log not found" 878 fi 879 880 warning "VM LOG:" 881 if [[ -r $VM_DIR/$vm_num/serial.log ]]; then 882 cat $VM_DIR/$vm_num/serial.log 883 else 884 warning "LOG serial.log not found" 885 fi 886 887 warning "SEABIOS LOG:" 888 if [[ -r $VM_DIR/$vm_num/seabios.log ]]; then 889 cat $VM_DIR/$vm_num/seabios.log 890 else 891 warning "LOG seabios.log not found" 892 fi 893 warning "================" 894} 895 896# Wait for all created VMs to boot. 897# param $1 max wait time 898function vm_wait_for_boot() { 899 assert_number $1 900 901 xtrace_disable 902 903 local all_booted=false 904 local timeout_time=$1 905 [[ $timeout_time -lt 10 ]] && timeout_time=10 906 local timeout_time 907 timeout_time=$(date -d "+$timeout_time seconds" +%s) 908 909 notice "Waiting for VMs to boot" 910 shift 911 if [[ "$*" == "" ]]; then 912 local vms_to_check="$VM_DIR/[0-9]*" 913 else 914 local vms_to_check="" 915 for vm in "$@"; do 916 vms_to_check+=" $VM_DIR/$vm" 917 done 918 fi 919 920 for vm in $vms_to_check; do 921 local vm_num 922 vm_num=$(basename $vm) 923 local i=0 924 notice "waiting for VM$vm_num ($vm)" 925 while ! vm_os_booted $vm_num; do 926 if ! vm_is_running $vm_num; then 927 warning "VM $vm_num is not running" 928 vm_print_logs $vm_num 929 xtrace_restore 930 return 1 931 fi 932 933 if [[ $(date +%s) -gt $timeout_time ]]; then 934 warning "timeout waiting for machines to boot" 935 vm_print_logs $vm_num 936 xtrace_restore 937 return 1 938 fi 939 if ((i > 30)); then 940 local i=0 941 echo 942 fi 943 echo -n "." 944 sleep 1 945 done 946 echo "" 947 notice "VM$vm_num ready" 948 #Change Timeout for stopping services to prevent lengthy powerdowns 949 #Check that remote system is not Cygwin in case of Windows VMs 950 local vm_os 951 vm_os=$(vm_exec $vm_num "uname -o") 952 if [[ "$vm_os" != "Cygwin" ]]; then 953 vm_exec $vm_num "echo 'DefaultTimeoutStopSec=10' >> /etc/systemd/system.conf; systemctl daemon-reexec" 954 fi 955 done 956 957 notice "all VMs ready" 958 xtrace_restore 959 return 0 960} 961 962function vm_start_fio_server() { 963 local OPTIND optchar 964 local readonly='' 965 local fio_bin='' 966 while getopts ':-:' optchar; do 967 case "$optchar" in 968 -) 969 case "$OPTARG" in 970 fio-bin=*) local fio_bin="${OPTARG#*=}" ;; 971 readonly) local readonly="--readonly" ;; 972 *) error "Invalid argument '$OPTARG'" && return 1 ;; 973 esac 974 ;; 975 *) error "Invalid argument '$OPTARG'" && return 1 ;; 976 esac 977 done 978 979 shift $((OPTIND - 1)) 980 for vm_num in "$@"; do 981 notice "Starting fio server on VM$vm_num" 982 if [[ $fio_bin != "" ]]; then 983 vm_exec $vm_num 'cat > /root/fio; chmod +x /root/fio' < $fio_bin 984 vm_exec $vm_num /root/fio $readonly --eta=never --server --daemonize=/root/fio.pid 985 else 986 vm_exec $vm_num fio $readonly --eta=never --server --daemonize=/root/fio.pid 987 fi 988 done 989} 990 991function vm_check_scsi_location() { 992 # Script to find wanted disc 993 local script='shopt -s nullglob; 994 for entry in /sys/block/sd*; do 995 disk_type="$(cat $entry/device/vendor)"; 996 if [[ $disk_type == INTEL* ]] || [[ $disk_type == RAWSCSI* ]] || [[ $disk_type == LIO-ORG* ]]; then 997 fname=$(basename $entry); 998 echo -n " $fname"; 999 fi; 1000 done' 1001 1002 SCSI_DISK="$(echo "$script" | vm_exec $1 bash -s)" 1003 1004 if [[ -z "$SCSI_DISK" ]]; then 1005 error "no test disk found!" 1006 return 1 1007 fi 1008} 1009 1010# Script to perform scsi device reset on all disks in VM 1011# param $1 VM num 1012# param $2..$n Disks to perform reset on 1013function vm_reset_scsi_devices() { 1014 for disk in "${@:2}"; do 1015 notice "VM$1 Performing device reset on disk $disk" 1016 vm_exec $1 sg_reset /dev/$disk -vNd 1017 done 1018} 1019 1020function vm_check_blk_location() { 1021 local script='shopt -s nullglob; cd /sys/block; echo vd*' 1022 SCSI_DISK="$(echo "$script" | vm_exec $1 bash -s)" 1023 1024 if [[ -z "$SCSI_DISK" ]]; then 1025 error "no blk test disk found!" 1026 return 1 1027 fi 1028} 1029 1030function vm_check_nvme_location() { 1031 SCSI_DISK="$(vm_exec $1 "grep -l SPDK /sys/class/nvme/*/model" | awk -F/ '{print $5"n1"}')" 1032 if [[ -z "$SCSI_DISK" ]]; then 1033 error "no vfio-user nvme test disk found!" 1034 return 1 1035 fi 1036} 1037 1038function run_fio() { 1039 local arg 1040 local job_file="" 1041 local fio_bin="" 1042 local vms=() 1043 local out="" 1044 local vm 1045 local run_server_mode=true 1046 local run_plugin_mode=false 1047 local fio_start_cmd 1048 local fio_output_format="normal" 1049 local fio_gtod_reduce=false 1050 local wait_for_fio=true 1051 1052 for arg in "$@"; do 1053 case "$arg" in 1054 --job-file=*) local job_file="${arg#*=}" ;; 1055 --fio-bin=*) local fio_bin="${arg#*=}" ;; 1056 --vm=*) vms+=("${arg#*=}") ;; 1057 --out=*) 1058 local out="${arg#*=}" 1059 mkdir -p $out 1060 ;; 1061 --local) run_server_mode=false ;; 1062 --plugin) 1063 notice "Using plugin mode. Disabling server mode." 1064 run_plugin_mode=true 1065 run_server_mode=false 1066 ;; 1067 --json) fio_output_format="json" ;; 1068 --hide-results) hide_results=true ;; 1069 --no-wait-for-fio) wait_for_fio=false ;; 1070 --gtod-reduce) fio_gtod_reduce=true ;; 1071 *) 1072 error "Invalid argument '$arg'" 1073 return 1 1074 ;; 1075 esac 1076 done 1077 1078 if [[ -n "$fio_bin" && ! -r "$fio_bin" ]]; then 1079 error "FIO binary '$fio_bin' does not exist" 1080 return 1 1081 fi 1082 1083 if [[ -z "$fio_bin" ]]; then 1084 fio_bin="fio" 1085 fi 1086 1087 if [[ ! -r "$job_file" ]]; then 1088 error "Fio job '$job_file' does not exist" 1089 return 1 1090 fi 1091 1092 fio_start_cmd="$fio_bin --eta=never " 1093 1094 local job_fname 1095 job_fname=$(basename "$job_file") 1096 log_fname="${job_fname%%.*}.log" 1097 fio_start_cmd+=" --output=$out/$log_fname --output-format=$fio_output_format " 1098 1099 # prepare job file for each VM 1100 for vm in "${vms[@]}"; do 1101 local vm_num=${vm%%:*} 1102 local vmdisks=${vm#*:} 1103 1104 sed "s@filename=@filename=$vmdisks@" $job_file | vm_exec $vm_num "cat > /root/$job_fname" 1105 1106 if $fio_gtod_reduce; then 1107 vm_exec $vm_num "echo 'gtod_reduce=1' >> /root/$job_fname" 1108 fi 1109 1110 vm_exec $vm_num cat /root/$job_fname 1111 1112 if $run_server_mode; then 1113 fio_start_cmd+="--client=127.0.0.1,$(vm_fio_socket $vm_num) --remote-config /root/$job_fname " 1114 fi 1115 1116 if ! $run_server_mode; then 1117 if [[ -n "$fio_bin" ]]; then 1118 if ! $run_plugin_mode; then 1119 vm_exec $vm_num 'cat > /root/fio; chmod +x /root/fio' < $fio_bin 1120 vm_fio_bin="/root/fio" 1121 else 1122 vm_fio_bin="/usr/src/fio/fio" 1123 fi 1124 fi 1125 1126 notice "Running local fio on VM $vm_num" 1127 vm_exec $vm_num "$vm_fio_bin --output=/root/$log_fname --output-format=$fio_output_format /root/$job_fname & echo \$! > /root/fio.pid" & 1128 vm_exec_pids+=("$!") 1129 fi 1130 done 1131 1132 if ! $run_server_mode; then 1133 if ! $wait_for_fio; then 1134 return 0 1135 fi 1136 echo "Waiting for guest fio instances to finish.." 1137 wait "${vm_exec_pids[@]}" 1138 1139 for vm in "${vms[@]}"; do 1140 local vm_num=${vm%%:*} 1141 vm_exec $vm_num cat /root/$log_fname > "$out/vm${vm_num}_${log_fname}" 1142 done 1143 return 0 1144 fi 1145 1146 $fio_start_cmd 1147 sleep 1 1148 1149 if [[ "$fio_output_format" == "json" ]]; then 1150 # Fio in client-server mode produces a lot of "trash" output 1151 # preceding JSON structure, making it not possible to parse. 1152 # Remove these lines from file. 1153 # shellcheck disable=SC2005 1154 echo "$(grep -vP '^[<\w]' "$out/$log_fname")" > "$out/$log_fname" 1155 fi 1156 1157 if [[ ! $hide_results ]]; then 1158 cat $out/$log_fname 1159 fi 1160} 1161 1162# Parsing fio results for json output and client-server mode only! 1163function parse_fio_results() { 1164 local fio_log_dir=$1 1165 local fio_log_filename=$2 1166 local fio_csv_filename 1167 1168 # Variables used in parsing loop 1169 local log_file 1170 local rwmode mixread mixwrite 1171 local lat_key lat_divisor 1172 local client_stats iops bw 1173 local read_avg_lat read_min_lat read_max_lat 1174 local write_avg_lat write_min_lat write_min_lat 1175 local clients 1176 1177 declare -A results 1178 results["iops"]=0 1179 results["bw"]=0 1180 results["avg_lat"]=0 1181 results["min_lat"]=0 1182 results["max_lat"]=0 1183 1184 # Loop using the log filename to see if there are any other 1185 # matching files. This is in case we ran fio test multiple times. 1186 log_files=("$fio_log_dir/$fio_log_filename"*) 1187 for log_file in "${log_files[@]}"; do 1188 # Save entire array to avoid opening $log_file multiple times 1189 clients=$(jq -r '.client_stats' "$log_file") 1190 [[ -n $clients ]] 1191 rwmode=$(jq -r '.[0]["job options"]["rw"]' <<< "$clients") 1192 mixread=1 1193 mixwrite=1 1194 if [[ $rwmode = *"rw"* ]]; then 1195 mixread=$(jq -r '.[0]["job options"]["rwmixread"]' <<< "$clients") 1196 mixread=$(bc -l <<< "scale=3; $mixread/100") 1197 mixwrite=$(bc -l <<< "scale=3; 1-$mixread") 1198 fi 1199 1200 client_stats=$(jq -r '.[] | select(.jobname == "All clients")' <<< "$clients") 1201 if [[ -z $client_stats ]]; then 1202 # Potentially single client (single VM) 1203 client_stats=$(jq -r '.[]' <<< "$clients") 1204 fi 1205 1206 # Check latency unit and later normalize to microseconds 1207 lat_key="lat_us" 1208 lat_divisor=1 1209 if jq -er '.read["lat_ns"]' &> /dev/null <<< $client_stats; then 1210 lat_key="lat_ns" 1211 lat_divisor=1000 1212 fi 1213 1214 # Horrific bash float point arithmetic operations below. 1215 # Viewer discretion is advised. 1216 iops=$(jq -r '[.read["iops"],.write["iops"]] | add' <<< $client_stats) 1217 bw=$(jq -r '[.read["bw"],.write["bw"]] | add' <<< $client_stats) 1218 read_avg_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["mean"]' <<< $client_stats) 1219 read_min_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["min"]' <<< $client_stats) 1220 read_max_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["max"]' <<< $client_stats) 1221 write_avg_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["mean"]' <<< $client_stats) 1222 write_min_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["min"]' <<< $client_stats) 1223 write_max_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["max"]' <<< $client_stats) 1224 1225 results["iops"]=$(bc -l <<< "${results[iops]} + $iops") 1226 results["bw"]=$(bc -l <<< "${results[bw]} + $bw") 1227 results["avg_lat"]=$(bc -l <<< "${results[avg_lat]} + ($mixread*$read_avg_lat + $mixwrite*$write_avg_lat)/$lat_divisor") 1228 results["min_lat"]=$(bc -l <<< "${results[min_lat]} + ($mixread*$read_min_lat + $mixwrite*$write_min_lat)/$lat_divisor") 1229 results["max_lat"]=$(bc -l <<< "${results[max_lat]} + ($mixread*$read_max_lat + $mixwrite*$write_max_lat)/$lat_divisor") 1230 done 1231 1232 results["iops"]=$(bc -l <<< "scale=3; ${results[iops]} / ${#log_files[@]}") 1233 results["bw"]=$(bc -l <<< "scale=3; ${results[bw]} / ${#log_files[@]}") 1234 results["avg_lat"]=$(bc -l <<< "scale=3; ${results[avg_lat]} / ${#log_files[@]}") 1235 results["min_lat"]=$(bc -l <<< "scale=3; ${results[min_lat]} / ${#log_files[@]}") 1236 results["max_lat"]=$(bc -l <<< "scale=3; ${results[max_lat]} / ${#log_files[@]}") 1237 1238 fio_csv_filename="${fio_log_filename%%.*}.csv" 1239 cat <<- EOF > "$fio_log_dir/$fio_csv_filename" 1240 iops,bw,avg_lat,min_lat,max_lat 1241 ${results["iops"]},${results["bw"]},${results["avg_lat"]},${results["min_lat"]},${results["max_lat"]} 1242 EOF 1243} 1244 1245# Shutdown or kill any running VM and SPDK APP. 1246# 1247function at_app_exit() { 1248 local vhost_name 1249 1250 notice "APP EXITING" 1251 notice "killing all VMs" 1252 vm_kill_all 1253 # Kill vhost application 1254 notice "killing vhost app" 1255 1256 for vhost_name in "$TARGET_DIR"/*; do 1257 vhost_kill "$(basename "$vhost_name")" 1258 done 1259 1260 notice "EXIT DONE" 1261} 1262 1263function error_exit() { 1264 trap - ERR 1265 print_backtrace 1266 set +e 1267 error "Error on $1 $2" 1268 1269 at_app_exit 1270 exit 1 1271} 1272