1: ${SPDK_VHOST_VERBOSE=false} 2: ${VHOST_DIR="$HOME/vhost_test"} 3: ${QEMU_BIN="qemu-system-x86_64"} 4: ${QEMU_IMG_BIN="qemu-img"} 5 6TEST_DIR=$(readlink -f $rootdir/..) 7VM_DIR=$VHOST_DIR/vms 8TARGET_DIR=$VHOST_DIR/vhost 9VM_PASSWORD="root" 10 11VM_IMAGE=$HOME/spdk_test_image.qcow2 12 13if ! hash $QEMU_IMG_BIN $QEMU_BIN; then 14 error 'QEMU is not installed on this system. Unable to run vhost tests.' 15 exit 1 16fi 17 18mkdir -p $VHOST_DIR 19mkdir -p $VM_DIR 20mkdir -p $TARGET_DIR 21 22# 23# Source config describing QEMU and VHOST cores and NUMA 24# 25source $rootdir/test/vhost/common/autotest.config 26 27function vhosttestinit() { 28 if [ "$TEST_MODE" == "iso" ]; then 29 $rootdir/scripts/setup.sh 30 31 # Look for the VM image 32 if [[ ! -f $VM_IMAGE ]]; then 33 echo "VM image not found at $VM_IMAGE" 34 echo "Download to $HOME? [yn]" 35 read -r download 36 if [ "$download" = "y" ]; then 37 curl https://ci.spdk.io/download/test_resources/vhost_vm_image.tar.gz | tar xz -C $HOME 38 fi 39 fi 40 fi 41 42 # Look for the VM image 43 if [[ "$1" != "--no_vm" ]] && [[ ! -f $VM_IMAGE ]]; then 44 error "VM image not found at $VM_IMAGE" 45 exit 1 46 fi 47} 48 49function vhosttestfini() { 50 if [ "$TEST_MODE" == "iso" ]; then 51 $rootdir/scripts/setup.sh reset 52 fi 53} 54 55function message() { 56 local verbose_out 57 if ! $SPDK_VHOST_VERBOSE; then 58 verbose_out="" 59 elif [[ ${FUNCNAME[2]} == "source" ]]; then 60 verbose_out=" (file $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[1]})" 61 else 62 verbose_out=" (function ${FUNCNAME[2]}:${BASH_LINENO[1]})" 63 fi 64 65 local msg_type="$1" 66 shift 67 echo -e "${msg_type}${verbose_out}: $*" 68} 69 70function fail() { 71 echo "===========" >&2 72 message "FAIL" "$@" >&2 73 echo "===========" >&2 74 exit 1 75} 76 77function error() { 78 echo "===========" >&2 79 message "ERROR" "$@" >&2 80 echo "===========" >&2 81 # Don't 'return 1' since the stack trace will be incomplete (why?) missing upper command. 82 false 83} 84 85function warning() { 86 message "WARN" "$@" >&2 87} 88 89function notice() { 90 message "INFO" "$@" 91} 92 93function check_qemu_packedring_support() { 94 qemu_version=$($QEMU_BIN -version | grep -Po "(?<=version )\d+.\d+.\d+") 95 if [[ "$qemu_version" < "4.2.0" ]]; then 96 error "This qemu binary does not support packed ring" 97 fi 98} 99 100function get_vhost_dir() { 101 local vhost_name="$1" 102 103 if [[ -z "$vhost_name" ]]; then 104 error "vhost name must be provided to get_vhost_dir" 105 return 1 106 fi 107 108 echo "$TARGET_DIR/${vhost_name}" 109} 110 111function vhost_run() { 112 local vhost_name="$1" 113 local run_gen_nvme=true 114 115 if [[ -z "$vhost_name" ]]; then 116 error "vhost name must be provided to vhost_run" 117 return 1 118 fi 119 shift 120 121 if [[ "$1" == "--no-gen-nvme" ]]; then 122 notice "Skipping gen_nvmf.sh NVMe bdev configuration" 123 run_gen_nvme=false 124 shift 125 fi 126 127 local vhost_dir 128 vhost_dir="$(get_vhost_dir $vhost_name)" 129 local vhost_app="$SPDK_BIN_DIR/vhost" 130 local vhost_log_file="$vhost_dir/vhost.log" 131 local vhost_pid_file="$vhost_dir/vhost.pid" 132 local vhost_socket="$vhost_dir/usvhost" 133 notice "starting vhost app in background" 134 [[ -r "$vhost_pid_file" ]] && vhost_kill $vhost_name 135 [[ -d $vhost_dir ]] && rm -f $vhost_dir/* 136 mkdir -p $vhost_dir 137 138 if [[ ! -x $vhost_app ]]; then 139 error "application not found: $vhost_app" 140 return 1 141 fi 142 143 local cmd="$vhost_app -r $vhost_dir/rpc.sock $*" 144 145 notice "Loging to: $vhost_log_file" 146 notice "Socket: $vhost_socket" 147 notice "Command: $cmd" 148 149 timing_enter vhost_start 150 cd $vhost_dir 151 $cmd & 152 vhost_pid=$! 153 echo $vhost_pid > $vhost_pid_file 154 155 notice "waiting for app to run..." 156 waitforlisten "$vhost_pid" "$vhost_dir/rpc.sock" 157 #do not generate nvmes if pci access is disabled 158 if [[ "$cmd" != *"--no-pci"* ]] && [[ "$cmd" != *"-u"* ]] && $run_gen_nvme; then 159 $rootdir/scripts/gen_nvme.sh | $rootdir/scripts/rpc.py -s $vhost_dir/rpc.sock load_subsystem_config 160 fi 161 162 notice "vhost started - pid=$vhost_pid" 163 timing_exit vhost_start 164} 165 166function vhost_kill() { 167 local rc=0 168 local vhost_name="$1" 169 170 if [[ -z "$vhost_name" ]]; then 171 error "Must provide vhost name to vhost_kill" 172 return 0 173 fi 174 175 local vhost_dir 176 vhost_dir="$(get_vhost_dir $vhost_name)" 177 local vhost_pid_file="$vhost_dir/vhost.pid" 178 179 if [[ ! -r $vhost_pid_file ]]; then 180 warning "no vhost pid file found" 181 return 0 182 fi 183 184 timing_enter vhost_kill 185 local vhost_pid 186 vhost_pid="$(cat $vhost_pid_file)" 187 notice "killing vhost (PID $vhost_pid) app" 188 189 if kill -INT $vhost_pid > /dev/null; then 190 notice "sent SIGINT to vhost app - waiting 60 seconds to exit" 191 for ((i = 0; i < 60; i++)); do 192 if kill -0 $vhost_pid; then 193 echo "." 194 sleep 1 195 else 196 break 197 fi 198 done 199 if kill -0 $vhost_pid; then 200 error "ERROR: vhost was NOT killed - sending SIGABRT" 201 kill -ABRT $vhost_pid 202 rm $vhost_pid_file 203 rc=1 204 else 205 while kill -0 $vhost_pid; do 206 echo "." 207 done 208 fi 209 elif kill -0 $vhost_pid; then 210 error "vhost NOT killed - you need to kill it manually" 211 rc=1 212 else 213 notice "vhost was not running" 214 fi 215 216 timing_exit vhost_kill 217 if [[ $rc == 0 ]]; then 218 rm $vhost_pid_file 219 fi 220 221 rm -rf "$vhost_dir" 222 223 return $rc 224} 225 226function vhost_rpc() { 227 local vhost_name="$1" 228 229 if [[ -z "$vhost_name" ]]; then 230 error "vhost name must be provided to vhost_rpc" 231 return 1 232 fi 233 shift 234 235 $rootdir/scripts/rpc.py -s $(get_vhost_dir $vhost_name)/rpc.sock "$@" 236} 237 238### 239# Mgmt functions 240### 241 242function assert_number() { 243 [[ "$1" =~ [0-9]+ ]] && return 0 244 245 error "Invalid or missing paramter: need number but got '$1'" 246 return 1 247} 248 249# Run command on vm with given password 250# First argument - vm number 251# Second argument - ssh password for vm 252# 253function vm_sshpass() { 254 vm_num_is_valid $1 || return 1 255 256 local ssh_cmd 257 ssh_cmd="sshpass -p $2 ssh \ 258 -o UserKnownHostsFile=/dev/null \ 259 -o StrictHostKeyChecking=no \ 260 -o User=root \ 261 -p $(vm_ssh_socket $1) $VM_SSH_OPTIONS 127.0.0.1" 262 263 shift 2 264 $ssh_cmd "$@" 265} 266 267# Helper to validate VM number 268# param $1 VM number 269# 270function vm_num_is_valid() { 271 [[ "$1" =~ ^[0-9]+$ ]] && return 0 272 273 error "Invalid or missing paramter: vm number '$1'" 274 return 1 275} 276 277# Print network socket for given VM number 278# param $1 virtual machine number 279# 280function vm_ssh_socket() { 281 vm_num_is_valid $1 || return 1 282 local vm_dir="$VM_DIR/$1" 283 284 cat $vm_dir/ssh_socket 285} 286 287function vm_fio_socket() { 288 vm_num_is_valid $1 || return 1 289 local vm_dir="$VM_DIR/$1" 290 291 cat $vm_dir/fio_socket 292} 293 294# Execute command on given VM 295# param $1 virtual machine number 296# 297function vm_exec() { 298 vm_num_is_valid $1 || return 1 299 300 local vm_num="$1" 301 shift 302 303 sshpass -p "$VM_PASSWORD" ssh \ 304 -o UserKnownHostsFile=/dev/null \ 305 -o StrictHostKeyChecking=no \ 306 -o User=root \ 307 -p $(vm_ssh_socket $vm_num) $VM_SSH_OPTIONS 127.0.0.1 \ 308 "$@" 309} 310 311# Execute scp command on given VM 312# param $1 virtual machine number 313# 314function vm_scp() { 315 vm_num_is_valid $1 || return 1 316 317 local vm_num="$1" 318 shift 319 320 sshpass -p "$VM_PASSWORD" scp \ 321 -o UserKnownHostsFile=/dev/null \ 322 -o StrictHostKeyChecking=no \ 323 -o User=root \ 324 -P $(vm_ssh_socket $vm_num) $VM_SSH_OPTIONS \ 325 "$@" 326} 327 328# check if specified VM is running 329# param $1 VM num 330function vm_is_running() { 331 vm_num_is_valid $1 || return 1 332 local vm_dir="$VM_DIR/$1" 333 334 if [[ ! -r $vm_dir/qemu.pid ]]; then 335 return 1 336 fi 337 338 local vm_pid 339 vm_pid="$(cat $vm_dir/qemu.pid)" 340 341 if /bin/kill -0 $vm_pid; then 342 return 0 343 else 344 if [[ $EUID -ne 0 ]]; then 345 warning "not root - assuming VM running since can't be checked" 346 return 0 347 fi 348 349 # not running - remove pid file 350 rm $vm_dir/qemu.pid 351 return 1 352 fi 353} 354 355# check if specified VM is running 356# param $1 VM num 357function vm_os_booted() { 358 vm_num_is_valid $1 || return 1 359 local vm_dir="$VM_DIR/$1" 360 361 if [[ ! -r $vm_dir/qemu.pid ]]; then 362 error "VM $1 is not running" 363 return 1 364 fi 365 366 if ! VM_SSH_OPTIONS="-o ControlMaster=no" vm_exec $1 "true" 2> /dev/null; then 367 # Shutdown existing master. Ignore errors as it might not exist. 368 VM_SSH_OPTIONS="-O exit" vm_exec $1 "true" 2> /dev/null 369 return 1 370 fi 371 372 return 0 373} 374 375# Shutdown given VM 376# param $1 virtual machine number 377# return non-zero in case of error. 378function vm_shutdown() { 379 vm_num_is_valid $1 || return 1 380 local vm_dir="$VM_DIR/$1" 381 if [[ ! -d "$vm_dir" ]]; then 382 error "VM$1 ($vm_dir) not exist - setup it first" 383 return 1 384 fi 385 386 if ! vm_is_running $1; then 387 notice "VM$1 ($vm_dir) is not running" 388 return 0 389 fi 390 391 # Temporarily disabling exit flag for next ssh command, since it will 392 # "fail" due to shutdown 393 notice "Shutting down virtual machine $vm_dir" 394 set +e 395 vm_exec $1 "nohup sh -c 'shutdown -h -P now'" || true 396 notice "VM$1 is shutting down - wait a while to complete" 397 set -e 398} 399 400# Kill given VM 401# param $1 virtual machine number 402# 403function vm_kill() { 404 vm_num_is_valid $1 || return 1 405 local vm_dir="$VM_DIR/$1" 406 407 if [[ ! -r $vm_dir/qemu.pid ]]; then 408 return 0 409 fi 410 411 local vm_pid 412 vm_pid="$(cat $vm_dir/qemu.pid)" 413 414 notice "Killing virtual machine $vm_dir (pid=$vm_pid)" 415 # First kill should fail, second one must fail 416 if /bin/kill $vm_pid; then 417 notice "process $vm_pid killed" 418 rm $vm_dir/qemu.pid 419 rm -rf $vm_dir 420 elif vm_is_running $1; then 421 error "Process $vm_pid NOT killed" 422 return 1 423 fi 424} 425 426# List all VM numbers in VM_DIR 427# 428function vm_list_all() { 429 local vms 430 vms="$( 431 shopt -s nullglob 432 echo $VM_DIR/[0-9]* 433 )" 434 if [[ -n "$vms" ]]; then 435 basename --multiple $vms 436 fi 437} 438 439# Kills all VM in $VM_DIR 440# 441function vm_kill_all() { 442 local vm 443 for vm in $(vm_list_all); do 444 vm_kill $vm 445 done 446 447 rm -rf $VM_DIR 448} 449 450# Shutdown all VM in $VM_DIR 451# 452function vm_shutdown_all() { 453 # XXX: temporarily disable to debug shutdown issue 454 # xtrace_disable 455 456 local vms 457 vms=$(vm_list_all) 458 local vm 459 460 for vm in $vms; do 461 vm_shutdown $vm 462 done 463 464 notice "Waiting for VMs to shutdown..." 465 local timeo=30 466 while [[ $timeo -gt 0 ]]; do 467 local all_vms_down=1 468 for vm in $vms; do 469 if vm_is_running $vm; then 470 all_vms_down=0 471 break 472 fi 473 done 474 475 if [[ $all_vms_down == 1 ]]; then 476 notice "All VMs successfully shut down" 477 xtrace_restore 478 return 0 479 fi 480 481 ((timeo -= 1)) 482 sleep 1 483 done 484 485 rm -rf $VM_DIR 486 487 xtrace_restore 488} 489 490function vm_setup() { 491 xtrace_disable 492 local OPTIND optchar vm_num 493 494 local os="" 495 local os_mode="" 496 local qemu_args=() 497 local disk_type_g=NOT_DEFINED 498 local read_only="false" 499 # List created of a strings separated with a ":" 500 local disks=() 501 local raw_cache="" 502 local vm_incoming="" 503 local vm_migrate_to="" 504 local force_vm="" 505 local guest_memory=1024 506 local queue_number="" 507 local vhost_dir 508 local packed=false 509 vhost_dir="$(get_vhost_dir 0)" 510 while getopts ':-:' optchar; do 511 case "$optchar" in 512 -) 513 case "$OPTARG" in 514 os=*) os="${OPTARG#*=}" ;; 515 os-mode=*) os_mode="${OPTARG#*=}" ;; 516 qemu-args=*) qemu_args+=("${OPTARG#*=}") ;; 517 disk-type=*) disk_type_g="${OPTARG#*=}" ;; 518 read-only=*) read_only="${OPTARG#*=}" ;; 519 disks=*) IFS=":" read -ra disks <<< "${OPTARG#*=}" ;; 520 raw-cache=*) raw_cache=",cache${OPTARG#*=}" ;; 521 force=*) force_vm=${OPTARG#*=} ;; 522 memory=*) guest_memory=${OPTARG#*=} ;; 523 queue_num=*) queue_number=${OPTARG#*=} ;; 524 incoming=*) vm_incoming="${OPTARG#*=}" ;; 525 migrate-to=*) vm_migrate_to="${OPTARG#*=}" ;; 526 vhost-name=*) vhost_dir="$(get_vhost_dir ${OPTARG#*=})" ;; 527 spdk-boot=*) local boot_from="${OPTARG#*=}" ;; 528 packed) packed=true ;; 529 *) 530 error "unknown argument $OPTARG" 531 return 1 532 ;; 533 esac 534 ;; 535 *) 536 error "vm_create Unknown param $OPTARG" 537 return 1 538 ;; 539 esac 540 done 541 542 # Find next directory we can use 543 if [[ -n $force_vm ]]; then 544 vm_num=$force_vm 545 546 vm_num_is_valid $vm_num || return 1 547 local vm_dir="$VM_DIR/$vm_num" 548 [[ -d $vm_dir ]] && warning "removing existing VM in '$vm_dir'" 549 else 550 local vm_dir="" 551 552 set +x 553 for ((i = 0; i <= 256; i++)); do 554 local vm_dir="$VM_DIR/$i" 555 [[ ! -d $vm_dir ]] && break 556 done 557 xtrace_restore 558 559 vm_num=$i 560 fi 561 562 if [[ $vm_num -eq 256 ]]; then 563 error "no free VM found. do some cleanup (256 VMs created, are you insane?)" 564 return 1 565 fi 566 567 if [[ -n "$vm_migrate_to" && -n "$vm_incoming" ]]; then 568 error "'--incoming' and '--migrate-to' cannot be used together" 569 return 1 570 elif [[ -n "$vm_incoming" ]]; then 571 if [[ -n "$os_mode" || -n "$os" ]]; then 572 error "'--incoming' can't be used together with '--os' nor '--os-mode'" 573 return 1 574 fi 575 576 os_mode="original" 577 os="$VM_DIR/$vm_incoming/os.qcow2" 578 elif [[ -n "$vm_migrate_to" ]]; then 579 [[ "$os_mode" != "backing" ]] && warning "Using 'backing' mode for OS since '--migrate-to' is used" 580 os_mode=backing 581 fi 582 583 notice "Creating new VM in $vm_dir" 584 mkdir -p $vm_dir 585 586 if [[ "$os_mode" == "backing" ]]; then 587 notice "Creating backing file for OS image file: $os" 588 if ! $QEMU_IMG_BIN create -f qcow2 -b $os $vm_dir/os.qcow2; then 589 error "Failed to create OS backing file in '$vm_dir/os.qcow2' using '$os'" 590 return 1 591 fi 592 593 local os=$vm_dir/os.qcow2 594 elif [[ "$os_mode" == "original" ]]; then 595 warning "Using original OS image file: $os" 596 elif [[ "$os_mode" != "snapshot" ]]; then 597 if [[ -z "$os_mode" ]]; then 598 notice "No '--os-mode' parameter provided - using 'snapshot'" 599 os_mode="snapshot" 600 else 601 error "Invalid '--os-mode=$os_mode'" 602 return 1 603 fi 604 fi 605 606 local qemu_mask_param="VM_${vm_num}_qemu_mask" 607 local qemu_numa_node_param="VM_${vm_num}_qemu_numa_node" 608 609 if [[ -z "${!qemu_mask_param}" ]] || [[ -z "${!qemu_numa_node_param}" ]]; then 610 error "Parameters ${qemu_mask_param} or ${qemu_numa_node_param} not found in autotest.config file" 611 return 1 612 fi 613 614 local task_mask=${!qemu_mask_param} 615 616 notice "TASK MASK: $task_mask" 617 local cmd=(taskset -a -c "$task_mask" "$QEMU_BIN") 618 local vm_socket_offset=$((10000 + 100 * vm_num)) 619 620 local ssh_socket=$((vm_socket_offset + 0)) 621 local fio_socket=$((vm_socket_offset + 1)) 622 local monitor_port=$((vm_socket_offset + 2)) 623 local migration_port=$((vm_socket_offset + 3)) 624 local gdbserver_socket=$((vm_socket_offset + 4)) 625 local vnc_socket=$((100 + vm_num)) 626 local qemu_pid_file="$vm_dir/qemu.pid" 627 local cpu_num=0 628 629 set +x 630 # cpu list for taskset can be comma separated or range 631 # or both at the same time, so first split on commas 632 cpu_list=$(echo $task_mask | tr "," "\n") 633 queue_number=0 634 for c in $cpu_list; do 635 # if range is detected - count how many cpus 636 if [[ $c =~ [0-9]+-[0-9]+ ]]; then 637 val=$((c - 1)) 638 val=${val#-} 639 else 640 val=1 641 fi 642 cpu_num=$((cpu_num + val)) 643 queue_number=$((queue_number + val)) 644 done 645 646 if [ -z $queue_number ]; then 647 queue_number=$cpu_num 648 fi 649 650 xtrace_restore 651 652 local node_num=${!qemu_numa_node_param} 653 local boot_disk_present=false 654 notice "NUMA NODE: $node_num" 655 cmd+=(-m "$guest_memory" --enable-kvm -cpu host -smp "$cpu_num" -vga std -vnc ":$vnc_socket" -daemonize) 656 cmd+=(-object "memory-backend-file,id=mem,size=${guest_memory}M,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes=$node_num,policy=bind") 657 [[ $os_mode == snapshot ]] && cmd+=(-snapshot) 658 [[ -n "$vm_incoming" ]] && cmd+=(-incoming "tcp:0:$migration_port") 659 cmd+=(-monitor "telnet:127.0.0.1:$monitor_port,server,nowait") 660 cmd+=(-numa "node,memdev=mem") 661 cmd+=(-pidfile "$qemu_pid_file") 662 cmd+=(-serial "file:$vm_dir/serial.log") 663 cmd+=(-D "$vm_dir/qemu.log") 664 cmd+=(-chardev "file,path=$vm_dir/seabios.log,id=seabios" -device "isa-debugcon,iobase=0x402,chardev=seabios") 665 cmd+=(-net "user,hostfwd=tcp::$ssh_socket-:22,hostfwd=tcp::$fio_socket-:8765") 666 cmd+=(-net nic) 667 if [[ -z "$boot_from" ]]; then 668 cmd+=(-drive "file=$os,if=none,id=os_disk") 669 cmd+=(-device "ide-hd,drive=os_disk,bootindex=0") 670 fi 671 672 if ((${#disks[@]} == 0)) && [[ $disk_type_g == virtio* ]]; then 673 disks=("default_virtio.img") 674 elif ((${#disks[@]} == 0)); then 675 error "No disks defined, aborting" 676 return 1 677 fi 678 679 for disk in "${disks[@]}"; do 680 # Each disk can define its type in a form of a disk_name,type. The remaining parts 681 # of the string are dropped. 682 IFS="," read -r disk disk_type _ <<< "$disk" 683 [[ -z $disk_type ]] && disk_type=$disk_type_g 684 685 case $disk_type in 686 virtio) 687 local raw_name="RAWSCSI" 688 local raw_disk=$vm_dir/test.img 689 690 if [[ -n $disk ]]; then 691 [[ ! -b $disk ]] && touch $disk 692 local raw_disk 693 raw_disk=$(readlink -f $disk) 694 fi 695 696 # Create disk file if it not exist or it is smaller than 1G 697 if { [[ -f $raw_disk ]] && [[ $(stat --printf="%s" $raw_disk) -lt $((1024 * 1024 * 1024)) ]]; } \ 698 || [[ ! -e $raw_disk ]]; then 699 if [[ $raw_disk =~ /dev/.* ]]; then 700 error \ 701 "ERROR: Virtio disk point to missing device ($raw_disk) -\n" \ 702 " this is probably not what you want." 703 return 1 704 fi 705 706 notice "Creating Virtio disc $raw_disk" 707 dd if=/dev/zero of=$raw_disk bs=1024k count=1024 708 else 709 notice "Using existing image $raw_disk" 710 fi 711 712 cmd+=(-device "virtio-scsi-pci,num_queues=$queue_number") 713 cmd+=(-device "scsi-hd,drive=hd$i,vendor=$raw_name") 714 cmd+=(-drive "if=none,id=hd$i,file=$raw_disk,format=raw$raw_cache") 715 ;; 716 spdk_vhost_scsi) 717 notice "using socket $vhost_dir/naa.$disk.$vm_num" 718 cmd+=(-chardev "socket,id=char_$disk,path=$vhost_dir/naa.$disk.$vm_num") 719 cmd+=(-device "vhost-user-scsi-pci,id=scsi_$disk,num_queues=$queue_number,chardev=char_$disk") 720 if [[ "$disk" == "$boot_from" ]]; then 721 cmd[-1]+=,bootindex=0 722 boot_disk_present=true 723 fi 724 ;; 725 spdk_vhost_blk) 726 notice "using socket $vhost_dir/naa.$disk.$vm_num" 727 cmd+=(-chardev "socket,id=char_$disk,path=$vhost_dir/naa.$disk.$vm_num") 728 cmd+=(-device "vhost-user-blk-pci,num-queues=$queue_number,chardev=char_$disk") 729 if [[ "$disk" == "$boot_from" ]]; then 730 cmd[-1]+=,bootindex=0 731 boot_disk_present=true 732 fi 733 734 if $packed; then 735 check_qemu_packedring_support 736 notice "Enabling packed ring support for VM $vm_num, controller $vhost_dir/naa.$disk.$vm_num" 737 cmd[-1]+=,packed=on 738 fi 739 ;; 740 kernel_vhost) 741 if [[ -z $disk ]]; then 742 error "need WWN for $disk_type" 743 return 1 744 elif [[ ! $disk =~ ^[[:alpha:]]{3}[.][[:xdigit:]]+$ ]]; then 745 error "$disk_type - disk(wnn)=$disk does not look like WNN number" 746 return 1 747 fi 748 notice "Using kernel vhost disk wwn=$disk" 749 cmd+=(-device "vhost-scsi-pci,wwpn=$disk,num_queues=$queue_number") 750 ;; 751 *) 752 error "unknown mode '$disk_type', use: virtio, spdk_vhost_scsi, spdk_vhost_blk or kernel_vhost" 753 return 1 754 ;; 755 esac 756 done 757 758 if [[ -n $boot_from ]] && [[ $boot_disk_present == false ]]; then 759 error "Boot from $boot_from is selected but device is not present" 760 return 1 761 fi 762 763 ((${#qemu_args[@]})) && cmd+=("${qemu_args[@]}") 764 notice "Saving to $vm_dir/run.sh" 765 cat <<- RUN > "$vm_dir/run.sh" 766 #!/bin/bash 767 qemu_log () { 768 echo "=== qemu.log ===" 769 [[ -s $vm_dir/qemu.log ]] && cat $vm_dir/qemu.log 770 echo "=== qemu.log ===" 771 } 772 773 if [[ \$EUID -ne 0 ]]; then 774 echo "Go away user come back as root" 775 exit 1 776 fi 777 778 trap "qemu_log" EXIT 779 780 qemu_cmd=($(printf '%s\n' "${cmd[@]}")) 781 chmod +r $vm_dir/* 782 echo "Running VM in $vm_dir" 783 rm -f $qemu_pid_file 784 "\${qemu_cmd[@]}" 785 786 echo "Waiting for QEMU pid file" 787 sleep 1 788 [[ ! -f $qemu_pid_file ]] && sleep 1 789 [[ ! -f $qemu_pid_file ]] && echo "ERROR: no qemu pid file found" && exit 1 790 exit 0 791 # EOF 792 RUN 793 chmod +x $vm_dir/run.sh 794 795 # Save generated sockets redirection 796 echo $ssh_socket > $vm_dir/ssh_socket 797 echo $fio_socket > $vm_dir/fio_socket 798 echo $monitor_port > $vm_dir/monitor_port 799 800 rm -f $vm_dir/migration_port 801 [[ -z $vm_incoming ]] || echo $migration_port > $vm_dir/migration_port 802 803 echo $gdbserver_socket > $vm_dir/gdbserver_socket 804 echo $vnc_socket >> $vm_dir/vnc_socket 805 806 [[ -z $vm_incoming ]] || ln -fs $VM_DIR/$vm_incoming $vm_dir/vm_incoming 807 [[ -z $vm_migrate_to ]] || ln -fs $VM_DIR/$vm_migrate_to $vm_dir/vm_migrate_to 808} 809 810function vm_run() { 811 local OPTIND optchar vm 812 local run_all=false 813 local vms_to_run="" 814 815 while getopts 'a-:' optchar; do 816 case "$optchar" in 817 a) run_all=true ;; 818 *) 819 error "Unknown param $OPTARG" 820 return 1 821 ;; 822 esac 823 done 824 825 if $run_all; then 826 vms_to_run="$(vm_list_all)" 827 else 828 shift $((OPTIND - 1)) 829 for vm in "$@"; do 830 vm_num_is_valid $1 || return 1 831 if [[ ! -x $VM_DIR/$vm/run.sh ]]; then 832 error "VM$vm not defined - setup it first" 833 return 1 834 fi 835 vms_to_run+=" $vm" 836 done 837 fi 838 839 for vm in $vms_to_run; do 840 if vm_is_running $vm; then 841 warning "VM$vm ($VM_DIR/$vm) already running" 842 continue 843 fi 844 845 notice "running $VM_DIR/$vm/run.sh" 846 if ! $VM_DIR/$vm/run.sh; then 847 error "FAILED to run vm $vm" 848 return 1 849 fi 850 done 851} 852 853function vm_print_logs() { 854 vm_num=$1 855 warning "================" 856 warning "QEMU LOG:" 857 if [[ -r $VM_DIR/$vm_num/qemu.log ]]; then 858 cat $VM_DIR/$vm_num/qemu.log 859 else 860 warning "LOG qemu.log not found" 861 fi 862 863 warning "VM LOG:" 864 if [[ -r $VM_DIR/$vm_num/serial.log ]]; then 865 cat $VM_DIR/$vm_num/serial.log 866 else 867 warning "LOG serial.log not found" 868 fi 869 870 warning "SEABIOS LOG:" 871 if [[ -r $VM_DIR/$vm_num/seabios.log ]]; then 872 cat $VM_DIR/$vm_num/seabios.log 873 else 874 warning "LOG seabios.log not found" 875 fi 876 warning "================" 877} 878 879# Wait for all created VMs to boot. 880# param $1 max wait time 881function vm_wait_for_boot() { 882 assert_number $1 883 884 xtrace_disable 885 886 local all_booted=false 887 local timeout_time=$1 888 [[ $timeout_time -lt 10 ]] && timeout_time=10 889 local timeout_time 890 timeout_time=$(date -d "+$timeout_time seconds" +%s) 891 892 notice "Waiting for VMs to boot" 893 shift 894 if [[ "$*" == "" ]]; then 895 local vms_to_check="$VM_DIR/[0-9]*" 896 else 897 local vms_to_check="" 898 for vm in "$@"; do 899 vms_to_check+=" $VM_DIR/$vm" 900 done 901 fi 902 903 for vm in $vms_to_check; do 904 local vm_num 905 vm_num=$(basename $vm) 906 local i=0 907 notice "waiting for VM$vm_num ($vm)" 908 while ! vm_os_booted $vm_num; do 909 if ! vm_is_running $vm_num; then 910 warning "VM $vm_num is not running" 911 vm_print_logs $vm_num 912 xtrace_restore 913 return 1 914 fi 915 916 if [[ $(date +%s) -gt $timeout_time ]]; then 917 warning "timeout waiting for machines to boot" 918 vm_print_logs $vm_num 919 xtrace_restore 920 return 1 921 fi 922 if ((i > 30)); then 923 local i=0 924 echo 925 fi 926 echo -n "." 927 sleep 1 928 done 929 echo "" 930 notice "VM$vm_num ready" 931 #Change Timeout for stopping services to prevent lengthy powerdowns 932 #Check that remote system is not Cygwin in case of Windows VMs 933 local vm_os 934 vm_os=$(vm_exec $vm_num "uname -o") 935 if [[ "$vm_os" != "Cygwin" ]]; then 936 vm_exec $vm_num "echo 'DefaultTimeoutStopSec=10' >> /etc/systemd/system.conf; systemctl daemon-reexec" 937 fi 938 done 939 940 notice "all VMs ready" 941 xtrace_restore 942 return 0 943} 944 945function vm_start_fio_server() { 946 local OPTIND optchar 947 local readonly='' 948 local fio_bin='' 949 while getopts ':-:' optchar; do 950 case "$optchar" in 951 -) 952 case "$OPTARG" in 953 fio-bin=*) local fio_bin="${OPTARG#*=}" ;; 954 readonly) local readonly="--readonly" ;; 955 *) error "Invalid argument '$OPTARG'" && return 1 ;; 956 esac 957 ;; 958 *) error "Invalid argument '$OPTARG'" && return 1 ;; 959 esac 960 done 961 962 shift $((OPTIND - 1)) 963 for vm_num in "$@"; do 964 notice "Starting fio server on VM$vm_num" 965 if [[ $fio_bin != "" ]]; then 966 vm_exec $vm_num 'cat > /root/fio; chmod +x /root/fio' < $fio_bin 967 vm_exec $vm_num /root/fio $readonly --eta=never --server --daemonize=/root/fio.pid 968 else 969 vm_exec $vm_num fio $readonly --eta=never --server --daemonize=/root/fio.pid 970 fi 971 done 972} 973 974function vm_check_scsi_location() { 975 # Script to find wanted disc 976 local script='shopt -s nullglob; 977 for entry in /sys/block/sd*; do 978 disk_type="$(cat $entry/device/vendor)"; 979 if [[ $disk_type == INTEL* ]] || [[ $disk_type == RAWSCSI* ]] || [[ $disk_type == LIO-ORG* ]]; then 980 fname=$(basename $entry); 981 echo -n " $fname"; 982 fi; 983 done' 984 985 SCSI_DISK="$(echo "$script" | vm_exec $1 bash -s)" 986 987 if [[ -z "$SCSI_DISK" ]]; then 988 error "no test disk found!" 989 return 1 990 fi 991} 992 993# Script to perform scsi device reset on all disks in VM 994# param $1 VM num 995# param $2..$n Disks to perform reset on 996function vm_reset_scsi_devices() { 997 for disk in "${@:2}"; do 998 notice "VM$1 Performing device reset on disk $disk" 999 vm_exec $1 sg_reset /dev/$disk -vNd 1000 done 1001} 1002 1003function vm_check_blk_location() { 1004 local script='shopt -s nullglob; cd /sys/block; echo vd*' 1005 SCSI_DISK="$(echo "$script" | vm_exec $1 bash -s)" 1006 1007 if [[ -z "$SCSI_DISK" ]]; then 1008 error "no blk test disk found!" 1009 return 1 1010 fi 1011} 1012 1013function run_fio() { 1014 local arg 1015 local job_file="" 1016 local fio_bin="" 1017 local vms=() 1018 local out="" 1019 local vm 1020 local run_server_mode=true 1021 local run_plugin_mode=false 1022 local fio_start_cmd 1023 local fio_output_format="normal" 1024 local fio_gtod_reduce=false 1025 local wait_for_fio=true 1026 1027 for arg in "$@"; do 1028 case "$arg" in 1029 --job-file=*) local job_file="${arg#*=}" ;; 1030 --fio-bin=*) local fio_bin="${arg#*=}" ;; 1031 --vm=*) vms+=("${arg#*=}") ;; 1032 --out=*) 1033 local out="${arg#*=}" 1034 mkdir -p $out 1035 ;; 1036 --local) run_server_mode=false ;; 1037 --plugin) 1038 notice "Using plugin mode. Disabling server mode." 1039 run_plugin_mode=true 1040 run_server_mode=false 1041 ;; 1042 --json) fio_output_format="json" ;; 1043 --hide-results) hide_results=true ;; 1044 --no-wait-for-fio) wait_for_fio=false ;; 1045 --gtod-reduce) fio_gtod_reduce=true ;; 1046 *) 1047 error "Invalid argument '$arg'" 1048 return 1 1049 ;; 1050 esac 1051 done 1052 1053 if [[ -n "$fio_bin" && ! -r "$fio_bin" ]]; then 1054 error "FIO binary '$fio_bin' does not exist" 1055 return 1 1056 fi 1057 1058 if [[ -z "$fio_bin" ]]; then 1059 fio_bin="fio" 1060 fi 1061 1062 if [[ ! -r "$job_file" ]]; then 1063 error "Fio job '$job_file' does not exist" 1064 return 1 1065 fi 1066 1067 fio_start_cmd="$fio_bin --eta=never " 1068 1069 local job_fname 1070 job_fname=$(basename "$job_file") 1071 log_fname="${job_fname%%.*}.log" 1072 fio_start_cmd+=" --output=$out/$log_fname --output-format=$fio_output_format " 1073 1074 # prepare job file for each VM 1075 for vm in "${vms[@]}"; do 1076 local vm_num=${vm%%:*} 1077 local vmdisks=${vm#*:} 1078 1079 sed "s@filename=@filename=$vmdisks@" $job_file | vm_exec $vm_num "cat > /root/$job_fname" 1080 1081 if $fio_gtod_reduce; then 1082 vm_exec $vm_num "echo 'gtod_reduce=1' >> /root/$job_fname" 1083 fi 1084 1085 vm_exec $vm_num cat /root/$job_fname 1086 1087 if $run_server_mode; then 1088 fio_start_cmd+="--client=127.0.0.1,$(vm_fio_socket $vm_num) --remote-config /root/$job_fname " 1089 fi 1090 1091 if ! $run_server_mode; then 1092 if [[ -n "$fio_bin" ]]; then 1093 if ! $run_plugin_mode; then 1094 vm_exec $vm_num 'cat > /root/fio; chmod +x /root/fio' < $fio_bin 1095 vm_fio_bin="/root/fio" 1096 else 1097 vm_fio_bin="/usr/src/fio/fio" 1098 fi 1099 fi 1100 1101 notice "Running local fio on VM $vm_num" 1102 vm_exec $vm_num "$vm_fio_bin --output=/root/$log_fname --output-format=$fio_output_format /root/$job_fname & echo \$! > /root/fio.pid" & 1103 vm_exec_pids+=("$!") 1104 fi 1105 done 1106 1107 if ! $run_server_mode; then 1108 if ! $wait_for_fio; then 1109 return 0 1110 fi 1111 echo "Waiting for guest fio instances to finish.." 1112 wait "${vm_exec_pids[@]}" 1113 1114 for vm in "${vms[@]}"; do 1115 local vm_num=${vm%%:*} 1116 vm_exec $vm_num cat /root/$log_fname > "$out/vm${vm_num}_${log_fname}" 1117 done 1118 return 0 1119 fi 1120 1121 $fio_start_cmd 1122 sleep 1 1123 1124 if [[ "$fio_output_format" == "json" ]]; then 1125 # Fio in client-server mode produces a lot of "trash" output 1126 # preceding JSON structure, making it not possible to parse. 1127 # Remove these lines from file. 1128 # shellcheck disable=SC2005 1129 echo "$(grep -vP '^[<\w]' "$out/$log_fname")" > "$out/$log_fname" 1130 fi 1131 1132 if [[ ! $hide_results ]]; then 1133 cat $out/$log_fname 1134 fi 1135} 1136 1137# Parsing fio results for json output and client-server mode only! 1138function parse_fio_results() { 1139 local fio_log_dir=$1 1140 local fio_log_filename=$2 1141 local fio_csv_filename 1142 1143 # Variables used in parsing loop 1144 local log_file 1145 local rwmode mixread mixwrite 1146 local lat_key lat_divisor 1147 local client_stats iops bw 1148 local read_avg_lat read_min_lat read_max_lat 1149 local write_avg_lat write_min_lat write_min_lat 1150 1151 declare -A results 1152 results["iops"]=0 1153 results["bw"]=0 1154 results["avg_lat"]=0 1155 results["min_lat"]=0 1156 results["max_lat"]=0 1157 1158 # Loop using the log filename to see if there are any other 1159 # matching files. This is in case we ran fio test multiple times. 1160 log_files=("$fio_log_dir/$fio_log_filename"*) 1161 for log_file in "${log_files[@]}"; do 1162 rwmode=$(jq -r '.["client_stats"][0]["job options"]["rw"]' "$log_file") 1163 mixread=1 1164 mixwrite=1 1165 if [[ $rwmode = *"rw"* ]]; then 1166 mixread=$(jq -r '.["client_stats"][0]["job options"]["rwmixread"]' "$log_file") 1167 mixread=$(bc -l <<< "scale=3; $mixread/100") 1168 mixwrite=$(bc -l <<< "scale=3; 1-$mixread") 1169 fi 1170 1171 client_stats=$(jq -r '.["client_stats"][] | select(.jobname == "All clients")' "$log_file") 1172 1173 # Check latency unit and later normalize to microseconds 1174 lat_key="lat_us" 1175 lat_divisor=1 1176 if jq -er '.read["lat_ns"]' &> /dev/null <<< $client_stats; then 1177 lat_key="lat_ns" 1178 lat_divisor=1000 1179 fi 1180 1181 # Horrific bash float point arithmetic oprations below. 1182 # Viewer discretion is advised. 1183 iops=$(jq -r '[.read["iops"],.write["iops"]] | add' <<< $client_stats) 1184 bw=$(jq -r '[.read["bw"],.write["bw"]] | add' <<< $client_stats) 1185 read_avg_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["mean"]' <<< $client_stats) 1186 read_min_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["min"]' <<< $client_stats) 1187 read_max_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["max"]' <<< $client_stats) 1188 write_avg_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["mean"]' <<< $client_stats) 1189 write_min_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["min"]' <<< $client_stats) 1190 write_max_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["max"]' <<< $client_stats) 1191 1192 results["iops"]=$(bc -l <<< "${results[iops]} + $iops") 1193 results["bw"]=$(bc -l <<< "${results[bw]} + $bw") 1194 results["avg_lat"]=$(bc -l <<< "${results[avg_lat]} + ($mixread*$read_avg_lat + $mixwrite*$write_avg_lat)/$lat_divisor") 1195 results["min_lat"]=$(bc -l <<< "${results[min_lat]} + ($mixread*$read_min_lat + $mixwrite*$write_min_lat)/$lat_divisor") 1196 results["max_lat"]=$(bc -l <<< "${results[max_lat]} + ($mixread*$read_max_lat + $mixwrite*$write_max_lat)/$lat_divisor") 1197 done 1198 1199 results["iops"]=$(bc -l <<< "scale=3; ${results[iops]} / ${#log_files[@]}") 1200 results["bw"]=$(bc -l <<< "scale=3; ${results[bw]} / ${#log_files[@]}") 1201 results["avg_lat"]=$(bc -l <<< "scale=3; ${results[avg_lat]} / ${#log_files[@]}") 1202 results["min_lat"]=$(bc -l <<< "scale=3; ${results[min_lat]} / ${#log_files[@]}") 1203 results["max_lat"]=$(bc -l <<< "scale=3; ${results[max_lat]} / ${#log_files[@]}") 1204 1205 fio_csv_filename="${fio_log_filename%%.*}.csv" 1206 cat <<- EOF > "$fio_log_dir/$fio_csv_filename" 1207 iops,bw,avg_lat,min_lat,max_lat 1208 ${results["iops"]},${results["bw"]},${results["avg_lat"]},${results["min_lat"]},${results["max_lat"]} 1209 EOF 1210} 1211 1212# Shutdown or kill any running VM and SPDK APP. 1213# 1214function at_app_exit() { 1215 local vhost_name 1216 1217 notice "APP EXITING" 1218 notice "killing all VMs" 1219 vm_kill_all 1220 # Kill vhost application 1221 notice "killing vhost app" 1222 1223 for vhost_name in "$TARGET_DIR"/*; do 1224 vhost_kill $vhost_name 1225 done 1226 1227 notice "EXIT DONE" 1228} 1229 1230function error_exit() { 1231 trap - ERR 1232 print_backtrace 1233 set +e 1234 error "Error on $1 $2" 1235 1236 at_app_exit 1237 exit 1 1238} 1239