1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2017 Intel Corporation 4# All rights reserved. 5# 6testdir=$(readlink -f $(dirname $0)) 7rootdir=$(readlink -f $testdir/../../..) 8source $rootdir/test/common/autotest_common.sh 9source $rootdir/test/vhost/common.sh 10source $rootdir/scripts/common.sh 11 12rpc_py="$rootdir/scripts/rpc.py -s $(get_vhost_dir 0)/rpc.sock" 13 14vm_count=1 15ctrl_type="spdk_vhost_scsi" 16use_fs=false 17distribute_cores=false 18 19function usage() { 20 [[ -n $2 ]] && ( 21 echo "$2" 22 echo "" 23 ) 24 echo "Shortcut script for doing automated test" 25 echo "Usage: $(basename $1) [OPTIONS]" 26 echo 27 echo "-h, --help Print help and exit" 28 echo " --fio-bin=PATH Path to FIO binary.;" 29 echo " --vm-count=INT Virtual machines to use in test;" 30 echo " Each VM will get one lvol bdev on each NVMe." 31 echo " Default: 1" 32 echo " --ctrl-type=TYPE Controller type to use for test:" 33 echo " spdk_vhost_scsi - use spdk vhost scsi" 34 echo " spdk_vhost_blk - use spdk vhost block" 35 echo " --thin-provisioning Create lvol bdevs thin provisioned instead of" 36 echo " allocating space up front" 37 echo " --distribute-cores Use custom config file and run vhost controllers" 38 echo " on different CPU cores instead of single core." 39 echo " Default: False" 40 echo "-x set -x for script debug" 41 exit 0 42} 43 44function clean_lvol_cfg() { 45 notice "Removing lvol bdevs" 46 for lvol_bdev in "${lvol_bdevs[@]}"; do 47 $rpc_py -t 120 bdev_lvol_delete $lvol_bdev 48 notice "lvol bdev $lvol_bdev removed" 49 done 50 51 notice "Removing lvol stores" 52 $rpc_py -t 120 bdev_lvol_delete_lvstore -u "$ls_guid" 53 notice "lvol store $ls_guid removed" 54} 55 56while getopts 'xh-:' optchar; do 57 case "$optchar" in 58 -) 59 case "$OPTARG" in 60 help) usage $0 ;; 61 fio-bin=*) fio_bin="--fio-bin=${OPTARG#*=}" ;; 62 vm-count=*) vm_count="${OPTARG#*=}" ;; 63 ctrl-type=*) ctrl_type="${OPTARG#*=}" ;; 64 distribute-cores) distribute_cores=true ;; 65 thin-provisioning) thin=" -t " ;; 66 *) usage $0 "Invalid argument '$OPTARG'" ;; 67 esac 68 ;; 69 h) usage $0 ;; 70 x) 71 set -x 72 x="-x" 73 ;; 74 *) usage $0 "Invalid argument '$OPTARG'" ;; 75 esac 76done 77 78vhosttestinit 79 80# $vm_count VMs, 1CPU per VM, 4CPUs for SPDK, all pinned to node0 81source <(gen_cpu_vm_spdk_config "$vm_count" 1 4 "" 0) 82spdk_mask=$vhost_0_reactor_mask 83 84trap 'error_exit "${FUNCNAME}" "${LINENO}"' SIGTERM SIGABRT ERR 85 86vm_kill_all 87 88notice "running SPDK vhost" 89vhost_run -n "0" -- --cpumask "$spdk_mask" 90notice "..." 91 92trap 'clean_lvol_cfg; error_exit "${FUNCNAME}" "${LINENO}"' SIGTERM SIGABRT ERR 93 94lvol_bdevs=() 95used_vms="" 96 97id=0 98# Create base lvol store on NVMe 99notice "Creating lvol store on device Nvme${id}n1" 100ls_guid=$($rpc_py bdev_lvol_create_lvstore Nvme0n1 lvs_$id -c 4194304) 101# Create base lvol bdevs 102for ((j = 0; j < vm_count; j++)); do 103 notice "Creating lvol bdev for VM $id on lvol store $ls_guid" 104 free_mb=$(get_lvs_free_mb "$ls_guid") 105 size=$((free_mb / (vm_count - j))) 106 lb_name=$($rpc_py bdev_lvol_create -u $ls_guid lbd_vm_$j $size $thin) 107 lvol_bdevs+=("$lb_name") 108done 109 110bdev_info=$($rpc_py bdev_get_bdevs) 111notice "Configuration after initial set-up:" 112$rpc_py bdev_lvol_get_lvstores 113echo "$bdev_info" 114 115# Set up VMs 116for ((i = 0; i < vm_count; i++)); do 117 vm="vm_$i" 118 119 # Get all lvol bdevs associated with this VM number 120 bdevs=$(jq -r "map(select(.aliases[] | contains(\"$vm\")) | \ 121 .aliases[]) | join(\" \")" <<< "$bdev_info") 122 bdevs=($bdevs) 123 124 setup_cmd="vm_setup --disk-type=$ctrl_type --force=$i" 125 setup_cmd+=" --os=$VM_IMAGE" 126 127 # Create single SCSI controller or multiple BLK controllers for this VM 128 mask_arg=("--cpumask" "$spdk_mask") 129 130 if [[ "$ctrl_type" == "spdk_vhost_scsi" ]]; then 131 $rpc_py vhost_create_scsi_controller naa.0.$i "${mask_arg[@]}" 132 for ((j = 0; j < ${#bdevs[@]}; j++)); do 133 $rpc_py vhost_scsi_controller_add_target naa.0.$i $j ${bdevs[$j]} 134 done 135 setup_cmd+=" --disks=0" 136 elif [[ "$ctrl_type" == "spdk_vhost_blk" ]]; then 137 disk="" 138 for ((j = 0; j < ${#bdevs[@]}; j++)); do 139 $rpc_py vhost_create_blk_controller naa.$j.$i ${bdevs[$j]} "${mask_arg[@]}" 140 disk+="${j}:" 141 done 142 disk="${disk::-1}" 143 setup_cmd+=" --disks=$disk" 144 fi 145 146 $setup_cmd 147 used_vms+=" $i" 148done 149 150$rpc_py vhost_get_controllers 151 152# Run VMs 153vm_run $used_vms 154vm_wait_for_boot 300 $used_vms 155 156# Get disk names from VMs and run FIO traffic 157 158fio_disks="" 159for vm_num in $used_vms; do 160 qemu_mask_param="VM_${vm_num}_qemu_mask" 161 162 host_name="VM-$vm_num-${!qemu_mask_param}" 163 vm_exec $vm_num "hostname $host_name" 164 vm_start_fio_server $fio_bin $vm_num 165 166 if [[ "$ctrl_type" == "spdk_vhost_scsi" ]]; then 167 vm_check_scsi_location $vm_num 168 elif [[ "$ctrl_type" == "spdk_vhost_blk" ]]; then 169 vm_check_blk_location $vm_num 170 fi 171 172 fio_disks+=" --vm=${vm_num}$(printf ':/dev/%s' $SCSI_DISK)" 173done 174 175if [[ $RUN_NIGHTLY -eq 1 ]]; then 176 job_file="default_integrity_nightly.job" 177else 178 job_file="default_integrity.job" 179fi 180# Run FIO traffic 181run_fio $fio_bin --job-file=$rootdir/test/vhost/common/fio_jobs/$job_file --out="$VHOST_DIR/fio_results" $fio_disks 182 183notice "Shutting down virtual machines..." 184vm_shutdown_all 185sleep 2 186 187notice "Cleaning up vhost - remove LUNs, controllers, lvol bdevs and lvol stores" 188if [[ "$ctrl_type" == "spdk_vhost_scsi" ]]; then 189 for ((i = 0; i < vm_count; i++)); do 190 notice "Removing devices from vhost SCSI controller naa.0.$i" 191 for ((j = 0; j < ${#bdevs[@]}; j++)); do 192 $rpc_py vhost_scsi_controller_remove_target naa.0.$i $j 193 notice "Removed device $j" 194 done 195 notice "Removing vhost SCSI controller naa.0.$i" 196 $rpc_py vhost_delete_controller naa.0.$i 197 done 198elif [[ "$ctrl_type" == "spdk_vhost_blk" ]]; then 199 for ((i = 0; i < vm_count; i++)); do 200 for ((j = 0; j < ${#bdevs[@]}; j++)); do 201 notice "Removing vhost BLK controller naa.$j.$i" 202 $rpc_py vhost_delete_controller naa.$j.$i 203 notice "Removed naa.$j.$i" 204 done 205 done 206fi 207 208clean_lvol_cfg 209 210$rpc_py bdev_lvol_get_lvstores 211$rpc_py bdev_get_bdevs 212$rpc_py vhost_get_controllers 213 214notice "Shutting down SPDK vhost app..." 215vhost_kill 0 216 217vhosttestfini 218