1#!/usr/bin/env bash 2shopt -s nullglob 3 4rootdir=$(readlink -f $(dirname $0))/.. 5igb_driverdir=$rootdir/dpdk/build-tmp/kernel/linux/igb_uio/ 6allowed_drivers=("igb_uio" "uio_pci_generic") 7 8reload_intel_qat() { 9 # We need to make sure the out-of-tree intel_qat driver, provided via vm_setup.sh, is in 10 # use. Otherwise, some dependency modules loaded by qat_service may fail causing some 11 # disturbance later on during the tests - in particular, it's been seen that the adf_ctl 12 # was returning inconsistent data (wrong pci addresses), confusing the service into 13 # believing SR-IOV is not enabled. 14 15 # If this file doesn't exist, then either intel_qat is a kernel built-in or is not loaded. 16 # Nothing to do in such cases, qat_service will load the module for us. 17 [[ -e /sys/module/intel_qat/taint ]] || return 0 18 19 local t v 20 t=$(< /sys/module/intel_qat/taint) 21 v=$(< /sys/module/intel_qat/version) 22 23 # OE - out-of-tree, unsigned. By the very default, drivers built via vm_setup.sh are not 24 # signed. 25 [[ -z $t || $t != *"OE"* ]] || return 0 26 27 # Check the version of loaded module against the version of the same module as seen 28 # from .dep. perspective. if these are the same the most likely something is broken 29 # with the dependencies. We report a failure in such a case since reloading the module 30 # won't do any good anyway. 31 32 if [[ $(modinfo -F version intel_qat) == "$v" ]]; then 33 cat <<- WARN 34 Upstream intel_qat driver detected. Same version of the driver is seen 35 in modules dependencies: $v. This may mean QAT build didn't update 36 dependencies properly. QAT setup may fail, please, rebuild the QAT 37 driver. 38 WARN 39 return 0 40 fi 41 42 # Ok, intel_qat is an upstream module, replace it with the out-of-tree one. 43 echo "Reloading intel_qat module" 44 45 local h=(/sys/module/intel_qat/holders/*) 46 h=("${h[@]##*/}") 47 48 modprobe -r "${h[@]}" intel_qat 49 # qat_service does that too, but be vigilant 50 modprobe -a intel_qat "${h[@]}" 51} 52 53# This script requires an igb_uio kernel module binary located at $igb_driverdir/igb_uio.ko 54# Please also note that this script is not intended to be comprehensive or production quality. 55# It supports configuring a single card (the Intel QAT 8970) for use with the SPDK 56 57bad_driver=true 58driver_to_bind=uio_pci_generic 59num_vfs=16 60 61qat_pci_bdfs=($(lspci -Dd:37c8 | awk '{print $1}')) 62if [ ${#qat_pci_bdfs[@]} -eq 0 ]; then 63 echo "No QAT devices found. Exiting" 64 exit 0 65fi 66 67if [ -n "$1" ]; then 68 driver_to_bind=$1 69fi 70 71for driver in "${allowed_drivers[@]}"; do 72 if [ $driver == $driver_to_bind ]; then 73 bad_driver=false 74 fi 75done 76 77if $bad_driver; then 78 echo "Unrecognized driver. Please specify an accepted driver (listed below):" 79 echo "${allowed_drivers[@]}" 80 exit 1 81fi 82 83reload_intel_qat 84 85# try starting the qat service. If this doesn't work, just treat it as a warning for now. 86if ! service qat_service start; then 87 echo "failed to start the qat service. Something may be wrong with your 01.org driver." 88fi 89 90# configure virtual functions for the QAT cards. 91for qat_bdf in "${qat_pci_bdfs[@]}"; do 92 if [[ ! -e /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs ]]; then 93 echo "($qat_bdf) sriov_numvfs interface missing, is SR-IOV enabled?" 94 continue 95 fi 96 echo "$num_vfs" > /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs 97 num_vfs_set=$(cat /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs) 98 if ((num_vfs != num_vfs_set)); then 99 echo "Number of VFs set to $num_vfs_set, expected $num_vfs" 100 else 101 echo "$qat_bdf set to $num_vfs VFs" 102 fi 103done 104 105# Confirm we have all of the virtual functions we asked for. 106 107qat_vf_bdfs=($(lspci -Dd:37c9 | awk '{print $1}')) 108if ((${#qat_vf_bdfs[@]} != ${#qat_pci_bdfs[@]} * num_vfs)); then 109 echo "Failed to prepare the VFs. Aborting" 110 exit 1 111fi 112 113# Unbind old driver if necessary. 114for vf in "${qat_vf_bdfs[@]}"; do 115 old_driver=$(basename $(readlink -f /sys/bus/pci/devices/${vf}/driver)) 116 if [ $old_driver != "driver" ]; then 117 echo "unbinding driver $old_driver from qat VF at BDF $vf" 118 echo -n $vf > /sys/bus/pci/drivers/$old_driver/unbind 119 fi 120done 121 122modprobe uio 123 124# Insert the dpdk uio kernel module. 125if [ $driver_to_bind == "igb_uio" ]; then 126 if ! lsmod | grep -q igb_uio; then 127 if ! insmod $igb_driverdir/igb_uio.ko; then 128 echo "Unable to insert the igb_uio kernel module. Aborting." 129 exit 1 130 fi 131 fi 132elif [ "$driver_to_bind" == "uio_pci_generic" ]; then 133 modprobe uio_pci_generic 134else 135 echo "Unsure how to work with driver $driver_to_bind. Please configure it in qat_setup.sh" 136 exit 1 137fi 138 139echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/new_id 140for vf in "${qat_vf_bdfs[@]}"; do 141 if ! ls -l /sys/bus/pci/devices/$vf/driver | grep -q $driver_to_bind; then 142 echo "unable to bind the driver to the device at bdf $vf" 143 if [ "$driver_to_bind" == "uio_pci_generic" ]; then 144 echo "Your kernel's uio_pci_generic module does not support binding to virtual functions." 145 echo "It likely is missing Linux git commit ID acec09e67 which is needed to bind" 146 echo "uio_pci_generic to virtual functions which have no legacy interrupt vector." 147 echo "Please rebuild spdk with --with-igb-uio-driver and re-run this script specifying the igb_uio driver." 148 fi 149 exit 1 150 fi 151done 152echo "Properly configured the qat device with driver $driver_to_bind." 153