1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2018 Intel Corporation 4# All rights reserved. 5# 6shopt -s nullglob 7 8rootdir=$(readlink -f $(dirname $0))/.. 9allowed_drivers=("igb_uio" "uio_pci_generic") 10 11reload_intel_qat() { 12 # We need to make sure the out-of-tree intel_qat driver, provided via vm_setup.sh, is in 13 # use. Otherwise, some dependency modules loaded by qat_service may fail causing some 14 # disturbance later on during the tests - in particular, it's been seen that the adf_ctl 15 # was returning inconsistent data (wrong pci addresses), confusing the service into 16 # believing SR-IOV is not enabled. 17 18 # If this file doesn't exist, then either intel_qat is a kernel built-in or is not loaded. 19 # Nothing to do in such cases, qat_service will load the module for us. 20 [[ -e /sys/module/intel_qat/taint ]] || return 0 21 22 local t v 23 t=$(< /sys/module/intel_qat/taint) 24 v=$(< /sys/module/intel_qat/version) 25 26 # OE - out-of-tree, unsigned. By the very default, drivers built via vm_setup.sh are not 27 # signed. 28 [[ -z $t || $t != *"OE"* ]] || return 0 29 30 # Check the version of loaded module against the version of the same module as seen 31 # from .dep. perspective. if these are the same the most likely something is broken 32 # with the dependencies. We report a failure in such a case since reloading the module 33 # won't do any good anyway. 34 35 if [[ $(modinfo -F version intel_qat) == "$v" ]]; then 36 cat <<- WARN 37 Upstream intel_qat driver detected. Same version of the driver is seen 38 in modules dependencies: $v. This may mean QAT build didn't update 39 dependencies properly. QAT setup may fail, please, rebuild the QAT 40 driver. 41 WARN 42 return 0 43 fi 44 45 # Ok, intel_qat is an upstream module, replace it with the out-of-tree one. 46 echo "Reloading intel_qat module" 47 48 local h=(/sys/module/intel_qat/holders/*) 49 h=("${h[@]##*/}") 50 51 modprobe -r "${h[@]}" intel_qat 52 # qat_service does that too, but be vigilant 53 modprobe -a intel_qat "${h[@]}" 54} 55 56# Please note that this script is not intended to be comprehensive or production quality. 57# It supports configuring a single card (the Intel QAT 8970) for use with the SPDK 58 59bad_driver=true 60driver_to_bind=uio_pci_generic 61num_vfs=16 62 63qat_pci_bdfs=($(lspci -Dd:37c8 | awk '{print $1}')) 64if [ ${#qat_pci_bdfs[@]} -eq 0 ]; then 65 echo "No QAT devices found. Exiting" 66 exit 1 67fi 68 69if [ -n "$1" ]; then 70 driver_to_bind=$1 71fi 72 73for driver in "${allowed_drivers[@]}"; do 74 if [ $driver == $driver_to_bind ]; then 75 bad_driver=false 76 fi 77done 78 79if $bad_driver; then 80 echo "Unrecognized driver. Please specify an accepted driver (listed below):" 81 echo "${allowed_drivers[@]}" 82 exit 1 83fi 84 85reload_intel_qat 86 87# try starting the qat service. If this doesn't work, just treat it as a warning for now. 88if ! service qat_service start; then 89 echo "failed to start the qat service. Something may be wrong with your 01.org driver." 90fi 91 92# configure virtual functions for the QAT cards. 93for qat_bdf in "${qat_pci_bdfs[@]}"; do 94 if [[ ! -e /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs ]]; then 95 echo "($qat_bdf) sriov_numvfs interface missing, is SR-IOV enabled?" 96 continue 97 fi 98 echo "$num_vfs" > /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs 99 num_vfs_set=$(cat /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs) 100 if ((num_vfs != num_vfs_set)); then 101 echo "Number of VFs set to $num_vfs_set, expected $num_vfs" 102 else 103 echo "$qat_bdf set to $num_vfs VFs" 104 fi 105done 106 107# Confirm we have all of the virtual functions we asked for. 108 109qat_vf_bdfs=($(lspci -Dd:37c9 | awk '{print $1}')) 110if ((${#qat_vf_bdfs[@]} != ${#qat_pci_bdfs[@]} * num_vfs)); then 111 echo "Failed to prepare the VFs. Aborting" 112 exit 1 113fi 114 115# Unbind old driver if necessary. 116for vf in "${qat_vf_bdfs[@]}"; do 117 old_driver=$(basename $(readlink -f /sys/bus/pci/devices/${vf}/driver)) 118 if [ $old_driver != "driver" ]; then 119 echo "unbinding driver $old_driver from qat VF at BDF $vf" 120 echo -n $vf > /sys/bus/pci/drivers/$old_driver/unbind 121 fi 122done 123 124modprobe uio 125 126# Insert the dpdk uio kernel module. 127if [ $driver_to_bind == "igb_uio" ]; then 128 if ! modprobe igb_uio; then 129 echo "Unable to insert the igb_uio kernel module. Aborting." 130 exit 1 131 fi 132elif [ "$driver_to_bind" == "uio_pci_generic" ]; then 133 modprobe uio_pci_generic 134else 135 echo "Unsure how to work with driver $driver_to_bind. Please configure it in qat_setup.sh" 136 exit 1 137fi 138 139echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/remove_id 2> /dev/null 140echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/new_id 141for vf in "${qat_vf_bdfs[@]}"; do 142 if ! ls -l /sys/bus/pci/devices/$vf/driver | grep -q $driver_to_bind; then 143 echo "unable to bind the driver to the device at bdf $vf" 144 if [ "$driver_to_bind" == "uio_pci_generic" ]; then 145 echo "Your kernel's uio_pci_generic module does not support binding to virtual functions." 146 echo "It likely is missing Linux git commit ID acec09e67 which is needed to bind" 147 echo "uio_pci_generic to virtual functions which have no legacy interrupt vector." 148 echo "Please build DPDK kernel module for igb_uio and re-run this script specifying the igb_uio driver." 149 fi 150 exit 1 151 fi 152done 153echo "Properly configured the qat device with driver $driver_to_bind." 154