1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2020 Intel Corporation 4# All rights reserved. 5# 6[[ $(uname -s) == Linux ]] || exit 0 7 8shopt -s extglob nullglob 9 10declare -r rdma_rxe=/sys/module/rdma_rxe 11declare -r rdma_rxe_add=$rdma_rxe/parameters/add 12declare -r rdma_rxe_rm=$rdma_rxe/parameters/remove 13 14declare -r infiniband=/sys/class/infiniband 15declare -r infiniband_verbs=/sys/class/infiniband_verbs 16declare -r net=/sys/class/net 17 18declare -A net_devices 19declare -A net_to_rxe 20declare -A rxe_to_net 21 22uevent() ( 23 [[ -e $1/uevent ]] || return 0 24 25 source "$1/uevent" 26 27 if [[ -v $2 ]]; then 28 echo "${!2}" 29 elif [[ -n $3 ]]; then 30 echo "$3" 31 fi 32) 33 34modprobeq() { 35 modprobe -q "$@" 36} 37 38get_ipv4() { 39 local ip 40 41 # Get only the first ip 42 read -r _ _ _ ip _ < <(ip -o -4 addr show dev "$1") 43 if [[ -n $ip ]]; then 44 echo "${ip%/*}" 45 else 46 echo " " 47 fi 48} 49 50get_rxe_mtu() { 51 local rxe=$1 52 local mtu 53 local uverb 54 55 for uverb in "$infiniband_verbs/uverbs"*; do 56 if [[ $(< "$uverb/ibdev") == "$rxe" ]] \ 57 && [[ -c /dev/infiniband/${uverb##*/} ]]; then 58 [[ $(ibv_devinfo -d "$rxe") =~ active_mtu:(.*\ \(.*\)) ]] 59 echo "${BASH_REMATCH[1]:-(?)}" 60 return 0 61 fi 62 done 63} 64 65start() { 66 local modules module 67 68 modules=( 69 "ib_core" 70 "ib_uverbs" 71 "rdma_ucm" 72 "rdma_rxe" 73 ) 74 75 for module in "${modules[@]}"; do 76 [[ -e /sys/module/$module ]] && continue 77 if [[ ! -e $(modinfo -F filename "$module") ]]; then 78 return 0 79 fi 80 done 2> /dev/null 81 82 modprobeq -a "${modules[@]}" || return 1 83 add_rxe all 84} 85 86stop() { 87 remove_rxe 88 89 if ! modprobeq -r rdma_rxe \ 90 || [[ -e $rdma_rxe ]]; then 91 printf 'unable to unload drivers, reboot required\n' 92 fi 93} 94 95status_header() { 96 local header=("Name" "Link" "Driver" "Speed" "NMTU" "IPv4_addr" "RDEV" "RMTU") 97 98 size_print_fields "${header[@]}" 99} 100 101status() { 102 if [[ ! -e $rdma_rxe ]]; then 103 printf 'rdma_rxe module not loaded\n' >&2 104 fi 105 106 local dev 107 local link_map 108 109 link_map[0]=no 110 link_map[1]=yes 111 112 status_header 113 114 local name link driver speed mtu ip rxe rxe_dev active_mtu 115 for dev in "${net_devices[@]}"; do 116 name="" link="" driver="" 117 speed="" mtu="" ip="" 118 rxe_dev="" active_mtu="" 119 120 name=${dev##*/} 121 rxe_dev=${net_to_rxe["$name"]} 122 active_mtu=$(get_rxe_mtu "$rxe_dev") 123 124 link=${link_map[$(< "$dev/carrier")]} 125 126 if [[ -e $dev/device/driver ]]; then 127 driver=$(readlink -f "$dev/device/driver") 128 driver=${driver##*/} 129 elif [[ -e /sys/devices/virtual/net/${dev##*/} ]]; then 130 # Try to be smart and get the type of the device instead 131 driver=$(uevent "$dev" "DEVTYPE" "virtual") 132 fi 133 134 if [[ $link == yes ]]; then 135 speed=$(< "$dev/speed") 136 if ((speed >= 1000)); then 137 speed=$((speed / 1000))GigE 138 elif ((speed > 0)); then 139 speed=${speed}Mb/s 140 else 141 speed="" 142 fi 143 fi 144 145 mtu=$(< "$dev/mtu") 146 ip=$(get_ipv4 "$name") 147 148 size_print_fields \ 149 "$name" \ 150 "$link" \ 151 "$driver" \ 152 "$speed" \ 153 "$mtu" \ 154 "$ip" \ 155 "$rxe_dev" \ 156 "$active_mtu" 157 done 2> /dev/null 158 print_status 159} 160 161size_print_fields() { 162 local fields=("$@") field 163 local -g lengths lines lineno 164 165 for field in "${!fields[@]}"; do 166 if [[ -z ${fields[field]} ]]; then 167 fields[field]="###" 168 fi 169 if [[ -z ${lengths[field]} ]]; then 170 lengths[field]=${#fields[field]} 171 else 172 lengths[field]=$((lengths[field] > ${#fields[field]} ? lengths[field] : ${#fields[field]})) 173 fi 174 done 175 176 eval "local -g _line_$lineno=(\"\${fields[@]}\")" 177 lines+=("_line_${lineno}[@]") 178 ((++lineno)) 179} 180 181print_status() { 182 local field field_ref fieldidx 183 local pad 184 185 if [[ -n $NO_HEADER ]]; then 186 unset -v "lines[0]" 187 fi 188 189 for field_ref in "${lines[@]}"; do 190 printf ' ' 191 fieldidx=0 192 for field in "${!field_ref}"; do 193 if [[ -n $field ]]; then 194 pad=$((lengths[fieldidx] - ${#field} + 2)) 195 else 196 pad=$((lengths[fieldidx] + 2)) 197 fi 198 if [[ -n $field && $field != "###" ]]; then 199 printf '%s' "$field" 200 else 201 printf ' ' 202 fi 203 printf '%*s' "$pad" "" 204 ((++fieldidx)) 205 done 206 printf '\n' 207 done 208} 209 210add_rxe() { 211 local dev net_devs 212 213 [[ -e $rdma_rxe_add ]] || return 0 214 215 if [[ -z $1 || $1 == all ]]; then 216 net_devs=("${!net_devices[@]}") 217 elif [[ -n ${net_to_rxe["$1"]} ]]; then 218 printf '%s interface already in use (%s)\n' \ 219 "$1" "${net_to_rxe["$1"]}" 220 return 0 221 elif [[ -n ${net_devices["$1"]} ]]; then 222 net_devs=("$1") 223 else 224 printf '%s interface does not exist\n' "$1" 225 return 1 226 fi 227 228 for dev in "${net_devs[@]}"; do 229 if [[ -z ${net_to_rxe["$dev"]} ]]; then 230 echo "${dev##*/}" > "$rdma_rxe_add" 231 fi 232 link_up "${dev##*/}" 233 done 2> /dev/null 234} 235 236remove_rxe() { 237 local rxes rxe 238 239 [[ -e $rdma_rxe_rm ]] || return 0 240 241 rxes=("${!rxe_to_net[@]}") 242 if [[ -z $1 || $1 == all ]]; then 243 rxes=("${!rxe_to_net[@]}") 244 elif [[ -z ${rxe_to_net["$1"]} ]]; then 245 printf '%s rxe interface does not exist\n' "$1" 246 return 0 247 elif [[ -n ${rxe_to_net["$1"]} ]]; then 248 rxes=("$1") 249 fi 250 251 for rxe in "${rxes[@]}"; do 252 echo "$rxe" > "$rdma_rxe_rm" 253 done 2> /dev/null 254} 255 256link_up() { 257 [[ -e $net/$1 ]] || return 0 258 259 echo $(($(< "$net/$1/flags") | 0x1)) > "$net/$1/flags" 260} 261 262collect_net_devices() { 263 local net_dev 264 265 for net_dev in "$net/"!(bonding_masters); do 266 (($(< "$net_dev/type") != 1)) && continue 267 net_devices["${net_dev##*/}"]=$net_dev 268 done 269} 270 271collect_rxe_devices() { 272 local rxe_dev net_dev 273 274 for rxe_dev in "$infiniband/"*; do 275 if [[ -e $rxe_dev/parent ]]; then 276 # Soft 277 net_dev=$(< "$rxe_dev/parent") 278 elif [[ -e $rxe_dev/device/net ]]; then 279 # HW 280 net_dev=$(readlink -f "$rxe_dev/device/net/"*) 281 net_dev=${net_dev##*/} 282 else 283 continue 284 fi 2> /dev/null 285 286 [[ -n ${net_devices["$net_dev"]} ]] || continue 287 net_to_rxe["$net_dev"]=${rxe_dev##*/} 288 rxe_to_net["${rxe_dev##*/}"]=$net_dev 289 done 290} 291 292collect_net_devices 293collect_rxe_devices 294 295case "${1:-status}" in 296 start) 297 start 298 ;; 299 stop) 300 stop 301 ;; 302 add) 303 add_rxe "${2:-all}" 304 ;; 305 remove) 306 remove_rxe "${2:-all}" 307 ;; 308 status) 309 IFS= read -r match < <( 310 IFS="|" 311 printf '%s\n' "${*:2}" 312 ) 313 status | grep -E "${match:-.}" 314 ;; 315 rxe-net) 316 ((${#rxe_to_net[@]} > 0)) && printf '%s\n' "${rxe_to_net[@]}" 317 ;; 318 *) 319 printf 'Invalid argument (%s)\n' "$1" 320 ;; 321esac 322