1#! /usr/bin/env python 2# 3# BSD LICENSE 4# 5# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 6# All rights reserved. 7# 8# Redistribution and use in source and binary forms, with or without 9# modification, are permitted provided that the following conditions 10# are met: 11# 12# * Redistributions of source code must retain the above copyright 13# notice, this list of conditions and the following disclaimer. 14# * Redistributions in binary form must reproduce the above copyright 15# notice, this list of conditions and the following disclaimer in 16# the documentation and/or other materials provided with the 17# distribution. 18# * Neither the name of Intel Corporation nor the names of its 19# contributors may be used to endorse or promote products derived 20# from this software without specific prior written permission. 21# 22# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33# 34 35import sys 36import os 37import getopt 38import subprocess 39from os.path import exists, abspath, dirname, basename 40 41# The PCI base class for NETWORK devices 42NETWORK_BASE_CLASS = "02" 43CRYPTO_BASE_CLASS = "0b" 44 45# global dict ethernet devices present. Dictionary indexed by PCI address. 46# Each device within this is itself a dictionary of device properties 47devices = {} 48# list of supported DPDK drivers 49dpdk_drivers = ["igb_uio", "vfio-pci", "uio_pci_generic"] 50 51# command-line arg flags 52b_flag = None 53status_flag = False 54force_flag = False 55args = [] 56 57 58def usage(): 59 '''Print usage information for the program''' 60 argv0 = basename(sys.argv[0]) 61 print(""" 62Usage: 63------ 64 65 %(argv0)s [options] DEVICE1 DEVICE2 .... 66 67where DEVICE1, DEVICE2 etc, are specified via PCI "domain:bus:slot.func" syntax 68or "bus:slot.func" syntax. For devices bound to Linux kernel drivers, they may 69also be referred to by Linux interface name e.g. eth0, eth1, em0, em1, etc. 70 71Options: 72 --help, --usage: 73 Display usage information and quit 74 75 -s, --status: 76 Print the current status of all known network and crypto devices. 77 For each device, it displays the PCI domain, bus, slot and function, 78 along with a text description of the device. Depending upon whether the 79 device is being used by a kernel driver, the igb_uio driver, or no 80 driver, other relevant information will be displayed: 81 * the Linux interface name e.g. if=eth0 82 * the driver being used e.g. drv=igb_uio 83 * any suitable drivers not currently using that device 84 e.g. unused=igb_uio 85 NOTE: if this flag is passed along with a bind/unbind option, the 86 status display will always occur after the other operations have taken 87 place. 88 89 -b driver, --bind=driver: 90 Select the driver to use or \"none\" to unbind the device 91 92 -u, --unbind: 93 Unbind a device (Equivalent to \"-b none\") 94 95 --force: 96 By default, network devices which are used by Linux - as indicated by 97 having routes in the routing table - cannot be modified. Using the 98 --force flag overrides this behavior, allowing active links to be 99 forcibly unbound. 100 WARNING: This can lead to loss of network connection and should be used 101 with caution. 102 103Examples: 104--------- 105 106To display current device status: 107 %(argv0)s --status 108 109To bind eth1 from the current driver and move to use igb_uio 110 %(argv0)s --bind=igb_uio eth1 111 112To unbind 0000:01:00.0 from using any driver 113 %(argv0)s -u 0000:01:00.0 114 115To bind 0000:02:00.0 and 0000:02:00.1 to the ixgbe kernel driver 116 %(argv0)s -b ixgbe 02:00.0 02:00.1 117 118 """ % locals()) # replace items from local variables 119 120 121# This is roughly compatible with check_output function in subprocess module 122# which is only available in python 2.7. 123def check_output(args, stderr=None): 124 '''Run a command and capture its output''' 125 return subprocess.Popen(args, stdout=subprocess.PIPE, 126 stderr=stderr).communicate()[0] 127 128 129def find_module(mod): 130 '''find the .ko file for kernel module named mod. 131 Searches the $RTE_SDK/$RTE_TARGET directory, the kernel 132 modules directory and finally under the parent directory of 133 the script ''' 134 # check $RTE_SDK/$RTE_TARGET directory 135 if 'RTE_SDK' in os.environ and 'RTE_TARGET' in os.environ: 136 path = "%s/%s/kmod/%s.ko" % (os.environ['RTE_SDK'], 137 os.environ['RTE_TARGET'], mod) 138 if exists(path): 139 return path 140 141 # check using depmod 142 try: 143 depmod_out = check_output(["modinfo", "-n", mod], 144 stderr=subprocess.STDOUT).lower() 145 if "error" not in depmod_out: 146 path = depmod_out.strip() 147 if exists(path): 148 return path 149 except: # if modinfo can't find module, it fails, so continue 150 pass 151 152 # check for a copy based off current path 153 tools_dir = dirname(abspath(sys.argv[0])) 154 if tools_dir.endswith("tools"): 155 base_dir = dirname(tools_dir) 156 find_out = check_output(["find", base_dir, "-name", mod + ".ko"]) 157 if len(find_out) > 0: # something matched 158 path = find_out.splitlines()[0] 159 if exists(path): 160 return path 161 162 163def check_modules(): 164 '''Checks that igb_uio is loaded''' 165 global dpdk_drivers 166 167 # list of supported modules 168 mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers] 169 170 # first check if module is loaded 171 try: 172 # Get list of sysfs modules (both built-in and dynamically loaded) 173 sysfs_path = '/sys/module/' 174 175 # Get the list of directories in sysfs_path 176 sysfs_mods = [os.path.join(sysfs_path, o) for o 177 in os.listdir(sysfs_path) 178 if os.path.isdir(os.path.join(sysfs_path, o))] 179 180 # Extract the last element of '/sys/module/abc' in the array 181 sysfs_mods = [a.split('/')[-1] for a in sysfs_mods] 182 183 # special case for vfio_pci (module is named vfio-pci, 184 # but its .ko is named vfio_pci) 185 sysfs_mods = map(lambda a: 186 a if a != 'vfio_pci' else 'vfio-pci', sysfs_mods) 187 188 for mod in mods: 189 if mod["Name"] in sysfs_mods: 190 mod["Found"] = True 191 except: 192 pass 193 194 # check if we have at least one loaded module 195 if True not in [mod["Found"] for mod in mods] and b_flag is not None: 196 if b_flag in dpdk_drivers: 197 print("Error - no supported modules(DPDK driver) are loaded") 198 sys.exit(1) 199 else: 200 print("Warning - no supported modules(DPDK driver) are loaded") 201 202 # change DPDK driver list to only contain drivers that are loaded 203 dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]] 204 205 206def has_driver(dev_id): 207 '''return true if a device is assigned to a driver. False otherwise''' 208 return "Driver_str" in devices[dev_id] 209 210 211def get_pci_device_details(dev_id): 212 '''This function gets additional details for a PCI device''' 213 device = {} 214 215 extra_info = check_output(["lspci", "-vmmks", dev_id]).splitlines() 216 217 # parse lspci details 218 for line in extra_info: 219 if len(line) == 0: 220 continue 221 name, value = line.decode().split("\t", 1) 222 name = name.strip(":") + "_str" 223 device[name] = value 224 # check for a unix interface name 225 device["Interface"] = "" 226 for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id): 227 if "net" in dirs: 228 device["Interface"] = \ 229 ",".join(os.listdir(os.path.join(base, "net"))) 230 break 231 # check if a port is used for ssh connection 232 device["Ssh_if"] = False 233 device["Active"] = "" 234 235 return device 236 237 238def get_nic_details(): 239 '''This function populates the "devices" dictionary. The keys used are 240 the pci addresses (domain:bus:slot.func). The values are themselves 241 dictionaries - one for each NIC.''' 242 global devices 243 global dpdk_drivers 244 245 # clear any old data 246 devices = {} 247 # first loop through and read details for all devices 248 # request machine readable format, with numeric IDs 249 dev = {} 250 dev_lines = check_output(["lspci", "-Dvmmn"]).splitlines() 251 for dev_line in dev_lines: 252 if len(dev_line) == 0: 253 if dev["Class"][0:2] == NETWORK_BASE_CLASS: 254 # convert device and vendor ids to numbers, then add to global 255 dev["Vendor"] = int(dev["Vendor"], 16) 256 dev["Device"] = int(dev["Device"], 16) 257 # use dict to make copy of dev 258 devices[dev["Slot"]] = dict(dev) 259 else: 260 name, value = dev_line.decode().split("\t", 1) 261 dev[name.rstrip(":")] = value 262 263 # check what is the interface if any for an ssh connection if 264 # any to this host, so we can mark it later. 265 ssh_if = [] 266 route = check_output(["ip", "-o", "route"]) 267 # filter out all lines for 169.254 routes 268 route = "\n".join(filter(lambda ln: not ln.startswith("169.254"), 269 route.decode().splitlines())) 270 rt_info = route.split() 271 for i in range(len(rt_info) - 1): 272 if rt_info[i] == "dev": 273 ssh_if.append(rt_info[i+1]) 274 275 # based on the basic info, get extended text details 276 for d in devices.keys(): 277 # get additional info and add it to existing data 278 devices[d] = devices[d].copy() 279 devices[d].update(get_pci_device_details(d).items()) 280 281 for _if in ssh_if: 282 if _if in devices[d]["Interface"].split(","): 283 devices[d]["Ssh_if"] = True 284 devices[d]["Active"] = "*Active*" 285 break 286 287 # add igb_uio to list of supporting modules if needed 288 if "Module_str" in devices[d]: 289 for driver in dpdk_drivers: 290 if driver not in devices[d]["Module_str"]: 291 devices[d]["Module_str"] = \ 292 devices[d]["Module_str"] + ",%s" % driver 293 else: 294 devices[d]["Module_str"] = ",".join(dpdk_drivers) 295 296 # make sure the driver and module strings do not have any duplicates 297 if has_driver(d): 298 modules = devices[d]["Module_str"].split(",") 299 if devices[d]["Driver_str"] in modules: 300 modules.remove(devices[d]["Driver_str"]) 301 devices[d]["Module_str"] = ",".join(modules) 302 303 304def get_crypto_details(): 305 '''This function populates the "devices" dictionary. The keys used are 306 the pci addresses (domain:bus:slot.func). The values are themselves 307 dictionaries - one for each NIC.''' 308 global devices 309 global dpdk_drivers 310 311 # clear any old data 312 # devices = {} 313 # first loop through and read details for all devices 314 # request machine readable format, with numeric IDs 315 dev = {} 316 dev_lines = check_output(["lspci", "-Dvmmn"]).splitlines() 317 for dev_line in dev_lines: 318 if len(dev_line) == 0: 319 if dev["Class"][0:2] == CRYPTO_BASE_CLASS: 320 # convert device and vendor ids to numbers, then add to global 321 dev["Vendor"] = int(dev["Vendor"], 16) 322 dev["Device"] = int(dev["Device"], 16) 323 # use dict to make copy of dev 324 devices[dev["Slot"]] = dict(dev) 325 else: 326 name, value = dev_line.decode().split("\t", 1) 327 dev[name.rstrip(":")] = value 328 329 # based on the basic info, get extended text details 330 for d in devices.keys(): 331 if devices[d]["Class"][0:2] != CRYPTO_BASE_CLASS: 332 continue 333 334 # get additional info and add it to existing data 335 devices[d] = devices[d].copy() 336 devices[d].update(get_pci_device_details(d).items()) 337 338 # add igb_uio to list of supporting modules if needed 339 if "Module_str" in devices[d]: 340 for driver in dpdk_drivers: 341 if driver not in devices[d]["Module_str"]: 342 devices[d]["Module_str"] = \ 343 devices[d]["Module_str"] + ",%s" % driver 344 else: 345 devices[d]["Module_str"] = ",".join(dpdk_drivers) 346 347 # make sure the driver and module strings do not have any duplicates 348 if has_driver(d): 349 modules = devices[d]["Module_str"].split(",") 350 if devices[d]["Driver_str"] in modules: 351 modules.remove(devices[d]["Driver_str"]) 352 devices[d]["Module_str"] = ",".join(modules) 353 354 355def dev_id_from_dev_name(dev_name): 356 '''Take a device "name" - a string passed in by user to identify a NIC 357 device, and determine the device id - i.e. the domain:bus:slot.func - for 358 it, which can then be used to index into the devices array''' 359 360 # check if it's already a suitable index 361 if dev_name in devices: 362 return dev_name 363 # check if it's an index just missing the domain part 364 elif "0000:" + dev_name in devices: 365 return "0000:" + dev_name 366 else: 367 # check if it's an interface name, e.g. eth1 368 for d in devices.keys(): 369 if dev_name in devices[d]["Interface"].split(","): 370 return devices[d]["Slot"] 371 # if nothing else matches - error 372 print("Unknown device: %s. " 373 "Please specify device in \"bus:slot.func\" format" % dev_name) 374 sys.exit(1) 375 376 377def unbind_one(dev_id, force): 378 '''Unbind the device identified by "dev_id" from its current driver''' 379 dev = devices[dev_id] 380 if not has_driver(dev_id): 381 print("%s %s %s is not currently managed by any driver\n" % 382 (dev["Slot"], dev["Device_str"], dev["Interface"])) 383 return 384 385 # prevent us disconnecting ourselves 386 if dev["Ssh_if"] and not force: 387 print("Routing table indicates that interface %s is active. " 388 "Skipping unbind" % (dev_id)) 389 return 390 391 # write to /sys to unbind 392 filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] 393 try: 394 f = open(filename, "a") 395 except: 396 print("Error: unbind failed for %s - Cannot open %s" 397 % (dev_id, filename)) 398 sys.exit(1) 399 f.write(dev_id) 400 f.close() 401 402 403def bind_one(dev_id, driver, force): 404 '''Bind the device given by "dev_id" to the driver "driver". If the device 405 is already bound to a different driver, it will be unbound first''' 406 dev = devices[dev_id] 407 saved_driver = None # used to rollback any unbind in case of failure 408 409 # prevent disconnection of our ssh session 410 if dev["Ssh_if"] and not force: 411 print("Routing table indicates that interface %s is active. " 412 "Not modifying" % (dev_id)) 413 return 414 415 # unbind any existing drivers we don't want 416 if has_driver(dev_id): 417 if dev["Driver_str"] == driver: 418 print("%s already bound to driver %s, skipping\n" 419 % (dev_id, driver)) 420 return 421 else: 422 saved_driver = dev["Driver_str"] 423 unbind_one(dev_id, force) 424 dev["Driver_str"] = "" # clear driver string 425 426 # if we are binding to one of DPDK drivers, add PCI id's to that driver 427 if driver in dpdk_drivers: 428 filename = "/sys/bus/pci/drivers/%s/new_id" % driver 429 try: 430 f = open(filename, "w") 431 except: 432 print("Error: bind failed for %s - Cannot open %s" 433 % (dev_id, filename)) 434 return 435 try: 436 f.write("%04x %04x" % (dev["Vendor"], dev["Device"])) 437 f.close() 438 except: 439 print("Error: bind failed for %s - Cannot write new PCI ID to " 440 "driver %s" % (dev_id, driver)) 441 return 442 443 # do the bind by writing to /sys 444 filename = "/sys/bus/pci/drivers/%s/bind" % driver 445 try: 446 f = open(filename, "a") 447 except: 448 print("Error: bind failed for %s - Cannot open %s" 449 % (dev_id, filename)) 450 if saved_driver is not None: # restore any previous driver 451 bind_one(dev_id, saved_driver, force) 452 return 453 try: 454 f.write(dev_id) 455 f.close() 456 except: 457 # for some reason, closing dev_id after adding a new PCI ID to new_id 458 # results in IOError. however, if the device was successfully bound, 459 # we don't care for any errors and can safely ignore IOError 460 tmp = get_pci_device_details(dev_id) 461 if "Driver_str" in tmp and tmp["Driver_str"] == driver: 462 return 463 print("Error: bind failed for %s - Cannot bind to driver %s" 464 % (dev_id, driver)) 465 if saved_driver is not None: # restore any previous driver 466 bind_one(dev_id, saved_driver, force) 467 return 468 469 470def unbind_all(dev_list, force=False): 471 """Unbind method, takes a list of device locations""" 472 dev_list = map(dev_id_from_dev_name, dev_list) 473 for d in dev_list: 474 unbind_one(d, force) 475 476 477def bind_all(dev_list, driver, force=False): 478 """Bind method, takes a list of device locations""" 479 global devices 480 481 dev_list = map(dev_id_from_dev_name, dev_list) 482 483 for d in dev_list: 484 bind_one(d, driver, force) 485 486 # when binding devices to a generic driver (i.e. one that doesn't have a 487 # PCI ID table), some devices that are not bound to any other driver could 488 # be bound even if no one has asked them to. hence, we check the list of 489 # drivers again, and see if some of the previously-unbound devices were 490 # erroneously bound. 491 for d in devices.keys(): 492 # skip devices that were already bound or that we know should be bound 493 if "Driver_str" in devices[d] or d in dev_list: 494 continue 495 496 # update information about this device 497 devices[d] = dict(devices[d].items() + 498 get_pci_device_details(d).items()) 499 500 # check if updated information indicates that the device was bound 501 if "Driver_str" in devices[d]: 502 unbind_one(d, force) 503 504 505def display_devices(title, dev_list, extra_params=None): 506 '''Displays to the user the details of a list of devices given in 507 "dev_list". The "extra_params" parameter, if given, should contain a string 508 with %()s fields in it for replacement by the named fields in each 509 device's dictionary.''' 510 strings = [] # this holds the strings to print. We sort before printing 511 print("\n%s" % title) 512 print("="*len(title)) 513 if len(dev_list) == 0: 514 strings.append("<none>") 515 else: 516 for dev in dev_list: 517 if extra_params is not None: 518 strings.append("%s '%s' %s" % (dev["Slot"], 519 dev["Device_str"], 520 extra_params % dev)) 521 else: 522 strings.append("%s '%s'" % (dev["Slot"], dev["Device_str"])) 523 # sort before printing, so that the entries appear in PCI order 524 strings.sort() 525 print("\n".join(strings)) # print one per line 526 527 528def show_status(): 529 '''Function called when the script is passed the "--status" option. 530 Displays to the user what devices are bound to the igb_uio driver, the 531 kernel driver or to no driver''' 532 global dpdk_drivers 533 kernel_drv = [] 534 dpdk_drv = [] 535 no_drv = [] 536 537 # split our list of network devices into the three categories above 538 for d in devices.keys(): 539 if NETWORK_BASE_CLASS in devices[d]["Class"]: 540 if not has_driver(d): 541 no_drv.append(devices[d]) 542 continue 543 if devices[d]["Driver_str"] in dpdk_drivers: 544 dpdk_drv.append(devices[d]) 545 else: 546 kernel_drv.append(devices[d]) 547 548 # print each category separately, so we can clearly see what's used by DPDK 549 display_devices("Network devices using DPDK-compatible driver", dpdk_drv, 550 "drv=%(Driver_str)s unused=%(Module_str)s") 551 display_devices("Network devices using kernel driver", kernel_drv, 552 "if=%(Interface)s drv=%(Driver_str)s " 553 "unused=%(Module_str)s %(Active)s") 554 display_devices("Other network devices", no_drv, "unused=%(Module_str)s") 555 556 # split our list of crypto devices into the three categories above 557 kernel_drv = [] 558 dpdk_drv = [] 559 no_drv = [] 560 561 for d in devices.keys(): 562 if CRYPTO_BASE_CLASS in devices[d]["Class"]: 563 if not has_driver(d): 564 no_drv.append(devices[d]) 565 continue 566 if devices[d]["Driver_str"] in dpdk_drivers: 567 dpdk_drv.append(devices[d]) 568 else: 569 kernel_drv.append(devices[d]) 570 571 display_devices("Crypto devices using DPDK-compatible driver", dpdk_drv, 572 "drv=%(Driver_str)s unused=%(Module_str)s") 573 display_devices("Crypto devices using kernel driver", kernel_drv, 574 "drv=%(Driver_str)s " 575 "unused=%(Module_str)s") 576 display_devices("Other crypto devices", no_drv, "unused=%(Module_str)s") 577 578 579def parse_args(): 580 '''Parses the command-line arguments given by the user and takes the 581 appropriate action for each''' 582 global b_flag 583 global status_flag 584 global force_flag 585 global args 586 if len(sys.argv) <= 1: 587 usage() 588 sys.exit(0) 589 590 try: 591 opts, args = getopt.getopt(sys.argv[1:], "b:us", 592 ["help", "usage", "status", "force", 593 "bind=", "unbind"]) 594 except getopt.GetoptError as error: 595 print(str(error)) 596 print("Run '%s --usage' for further information" % sys.argv[0]) 597 sys.exit(1) 598 599 for opt, arg in opts: 600 if opt == "--help" or opt == "--usage": 601 usage() 602 sys.exit(0) 603 if opt == "--status" or opt == "-s": 604 status_flag = True 605 if opt == "--force": 606 force_flag = True 607 if opt == "-b" or opt == "-u" or opt == "--bind" or opt == "--unbind": 608 if b_flag is not None: 609 print("Error - Only one bind or unbind may be specified\n") 610 sys.exit(1) 611 if opt == "-u" or opt == "--unbind": 612 b_flag = "none" 613 else: 614 b_flag = arg 615 616 617def do_arg_actions(): 618 '''do the actual action requested by the user''' 619 global b_flag 620 global status_flag 621 global force_flag 622 global args 623 624 if b_flag is None and not status_flag: 625 print("Error: No action specified for devices." 626 "Please give a -b or -u option") 627 print("Run '%s --usage' for further information" % sys.argv[0]) 628 sys.exit(1) 629 630 if b_flag is not None and len(args) == 0: 631 print("Error: No devices specified.") 632 print("Run '%s --usage' for further information" % sys.argv[0]) 633 sys.exit(1) 634 635 if b_flag == "none" or b_flag == "None": 636 unbind_all(args, force_flag) 637 elif b_flag is not None: 638 bind_all(args, b_flag, force_flag) 639 if status_flag: 640 if b_flag is not None: 641 get_nic_details() # refresh if we have changed anything 642 get_crypto_details() # refresh if we have changed anything 643 show_status() 644 645 646def main(): 647 '''program main function''' 648 parse_args() 649 check_modules() 650 get_nic_details() 651 get_crypto_details() 652 do_arg_actions() 653 654if __name__ == "__main__": 655 main() 656