1#! /usr/bin/env python 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright(c) 2010-2014 Intel Corporation 4# 5 6import sys 7import os 8import getopt 9import subprocess 10from os.path import exists, abspath, dirname, basename 11 12# The PCI base class for all devices 13network_class = {'Class': '02', 'Vendor': None, 'Device': None, 14 'SVendor': None, 'SDevice': None} 15encryption_class = {'Class': '10', 'Vendor': None, 'Device': None, 16 'SVendor': None, 'SDevice': None} 17intel_processor_class = {'Class': '0b', 'Vendor': '8086', 'Device': None, 18 'SVendor': None, 'SDevice': None} 19cavium_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a04b,a04d', 20 'SVendor': None, 'SDevice': None} 21cavium_fpa = {'Class': '08', 'Vendor': '177d', 'Device': 'a053', 22 'SVendor': None, 'SDevice': None} 23cavium_pkx = {'Class': '08', 'Vendor': '177d', 'Device': 'a0dd,a049', 24 'SVendor': None, 'SDevice': None} 25 26network_devices = [network_class, cavium_pkx] 27crypto_devices = [encryption_class, intel_processor_class] 28eventdev_devices = [cavium_sso] 29mempool_devices = [cavium_fpa] 30 31# global dict ethernet devices present. Dictionary indexed by PCI address. 32# Each device within this is itself a dictionary of device properties 33devices = {} 34# list of supported DPDK drivers 35dpdk_drivers = ["igb_uio", "vfio-pci", "uio_pci_generic"] 36 37# command-line arg flags 38b_flag = None 39status_flag = False 40force_flag = False 41args = [] 42 43 44def usage(): 45 '''Print usage information for the program''' 46 argv0 = basename(sys.argv[0]) 47 print(""" 48Usage: 49------ 50 51 %(argv0)s [options] DEVICE1 DEVICE2 .... 52 53where DEVICE1, DEVICE2 etc, are specified via PCI "domain:bus:slot.func" syntax 54or "bus:slot.func" syntax. For devices bound to Linux kernel drivers, they may 55also be referred to by Linux interface name e.g. eth0, eth1, em0, em1, etc. 56 57Options: 58 --help, --usage: 59 Display usage information and quit 60 61 -s, --status: 62 Print the current status of all known network, crypto, event 63 and mempool devices. 64 For each device, it displays the PCI domain, bus, slot and function, 65 along with a text description of the device. Depending upon whether the 66 device is being used by a kernel driver, the igb_uio driver, or no 67 driver, other relevant information will be displayed: 68 * the Linux interface name e.g. if=eth0 69 * the driver being used e.g. drv=igb_uio 70 * any suitable drivers not currently using that device 71 e.g. unused=igb_uio 72 NOTE: if this flag is passed along with a bind/unbind option, the 73 status display will always occur after the other operations have taken 74 place. 75 76 --status-dev: 77 Print the status of given device group. Supported device groups are: 78 "net", "crypto", "event" and "mempool" 79 80 -b driver, --bind=driver: 81 Select the driver to use or \"none\" to unbind the device 82 83 -u, --unbind: 84 Unbind a device (Equivalent to \"-b none\") 85 86 --force: 87 By default, network devices which are used by Linux - as indicated by 88 having routes in the routing table - cannot be modified. Using the 89 --force flag overrides this behavior, allowing active links to be 90 forcibly unbound. 91 WARNING: This can lead to loss of network connection and should be used 92 with caution. 93 94Examples: 95--------- 96 97To display current device status: 98 %(argv0)s --status 99 100To display current network device status: 101 %(argv0)s --status-dev net 102 103To bind eth1 from the current driver and move to use igb_uio 104 %(argv0)s --bind=igb_uio eth1 105 106To unbind 0000:01:00.0 from using any driver 107 %(argv0)s -u 0000:01:00.0 108 109To bind 0000:02:00.0 and 0000:02:00.1 to the ixgbe kernel driver 110 %(argv0)s -b ixgbe 02:00.0 02:00.1 111 112 """ % locals()) # replace items from local variables 113 114 115# This is roughly compatible with check_output function in subprocess module 116# which is only available in python 2.7. 117def check_output(args, stderr=None): 118 '''Run a command and capture its output''' 119 return subprocess.Popen(args, stdout=subprocess.PIPE, 120 stderr=stderr).communicate()[0] 121 122 123def find_module(mod): 124 '''find the .ko file for kernel module named mod. 125 Searches the $RTE_SDK/$RTE_TARGET directory, the kernel 126 modules directory and finally under the parent directory of 127 the script ''' 128 # check $RTE_SDK/$RTE_TARGET directory 129 if 'RTE_SDK' in os.environ and 'RTE_TARGET' in os.environ: 130 path = "%s/%s/kmod/%s.ko" % (os.environ['RTE_SDK'], 131 os.environ['RTE_TARGET'], mod) 132 if exists(path): 133 return path 134 135 # check using depmod 136 try: 137 with open(os.devnull, "w") as fnull: 138 path = check_output(["modinfo", "-n", mod], stderr=fnull).strip() 139 140 if path and exists(path): 141 return path 142 except: # if modinfo can't find module, it fails, so continue 143 pass 144 145 # check for a copy based off current path 146 tools_dir = dirname(abspath(sys.argv[0])) 147 if tools_dir.endswith("tools"): 148 base_dir = dirname(tools_dir) 149 find_out = check_output(["find", base_dir, "-name", mod + ".ko"]) 150 if len(find_out) > 0: # something matched 151 path = find_out.splitlines()[0] 152 if exists(path): 153 return path 154 155 156def check_modules(): 157 '''Checks that igb_uio is loaded''' 158 global dpdk_drivers 159 160 # list of supported modules 161 mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers] 162 163 # first check if module is loaded 164 try: 165 # Get list of sysfs modules (both built-in and dynamically loaded) 166 sysfs_path = '/sys/module/' 167 168 # Get the list of directories in sysfs_path 169 sysfs_mods = [os.path.join(sysfs_path, o) for o 170 in os.listdir(sysfs_path) 171 if os.path.isdir(os.path.join(sysfs_path, o))] 172 173 # Extract the last element of '/sys/module/abc' in the array 174 sysfs_mods = [a.split('/')[-1] for a in sysfs_mods] 175 176 # special case for vfio_pci (module is named vfio-pci, 177 # but its .ko is named vfio_pci) 178 sysfs_mods = [a if a != 'vfio_pci' else 'vfio-pci' for a in sysfs_mods] 179 180 for mod in mods: 181 if mod["Name"] in sysfs_mods: 182 mod["Found"] = True 183 except: 184 pass 185 186 # check if we have at least one loaded module 187 if True not in [mod["Found"] for mod in mods] and b_flag is not None: 188 if b_flag in dpdk_drivers: 189 print("Error - no supported modules(DPDK driver) are loaded") 190 sys.exit(1) 191 else: 192 print("Warning - no supported modules(DPDK driver) are loaded") 193 194 # change DPDK driver list to only contain drivers that are loaded 195 dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]] 196 197 198def has_driver(dev_id): 199 '''return true if a device is assigned to a driver. False otherwise''' 200 return "Driver_str" in devices[dev_id] 201 202 203def get_pci_device_details(dev_id, probe_lspci): 204 '''This function gets additional details for a PCI device''' 205 device = {} 206 207 if probe_lspci: 208 extra_info = check_output(["lspci", "-vmmks", dev_id]).splitlines() 209 210 # parse lspci details 211 for line in extra_info: 212 if len(line) == 0: 213 continue 214 name, value = line.decode().split("\t", 1) 215 name = name.strip(":") + "_str" 216 device[name] = value 217 # check for a unix interface name 218 device["Interface"] = "" 219 for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id): 220 if "net" in dirs: 221 device["Interface"] = \ 222 ",".join(os.listdir(os.path.join(base, "net"))) 223 break 224 # check if a port is used for ssh connection 225 device["Ssh_if"] = False 226 device["Active"] = "" 227 228 return device 229 230def clear_data(): 231 '''This function clears any old data''' 232 devices = {} 233 234def get_device_details(devices_type): 235 '''This function populates the "devices" dictionary. The keys used are 236 the pci addresses (domain:bus:slot.func). The values are themselves 237 dictionaries - one for each NIC.''' 238 global devices 239 global dpdk_drivers 240 241 # first loop through and read details for all devices 242 # request machine readable format, with numeric IDs and String 243 dev = {} 244 dev_lines = check_output(["lspci", "-Dvmmnnk"]).splitlines() 245 for dev_line in dev_lines: 246 if len(dev_line) == 0: 247 if device_type_match(dev, devices_type): 248 # Replace "Driver" with "Driver_str" to have consistency of 249 # of dictionary key names 250 if "Driver" in dev.keys(): 251 dev["Driver_str"] = dev.pop("Driver") 252 # use dict to make copy of dev 253 devices[dev["Slot"]] = dict(dev) 254 # Clear previous device's data 255 dev = {} 256 else: 257 name, value = dev_line.decode().split("\t", 1) 258 value_list = value.rsplit(' ', 1) 259 if len(value_list) > 1: 260 # String stored in <name>_str 261 dev[name.rstrip(":") + '_str'] = value_list[0] 262 # Numeric IDs 263 dev[name.rstrip(":")] = value_list[len(value_list) - 1] \ 264 .rstrip("]").lstrip("[") 265 266 if devices_type == network_devices: 267 # check what is the interface if any for an ssh connection if 268 # any to this host, so we can mark it later. 269 ssh_if = [] 270 route = check_output(["ip", "-o", "route"]) 271 # filter out all lines for 169.254 routes 272 route = "\n".join(filter(lambda ln: not ln.startswith("169.254"), 273 route.decode().splitlines())) 274 rt_info = route.split() 275 for i in range(len(rt_info) - 1): 276 if rt_info[i] == "dev": 277 ssh_if.append(rt_info[i+1]) 278 279 # based on the basic info, get extended text details 280 for d in devices.keys(): 281 if not device_type_match(devices[d], devices_type): 282 continue 283 284 # get additional info and add it to existing data 285 devices[d] = devices[d].copy() 286 # No need to probe lspci 287 devices[d].update(get_pci_device_details(d, False).items()) 288 289 if devices_type == network_devices: 290 for _if in ssh_if: 291 if _if in devices[d]["Interface"].split(","): 292 devices[d]["Ssh_if"] = True 293 devices[d]["Active"] = "*Active*" 294 break 295 296 # add igb_uio to list of supporting modules if needed 297 if "Module_str" in devices[d]: 298 for driver in dpdk_drivers: 299 if driver not in devices[d]["Module_str"]: 300 devices[d]["Module_str"] = \ 301 devices[d]["Module_str"] + ",%s" % driver 302 else: 303 devices[d]["Module_str"] = ",".join(dpdk_drivers) 304 305 # make sure the driver and module strings do not have any duplicates 306 if has_driver(d): 307 modules = devices[d]["Module_str"].split(",") 308 if devices[d]["Driver_str"] in modules: 309 modules.remove(devices[d]["Driver_str"]) 310 devices[d]["Module_str"] = ",".join(modules) 311 312 313def device_type_match(dev, devices_type): 314 for i in range(len(devices_type)): 315 param_count = len( 316 [x for x in devices_type[i].values() if x is not None]) 317 match_count = 0 318 if dev["Class"][0:2] == devices_type[i]["Class"]: 319 match_count = match_count + 1 320 for key in devices_type[i].keys(): 321 if key != 'Class' and devices_type[i][key]: 322 value_list = devices_type[i][key].split(',') 323 for value in value_list: 324 if value.strip(' ') == dev[key]: 325 match_count = match_count + 1 326 # count must be the number of non None parameters to match 327 if match_count == param_count: 328 return True 329 return False 330 331def dev_id_from_dev_name(dev_name): 332 '''Take a device "name" - a string passed in by user to identify a NIC 333 device, and determine the device id - i.e. the domain:bus:slot.func - for 334 it, which can then be used to index into the devices array''' 335 336 # check if it's already a suitable index 337 if dev_name in devices: 338 return dev_name 339 # check if it's an index just missing the domain part 340 elif "0000:" + dev_name in devices: 341 return "0000:" + dev_name 342 else: 343 # check if it's an interface name, e.g. eth1 344 for d in devices.keys(): 345 if dev_name in devices[d]["Interface"].split(","): 346 return devices[d]["Slot"] 347 # if nothing else matches - error 348 print("Unknown device: %s. " 349 "Please specify device in \"bus:slot.func\" format" % dev_name) 350 sys.exit(1) 351 352 353def unbind_one(dev_id, force): 354 '''Unbind the device identified by "dev_id" from its current driver''' 355 dev = devices[dev_id] 356 if not has_driver(dev_id): 357 print("%s %s %s is not currently managed by any driver\n" % 358 (dev["Slot"], dev["Device_str"], dev["Interface"])) 359 return 360 361 # prevent us disconnecting ourselves 362 if dev["Ssh_if"] and not force: 363 print("Routing table indicates that interface %s is active. " 364 "Skipping unbind" % (dev_id)) 365 return 366 367 # write to /sys to unbind 368 filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] 369 try: 370 f = open(filename, "a") 371 except: 372 print("Error: unbind failed for %s - Cannot open %s" 373 % (dev_id, filename)) 374 sys.exit(1) 375 f.write(dev_id) 376 f.close() 377 378 379def bind_one(dev_id, driver, force): 380 '''Bind the device given by "dev_id" to the driver "driver". If the device 381 is already bound to a different driver, it will be unbound first''' 382 dev = devices[dev_id] 383 saved_driver = None # used to rollback any unbind in case of failure 384 385 # prevent disconnection of our ssh session 386 if dev["Ssh_if"] and not force: 387 print("Routing table indicates that interface %s is active. " 388 "Not modifying" % (dev_id)) 389 return 390 391 # unbind any existing drivers we don't want 392 if has_driver(dev_id): 393 if dev["Driver_str"] == driver: 394 print("%s already bound to driver %s, skipping\n" 395 % (dev_id, driver)) 396 return 397 else: 398 saved_driver = dev["Driver_str"] 399 unbind_one(dev_id, force) 400 dev["Driver_str"] = "" # clear driver string 401 402 # For kernels >= 3.15 driver_override can be used to specify the driver 403 # for a device rather than relying on the driver to provide a positive 404 # match of the device. The existing process of looking up 405 # the vendor and device ID, adding them to the driver new_id, 406 # will erroneously bind other devices too which has the additional burden 407 # of unbinding those devices 408 if driver in dpdk_drivers: 409 filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id 410 if os.path.exists(filename): 411 try: 412 f = open(filename, "w") 413 except: 414 print("Error: bind failed for %s - Cannot open %s" 415 % (dev_id, filename)) 416 return 417 try: 418 f.write("%s" % driver) 419 f.close() 420 except: 421 print("Error: bind failed for %s - Cannot write driver %s to " 422 "PCI ID " % (dev_id, driver)) 423 return 424 # For kernels < 3.15 use new_id to add PCI id's to the driver 425 else: 426 filename = "/sys/bus/pci/drivers/%s/new_id" % driver 427 try: 428 f = open(filename, "w") 429 except: 430 print("Error: bind failed for %s - Cannot open %s" 431 % (dev_id, filename)) 432 return 433 try: 434 # Convert Device and Vendor Id to int to write to new_id 435 f.write("%04x %04x" % (int(dev["Vendor"],16), 436 int(dev["Device"], 16))) 437 f.close() 438 except: 439 print("Error: bind failed for %s - Cannot write new PCI ID to " 440 "driver %s" % (dev_id, driver)) 441 return 442 443 # do the bind by writing to /sys 444 filename = "/sys/bus/pci/drivers/%s/bind" % driver 445 try: 446 f = open(filename, "a") 447 except: 448 print("Error: bind failed for %s - Cannot open %s" 449 % (dev_id, filename)) 450 if saved_driver is not None: # restore any previous driver 451 bind_one(dev_id, saved_driver, force) 452 return 453 try: 454 f.write(dev_id) 455 f.close() 456 except: 457 # for some reason, closing dev_id after adding a new PCI ID to new_id 458 # results in IOError. however, if the device was successfully bound, 459 # we don't care for any errors and can safely ignore IOError 460 tmp = get_pci_device_details(dev_id, True) 461 if "Driver_str" in tmp and tmp["Driver_str"] == driver: 462 return 463 print("Error: bind failed for %s - Cannot bind to driver %s" 464 % (dev_id, driver)) 465 if saved_driver is not None: # restore any previous driver 466 bind_one(dev_id, saved_driver, force) 467 return 468 469 # For kernels > 3.15 driver_override is used to bind a device to a driver. 470 # Before unbinding it, overwrite driver_override with empty string so that 471 # the device can be bound to any other driver 472 filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id 473 if os.path.exists(filename): 474 try: 475 f = open(filename, "w") 476 except: 477 print("Error: unbind failed for %s - Cannot open %s" 478 % (dev_id, filename)) 479 sys.exit(1) 480 try: 481 f.write("\00") 482 f.close() 483 except: 484 print("Error: unbind failed for %s - Cannot open %s" 485 % (dev_id, filename)) 486 sys.exit(1) 487 488 489def unbind_all(dev_list, force=False): 490 """Unbind method, takes a list of device locations""" 491 492 if dev_list[0] == "dpdk": 493 for d in devices.keys(): 494 if "Driver_str" in devices[d]: 495 if devices[d]["Driver_str"] in dpdk_drivers: 496 unbind_one(devices[d]["Slot"], force) 497 return 498 499 dev_list = map(dev_id_from_dev_name, dev_list) 500 for d in dev_list: 501 unbind_one(d, force) 502 503 504def bind_all(dev_list, driver, force=False): 505 """Bind method, takes a list of device locations""" 506 global devices 507 508 dev_list = map(dev_id_from_dev_name, dev_list) 509 510 for d in dev_list: 511 bind_one(d, driver, force) 512 513 # For kernels < 3.15 when binding devices to a generic driver 514 # (i.e. one that doesn't have a PCI ID table) using new_id, some devices 515 # that are not bound to any other driver could be bound even if no one has 516 # asked them to. hence, we check the list of drivers again, and see if 517 # some of the previously-unbound devices were erroneously bound. 518 if not os.path.exists("/sys/bus/pci/devices/%s/driver_override" % d): 519 for d in devices.keys(): 520 # skip devices that were already bound or that we know should be bound 521 if "Driver_str" in devices[d] or d in dev_list: 522 continue 523 524 # update information about this device 525 devices[d] = dict(devices[d].items() + 526 get_pci_device_details(d, True).items()) 527 528 # check if updated information indicates that the device was bound 529 if "Driver_str" in devices[d]: 530 unbind_one(d, force) 531 532 533def display_devices(title, dev_list, extra_params=None): 534 '''Displays to the user the details of a list of devices given in 535 "dev_list". The "extra_params" parameter, if given, should contain a string 536 with %()s fields in it for replacement by the named fields in each 537 device's dictionary.''' 538 strings = [] # this holds the strings to print. We sort before printing 539 print("\n%s" % title) 540 print("="*len(title)) 541 if len(dev_list) == 0: 542 strings.append("<none>") 543 else: 544 for dev in dev_list: 545 if extra_params is not None: 546 strings.append("%s '%s %s' %s" % (dev["Slot"], 547 dev["Device_str"], 548 dev["Device"], 549 extra_params % dev)) 550 else: 551 strings.append("%s '%s'" % (dev["Slot"], dev["Device_str"])) 552 # sort before printing, so that the entries appear in PCI order 553 strings.sort() 554 print("\n".join(strings)) # print one per line 555 556def show_device_status(devices_type, device_name): 557 global dpdk_drivers 558 kernel_drv = [] 559 dpdk_drv = [] 560 no_drv = [] 561 562 # split our list of network devices into the three categories above 563 for d in devices.keys(): 564 if device_type_match(devices[d], devices_type): 565 if not has_driver(d): 566 no_drv.append(devices[d]) 567 continue 568 if devices[d]["Driver_str"] in dpdk_drivers: 569 dpdk_drv.append(devices[d]) 570 else: 571 kernel_drv.append(devices[d]) 572 573 # print each category separately, so we can clearly see what's used by DPDK 574 display_devices("%s devices using DPDK-compatible driver" % device_name, 575 dpdk_drv, "drv=%(Driver_str)s unused=%(Module_str)s") 576 display_devices("%s devices using kernel driver" % device_name, kernel_drv, 577 "if=%(Interface)s drv=%(Driver_str)s " 578 "unused=%(Module_str)s %(Active)s") 579 display_devices("Other %s devices" % device_name, no_drv, 580 "unused=%(Module_str)s") 581 582def show_status(): 583 '''Function called when the script is passed the "--status" option. 584 Displays to the user what devices are bound to the igb_uio driver, the 585 kernel driver or to no driver''' 586 587 if status_dev == "net" or status_dev == "all": 588 show_device_status(network_devices, "Network") 589 590 if status_dev == "crypto" or status_dev == "all": 591 show_device_status(crypto_devices, "Crypto") 592 593 if status_dev == "event" or status_dev == "all": 594 show_device_status(eventdev_devices, "Eventdev") 595 596 if status_dev == "mempool" or status_dev == "all": 597 show_device_status(mempool_devices, "Mempool") 598 599def parse_args(): 600 '''Parses the command-line arguments given by the user and takes the 601 appropriate action for each''' 602 global b_flag 603 global status_flag 604 global status_dev 605 global force_flag 606 global args 607 if len(sys.argv) <= 1: 608 usage() 609 sys.exit(0) 610 611 try: 612 opts, args = getopt.getopt(sys.argv[1:], "b:us", 613 ["help", "usage", "status", "status-dev=", 614 "force", "bind=", "unbind", ]) 615 except getopt.GetoptError as error: 616 print(str(error)) 617 print("Run '%s --usage' for further information" % sys.argv[0]) 618 sys.exit(1) 619 620 for opt, arg in opts: 621 if opt == "--help" or opt == "--usage": 622 usage() 623 sys.exit(0) 624 if opt == "--status-dev": 625 status_flag = True 626 status_dev = arg 627 if opt == "--status" or opt == "-s": 628 status_flag = True 629 status_dev = "all" 630 if opt == "--force": 631 force_flag = True 632 if opt == "-b" or opt == "-u" or opt == "--bind" or opt == "--unbind": 633 if b_flag is not None: 634 print("Error - Only one bind or unbind may be specified\n") 635 sys.exit(1) 636 if opt == "-u" or opt == "--unbind": 637 b_flag = "none" 638 else: 639 b_flag = arg 640 641 642def do_arg_actions(): 643 '''do the actual action requested by the user''' 644 global b_flag 645 global status_flag 646 global force_flag 647 global args 648 649 if b_flag is None and not status_flag: 650 print("Error: No action specified for devices." 651 "Please give a -b or -u option") 652 print("Run '%s --usage' for further information" % sys.argv[0]) 653 sys.exit(1) 654 655 if b_flag is not None and len(args) == 0: 656 print("Error: No devices specified.") 657 print("Run '%s --usage' for further information" % sys.argv[0]) 658 sys.exit(1) 659 660 if b_flag == "none" or b_flag == "None": 661 unbind_all(args, force_flag) 662 elif b_flag is not None: 663 bind_all(args, b_flag, force_flag) 664 if status_flag: 665 if b_flag is not None: 666 clear_data() 667 # refresh if we have changed anything 668 get_device_details(network_devices) 669 get_device_details(crypto_devices) 670 get_device_details(eventdev_devices) 671 get_device_details(mempool_devices) 672 show_status() 673 674 675def main(): 676 '''program main function''' 677 parse_args() 678 check_modules() 679 clear_data() 680 get_device_details(network_devices) 681 get_device_details(crypto_devices) 682 get_device_details(eventdev_devices) 683 get_device_details(mempool_devices) 684 do_arg_actions() 685 686if __name__ == "__main__": 687 main() 688