1#!/usr/bin/env python3 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright(c) 2020 Intel Corporation 4 5""" 6Configure an entire Intel DSA instance, using idxd kernel driver, for DPDK use 7""" 8 9import sys 10import argparse 11import os 12import os.path 13 14 15class SysfsDir: 16 verbose = False 17 18 "Used to read/write paths in a sysfs directory" 19 def __init__(self, path): 20 self.path = path 21 22 def read_int(self, filename): 23 "Return a value from sysfs file" 24 if SysfsDir.verbose: 25 print(f"Reading '{filename}' in {self.path}") 26 with open(os.path.join(self.path, filename)) as f: 27 return int(f.readline()) 28 29 def write_values(self, values): 30 "write dictionary, where key is filename and value is value to write" 31 for filename, contents in values.items(): 32 if SysfsDir.verbose: 33 print(f"Writing '{contents}' to '{filename}' in {self.path}") 34 with open(os.path.join(self.path, filename), "w") as f: 35 f.write(str(contents)) 36 37 38def get_drv_dir(dtype): 39 "Get the sysfs path for the driver, either 'idxd' or 'user'" 40 drv_dir = "/sys/bus/dsa/drivers/" + dtype 41 if not os.path.exists(drv_dir): 42 return "/sys/bus/dsa/drivers/dsa" 43 return drv_dir 44 45 46def reset_device(dsa_id): 47 "Reset the DSA device and all its queues" 48 drv_dir = SysfsDir(get_drv_dir("idxd")) 49 drv_dir.write_values({"unbind": f"dsa{dsa_id}"}) 50 51 52def get_pci_dir(pci): 53 "Search for the sysfs directory of the PCI device" 54 base_dir = '/sys/bus/pci/devices/' 55 for path, dirs, files in os.walk(base_dir): 56 for dir in dirs: 57 if pci in dir: 58 return os.path.join(base_dir, dir) 59 sys.exit(f"Could not find sysfs directory for device {pci}") 60 61 62def get_dsa_id(pci): 63 "Get the DSA instance ID using the PCI address of the device" 64 pci_dir = get_pci_dir(pci) 65 for path, dirs, files in os.walk(pci_dir): 66 for dir in dirs: 67 if dir.startswith('dsa') and 'wq' not in dir: 68 return int(dir[3:]) 69 sys.exit(f"Could not get device ID for device {pci}") 70 71 72def parse_wq_opts(wq_opts): 73 "Parse user-specified queue configuration, creating a dict of options" 74 try: 75 return {o.split('=')[0]: o.split('=')[1] for o in wq_opts} 76 except ValueError: 77 sys.exit("Invalid --wq-option format, use format 'option=value'") 78 79 80def configure_dsa(dsa_id, args): 81 "Configure the DSA instance with appropriate number of queues" 82 dsa_dir = SysfsDir(f"/sys/bus/dsa/devices/dsa{dsa_id}") 83 84 max_groups = dsa_dir.read_int("max_groups") 85 max_engines = dsa_dir.read_int("max_engines") 86 max_queues = dsa_dir.read_int("max_work_queues") 87 max_work_queues_size = dsa_dir.read_int("max_work_queues_size") 88 89 nb_queues = min(args.q, max_queues) 90 if args.q > nb_queues: 91 print(f"Setting number of queues to max supported value: {max_queues}") 92 93 # we want one engine per group, and no more engines than queues 94 nb_groups = min(max_engines, max_groups, nb_queues) 95 for grp in range(nb_groups): 96 dsa_dir.write_values({f"engine{dsa_id}.{grp}/group_id": grp}) 97 98 # configure each queue 99 for q in range(nb_queues): 100 wqcfg = {"group_id": q % nb_groups, 101 "type": "user", 102 "mode": "dedicated", 103 "name": f"{args.prefix}_wq{dsa_id}.{q}", 104 "priority": 1, 105 "max_batch_size": 1024, 106 "size": int(max_work_queues_size / nb_queues)} 107 wq_dir = SysfsDir(os.path.join(dsa_dir.path, f"wq{dsa_id}.{q}")) 108 if os.path.exists(os.path.join(wq_dir.path, f"driver_name")): 109 wqcfg.update({"driver_name": "user"}) 110 wqcfg.update(parse_wq_opts(args.wq_option)) 111 wq_dir.write_values(wqcfg) 112 113 # enable device and then queues 114 idxd_dir = SysfsDir(get_drv_dir("idxd")) 115 idxd_dir.write_values({"bind": f"dsa{dsa_id}"}) 116 117 user_dir = SysfsDir(get_drv_dir("user")) 118 for q in range(nb_queues): 119 user_dir.write_values({"bind": f"wq{dsa_id}.{q}"}) 120 121 122def main(args): 123 "Main function, does arg parsing and calls config function" 124 arg_p = argparse.ArgumentParser( 125 description="Configure whole DSA device instance for DPDK use") 126 arg_p.add_argument('dsa_id', 127 help="Specify DSA instance either via DSA instance number or PCI address") 128 arg_p.add_argument('-q', metavar='queues', type=int, default=255, 129 help="Number of queues to set up") 130 arg_p.add_argument('--name-prefix', metavar='prefix', dest='prefix', 131 default="dpdk", 132 help="Prefix for workqueue name to mark for DPDK use [default: 'dpdk']") 133 arg_p.add_argument('--wq-option', action='append', default=[], 134 help="Provide additional config option for queues (format 'x=y')") 135 arg_p.add_argument('--verbose', '-v', action='store_true', 136 help="Provide addition info on tasks being performed") 137 arg_p.add_argument('--reset', action='store_true', 138 help="Reset DSA device and its queues") 139 parsed_args = arg_p.parse_args(args[1:]) 140 141 dsa_id = parsed_args.dsa_id 142 dsa_id = get_dsa_id(dsa_id) if ':' in dsa_id else dsa_id 143 144 SysfsDir.verbose = parsed_args.verbose 145 if parsed_args.reset: 146 reset_device(dsa_id) 147 else: 148 configure_dsa(dsa_id, parsed_args) 149 150 151if __name__ == "__main__": 152 main(sys.argv) 153