1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <string.h> 6 #include <inttypes.h> 7 #include <stdint.h> 8 #include <stdbool.h> 9 #include <stdlib.h> 10 #include <stdio.h> 11 #include <sys/queue.h> 12 #include <rte_errno.h> 13 #include <rte_interrupts.h> 14 #include <rte_log.h> 15 #include <rte_bus.h> 16 #include <rte_per_lcore.h> 17 #include <rte_memory.h> 18 #include <rte_eal.h> 19 #include <rte_eal_paging.h> 20 #include <rte_lcore.h> 21 #include <rte_string_fns.h> 22 #include <rte_common.h> 23 #include <rte_devargs.h> 24 25 #include "private.h" 26 #include "rte_bus_auxiliary.h" 27 28 static struct rte_devargs * 29 auxiliary_devargs_lookup(const char *name) 30 { 31 struct rte_devargs *devargs; 32 33 RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) { 34 if (strcmp(devargs->name, name) == 0) 35 return devargs; 36 } 37 return NULL; 38 } 39 40 /* 41 * Test whether the auxiliary device exist. 42 * 43 * Stub for OS not supporting auxiliary bus. 44 */ 45 __rte_weak bool 46 auxiliary_dev_exists(const char *name) 47 { 48 RTE_SET_USED(name); 49 return false; 50 } 51 52 /* 53 * Scan the devices in the auxiliary bus. 54 * 55 * Stub for OS not supporting auxiliary bus. 56 */ 57 __rte_weak int 58 auxiliary_scan(void) 59 { 60 return 0; 61 } 62 63 /* 64 * Update a device's devargs being scanned. 65 */ 66 void 67 auxiliary_on_scan(struct rte_auxiliary_device *aux_dev) 68 { 69 aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name); 70 } 71 72 /* 73 * Match the auxiliary driver and device using driver function. 74 */ 75 bool 76 auxiliary_match(const struct rte_auxiliary_driver *aux_drv, 77 const struct rte_auxiliary_device *aux_dev) 78 { 79 if (aux_drv->match == NULL) 80 return false; 81 return aux_drv->match(aux_dev->name); 82 } 83 84 /* 85 * Call the probe() function of the driver. 86 */ 87 static int 88 rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv, 89 struct rte_auxiliary_device *dev) 90 { 91 enum rte_iova_mode iova_mode; 92 int ret; 93 94 if (drv == NULL || dev == NULL) 95 return -EINVAL; 96 97 /* Check if driver supports it. */ 98 if (!auxiliary_match(drv, dev)) 99 /* Match of device and driver failed */ 100 return 1; 101 102 /* No initialization when marked as blocked, return without error. */ 103 if (dev->device.devargs != NULL && 104 dev->device.devargs->policy == RTE_DEV_BLOCKED) { 105 AUXILIARY_LOG(INFO, "Device is blocked, not initializing"); 106 return -1; 107 } 108 109 if (dev->device.numa_node < 0) { 110 if (rte_socket_count() > 1) 111 AUXILIARY_LOG(INFO, "Device %s is not NUMA-aware, defaulting socket to 0", 112 dev->name); 113 dev->device.numa_node = 0; 114 } 115 116 iova_mode = rte_eal_iova_mode(); 117 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 && 118 iova_mode != RTE_IOVA_VA) { 119 AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing", 120 drv->driver.name); 121 return -EINVAL; 122 } 123 124 dev->driver = drv; 125 126 AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (NUMA node %i)", 127 drv->driver.name, dev->name, dev->device.numa_node); 128 ret = drv->probe(drv, dev); 129 if (ret != 0) 130 dev->driver = NULL; 131 else 132 dev->device.driver = &drv->driver; 133 134 return ret; 135 } 136 137 /* 138 * Call the remove() function of the driver. 139 */ 140 static int 141 rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev) 142 { 143 struct rte_auxiliary_driver *drv; 144 int ret = 0; 145 146 if (dev == NULL) 147 return -EINVAL; 148 149 drv = dev->driver; 150 151 AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA node %i", 152 drv->driver.name, dev->name, dev->device.numa_node); 153 154 if (drv->remove != NULL) { 155 ret = drv->remove(dev); 156 if (ret < 0) 157 return ret; 158 } 159 160 /* clear driver structure */ 161 dev->driver = NULL; 162 dev->device.driver = NULL; 163 164 return 0; 165 } 166 167 /* 168 * Call the probe() function of all registered drivers for the given device. 169 * Return < 0 if initialization failed. 170 * Return 1 if no driver is found for this device. 171 */ 172 static int 173 auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev) 174 { 175 struct rte_auxiliary_driver *drv; 176 int rc; 177 178 if (dev == NULL) 179 return -EINVAL; 180 181 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) { 182 if (!drv->match(dev->name)) 183 continue; 184 185 rc = rte_auxiliary_probe_one_driver(drv, dev); 186 if (rc < 0) 187 /* negative value is an error */ 188 return rc; 189 if (rc > 0) 190 /* positive value means driver doesn't support it */ 191 continue; 192 return 0; 193 } 194 return 1; 195 } 196 197 /* 198 * Scan the content of the auxiliary bus, and call the probe function for 199 * all registered drivers to try to probe discovered devices. 200 */ 201 static int 202 auxiliary_probe(void) 203 { 204 struct rte_auxiliary_device *dev = NULL; 205 size_t probed = 0, failed = 0; 206 int ret = 0; 207 208 FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) { 209 probed++; 210 211 ret = auxiliary_probe_all_drivers(dev); 212 if (ret < 0) { 213 if (ret != -EEXIST) { 214 AUXILIARY_LOG(ERR, "Requested device %s cannot be used", 215 dev->name); 216 rte_errno = errno; 217 failed++; 218 } 219 ret = 0; 220 } 221 } 222 223 return (probed && probed == failed) ? -1 : 0; 224 } 225 226 static int 227 auxiliary_parse(const char *name, void *addr) 228 { 229 struct rte_auxiliary_driver *drv = NULL; 230 const char **out = addr; 231 232 /* Allow empty device name "auxiliary:" to bypass entire bus scan. */ 233 if (strlen(name) == 0) 234 return 0; 235 236 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) { 237 if (drv->match(name)) 238 break; 239 } 240 if (drv != NULL && addr != NULL) 241 *out = name; 242 return drv != NULL ? 0 : -1; 243 } 244 245 /* Register a driver */ 246 void 247 rte_auxiliary_register(struct rte_auxiliary_driver *driver) 248 { 249 TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next); 250 driver->bus = &auxiliary_bus; 251 } 252 253 /* Unregister a driver */ 254 void 255 rte_auxiliary_unregister(struct rte_auxiliary_driver *driver) 256 { 257 TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next); 258 driver->bus = NULL; 259 } 260 261 /* Add a device to auxiliary bus */ 262 void 263 auxiliary_add_device(struct rte_auxiliary_device *aux_dev) 264 { 265 TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next); 266 } 267 268 /* Insert a device into a predefined position in auxiliary bus */ 269 void 270 auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev, 271 struct rte_auxiliary_device *new_aux_dev) 272 { 273 TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next); 274 } 275 276 /* Remove a device from auxiliary bus */ 277 static void 278 rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev) 279 { 280 TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next); 281 } 282 283 static struct rte_device * 284 auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, 285 const void *data) 286 { 287 const struct rte_auxiliary_device *pstart; 288 struct rte_auxiliary_device *adev; 289 290 if (start != NULL) { 291 pstart = RTE_DEV_TO_AUXILIARY_CONST(start); 292 adev = TAILQ_NEXT(pstart, next); 293 } else { 294 adev = TAILQ_FIRST(&auxiliary_bus.device_list); 295 } 296 while (adev != NULL) { 297 if (cmp(&adev->device, data) == 0) 298 return &adev->device; 299 adev = TAILQ_NEXT(adev, next); 300 } 301 return NULL; 302 } 303 304 static int 305 auxiliary_plug(struct rte_device *dev) 306 { 307 if (!auxiliary_dev_exists(dev->name)) 308 return -ENOENT; 309 return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev)); 310 } 311 312 static int 313 auxiliary_unplug(struct rte_device *dev) 314 { 315 struct rte_auxiliary_device *adev; 316 int ret; 317 318 adev = RTE_DEV_TO_AUXILIARY(dev); 319 ret = rte_auxiliary_driver_remove_dev(adev); 320 if (ret == 0) { 321 rte_auxiliary_remove_device(adev); 322 rte_devargs_remove(dev->devargs); 323 free(adev); 324 } 325 return ret; 326 } 327 328 static int 329 auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 330 { 331 struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev); 332 333 if (dev == NULL || aux_dev->driver == NULL) { 334 rte_errno = EINVAL; 335 return -1; 336 } 337 if (aux_dev->driver->dma_map == NULL) { 338 rte_errno = ENOTSUP; 339 return -1; 340 } 341 return aux_dev->driver->dma_map(aux_dev, addr, iova, len); 342 } 343 344 static int 345 auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, 346 size_t len) 347 { 348 struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev); 349 350 if (dev == NULL || aux_dev->driver == NULL) { 351 rte_errno = EINVAL; 352 return -1; 353 } 354 if (aux_dev->driver->dma_unmap == NULL) { 355 rte_errno = ENOTSUP; 356 return -1; 357 } 358 return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len); 359 } 360 361 bool 362 auxiliary_is_ignored_device(const char *name) 363 { 364 struct rte_devargs *devargs = auxiliary_devargs_lookup(name); 365 366 switch (auxiliary_bus.bus.conf.scan_mode) { 367 case RTE_BUS_SCAN_ALLOWLIST: 368 if (devargs && devargs->policy == RTE_DEV_ALLOWED) 369 return false; 370 break; 371 case RTE_BUS_SCAN_UNDEFINED: 372 case RTE_BUS_SCAN_BLOCKLIST: 373 if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED) 374 return false; 375 break; 376 } 377 return true; 378 } 379 380 static enum rte_iova_mode 381 auxiliary_get_iommu_class(void) 382 { 383 const struct rte_auxiliary_driver *drv; 384 385 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) { 386 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0) 387 return RTE_IOVA_VA; 388 } 389 390 return RTE_IOVA_DC; 391 } 392 393 struct rte_auxiliary_bus auxiliary_bus = { 394 .bus = { 395 .scan = auxiliary_scan, 396 .probe = auxiliary_probe, 397 .find_device = auxiliary_find_device, 398 .plug = auxiliary_plug, 399 .unplug = auxiliary_unplug, 400 .parse = auxiliary_parse, 401 .dma_map = auxiliary_dma_map, 402 .dma_unmap = auxiliary_dma_unmap, 403 .get_iommu_class = auxiliary_get_iommu_class, 404 .dev_iterate = auxiliary_dev_iterate, 405 }, 406 .device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list), 407 .driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list), 408 }; 409 410 RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus); 411 RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE); 412