1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022-2023, Advanced Micro Devices, Inc. 3 */ 4 5 /* 6 * Architecture Overview 7 * ===================== 8 * CDX is a Hardware Architecture designed for AMD FPGA devices. It 9 * consists of sophisticated mechanism for interaction between FPGA, 10 * Firmware and the APUs (Application CPUs). 11 * 12 * Firmware resides on RPU (Realtime CPUs) which interacts with 13 * the FPGA program manager and the APUs. The RPU provides memory-mapped 14 * interface (RPU if) which is used to communicate with APUs. 15 * 16 * The diagram below shows an overview of the AMD CDX architecture: 17 * 18 * +--------------------------------------+ 19 * | DPDK | 20 * | DPDK CDX drivers | 21 * | | | 22 * | DPDK AMD CDX bus | 23 * | | | 24 * +-----------------------------|--------+ 25 * | 26 * +-----------------------------|--------+ 27 * | Application CPUs (APU) | | 28 * | | | 29 * | VFIO CDX driver | 30 * | Linux OS | | 31 * | Linux AMD CDX bus | 32 * | | | 33 * +-----------------------------|--------+ 34 * | 35 * | 36 * +------------------------| RPU if |----+ 37 * | | | 38 * | V | 39 * | Realtime CPUs (RPU) | 40 * | | 41 * +--------------------------------------+ 42 * | 43 * +---------------------|----------------+ 44 * | FPGA | | 45 * | +-----------------------+ | 46 * | | | | | 47 * | +-------+ +-------+ +-------+ | 48 * | | dev 1 | | dev 2 | | dev 3 | | 49 * | +-------+ +-------+ +-------+ | 50 * +--------------------------------------+ 51 * 52 * The RPU firmware extracts the device information from the loaded FPGA 53 * image and implements a mechanism that allows the APU drivers to 54 * enumerate such devices (device personality and resource details) via 55 * a dedicated communication channel. 56 * 57 * VFIO CDX driver provides the CDX device resources like MMIO and interrupts 58 * to map to user-space. DPDK CDX bus uses sysfs interface and the vfio-cdx 59 * driver to discover and initialize the CDX devices for user-space 60 * applications. 61 */ 62 63 /** 64 * @file 65 * CDX probing using Linux sysfs. 66 */ 67 68 #include <string.h> 69 #include <dirent.h> 70 71 #include <rte_eal_paging.h> 72 #include <rte_errno.h> 73 #include <rte_devargs.h> 74 #include <rte_kvargs.h> 75 #include <rte_malloc.h> 76 #include <rte_vfio.h> 77 78 #include <eal_filesystem.h> 79 80 #include "bus_cdx_driver.h" 81 #include "cdx_logs.h" 82 #include "private.h" 83 84 #define CDX_BUS_NAME cdx 85 #define CDX_DEV_PREFIX "cdx-" 86 87 /* CDX Bus iterators */ 88 #define FOREACH_DEVICE_ON_CDXBUS(p) \ 89 RTE_TAILQ_FOREACH(p, &rte_cdx_bus.device_list, next) 90 91 #define FOREACH_DRIVER_ON_CDXBUS(p) \ 92 RTE_TAILQ_FOREACH(p, &rte_cdx_bus.driver_list, next) 93 94 struct rte_cdx_bus rte_cdx_bus; 95 96 enum cdx_params { 97 RTE_CDX_PARAM_NAME, 98 }; 99 100 static const char * const cdx_params_keys[] = { 101 [RTE_CDX_PARAM_NAME] = "name", 102 NULL, 103 }; 104 105 /* Add a device to CDX bus */ 106 static void 107 cdx_add_device(struct rte_cdx_device *cdx_dev) 108 { 109 TAILQ_INSERT_TAIL(&rte_cdx_bus.device_list, cdx_dev, next); 110 } 111 112 static int 113 cdx_get_kernel_driver_by_path(const char *filename, char *driver_name, 114 size_t len) 115 { 116 int count; 117 char path[PATH_MAX]; 118 char *name; 119 120 if (!filename || !driver_name) 121 return -1; 122 123 count = readlink(filename, path, PATH_MAX); 124 if (count >= PATH_MAX) 125 return -1; 126 127 /* For device does not have a driver */ 128 if (count < 0) 129 return 1; 130 131 path[count] = '\0'; 132 133 name = strrchr(path, '/'); 134 if (name) { 135 strlcpy(driver_name, name + 1, len); 136 return 0; 137 } 138 139 return -1; 140 } 141 142 int rte_cdx_map_device(struct rte_cdx_device *dev) 143 { 144 return cdx_vfio_map_resource(dev); 145 } 146 147 void rte_cdx_unmap_device(struct rte_cdx_device *dev) 148 { 149 cdx_vfio_unmap_resource(dev); 150 } 151 152 static struct rte_devargs * 153 cdx_devargs_lookup(const char *dev_name) 154 { 155 struct rte_devargs *devargs; 156 157 RTE_EAL_DEVARGS_FOREACH("cdx", devargs) { 158 if (strcmp(devargs->name, dev_name) == 0) 159 return devargs; 160 } 161 return NULL; 162 } 163 164 static bool 165 cdx_ignore_device(const char *dev_name) 166 { 167 struct rte_devargs *devargs = cdx_devargs_lookup(dev_name); 168 169 switch (rte_cdx_bus.bus.conf.scan_mode) { 170 case RTE_BUS_SCAN_ALLOWLIST: 171 if (devargs && devargs->policy == RTE_DEV_ALLOWED) 172 return false; 173 break; 174 case RTE_BUS_SCAN_UNDEFINED: 175 case RTE_BUS_SCAN_BLOCKLIST: 176 if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED) 177 return false; 178 break; 179 } 180 return true; 181 } 182 183 /* 184 * Scan one cdx sysfs entry, and fill the devices list from it. 185 * It checks if the CDX device is bound to vfio-cdx driver. In case 186 * the device is vfio bound, it reads the vendor and device id and 187 * stores it for device-driver matching. 188 */ 189 static int 190 cdx_scan_one(const char *dirname, const char *dev_name) 191 { 192 char filename[PATH_MAX]; 193 struct rte_cdx_device *dev = NULL; 194 char driver[PATH_MAX]; 195 unsigned long tmp; 196 int ret; 197 198 dev = calloc(1, sizeof(*dev)); 199 if (!dev) 200 return -ENOMEM; 201 202 dev->device.bus = &rte_cdx_bus.bus; 203 memcpy(dev->name, dev_name, RTE_DEV_NAME_MAX_LEN); 204 dev->device.name = dev->name; 205 206 /* parse driver */ 207 snprintf(filename, sizeof(filename), "%s/driver", dirname); 208 ret = cdx_get_kernel_driver_by_path(filename, driver, sizeof(driver)); 209 if (ret < 0) { 210 CDX_BUS_ERR("Fail to get kernel driver"); 211 free(dev); 212 return -1; 213 } 214 215 /* Allocate interrupt instance for cdx device */ 216 dev->intr_handle = 217 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE); 218 if (dev->intr_handle == NULL) { 219 CDX_BUS_ERR("Failed to create interrupt instance for %s", 220 dev->device.name); 221 free(dev); 222 return -ENOMEM; 223 } 224 225 /* 226 * Check if device is bound to 'vfio-cdx' driver, so that user-space 227 * can gracefully access the device. 228 */ 229 if (ret || strcmp(driver, "vfio-cdx")) { 230 ret = 0; 231 goto err; 232 } 233 234 /* get vendor id */ 235 snprintf(filename, sizeof(filename), "%s/vendor", dirname); 236 if (eal_parse_sysfs_value(filename, &tmp) < 0) { 237 ret = -1; 238 goto err; 239 } 240 dev->id.vendor_id = (uint16_t)tmp; 241 242 /* get device id */ 243 snprintf(filename, sizeof(filename), "%s/device", dirname); 244 if (eal_parse_sysfs_value(filename, &tmp) < 0) { 245 ret = -1; 246 goto err; 247 } 248 dev->id.device_id = (uint16_t)tmp; 249 250 cdx_add_device(dev); 251 252 return 0; 253 254 err: 255 rte_intr_instance_free(dev->intr_handle); 256 free(dev); 257 return ret; 258 } 259 260 /* 261 * Scan the content of the CDX bus, and the devices in the devices 262 * list. 263 */ 264 static int 265 cdx_scan(void) 266 { 267 struct dirent *e; 268 DIR *dir; 269 char dirname[PATH_MAX]; 270 271 dir = opendir(RTE_CDX_BUS_DEVICES_PATH); 272 if (dir == NULL) { 273 CDX_BUS_INFO("%s(): opendir failed: %s", __func__, 274 strerror(errno)); 275 return 0; 276 } 277 278 while ((e = readdir(dir)) != NULL) { 279 if (e->d_name[0] == '.') 280 continue; 281 282 if (cdx_ignore_device(e->d_name)) 283 continue; 284 285 snprintf(dirname, sizeof(dirname), "%s/%s", 286 RTE_CDX_BUS_DEVICES_PATH, e->d_name); 287 288 if (cdx_scan_one(dirname, e->d_name) < 0) 289 goto error; 290 } 291 closedir(dir); 292 return 0; 293 294 error: 295 closedir(dir); 296 return -1; 297 } 298 299 /* map a particular resource from a file */ 300 void * 301 cdx_map_resource(void *requested_addr, int fd, uint64_t offset, size_t size, 302 int additional_flags) 303 { 304 void *mapaddr; 305 306 /* Map the cdx MMIO memory resource of device */ 307 mapaddr = rte_mem_map(requested_addr, size, 308 RTE_PROT_READ | RTE_PROT_WRITE, 309 RTE_MAP_SHARED | additional_flags, fd, offset); 310 if (mapaddr == NULL) { 311 CDX_BUS_ERR("%s(): cannot map resource(%d, %p, 0x%zx, 0x%"PRIx64"): %s (%p)", 312 __func__, fd, requested_addr, size, offset, 313 rte_strerror(rte_errno), mapaddr); 314 } 315 CDX_BUS_DEBUG("CDX MMIO memory mapped at %p", mapaddr); 316 317 return mapaddr; 318 } 319 320 /* unmap a particular resource */ 321 void 322 cdx_unmap_resource(void *requested_addr, size_t size) 323 { 324 if (requested_addr == NULL) 325 return; 326 327 CDX_BUS_DEBUG("Unmapping CDX memory at %p", requested_addr); 328 329 /* Unmap the CDX memory resource of device */ 330 if (rte_mem_unmap(requested_addr, size)) { 331 CDX_BUS_ERR("%s(): cannot mem unmap(%p, %#zx): %s", __func__, 332 requested_addr, size, rte_strerror(rte_errno)); 333 } 334 } 335 /* 336 * Match the CDX Driver and Device using device id and vendor id. 337 */ 338 static bool 339 cdx_match(const struct rte_cdx_driver *cdx_drv, 340 const struct rte_cdx_device *cdx_dev) 341 { 342 const struct rte_cdx_id *id_table; 343 344 for (id_table = cdx_drv->id_table; id_table->vendor_id != 0; 345 id_table++) { 346 /* check if device's identifiers match the driver's ones */ 347 if (id_table->vendor_id != cdx_dev->id.vendor_id && 348 id_table->vendor_id != RTE_CDX_ANY_ID) 349 continue; 350 if (id_table->device_id != cdx_dev->id.device_id && 351 id_table->device_id != RTE_CDX_ANY_ID) 352 continue; 353 354 return 1; 355 } 356 357 return 0; 358 } 359 360 /* 361 * If vendor id and device id match, call the probe() function of the 362 * driver. 363 */ 364 static int 365 cdx_probe_one_driver(struct rte_cdx_driver *dr, 366 struct rte_cdx_device *dev) 367 { 368 const char *dev_name = dev->name; 369 bool already_probed; 370 int ret; 371 372 /* The device is not blocked; Check if driver supports it */ 373 if (!cdx_match(dr, dev)) 374 /* Match of device and driver failed */ 375 return 1; 376 377 already_probed = rte_dev_is_probed(&dev->device); 378 if (already_probed) { 379 CDX_BUS_INFO("Device %s is already probed", dev_name); 380 return -EEXIST; 381 } 382 383 CDX_BUS_DEBUG(" probe device %s using driver: %s", dev_name, 384 dr->driver.name); 385 386 if (dr->drv_flags & RTE_CDX_DRV_NEED_MAPPING) { 387 ret = cdx_vfio_map_resource(dev); 388 if (ret != 0) { 389 CDX_BUS_ERR("CDX map device failed: %d", ret); 390 goto error_map_device; 391 } 392 } 393 394 /* call the driver probe() function */ 395 ret = dr->probe(dr, dev); 396 if (ret) { 397 CDX_BUS_ERR("Probe CDX driver: %s device: %s failed: %d", 398 dr->driver.name, dev_name, ret); 399 goto error_probe; 400 } else { 401 dev->device.driver = &dr->driver; 402 } 403 dev->driver = dr; 404 405 return ret; 406 407 error_probe: 408 cdx_vfio_unmap_resource(dev); 409 rte_intr_instance_free(dev->intr_handle); 410 dev->intr_handle = NULL; 411 error_map_device: 412 return ret; 413 } 414 415 /* 416 * If vendor/device ID match, call the probe() function of all 417 * registered driver for the given device. Return < 0 if initialization 418 * failed, return 1 if no driver is found for this device. 419 */ 420 static int 421 cdx_probe_all_drivers(struct rte_cdx_device *dev) 422 { 423 struct rte_cdx_driver *dr = NULL; 424 int rc = 0; 425 426 FOREACH_DRIVER_ON_CDXBUS(dr) { 427 rc = cdx_probe_one_driver(dr, dev); 428 if (rc < 0) 429 /* negative value is an error */ 430 return rc; 431 if (rc > 0) 432 /* positive value means driver doesn't support it */ 433 continue; 434 return 0; 435 } 436 return 1; 437 } 438 439 /* 440 * Scan the content of the CDX bus, and call the probe() function for 441 * all registered drivers that have a matching entry in its id_table 442 * for discovered devices. 443 */ 444 static int 445 cdx_probe(void) 446 { 447 struct rte_cdx_device *dev = NULL; 448 size_t probed = 0, failed = 0; 449 int ret = 0; 450 451 FOREACH_DEVICE_ON_CDXBUS(dev) { 452 probed++; 453 454 ret = cdx_probe_all_drivers(dev); 455 if (ret < 0) { 456 CDX_BUS_ERR("Requested device %s cannot be used", 457 dev->name); 458 rte_errno = errno; 459 failed++; 460 } 461 } 462 463 return (probed && probed == failed) ? -1 : 0; 464 } 465 466 static int 467 cdx_parse(const char *name, void *addr) 468 { 469 const char **out = addr; 470 int ret; 471 472 ret = strncmp(name, CDX_DEV_PREFIX, strlen(CDX_DEV_PREFIX)); 473 474 if (ret == 0 && addr) 475 *out = name; 476 477 return ret; 478 } 479 480 /* register a driver */ 481 void 482 rte_cdx_register(struct rte_cdx_driver *driver) 483 { 484 TAILQ_INSERT_TAIL(&rte_cdx_bus.driver_list, driver, next); 485 driver->bus = &rte_cdx_bus; 486 } 487 488 /* unregister a driver */ 489 void 490 rte_cdx_unregister(struct rte_cdx_driver *driver) 491 { 492 TAILQ_REMOVE(&rte_cdx_bus.driver_list, driver, next); 493 driver->bus = NULL; 494 } 495 496 static struct rte_device * 497 cdx_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, 498 const void *data) 499 { 500 const struct rte_cdx_device *cdx_start; 501 struct rte_cdx_device *cdx_dev; 502 503 if (start != NULL) { 504 cdx_start = RTE_DEV_TO_CDX_DEV_CONST(start); 505 cdx_dev = TAILQ_NEXT(cdx_start, next); 506 } else { 507 cdx_dev = TAILQ_FIRST(&rte_cdx_bus.device_list); 508 } 509 while (cdx_dev != NULL) { 510 if (cmp(&cdx_dev->device, data) == 0) 511 return &cdx_dev->device; 512 cdx_dev = TAILQ_NEXT(cdx_dev, next); 513 } 514 return NULL; 515 } 516 517 /* Remove a device from CDX bus */ 518 static void 519 cdx_remove_device(struct rte_cdx_device *cdx_dev) 520 { 521 TAILQ_REMOVE(&rte_cdx_bus.device_list, cdx_dev, next); 522 } 523 524 /* 525 * If vendor/device ID match, call the remove() function of the 526 * driver. 527 */ 528 static int 529 cdx_detach_dev(struct rte_cdx_device *dev) 530 { 531 struct rte_cdx_driver *dr; 532 int ret = 0; 533 534 if (dev == NULL) 535 return -EINVAL; 536 537 dr = dev->driver; 538 539 CDX_BUS_DEBUG("detach device %s using driver: %s", 540 dev->device.name, dr->driver.name); 541 542 if (dr->remove) { 543 ret = dr->remove(dev); 544 if (ret < 0) 545 return ret; 546 } 547 548 /* clear driver structure */ 549 dev->driver = NULL; 550 dev->device.driver = NULL; 551 552 rte_cdx_unmap_device(dev); 553 554 rte_intr_instance_free(dev->intr_handle); 555 dev->intr_handle = NULL; 556 557 return 0; 558 } 559 560 static int 561 cdx_plug(struct rte_device *dev) 562 { 563 return cdx_probe_all_drivers(RTE_DEV_TO_CDX_DEV(dev)); 564 } 565 566 static int 567 cdx_unplug(struct rte_device *dev) 568 { 569 struct rte_cdx_device *cdx_dev; 570 int ret; 571 572 cdx_dev = RTE_DEV_TO_CDX_DEV(dev); 573 ret = cdx_detach_dev(cdx_dev); 574 if (ret == 0) { 575 cdx_remove_device(cdx_dev); 576 rte_devargs_remove(dev->devargs); 577 free(cdx_dev); 578 } 579 return ret; 580 } 581 582 static int 583 cdx_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 584 { 585 RTE_SET_USED(dev); 586 587 return rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD, 588 (uintptr_t)addr, iova, len); 589 } 590 591 static int 592 cdx_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 593 { 594 RTE_SET_USED(dev); 595 596 return rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD, 597 (uintptr_t)addr, iova, len); 598 } 599 600 static enum rte_iova_mode 601 cdx_get_iommu_class(void) 602 { 603 if (TAILQ_EMPTY(&rte_cdx_bus.device_list)) 604 return RTE_IOVA_DC; 605 606 return RTE_IOVA_VA; 607 } 608 609 static int 610 cdx_dev_match(const struct rte_device *dev, 611 const void *_kvlist) 612 { 613 const struct rte_kvargs *kvlist = _kvlist; 614 const char *key = cdx_params_keys[RTE_CDX_PARAM_NAME]; 615 const char *name; 616 617 /* no kvlist arg, all devices match */ 618 if (kvlist == NULL) 619 return 0; 620 621 /* if key is present in kvlist and does not match, filter device */ 622 name = rte_kvargs_get(kvlist, key); 623 if (name != NULL && strcmp(name, dev->name)) 624 return -1; 625 626 return 0; 627 } 628 629 static void * 630 cdx_dev_iterate(const void *start, 631 const char *str, 632 const struct rte_dev_iterator *it __rte_unused) 633 { 634 rte_bus_find_device_t find_device; 635 struct rte_kvargs *kvargs = NULL; 636 struct rte_device *dev; 637 638 if (str != NULL) { 639 kvargs = rte_kvargs_parse(str, cdx_params_keys); 640 if (kvargs == NULL) { 641 CDX_BUS_ERR("cannot parse argument list %s", str); 642 rte_errno = EINVAL; 643 return NULL; 644 } 645 } 646 find_device = rte_cdx_bus.bus.find_device; 647 dev = find_device(start, cdx_dev_match, kvargs); 648 rte_kvargs_free(kvargs); 649 return dev; 650 } 651 652 struct rte_cdx_bus rte_cdx_bus = { 653 .bus = { 654 .scan = cdx_scan, 655 .probe = cdx_probe, 656 .find_device = cdx_find_device, 657 .plug = cdx_plug, 658 .unplug = cdx_unplug, 659 .parse = cdx_parse, 660 .dma_map = cdx_dma_map, 661 .dma_unmap = cdx_dma_unmap, 662 .get_iommu_class = cdx_get_iommu_class, 663 .dev_iterate = cdx_dev_iterate, 664 }, 665 .device_list = TAILQ_HEAD_INITIALIZER(rte_cdx_bus.device_list), 666 .driver_list = TAILQ_HEAD_INITIALIZER(rte_cdx_bus.driver_list), 667 }; 668 669 RTE_REGISTER_BUS(cdx, rte_cdx_bus.bus); 670 RTE_LOG_REGISTER_DEFAULT(cdx_logtype_bus, NOTICE); 671