1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
3 */
4
5 /*
6 * Architecture Overview
7 * =====================
8 * CDX is a Hardware Architecture designed for AMD FPGA devices. It
9 * consists of sophisticated mechanism for interaction between FPGA,
10 * Firmware and the APUs (Application CPUs).
11 *
12 * Firmware resides on RPU (Realtime CPUs) which interacts with
13 * the FPGA program manager and the APUs. The RPU provides memory-mapped
14 * interface (RPU if) which is used to communicate with APUs.
15 *
16 * The diagram below shows an overview of the AMD CDX architecture:
17 *
18 * +--------------------------------------+
19 * | DPDK |
20 * | DPDK CDX drivers |
21 * | | |
22 * | DPDK AMD CDX bus |
23 * | | |
24 * +-----------------------------|--------+
25 * |
26 * +-----------------------------|--------+
27 * | Application CPUs (APU) | |
28 * | | |
29 * | VFIO CDX driver |
30 * | Linux OS | |
31 * | Linux AMD CDX bus |
32 * | | |
33 * +-----------------------------|--------+
34 * |
35 * |
36 * +------------------------| RPU if |----+
37 * | | |
38 * | V |
39 * | Realtime CPUs (RPU) |
40 * | |
41 * +--------------------------------------+
42 * |
43 * +---------------------|----------------+
44 * | FPGA | |
45 * | +-----------------------+ |
46 * | | | | |
47 * | +-------+ +-------+ +-------+ |
48 * | | dev 1 | | dev 2 | | dev 3 | |
49 * | +-------+ +-------+ +-------+ |
50 * +--------------------------------------+
51 *
52 * The RPU firmware extracts the device information from the loaded FPGA
53 * image and implements a mechanism that allows the APU drivers to
54 * enumerate such devices (device personality and resource details) via
55 * a dedicated communication channel.
56 *
57 * VFIO CDX driver provides the CDX device resources like MMIO and interrupts
58 * to map to user-space. DPDK CDX bus uses sysfs interface and the vfio-cdx
59 * driver to discover and initialize the CDX devices for user-space
60 * applications.
61 */
62
63 /**
64 * @file
65 * CDX probing using Linux sysfs.
66 */
67
68 #include <string.h>
69 #include <dirent.h>
70
71 #include <rte_eal_paging.h>
72 #include <rte_errno.h>
73 #include <rte_devargs.h>
74 #include <rte_kvargs.h>
75 #include <rte_malloc.h>
76 #include <rte_vfio.h>
77
78 #include <eal_filesystem.h>
79
80 #include "bus_cdx_driver.h"
81 #include "cdx_logs.h"
82 #include "private.h"
83
84 #define CDX_BUS_NAME cdx
85 #define CDX_DEV_PREFIX "cdx-"
86
87 /* CDX Bus iterators */
88 #define FOREACH_DEVICE_ON_CDXBUS(p) \
89 RTE_TAILQ_FOREACH(p, &rte_cdx_bus.device_list, next)
90
91 #define FOREACH_DRIVER_ON_CDXBUS(p) \
92 RTE_TAILQ_FOREACH(p, &rte_cdx_bus.driver_list, next)
93
94 struct rte_cdx_bus rte_cdx_bus;
95
96 enum cdx_params {
97 RTE_CDX_PARAM_NAME,
98 };
99
100 static const char * const cdx_params_keys[] = {
101 [RTE_CDX_PARAM_NAME] = "name",
102 NULL,
103 };
104
105 /* Add a device to CDX bus */
106 static void
cdx_add_device(struct rte_cdx_device * cdx_dev)107 cdx_add_device(struct rte_cdx_device *cdx_dev)
108 {
109 TAILQ_INSERT_TAIL(&rte_cdx_bus.device_list, cdx_dev, next);
110 }
111
112 static int
cdx_get_kernel_driver_by_path(const char * filename,char * driver_name,size_t len)113 cdx_get_kernel_driver_by_path(const char *filename, char *driver_name,
114 size_t len)
115 {
116 int count;
117 char path[PATH_MAX];
118 char *name;
119
120 if (!filename || !driver_name)
121 return -1;
122
123 count = readlink(filename, path, PATH_MAX);
124 if (count >= PATH_MAX)
125 return -1;
126
127 /* For device does not have a driver */
128 if (count < 0)
129 return 1;
130
131 path[count] = '\0';
132
133 name = strrchr(path, '/');
134 if (name) {
135 strlcpy(driver_name, name + 1, len);
136 return 0;
137 }
138
139 return -1;
140 }
141
rte_cdx_map_device(struct rte_cdx_device * dev)142 int rte_cdx_map_device(struct rte_cdx_device *dev)
143 {
144 return cdx_vfio_map_resource(dev);
145 }
146
rte_cdx_unmap_device(struct rte_cdx_device * dev)147 void rte_cdx_unmap_device(struct rte_cdx_device *dev)
148 {
149 cdx_vfio_unmap_resource(dev);
150 }
151
152 static struct rte_devargs *
cdx_devargs_lookup(const char * dev_name)153 cdx_devargs_lookup(const char *dev_name)
154 {
155 struct rte_devargs *devargs;
156
157 RTE_EAL_DEVARGS_FOREACH("cdx", devargs) {
158 if (strcmp(devargs->name, dev_name) == 0)
159 return devargs;
160 }
161 return NULL;
162 }
163
164 static bool
cdx_ignore_device(const char * dev_name)165 cdx_ignore_device(const char *dev_name)
166 {
167 struct rte_devargs *devargs = cdx_devargs_lookup(dev_name);
168
169 switch (rte_cdx_bus.bus.conf.scan_mode) {
170 case RTE_BUS_SCAN_ALLOWLIST:
171 if (devargs && devargs->policy == RTE_DEV_ALLOWED)
172 return false;
173 break;
174 case RTE_BUS_SCAN_UNDEFINED:
175 case RTE_BUS_SCAN_BLOCKLIST:
176 if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
177 return false;
178 break;
179 }
180 return true;
181 }
182
183 /*
184 * Scan one cdx sysfs entry, and fill the devices list from it.
185 * It checks if the CDX device is bound to vfio-cdx driver. In case
186 * the device is vfio bound, it reads the vendor and device id and
187 * stores it for device-driver matching.
188 */
189 static int
cdx_scan_one(const char * dirname,const char * dev_name)190 cdx_scan_one(const char *dirname, const char *dev_name)
191 {
192 char filename[PATH_MAX];
193 struct rte_cdx_device *dev = NULL;
194 char driver[PATH_MAX];
195 unsigned long tmp;
196 int ret;
197
198 dev = calloc(1, sizeof(*dev));
199 if (!dev)
200 return -ENOMEM;
201
202 dev->device.bus = &rte_cdx_bus.bus;
203 memcpy(dev->name, dev_name, RTE_DEV_NAME_MAX_LEN);
204 dev->device.name = dev->name;
205
206 /* parse driver */
207 snprintf(filename, sizeof(filename), "%s/driver", dirname);
208 ret = cdx_get_kernel_driver_by_path(filename, driver, sizeof(driver));
209 if (ret < 0) {
210 CDX_BUS_ERR("Fail to get kernel driver");
211 free(dev);
212 return -1;
213 }
214
215 /* Allocate interrupt instance for cdx device */
216 dev->intr_handle =
217 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE);
218 if (dev->intr_handle == NULL) {
219 CDX_BUS_ERR("Failed to create interrupt instance for %s",
220 dev->device.name);
221 free(dev);
222 return -ENOMEM;
223 }
224
225 /*
226 * Check if device is bound to 'vfio-cdx' driver, so that user-space
227 * can gracefully access the device.
228 */
229 if (ret || strcmp(driver, "vfio-cdx")) {
230 ret = 0;
231 goto err;
232 }
233
234 /* get vendor id */
235 snprintf(filename, sizeof(filename), "%s/vendor", dirname);
236 if (eal_parse_sysfs_value(filename, &tmp) < 0) {
237 ret = -1;
238 goto err;
239 }
240 dev->id.vendor_id = (uint16_t)tmp;
241
242 /* get device id */
243 snprintf(filename, sizeof(filename), "%s/device", dirname);
244 if (eal_parse_sysfs_value(filename, &tmp) < 0) {
245 ret = -1;
246 goto err;
247 }
248 dev->id.device_id = (uint16_t)tmp;
249
250 cdx_add_device(dev);
251
252 return 0;
253
254 err:
255 rte_intr_instance_free(dev->intr_handle);
256 free(dev);
257 return ret;
258 }
259
260 /*
261 * Scan the content of the CDX bus, and the devices in the devices
262 * list.
263 */
264 static int
cdx_scan(void)265 cdx_scan(void)
266 {
267 struct dirent *e;
268 DIR *dir;
269 char dirname[PATH_MAX];
270
271 dir = opendir(RTE_CDX_BUS_DEVICES_PATH);
272 if (dir == NULL) {
273 CDX_BUS_INFO("%s(): opendir failed: %s", __func__,
274 strerror(errno));
275 return 0;
276 }
277
278 while ((e = readdir(dir)) != NULL) {
279 if (e->d_name[0] == '.')
280 continue;
281
282 if (cdx_ignore_device(e->d_name))
283 continue;
284
285 snprintf(dirname, sizeof(dirname), "%s/%s",
286 RTE_CDX_BUS_DEVICES_PATH, e->d_name);
287
288 if (cdx_scan_one(dirname, e->d_name) < 0)
289 goto error;
290 }
291 closedir(dir);
292 return 0;
293
294 error:
295 closedir(dir);
296 return -1;
297 }
298
299 /* map a particular resource from a file */
300 void *
cdx_map_resource(void * requested_addr,int fd,uint64_t offset,size_t size,int additional_flags)301 cdx_map_resource(void *requested_addr, int fd, uint64_t offset, size_t size,
302 int additional_flags)
303 {
304 void *mapaddr;
305
306 /* Map the cdx MMIO memory resource of device */
307 mapaddr = rte_mem_map(requested_addr, size,
308 RTE_PROT_READ | RTE_PROT_WRITE,
309 RTE_MAP_SHARED | additional_flags, fd, offset);
310 if (mapaddr == NULL) {
311 CDX_BUS_ERR("%s(): cannot map resource(%d, %p, 0x%zx, 0x%"PRIx64"): %s (%p)",
312 __func__, fd, requested_addr, size, offset,
313 rte_strerror(rte_errno), mapaddr);
314 }
315 CDX_BUS_DEBUG("CDX MMIO memory mapped at %p", mapaddr);
316
317 return mapaddr;
318 }
319
320 /* unmap a particular resource */
321 void
cdx_unmap_resource(void * requested_addr,size_t size)322 cdx_unmap_resource(void *requested_addr, size_t size)
323 {
324 if (requested_addr == NULL)
325 return;
326
327 CDX_BUS_DEBUG("Unmapping CDX memory at %p", requested_addr);
328
329 /* Unmap the CDX memory resource of device */
330 if (rte_mem_unmap(requested_addr, size)) {
331 CDX_BUS_ERR("%s(): cannot mem unmap(%p, %#zx): %s", __func__,
332 requested_addr, size, rte_strerror(rte_errno));
333 }
334 }
335 /*
336 * Match the CDX Driver and Device using device id and vendor id.
337 */
338 static bool
cdx_match(const struct rte_cdx_driver * cdx_drv,const struct rte_cdx_device * cdx_dev)339 cdx_match(const struct rte_cdx_driver *cdx_drv,
340 const struct rte_cdx_device *cdx_dev)
341 {
342 const struct rte_cdx_id *id_table;
343
344 for (id_table = cdx_drv->id_table; id_table->vendor_id != 0;
345 id_table++) {
346 /* check if device's identifiers match the driver's ones */
347 if (id_table->vendor_id != cdx_dev->id.vendor_id &&
348 id_table->vendor_id != RTE_CDX_ANY_ID)
349 continue;
350 if (id_table->device_id != cdx_dev->id.device_id &&
351 id_table->device_id != RTE_CDX_ANY_ID)
352 continue;
353
354 return 1;
355 }
356
357 return 0;
358 }
359
360 /*
361 * If vendor id and device id match, call the probe() function of the
362 * driver.
363 */
364 static int
cdx_probe_one_driver(struct rte_cdx_driver * dr,struct rte_cdx_device * dev)365 cdx_probe_one_driver(struct rte_cdx_driver *dr,
366 struct rte_cdx_device *dev)
367 {
368 const char *dev_name = dev->name;
369 bool already_probed;
370 int ret;
371
372 /* The device is not blocked; Check if driver supports it */
373 if (!cdx_match(dr, dev))
374 /* Match of device and driver failed */
375 return 1;
376
377 already_probed = rte_dev_is_probed(&dev->device);
378 if (already_probed) {
379 CDX_BUS_INFO("Device %s is already probed", dev_name);
380 return -EEXIST;
381 }
382
383 CDX_BUS_DEBUG(" probe device %s using driver: %s", dev_name,
384 dr->driver.name);
385
386 if (dr->drv_flags & RTE_CDX_DRV_NEED_MAPPING) {
387 ret = cdx_vfio_map_resource(dev);
388 if (ret != 0) {
389 CDX_BUS_ERR("CDX map device failed: %d", ret);
390 goto error_map_device;
391 }
392 }
393
394 /* call the driver probe() function */
395 ret = dr->probe(dr, dev);
396 if (ret) {
397 CDX_BUS_ERR("Probe CDX driver: %s device: %s failed: %d",
398 dr->driver.name, dev_name, ret);
399 goto error_probe;
400 } else {
401 dev->device.driver = &dr->driver;
402 }
403 dev->driver = dr;
404
405 return ret;
406
407 error_probe:
408 cdx_vfio_unmap_resource(dev);
409 rte_intr_instance_free(dev->intr_handle);
410 dev->intr_handle = NULL;
411 error_map_device:
412 return ret;
413 }
414
415 /*
416 * If vendor/device ID match, call the probe() function of all
417 * registered driver for the given device. Return < 0 if initialization
418 * failed, return 1 if no driver is found for this device.
419 */
420 static int
cdx_probe_all_drivers(struct rte_cdx_device * dev)421 cdx_probe_all_drivers(struct rte_cdx_device *dev)
422 {
423 struct rte_cdx_driver *dr = NULL;
424 int rc = 0;
425
426 FOREACH_DRIVER_ON_CDXBUS(dr) {
427 rc = cdx_probe_one_driver(dr, dev);
428 if (rc < 0)
429 /* negative value is an error */
430 return rc;
431 if (rc > 0)
432 /* positive value means driver doesn't support it */
433 continue;
434 return 0;
435 }
436 return 1;
437 }
438
439 /*
440 * Scan the content of the CDX bus, and call the probe() function for
441 * all registered drivers that have a matching entry in its id_table
442 * for discovered devices.
443 */
444 static int
cdx_probe(void)445 cdx_probe(void)
446 {
447 struct rte_cdx_device *dev = NULL;
448 size_t probed = 0, failed = 0;
449 int ret = 0;
450
451 FOREACH_DEVICE_ON_CDXBUS(dev) {
452 probed++;
453
454 ret = cdx_probe_all_drivers(dev);
455 if (ret < 0) {
456 CDX_BUS_ERR("Requested device %s cannot be used",
457 dev->name);
458 rte_errno = errno;
459 failed++;
460 }
461 }
462
463 return (probed && probed == failed) ? -1 : 0;
464 }
465
466 static int
cdx_parse(const char * name,void * addr)467 cdx_parse(const char *name, void *addr)
468 {
469 const char **out = addr;
470 int ret;
471
472 ret = strncmp(name, CDX_DEV_PREFIX, strlen(CDX_DEV_PREFIX));
473
474 if (ret == 0 && addr)
475 *out = name;
476
477 return ret;
478 }
479
480 /* register a driver */
481 void
rte_cdx_register(struct rte_cdx_driver * driver)482 rte_cdx_register(struct rte_cdx_driver *driver)
483 {
484 TAILQ_INSERT_TAIL(&rte_cdx_bus.driver_list, driver, next);
485 driver->bus = &rte_cdx_bus;
486 }
487
488 /* unregister a driver */
489 void
rte_cdx_unregister(struct rte_cdx_driver * driver)490 rte_cdx_unregister(struct rte_cdx_driver *driver)
491 {
492 TAILQ_REMOVE(&rte_cdx_bus.driver_list, driver, next);
493 driver->bus = NULL;
494 }
495
496 static struct rte_device *
cdx_find_device(const struct rte_device * start,rte_dev_cmp_t cmp,const void * data)497 cdx_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
498 const void *data)
499 {
500 const struct rte_cdx_device *cdx_start;
501 struct rte_cdx_device *cdx_dev;
502
503 if (start != NULL) {
504 cdx_start = RTE_DEV_TO_CDX_DEV_CONST(start);
505 cdx_dev = TAILQ_NEXT(cdx_start, next);
506 } else {
507 cdx_dev = TAILQ_FIRST(&rte_cdx_bus.device_list);
508 }
509 while (cdx_dev != NULL) {
510 if (cmp(&cdx_dev->device, data) == 0)
511 return &cdx_dev->device;
512 cdx_dev = TAILQ_NEXT(cdx_dev, next);
513 }
514 return NULL;
515 }
516
517 /* Remove a device from CDX bus */
518 static void
cdx_remove_device(struct rte_cdx_device * cdx_dev)519 cdx_remove_device(struct rte_cdx_device *cdx_dev)
520 {
521 TAILQ_REMOVE(&rte_cdx_bus.device_list, cdx_dev, next);
522 }
523
524 /*
525 * If vendor/device ID match, call the remove() function of the
526 * driver.
527 */
528 static int
cdx_detach_dev(struct rte_cdx_device * dev)529 cdx_detach_dev(struct rte_cdx_device *dev)
530 {
531 struct rte_cdx_driver *dr;
532 int ret = 0;
533
534 if (dev == NULL)
535 return -EINVAL;
536
537 dr = dev->driver;
538
539 CDX_BUS_DEBUG("detach device %s using driver: %s",
540 dev->device.name, dr->driver.name);
541
542 if (dr->remove) {
543 ret = dr->remove(dev);
544 if (ret < 0)
545 return ret;
546 }
547
548 /* clear driver structure */
549 dev->driver = NULL;
550 dev->device.driver = NULL;
551
552 rte_cdx_unmap_device(dev);
553
554 rte_intr_instance_free(dev->intr_handle);
555 dev->intr_handle = NULL;
556
557 return 0;
558 }
559
560 static int
cdx_plug(struct rte_device * dev)561 cdx_plug(struct rte_device *dev)
562 {
563 return cdx_probe_all_drivers(RTE_DEV_TO_CDX_DEV(dev));
564 }
565
566 static int
cdx_unplug(struct rte_device * dev)567 cdx_unplug(struct rte_device *dev)
568 {
569 struct rte_cdx_device *cdx_dev;
570 int ret;
571
572 cdx_dev = RTE_DEV_TO_CDX_DEV(dev);
573 ret = cdx_detach_dev(cdx_dev);
574 if (ret == 0) {
575 cdx_remove_device(cdx_dev);
576 rte_devargs_remove(dev->devargs);
577 free(cdx_dev);
578 }
579 return ret;
580 }
581
582 static int
cdx_dma_map(struct rte_device * dev,void * addr,uint64_t iova,size_t len)583 cdx_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
584 {
585 RTE_SET_USED(dev);
586
587 return rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD,
588 (uintptr_t)addr, iova, len);
589 }
590
591 static int
cdx_dma_unmap(struct rte_device * dev,void * addr,uint64_t iova,size_t len)592 cdx_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
593 {
594 RTE_SET_USED(dev);
595
596 return rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
597 (uintptr_t)addr, iova, len);
598 }
599
600 static enum rte_iova_mode
cdx_get_iommu_class(void)601 cdx_get_iommu_class(void)
602 {
603 if (TAILQ_EMPTY(&rte_cdx_bus.device_list))
604 return RTE_IOVA_DC;
605
606 return RTE_IOVA_VA;
607 }
608
609 static int
cdx_dev_match(const struct rte_device * dev,const void * _kvlist)610 cdx_dev_match(const struct rte_device *dev,
611 const void *_kvlist)
612 {
613 const struct rte_kvargs *kvlist = _kvlist;
614 const char *key = cdx_params_keys[RTE_CDX_PARAM_NAME];
615 const char *name;
616
617 /* no kvlist arg, all devices match */
618 if (kvlist == NULL)
619 return 0;
620
621 /* if key is present in kvlist and does not match, filter device */
622 name = rte_kvargs_get(kvlist, key);
623 if (name != NULL && strcmp(name, dev->name))
624 return -1;
625
626 return 0;
627 }
628
629 static void *
cdx_dev_iterate(const void * start,const char * str,const struct rte_dev_iterator * it __rte_unused)630 cdx_dev_iterate(const void *start,
631 const char *str,
632 const struct rte_dev_iterator *it __rte_unused)
633 {
634 rte_bus_find_device_t find_device;
635 struct rte_kvargs *kvargs = NULL;
636 struct rte_device *dev;
637
638 if (str != NULL) {
639 kvargs = rte_kvargs_parse(str, cdx_params_keys);
640 if (kvargs == NULL) {
641 CDX_BUS_ERR("cannot parse argument list %s", str);
642 rte_errno = EINVAL;
643 return NULL;
644 }
645 }
646 find_device = rte_cdx_bus.bus.find_device;
647 dev = find_device(start, cdx_dev_match, kvargs);
648 rte_kvargs_free(kvargs);
649 return dev;
650 }
651
652 struct rte_cdx_bus rte_cdx_bus = {
653 .bus = {
654 .scan = cdx_scan,
655 .probe = cdx_probe,
656 .find_device = cdx_find_device,
657 .plug = cdx_plug,
658 .unplug = cdx_unplug,
659 .parse = cdx_parse,
660 .dma_map = cdx_dma_map,
661 .dma_unmap = cdx_dma_unmap,
662 .get_iommu_class = cdx_get_iommu_class,
663 .dev_iterate = cdx_dev_iterate,
664 },
665 .device_list = TAILQ_HEAD_INITIALIZER(rte_cdx_bus.device_list),
666 .driver_list = TAILQ_HEAD_INITIALIZER(rte_cdx_bus.driver_list),
667 };
668
669 RTE_REGISTER_BUS(cdx, rte_cdx_bus.bus);
670 RTE_LOG_REGISTER_DEFAULT(cdx_logtype_bus, NOTICE);
671