1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2023 Marvell. 3 */ 4 5 #include <dirent.h> 6 #include <inttypes.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <sys/ioctl.h> 10 #include <sys/mman.h> 11 #include <sys/queue.h> 12 #include <unistd.h> 13 14 #include <bus_driver.h> 15 #include <bus_platform_driver.h> 16 #include <eal_filesystem.h> 17 #include <rte_bus.h> 18 #include <rte_devargs.h> 19 #include <rte_errno.h> 20 #include <rte_log.h> 21 #include <rte_memory.h> 22 #include <rte_string_fns.h> 23 #include <rte_vfio.h> 24 25 #include "private.h" 26 27 #ifdef VFIO_PRESENT 28 29 #define PLATFORM_BUS_DEVICES_PATH "/sys/bus/platform/devices" 30 31 void 32 rte_platform_register(struct rte_platform_driver *pdrv) 33 { 34 TAILQ_INSERT_TAIL(&platform_bus.driver_list, pdrv, next); 35 } 36 37 void 38 rte_platform_unregister(struct rte_platform_driver *pdrv) 39 { 40 TAILQ_REMOVE(&platform_bus.driver_list, pdrv, next); 41 } 42 43 static struct rte_devargs * 44 dev_devargs(const char *dev_name) 45 { 46 struct rte_devargs *devargs; 47 48 RTE_EAL_DEVARGS_FOREACH("platform", devargs) { 49 if (!strcmp(devargs->name, dev_name)) 50 return devargs; 51 } 52 53 return NULL; 54 } 55 56 static bool 57 dev_allowed(const char *dev_name) 58 { 59 struct rte_devargs *devargs; 60 61 devargs = dev_devargs(dev_name); 62 if (devargs == NULL) 63 return true; 64 65 switch (platform_bus.bus.conf.scan_mode) { 66 case RTE_BUS_SCAN_UNDEFINED: 67 case RTE_BUS_SCAN_ALLOWLIST: 68 if (devargs->policy == RTE_DEV_ALLOWED) 69 return true; 70 break; 71 case RTE_BUS_SCAN_BLOCKLIST: 72 if (devargs->policy == RTE_DEV_BLOCKED) 73 return false; 74 break; 75 } 76 77 return true; 78 } 79 80 static int 81 dev_add(const char *dev_name) 82 { 83 struct rte_platform_device *pdev, *tmp; 84 char path[PATH_MAX]; 85 unsigned long val; 86 87 pdev = calloc(1, sizeof(*pdev)); 88 if (pdev == NULL) 89 return -ENOMEM; 90 91 rte_strscpy(pdev->name, dev_name, sizeof(pdev->name)); 92 pdev->device.name = pdev->name; 93 pdev->device.devargs = dev_devargs(dev_name); 94 pdev->device.bus = &platform_bus.bus; 95 snprintf(path, sizeof(path), PLATFORM_BUS_DEVICES_PATH "/%s/numa_node", dev_name); 96 pdev->device.numa_node = eal_parse_sysfs_value(path, &val) ? rte_socket_id() : val; 97 98 FOREACH_DEVICE_ON_PLATFORM_BUS(tmp) { 99 if (!strcmp(tmp->name, pdev->name)) { 100 PLATFORM_LOG(INFO, "device %s already added\n", pdev->name); 101 102 if (tmp->device.devargs != pdev->device.devargs) 103 rte_devargs_remove(pdev->device.devargs); 104 105 free(pdev); 106 } 107 } 108 109 TAILQ_INSERT_HEAD(&platform_bus.device_list, pdev, next); 110 111 PLATFORM_LOG(INFO, "adding device %s to the list\n", dev_name); 112 113 return 0; 114 } 115 116 static char * 117 dev_kernel_driver_name(const char *dev_name) 118 { 119 char path[PATH_MAX], buf[BUFSIZ] = { }; 120 char *kdrv; 121 int ret; 122 123 snprintf(path, sizeof(path), PLATFORM_BUS_DEVICES_PATH "/%s/driver", dev_name); 124 /* save space for NUL */ 125 ret = readlink(path, buf, sizeof(buf) - 1); 126 if (ret <= 0) 127 return NULL; 128 129 /* last token is kernel driver name */ 130 kdrv = strrchr(buf, '/'); 131 if (kdrv != NULL) 132 return strdup(kdrv + 1); 133 134 return NULL; 135 } 136 137 static bool 138 dev_is_bound_vfio_platform(const char *dev_name) 139 { 140 char *kdrv; 141 int ret; 142 143 kdrv = dev_kernel_driver_name(dev_name); 144 if (!kdrv) 145 return false; 146 147 ret = strcmp(kdrv, "vfio-platform"); 148 free(kdrv); 149 150 return ret == 0; 151 } 152 153 static int 154 platform_bus_scan(void) 155 { 156 const struct dirent *ent; 157 const char *dev_name; 158 int ret = 0; 159 DIR *dp; 160 161 dp = opendir(PLATFORM_BUS_DEVICES_PATH); 162 if (dp == NULL) { 163 PLATFORM_LOG(INFO, "failed to open %s\n", PLATFORM_BUS_DEVICES_PATH); 164 return -errno; 165 } 166 167 while ((ent = readdir(dp))) { 168 dev_name = ent->d_name; 169 if (dev_name[0] == '.') 170 continue; 171 172 if (!dev_allowed(dev_name)) 173 continue; 174 175 if (!dev_is_bound_vfio_platform(dev_name)) 176 continue; 177 178 ret = dev_add(dev_name); 179 if (ret) 180 break; 181 } 182 183 closedir(dp); 184 185 return ret; 186 } 187 188 static int 189 device_map_resource_offset(struct rte_platform_device *pdev, struct rte_platform_resource *res, 190 size_t offset) 191 { 192 res->mem.addr = mmap(NULL, res->mem.len, PROT_READ | PROT_WRITE, MAP_SHARED, pdev->dev_fd, 193 offset); 194 if (res->mem.addr == MAP_FAILED) 195 return -errno; 196 197 PLATFORM_LOG(DEBUG, "adding resource va = %p len = %"PRIu64" name = %s\n", res->mem.addr, 198 res->mem.len, res->name); 199 200 return 0; 201 } 202 203 static void 204 device_unmap_resources(struct rte_platform_device *pdev) 205 { 206 struct rte_platform_resource *res; 207 unsigned int i; 208 209 for (i = 0; i < pdev->num_resource; i++) { 210 res = &pdev->resource[i]; 211 munmap(res->mem.addr, res->mem.len); 212 free(res->name); 213 } 214 215 free(pdev->resource); 216 pdev->resource = NULL; 217 pdev->num_resource = 0; 218 } 219 220 static int 221 read_sysfs_string(const char *path, char *buf, size_t size) 222 { 223 FILE *f; 224 char *p; 225 226 f = fopen(path, "r"); 227 if (f == NULL) 228 return -errno; 229 230 if (fgets(buf, size, f) == NULL) { 231 fclose(f); 232 return -ENODATA; 233 } 234 235 fclose(f); 236 237 p = strrchr(buf, '\n'); 238 if (p != NULL) 239 *p = '\0'; 240 241 return 0; 242 } 243 244 static char * 245 of_resource_name(const char *dev_name, int index) 246 { 247 char path[PATH_MAX], buf[BUFSIZ] = { }; 248 int num = 0, ret; 249 char *name; 250 251 snprintf(path, sizeof(path), PLATFORM_BUS_DEVICES_PATH "/%s/of_node/reg-names", dev_name); 252 ret = read_sysfs_string(path, buf, sizeof(buf) - 1); 253 if (ret) 254 return NULL; 255 256 for (name = buf; *name != 0; name += strlen(name) + 1) { 257 if (num++ != index) 258 continue; 259 return strdup(name); 260 } 261 262 return NULL; 263 } 264 265 static int 266 device_map_resources(struct rte_platform_device *pdev, unsigned int num) 267 { 268 struct rte_platform_resource *res; 269 unsigned int i; 270 int ret; 271 272 if (num == 0) { 273 PLATFORM_LOG(WARNING, "device %s has no resources\n", pdev->name); 274 return 0; 275 } 276 277 pdev->resource = calloc(num, sizeof(*pdev->resource)); 278 if (pdev->resource == NULL) 279 return -ENOMEM; 280 281 for (i = 0; i < num; i++) { 282 struct vfio_region_info reg_info = { 283 .argsz = sizeof(reg_info), 284 .index = i, 285 }; 286 287 ret = ioctl(pdev->dev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info); 288 if (ret) { 289 PLATFORM_LOG(ERR, "failed to get region info at %d\n", i); 290 ret = -errno; 291 goto out; 292 } 293 294 res = &pdev->resource[i]; 295 res->name = of_resource_name(pdev->name, reg_info.index); 296 res->mem.len = reg_info.size; 297 ret = device_map_resource_offset(pdev, res, reg_info.offset); 298 if (ret) { 299 PLATFORM_LOG(ERR, "failed to ioremap resource at %d\n", i); 300 goto out; 301 } 302 303 pdev->num_resource++; 304 } 305 306 return 0; 307 out: 308 device_unmap_resources(pdev); 309 310 return ret; 311 } 312 313 static void 314 device_cleanup(struct rte_platform_device *pdev) 315 { 316 device_unmap_resources(pdev); 317 rte_vfio_release_device(PLATFORM_BUS_DEVICES_PATH, pdev->name, pdev->dev_fd); 318 } 319 320 static int 321 device_setup(struct rte_platform_device *pdev) 322 { 323 struct vfio_device_info dev_info = { .argsz = sizeof(dev_info), }; 324 const char *name = pdev->name; 325 int ret; 326 327 ret = rte_vfio_setup_device(PLATFORM_BUS_DEVICES_PATH, name, &pdev->dev_fd, &dev_info); 328 if (ret) { 329 PLATFORM_LOG(ERR, "failed to setup %s\n", name); 330 return -ENODEV; 331 } 332 333 /* This is an extra check to confirm that platform device was initialized 334 * by a kernel vfio-platform driver. On kernels that predate vfio-platform 335 * driver this flag obviously does not exist. In such scenarios this 336 * check needs to be removed otherwise compilation fails. 337 * 338 * Now, on such old kernels code will never reach here because 339 * there is another check much earlier which verifies whether 340 * device has been bound to vfio-platform driver. 341 */ 342 #ifdef VFIO_DEVICE_FLAGS_PLATFORM 343 if (!(dev_info.flags & VFIO_DEVICE_FLAGS_PLATFORM)) { 344 PLATFORM_LOG(ERR, "device not backed by vfio-platform\n"); 345 ret = -ENOTSUP; 346 goto out; 347 } 348 #endif 349 350 ret = device_map_resources(pdev, dev_info.num_regions); 351 if (ret) { 352 PLATFORM_LOG(ERR, "failed to setup platform resources\n"); 353 goto out; 354 } 355 356 return 0; 357 out: 358 device_cleanup(pdev); 359 360 return ret; 361 } 362 363 static int 364 driver_call_probe(struct rte_platform_driver *pdrv, struct rte_platform_device *pdev) 365 { 366 int ret; 367 368 if (rte_dev_is_probed(&pdev->device)) 369 return -EBUSY; 370 371 if (pdrv->probe != NULL) { 372 pdev->driver = pdrv; 373 ret = pdrv->probe(pdev); 374 if (ret) 375 return ret; 376 } 377 378 pdev->device.driver = &pdrv->driver; 379 380 return 0; 381 } 382 383 static int 384 driver_probe_device(struct rte_platform_driver *pdrv, struct rte_platform_device *pdev) 385 { 386 enum rte_iova_mode iova_mode; 387 int ret; 388 389 iova_mode = rte_eal_iova_mode(); 390 if (pdrv->drv_flags & RTE_PLATFORM_DRV_NEED_IOVA_AS_VA && iova_mode != RTE_IOVA_VA) { 391 PLATFORM_LOG(ERR, "driver %s expects VA IOVA mode but current mode is PA\n", 392 pdrv->driver.name); 393 return -EINVAL; 394 } 395 396 ret = device_setup(pdev); 397 if (ret) 398 return ret; 399 400 ret = driver_call_probe(pdrv, pdev); 401 if (ret) 402 device_cleanup(pdev); 403 404 return ret; 405 } 406 407 static bool 408 driver_match_device(struct rte_platform_driver *pdrv, struct rte_platform_device *pdev) 409 { 410 bool match = false; 411 char *kdrv; 412 413 kdrv = dev_kernel_driver_name(pdev->name); 414 if (!kdrv) 415 return false; 416 417 /* match by driver name */ 418 if (!strcmp(kdrv, pdrv->driver.name)) { 419 match = true; 420 goto out; 421 } 422 423 /* match by driver alias */ 424 if (pdrv->driver.alias != NULL && !strcmp(kdrv, pdrv->driver.alias)) { 425 match = true; 426 goto out; 427 } 428 429 /* match by device name */ 430 if (!strcmp(pdev->name, pdrv->driver.name)) 431 match = true; 432 433 out: 434 free(kdrv); 435 436 return match; 437 } 438 439 static int 440 device_attach(struct rte_platform_device *pdev) 441 { 442 struct rte_platform_driver *pdrv; 443 444 FOREACH_DRIVER_ON_PLATFORM_BUS(pdrv) { 445 if (driver_match_device(pdrv, pdev)) 446 break; 447 } 448 449 if (pdrv == NULL) 450 return -ENODEV; 451 452 return driver_probe_device(pdrv, pdev); 453 } 454 455 static int 456 platform_bus_probe(void) 457 { 458 struct rte_platform_device *pdev; 459 int ret; 460 461 FOREACH_DEVICE_ON_PLATFORM_BUS(pdev) { 462 ret = device_attach(pdev); 463 if (ret == -EBUSY) { 464 PLATFORM_LOG(DEBUG, "device %s already probed\n", pdev->name); 465 continue; 466 } 467 if (ret) 468 PLATFORM_LOG(ERR, "failed to probe %s\n", pdev->name); 469 } 470 471 return 0; 472 } 473 474 static struct rte_device * 475 platform_bus_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, const void *data) 476 { 477 struct rte_platform_device *pdev; 478 479 pdev = start ? RTE_TAILQ_NEXT(RTE_DEV_TO_PLATFORM_DEV_CONST(start), next) : 480 RTE_TAILQ_FIRST(&platform_bus.device_list); 481 while (pdev) { 482 if (cmp(&pdev->device, data) == 0) 483 return &pdev->device; 484 485 pdev = RTE_TAILQ_NEXT(pdev, next); 486 } 487 488 return NULL; 489 } 490 491 static int 492 platform_bus_plug(struct rte_device *dev) 493 { 494 struct rte_platform_device *pdev; 495 496 if (!dev_allowed(dev->name)) 497 return -EPERM; 498 499 if (!dev_is_bound_vfio_platform(dev->name)) 500 return -EPERM; 501 502 pdev = RTE_DEV_TO_PLATFORM_DEV(dev); 503 if (pdev == NULL) 504 return -EINVAL; 505 506 return device_attach(pdev); 507 } 508 509 static void 510 device_release_driver(struct rte_platform_device *pdev) 511 { 512 struct rte_platform_driver *pdrv; 513 int ret; 514 515 pdrv = pdev->driver; 516 if (pdrv != NULL && pdrv->remove != NULL) { 517 ret = pdrv->remove(pdev); 518 if (ret) 519 PLATFORM_LOG(WARNING, "failed to remove %s\n", pdev->name); 520 } 521 522 pdev->device.driver = NULL; 523 pdev->driver = NULL; 524 } 525 526 static int 527 platform_bus_unplug(struct rte_device *dev) 528 { 529 struct rte_platform_device *pdev; 530 531 pdev = RTE_DEV_TO_PLATFORM_DEV(dev); 532 if (pdev == NULL) 533 return -EINVAL; 534 535 device_release_driver(pdev); 536 device_cleanup(pdev); 537 rte_devargs_remove(pdev->device.devargs); 538 free(pdev); 539 540 return 0; 541 } 542 543 static int 544 platform_bus_parse(const char *name, void *addr) 545 { 546 struct rte_platform_device pdev = { }; 547 struct rte_platform_driver *pdrv; 548 const char **out = addr; 549 550 rte_strscpy(pdev.name, name, sizeof(pdev.name)); 551 552 FOREACH_DRIVER_ON_PLATFORM_BUS(pdrv) { 553 if (driver_match_device(pdrv, &pdev)) 554 break; 555 } 556 557 if (pdrv != NULL && addr != NULL) 558 *out = name; 559 560 return pdrv != NULL ? 0 : -ENODEV; 561 } 562 563 static int 564 platform_bus_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 565 { 566 struct rte_platform_device *pdev; 567 568 pdev = RTE_DEV_TO_PLATFORM_DEV(dev); 569 if (pdev == NULL || pdev->driver == NULL) { 570 rte_errno = EINVAL; 571 return -1; 572 } 573 574 if (pdev->driver->dma_map != NULL) 575 return pdev->driver->dma_map(pdev, addr, iova, len); 576 577 return rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD, (uint64_t)addr, iova, len); 578 } 579 580 static int 581 platform_bus_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) 582 { 583 struct rte_platform_device *pdev; 584 585 pdev = RTE_DEV_TO_PLATFORM_DEV(dev); 586 if (pdev == NULL || pdev->driver == NULL) { 587 rte_errno = EINVAL; 588 return -1; 589 } 590 591 if (pdev->driver->dma_unmap != NULL) 592 return pdev->driver->dma_unmap(pdev, addr, iova, len); 593 594 return rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD, (uint64_t)addr, iova, 595 len); 596 } 597 598 static enum rte_iova_mode 599 platform_bus_get_iommu_class(void) 600 { 601 struct rte_platform_driver *pdrv; 602 struct rte_platform_device *pdev; 603 604 FOREACH_DEVICE_ON_PLATFORM_BUS(pdev) { 605 pdrv = pdev->driver; 606 if (pdrv != NULL && pdrv->drv_flags & RTE_PLATFORM_DRV_NEED_IOVA_AS_VA) 607 return RTE_IOVA_VA; 608 } 609 610 return RTE_IOVA_DC; 611 } 612 613 static int 614 platform_bus_cleanup(void) 615 { 616 struct rte_platform_device *pdev, *tmp; 617 618 RTE_TAILQ_FOREACH_SAFE(pdev, &platform_bus.device_list, next, tmp) { 619 platform_bus_unplug(&pdev->device); 620 TAILQ_REMOVE(&platform_bus.device_list, pdev, next); 621 } 622 623 return 0; 624 } 625 626 struct rte_platform_bus platform_bus = { 627 .bus = { 628 .scan = platform_bus_scan, 629 .probe = platform_bus_probe, 630 .find_device = platform_bus_find_device, 631 .plug = platform_bus_plug, 632 .unplug = platform_bus_unplug, 633 .parse = platform_bus_parse, 634 .dma_map = platform_bus_dma_map, 635 .dma_unmap = platform_bus_dma_unmap, 636 .get_iommu_class = platform_bus_get_iommu_class, 637 .dev_iterate = platform_bus_dev_iterate, 638 .cleanup = platform_bus_cleanup, 639 }, 640 .device_list = TAILQ_HEAD_INITIALIZER(platform_bus.device_list), 641 .driver_list = TAILQ_HEAD_INITIALIZER(platform_bus.driver_list), 642 }; 643 644 RTE_REGISTER_BUS(platform, platform_bus.bus); 645 RTE_LOG_REGISTER_DEFAULT(platform_bus_logtype, NOTICE); 646 647 #endif /* VFIO_PRESENT */ 648