1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright 2016,2018-2021 NXP 4 * 5 */ 6 7 #include <string.h> 8 #include <dirent.h> 9 #include <stdalign.h> 10 #include <stdbool.h> 11 12 #include <rte_log.h> 13 #include <bus_driver.h> 14 #include <rte_malloc.h> 15 #include <rte_devargs.h> 16 #include <rte_memcpy.h> 17 #include <ethdev_driver.h> 18 #include <rte_mbuf_dyn.h> 19 20 #include "private.h" 21 #include <fslmc_vfio.h> 22 #include "fslmc_logs.h" 23 24 #include <dpaax_iova_table.h> 25 26 #define VFIO_IOMMU_GROUP_PATH "/sys/kernel/iommu_groups" 27 #define FSLMC_BUS_NAME fslmc 28 29 struct rte_fslmc_bus rte_fslmc_bus; 30 31 #define DPAA2_SEQN_DYNFIELD_NAME "dpaa2_seqn_dynfield" 32 int dpaa2_seqn_dynfield_offset = -1; 33 34 uint32_t 35 rte_fslmc_get_device_count(enum rte_dpaa2_dev_type device_type) 36 { 37 if (device_type >= DPAA2_DEVTYPE_MAX) 38 return 0; 39 return rte_fslmc_bus.device_count[device_type]; 40 } 41 42 static void 43 cleanup_fslmc_device_list(void) 44 { 45 struct rte_dpaa2_device *dev; 46 struct rte_dpaa2_device *t_dev; 47 48 RTE_TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, t_dev) { 49 TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next); 50 rte_intr_instance_free(dev->intr_handle); 51 free(dev); 52 dev = NULL; 53 } 54 } 55 56 static int 57 compare_dpaa2_devname(struct rte_dpaa2_device *dev1, 58 struct rte_dpaa2_device *dev2) 59 { 60 int comp; 61 62 if (dev1->dev_type > dev2->dev_type) { 63 comp = 1; 64 } else if (dev1->dev_type < dev2->dev_type) { 65 comp = -1; 66 } else { 67 /* Check the ID as types match */ 68 if (dev1->object_id > dev2->object_id) 69 comp = 1; 70 else if (dev1->object_id < dev2->object_id) 71 comp = -1; 72 else 73 comp = 0; /* Duplicate device name */ 74 } 75 76 return comp; 77 } 78 79 static void 80 insert_in_device_list(struct rte_dpaa2_device *newdev) 81 { 82 int comp, inserted = 0; 83 struct rte_dpaa2_device *dev = NULL; 84 struct rte_dpaa2_device *tdev = NULL; 85 86 RTE_TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, tdev) { 87 comp = compare_dpaa2_devname(newdev, dev); 88 if (comp < 0) { 89 TAILQ_INSERT_BEFORE(dev, newdev, next); 90 inserted = 1; 91 break; 92 } 93 } 94 95 if (!inserted) 96 TAILQ_INSERT_TAIL(&rte_fslmc_bus.device_list, newdev, next); 97 } 98 99 static struct rte_devargs * 100 fslmc_devargs_lookup(struct rte_dpaa2_device *dev) 101 { 102 struct rte_devargs *devargs; 103 char dev_name[32]; 104 105 RTE_EAL_DEVARGS_FOREACH("fslmc", devargs) { 106 devargs->bus->parse(devargs->name, &dev_name); 107 if (strcmp(dev_name, dev->device.name) == 0) { 108 DPAA2_BUS_INFO("**Devargs matched %s", dev_name); 109 return devargs; 110 } 111 } 112 return NULL; 113 } 114 115 static void 116 dump_device_list(void) 117 { 118 struct rte_dpaa2_device *dev; 119 120 /* Only if the log level has been set to Debugging, print list */ 121 if (rte_log_can_log(dpaa2_logtype_bus, RTE_LOG_DEBUG)) { 122 DPAA2_BUS_LOG(DEBUG, "List of devices scanned on bus:"); 123 TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { 124 DPAA2_BUS_LOG(DEBUG, "\t\t%s", dev->device.name); 125 } 126 } 127 } 128 129 static int 130 scan_one_fslmc_device(char *dev_name) 131 { 132 char *dup_dev_name, *t_ptr; 133 struct rte_dpaa2_device *dev = NULL; 134 int ret = -1; 135 136 if (!dev_name) 137 return ret; 138 139 /* Creating a temporary copy to perform cut-parse over string */ 140 dup_dev_name = strdup(dev_name); 141 if (!dup_dev_name) { 142 DPAA2_BUS_ERR("Unable to allocate device name memory"); 143 return -ENOMEM; 144 } 145 146 /* For all other devices, we allocate rte_dpaa2_device. 147 * For those devices where there is no driver, probe would release 148 * the memory associated with the rte_dpaa2_device after necessary 149 * initialization. 150 */ 151 dev = calloc(1, sizeof(struct rte_dpaa2_device)); 152 if (!dev) { 153 DPAA2_BUS_ERR("Unable to allocate device object"); 154 free(dup_dev_name); 155 return -ENOMEM; 156 } 157 158 dev->device.bus = &rte_fslmc_bus.bus; 159 dev->device.numa_node = SOCKET_ID_ANY; 160 161 /* Allocate interrupt instance */ 162 dev->intr_handle = 163 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE); 164 if (dev->intr_handle == NULL) { 165 DPAA2_BUS_ERR("Failed to allocate intr handle"); 166 ret = -ENOMEM; 167 goto cleanup; 168 } 169 170 /* Parse the device name and ID */ 171 t_ptr = strtok(dup_dev_name, "."); 172 if (!t_ptr) { 173 DPAA2_BUS_ERR("Invalid device found: (%s)", dup_dev_name); 174 ret = -EINVAL; 175 goto cleanup; 176 } 177 if (!strncmp("dpni", t_ptr, 4)) 178 dev->dev_type = DPAA2_ETH; 179 else if (!strncmp("dpseci", t_ptr, 6)) 180 dev->dev_type = DPAA2_CRYPTO; 181 else if (!strncmp("dpcon", t_ptr, 5)) 182 dev->dev_type = DPAA2_CON; 183 else if (!strncmp("dpbp", t_ptr, 4)) 184 dev->dev_type = DPAA2_BPOOL; 185 else if (!strncmp("dpio", t_ptr, 4)) 186 dev->dev_type = DPAA2_IO; 187 else if (!strncmp("dpci", t_ptr, 4)) 188 dev->dev_type = DPAA2_CI; 189 else if (!strncmp("dpmcp", t_ptr, 5)) 190 dev->dev_type = DPAA2_MPORTAL; 191 else if (!strncmp("dpdmai", t_ptr, 6)) 192 dev->dev_type = DPAA2_QDMA; 193 else if (!strncmp("dpdmux", t_ptr, 6)) 194 dev->dev_type = DPAA2_MUX; 195 else if (!strncmp("dprtc", t_ptr, 5)) 196 dev->dev_type = DPAA2_DPRTC; 197 else if (!strncmp("dprc", t_ptr, 4)) 198 dev->dev_type = DPAA2_DPRC; 199 else 200 dev->dev_type = DPAA2_UNKNOWN; 201 202 t_ptr = strtok(NULL, "."); 203 if (!t_ptr) { 204 DPAA2_BUS_ERR("Skipping invalid device (%s)", dup_dev_name); 205 ret = 0; 206 goto cleanup; 207 } 208 209 sscanf(t_ptr, "%hu", &dev->object_id); 210 dev->device.name = strdup(dev_name); 211 if (!dev->device.name) { 212 DPAA2_BUS_ERR("Unable to clone device name. Out of memory"); 213 ret = -ENOMEM; 214 goto cleanup; 215 } 216 dev->device.devargs = fslmc_devargs_lookup(dev); 217 218 /* Update the device found into the device_count table */ 219 rte_fslmc_bus.device_count[dev->dev_type]++; 220 221 /* Add device in the fslmc device list */ 222 insert_in_device_list(dev); 223 224 /* Don't need the duplicated device filesystem entry anymore */ 225 free(dup_dev_name); 226 227 return 0; 228 cleanup: 229 free(dup_dev_name); 230 if (dev) { 231 rte_intr_instance_free(dev->intr_handle); 232 free(dev); 233 } 234 return ret; 235 } 236 237 static int 238 rte_fslmc_parse(const char *name, void *addr) 239 { 240 uint16_t dev_id; 241 char *t_ptr; 242 const char *sep; 243 uint8_t sep_exists = 0; 244 int ret = -1; 245 246 /* There are multiple ways this can be called, with bus:dev, name=dev 247 * or just dev. In all cases, the 'addr' is actually a string. 248 */ 249 sep = strchr(name, ':'); 250 if (!sep) { 251 /* check for '=' */ 252 sep = strchr(name, '='); 253 if (!sep) 254 sep_exists = 0; 255 else 256 sep_exists = 1; 257 } else 258 sep_exists = 1; 259 260 /* Check if starting part if either of 'fslmc:' or 'name=', separator 261 * exists. 262 */ 263 if (sep_exists) { 264 /* If either of "fslmc" or "name" are starting part */ 265 if (!strncmp(name, RTE_STR(FSLMC_BUS_NAME), 266 strlen(RTE_STR(FSLMC_BUS_NAME))) || 267 (!strncmp(name, "name", strlen("name")))) { 268 goto jump_out; 269 } else { 270 DPAA2_BUS_DEBUG("Invalid device for matching (%s).", 271 name); 272 ret = -EINVAL; 273 goto err_out; 274 } 275 } else 276 sep = name; 277 278 jump_out: 279 /* Validate device name */ 280 if (strncmp("dpni", sep, 4) && 281 strncmp("dpseci", sep, 6) && 282 strncmp("dpcon", sep, 5) && 283 strncmp("dpbp", sep, 4) && 284 strncmp("dpio", sep, 4) && 285 strncmp("dpci", sep, 4) && 286 strncmp("dpmcp", sep, 5) && 287 strncmp("dpdmai", sep, 6) && 288 strncmp("dpdmux", sep, 6)) { 289 DPAA2_BUS_DEBUG("Unknown or unsupported device (%s)", sep); 290 ret = -EINVAL; 291 goto err_out; 292 } 293 294 t_ptr = strchr(sep, '.'); 295 if (!t_ptr || sscanf(t_ptr + 1, "%hu", &dev_id) != 1) { 296 DPAA2_BUS_ERR("Missing device id in device name (%s)", sep); 297 ret = -EINVAL; 298 goto err_out; 299 } 300 301 if (addr) 302 strcpy(addr, sep); 303 304 ret = 0; 305 err_out: 306 return ret; 307 } 308 309 static int 310 rte_fslmc_scan(void) 311 { 312 int ret; 313 char fslmc_dirpath[PATH_MAX]; 314 DIR *dir; 315 struct dirent *entry; 316 static int process_once; 317 int groupid; 318 char *group_name; 319 320 if (process_once) { 321 DPAA2_BUS_DEBUG("Fslmc bus already scanned. Not rescanning"); 322 return 0; 323 } 324 process_once = 1; 325 326 /* Now we only support single group per process.*/ 327 group_name = getenv("DPRC"); 328 if (!group_name) { 329 DPAA2_BUS_DEBUG("DPAA2: DPRC not available"); 330 ret = -EINVAL; 331 goto scan_fail; 332 } 333 334 ret = fslmc_get_container_group(group_name, &groupid); 335 if (ret != 0) 336 goto scan_fail; 337 338 /* Scan devices on the group */ 339 sprintf(fslmc_dirpath, "%s/%s", SYSFS_FSL_MC_DEVICES, group_name); 340 dir = opendir(fslmc_dirpath); 341 if (!dir) { 342 DPAA2_BUS_ERR("Unable to open VFIO group directory"); 343 goto scan_fail; 344 } 345 346 /* Scan the DPRC container object */ 347 ret = scan_one_fslmc_device(group_name); 348 if (ret != 0) { 349 /* Error in parsing directory - exit gracefully */ 350 goto scan_fail_cleanup; 351 } 352 353 while ((entry = readdir(dir)) != NULL) { 354 if (entry->d_name[0] == '.' || entry->d_type != DT_DIR) 355 continue; 356 357 ret = scan_one_fslmc_device(entry->d_name); 358 if (ret != 0) { 359 /* Error in parsing directory - exit gracefully */ 360 goto scan_fail_cleanup; 361 } 362 } 363 364 closedir(dir); 365 366 DPAA2_BUS_INFO("FSLMC Bus scan completed"); 367 /* If debugging is enabled, device list is dumped to log output */ 368 dump_device_list(); 369 370 return 0; 371 372 scan_fail_cleanup: 373 closedir(dir); 374 375 /* Remove all devices in the list */ 376 cleanup_fslmc_device_list(); 377 scan_fail: 378 DPAA2_BUS_DEBUG("FSLMC Bus Not Available. Skipping (%d)", ret); 379 /* Irrespective of failure, scan only return success */ 380 return 0; 381 } 382 383 static int 384 rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv, 385 struct rte_dpaa2_device *dpaa2_dev) 386 { 387 if (dpaa2_drv->drv_type == dpaa2_dev->dev_type) 388 return 0; 389 390 return 1; 391 } 392 393 static int 394 rte_fslmc_close(void) 395 { 396 int ret = 0; 397 398 ret = fslmc_vfio_close_group(); 399 if (ret) 400 DPAA2_BUS_INFO("Unable to close devices %d", ret); 401 402 return 0; 403 } 404 405 static int 406 rte_fslmc_probe(void) 407 { 408 int ret = 0; 409 int probe_all; 410 411 struct rte_dpaa2_device *dev; 412 struct rte_dpaa2_driver *drv; 413 414 static const struct rte_mbuf_dynfield dpaa2_seqn_dynfield_desc = { 415 .name = DPAA2_SEQN_DYNFIELD_NAME, 416 .size = sizeof(dpaa2_seqn_t), 417 .align = alignof(dpaa2_seqn_t), 418 }; 419 420 if (TAILQ_EMPTY(&rte_fslmc_bus.device_list)) 421 return 0; 422 423 dpaa2_seqn_dynfield_offset = 424 rte_mbuf_dynfield_register(&dpaa2_seqn_dynfield_desc); 425 if (dpaa2_seqn_dynfield_offset < 0) { 426 DPAA2_BUS_ERR("Failed to register mbuf field for dpaa sequence number"); 427 return 0; 428 } 429 430 ret = fslmc_vfio_setup_group(); 431 if (ret) { 432 DPAA2_BUS_ERR("Unable to setup VFIO %d", ret); 433 return 0; 434 } 435 436 /* Map existing segments as well as, in case of hotpluggable memory, 437 * install callback handler. 438 */ 439 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 440 ret = fslmc_vfio_dmamap(); 441 if (ret) { 442 DPAA2_BUS_ERR("Unable to DMA map existing VAs: (%d)", 443 ret); 444 /* Not continuing ahead */ 445 DPAA2_BUS_ERR("FSLMC VFIO Mapping failed"); 446 return 0; 447 } 448 } 449 450 ret = fslmc_vfio_process_group(); 451 if (ret) { 452 DPAA2_BUS_ERR("Unable to setup devices %d", ret); 453 return 0; 454 } 455 456 probe_all = rte_fslmc_bus.bus.conf.scan_mode != RTE_BUS_SCAN_ALLOWLIST; 457 458 TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { 459 TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) { 460 ret = rte_fslmc_match(drv, dev); 461 if (ret) 462 continue; 463 464 if (!drv->probe) 465 continue; 466 467 if (rte_dev_is_probed(&dev->device)) 468 continue; 469 470 if (dev->device.devargs && 471 dev->device.devargs->policy == RTE_DEV_BLOCKED) { 472 DPAA2_BUS_LOG(DEBUG, "%s Blocked, skipping", 473 dev->device.name); 474 continue; 475 } 476 477 if (probe_all || 478 (dev->device.devargs && 479 dev->device.devargs->policy == RTE_DEV_ALLOWED)) { 480 ret = drv->probe(drv, dev); 481 if (ret) { 482 DPAA2_BUS_ERR("Unable to probe"); 483 } else { 484 dev->driver = drv; 485 dev->device.driver = &drv->driver; 486 } 487 } 488 break; 489 } 490 } 491 492 return 0; 493 } 494 495 static struct rte_device * 496 rte_fslmc_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, 497 const void *data) 498 { 499 const struct rte_dpaa2_device *dstart; 500 struct rte_dpaa2_device *dev; 501 502 DPAA2_BUS_DEBUG("Finding a device named %s", (const char *)data); 503 504 /* find_device is always called with an opaque object which should be 505 * passed along to the 'cmp' function iterating over all device obj 506 * on the bus. 507 */ 508 509 if (start != NULL) { 510 dstart = RTE_DEV_TO_FSLMC_CONST(start); 511 dev = TAILQ_NEXT(dstart, next); 512 } else { 513 dev = TAILQ_FIRST(&rte_fslmc_bus.device_list); 514 } 515 while (dev != NULL) { 516 if (cmp(&dev->device, data) == 0) { 517 DPAA2_BUS_DEBUG("Found device (%s)", 518 dev->device.name); 519 return &dev->device; 520 } 521 dev = TAILQ_NEXT(dev, next); 522 } 523 524 return NULL; 525 } 526 527 /*register a fslmc bus based dpaa2 driver */ 528 void 529 rte_fslmc_driver_register(struct rte_dpaa2_driver *driver) 530 { 531 RTE_VERIFY(driver); 532 533 TAILQ_INSERT_TAIL(&rte_fslmc_bus.driver_list, driver, next); 534 } 535 536 /*un-register a fslmc bus based dpaa2 driver */ 537 void 538 rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver) 539 { 540 TAILQ_REMOVE(&rte_fslmc_bus.driver_list, driver, next); 541 } 542 543 /* 544 * All device has iova as va 545 */ 546 static inline int 547 fslmc_all_device_support_iova(void) 548 { 549 int ret = 0; 550 struct rte_dpaa2_device *dev; 551 struct rte_dpaa2_driver *drv; 552 553 TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { 554 TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) { 555 ret = rte_fslmc_match(drv, dev); 556 if (ret) 557 continue; 558 /* if the driver is not supporting IOVA */ 559 if (!(drv->drv_flags & RTE_DPAA2_DRV_IOVA_AS_VA)) 560 return 0; 561 } 562 } 563 return 1; 564 } 565 566 /* 567 * Get iommu class of DPAA2 devices on the bus. 568 */ 569 static enum rte_iova_mode 570 rte_dpaa2_get_iommu_class(void) 571 { 572 bool is_vfio_noiommu_enabled = 1; 573 bool has_iova_va; 574 575 if (rte_eal_iova_mode() == RTE_IOVA_PA) 576 return RTE_IOVA_PA; 577 578 if (TAILQ_EMPTY(&rte_fslmc_bus.device_list)) 579 return RTE_IOVA_DC; 580 581 /* check if all devices on the bus support Virtual addressing or not */ 582 has_iova_va = fslmc_all_device_support_iova(); 583 584 #ifdef VFIO_PRESENT 585 is_vfio_noiommu_enabled = rte_vfio_noiommu_is_enabled() == true ? 586 true : false; 587 #endif 588 589 if (has_iova_va && !is_vfio_noiommu_enabled) 590 return RTE_IOVA_VA; 591 592 return RTE_IOVA_PA; 593 } 594 595 static int 596 fslmc_bus_plug(struct rte_device *dev __rte_unused) 597 { 598 /* No operation is performed while plugging the device */ 599 return 0; 600 } 601 602 static int 603 fslmc_bus_unplug(struct rte_device *dev __rte_unused) 604 { 605 /* No operation is performed while unplugging the device */ 606 return 0; 607 } 608 609 static void * 610 fslmc_bus_dev_iterate(const void *start, const char *str, 611 const struct rte_dev_iterator *it __rte_unused) 612 { 613 const struct rte_dpaa2_device *dstart; 614 struct rte_dpaa2_device *dev; 615 char *dup, *dev_name = NULL; 616 617 if (str == NULL) { 618 DPAA2_BUS_DEBUG("No device string"); 619 return NULL; 620 } 621 622 /* Expectation is that device would be name=device_name */ 623 if (strncmp(str, "name=", 5) != 0) { 624 DPAA2_BUS_DEBUG("Invalid device string (%s)", str); 625 return NULL; 626 } 627 628 /* Now that name=device_name format is available, split */ 629 dup = strdup(str); 630 if (dup == NULL) { 631 DPAA2_BUS_DEBUG("Dup string (%s) failed!", str); 632 return NULL; 633 } 634 dev_name = dup + strlen("name="); 635 636 if (start != NULL) { 637 dstart = RTE_DEV_TO_FSLMC_CONST(start); 638 dev = TAILQ_NEXT(dstart, next); 639 } else { 640 dev = TAILQ_FIRST(&rte_fslmc_bus.device_list); 641 } 642 643 while (dev != NULL) { 644 if (strcmp(dev->device.name, dev_name) == 0) { 645 free(dup); 646 return &dev->device; 647 } 648 dev = TAILQ_NEXT(dev, next); 649 } 650 651 free(dup); 652 return NULL; 653 } 654 655 struct rte_fslmc_bus rte_fslmc_bus = { 656 .bus = { 657 .scan = rte_fslmc_scan, 658 .probe = rte_fslmc_probe, 659 .cleanup = rte_fslmc_close, 660 .parse = rte_fslmc_parse, 661 .find_device = rte_fslmc_find_device, 662 .get_iommu_class = rte_dpaa2_get_iommu_class, 663 .plug = fslmc_bus_plug, 664 .unplug = fslmc_bus_unplug, 665 .dev_iterate = fslmc_bus_dev_iterate, 666 }, 667 .device_list = TAILQ_HEAD_INITIALIZER(rte_fslmc_bus.device_list), 668 .driver_list = TAILQ_HEAD_INITIALIZER(rte_fslmc_bus.driver_list), 669 .device_count = {0}, 670 }; 671 672 RTE_REGISTER_BUS(FSLMC_BUS_NAME, rte_fslmc_bus.bus); 673 RTE_LOG_REGISTER_DEFAULT(dpaa2_logtype_bus, NOTICE); 674