1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies Ltd 3 */ 4 5 #include <stdlib.h> 6 7 #include <rte_malloc.h> 8 #include <rte_devargs.h> 9 #include <rte_errno.h> 10 #include <rte_class.h> 11 12 #include "mlx5_common_log.h" 13 #include "mlx5_common_pci.h" 14 #include "mlx5_common_private.h" 15 16 static struct rte_pci_driver mlx5_common_pci_driver; 17 18 /********** Legacy PCI bus driver, to be removed ********/ 19 20 struct mlx5_pci_device { 21 struct rte_pci_device *pci_dev; 22 TAILQ_ENTRY(mlx5_pci_device) next; 23 uint32_t classes_loaded; 24 }; 25 26 /* Head of list of drivers. */ 27 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list = 28 TAILQ_HEAD_INITIALIZER(drv_list); 29 30 /* Head of mlx5 pci devices. */ 31 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list = 32 TAILQ_HEAD_INITIALIZER(devices_list); 33 34 static const struct { 35 const char *name; 36 unsigned int driver_class; 37 } mlx5_classes[] = { 38 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA }, 39 { .name = "eth", .driver_class = MLX5_CLASS_ETH }, 40 /* Keep name "net" for backward compatibility. */ 41 { .name = "net", .driver_class = MLX5_CLASS_ETH }, 42 { .name = "regex", .driver_class = MLX5_CLASS_REGEX }, 43 { .name = "compress", .driver_class = MLX5_CLASS_COMPRESS }, 44 { .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO }, 45 }; 46 47 static const unsigned int mlx5_class_combinations[] = { 48 MLX5_CLASS_ETH, 49 MLX5_CLASS_VDPA, 50 MLX5_CLASS_REGEX, 51 MLX5_CLASS_COMPRESS, 52 MLX5_CLASS_CRYPTO, 53 MLX5_CLASS_ETH | MLX5_CLASS_REGEX, 54 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX, 55 MLX5_CLASS_ETH | MLX5_CLASS_COMPRESS, 56 MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS, 57 MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 58 MLX5_CLASS_ETH | MLX5_CLASS_CRYPTO, 59 MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 60 MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO, 61 MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 62 MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 63 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 64 MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 65 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 66 MLX5_CLASS_ETH | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 67 MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 68 MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS | 69 MLX5_CLASS_CRYPTO, 70 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS | 71 MLX5_CLASS_CRYPTO, 72 /* New class combination should be added here. */ 73 }; 74 75 static int 76 class_name_to_value(const char *class_name) 77 { 78 unsigned int i; 79 80 for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 81 if (strcmp(class_name, mlx5_classes[i].name) == 0) 82 return mlx5_classes[i].driver_class; 83 } 84 return -EINVAL; 85 } 86 87 static struct mlx5_pci_driver * 88 driver_get(uint32_t class) 89 { 90 struct mlx5_pci_driver *driver; 91 92 TAILQ_FOREACH(driver, &drv_list, next) { 93 if (driver->driver_class == class) 94 return driver; 95 } 96 return NULL; 97 } 98 99 static int 100 bus_cmdline_options_handler(__rte_unused const char *key, 101 const char *class_names, void *opaque) 102 { 103 int *ret = opaque; 104 char *nstr_org; 105 int class_val; 106 char *found; 107 char *nstr; 108 char *refstr = NULL; 109 110 *ret = 0; 111 nstr = strdup(class_names); 112 if (!nstr) { 113 *ret = -ENOMEM; 114 return *ret; 115 } 116 nstr_org = nstr; 117 found = strtok_r(nstr, ":", &refstr); 118 if (!found) 119 goto err; 120 do { 121 /* Extract each individual class name. Multiple 122 * class key,value is supplied as class=net:vdpa:foo:bar. 123 */ 124 class_val = class_name_to_value(found); 125 /* Check if its a valid class. */ 126 if (class_val < 0) { 127 *ret = -EINVAL; 128 goto err; 129 } 130 *ret |= class_val; 131 found = strtok_r(NULL, ":", &refstr); 132 } while (found); 133 err: 134 free(nstr_org); 135 if (*ret < 0) 136 DRV_LOG(ERR, "Invalid mlx5 class options %s." 137 " Maybe typo in device class argument setting?", 138 class_names); 139 return *ret; 140 } 141 142 static int 143 parse_class_options(const struct rte_devargs *devargs) 144 { 145 const char *key = RTE_DEVARGS_KEY_CLASS; 146 struct rte_kvargs *kvlist; 147 int ret = 0; 148 149 if (devargs == NULL) 150 return 0; 151 kvlist = rte_kvargs_parse(devargs->args, NULL); 152 if (kvlist == NULL) 153 return 0; 154 if (rte_kvargs_count(kvlist, key)) 155 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler, 156 &ret); 157 rte_kvargs_free(kvlist); 158 return ret; 159 } 160 161 static bool 162 mlx5_bus_match(const struct mlx5_pci_driver *drv, 163 const struct rte_pci_device *pci_dev) 164 { 165 const struct rte_pci_id *id_table; 166 167 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0; 168 id_table++) { 169 /* Check if device's ids match the class driver's ids. */ 170 if (id_table->vendor_id != pci_dev->id.vendor_id && 171 id_table->vendor_id != RTE_PCI_ANY_ID) 172 continue; 173 if (id_table->device_id != pci_dev->id.device_id && 174 id_table->device_id != RTE_PCI_ANY_ID) 175 continue; 176 if (id_table->subsystem_vendor_id != 177 pci_dev->id.subsystem_vendor_id && 178 id_table->subsystem_vendor_id != RTE_PCI_ANY_ID) 179 continue; 180 if (id_table->subsystem_device_id != 181 pci_dev->id.subsystem_device_id && 182 id_table->subsystem_device_id != RTE_PCI_ANY_ID) 183 continue; 184 if (id_table->class_id != pci_dev->id.class_id && 185 id_table->class_id != RTE_CLASS_ANY_ID) 186 continue; 187 return true; 188 } 189 return false; 190 } 191 192 static int 193 is_valid_class_combination(uint32_t user_classes) 194 { 195 unsigned int i; 196 197 /* Verify if user specified valid supported combination. */ 198 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) { 199 if (mlx5_class_combinations[i] == user_classes) 200 return 0; 201 } 202 /* Not found any valid class combination. */ 203 return -EINVAL; 204 } 205 206 static struct mlx5_pci_device * 207 pci_to_mlx5_device(const struct rte_pci_device *pci_dev) 208 { 209 struct mlx5_pci_device *dev; 210 211 TAILQ_FOREACH(dev, &devices_list, next) { 212 if (dev->pci_dev == pci_dev) 213 return dev; 214 } 215 return NULL; 216 } 217 218 static bool 219 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class) 220 { 221 return (device->classes_loaded & class) ? true : false; 222 } 223 224 static void 225 dev_release(struct mlx5_pci_device *dev) 226 { 227 TAILQ_REMOVE(&devices_list, dev, next); 228 rte_free(dev); 229 } 230 231 static int 232 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes) 233 { 234 struct mlx5_pci_driver *driver; 235 int local_ret = -ENODEV; 236 unsigned int i = 0; 237 int ret = 0; 238 239 enabled_classes &= dev->classes_loaded; 240 while (enabled_classes) { 241 driver = driver_get(RTE_BIT64(i)); 242 if (driver) { 243 local_ret = driver->pci_driver.remove(dev->pci_dev); 244 if (!local_ret) 245 dev->classes_loaded &= ~RTE_BIT64(i); 246 else if (ret == 0) 247 ret = local_ret; 248 } 249 enabled_classes &= ~RTE_BIT64(i); 250 i++; 251 } 252 if (local_ret) 253 ret = local_ret; 254 return ret; 255 } 256 257 static int 258 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv, 259 struct rte_pci_device *pci_dev, uint32_t user_classes) 260 { 261 struct mlx5_pci_driver *driver; 262 uint32_t enabled_classes = 0; 263 bool already_loaded; 264 int ret; 265 266 TAILQ_FOREACH(driver, &drv_list, next) { 267 if ((driver->driver_class & user_classes) == 0) 268 continue; 269 if (!mlx5_bus_match(driver, pci_dev)) 270 continue; 271 already_loaded = dev->classes_loaded & driver->driver_class; 272 if (already_loaded && 273 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) { 274 DRV_LOG(ERR, "Device %s is already probed", 275 pci_dev->device.name); 276 ret = -EEXIST; 277 goto probe_err; 278 } 279 ret = driver->pci_driver.probe(pci_drv, pci_dev); 280 if (ret < 0) { 281 DRV_LOG(ERR, "Failed to load driver %s", 282 driver->pci_driver.driver.name); 283 goto probe_err; 284 } 285 enabled_classes |= driver->driver_class; 286 } 287 dev->classes_loaded |= enabled_classes; 288 return 0; 289 probe_err: 290 /* Only unload drivers which are enabled which were enabled 291 * in this probe instance. 292 */ 293 drivers_remove(dev, enabled_classes); 294 return ret; 295 } 296 297 /** 298 * DPDK callback to register to probe multiple drivers for a PCI device. 299 * 300 * @param[in] pci_drv 301 * PCI driver structure. 302 * @param[in] dev 303 * PCI device information. 304 * 305 * @return 306 * 0 on success, a negative errno value otherwise and rte_errno is set. 307 */ 308 static int 309 mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 310 struct rte_pci_device *pci_dev) 311 { 312 struct mlx5_pci_device *dev; 313 uint32_t user_classes = 0; 314 bool new_device = false; 315 int ret; 316 317 ret = parse_class_options(pci_dev->device.devargs); 318 if (ret < 0) 319 return ret; 320 user_classes = ret; 321 if (user_classes) { 322 /* Validate combination here. */ 323 ret = is_valid_class_combination(user_classes); 324 if (ret) { 325 DRV_LOG(ERR, "Unsupported mlx5 classes supplied."); 326 return ret; 327 } 328 } else { 329 /* Default to net class. */ 330 user_classes = MLX5_CLASS_ETH; 331 } 332 dev = pci_to_mlx5_device(pci_dev); 333 if (!dev) { 334 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0); 335 if (!dev) 336 return -ENOMEM; 337 dev->pci_dev = pci_dev; 338 TAILQ_INSERT_HEAD(&devices_list, dev, next); 339 new_device = true; 340 } 341 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes); 342 if (ret) 343 goto class_err; 344 return 0; 345 class_err: 346 if (new_device) 347 dev_release(dev); 348 return ret; 349 } 350 351 /** 352 * DPDK callback to remove one or more drivers for a PCI device. 353 * 354 * This function removes all drivers probed for a given PCI device. 355 * 356 * @param[in] pci_dev 357 * Pointer to the PCI device. 358 * 359 * @return 360 * 0 on success, the function cannot fail. 361 */ 362 static int 363 mlx5_pci_remove(struct rte_pci_device *pci_dev) 364 { 365 struct mlx5_pci_device *dev; 366 int ret; 367 368 dev = pci_to_mlx5_device(pci_dev); 369 if (!dev) 370 return -ENODEV; 371 /* Matching device found, cleanup and unload drivers. */ 372 ret = drivers_remove(dev, dev->classes_loaded); 373 if (!ret) 374 dev_release(dev); 375 return ret; 376 } 377 378 static int 379 mlx5_pci_dma_map(struct rte_pci_device *pci_dev, void *addr, 380 uint64_t iova, size_t len) 381 { 382 struct mlx5_pci_driver *driver = NULL; 383 struct mlx5_pci_driver *temp; 384 struct mlx5_pci_device *dev; 385 int ret = -EINVAL; 386 387 dev = pci_to_mlx5_device(pci_dev); 388 if (!dev) 389 return -ENODEV; 390 TAILQ_FOREACH(driver, &drv_list, next) { 391 if (device_class_enabled(dev, driver->driver_class) && 392 driver->pci_driver.dma_map) { 393 ret = driver->pci_driver.dma_map(pci_dev, addr, 394 iova, len); 395 if (ret) 396 goto map_err; 397 } 398 } 399 return ret; 400 map_err: 401 TAILQ_FOREACH(temp, &drv_list, next) { 402 if (temp == driver) 403 break; 404 if (device_class_enabled(dev, temp->driver_class) && 405 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap) 406 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len); 407 } 408 return ret; 409 } 410 411 static int 412 mlx5_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr, 413 uint64_t iova, size_t len) 414 { 415 struct mlx5_pci_driver *driver; 416 struct mlx5_pci_device *dev; 417 int local_ret = -EINVAL; 418 int ret; 419 420 dev = pci_to_mlx5_device(pci_dev); 421 if (!dev) 422 return -ENODEV; 423 ret = 0; 424 /* There is no unmap error recovery in current implementation. */ 425 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) { 426 if (device_class_enabled(dev, driver->driver_class) && 427 driver->pci_driver.dma_unmap) { 428 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr, 429 iova, len); 430 if (local_ret && (ret == 0)) 431 ret = local_ret; 432 } 433 } 434 if (local_ret) 435 ret = local_ret; 436 return ret; 437 } 438 439 /* PCI ID table is build dynamically based on registered mlx5 drivers. */ 440 static struct rte_pci_id *mlx5_pci_id_table; 441 442 static struct rte_pci_driver mlx5_pci_driver = { 443 .driver = { 444 .name = MLX5_PCI_DRIVER_NAME, 445 }, 446 .probe = mlx5_pci_probe, 447 .remove = mlx5_pci_remove, 448 .dma_map = mlx5_pci_dma_map, 449 .dma_unmap = mlx5_pci_dma_unmap, 450 }; 451 452 static int 453 pci_id_table_size_get(const struct rte_pci_id *id_table) 454 { 455 int table_size = 0; 456 457 for (; id_table->vendor_id != 0; id_table++) 458 table_size++; 459 return table_size; 460 } 461 462 static bool 463 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table, 464 int next_idx) 465 { 466 int current_size = next_idx - 1; 467 int i; 468 469 for (i = 0; i < current_size; i++) { 470 if (id->device_id == table[i].device_id && 471 id->vendor_id == table[i].vendor_id && 472 id->subsystem_vendor_id == table[i].subsystem_vendor_id && 473 id->subsystem_device_id == table[i].subsystem_device_id) 474 return true; 475 } 476 return false; 477 } 478 479 static void 480 pci_id_insert(struct rte_pci_id *new_table, int *next_idx, 481 const struct rte_pci_id *id_table) 482 { 483 /* Traverse the id_table, check if entry exists in new_table; 484 * Add non duplicate entries to new table. 485 */ 486 for (; id_table->vendor_id != 0; id_table++) { 487 if (!pci_id_exists(id_table, new_table, *next_idx)) { 488 /* New entry; add to the table. */ 489 new_table[*next_idx] = *id_table; 490 (*next_idx)++; 491 } 492 } 493 } 494 495 static int 496 pci_ids_table_update(const struct rte_pci_id *driver_id_table) 497 { 498 const struct rte_pci_id *id_iter; 499 struct rte_pci_id *updated_table; 500 struct rte_pci_id *old_table; 501 int num_ids = 0; 502 int i = 0; 503 504 old_table = mlx5_pci_id_table; 505 if (old_table) 506 num_ids = pci_id_table_size_get(old_table); 507 num_ids += pci_id_table_size_get(driver_id_table); 508 /* Increase size by one for the termination entry of vendor_id = 0. */ 509 num_ids += 1; 510 updated_table = calloc(num_ids, sizeof(*updated_table)); 511 if (!updated_table) 512 return -ENOMEM; 513 if (old_table == NULL) { 514 /* Copy the first driver's ID table. */ 515 for (id_iter = driver_id_table; id_iter->vendor_id != 0; 516 id_iter++, i++) 517 updated_table[i] = *id_iter; 518 } else { 519 /* First copy existing table entries. */ 520 for (id_iter = old_table; id_iter->vendor_id != 0; 521 id_iter++, i++) 522 updated_table[i] = *id_iter; 523 /* New id to be added at the end of current ID table. */ 524 pci_id_insert(updated_table, &i, driver_id_table); 525 } 526 /* Terminate table with empty entry. */ 527 updated_table[i].vendor_id = 0; 528 mlx5_pci_driver.id_table = updated_table; 529 mlx5_common_pci_driver.id_table = updated_table; 530 mlx5_pci_id_table = updated_table; 531 if (old_table) 532 free(old_table); 533 return 0; 534 } 535 536 void 537 mlx5_pci_driver_register(struct mlx5_pci_driver *driver) 538 { 539 int ret; 540 541 ret = pci_ids_table_update(driver->pci_driver.id_table); 542 if (ret) 543 return; 544 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags; 545 TAILQ_INSERT_TAIL(&drv_list, driver, next); 546 } 547 548 /********** New common PCI bus driver ********/ 549 550 bool 551 mlx5_dev_is_pci(const struct rte_device *dev) 552 { 553 return strcmp(dev->bus->name, "pci") == 0; 554 } 555 556 bool 557 mlx5_dev_pci_match(const struct mlx5_class_driver *drv, 558 const struct rte_device *dev) 559 { 560 const struct rte_pci_device *pci_dev; 561 const struct rte_pci_id *id_table; 562 563 if (!mlx5_dev_is_pci(dev)) 564 return false; 565 pci_dev = RTE_DEV_TO_PCI_CONST(dev); 566 for (id_table = drv->id_table; id_table->vendor_id != 0; 567 id_table++) { 568 /* Check if device's ids match the class driver's ids. */ 569 if (id_table->vendor_id != pci_dev->id.vendor_id && 570 id_table->vendor_id != RTE_PCI_ANY_ID) 571 continue; 572 if (id_table->device_id != pci_dev->id.device_id && 573 id_table->device_id != RTE_PCI_ANY_ID) 574 continue; 575 if (id_table->subsystem_vendor_id != 576 pci_dev->id.subsystem_vendor_id && 577 id_table->subsystem_vendor_id != RTE_PCI_ANY_ID) 578 continue; 579 if (id_table->subsystem_device_id != 580 pci_dev->id.subsystem_device_id && 581 id_table->subsystem_device_id != RTE_PCI_ANY_ID) 582 continue; 583 if (id_table->class_id != pci_dev->id.class_id && 584 id_table->class_id != RTE_CLASS_ANY_ID) 585 continue; 586 return true; 587 } 588 return false; 589 } 590 591 static int 592 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 593 struct rte_pci_device *pci_dev) 594 { 595 return mlx5_common_dev_probe(&pci_dev->device); 596 } 597 598 static int 599 mlx5_common_pci_remove(struct rte_pci_device *pci_dev) 600 { 601 return mlx5_common_dev_remove(&pci_dev->device); 602 } 603 604 static int 605 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr, 606 uint64_t iova, size_t len) 607 { 608 return mlx5_common_dev_dma_map(&pci_dev->device, addr, iova, len); 609 } 610 611 static int 612 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr, 613 uint64_t iova, size_t len) 614 { 615 return mlx5_common_dev_dma_unmap(&pci_dev->device, addr, iova, len); 616 } 617 618 void 619 mlx5_common_driver_on_register_pci(struct mlx5_class_driver *driver) 620 { 621 if (driver->id_table != NULL) { 622 if (pci_ids_table_update(driver->id_table) != 0) 623 return; 624 } 625 if (driver->probe_again) 626 mlx5_common_pci_driver.drv_flags |= RTE_PCI_DRV_PROBE_AGAIN; 627 if (driver->intr_lsc) 628 mlx5_common_pci_driver.drv_flags |= RTE_PCI_DRV_INTR_LSC; 629 if (driver->intr_rmv) 630 mlx5_common_pci_driver.drv_flags |= RTE_PCI_DRV_INTR_RMV; 631 } 632 633 static struct rte_pci_driver mlx5_common_pci_driver = { 634 .driver = { 635 .name = MLX5_PCI_DRIVER_NAME, 636 }, 637 .probe = mlx5_common_pci_probe, 638 .remove = mlx5_common_pci_remove, 639 .dma_map = mlx5_common_pci_dma_map, 640 .dma_unmap = mlx5_common_pci_dma_unmap, 641 }; 642 643 void mlx5_common_pci_init(void) 644 { 645 const struct rte_pci_id empty_table[] = { 646 { 647 .vendor_id = 0 648 }, 649 }; 650 651 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD 652 * including this one can register the PCI table first. If any other 653 * PMD(s) have registered the PCI ID table, No need to register an empty 654 * default one. 655 */ 656 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table)) 657 return; 658 rte_pci_register(&mlx5_common_pci_driver); 659 } 660 661 RTE_FINI(mlx5_common_pci_finish) 662 { 663 if (mlx5_pci_id_table != NULL) { 664 /* Constructor doesn't register with PCI bus if it failed 665 * to build the table. 666 */ 667 rte_pci_unregister(&mlx5_common_pci_driver); 668 free(mlx5_pci_id_table); 669 } 670 } 671 672 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__); 673