1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies Ltd 3 */ 4 5 #include <stdlib.h> 6 #include <rte_malloc.h> 7 #include <rte_class.h> 8 9 #include "mlx5_common_log.h" 10 #include "mlx5_common_pci.h" 11 12 struct mlx5_pci_device { 13 struct rte_pci_device *pci_dev; 14 TAILQ_ENTRY(mlx5_pci_device) next; 15 uint32_t classes_loaded; 16 }; 17 18 /* Head of list of drivers. */ 19 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list = 20 TAILQ_HEAD_INITIALIZER(drv_list); 21 22 /* Head of mlx5 pci devices. */ 23 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list = 24 TAILQ_HEAD_INITIALIZER(devices_list); 25 26 static const struct { 27 const char *name; 28 unsigned int driver_class; 29 } mlx5_classes[] = { 30 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA }, 31 { .name = "eth", .driver_class = MLX5_CLASS_ETH }, 32 /* Keep name "net" for backward compatibility. */ 33 { .name = "net", .driver_class = MLX5_CLASS_ETH }, 34 { .name = "regex", .driver_class = MLX5_CLASS_REGEX }, 35 { .name = "compress", .driver_class = MLX5_CLASS_COMPRESS }, 36 { .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO }, 37 }; 38 39 static const unsigned int mlx5_class_combinations[] = { 40 MLX5_CLASS_ETH, 41 MLX5_CLASS_VDPA, 42 MLX5_CLASS_REGEX, 43 MLX5_CLASS_COMPRESS, 44 MLX5_CLASS_CRYPTO, 45 MLX5_CLASS_ETH | MLX5_CLASS_REGEX, 46 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX, 47 MLX5_CLASS_ETH | MLX5_CLASS_COMPRESS, 48 MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS, 49 MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 50 MLX5_CLASS_ETH | MLX5_CLASS_CRYPTO, 51 MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 52 MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO, 53 MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 54 MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 55 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 56 MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 57 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 58 MLX5_CLASS_ETH | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 59 MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 60 MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS | 61 MLX5_CLASS_CRYPTO, 62 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS | 63 MLX5_CLASS_CRYPTO, 64 /* New class combination should be added here. */ 65 }; 66 67 static int 68 class_name_to_value(const char *class_name) 69 { 70 unsigned int i; 71 72 for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 73 if (strcmp(class_name, mlx5_classes[i].name) == 0) 74 return mlx5_classes[i].driver_class; 75 } 76 return -EINVAL; 77 } 78 79 static struct mlx5_pci_driver * 80 driver_get(uint32_t class) 81 { 82 struct mlx5_pci_driver *driver; 83 84 TAILQ_FOREACH(driver, &drv_list, next) { 85 if (driver->driver_class == class) 86 return driver; 87 } 88 return NULL; 89 } 90 91 static int 92 bus_cmdline_options_handler(__rte_unused const char *key, 93 const char *class_names, void *opaque) 94 { 95 int *ret = opaque; 96 char *nstr_org; 97 int class_val; 98 char *found; 99 char *nstr; 100 char *refstr = NULL; 101 102 *ret = 0; 103 nstr = strdup(class_names); 104 if (!nstr) { 105 *ret = -ENOMEM; 106 return *ret; 107 } 108 nstr_org = nstr; 109 found = strtok_r(nstr, ":", &refstr); 110 if (!found) 111 goto err; 112 do { 113 /* Extract each individual class name. Multiple 114 * class key,value is supplied as class=net:vdpa:foo:bar. 115 */ 116 class_val = class_name_to_value(found); 117 /* Check if its a valid class. */ 118 if (class_val < 0) { 119 *ret = -EINVAL; 120 goto err; 121 } 122 *ret |= class_val; 123 found = strtok_r(NULL, ":", &refstr); 124 } while (found); 125 err: 126 free(nstr_org); 127 if (*ret < 0) 128 DRV_LOG(ERR, "Invalid mlx5 class options %s." 129 " Maybe typo in device class argument setting?", 130 class_names); 131 return *ret; 132 } 133 134 static int 135 parse_class_options(const struct rte_devargs *devargs) 136 { 137 const char *key = RTE_DEVARGS_KEY_CLASS; 138 struct rte_kvargs *kvlist; 139 int ret = 0; 140 141 if (devargs == NULL) 142 return 0; 143 kvlist = rte_kvargs_parse(devargs->args, NULL); 144 if (kvlist == NULL) 145 return 0; 146 if (rte_kvargs_count(kvlist, key)) 147 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler, 148 &ret); 149 rte_kvargs_free(kvlist); 150 return ret; 151 } 152 153 static bool 154 mlx5_bus_match(const struct mlx5_pci_driver *drv, 155 const struct rte_pci_device *pci_dev) 156 { 157 const struct rte_pci_id *id_table; 158 159 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0; 160 id_table++) { 161 /* Check if device's ids match the class driver's ids. */ 162 if (id_table->vendor_id != pci_dev->id.vendor_id && 163 id_table->vendor_id != RTE_PCI_ANY_ID) 164 continue; 165 if (id_table->device_id != pci_dev->id.device_id && 166 id_table->device_id != RTE_PCI_ANY_ID) 167 continue; 168 if (id_table->subsystem_vendor_id != 169 pci_dev->id.subsystem_vendor_id && 170 id_table->subsystem_vendor_id != RTE_PCI_ANY_ID) 171 continue; 172 if (id_table->subsystem_device_id != 173 pci_dev->id.subsystem_device_id && 174 id_table->subsystem_device_id != RTE_PCI_ANY_ID) 175 continue; 176 if (id_table->class_id != pci_dev->id.class_id && 177 id_table->class_id != RTE_CLASS_ANY_ID) 178 continue; 179 return true; 180 } 181 return false; 182 } 183 184 static int 185 is_valid_class_combination(uint32_t user_classes) 186 { 187 unsigned int i; 188 189 /* Verify if user specified valid supported combination. */ 190 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) { 191 if (mlx5_class_combinations[i] == user_classes) 192 return 0; 193 } 194 /* Not found any valid class combination. */ 195 return -EINVAL; 196 } 197 198 static struct mlx5_pci_device * 199 pci_to_mlx5_device(const struct rte_pci_device *pci_dev) 200 { 201 struct mlx5_pci_device *dev; 202 203 TAILQ_FOREACH(dev, &devices_list, next) { 204 if (dev->pci_dev == pci_dev) 205 return dev; 206 } 207 return NULL; 208 } 209 210 static bool 211 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class) 212 { 213 return (device->classes_loaded & class) ? true : false; 214 } 215 216 static void 217 dev_release(struct mlx5_pci_device *dev) 218 { 219 TAILQ_REMOVE(&devices_list, dev, next); 220 rte_free(dev); 221 } 222 223 static int 224 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes) 225 { 226 struct mlx5_pci_driver *driver; 227 int local_ret = -ENODEV; 228 unsigned int i = 0; 229 int ret = 0; 230 231 enabled_classes &= dev->classes_loaded; 232 while (enabled_classes) { 233 driver = driver_get(RTE_BIT64(i)); 234 if (driver) { 235 local_ret = driver->pci_driver.remove(dev->pci_dev); 236 if (!local_ret) 237 dev->classes_loaded &= ~RTE_BIT64(i); 238 else if (ret == 0) 239 ret = local_ret; 240 } 241 enabled_classes &= ~RTE_BIT64(i); 242 i++; 243 } 244 if (local_ret) 245 ret = local_ret; 246 return ret; 247 } 248 249 static int 250 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv, 251 struct rte_pci_device *pci_dev, uint32_t user_classes) 252 { 253 struct mlx5_pci_driver *driver; 254 uint32_t enabled_classes = 0; 255 bool already_loaded; 256 int ret; 257 258 TAILQ_FOREACH(driver, &drv_list, next) { 259 if ((driver->driver_class & user_classes) == 0) 260 continue; 261 if (!mlx5_bus_match(driver, pci_dev)) 262 continue; 263 already_loaded = dev->classes_loaded & driver->driver_class; 264 if (already_loaded && 265 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) { 266 DRV_LOG(ERR, "Device %s is already probed", 267 pci_dev->device.name); 268 ret = -EEXIST; 269 goto probe_err; 270 } 271 ret = driver->pci_driver.probe(pci_drv, pci_dev); 272 if (ret < 0) { 273 DRV_LOG(ERR, "Failed to load driver %s", 274 driver->pci_driver.driver.name); 275 goto probe_err; 276 } 277 enabled_classes |= driver->driver_class; 278 } 279 dev->classes_loaded |= enabled_classes; 280 return 0; 281 probe_err: 282 /* Only unload drivers which are enabled which were enabled 283 * in this probe instance. 284 */ 285 drivers_remove(dev, enabled_classes); 286 return ret; 287 } 288 289 /** 290 * DPDK callback to register to probe multiple drivers for a PCI device. 291 * 292 * @param[in] pci_drv 293 * PCI driver structure. 294 * @param[in] dev 295 * PCI device information. 296 * 297 * @return 298 * 0 on success, a negative errno value otherwise and rte_errno is set. 299 */ 300 static int 301 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 302 struct rte_pci_device *pci_dev) 303 { 304 struct mlx5_pci_device *dev; 305 uint32_t user_classes = 0; 306 bool new_device = false; 307 int ret; 308 309 ret = parse_class_options(pci_dev->device.devargs); 310 if (ret < 0) 311 return ret; 312 user_classes = ret; 313 if (user_classes) { 314 /* Validate combination here. */ 315 ret = is_valid_class_combination(user_classes); 316 if (ret) { 317 DRV_LOG(ERR, "Unsupported mlx5 classes supplied."); 318 return ret; 319 } 320 } else { 321 /* Default to net class. */ 322 user_classes = MLX5_CLASS_ETH; 323 } 324 dev = pci_to_mlx5_device(pci_dev); 325 if (!dev) { 326 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0); 327 if (!dev) 328 return -ENOMEM; 329 dev->pci_dev = pci_dev; 330 TAILQ_INSERT_HEAD(&devices_list, dev, next); 331 new_device = true; 332 } 333 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes); 334 if (ret) 335 goto class_err; 336 return 0; 337 class_err: 338 if (new_device) 339 dev_release(dev); 340 return ret; 341 } 342 343 /** 344 * DPDK callback to remove one or more drivers for a PCI device. 345 * 346 * This function removes all drivers probed for a given PCI device. 347 * 348 * @param[in] pci_dev 349 * Pointer to the PCI device. 350 * 351 * @return 352 * 0 on success, the function cannot fail. 353 */ 354 static int 355 mlx5_common_pci_remove(struct rte_pci_device *pci_dev) 356 { 357 struct mlx5_pci_device *dev; 358 int ret; 359 360 dev = pci_to_mlx5_device(pci_dev); 361 if (!dev) 362 return -ENODEV; 363 /* Matching device found, cleanup and unload drivers. */ 364 ret = drivers_remove(dev, dev->classes_loaded); 365 if (!ret) 366 dev_release(dev); 367 return ret; 368 } 369 370 static int 371 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr, 372 uint64_t iova, size_t len) 373 { 374 struct mlx5_pci_driver *driver = NULL; 375 struct mlx5_pci_driver *temp; 376 struct mlx5_pci_device *dev; 377 int ret = -EINVAL; 378 379 dev = pci_to_mlx5_device(pci_dev); 380 if (!dev) 381 return -ENODEV; 382 TAILQ_FOREACH(driver, &drv_list, next) { 383 if (device_class_enabled(dev, driver->driver_class) && 384 driver->pci_driver.dma_map) { 385 ret = driver->pci_driver.dma_map(pci_dev, addr, 386 iova, len); 387 if (ret) 388 goto map_err; 389 } 390 } 391 return ret; 392 map_err: 393 TAILQ_FOREACH(temp, &drv_list, next) { 394 if (temp == driver) 395 break; 396 if (device_class_enabled(dev, temp->driver_class) && 397 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap) 398 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len); 399 } 400 return ret; 401 } 402 403 static int 404 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr, 405 uint64_t iova, size_t len) 406 { 407 struct mlx5_pci_driver *driver; 408 struct mlx5_pci_device *dev; 409 int local_ret = -EINVAL; 410 int ret; 411 412 dev = pci_to_mlx5_device(pci_dev); 413 if (!dev) 414 return -ENODEV; 415 ret = 0; 416 /* There is no unmap error recovery in current implementation. */ 417 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) { 418 if (device_class_enabled(dev, driver->driver_class) && 419 driver->pci_driver.dma_unmap) { 420 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr, 421 iova, len); 422 if (local_ret && (ret == 0)) 423 ret = local_ret; 424 } 425 } 426 if (local_ret) 427 ret = local_ret; 428 return ret; 429 } 430 431 /* PCI ID table is build dynamically based on registered mlx5 drivers. */ 432 static struct rte_pci_id *mlx5_pci_id_table; 433 434 static struct rte_pci_driver mlx5_pci_driver = { 435 .driver = { 436 .name = MLX5_PCI_DRIVER_NAME, 437 }, 438 .probe = mlx5_common_pci_probe, 439 .remove = mlx5_common_pci_remove, 440 .dma_map = mlx5_common_pci_dma_map, 441 .dma_unmap = mlx5_common_pci_dma_unmap, 442 }; 443 444 static int 445 pci_id_table_size_get(const struct rte_pci_id *id_table) 446 { 447 int table_size = 0; 448 449 for (; id_table->vendor_id != 0; id_table++) 450 table_size++; 451 return table_size; 452 } 453 454 static bool 455 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table, 456 int next_idx) 457 { 458 int current_size = next_idx - 1; 459 int i; 460 461 for (i = 0; i < current_size; i++) { 462 if (id->device_id == table[i].device_id && 463 id->vendor_id == table[i].vendor_id && 464 id->subsystem_vendor_id == table[i].subsystem_vendor_id && 465 id->subsystem_device_id == table[i].subsystem_device_id) 466 return true; 467 } 468 return false; 469 } 470 471 static void 472 pci_id_insert(struct rte_pci_id *new_table, int *next_idx, 473 const struct rte_pci_id *id_table) 474 { 475 /* Traverse the id_table, check if entry exists in new_table; 476 * Add non duplicate entries to new table. 477 */ 478 for (; id_table->vendor_id != 0; id_table++) { 479 if (!pci_id_exists(id_table, new_table, *next_idx)) { 480 /* New entry; add to the table. */ 481 new_table[*next_idx] = *id_table; 482 (*next_idx)++; 483 } 484 } 485 } 486 487 static int 488 pci_ids_table_update(const struct rte_pci_id *driver_id_table) 489 { 490 const struct rte_pci_id *id_iter; 491 struct rte_pci_id *updated_table; 492 struct rte_pci_id *old_table; 493 int num_ids = 0; 494 int i = 0; 495 496 old_table = mlx5_pci_id_table; 497 if (old_table) 498 num_ids = pci_id_table_size_get(old_table); 499 num_ids += pci_id_table_size_get(driver_id_table); 500 /* Increase size by one for the termination entry of vendor_id = 0. */ 501 num_ids += 1; 502 updated_table = calloc(num_ids, sizeof(*updated_table)); 503 if (!updated_table) 504 return -ENOMEM; 505 if (TAILQ_EMPTY(&drv_list)) { 506 /* Copy the first driver's ID table. */ 507 for (id_iter = driver_id_table; id_iter->vendor_id != 0; 508 id_iter++, i++) 509 updated_table[i] = *id_iter; 510 } else { 511 /* First copy existing table entries. */ 512 for (id_iter = old_table; id_iter->vendor_id != 0; 513 id_iter++, i++) 514 updated_table[i] = *id_iter; 515 /* New id to be added at the end of current ID table. */ 516 pci_id_insert(updated_table, &i, driver_id_table); 517 } 518 /* Terminate table with empty entry. */ 519 updated_table[i].vendor_id = 0; 520 mlx5_pci_driver.id_table = updated_table; 521 mlx5_pci_id_table = updated_table; 522 if (old_table) 523 free(old_table); 524 return 0; 525 } 526 527 void 528 mlx5_pci_driver_register(struct mlx5_pci_driver *driver) 529 { 530 int ret; 531 532 ret = pci_ids_table_update(driver->pci_driver.id_table); 533 if (ret) 534 return; 535 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags; 536 TAILQ_INSERT_TAIL(&drv_list, driver, next); 537 } 538 539 void mlx5_common_pci_init(void) 540 { 541 const struct rte_pci_id empty_table[] = { 542 { 543 .vendor_id = 0 544 }, 545 }; 546 547 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD 548 * including this one can register the PCI table first. If any other 549 * PMD(s) have registered the PCI ID table, No need to register an empty 550 * default one. 551 */ 552 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table)) 553 return; 554 rte_pci_register(&mlx5_pci_driver); 555 } 556 557 RTE_FINI(mlx5_common_pci_finish) 558 { 559 if (mlx5_pci_id_table != NULL) { 560 /* Constructor doesn't register with PCI bus if it failed 561 * to build the table. 562 */ 563 rte_pci_unregister(&mlx5_pci_driver); 564 free(mlx5_pci_id_table); 565 } 566 } 567 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__); 568