1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies Ltd 3 */ 4 5 #include <stdlib.h> 6 #include <rte_malloc.h> 7 #include <rte_class.h> 8 9 #include "mlx5_common_log.h" 10 #include "mlx5_common_pci.h" 11 12 struct mlx5_pci_device { 13 struct rte_pci_device *pci_dev; 14 TAILQ_ENTRY(mlx5_pci_device) next; 15 uint32_t classes_loaded; 16 }; 17 18 /* Head of list of drivers. */ 19 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list = 20 TAILQ_HEAD_INITIALIZER(drv_list); 21 22 /* Head of mlx5 pci devices. */ 23 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list = 24 TAILQ_HEAD_INITIALIZER(devices_list); 25 26 static const struct { 27 const char *name; 28 unsigned int driver_class; 29 } mlx5_classes[] = { 30 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA }, 31 { .name = "net", .driver_class = MLX5_CLASS_NET }, 32 { .name = "regex", .driver_class = MLX5_CLASS_REGEX }, 33 { .name = "compress", .driver_class = MLX5_CLASS_COMPRESS }, 34 { .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO }, 35 }; 36 37 static const unsigned int mlx5_class_combinations[] = { 38 MLX5_CLASS_NET, 39 MLX5_CLASS_VDPA, 40 MLX5_CLASS_REGEX, 41 MLX5_CLASS_COMPRESS, 42 MLX5_CLASS_CRYPTO, 43 MLX5_CLASS_NET | MLX5_CLASS_REGEX, 44 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX, 45 MLX5_CLASS_NET | MLX5_CLASS_COMPRESS, 46 MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS, 47 MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 48 MLX5_CLASS_NET | MLX5_CLASS_CRYPTO, 49 MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO, 50 MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 51 MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 52 MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 53 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 54 MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 55 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 56 MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 57 MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 58 MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS | 59 MLX5_CLASS_CRYPTO, 60 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS | 61 MLX5_CLASS_CRYPTO, 62 /* New class combination should be added here. */ 63 }; 64 65 static int 66 class_name_to_value(const char *class_name) 67 { 68 unsigned int i; 69 70 for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 71 if (strcmp(class_name, mlx5_classes[i].name) == 0) 72 return mlx5_classes[i].driver_class; 73 } 74 return -EINVAL; 75 } 76 77 static struct mlx5_pci_driver * 78 driver_get(uint32_t class) 79 { 80 struct mlx5_pci_driver *driver; 81 82 TAILQ_FOREACH(driver, &drv_list, next) { 83 if (driver->driver_class == class) 84 return driver; 85 } 86 return NULL; 87 } 88 89 static int 90 bus_cmdline_options_handler(__rte_unused const char *key, 91 const char *class_names, void *opaque) 92 { 93 int *ret = opaque; 94 char *nstr_org; 95 int class_val; 96 char *found; 97 char *nstr; 98 char *refstr = NULL; 99 100 *ret = 0; 101 nstr = strdup(class_names); 102 if (!nstr) { 103 *ret = -ENOMEM; 104 return *ret; 105 } 106 nstr_org = nstr; 107 found = strtok_r(nstr, ":", &refstr); 108 if (!found) 109 goto err; 110 do { 111 /* Extract each individual class name. Multiple 112 * class key,value is supplied as class=net:vdpa:foo:bar. 113 */ 114 class_val = class_name_to_value(found); 115 /* Check if its a valid class. */ 116 if (class_val < 0) { 117 *ret = -EINVAL; 118 goto err; 119 } 120 *ret |= class_val; 121 found = strtok_r(NULL, ":", &refstr); 122 } while (found); 123 err: 124 free(nstr_org); 125 if (*ret < 0) 126 DRV_LOG(ERR, "Invalid mlx5 class options %s." 127 " Maybe typo in device class argument setting?", 128 class_names); 129 return *ret; 130 } 131 132 static int 133 parse_class_options(const struct rte_devargs *devargs) 134 { 135 const char *key = RTE_DEVARGS_KEY_CLASS; 136 struct rte_kvargs *kvlist; 137 int ret = 0; 138 139 if (devargs == NULL) 140 return 0; 141 kvlist = rte_kvargs_parse(devargs->args, NULL); 142 if (kvlist == NULL) 143 return 0; 144 if (rte_kvargs_count(kvlist, key)) 145 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler, 146 &ret); 147 rte_kvargs_free(kvlist); 148 return ret; 149 } 150 151 static bool 152 mlx5_bus_match(const struct mlx5_pci_driver *drv, 153 const struct rte_pci_device *pci_dev) 154 { 155 const struct rte_pci_id *id_table; 156 157 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0; 158 id_table++) { 159 /* Check if device's ids match the class driver's ids. */ 160 if (id_table->vendor_id != pci_dev->id.vendor_id && 161 id_table->vendor_id != RTE_PCI_ANY_ID) 162 continue; 163 if (id_table->device_id != pci_dev->id.device_id && 164 id_table->device_id != RTE_PCI_ANY_ID) 165 continue; 166 if (id_table->subsystem_vendor_id != 167 pci_dev->id.subsystem_vendor_id && 168 id_table->subsystem_vendor_id != RTE_PCI_ANY_ID) 169 continue; 170 if (id_table->subsystem_device_id != 171 pci_dev->id.subsystem_device_id && 172 id_table->subsystem_device_id != RTE_PCI_ANY_ID) 173 continue; 174 if (id_table->class_id != pci_dev->id.class_id && 175 id_table->class_id != RTE_CLASS_ANY_ID) 176 continue; 177 return true; 178 } 179 return false; 180 } 181 182 static int 183 is_valid_class_combination(uint32_t user_classes) 184 { 185 unsigned int i; 186 187 /* Verify if user specified valid supported combination. */ 188 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) { 189 if (mlx5_class_combinations[i] == user_classes) 190 return 0; 191 } 192 /* Not found any valid class combination. */ 193 return -EINVAL; 194 } 195 196 static struct mlx5_pci_device * 197 pci_to_mlx5_device(const struct rte_pci_device *pci_dev) 198 { 199 struct mlx5_pci_device *dev; 200 201 TAILQ_FOREACH(dev, &devices_list, next) { 202 if (dev->pci_dev == pci_dev) 203 return dev; 204 } 205 return NULL; 206 } 207 208 static bool 209 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class) 210 { 211 return (device->classes_loaded & class) ? true : false; 212 } 213 214 static void 215 dev_release(struct mlx5_pci_device *dev) 216 { 217 TAILQ_REMOVE(&devices_list, dev, next); 218 rte_free(dev); 219 } 220 221 static int 222 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes) 223 { 224 struct mlx5_pci_driver *driver; 225 int local_ret = -ENODEV; 226 unsigned int i = 0; 227 int ret = 0; 228 229 enabled_classes &= dev->classes_loaded; 230 while (enabled_classes) { 231 driver = driver_get(RTE_BIT64(i)); 232 if (driver) { 233 local_ret = driver->pci_driver.remove(dev->pci_dev); 234 if (!local_ret) 235 dev->classes_loaded &= ~RTE_BIT64(i); 236 else if (ret == 0) 237 ret = local_ret; 238 } 239 enabled_classes &= ~RTE_BIT64(i); 240 i++; 241 } 242 if (local_ret) 243 ret = local_ret; 244 return ret; 245 } 246 247 static int 248 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv, 249 struct rte_pci_device *pci_dev, uint32_t user_classes) 250 { 251 struct mlx5_pci_driver *driver; 252 uint32_t enabled_classes = 0; 253 bool already_loaded; 254 int ret; 255 256 TAILQ_FOREACH(driver, &drv_list, next) { 257 if ((driver->driver_class & user_classes) == 0) 258 continue; 259 if (!mlx5_bus_match(driver, pci_dev)) 260 continue; 261 already_loaded = dev->classes_loaded & driver->driver_class; 262 if (already_loaded && 263 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) { 264 DRV_LOG(ERR, "Device %s is already probed", 265 pci_dev->device.name); 266 ret = -EEXIST; 267 goto probe_err; 268 } 269 ret = driver->pci_driver.probe(pci_drv, pci_dev); 270 if (ret < 0) { 271 DRV_LOG(ERR, "Failed to load driver %s", 272 driver->pci_driver.driver.name); 273 goto probe_err; 274 } 275 enabled_classes |= driver->driver_class; 276 } 277 dev->classes_loaded |= enabled_classes; 278 return 0; 279 probe_err: 280 /* Only unload drivers which are enabled which were enabled 281 * in this probe instance. 282 */ 283 drivers_remove(dev, enabled_classes); 284 return ret; 285 } 286 287 /** 288 * DPDK callback to register to probe multiple drivers for a PCI device. 289 * 290 * @param[in] pci_drv 291 * PCI driver structure. 292 * @param[in] dev 293 * PCI device information. 294 * 295 * @return 296 * 0 on success, a negative errno value otherwise and rte_errno is set. 297 */ 298 static int 299 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 300 struct rte_pci_device *pci_dev) 301 { 302 struct mlx5_pci_device *dev; 303 uint32_t user_classes = 0; 304 bool new_device = false; 305 int ret; 306 307 ret = parse_class_options(pci_dev->device.devargs); 308 if (ret < 0) 309 return ret; 310 user_classes = ret; 311 if (user_classes) { 312 /* Validate combination here. */ 313 ret = is_valid_class_combination(user_classes); 314 if (ret) { 315 DRV_LOG(ERR, "Unsupported mlx5 classes supplied."); 316 return ret; 317 } 318 } else { 319 /* Default to net class. */ 320 user_classes = MLX5_CLASS_NET; 321 } 322 dev = pci_to_mlx5_device(pci_dev); 323 if (!dev) { 324 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0); 325 if (!dev) 326 return -ENOMEM; 327 dev->pci_dev = pci_dev; 328 TAILQ_INSERT_HEAD(&devices_list, dev, next); 329 new_device = true; 330 } 331 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes); 332 if (ret) 333 goto class_err; 334 return 0; 335 class_err: 336 if (new_device) 337 dev_release(dev); 338 return ret; 339 } 340 341 /** 342 * DPDK callback to remove one or more drivers for a PCI device. 343 * 344 * This function removes all drivers probed for a given PCI device. 345 * 346 * @param[in] pci_dev 347 * Pointer to the PCI device. 348 * 349 * @return 350 * 0 on success, the function cannot fail. 351 */ 352 static int 353 mlx5_common_pci_remove(struct rte_pci_device *pci_dev) 354 { 355 struct mlx5_pci_device *dev; 356 int ret; 357 358 dev = pci_to_mlx5_device(pci_dev); 359 if (!dev) 360 return -ENODEV; 361 /* Matching device found, cleanup and unload drivers. */ 362 ret = drivers_remove(dev, dev->classes_loaded); 363 if (!ret) 364 dev_release(dev); 365 return ret; 366 } 367 368 static int 369 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr, 370 uint64_t iova, size_t len) 371 { 372 struct mlx5_pci_driver *driver = NULL; 373 struct mlx5_pci_driver *temp; 374 struct mlx5_pci_device *dev; 375 int ret = -EINVAL; 376 377 dev = pci_to_mlx5_device(pci_dev); 378 if (!dev) 379 return -ENODEV; 380 TAILQ_FOREACH(driver, &drv_list, next) { 381 if (device_class_enabled(dev, driver->driver_class) && 382 driver->pci_driver.dma_map) { 383 ret = driver->pci_driver.dma_map(pci_dev, addr, 384 iova, len); 385 if (ret) 386 goto map_err; 387 } 388 } 389 return ret; 390 map_err: 391 TAILQ_FOREACH(temp, &drv_list, next) { 392 if (temp == driver) 393 break; 394 if (device_class_enabled(dev, temp->driver_class) && 395 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap) 396 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len); 397 } 398 return ret; 399 } 400 401 static int 402 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr, 403 uint64_t iova, size_t len) 404 { 405 struct mlx5_pci_driver *driver; 406 struct mlx5_pci_device *dev; 407 int local_ret = -EINVAL; 408 int ret; 409 410 dev = pci_to_mlx5_device(pci_dev); 411 if (!dev) 412 return -ENODEV; 413 ret = 0; 414 /* There is no unmap error recovery in current implementation. */ 415 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) { 416 if (device_class_enabled(dev, driver->driver_class) && 417 driver->pci_driver.dma_unmap) { 418 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr, 419 iova, len); 420 if (local_ret && (ret == 0)) 421 ret = local_ret; 422 } 423 } 424 if (local_ret) 425 ret = local_ret; 426 return ret; 427 } 428 429 /* PCI ID table is build dynamically based on registered mlx5 drivers. */ 430 static struct rte_pci_id *mlx5_pci_id_table; 431 432 static struct rte_pci_driver mlx5_pci_driver = { 433 .driver = { 434 .name = MLX5_PCI_DRIVER_NAME, 435 }, 436 .probe = mlx5_common_pci_probe, 437 .remove = mlx5_common_pci_remove, 438 .dma_map = mlx5_common_pci_dma_map, 439 .dma_unmap = mlx5_common_pci_dma_unmap, 440 }; 441 442 static int 443 pci_id_table_size_get(const struct rte_pci_id *id_table) 444 { 445 int table_size = 0; 446 447 for (; id_table->vendor_id != 0; id_table++) 448 table_size++; 449 return table_size; 450 } 451 452 static bool 453 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table, 454 int next_idx) 455 { 456 int current_size = next_idx - 1; 457 int i; 458 459 for (i = 0; i < current_size; i++) { 460 if (id->device_id == table[i].device_id && 461 id->vendor_id == table[i].vendor_id && 462 id->subsystem_vendor_id == table[i].subsystem_vendor_id && 463 id->subsystem_device_id == table[i].subsystem_device_id) 464 return true; 465 } 466 return false; 467 } 468 469 static void 470 pci_id_insert(struct rte_pci_id *new_table, int *next_idx, 471 const struct rte_pci_id *id_table) 472 { 473 /* Traverse the id_table, check if entry exists in new_table; 474 * Add non duplicate entries to new table. 475 */ 476 for (; id_table->vendor_id != 0; id_table++) { 477 if (!pci_id_exists(id_table, new_table, *next_idx)) { 478 /* New entry; add to the table. */ 479 new_table[*next_idx] = *id_table; 480 (*next_idx)++; 481 } 482 } 483 } 484 485 static int 486 pci_ids_table_update(const struct rte_pci_id *driver_id_table) 487 { 488 const struct rte_pci_id *id_iter; 489 struct rte_pci_id *updated_table; 490 struct rte_pci_id *old_table; 491 int num_ids = 0; 492 int i = 0; 493 494 old_table = mlx5_pci_id_table; 495 if (old_table) 496 num_ids = pci_id_table_size_get(old_table); 497 num_ids += pci_id_table_size_get(driver_id_table); 498 /* Increase size by one for the termination entry of vendor_id = 0. */ 499 num_ids += 1; 500 updated_table = calloc(num_ids, sizeof(*updated_table)); 501 if (!updated_table) 502 return -ENOMEM; 503 if (TAILQ_EMPTY(&drv_list)) { 504 /* Copy the first driver's ID table. */ 505 for (id_iter = driver_id_table; id_iter->vendor_id != 0; 506 id_iter++, i++) 507 updated_table[i] = *id_iter; 508 } else { 509 /* First copy existing table entries. */ 510 for (id_iter = old_table; id_iter->vendor_id != 0; 511 id_iter++, i++) 512 updated_table[i] = *id_iter; 513 /* New id to be added at the end of current ID table. */ 514 pci_id_insert(updated_table, &i, driver_id_table); 515 } 516 /* Terminate table with empty entry. */ 517 updated_table[i].vendor_id = 0; 518 mlx5_pci_driver.id_table = updated_table; 519 mlx5_pci_id_table = updated_table; 520 if (old_table) 521 free(old_table); 522 return 0; 523 } 524 525 void 526 mlx5_pci_driver_register(struct mlx5_pci_driver *driver) 527 { 528 int ret; 529 530 ret = pci_ids_table_update(driver->pci_driver.id_table); 531 if (ret) 532 return; 533 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags; 534 TAILQ_INSERT_TAIL(&drv_list, driver, next); 535 } 536 537 void mlx5_common_pci_init(void) 538 { 539 const struct rte_pci_id empty_table[] = { 540 { 541 .vendor_id = 0 542 }, 543 }; 544 545 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD 546 * including this one can register the PCI table first. If any other 547 * PMD(s) have registered the PCI ID table, No need to register an empty 548 * default one. 549 */ 550 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table)) 551 return; 552 rte_pci_register(&mlx5_pci_driver); 553 } 554 555 RTE_FINI(mlx5_common_pci_finish) 556 { 557 if (mlx5_pci_id_table != NULL) { 558 /* Constructor doesn't register with PCI bus if it failed 559 * to build the table. 560 */ 561 rte_pci_unregister(&mlx5_pci_driver); 562 free(mlx5_pci_id_table); 563 } 564 } 565 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__); 566