1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies Ltd 3 */ 4 5 #include <stdlib.h> 6 #include <rte_malloc.h> 7 #include <rte_class.h> 8 9 #include "mlx5_common_log.h" 10 #include "mlx5_common_pci.h" 11 12 struct mlx5_pci_device { 13 struct rte_pci_device *pci_dev; 14 TAILQ_ENTRY(mlx5_pci_device) next; 15 uint32_t classes_loaded; 16 }; 17 18 /* Head of list of drivers. */ 19 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list = 20 TAILQ_HEAD_INITIALIZER(drv_list); 21 22 /* Head of mlx5 pci devices. */ 23 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list = 24 TAILQ_HEAD_INITIALIZER(devices_list); 25 26 static const struct { 27 const char *name; 28 unsigned int driver_class; 29 } mlx5_classes[] = { 30 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA }, 31 { .name = "net", .driver_class = MLX5_CLASS_NET }, 32 { .name = "regex", .driver_class = MLX5_CLASS_REGEX }, 33 { .name = "compress", .driver_class = MLX5_CLASS_COMPRESS }, 34 }; 35 36 static const unsigned int mlx5_class_combinations[] = { 37 MLX5_CLASS_NET, 38 MLX5_CLASS_VDPA, 39 MLX5_CLASS_REGEX, 40 MLX5_CLASS_COMPRESS, 41 MLX5_CLASS_NET | MLX5_CLASS_REGEX, 42 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX, 43 MLX5_CLASS_NET | MLX5_CLASS_COMPRESS, 44 MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS, 45 MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 46 MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 47 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 48 /* New class combination should be added here. */ 49 }; 50 51 static int 52 class_name_to_value(const char *class_name) 53 { 54 unsigned int i; 55 56 for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 57 if (strcmp(class_name, mlx5_classes[i].name) == 0) 58 return mlx5_classes[i].driver_class; 59 } 60 return -EINVAL; 61 } 62 63 static struct mlx5_pci_driver * 64 driver_get(uint32_t class) 65 { 66 struct mlx5_pci_driver *driver; 67 68 TAILQ_FOREACH(driver, &drv_list, next) { 69 if (driver->driver_class == class) 70 return driver; 71 } 72 return NULL; 73 } 74 75 static int 76 bus_cmdline_options_handler(__rte_unused const char *key, 77 const char *class_names, void *opaque) 78 { 79 int *ret = opaque; 80 char *nstr_org; 81 int class_val; 82 char *found; 83 char *nstr; 84 char *refstr = NULL; 85 86 *ret = 0; 87 nstr = strdup(class_names); 88 if (!nstr) { 89 *ret = -ENOMEM; 90 return *ret; 91 } 92 nstr_org = nstr; 93 found = strtok_r(nstr, ":", &refstr); 94 if (!found) 95 goto err; 96 do { 97 /* Extract each individual class name. Multiple 98 * class key,value is supplied as class=net:vdpa:foo:bar. 99 */ 100 class_val = class_name_to_value(found); 101 /* Check if its a valid class. */ 102 if (class_val < 0) { 103 *ret = -EINVAL; 104 goto err; 105 } 106 *ret |= class_val; 107 found = strtok_r(NULL, ":", &refstr); 108 } while (found); 109 err: 110 free(nstr_org); 111 if (*ret < 0) 112 DRV_LOG(ERR, "Invalid mlx5 class options %s." 113 " Maybe typo in device class argument setting?", 114 class_names); 115 return *ret; 116 } 117 118 static int 119 parse_class_options(const struct rte_devargs *devargs) 120 { 121 const char *key = MLX5_CLASS_ARG_NAME; 122 struct rte_kvargs *kvlist; 123 int ret = 0; 124 125 if (devargs == NULL) 126 return 0; 127 kvlist = rte_kvargs_parse(devargs->args, NULL); 128 if (kvlist == NULL) 129 return 0; 130 if (rte_kvargs_count(kvlist, key)) 131 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler, 132 &ret); 133 rte_kvargs_free(kvlist); 134 return ret; 135 } 136 137 static bool 138 mlx5_bus_match(const struct mlx5_pci_driver *drv, 139 const struct rte_pci_device *pci_dev) 140 { 141 const struct rte_pci_id *id_table; 142 143 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0; 144 id_table++) { 145 /* Check if device's ids match the class driver's ids. */ 146 if (id_table->vendor_id != pci_dev->id.vendor_id && 147 id_table->vendor_id != RTE_PCI_ANY_ID) 148 continue; 149 if (id_table->device_id != pci_dev->id.device_id && 150 id_table->device_id != RTE_PCI_ANY_ID) 151 continue; 152 if (id_table->subsystem_vendor_id != 153 pci_dev->id.subsystem_vendor_id && 154 id_table->subsystem_vendor_id != RTE_PCI_ANY_ID) 155 continue; 156 if (id_table->subsystem_device_id != 157 pci_dev->id.subsystem_device_id && 158 id_table->subsystem_device_id != RTE_PCI_ANY_ID) 159 continue; 160 if (id_table->class_id != pci_dev->id.class_id && 161 id_table->class_id != RTE_CLASS_ANY_ID) 162 continue; 163 return true; 164 } 165 return false; 166 } 167 168 static int 169 is_valid_class_combination(uint32_t user_classes) 170 { 171 unsigned int i; 172 173 /* Verify if user specified valid supported combination. */ 174 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) { 175 if (mlx5_class_combinations[i] == user_classes) 176 return 0; 177 } 178 /* Not found any valid class combination. */ 179 return -EINVAL; 180 } 181 182 static struct mlx5_pci_device * 183 pci_to_mlx5_device(const struct rte_pci_device *pci_dev) 184 { 185 struct mlx5_pci_device *dev; 186 187 TAILQ_FOREACH(dev, &devices_list, next) { 188 if (dev->pci_dev == pci_dev) 189 return dev; 190 } 191 return NULL; 192 } 193 194 static bool 195 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class) 196 { 197 return (device->classes_loaded & class) ? true : false; 198 } 199 200 static void 201 dev_release(struct mlx5_pci_device *dev) 202 { 203 TAILQ_REMOVE(&devices_list, dev, next); 204 rte_free(dev); 205 } 206 207 static int 208 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes) 209 { 210 struct mlx5_pci_driver *driver; 211 int local_ret = -ENODEV; 212 unsigned int i = 0; 213 int ret = 0; 214 215 enabled_classes &= dev->classes_loaded; 216 while (enabled_classes) { 217 driver = driver_get(RTE_BIT64(i)); 218 if (driver) { 219 local_ret = driver->pci_driver.remove(dev->pci_dev); 220 if (!local_ret) 221 dev->classes_loaded &= ~RTE_BIT64(i); 222 else if (ret == 0) 223 ret = local_ret; 224 } 225 enabled_classes &= ~RTE_BIT64(i); 226 i++; 227 } 228 if (local_ret) 229 ret = local_ret; 230 return ret; 231 } 232 233 static int 234 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv, 235 struct rte_pci_device *pci_dev, uint32_t user_classes) 236 { 237 struct mlx5_pci_driver *driver; 238 uint32_t enabled_classes = 0; 239 bool already_loaded; 240 int ret; 241 242 TAILQ_FOREACH(driver, &drv_list, next) { 243 if ((driver->driver_class & user_classes) == 0) 244 continue; 245 if (!mlx5_bus_match(driver, pci_dev)) 246 continue; 247 already_loaded = dev->classes_loaded & driver->driver_class; 248 if (already_loaded && 249 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) { 250 DRV_LOG(ERR, "Device %s is already probed", 251 pci_dev->device.name); 252 ret = -EEXIST; 253 goto probe_err; 254 } 255 ret = driver->pci_driver.probe(pci_drv, pci_dev); 256 if (ret < 0) { 257 DRV_LOG(ERR, "Failed to load driver %s", 258 driver->pci_driver.driver.name); 259 goto probe_err; 260 } 261 enabled_classes |= driver->driver_class; 262 } 263 dev->classes_loaded |= enabled_classes; 264 return 0; 265 probe_err: 266 /* Only unload drivers which are enabled which were enabled 267 * in this probe instance. 268 */ 269 drivers_remove(dev, enabled_classes); 270 return ret; 271 } 272 273 /** 274 * DPDK callback to register to probe multiple drivers for a PCI device. 275 * 276 * @param[in] pci_drv 277 * PCI driver structure. 278 * @param[in] dev 279 * PCI device information. 280 * 281 * @return 282 * 0 on success, a negative errno value otherwise and rte_errno is set. 283 */ 284 static int 285 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 286 struct rte_pci_device *pci_dev) 287 { 288 struct mlx5_pci_device *dev; 289 uint32_t user_classes = 0; 290 bool new_device = false; 291 int ret; 292 293 ret = parse_class_options(pci_dev->device.devargs); 294 if (ret < 0) 295 return ret; 296 user_classes = ret; 297 if (user_classes) { 298 /* Validate combination here. */ 299 ret = is_valid_class_combination(user_classes); 300 if (ret) { 301 DRV_LOG(ERR, "Unsupported mlx5 classes supplied."); 302 return ret; 303 } 304 } else { 305 /* Default to net class. */ 306 user_classes = MLX5_CLASS_NET; 307 } 308 dev = pci_to_mlx5_device(pci_dev); 309 if (!dev) { 310 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0); 311 if (!dev) 312 return -ENOMEM; 313 dev->pci_dev = pci_dev; 314 TAILQ_INSERT_HEAD(&devices_list, dev, next); 315 new_device = true; 316 } 317 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes); 318 if (ret) 319 goto class_err; 320 return 0; 321 class_err: 322 if (new_device) 323 dev_release(dev); 324 return ret; 325 } 326 327 /** 328 * DPDK callback to remove one or more drivers for a PCI device. 329 * 330 * This function removes all drivers probed for a given PCI device. 331 * 332 * @param[in] pci_dev 333 * Pointer to the PCI device. 334 * 335 * @return 336 * 0 on success, the function cannot fail. 337 */ 338 static int 339 mlx5_common_pci_remove(struct rte_pci_device *pci_dev) 340 { 341 struct mlx5_pci_device *dev; 342 int ret; 343 344 dev = pci_to_mlx5_device(pci_dev); 345 if (!dev) 346 return -ENODEV; 347 /* Matching device found, cleanup and unload drivers. */ 348 ret = drivers_remove(dev, dev->classes_loaded); 349 if (!ret) 350 dev_release(dev); 351 return ret; 352 } 353 354 static int 355 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr, 356 uint64_t iova, size_t len) 357 { 358 struct mlx5_pci_driver *driver = NULL; 359 struct mlx5_pci_driver *temp; 360 struct mlx5_pci_device *dev; 361 int ret = -EINVAL; 362 363 dev = pci_to_mlx5_device(pci_dev); 364 if (!dev) 365 return -ENODEV; 366 TAILQ_FOREACH(driver, &drv_list, next) { 367 if (device_class_enabled(dev, driver->driver_class) && 368 driver->pci_driver.dma_map) { 369 ret = driver->pci_driver.dma_map(pci_dev, addr, 370 iova, len); 371 if (ret) 372 goto map_err; 373 } 374 } 375 return ret; 376 map_err: 377 TAILQ_FOREACH(temp, &drv_list, next) { 378 if (temp == driver) 379 break; 380 if (device_class_enabled(dev, temp->driver_class) && 381 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap) 382 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len); 383 } 384 return ret; 385 } 386 387 static int 388 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr, 389 uint64_t iova, size_t len) 390 { 391 struct mlx5_pci_driver *driver; 392 struct mlx5_pci_device *dev; 393 int local_ret = -EINVAL; 394 int ret; 395 396 dev = pci_to_mlx5_device(pci_dev); 397 if (!dev) 398 return -ENODEV; 399 ret = 0; 400 /* There is no unmap error recovery in current implementation. */ 401 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) { 402 if (device_class_enabled(dev, driver->driver_class) && 403 driver->pci_driver.dma_unmap) { 404 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr, 405 iova, len); 406 if (local_ret && (ret == 0)) 407 ret = local_ret; 408 } 409 } 410 if (local_ret) 411 ret = local_ret; 412 return ret; 413 } 414 415 /* PCI ID table is build dynamically based on registered mlx5 drivers. */ 416 static struct rte_pci_id *mlx5_pci_id_table; 417 418 static struct rte_pci_driver mlx5_pci_driver = { 419 .driver = { 420 .name = MLX5_PCI_DRIVER_NAME, 421 }, 422 .probe = mlx5_common_pci_probe, 423 .remove = mlx5_common_pci_remove, 424 .dma_map = mlx5_common_pci_dma_map, 425 .dma_unmap = mlx5_common_pci_dma_unmap, 426 }; 427 428 static int 429 pci_id_table_size_get(const struct rte_pci_id *id_table) 430 { 431 int table_size = 0; 432 433 for (; id_table->vendor_id != 0; id_table++) 434 table_size++; 435 return table_size; 436 } 437 438 static bool 439 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table, 440 int next_idx) 441 { 442 int current_size = next_idx - 1; 443 int i; 444 445 for (i = 0; i < current_size; i++) { 446 if (id->device_id == table[i].device_id && 447 id->vendor_id == table[i].vendor_id && 448 id->subsystem_vendor_id == table[i].subsystem_vendor_id && 449 id->subsystem_device_id == table[i].subsystem_device_id) 450 return true; 451 } 452 return false; 453 } 454 455 static void 456 pci_id_insert(struct rte_pci_id *new_table, int *next_idx, 457 const struct rte_pci_id *id_table) 458 { 459 /* Traverse the id_table, check if entry exists in new_table; 460 * Add non duplicate entries to new table. 461 */ 462 for (; id_table->vendor_id != 0; id_table++) { 463 if (!pci_id_exists(id_table, new_table, *next_idx)) { 464 /* New entry; add to the table. */ 465 new_table[*next_idx] = *id_table; 466 (*next_idx)++; 467 } 468 } 469 } 470 471 static int 472 pci_ids_table_update(const struct rte_pci_id *driver_id_table) 473 { 474 const struct rte_pci_id *id_iter; 475 struct rte_pci_id *updated_table; 476 struct rte_pci_id *old_table; 477 int num_ids = 0; 478 int i = 0; 479 480 old_table = mlx5_pci_id_table; 481 if (old_table) 482 num_ids = pci_id_table_size_get(old_table); 483 num_ids += pci_id_table_size_get(driver_id_table); 484 /* Increase size by one for the termination entry of vendor_id = 0. */ 485 num_ids += 1; 486 updated_table = calloc(num_ids, sizeof(*updated_table)); 487 if (!updated_table) 488 return -ENOMEM; 489 if (TAILQ_EMPTY(&drv_list)) { 490 /* Copy the first driver's ID table. */ 491 for (id_iter = driver_id_table; id_iter->vendor_id != 0; 492 id_iter++, i++) 493 updated_table[i] = *id_iter; 494 } else { 495 /* First copy existing table entries. */ 496 for (id_iter = old_table; id_iter->vendor_id != 0; 497 id_iter++, i++) 498 updated_table[i] = *id_iter; 499 /* New id to be added at the end of current ID table. */ 500 pci_id_insert(updated_table, &i, driver_id_table); 501 } 502 /* Terminate table with empty entry. */ 503 updated_table[i].vendor_id = 0; 504 mlx5_pci_driver.id_table = updated_table; 505 mlx5_pci_id_table = updated_table; 506 if (old_table) 507 free(old_table); 508 return 0; 509 } 510 511 void 512 mlx5_pci_driver_register(struct mlx5_pci_driver *driver) 513 { 514 int ret; 515 516 ret = pci_ids_table_update(driver->pci_driver.id_table); 517 if (ret) 518 return; 519 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags; 520 TAILQ_INSERT_TAIL(&drv_list, driver, next); 521 } 522 523 void mlx5_common_pci_init(void) 524 { 525 const struct rte_pci_id empty_table[] = { 526 { 527 .vendor_id = 0 528 }, 529 }; 530 531 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD 532 * including this one can register the PCI table first. If any other 533 * PMD(s) have registered the PCI ID table, No need to register an empty 534 * default one. 535 */ 536 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table)) 537 return; 538 rte_pci_register(&mlx5_pci_driver); 539 } 540 541 RTE_FINI(mlx5_common_pci_finish) 542 { 543 if (mlx5_pci_id_table != NULL) { 544 /* Constructor doesn't register with PCI bus if it failed 545 * to build the table. 546 */ 547 rte_pci_unregister(&mlx5_pci_driver); 548 free(mlx5_pci_id_table); 549 } 550 } 551 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__); 552