1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies Ltd 3 */ 4 5 #include <stdlib.h> 6 #include <rte_malloc.h> 7 #include "mlx5_common_utils.h" 8 #include "mlx5_common_pci.h" 9 10 struct mlx5_pci_device { 11 struct rte_pci_device *pci_dev; 12 TAILQ_ENTRY(mlx5_pci_device) next; 13 uint32_t classes_loaded; 14 }; 15 16 /* Head of list of drivers. */ 17 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list = 18 TAILQ_HEAD_INITIALIZER(drv_list); 19 20 /* Head of mlx5 pci devices. */ 21 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list = 22 TAILQ_HEAD_INITIALIZER(devices_list); 23 24 static const struct { 25 const char *name; 26 unsigned int driver_class; 27 } mlx5_classes[] = { 28 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA }, 29 { .name = "net", .driver_class = MLX5_CLASS_NET }, 30 { .name = "regex", .driver_class = MLX5_CLASS_REGEX }, 31 { .name = "compress", .driver_class = MLX5_CLASS_COMPRESS }, 32 }; 33 34 static const unsigned int mlx5_class_combinations[] = { 35 MLX5_CLASS_NET, 36 MLX5_CLASS_VDPA, 37 MLX5_CLASS_REGEX, 38 MLX5_CLASS_COMPRESS, 39 MLX5_CLASS_NET | MLX5_CLASS_REGEX, 40 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX, 41 MLX5_CLASS_NET | MLX5_CLASS_COMPRESS, 42 MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS, 43 MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 44 MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 45 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 46 /* New class combination should be added here. */ 47 }; 48 49 static int 50 class_name_to_value(const char *class_name) 51 { 52 unsigned int i; 53 54 for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 55 if (strcmp(class_name, mlx5_classes[i].name) == 0) 56 return mlx5_classes[i].driver_class; 57 } 58 return -EINVAL; 59 } 60 61 static struct mlx5_pci_driver * 62 driver_get(uint32_t class) 63 { 64 struct mlx5_pci_driver *driver; 65 66 TAILQ_FOREACH(driver, &drv_list, next) { 67 if (driver->driver_class == class) 68 return driver; 69 } 70 return NULL; 71 } 72 73 static int 74 bus_cmdline_options_handler(__rte_unused const char *key, 75 const char *class_names, void *opaque) 76 { 77 int *ret = opaque; 78 char *nstr_org; 79 int class_val; 80 char *found; 81 char *nstr; 82 char *refstr = NULL; 83 84 *ret = 0; 85 nstr = strdup(class_names); 86 if (!nstr) { 87 *ret = -ENOMEM; 88 return *ret; 89 } 90 nstr_org = nstr; 91 found = strtok_r(nstr, ":", &refstr); 92 if (!found) 93 goto err; 94 do { 95 /* Extract each individual class name. Multiple 96 * class key,value is supplied as class=net:vdpa:foo:bar. 97 */ 98 class_val = class_name_to_value(found); 99 /* Check if its a valid class. */ 100 if (class_val < 0) { 101 *ret = -EINVAL; 102 goto err; 103 } 104 *ret |= class_val; 105 found = strtok_r(NULL, ":", &refstr); 106 } while (found); 107 err: 108 free(nstr_org); 109 if (*ret < 0) 110 DRV_LOG(ERR, "Invalid mlx5 class options %s." 111 " Maybe typo in device class argument setting?", 112 class_names); 113 return *ret; 114 } 115 116 static int 117 parse_class_options(const struct rte_devargs *devargs) 118 { 119 const char *key = MLX5_CLASS_ARG_NAME; 120 struct rte_kvargs *kvlist; 121 int ret = 0; 122 123 if (devargs == NULL) 124 return 0; 125 kvlist = rte_kvargs_parse(devargs->args, NULL); 126 if (kvlist == NULL) 127 return 0; 128 if (rte_kvargs_count(kvlist, key)) 129 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler, 130 &ret); 131 rte_kvargs_free(kvlist); 132 return ret; 133 } 134 135 static bool 136 mlx5_bus_match(const struct mlx5_pci_driver *drv, 137 const struct rte_pci_device *pci_dev) 138 { 139 const struct rte_pci_id *id_table; 140 141 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0; 142 id_table++) { 143 /* Check if device's ids match the class driver's ids. */ 144 if (id_table->vendor_id != pci_dev->id.vendor_id && 145 id_table->vendor_id != RTE_PCI_ANY_ID) 146 continue; 147 if (id_table->device_id != pci_dev->id.device_id && 148 id_table->device_id != RTE_PCI_ANY_ID) 149 continue; 150 if (id_table->subsystem_vendor_id != 151 pci_dev->id.subsystem_vendor_id && 152 id_table->subsystem_vendor_id != RTE_PCI_ANY_ID) 153 continue; 154 if (id_table->subsystem_device_id != 155 pci_dev->id.subsystem_device_id && 156 id_table->subsystem_device_id != RTE_PCI_ANY_ID) 157 continue; 158 if (id_table->class_id != pci_dev->id.class_id && 159 id_table->class_id != RTE_CLASS_ANY_ID) 160 continue; 161 return true; 162 } 163 return false; 164 } 165 166 static int 167 is_valid_class_combination(uint32_t user_classes) 168 { 169 unsigned int i; 170 171 /* Verify if user specified valid supported combination. */ 172 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) { 173 if (mlx5_class_combinations[i] == user_classes) 174 return 0; 175 } 176 /* Not found any valid class combination. */ 177 return -EINVAL; 178 } 179 180 static struct mlx5_pci_device * 181 pci_to_mlx5_device(const struct rte_pci_device *pci_dev) 182 { 183 struct mlx5_pci_device *dev; 184 185 TAILQ_FOREACH(dev, &devices_list, next) { 186 if (dev->pci_dev == pci_dev) 187 return dev; 188 } 189 return NULL; 190 } 191 192 static bool 193 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class) 194 { 195 return (device->classes_loaded & class) ? true : false; 196 } 197 198 static void 199 dev_release(struct mlx5_pci_device *dev) 200 { 201 TAILQ_REMOVE(&devices_list, dev, next); 202 rte_free(dev); 203 } 204 205 static int 206 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes) 207 { 208 struct mlx5_pci_driver *driver; 209 int local_ret = -ENODEV; 210 unsigned int i = 0; 211 int ret = 0; 212 213 enabled_classes &= dev->classes_loaded; 214 while (enabled_classes) { 215 driver = driver_get(RTE_BIT64(i)); 216 if (driver) { 217 local_ret = driver->pci_driver.remove(dev->pci_dev); 218 if (!local_ret) 219 dev->classes_loaded &= ~RTE_BIT64(i); 220 else if (ret == 0) 221 ret = local_ret; 222 } 223 enabled_classes &= ~RTE_BIT64(i); 224 i++; 225 } 226 if (local_ret) 227 ret = local_ret; 228 return ret; 229 } 230 231 static int 232 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv, 233 struct rte_pci_device *pci_dev, uint32_t user_classes) 234 { 235 struct mlx5_pci_driver *driver; 236 uint32_t enabled_classes = 0; 237 bool already_loaded; 238 int ret; 239 240 TAILQ_FOREACH(driver, &drv_list, next) { 241 if ((driver->driver_class & user_classes) == 0) 242 continue; 243 if (!mlx5_bus_match(driver, pci_dev)) 244 continue; 245 already_loaded = dev->classes_loaded & driver->driver_class; 246 if (already_loaded && 247 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) { 248 DRV_LOG(ERR, "Device %s is already probed", 249 pci_dev->device.name); 250 ret = -EEXIST; 251 goto probe_err; 252 } 253 ret = driver->pci_driver.probe(pci_drv, pci_dev); 254 if (ret < 0) { 255 DRV_LOG(ERR, "Failed to load driver %s", 256 driver->pci_driver.driver.name); 257 goto probe_err; 258 } 259 enabled_classes |= driver->driver_class; 260 } 261 dev->classes_loaded |= enabled_classes; 262 return 0; 263 probe_err: 264 /* Only unload drivers which are enabled which were enabled 265 * in this probe instance. 266 */ 267 drivers_remove(dev, enabled_classes); 268 return ret; 269 } 270 271 /** 272 * DPDK callback to register to probe multiple drivers for a PCI device. 273 * 274 * @param[in] pci_drv 275 * PCI driver structure. 276 * @param[in] dev 277 * PCI device information. 278 * 279 * @return 280 * 0 on success, a negative errno value otherwise and rte_errno is set. 281 */ 282 static int 283 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 284 struct rte_pci_device *pci_dev) 285 { 286 struct mlx5_pci_device *dev; 287 uint32_t user_classes = 0; 288 bool new_device = false; 289 int ret; 290 291 ret = parse_class_options(pci_dev->device.devargs); 292 if (ret < 0) 293 return ret; 294 user_classes = ret; 295 if (user_classes) { 296 /* Validate combination here. */ 297 ret = is_valid_class_combination(user_classes); 298 if (ret) { 299 DRV_LOG(ERR, "Unsupported mlx5 classes supplied."); 300 return ret; 301 } 302 } else { 303 /* Default to net class. */ 304 user_classes = MLX5_CLASS_NET; 305 } 306 dev = pci_to_mlx5_device(pci_dev); 307 if (!dev) { 308 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0); 309 if (!dev) 310 return -ENOMEM; 311 dev->pci_dev = pci_dev; 312 TAILQ_INSERT_HEAD(&devices_list, dev, next); 313 new_device = true; 314 } 315 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes); 316 if (ret) 317 goto class_err; 318 return 0; 319 class_err: 320 if (new_device) 321 dev_release(dev); 322 return ret; 323 } 324 325 /** 326 * DPDK callback to remove one or more drivers for a PCI device. 327 * 328 * This function removes all drivers probed for a given PCI device. 329 * 330 * @param[in] pci_dev 331 * Pointer to the PCI device. 332 * 333 * @return 334 * 0 on success, the function cannot fail. 335 */ 336 static int 337 mlx5_common_pci_remove(struct rte_pci_device *pci_dev) 338 { 339 struct mlx5_pci_device *dev; 340 int ret; 341 342 dev = pci_to_mlx5_device(pci_dev); 343 if (!dev) 344 return -ENODEV; 345 /* Matching device found, cleanup and unload drivers. */ 346 ret = drivers_remove(dev, dev->classes_loaded); 347 if (!ret) 348 dev_release(dev); 349 return ret; 350 } 351 352 static int 353 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr, 354 uint64_t iova, size_t len) 355 { 356 struct mlx5_pci_driver *driver = NULL; 357 struct mlx5_pci_driver *temp; 358 struct mlx5_pci_device *dev; 359 int ret = -EINVAL; 360 361 dev = pci_to_mlx5_device(pci_dev); 362 if (!dev) 363 return -ENODEV; 364 TAILQ_FOREACH(driver, &drv_list, next) { 365 if (device_class_enabled(dev, driver->driver_class) && 366 driver->pci_driver.dma_map) { 367 ret = driver->pci_driver.dma_map(pci_dev, addr, 368 iova, len); 369 if (ret) 370 goto map_err; 371 } 372 } 373 return ret; 374 map_err: 375 TAILQ_FOREACH(temp, &drv_list, next) { 376 if (temp == driver) 377 break; 378 if (device_class_enabled(dev, temp->driver_class) && 379 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap) 380 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len); 381 } 382 return ret; 383 } 384 385 static int 386 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr, 387 uint64_t iova, size_t len) 388 { 389 struct mlx5_pci_driver *driver; 390 struct mlx5_pci_device *dev; 391 int local_ret = -EINVAL; 392 int ret; 393 394 dev = pci_to_mlx5_device(pci_dev); 395 if (!dev) 396 return -ENODEV; 397 ret = 0; 398 /* There is no unmap error recovery in current implementation. */ 399 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) { 400 if (device_class_enabled(dev, driver->driver_class) && 401 driver->pci_driver.dma_unmap) { 402 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr, 403 iova, len); 404 if (local_ret && (ret == 0)) 405 ret = local_ret; 406 } 407 } 408 if (local_ret) 409 ret = local_ret; 410 return ret; 411 } 412 413 /* PCI ID table is build dynamically based on registered mlx5 drivers. */ 414 static struct rte_pci_id *mlx5_pci_id_table; 415 416 static struct rte_pci_driver mlx5_pci_driver = { 417 .driver = { 418 .name = MLX5_PCI_DRIVER_NAME, 419 }, 420 .probe = mlx5_common_pci_probe, 421 .remove = mlx5_common_pci_remove, 422 .dma_map = mlx5_common_pci_dma_map, 423 .dma_unmap = mlx5_common_pci_dma_unmap, 424 }; 425 426 static int 427 pci_id_table_size_get(const struct rte_pci_id *id_table) 428 { 429 int table_size = 0; 430 431 for (; id_table->vendor_id != 0; id_table++) 432 table_size++; 433 return table_size; 434 } 435 436 static bool 437 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table, 438 int next_idx) 439 { 440 int current_size = next_idx - 1; 441 int i; 442 443 for (i = 0; i < current_size; i++) { 444 if (id->device_id == table[i].device_id && 445 id->vendor_id == table[i].vendor_id && 446 id->subsystem_vendor_id == table[i].subsystem_vendor_id && 447 id->subsystem_device_id == table[i].subsystem_device_id) 448 return true; 449 } 450 return false; 451 } 452 453 static void 454 pci_id_insert(struct rte_pci_id *new_table, int *next_idx, 455 const struct rte_pci_id *id_table) 456 { 457 /* Traverse the id_table, check if entry exists in new_table; 458 * Add non duplicate entries to new table. 459 */ 460 for (; id_table->vendor_id != 0; id_table++) { 461 if (!pci_id_exists(id_table, new_table, *next_idx)) { 462 /* New entry; add to the table. */ 463 new_table[*next_idx] = *id_table; 464 (*next_idx)++; 465 } 466 } 467 } 468 469 static int 470 pci_ids_table_update(const struct rte_pci_id *driver_id_table) 471 { 472 const struct rte_pci_id *id_iter; 473 struct rte_pci_id *updated_table; 474 struct rte_pci_id *old_table; 475 int num_ids = 0; 476 int i = 0; 477 478 old_table = mlx5_pci_id_table; 479 if (old_table) 480 num_ids = pci_id_table_size_get(old_table); 481 num_ids += pci_id_table_size_get(driver_id_table); 482 /* Increase size by one for the termination entry of vendor_id = 0. */ 483 num_ids += 1; 484 updated_table = calloc(num_ids, sizeof(*updated_table)); 485 if (!updated_table) 486 return -ENOMEM; 487 if (TAILQ_EMPTY(&drv_list)) { 488 /* Copy the first driver's ID table. */ 489 for (id_iter = driver_id_table; id_iter->vendor_id != 0; 490 id_iter++, i++) 491 updated_table[i] = *id_iter; 492 } else { 493 /* First copy existing table entries. */ 494 for (id_iter = old_table; id_iter->vendor_id != 0; 495 id_iter++, i++) 496 updated_table[i] = *id_iter; 497 /* New id to be added at the end of current ID table. */ 498 pci_id_insert(updated_table, &i, driver_id_table); 499 } 500 /* Terminate table with empty entry. */ 501 updated_table[i].vendor_id = 0; 502 mlx5_pci_driver.id_table = updated_table; 503 mlx5_pci_id_table = updated_table; 504 if (old_table) 505 free(old_table); 506 return 0; 507 } 508 509 void 510 mlx5_pci_driver_register(struct mlx5_pci_driver *driver) 511 { 512 int ret; 513 514 ret = pci_ids_table_update(driver->pci_driver.id_table); 515 if (ret) 516 return; 517 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags; 518 TAILQ_INSERT_TAIL(&drv_list, driver, next); 519 } 520 521 void mlx5_common_pci_init(void) 522 { 523 const struct rte_pci_id empty_table[] = { 524 { 525 .vendor_id = 0 526 }, 527 }; 528 529 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD 530 * including this one can register the PCI table first. If any other 531 * PMD(s) have registered the PCI ID table, No need to register an empty 532 * default one. 533 */ 534 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table)) 535 return; 536 rte_pci_register(&mlx5_pci_driver); 537 } 538 539 RTE_FINI(mlx5_common_pci_finish) 540 { 541 if (mlx5_pci_id_table != NULL) { 542 /* Constructor doesn't register with PCI bus if it failed 543 * to build the table. 544 */ 545 rte_pci_unregister(&mlx5_pci_driver); 546 free(mlx5_pci_id_table); 547 } 548 } 549 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__); 550