1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies Ltd 3 */ 4 5 #include <stdlib.h> 6 #include <rte_malloc.h> 7 #include "mlx5_common_utils.h" 8 #include "mlx5_common_pci.h" 9 10 struct mlx5_pci_device { 11 struct rte_pci_device *pci_dev; 12 TAILQ_ENTRY(mlx5_pci_device) next; 13 uint32_t classes_loaded; 14 }; 15 16 /* Head of list of drivers. */ 17 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list = 18 TAILQ_HEAD_INITIALIZER(drv_list); 19 20 /* Head of mlx5 pci devices. */ 21 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list = 22 TAILQ_HEAD_INITIALIZER(devices_list); 23 24 static const struct { 25 const char *name; 26 unsigned int driver_class; 27 } mlx5_classes[] = { 28 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA }, 29 { .name = "net", .driver_class = MLX5_CLASS_NET }, 30 { .name = "regex", .driver_class = MLX5_CLASS_REGEX }, 31 }; 32 33 static const unsigned int mlx5_class_combinations[] = { 34 MLX5_CLASS_NET, 35 MLX5_CLASS_VDPA, 36 MLX5_CLASS_REGEX, 37 MLX5_CLASS_NET | MLX5_CLASS_REGEX, 38 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX, 39 /* New class combination should be added here. */ 40 }; 41 42 static int 43 class_name_to_value(const char *class_name) 44 { 45 unsigned int i; 46 47 for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 48 if (strcmp(class_name, mlx5_classes[i].name) == 0) 49 return mlx5_classes[i].driver_class; 50 } 51 return -EINVAL; 52 } 53 54 static struct mlx5_pci_driver * 55 driver_get(uint32_t class) 56 { 57 struct mlx5_pci_driver *driver; 58 59 TAILQ_FOREACH(driver, &drv_list, next) { 60 if (driver->driver_class == class) 61 return driver; 62 } 63 return NULL; 64 } 65 66 static int 67 bus_cmdline_options_handler(__rte_unused const char *key, 68 const char *class_names, void *opaque) 69 { 70 int *ret = opaque; 71 char *nstr_org; 72 int class_val; 73 char *found; 74 char *nstr; 75 76 *ret = 0; 77 nstr = strdup(class_names); 78 if (!nstr) { 79 *ret = -ENOMEM; 80 return *ret; 81 } 82 nstr_org = nstr; 83 while (nstr) { 84 /* Extract each individual class name. Multiple 85 * class key,value is supplied as class=net:vdpa:foo:bar. 86 */ 87 found = strsep(&nstr, ":"); 88 if (!found) 89 continue; 90 /* Check if its a valid class. */ 91 class_val = class_name_to_value(found); 92 if (class_val < 0) { 93 *ret = -EINVAL; 94 goto err; 95 } 96 *ret |= class_val; 97 } 98 err: 99 free(nstr_org); 100 if (*ret < 0) 101 DRV_LOG(ERR, "Invalid mlx5 class options %s." 102 " Maybe typo in device class argument setting?", 103 class_names); 104 return *ret; 105 } 106 107 static int 108 parse_class_options(const struct rte_devargs *devargs) 109 { 110 const char *key = MLX5_CLASS_ARG_NAME; 111 struct rte_kvargs *kvlist; 112 int ret = 0; 113 114 if (devargs == NULL) 115 return 0; 116 kvlist = rte_kvargs_parse(devargs->args, NULL); 117 if (kvlist == NULL) 118 return 0; 119 if (rte_kvargs_count(kvlist, key)) 120 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler, 121 &ret); 122 rte_kvargs_free(kvlist); 123 return ret; 124 } 125 126 static bool 127 mlx5_bus_match(const struct mlx5_pci_driver *drv, 128 const struct rte_pci_device *pci_dev) 129 { 130 const struct rte_pci_id *id_table; 131 132 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0; 133 id_table++) { 134 /* Check if device's ids match the class driver's ids. */ 135 if (id_table->vendor_id != pci_dev->id.vendor_id && 136 id_table->vendor_id != PCI_ANY_ID) 137 continue; 138 if (id_table->device_id != pci_dev->id.device_id && 139 id_table->device_id != PCI_ANY_ID) 140 continue; 141 if (id_table->subsystem_vendor_id != 142 pci_dev->id.subsystem_vendor_id && 143 id_table->subsystem_vendor_id != PCI_ANY_ID) 144 continue; 145 if (id_table->subsystem_device_id != 146 pci_dev->id.subsystem_device_id && 147 id_table->subsystem_device_id != PCI_ANY_ID) 148 continue; 149 if (id_table->class_id != pci_dev->id.class_id && 150 id_table->class_id != RTE_CLASS_ANY_ID) 151 continue; 152 return true; 153 } 154 return false; 155 } 156 157 static int 158 is_valid_class_combination(uint32_t user_classes) 159 { 160 unsigned int i; 161 162 /* Verify if user specified valid supported combination. */ 163 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) { 164 if (mlx5_class_combinations[i] == user_classes) 165 return 0; 166 } 167 /* Not found any valid class combination. */ 168 return -EINVAL; 169 } 170 171 static struct mlx5_pci_device * 172 pci_to_mlx5_device(const struct rte_pci_device *pci_dev) 173 { 174 struct mlx5_pci_device *dev; 175 176 TAILQ_FOREACH(dev, &devices_list, next) { 177 if (dev->pci_dev == pci_dev) 178 return dev; 179 } 180 return NULL; 181 } 182 183 static bool 184 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class) 185 { 186 return (device->classes_loaded & class) ? true : false; 187 } 188 189 static void 190 dev_release(struct mlx5_pci_device *dev) 191 { 192 TAILQ_REMOVE(&devices_list, dev, next); 193 rte_free(dev); 194 } 195 196 static int 197 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes) 198 { 199 struct mlx5_pci_driver *driver; 200 int local_ret = -ENODEV; 201 unsigned int i = 0; 202 int ret = 0; 203 204 enabled_classes &= dev->classes_loaded; 205 while (enabled_classes) { 206 driver = driver_get(RTE_BIT64(i)); 207 if (driver) { 208 local_ret = driver->pci_driver.remove(dev->pci_dev); 209 if (!local_ret) 210 dev->classes_loaded &= ~RTE_BIT64(i); 211 else if (ret == 0) 212 ret = local_ret; 213 } 214 enabled_classes &= ~RTE_BIT64(i); 215 i++; 216 } 217 if (local_ret) 218 ret = local_ret; 219 return ret; 220 } 221 222 static int 223 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv, 224 struct rte_pci_device *pci_dev, uint32_t user_classes) 225 { 226 struct mlx5_pci_driver *driver; 227 uint32_t enabled_classes = 0; 228 bool already_loaded; 229 int ret; 230 231 TAILQ_FOREACH(driver, &drv_list, next) { 232 if ((driver->driver_class & user_classes) == 0) 233 continue; 234 if (!mlx5_bus_match(driver, pci_dev)) 235 continue; 236 already_loaded = dev->classes_loaded & driver->driver_class; 237 if (already_loaded && 238 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) { 239 DRV_LOG(ERR, "Device %s is already probed\n", 240 pci_dev->device.name); 241 ret = -EEXIST; 242 goto probe_err; 243 } 244 ret = driver->pci_driver.probe(pci_drv, pci_dev); 245 if (ret < 0) { 246 DRV_LOG(ERR, "Failed to load driver = %s.\n", 247 driver->pci_driver.driver.name); 248 goto probe_err; 249 } 250 enabled_classes |= driver->driver_class; 251 } 252 dev->classes_loaded |= enabled_classes; 253 return 0; 254 probe_err: 255 /* Only unload drivers which are enabled which were enabled 256 * in this probe instance. 257 */ 258 drivers_remove(dev, enabled_classes); 259 return ret; 260 } 261 262 /** 263 * DPDK callback to register to probe multiple drivers for a PCI device. 264 * 265 * @param[in] pci_drv 266 * PCI driver structure. 267 * @param[in] dev 268 * PCI device information. 269 * 270 * @return 271 * 0 on success, a negative errno value otherwise and rte_errno is set. 272 */ 273 static int 274 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 275 struct rte_pci_device *pci_dev) 276 { 277 struct mlx5_pci_device *dev; 278 uint32_t user_classes = 0; 279 bool new_device = false; 280 int ret; 281 282 ret = parse_class_options(pci_dev->device.devargs); 283 if (ret < 0) 284 return ret; 285 user_classes = ret; 286 if (user_classes) { 287 /* Validate combination here. */ 288 ret = is_valid_class_combination(user_classes); 289 if (ret) { 290 DRV_LOG(ERR, "Unsupported mlx5 classes supplied."); 291 return ret; 292 } 293 } else { 294 /* Default to net class. */ 295 user_classes = MLX5_CLASS_NET; 296 } 297 dev = pci_to_mlx5_device(pci_dev); 298 if (!dev) { 299 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0); 300 if (!dev) 301 return -ENOMEM; 302 dev->pci_dev = pci_dev; 303 TAILQ_INSERT_HEAD(&devices_list, dev, next); 304 new_device = true; 305 } 306 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes); 307 if (ret) 308 goto class_err; 309 return 0; 310 class_err: 311 if (new_device) 312 dev_release(dev); 313 return ret; 314 } 315 316 /** 317 * DPDK callback to remove one or more drivers for a PCI device. 318 * 319 * This function removes all drivers probed for a given PCI device. 320 * 321 * @param[in] pci_dev 322 * Pointer to the PCI device. 323 * 324 * @return 325 * 0 on success, the function cannot fail. 326 */ 327 static int 328 mlx5_common_pci_remove(struct rte_pci_device *pci_dev) 329 { 330 struct mlx5_pci_device *dev; 331 int ret; 332 333 dev = pci_to_mlx5_device(pci_dev); 334 if (!dev) 335 return -ENODEV; 336 /* Matching device found, cleanup and unload drivers. */ 337 ret = drivers_remove(dev, dev->classes_loaded); 338 if (!ret) 339 dev_release(dev); 340 return ret; 341 } 342 343 static int 344 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr, 345 uint64_t iova, size_t len) 346 { 347 struct mlx5_pci_driver *driver = NULL; 348 struct mlx5_pci_driver *temp; 349 struct mlx5_pci_device *dev; 350 int ret = -EINVAL; 351 352 dev = pci_to_mlx5_device(pci_dev); 353 if (!dev) 354 return -ENODEV; 355 TAILQ_FOREACH(driver, &drv_list, next) { 356 if (device_class_enabled(dev, driver->driver_class) && 357 driver->pci_driver.dma_map) { 358 ret = driver->pci_driver.dma_map(pci_dev, addr, 359 iova, len); 360 if (ret) 361 goto map_err; 362 } 363 } 364 return ret; 365 map_err: 366 TAILQ_FOREACH(temp, &drv_list, next) { 367 if (temp == driver) 368 break; 369 if (device_class_enabled(dev, temp->driver_class) && 370 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap) 371 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len); 372 } 373 return ret; 374 } 375 376 static int 377 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr, 378 uint64_t iova, size_t len) 379 { 380 struct mlx5_pci_driver *driver; 381 struct mlx5_pci_device *dev; 382 int local_ret = -EINVAL; 383 int ret; 384 385 dev = pci_to_mlx5_device(pci_dev); 386 if (!dev) 387 return -ENODEV; 388 ret = 0; 389 /* There is no unmap error recovery in current implementation. */ 390 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) { 391 if (device_class_enabled(dev, driver->driver_class) && 392 driver->pci_driver.dma_unmap) { 393 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr, 394 iova, len); 395 if (local_ret && (ret == 0)) 396 ret = local_ret; 397 } 398 } 399 if (local_ret) 400 ret = local_ret; 401 return ret; 402 } 403 404 /* PCI ID table is build dynamically based on registered mlx5 drivers. */ 405 static struct rte_pci_id *mlx5_pci_id_table; 406 407 static struct rte_pci_driver mlx5_pci_driver = { 408 .driver = { 409 .name = "mlx5_pci", 410 }, 411 .probe = mlx5_common_pci_probe, 412 .remove = mlx5_common_pci_remove, 413 .dma_map = mlx5_common_pci_dma_map, 414 .dma_unmap = mlx5_common_pci_dma_unmap, 415 }; 416 417 static int 418 pci_id_table_size_get(const struct rte_pci_id *id_table) 419 { 420 int table_size = 0; 421 422 for (; id_table->vendor_id != 0; id_table++) 423 table_size++; 424 return table_size; 425 } 426 427 static bool 428 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table, 429 int next_idx) 430 { 431 int current_size = next_idx - 1; 432 int i; 433 434 for (i = 0; i < current_size; i++) { 435 if (id->device_id == table[i].device_id && 436 id->vendor_id == table[i].vendor_id && 437 id->subsystem_vendor_id == table[i].subsystem_vendor_id && 438 id->subsystem_device_id == table[i].subsystem_device_id) 439 return true; 440 } 441 return false; 442 } 443 444 static void 445 pci_id_insert(struct rte_pci_id *new_table, int *next_idx, 446 const struct rte_pci_id *id_table) 447 { 448 /* Traverse the id_table, check if entry exists in new_table; 449 * Add non duplicate entries to new table. 450 */ 451 for (; id_table->vendor_id != 0; id_table++) { 452 if (!pci_id_exists(id_table, new_table, *next_idx)) { 453 /* New entry; add to the table. */ 454 new_table[*next_idx] = *id_table; 455 (*next_idx)++; 456 } 457 } 458 } 459 460 static int 461 pci_ids_table_update(const struct rte_pci_id *driver_id_table) 462 { 463 const struct rte_pci_id *id_iter; 464 struct rte_pci_id *updated_table; 465 struct rte_pci_id *old_table; 466 int num_ids = 0; 467 int i = 0; 468 469 old_table = mlx5_pci_id_table; 470 if (old_table) 471 num_ids = pci_id_table_size_get(old_table); 472 num_ids += pci_id_table_size_get(driver_id_table); 473 /* Increase size by one for the termination entry of vendor_id = 0. */ 474 num_ids += 1; 475 updated_table = calloc(num_ids, sizeof(*updated_table)); 476 if (!updated_table) 477 return -ENOMEM; 478 if (TAILQ_EMPTY(&drv_list)) { 479 /* Copy the first driver's ID table. */ 480 for (id_iter = driver_id_table; id_iter->vendor_id != 0; 481 id_iter++, i++) 482 updated_table[i] = *id_iter; 483 } else { 484 /* First copy existing table entries. */ 485 for (id_iter = old_table; id_iter->vendor_id != 0; 486 id_iter++, i++) 487 updated_table[i] = *id_iter; 488 /* New id to be added at the end of current ID table. */ 489 pci_id_insert(updated_table, &i, driver_id_table); 490 } 491 /* Terminate table with empty entry. */ 492 updated_table[i].vendor_id = 0; 493 mlx5_pci_driver.id_table = updated_table; 494 mlx5_pci_id_table = updated_table; 495 if (old_table) 496 free(old_table); 497 return 0; 498 } 499 500 void 501 mlx5_pci_driver_register(struct mlx5_pci_driver *driver) 502 { 503 int ret; 504 505 ret = pci_ids_table_update(driver->pci_driver.id_table); 506 if (ret) 507 return; 508 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags; 509 TAILQ_INSERT_TAIL(&drv_list, driver, next); 510 } 511 512 void mlx5_common_pci_init(void) 513 { 514 const struct rte_pci_id empty_table[] = { 515 { 516 .vendor_id = 0 517 }, 518 }; 519 520 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD 521 * including this one can register the PCI table first. If any other 522 * PMD(s) have registered the PCI ID table, No need to register an empty 523 * default one. 524 */ 525 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table)) 526 return; 527 rte_pci_register(&mlx5_pci_driver); 528 } 529 530 RTE_FINI(mlx5_common_pci_finish) 531 { 532 if (mlx5_pci_id_table != NULL) { 533 /* Constructor doesn't register with PCI bus if it failed 534 * to build the table. 535 */ 536 rte_pci_unregister(&mlx5_pci_driver); 537 free(mlx5_pci_id_table); 538 } 539 } 540 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__); 541