1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies Ltd 3 */ 4 5 #include <stdlib.h> 6 #include <rte_malloc.h> 7 #include "mlx5_common_utils.h" 8 #include "mlx5_common_pci.h" 9 10 struct mlx5_pci_device { 11 struct rte_pci_device *pci_dev; 12 TAILQ_ENTRY(mlx5_pci_device) next; 13 uint32_t classes_loaded; 14 }; 15 16 /* Head of list of drivers. */ 17 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list = 18 TAILQ_HEAD_INITIALIZER(drv_list); 19 20 /* Head of mlx5 pci devices. */ 21 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list = 22 TAILQ_HEAD_INITIALIZER(devices_list); 23 24 static const struct { 25 const char *name; 26 unsigned int driver_class; 27 } mlx5_classes[] = { 28 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA }, 29 { .name = "net", .driver_class = MLX5_CLASS_NET }, 30 { .name = "regex", .driver_class = MLX5_CLASS_REGEX }, 31 }; 32 33 static const unsigned int mlx5_class_combinations[] = { 34 MLX5_CLASS_NET, 35 MLX5_CLASS_VDPA, 36 MLX5_CLASS_REGEX, 37 MLX5_CLASS_NET | MLX5_CLASS_REGEX, 38 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX, 39 /* New class combination should be added here. */ 40 }; 41 42 static int 43 class_name_to_value(const char *class_name) 44 { 45 unsigned int i; 46 47 for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 48 if (strcmp(class_name, mlx5_classes[i].name) == 0) 49 return mlx5_classes[i].driver_class; 50 } 51 return -EINVAL; 52 } 53 54 static struct mlx5_pci_driver * 55 driver_get(uint32_t class) 56 { 57 struct mlx5_pci_driver *driver; 58 59 TAILQ_FOREACH(driver, &drv_list, next) { 60 if (driver->driver_class == class) 61 return driver; 62 } 63 return NULL; 64 } 65 66 static int 67 bus_cmdline_options_handler(__rte_unused const char *key, 68 const char *class_names, void *opaque) 69 { 70 int *ret = opaque; 71 char *nstr_org; 72 int class_val; 73 char *found; 74 char *nstr; 75 char *refstr = NULL; 76 77 *ret = 0; 78 nstr = strdup(class_names); 79 if (!nstr) { 80 *ret = -ENOMEM; 81 return *ret; 82 } 83 nstr_org = nstr; 84 found = strtok_r(nstr, ":", &refstr); 85 if (!found) 86 goto err; 87 do { 88 /* Extract each individual class name. Multiple 89 * class key,value is supplied as class=net:vdpa:foo:bar. 90 */ 91 class_val = class_name_to_value(found); 92 /* Check if its a valid class. */ 93 if (class_val < 0) { 94 *ret = -EINVAL; 95 goto err; 96 } 97 *ret |= class_val; 98 found = strtok_r(NULL, ":", &refstr); 99 } while (found); 100 err: 101 free(nstr_org); 102 if (*ret < 0) 103 DRV_LOG(ERR, "Invalid mlx5 class options %s." 104 " Maybe typo in device class argument setting?", 105 class_names); 106 return *ret; 107 } 108 109 static int 110 parse_class_options(const struct rte_devargs *devargs) 111 { 112 const char *key = MLX5_CLASS_ARG_NAME; 113 struct rte_kvargs *kvlist; 114 int ret = 0; 115 116 if (devargs == NULL) 117 return 0; 118 kvlist = rte_kvargs_parse(devargs->args, NULL); 119 if (kvlist == NULL) 120 return 0; 121 if (rte_kvargs_count(kvlist, key)) 122 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler, 123 &ret); 124 rte_kvargs_free(kvlist); 125 return ret; 126 } 127 128 static bool 129 mlx5_bus_match(const struct mlx5_pci_driver *drv, 130 const struct rte_pci_device *pci_dev) 131 { 132 const struct rte_pci_id *id_table; 133 134 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0; 135 id_table++) { 136 /* Check if device's ids match the class driver's ids. */ 137 if (id_table->vendor_id != pci_dev->id.vendor_id && 138 id_table->vendor_id != PCI_ANY_ID) 139 continue; 140 if (id_table->device_id != pci_dev->id.device_id && 141 id_table->device_id != PCI_ANY_ID) 142 continue; 143 if (id_table->subsystem_vendor_id != 144 pci_dev->id.subsystem_vendor_id && 145 id_table->subsystem_vendor_id != PCI_ANY_ID) 146 continue; 147 if (id_table->subsystem_device_id != 148 pci_dev->id.subsystem_device_id && 149 id_table->subsystem_device_id != PCI_ANY_ID) 150 continue; 151 if (id_table->class_id != pci_dev->id.class_id && 152 id_table->class_id != RTE_CLASS_ANY_ID) 153 continue; 154 return true; 155 } 156 return false; 157 } 158 159 static int 160 is_valid_class_combination(uint32_t user_classes) 161 { 162 unsigned int i; 163 164 /* Verify if user specified valid supported combination. */ 165 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) { 166 if (mlx5_class_combinations[i] == user_classes) 167 return 0; 168 } 169 /* Not found any valid class combination. */ 170 return -EINVAL; 171 } 172 173 static struct mlx5_pci_device * 174 pci_to_mlx5_device(const struct rte_pci_device *pci_dev) 175 { 176 struct mlx5_pci_device *dev; 177 178 TAILQ_FOREACH(dev, &devices_list, next) { 179 if (dev->pci_dev == pci_dev) 180 return dev; 181 } 182 return NULL; 183 } 184 185 static bool 186 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class) 187 { 188 return (device->classes_loaded & class) ? true : false; 189 } 190 191 static void 192 dev_release(struct mlx5_pci_device *dev) 193 { 194 TAILQ_REMOVE(&devices_list, dev, next); 195 rte_free(dev); 196 } 197 198 static int 199 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes) 200 { 201 struct mlx5_pci_driver *driver; 202 int local_ret = -ENODEV; 203 unsigned int i = 0; 204 int ret = 0; 205 206 enabled_classes &= dev->classes_loaded; 207 while (enabled_classes) { 208 driver = driver_get(RTE_BIT64(i)); 209 if (driver) { 210 local_ret = driver->pci_driver.remove(dev->pci_dev); 211 if (!local_ret) 212 dev->classes_loaded &= ~RTE_BIT64(i); 213 else if (ret == 0) 214 ret = local_ret; 215 } 216 enabled_classes &= ~RTE_BIT64(i); 217 i++; 218 } 219 if (local_ret) 220 ret = local_ret; 221 return ret; 222 } 223 224 static int 225 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv, 226 struct rte_pci_device *pci_dev, uint32_t user_classes) 227 { 228 struct mlx5_pci_driver *driver; 229 uint32_t enabled_classes = 0; 230 bool already_loaded; 231 int ret; 232 233 TAILQ_FOREACH(driver, &drv_list, next) { 234 if ((driver->driver_class & user_classes) == 0) 235 continue; 236 if (!mlx5_bus_match(driver, pci_dev)) 237 continue; 238 already_loaded = dev->classes_loaded & driver->driver_class; 239 if (already_loaded && 240 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) { 241 DRV_LOG(ERR, "Device %s is already probed\n", 242 pci_dev->device.name); 243 ret = -EEXIST; 244 goto probe_err; 245 } 246 ret = driver->pci_driver.probe(pci_drv, pci_dev); 247 if (ret < 0) { 248 DRV_LOG(ERR, "Failed to load driver = %s.\n", 249 driver->pci_driver.driver.name); 250 goto probe_err; 251 } 252 enabled_classes |= driver->driver_class; 253 } 254 dev->classes_loaded |= enabled_classes; 255 return 0; 256 probe_err: 257 /* Only unload drivers which are enabled which were enabled 258 * in this probe instance. 259 */ 260 drivers_remove(dev, enabled_classes); 261 return ret; 262 } 263 264 /** 265 * DPDK callback to register to probe multiple drivers for a PCI device. 266 * 267 * @param[in] pci_drv 268 * PCI driver structure. 269 * @param[in] dev 270 * PCI device information. 271 * 272 * @return 273 * 0 on success, a negative errno value otherwise and rte_errno is set. 274 */ 275 static int 276 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 277 struct rte_pci_device *pci_dev) 278 { 279 struct mlx5_pci_device *dev; 280 uint32_t user_classes = 0; 281 bool new_device = false; 282 int ret; 283 284 ret = parse_class_options(pci_dev->device.devargs); 285 if (ret < 0) 286 return ret; 287 user_classes = ret; 288 if (user_classes) { 289 /* Validate combination here. */ 290 ret = is_valid_class_combination(user_classes); 291 if (ret) { 292 DRV_LOG(ERR, "Unsupported mlx5 classes supplied."); 293 return ret; 294 } 295 } else { 296 /* Default to net class. */ 297 user_classes = MLX5_CLASS_NET; 298 } 299 dev = pci_to_mlx5_device(pci_dev); 300 if (!dev) { 301 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0); 302 if (!dev) 303 return -ENOMEM; 304 dev->pci_dev = pci_dev; 305 TAILQ_INSERT_HEAD(&devices_list, dev, next); 306 new_device = true; 307 } 308 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes); 309 if (ret) 310 goto class_err; 311 return 0; 312 class_err: 313 if (new_device) 314 dev_release(dev); 315 return ret; 316 } 317 318 /** 319 * DPDK callback to remove one or more drivers for a PCI device. 320 * 321 * This function removes all drivers probed for a given PCI device. 322 * 323 * @param[in] pci_dev 324 * Pointer to the PCI device. 325 * 326 * @return 327 * 0 on success, the function cannot fail. 328 */ 329 static int 330 mlx5_common_pci_remove(struct rte_pci_device *pci_dev) 331 { 332 struct mlx5_pci_device *dev; 333 int ret; 334 335 dev = pci_to_mlx5_device(pci_dev); 336 if (!dev) 337 return -ENODEV; 338 /* Matching device found, cleanup and unload drivers. */ 339 ret = drivers_remove(dev, dev->classes_loaded); 340 if (!ret) 341 dev_release(dev); 342 return ret; 343 } 344 345 static int 346 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr, 347 uint64_t iova, size_t len) 348 { 349 struct mlx5_pci_driver *driver = NULL; 350 struct mlx5_pci_driver *temp; 351 struct mlx5_pci_device *dev; 352 int ret = -EINVAL; 353 354 dev = pci_to_mlx5_device(pci_dev); 355 if (!dev) 356 return -ENODEV; 357 TAILQ_FOREACH(driver, &drv_list, next) { 358 if (device_class_enabled(dev, driver->driver_class) && 359 driver->pci_driver.dma_map) { 360 ret = driver->pci_driver.dma_map(pci_dev, addr, 361 iova, len); 362 if (ret) 363 goto map_err; 364 } 365 } 366 return ret; 367 map_err: 368 TAILQ_FOREACH(temp, &drv_list, next) { 369 if (temp == driver) 370 break; 371 if (device_class_enabled(dev, temp->driver_class) && 372 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap) 373 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len); 374 } 375 return ret; 376 } 377 378 static int 379 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr, 380 uint64_t iova, size_t len) 381 { 382 struct mlx5_pci_driver *driver; 383 struct mlx5_pci_device *dev; 384 int local_ret = -EINVAL; 385 int ret; 386 387 dev = pci_to_mlx5_device(pci_dev); 388 if (!dev) 389 return -ENODEV; 390 ret = 0; 391 /* There is no unmap error recovery in current implementation. */ 392 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) { 393 if (device_class_enabled(dev, driver->driver_class) && 394 driver->pci_driver.dma_unmap) { 395 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr, 396 iova, len); 397 if (local_ret && (ret == 0)) 398 ret = local_ret; 399 } 400 } 401 if (local_ret) 402 ret = local_ret; 403 return ret; 404 } 405 406 /* PCI ID table is build dynamically based on registered mlx5 drivers. */ 407 static struct rte_pci_id *mlx5_pci_id_table; 408 409 static struct rte_pci_driver mlx5_pci_driver = { 410 .driver = { 411 .name = "mlx5_pci", 412 }, 413 .probe = mlx5_common_pci_probe, 414 .remove = mlx5_common_pci_remove, 415 .dma_map = mlx5_common_pci_dma_map, 416 .dma_unmap = mlx5_common_pci_dma_unmap, 417 }; 418 419 static int 420 pci_id_table_size_get(const struct rte_pci_id *id_table) 421 { 422 int table_size = 0; 423 424 for (; id_table->vendor_id != 0; id_table++) 425 table_size++; 426 return table_size; 427 } 428 429 static bool 430 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table, 431 int next_idx) 432 { 433 int current_size = next_idx - 1; 434 int i; 435 436 for (i = 0; i < current_size; i++) { 437 if (id->device_id == table[i].device_id && 438 id->vendor_id == table[i].vendor_id && 439 id->subsystem_vendor_id == table[i].subsystem_vendor_id && 440 id->subsystem_device_id == table[i].subsystem_device_id) 441 return true; 442 } 443 return false; 444 } 445 446 static void 447 pci_id_insert(struct rte_pci_id *new_table, int *next_idx, 448 const struct rte_pci_id *id_table) 449 { 450 /* Traverse the id_table, check if entry exists in new_table; 451 * Add non duplicate entries to new table. 452 */ 453 for (; id_table->vendor_id != 0; id_table++) { 454 if (!pci_id_exists(id_table, new_table, *next_idx)) { 455 /* New entry; add to the table. */ 456 new_table[*next_idx] = *id_table; 457 (*next_idx)++; 458 } 459 } 460 } 461 462 static int 463 pci_ids_table_update(const struct rte_pci_id *driver_id_table) 464 { 465 const struct rte_pci_id *id_iter; 466 struct rte_pci_id *updated_table; 467 struct rte_pci_id *old_table; 468 int num_ids = 0; 469 int i = 0; 470 471 old_table = mlx5_pci_id_table; 472 if (old_table) 473 num_ids = pci_id_table_size_get(old_table); 474 num_ids += pci_id_table_size_get(driver_id_table); 475 /* Increase size by one for the termination entry of vendor_id = 0. */ 476 num_ids += 1; 477 updated_table = calloc(num_ids, sizeof(*updated_table)); 478 if (!updated_table) 479 return -ENOMEM; 480 if (TAILQ_EMPTY(&drv_list)) { 481 /* Copy the first driver's ID table. */ 482 for (id_iter = driver_id_table; id_iter->vendor_id != 0; 483 id_iter++, i++) 484 updated_table[i] = *id_iter; 485 } else { 486 /* First copy existing table entries. */ 487 for (id_iter = old_table; id_iter->vendor_id != 0; 488 id_iter++, i++) 489 updated_table[i] = *id_iter; 490 /* New id to be added at the end of current ID table. */ 491 pci_id_insert(updated_table, &i, driver_id_table); 492 } 493 /* Terminate table with empty entry. */ 494 updated_table[i].vendor_id = 0; 495 mlx5_pci_driver.id_table = updated_table; 496 mlx5_pci_id_table = updated_table; 497 if (old_table) 498 free(old_table); 499 return 0; 500 } 501 502 void 503 mlx5_pci_driver_register(struct mlx5_pci_driver *driver) 504 { 505 int ret; 506 507 ret = pci_ids_table_update(driver->pci_driver.id_table); 508 if (ret) 509 return; 510 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags; 511 TAILQ_INSERT_TAIL(&drv_list, driver, next); 512 } 513 514 void mlx5_common_pci_init(void) 515 { 516 const struct rte_pci_id empty_table[] = { 517 { 518 .vendor_id = 0 519 }, 520 }; 521 522 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD 523 * including this one can register the PCI table first. If any other 524 * PMD(s) have registered the PCI ID table, No need to register an empty 525 * default one. 526 */ 527 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table)) 528 return; 529 rte_pci_register(&mlx5_pci_driver); 530 } 531 532 RTE_FINI(mlx5_common_pci_finish) 533 { 534 if (mlx5_pci_id_table != NULL) { 535 /* Constructor doesn't register with PCI bus if it failed 536 * to build the table. 537 */ 538 rte_pci_unregister(&mlx5_pci_driver); 539 free(mlx5_pci_id_table); 540 } 541 } 542 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__); 543