18a41f4deSParav Pandit /* SPDX-License-Identifier: BSD-3-Clause 28a41f4deSParav Pandit * Copyright 2020 Mellanox Technologies Ltd 38a41f4deSParav Pandit */ 48a41f4deSParav Pandit 58a41f4deSParav Pandit #include <stdlib.h> 68a41f4deSParav Pandit #include <rte_malloc.h> 725245d5dSShiri Kuzin #include <rte_class.h> 825245d5dSShiri Kuzin 925245d5dSShiri Kuzin #include "mlx5_common_log.h" 108a41f4deSParav Pandit #include "mlx5_common_pci.h" 118a41f4deSParav Pandit 128a41f4deSParav Pandit struct mlx5_pci_device { 138a41f4deSParav Pandit struct rte_pci_device *pci_dev; 148a41f4deSParav Pandit TAILQ_ENTRY(mlx5_pci_device) next; 158a41f4deSParav Pandit uint32_t classes_loaded; 168a41f4deSParav Pandit }; 178a41f4deSParav Pandit 188a41f4deSParav Pandit /* Head of list of drivers. */ 198a41f4deSParav Pandit static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list = 208a41f4deSParav Pandit TAILQ_HEAD_INITIALIZER(drv_list); 218a41f4deSParav Pandit 228a41f4deSParav Pandit /* Head of mlx5 pci devices. */ 238a41f4deSParav Pandit static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list = 248a41f4deSParav Pandit TAILQ_HEAD_INITIALIZER(devices_list); 258a41f4deSParav Pandit 268a41f4deSParav Pandit static const struct { 278a41f4deSParav Pandit const char *name; 288a41f4deSParav Pandit unsigned int driver_class; 298a41f4deSParav Pandit } mlx5_classes[] = { 308a41f4deSParav Pandit { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA }, 31*a99f2f90SXueming Li { .name = "eth", .driver_class = MLX5_CLASS_ETH }, 32*a99f2f90SXueming Li /* Keep name "net" for backward compatibility. */ 33*a99f2f90SXueming Li { .name = "net", .driver_class = MLX5_CLASS_ETH }, 348a41f4deSParav Pandit { .name = "regex", .driver_class = MLX5_CLASS_REGEX }, 35832a4cf1SMatan Azrad { .name = "compress", .driver_class = MLX5_CLASS_COMPRESS }, 36a7c86884SShiri Kuzin { .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO }, 378a41f4deSParav Pandit }; 388a41f4deSParav Pandit 398a41f4deSParav Pandit static const unsigned int mlx5_class_combinations[] = { 40*a99f2f90SXueming Li MLX5_CLASS_ETH, 418a41f4deSParav Pandit MLX5_CLASS_VDPA, 428a41f4deSParav Pandit MLX5_CLASS_REGEX, 43832a4cf1SMatan Azrad MLX5_CLASS_COMPRESS, 44a7c86884SShiri Kuzin MLX5_CLASS_CRYPTO, 45*a99f2f90SXueming Li MLX5_CLASS_ETH | MLX5_CLASS_REGEX, 468a41f4deSParav Pandit MLX5_CLASS_VDPA | MLX5_CLASS_REGEX, 47*a99f2f90SXueming Li MLX5_CLASS_ETH | MLX5_CLASS_COMPRESS, 48832a4cf1SMatan Azrad MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS, 49832a4cf1SMatan Azrad MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 50*a99f2f90SXueming Li MLX5_CLASS_ETH | MLX5_CLASS_CRYPTO, 51*a99f2f90SXueming Li MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 52a7c86884SShiri Kuzin MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO, 53a7c86884SShiri Kuzin MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 54a7c86884SShiri Kuzin MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 55832a4cf1SMatan Azrad MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS, 56*a99f2f90SXueming Li MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 57a7c86884SShiri Kuzin MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO, 58*a99f2f90SXueming Li MLX5_CLASS_ETH | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 59a7c86884SShiri Kuzin MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO, 60*a99f2f90SXueming Li MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS | 61a7c86884SShiri Kuzin MLX5_CLASS_CRYPTO, 62a7c86884SShiri Kuzin MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS | 63a7c86884SShiri Kuzin MLX5_CLASS_CRYPTO, 648a41f4deSParav Pandit /* New class combination should be added here. */ 658a41f4deSParav Pandit }; 668a41f4deSParav Pandit 678a41f4deSParav Pandit static int 688a41f4deSParav Pandit class_name_to_value(const char *class_name) 698a41f4deSParav Pandit { 708a41f4deSParav Pandit unsigned int i; 718a41f4deSParav Pandit 728a41f4deSParav Pandit for (i = 0; i < RTE_DIM(mlx5_classes); i++) { 738a41f4deSParav Pandit if (strcmp(class_name, mlx5_classes[i].name) == 0) 748a41f4deSParav Pandit return mlx5_classes[i].driver_class; 758a41f4deSParav Pandit } 768a41f4deSParav Pandit return -EINVAL; 778a41f4deSParav Pandit } 788a41f4deSParav Pandit 798a41f4deSParav Pandit static struct mlx5_pci_driver * 808a41f4deSParav Pandit driver_get(uint32_t class) 818a41f4deSParav Pandit { 828a41f4deSParav Pandit struct mlx5_pci_driver *driver; 838a41f4deSParav Pandit 848a41f4deSParav Pandit TAILQ_FOREACH(driver, &drv_list, next) { 858a41f4deSParav Pandit if (driver->driver_class == class) 868a41f4deSParav Pandit return driver; 878a41f4deSParav Pandit } 888a41f4deSParav Pandit return NULL; 898a41f4deSParav Pandit } 908a41f4deSParav Pandit 918a41f4deSParav Pandit static int 928a41f4deSParav Pandit bus_cmdline_options_handler(__rte_unused const char *key, 938a41f4deSParav Pandit const char *class_names, void *opaque) 948a41f4deSParav Pandit { 958a41f4deSParav Pandit int *ret = opaque; 968a41f4deSParav Pandit char *nstr_org; 978a41f4deSParav Pandit int class_val; 988a41f4deSParav Pandit char *found; 998a41f4deSParav Pandit char *nstr; 1003475aeacSOphir Munk char *refstr = NULL; 1018a41f4deSParav Pandit 1028a41f4deSParav Pandit *ret = 0; 1038a41f4deSParav Pandit nstr = strdup(class_names); 1048a41f4deSParav Pandit if (!nstr) { 1058a41f4deSParav Pandit *ret = -ENOMEM; 1068a41f4deSParav Pandit return *ret; 1078a41f4deSParav Pandit } 1088a41f4deSParav Pandit nstr_org = nstr; 1093475aeacSOphir Munk found = strtok_r(nstr, ":", &refstr); 1103475aeacSOphir Munk if (!found) 1113475aeacSOphir Munk goto err; 1123475aeacSOphir Munk do { 1138a41f4deSParav Pandit /* Extract each individual class name. Multiple 1148a41f4deSParav Pandit * class key,value is supplied as class=net:vdpa:foo:bar. 1158a41f4deSParav Pandit */ 1168a41f4deSParav Pandit class_val = class_name_to_value(found); 1173475aeacSOphir Munk /* Check if its a valid class. */ 1188a41f4deSParav Pandit if (class_val < 0) { 1198a41f4deSParav Pandit *ret = -EINVAL; 1208a41f4deSParav Pandit goto err; 1218a41f4deSParav Pandit } 1228a41f4deSParav Pandit *ret |= class_val; 1233475aeacSOphir Munk found = strtok_r(NULL, ":", &refstr); 1243475aeacSOphir Munk } while (found); 1258a41f4deSParav Pandit err: 1268a41f4deSParav Pandit free(nstr_org); 1278a41f4deSParav Pandit if (*ret < 0) 1288a41f4deSParav Pandit DRV_LOG(ERR, "Invalid mlx5 class options %s." 1298a41f4deSParav Pandit " Maybe typo in device class argument setting?", 1308a41f4deSParav Pandit class_names); 1318a41f4deSParav Pandit return *ret; 1328a41f4deSParav Pandit } 1338a41f4deSParav Pandit 1348a41f4deSParav Pandit static int 1358a41f4deSParav Pandit parse_class_options(const struct rte_devargs *devargs) 1368a41f4deSParav Pandit { 13735d4f17bSXueming Li const char *key = RTE_DEVARGS_KEY_CLASS; 1388a41f4deSParav Pandit struct rte_kvargs *kvlist; 1398a41f4deSParav Pandit int ret = 0; 1408a41f4deSParav Pandit 1418a41f4deSParav Pandit if (devargs == NULL) 1428a41f4deSParav Pandit return 0; 1438a41f4deSParav Pandit kvlist = rte_kvargs_parse(devargs->args, NULL); 1448a41f4deSParav Pandit if (kvlist == NULL) 1458a41f4deSParav Pandit return 0; 1468a41f4deSParav Pandit if (rte_kvargs_count(kvlist, key)) 1478a41f4deSParav Pandit rte_kvargs_process(kvlist, key, bus_cmdline_options_handler, 1488a41f4deSParav Pandit &ret); 1498a41f4deSParav Pandit rte_kvargs_free(kvlist); 1508a41f4deSParav Pandit return ret; 1518a41f4deSParav Pandit } 1528a41f4deSParav Pandit 1538a41f4deSParav Pandit static bool 1548a41f4deSParav Pandit mlx5_bus_match(const struct mlx5_pci_driver *drv, 1558a41f4deSParav Pandit const struct rte_pci_device *pci_dev) 1568a41f4deSParav Pandit { 1578a41f4deSParav Pandit const struct rte_pci_id *id_table; 1588a41f4deSParav Pandit 1598a41f4deSParav Pandit for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0; 1608a41f4deSParav Pandit id_table++) { 1618a41f4deSParav Pandit /* Check if device's ids match the class driver's ids. */ 1628a41f4deSParav Pandit if (id_table->vendor_id != pci_dev->id.vendor_id && 1634d509afaSThomas Monjalon id_table->vendor_id != RTE_PCI_ANY_ID) 1648a41f4deSParav Pandit continue; 1658a41f4deSParav Pandit if (id_table->device_id != pci_dev->id.device_id && 1664d509afaSThomas Monjalon id_table->device_id != RTE_PCI_ANY_ID) 1678a41f4deSParav Pandit continue; 1688a41f4deSParav Pandit if (id_table->subsystem_vendor_id != 1698a41f4deSParav Pandit pci_dev->id.subsystem_vendor_id && 1704d509afaSThomas Monjalon id_table->subsystem_vendor_id != RTE_PCI_ANY_ID) 1718a41f4deSParav Pandit continue; 1728a41f4deSParav Pandit if (id_table->subsystem_device_id != 1738a41f4deSParav Pandit pci_dev->id.subsystem_device_id && 1744d509afaSThomas Monjalon id_table->subsystem_device_id != RTE_PCI_ANY_ID) 1758a41f4deSParav Pandit continue; 1768a41f4deSParav Pandit if (id_table->class_id != pci_dev->id.class_id && 1778a41f4deSParav Pandit id_table->class_id != RTE_CLASS_ANY_ID) 1788a41f4deSParav Pandit continue; 1798a41f4deSParav Pandit return true; 1808a41f4deSParav Pandit } 1818a41f4deSParav Pandit return false; 1828a41f4deSParav Pandit } 1838a41f4deSParav Pandit 1848a41f4deSParav Pandit static int 1858a41f4deSParav Pandit is_valid_class_combination(uint32_t user_classes) 1868a41f4deSParav Pandit { 1878a41f4deSParav Pandit unsigned int i; 1888a41f4deSParav Pandit 1898a41f4deSParav Pandit /* Verify if user specified valid supported combination. */ 1908a41f4deSParav Pandit for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) { 1918a41f4deSParav Pandit if (mlx5_class_combinations[i] == user_classes) 1928a41f4deSParav Pandit return 0; 1938a41f4deSParav Pandit } 1948a41f4deSParav Pandit /* Not found any valid class combination. */ 1958a41f4deSParav Pandit return -EINVAL; 1968a41f4deSParav Pandit } 1978a41f4deSParav Pandit 1988a41f4deSParav Pandit static struct mlx5_pci_device * 1998a41f4deSParav Pandit pci_to_mlx5_device(const struct rte_pci_device *pci_dev) 2008a41f4deSParav Pandit { 2018a41f4deSParav Pandit struct mlx5_pci_device *dev; 2028a41f4deSParav Pandit 2038a41f4deSParav Pandit TAILQ_FOREACH(dev, &devices_list, next) { 2048a41f4deSParav Pandit if (dev->pci_dev == pci_dev) 2058a41f4deSParav Pandit return dev; 2068a41f4deSParav Pandit } 2078a41f4deSParav Pandit return NULL; 2088a41f4deSParav Pandit } 2098a41f4deSParav Pandit 2108a41f4deSParav Pandit static bool 2118a41f4deSParav Pandit device_class_enabled(const struct mlx5_pci_device *device, uint32_t class) 2128a41f4deSParav Pandit { 2138a41f4deSParav Pandit return (device->classes_loaded & class) ? true : false; 2148a41f4deSParav Pandit } 2158a41f4deSParav Pandit 2168a41f4deSParav Pandit static void 2178a41f4deSParav Pandit dev_release(struct mlx5_pci_device *dev) 2188a41f4deSParav Pandit { 2198a41f4deSParav Pandit TAILQ_REMOVE(&devices_list, dev, next); 2208a41f4deSParav Pandit rte_free(dev); 2218a41f4deSParav Pandit } 2228a41f4deSParav Pandit 2238a41f4deSParav Pandit static int 2248a41f4deSParav Pandit drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes) 2258a41f4deSParav Pandit { 2268a41f4deSParav Pandit struct mlx5_pci_driver *driver; 2278a41f4deSParav Pandit int local_ret = -ENODEV; 2288a41f4deSParav Pandit unsigned int i = 0; 2298a41f4deSParav Pandit int ret = 0; 2308a41f4deSParav Pandit 2318a41f4deSParav Pandit enabled_classes &= dev->classes_loaded; 2328a41f4deSParav Pandit while (enabled_classes) { 2338a41f4deSParav Pandit driver = driver_get(RTE_BIT64(i)); 2348a41f4deSParav Pandit if (driver) { 2358a41f4deSParav Pandit local_ret = driver->pci_driver.remove(dev->pci_dev); 2368a41f4deSParav Pandit if (!local_ret) 2378a41f4deSParav Pandit dev->classes_loaded &= ~RTE_BIT64(i); 2388a41f4deSParav Pandit else if (ret == 0) 2398a41f4deSParav Pandit ret = local_ret; 2408a41f4deSParav Pandit } 2418a41f4deSParav Pandit enabled_classes &= ~RTE_BIT64(i); 2428a41f4deSParav Pandit i++; 2438a41f4deSParav Pandit } 2448a41f4deSParav Pandit if (local_ret) 2458a41f4deSParav Pandit ret = local_ret; 2468a41f4deSParav Pandit return ret; 2478a41f4deSParav Pandit } 2488a41f4deSParav Pandit 2498a41f4deSParav Pandit static int 2508a41f4deSParav Pandit drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv, 2518a41f4deSParav Pandit struct rte_pci_device *pci_dev, uint32_t user_classes) 2528a41f4deSParav Pandit { 2538a41f4deSParav Pandit struct mlx5_pci_driver *driver; 2548a41f4deSParav Pandit uint32_t enabled_classes = 0; 2558a41f4deSParav Pandit bool already_loaded; 2568a41f4deSParav Pandit int ret; 2578a41f4deSParav Pandit 2588a41f4deSParav Pandit TAILQ_FOREACH(driver, &drv_list, next) { 2598a41f4deSParav Pandit if ((driver->driver_class & user_classes) == 0) 2608a41f4deSParav Pandit continue; 2618a41f4deSParav Pandit if (!mlx5_bus_match(driver, pci_dev)) 2628a41f4deSParav Pandit continue; 2638a41f4deSParav Pandit already_loaded = dev->classes_loaded & driver->driver_class; 2648a41f4deSParav Pandit if (already_loaded && 2658a41f4deSParav Pandit !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) { 2661b9e9826SThomas Monjalon DRV_LOG(ERR, "Device %s is already probed", 2678a41f4deSParav Pandit pci_dev->device.name); 2688a41f4deSParav Pandit ret = -EEXIST; 2698a41f4deSParav Pandit goto probe_err; 2708a41f4deSParav Pandit } 2718a41f4deSParav Pandit ret = driver->pci_driver.probe(pci_drv, pci_dev); 2728a41f4deSParav Pandit if (ret < 0) { 2731b9e9826SThomas Monjalon DRV_LOG(ERR, "Failed to load driver %s", 2748a41f4deSParav Pandit driver->pci_driver.driver.name); 2758a41f4deSParav Pandit goto probe_err; 2768a41f4deSParav Pandit } 2778a41f4deSParav Pandit enabled_classes |= driver->driver_class; 2788a41f4deSParav Pandit } 2798a41f4deSParav Pandit dev->classes_loaded |= enabled_classes; 2808a41f4deSParav Pandit return 0; 2818a41f4deSParav Pandit probe_err: 2828a41f4deSParav Pandit /* Only unload drivers which are enabled which were enabled 2838a41f4deSParav Pandit * in this probe instance. 2848a41f4deSParav Pandit */ 2858a41f4deSParav Pandit drivers_remove(dev, enabled_classes); 2868a41f4deSParav Pandit return ret; 2878a41f4deSParav Pandit } 2888a41f4deSParav Pandit 2898a41f4deSParav Pandit /** 2908a41f4deSParav Pandit * DPDK callback to register to probe multiple drivers for a PCI device. 2918a41f4deSParav Pandit * 2928a41f4deSParav Pandit * @param[in] pci_drv 2938a41f4deSParav Pandit * PCI driver structure. 2948a41f4deSParav Pandit * @param[in] dev 2958a41f4deSParav Pandit * PCI device information. 2968a41f4deSParav Pandit * 2978a41f4deSParav Pandit * @return 2988a41f4deSParav Pandit * 0 on success, a negative errno value otherwise and rte_errno is set. 2998a41f4deSParav Pandit */ 3008a41f4deSParav Pandit static int 3018a41f4deSParav Pandit mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 3028a41f4deSParav Pandit struct rte_pci_device *pci_dev) 3038a41f4deSParav Pandit { 3048a41f4deSParav Pandit struct mlx5_pci_device *dev; 3058a41f4deSParav Pandit uint32_t user_classes = 0; 3068a41f4deSParav Pandit bool new_device = false; 3078a41f4deSParav Pandit int ret; 3088a41f4deSParav Pandit 3098a41f4deSParav Pandit ret = parse_class_options(pci_dev->device.devargs); 3108a41f4deSParav Pandit if (ret < 0) 3118a41f4deSParav Pandit return ret; 3128a41f4deSParav Pandit user_classes = ret; 3138a41f4deSParav Pandit if (user_classes) { 3148a41f4deSParav Pandit /* Validate combination here. */ 3158a41f4deSParav Pandit ret = is_valid_class_combination(user_classes); 3168a41f4deSParav Pandit if (ret) { 3178a41f4deSParav Pandit DRV_LOG(ERR, "Unsupported mlx5 classes supplied."); 3188a41f4deSParav Pandit return ret; 3198a41f4deSParav Pandit } 3208a41f4deSParav Pandit } else { 3218a41f4deSParav Pandit /* Default to net class. */ 322*a99f2f90SXueming Li user_classes = MLX5_CLASS_ETH; 3238a41f4deSParav Pandit } 3248a41f4deSParav Pandit dev = pci_to_mlx5_device(pci_dev); 3258a41f4deSParav Pandit if (!dev) { 3268a41f4deSParav Pandit dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0); 3278a41f4deSParav Pandit if (!dev) 3288a41f4deSParav Pandit return -ENOMEM; 3298a41f4deSParav Pandit dev->pci_dev = pci_dev; 3308a41f4deSParav Pandit TAILQ_INSERT_HEAD(&devices_list, dev, next); 3318a41f4deSParav Pandit new_device = true; 3328a41f4deSParav Pandit } 3338a41f4deSParav Pandit ret = drivers_probe(dev, pci_drv, pci_dev, user_classes); 3348a41f4deSParav Pandit if (ret) 3358a41f4deSParav Pandit goto class_err; 3368a41f4deSParav Pandit return 0; 3378a41f4deSParav Pandit class_err: 3388a41f4deSParav Pandit if (new_device) 3398a41f4deSParav Pandit dev_release(dev); 3408a41f4deSParav Pandit return ret; 3418a41f4deSParav Pandit } 3428a41f4deSParav Pandit 3438a41f4deSParav Pandit /** 3448a41f4deSParav Pandit * DPDK callback to remove one or more drivers for a PCI device. 3458a41f4deSParav Pandit * 3468a41f4deSParav Pandit * This function removes all drivers probed for a given PCI device. 3478a41f4deSParav Pandit * 3488a41f4deSParav Pandit * @param[in] pci_dev 3498a41f4deSParav Pandit * Pointer to the PCI device. 3508a41f4deSParav Pandit * 3518a41f4deSParav Pandit * @return 3528a41f4deSParav Pandit * 0 on success, the function cannot fail. 3538a41f4deSParav Pandit */ 3548a41f4deSParav Pandit static int 3558a41f4deSParav Pandit mlx5_common_pci_remove(struct rte_pci_device *pci_dev) 3568a41f4deSParav Pandit { 3578a41f4deSParav Pandit struct mlx5_pci_device *dev; 3588a41f4deSParav Pandit int ret; 3598a41f4deSParav Pandit 3608a41f4deSParav Pandit dev = pci_to_mlx5_device(pci_dev); 3618a41f4deSParav Pandit if (!dev) 3628a41f4deSParav Pandit return -ENODEV; 3638a41f4deSParav Pandit /* Matching device found, cleanup and unload drivers. */ 3648a41f4deSParav Pandit ret = drivers_remove(dev, dev->classes_loaded); 3658a41f4deSParav Pandit if (!ret) 3668a41f4deSParav Pandit dev_release(dev); 3678a41f4deSParav Pandit return ret; 3688a41f4deSParav Pandit } 3698a41f4deSParav Pandit 3708a41f4deSParav Pandit static int 3718a41f4deSParav Pandit mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr, 3728a41f4deSParav Pandit uint64_t iova, size_t len) 3738a41f4deSParav Pandit { 3748a41f4deSParav Pandit struct mlx5_pci_driver *driver = NULL; 3758a41f4deSParav Pandit struct mlx5_pci_driver *temp; 3768a41f4deSParav Pandit struct mlx5_pci_device *dev; 3778a41f4deSParav Pandit int ret = -EINVAL; 3788a41f4deSParav Pandit 3798a41f4deSParav Pandit dev = pci_to_mlx5_device(pci_dev); 3808a41f4deSParav Pandit if (!dev) 3818a41f4deSParav Pandit return -ENODEV; 3828a41f4deSParav Pandit TAILQ_FOREACH(driver, &drv_list, next) { 3838a41f4deSParav Pandit if (device_class_enabled(dev, driver->driver_class) && 3848a41f4deSParav Pandit driver->pci_driver.dma_map) { 3858a41f4deSParav Pandit ret = driver->pci_driver.dma_map(pci_dev, addr, 3868a41f4deSParav Pandit iova, len); 3878a41f4deSParav Pandit if (ret) 3888a41f4deSParav Pandit goto map_err; 3898a41f4deSParav Pandit } 3908a41f4deSParav Pandit } 3918a41f4deSParav Pandit return ret; 3928a41f4deSParav Pandit map_err: 3938a41f4deSParav Pandit TAILQ_FOREACH(temp, &drv_list, next) { 3948a41f4deSParav Pandit if (temp == driver) 3958a41f4deSParav Pandit break; 3968a41f4deSParav Pandit if (device_class_enabled(dev, temp->driver_class) && 3978a41f4deSParav Pandit temp->pci_driver.dma_map && temp->pci_driver.dma_unmap) 3988a41f4deSParav Pandit temp->pci_driver.dma_unmap(pci_dev, addr, iova, len); 3998a41f4deSParav Pandit } 4008a41f4deSParav Pandit return ret; 4018a41f4deSParav Pandit } 4028a41f4deSParav Pandit 4038a41f4deSParav Pandit static int 4048a41f4deSParav Pandit mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr, 4058a41f4deSParav Pandit uint64_t iova, size_t len) 4068a41f4deSParav Pandit { 4078a41f4deSParav Pandit struct mlx5_pci_driver *driver; 4088a41f4deSParav Pandit struct mlx5_pci_device *dev; 4098a41f4deSParav Pandit int local_ret = -EINVAL; 4108a41f4deSParav Pandit int ret; 4118a41f4deSParav Pandit 4128a41f4deSParav Pandit dev = pci_to_mlx5_device(pci_dev); 4138a41f4deSParav Pandit if (!dev) 4148a41f4deSParav Pandit return -ENODEV; 4158a41f4deSParav Pandit ret = 0; 4168a41f4deSParav Pandit /* There is no unmap error recovery in current implementation. */ 4178a41f4deSParav Pandit TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) { 4188a41f4deSParav Pandit if (device_class_enabled(dev, driver->driver_class) && 4198a41f4deSParav Pandit driver->pci_driver.dma_unmap) { 4208a41f4deSParav Pandit local_ret = driver->pci_driver.dma_unmap(pci_dev, addr, 4218a41f4deSParav Pandit iova, len); 4228a41f4deSParav Pandit if (local_ret && (ret == 0)) 4238a41f4deSParav Pandit ret = local_ret; 4248a41f4deSParav Pandit } 4258a41f4deSParav Pandit } 4268a41f4deSParav Pandit if (local_ret) 4278a41f4deSParav Pandit ret = local_ret; 4288a41f4deSParav Pandit return ret; 4298a41f4deSParav Pandit } 4308a41f4deSParav Pandit 4318a41f4deSParav Pandit /* PCI ID table is build dynamically based on registered mlx5 drivers. */ 4328a41f4deSParav Pandit static struct rte_pci_id *mlx5_pci_id_table; 4338a41f4deSParav Pandit 4348a41f4deSParav Pandit static struct rte_pci_driver mlx5_pci_driver = { 4358a41f4deSParav Pandit .driver = { 436188773a2SAsaf Penso .name = MLX5_PCI_DRIVER_NAME, 4378a41f4deSParav Pandit }, 4388a41f4deSParav Pandit .probe = mlx5_common_pci_probe, 4398a41f4deSParav Pandit .remove = mlx5_common_pci_remove, 4408a41f4deSParav Pandit .dma_map = mlx5_common_pci_dma_map, 4418a41f4deSParav Pandit .dma_unmap = mlx5_common_pci_dma_unmap, 4428a41f4deSParav Pandit }; 4438a41f4deSParav Pandit 4448a41f4deSParav Pandit static int 4458a41f4deSParav Pandit pci_id_table_size_get(const struct rte_pci_id *id_table) 4468a41f4deSParav Pandit { 4478a41f4deSParav Pandit int table_size = 0; 4488a41f4deSParav Pandit 4498a41f4deSParav Pandit for (; id_table->vendor_id != 0; id_table++) 4508a41f4deSParav Pandit table_size++; 4518a41f4deSParav Pandit return table_size; 4528a41f4deSParav Pandit } 4538a41f4deSParav Pandit 4548a41f4deSParav Pandit static bool 4558a41f4deSParav Pandit pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table, 4568a41f4deSParav Pandit int next_idx) 4578a41f4deSParav Pandit { 4588a41f4deSParav Pandit int current_size = next_idx - 1; 4598a41f4deSParav Pandit int i; 4608a41f4deSParav Pandit 4618a41f4deSParav Pandit for (i = 0; i < current_size; i++) { 4628a41f4deSParav Pandit if (id->device_id == table[i].device_id && 4638a41f4deSParav Pandit id->vendor_id == table[i].vendor_id && 4648a41f4deSParav Pandit id->subsystem_vendor_id == table[i].subsystem_vendor_id && 4658a41f4deSParav Pandit id->subsystem_device_id == table[i].subsystem_device_id) 4668a41f4deSParav Pandit return true; 4678a41f4deSParav Pandit } 4688a41f4deSParav Pandit return false; 4698a41f4deSParav Pandit } 4708a41f4deSParav Pandit 4718a41f4deSParav Pandit static void 4728a41f4deSParav Pandit pci_id_insert(struct rte_pci_id *new_table, int *next_idx, 4738a41f4deSParav Pandit const struct rte_pci_id *id_table) 4748a41f4deSParav Pandit { 4758a41f4deSParav Pandit /* Traverse the id_table, check if entry exists in new_table; 4768a41f4deSParav Pandit * Add non duplicate entries to new table. 4778a41f4deSParav Pandit */ 4788a41f4deSParav Pandit for (; id_table->vendor_id != 0; id_table++) { 4798a41f4deSParav Pandit if (!pci_id_exists(id_table, new_table, *next_idx)) { 4808a41f4deSParav Pandit /* New entry; add to the table. */ 4818a41f4deSParav Pandit new_table[*next_idx] = *id_table; 4828a41f4deSParav Pandit (*next_idx)++; 4838a41f4deSParav Pandit } 4848a41f4deSParav Pandit } 4858a41f4deSParav Pandit } 4868a41f4deSParav Pandit 4878a41f4deSParav Pandit static int 4888a41f4deSParav Pandit pci_ids_table_update(const struct rte_pci_id *driver_id_table) 4898a41f4deSParav Pandit { 4908a41f4deSParav Pandit const struct rte_pci_id *id_iter; 4918a41f4deSParav Pandit struct rte_pci_id *updated_table; 4928a41f4deSParav Pandit struct rte_pci_id *old_table; 4938a41f4deSParav Pandit int num_ids = 0; 4948a41f4deSParav Pandit int i = 0; 4958a41f4deSParav Pandit 4968a41f4deSParav Pandit old_table = mlx5_pci_id_table; 4978a41f4deSParav Pandit if (old_table) 4988a41f4deSParav Pandit num_ids = pci_id_table_size_get(old_table); 4998a41f4deSParav Pandit num_ids += pci_id_table_size_get(driver_id_table); 5008a41f4deSParav Pandit /* Increase size by one for the termination entry of vendor_id = 0. */ 5018a41f4deSParav Pandit num_ids += 1; 5028a41f4deSParav Pandit updated_table = calloc(num_ids, sizeof(*updated_table)); 5038a41f4deSParav Pandit if (!updated_table) 5048a41f4deSParav Pandit return -ENOMEM; 5058a41f4deSParav Pandit if (TAILQ_EMPTY(&drv_list)) { 5068a41f4deSParav Pandit /* Copy the first driver's ID table. */ 5078a41f4deSParav Pandit for (id_iter = driver_id_table; id_iter->vendor_id != 0; 5088a41f4deSParav Pandit id_iter++, i++) 5098a41f4deSParav Pandit updated_table[i] = *id_iter; 5108a41f4deSParav Pandit } else { 5118a41f4deSParav Pandit /* First copy existing table entries. */ 5128a41f4deSParav Pandit for (id_iter = old_table; id_iter->vendor_id != 0; 5138a41f4deSParav Pandit id_iter++, i++) 5148a41f4deSParav Pandit updated_table[i] = *id_iter; 5158a41f4deSParav Pandit /* New id to be added at the end of current ID table. */ 5168a41f4deSParav Pandit pci_id_insert(updated_table, &i, driver_id_table); 5178a41f4deSParav Pandit } 5188a41f4deSParav Pandit /* Terminate table with empty entry. */ 5198a41f4deSParav Pandit updated_table[i].vendor_id = 0; 5208a41f4deSParav Pandit mlx5_pci_driver.id_table = updated_table; 5218a41f4deSParav Pandit mlx5_pci_id_table = updated_table; 5228a41f4deSParav Pandit if (old_table) 5238a41f4deSParav Pandit free(old_table); 5248a41f4deSParav Pandit return 0; 5258a41f4deSParav Pandit } 5268a41f4deSParav Pandit 5278a41f4deSParav Pandit void 5288a41f4deSParav Pandit mlx5_pci_driver_register(struct mlx5_pci_driver *driver) 5298a41f4deSParav Pandit { 5308a41f4deSParav Pandit int ret; 5318a41f4deSParav Pandit 5328a41f4deSParav Pandit ret = pci_ids_table_update(driver->pci_driver.id_table); 5338a41f4deSParav Pandit if (ret) 5348a41f4deSParav Pandit return; 5358a41f4deSParav Pandit mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags; 5368a41f4deSParav Pandit TAILQ_INSERT_TAIL(&drv_list, driver, next); 5378a41f4deSParav Pandit } 5388a41f4deSParav Pandit 5398a41f4deSParav Pandit void mlx5_common_pci_init(void) 5408a41f4deSParav Pandit { 5418a41f4deSParav Pandit const struct rte_pci_id empty_table[] = { 5428a41f4deSParav Pandit { 5438a41f4deSParav Pandit .vendor_id = 0 5448a41f4deSParav Pandit }, 5458a41f4deSParav Pandit }; 5468a41f4deSParav Pandit 5478a41f4deSParav Pandit /* All mlx5 PMDs constructor runs at same priority. So any of the PMD 5488a41f4deSParav Pandit * including this one can register the PCI table first. If any other 5498a41f4deSParav Pandit * PMD(s) have registered the PCI ID table, No need to register an empty 5508a41f4deSParav Pandit * default one. 5518a41f4deSParav Pandit */ 5528a41f4deSParav Pandit if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table)) 5538a41f4deSParav Pandit return; 5548a41f4deSParav Pandit rte_pci_register(&mlx5_pci_driver); 5558a41f4deSParav Pandit } 5568a41f4deSParav Pandit 5578a41f4deSParav Pandit RTE_FINI(mlx5_common_pci_finish) 5588a41f4deSParav Pandit { 5598a41f4deSParav Pandit if (mlx5_pci_id_table != NULL) { 5608a41f4deSParav Pandit /* Constructor doesn't register with PCI bus if it failed 5618a41f4deSParav Pandit * to build the table. 5628a41f4deSParav Pandit */ 5638a41f4deSParav Pandit rte_pci_unregister(&mlx5_pci_driver); 5648a41f4deSParav Pandit free(mlx5_pci_id_table); 5658a41f4deSParav Pandit } 5668a41f4deSParav Pandit } 5678a41f4deSParav Pandit RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__); 568