xref: /dpdk/drivers/common/mlx5/mlx5_common_pci.c (revision a99f2f905479271a4878bb5d06dfaff33eb38057)
18a41f4deSParav Pandit /* SPDX-License-Identifier: BSD-3-Clause
28a41f4deSParav Pandit  * Copyright 2020 Mellanox Technologies Ltd
38a41f4deSParav Pandit  */
48a41f4deSParav Pandit 
58a41f4deSParav Pandit #include <stdlib.h>
68a41f4deSParav Pandit #include <rte_malloc.h>
725245d5dSShiri Kuzin #include <rte_class.h>
825245d5dSShiri Kuzin 
925245d5dSShiri Kuzin #include "mlx5_common_log.h"
108a41f4deSParav Pandit #include "mlx5_common_pci.h"
118a41f4deSParav Pandit 
128a41f4deSParav Pandit struct mlx5_pci_device {
138a41f4deSParav Pandit 	struct rte_pci_device *pci_dev;
148a41f4deSParav Pandit 	TAILQ_ENTRY(mlx5_pci_device) next;
158a41f4deSParav Pandit 	uint32_t classes_loaded;
168a41f4deSParav Pandit };
178a41f4deSParav Pandit 
188a41f4deSParav Pandit /* Head of list of drivers. */
198a41f4deSParav Pandit static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list =
208a41f4deSParav Pandit 				TAILQ_HEAD_INITIALIZER(drv_list);
218a41f4deSParav Pandit 
228a41f4deSParav Pandit /* Head of mlx5 pci devices. */
238a41f4deSParav Pandit static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list =
248a41f4deSParav Pandit 				TAILQ_HEAD_INITIALIZER(devices_list);
258a41f4deSParav Pandit 
268a41f4deSParav Pandit static const struct {
278a41f4deSParav Pandit 	const char *name;
288a41f4deSParav Pandit 	unsigned int driver_class;
298a41f4deSParav Pandit } mlx5_classes[] = {
308a41f4deSParav Pandit 	{ .name = "vdpa", .driver_class = MLX5_CLASS_VDPA },
31*a99f2f90SXueming Li 	{ .name = "eth", .driver_class = MLX5_CLASS_ETH },
32*a99f2f90SXueming Li 	/* Keep name "net" for backward compatibility. */
33*a99f2f90SXueming Li 	{ .name = "net", .driver_class = MLX5_CLASS_ETH },
348a41f4deSParav Pandit 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
35832a4cf1SMatan Azrad 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
36a7c86884SShiri Kuzin 	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
378a41f4deSParav Pandit };
388a41f4deSParav Pandit 
398a41f4deSParav Pandit static const unsigned int mlx5_class_combinations[] = {
40*a99f2f90SXueming Li 	MLX5_CLASS_ETH,
418a41f4deSParav Pandit 	MLX5_CLASS_VDPA,
428a41f4deSParav Pandit 	MLX5_CLASS_REGEX,
43832a4cf1SMatan Azrad 	MLX5_CLASS_COMPRESS,
44a7c86884SShiri Kuzin 	MLX5_CLASS_CRYPTO,
45*a99f2f90SXueming Li 	MLX5_CLASS_ETH | MLX5_CLASS_REGEX,
468a41f4deSParav Pandit 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
47*a99f2f90SXueming Li 	MLX5_CLASS_ETH | MLX5_CLASS_COMPRESS,
48832a4cf1SMatan Azrad 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
49832a4cf1SMatan Azrad 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
50*a99f2f90SXueming Li 	MLX5_CLASS_ETH | MLX5_CLASS_CRYPTO,
51*a99f2f90SXueming Li 	MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
52a7c86884SShiri Kuzin 	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
53a7c86884SShiri Kuzin 	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
54a7c86884SShiri Kuzin 	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
55832a4cf1SMatan Azrad 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
56*a99f2f90SXueming Li 	MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
57a7c86884SShiri Kuzin 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
58*a99f2f90SXueming Li 	MLX5_CLASS_ETH | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
59a7c86884SShiri Kuzin 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
60*a99f2f90SXueming Li 	MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
61a7c86884SShiri Kuzin 	MLX5_CLASS_CRYPTO,
62a7c86884SShiri Kuzin 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
63a7c86884SShiri Kuzin 	MLX5_CLASS_CRYPTO,
648a41f4deSParav Pandit 	/* New class combination should be added here. */
658a41f4deSParav Pandit };
668a41f4deSParav Pandit 
678a41f4deSParav Pandit static int
688a41f4deSParav Pandit class_name_to_value(const char *class_name)
698a41f4deSParav Pandit {
708a41f4deSParav Pandit 	unsigned int i;
718a41f4deSParav Pandit 
728a41f4deSParav Pandit 	for (i = 0; i < RTE_DIM(mlx5_classes); i++) {
738a41f4deSParav Pandit 		if (strcmp(class_name, mlx5_classes[i].name) == 0)
748a41f4deSParav Pandit 			return mlx5_classes[i].driver_class;
758a41f4deSParav Pandit 	}
768a41f4deSParav Pandit 	return -EINVAL;
778a41f4deSParav Pandit }
788a41f4deSParav Pandit 
798a41f4deSParav Pandit static struct mlx5_pci_driver *
808a41f4deSParav Pandit driver_get(uint32_t class)
818a41f4deSParav Pandit {
828a41f4deSParav Pandit 	struct mlx5_pci_driver *driver;
838a41f4deSParav Pandit 
848a41f4deSParav Pandit 	TAILQ_FOREACH(driver, &drv_list, next) {
858a41f4deSParav Pandit 		if (driver->driver_class == class)
868a41f4deSParav Pandit 			return driver;
878a41f4deSParav Pandit 	}
888a41f4deSParav Pandit 	return NULL;
898a41f4deSParav Pandit }
908a41f4deSParav Pandit 
918a41f4deSParav Pandit static int
928a41f4deSParav Pandit bus_cmdline_options_handler(__rte_unused const char *key,
938a41f4deSParav Pandit 			    const char *class_names, void *opaque)
948a41f4deSParav Pandit {
958a41f4deSParav Pandit 	int *ret = opaque;
968a41f4deSParav Pandit 	char *nstr_org;
978a41f4deSParav Pandit 	int class_val;
988a41f4deSParav Pandit 	char *found;
998a41f4deSParav Pandit 	char *nstr;
1003475aeacSOphir Munk 	char *refstr = NULL;
1018a41f4deSParav Pandit 
1028a41f4deSParav Pandit 	*ret = 0;
1038a41f4deSParav Pandit 	nstr = strdup(class_names);
1048a41f4deSParav Pandit 	if (!nstr) {
1058a41f4deSParav Pandit 		*ret = -ENOMEM;
1068a41f4deSParav Pandit 		return *ret;
1078a41f4deSParav Pandit 	}
1088a41f4deSParav Pandit 	nstr_org = nstr;
1093475aeacSOphir Munk 	found = strtok_r(nstr, ":", &refstr);
1103475aeacSOphir Munk 	if (!found)
1113475aeacSOphir Munk 		goto err;
1123475aeacSOphir Munk 	do {
1138a41f4deSParav Pandit 		/* Extract each individual class name. Multiple
1148a41f4deSParav Pandit 		 * class key,value is supplied as class=net:vdpa:foo:bar.
1158a41f4deSParav Pandit 		 */
1168a41f4deSParav Pandit 		class_val = class_name_to_value(found);
1173475aeacSOphir Munk 		/* Check if its a valid class. */
1188a41f4deSParav Pandit 		if (class_val < 0) {
1198a41f4deSParav Pandit 			*ret = -EINVAL;
1208a41f4deSParav Pandit 			goto err;
1218a41f4deSParav Pandit 		}
1228a41f4deSParav Pandit 		*ret |= class_val;
1233475aeacSOphir Munk 		found = strtok_r(NULL, ":", &refstr);
1243475aeacSOphir Munk 	} while (found);
1258a41f4deSParav Pandit err:
1268a41f4deSParav Pandit 	free(nstr_org);
1278a41f4deSParav Pandit 	if (*ret < 0)
1288a41f4deSParav Pandit 		DRV_LOG(ERR, "Invalid mlx5 class options %s."
1298a41f4deSParav Pandit 			" Maybe typo in device class argument setting?",
1308a41f4deSParav Pandit 			class_names);
1318a41f4deSParav Pandit 	return *ret;
1328a41f4deSParav Pandit }
1338a41f4deSParav Pandit 
1348a41f4deSParav Pandit static int
1358a41f4deSParav Pandit parse_class_options(const struct rte_devargs *devargs)
1368a41f4deSParav Pandit {
13735d4f17bSXueming Li 	const char *key = RTE_DEVARGS_KEY_CLASS;
1388a41f4deSParav Pandit 	struct rte_kvargs *kvlist;
1398a41f4deSParav Pandit 	int ret = 0;
1408a41f4deSParav Pandit 
1418a41f4deSParav Pandit 	if (devargs == NULL)
1428a41f4deSParav Pandit 		return 0;
1438a41f4deSParav Pandit 	kvlist = rte_kvargs_parse(devargs->args, NULL);
1448a41f4deSParav Pandit 	if (kvlist == NULL)
1458a41f4deSParav Pandit 		return 0;
1468a41f4deSParav Pandit 	if (rte_kvargs_count(kvlist, key))
1478a41f4deSParav Pandit 		rte_kvargs_process(kvlist, key, bus_cmdline_options_handler,
1488a41f4deSParav Pandit 				   &ret);
1498a41f4deSParav Pandit 	rte_kvargs_free(kvlist);
1508a41f4deSParav Pandit 	return ret;
1518a41f4deSParav Pandit }
1528a41f4deSParav Pandit 
1538a41f4deSParav Pandit static bool
1548a41f4deSParav Pandit mlx5_bus_match(const struct mlx5_pci_driver *drv,
1558a41f4deSParav Pandit 	       const struct rte_pci_device *pci_dev)
1568a41f4deSParav Pandit {
1578a41f4deSParav Pandit 	const struct rte_pci_id *id_table;
1588a41f4deSParav Pandit 
1598a41f4deSParav Pandit 	for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0;
1608a41f4deSParav Pandit 	     id_table++) {
1618a41f4deSParav Pandit 		/* Check if device's ids match the class driver's ids. */
1628a41f4deSParav Pandit 		if (id_table->vendor_id != pci_dev->id.vendor_id &&
1634d509afaSThomas Monjalon 		    id_table->vendor_id != RTE_PCI_ANY_ID)
1648a41f4deSParav Pandit 			continue;
1658a41f4deSParav Pandit 		if (id_table->device_id != pci_dev->id.device_id &&
1664d509afaSThomas Monjalon 		    id_table->device_id != RTE_PCI_ANY_ID)
1678a41f4deSParav Pandit 			continue;
1688a41f4deSParav Pandit 		if (id_table->subsystem_vendor_id !=
1698a41f4deSParav Pandit 		    pci_dev->id.subsystem_vendor_id &&
1704d509afaSThomas Monjalon 		    id_table->subsystem_vendor_id != RTE_PCI_ANY_ID)
1718a41f4deSParav Pandit 			continue;
1728a41f4deSParav Pandit 		if (id_table->subsystem_device_id !=
1738a41f4deSParav Pandit 		    pci_dev->id.subsystem_device_id &&
1744d509afaSThomas Monjalon 		    id_table->subsystem_device_id != RTE_PCI_ANY_ID)
1758a41f4deSParav Pandit 			continue;
1768a41f4deSParav Pandit 		if (id_table->class_id != pci_dev->id.class_id &&
1778a41f4deSParav Pandit 		    id_table->class_id != RTE_CLASS_ANY_ID)
1788a41f4deSParav Pandit 			continue;
1798a41f4deSParav Pandit 		return true;
1808a41f4deSParav Pandit 	}
1818a41f4deSParav Pandit 	return false;
1828a41f4deSParav Pandit }
1838a41f4deSParav Pandit 
1848a41f4deSParav Pandit static int
1858a41f4deSParav Pandit is_valid_class_combination(uint32_t user_classes)
1868a41f4deSParav Pandit {
1878a41f4deSParav Pandit 	unsigned int i;
1888a41f4deSParav Pandit 
1898a41f4deSParav Pandit 	/* Verify if user specified valid supported combination. */
1908a41f4deSParav Pandit 	for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) {
1918a41f4deSParav Pandit 		if (mlx5_class_combinations[i] == user_classes)
1928a41f4deSParav Pandit 			return 0;
1938a41f4deSParav Pandit 	}
1948a41f4deSParav Pandit 	/* Not found any valid class combination. */
1958a41f4deSParav Pandit 	return -EINVAL;
1968a41f4deSParav Pandit }
1978a41f4deSParav Pandit 
1988a41f4deSParav Pandit static struct mlx5_pci_device *
1998a41f4deSParav Pandit pci_to_mlx5_device(const struct rte_pci_device *pci_dev)
2008a41f4deSParav Pandit {
2018a41f4deSParav Pandit 	struct mlx5_pci_device *dev;
2028a41f4deSParav Pandit 
2038a41f4deSParav Pandit 	TAILQ_FOREACH(dev, &devices_list, next) {
2048a41f4deSParav Pandit 		if (dev->pci_dev == pci_dev)
2058a41f4deSParav Pandit 			return dev;
2068a41f4deSParav Pandit 	}
2078a41f4deSParav Pandit 	return NULL;
2088a41f4deSParav Pandit }
2098a41f4deSParav Pandit 
2108a41f4deSParav Pandit static bool
2118a41f4deSParav Pandit device_class_enabled(const struct mlx5_pci_device *device, uint32_t class)
2128a41f4deSParav Pandit {
2138a41f4deSParav Pandit 	return (device->classes_loaded & class) ? true : false;
2148a41f4deSParav Pandit }
2158a41f4deSParav Pandit 
2168a41f4deSParav Pandit static void
2178a41f4deSParav Pandit dev_release(struct mlx5_pci_device *dev)
2188a41f4deSParav Pandit {
2198a41f4deSParav Pandit 	TAILQ_REMOVE(&devices_list, dev, next);
2208a41f4deSParav Pandit 	rte_free(dev);
2218a41f4deSParav Pandit }
2228a41f4deSParav Pandit 
2238a41f4deSParav Pandit static int
2248a41f4deSParav Pandit drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes)
2258a41f4deSParav Pandit {
2268a41f4deSParav Pandit 	struct mlx5_pci_driver *driver;
2278a41f4deSParav Pandit 	int local_ret = -ENODEV;
2288a41f4deSParav Pandit 	unsigned int i = 0;
2298a41f4deSParav Pandit 	int ret = 0;
2308a41f4deSParav Pandit 
2318a41f4deSParav Pandit 	enabled_classes &= dev->classes_loaded;
2328a41f4deSParav Pandit 	while (enabled_classes) {
2338a41f4deSParav Pandit 		driver = driver_get(RTE_BIT64(i));
2348a41f4deSParav Pandit 		if (driver) {
2358a41f4deSParav Pandit 			local_ret = driver->pci_driver.remove(dev->pci_dev);
2368a41f4deSParav Pandit 			if (!local_ret)
2378a41f4deSParav Pandit 				dev->classes_loaded &= ~RTE_BIT64(i);
2388a41f4deSParav Pandit 			else if (ret == 0)
2398a41f4deSParav Pandit 				ret = local_ret;
2408a41f4deSParav Pandit 		}
2418a41f4deSParav Pandit 		enabled_classes &= ~RTE_BIT64(i);
2428a41f4deSParav Pandit 		i++;
2438a41f4deSParav Pandit 	}
2448a41f4deSParav Pandit 	if (local_ret)
2458a41f4deSParav Pandit 		ret = local_ret;
2468a41f4deSParav Pandit 	return ret;
2478a41f4deSParav Pandit }
2488a41f4deSParav Pandit 
2498a41f4deSParav Pandit static int
2508a41f4deSParav Pandit drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv,
2518a41f4deSParav Pandit 	      struct rte_pci_device *pci_dev, uint32_t user_classes)
2528a41f4deSParav Pandit {
2538a41f4deSParav Pandit 	struct mlx5_pci_driver *driver;
2548a41f4deSParav Pandit 	uint32_t enabled_classes = 0;
2558a41f4deSParav Pandit 	bool already_loaded;
2568a41f4deSParav Pandit 	int ret;
2578a41f4deSParav Pandit 
2588a41f4deSParav Pandit 	TAILQ_FOREACH(driver, &drv_list, next) {
2598a41f4deSParav Pandit 		if ((driver->driver_class & user_classes) == 0)
2608a41f4deSParav Pandit 			continue;
2618a41f4deSParav Pandit 		if (!mlx5_bus_match(driver, pci_dev))
2628a41f4deSParav Pandit 			continue;
2638a41f4deSParav Pandit 		already_loaded = dev->classes_loaded & driver->driver_class;
2648a41f4deSParav Pandit 		if (already_loaded &&
2658a41f4deSParav Pandit 		    !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) {
2661b9e9826SThomas Monjalon 			DRV_LOG(ERR, "Device %s is already probed",
2678a41f4deSParav Pandit 				pci_dev->device.name);
2688a41f4deSParav Pandit 			ret = -EEXIST;
2698a41f4deSParav Pandit 			goto probe_err;
2708a41f4deSParav Pandit 		}
2718a41f4deSParav Pandit 		ret = driver->pci_driver.probe(pci_drv, pci_dev);
2728a41f4deSParav Pandit 		if (ret < 0) {
2731b9e9826SThomas Monjalon 			DRV_LOG(ERR, "Failed to load driver %s",
2748a41f4deSParav Pandit 				driver->pci_driver.driver.name);
2758a41f4deSParav Pandit 			goto probe_err;
2768a41f4deSParav Pandit 		}
2778a41f4deSParav Pandit 		enabled_classes |= driver->driver_class;
2788a41f4deSParav Pandit 	}
2798a41f4deSParav Pandit 	dev->classes_loaded |= enabled_classes;
2808a41f4deSParav Pandit 	return 0;
2818a41f4deSParav Pandit probe_err:
2828a41f4deSParav Pandit 	/* Only unload drivers which are enabled which were enabled
2838a41f4deSParav Pandit 	 * in this probe instance.
2848a41f4deSParav Pandit 	 */
2858a41f4deSParav Pandit 	drivers_remove(dev, enabled_classes);
2868a41f4deSParav Pandit 	return ret;
2878a41f4deSParav Pandit }
2888a41f4deSParav Pandit 
2898a41f4deSParav Pandit /**
2908a41f4deSParav Pandit  * DPDK callback to register to probe multiple drivers for a PCI device.
2918a41f4deSParav Pandit  *
2928a41f4deSParav Pandit  * @param[in] pci_drv
2938a41f4deSParav Pandit  *   PCI driver structure.
2948a41f4deSParav Pandit  * @param[in] dev
2958a41f4deSParav Pandit  *   PCI device information.
2968a41f4deSParav Pandit  *
2978a41f4deSParav Pandit  * @return
2988a41f4deSParav Pandit  *   0 on success, a negative errno value otherwise and rte_errno is set.
2998a41f4deSParav Pandit  */
3008a41f4deSParav Pandit static int
3018a41f4deSParav Pandit mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3028a41f4deSParav Pandit 		      struct rte_pci_device *pci_dev)
3038a41f4deSParav Pandit {
3048a41f4deSParav Pandit 	struct mlx5_pci_device *dev;
3058a41f4deSParav Pandit 	uint32_t user_classes = 0;
3068a41f4deSParav Pandit 	bool new_device = false;
3078a41f4deSParav Pandit 	int ret;
3088a41f4deSParav Pandit 
3098a41f4deSParav Pandit 	ret = parse_class_options(pci_dev->device.devargs);
3108a41f4deSParav Pandit 	if (ret < 0)
3118a41f4deSParav Pandit 		return ret;
3128a41f4deSParav Pandit 	user_classes = ret;
3138a41f4deSParav Pandit 	if (user_classes) {
3148a41f4deSParav Pandit 		/* Validate combination here. */
3158a41f4deSParav Pandit 		ret = is_valid_class_combination(user_classes);
3168a41f4deSParav Pandit 		if (ret) {
3178a41f4deSParav Pandit 			DRV_LOG(ERR, "Unsupported mlx5 classes supplied.");
3188a41f4deSParav Pandit 			return ret;
3198a41f4deSParav Pandit 		}
3208a41f4deSParav Pandit 	} else {
3218a41f4deSParav Pandit 		/* Default to net class. */
322*a99f2f90SXueming Li 		user_classes = MLX5_CLASS_ETH;
3238a41f4deSParav Pandit 	}
3248a41f4deSParav Pandit 	dev = pci_to_mlx5_device(pci_dev);
3258a41f4deSParav Pandit 	if (!dev) {
3268a41f4deSParav Pandit 		dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0);
3278a41f4deSParav Pandit 		if (!dev)
3288a41f4deSParav Pandit 			return -ENOMEM;
3298a41f4deSParav Pandit 		dev->pci_dev = pci_dev;
3308a41f4deSParav Pandit 		TAILQ_INSERT_HEAD(&devices_list, dev, next);
3318a41f4deSParav Pandit 		new_device = true;
3328a41f4deSParav Pandit 	}
3338a41f4deSParav Pandit 	ret = drivers_probe(dev, pci_drv, pci_dev, user_classes);
3348a41f4deSParav Pandit 	if (ret)
3358a41f4deSParav Pandit 		goto class_err;
3368a41f4deSParav Pandit 	return 0;
3378a41f4deSParav Pandit class_err:
3388a41f4deSParav Pandit 	if (new_device)
3398a41f4deSParav Pandit 		dev_release(dev);
3408a41f4deSParav Pandit 	return ret;
3418a41f4deSParav Pandit }
3428a41f4deSParav Pandit 
3438a41f4deSParav Pandit /**
3448a41f4deSParav Pandit  * DPDK callback to remove one or more drivers for a PCI device.
3458a41f4deSParav Pandit  *
3468a41f4deSParav Pandit  * This function removes all drivers probed for a given PCI device.
3478a41f4deSParav Pandit  *
3488a41f4deSParav Pandit  * @param[in] pci_dev
3498a41f4deSParav Pandit  *   Pointer to the PCI device.
3508a41f4deSParav Pandit  *
3518a41f4deSParav Pandit  * @return
3528a41f4deSParav Pandit  *   0 on success, the function cannot fail.
3538a41f4deSParav Pandit  */
3548a41f4deSParav Pandit static int
3558a41f4deSParav Pandit mlx5_common_pci_remove(struct rte_pci_device *pci_dev)
3568a41f4deSParav Pandit {
3578a41f4deSParav Pandit 	struct mlx5_pci_device *dev;
3588a41f4deSParav Pandit 	int ret;
3598a41f4deSParav Pandit 
3608a41f4deSParav Pandit 	dev = pci_to_mlx5_device(pci_dev);
3618a41f4deSParav Pandit 	if (!dev)
3628a41f4deSParav Pandit 		return -ENODEV;
3638a41f4deSParav Pandit 	/* Matching device found, cleanup and unload drivers. */
3648a41f4deSParav Pandit 	ret = drivers_remove(dev, dev->classes_loaded);
3658a41f4deSParav Pandit 	if (!ret)
3668a41f4deSParav Pandit 		dev_release(dev);
3678a41f4deSParav Pandit 	return ret;
3688a41f4deSParav Pandit }
3698a41f4deSParav Pandit 
3708a41f4deSParav Pandit static int
3718a41f4deSParav Pandit mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr,
3728a41f4deSParav Pandit 			uint64_t iova, size_t len)
3738a41f4deSParav Pandit {
3748a41f4deSParav Pandit 	struct mlx5_pci_driver *driver = NULL;
3758a41f4deSParav Pandit 	struct mlx5_pci_driver *temp;
3768a41f4deSParav Pandit 	struct mlx5_pci_device *dev;
3778a41f4deSParav Pandit 	int ret = -EINVAL;
3788a41f4deSParav Pandit 
3798a41f4deSParav Pandit 	dev = pci_to_mlx5_device(pci_dev);
3808a41f4deSParav Pandit 	if (!dev)
3818a41f4deSParav Pandit 		return -ENODEV;
3828a41f4deSParav Pandit 	TAILQ_FOREACH(driver, &drv_list, next) {
3838a41f4deSParav Pandit 		if (device_class_enabled(dev, driver->driver_class) &&
3848a41f4deSParav Pandit 		    driver->pci_driver.dma_map) {
3858a41f4deSParav Pandit 			ret = driver->pci_driver.dma_map(pci_dev, addr,
3868a41f4deSParav Pandit 							 iova, len);
3878a41f4deSParav Pandit 			if (ret)
3888a41f4deSParav Pandit 				goto map_err;
3898a41f4deSParav Pandit 		}
3908a41f4deSParav Pandit 	}
3918a41f4deSParav Pandit 	return ret;
3928a41f4deSParav Pandit map_err:
3938a41f4deSParav Pandit 	TAILQ_FOREACH(temp, &drv_list, next) {
3948a41f4deSParav Pandit 		if (temp == driver)
3958a41f4deSParav Pandit 			break;
3968a41f4deSParav Pandit 		if (device_class_enabled(dev, temp->driver_class) &&
3978a41f4deSParav Pandit 		    temp->pci_driver.dma_map && temp->pci_driver.dma_unmap)
3988a41f4deSParav Pandit 			temp->pci_driver.dma_unmap(pci_dev, addr, iova, len);
3998a41f4deSParav Pandit 	}
4008a41f4deSParav Pandit 	return ret;
4018a41f4deSParav Pandit }
4028a41f4deSParav Pandit 
4038a41f4deSParav Pandit static int
4048a41f4deSParav Pandit mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr,
4058a41f4deSParav Pandit 			  uint64_t iova, size_t len)
4068a41f4deSParav Pandit {
4078a41f4deSParav Pandit 	struct mlx5_pci_driver *driver;
4088a41f4deSParav Pandit 	struct mlx5_pci_device *dev;
4098a41f4deSParav Pandit 	int local_ret = -EINVAL;
4108a41f4deSParav Pandit 	int ret;
4118a41f4deSParav Pandit 
4128a41f4deSParav Pandit 	dev = pci_to_mlx5_device(pci_dev);
4138a41f4deSParav Pandit 	if (!dev)
4148a41f4deSParav Pandit 		return -ENODEV;
4158a41f4deSParav Pandit 	ret = 0;
4168a41f4deSParav Pandit 	/* There is no unmap error recovery in current implementation. */
4178a41f4deSParav Pandit 	TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) {
4188a41f4deSParav Pandit 		if (device_class_enabled(dev, driver->driver_class) &&
4198a41f4deSParav Pandit 		    driver->pci_driver.dma_unmap) {
4208a41f4deSParav Pandit 			local_ret = driver->pci_driver.dma_unmap(pci_dev, addr,
4218a41f4deSParav Pandit 								 iova, len);
4228a41f4deSParav Pandit 			if (local_ret && (ret == 0))
4238a41f4deSParav Pandit 				ret = local_ret;
4248a41f4deSParav Pandit 		}
4258a41f4deSParav Pandit 	}
4268a41f4deSParav Pandit 	if (local_ret)
4278a41f4deSParav Pandit 		ret = local_ret;
4288a41f4deSParav Pandit 	return ret;
4298a41f4deSParav Pandit }
4308a41f4deSParav Pandit 
4318a41f4deSParav Pandit /* PCI ID table is build dynamically based on registered mlx5 drivers. */
4328a41f4deSParav Pandit static struct rte_pci_id *mlx5_pci_id_table;
4338a41f4deSParav Pandit 
4348a41f4deSParav Pandit static struct rte_pci_driver mlx5_pci_driver = {
4358a41f4deSParav Pandit 	.driver = {
436188773a2SAsaf Penso 		.name = MLX5_PCI_DRIVER_NAME,
4378a41f4deSParav Pandit 	},
4388a41f4deSParav Pandit 	.probe = mlx5_common_pci_probe,
4398a41f4deSParav Pandit 	.remove = mlx5_common_pci_remove,
4408a41f4deSParav Pandit 	.dma_map = mlx5_common_pci_dma_map,
4418a41f4deSParav Pandit 	.dma_unmap = mlx5_common_pci_dma_unmap,
4428a41f4deSParav Pandit };
4438a41f4deSParav Pandit 
4448a41f4deSParav Pandit static int
4458a41f4deSParav Pandit pci_id_table_size_get(const struct rte_pci_id *id_table)
4468a41f4deSParav Pandit {
4478a41f4deSParav Pandit 	int table_size = 0;
4488a41f4deSParav Pandit 
4498a41f4deSParav Pandit 	for (; id_table->vendor_id != 0; id_table++)
4508a41f4deSParav Pandit 		table_size++;
4518a41f4deSParav Pandit 	return table_size;
4528a41f4deSParav Pandit }
4538a41f4deSParav Pandit 
4548a41f4deSParav Pandit static bool
4558a41f4deSParav Pandit pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table,
4568a41f4deSParav Pandit 	      int next_idx)
4578a41f4deSParav Pandit {
4588a41f4deSParav Pandit 	int current_size = next_idx - 1;
4598a41f4deSParav Pandit 	int i;
4608a41f4deSParav Pandit 
4618a41f4deSParav Pandit 	for (i = 0; i < current_size; i++) {
4628a41f4deSParav Pandit 		if (id->device_id == table[i].device_id &&
4638a41f4deSParav Pandit 		    id->vendor_id == table[i].vendor_id &&
4648a41f4deSParav Pandit 		    id->subsystem_vendor_id == table[i].subsystem_vendor_id &&
4658a41f4deSParav Pandit 		    id->subsystem_device_id == table[i].subsystem_device_id)
4668a41f4deSParav Pandit 			return true;
4678a41f4deSParav Pandit 	}
4688a41f4deSParav Pandit 	return false;
4698a41f4deSParav Pandit }
4708a41f4deSParav Pandit 
4718a41f4deSParav Pandit static void
4728a41f4deSParav Pandit pci_id_insert(struct rte_pci_id *new_table, int *next_idx,
4738a41f4deSParav Pandit 	      const struct rte_pci_id *id_table)
4748a41f4deSParav Pandit {
4758a41f4deSParav Pandit 	/* Traverse the id_table, check if entry exists in new_table;
4768a41f4deSParav Pandit 	 * Add non duplicate entries to new table.
4778a41f4deSParav Pandit 	 */
4788a41f4deSParav Pandit 	for (; id_table->vendor_id != 0; id_table++) {
4798a41f4deSParav Pandit 		if (!pci_id_exists(id_table, new_table, *next_idx)) {
4808a41f4deSParav Pandit 			/* New entry; add to the table. */
4818a41f4deSParav Pandit 			new_table[*next_idx] = *id_table;
4828a41f4deSParav Pandit 			(*next_idx)++;
4838a41f4deSParav Pandit 		}
4848a41f4deSParav Pandit 	}
4858a41f4deSParav Pandit }
4868a41f4deSParav Pandit 
4878a41f4deSParav Pandit static int
4888a41f4deSParav Pandit pci_ids_table_update(const struct rte_pci_id *driver_id_table)
4898a41f4deSParav Pandit {
4908a41f4deSParav Pandit 	const struct rte_pci_id *id_iter;
4918a41f4deSParav Pandit 	struct rte_pci_id *updated_table;
4928a41f4deSParav Pandit 	struct rte_pci_id *old_table;
4938a41f4deSParav Pandit 	int num_ids = 0;
4948a41f4deSParav Pandit 	int i = 0;
4958a41f4deSParav Pandit 
4968a41f4deSParav Pandit 	old_table = mlx5_pci_id_table;
4978a41f4deSParav Pandit 	if (old_table)
4988a41f4deSParav Pandit 		num_ids = pci_id_table_size_get(old_table);
4998a41f4deSParav Pandit 	num_ids += pci_id_table_size_get(driver_id_table);
5008a41f4deSParav Pandit 	/* Increase size by one for the termination entry of vendor_id = 0. */
5018a41f4deSParav Pandit 	num_ids += 1;
5028a41f4deSParav Pandit 	updated_table = calloc(num_ids, sizeof(*updated_table));
5038a41f4deSParav Pandit 	if (!updated_table)
5048a41f4deSParav Pandit 		return -ENOMEM;
5058a41f4deSParav Pandit 	if (TAILQ_EMPTY(&drv_list)) {
5068a41f4deSParav Pandit 		/* Copy the first driver's ID table. */
5078a41f4deSParav Pandit 		for (id_iter = driver_id_table; id_iter->vendor_id != 0;
5088a41f4deSParav Pandit 		     id_iter++, i++)
5098a41f4deSParav Pandit 			updated_table[i] = *id_iter;
5108a41f4deSParav Pandit 	} else {
5118a41f4deSParav Pandit 		/* First copy existing table entries. */
5128a41f4deSParav Pandit 		for (id_iter = old_table; id_iter->vendor_id != 0;
5138a41f4deSParav Pandit 		     id_iter++, i++)
5148a41f4deSParav Pandit 			updated_table[i] = *id_iter;
5158a41f4deSParav Pandit 		/* New id to be added at the end of current ID table. */
5168a41f4deSParav Pandit 		pci_id_insert(updated_table, &i, driver_id_table);
5178a41f4deSParav Pandit 	}
5188a41f4deSParav Pandit 	/* Terminate table with empty entry. */
5198a41f4deSParav Pandit 	updated_table[i].vendor_id = 0;
5208a41f4deSParav Pandit 	mlx5_pci_driver.id_table = updated_table;
5218a41f4deSParav Pandit 	mlx5_pci_id_table = updated_table;
5228a41f4deSParav Pandit 	if (old_table)
5238a41f4deSParav Pandit 		free(old_table);
5248a41f4deSParav Pandit 	return 0;
5258a41f4deSParav Pandit }
5268a41f4deSParav Pandit 
5278a41f4deSParav Pandit void
5288a41f4deSParav Pandit mlx5_pci_driver_register(struct mlx5_pci_driver *driver)
5298a41f4deSParav Pandit {
5308a41f4deSParav Pandit 	int ret;
5318a41f4deSParav Pandit 
5328a41f4deSParav Pandit 	ret = pci_ids_table_update(driver->pci_driver.id_table);
5338a41f4deSParav Pandit 	if (ret)
5348a41f4deSParav Pandit 		return;
5358a41f4deSParav Pandit 	mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags;
5368a41f4deSParav Pandit 	TAILQ_INSERT_TAIL(&drv_list, driver, next);
5378a41f4deSParav Pandit }
5388a41f4deSParav Pandit 
5398a41f4deSParav Pandit void mlx5_common_pci_init(void)
5408a41f4deSParav Pandit {
5418a41f4deSParav Pandit 	const struct rte_pci_id empty_table[] = {
5428a41f4deSParav Pandit 		{
5438a41f4deSParav Pandit 			.vendor_id = 0
5448a41f4deSParav Pandit 		},
5458a41f4deSParav Pandit 	};
5468a41f4deSParav Pandit 
5478a41f4deSParav Pandit 	/* All mlx5 PMDs constructor runs at same priority. So any of the PMD
5488a41f4deSParav Pandit 	 * including this one can register the PCI table first. If any other
5498a41f4deSParav Pandit 	 * PMD(s) have registered the PCI ID table, No need to register an empty
5508a41f4deSParav Pandit 	 * default one.
5518a41f4deSParav Pandit 	 */
5528a41f4deSParav Pandit 	if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table))
5538a41f4deSParav Pandit 		return;
5548a41f4deSParav Pandit 	rte_pci_register(&mlx5_pci_driver);
5558a41f4deSParav Pandit }
5568a41f4deSParav Pandit 
5578a41f4deSParav Pandit RTE_FINI(mlx5_common_pci_finish)
5588a41f4deSParav Pandit {
5598a41f4deSParav Pandit 	if (mlx5_pci_id_table != NULL) {
5608a41f4deSParav Pandit 		/* Constructor doesn't register with PCI bus if it failed
5618a41f4deSParav Pandit 		 * to build the table.
5628a41f4deSParav Pandit 		 */
5638a41f4deSParav Pandit 		rte_pci_unregister(&mlx5_pci_driver);
5648a41f4deSParav Pandit 		free(mlx5_pci_id_table);
5658a41f4deSParav Pandit 	}
5668a41f4deSParav Pandit }
5678a41f4deSParav Pandit RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__);
568