xref: /dpdk/drivers/net/mlx5/linux/mlx5_os.c (revision 391b8bcc81713d0da7e68eb39a64b00e69fc3543)
1f44b09f9SOphir Munk /* SPDX-License-Identifier: BSD-3-Clause
2f44b09f9SOphir Munk  * Copyright 2015 6WIND S.A.
3f44b09f9SOphir Munk  * Copyright 2020 Mellanox Technologies, Ltd
4f44b09f9SOphir Munk  */
5f44b09f9SOphir Munk 
6f44b09f9SOphir Munk #include <stddef.h>
7f44b09f9SOphir Munk #include <unistd.h>
8f44b09f9SOphir Munk #include <string.h>
9f44b09f9SOphir Munk #include <stdint.h>
10f44b09f9SOphir Munk #include <stdlib.h>
11f44b09f9SOphir Munk #include <errno.h>
12f44b09f9SOphir Munk #include <net/if.h>
13f44b09f9SOphir Munk #include <sys/mman.h>
14f44b09f9SOphir Munk #include <linux/rtnetlink.h>
1573bf9235SOphir Munk #include <linux/sockios.h>
1673bf9235SOphir Munk #include <linux/ethtool.h>
17f44b09f9SOphir Munk #include <fcntl.h>
18f44b09f9SOphir Munk 
19f44b09f9SOphir Munk /* Verbs header. */
20f44b09f9SOphir Munk /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
21f44b09f9SOphir Munk #ifdef PEDANTIC
22f44b09f9SOphir Munk #pragma GCC diagnostic ignored "-Wpedantic"
23f44b09f9SOphir Munk #endif
24f44b09f9SOphir Munk #include <infiniband/verbs.h>
25f44b09f9SOphir Munk #ifdef PEDANTIC
26f44b09f9SOphir Munk #pragma GCC diagnostic error "-Wpedantic"
27f44b09f9SOphir Munk #endif
28f44b09f9SOphir Munk 
29f44b09f9SOphir Munk #include <rte_malloc.h>
30f44b09f9SOphir Munk #include <rte_ethdev_driver.h>
31f44b09f9SOphir Munk #include <rte_ethdev_pci.h>
32f44b09f9SOphir Munk #include <rte_pci.h>
33f44b09f9SOphir Munk #include <rte_bus_pci.h>
34f44b09f9SOphir Munk #include <rte_common.h>
35f44b09f9SOphir Munk #include <rte_kvargs.h>
36f44b09f9SOphir Munk #include <rte_rwlock.h>
37f44b09f9SOphir Munk #include <rte_spinlock.h>
38f44b09f9SOphir Munk #include <rte_string_fns.h>
39f44b09f9SOphir Munk #include <rte_alarm.h>
40f44b09f9SOphir Munk 
41f44b09f9SOphir Munk #include <mlx5_glue.h>
42f44b09f9SOphir Munk #include <mlx5_devx_cmds.h>
43f44b09f9SOphir Munk #include <mlx5_common.h>
442eb4d010SOphir Munk #include <mlx5_common_mp.h>
45d5ed8aa9SOphir Munk #include <mlx5_common_mr.h>
46f44b09f9SOphir Munk 
47f44b09f9SOphir Munk #include "mlx5_defs.h"
48f44b09f9SOphir Munk #include "mlx5.h"
49*391b8bccSOphir Munk #include "mlx5_common_os.h"
50f44b09f9SOphir Munk #include "mlx5_utils.h"
51f44b09f9SOphir Munk #include "mlx5_rxtx.h"
52f44b09f9SOphir Munk #include "mlx5_autoconf.h"
53f44b09f9SOphir Munk #include "mlx5_mr.h"
54f44b09f9SOphir Munk #include "mlx5_flow.h"
55f44b09f9SOphir Munk #include "rte_pmd_mlx5.h"
564f96d913SOphir Munk #include "mlx5_verbs.h"
57f44b09f9SOphir Munk 
582eb4d010SOphir Munk #define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
592eb4d010SOphir Munk 
602eb4d010SOphir Munk #ifndef HAVE_IBV_MLX5_MOD_MPW
612eb4d010SOphir Munk #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
622eb4d010SOphir Munk #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
632eb4d010SOphir Munk #endif
642eb4d010SOphir Munk 
652eb4d010SOphir Munk #ifndef HAVE_IBV_MLX5_MOD_CQE_128B_COMP
662eb4d010SOphir Munk #define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4)
672eb4d010SOphir Munk #endif
682eb4d010SOphir Munk 
69f44b09f9SOphir Munk /**
70e85f623eSOphir Munk  * Get mlx5 device attributes. The glue function query_device_ex() is called
71e85f623eSOphir Munk  * with out parameter of type 'struct ibv_device_attr_ex *'. Then fill in mlx5
72e85f623eSOphir Munk  * device attributes from the glue out parameter.
73e85f623eSOphir Munk  *
74e85f623eSOphir Munk  * @param dev
75e85f623eSOphir Munk  *   Pointer to ibv context.
76e85f623eSOphir Munk  *
77e85f623eSOphir Munk  * @param device_attr
78e85f623eSOphir Munk  *   Pointer to mlx5 device attributes.
79e85f623eSOphir Munk  *
80e85f623eSOphir Munk  * @return
81e85f623eSOphir Munk  *   0 on success, non zero error number otherwise
82e85f623eSOphir Munk  */
83e85f623eSOphir Munk int
84e85f623eSOphir Munk mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *device_attr)
85e85f623eSOphir Munk {
86e85f623eSOphir Munk 	int err;
87e85f623eSOphir Munk 	struct ibv_device_attr_ex attr_ex;
88e85f623eSOphir Munk 	memset(device_attr, 0, sizeof(*device_attr));
89e85f623eSOphir Munk 	err = mlx5_glue->query_device_ex(ctx, NULL, &attr_ex);
90e85f623eSOphir Munk 	if (err)
91e85f623eSOphir Munk 		return err;
92e85f623eSOphir Munk 
93e85f623eSOphir Munk 	device_attr->device_cap_flags_ex = attr_ex.device_cap_flags_ex;
94e85f623eSOphir Munk 	device_attr->max_qp_wr = attr_ex.orig_attr.max_qp_wr;
95e85f623eSOphir Munk 	device_attr->max_sge = attr_ex.orig_attr.max_sge;
96e85f623eSOphir Munk 	device_attr->max_cq = attr_ex.orig_attr.max_cq;
97e85f623eSOphir Munk 	device_attr->max_qp = attr_ex.orig_attr.max_qp;
98e85f623eSOphir Munk 	device_attr->raw_packet_caps = attr_ex.raw_packet_caps;
99e85f623eSOphir Munk 	device_attr->max_rwq_indirection_table_size =
100e85f623eSOphir Munk 		attr_ex.rss_caps.max_rwq_indirection_table_size;
101e85f623eSOphir Munk 	device_attr->max_tso = attr_ex.tso_caps.max_tso;
102e85f623eSOphir Munk 	device_attr->tso_supported_qpts = attr_ex.tso_caps.supported_qpts;
103e85f623eSOphir Munk 
104e85f623eSOphir Munk 	struct mlx5dv_context dv_attr = { .comp_mask = 0 };
105e85f623eSOphir Munk 	err = mlx5_glue->dv_query_device(ctx, &dv_attr);
106e85f623eSOphir Munk 	if (err)
107e85f623eSOphir Munk 		return err;
108e85f623eSOphir Munk 
109e85f623eSOphir Munk 	device_attr->flags = dv_attr.flags;
110e85f623eSOphir Munk 	device_attr->comp_mask = dv_attr.comp_mask;
111e85f623eSOphir Munk #ifdef HAVE_IBV_MLX5_MOD_SWP
112e85f623eSOphir Munk 	device_attr->sw_parsing_offloads =
113e85f623eSOphir Munk 		dv_attr.sw_parsing_caps.sw_parsing_offloads;
114e85f623eSOphir Munk #endif
115e85f623eSOphir Munk 	device_attr->min_single_stride_log_num_of_bytes =
116e85f623eSOphir Munk 		dv_attr.striding_rq_caps.min_single_stride_log_num_of_bytes;
117e85f623eSOphir Munk 	device_attr->max_single_stride_log_num_of_bytes =
118e85f623eSOphir Munk 		dv_attr.striding_rq_caps.max_single_stride_log_num_of_bytes;
119e85f623eSOphir Munk 	device_attr->min_single_wqe_log_num_of_strides =
120e85f623eSOphir Munk 		dv_attr.striding_rq_caps.min_single_wqe_log_num_of_strides;
121e85f623eSOphir Munk 	device_attr->max_single_wqe_log_num_of_strides =
122e85f623eSOphir Munk 		dv_attr.striding_rq_caps.max_single_wqe_log_num_of_strides;
123e85f623eSOphir Munk 	device_attr->stride_supported_qpts =
124e85f623eSOphir Munk 		dv_attr.striding_rq_caps.supported_qpts;
125e85f623eSOphir Munk #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
126e85f623eSOphir Munk 	device_attr->tunnel_offloads_caps = dv_attr.tunnel_offloads_caps;
127e85f623eSOphir Munk #endif
128e85f623eSOphir Munk 
129e85f623eSOphir Munk 	return err;
130e85f623eSOphir Munk }
1312eb4d010SOphir Munk 
1322eb4d010SOphir Munk /**
1332eb4d010SOphir Munk  * Verbs callback to allocate a memory. This function should allocate the space
1342eb4d010SOphir Munk  * according to the size provided residing inside a huge page.
1352eb4d010SOphir Munk  * Please note that all allocation must respect the alignment from libmlx5
1362eb4d010SOphir Munk  * (i.e. currently sysconf(_SC_PAGESIZE)).
1372eb4d010SOphir Munk  *
1382eb4d010SOphir Munk  * @param[in] size
1392eb4d010SOphir Munk  *   The size in bytes of the memory to allocate.
1402eb4d010SOphir Munk  * @param[in] data
1412eb4d010SOphir Munk  *   A pointer to the callback data.
1422eb4d010SOphir Munk  *
1432eb4d010SOphir Munk  * @return
1442eb4d010SOphir Munk  *   Allocated buffer, NULL otherwise and rte_errno is set.
1452eb4d010SOphir Munk  */
1462eb4d010SOphir Munk static void *
1472eb4d010SOphir Munk mlx5_alloc_verbs_buf(size_t size, void *data)
1482eb4d010SOphir Munk {
1492eb4d010SOphir Munk 	struct mlx5_priv *priv = data;
1502eb4d010SOphir Munk 	void *ret;
1512eb4d010SOphir Munk 	size_t alignment = sysconf(_SC_PAGESIZE);
1522eb4d010SOphir Munk 	unsigned int socket = SOCKET_ID_ANY;
1532eb4d010SOphir Munk 
1542eb4d010SOphir Munk 	if (priv->verbs_alloc_ctx.type == MLX5_VERBS_ALLOC_TYPE_TX_QUEUE) {
1552eb4d010SOphir Munk 		const struct mlx5_txq_ctrl *ctrl = priv->verbs_alloc_ctx.obj;
1562eb4d010SOphir Munk 
1572eb4d010SOphir Munk 		socket = ctrl->socket;
1582eb4d010SOphir Munk 	} else if (priv->verbs_alloc_ctx.type ==
1592eb4d010SOphir Munk 		   MLX5_VERBS_ALLOC_TYPE_RX_QUEUE) {
1602eb4d010SOphir Munk 		const struct mlx5_rxq_ctrl *ctrl = priv->verbs_alloc_ctx.obj;
1612eb4d010SOphir Munk 
1622eb4d010SOphir Munk 		socket = ctrl->socket;
1632eb4d010SOphir Munk 	}
1642eb4d010SOphir Munk 	MLX5_ASSERT(data != NULL);
1652eb4d010SOphir Munk 	ret = rte_malloc_socket(__func__, size, alignment, socket);
1662eb4d010SOphir Munk 	if (!ret && size)
1672eb4d010SOphir Munk 		rte_errno = ENOMEM;
1682eb4d010SOphir Munk 	return ret;
1692eb4d010SOphir Munk }
1702eb4d010SOphir Munk 
1712eb4d010SOphir Munk /**
1722eb4d010SOphir Munk  * Verbs callback to free a memory.
1732eb4d010SOphir Munk  *
1742eb4d010SOphir Munk  * @param[in] ptr
1752eb4d010SOphir Munk  *   A pointer to the memory to free.
1762eb4d010SOphir Munk  * @param[in] data
1772eb4d010SOphir Munk  *   A pointer to the callback data.
1782eb4d010SOphir Munk  */
1792eb4d010SOphir Munk static void
1802eb4d010SOphir Munk mlx5_free_verbs_buf(void *ptr, void *data __rte_unused)
1812eb4d010SOphir Munk {
1822eb4d010SOphir Munk 	MLX5_ASSERT(data != NULL);
1832eb4d010SOphir Munk 	rte_free(ptr);
1842eb4d010SOphir Munk }
1852eb4d010SOphir Munk 
1862eb4d010SOphir Munk /**
1872eb4d010SOphir Munk  * Initialize DR related data within private structure.
1882eb4d010SOphir Munk  * Routine checks the reference counter and does actual
1892eb4d010SOphir Munk  * resources creation/initialization only if counter is zero.
1902eb4d010SOphir Munk  *
1912eb4d010SOphir Munk  * @param[in] priv
1922eb4d010SOphir Munk  *   Pointer to the private device data structure.
1932eb4d010SOphir Munk  *
1942eb4d010SOphir Munk  * @return
1952eb4d010SOphir Munk  *   Zero on success, positive error code otherwise.
1962eb4d010SOphir Munk  */
1972eb4d010SOphir Munk static int
1982eb4d010SOphir Munk mlx5_alloc_shared_dr(struct mlx5_priv *priv)
1992eb4d010SOphir Munk {
2002eb4d010SOphir Munk 	struct mlx5_dev_ctx_shared *sh = priv->sh;
2012eb4d010SOphir Munk 	char s[MLX5_HLIST_NAMESIZE];
2022eb4d010SOphir Munk 	int err = 0;
2032eb4d010SOphir Munk 
2042eb4d010SOphir Munk 	if (!sh->flow_tbls)
2052eb4d010SOphir Munk 		err = mlx5_alloc_table_hash_list(priv);
2062eb4d010SOphir Munk 	else
2072eb4d010SOphir Munk 		DRV_LOG(DEBUG, "sh->flow_tbls[%p] already created, reuse\n",
2082eb4d010SOphir Munk 			(void *)sh->flow_tbls);
2092eb4d010SOphir Munk 	if (err)
2102eb4d010SOphir Munk 		return err;
2112eb4d010SOphir Munk 	/* Create tags hash list table. */
2122eb4d010SOphir Munk 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
2132eb4d010SOphir Munk 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE);
2142eb4d010SOphir Munk 	if (!sh->tag_table) {
21563783b01SDavid Marchand 		DRV_LOG(ERR, "tags with hash creation failed.");
2162eb4d010SOphir Munk 		err = ENOMEM;
2172eb4d010SOphir Munk 		goto error;
2182eb4d010SOphir Munk 	}
2192eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR
2202eb4d010SOphir Munk 	void *domain;
2212eb4d010SOphir Munk 
2222eb4d010SOphir Munk 	if (sh->dv_refcnt) {
2232eb4d010SOphir Munk 		/* Shared DV/DR structures is already initialized. */
2242eb4d010SOphir Munk 		sh->dv_refcnt++;
2252eb4d010SOphir Munk 		priv->dr_shared = 1;
2262eb4d010SOphir Munk 		return 0;
2272eb4d010SOphir Munk 	}
2282eb4d010SOphir Munk 	/* Reference counter is zero, we should initialize structures. */
2292eb4d010SOphir Munk 	domain = mlx5_glue->dr_create_domain(sh->ctx,
2302eb4d010SOphir Munk 					     MLX5DV_DR_DOMAIN_TYPE_NIC_RX);
2312eb4d010SOphir Munk 	if (!domain) {
2322eb4d010SOphir Munk 		DRV_LOG(ERR, "ingress mlx5dv_dr_create_domain failed");
2332eb4d010SOphir Munk 		err = errno;
2342eb4d010SOphir Munk 		goto error;
2352eb4d010SOphir Munk 	}
2362eb4d010SOphir Munk 	sh->rx_domain = domain;
2372eb4d010SOphir Munk 	domain = mlx5_glue->dr_create_domain(sh->ctx,
2382eb4d010SOphir Munk 					     MLX5DV_DR_DOMAIN_TYPE_NIC_TX);
2392eb4d010SOphir Munk 	if (!domain) {
2402eb4d010SOphir Munk 		DRV_LOG(ERR, "egress mlx5dv_dr_create_domain failed");
2412eb4d010SOphir Munk 		err = errno;
2422eb4d010SOphir Munk 		goto error;
2432eb4d010SOphir Munk 	}
2442eb4d010SOphir Munk 	pthread_mutex_init(&sh->dv_mutex, NULL);
2452eb4d010SOphir Munk 	sh->tx_domain = domain;
2462eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_ESWITCH
2472eb4d010SOphir Munk 	if (priv->config.dv_esw_en) {
2482eb4d010SOphir Munk 		domain  = mlx5_glue->dr_create_domain
2492eb4d010SOphir Munk 			(sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB);
2502eb4d010SOphir Munk 		if (!domain) {
2512eb4d010SOphir Munk 			DRV_LOG(ERR, "FDB mlx5dv_dr_create_domain failed");
2522eb4d010SOphir Munk 			err = errno;
2532eb4d010SOphir Munk 			goto error;
2542eb4d010SOphir Munk 		}
2552eb4d010SOphir Munk 		sh->fdb_domain = domain;
2562eb4d010SOphir Munk 		sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop();
2572eb4d010SOphir Munk 	}
2582eb4d010SOphir Munk #endif
2592eb4d010SOphir Munk 	if (priv->config.reclaim_mode == MLX5_RCM_AGGR) {
2602eb4d010SOphir Munk 		mlx5_glue->dr_reclaim_domain_memory(sh->rx_domain, 1);
2612eb4d010SOphir Munk 		mlx5_glue->dr_reclaim_domain_memory(sh->tx_domain, 1);
2622eb4d010SOphir Munk 		if (sh->fdb_domain)
2632eb4d010SOphir Munk 			mlx5_glue->dr_reclaim_domain_memory(sh->fdb_domain, 1);
2642eb4d010SOphir Munk 	}
2652eb4d010SOphir Munk 	sh->pop_vlan_action = mlx5_glue->dr_create_flow_action_pop_vlan();
2662eb4d010SOphir Munk #endif /* HAVE_MLX5DV_DR */
2672eb4d010SOphir Munk 	sh->dv_refcnt++;
2682eb4d010SOphir Munk 	priv->dr_shared = 1;
2692eb4d010SOphir Munk 	return 0;
2702eb4d010SOphir Munk error:
2712eb4d010SOphir Munk 	/* Rollback the created objects. */
2722eb4d010SOphir Munk 	if (sh->rx_domain) {
2732eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->rx_domain);
2742eb4d010SOphir Munk 		sh->rx_domain = NULL;
2752eb4d010SOphir Munk 	}
2762eb4d010SOphir Munk 	if (sh->tx_domain) {
2772eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->tx_domain);
2782eb4d010SOphir Munk 		sh->tx_domain = NULL;
2792eb4d010SOphir Munk 	}
2802eb4d010SOphir Munk 	if (sh->fdb_domain) {
2812eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->fdb_domain);
2822eb4d010SOphir Munk 		sh->fdb_domain = NULL;
2832eb4d010SOphir Munk 	}
2842eb4d010SOphir Munk 	if (sh->esw_drop_action) {
2852eb4d010SOphir Munk 		mlx5_glue->destroy_flow_action(sh->esw_drop_action);
2862eb4d010SOphir Munk 		sh->esw_drop_action = NULL;
2872eb4d010SOphir Munk 	}
2882eb4d010SOphir Munk 	if (sh->pop_vlan_action) {
2892eb4d010SOphir Munk 		mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
2902eb4d010SOphir Munk 		sh->pop_vlan_action = NULL;
2912eb4d010SOphir Munk 	}
2922eb4d010SOphir Munk 	if (sh->tag_table) {
2932eb4d010SOphir Munk 		/* tags should be destroyed with flow before. */
2942eb4d010SOphir Munk 		mlx5_hlist_destroy(sh->tag_table, NULL, NULL);
2952eb4d010SOphir Munk 		sh->tag_table = NULL;
2962eb4d010SOphir Munk 	}
2972eb4d010SOphir Munk 	mlx5_free_table_hash_list(priv);
2982eb4d010SOphir Munk 	return err;
2992eb4d010SOphir Munk }
3002eb4d010SOphir Munk 
3012eb4d010SOphir Munk /**
3022eb4d010SOphir Munk  * Destroy DR related data within private structure.
3032eb4d010SOphir Munk  *
3042eb4d010SOphir Munk  * @param[in] priv
3052eb4d010SOphir Munk  *   Pointer to the private device data structure.
3062eb4d010SOphir Munk  */
3072eb4d010SOphir Munk void
3082eb4d010SOphir Munk mlx5_os_free_shared_dr(struct mlx5_priv *priv)
3092eb4d010SOphir Munk {
3102eb4d010SOphir Munk 	struct mlx5_dev_ctx_shared *sh;
3112eb4d010SOphir Munk 
3122eb4d010SOphir Munk 	if (!priv->dr_shared)
3132eb4d010SOphir Munk 		return;
3142eb4d010SOphir Munk 	priv->dr_shared = 0;
3152eb4d010SOphir Munk 	sh = priv->sh;
3162eb4d010SOphir Munk 	MLX5_ASSERT(sh);
3172eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR
3182eb4d010SOphir Munk 	MLX5_ASSERT(sh->dv_refcnt);
3192eb4d010SOphir Munk 	if (sh->dv_refcnt && --sh->dv_refcnt)
3202eb4d010SOphir Munk 		return;
3212eb4d010SOphir Munk 	if (sh->rx_domain) {
3222eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->rx_domain);
3232eb4d010SOphir Munk 		sh->rx_domain = NULL;
3242eb4d010SOphir Munk 	}
3252eb4d010SOphir Munk 	if (sh->tx_domain) {
3262eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->tx_domain);
3272eb4d010SOphir Munk 		sh->tx_domain = NULL;
3282eb4d010SOphir Munk 	}
3292eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_ESWITCH
3302eb4d010SOphir Munk 	if (sh->fdb_domain) {
3312eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->fdb_domain);
3322eb4d010SOphir Munk 		sh->fdb_domain = NULL;
3332eb4d010SOphir Munk 	}
3342eb4d010SOphir Munk 	if (sh->esw_drop_action) {
3352eb4d010SOphir Munk 		mlx5_glue->destroy_flow_action(sh->esw_drop_action);
3362eb4d010SOphir Munk 		sh->esw_drop_action = NULL;
3372eb4d010SOphir Munk 	}
3382eb4d010SOphir Munk #endif
3392eb4d010SOphir Munk 	if (sh->pop_vlan_action) {
3402eb4d010SOphir Munk 		mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
3412eb4d010SOphir Munk 		sh->pop_vlan_action = NULL;
3422eb4d010SOphir Munk 	}
3432eb4d010SOphir Munk 	pthread_mutex_destroy(&sh->dv_mutex);
3442eb4d010SOphir Munk #endif /* HAVE_MLX5DV_DR */
3452eb4d010SOphir Munk 	if (sh->tag_table) {
3462eb4d010SOphir Munk 		/* tags should be destroyed with flow before. */
3472eb4d010SOphir Munk 		mlx5_hlist_destroy(sh->tag_table, NULL, NULL);
3482eb4d010SOphir Munk 		sh->tag_table = NULL;
3492eb4d010SOphir Munk 	}
3502eb4d010SOphir Munk 	mlx5_free_table_hash_list(priv);
3512eb4d010SOphir Munk }
3522eb4d010SOphir Munk 
3532eb4d010SOphir Munk /**
3542eb4d010SOphir Munk  * Spawn an Ethernet device from Verbs information.
3552eb4d010SOphir Munk  *
3562eb4d010SOphir Munk  * @param dpdk_dev
3572eb4d010SOphir Munk  *   Backing DPDK device.
3582eb4d010SOphir Munk  * @param spawn
3592eb4d010SOphir Munk  *   Verbs device parameters (name, port, switch_info) to spawn.
3602eb4d010SOphir Munk  * @param config
3612eb4d010SOphir Munk  *   Device configuration parameters.
3622eb4d010SOphir Munk  *
3632eb4d010SOphir Munk  * @return
3642eb4d010SOphir Munk  *   A valid Ethernet device object on success, NULL otherwise and rte_errno
3652eb4d010SOphir Munk  *   is set. The following errors are defined:
3662eb4d010SOphir Munk  *
3672eb4d010SOphir Munk  *   EBUSY: device is not supposed to be spawned.
3682eb4d010SOphir Munk  *   EEXIST: device is already spawned
3692eb4d010SOphir Munk  */
3702eb4d010SOphir Munk static struct rte_eth_dev *
3712eb4d010SOphir Munk mlx5_dev_spawn(struct rte_device *dpdk_dev,
3722eb4d010SOphir Munk 	       struct mlx5_dev_spawn_data *spawn,
3732eb4d010SOphir Munk 	       struct mlx5_dev_config config)
3742eb4d010SOphir Munk {
3752eb4d010SOphir Munk 	const struct mlx5_switch_info *switch_info = &spawn->info;
3762eb4d010SOphir Munk 	struct mlx5_dev_ctx_shared *sh = NULL;
3772eb4d010SOphir Munk 	struct ibv_port_attr port_attr;
3782eb4d010SOphir Munk 	struct mlx5dv_context dv_attr = { .comp_mask = 0 };
3792eb4d010SOphir Munk 	struct rte_eth_dev *eth_dev = NULL;
3802eb4d010SOphir Munk 	struct mlx5_priv *priv = NULL;
3812eb4d010SOphir Munk 	int err = 0;
3822eb4d010SOphir Munk 	unsigned int hw_padding = 0;
3832eb4d010SOphir Munk 	unsigned int mps;
3842eb4d010SOphir Munk 	unsigned int cqe_comp;
3852eb4d010SOphir Munk 	unsigned int cqe_pad = 0;
3862eb4d010SOphir Munk 	unsigned int tunnel_en = 0;
3872eb4d010SOphir Munk 	unsigned int mpls_en = 0;
3882eb4d010SOphir Munk 	unsigned int swp = 0;
3892eb4d010SOphir Munk 	unsigned int mprq = 0;
3902eb4d010SOphir Munk 	unsigned int mprq_min_stride_size_n = 0;
3912eb4d010SOphir Munk 	unsigned int mprq_max_stride_size_n = 0;
3922eb4d010SOphir Munk 	unsigned int mprq_min_stride_num_n = 0;
3932eb4d010SOphir Munk 	unsigned int mprq_max_stride_num_n = 0;
3942eb4d010SOphir Munk 	struct rte_ether_addr mac;
3952eb4d010SOphir Munk 	char name[RTE_ETH_NAME_MAX_LEN];
3962eb4d010SOphir Munk 	int own_domain_id = 0;
3972eb4d010SOphir Munk 	uint16_t port_id;
3982eb4d010SOphir Munk 	unsigned int i;
3992eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_DEVX_PORT
4002eb4d010SOphir Munk 	struct mlx5dv_devx_port devx_port = { .comp_mask = 0 };
4012eb4d010SOphir Munk #endif
4022eb4d010SOphir Munk 
4032eb4d010SOphir Munk 	/* Determine if this port representor is supposed to be spawned. */
4042eb4d010SOphir Munk 	if (switch_info->representor && dpdk_dev->devargs) {
4052eb4d010SOphir Munk 		struct rte_eth_devargs eth_da;
4062eb4d010SOphir Munk 
4072eb4d010SOphir Munk 		err = rte_eth_devargs_parse(dpdk_dev->devargs->args, &eth_da);
4082eb4d010SOphir Munk 		if (err) {
4092eb4d010SOphir Munk 			rte_errno = -err;
4102eb4d010SOphir Munk 			DRV_LOG(ERR, "failed to process device arguments: %s",
4112eb4d010SOphir Munk 				strerror(rte_errno));
4122eb4d010SOphir Munk 			return NULL;
4132eb4d010SOphir Munk 		}
4142eb4d010SOphir Munk 		for (i = 0; i < eth_da.nb_representor_ports; ++i)
4152eb4d010SOphir Munk 			if (eth_da.representor_ports[i] ==
4162eb4d010SOphir Munk 			    (uint16_t)switch_info->port_name)
4172eb4d010SOphir Munk 				break;
4182eb4d010SOphir Munk 		if (i == eth_da.nb_representor_ports) {
4192eb4d010SOphir Munk 			rte_errno = EBUSY;
4202eb4d010SOphir Munk 			return NULL;
4212eb4d010SOphir Munk 		}
4222eb4d010SOphir Munk 	}
4232eb4d010SOphir Munk 	/* Build device name. */
4242eb4d010SOphir Munk 	if (spawn->pf_bond <  0) {
4252eb4d010SOphir Munk 		/* Single device. */
4262eb4d010SOphir Munk 		if (!switch_info->representor)
4272eb4d010SOphir Munk 			strlcpy(name, dpdk_dev->name, sizeof(name));
4282eb4d010SOphir Munk 		else
4292eb4d010SOphir Munk 			snprintf(name, sizeof(name), "%s_representor_%u",
4302eb4d010SOphir Munk 				 dpdk_dev->name, switch_info->port_name);
4312eb4d010SOphir Munk 	} else {
4322eb4d010SOphir Munk 		/* Bonding device. */
4332eb4d010SOphir Munk 		if (!switch_info->representor)
4342eb4d010SOphir Munk 			snprintf(name, sizeof(name), "%s_%s",
435834a9019SOphir Munk 				 dpdk_dev->name,
436834a9019SOphir Munk 				 mlx5_os_get_dev_device_name(spawn->phys_dev));
4372eb4d010SOphir Munk 		else
4382eb4d010SOphir Munk 			snprintf(name, sizeof(name), "%s_%s_representor_%u",
439834a9019SOphir Munk 				 dpdk_dev->name,
440834a9019SOphir Munk 				 mlx5_os_get_dev_device_name(spawn->phys_dev),
4412eb4d010SOphir Munk 				 switch_info->port_name);
4422eb4d010SOphir Munk 	}
4432eb4d010SOphir Munk 	/* check if the device is already spawned */
4442eb4d010SOphir Munk 	if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) {
4452eb4d010SOphir Munk 		rte_errno = EEXIST;
4462eb4d010SOphir Munk 		return NULL;
4472eb4d010SOphir Munk 	}
4482eb4d010SOphir Munk 	DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name);
4492eb4d010SOphir Munk 	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
4502eb4d010SOphir Munk 		struct mlx5_mp_id mp_id;
4512eb4d010SOphir Munk 
4522eb4d010SOphir Munk 		eth_dev = rte_eth_dev_attach_secondary(name);
4532eb4d010SOphir Munk 		if (eth_dev == NULL) {
4542eb4d010SOphir Munk 			DRV_LOG(ERR, "can not attach rte ethdev");
4552eb4d010SOphir Munk 			rte_errno = ENOMEM;
4562eb4d010SOphir Munk 			return NULL;
4572eb4d010SOphir Munk 		}
4582eb4d010SOphir Munk 		eth_dev->device = dpdk_dev;
459042f5c94SOphir Munk 		eth_dev->dev_ops = &mlx5_os_dev_sec_ops;
4602eb4d010SOphir Munk 		err = mlx5_proc_priv_init(eth_dev);
4612eb4d010SOphir Munk 		if (err)
4622eb4d010SOphir Munk 			return NULL;
4632eb4d010SOphir Munk 		mp_id.port_id = eth_dev->data->port_id;
4642eb4d010SOphir Munk 		strlcpy(mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN);
4652eb4d010SOphir Munk 		/* Receive command fd from primary process */
4662eb4d010SOphir Munk 		err = mlx5_mp_req_verbs_cmd_fd(&mp_id);
4672eb4d010SOphir Munk 		if (err < 0)
4682eb4d010SOphir Munk 			goto err_secondary;
4692eb4d010SOphir Munk 		/* Remap UAR for Tx queues. */
4702eb4d010SOphir Munk 		err = mlx5_tx_uar_init_secondary(eth_dev, err);
4712eb4d010SOphir Munk 		if (err)
4722eb4d010SOphir Munk 			goto err_secondary;
4732eb4d010SOphir Munk 		/*
4742eb4d010SOphir Munk 		 * Ethdev pointer is still required as input since
4752eb4d010SOphir Munk 		 * the primary device is not accessible from the
4762eb4d010SOphir Munk 		 * secondary process.
4772eb4d010SOphir Munk 		 */
4782eb4d010SOphir Munk 		eth_dev->rx_pkt_burst = mlx5_select_rx_function(eth_dev);
4792eb4d010SOphir Munk 		eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev);
4802eb4d010SOphir Munk 		return eth_dev;
4812eb4d010SOphir Munk err_secondary:
4822eb4d010SOphir Munk 		mlx5_dev_close(eth_dev);
4832eb4d010SOphir Munk 		return NULL;
4842eb4d010SOphir Munk 	}
4852eb4d010SOphir Munk 	/*
4862eb4d010SOphir Munk 	 * Some parameters ("tx_db_nc" in particularly) are needed in
4872eb4d010SOphir Munk 	 * advance to create dv/verbs device context. We proceed the
4882eb4d010SOphir Munk 	 * devargs here to get ones, and later proceed devargs again
4892eb4d010SOphir Munk 	 * to override some hardware settings.
4902eb4d010SOphir Munk 	 */
4912eb4d010SOphir Munk 	err = mlx5_args(&config, dpdk_dev->devargs);
4922eb4d010SOphir Munk 	if (err) {
4932eb4d010SOphir Munk 		err = rte_errno;
4942eb4d010SOphir Munk 		DRV_LOG(ERR, "failed to process device arguments: %s",
4952eb4d010SOphir Munk 			strerror(rte_errno));
4962eb4d010SOphir Munk 		goto error;
4972eb4d010SOphir Munk 	}
49891389890SOphir Munk 	sh = mlx5_alloc_shared_dev_ctx(spawn, &config);
4992eb4d010SOphir Munk 	if (!sh)
5002eb4d010SOphir Munk 		return NULL;
5012eb4d010SOphir Munk 	config.devx = sh->devx;
5022eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
5032eb4d010SOphir Munk 	config.dest_tir = 1;
5042eb4d010SOphir Munk #endif
5052eb4d010SOphir Munk #ifdef HAVE_IBV_MLX5_MOD_SWP
5062eb4d010SOphir Munk 	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP;
5072eb4d010SOphir Munk #endif
5082eb4d010SOphir Munk 	/*
5092eb4d010SOphir Munk 	 * Multi-packet send is supported by ConnectX-4 Lx PF as well
5102eb4d010SOphir Munk 	 * as all ConnectX-5 devices.
5112eb4d010SOphir Munk 	 */
5122eb4d010SOphir Munk #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
5132eb4d010SOphir Munk 	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS;
5142eb4d010SOphir Munk #endif
5152eb4d010SOphir Munk #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
5162eb4d010SOphir Munk 	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
5172eb4d010SOphir Munk #endif
5182eb4d010SOphir Munk 	mlx5_glue->dv_query_device(sh->ctx, &dv_attr);
5192eb4d010SOphir Munk 	if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) {
5202eb4d010SOphir Munk 		if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) {
5212eb4d010SOphir Munk 			DRV_LOG(DEBUG, "enhanced MPW is supported");
5222eb4d010SOphir Munk 			mps = MLX5_MPW_ENHANCED;
5232eb4d010SOphir Munk 		} else {
5242eb4d010SOphir Munk 			DRV_LOG(DEBUG, "MPW is supported");
5252eb4d010SOphir Munk 			mps = MLX5_MPW;
5262eb4d010SOphir Munk 		}
5272eb4d010SOphir Munk 	} else {
5282eb4d010SOphir Munk 		DRV_LOG(DEBUG, "MPW isn't supported");
5292eb4d010SOphir Munk 		mps = MLX5_MPW_DISABLED;
5302eb4d010SOphir Munk 	}
5312eb4d010SOphir Munk #ifdef HAVE_IBV_MLX5_MOD_SWP
5322eb4d010SOphir Munk 	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP)
5332eb4d010SOphir Munk 		swp = dv_attr.sw_parsing_caps.sw_parsing_offloads;
5342eb4d010SOphir Munk 	DRV_LOG(DEBUG, "SWP support: %u", swp);
5352eb4d010SOphir Munk #endif
5362eb4d010SOphir Munk 	config.swp = !!swp;
5372eb4d010SOphir Munk #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
5382eb4d010SOphir Munk 	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) {
5392eb4d010SOphir Munk 		struct mlx5dv_striding_rq_caps mprq_caps =
5402eb4d010SOphir Munk 			dv_attr.striding_rq_caps;
5412eb4d010SOphir Munk 
5422eb4d010SOphir Munk 		DRV_LOG(DEBUG, "\tmin_single_stride_log_num_of_bytes: %d",
5432eb4d010SOphir Munk 			mprq_caps.min_single_stride_log_num_of_bytes);
5442eb4d010SOphir Munk 		DRV_LOG(DEBUG, "\tmax_single_stride_log_num_of_bytes: %d",
5452eb4d010SOphir Munk 			mprq_caps.max_single_stride_log_num_of_bytes);
5462eb4d010SOphir Munk 		DRV_LOG(DEBUG, "\tmin_single_wqe_log_num_of_strides: %d",
5472eb4d010SOphir Munk 			mprq_caps.min_single_wqe_log_num_of_strides);
5482eb4d010SOphir Munk 		DRV_LOG(DEBUG, "\tmax_single_wqe_log_num_of_strides: %d",
5492eb4d010SOphir Munk 			mprq_caps.max_single_wqe_log_num_of_strides);
5502eb4d010SOphir Munk 		DRV_LOG(DEBUG, "\tsupported_qpts: %d",
5512eb4d010SOphir Munk 			mprq_caps.supported_qpts);
5522eb4d010SOphir Munk 		DRV_LOG(DEBUG, "device supports Multi-Packet RQ");
5532eb4d010SOphir Munk 		mprq = 1;
5542eb4d010SOphir Munk 		mprq_min_stride_size_n =
5552eb4d010SOphir Munk 			mprq_caps.min_single_stride_log_num_of_bytes;
5562eb4d010SOphir Munk 		mprq_max_stride_size_n =
5572eb4d010SOphir Munk 			mprq_caps.max_single_stride_log_num_of_bytes;
5582eb4d010SOphir Munk 		mprq_min_stride_num_n =
5592eb4d010SOphir Munk 			mprq_caps.min_single_wqe_log_num_of_strides;
5602eb4d010SOphir Munk 		mprq_max_stride_num_n =
5612eb4d010SOphir Munk 			mprq_caps.max_single_wqe_log_num_of_strides;
5622eb4d010SOphir Munk 	}
5632eb4d010SOphir Munk #endif
5642eb4d010SOphir Munk 	if (RTE_CACHE_LINE_SIZE == 128 &&
5652eb4d010SOphir Munk 	    !(dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP))
5662eb4d010SOphir Munk 		cqe_comp = 0;
5672eb4d010SOphir Munk 	else
5682eb4d010SOphir Munk 		cqe_comp = 1;
5692eb4d010SOphir Munk 	config.cqe_comp = cqe_comp;
5702eb4d010SOphir Munk #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD
5712eb4d010SOphir Munk 	/* Whether device supports 128B Rx CQE padding. */
5722eb4d010SOphir Munk 	cqe_pad = RTE_CACHE_LINE_SIZE == 128 &&
5732eb4d010SOphir Munk 		  (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD);
5742eb4d010SOphir Munk #endif
5752eb4d010SOphir Munk #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
5762eb4d010SOphir Munk 	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) {
5772eb4d010SOphir Munk 		tunnel_en = ((dv_attr.tunnel_offloads_caps &
5782eb4d010SOphir Munk 			      MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN) &&
5792eb4d010SOphir Munk 			     (dv_attr.tunnel_offloads_caps &
5802eb4d010SOphir Munk 			      MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE) &&
5812eb4d010SOphir Munk 			     (dv_attr.tunnel_offloads_caps &
5822eb4d010SOphir Munk 			      MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GENEVE));
5832eb4d010SOphir Munk 	}
5842eb4d010SOphir Munk 	DRV_LOG(DEBUG, "tunnel offloading is %ssupported",
5852eb4d010SOphir Munk 		tunnel_en ? "" : "not ");
5862eb4d010SOphir Munk #else
5872eb4d010SOphir Munk 	DRV_LOG(WARNING,
5882eb4d010SOphir Munk 		"tunnel offloading disabled due to old OFED/rdma-core version");
5892eb4d010SOphir Munk #endif
5902eb4d010SOphir Munk 	config.tunnel_en = tunnel_en;
5912eb4d010SOphir Munk #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
5922eb4d010SOphir Munk 	mpls_en = ((dv_attr.tunnel_offloads_caps &
5932eb4d010SOphir Munk 		    MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) &&
5942eb4d010SOphir Munk 		   (dv_attr.tunnel_offloads_caps &
5952eb4d010SOphir Munk 		    MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP));
5962eb4d010SOphir Munk 	DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is %ssupported",
5972eb4d010SOphir Munk 		mpls_en ? "" : "not ");
5982eb4d010SOphir Munk #else
5992eb4d010SOphir Munk 	DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to"
6002eb4d010SOphir Munk 		" old OFED/rdma-core version or firmware configuration");
6012eb4d010SOphir Munk #endif
6022eb4d010SOphir Munk 	config.mpls_en = mpls_en;
6032eb4d010SOphir Munk 	/* Check port status. */
604834a9019SOphir Munk 	err = mlx5_glue->query_port(sh->ctx, spawn->phys_port, &port_attr);
6052eb4d010SOphir Munk 	if (err) {
6062eb4d010SOphir Munk 		DRV_LOG(ERR, "port query failed: %s", strerror(err));
6072eb4d010SOphir Munk 		goto error;
6082eb4d010SOphir Munk 	}
6092eb4d010SOphir Munk 	if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
6102eb4d010SOphir Munk 		DRV_LOG(ERR, "port is not configured in Ethernet mode");
6112eb4d010SOphir Munk 		err = EINVAL;
6122eb4d010SOphir Munk 		goto error;
6132eb4d010SOphir Munk 	}
6142eb4d010SOphir Munk 	if (port_attr.state != IBV_PORT_ACTIVE)
6152eb4d010SOphir Munk 		DRV_LOG(DEBUG, "port is not active: \"%s\" (%d)",
6162eb4d010SOphir Munk 			mlx5_glue->port_state_str(port_attr.state),
6172eb4d010SOphir Munk 			port_attr.state);
6182eb4d010SOphir Munk 	/* Allocate private eth device data. */
6192eb4d010SOphir Munk 	priv = rte_zmalloc("ethdev private structure",
6202eb4d010SOphir Munk 			   sizeof(*priv),
6212eb4d010SOphir Munk 			   RTE_CACHE_LINE_SIZE);
6222eb4d010SOphir Munk 	if (priv == NULL) {
6232eb4d010SOphir Munk 		DRV_LOG(ERR, "priv allocation failure");
6242eb4d010SOphir Munk 		err = ENOMEM;
6252eb4d010SOphir Munk 		goto error;
6262eb4d010SOphir Munk 	}
6272eb4d010SOphir Munk 	priv->sh = sh;
62891389890SOphir Munk 	priv->dev_port = spawn->phys_port;
6292eb4d010SOphir Munk 	priv->pci_dev = spawn->pci_dev;
6302eb4d010SOphir Munk 	priv->mtu = RTE_ETHER_MTU;
6312eb4d010SOphir Munk 	priv->mp_id.port_id = port_id;
6322eb4d010SOphir Munk 	strlcpy(priv->mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN);
6332eb4d010SOphir Munk #ifndef RTE_ARCH_64
6342eb4d010SOphir Munk 	/* Initialize UAR access locks for 32bit implementations. */
6352eb4d010SOphir Munk 	rte_spinlock_init(&priv->uar_lock_cq);
6362eb4d010SOphir Munk 	for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++)
6372eb4d010SOphir Munk 		rte_spinlock_init(&priv->uar_lock[i]);
6382eb4d010SOphir Munk #endif
6392eb4d010SOphir Munk 	/* Some internal functions rely on Netlink sockets, open them now. */
6402eb4d010SOphir Munk 	priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA);
6412eb4d010SOphir Munk 	priv->nl_socket_route =	mlx5_nl_init(NETLINK_ROUTE);
6422eb4d010SOphir Munk 	priv->representor = !!switch_info->representor;
6432eb4d010SOphir Munk 	priv->master = !!switch_info->master;
6442eb4d010SOphir Munk 	priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
6452eb4d010SOphir Munk 	priv->vport_meta_tag = 0;
6462eb4d010SOphir Munk 	priv->vport_meta_mask = 0;
6472eb4d010SOphir Munk 	priv->pf_bond = spawn->pf_bond;
6482eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_DEVX_PORT
6492eb4d010SOphir Munk 	/*
6502eb4d010SOphir Munk 	 * The DevX port query API is implemented. E-Switch may use
6512eb4d010SOphir Munk 	 * either vport or reg_c[0] metadata register to match on
6522eb4d010SOphir Munk 	 * vport index. The engaged part of metadata register is
6532eb4d010SOphir Munk 	 * defined by mask.
6542eb4d010SOphir Munk 	 */
6552eb4d010SOphir Munk 	if (switch_info->representor || switch_info->master) {
6562eb4d010SOphir Munk 		devx_port.comp_mask = MLX5DV_DEVX_PORT_VPORT |
6572eb4d010SOphir Munk 				      MLX5DV_DEVX_PORT_MATCH_REG_C_0;
658834a9019SOphir Munk 		err = mlx5_glue->devx_port_query(sh->ctx, spawn->phys_port,
6592eb4d010SOphir Munk 						 &devx_port);
6602eb4d010SOphir Munk 		if (err) {
6612eb4d010SOphir Munk 			DRV_LOG(WARNING,
6622eb4d010SOphir Munk 				"can't query devx port %d on device %s",
663834a9019SOphir Munk 				spawn->phys_port,
664834a9019SOphir Munk 				mlx5_os_get_dev_device_name(spawn->phys_dev));
6652eb4d010SOphir Munk 			devx_port.comp_mask = 0;
6662eb4d010SOphir Munk 		}
6672eb4d010SOphir Munk 	}
6682eb4d010SOphir Munk 	if (devx_port.comp_mask & MLX5DV_DEVX_PORT_MATCH_REG_C_0) {
6692eb4d010SOphir Munk 		priv->vport_meta_tag = devx_port.reg_c_0.value;
6702eb4d010SOphir Munk 		priv->vport_meta_mask = devx_port.reg_c_0.mask;
6712eb4d010SOphir Munk 		if (!priv->vport_meta_mask) {
6722eb4d010SOphir Munk 			DRV_LOG(ERR, "vport zero mask for port %d"
6732eb4d010SOphir Munk 				     " on bonding device %s",
674834a9019SOphir Munk 				     spawn->phys_port,
675834a9019SOphir Munk 				     mlx5_os_get_dev_device_name
676834a9019SOphir Munk 							(spawn->phys_dev));
6772eb4d010SOphir Munk 			err = ENOTSUP;
6782eb4d010SOphir Munk 			goto error;
6792eb4d010SOphir Munk 		}
6802eb4d010SOphir Munk 		if (priv->vport_meta_tag & ~priv->vport_meta_mask) {
6812eb4d010SOphir Munk 			DRV_LOG(ERR, "invalid vport tag for port %d"
6822eb4d010SOphir Munk 				     " on bonding device %s",
683834a9019SOphir Munk 				     spawn->phys_port,
684834a9019SOphir Munk 				     mlx5_os_get_dev_device_name
685834a9019SOphir Munk 							(spawn->phys_dev));
6862eb4d010SOphir Munk 			err = ENOTSUP;
6872eb4d010SOphir Munk 			goto error;
6882eb4d010SOphir Munk 		}
6892eb4d010SOphir Munk 	}
6902eb4d010SOphir Munk 	if (devx_port.comp_mask & MLX5DV_DEVX_PORT_VPORT) {
6912eb4d010SOphir Munk 		priv->vport_id = devx_port.vport_num;
6922eb4d010SOphir Munk 	} else if (spawn->pf_bond >= 0) {
6932eb4d010SOphir Munk 		DRV_LOG(ERR, "can't deduce vport index for port %d"
6942eb4d010SOphir Munk 			     " on bonding device %s",
695834a9019SOphir Munk 			     spawn->phys_port,
696834a9019SOphir Munk 			     mlx5_os_get_dev_device_name(spawn->phys_dev));
6972eb4d010SOphir Munk 		err = ENOTSUP;
6982eb4d010SOphir Munk 		goto error;
6992eb4d010SOphir Munk 	} else {
7002eb4d010SOphir Munk 		/* Suppose vport index in compatible way. */
7012eb4d010SOphir Munk 		priv->vport_id = switch_info->representor ?
7022eb4d010SOphir Munk 				 switch_info->port_name + 1 : -1;
7032eb4d010SOphir Munk 	}
7042eb4d010SOphir Munk #else
7052eb4d010SOphir Munk 	/*
7062eb4d010SOphir Munk 	 * Kernel/rdma_core support single E-Switch per PF configurations
7072eb4d010SOphir Munk 	 * only and vport_id field contains the vport index for
7082eb4d010SOphir Munk 	 * associated VF, which is deduced from representor port name.
7092eb4d010SOphir Munk 	 * For example, let's have the IB device port 10, it has
7102eb4d010SOphir Munk 	 * attached network device eth0, which has port name attribute
7112eb4d010SOphir Munk 	 * pf0vf2, we can deduce the VF number as 2, and set vport index
7122eb4d010SOphir Munk 	 * as 3 (2+1). This assigning schema should be changed if the
7132eb4d010SOphir Munk 	 * multiple E-Switch instances per PF configurations or/and PCI
7142eb4d010SOphir Munk 	 * subfunctions are added.
7152eb4d010SOphir Munk 	 */
7162eb4d010SOphir Munk 	priv->vport_id = switch_info->representor ?
7172eb4d010SOphir Munk 			 switch_info->port_name + 1 : -1;
7182eb4d010SOphir Munk #endif
7192eb4d010SOphir Munk 	/* representor_id field keeps the unmodified VF index. */
7202eb4d010SOphir Munk 	priv->representor_id = switch_info->representor ?
7212eb4d010SOphir Munk 			       switch_info->port_name : -1;
7222eb4d010SOphir Munk 	/*
7232eb4d010SOphir Munk 	 * Look for sibling devices in order to reuse their switch domain
7242eb4d010SOphir Munk 	 * if any, otherwise allocate one.
7252eb4d010SOphir Munk 	 */
7262eb4d010SOphir Munk 	MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
7272eb4d010SOphir Munk 		const struct mlx5_priv *opriv =
7282eb4d010SOphir Munk 			rte_eth_devices[port_id].data->dev_private;
7292eb4d010SOphir Munk 
7302eb4d010SOphir Munk 		if (!opriv ||
7312eb4d010SOphir Munk 		    opriv->sh != priv->sh ||
7322eb4d010SOphir Munk 			opriv->domain_id ==
7332eb4d010SOphir Munk 			RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
7342eb4d010SOphir Munk 			continue;
7352eb4d010SOphir Munk 		priv->domain_id = opriv->domain_id;
7362eb4d010SOphir Munk 		break;
7372eb4d010SOphir Munk 	}
7382eb4d010SOphir Munk 	if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
7392eb4d010SOphir Munk 		err = rte_eth_switch_domain_alloc(&priv->domain_id);
7402eb4d010SOphir Munk 		if (err) {
7412eb4d010SOphir Munk 			err = rte_errno;
7422eb4d010SOphir Munk 			DRV_LOG(ERR, "unable to allocate switch domain: %s",
7432eb4d010SOphir Munk 				strerror(rte_errno));
7442eb4d010SOphir Munk 			goto error;
7452eb4d010SOphir Munk 		}
7462eb4d010SOphir Munk 		own_domain_id = 1;
7472eb4d010SOphir Munk 	}
7482eb4d010SOphir Munk 	/* Override some values set by hardware configuration. */
7492eb4d010SOphir Munk 	mlx5_args(&config, dpdk_dev->devargs);
7502eb4d010SOphir Munk 	err = mlx5_dev_check_sibling_config(priv, &config);
7512eb4d010SOphir Munk 	if (err)
7522eb4d010SOphir Munk 		goto error;
7532eb4d010SOphir Munk 	config.hw_csum = !!(sh->device_attr.device_cap_flags_ex &
7542eb4d010SOphir Munk 			    IBV_DEVICE_RAW_IP_CSUM);
7552eb4d010SOphir Munk 	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
7562eb4d010SOphir Munk 		(config.hw_csum ? "" : "not "));
7572eb4d010SOphir Munk #if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
7582eb4d010SOphir Munk 	!defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
7592eb4d010SOphir Munk 	DRV_LOG(DEBUG, "counters are not supported");
7602eb4d010SOphir Munk #endif
7612eb4d010SOphir Munk #if !defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_MLX5DV_DR)
7622eb4d010SOphir Munk 	if (config.dv_flow_en) {
7632eb4d010SOphir Munk 		DRV_LOG(WARNING, "DV flow is not supported");
7642eb4d010SOphir Munk 		config.dv_flow_en = 0;
7652eb4d010SOphir Munk 	}
7662eb4d010SOphir Munk #endif
7672eb4d010SOphir Munk 	config.ind_table_max_size =
7682eb4d010SOphir Munk 		sh->device_attr.max_rwq_indirection_table_size;
7692eb4d010SOphir Munk 	/*
7702eb4d010SOphir Munk 	 * Remove this check once DPDK supports larger/variable
7712eb4d010SOphir Munk 	 * indirection tables.
7722eb4d010SOphir Munk 	 */
7732eb4d010SOphir Munk 	if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512)
7742eb4d010SOphir Munk 		config.ind_table_max_size = ETH_RSS_RETA_SIZE_512;
7752eb4d010SOphir Munk 	DRV_LOG(DEBUG, "maximum Rx indirection table size is %u",
7762eb4d010SOphir Munk 		config.ind_table_max_size);
7772eb4d010SOphir Munk 	config.hw_vlan_strip = !!(sh->device_attr.raw_packet_caps &
7782eb4d010SOphir Munk 				  IBV_RAW_PACKET_CAP_CVLAN_STRIPPING);
7792eb4d010SOphir Munk 	DRV_LOG(DEBUG, "VLAN stripping is %ssupported",
7802eb4d010SOphir Munk 		(config.hw_vlan_strip ? "" : "not "));
7812eb4d010SOphir Munk 	config.hw_fcs_strip = !!(sh->device_attr.raw_packet_caps &
7822eb4d010SOphir Munk 				 IBV_RAW_PACKET_CAP_SCATTER_FCS);
7832eb4d010SOphir Munk 	DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported",
7842eb4d010SOphir Munk 		(config.hw_fcs_strip ? "" : "not "));
7852eb4d010SOphir Munk #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)
7862eb4d010SOphir Munk 	hw_padding = !!sh->device_attr.rx_pad_end_addr_align;
7872eb4d010SOphir Munk #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)
7882eb4d010SOphir Munk 	hw_padding = !!(sh->device_attr.device_cap_flags_ex &
7892eb4d010SOphir Munk 			IBV_DEVICE_PCI_WRITE_END_PADDING);
7902eb4d010SOphir Munk #endif
7912eb4d010SOphir Munk 	if (config.hw_padding && !hw_padding) {
7922eb4d010SOphir Munk 		DRV_LOG(DEBUG, "Rx end alignment padding isn't supported");
7932eb4d010SOphir Munk 		config.hw_padding = 0;
7942eb4d010SOphir Munk 	} else if (config.hw_padding) {
7952eb4d010SOphir Munk 		DRV_LOG(DEBUG, "Rx end alignment padding is enabled");
7962eb4d010SOphir Munk 	}
7972eb4d010SOphir Munk 	config.tso = (sh->device_attr.max_tso > 0 &&
7982eb4d010SOphir Munk 		      (sh->device_attr.tso_supported_qpts &
7992eb4d010SOphir Munk 		       (1 << IBV_QPT_RAW_PACKET)));
8002eb4d010SOphir Munk 	if (config.tso)
8012eb4d010SOphir Munk 		config.tso_max_payload_sz = sh->device_attr.max_tso;
8022eb4d010SOphir Munk 	/*
8032eb4d010SOphir Munk 	 * MPW is disabled by default, while the Enhanced MPW is enabled
8042eb4d010SOphir Munk 	 * by default.
8052eb4d010SOphir Munk 	 */
8062eb4d010SOphir Munk 	if (config.mps == MLX5_ARG_UNSET)
8072eb4d010SOphir Munk 		config.mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED :
8082eb4d010SOphir Munk 							  MLX5_MPW_DISABLED;
8092eb4d010SOphir Munk 	else
8102eb4d010SOphir Munk 		config.mps = config.mps ? mps : MLX5_MPW_DISABLED;
8112eb4d010SOphir Munk 	DRV_LOG(INFO, "%sMPS is %s",
8122eb4d010SOphir Munk 		config.mps == MLX5_MPW_ENHANCED ? "enhanced " :
8132eb4d010SOphir Munk 		config.mps == MLX5_MPW ? "legacy " : "",
8142eb4d010SOphir Munk 		config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled");
8152eb4d010SOphir Munk 	if (config.cqe_comp && !cqe_comp) {
8162eb4d010SOphir Munk 		DRV_LOG(WARNING, "Rx CQE compression isn't supported");
8172eb4d010SOphir Munk 		config.cqe_comp = 0;
8182eb4d010SOphir Munk 	}
8192eb4d010SOphir Munk 	if (config.cqe_pad && !cqe_pad) {
8202eb4d010SOphir Munk 		DRV_LOG(WARNING, "Rx CQE padding isn't supported");
8212eb4d010SOphir Munk 		config.cqe_pad = 0;
8222eb4d010SOphir Munk 	} else if (config.cqe_pad) {
8232eb4d010SOphir Munk 		DRV_LOG(INFO, "Rx CQE padding is enabled");
8242eb4d010SOphir Munk 	}
8252eb4d010SOphir Munk 	if (config.devx) {
8262eb4d010SOphir Munk 		priv->counter_fallback = 0;
8272eb4d010SOphir Munk 		err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config.hca_attr);
8282eb4d010SOphir Munk 		if (err) {
8292eb4d010SOphir Munk 			err = -err;
8302eb4d010SOphir Munk 			goto error;
8312eb4d010SOphir Munk 		}
8322eb4d010SOphir Munk 		if (!config.hca_attr.flow_counters_dump)
8332eb4d010SOphir Munk 			priv->counter_fallback = 1;
8342eb4d010SOphir Munk #ifndef HAVE_IBV_DEVX_ASYNC
8352eb4d010SOphir Munk 		priv->counter_fallback = 1;
8362eb4d010SOphir Munk #endif
8372eb4d010SOphir Munk 		if (priv->counter_fallback)
8382eb4d010SOphir Munk 			DRV_LOG(INFO, "Use fall-back DV counter management");
8392eb4d010SOphir Munk 		/* Check for LRO support. */
8402eb4d010SOphir Munk 		if (config.dest_tir && config.hca_attr.lro_cap &&
8412eb4d010SOphir Munk 		    config.dv_flow_en) {
8422eb4d010SOphir Munk 			/* TBD check tunnel lro caps. */
8432eb4d010SOphir Munk 			config.lro.supported = config.hca_attr.lro_cap;
8442eb4d010SOphir Munk 			DRV_LOG(DEBUG, "Device supports LRO");
8452eb4d010SOphir Munk 			/*
8462eb4d010SOphir Munk 			 * If LRO timeout is not configured by application,
8472eb4d010SOphir Munk 			 * use the minimal supported value.
8482eb4d010SOphir Munk 			 */
8492eb4d010SOphir Munk 			if (!config.lro.timeout)
8502eb4d010SOphir Munk 				config.lro.timeout =
8512eb4d010SOphir Munk 				config.hca_attr.lro_timer_supported_periods[0];
8522eb4d010SOphir Munk 			DRV_LOG(DEBUG, "LRO session timeout set to %d usec",
8532eb4d010SOphir Munk 				config.lro.timeout);
8542eb4d010SOphir Munk 		}
8552eb4d010SOphir Munk #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
8562eb4d010SOphir Munk 		if (config.hca_attr.qos.sup && config.hca_attr.qos.srtcm_sup &&
8572eb4d010SOphir Munk 		    config.dv_flow_en) {
8582eb4d010SOphir Munk 			uint8_t reg_c_mask =
8592eb4d010SOphir Munk 				config.hca_attr.qos.flow_meter_reg_c_ids;
8602eb4d010SOphir Munk 			/*
8612eb4d010SOphir Munk 			 * Meter needs two REG_C's for color match and pre-sfx
8622eb4d010SOphir Munk 			 * flow match. Here get the REG_C for color match.
8632eb4d010SOphir Munk 			 * REG_C_0 and REG_C_1 is reserved for metadata feature.
8642eb4d010SOphir Munk 			 */
8652eb4d010SOphir Munk 			reg_c_mask &= 0xfc;
8662eb4d010SOphir Munk 			if (__builtin_popcount(reg_c_mask) < 1) {
8672eb4d010SOphir Munk 				priv->mtr_en = 0;
8682eb4d010SOphir Munk 				DRV_LOG(WARNING, "No available register for"
8692eb4d010SOphir Munk 					" meter.");
8702eb4d010SOphir Munk 			} else {
8712eb4d010SOphir Munk 				priv->mtr_color_reg = ffs(reg_c_mask) - 1 +
8722eb4d010SOphir Munk 						      REG_C_0;
8732eb4d010SOphir Munk 				priv->mtr_en = 1;
8742eb4d010SOphir Munk 				priv->mtr_reg_share =
8752eb4d010SOphir Munk 				      config.hca_attr.qos.flow_meter_reg_share;
8762eb4d010SOphir Munk 				DRV_LOG(DEBUG, "The REG_C meter uses is %d",
8772eb4d010SOphir Munk 					priv->mtr_color_reg);
8782eb4d010SOphir Munk 			}
8792eb4d010SOphir Munk 		}
8802eb4d010SOphir Munk #endif
8812eb4d010SOphir Munk 	}
8822eb4d010SOphir Munk 	if (config.mprq.enabled && mprq) {
8832eb4d010SOphir Munk 		if (config.mprq.stride_num_n &&
8842eb4d010SOphir Munk 		    (config.mprq.stride_num_n > mprq_max_stride_num_n ||
8852eb4d010SOphir Munk 		     config.mprq.stride_num_n < mprq_min_stride_num_n)) {
8862eb4d010SOphir Munk 			config.mprq.stride_num_n =
8872eb4d010SOphir Munk 				RTE_MIN(RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
8882eb4d010SOphir Munk 						mprq_min_stride_num_n),
8892eb4d010SOphir Munk 					mprq_max_stride_num_n);
8902eb4d010SOphir Munk 			DRV_LOG(WARNING,
8912eb4d010SOphir Munk 				"the number of strides"
8922eb4d010SOphir Munk 				" for Multi-Packet RQ is out of range,"
8932eb4d010SOphir Munk 				" setting default value (%u)",
8942eb4d010SOphir Munk 				1 << config.mprq.stride_num_n);
8952eb4d010SOphir Munk 		}
8962eb4d010SOphir Munk 		if (config.mprq.stride_size_n &&
8972eb4d010SOphir Munk 		    (config.mprq.stride_size_n > mprq_max_stride_size_n ||
8982eb4d010SOphir Munk 		     config.mprq.stride_size_n < mprq_min_stride_size_n)) {
8992eb4d010SOphir Munk 			config.mprq.stride_size_n =
9002eb4d010SOphir Munk 				RTE_MIN(RTE_MAX(MLX5_MPRQ_STRIDE_SIZE_N,
9012eb4d010SOphir Munk 						mprq_min_stride_size_n),
9022eb4d010SOphir Munk 					mprq_max_stride_size_n);
9032eb4d010SOphir Munk 			DRV_LOG(WARNING,
9042eb4d010SOphir Munk 				"the size of a stride"
9052eb4d010SOphir Munk 				" for Multi-Packet RQ is out of range,"
9062eb4d010SOphir Munk 				" setting default value (%u)",
9072eb4d010SOphir Munk 				1 << config.mprq.stride_size_n);
9082eb4d010SOphir Munk 		}
9092eb4d010SOphir Munk 		config.mprq.min_stride_size_n = mprq_min_stride_size_n;
9102eb4d010SOphir Munk 		config.mprq.max_stride_size_n = mprq_max_stride_size_n;
9112eb4d010SOphir Munk 	} else if (config.mprq.enabled && !mprq) {
9122eb4d010SOphir Munk 		DRV_LOG(WARNING, "Multi-Packet RQ isn't supported");
9132eb4d010SOphir Munk 		config.mprq.enabled = 0;
9142eb4d010SOphir Munk 	}
9152eb4d010SOphir Munk 	if (config.max_dump_files_num == 0)
9162eb4d010SOphir Munk 		config.max_dump_files_num = 128;
9172eb4d010SOphir Munk 	eth_dev = rte_eth_dev_allocate(name);
9182eb4d010SOphir Munk 	if (eth_dev == NULL) {
9192eb4d010SOphir Munk 		DRV_LOG(ERR, "can not allocate rte ethdev");
9202eb4d010SOphir Munk 		err = ENOMEM;
9212eb4d010SOphir Munk 		goto error;
9222eb4d010SOphir Munk 	}
9232eb4d010SOphir Munk 	/* Flag to call rte_eth_dev_release_port() in rte_eth_dev_close(). */
9242eb4d010SOphir Munk 	eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
9252eb4d010SOphir Munk 	if (priv->representor) {
9262eb4d010SOphir Munk 		eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
9272eb4d010SOphir Munk 		eth_dev->data->representor_id = priv->representor_id;
9282eb4d010SOphir Munk 	}
9292eb4d010SOphir Munk 	/*
9302eb4d010SOphir Munk 	 * Store associated network device interface index. This index
9312eb4d010SOphir Munk 	 * is permanent throughout the lifetime of device. So, we may store
9322eb4d010SOphir Munk 	 * the ifindex here and use the cached value further.
9332eb4d010SOphir Munk 	 */
9342eb4d010SOphir Munk 	MLX5_ASSERT(spawn->ifindex);
9352eb4d010SOphir Munk 	priv->if_index = spawn->ifindex;
9362eb4d010SOphir Munk 	eth_dev->data->dev_private = priv;
9372eb4d010SOphir Munk 	priv->dev_data = eth_dev->data;
9382eb4d010SOphir Munk 	eth_dev->data->mac_addrs = priv->mac;
9392eb4d010SOphir Munk 	eth_dev->device = dpdk_dev;
9402eb4d010SOphir Munk 	/* Configure the first MAC address by default. */
9412eb4d010SOphir Munk 	if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
9422eb4d010SOphir Munk 		DRV_LOG(ERR,
9432eb4d010SOphir Munk 			"port %u cannot get MAC address, is mlx5_en"
9442eb4d010SOphir Munk 			" loaded? (errno: %s)",
9452eb4d010SOphir Munk 			eth_dev->data->port_id, strerror(rte_errno));
9462eb4d010SOphir Munk 		err = ENODEV;
9472eb4d010SOphir Munk 		goto error;
9482eb4d010SOphir Munk 	}
9492eb4d010SOphir Munk 	DRV_LOG(INFO,
9502eb4d010SOphir Munk 		"port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
9512eb4d010SOphir Munk 		eth_dev->data->port_id,
9522eb4d010SOphir Munk 		mac.addr_bytes[0], mac.addr_bytes[1],
9532eb4d010SOphir Munk 		mac.addr_bytes[2], mac.addr_bytes[3],
9542eb4d010SOphir Munk 		mac.addr_bytes[4], mac.addr_bytes[5]);
9552eb4d010SOphir Munk #ifdef RTE_LIBRTE_MLX5_DEBUG
9562eb4d010SOphir Munk 	{
9572eb4d010SOphir Munk 		char ifname[IF_NAMESIZE];
9582eb4d010SOphir Munk 
9592eb4d010SOphir Munk 		if (mlx5_get_ifname(eth_dev, &ifname) == 0)
9602eb4d010SOphir Munk 			DRV_LOG(DEBUG, "port %u ifname is \"%s\"",
9612eb4d010SOphir Munk 				eth_dev->data->port_id, ifname);
9622eb4d010SOphir Munk 		else
9632eb4d010SOphir Munk 			DRV_LOG(DEBUG, "port %u ifname is unknown",
9642eb4d010SOphir Munk 				eth_dev->data->port_id);
9652eb4d010SOphir Munk 	}
9662eb4d010SOphir Munk #endif
9672eb4d010SOphir Munk 	/* Get actual MTU if possible. */
9682eb4d010SOphir Munk 	err = mlx5_get_mtu(eth_dev, &priv->mtu);
9692eb4d010SOphir Munk 	if (err) {
9702eb4d010SOphir Munk 		err = rte_errno;
9712eb4d010SOphir Munk 		goto error;
9722eb4d010SOphir Munk 	}
9732eb4d010SOphir Munk 	DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id,
9742eb4d010SOphir Munk 		priv->mtu);
9752eb4d010SOphir Munk 	/* Initialize burst functions to prevent crashes before link-up. */
9762eb4d010SOphir Munk 	eth_dev->rx_pkt_burst = removed_rx_burst;
9772eb4d010SOphir Munk 	eth_dev->tx_pkt_burst = removed_tx_burst;
978042f5c94SOphir Munk 	eth_dev->dev_ops = &mlx5_os_dev_ops;
9792eb4d010SOphir Munk 	/* Register MAC address. */
9802eb4d010SOphir Munk 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
9812eb4d010SOphir Munk 	if (config.vf && config.vf_nl_en)
9822eb4d010SOphir Munk 		mlx5_nl_mac_addr_sync(priv->nl_socket_route,
9832eb4d010SOphir Munk 				      mlx5_ifindex(eth_dev),
9842eb4d010SOphir Munk 				      eth_dev->data->mac_addrs,
9852eb4d010SOphir Munk 				      MLX5_MAX_MAC_ADDRESSES);
9862eb4d010SOphir Munk 	priv->flows = 0;
9872eb4d010SOphir Munk 	priv->ctrl_flows = 0;
9882eb4d010SOphir Munk 	TAILQ_INIT(&priv->flow_meters);
9892eb4d010SOphir Munk 	TAILQ_INIT(&priv->flow_meter_profiles);
9902eb4d010SOphir Munk 	/* Hint libmlx5 to use PMD allocator for data plane resources */
9912eb4d010SOphir Munk 	struct mlx5dv_ctx_allocators alctr = {
9922eb4d010SOphir Munk 		.alloc = &mlx5_alloc_verbs_buf,
9932eb4d010SOphir Munk 		.free = &mlx5_free_verbs_buf,
9942eb4d010SOphir Munk 		.data = priv,
9952eb4d010SOphir Munk 	};
9962eb4d010SOphir Munk 	mlx5_glue->dv_set_context_attr(sh->ctx,
9972eb4d010SOphir Munk 				       MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
9982eb4d010SOphir Munk 				       (void *)((uintptr_t)&alctr));
9992eb4d010SOphir Munk 	/* Bring Ethernet device up. */
10002eb4d010SOphir Munk 	DRV_LOG(DEBUG, "port %u forcing Ethernet interface up",
10012eb4d010SOphir Munk 		eth_dev->data->port_id);
10022eb4d010SOphir Munk 	mlx5_set_link_up(eth_dev);
10032eb4d010SOphir Munk 	/*
10042eb4d010SOphir Munk 	 * Even though the interrupt handler is not installed yet,
10052eb4d010SOphir Munk 	 * interrupts will still trigger on the async_fd from
10062eb4d010SOphir Munk 	 * Verbs context returned by ibv_open_device().
10072eb4d010SOphir Munk 	 */
10082eb4d010SOphir Munk 	mlx5_link_update(eth_dev, 0);
10092eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_ESWITCH
10102eb4d010SOphir Munk 	if (!(config.hca_attr.eswitch_manager && config.dv_flow_en &&
10112eb4d010SOphir Munk 	      (switch_info->representor || switch_info->master)))
10122eb4d010SOphir Munk 		config.dv_esw_en = 0;
10132eb4d010SOphir Munk #else
10142eb4d010SOphir Munk 	config.dv_esw_en = 0;
10152eb4d010SOphir Munk #endif
10162eb4d010SOphir Munk 	/* Detect minimal data bytes to inline. */
10172eb4d010SOphir Munk 	mlx5_set_min_inline(spawn, &config);
10182eb4d010SOphir Munk 	/* Store device configuration on private structure. */
10192eb4d010SOphir Munk 	priv->config = config;
10202eb4d010SOphir Munk 	/* Create context for virtual machine VLAN workaround. */
10212eb4d010SOphir Munk 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
10222eb4d010SOphir Munk 	if (config.dv_flow_en) {
10232eb4d010SOphir Munk 		err = mlx5_alloc_shared_dr(priv);
10242eb4d010SOphir Munk 		if (err)
10252eb4d010SOphir Munk 			goto error;
10262eb4d010SOphir Munk 		/*
10272eb4d010SOphir Munk 		 * RSS id is shared with meter flow id. Meter flow id can only
10282eb4d010SOphir Munk 		 * use the 24 MSB of the register.
10292eb4d010SOphir Munk 		 */
10302eb4d010SOphir Munk 		priv->qrss_id_pool = mlx5_flow_id_pool_alloc(UINT32_MAX >>
10312eb4d010SOphir Munk 				     MLX5_MTR_COLOR_BITS);
10322eb4d010SOphir Munk 		if (!priv->qrss_id_pool) {
10332eb4d010SOphir Munk 			DRV_LOG(ERR, "can't create flow id pool");
10342eb4d010SOphir Munk 			err = ENOMEM;
10352eb4d010SOphir Munk 			goto error;
10362eb4d010SOphir Munk 		}
10372eb4d010SOphir Munk 	}
10382eb4d010SOphir Munk 	/* Supported Verbs flow priority number detection. */
10392eb4d010SOphir Munk 	err = mlx5_flow_discover_priorities(eth_dev);
10402eb4d010SOphir Munk 	if (err < 0) {
10412eb4d010SOphir Munk 		err = -err;
10422eb4d010SOphir Munk 		goto error;
10432eb4d010SOphir Munk 	}
10442eb4d010SOphir Munk 	priv->config.flow_prio = err;
10452eb4d010SOphir Munk 	if (!priv->config.dv_esw_en &&
10462eb4d010SOphir Munk 	    priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
10472eb4d010SOphir Munk 		DRV_LOG(WARNING, "metadata mode %u is not supported "
10482eb4d010SOphir Munk 				 "(no E-Switch)", priv->config.dv_xmeta_en);
10492eb4d010SOphir Munk 		priv->config.dv_xmeta_en = MLX5_XMETA_MODE_LEGACY;
10502eb4d010SOphir Munk 	}
10512eb4d010SOphir Munk 	mlx5_set_metadata_mask(eth_dev);
10522eb4d010SOphir Munk 	if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
10532eb4d010SOphir Munk 	    !priv->sh->dv_regc0_mask) {
10542eb4d010SOphir Munk 		DRV_LOG(ERR, "metadata mode %u is not supported "
10552eb4d010SOphir Munk 			     "(no metadata reg_c[0] is available)",
10562eb4d010SOphir Munk 			     priv->config.dv_xmeta_en);
10572eb4d010SOphir Munk 			err = ENOTSUP;
10582eb4d010SOphir Munk 			goto error;
10592eb4d010SOphir Munk 	}
10602eb4d010SOphir Munk 	/*
10612eb4d010SOphir Munk 	 * Allocate the buffer for flow creating, just once.
10622eb4d010SOphir Munk 	 * The allocation must be done before any flow creating.
10632eb4d010SOphir Munk 	 */
10642eb4d010SOphir Munk 	mlx5_flow_alloc_intermediate(eth_dev);
10652eb4d010SOphir Munk 	/* Query availability of metadata reg_c's. */
10662eb4d010SOphir Munk 	err = mlx5_flow_discover_mreg_c(eth_dev);
10672eb4d010SOphir Munk 	if (err < 0) {
10682eb4d010SOphir Munk 		err = -err;
10692eb4d010SOphir Munk 		goto error;
10702eb4d010SOphir Munk 	}
10712eb4d010SOphir Munk 	if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
10722eb4d010SOphir Munk 		DRV_LOG(DEBUG,
10732eb4d010SOphir Munk 			"port %u extensive metadata register is not supported",
10742eb4d010SOphir Munk 			eth_dev->data->port_id);
10752eb4d010SOphir Munk 		if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
10762eb4d010SOphir Munk 			DRV_LOG(ERR, "metadata mode %u is not supported "
10772eb4d010SOphir Munk 				     "(no metadata registers available)",
10782eb4d010SOphir Munk 				     priv->config.dv_xmeta_en);
10792eb4d010SOphir Munk 			err = ENOTSUP;
10802eb4d010SOphir Munk 			goto error;
10812eb4d010SOphir Munk 		}
10822eb4d010SOphir Munk 	}
10832eb4d010SOphir Munk 	if (priv->config.dv_flow_en &&
10842eb4d010SOphir Munk 	    priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
10852eb4d010SOphir Munk 	    mlx5_flow_ext_mreg_supported(eth_dev) &&
10862eb4d010SOphir Munk 	    priv->sh->dv_regc0_mask) {
10872eb4d010SOphir Munk 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
10882eb4d010SOphir Munk 						      MLX5_FLOW_MREG_HTABLE_SZ);
10892eb4d010SOphir Munk 		if (!priv->mreg_cp_tbl) {
10902eb4d010SOphir Munk 			err = ENOMEM;
10912eb4d010SOphir Munk 			goto error;
10922eb4d010SOphir Munk 		}
10932eb4d010SOphir Munk 	}
10942eb4d010SOphir Munk 	return eth_dev;
10952eb4d010SOphir Munk error:
10962eb4d010SOphir Munk 	if (priv) {
10972eb4d010SOphir Munk 		if (priv->mreg_cp_tbl)
10982eb4d010SOphir Munk 			mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
10992eb4d010SOphir Munk 		if (priv->sh)
11002eb4d010SOphir Munk 			mlx5_os_free_shared_dr(priv);
11012eb4d010SOphir Munk 		if (priv->nl_socket_route >= 0)
11022eb4d010SOphir Munk 			close(priv->nl_socket_route);
11032eb4d010SOphir Munk 		if (priv->nl_socket_rdma >= 0)
11042eb4d010SOphir Munk 			close(priv->nl_socket_rdma);
11052eb4d010SOphir Munk 		if (priv->vmwa_context)
11062eb4d010SOphir Munk 			mlx5_vlan_vmwa_exit(priv->vmwa_context);
11072eb4d010SOphir Munk 		if (priv->qrss_id_pool)
11082eb4d010SOphir Munk 			mlx5_flow_id_pool_release(priv->qrss_id_pool);
11092eb4d010SOphir Munk 		if (own_domain_id)
11102eb4d010SOphir Munk 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
11112eb4d010SOphir Munk 		rte_free(priv);
11122eb4d010SOphir Munk 		if (eth_dev != NULL)
11132eb4d010SOphir Munk 			eth_dev->data->dev_private = NULL;
11142eb4d010SOphir Munk 	}
11152eb4d010SOphir Munk 	if (eth_dev != NULL) {
11162eb4d010SOphir Munk 		/* mac_addrs must not be freed alone because part of
11172eb4d010SOphir Munk 		 * dev_private
11182eb4d010SOphir Munk 		 **/
11192eb4d010SOphir Munk 		eth_dev->data->mac_addrs = NULL;
11202eb4d010SOphir Munk 		rte_eth_dev_release_port(eth_dev);
11212eb4d010SOphir Munk 	}
11222eb4d010SOphir Munk 	if (sh)
112391389890SOphir Munk 		mlx5_free_shared_dev_ctx(sh);
11242eb4d010SOphir Munk 	MLX5_ASSERT(err > 0);
11252eb4d010SOphir Munk 	rte_errno = err;
11262eb4d010SOphir Munk 	return NULL;
11272eb4d010SOphir Munk }
11282eb4d010SOphir Munk 
11292eb4d010SOphir Munk /**
11302eb4d010SOphir Munk  * Comparison callback to sort device data.
11312eb4d010SOphir Munk  *
11322eb4d010SOphir Munk  * This is meant to be used with qsort().
11332eb4d010SOphir Munk  *
11342eb4d010SOphir Munk  * @param a[in]
11352eb4d010SOphir Munk  *   Pointer to pointer to first data object.
11362eb4d010SOphir Munk  * @param b[in]
11372eb4d010SOphir Munk  *   Pointer to pointer to second data object.
11382eb4d010SOphir Munk  *
11392eb4d010SOphir Munk  * @return
11402eb4d010SOphir Munk  *   0 if both objects are equal, less than 0 if the first argument is less
11412eb4d010SOphir Munk  *   than the second, greater than 0 otherwise.
11422eb4d010SOphir Munk  */
11432eb4d010SOphir Munk static int
11442eb4d010SOphir Munk mlx5_dev_spawn_data_cmp(const void *a, const void *b)
11452eb4d010SOphir Munk {
11462eb4d010SOphir Munk 	const struct mlx5_switch_info *si_a =
11472eb4d010SOphir Munk 		&((const struct mlx5_dev_spawn_data *)a)->info;
11482eb4d010SOphir Munk 	const struct mlx5_switch_info *si_b =
11492eb4d010SOphir Munk 		&((const struct mlx5_dev_spawn_data *)b)->info;
11502eb4d010SOphir Munk 	int ret;
11512eb4d010SOphir Munk 
11522eb4d010SOphir Munk 	/* Master device first. */
11532eb4d010SOphir Munk 	ret = si_b->master - si_a->master;
11542eb4d010SOphir Munk 	if (ret)
11552eb4d010SOphir Munk 		return ret;
11562eb4d010SOphir Munk 	/* Then representor devices. */
11572eb4d010SOphir Munk 	ret = si_b->representor - si_a->representor;
11582eb4d010SOphir Munk 	if (ret)
11592eb4d010SOphir Munk 		return ret;
11602eb4d010SOphir Munk 	/* Unidentified devices come last in no specific order. */
11612eb4d010SOphir Munk 	if (!si_a->representor)
11622eb4d010SOphir Munk 		return 0;
11632eb4d010SOphir Munk 	/* Order representors by name. */
11642eb4d010SOphir Munk 	return si_a->port_name - si_b->port_name;
11652eb4d010SOphir Munk }
11662eb4d010SOphir Munk 
11672eb4d010SOphir Munk /**
11682eb4d010SOphir Munk  * Match PCI information for possible slaves of bonding device.
11692eb4d010SOphir Munk  *
11702eb4d010SOphir Munk  * @param[in] ibv_dev
11712eb4d010SOphir Munk  *   Pointer to Infiniband device structure.
11722eb4d010SOphir Munk  * @param[in] pci_dev
11732eb4d010SOphir Munk  *   Pointer to PCI device structure to match PCI address.
11742eb4d010SOphir Munk  * @param[in] nl_rdma
11752eb4d010SOphir Munk  *   Netlink RDMA group socket handle.
11762eb4d010SOphir Munk  *
11772eb4d010SOphir Munk  * @return
11782eb4d010SOphir Munk  *   negative value if no bonding device found, otherwise
11792eb4d010SOphir Munk  *   positive index of slave PF in bonding.
11802eb4d010SOphir Munk  */
11812eb4d010SOphir Munk static int
11822eb4d010SOphir Munk mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev,
11832eb4d010SOphir Munk 			   const struct rte_pci_device *pci_dev,
11842eb4d010SOphir Munk 			   int nl_rdma)
11852eb4d010SOphir Munk {
11862eb4d010SOphir Munk 	char ifname[IF_NAMESIZE + 1];
11872eb4d010SOphir Munk 	unsigned int ifindex;
11882eb4d010SOphir Munk 	unsigned int np, i;
11892eb4d010SOphir Munk 	FILE *file = NULL;
11902eb4d010SOphir Munk 	int pf = -1;
11912eb4d010SOphir Munk 
11922eb4d010SOphir Munk 	/*
11932eb4d010SOphir Munk 	 * Try to get master device name. If something goes
11942eb4d010SOphir Munk 	 * wrong suppose the lack of kernel support and no
11952eb4d010SOphir Munk 	 * bonding devices.
11962eb4d010SOphir Munk 	 */
11972eb4d010SOphir Munk 	if (nl_rdma < 0)
11982eb4d010SOphir Munk 		return -1;
11992eb4d010SOphir Munk 	if (!strstr(ibv_dev->name, "bond"))
12002eb4d010SOphir Munk 		return -1;
12012eb4d010SOphir Munk 	np = mlx5_nl_portnum(nl_rdma, ibv_dev->name);
12022eb4d010SOphir Munk 	if (!np)
12032eb4d010SOphir Munk 		return -1;
12042eb4d010SOphir Munk 	/*
12052eb4d010SOphir Munk 	 * The Master device might not be on the predefined
12062eb4d010SOphir Munk 	 * port (not on port index 1, it is not garanted),
12072eb4d010SOphir Munk 	 * we have to scan all Infiniband device port and
12082eb4d010SOphir Munk 	 * find master.
12092eb4d010SOphir Munk 	 */
12102eb4d010SOphir Munk 	for (i = 1; i <= np; ++i) {
12112eb4d010SOphir Munk 		/* Check whether Infiniband port is populated. */
12122eb4d010SOphir Munk 		ifindex = mlx5_nl_ifindex(nl_rdma, ibv_dev->name, i);
12132eb4d010SOphir Munk 		if (!ifindex)
12142eb4d010SOphir Munk 			continue;
12152eb4d010SOphir Munk 		if (!if_indextoname(ifindex, ifname))
12162eb4d010SOphir Munk 			continue;
12172eb4d010SOphir Munk 		/* Try to read bonding slave names from sysfs. */
12182eb4d010SOphir Munk 		MKSTR(slaves,
12192eb4d010SOphir Munk 		      "/sys/class/net/%s/master/bonding/slaves", ifname);
12202eb4d010SOphir Munk 		file = fopen(slaves, "r");
12212eb4d010SOphir Munk 		if (file)
12222eb4d010SOphir Munk 			break;
12232eb4d010SOphir Munk 	}
12242eb4d010SOphir Munk 	if (!file)
12252eb4d010SOphir Munk 		return -1;
12262eb4d010SOphir Munk 	/* Use safe format to check maximal buffer length. */
12272eb4d010SOphir Munk 	MLX5_ASSERT(atol(RTE_STR(IF_NAMESIZE)) == IF_NAMESIZE);
12282eb4d010SOphir Munk 	while (fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", ifname) == 1) {
12292eb4d010SOphir Munk 		char tmp_str[IF_NAMESIZE + 32];
12302eb4d010SOphir Munk 		struct rte_pci_addr pci_addr;
12312eb4d010SOphir Munk 		struct mlx5_switch_info	info;
12322eb4d010SOphir Munk 
12332eb4d010SOphir Munk 		/* Process slave interface names in the loop. */
12342eb4d010SOphir Munk 		snprintf(tmp_str, sizeof(tmp_str),
12352eb4d010SOphir Munk 			 "/sys/class/net/%s", ifname);
12362eb4d010SOphir Munk 		if (mlx5_dev_to_pci_addr(tmp_str, &pci_addr)) {
12372eb4d010SOphir Munk 			DRV_LOG(WARNING, "can not get PCI address"
12382eb4d010SOphir Munk 					 " for netdev \"%s\"", ifname);
12392eb4d010SOphir Munk 			continue;
12402eb4d010SOphir Munk 		}
12412eb4d010SOphir Munk 		if (pci_dev->addr.domain != pci_addr.domain ||
12422eb4d010SOphir Munk 		    pci_dev->addr.bus != pci_addr.bus ||
12432eb4d010SOphir Munk 		    pci_dev->addr.devid != pci_addr.devid ||
12442eb4d010SOphir Munk 		    pci_dev->addr.function != pci_addr.function)
12452eb4d010SOphir Munk 			continue;
12462eb4d010SOphir Munk 		/* Slave interface PCI address match found. */
12472eb4d010SOphir Munk 		fclose(file);
12482eb4d010SOphir Munk 		snprintf(tmp_str, sizeof(tmp_str),
12492eb4d010SOphir Munk 			 "/sys/class/net/%s/phys_port_name", ifname);
12502eb4d010SOphir Munk 		file = fopen(tmp_str, "rb");
12512eb4d010SOphir Munk 		if (!file)
12522eb4d010SOphir Munk 			break;
12532eb4d010SOphir Munk 		info.name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET;
12542eb4d010SOphir Munk 		if (fscanf(file, "%32s", tmp_str) == 1)
12552eb4d010SOphir Munk 			mlx5_translate_port_name(tmp_str, &info);
12562eb4d010SOphir Munk 		if (info.name_type == MLX5_PHYS_PORT_NAME_TYPE_LEGACY ||
12572eb4d010SOphir Munk 		    info.name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK)
12582eb4d010SOphir Munk 			pf = info.port_name;
12592eb4d010SOphir Munk 		break;
12602eb4d010SOphir Munk 	}
12612eb4d010SOphir Munk 	if (file)
12622eb4d010SOphir Munk 		fclose(file);
12632eb4d010SOphir Munk 	return pf;
12642eb4d010SOphir Munk }
12652eb4d010SOphir Munk 
12662eb4d010SOphir Munk /**
12672eb4d010SOphir Munk  * DPDK callback to register a PCI device.
12682eb4d010SOphir Munk  *
12692eb4d010SOphir Munk  * This function spawns Ethernet devices out of a given PCI device.
12702eb4d010SOphir Munk  *
12712eb4d010SOphir Munk  * @param[in] pci_drv
12722eb4d010SOphir Munk  *   PCI driver structure (mlx5_driver).
12732eb4d010SOphir Munk  * @param[in] pci_dev
12742eb4d010SOphir Munk  *   PCI device information.
12752eb4d010SOphir Munk  *
12762eb4d010SOphir Munk  * @return
12772eb4d010SOphir Munk  *   0 on success, a negative errno value otherwise and rte_errno is set.
12782eb4d010SOphir Munk  */
12792eb4d010SOphir Munk int
12802eb4d010SOphir Munk mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
12812eb4d010SOphir Munk 		  struct rte_pci_device *pci_dev)
12822eb4d010SOphir Munk {
12832eb4d010SOphir Munk 	struct ibv_device **ibv_list;
12842eb4d010SOphir Munk 	/*
12852eb4d010SOphir Munk 	 * Number of found IB Devices matching with requested PCI BDF.
12862eb4d010SOphir Munk 	 * nd != 1 means there are multiple IB devices over the same
12872eb4d010SOphir Munk 	 * PCI device and we have representors and master.
12882eb4d010SOphir Munk 	 */
12892eb4d010SOphir Munk 	unsigned int nd = 0;
12902eb4d010SOphir Munk 	/*
12912eb4d010SOphir Munk 	 * Number of found IB device Ports. nd = 1 and np = 1..n means
12922eb4d010SOphir Munk 	 * we have the single multiport IB device, and there may be
12932eb4d010SOphir Munk 	 * representors attached to some of found ports.
12942eb4d010SOphir Munk 	 */
12952eb4d010SOphir Munk 	unsigned int np = 0;
12962eb4d010SOphir Munk 	/*
12972eb4d010SOphir Munk 	 * Number of DPDK ethernet devices to Spawn - either over
12982eb4d010SOphir Munk 	 * multiple IB devices or multiple ports of single IB device.
12992eb4d010SOphir Munk 	 * Actually this is the number of iterations to spawn.
13002eb4d010SOphir Munk 	 */
13012eb4d010SOphir Munk 	unsigned int ns = 0;
13022eb4d010SOphir Munk 	/*
13032eb4d010SOphir Munk 	 * Bonding device
13042eb4d010SOphir Munk 	 *   < 0 - no bonding device (single one)
13052eb4d010SOphir Munk 	 *  >= 0 - bonding device (value is slave PF index)
13062eb4d010SOphir Munk 	 */
13072eb4d010SOphir Munk 	int bd = -1;
13082eb4d010SOphir Munk 	struct mlx5_dev_spawn_data *list = NULL;
13092eb4d010SOphir Munk 	struct mlx5_dev_config dev_config;
13102eb4d010SOphir Munk 	int ret;
13112eb4d010SOphir Munk 
13122eb4d010SOphir Munk 	if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_NET) {
13132eb4d010SOphir Munk 		DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5"
13142eb4d010SOphir Munk 			" driver.");
13152eb4d010SOphir Munk 		return 1;
13162eb4d010SOphir Munk 	}
13172eb4d010SOphir Munk 	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
13182eb4d010SOphir Munk 		mlx5_pmd_socket_init();
13192eb4d010SOphir Munk 	ret = mlx5_init_once();
13202eb4d010SOphir Munk 	if (ret) {
13212eb4d010SOphir Munk 		DRV_LOG(ERR, "unable to init PMD global data: %s",
13222eb4d010SOphir Munk 			strerror(rte_errno));
13232eb4d010SOphir Munk 		return -rte_errno;
13242eb4d010SOphir Munk 	}
13252eb4d010SOphir Munk 	MLX5_ASSERT(pci_drv == &mlx5_driver);
13262eb4d010SOphir Munk 	errno = 0;
13272eb4d010SOphir Munk 	ibv_list = mlx5_glue->get_device_list(&ret);
13282eb4d010SOphir Munk 	if (!ibv_list) {
13292eb4d010SOphir Munk 		rte_errno = errno ? errno : ENOSYS;
13302eb4d010SOphir Munk 		DRV_LOG(ERR, "cannot list devices, is ib_uverbs loaded?");
13312eb4d010SOphir Munk 		return -rte_errno;
13322eb4d010SOphir Munk 	}
13332eb4d010SOphir Munk 	/*
13342eb4d010SOphir Munk 	 * First scan the list of all Infiniband devices to find
13352eb4d010SOphir Munk 	 * matching ones, gathering into the list.
13362eb4d010SOphir Munk 	 */
13372eb4d010SOphir Munk 	struct ibv_device *ibv_match[ret + 1];
13382eb4d010SOphir Munk 	int nl_route = mlx5_nl_init(NETLINK_ROUTE);
13392eb4d010SOphir Munk 	int nl_rdma = mlx5_nl_init(NETLINK_RDMA);
13402eb4d010SOphir Munk 	unsigned int i;
13412eb4d010SOphir Munk 
13422eb4d010SOphir Munk 	while (ret-- > 0) {
13432eb4d010SOphir Munk 		struct rte_pci_addr pci_addr;
13442eb4d010SOphir Munk 
13452eb4d010SOphir Munk 		DRV_LOG(DEBUG, "checking device \"%s\"", ibv_list[ret]->name);
13462eb4d010SOphir Munk 		bd = mlx5_device_bond_pci_match
13472eb4d010SOphir Munk 				(ibv_list[ret], pci_dev, nl_rdma);
13482eb4d010SOphir Munk 		if (bd >= 0) {
13492eb4d010SOphir Munk 			/*
13502eb4d010SOphir Munk 			 * Bonding device detected. Only one match is allowed,
13512eb4d010SOphir Munk 			 * the bonding is supported over multi-port IB device,
13522eb4d010SOphir Munk 			 * there should be no matches on representor PCI
13532eb4d010SOphir Munk 			 * functions or non VF LAG bonding devices with
13542eb4d010SOphir Munk 			 * specified address.
13552eb4d010SOphir Munk 			 */
13562eb4d010SOphir Munk 			if (nd) {
13572eb4d010SOphir Munk 				DRV_LOG(ERR,
13582eb4d010SOphir Munk 					"multiple PCI match on bonding device"
13592eb4d010SOphir Munk 					"\"%s\" found", ibv_list[ret]->name);
13602eb4d010SOphir Munk 				rte_errno = ENOENT;
13612eb4d010SOphir Munk 				ret = -rte_errno;
13622eb4d010SOphir Munk 				goto exit;
13632eb4d010SOphir Munk 			}
13642eb4d010SOphir Munk 			DRV_LOG(INFO, "PCI information matches for"
13652eb4d010SOphir Munk 				      " slave %d bonding device \"%s\"",
13662eb4d010SOphir Munk 				      bd, ibv_list[ret]->name);
13672eb4d010SOphir Munk 			ibv_match[nd++] = ibv_list[ret];
13682eb4d010SOphir Munk 			break;
13692eb4d010SOphir Munk 		}
13702eb4d010SOphir Munk 		if (mlx5_dev_to_pci_addr
13712eb4d010SOphir Munk 			(ibv_list[ret]->ibdev_path, &pci_addr))
13722eb4d010SOphir Munk 			continue;
13732eb4d010SOphir Munk 		if (pci_dev->addr.domain != pci_addr.domain ||
13742eb4d010SOphir Munk 		    pci_dev->addr.bus != pci_addr.bus ||
13752eb4d010SOphir Munk 		    pci_dev->addr.devid != pci_addr.devid ||
13762eb4d010SOphir Munk 		    pci_dev->addr.function != pci_addr.function)
13772eb4d010SOphir Munk 			continue;
13782eb4d010SOphir Munk 		DRV_LOG(INFO, "PCI information matches for device \"%s\"",
13792eb4d010SOphir Munk 			ibv_list[ret]->name);
13802eb4d010SOphir Munk 		ibv_match[nd++] = ibv_list[ret];
13812eb4d010SOphir Munk 	}
13822eb4d010SOphir Munk 	ibv_match[nd] = NULL;
13832eb4d010SOphir Munk 	if (!nd) {
13842eb4d010SOphir Munk 		/* No device matches, just complain and bail out. */
13852eb4d010SOphir Munk 		DRV_LOG(WARNING,
13862eb4d010SOphir Munk 			"no Verbs device matches PCI device " PCI_PRI_FMT ","
13872eb4d010SOphir Munk 			" are kernel drivers loaded?",
13882eb4d010SOphir Munk 			pci_dev->addr.domain, pci_dev->addr.bus,
13892eb4d010SOphir Munk 			pci_dev->addr.devid, pci_dev->addr.function);
13902eb4d010SOphir Munk 		rte_errno = ENOENT;
13912eb4d010SOphir Munk 		ret = -rte_errno;
13922eb4d010SOphir Munk 		goto exit;
13932eb4d010SOphir Munk 	}
13942eb4d010SOphir Munk 	if (nd == 1) {
13952eb4d010SOphir Munk 		/*
13962eb4d010SOphir Munk 		 * Found single matching device may have multiple ports.
13972eb4d010SOphir Munk 		 * Each port may be representor, we have to check the port
13982eb4d010SOphir Munk 		 * number and check the representors existence.
13992eb4d010SOphir Munk 		 */
14002eb4d010SOphir Munk 		if (nl_rdma >= 0)
14012eb4d010SOphir Munk 			np = mlx5_nl_portnum(nl_rdma, ibv_match[0]->name);
14022eb4d010SOphir Munk 		if (!np)
14032eb4d010SOphir Munk 			DRV_LOG(WARNING, "can not get IB device \"%s\""
14042eb4d010SOphir Munk 					 " ports number", ibv_match[0]->name);
14052eb4d010SOphir Munk 		if (bd >= 0 && !np) {
14062eb4d010SOphir Munk 			DRV_LOG(ERR, "can not get ports"
14072eb4d010SOphir Munk 				     " for bonding device");
14082eb4d010SOphir Munk 			rte_errno = ENOENT;
14092eb4d010SOphir Munk 			ret = -rte_errno;
14102eb4d010SOphir Munk 			goto exit;
14112eb4d010SOphir Munk 		}
14122eb4d010SOphir Munk 	}
14132eb4d010SOphir Munk #ifndef HAVE_MLX5DV_DR_DEVX_PORT
14142eb4d010SOphir Munk 	if (bd >= 0) {
14152eb4d010SOphir Munk 		/*
14162eb4d010SOphir Munk 		 * This may happen if there is VF LAG kernel support and
14172eb4d010SOphir Munk 		 * application is compiled with older rdma_core library.
14182eb4d010SOphir Munk 		 */
14192eb4d010SOphir Munk 		DRV_LOG(ERR,
14202eb4d010SOphir Munk 			"No kernel/verbs support for VF LAG bonding found.");
14212eb4d010SOphir Munk 		rte_errno = ENOTSUP;
14222eb4d010SOphir Munk 		ret = -rte_errno;
14232eb4d010SOphir Munk 		goto exit;
14242eb4d010SOphir Munk 	}
14252eb4d010SOphir Munk #endif
14262eb4d010SOphir Munk 	/*
14272eb4d010SOphir Munk 	 * Now we can determine the maximal
14282eb4d010SOphir Munk 	 * amount of devices to be spawned.
14292eb4d010SOphir Munk 	 */
14302eb4d010SOphir Munk 	list = rte_zmalloc("device spawn data",
14312eb4d010SOphir Munk 			 sizeof(struct mlx5_dev_spawn_data) *
14322eb4d010SOphir Munk 			 (np ? np : nd),
14332eb4d010SOphir Munk 			 RTE_CACHE_LINE_SIZE);
14342eb4d010SOphir Munk 	if (!list) {
14352eb4d010SOphir Munk 		DRV_LOG(ERR, "spawn data array allocation failure");
14362eb4d010SOphir Munk 		rte_errno = ENOMEM;
14372eb4d010SOphir Munk 		ret = -rte_errno;
14382eb4d010SOphir Munk 		goto exit;
14392eb4d010SOphir Munk 	}
14402eb4d010SOphir Munk 	if (bd >= 0 || np > 1) {
14412eb4d010SOphir Munk 		/*
14422eb4d010SOphir Munk 		 * Single IB device with multiple ports found,
14432eb4d010SOphir Munk 		 * it may be E-Switch master device and representors.
14442eb4d010SOphir Munk 		 * We have to perform identification through the ports.
14452eb4d010SOphir Munk 		 */
14462eb4d010SOphir Munk 		MLX5_ASSERT(nl_rdma >= 0);
14472eb4d010SOphir Munk 		MLX5_ASSERT(ns == 0);
14482eb4d010SOphir Munk 		MLX5_ASSERT(nd == 1);
14492eb4d010SOphir Munk 		MLX5_ASSERT(np);
14502eb4d010SOphir Munk 		for (i = 1; i <= np; ++i) {
14512eb4d010SOphir Munk 			list[ns].max_port = np;
1452834a9019SOphir Munk 			list[ns].phys_port = i;
1453834a9019SOphir Munk 			list[ns].phys_dev = ibv_match[0];
14542eb4d010SOphir Munk 			list[ns].eth_dev = NULL;
14552eb4d010SOphir Munk 			list[ns].pci_dev = pci_dev;
14562eb4d010SOphir Munk 			list[ns].pf_bond = bd;
14572eb4d010SOphir Munk 			list[ns].ifindex = mlx5_nl_ifindex
1458834a9019SOphir Munk 				(nl_rdma,
1459834a9019SOphir Munk 				mlx5_os_get_dev_device_name
1460834a9019SOphir Munk 						(list[ns].phys_dev), i);
14612eb4d010SOphir Munk 			if (!list[ns].ifindex) {
14622eb4d010SOphir Munk 				/*
14632eb4d010SOphir Munk 				 * No network interface index found for the
14642eb4d010SOphir Munk 				 * specified port, it means there is no
14652eb4d010SOphir Munk 				 * representor on this port. It's OK,
14662eb4d010SOphir Munk 				 * there can be disabled ports, for example
14672eb4d010SOphir Munk 				 * if sriov_numvfs < sriov_totalvfs.
14682eb4d010SOphir Munk 				 */
14692eb4d010SOphir Munk 				continue;
14702eb4d010SOphir Munk 			}
14712eb4d010SOphir Munk 			ret = -1;
14722eb4d010SOphir Munk 			if (nl_route >= 0)
14732eb4d010SOphir Munk 				ret = mlx5_nl_switch_info
14742eb4d010SOphir Munk 					       (nl_route,
14752eb4d010SOphir Munk 						list[ns].ifindex,
14762eb4d010SOphir Munk 						&list[ns].info);
14772eb4d010SOphir Munk 			if (ret || (!list[ns].info.representor &&
14782eb4d010SOphir Munk 				    !list[ns].info.master)) {
14792eb4d010SOphir Munk 				/*
14802eb4d010SOphir Munk 				 * We failed to recognize representors with
14812eb4d010SOphir Munk 				 * Netlink, let's try to perform the task
14822eb4d010SOphir Munk 				 * with sysfs.
14832eb4d010SOphir Munk 				 */
14842eb4d010SOphir Munk 				ret =  mlx5_sysfs_switch_info
14852eb4d010SOphir Munk 						(list[ns].ifindex,
14862eb4d010SOphir Munk 						 &list[ns].info);
14872eb4d010SOphir Munk 			}
14882eb4d010SOphir Munk 			if (!ret && bd >= 0) {
14892eb4d010SOphir Munk 				switch (list[ns].info.name_type) {
14902eb4d010SOphir Munk 				case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
14912eb4d010SOphir Munk 					if (list[ns].info.port_name == bd)
14922eb4d010SOphir Munk 						ns++;
14932eb4d010SOphir Munk 					break;
14942eb4d010SOphir Munk 				case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
14952eb4d010SOphir Munk 					if (list[ns].info.pf_num == bd)
14962eb4d010SOphir Munk 						ns++;
14972eb4d010SOphir Munk 					break;
14982eb4d010SOphir Munk 				default:
14992eb4d010SOphir Munk 					break;
15002eb4d010SOphir Munk 				}
15012eb4d010SOphir Munk 				continue;
15022eb4d010SOphir Munk 			}
15032eb4d010SOphir Munk 			if (!ret && (list[ns].info.representor ^
15042eb4d010SOphir Munk 				     list[ns].info.master))
15052eb4d010SOphir Munk 				ns++;
15062eb4d010SOphir Munk 		}
15072eb4d010SOphir Munk 		if (!ns) {
15082eb4d010SOphir Munk 			DRV_LOG(ERR,
15092eb4d010SOphir Munk 				"unable to recognize master/representors"
15102eb4d010SOphir Munk 				" on the IB device with multiple ports");
15112eb4d010SOphir Munk 			rte_errno = ENOENT;
15122eb4d010SOphir Munk 			ret = -rte_errno;
15132eb4d010SOphir Munk 			goto exit;
15142eb4d010SOphir Munk 		}
15152eb4d010SOphir Munk 	} else {
15162eb4d010SOphir Munk 		/*
15172eb4d010SOphir Munk 		 * The existence of several matching entries (nd > 1) means
15182eb4d010SOphir Munk 		 * port representors have been instantiated. No existing Verbs
15192eb4d010SOphir Munk 		 * call nor sysfs entries can tell them apart, this can only
15202eb4d010SOphir Munk 		 * be done through Netlink calls assuming kernel drivers are
15212eb4d010SOphir Munk 		 * recent enough to support them.
15222eb4d010SOphir Munk 		 *
15232eb4d010SOphir Munk 		 * In the event of identification failure through Netlink,
15242eb4d010SOphir Munk 		 * try again through sysfs, then:
15252eb4d010SOphir Munk 		 *
15262eb4d010SOphir Munk 		 * 1. A single IB device matches (nd == 1) with single
15272eb4d010SOphir Munk 		 *    port (np=0/1) and is not a representor, assume
15282eb4d010SOphir Munk 		 *    no switch support.
15292eb4d010SOphir Munk 		 *
15302eb4d010SOphir Munk 		 * 2. Otherwise no safe assumptions can be made;
15312eb4d010SOphir Munk 		 *    complain louder and bail out.
15322eb4d010SOphir Munk 		 */
15332eb4d010SOphir Munk 		for (i = 0; i != nd; ++i) {
15342eb4d010SOphir Munk 			memset(&list[ns].info, 0, sizeof(list[ns].info));
15352eb4d010SOphir Munk 			list[ns].max_port = 1;
1536834a9019SOphir Munk 			list[ns].phys_port = 1;
1537834a9019SOphir Munk 			list[ns].phys_dev = ibv_match[i];
15382eb4d010SOphir Munk 			list[ns].eth_dev = NULL;
15392eb4d010SOphir Munk 			list[ns].pci_dev = pci_dev;
15402eb4d010SOphir Munk 			list[ns].pf_bond = -1;
15412eb4d010SOphir Munk 			list[ns].ifindex = 0;
15422eb4d010SOphir Munk 			if (nl_rdma >= 0)
15432eb4d010SOphir Munk 				list[ns].ifindex = mlx5_nl_ifindex
1544834a9019SOphir Munk 				(nl_rdma,
1545834a9019SOphir Munk 				mlx5_os_get_dev_device_name
1546834a9019SOphir Munk 						(list[ns].phys_dev), 1);
15472eb4d010SOphir Munk 			if (!list[ns].ifindex) {
15482eb4d010SOphir Munk 				char ifname[IF_NAMESIZE];
15492eb4d010SOphir Munk 
15502eb4d010SOphir Munk 				/*
15512eb4d010SOphir Munk 				 * Netlink failed, it may happen with old
15522eb4d010SOphir Munk 				 * ib_core kernel driver (before 4.16).
15532eb4d010SOphir Munk 				 * We can assume there is old driver because
15542eb4d010SOphir Munk 				 * here we are processing single ports IB
15552eb4d010SOphir Munk 				 * devices. Let's try sysfs to retrieve
15562eb4d010SOphir Munk 				 * the ifindex. The method works for
15572eb4d010SOphir Munk 				 * master device only.
15582eb4d010SOphir Munk 				 */
15592eb4d010SOphir Munk 				if (nd > 1) {
15602eb4d010SOphir Munk 					/*
15612eb4d010SOphir Munk 					 * Multiple devices found, assume
15622eb4d010SOphir Munk 					 * representors, can not distinguish
15632eb4d010SOphir Munk 					 * master/representor and retrieve
15642eb4d010SOphir Munk 					 * ifindex via sysfs.
15652eb4d010SOphir Munk 					 */
15662eb4d010SOphir Munk 					continue;
15672eb4d010SOphir Munk 				}
1568aec086c9SMatan Azrad 				ret = mlx5_get_ifname_sysfs
1569aec086c9SMatan Azrad 					(ibv_match[i]->ibdev_path, ifname);
15702eb4d010SOphir Munk 				if (!ret)
15712eb4d010SOphir Munk 					list[ns].ifindex =
15722eb4d010SOphir Munk 						if_nametoindex(ifname);
15732eb4d010SOphir Munk 				if (!list[ns].ifindex) {
15742eb4d010SOphir Munk 					/*
15752eb4d010SOphir Munk 					 * No network interface index found
15762eb4d010SOphir Munk 					 * for the specified device, it means
15772eb4d010SOphir Munk 					 * there it is neither representor
15782eb4d010SOphir Munk 					 * nor master.
15792eb4d010SOphir Munk 					 */
15802eb4d010SOphir Munk 					continue;
15812eb4d010SOphir Munk 				}
15822eb4d010SOphir Munk 			}
15832eb4d010SOphir Munk 			ret = -1;
15842eb4d010SOphir Munk 			if (nl_route >= 0)
15852eb4d010SOphir Munk 				ret = mlx5_nl_switch_info
15862eb4d010SOphir Munk 					       (nl_route,
15872eb4d010SOphir Munk 						list[ns].ifindex,
15882eb4d010SOphir Munk 						&list[ns].info);
15892eb4d010SOphir Munk 			if (ret || (!list[ns].info.representor &&
15902eb4d010SOphir Munk 				    !list[ns].info.master)) {
15912eb4d010SOphir Munk 				/*
15922eb4d010SOphir Munk 				 * We failed to recognize representors with
15932eb4d010SOphir Munk 				 * Netlink, let's try to perform the task
15942eb4d010SOphir Munk 				 * with sysfs.
15952eb4d010SOphir Munk 				 */
15962eb4d010SOphir Munk 				ret =  mlx5_sysfs_switch_info
15972eb4d010SOphir Munk 						(list[ns].ifindex,
15982eb4d010SOphir Munk 						 &list[ns].info);
15992eb4d010SOphir Munk 			}
16002eb4d010SOphir Munk 			if (!ret && (list[ns].info.representor ^
16012eb4d010SOphir Munk 				     list[ns].info.master)) {
16022eb4d010SOphir Munk 				ns++;
16032eb4d010SOphir Munk 			} else if ((nd == 1) &&
16042eb4d010SOphir Munk 				   !list[ns].info.representor &&
16052eb4d010SOphir Munk 				   !list[ns].info.master) {
16062eb4d010SOphir Munk 				/*
16072eb4d010SOphir Munk 				 * Single IB device with
16082eb4d010SOphir Munk 				 * one physical port and
16092eb4d010SOphir Munk 				 * attached network device.
16102eb4d010SOphir Munk 				 * May be SRIOV is not enabled
16112eb4d010SOphir Munk 				 * or there is no representors.
16122eb4d010SOphir Munk 				 */
16132eb4d010SOphir Munk 				DRV_LOG(INFO, "no E-Switch support detected");
16142eb4d010SOphir Munk 				ns++;
16152eb4d010SOphir Munk 				break;
16162eb4d010SOphir Munk 			}
16172eb4d010SOphir Munk 		}
16182eb4d010SOphir Munk 		if (!ns) {
16192eb4d010SOphir Munk 			DRV_LOG(ERR,
16202eb4d010SOphir Munk 				"unable to recognize master/representors"
16212eb4d010SOphir Munk 				" on the multiple IB devices");
16222eb4d010SOphir Munk 			rte_errno = ENOENT;
16232eb4d010SOphir Munk 			ret = -rte_errno;
16242eb4d010SOphir Munk 			goto exit;
16252eb4d010SOphir Munk 		}
16262eb4d010SOphir Munk 	}
16272eb4d010SOphir Munk 	MLX5_ASSERT(ns);
16282eb4d010SOphir Munk 	/*
16292eb4d010SOphir Munk 	 * Sort list to probe devices in natural order for users convenience
16302eb4d010SOphir Munk 	 * (i.e. master first, then representors from lowest to highest ID).
16312eb4d010SOphir Munk 	 */
16322eb4d010SOphir Munk 	qsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp);
16332eb4d010SOphir Munk 	/* Default configuration. */
16342eb4d010SOphir Munk 	dev_config = (struct mlx5_dev_config){
16352eb4d010SOphir Munk 		.hw_padding = 0,
16362eb4d010SOphir Munk 		.mps = MLX5_ARG_UNSET,
16372eb4d010SOphir Munk 		.dbnc = MLX5_ARG_UNSET,
16382eb4d010SOphir Munk 		.rx_vec_en = 1,
16392eb4d010SOphir Munk 		.txq_inline_max = MLX5_ARG_UNSET,
16402eb4d010SOphir Munk 		.txq_inline_min = MLX5_ARG_UNSET,
16412eb4d010SOphir Munk 		.txq_inline_mpw = MLX5_ARG_UNSET,
16422eb4d010SOphir Munk 		.txqs_inline = MLX5_ARG_UNSET,
16432eb4d010SOphir Munk 		.vf_nl_en = 1,
16442eb4d010SOphir Munk 		.mr_ext_memseg_en = 1,
16452eb4d010SOphir Munk 		.mprq = {
16462eb4d010SOphir Munk 			.enabled = 0, /* Disabled by default. */
16472eb4d010SOphir Munk 			.stride_num_n = 0,
16482eb4d010SOphir Munk 			.stride_size_n = 0,
16492eb4d010SOphir Munk 			.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN,
16502eb4d010SOphir Munk 			.min_rxqs_num = MLX5_MPRQ_MIN_RXQS,
16512eb4d010SOphir Munk 		},
16522eb4d010SOphir Munk 		.dv_esw_en = 1,
16532eb4d010SOphir Munk 		.dv_flow_en = 1,
16542eb4d010SOphir Munk 		.log_hp_size = MLX5_ARG_UNSET,
16552eb4d010SOphir Munk 	};
16562eb4d010SOphir Munk 	/* Device specific configuration. */
16572eb4d010SOphir Munk 	switch (pci_dev->id.device_id) {
16582eb4d010SOphir Munk 	case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
16592eb4d010SOphir Munk 	case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
16602eb4d010SOphir Munk 	case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
16612eb4d010SOphir Munk 	case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
16622eb4d010SOphir Munk 	case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF:
16632eb4d010SOphir Munk 	case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF:
16642eb4d010SOphir Munk 	case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF:
16652eb4d010SOphir Munk 		dev_config.vf = 1;
16662eb4d010SOphir Munk 		break;
16672eb4d010SOphir Munk 	default:
16682eb4d010SOphir Munk 		break;
16692eb4d010SOphir Munk 	}
16702eb4d010SOphir Munk 	for (i = 0; i != ns; ++i) {
16712eb4d010SOphir Munk 		uint32_t restore;
16722eb4d010SOphir Munk 
16732eb4d010SOphir Munk 		list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device,
16742eb4d010SOphir Munk 						 &list[i],
16752eb4d010SOphir Munk 						 dev_config);
16762eb4d010SOphir Munk 		if (!list[i].eth_dev) {
16772eb4d010SOphir Munk 			if (rte_errno != EBUSY && rte_errno != EEXIST)
16782eb4d010SOphir Munk 				break;
16792eb4d010SOphir Munk 			/* Device is disabled or already spawned. Ignore it. */
16802eb4d010SOphir Munk 			continue;
16812eb4d010SOphir Munk 		}
16822eb4d010SOphir Munk 		restore = list[i].eth_dev->data->dev_flags;
16832eb4d010SOphir Munk 		rte_eth_copy_pci_info(list[i].eth_dev, pci_dev);
16842eb4d010SOphir Munk 		/* Restore non-PCI flags cleared by the above call. */
16852eb4d010SOphir Munk 		list[i].eth_dev->data->dev_flags |= restore;
16862eb4d010SOphir Munk 		rte_eth_dev_probing_finish(list[i].eth_dev);
16872eb4d010SOphir Munk 	}
16882eb4d010SOphir Munk 	if (i != ns) {
16892eb4d010SOphir Munk 		DRV_LOG(ERR,
16902eb4d010SOphir Munk 			"probe of PCI device " PCI_PRI_FMT " aborted after"
16912eb4d010SOphir Munk 			" encountering an error: %s",
16922eb4d010SOphir Munk 			pci_dev->addr.domain, pci_dev->addr.bus,
16932eb4d010SOphir Munk 			pci_dev->addr.devid, pci_dev->addr.function,
16942eb4d010SOphir Munk 			strerror(rte_errno));
16952eb4d010SOphir Munk 		ret = -rte_errno;
16962eb4d010SOphir Munk 		/* Roll back. */
16972eb4d010SOphir Munk 		while (i--) {
16982eb4d010SOphir Munk 			if (!list[i].eth_dev)
16992eb4d010SOphir Munk 				continue;
17002eb4d010SOphir Munk 			mlx5_dev_close(list[i].eth_dev);
17012eb4d010SOphir Munk 			/* mac_addrs must not be freed because in dev_private */
17022eb4d010SOphir Munk 			list[i].eth_dev->data->mac_addrs = NULL;
17032eb4d010SOphir Munk 			claim_zero(rte_eth_dev_release_port(list[i].eth_dev));
17042eb4d010SOphir Munk 		}
17052eb4d010SOphir Munk 		/* Restore original error. */
17062eb4d010SOphir Munk 		rte_errno = -ret;
17072eb4d010SOphir Munk 	} else {
17082eb4d010SOphir Munk 		ret = 0;
17092eb4d010SOphir Munk 	}
17102eb4d010SOphir Munk exit:
17112eb4d010SOphir Munk 	/*
17122eb4d010SOphir Munk 	 * Do the routine cleanup:
17132eb4d010SOphir Munk 	 * - close opened Netlink sockets
17142eb4d010SOphir Munk 	 * - free allocated spawn data array
17152eb4d010SOphir Munk 	 * - free the Infiniband device list
17162eb4d010SOphir Munk 	 */
17172eb4d010SOphir Munk 	if (nl_rdma >= 0)
17182eb4d010SOphir Munk 		close(nl_rdma);
17192eb4d010SOphir Munk 	if (nl_route >= 0)
17202eb4d010SOphir Munk 		close(nl_route);
17212eb4d010SOphir Munk 	if (list)
17222eb4d010SOphir Munk 		rte_free(list);
17232eb4d010SOphir Munk 	MLX5_ASSERT(ibv_list);
17242eb4d010SOphir Munk 	mlx5_glue->free_device_list(ibv_list);
17252eb4d010SOphir Munk 	return ret;
17262eb4d010SOphir Munk }
17272eb4d010SOphir Munk 
17282eb4d010SOphir Munk static int
17292eb4d010SOphir Munk mlx5_config_doorbell_mapping_env(const struct mlx5_dev_config *config)
17302eb4d010SOphir Munk {
17312eb4d010SOphir Munk 	char *env;
17322eb4d010SOphir Munk 	int value;
17332eb4d010SOphir Munk 
17342eb4d010SOphir Munk 	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
17352eb4d010SOphir Munk 	/* Get environment variable to store. */
17362eb4d010SOphir Munk 	env = getenv(MLX5_SHUT_UP_BF);
17372eb4d010SOphir Munk 	value = env ? !!strcmp(env, "0") : MLX5_ARG_UNSET;
17382eb4d010SOphir Munk 	if (config->dbnc == MLX5_ARG_UNSET)
17392eb4d010SOphir Munk 		setenv(MLX5_SHUT_UP_BF, MLX5_SHUT_UP_BF_DEFAULT, 1);
17402eb4d010SOphir Munk 	else
17412eb4d010SOphir Munk 		setenv(MLX5_SHUT_UP_BF,
17422eb4d010SOphir Munk 		       config->dbnc == MLX5_TXDB_NCACHED ? "1" : "0", 1);
17432eb4d010SOphir Munk 	return value;
17442eb4d010SOphir Munk }
17452eb4d010SOphir Munk 
17462eb4d010SOphir Munk static void
17472eb4d010SOphir Munk mlx5_restore_doorbell_mapping_env(int value)
17482eb4d010SOphir Munk {
17492eb4d010SOphir Munk 	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
17502eb4d010SOphir Munk 	/* Restore the original environment variable state. */
17512eb4d010SOphir Munk 	if (value == MLX5_ARG_UNSET)
17522eb4d010SOphir Munk 		unsetenv(MLX5_SHUT_UP_BF);
17532eb4d010SOphir Munk 	else
17542eb4d010SOphir Munk 		setenv(MLX5_SHUT_UP_BF, value ? "1" : "0", 1);
17552eb4d010SOphir Munk }
17562eb4d010SOphir Munk 
17572eb4d010SOphir Munk /**
17582eb4d010SOphir Munk  * Extract pdn of PD object using DV API.
17592eb4d010SOphir Munk  *
17602eb4d010SOphir Munk  * @param[in] pd
17612eb4d010SOphir Munk  *   Pointer to the verbs PD object.
17622eb4d010SOphir Munk  * @param[out] pdn
17632eb4d010SOphir Munk  *   Pointer to the PD object number variable.
17642eb4d010SOphir Munk  *
17652eb4d010SOphir Munk  * @return
17662eb4d010SOphir Munk  *   0 on success, error value otherwise.
17672eb4d010SOphir Munk  */
17682eb4d010SOphir Munk int
17692eb4d010SOphir Munk mlx5_os_get_pdn(void *pd, uint32_t *pdn)
17702eb4d010SOphir Munk {
17712eb4d010SOphir Munk #ifdef HAVE_IBV_FLOW_DV_SUPPORT
17722eb4d010SOphir Munk 	struct mlx5dv_obj obj;
17732eb4d010SOphir Munk 	struct mlx5dv_pd pd_info;
17742eb4d010SOphir Munk 	int ret = 0;
17752eb4d010SOphir Munk 
17762eb4d010SOphir Munk 	obj.pd.in = pd;
17772eb4d010SOphir Munk 	obj.pd.out = &pd_info;
17782eb4d010SOphir Munk 	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
17792eb4d010SOphir Munk 	if (ret) {
17802eb4d010SOphir Munk 		DRV_LOG(DEBUG, "Fail to get PD object info");
17812eb4d010SOphir Munk 		return ret;
17822eb4d010SOphir Munk 	}
17832eb4d010SOphir Munk 	*pdn = pd_info.pdn;
17842eb4d010SOphir Munk 	return 0;
17852eb4d010SOphir Munk #else
17862eb4d010SOphir Munk 	(void)pd;
17872eb4d010SOphir Munk 	(void)pdn;
17882eb4d010SOphir Munk 	return -ENOTSUP;
17892eb4d010SOphir Munk #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
17902eb4d010SOphir Munk }
17912eb4d010SOphir Munk 
17922eb4d010SOphir Munk /**
17932eb4d010SOphir Munk  * Function API to open IB device.
17942eb4d010SOphir Munk  *
17952eb4d010SOphir Munk  * This function calls the Linux glue APIs to open a device.
17962eb4d010SOphir Munk  *
17972eb4d010SOphir Munk  * @param[in] spawn
17982eb4d010SOphir Munk  *   Pointer to the IB device attributes (name, port, etc).
17992eb4d010SOphir Munk  * @param[out] config
18002eb4d010SOphir Munk  *   Pointer to device configuration structure.
18012eb4d010SOphir Munk  * @param[out] sh
18022eb4d010SOphir Munk  *   Pointer to shared context structure.
18032eb4d010SOphir Munk  *
18042eb4d010SOphir Munk  * @return
18052eb4d010SOphir Munk  *   0 on success, a positive error value otherwise.
18062eb4d010SOphir Munk  */
18072eb4d010SOphir Munk int
18082eb4d010SOphir Munk mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
18092eb4d010SOphir Munk 		     const struct mlx5_dev_config *config,
18102eb4d010SOphir Munk 		     struct mlx5_dev_ctx_shared *sh)
18112eb4d010SOphir Munk {
18122eb4d010SOphir Munk 	int dbmap_env;
18132eb4d010SOphir Munk 	int err = 0;
18142eb4d010SOphir Munk 	/*
18152eb4d010SOphir Munk 	 * Configure environment variable "MLX5_BF_SHUT_UP"
18162eb4d010SOphir Munk 	 * before the device creation. The rdma_core library
18172eb4d010SOphir Munk 	 * checks the variable at device creation and
18182eb4d010SOphir Munk 	 * stores the result internally.
18192eb4d010SOphir Munk 	 */
18202eb4d010SOphir Munk 	dbmap_env = mlx5_config_doorbell_mapping_env(config);
18212eb4d010SOphir Munk 	/* Try to open IB device with DV first, then usual Verbs. */
18222eb4d010SOphir Munk 	errno = 0;
1823834a9019SOphir Munk 	sh->ctx = mlx5_glue->dv_open_device(spawn->phys_dev);
18242eb4d010SOphir Munk 	if (sh->ctx) {
18252eb4d010SOphir Munk 		sh->devx = 1;
18262eb4d010SOphir Munk 		DRV_LOG(DEBUG, "DevX is supported");
18272eb4d010SOphir Munk 		/* The device is created, no need for environment. */
18282eb4d010SOphir Munk 		mlx5_restore_doorbell_mapping_env(dbmap_env);
18292eb4d010SOphir Munk 	} else {
18302eb4d010SOphir Munk 		/* The environment variable is still configured. */
1831834a9019SOphir Munk 		sh->ctx = mlx5_glue->open_device(spawn->phys_dev);
18322eb4d010SOphir Munk 		err = errno ? errno : ENODEV;
18332eb4d010SOphir Munk 		/*
18342eb4d010SOphir Munk 		 * The environment variable is not needed anymore,
18352eb4d010SOphir Munk 		 * all device creation attempts are completed.
18362eb4d010SOphir Munk 		 */
18372eb4d010SOphir Munk 		mlx5_restore_doorbell_mapping_env(dbmap_env);
18382eb4d010SOphir Munk 		if (!sh->ctx)
18392eb4d010SOphir Munk 			return err;
18402eb4d010SOphir Munk 		DRV_LOG(DEBUG, "DevX is NOT supported");
18412eb4d010SOphir Munk 		err = 0;
18422eb4d010SOphir Munk 	}
18432eb4d010SOphir Munk 	return err;
18442eb4d010SOphir Munk }
18452eb4d010SOphir Munk 
18462eb4d010SOphir Munk /**
18472eb4d010SOphir Munk  * Install shared asynchronous device events handler.
18482eb4d010SOphir Munk  * This function is implemented to support event sharing
18492eb4d010SOphir Munk  * between multiple ports of single IB device.
18502eb4d010SOphir Munk  *
18512eb4d010SOphir Munk  * @param sh
18522eb4d010SOphir Munk  *   Pointer to mlx5_dev_ctx_shared object.
18532eb4d010SOphir Munk  */
18542eb4d010SOphir Munk void
18552eb4d010SOphir Munk mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh)
18562eb4d010SOphir Munk {
18572eb4d010SOphir Munk 	int ret;
18582eb4d010SOphir Munk 	int flags;
18592eb4d010SOphir Munk 
18602eb4d010SOphir Munk 	sh->intr_handle.fd = -1;
18612eb4d010SOphir Munk 	flags = fcntl(((struct ibv_context *)sh->ctx)->async_fd, F_GETFL);
18622eb4d010SOphir Munk 	ret = fcntl(((struct ibv_context *)sh->ctx)->async_fd,
18632eb4d010SOphir Munk 		    F_SETFL, flags | O_NONBLOCK);
18642eb4d010SOphir Munk 	if (ret) {
18652eb4d010SOphir Munk 		DRV_LOG(INFO, "failed to change file descriptor async event"
18662eb4d010SOphir Munk 			" queue");
18672eb4d010SOphir Munk 	} else {
18682eb4d010SOphir Munk 		sh->intr_handle.fd = ((struct ibv_context *)sh->ctx)->async_fd;
18692eb4d010SOphir Munk 		sh->intr_handle.type = RTE_INTR_HANDLE_EXT;
18702eb4d010SOphir Munk 		if (rte_intr_callback_register(&sh->intr_handle,
18712eb4d010SOphir Munk 					mlx5_dev_interrupt_handler, sh)) {
18722eb4d010SOphir Munk 			DRV_LOG(INFO, "Fail to install the shared interrupt.");
18732eb4d010SOphir Munk 			sh->intr_handle.fd = -1;
18742eb4d010SOphir Munk 		}
18752eb4d010SOphir Munk 	}
18762eb4d010SOphir Munk 	if (sh->devx) {
18772eb4d010SOphir Munk #ifdef HAVE_IBV_DEVX_ASYNC
18782eb4d010SOphir Munk 		sh->intr_handle_devx.fd = -1;
187921b7c452SOphir Munk 		sh->devx_comp =
188021b7c452SOphir Munk 			(void *)mlx5_glue->devx_create_cmd_comp(sh->ctx);
188121b7c452SOphir Munk 		struct mlx5dv_devx_cmd_comp *devx_comp = sh->devx_comp;
188221b7c452SOphir Munk 		if (!devx_comp) {
18832eb4d010SOphir Munk 			DRV_LOG(INFO, "failed to allocate devx_comp.");
18842eb4d010SOphir Munk 			return;
18852eb4d010SOphir Munk 		}
188621b7c452SOphir Munk 		flags = fcntl(devx_comp->fd, F_GETFL);
188721b7c452SOphir Munk 		ret = fcntl(devx_comp->fd, F_SETFL, flags | O_NONBLOCK);
18882eb4d010SOphir Munk 		if (ret) {
18892eb4d010SOphir Munk 			DRV_LOG(INFO, "failed to change file descriptor"
18902eb4d010SOphir Munk 				" devx comp");
18912eb4d010SOphir Munk 			return;
18922eb4d010SOphir Munk 		}
189321b7c452SOphir Munk 		sh->intr_handle_devx.fd = devx_comp->fd;
18942eb4d010SOphir Munk 		sh->intr_handle_devx.type = RTE_INTR_HANDLE_EXT;
18952eb4d010SOphir Munk 		if (rte_intr_callback_register(&sh->intr_handle_devx,
18962eb4d010SOphir Munk 					mlx5_dev_interrupt_handler_devx, sh)) {
18972eb4d010SOphir Munk 			DRV_LOG(INFO, "Fail to install the devx shared"
18982eb4d010SOphir Munk 				" interrupt.");
18992eb4d010SOphir Munk 			sh->intr_handle_devx.fd = -1;
19002eb4d010SOphir Munk 		}
19012eb4d010SOphir Munk #endif /* HAVE_IBV_DEVX_ASYNC */
19022eb4d010SOphir Munk 	}
19032eb4d010SOphir Munk }
19042eb4d010SOphir Munk 
19052eb4d010SOphir Munk /**
19062eb4d010SOphir Munk  * Uninstall shared asynchronous device events handler.
19072eb4d010SOphir Munk  * This function is implemented to support event sharing
19082eb4d010SOphir Munk  * between multiple ports of single IB device.
19092eb4d010SOphir Munk  *
19102eb4d010SOphir Munk  * @param dev
19112eb4d010SOphir Munk  *   Pointer to mlx5_dev_ctx_shared object.
19122eb4d010SOphir Munk  */
19132eb4d010SOphir Munk void
19142eb4d010SOphir Munk mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh)
19152eb4d010SOphir Munk {
19162eb4d010SOphir Munk 	if (sh->intr_handle.fd >= 0)
19172eb4d010SOphir Munk 		mlx5_intr_callback_unregister(&sh->intr_handle,
19182eb4d010SOphir Munk 					      mlx5_dev_interrupt_handler, sh);
19192eb4d010SOphir Munk #ifdef HAVE_IBV_DEVX_ASYNC
19202eb4d010SOphir Munk 	if (sh->intr_handle_devx.fd >= 0)
19212eb4d010SOphir Munk 		rte_intr_callback_unregister(&sh->intr_handle_devx,
19222eb4d010SOphir Munk 				  mlx5_dev_interrupt_handler_devx, sh);
19232eb4d010SOphir Munk 	if (sh->devx_comp)
19242eb4d010SOphir Munk 		mlx5_glue->devx_destroy_cmd_comp(sh->devx_comp);
19252eb4d010SOphir Munk #endif
19262eb4d010SOphir Munk }
1927042f5c94SOphir Munk 
192873bf9235SOphir Munk /**
192973bf9235SOphir Munk  * Read statistics by a named counter.
193073bf9235SOphir Munk  *
193173bf9235SOphir Munk  * @param[in] priv
193273bf9235SOphir Munk  *   Pointer to the private device data structure.
193373bf9235SOphir Munk  * @param[in] ctr_name
193473bf9235SOphir Munk  *   Pointer to the name of the statistic counter to read
193573bf9235SOphir Munk  * @param[out] stat
193673bf9235SOphir Munk  *   Pointer to read statistic value.
193773bf9235SOphir Munk  * @return
193873bf9235SOphir Munk  *   0 on success and stat is valud, 1 if failed to read the value
193973bf9235SOphir Munk  *   rte_errno is set.
194073bf9235SOphir Munk  *
194173bf9235SOphir Munk  */
194273bf9235SOphir Munk int
194373bf9235SOphir Munk mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name,
194473bf9235SOphir Munk 		      uint64_t *stat)
194573bf9235SOphir Munk {
194673bf9235SOphir Munk 	int fd;
194773bf9235SOphir Munk 
194873bf9235SOphir Munk 	if (priv->sh) {
194973bf9235SOphir Munk 		MKSTR(path, "%s/ports/%d/hw_counters/%s",
195073bf9235SOphir Munk 			  priv->sh->ibdev_path,
195173bf9235SOphir Munk 			  priv->dev_port,
195273bf9235SOphir Munk 			  ctr_name);
195373bf9235SOphir Munk 		fd = open(path, O_RDONLY);
195473bf9235SOphir Munk 		if (fd != -1) {
195573bf9235SOphir Munk 			char buf[21] = {'\0'};
195673bf9235SOphir Munk 			ssize_t n = read(fd, buf, sizeof(buf));
195773bf9235SOphir Munk 
195873bf9235SOphir Munk 			close(fd);
195973bf9235SOphir Munk 			if (n != -1) {
196073bf9235SOphir Munk 				*stat = strtoull(buf, NULL, 10);
196173bf9235SOphir Munk 				return 0;
196273bf9235SOphir Munk 			}
196373bf9235SOphir Munk 		}
196473bf9235SOphir Munk 	}
196573bf9235SOphir Munk 	*stat = 0;
196673bf9235SOphir Munk 	return 1;
196773bf9235SOphir Munk }
196873bf9235SOphir Munk 
196973bf9235SOphir Munk /**
197073bf9235SOphir Munk  * Read device counters table.
197173bf9235SOphir Munk  *
197273bf9235SOphir Munk  * @param dev
197373bf9235SOphir Munk  *   Pointer to Ethernet device.
197473bf9235SOphir Munk  * @param[out] stats
197573bf9235SOphir Munk  *   Counters table output buffer.
197673bf9235SOphir Munk  *
197773bf9235SOphir Munk  * @return
197873bf9235SOphir Munk  *   0 on success and stats is filled, negative errno value otherwise and
197973bf9235SOphir Munk  *   rte_errno is set.
198073bf9235SOphir Munk  */
198173bf9235SOphir Munk int
198273bf9235SOphir Munk mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
198373bf9235SOphir Munk {
198473bf9235SOphir Munk 	struct mlx5_priv *priv = dev->data->dev_private;
198573bf9235SOphir Munk 	struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
198673bf9235SOphir Munk 	unsigned int i;
198773bf9235SOphir Munk 	struct ifreq ifr;
198873bf9235SOphir Munk 	unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t);
198973bf9235SOphir Munk 	unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz];
199073bf9235SOphir Munk 	struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf;
199173bf9235SOphir Munk 	int ret;
199273bf9235SOphir Munk 
199373bf9235SOphir Munk 	et_stats->cmd = ETHTOOL_GSTATS;
199473bf9235SOphir Munk 	et_stats->n_stats = xstats_ctrl->stats_n;
199573bf9235SOphir Munk 	ifr.ifr_data = (caddr_t)et_stats;
199673bf9235SOphir Munk 	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
199773bf9235SOphir Munk 	if (ret) {
199873bf9235SOphir Munk 		DRV_LOG(WARNING,
199973bf9235SOphir Munk 			"port %u unable to read statistic values from device",
200073bf9235SOphir Munk 			dev->data->port_id);
200173bf9235SOphir Munk 		return ret;
200273bf9235SOphir Munk 	}
200373bf9235SOphir Munk 	for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) {
200473bf9235SOphir Munk 		if (xstats_ctrl->info[i].dev) {
200573bf9235SOphir Munk 			ret = mlx5_os_read_dev_stat(priv,
200673bf9235SOphir Munk 					    xstats_ctrl->info[i].ctr_name,
200773bf9235SOphir Munk 					    &stats[i]);
200873bf9235SOphir Munk 			/* return last xstats counter if fail to read. */
200973bf9235SOphir Munk 			if (ret == 0)
201073bf9235SOphir Munk 				xstats_ctrl->xstats[i] = stats[i];
201173bf9235SOphir Munk 			else
201273bf9235SOphir Munk 				stats[i] = xstats_ctrl->xstats[i];
201373bf9235SOphir Munk 		} else {
201473bf9235SOphir Munk 			stats[i] = (uint64_t)
201573bf9235SOphir Munk 				et_stats->data[xstats_ctrl->dev_table_idx[i]];
201673bf9235SOphir Munk 		}
201773bf9235SOphir Munk 	}
201873bf9235SOphir Munk 	return 0;
201973bf9235SOphir Munk }
202073bf9235SOphir Munk 
202173bf9235SOphir Munk /**
202273bf9235SOphir Munk  * Query the number of statistics provided by ETHTOOL.
202373bf9235SOphir Munk  *
202473bf9235SOphir Munk  * @param dev
202573bf9235SOphir Munk  *   Pointer to Ethernet device.
202673bf9235SOphir Munk  *
202773bf9235SOphir Munk  * @return
202873bf9235SOphir Munk  *   Number of statistics on success, negative errno value otherwise and
202973bf9235SOphir Munk  *   rte_errno is set.
203073bf9235SOphir Munk  */
203173bf9235SOphir Munk int
203273bf9235SOphir Munk mlx5_os_get_stats_n(struct rte_eth_dev *dev)
203373bf9235SOphir Munk {
203473bf9235SOphir Munk 	struct ethtool_drvinfo drvinfo;
203573bf9235SOphir Munk 	struct ifreq ifr;
203673bf9235SOphir Munk 	int ret;
203773bf9235SOphir Munk 
203873bf9235SOphir Munk 	drvinfo.cmd = ETHTOOL_GDRVINFO;
203973bf9235SOphir Munk 	ifr.ifr_data = (caddr_t)&drvinfo;
204073bf9235SOphir Munk 	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
204173bf9235SOphir Munk 	if (ret) {
204273bf9235SOphir Munk 		DRV_LOG(WARNING, "port %u unable to query number of statistics",
204373bf9235SOphir Munk 			dev->data->port_id);
204473bf9235SOphir Munk 		return ret;
204573bf9235SOphir Munk 	}
204673bf9235SOphir Munk 	return drvinfo.n_stats;
204773bf9235SOphir Munk }
204873bf9235SOphir Munk 
204973bf9235SOphir Munk static const struct mlx5_counter_ctrl mlx5_counters_init[] = {
205073bf9235SOphir Munk 	{
205173bf9235SOphir Munk 		.dpdk_name = "rx_port_unicast_bytes",
205273bf9235SOphir Munk 		.ctr_name = "rx_vport_unicast_bytes",
205373bf9235SOphir Munk 	},
205473bf9235SOphir Munk 	{
205573bf9235SOphir Munk 		.dpdk_name = "rx_port_multicast_bytes",
205673bf9235SOphir Munk 		.ctr_name = "rx_vport_multicast_bytes",
205773bf9235SOphir Munk 	},
205873bf9235SOphir Munk 	{
205973bf9235SOphir Munk 		.dpdk_name = "rx_port_broadcast_bytes",
206073bf9235SOphir Munk 		.ctr_name = "rx_vport_broadcast_bytes",
206173bf9235SOphir Munk 	},
206273bf9235SOphir Munk 	{
206373bf9235SOphir Munk 		.dpdk_name = "rx_port_unicast_packets",
206473bf9235SOphir Munk 		.ctr_name = "rx_vport_unicast_packets",
206573bf9235SOphir Munk 	},
206673bf9235SOphir Munk 	{
206773bf9235SOphir Munk 		.dpdk_name = "rx_port_multicast_packets",
206873bf9235SOphir Munk 		.ctr_name = "rx_vport_multicast_packets",
206973bf9235SOphir Munk 	},
207073bf9235SOphir Munk 	{
207173bf9235SOphir Munk 		.dpdk_name = "rx_port_broadcast_packets",
207273bf9235SOphir Munk 		.ctr_name = "rx_vport_broadcast_packets",
207373bf9235SOphir Munk 	},
207473bf9235SOphir Munk 	{
207573bf9235SOphir Munk 		.dpdk_name = "tx_port_unicast_bytes",
207673bf9235SOphir Munk 		.ctr_name = "tx_vport_unicast_bytes",
207773bf9235SOphir Munk 	},
207873bf9235SOphir Munk 	{
207973bf9235SOphir Munk 		.dpdk_name = "tx_port_multicast_bytes",
208073bf9235SOphir Munk 		.ctr_name = "tx_vport_multicast_bytes",
208173bf9235SOphir Munk 	},
208273bf9235SOphir Munk 	{
208373bf9235SOphir Munk 		.dpdk_name = "tx_port_broadcast_bytes",
208473bf9235SOphir Munk 		.ctr_name = "tx_vport_broadcast_bytes",
208573bf9235SOphir Munk 	},
208673bf9235SOphir Munk 	{
208773bf9235SOphir Munk 		.dpdk_name = "tx_port_unicast_packets",
208873bf9235SOphir Munk 		.ctr_name = "tx_vport_unicast_packets",
208973bf9235SOphir Munk 	},
209073bf9235SOphir Munk 	{
209173bf9235SOphir Munk 		.dpdk_name = "tx_port_multicast_packets",
209273bf9235SOphir Munk 		.ctr_name = "tx_vport_multicast_packets",
209373bf9235SOphir Munk 	},
209473bf9235SOphir Munk 	{
209573bf9235SOphir Munk 		.dpdk_name = "tx_port_broadcast_packets",
209673bf9235SOphir Munk 		.ctr_name = "tx_vport_broadcast_packets",
209773bf9235SOphir Munk 	},
209873bf9235SOphir Munk 	{
209973bf9235SOphir Munk 		.dpdk_name = "rx_wqe_err",
210073bf9235SOphir Munk 		.ctr_name = "rx_wqe_err",
210173bf9235SOphir Munk 	},
210273bf9235SOphir Munk 	{
210373bf9235SOphir Munk 		.dpdk_name = "rx_crc_errors_phy",
210473bf9235SOphir Munk 		.ctr_name = "rx_crc_errors_phy",
210573bf9235SOphir Munk 	},
210673bf9235SOphir Munk 	{
210773bf9235SOphir Munk 		.dpdk_name = "rx_in_range_len_errors_phy",
210873bf9235SOphir Munk 		.ctr_name = "rx_in_range_len_errors_phy",
210973bf9235SOphir Munk 	},
211073bf9235SOphir Munk 	{
211173bf9235SOphir Munk 		.dpdk_name = "rx_symbol_err_phy",
211273bf9235SOphir Munk 		.ctr_name = "rx_symbol_err_phy",
211373bf9235SOphir Munk 	},
211473bf9235SOphir Munk 	{
211573bf9235SOphir Munk 		.dpdk_name = "tx_errors_phy",
211673bf9235SOphir Munk 		.ctr_name = "tx_errors_phy",
211773bf9235SOphir Munk 	},
211873bf9235SOphir Munk 	{
211973bf9235SOphir Munk 		.dpdk_name = "rx_out_of_buffer",
212073bf9235SOphir Munk 		.ctr_name = "out_of_buffer",
212173bf9235SOphir Munk 		.dev = 1,
212273bf9235SOphir Munk 	},
212373bf9235SOphir Munk 	{
212473bf9235SOphir Munk 		.dpdk_name = "tx_packets_phy",
212573bf9235SOphir Munk 		.ctr_name = "tx_packets_phy",
212673bf9235SOphir Munk 	},
212773bf9235SOphir Munk 	{
212873bf9235SOphir Munk 		.dpdk_name = "rx_packets_phy",
212973bf9235SOphir Munk 		.ctr_name = "rx_packets_phy",
213073bf9235SOphir Munk 	},
213173bf9235SOphir Munk 	{
213273bf9235SOphir Munk 		.dpdk_name = "tx_discards_phy",
213373bf9235SOphir Munk 		.ctr_name = "tx_discards_phy",
213473bf9235SOphir Munk 	},
213573bf9235SOphir Munk 	{
213673bf9235SOphir Munk 		.dpdk_name = "rx_discards_phy",
213773bf9235SOphir Munk 		.ctr_name = "rx_discards_phy",
213873bf9235SOphir Munk 	},
213973bf9235SOphir Munk 	{
214073bf9235SOphir Munk 		.dpdk_name = "tx_bytes_phy",
214173bf9235SOphir Munk 		.ctr_name = "tx_bytes_phy",
214273bf9235SOphir Munk 	},
214373bf9235SOphir Munk 	{
214473bf9235SOphir Munk 		.dpdk_name = "rx_bytes_phy",
214573bf9235SOphir Munk 		.ctr_name = "rx_bytes_phy",
214673bf9235SOphir Munk 	},
214773bf9235SOphir Munk 	/* Representor only */
214873bf9235SOphir Munk 	{
214973bf9235SOphir Munk 		.dpdk_name = "rx_packets",
215073bf9235SOphir Munk 		.ctr_name = "vport_rx_packets",
215173bf9235SOphir Munk 	},
215273bf9235SOphir Munk 	{
215373bf9235SOphir Munk 		.dpdk_name = "rx_bytes",
215473bf9235SOphir Munk 		.ctr_name = "vport_rx_bytes",
215573bf9235SOphir Munk 	},
215673bf9235SOphir Munk 	{
215773bf9235SOphir Munk 		.dpdk_name = "tx_packets",
215873bf9235SOphir Munk 		.ctr_name = "vport_tx_packets",
215973bf9235SOphir Munk 	},
216073bf9235SOphir Munk 	{
216173bf9235SOphir Munk 		.dpdk_name = "tx_bytes",
216273bf9235SOphir Munk 		.ctr_name = "vport_tx_bytes",
216373bf9235SOphir Munk 	},
216473bf9235SOphir Munk };
216573bf9235SOphir Munk 
216673bf9235SOphir Munk static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
216773bf9235SOphir Munk 
216873bf9235SOphir Munk /**
216973bf9235SOphir Munk  * Init the structures to read device counters.
217073bf9235SOphir Munk  *
217173bf9235SOphir Munk  * @param dev
217273bf9235SOphir Munk  *   Pointer to Ethernet device.
217373bf9235SOphir Munk  */
217473bf9235SOphir Munk void
217573bf9235SOphir Munk mlx5_os_stats_init(struct rte_eth_dev *dev)
217673bf9235SOphir Munk {
217773bf9235SOphir Munk 	struct mlx5_priv *priv = dev->data->dev_private;
217873bf9235SOphir Munk 	struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
217973bf9235SOphir Munk 	struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl;
218073bf9235SOphir Munk 	unsigned int i;
218173bf9235SOphir Munk 	unsigned int j;
218273bf9235SOphir Munk 	struct ifreq ifr;
218373bf9235SOphir Munk 	struct ethtool_gstrings *strings = NULL;
218473bf9235SOphir Munk 	unsigned int dev_stats_n;
218573bf9235SOphir Munk 	unsigned int str_sz;
218673bf9235SOphir Munk 	int ret;
218773bf9235SOphir Munk 
218873bf9235SOphir Munk 	/* So that it won't aggregate for each init. */
218973bf9235SOphir Munk 	xstats_ctrl->mlx5_stats_n = 0;
219073bf9235SOphir Munk 	ret = mlx5_os_get_stats_n(dev);
219173bf9235SOphir Munk 	if (ret < 0) {
219273bf9235SOphir Munk 		DRV_LOG(WARNING, "port %u no extended statistics available",
219373bf9235SOphir Munk 			dev->data->port_id);
219473bf9235SOphir Munk 		return;
219573bf9235SOphir Munk 	}
219673bf9235SOphir Munk 	dev_stats_n = ret;
219773bf9235SOphir Munk 	/* Allocate memory to grab stat names and values. */
219873bf9235SOphir Munk 	str_sz = dev_stats_n * ETH_GSTRING_LEN;
219973bf9235SOphir Munk 	strings = (struct ethtool_gstrings *)
220073bf9235SOphir Munk 		  rte_malloc("xstats_strings",
220173bf9235SOphir Munk 			     str_sz + sizeof(struct ethtool_gstrings), 0);
220273bf9235SOphir Munk 	if (!strings) {
220373bf9235SOphir Munk 		DRV_LOG(WARNING, "port %u unable to allocate memory for xstats",
220473bf9235SOphir Munk 		     dev->data->port_id);
220573bf9235SOphir Munk 		return;
220673bf9235SOphir Munk 	}
220773bf9235SOphir Munk 	strings->cmd = ETHTOOL_GSTRINGS;
220873bf9235SOphir Munk 	strings->string_set = ETH_SS_STATS;
220973bf9235SOphir Munk 	strings->len = dev_stats_n;
221073bf9235SOphir Munk 	ifr.ifr_data = (caddr_t)strings;
221173bf9235SOphir Munk 	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
221273bf9235SOphir Munk 	if (ret) {
221373bf9235SOphir Munk 		DRV_LOG(WARNING, "port %u unable to get statistic names",
221473bf9235SOphir Munk 			dev->data->port_id);
221573bf9235SOphir Munk 		goto free;
221673bf9235SOphir Munk 	}
221773bf9235SOphir Munk 	for (i = 0; i != dev_stats_n; ++i) {
221873bf9235SOphir Munk 		const char *curr_string = (const char *)
221973bf9235SOphir Munk 			&strings->data[i * ETH_GSTRING_LEN];
222073bf9235SOphir Munk 
222173bf9235SOphir Munk 		for (j = 0; j != xstats_n; ++j) {
222273bf9235SOphir Munk 			if (!strcmp(mlx5_counters_init[j].ctr_name,
222373bf9235SOphir Munk 				    curr_string)) {
222473bf9235SOphir Munk 				unsigned int idx = xstats_ctrl->mlx5_stats_n++;
222573bf9235SOphir Munk 
222673bf9235SOphir Munk 				xstats_ctrl->dev_table_idx[idx] = i;
222773bf9235SOphir Munk 				xstats_ctrl->info[idx] = mlx5_counters_init[j];
222873bf9235SOphir Munk 				break;
222973bf9235SOphir Munk 			}
223073bf9235SOphir Munk 		}
223173bf9235SOphir Munk 	}
223273bf9235SOphir Munk 	/* Add dev counters. */
223373bf9235SOphir Munk 	for (i = 0; i != xstats_n; ++i) {
223473bf9235SOphir Munk 		if (mlx5_counters_init[i].dev) {
223573bf9235SOphir Munk 			unsigned int idx = xstats_ctrl->mlx5_stats_n++;
223673bf9235SOphir Munk 
223773bf9235SOphir Munk 			xstats_ctrl->info[idx] = mlx5_counters_init[i];
223873bf9235SOphir Munk 			xstats_ctrl->hw_stats[idx] = 0;
223973bf9235SOphir Munk 		}
224073bf9235SOphir Munk 	}
224173bf9235SOphir Munk 	MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS);
224273bf9235SOphir Munk 	xstats_ctrl->stats_n = dev_stats_n;
224373bf9235SOphir Munk 	/* Copy to base at first time. */
224473bf9235SOphir Munk 	ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base);
224573bf9235SOphir Munk 	if (ret)
224673bf9235SOphir Munk 		DRV_LOG(ERR, "port %u cannot read device counters: %s",
224773bf9235SOphir Munk 			dev->data->port_id, strerror(rte_errno));
224873bf9235SOphir Munk 	mlx5_os_read_dev_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base);
224973bf9235SOphir Munk 	stats_ctrl->imissed = 0;
225073bf9235SOphir Munk free:
225173bf9235SOphir Munk 	rte_free(strings);
225273bf9235SOphir Munk }
225373bf9235SOphir Munk 
2254d5ed8aa9SOphir Munk /**
2255d5ed8aa9SOphir Munk  * Set the reg_mr and dereg_mr call backs
2256d5ed8aa9SOphir Munk  *
2257d5ed8aa9SOphir Munk  * @param reg_mr_cb[out]
2258d5ed8aa9SOphir Munk  *   Pointer to reg_mr func
2259d5ed8aa9SOphir Munk  * @param dereg_mr_cb[out]
2260d5ed8aa9SOphir Munk  *   Pointer to dereg_mr func
2261d5ed8aa9SOphir Munk  *
2262d5ed8aa9SOphir Munk  */
2263d5ed8aa9SOphir Munk void
2264d5ed8aa9SOphir Munk mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb,
2265d5ed8aa9SOphir Munk 		      mlx5_dereg_mr_t *dereg_mr_cb)
2266d5ed8aa9SOphir Munk {
22674f96d913SOphir Munk 	*reg_mr_cb = mlx5_verbs_ops.reg_mr;
22684f96d913SOphir Munk 	*dereg_mr_cb = mlx5_verbs_ops.dereg_mr;
2269d5ed8aa9SOphir Munk }
2270d5ed8aa9SOphir Munk 
2271042f5c94SOphir Munk const struct eth_dev_ops mlx5_os_dev_ops = {
2272042f5c94SOphir Munk 	.dev_configure = mlx5_dev_configure,
2273042f5c94SOphir Munk 	.dev_start = mlx5_dev_start,
2274042f5c94SOphir Munk 	.dev_stop = mlx5_dev_stop,
2275042f5c94SOphir Munk 	.dev_set_link_down = mlx5_set_link_down,
2276042f5c94SOphir Munk 	.dev_set_link_up = mlx5_set_link_up,
2277042f5c94SOphir Munk 	.dev_close = mlx5_dev_close,
2278042f5c94SOphir Munk 	.promiscuous_enable = mlx5_promiscuous_enable,
2279042f5c94SOphir Munk 	.promiscuous_disable = mlx5_promiscuous_disable,
2280042f5c94SOphir Munk 	.allmulticast_enable = mlx5_allmulticast_enable,
2281042f5c94SOphir Munk 	.allmulticast_disable = mlx5_allmulticast_disable,
2282042f5c94SOphir Munk 	.link_update = mlx5_link_update,
2283042f5c94SOphir Munk 	.stats_get = mlx5_stats_get,
2284042f5c94SOphir Munk 	.stats_reset = mlx5_stats_reset,
2285042f5c94SOphir Munk 	.xstats_get = mlx5_xstats_get,
2286042f5c94SOphir Munk 	.xstats_reset = mlx5_xstats_reset,
2287042f5c94SOphir Munk 	.xstats_get_names = mlx5_xstats_get_names,
2288042f5c94SOphir Munk 	.fw_version_get = mlx5_fw_version_get,
2289042f5c94SOphir Munk 	.dev_infos_get = mlx5_dev_infos_get,
2290042f5c94SOphir Munk 	.read_clock = mlx5_read_clock,
2291042f5c94SOphir Munk 	.dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get,
2292042f5c94SOphir Munk 	.vlan_filter_set = mlx5_vlan_filter_set,
2293042f5c94SOphir Munk 	.rx_queue_setup = mlx5_rx_queue_setup,
2294042f5c94SOphir Munk 	.rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup,
2295042f5c94SOphir Munk 	.tx_queue_setup = mlx5_tx_queue_setup,
2296042f5c94SOphir Munk 	.tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup,
2297042f5c94SOphir Munk 	.rx_queue_release = mlx5_rx_queue_release,
2298042f5c94SOphir Munk 	.tx_queue_release = mlx5_tx_queue_release,
2299042f5c94SOphir Munk 	.flow_ctrl_get = mlx5_dev_get_flow_ctrl,
2300042f5c94SOphir Munk 	.flow_ctrl_set = mlx5_dev_set_flow_ctrl,
2301042f5c94SOphir Munk 	.mac_addr_remove = mlx5_mac_addr_remove,
2302042f5c94SOphir Munk 	.mac_addr_add = mlx5_mac_addr_add,
2303042f5c94SOphir Munk 	.mac_addr_set = mlx5_mac_addr_set,
2304042f5c94SOphir Munk 	.set_mc_addr_list = mlx5_set_mc_addr_list,
2305042f5c94SOphir Munk 	.mtu_set = mlx5_dev_set_mtu,
2306042f5c94SOphir Munk 	.vlan_strip_queue_set = mlx5_vlan_strip_queue_set,
2307042f5c94SOphir Munk 	.vlan_offload_set = mlx5_vlan_offload_set,
2308042f5c94SOphir Munk 	.reta_update = mlx5_dev_rss_reta_update,
2309042f5c94SOphir Munk 	.reta_query = mlx5_dev_rss_reta_query,
2310042f5c94SOphir Munk 	.rss_hash_update = mlx5_rss_hash_update,
2311042f5c94SOphir Munk 	.rss_hash_conf_get = mlx5_rss_hash_conf_get,
2312042f5c94SOphir Munk 	.filter_ctrl = mlx5_dev_filter_ctrl,
2313042f5c94SOphir Munk 	.rx_descriptor_status = mlx5_rx_descriptor_status,
2314042f5c94SOphir Munk 	.tx_descriptor_status = mlx5_tx_descriptor_status,
2315042f5c94SOphir Munk 	.rxq_info_get = mlx5_rxq_info_get,
2316042f5c94SOphir Munk 	.txq_info_get = mlx5_txq_info_get,
2317042f5c94SOphir Munk 	.rx_burst_mode_get = mlx5_rx_burst_mode_get,
2318042f5c94SOphir Munk 	.tx_burst_mode_get = mlx5_tx_burst_mode_get,
2319042f5c94SOphir Munk 	.rx_queue_count = mlx5_rx_queue_count,
2320042f5c94SOphir Munk 	.rx_queue_intr_enable = mlx5_rx_intr_enable,
2321042f5c94SOphir Munk 	.rx_queue_intr_disable = mlx5_rx_intr_disable,
2322042f5c94SOphir Munk 	.is_removed = mlx5_is_removed,
2323042f5c94SOphir Munk 	.udp_tunnel_port_add  = mlx5_udp_tunnel_port_add,
2324042f5c94SOphir Munk 	.get_module_info = mlx5_get_module_info,
2325042f5c94SOphir Munk 	.get_module_eeprom = mlx5_get_module_eeprom,
2326042f5c94SOphir Munk 	.hairpin_cap_get = mlx5_hairpin_cap_get,
2327042f5c94SOphir Munk 	.mtr_ops_get = mlx5_flow_meter_ops_get,
2328042f5c94SOphir Munk };
2329042f5c94SOphir Munk 
2330042f5c94SOphir Munk /* Available operations from secondary process. */
2331042f5c94SOphir Munk const struct eth_dev_ops mlx5_os_dev_sec_ops = {
2332042f5c94SOphir Munk 	.stats_get = mlx5_stats_get,
2333042f5c94SOphir Munk 	.stats_reset = mlx5_stats_reset,
2334042f5c94SOphir Munk 	.xstats_get = mlx5_xstats_get,
2335042f5c94SOphir Munk 	.xstats_reset = mlx5_xstats_reset,
2336042f5c94SOphir Munk 	.xstats_get_names = mlx5_xstats_get_names,
2337042f5c94SOphir Munk 	.fw_version_get = mlx5_fw_version_get,
2338042f5c94SOphir Munk 	.dev_infos_get = mlx5_dev_infos_get,
2339042f5c94SOphir Munk 	.rx_descriptor_status = mlx5_rx_descriptor_status,
2340042f5c94SOphir Munk 	.tx_descriptor_status = mlx5_tx_descriptor_status,
2341042f5c94SOphir Munk 	.rxq_info_get = mlx5_rxq_info_get,
2342042f5c94SOphir Munk 	.txq_info_get = mlx5_txq_info_get,
2343042f5c94SOphir Munk 	.rx_burst_mode_get = mlx5_rx_burst_mode_get,
2344042f5c94SOphir Munk 	.tx_burst_mode_get = mlx5_tx_burst_mode_get,
2345042f5c94SOphir Munk 	.get_module_info = mlx5_get_module_info,
2346042f5c94SOphir Munk 	.get_module_eeprom = mlx5_get_module_eeprom,
2347042f5c94SOphir Munk };
2348042f5c94SOphir Munk 
2349042f5c94SOphir Munk /* Available operations in flow isolated mode. */
2350042f5c94SOphir Munk const struct eth_dev_ops mlx5_os_dev_ops_isolate = {
2351042f5c94SOphir Munk 	.dev_configure = mlx5_dev_configure,
2352042f5c94SOphir Munk 	.dev_start = mlx5_dev_start,
2353042f5c94SOphir Munk 	.dev_stop = mlx5_dev_stop,
2354042f5c94SOphir Munk 	.dev_set_link_down = mlx5_set_link_down,
2355042f5c94SOphir Munk 	.dev_set_link_up = mlx5_set_link_up,
2356042f5c94SOphir Munk 	.dev_close = mlx5_dev_close,
2357042f5c94SOphir Munk 	.promiscuous_enable = mlx5_promiscuous_enable,
2358042f5c94SOphir Munk 	.promiscuous_disable = mlx5_promiscuous_disable,
2359042f5c94SOphir Munk 	.allmulticast_enable = mlx5_allmulticast_enable,
2360042f5c94SOphir Munk 	.allmulticast_disable = mlx5_allmulticast_disable,
2361042f5c94SOphir Munk 	.link_update = mlx5_link_update,
2362042f5c94SOphir Munk 	.stats_get = mlx5_stats_get,
2363042f5c94SOphir Munk 	.stats_reset = mlx5_stats_reset,
2364042f5c94SOphir Munk 	.xstats_get = mlx5_xstats_get,
2365042f5c94SOphir Munk 	.xstats_reset = mlx5_xstats_reset,
2366042f5c94SOphir Munk 	.xstats_get_names = mlx5_xstats_get_names,
2367042f5c94SOphir Munk 	.fw_version_get = mlx5_fw_version_get,
2368042f5c94SOphir Munk 	.dev_infos_get = mlx5_dev_infos_get,
2369042f5c94SOphir Munk 	.dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get,
2370042f5c94SOphir Munk 	.vlan_filter_set = mlx5_vlan_filter_set,
2371042f5c94SOphir Munk 	.rx_queue_setup = mlx5_rx_queue_setup,
2372042f5c94SOphir Munk 	.rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup,
2373042f5c94SOphir Munk 	.tx_queue_setup = mlx5_tx_queue_setup,
2374042f5c94SOphir Munk 	.tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup,
2375042f5c94SOphir Munk 	.rx_queue_release = mlx5_rx_queue_release,
2376042f5c94SOphir Munk 	.tx_queue_release = mlx5_tx_queue_release,
2377042f5c94SOphir Munk 	.flow_ctrl_get = mlx5_dev_get_flow_ctrl,
2378042f5c94SOphir Munk 	.flow_ctrl_set = mlx5_dev_set_flow_ctrl,
2379042f5c94SOphir Munk 	.mac_addr_remove = mlx5_mac_addr_remove,
2380042f5c94SOphir Munk 	.mac_addr_add = mlx5_mac_addr_add,
2381042f5c94SOphir Munk 	.mac_addr_set = mlx5_mac_addr_set,
2382042f5c94SOphir Munk 	.set_mc_addr_list = mlx5_set_mc_addr_list,
2383042f5c94SOphir Munk 	.mtu_set = mlx5_dev_set_mtu,
2384042f5c94SOphir Munk 	.vlan_strip_queue_set = mlx5_vlan_strip_queue_set,
2385042f5c94SOphir Munk 	.vlan_offload_set = mlx5_vlan_offload_set,
2386042f5c94SOphir Munk 	.filter_ctrl = mlx5_dev_filter_ctrl,
2387042f5c94SOphir Munk 	.rx_descriptor_status = mlx5_rx_descriptor_status,
2388042f5c94SOphir Munk 	.tx_descriptor_status = mlx5_tx_descriptor_status,
2389042f5c94SOphir Munk 	.rxq_info_get = mlx5_rxq_info_get,
2390042f5c94SOphir Munk 	.txq_info_get = mlx5_txq_info_get,
2391042f5c94SOphir Munk 	.rx_burst_mode_get = mlx5_rx_burst_mode_get,
2392042f5c94SOphir Munk 	.tx_burst_mode_get = mlx5_tx_burst_mode_get,
2393042f5c94SOphir Munk 	.rx_queue_intr_enable = mlx5_rx_intr_enable,
2394042f5c94SOphir Munk 	.rx_queue_intr_disable = mlx5_rx_intr_disable,
2395042f5c94SOphir Munk 	.is_removed = mlx5_is_removed,
2396042f5c94SOphir Munk 	.get_module_info = mlx5_get_module_info,
2397042f5c94SOphir Munk 	.get_module_eeprom = mlx5_get_module_eeprom,
2398042f5c94SOphir Munk 	.hairpin_cap_get = mlx5_hairpin_cap_get,
2399042f5c94SOphir Munk 	.mtr_ops_get = mlx5_flow_meter_ops_get,
2400042f5c94SOphir Munk };
2401