xref: /dpdk/drivers/net/mlx5/linux/mlx5_os.c (revision 4cbeba6fdbc4d58da40c2051a8543ca69d2eac17)
1f44b09f9SOphir Munk /* SPDX-License-Identifier: BSD-3-Clause
2f44b09f9SOphir Munk  * Copyright 2015 6WIND S.A.
3f44b09f9SOphir Munk  * Copyright 2020 Mellanox Technologies, Ltd
4f44b09f9SOphir Munk  */
5f44b09f9SOphir Munk 
6f44b09f9SOphir Munk #include <stddef.h>
7f44b09f9SOphir Munk #include <unistd.h>
8f44b09f9SOphir Munk #include <string.h>
9f44b09f9SOphir Munk #include <stdint.h>
10f44b09f9SOphir Munk #include <stdlib.h>
11f44b09f9SOphir Munk #include <errno.h>
12f44b09f9SOphir Munk #include <net/if.h>
13f44b09f9SOphir Munk #include <linux/rtnetlink.h>
1473bf9235SOphir Munk #include <linux/sockios.h>
1573bf9235SOphir Munk #include <linux/ethtool.h>
16f44b09f9SOphir Munk #include <fcntl.h>
17f44b09f9SOphir Munk 
18f44b09f9SOphir Munk #include <rte_malloc.h>
19df96fd0dSBruce Richardson #include <ethdev_driver.h>
20df96fd0dSBruce Richardson #include <ethdev_pci.h>
21f44b09f9SOphir Munk #include <rte_pci.h>
22a04322f6SDavid Marchand #include <bus_driver.h>
231f37cb2bSDavid Marchand #include <bus_pci_driver.h>
24b3f89090SDavid Marchand #include <bus_auxiliary_driver.h>
25f44b09f9SOphir Munk #include <rte_common.h>
26f44b09f9SOphir Munk #include <rte_kvargs.h>
27f44b09f9SOphir Munk #include <rte_rwlock.h>
28f44b09f9SOphir Munk #include <rte_spinlock.h>
29f44b09f9SOphir Munk #include <rte_string_fns.h>
30f44b09f9SOphir Munk #include <rte_alarm.h>
312aba9fc7SOphir Munk #include <rte_eal_paging.h>
32f44b09f9SOphir Munk 
33f44b09f9SOphir Munk #include <mlx5_glue.h>
34f44b09f9SOphir Munk #include <mlx5_devx_cmds.h>
35f44b09f9SOphir Munk #include <mlx5_common.h>
362eb4d010SOphir Munk #include <mlx5_common_mp.h>
37d5ed8aa9SOphir Munk #include <mlx5_common_mr.h>
385522da6bSSuanming Mou #include <mlx5_malloc.h>
39f44b09f9SOphir Munk 
40f44b09f9SOphir Munk #include "mlx5_defs.h"
41f44b09f9SOphir Munk #include "mlx5.h"
42391b8bccSOphir Munk #include "mlx5_common_os.h"
43f44b09f9SOphir Munk #include "mlx5_utils.h"
44f44b09f9SOphir Munk #include "mlx5_rxtx.h"
45151cbe3aSMichael Baum #include "mlx5_rx.h"
46377b69fbSMichael Baum #include "mlx5_tx.h"
47f44b09f9SOphir Munk #include "mlx5_autoconf.h"
48f44b09f9SOphir Munk #include "mlx5_flow.h"
49f44b09f9SOphir Munk #include "rte_pmd_mlx5.h"
504f96d913SOphir Munk #include "mlx5_verbs.h"
51f00f6562SOphir Munk #include "mlx5_nl.h"
526deb19e1SMichael Baum #include "mlx5_devx.h"
53f44b09f9SOphir Munk 
542eb4d010SOphir Munk #ifndef HAVE_IBV_MLX5_MOD_MPW
552eb4d010SOphir Munk #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
562eb4d010SOphir Munk #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
572eb4d010SOphir Munk #endif
582eb4d010SOphir Munk 
592eb4d010SOphir Munk #ifndef HAVE_IBV_MLX5_MOD_CQE_128B_COMP
602eb4d010SOphir Munk #define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4)
612eb4d010SOphir Munk #endif
622eb4d010SOphir Munk 
632e86c4e5SOphir Munk static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
642e86c4e5SOphir Munk 
652e86c4e5SOphir Munk /* Spinlock for mlx5_shared_data allocation. */
662e86c4e5SOphir Munk static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
672e86c4e5SOphir Munk 
682e86c4e5SOphir Munk /* Process local data for secondary processes. */
692e86c4e5SOphir Munk static struct mlx5_local_data mlx5_local_data;
702e86c4e5SOphir Munk 
71b4edeaf3SSuanming Mou /* rte flow indexed pool configuration. */
72b4edeaf3SSuanming Mou static struct mlx5_indexed_pool_config icfg[] = {
73b4edeaf3SSuanming Mou 	{
74b4edeaf3SSuanming Mou 		.size = sizeof(struct rte_flow),
75b4edeaf3SSuanming Mou 		.trunk_size = 64,
76b4edeaf3SSuanming Mou 		.need_lock = 1,
77b4edeaf3SSuanming Mou 		.release_mem_en = 0,
78b4edeaf3SSuanming Mou 		.malloc = mlx5_malloc,
79b4edeaf3SSuanming Mou 		.free = mlx5_free,
80b4edeaf3SSuanming Mou 		.per_core_cache = 0,
81b4edeaf3SSuanming Mou 		.type = "ctl_flow_ipool",
82b4edeaf3SSuanming Mou 	},
83b4edeaf3SSuanming Mou 	{
84b4edeaf3SSuanming Mou 		.size = sizeof(struct rte_flow),
85b4edeaf3SSuanming Mou 		.trunk_size = 64,
86b4edeaf3SSuanming Mou 		.grow_trunk = 3,
87b4edeaf3SSuanming Mou 		.grow_shift = 2,
88b4edeaf3SSuanming Mou 		.need_lock = 1,
89b4edeaf3SSuanming Mou 		.release_mem_en = 0,
90b4edeaf3SSuanming Mou 		.malloc = mlx5_malloc,
91b4edeaf3SSuanming Mou 		.free = mlx5_free,
92b4edeaf3SSuanming Mou 		.per_core_cache = 1 << 14,
93b4edeaf3SSuanming Mou 		.type = "rte_flow_ipool",
94b4edeaf3SSuanming Mou 	},
95b4edeaf3SSuanming Mou 	{
96b4edeaf3SSuanming Mou 		.size = sizeof(struct rte_flow),
97b4edeaf3SSuanming Mou 		.trunk_size = 64,
98b4edeaf3SSuanming Mou 		.grow_trunk = 3,
99b4edeaf3SSuanming Mou 		.grow_shift = 2,
100b4edeaf3SSuanming Mou 		.need_lock = 1,
101b4edeaf3SSuanming Mou 		.release_mem_en = 0,
102b4edeaf3SSuanming Mou 		.malloc = mlx5_malloc,
103b4edeaf3SSuanming Mou 		.free = mlx5_free,
104b4edeaf3SSuanming Mou 		.per_core_cache = 0,
105b4edeaf3SSuanming Mou 		.type = "mcp_flow_ipool",
106b4edeaf3SSuanming Mou 	},
107b4edeaf3SSuanming Mou };
108b4edeaf3SSuanming Mou 
109f44b09f9SOphir Munk /**
11008d1838fSDekel Peled  * Set the completion channel file descriptor interrupt as non-blocking.
11108d1838fSDekel Peled  *
11208d1838fSDekel Peled  * @param[in] rxq_obj
11308d1838fSDekel Peled  *   Pointer to RQ channel object, which includes the channel fd
11408d1838fSDekel Peled  *
11508d1838fSDekel Peled  * @param[out] fd
1167be78d02SJosh Soref  *   The file descriptor (representing the interrupt) used in this channel.
11708d1838fSDekel Peled  *
11808d1838fSDekel Peled  * @return
11908d1838fSDekel Peled  *   0 on successfully setting the fd to non-blocking, non-zero otherwise.
12008d1838fSDekel Peled  */
12108d1838fSDekel Peled int
12208d1838fSDekel Peled mlx5_os_set_nonblock_channel_fd(int fd)
12308d1838fSDekel Peled {
12408d1838fSDekel Peled 	int flags;
12508d1838fSDekel Peled 
12608d1838fSDekel Peled 	flags = fcntl(fd, F_GETFL);
12708d1838fSDekel Peled 	return fcntl(fd, F_SETFL, flags | O_NONBLOCK);
12808d1838fSDekel Peled }
12908d1838fSDekel Peled 
13008d1838fSDekel Peled /**
131e85f623eSOphir Munk  * Get mlx5 device attributes. The glue function query_device_ex() is called
132e85f623eSOphir Munk  * with out parameter of type 'struct ibv_device_attr_ex *'. Then fill in mlx5
133e85f623eSOphir Munk  * device attributes from the glue out parameter.
134e85f623eSOphir Munk  *
13591d1cfafSMichael Baum  * @param sh
13691d1cfafSMichael Baum  *   Pointer to shared device context.
137e85f623eSOphir Munk  *
138e85f623eSOphir Munk  * @return
1396be4c57aSMichael Baum  *   0 on success, a negative errno value otherwise and rte_errno is set.
140e85f623eSOphir Munk  */
141e85f623eSOphir Munk int
14291d1cfafSMichael Baum mlx5_os_capabilities_prepare(struct mlx5_dev_ctx_shared *sh)
143e85f623eSOphir Munk {
144e85f623eSOphir Munk 	int err;
14587af0d1eSMichael Baum 	struct mlx5_common_device *cdev = sh->cdev;
14687af0d1eSMichael Baum 	struct mlx5_hca_attr *hca_attr = &cdev->config.hca_attr;
14791d1cfafSMichael Baum 	struct ibv_device_attr_ex attr_ex = { .comp_mask = 0 };
14891d1cfafSMichael Baum 	struct mlx5dv_context dv_attr = { .comp_mask = 0 };
149fe46b20cSMichael Baum 
15087af0d1eSMichael Baum 	err = mlx5_glue->query_device_ex(cdev->ctx, NULL, &attr_ex);
1516be4c57aSMichael Baum 	if (err) {
1526be4c57aSMichael Baum 		rte_errno = errno;
1536be4c57aSMichael Baum 		return -rte_errno;
1546be4c57aSMichael Baum 	}
1558f464810SMichael Baum #ifdef HAVE_IBV_MLX5_MOD_SWP
1568f464810SMichael Baum 	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP;
1578f464810SMichael Baum #endif
1588f464810SMichael Baum #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1598f464810SMichael Baum 	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS;
1608f464810SMichael Baum #endif
1618f464810SMichael Baum #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
1628f464810SMichael Baum 	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
1638f464810SMichael Baum #endif
164*4cbeba6fSSuanming Mou #ifdef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
165*4cbeba6fSSuanming Mou 	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_REG_C0;
166*4cbeba6fSSuanming Mou #endif
16787af0d1eSMichael Baum 	err = mlx5_glue->dv_query_device(cdev->ctx, &dv_attr);
1686be4c57aSMichael Baum 	if (err) {
1696be4c57aSMichael Baum 		rte_errno = errno;
1706be4c57aSMichael Baum 		return -rte_errno;
1716be4c57aSMichael Baum 	}
17291d1cfafSMichael Baum 	memset(&sh->dev_cap, 0, sizeof(struct mlx5_dev_cap));
17387af0d1eSMichael Baum 	if (mlx5_dev_is_pci(cdev->dev))
17487af0d1eSMichael Baum 		sh->dev_cap.vf = mlx5_dev_is_vf_pci(RTE_DEV_TO_PCI(cdev->dev));
17587af0d1eSMichael Baum 	else
17687af0d1eSMichael Baum 		sh->dev_cap.sf = 1;
17791d1cfafSMichael Baum 	sh->dev_cap.max_qp_wr = attr_ex.orig_attr.max_qp_wr;
17891d1cfafSMichael Baum 	sh->dev_cap.max_sge = attr_ex.orig_attr.max_sge;
17991d1cfafSMichael Baum 	sh->dev_cap.max_cq = attr_ex.orig_attr.max_cq;
18091d1cfafSMichael Baum 	sh->dev_cap.max_qp = attr_ex.orig_attr.max_qp;
18187af0d1eSMichael Baum #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
18287af0d1eSMichael Baum 	sh->dev_cap.dest_tir = 1;
18387af0d1eSMichael Baum #endif
18487af0d1eSMichael Baum #if defined(HAVE_IBV_FLOW_DV_SUPPORT) && defined(HAVE_MLX5DV_DR)
18587af0d1eSMichael Baum 	DRV_LOG(DEBUG, "DV flow is supported.");
18687af0d1eSMichael Baum 	sh->dev_cap.dv_flow_en = 1;
18787af0d1eSMichael Baum #endif
18887af0d1eSMichael Baum #ifdef HAVE_MLX5DV_DR_ESWITCH
18987af0d1eSMichael Baum 	if (hca_attr->eswitch_manager && sh->dev_cap.dv_flow_en && sh->esw_mode)
19087af0d1eSMichael Baum 		sh->dev_cap.dv_esw_en = 1;
19187af0d1eSMichael Baum #endif
19287af0d1eSMichael Baum 	/*
19387af0d1eSMichael Baum 	 * Multi-packet send is supported by ConnectX-4 Lx PF as well
19487af0d1eSMichael Baum 	 * as all ConnectX-5 devices.
19587af0d1eSMichael Baum 	 */
19687af0d1eSMichael Baum 	if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) {
19787af0d1eSMichael Baum 		if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) {
19887af0d1eSMichael Baum 			DRV_LOG(DEBUG, "Enhanced MPW is supported.");
19987af0d1eSMichael Baum 			sh->dev_cap.mps = MLX5_MPW_ENHANCED;
20087af0d1eSMichael Baum 		} else {
20187af0d1eSMichael Baum 			DRV_LOG(DEBUG, "MPW is supported.");
20287af0d1eSMichael Baum 			sh->dev_cap.mps = MLX5_MPW;
20387af0d1eSMichael Baum 		}
20487af0d1eSMichael Baum 	} else {
20587af0d1eSMichael Baum 		DRV_LOG(DEBUG, "MPW isn't supported.");
20687af0d1eSMichael Baum 		sh->dev_cap.mps = MLX5_MPW_DISABLED;
20787af0d1eSMichael Baum 	}
20887af0d1eSMichael Baum #if (RTE_CACHE_LINE_SIZE == 128)
20987af0d1eSMichael Baum 	if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP)
21087af0d1eSMichael Baum 		sh->dev_cap.cqe_comp = 1;
21187af0d1eSMichael Baum 	DRV_LOG(DEBUG, "Rx CQE 128B compression is %ssupported.",
21287af0d1eSMichael Baum 		sh->dev_cap.cqe_comp ? "" : "not ");
21387af0d1eSMichael Baum #else
21487af0d1eSMichael Baum 	sh->dev_cap.cqe_comp = 1;
21587af0d1eSMichael Baum #endif
21687af0d1eSMichael Baum #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
21787af0d1eSMichael Baum 	sh->dev_cap.mpls_en =
21887af0d1eSMichael Baum 		((dv_attr.tunnel_offloads_caps &
21987af0d1eSMichael Baum 		  MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) &&
22087af0d1eSMichael Baum 		 (dv_attr.tunnel_offloads_caps &
22187af0d1eSMichael Baum 		  MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP));
22287af0d1eSMichael Baum 	DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is %ssupported.",
22387af0d1eSMichael Baum 		sh->dev_cap.mpls_en ? "" : "not ");
22487af0d1eSMichael Baum #else
22587af0d1eSMichael Baum 	DRV_LOG(WARNING,
22687af0d1eSMichael Baum 		"MPLS over GRE/UDP tunnel offloading disabled due to old OFED/rdma-core version or firmware configuration");
22787af0d1eSMichael Baum #endif
22887af0d1eSMichael Baum #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)
22987af0d1eSMichael Baum 	sh->dev_cap.hw_padding = !!attr_ex.rx_pad_end_addr_align;
23087af0d1eSMichael Baum #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)
23187af0d1eSMichael Baum 	sh->dev_cap.hw_padding = !!(attr_ex.device_cap_flags_ex &
23287af0d1eSMichael Baum 				    IBV_DEVICE_PCI_WRITE_END_PADDING);
23387af0d1eSMichael Baum #endif
23487af0d1eSMichael Baum 	sh->dev_cap.hw_csum =
23587af0d1eSMichael Baum 		!!(attr_ex.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM);
23687af0d1eSMichael Baum 	DRV_LOG(DEBUG, "Checksum offloading is %ssupported.",
23787af0d1eSMichael Baum 		sh->dev_cap.hw_csum ? "" : "not ");
23887af0d1eSMichael Baum 	sh->dev_cap.hw_vlan_strip = !!(attr_ex.raw_packet_caps &
23987af0d1eSMichael Baum 				       IBV_RAW_PACKET_CAP_CVLAN_STRIPPING);
24087af0d1eSMichael Baum 	DRV_LOG(DEBUG, "VLAN stripping is %ssupported.",
24187af0d1eSMichael Baum 		(sh->dev_cap.hw_vlan_strip ? "" : "not "));
24287af0d1eSMichael Baum 	sh->dev_cap.hw_fcs_strip = !!(attr_ex.raw_packet_caps &
24387af0d1eSMichael Baum 				      IBV_RAW_PACKET_CAP_SCATTER_FCS);
24487af0d1eSMichael Baum #if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
24587af0d1eSMichael Baum 	!defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
24687af0d1eSMichael Baum 	DRV_LOG(DEBUG, "Counters are not supported.");
24787af0d1eSMichael Baum #endif
24887af0d1eSMichael Baum 	/*
24987af0d1eSMichael Baum 	 * DPDK doesn't support larger/variable indirection tables.
25087af0d1eSMichael Baum 	 * Once DPDK supports it, take max size from device attr.
25187af0d1eSMichael Baum 	 */
25287af0d1eSMichael Baum 	sh->dev_cap.ind_table_max_size =
25387af0d1eSMichael Baum 			RTE_MIN(attr_ex.rss_caps.max_rwq_indirection_table_size,
25487af0d1eSMichael Baum 				(unsigned int)RTE_ETH_RSS_RETA_SIZE_512);
25587af0d1eSMichael Baum 	DRV_LOG(DEBUG, "Maximum Rx indirection table size is %u",
25687af0d1eSMichael Baum 		sh->dev_cap.ind_table_max_size);
25787af0d1eSMichael Baum 	sh->dev_cap.tso = (attr_ex.tso_caps.max_tso > 0 &&
25887af0d1eSMichael Baum 			   (attr_ex.tso_caps.supported_qpts &
25987af0d1eSMichael Baum 			    (1 << IBV_QPT_RAW_PACKET)));
26087af0d1eSMichael Baum 	if (sh->dev_cap.tso)
26187af0d1eSMichael Baum 		sh->dev_cap.tso_max_payload_sz = attr_ex.tso_caps.max_tso;
26291d1cfafSMichael Baum 	strlcpy(sh->dev_cap.fw_ver, attr_ex.orig_attr.fw_ver,
26391d1cfafSMichael Baum 		sizeof(sh->dev_cap.fw_ver));
264e85f623eSOphir Munk #ifdef HAVE_IBV_MLX5_MOD_SWP
26587af0d1eSMichael Baum 	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP)
26687af0d1eSMichael Baum 		sh->dev_cap.swp = dv_attr.sw_parsing_caps.sw_parsing_offloads &
26787af0d1eSMichael Baum 				  (MLX5_SW_PARSING_CAP |
26887af0d1eSMichael Baum 				   MLX5_SW_PARSING_CSUM_CAP |
26987af0d1eSMichael Baum 				   MLX5_SW_PARSING_TSO_CAP);
27087af0d1eSMichael Baum 	DRV_LOG(DEBUG, "SWP support: %u", sh->dev_cap.swp);
271e85f623eSOphir Munk #endif
2728f464810SMichael Baum #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
27387af0d1eSMichael Baum 	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) {
27487af0d1eSMichael Baum 		struct mlx5dv_striding_rq_caps *strd_rq_caps =
27587af0d1eSMichael Baum 				&dv_attr.striding_rq_caps;
27687af0d1eSMichael Baum 
27787af0d1eSMichael Baum 		sh->dev_cap.mprq.enabled = 1;
27887af0d1eSMichael Baum 		sh->dev_cap.mprq.log_min_stride_size =
27987af0d1eSMichael Baum 			strd_rq_caps->min_single_stride_log_num_of_bytes;
28087af0d1eSMichael Baum 		sh->dev_cap.mprq.log_max_stride_size =
28187af0d1eSMichael Baum 			strd_rq_caps->max_single_stride_log_num_of_bytes;
28287af0d1eSMichael Baum 		sh->dev_cap.mprq.log_min_stride_num =
28387af0d1eSMichael Baum 			strd_rq_caps->min_single_wqe_log_num_of_strides;
28487af0d1eSMichael Baum 		sh->dev_cap.mprq.log_max_stride_num =
28587af0d1eSMichael Baum 			strd_rq_caps->max_single_wqe_log_num_of_strides;
28687af0d1eSMichael Baum 		sh->dev_cap.mprq.log_min_stride_wqe_size =
28787af0d1eSMichael Baum 					cdev->config.devx ?
28887af0d1eSMichael Baum 					hca_attr->log_min_stride_wqe_sz :
28987af0d1eSMichael Baum 					MLX5_MPRQ_LOG_MIN_STRIDE_WQE_SIZE;
29087af0d1eSMichael Baum 		DRV_LOG(DEBUG, "\tmin_single_stride_log_num_of_bytes: %u",
29187af0d1eSMichael Baum 			sh->dev_cap.mprq.log_min_stride_size);
29287af0d1eSMichael Baum 		DRV_LOG(DEBUG, "\tmax_single_stride_log_num_of_bytes: %u",
29387af0d1eSMichael Baum 			sh->dev_cap.mprq.log_max_stride_size);
29487af0d1eSMichael Baum 		DRV_LOG(DEBUG, "\tmin_single_wqe_log_num_of_strides: %u",
29587af0d1eSMichael Baum 			sh->dev_cap.mprq.log_min_stride_num);
29687af0d1eSMichael Baum 		DRV_LOG(DEBUG, "\tmax_single_wqe_log_num_of_strides: %u",
29787af0d1eSMichael Baum 			sh->dev_cap.mprq.log_max_stride_num);
29887af0d1eSMichael Baum 		DRV_LOG(DEBUG, "\tmin_stride_wqe_log_size: %u",
29987af0d1eSMichael Baum 			sh->dev_cap.mprq.log_min_stride_wqe_size);
30087af0d1eSMichael Baum 		DRV_LOG(DEBUG, "\tsupported_qpts: %d",
30187af0d1eSMichael Baum 			strd_rq_caps->supported_qpts);
30287af0d1eSMichael Baum 		DRV_LOG(DEBUG, "Device supports Multi-Packet RQ.");
30387af0d1eSMichael Baum 	}
3048f464810SMichael Baum #endif
305e85f623eSOphir Munk #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
30687af0d1eSMichael Baum 	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) {
30787af0d1eSMichael Baum 		sh->dev_cap.tunnel_en = dv_attr.tunnel_offloads_caps &
30887af0d1eSMichael Baum 					(MLX5_TUNNELED_OFFLOADS_VXLAN_CAP |
30987af0d1eSMichael Baum 					 MLX5_TUNNELED_OFFLOADS_GRE_CAP |
31087af0d1eSMichael Baum 					 MLX5_TUNNELED_OFFLOADS_GENEVE_CAP);
31187af0d1eSMichael Baum 	}
31287af0d1eSMichael Baum 	if (sh->dev_cap.tunnel_en) {
31387af0d1eSMichael Baum 		DRV_LOG(DEBUG, "Tunnel offloading is supported for %s%s%s",
31487af0d1eSMichael Baum 			sh->dev_cap.tunnel_en &
31587af0d1eSMichael Baum 			MLX5_TUNNELED_OFFLOADS_VXLAN_CAP ? "[VXLAN]" : "",
31687af0d1eSMichael Baum 			sh->dev_cap.tunnel_en &
31787af0d1eSMichael Baum 			MLX5_TUNNELED_OFFLOADS_GRE_CAP ? "[GRE]" : "",
31887af0d1eSMichael Baum 			sh->dev_cap.tunnel_en &
31987af0d1eSMichael Baum 			MLX5_TUNNELED_OFFLOADS_GENEVE_CAP ? "[GENEVE]" : "");
32087af0d1eSMichael Baum 	} else {
32187af0d1eSMichael Baum 		DRV_LOG(DEBUG, "Tunnel offloading is not supported.");
32287af0d1eSMichael Baum 	}
32387af0d1eSMichael Baum #else
32487af0d1eSMichael Baum 	DRV_LOG(WARNING,
32587af0d1eSMichael Baum 		"Tunnel offloading disabled due to old OFED/rdma-core version");
326e85f623eSOphir Munk #endif
32787af0d1eSMichael Baum 	if (!sh->cdev->config.devx)
32887af0d1eSMichael Baum 		return 0;
32987af0d1eSMichael Baum 	/* Check capabilities for Packet Pacing. */
33087af0d1eSMichael Baum 	DRV_LOG(DEBUG, "Timestamp counter frequency %u kHz.",
33187af0d1eSMichael Baum 		hca_attr->dev_freq_khz);
33287af0d1eSMichael Baum 	DRV_LOG(DEBUG, "Packet pacing is %ssupported.",
33387af0d1eSMichael Baum 		hca_attr->qos.packet_pacing ? "" : "not ");
33487af0d1eSMichael Baum 	DRV_LOG(DEBUG, "Cross channel ops are %ssupported.",
33587af0d1eSMichael Baum 		hca_attr->cross_channel ? "" : "not ");
33687af0d1eSMichael Baum 	DRV_LOG(DEBUG, "WQE index ignore is %ssupported.",
33787af0d1eSMichael Baum 		hca_attr->wqe_index_ignore ? "" : "not ");
33887af0d1eSMichael Baum 	DRV_LOG(DEBUG, "Non-wire SQ feature is %ssupported.",
33987af0d1eSMichael Baum 		hca_attr->non_wire_sq ? "" : "not ");
34087af0d1eSMichael Baum 	DRV_LOG(DEBUG, "Static WQE SQ feature is %ssupported (%d)",
34187af0d1eSMichael Baum 		hca_attr->log_max_static_sq_wq ? "" : "not ",
34287af0d1eSMichael Baum 		hca_attr->log_max_static_sq_wq);
34387af0d1eSMichael Baum 	DRV_LOG(DEBUG, "WQE rate PP mode is %ssupported.",
34487af0d1eSMichael Baum 		hca_attr->qos.wqe_rate_pp ? "" : "not ");
34587af0d1eSMichael Baum 	sh->dev_cap.txpp_en = hca_attr->qos.packet_pacing;
34687af0d1eSMichael Baum 	if (!hca_attr->cross_channel) {
34787af0d1eSMichael Baum 		DRV_LOG(DEBUG,
34887af0d1eSMichael Baum 			"Cross channel operations are required for packet pacing.");
34987af0d1eSMichael Baum 		sh->dev_cap.txpp_en = 0;
35087af0d1eSMichael Baum 	}
35187af0d1eSMichael Baum 	if (!hca_attr->wqe_index_ignore) {
35287af0d1eSMichael Baum 		DRV_LOG(DEBUG,
35387af0d1eSMichael Baum 			"WQE index ignore feature is required for packet pacing.");
35487af0d1eSMichael Baum 		sh->dev_cap.txpp_en = 0;
35587af0d1eSMichael Baum 	}
35687af0d1eSMichael Baum 	if (!hca_attr->non_wire_sq) {
35787af0d1eSMichael Baum 		DRV_LOG(DEBUG,
35887af0d1eSMichael Baum 			"Non-wire SQ feature is required for packet pacing.");
35987af0d1eSMichael Baum 		sh->dev_cap.txpp_en = 0;
36087af0d1eSMichael Baum 	}
36187af0d1eSMichael Baum 	if (!hca_attr->log_max_static_sq_wq) {
36287af0d1eSMichael Baum 		DRV_LOG(DEBUG,
36387af0d1eSMichael Baum 			"Static WQE SQ feature is required for packet pacing.");
36487af0d1eSMichael Baum 		sh->dev_cap.txpp_en = 0;
36587af0d1eSMichael Baum 	}
36687af0d1eSMichael Baum 	if (!hca_attr->qos.wqe_rate_pp) {
36787af0d1eSMichael Baum 		DRV_LOG(DEBUG,
36887af0d1eSMichael Baum 			"WQE rate mode is required for packet pacing.");
36987af0d1eSMichael Baum 		sh->dev_cap.txpp_en = 0;
37087af0d1eSMichael Baum 	}
37187af0d1eSMichael Baum #ifndef HAVE_MLX5DV_DEVX_UAR_OFFSET
37287af0d1eSMichael Baum 	DRV_LOG(DEBUG,
37387af0d1eSMichael Baum 		"DevX does not provide UAR offset, can't create queues for packet pacing.");
37487af0d1eSMichael Baum 	sh->dev_cap.txpp_en = 0;
37587af0d1eSMichael Baum #endif
37687af0d1eSMichael Baum 	sh->dev_cap.scatter_fcs_w_decap_disable =
37787af0d1eSMichael Baum 					hca_attr->scatter_fcs_w_decap_disable;
37887af0d1eSMichael Baum 	sh->dev_cap.rq_delay_drop_en = hca_attr->rq_delay_drop;
37987af0d1eSMichael Baum 	mlx5_rt_timestamp_config(sh, hca_attr);
380*4cbeba6fSSuanming Mou #ifdef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
381*4cbeba6fSSuanming Mou 	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_REG_C0) {
382*4cbeba6fSSuanming Mou 		sh->dev_cap.esw_info.regc_value = dv_attr.reg_c0.value;
383*4cbeba6fSSuanming Mou 		sh->dev_cap.esw_info.regc_mask = dv_attr.reg_c0.mask;
384*4cbeba6fSSuanming Mou 	}
385*4cbeba6fSSuanming Mou #else
386*4cbeba6fSSuanming Mou 	sh->dev_cap.esw_info.regc_value = 0;
387*4cbeba6fSSuanming Mou 	sh->dev_cap.esw_info.regc_mask = 0;
388*4cbeba6fSSuanming Mou #endif
3896be4c57aSMichael Baum 	return 0;
390e85f623eSOphir Munk }
3912eb4d010SOphir Munk 
3922eb4d010SOphir Munk /**
393630a587bSRongwei Liu  * Detect misc5 support or not
394630a587bSRongwei Liu  *
395630a587bSRongwei Liu  * @param[in] priv
396630a587bSRongwei Liu  *   Device private data pointer
397630a587bSRongwei Liu  */
398630a587bSRongwei Liu #ifdef HAVE_MLX5DV_DR
399630a587bSRongwei Liu static void
400630a587bSRongwei Liu __mlx5_discovery_misc5_cap(struct mlx5_priv *priv)
401630a587bSRongwei Liu {
402630a587bSRongwei Liu #ifdef HAVE_IBV_FLOW_DV_SUPPORT
403630a587bSRongwei Liu 	/* Dummy VxLAN matcher to detect rdma-core misc5 cap
404630a587bSRongwei Liu 	 * Case: IPv4--->UDP--->VxLAN--->vni
405630a587bSRongwei Liu 	 */
406630a587bSRongwei Liu 	void *tbl;
407630a587bSRongwei Liu 	struct mlx5_flow_dv_match_params matcher_mask;
408630a587bSRongwei Liu 	void *match_m;
409630a587bSRongwei Liu 	void *matcher;
410630a587bSRongwei Liu 	void *headers_m;
411630a587bSRongwei Liu 	void *misc5_m;
412630a587bSRongwei Liu 	uint32_t *tunnel_header_m;
413630a587bSRongwei Liu 	struct mlx5dv_flow_matcher_attr dv_attr;
414630a587bSRongwei Liu 
415630a587bSRongwei Liu 	memset(&matcher_mask, 0, sizeof(matcher_mask));
416630a587bSRongwei Liu 	matcher_mask.size = sizeof(matcher_mask.buf);
417630a587bSRongwei Liu 	match_m = matcher_mask.buf;
418630a587bSRongwei Liu 	headers_m = MLX5_ADDR_OF(fte_match_param, match_m, outer_headers);
419630a587bSRongwei Liu 	misc5_m = MLX5_ADDR_OF(fte_match_param,
420630a587bSRongwei Liu 			       match_m, misc_parameters_5);
421630a587bSRongwei Liu 	tunnel_header_m = (uint32_t *)
422630a587bSRongwei Liu 				MLX5_ADDR_OF(fte_match_set_misc5,
423630a587bSRongwei Liu 				misc5_m, tunnel_header_1);
424630a587bSRongwei Liu 	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
425630a587bSRongwei Liu 	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 4);
426630a587bSRongwei Liu 	MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
427630a587bSRongwei Liu 	*tunnel_header_m = 0xffffff;
428630a587bSRongwei Liu 
429630a587bSRongwei Liu 	tbl = mlx5_glue->dr_create_flow_tbl(priv->sh->rx_domain, 1);
430630a587bSRongwei Liu 	if (!tbl) {
431630a587bSRongwei Liu 		DRV_LOG(INFO, "No SW steering support");
432630a587bSRongwei Liu 		return;
433630a587bSRongwei Liu 	}
434630a587bSRongwei Liu 	dv_attr.type = IBV_FLOW_ATTR_NORMAL,
435630a587bSRongwei Liu 	dv_attr.match_mask = (void *)&matcher_mask,
436630a587bSRongwei Liu 	dv_attr.match_criteria_enable =
437630a587bSRongwei Liu 			(1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT) |
438630a587bSRongwei Liu 			(1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT);
439630a587bSRongwei Liu 	dv_attr.priority = 3;
440630a587bSRongwei Liu #ifdef HAVE_MLX5DV_DR_ESWITCH
441630a587bSRongwei Liu 	void *misc2_m;
442a13ec19cSMichael Baum 	if (priv->sh->config.dv_esw_en) {
443630a587bSRongwei Liu 		/* FDB enabled reg_c_0 */
444630a587bSRongwei Liu 		dv_attr.match_criteria_enable |=
445630a587bSRongwei Liu 				(1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT);
446630a587bSRongwei Liu 		misc2_m = MLX5_ADDR_OF(fte_match_param,
447630a587bSRongwei Liu 				       match_m, misc_parameters_2);
448630a587bSRongwei Liu 		MLX5_SET(fte_match_set_misc2, misc2_m,
449630a587bSRongwei Liu 			 metadata_reg_c_0, 0xffff);
450630a587bSRongwei Liu 	}
451630a587bSRongwei Liu #endif
452ca1418ceSMichael Baum 	matcher = mlx5_glue->dv_create_flow_matcher(priv->sh->cdev->ctx,
453630a587bSRongwei Liu 						    &dv_attr, tbl);
454630a587bSRongwei Liu 	if (matcher) {
455630a587bSRongwei Liu 		priv->sh->misc5_cap = 1;
456630a587bSRongwei Liu 		mlx5_glue->dv_destroy_flow_matcher(matcher);
457630a587bSRongwei Liu 	}
458630a587bSRongwei Liu 	mlx5_glue->dr_destroy_flow_tbl(tbl);
459630a587bSRongwei Liu #else
460630a587bSRongwei Liu 	RTE_SET_USED(priv);
461630a587bSRongwei Liu #endif
462630a587bSRongwei Liu }
463630a587bSRongwei Liu #endif
464630a587bSRongwei Liu 
465630a587bSRongwei Liu /**
4662eb4d010SOphir Munk  * Initialize DR related data within private structure.
4672eb4d010SOphir Munk  * Routine checks the reference counter and does actual
4682eb4d010SOphir Munk  * resources creation/initialization only if counter is zero.
4692eb4d010SOphir Munk  *
4702eb4d010SOphir Munk  * @param[in] priv
4712eb4d010SOphir Munk  *   Pointer to the private device data structure.
4722eb4d010SOphir Munk  *
4732eb4d010SOphir Munk  * @return
4742eb4d010SOphir Munk  *   Zero on success, positive error code otherwise.
4752eb4d010SOphir Munk  */
4762eb4d010SOphir Munk static int
4772eb4d010SOphir Munk mlx5_alloc_shared_dr(struct mlx5_priv *priv)
4782eb4d010SOphir Munk {
4792eb4d010SOphir Munk 	struct mlx5_dev_ctx_shared *sh = priv->sh;
480961b6774SMatan Azrad 	char s[MLX5_NAME_SIZE] __rte_unused;
48116dbba25SXueming Li 	int err;
4822eb4d010SOphir Munk 
48316dbba25SXueming Li 	MLX5_ASSERT(sh && sh->refcnt);
48416dbba25SXueming Li 	if (sh->refcnt > 1)
48516dbba25SXueming Li 		return 0;
4862eb4d010SOphir Munk 	err = mlx5_alloc_table_hash_list(priv);
4872eb4d010SOphir Munk 	if (err)
488291140c6SSuanming Mou 		goto error;
48949dffadfSBing Zhao 	sh->default_miss_action =
49049dffadfSBing Zhao 			mlx5_glue->dr_create_flow_action_default_miss();
49149dffadfSBing Zhao 	if (!sh->default_miss_action)
49249dffadfSBing Zhao 		DRV_LOG(WARNING, "Default miss action is not supported.");
493291140c6SSuanming Mou 	/* The resources below are only valid with DV support. */
494291140c6SSuanming Mou #ifdef HAVE_IBV_FLOW_DV_SUPPORT
495612b619fSRongwei Liu 	/* Init shared flex parsers list, no need lcore_share */
496612b619fSRongwei Liu 	snprintf(s, sizeof(s), "%s_flex_parsers_list", sh->ibdev_name);
497612b619fSRongwei Liu 	sh->flex_parsers_dv = mlx5_list_create(s, sh, false,
498612b619fSRongwei Liu 					       mlx5_flex_parser_create_cb,
499612b619fSRongwei Liu 					       mlx5_flex_parser_match_cb,
500612b619fSRongwei Liu 					       mlx5_flex_parser_remove_cb,
501612b619fSRongwei Liu 					       mlx5_flex_parser_clone_cb,
502612b619fSRongwei Liu 					       mlx5_flex_parser_clone_free_cb);
503612b619fSRongwei Liu 	if (!sh->flex_parsers_dv)
504612b619fSRongwei Liu 		goto error;
505612b619fSRongwei Liu 	if (priv->sh->config.dv_flow_en == 2)
506612b619fSRongwei Liu 		return 0;
507491b7137SMatan Azrad 	/* Init port id action list. */
508e78e5408SMatan Azrad 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
509d03b7860SSuanming Mou 	sh->port_id_action_list = mlx5_list_create(s, sh, true,
5100fd5f82aSXueming Li 						   flow_dv_port_id_create_cb,
5110fd5f82aSXueming Li 						   flow_dv_port_id_match_cb,
512491b7137SMatan Azrad 						   flow_dv_port_id_remove_cb,
513491b7137SMatan Azrad 						   flow_dv_port_id_clone_cb,
514491b7137SMatan Azrad 						 flow_dv_port_id_clone_free_cb);
515679f46c7SMatan Azrad 	if (!sh->port_id_action_list)
516679f46c7SMatan Azrad 		goto error;
517491b7137SMatan Azrad 	/* Init push vlan action list. */
518e78e5408SMatan Azrad 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
519d03b7860SSuanming Mou 	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
5203422af2aSXueming Li 						    flow_dv_push_vlan_create_cb,
5213422af2aSXueming Li 						    flow_dv_push_vlan_match_cb,
522491b7137SMatan Azrad 						    flow_dv_push_vlan_remove_cb,
523491b7137SMatan Azrad 						    flow_dv_push_vlan_clone_cb,
524491b7137SMatan Azrad 					       flow_dv_push_vlan_clone_free_cb);
525679f46c7SMatan Azrad 	if (!sh->push_vlan_action_list)
526679f46c7SMatan Azrad 		goto error;
527491b7137SMatan Azrad 	/* Init sample action list. */
528e78e5408SMatan Azrad 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
529d03b7860SSuanming Mou 	sh->sample_action_list = mlx5_list_create(s, sh, true,
53019784141SSuanming Mou 						  flow_dv_sample_create_cb,
53119784141SSuanming Mou 						  flow_dv_sample_match_cb,
532491b7137SMatan Azrad 						  flow_dv_sample_remove_cb,
533491b7137SMatan Azrad 						  flow_dv_sample_clone_cb,
534491b7137SMatan Azrad 						  flow_dv_sample_clone_free_cb);
535679f46c7SMatan Azrad 	if (!sh->sample_action_list)
536679f46c7SMatan Azrad 		goto error;
537491b7137SMatan Azrad 	/* Init dest array action list. */
538e78e5408SMatan Azrad 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
539d03b7860SSuanming Mou 	sh->dest_array_list = mlx5_list_create(s, sh, true,
54019784141SSuanming Mou 					       flow_dv_dest_array_create_cb,
54119784141SSuanming Mou 					       flow_dv_dest_array_match_cb,
542491b7137SMatan Azrad 					       flow_dv_dest_array_remove_cb,
543491b7137SMatan Azrad 					       flow_dv_dest_array_clone_cb,
544491b7137SMatan Azrad 					      flow_dv_dest_array_clone_free_cb);
545679f46c7SMatan Azrad 	if (!sh->dest_array_list)
546679f46c7SMatan Azrad 		goto error;
547612b619fSRongwei Liu #else
548612b619fSRongwei Liu 	if (priv->sh->config.dv_flow_en == 2)
549612b619fSRongwei Liu 		return 0;
550291140c6SSuanming Mou #endif
5512eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR
5522eb4d010SOphir Munk 	void *domain;
5532eb4d010SOphir Munk 
5542eb4d010SOphir Munk 	/* Reference counter is zero, we should initialize structures. */
555ca1418ceSMichael Baum 	domain = mlx5_glue->dr_create_domain(sh->cdev->ctx,
5562eb4d010SOphir Munk 					     MLX5DV_DR_DOMAIN_TYPE_NIC_RX);
5572eb4d010SOphir Munk 	if (!domain) {
5582eb4d010SOphir Munk 		DRV_LOG(ERR, "ingress mlx5dv_dr_create_domain failed");
5592eb4d010SOphir Munk 		err = errno;
5602eb4d010SOphir Munk 		goto error;
5612eb4d010SOphir Munk 	}
5622eb4d010SOphir Munk 	sh->rx_domain = domain;
563ca1418ceSMichael Baum 	domain = mlx5_glue->dr_create_domain(sh->cdev->ctx,
5642eb4d010SOphir Munk 					     MLX5DV_DR_DOMAIN_TYPE_NIC_TX);
5652eb4d010SOphir Munk 	if (!domain) {
5662eb4d010SOphir Munk 		DRV_LOG(ERR, "egress mlx5dv_dr_create_domain failed");
5672eb4d010SOphir Munk 		err = errno;
5682eb4d010SOphir Munk 		goto error;
5692eb4d010SOphir Munk 	}
5702eb4d010SOphir Munk 	sh->tx_domain = domain;
5712eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_ESWITCH
572a13ec19cSMichael Baum 	if (sh->config.dv_esw_en) {
573ca1418ceSMichael Baum 		domain = mlx5_glue->dr_create_domain(sh->cdev->ctx,
574ca1418ceSMichael Baum 						     MLX5DV_DR_DOMAIN_TYPE_FDB);
5752eb4d010SOphir Munk 		if (!domain) {
5762eb4d010SOphir Munk 			DRV_LOG(ERR, "FDB mlx5dv_dr_create_domain failed");
5772eb4d010SOphir Munk 			err = errno;
5782eb4d010SOphir Munk 			goto error;
5792eb4d010SOphir Munk 		}
5802eb4d010SOphir Munk 		sh->fdb_domain = domain;
581da845ae9SViacheslav Ovsiienko 	}
582da845ae9SViacheslav Ovsiienko 	/*
583da845ae9SViacheslav Ovsiienko 	 * The drop action is just some dummy placeholder in rdma-core. It
584da845ae9SViacheslav Ovsiienko 	 * does not belong to domains and has no any attributes, and, can be
585da845ae9SViacheslav Ovsiienko 	 * shared by the entire device.
586da845ae9SViacheslav Ovsiienko 	 */
587da845ae9SViacheslav Ovsiienko 	sh->dr_drop_action = mlx5_glue->dr_create_flow_action_drop();
588da845ae9SViacheslav Ovsiienko 	if (!sh->dr_drop_action) {
589da845ae9SViacheslav Ovsiienko 		DRV_LOG(ERR, "FDB mlx5dv_dr_create_flow_action_drop");
590da845ae9SViacheslav Ovsiienko 		err = errno;
591da845ae9SViacheslav Ovsiienko 		goto error;
5922eb4d010SOphir Munk 	}
5932eb4d010SOphir Munk #endif
594a13ec19cSMichael Baum 	if (!sh->tunnel_hub && sh->config.dv_miss_info)
5954ec6360dSGregory Etelson 		err = mlx5_alloc_tunnel_hub(sh);
5964ec6360dSGregory Etelson 	if (err) {
5974ec6360dSGregory Etelson 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
5984ec6360dSGregory Etelson 		goto error;
5994ec6360dSGregory Etelson 	}
600a13ec19cSMichael Baum 	if (sh->config.reclaim_mode == MLX5_RCM_AGGR) {
6012eb4d010SOphir Munk 		mlx5_glue->dr_reclaim_domain_memory(sh->rx_domain, 1);
6022eb4d010SOphir Munk 		mlx5_glue->dr_reclaim_domain_memory(sh->tx_domain, 1);
6032eb4d010SOphir Munk 		if (sh->fdb_domain)
6042eb4d010SOphir Munk 			mlx5_glue->dr_reclaim_domain_memory(sh->fdb_domain, 1);
6052eb4d010SOphir Munk 	}
6062eb4d010SOphir Munk 	sh->pop_vlan_action = mlx5_glue->dr_create_flow_action_pop_vlan();
607a13ec19cSMichael Baum 	if (!sh->config.allow_duplicate_pattern) {
608e39226bdSJiawei Wang #ifndef HAVE_MLX5_DR_ALLOW_DUPLICATE
609e39226bdSJiawei Wang 		DRV_LOG(WARNING, "Disallow duplicate pattern is not supported - maybe old rdma-core version?");
610e39226bdSJiawei Wang #endif
611e39226bdSJiawei Wang 		mlx5_glue->dr_allow_duplicate_rules(sh->rx_domain, 0);
612e39226bdSJiawei Wang 		mlx5_glue->dr_allow_duplicate_rules(sh->tx_domain, 0);
613e39226bdSJiawei Wang 		if (sh->fdb_domain)
614e39226bdSJiawei Wang 			mlx5_glue->dr_allow_duplicate_rules(sh->fdb_domain, 0);
615e39226bdSJiawei Wang 	}
616630a587bSRongwei Liu 
617630a587bSRongwei Liu 	__mlx5_discovery_misc5_cap(priv);
6182eb4d010SOphir Munk #endif /* HAVE_MLX5DV_DR */
61909c25553SXueming Li 	LIST_INIT(&sh->shared_rxqs);
6202eb4d010SOphir Munk 	return 0;
6212eb4d010SOphir Munk error:
6222eb4d010SOphir Munk 	/* Rollback the created objects. */
6232eb4d010SOphir Munk 	if (sh->rx_domain) {
6242eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->rx_domain);
6252eb4d010SOphir Munk 		sh->rx_domain = NULL;
6262eb4d010SOphir Munk 	}
6272eb4d010SOphir Munk 	if (sh->tx_domain) {
6282eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->tx_domain);
6292eb4d010SOphir Munk 		sh->tx_domain = NULL;
6302eb4d010SOphir Munk 	}
6312eb4d010SOphir Munk 	if (sh->fdb_domain) {
6322eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->fdb_domain);
6332eb4d010SOphir Munk 		sh->fdb_domain = NULL;
6342eb4d010SOphir Munk 	}
635da845ae9SViacheslav Ovsiienko 	if (sh->dr_drop_action) {
636da845ae9SViacheslav Ovsiienko 		mlx5_glue->destroy_flow_action(sh->dr_drop_action);
637da845ae9SViacheslav Ovsiienko 		sh->dr_drop_action = NULL;
6382eb4d010SOphir Munk 	}
6392eb4d010SOphir Munk 	if (sh->pop_vlan_action) {
6402eb4d010SOphir Munk 		mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
6412eb4d010SOphir Munk 		sh->pop_vlan_action = NULL;
6422eb4d010SOphir Munk 	}
643bf615b07SSuanming Mou 	if (sh->encaps_decaps) {
644e69a5922SXueming Li 		mlx5_hlist_destroy(sh->encaps_decaps);
645bf615b07SSuanming Mou 		sh->encaps_decaps = NULL;
646bf615b07SSuanming Mou 	}
6473fe88961SSuanming Mou 	if (sh->modify_cmds) {
648e69a5922SXueming Li 		mlx5_hlist_destroy(sh->modify_cmds);
6493fe88961SSuanming Mou 		sh->modify_cmds = NULL;
6503fe88961SSuanming Mou 	}
6512eb4d010SOphir Munk 	if (sh->tag_table) {
6522eb4d010SOphir Munk 		/* tags should be destroyed with flow before. */
653e69a5922SXueming Li 		mlx5_hlist_destroy(sh->tag_table);
6542eb4d010SOphir Munk 		sh->tag_table = NULL;
6552eb4d010SOphir Munk 	}
6564ec6360dSGregory Etelson 	if (sh->tunnel_hub) {
6574ec6360dSGregory Etelson 		mlx5_release_tunnel_hub(sh, priv->dev_port);
6584ec6360dSGregory Etelson 		sh->tunnel_hub = NULL;
6594ec6360dSGregory Etelson 	}
6602eb4d010SOphir Munk 	mlx5_free_table_hash_list(priv);
661679f46c7SMatan Azrad 	if (sh->port_id_action_list) {
662679f46c7SMatan Azrad 		mlx5_list_destroy(sh->port_id_action_list);
663679f46c7SMatan Azrad 		sh->port_id_action_list = NULL;
664679f46c7SMatan Azrad 	}
665679f46c7SMatan Azrad 	if (sh->push_vlan_action_list) {
666679f46c7SMatan Azrad 		mlx5_list_destroy(sh->push_vlan_action_list);
667679f46c7SMatan Azrad 		sh->push_vlan_action_list = NULL;
668679f46c7SMatan Azrad 	}
669679f46c7SMatan Azrad 	if (sh->sample_action_list) {
670679f46c7SMatan Azrad 		mlx5_list_destroy(sh->sample_action_list);
671679f46c7SMatan Azrad 		sh->sample_action_list = NULL;
672679f46c7SMatan Azrad 	}
673679f46c7SMatan Azrad 	if (sh->dest_array_list) {
674679f46c7SMatan Azrad 		mlx5_list_destroy(sh->dest_array_list);
675679f46c7SMatan Azrad 		sh->dest_array_list = NULL;
676679f46c7SMatan Azrad 	}
6772eb4d010SOphir Munk 	return err;
6782eb4d010SOphir Munk }
6792eb4d010SOphir Munk 
6802eb4d010SOphir Munk /**
6812eb4d010SOphir Munk  * Destroy DR related data within private structure.
6822eb4d010SOphir Munk  *
6832eb4d010SOphir Munk  * @param[in] priv
6842eb4d010SOphir Munk  *   Pointer to the private device data structure.
6852eb4d010SOphir Munk  */
6862eb4d010SOphir Munk void
6872eb4d010SOphir Munk mlx5_os_free_shared_dr(struct mlx5_priv *priv)
6882eb4d010SOphir Munk {
68916dbba25SXueming Li 	struct mlx5_dev_ctx_shared *sh = priv->sh;
690b2cd3918SJiawei Wang #ifdef HAVE_MLX5DV_DR
691b2cd3918SJiawei Wang 	int i;
692b2cd3918SJiawei Wang #endif
6932eb4d010SOphir Munk 
69416dbba25SXueming Li 	MLX5_ASSERT(sh && sh->refcnt);
69516dbba25SXueming Li 	if (sh->refcnt > 1)
6962eb4d010SOphir Munk 		return;
69709c25553SXueming Li 	MLX5_ASSERT(LIST_EMPTY(&sh->shared_rxqs));
6982eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR
6992eb4d010SOphir Munk 	if (sh->rx_domain) {
7002eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->rx_domain);
7012eb4d010SOphir Munk 		sh->rx_domain = NULL;
7022eb4d010SOphir Munk 	}
7032eb4d010SOphir Munk 	if (sh->tx_domain) {
7042eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->tx_domain);
7052eb4d010SOphir Munk 		sh->tx_domain = NULL;
7062eb4d010SOphir Munk 	}
7072eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_ESWITCH
7082eb4d010SOphir Munk 	if (sh->fdb_domain) {
7092eb4d010SOphir Munk 		mlx5_glue->dr_destroy_domain(sh->fdb_domain);
7102eb4d010SOphir Munk 		sh->fdb_domain = NULL;
7112eb4d010SOphir Munk 	}
712da845ae9SViacheslav Ovsiienko 	if (sh->dr_drop_action) {
713da845ae9SViacheslav Ovsiienko 		mlx5_glue->destroy_flow_action(sh->dr_drop_action);
714da845ae9SViacheslav Ovsiienko 		sh->dr_drop_action = NULL;
7152eb4d010SOphir Munk 	}
7162eb4d010SOphir Munk #endif
7172eb4d010SOphir Munk 	if (sh->pop_vlan_action) {
7182eb4d010SOphir Munk 		mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
7192eb4d010SOphir Munk 		sh->pop_vlan_action = NULL;
7202eb4d010SOphir Munk 	}
721b2cd3918SJiawei Wang 	for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
722b2cd3918SJiawei Wang 		if (sh->send_to_kernel_action[i].action) {
723b2cd3918SJiawei Wang 			void *action = sh->send_to_kernel_action[i].action;
724f31a141eSMichael Savisko 
725f31a141eSMichael Savisko 			mlx5_glue->destroy_flow_action(action);
726b2cd3918SJiawei Wang 			sh->send_to_kernel_action[i].action = NULL;
727f31a141eSMichael Savisko 		}
728b2cd3918SJiawei Wang 		if (sh->send_to_kernel_action[i].tbl) {
729f31a141eSMichael Savisko 			struct mlx5_flow_tbl_resource *tbl =
730b2cd3918SJiawei Wang 					sh->send_to_kernel_action[i].tbl;
731f31a141eSMichael Savisko 
732f31a141eSMichael Savisko 			flow_dv_tbl_resource_release(sh, tbl);
733b2cd3918SJiawei Wang 			sh->send_to_kernel_action[i].tbl = NULL;
734b2cd3918SJiawei Wang 		}
735f31a141eSMichael Savisko 	}
7362eb4d010SOphir Munk #endif /* HAVE_MLX5DV_DR */
737b80726dcSSuanming Mou 	if (sh->default_miss_action)
738b80726dcSSuanming Mou 		mlx5_glue->destroy_flow_action
739b80726dcSSuanming Mou 				(sh->default_miss_action);
740bf615b07SSuanming Mou 	if (sh->encaps_decaps) {
741e69a5922SXueming Li 		mlx5_hlist_destroy(sh->encaps_decaps);
742bf615b07SSuanming Mou 		sh->encaps_decaps = NULL;
743bf615b07SSuanming Mou 	}
7443fe88961SSuanming Mou 	if (sh->modify_cmds) {
745e69a5922SXueming Li 		mlx5_hlist_destroy(sh->modify_cmds);
7463fe88961SSuanming Mou 		sh->modify_cmds = NULL;
7473fe88961SSuanming Mou 	}
7482eb4d010SOphir Munk 	if (sh->tag_table) {
7492eb4d010SOphir Munk 		/* tags should be destroyed with flow before. */
750e69a5922SXueming Li 		mlx5_hlist_destroy(sh->tag_table);
7512eb4d010SOphir Munk 		sh->tag_table = NULL;
7522eb4d010SOphir Munk 	}
7534ec6360dSGregory Etelson 	if (sh->tunnel_hub) {
7544ec6360dSGregory Etelson 		mlx5_release_tunnel_hub(sh, priv->dev_port);
7554ec6360dSGregory Etelson 		sh->tunnel_hub = NULL;
7564ec6360dSGregory Etelson 	}
7572eb4d010SOphir Munk 	mlx5_free_table_hash_list(priv);
758679f46c7SMatan Azrad 	if (sh->port_id_action_list) {
759679f46c7SMatan Azrad 		mlx5_list_destroy(sh->port_id_action_list);
760679f46c7SMatan Azrad 		sh->port_id_action_list = NULL;
761679f46c7SMatan Azrad 	}
762679f46c7SMatan Azrad 	if (sh->push_vlan_action_list) {
763679f46c7SMatan Azrad 		mlx5_list_destroy(sh->push_vlan_action_list);
764679f46c7SMatan Azrad 		sh->push_vlan_action_list = NULL;
765679f46c7SMatan Azrad 	}
766679f46c7SMatan Azrad 	if (sh->sample_action_list) {
767679f46c7SMatan Azrad 		mlx5_list_destroy(sh->sample_action_list);
768679f46c7SMatan Azrad 		sh->sample_action_list = NULL;
769679f46c7SMatan Azrad 	}
770679f46c7SMatan Azrad 	if (sh->dest_array_list) {
771679f46c7SMatan Azrad 		mlx5_list_destroy(sh->dest_array_list);
772679f46c7SMatan Azrad 		sh->dest_array_list = NULL;
773679f46c7SMatan Azrad 	}
7742eb4d010SOphir Munk }
7752eb4d010SOphir Munk 
7762eb4d010SOphir Munk /**
7772e86c4e5SOphir Munk  * Initialize shared data between primary and secondary process.
7782e86c4e5SOphir Munk  *
7792e86c4e5SOphir Munk  * A memzone is reserved by primary process and secondary processes attach to
7802e86c4e5SOphir Munk  * the memzone.
7812e86c4e5SOphir Munk  *
7822e86c4e5SOphir Munk  * @return
7832e86c4e5SOphir Munk  *   0 on success, a negative errno value otherwise and rte_errno is set.
7842e86c4e5SOphir Munk  */
7852e86c4e5SOphir Munk static int
7862e86c4e5SOphir Munk mlx5_init_shared_data(void)
7872e86c4e5SOphir Munk {
7882e86c4e5SOphir Munk 	const struct rte_memzone *mz;
7892e86c4e5SOphir Munk 	int ret = 0;
7902e86c4e5SOphir Munk 
7912e86c4e5SOphir Munk 	rte_spinlock_lock(&mlx5_shared_data_lock);
7922e86c4e5SOphir Munk 	if (mlx5_shared_data == NULL) {
7932e86c4e5SOphir Munk 		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
7942e86c4e5SOphir Munk 			/* Allocate shared memory. */
7952e86c4e5SOphir Munk 			mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA,
7962e86c4e5SOphir Munk 						 sizeof(*mlx5_shared_data),
7972e86c4e5SOphir Munk 						 SOCKET_ID_ANY, 0);
7982e86c4e5SOphir Munk 			if (mz == NULL) {
7992e86c4e5SOphir Munk 				DRV_LOG(ERR,
8002e86c4e5SOphir Munk 					"Cannot allocate mlx5 shared data");
8012e86c4e5SOphir Munk 				ret = -rte_errno;
8022e86c4e5SOphir Munk 				goto error;
8032e86c4e5SOphir Munk 			}
8042e86c4e5SOphir Munk 			mlx5_shared_data = mz->addr;
8052e86c4e5SOphir Munk 			memset(mlx5_shared_data, 0, sizeof(*mlx5_shared_data));
8062e86c4e5SOphir Munk 			rte_spinlock_init(&mlx5_shared_data->lock);
8072e86c4e5SOphir Munk 		} else {
8082e86c4e5SOphir Munk 			/* Lookup allocated shared memory. */
8092e86c4e5SOphir Munk 			mz = rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA);
8102e86c4e5SOphir Munk 			if (mz == NULL) {
8112e86c4e5SOphir Munk 				DRV_LOG(ERR,
8122e86c4e5SOphir Munk 					"Cannot attach mlx5 shared data");
8132e86c4e5SOphir Munk 				ret = -rte_errno;
8142e86c4e5SOphir Munk 				goto error;
8152e86c4e5SOphir Munk 			}
8162e86c4e5SOphir Munk 			mlx5_shared_data = mz->addr;
8172e86c4e5SOphir Munk 			memset(&mlx5_local_data, 0, sizeof(mlx5_local_data));
8182e86c4e5SOphir Munk 		}
8192e86c4e5SOphir Munk 	}
8202e86c4e5SOphir Munk error:
8212e86c4e5SOphir Munk 	rte_spinlock_unlock(&mlx5_shared_data_lock);
8222e86c4e5SOphir Munk 	return ret;
8232e86c4e5SOphir Munk }
8242e86c4e5SOphir Munk 
8252e86c4e5SOphir Munk /**
8262e86c4e5SOphir Munk  * PMD global initialization.
8272e86c4e5SOphir Munk  *
8282e86c4e5SOphir Munk  * Independent from individual device, this function initializes global
8292e86c4e5SOphir Munk  * per-PMD data structures distinguishing primary and secondary processes.
8302e86c4e5SOphir Munk  * Hence, each initialization is called once per a process.
8312e86c4e5SOphir Munk  *
8322e86c4e5SOphir Munk  * @return
8332e86c4e5SOphir Munk  *   0 on success, a negative errno value otherwise and rte_errno is set.
8342e86c4e5SOphir Munk  */
8352e86c4e5SOphir Munk static int
8362e86c4e5SOphir Munk mlx5_init_once(void)
8372e86c4e5SOphir Munk {
8382e86c4e5SOphir Munk 	struct mlx5_shared_data *sd;
8392e86c4e5SOphir Munk 	struct mlx5_local_data *ld = &mlx5_local_data;
8402e86c4e5SOphir Munk 	int ret = 0;
8412e86c4e5SOphir Munk 
8422e86c4e5SOphir Munk 	if (mlx5_init_shared_data())
8432e86c4e5SOphir Munk 		return -rte_errno;
8442e86c4e5SOphir Munk 	sd = mlx5_shared_data;
8452e86c4e5SOphir Munk 	MLX5_ASSERT(sd);
8462e86c4e5SOphir Munk 	rte_spinlock_lock(&sd->lock);
8472e86c4e5SOphir Munk 	switch (rte_eal_process_type()) {
8482e86c4e5SOphir Munk 	case RTE_PROC_PRIMARY:
8492e86c4e5SOphir Munk 		if (sd->init_done)
8502e86c4e5SOphir Munk 			break;
8512e86c4e5SOphir Munk 		ret = mlx5_mp_init_primary(MLX5_MP_NAME,
8522e86c4e5SOphir Munk 					   mlx5_mp_os_primary_handle);
8532e86c4e5SOphir Munk 		if (ret)
8542e86c4e5SOphir Munk 			goto out;
8552e86c4e5SOphir Munk 		sd->init_done = true;
8562e86c4e5SOphir Munk 		break;
8572e86c4e5SOphir Munk 	case RTE_PROC_SECONDARY:
8582e86c4e5SOphir Munk 		if (ld->init_done)
8592e86c4e5SOphir Munk 			break;
8602e86c4e5SOphir Munk 		ret = mlx5_mp_init_secondary(MLX5_MP_NAME,
8612e86c4e5SOphir Munk 					     mlx5_mp_os_secondary_handle);
8622e86c4e5SOphir Munk 		if (ret)
8632e86c4e5SOphir Munk 			goto out;
8642e86c4e5SOphir Munk 		++sd->secondary_cnt;
8652e86c4e5SOphir Munk 		ld->init_done = true;
8662e86c4e5SOphir Munk 		break;
8672e86c4e5SOphir Munk 	default:
8682e86c4e5SOphir Munk 		break;
8692e86c4e5SOphir Munk 	}
8702e86c4e5SOphir Munk out:
8712e86c4e5SOphir Munk 	rte_spinlock_unlock(&sd->lock);
8722e86c4e5SOphir Munk 	return ret;
8732e86c4e5SOphir Munk }
8742e86c4e5SOphir Munk 
8752e86c4e5SOphir Munk /**
87645633c46SSuanming Mou  * DR flow drop action support detect.
87745633c46SSuanming Mou  *
87845633c46SSuanming Mou  * @param dev
87945633c46SSuanming Mou  *   Pointer to rte_eth_dev structure.
88045633c46SSuanming Mou  *
88145633c46SSuanming Mou  */
88245633c46SSuanming Mou static void
88345633c46SSuanming Mou mlx5_flow_drop_action_config(struct rte_eth_dev *dev __rte_unused)
88445633c46SSuanming Mou {
88545633c46SSuanming Mou #ifdef HAVE_MLX5DV_DR
88645633c46SSuanming Mou 	struct mlx5_priv *priv = dev->data->dev_private;
88745633c46SSuanming Mou 
888a13ec19cSMichael Baum 	if (!priv->sh->config.dv_flow_en || !priv->sh->dr_drop_action)
88945633c46SSuanming Mou 		return;
89045633c46SSuanming Mou 	/**
89145633c46SSuanming Mou 	 * DR supports drop action placeholder when it is supported;
89245633c46SSuanming Mou 	 * otherwise, use the queue drop action.
89345633c46SSuanming Mou 	 */
8943c4338a4SJiawei Wang 	if (!priv->sh->drop_action_check_flag) {
8953c4338a4SJiawei Wang 		if (!mlx5_flow_discover_dr_action_support(dev))
896c1f0cdaeSDariusz Sosnowski 			priv->sh->dr_root_drop_action_en = 1;
8973c4338a4SJiawei Wang 		priv->sh->drop_action_check_flag = 1;
8983c4338a4SJiawei Wang 	}
899c1f0cdaeSDariusz Sosnowski 	if (priv->sh->dr_root_drop_action_en)
90045633c46SSuanming Mou 		priv->root_drop_action = priv->sh->dr_drop_action;
9013c4338a4SJiawei Wang 	else
9023c4338a4SJiawei Wang 		priv->root_drop_action = priv->drop_queue.hrxq->action;
90345633c46SSuanming Mou #endif
90445633c46SSuanming Mou }
90545633c46SSuanming Mou 
906e6988afdSMatan Azrad static void
907e6988afdSMatan Azrad mlx5_queue_counter_id_prepare(struct rte_eth_dev *dev)
908e6988afdSMatan Azrad {
909e6988afdSMatan Azrad 	struct mlx5_priv *priv = dev->data->dev_private;
910ca1418ceSMichael Baum 	void *ctx = priv->sh->cdev->ctx;
911e6988afdSMatan Azrad 
912e6988afdSMatan Azrad 	priv->q_counters = mlx5_devx_cmd_queue_counter_alloc(ctx);
913e6988afdSMatan Azrad 	if (!priv->q_counters) {
914e6988afdSMatan Azrad 		struct ibv_cq *cq = mlx5_glue->create_cq(ctx, 1, NULL, NULL, 0);
915e6988afdSMatan Azrad 		struct ibv_wq *wq;
916e6988afdSMatan Azrad 
917e6988afdSMatan Azrad 		DRV_LOG(DEBUG, "Port %d queue counter object cannot be created "
918e6988afdSMatan Azrad 			"by DevX - fall-back to use the kernel driver global "
919e6988afdSMatan Azrad 			"queue counter.", dev->data->port_id);
920e6988afdSMatan Azrad 		/* Create WQ by kernel and query its queue counter ID. */
921e6988afdSMatan Azrad 		if (cq) {
922e6988afdSMatan Azrad 			wq = mlx5_glue->create_wq(ctx,
923e6988afdSMatan Azrad 						  &(struct ibv_wq_init_attr){
924e6988afdSMatan Azrad 						    .wq_type = IBV_WQT_RQ,
925e6988afdSMatan Azrad 						    .max_wr = 1,
926e6988afdSMatan Azrad 						    .max_sge = 1,
927e35ccf24SMichael Baum 						    .pd = priv->sh->cdev->pd,
928e6988afdSMatan Azrad 						    .cq = cq,
929e6988afdSMatan Azrad 						});
930e6988afdSMatan Azrad 			if (wq) {
931e6988afdSMatan Azrad 				/* Counter is assigned only on RDY state. */
932e6988afdSMatan Azrad 				int ret = mlx5_glue->modify_wq(wq,
933e6988afdSMatan Azrad 						 &(struct ibv_wq_attr){
934e6988afdSMatan Azrad 						 .attr_mask = IBV_WQ_ATTR_STATE,
935e6988afdSMatan Azrad 						 .wq_state = IBV_WQS_RDY,
936e6988afdSMatan Azrad 						});
937e6988afdSMatan Azrad 
938e6988afdSMatan Azrad 				if (ret == 0)
939e6988afdSMatan Azrad 					mlx5_devx_cmd_wq_query(wq,
940e6988afdSMatan Azrad 							 &priv->counter_set_id);
941e6988afdSMatan Azrad 				claim_zero(mlx5_glue->destroy_wq(wq));
942e6988afdSMatan Azrad 			}
943e6988afdSMatan Azrad 			claim_zero(mlx5_glue->destroy_cq(cq));
944e6988afdSMatan Azrad 		}
945e6988afdSMatan Azrad 	} else {
946e6988afdSMatan Azrad 		priv->counter_set_id = priv->q_counters->id;
947e6988afdSMatan Azrad 	}
948e6988afdSMatan Azrad 	if (priv->counter_set_id == 0)
949e6988afdSMatan Azrad 		DRV_LOG(INFO, "Part of the port %d statistics will not be "
950e6988afdSMatan Azrad 			"available.", dev->data->port_id);
951e6988afdSMatan Azrad }
952e6988afdSMatan Azrad 
953994829e6SSuanming Mou /**
954f926cce3SXueming Li  * Check if representor spawn info match devargs.
955f926cce3SXueming Li  *
956f926cce3SXueming Li  * @param spawn
957f926cce3SXueming Li  *   Verbs device parameters (name, port, switch_info) to spawn.
958f926cce3SXueming Li  * @param eth_da
959f926cce3SXueming Li  *   Device devargs to probe.
960f926cce3SXueming Li  *
961f926cce3SXueming Li  * @return
962f926cce3SXueming Li  *   Match result.
963f926cce3SXueming Li  */
964f926cce3SXueming Li static bool
965f926cce3SXueming Li mlx5_representor_match(struct mlx5_dev_spawn_data *spawn,
966f926cce3SXueming Li 		       struct rte_eth_devargs *eth_da)
967f926cce3SXueming Li {
968f926cce3SXueming Li 	struct mlx5_switch_info *switch_info = &spawn->info;
969f926cce3SXueming Li 	unsigned int p, f;
970f926cce3SXueming Li 	uint16_t id;
97191766faeSXueming Li 	uint16_t repr_id = mlx5_representor_id_encode(switch_info,
97291766faeSXueming Li 						      eth_da->type);
973f926cce3SXueming Li 
97411c73de9SDariusz Sosnowski 	/*
97511c73de9SDariusz Sosnowski 	 * Assuming Multiport E-Switch device was detected,
97611c73de9SDariusz Sosnowski 	 * if spawned port is an uplink, check if the port
97711c73de9SDariusz Sosnowski 	 * was requested through representor devarg.
97811c73de9SDariusz Sosnowski 	 */
97911c73de9SDariusz Sosnowski 	if (mlx5_is_probed_port_on_mpesw_device(spawn) &&
98011c73de9SDariusz Sosnowski 	    switch_info->name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK) {
98111c73de9SDariusz Sosnowski 		for (p = 0; p < eth_da->nb_ports; ++p)
98211c73de9SDariusz Sosnowski 			if (switch_info->port_name == eth_da->ports[p])
98311c73de9SDariusz Sosnowski 				return true;
98411c73de9SDariusz Sosnowski 		rte_errno = EBUSY;
98511c73de9SDariusz Sosnowski 		return false;
98611c73de9SDariusz Sosnowski 	}
987f926cce3SXueming Li 	switch (eth_da->type) {
98811c73de9SDariusz Sosnowski 	case RTE_ETH_REPRESENTOR_PF:
98911c73de9SDariusz Sosnowski 		/*
99011c73de9SDariusz Sosnowski 		 * PF representors provided in devargs translate to uplink ports, but
99111c73de9SDariusz Sosnowski 		 * if and only if the device is a part of MPESW device.
99211c73de9SDariusz Sosnowski 		 */
99311c73de9SDariusz Sosnowski 		if (!mlx5_is_probed_port_on_mpesw_device(spawn)) {
99411c73de9SDariusz Sosnowski 			rte_errno = EBUSY;
99511c73de9SDariusz Sosnowski 			return false;
99611c73de9SDariusz Sosnowski 		}
99711c73de9SDariusz Sosnowski 		break;
998f926cce3SXueming Li 	case RTE_ETH_REPRESENTOR_SF:
99991766faeSXueming Li 		if (!(spawn->info.port_name == -1 &&
100091766faeSXueming Li 		      switch_info->name_type ==
100191766faeSXueming Li 				MLX5_PHYS_PORT_NAME_TYPE_PFHPF) &&
100291766faeSXueming Li 		    switch_info->name_type != MLX5_PHYS_PORT_NAME_TYPE_PFSF) {
1003f926cce3SXueming Li 			rte_errno = EBUSY;
1004f926cce3SXueming Li 			return false;
1005f926cce3SXueming Li 		}
1006f926cce3SXueming Li 		break;
1007f926cce3SXueming Li 	case RTE_ETH_REPRESENTOR_VF:
1008f926cce3SXueming Li 		/* Allows HPF representor index -1 as exception. */
1009f926cce3SXueming Li 		if (!(spawn->info.port_name == -1 &&
1010f926cce3SXueming Li 		      switch_info->name_type ==
1011f926cce3SXueming Li 				MLX5_PHYS_PORT_NAME_TYPE_PFHPF) &&
1012f926cce3SXueming Li 		    switch_info->name_type != MLX5_PHYS_PORT_NAME_TYPE_PFVF) {
1013f926cce3SXueming Li 			rte_errno = EBUSY;
1014f926cce3SXueming Li 			return false;
1015f926cce3SXueming Li 		}
1016f926cce3SXueming Li 		break;
1017f926cce3SXueming Li 	case RTE_ETH_REPRESENTOR_NONE:
1018f926cce3SXueming Li 		rte_errno = EBUSY;
1019f926cce3SXueming Li 		return false;
1020f926cce3SXueming Li 	default:
1021f926cce3SXueming Li 		rte_errno = ENOTSUP;
1022f926cce3SXueming Li 		DRV_LOG(ERR, "unsupported representor type");
1023f926cce3SXueming Li 		return false;
1024f926cce3SXueming Li 	}
1025f926cce3SXueming Li 	/* Check representor ID: */
1026f926cce3SXueming Li 	for (p = 0; p < eth_da->nb_ports; ++p) {
102711c73de9SDariusz Sosnowski 		if (!mlx5_is_probed_port_on_mpesw_device(spawn) && spawn->pf_bond < 0) {
1028f926cce3SXueming Li 			/* For non-LAG mode, allow and ignore pf. */
1029f926cce3SXueming Li 			switch_info->pf_num = eth_da->ports[p];
103091766faeSXueming Li 			repr_id = mlx5_representor_id_encode(switch_info,
103191766faeSXueming Li 							     eth_da->type);
1032f926cce3SXueming Li 		}
1033f926cce3SXueming Li 		for (f = 0; f < eth_da->nb_representor_ports; ++f) {
1034f926cce3SXueming Li 			id = MLX5_REPRESENTOR_ID
1035f926cce3SXueming Li 				(eth_da->ports[p], eth_da->type,
1036f926cce3SXueming Li 				 eth_da->representor_ports[f]);
1037f926cce3SXueming Li 			if (repr_id == id)
1038f926cce3SXueming Li 				return true;
1039f926cce3SXueming Li 		}
1040f926cce3SXueming Li 	}
1041f926cce3SXueming Li 	rte_errno = EBUSY;
1042f926cce3SXueming Li 	return false;
1043f926cce3SXueming Li }
1044f926cce3SXueming Li 
1045f926cce3SXueming Li /**
10462eb4d010SOphir Munk  * Spawn an Ethernet device from Verbs information.
10472eb4d010SOphir Munk  *
10482eb4d010SOphir Munk  * @param dpdk_dev
10492eb4d010SOphir Munk  *   Backing DPDK device.
10502eb4d010SOphir Munk  * @param spawn
10512eb4d010SOphir Munk  *   Verbs device parameters (name, port, switch_info) to spawn.
1052887183efSMichael Baum  * @param eth_da
1053cb95feefSXueming Li  *   Device arguments.
1054a729d2f0SMichael Baum  * @param mkvlist
1055a729d2f0SMichael Baum  *   Pointer to mlx5 kvargs control, can be NULL if there is no devargs.
10562eb4d010SOphir Munk  *
10572eb4d010SOphir Munk  * @return
10582eb4d010SOphir Munk  *   A valid Ethernet device object on success, NULL otherwise and rte_errno
10592eb4d010SOphir Munk  *   is set. The following errors are defined:
10602eb4d010SOphir Munk  *
10612eb4d010SOphir Munk  *   EBUSY: device is not supposed to be spawned.
10622eb4d010SOphir Munk  *   EEXIST: device is already spawned
10632eb4d010SOphir Munk  */
10642eb4d010SOphir Munk static struct rte_eth_dev *
10652eb4d010SOphir Munk mlx5_dev_spawn(struct rte_device *dpdk_dev,
10662eb4d010SOphir Munk 	       struct mlx5_dev_spawn_data *spawn,
1067a729d2f0SMichael Baum 	       struct rte_eth_devargs *eth_da,
1068a729d2f0SMichael Baum 	       struct mlx5_kvargs_ctrl *mkvlist)
10692eb4d010SOphir Munk {
10702eb4d010SOphir Munk 	const struct mlx5_switch_info *switch_info = &spawn->info;
10712eb4d010SOphir Munk 	struct mlx5_dev_ctx_shared *sh = NULL;
10723fd2961eSXueming Li 	struct ibv_port_attr port_attr = { .state = IBV_PORT_NOP };
10732eb4d010SOphir Munk 	struct rte_eth_dev *eth_dev = NULL;
10742eb4d010SOphir Munk 	struct mlx5_priv *priv = NULL;
10752eb4d010SOphir Munk 	int err = 0;
10762eb4d010SOphir Munk 	struct rte_ether_addr mac;
10772eb4d010SOphir Munk 	char name[RTE_ETH_NAME_MAX_LEN];
10782eb4d010SOphir Munk 	int own_domain_id = 0;
10792eb4d010SOphir Munk 	uint16_t port_id;
1080d0cf77e8SViacheslav Ovsiienko 	struct mlx5_port_info vport_info = { .query_flags = 0 };
108145a6df80SMichael Baum 	int nl_rdma;
1082b4edeaf3SSuanming Mou 	int i;
10832eb4d010SOphir Munk 
10842eb4d010SOphir Munk 	/* Determine if this port representor is supposed to be spawned. */
1085f926cce3SXueming Li 	if (switch_info->representor && dpdk_dev->devargs &&
1086f926cce3SXueming Li 	    !mlx5_representor_match(spawn, eth_da))
1087d6541676SXueming Li 		return NULL;
10882eb4d010SOphir Munk 	/* Build device name. */
108911c73de9SDariusz Sosnowski 	if (spawn->pf_bond >= 0) {
10902eb4d010SOphir Munk 		/* Bonding device. */
1091f926cce3SXueming Li 		if (!switch_info->representor) {
1092f926cce3SXueming Li 			err = snprintf(name, sizeof(name), "%s_%s",
1093887183efSMichael Baum 				       dpdk_dev->name, spawn->phys_dev_name);
1094f926cce3SXueming Li 		} else {
1095f926cce3SXueming Li 			err = snprintf(name, sizeof(name), "%s_%s_representor_c%dpf%d%s%u",
1096887183efSMichael Baum 				dpdk_dev->name, spawn->phys_dev_name,
1097f926cce3SXueming Li 				switch_info->ctrl_num,
1098f926cce3SXueming Li 				switch_info->pf_num,
1099cb95feefSXueming Li 				switch_info->name_type ==
1100cb95feefSXueming Li 				MLX5_PHYS_PORT_NAME_TYPE_PFSF ? "sf" : "vf",
11012eb4d010SOphir Munk 				switch_info->port_name);
11022eb4d010SOphir Munk 		}
110311c73de9SDariusz Sosnowski 	} else if (mlx5_is_probed_port_on_mpesw_device(spawn)) {
110411c73de9SDariusz Sosnowski 		/* MPESW device. */
110511c73de9SDariusz Sosnowski 		if (switch_info->name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK) {
110611c73de9SDariusz Sosnowski 			err = snprintf(name, sizeof(name), "%s_p%d",
110711c73de9SDariusz Sosnowski 				       dpdk_dev->name, spawn->mpesw_port);
110811c73de9SDariusz Sosnowski 		} else {
110911c73de9SDariusz Sosnowski 			err = snprintf(name, sizeof(name), "%s_representor_c%dpf%d%s%u",
111011c73de9SDariusz Sosnowski 				dpdk_dev->name,
111111c73de9SDariusz Sosnowski 				switch_info->ctrl_num,
111211c73de9SDariusz Sosnowski 				switch_info->pf_num,
111311c73de9SDariusz Sosnowski 				switch_info->name_type ==
111411c73de9SDariusz Sosnowski 				MLX5_PHYS_PORT_NAME_TYPE_PFSF ? "sf" : "vf",
111511c73de9SDariusz Sosnowski 				switch_info->port_name);
111611c73de9SDariusz Sosnowski 		}
111711c73de9SDariusz Sosnowski 	} else {
111811c73de9SDariusz Sosnowski 		/* Single device. */
111911c73de9SDariusz Sosnowski 		if (!switch_info->representor)
112011c73de9SDariusz Sosnowski 			strlcpy(name, dpdk_dev->name, sizeof(name));
112111c73de9SDariusz Sosnowski 		else
112211c73de9SDariusz Sosnowski 			err = snprintf(name, sizeof(name), "%s_representor_%s%u",
112311c73de9SDariusz Sosnowski 				 dpdk_dev->name,
112411c73de9SDariusz Sosnowski 				 switch_info->name_type ==
112511c73de9SDariusz Sosnowski 				 MLX5_PHYS_PORT_NAME_TYPE_PFSF ? "sf" : "vf",
112611c73de9SDariusz Sosnowski 				 switch_info->port_name);
1127f926cce3SXueming Li 	}
1128f926cce3SXueming Li 	if (err >= (int)sizeof(name))
1129f926cce3SXueming Li 		DRV_LOG(WARNING, "device name overflow %s", name);
11302eb4d010SOphir Munk 	/* check if the device is already spawned */
11312eb4d010SOphir Munk 	if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) {
1132a729d2f0SMichael Baum 		/*
1133a729d2f0SMichael Baum 		 * When device is already spawned, its devargs should be set
1134a729d2f0SMichael Baum 		 * as used. otherwise, mlx5_kvargs_validate() will fail.
1135a729d2f0SMichael Baum 		 */
1136a729d2f0SMichael Baum 		if (mkvlist)
1137a729d2f0SMichael Baum 			mlx5_port_args_set_used(name, port_id, mkvlist);
11382eb4d010SOphir Munk 		rte_errno = EEXIST;
11392eb4d010SOphir Munk 		return NULL;
11402eb4d010SOphir Munk 	}
11412eb4d010SOphir Munk 	DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name);
11422eb4d010SOphir Munk 	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
11432eb4d010SOphir Munk 		struct mlx5_mp_id mp_id;
1144bc5d8fdbSLong Li 		int fd;
11452eb4d010SOphir Munk 
11462eb4d010SOphir Munk 		eth_dev = rte_eth_dev_attach_secondary(name);
11472eb4d010SOphir Munk 		if (eth_dev == NULL) {
11482eb4d010SOphir Munk 			DRV_LOG(ERR, "can not attach rte ethdev");
11492eb4d010SOphir Munk 			rte_errno = ENOMEM;
11502eb4d010SOphir Munk 			return NULL;
11512eb4d010SOphir Munk 		}
11522eb4d010SOphir Munk 		eth_dev->device = dpdk_dev;
1153b012b4ceSOphir Munk 		eth_dev->dev_ops = &mlx5_dev_sec_ops;
1154cbfc6111SFerruh Yigit 		eth_dev->rx_descriptor_status = mlx5_rx_descriptor_status;
1155cbfc6111SFerruh Yigit 		eth_dev->tx_descriptor_status = mlx5_tx_descriptor_status;
11562eb4d010SOphir Munk 		err = mlx5_proc_priv_init(eth_dev);
11572eb4d010SOphir Munk 		if (err)
11582eb4d010SOphir Munk 			return NULL;
1159fec28ca0SDmitry Kozlyuk 		mlx5_mp_id_init(&mp_id, eth_dev->data->port_id);
11602eb4d010SOphir Munk 		/* Receive command fd from primary process */
1161bc5d8fdbSLong Li 		fd = mlx5_mp_req_verbs_cmd_fd(&mp_id);
1162bc5d8fdbSLong Li 		if (fd < 0)
11632eb4d010SOphir Munk 			goto err_secondary;
11642eb4d010SOphir Munk 		/* Remap UAR for Tx queues. */
1165bc5d8fdbSLong Li 		err = mlx5_tx_uar_init_secondary(eth_dev, fd);
1166bc5d8fdbSLong Li 		close(fd);
11672eb4d010SOphir Munk 		if (err)
11682eb4d010SOphir Munk 			goto err_secondary;
11692eb4d010SOphir Munk 		/*
11702eb4d010SOphir Munk 		 * Ethdev pointer is still required as input since
11712eb4d010SOphir Munk 		 * the primary device is not accessible from the
11722eb4d010SOphir Munk 		 * secondary process.
11732eb4d010SOphir Munk 		 */
11742eb4d010SOphir Munk 		eth_dev->rx_pkt_burst = mlx5_select_rx_function(eth_dev);
11752eb4d010SOphir Munk 		eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev);
11762eb4d010SOphir Munk 		return eth_dev;
11772eb4d010SOphir Munk err_secondary:
11782eb4d010SOphir Munk 		mlx5_dev_close(eth_dev);
11792eb4d010SOphir Munk 		return NULL;
11802eb4d010SOphir Munk 	}
1181a729d2f0SMichael Baum 	sh = mlx5_alloc_shared_dev_ctx(spawn, mkvlist);
11822eb4d010SOphir Munk 	if (!sh)
11832eb4d010SOphir Munk 		return NULL;
1184be66461cSDmitry Kozlyuk 	nl_rdma = mlx5_nl_init(NETLINK_RDMA, 0);
11852eb4d010SOphir Munk 	/* Check port status. */
11863fd2961eSXueming Li 	if (spawn->phys_port <= UINT8_MAX) {
11873fd2961eSXueming Li 		/* Legacy Verbs api only support u8 port number. */
1188ca1418ceSMichael Baum 		err = mlx5_glue->query_port(sh->cdev->ctx, spawn->phys_port,
1189ca1418ceSMichael Baum 					    &port_attr);
11902eb4d010SOphir Munk 		if (err) {
11912eb4d010SOphir Munk 			DRV_LOG(ERR, "port query failed: %s", strerror(err));
11922eb4d010SOphir Munk 			goto error;
11932eb4d010SOphir Munk 		}
11942eb4d010SOphir Munk 		if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
11952eb4d010SOphir Munk 			DRV_LOG(ERR, "port is not configured in Ethernet mode");
11962eb4d010SOphir Munk 			err = EINVAL;
11972eb4d010SOphir Munk 			goto error;
11982eb4d010SOphir Munk 		}
11993fd2961eSXueming Li 	} else if (nl_rdma >= 0) {
12003fd2961eSXueming Li 		/* IB doesn't allow more than 255 ports, must be Ethernet. */
12013fd2961eSXueming Li 		err = mlx5_nl_port_state(nl_rdma,
12023fd2961eSXueming Li 			spawn->phys_dev_name,
12033fd2961eSXueming Li 			spawn->phys_port);
12043fd2961eSXueming Li 		if (err < 0) {
12053fd2961eSXueming Li 			DRV_LOG(INFO, "Failed to get netlink port state: %s",
12063fd2961eSXueming Li 				strerror(rte_errno));
12073fd2961eSXueming Li 			err = -rte_errno;
12083fd2961eSXueming Li 			goto error;
12093fd2961eSXueming Li 		}
12103fd2961eSXueming Li 		port_attr.state = (enum ibv_port_state)err;
12113fd2961eSXueming Li 	}
12122eb4d010SOphir Munk 	if (port_attr.state != IBV_PORT_ACTIVE)
12133fd2961eSXueming Li 		DRV_LOG(INFO, "port is not active: \"%s\" (%d)",
12142eb4d010SOphir Munk 			mlx5_glue->port_state_str(port_attr.state),
12152eb4d010SOphir Munk 			port_attr.state);
12162eb4d010SOphir Munk 	/* Allocate private eth device data. */
12172175c4dcSSuanming Mou 	priv = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
12182eb4d010SOphir Munk 			   sizeof(*priv),
12192175c4dcSSuanming Mou 			   RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
12202eb4d010SOphir Munk 	if (priv == NULL) {
12212eb4d010SOphir Munk 		DRV_LOG(ERR, "priv allocation failure");
12222eb4d010SOphir Munk 		err = ENOMEM;
12232eb4d010SOphir Munk 		goto error;
12242eb4d010SOphir Munk 	}
122580f872eeSMichael Baum 	/*
122680f872eeSMichael Baum 	 * When user configures remote PD and CTX and device creates RxQ by
122780f872eeSMichael Baum 	 * DevX, external RxQ is both supported and requested.
122880f872eeSMichael Baum 	 */
122980f872eeSMichael Baum 	if (mlx5_imported_pd_and_ctx(sh->cdev) && mlx5_devx_obj_ops_en(sh)) {
123080f872eeSMichael Baum 		priv->ext_rxqs = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
12318e8b44f2SSuanming Mou 					     sizeof(struct mlx5_external_q) *
123280f872eeSMichael Baum 					     MLX5_MAX_EXT_RX_QUEUES, 0,
123380f872eeSMichael Baum 					     SOCKET_ID_ANY);
123480f872eeSMichael Baum 		if (priv->ext_rxqs == NULL) {
123580f872eeSMichael Baum 			DRV_LOG(ERR, "Fail to allocate external RxQ array.");
123680f872eeSMichael Baum 			err = ENOMEM;
123780f872eeSMichael Baum 			goto error;
123880f872eeSMichael Baum 		}
12391944fbc3SSuanming Mou 		priv->ext_txqs = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
12401944fbc3SSuanming Mou 					     sizeof(struct mlx5_external_q) *
12411944fbc3SSuanming Mou 					     MLX5_MAX_EXT_TX_QUEUES, 0,
12421944fbc3SSuanming Mou 					     SOCKET_ID_ANY);
12431944fbc3SSuanming Mou 		if (priv->ext_txqs == NULL) {
12441944fbc3SSuanming Mou 			DRV_LOG(ERR, "Fail to allocate external TxQ array.");
12451944fbc3SSuanming Mou 			err = ENOMEM;
12461944fbc3SSuanming Mou 			goto error;
12471944fbc3SSuanming Mou 		}
12481944fbc3SSuanming Mou 		DRV_LOG(DEBUG, "External queue is supported.");
124980f872eeSMichael Baum 	}
12502eb4d010SOphir Munk 	priv->sh = sh;
125191389890SOphir Munk 	priv->dev_port = spawn->phys_port;
12522eb4d010SOphir Munk 	priv->pci_dev = spawn->pci_dev;
12532eb4d010SOphir Munk 	priv->mtu = RTE_ETHER_MTU;
12542eb4d010SOphir Munk 	/* Some internal functions rely on Netlink sockets, open them now. */
12553fd2961eSXueming Li 	priv->nl_socket_rdma = nl_rdma;
1256be66461cSDmitry Kozlyuk 	priv->nl_socket_route =	mlx5_nl_init(NETLINK_ROUTE, 0);
12572eb4d010SOphir Munk 	priv->representor = !!switch_info->representor;
12582eb4d010SOphir Munk 	priv->master = !!switch_info->master;
12592eb4d010SOphir Munk 	priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
12602eb4d010SOphir Munk 	priv->vport_meta_tag = 0;
12612eb4d010SOphir Munk 	priv->vport_meta_mask = 0;
12622eb4d010SOphir Munk 	priv->pf_bond = spawn->pf_bond;
126311c73de9SDariusz Sosnowski 	priv->mpesw_port = spawn->mpesw_port;
126411c73de9SDariusz Sosnowski 	priv->mpesw_uplink = false;
126511c73de9SDariusz Sosnowski 	priv->mpesw_owner = spawn->info.mpesw_owner;
126611c73de9SDariusz Sosnowski 	if (mlx5_is_port_on_mpesw_device(priv))
126711c73de9SDariusz Sosnowski 		priv->mpesw_uplink = (spawn->info.name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK);
1268ce4062cbSGregory Etelson 
1269ce4062cbSGregory Etelson 	DRV_LOG(DEBUG,
127011c73de9SDariusz Sosnowski 		"dev_port=%u bus=%s pci=%s master=%d representor=%d pf_bond=%d "
127111c73de9SDariusz Sosnowski 		"mpesw_port=%d mpesw_uplink=%d",
1272ce4062cbSGregory Etelson 		priv->dev_port, dpdk_dev->bus->name,
1273ce4062cbSGregory Etelson 		priv->pci_dev ? priv->pci_dev->name : "NONE",
127411c73de9SDariusz Sosnowski 		priv->master, priv->representor, priv->pf_bond,
127511c73de9SDariusz Sosnowski 		priv->mpesw_port, priv->mpesw_uplink);
1276ce4062cbSGregory Etelson 
127711c73de9SDariusz Sosnowski 	if (mlx5_is_port_on_mpesw_device(priv) && priv->sh->config.dv_flow_en != 2) {
127811c73de9SDariusz Sosnowski 		DRV_LOG(ERR, "MPESW device is supported only with HWS");
127911c73de9SDariusz Sosnowski 		err = ENOTSUP;
128011c73de9SDariusz Sosnowski 		goto error;
128111c73de9SDariusz Sosnowski 	}
12822eb4d010SOphir Munk 	/*
1283d0cf77e8SViacheslav Ovsiienko 	 * If we have E-Switch we should determine the vport attributes.
1284d0cf77e8SViacheslav Ovsiienko 	 * E-Switch may use either source vport field or reg_c[0] metadata
1285d0cf77e8SViacheslav Ovsiienko 	 * register to match on vport index. The engaged part of metadata
1286d0cf77e8SViacheslav Ovsiienko 	 * register is defined by mask.
12872eb4d010SOphir Munk 	 */
1288cf004fd3SMichael Baum 	if (sh->esw_mode) {
1289ca1418ceSMichael Baum 		err = mlx5_glue->devx_port_query(sh->cdev->ctx,
1290d0cf77e8SViacheslav Ovsiienko 						 spawn->phys_port,
1291d0cf77e8SViacheslav Ovsiienko 						 &vport_info);
12922eb4d010SOphir Munk 		if (err) {
12932eb4d010SOphir Munk 			DRV_LOG(WARNING,
1294887183efSMichael Baum 				"Cannot query devx port %d on device %s",
1295887183efSMichael Baum 				spawn->phys_port, spawn->phys_dev_name);
1296d0cf77e8SViacheslav Ovsiienko 			vport_info.query_flags = 0;
12972eb4d010SOphir Munk 		}
12982eb4d010SOphir Munk 	}
1299d0cf77e8SViacheslav Ovsiienko 	if (vport_info.query_flags & MLX5_PORT_QUERY_REG_C0) {
1300d0cf77e8SViacheslav Ovsiienko 		priv->vport_meta_tag = vport_info.vport_meta_tag;
1301d0cf77e8SViacheslav Ovsiienko 		priv->vport_meta_mask = vport_info.vport_meta_mask;
13022eb4d010SOphir Munk 		if (!priv->vport_meta_mask) {
1303887183efSMichael Baum 			DRV_LOG(ERR,
1304887183efSMichael Baum 				"vport zero mask for port %d on bonding device %s",
1305887183efSMichael Baum 				spawn->phys_port, spawn->phys_dev_name);
13062eb4d010SOphir Munk 			err = ENOTSUP;
13072eb4d010SOphir Munk 			goto error;
13082eb4d010SOphir Munk 		}
13092eb4d010SOphir Munk 		if (priv->vport_meta_tag & ~priv->vport_meta_mask) {
1310887183efSMichael Baum 			DRV_LOG(ERR,
1311887183efSMichael Baum 				"Invalid vport tag for port %d on bonding device %s",
1312887183efSMichael Baum 				spawn->phys_port, spawn->phys_dev_name);
13132eb4d010SOphir Munk 			err = ENOTSUP;
13142eb4d010SOphir Munk 			goto error;
13152eb4d010SOphir Munk 		}
13162eb4d010SOphir Munk 	}
1317d0cf77e8SViacheslav Ovsiienko 	if (vport_info.query_flags & MLX5_PORT_QUERY_VPORT) {
1318d0cf77e8SViacheslav Ovsiienko 		priv->vport_id = vport_info.vport_id;
1319cf004fd3SMichael Baum 	} else if (spawn->pf_bond >= 0 && sh->esw_mode) {
1320887183efSMichael Baum 		DRV_LOG(ERR,
1321887183efSMichael Baum 			"Cannot deduce vport index for port %d on bonding device %s",
1322887183efSMichael Baum 			spawn->phys_port, spawn->phys_dev_name);
13232eb4d010SOphir Munk 		err = ENOTSUP;
13242eb4d010SOphir Munk 		goto error;
13252eb4d010SOphir Munk 	} else {
13262eb4d010SOphir Munk 		/*
1327d0cf77e8SViacheslav Ovsiienko 		 * Suppose vport index in compatible way. Kernel/rdma_core
1328d0cf77e8SViacheslav Ovsiienko 		 * support single E-Switch per PF configurations only and
1329d0cf77e8SViacheslav Ovsiienko 		 * vport_id field contains the vport index for associated VF,
1330d0cf77e8SViacheslav Ovsiienko 		 * which is deduced from representor port name.
13312eb4d010SOphir Munk 		 * For example, let's have the IB device port 10, it has
13322eb4d010SOphir Munk 		 * attached network device eth0, which has port name attribute
13332eb4d010SOphir Munk 		 * pf0vf2, we can deduce the VF number as 2, and set vport index
13342eb4d010SOphir Munk 		 * as 3 (2+1). This assigning schema should be changed if the
13352eb4d010SOphir Munk 		 * multiple E-Switch instances per PF configurations or/and PCI
13362eb4d010SOphir Munk 		 * subfunctions are added.
13372eb4d010SOphir Munk 		 */
13382eb4d010SOphir Munk 		priv->vport_id = switch_info->representor ?
13392eb4d010SOphir Munk 				 switch_info->port_name + 1 : -1;
1340d0cf77e8SViacheslav Ovsiienko 	}
134191766faeSXueming Li 	priv->representor_id = mlx5_representor_id_encode(switch_info,
134291766faeSXueming Li 							  eth_da->type);
13432eb4d010SOphir Munk 	/*
13442eb4d010SOphir Munk 	 * Look for sibling devices in order to reuse their switch domain
13452eb4d010SOphir Munk 	 * if any, otherwise allocate one.
13462eb4d010SOphir Munk 	 */
1347ce4062cbSGregory Etelson 	MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) {
13482eb4d010SOphir Munk 		const struct mlx5_priv *opriv =
13492eb4d010SOphir Munk 			rte_eth_devices[port_id].data->dev_private;
13502eb4d010SOphir Munk 
13512eb4d010SOphir Munk 		if (!opriv ||
13522eb4d010SOphir Munk 		    opriv->sh != priv->sh ||
13532eb4d010SOphir Munk 			opriv->domain_id ==
13542eb4d010SOphir Munk 			RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
13552eb4d010SOphir Munk 			continue;
13562eb4d010SOphir Munk 		priv->domain_id = opriv->domain_id;
1357ce4062cbSGregory Etelson 		DRV_LOG(DEBUG, "dev_port-%u inherit domain_id=%u\n",
1358ce4062cbSGregory Etelson 			priv->dev_port, priv->domain_id);
13592eb4d010SOphir Munk 		break;
13602eb4d010SOphir Munk 	}
13612eb4d010SOphir Munk 	if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
13622eb4d010SOphir Munk 		err = rte_eth_switch_domain_alloc(&priv->domain_id);
13632eb4d010SOphir Munk 		if (err) {
13642eb4d010SOphir Munk 			err = rte_errno;
13652eb4d010SOphir Munk 			DRV_LOG(ERR, "unable to allocate switch domain: %s",
13662eb4d010SOphir Munk 				strerror(rte_errno));
13672eb4d010SOphir Munk 			goto error;
13682eb4d010SOphir Munk 		}
13692eb4d010SOphir Munk 		own_domain_id = 1;
1370ce4062cbSGregory Etelson 		DRV_LOG(DEBUG, "dev_port-%u new domain_id=%u\n",
1371ce4062cbSGregory Etelson 			priv->dev_port, priv->domain_id);
13722eb4d010SOphir Munk 	}
13736dc0cbc6SMichael Baum 	if (sh->cdev->config.devx) {
137445a6df80SMichael Baum 		struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr;
137545a6df80SMichael Baum 
137653820561SMichael Baum 		sh->steering_format_version = hca_attr->steering_format_version;
137748041ccbSGregory Etelson #if defined(HAVE_MLX5_DR_CREATE_ACTION_ASO_EXT)
137853820561SMichael Baum 		if (hca_attr->qos.sup && hca_attr->qos.flow_meter_old &&
1379a13ec19cSMichael Baum 		    sh->config.dv_flow_en) {
138048041ccbSGregory Etelson 			if (sh->registers.aso_reg != REG_NON) {
13812eb4d010SOphir Munk 				priv->mtr_en = 1;
138253820561SMichael Baum 				priv->mtr_reg_share = hca_attr->qos.flow_meter;
13832eb4d010SOphir Munk 			}
13842eb4d010SOphir Munk 		}
138553820561SMichael Baum 		if (hca_attr->qos.sup && hca_attr->qos.flow_meter_aso_sup) {
138629efa63aSLi Zhang 			uint32_t log_obj_size =
138729efa63aSLi Zhang 				rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1);
138829efa63aSLi Zhang 			if (log_obj_size >=
138953820561SMichael Baum 			    hca_attr->qos.log_meter_aso_granularity &&
139029efa63aSLi Zhang 			    log_obj_size <=
139153820561SMichael Baum 			    hca_attr->qos.log_meter_aso_max_alloc)
139229efa63aSLi Zhang 				sh->meter_aso_en = 1;
139344432018SLi Zhang 		}
139444432018SLi Zhang 		if (priv->mtr_en) {
1395afb4aa4fSLi Zhang 			err = mlx5_aso_flow_mtrs_mng_init(priv->sh);
139629efa63aSLi Zhang 			if (err) {
139729efa63aSLi Zhang 				err = -err;
139829efa63aSLi Zhang 				goto error;
139929efa63aSLi Zhang 			}
140029efa63aSLi Zhang 		}
140153820561SMichael Baum 		if (hca_attr->flow.tunnel_header_0_1)
1402630a587bSRongwei Liu 			sh->tunnel_header_0_1 = 1;
14035c4d4917SSean Zhang 		if (hca_attr->flow.tunnel_header_2_3)
14045c4d4917SSean Zhang 			sh->tunnel_header_2_3 = 1;
140548041ccbSGregory Etelson #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO_EXT */
1406a2999c7bSDekel Peled #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
14075e9f9a28SGregory Etelson 		if (hca_attr->flow_hit_aso && sh->registers.aso_reg == REG_C_3) {
140831ef2982SDekel Peled 			sh->flow_hit_aso_en = 1;
140931ef2982SDekel Peled 			err = mlx5_flow_aso_age_mng_init(sh);
141031ef2982SDekel Peled 			if (err) {
141131ef2982SDekel Peled 				err = -err;
141231ef2982SDekel Peled 				goto error;
141331ef2982SDekel Peled 			}
141431ef2982SDekel Peled 			DRV_LOG(DEBUG, "Flow Hit ASO is supported.");
141531ef2982SDekel Peled 		}
1416a2999c7bSDekel Peled #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
1417ee9e5fadSBing Zhao #if defined (HAVE_MLX5_DR_CREATE_ACTION_ASO) && \
1418ee9e5fadSBing Zhao     defined (HAVE_MLX5_DR_ACTION_ASO_CT)
1419463170a7SSuanming Mou 		/* HWS create CT ASO SQ based on HWS configure queue number. */
1420463170a7SSuanming Mou 		if (sh->config.dv_flow_en != 2 &&
14215e9f9a28SGregory Etelson 		    hca_attr->ct_offload && sh->registers.aso_reg == REG_C_3) {
1422ee9e5fadSBing Zhao 			err = mlx5_flow_aso_ct_mng_init(sh);
1423ee9e5fadSBing Zhao 			if (err) {
1424ee9e5fadSBing Zhao 				err = -err;
1425ee9e5fadSBing Zhao 				goto error;
1426ee9e5fadSBing Zhao 			}
1427ee9e5fadSBing Zhao 			DRV_LOG(DEBUG, "CT ASO is supported.");
1428ee9e5fadSBing Zhao 			sh->ct_aso_en = 1;
1429ee9e5fadSBing Zhao 		}
1430ee9e5fadSBing Zhao #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO && HAVE_MLX5_DR_ACTION_ASO_CT */
143196b1f027SJiawei Wang #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_SAMPLE)
143253820561SMichael Baum 		if (hca_attr->log_max_ft_sampler_num > 0  &&
1433a13ec19cSMichael Baum 		    sh->config.dv_flow_en) {
143496b1f027SJiawei Wang 			priv->sampler_en = 1;
14351b9e9826SThomas Monjalon 			DRV_LOG(DEBUG, "Sampler enabled!");
143696b1f027SJiawei Wang 		} else {
143796b1f027SJiawei Wang 			priv->sampler_en = 0;
143853820561SMichael Baum 			if (!hca_attr->log_max_ft_sampler_num)
14391b9e9826SThomas Monjalon 				DRV_LOG(WARNING,
14401b9e9826SThomas Monjalon 					"No available register for sampler.");
144196b1f027SJiawei Wang 			else
14421b9e9826SThomas Monjalon 				DRV_LOG(DEBUG, "DV flow is not supported!");
144396b1f027SJiawei Wang 		}
144496b1f027SJiawei Wang #endif
144576895c7dSJiawei Wang 		if (hca_attr->lag_rx_port_affinity) {
144676895c7dSJiawei Wang 			sh->lag_rx_port_affinity_en = 1;
144776895c7dSJiawei Wang 			DRV_LOG(DEBUG, "LAG Rx Port Affinity enabled");
144876895c7dSJiawei Wang 		}
1449674afdf0SJiawei Wang 		priv->num_lag_ports = hca_attr->num_lag_ports;
1450674afdf0SJiawei Wang 		DRV_LOG(DEBUG, "The number of lag ports is %d", priv->num_lag_ports);
14512eb4d010SOphir Munk 	}
145245a6df80SMichael Baum 	/* Process parameters and store port configuration on priv structure. */
1453a729d2f0SMichael Baum 	err = mlx5_port_args_config(priv, mkvlist, &priv->config);
145445a6df80SMichael Baum 	if (err) {
145545a6df80SMichael Baum 		err = rte_errno;
145645a6df80SMichael Baum 		DRV_LOG(ERR, "Failed to process port configure: %s",
145745a6df80SMichael Baum 			strerror(rte_errno));
145845a6df80SMichael Baum 		goto error;
14593d3f4e6dSAlexander Kozyrev 	}
14602eb4d010SOphir Munk 	eth_dev = rte_eth_dev_allocate(name);
14612eb4d010SOphir Munk 	if (eth_dev == NULL) {
14622eb4d010SOphir Munk 		DRV_LOG(ERR, "can not allocate rte ethdev");
14632eb4d010SOphir Munk 		err = ENOMEM;
14642eb4d010SOphir Munk 		goto error;
14652eb4d010SOphir Munk 	}
14662eb4d010SOphir Munk 	if (priv->representor) {
14672eb4d010SOphir Munk 		eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
14682eb4d010SOphir Munk 		eth_dev->data->representor_id = priv->representor_id;
1469ff4e52efSViacheslav Galaktionov 		MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) {
1470ff4e52efSViacheslav Galaktionov 			struct mlx5_priv *opriv =
1471ff4e52efSViacheslav Galaktionov 				rte_eth_devices[port_id].data->dev_private;
1472ff4e52efSViacheslav Galaktionov 			if (opriv &&
1473ff4e52efSViacheslav Galaktionov 			    opriv->master &&
1474ff4e52efSViacheslav Galaktionov 			    opriv->domain_id == priv->domain_id &&
1475ff4e52efSViacheslav Galaktionov 			    opriv->sh == priv->sh) {
1476ff4e52efSViacheslav Galaktionov 				eth_dev->data->backer_port_id = port_id;
1477ff4e52efSViacheslav Galaktionov 				break;
1478ff4e52efSViacheslav Galaktionov 			}
1479ff4e52efSViacheslav Galaktionov 		}
1480ff4e52efSViacheslav Galaktionov 		if (port_id >= RTE_MAX_ETHPORTS)
1481ff4e52efSViacheslav Galaktionov 			eth_dev->data->backer_port_id = eth_dev->data->port_id;
14822eb4d010SOphir Munk 	}
148339ae7577SSuanming Mou 	priv->mp_id.port_id = eth_dev->data->port_id;
148439ae7577SSuanming Mou 	strlcpy(priv->mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN);
14852eb4d010SOphir Munk 	/*
14862eb4d010SOphir Munk 	 * Store associated network device interface index. This index
14872eb4d010SOphir Munk 	 * is permanent throughout the lifetime of device. So, we may store
14882eb4d010SOphir Munk 	 * the ifindex here and use the cached value further.
14892eb4d010SOphir Munk 	 */
14902eb4d010SOphir Munk 	MLX5_ASSERT(spawn->ifindex);
14912eb4d010SOphir Munk 	priv->if_index = spawn->ifindex;
1492a89f6433SRongwei Liu 	priv->lag_affinity_idx = sh->refcnt - 1;
14932eb4d010SOphir Munk 	eth_dev->data->dev_private = priv;
14942eb4d010SOphir Munk 	priv->dev_data = eth_dev->data;
14952eb4d010SOphir Munk 	eth_dev->data->mac_addrs = priv->mac;
14962eb4d010SOphir Munk 	eth_dev->device = dpdk_dev;
1497f30e69b4SFerruh Yigit 	eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
14982eb4d010SOphir Munk 	/* Configure the first MAC address by default. */
14992eb4d010SOphir Munk 	if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
15002eb4d010SOphir Munk 		DRV_LOG(ERR,
15012eb4d010SOphir Munk 			"port %u cannot get MAC address, is mlx5_en"
15022eb4d010SOphir Munk 			" loaded? (errno: %s)",
15032eb4d010SOphir Munk 			eth_dev->data->port_id, strerror(rte_errno));
15042eb4d010SOphir Munk 		err = ENODEV;
15052eb4d010SOphir Munk 		goto error;
15062eb4d010SOphir Munk 	}
15072eb4d010SOphir Munk 	DRV_LOG(INFO,
1508c2c4f87bSAman Deep Singh 		"port %u MAC address is " RTE_ETHER_ADDR_PRT_FMT,
1509a7db3afcSAman Deep Singh 		eth_dev->data->port_id, RTE_ETHER_ADDR_BYTES(&mac));
15102eb4d010SOphir Munk #ifdef RTE_LIBRTE_MLX5_DEBUG
15112eb4d010SOphir Munk 	{
151228743807STal Shnaiderman 		char ifname[MLX5_NAMESIZE];
15132eb4d010SOphir Munk 
15142eb4d010SOphir Munk 		if (mlx5_get_ifname(eth_dev, &ifname) == 0)
15152eb4d010SOphir Munk 			DRV_LOG(DEBUG, "port %u ifname is \"%s\"",
15162eb4d010SOphir Munk 				eth_dev->data->port_id, ifname);
15172eb4d010SOphir Munk 		else
15182eb4d010SOphir Munk 			DRV_LOG(DEBUG, "port %u ifname is unknown",
15192eb4d010SOphir Munk 				eth_dev->data->port_id);
15202eb4d010SOphir Munk 	}
15212eb4d010SOphir Munk #endif
15222eb4d010SOphir Munk 	/* Get actual MTU if possible. */
15232eb4d010SOphir Munk 	err = mlx5_get_mtu(eth_dev, &priv->mtu);
15242eb4d010SOphir Munk 	if (err) {
15252eb4d010SOphir Munk 		err = rte_errno;
15262eb4d010SOphir Munk 		goto error;
15272eb4d010SOphir Munk 	}
15282eb4d010SOphir Munk 	DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id,
15292eb4d010SOphir Munk 		priv->mtu);
15302eb4d010SOphir Munk 	/* Initialize burst functions to prevent crashes before link-up. */
1531a41f593fSFerruh Yigit 	eth_dev->rx_pkt_burst = rte_eth_pkt_burst_dummy;
1532a41f593fSFerruh Yigit 	eth_dev->tx_pkt_burst = rte_eth_pkt_burst_dummy;
1533b012b4ceSOphir Munk 	eth_dev->dev_ops = &mlx5_dev_ops;
1534cbfc6111SFerruh Yigit 	eth_dev->rx_descriptor_status = mlx5_rx_descriptor_status;
1535cbfc6111SFerruh Yigit 	eth_dev->tx_descriptor_status = mlx5_tx_descriptor_status;
1536cbfc6111SFerruh Yigit 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
15372eb4d010SOphir Munk 	/* Register MAC address. */
15382eb4d010SOphir Munk 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
1539a13ec19cSMichael Baum 	if (sh->dev_cap.vf && sh->config.vf_nl_en)
15402eb4d010SOphir Munk 		mlx5_nl_mac_addr_sync(priv->nl_socket_route,
15412eb4d010SOphir Munk 				      mlx5_ifindex(eth_dev),
15422eb4d010SOphir Munk 				      eth_dev->data->mac_addrs,
15432eb4d010SOphir Munk 				      MLX5_MAX_MAC_ADDRESSES);
15442eb4d010SOphir Munk 	priv->ctrl_flows = 0;
1545d163fc2dSXueming Li 	rte_spinlock_init(&priv->flow_list_lock);
15462eb4d010SOphir Munk 	TAILQ_INIT(&priv->flow_meters);
1547a295c69aSShun Hao 	priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR);
1548a295c69aSShun Hao 	if (!priv->mtr_profile_tbl)
1549a295c69aSShun Hao 		goto error;
15502eb4d010SOphir Munk 	/* Bring Ethernet device up. */
15512eb4d010SOphir Munk 	DRV_LOG(DEBUG, "port %u forcing Ethernet interface up",
15522eb4d010SOphir Munk 		eth_dev->data->port_id);
1553655c3c26SDmitry Kozlyuk 	/* Read link status in case it is up and there will be no event. */
15542eb4d010SOphir Munk 	mlx5_link_update(eth_dev, 0);
1555655c3c26SDmitry Kozlyuk 	/* Watch LSC interrupts between port probe and port start. */
1556655c3c26SDmitry Kozlyuk 	priv->sh->port[priv->dev_port - 1].nl_ih_port_id =
1557655c3c26SDmitry Kozlyuk 							eth_dev->data->port_id;
1558655c3c26SDmitry Kozlyuk 	mlx5_set_link_up(eth_dev);
1559b4edeaf3SSuanming Mou 	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
1560a13ec19cSMichael Baum 		icfg[i].release_mem_en = !!sh->config.reclaim_mode;
1561a13ec19cSMichael Baum 		if (sh->config.reclaim_mode)
1562b4edeaf3SSuanming Mou 			icfg[i].per_core_cache = 0;
1563b4edeaf3SSuanming Mou 		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
1564b4edeaf3SSuanming Mou 		if (!priv->flows[i])
1565b4edeaf3SSuanming Mou 			goto error;
1566b4edeaf3SSuanming Mou 	}
15672eb4d010SOphir Munk 	/* Create context for virtual machine VLAN workaround. */
15682eb4d010SOphir Munk 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
1569a13ec19cSMichael Baum 	if (sh->config.dv_flow_en) {
15702eb4d010SOphir Munk 		err = mlx5_alloc_shared_dr(priv);
15712eb4d010SOphir Munk 		if (err)
15722eb4d010SOphir Munk 			goto error;
1573db25cadcSViacheslav Ovsiienko 		if (mlx5_flex_item_port_init(eth_dev) < 0)
1574db25cadcSViacheslav Ovsiienko 			goto error;
15752eb4d010SOphir Munk 	}
1576c4b86201SMichael Baum 	if (mlx5_devx_obj_ops_en(sh)) {
15775eaf882eSMichael Baum 		priv->obj_ops = devx_obj_ops;
1578e6988afdSMatan Azrad 		mlx5_queue_counter_id_prepare(eth_dev);
157923233fd6SBing Zhao 		priv->obj_ops.lb_dummy_queue_create =
158023233fd6SBing Zhao 					mlx5_rxq_ibv_obj_dummy_lb_create;
158123233fd6SBing Zhao 		priv->obj_ops.lb_dummy_queue_release =
158223233fd6SBing Zhao 					mlx5_rxq_ibv_obj_dummy_lb_release;
1583614966c2SXueming Li 	} else if (spawn->max_port > UINT8_MAX) {
1584614966c2SXueming Li 		/* Verbs can't support ports larger than 255 by design. */
1585614966c2SXueming Li 		DRV_LOG(ERR, "must enable DV and ESW when RDMA link ports > 255");
1586614966c2SXueming Li 		err = ENOTSUP;
1587614966c2SXueming Li 		goto error;
15885eaf882eSMichael Baum 	} else {
15895eaf882eSMichael Baum 		priv->obj_ops = ibv_obj_ops;
15905eaf882eSMichael Baum 	}
1591a13ec19cSMichael Baum 	if (sh->config.tx_pp &&
159211cfe349SViacheslav Ovsiienko 	    priv->obj_ops.txq_obj_new != mlx5_txq_devx_obj_new) {
1593f17e4b4fSViacheslav Ovsiienko 		/*
1594f17e4b4fSViacheslav Ovsiienko 		 * HAVE_MLX5DV_DEVX_UAR_OFFSET is required to support
1595f17e4b4fSViacheslav Ovsiienko 		 * packet pacing and already checked above.
1596f17e4b4fSViacheslav Ovsiienko 		 * Hence, we should only make sure the SQs will be created
1597f17e4b4fSViacheslav Ovsiienko 		 * with DevX, not with Verbs.
1598f17e4b4fSViacheslav Ovsiienko 		 * Verbs allocates the SQ UAR on its own and it can't be shared
1599f17e4b4fSViacheslav Ovsiienko 		 * with Clock Queue UAR as required for Tx scheduling.
1600f17e4b4fSViacheslav Ovsiienko 		 */
1601f17e4b4fSViacheslav Ovsiienko 		DRV_LOG(ERR, "Verbs SQs, UAR can't be shared as required for packet pacing");
1602f17e4b4fSViacheslav Ovsiienko 		err = ENODEV;
1603f17e4b4fSViacheslav Ovsiienko 		goto error;
1604f17e4b4fSViacheslav Ovsiienko 	}
160565b3cd0dSSuanming Mou 	priv->drop_queue.hrxq = mlx5_drop_action_create(eth_dev);
160665b3cd0dSSuanming Mou 	if (!priv->drop_queue.hrxq)
160765b3cd0dSSuanming Mou 		goto error;
16083a2f674bSSuanming Mou 	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
16093a2f674bSSuanming Mou 				       mlx5_hrxq_create_cb,
16103a2f674bSSuanming Mou 				       mlx5_hrxq_match_cb,
16113a2f674bSSuanming Mou 				       mlx5_hrxq_remove_cb,
16123a2f674bSSuanming Mou 				       mlx5_hrxq_clone_cb,
16133a2f674bSSuanming Mou 				       mlx5_hrxq_clone_free_cb);
16143a2f674bSSuanming Mou 	if (!priv->hrxqs)
16153a2f674bSSuanming Mou 		goto error;
16160f4aa72bSSuanming Mou 	mlx5_set_metadata_mask(eth_dev);
16170f4aa72bSSuanming Mou 	if (sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
16180f4aa72bSSuanming Mou 	    !priv->sh->dv_regc0_mask) {
16190f4aa72bSSuanming Mou 		DRV_LOG(ERR, "metadata mode %u is not supported "
16200f4aa72bSSuanming Mou 			     "(no metadata reg_c[0] is available)",
16210f4aa72bSSuanming Mou 			     sh->config.dv_xmeta_en);
16220f4aa72bSSuanming Mou 			err = ENOTSUP;
16230f4aa72bSSuanming Mou 			goto error;
16240f4aa72bSSuanming Mou 	}
16253a2f674bSSuanming Mou 	rte_rwlock_init(&priv->ind_tbls_lock);
1626edc80bbfSGavin Li 	if (sh->phdev->config.ipv6_tc_fallback == MLX5_IPV6_TC_UNKNOWN) {
16272c2856f7SGavin Li 		sh->phdev->config.ipv6_tc_fallback = MLX5_IPV6_TC_OK;
1628edc80bbfSGavin Li 		if (!sh->cdev->config.hca_attr.modify_outer_ipv6_traffic_class ||
1629edc80bbfSGavin Li 		    (sh->config.dv_flow_en == 1 && mlx5_flow_discover_ipv6_tc_support(eth_dev)))
1630edc80bbfSGavin Li 			sh->phdev->config.ipv6_tc_fallback = MLX5_IPV6_TC_FALLBACK;
1631edc80bbfSGavin Li 	}
16325bd0e3e6SDariusz Sosnowski 	if (priv->sh->config.dv_flow_en == 2) {
16331939eb6fSDariusz Sosnowski #ifdef HAVE_MLX5_HWS_SUPPORT
163426e1eaf2SDariusz Sosnowski 		if (priv->sh->config.dv_esw_en) {
1635483181f7SDariusz Sosnowski 			uint32_t usable_bits;
1636483181f7SDariusz Sosnowski 			uint32_t required_bits;
1637483181f7SDariusz Sosnowski 
163826e1eaf2SDariusz Sosnowski 			if (priv->sh->dv_regc0_mask == UINT32_MAX) {
163926e1eaf2SDariusz Sosnowski 				DRV_LOG(ERR, "E-Switch port metadata is required when using HWS "
164026e1eaf2SDariusz Sosnowski 					     "but it is disabled (configure it through devlink)");
164126e1eaf2SDariusz Sosnowski 				err = ENOTSUP;
164226e1eaf2SDariusz Sosnowski 				goto error;
164326e1eaf2SDariusz Sosnowski 			}
164426e1eaf2SDariusz Sosnowski 			if (priv->sh->dv_regc0_mask == 0) {
164526e1eaf2SDariusz Sosnowski 				DRV_LOG(ERR, "E-Switch with HWS is not supported "
164626e1eaf2SDariusz Sosnowski 					     "(no available bits in reg_c[0])");
164726e1eaf2SDariusz Sosnowski 				err = ENOTSUP;
164826e1eaf2SDariusz Sosnowski 				goto error;
164926e1eaf2SDariusz Sosnowski 			}
16503d4e27fdSDavid Marchand 			usable_bits = rte_popcount32(priv->sh->dv_regc0_mask);
16513d4e27fdSDavid Marchand 			required_bits = rte_popcount32(priv->vport_meta_mask);
1652483181f7SDariusz Sosnowski 			if (usable_bits < required_bits) {
1653483181f7SDariusz Sosnowski 				DRV_LOG(ERR, "Not enough bits available in reg_c[0] to provide "
1654483181f7SDariusz Sosnowski 					     "representor matching.");
1655483181f7SDariusz Sosnowski 				err = ENOTSUP;
1656483181f7SDariusz Sosnowski 				goto error;
1657483181f7SDariusz Sosnowski 			}
165826e1eaf2SDariusz Sosnowski 		}
16595bd0e3e6SDariusz Sosnowski 		if (priv->vport_meta_mask)
16605bd0e3e6SDariusz Sosnowski 			flow_hw_set_port_info(eth_dev);
1661ddb68e47SBing Zhao 		if (priv->sh->config.dv_esw_en &&
1662ddb68e47SBing Zhao 		    priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1663ddb68e47SBing Zhao 		    priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) {
1664ddb68e47SBing Zhao 			DRV_LOG(ERR,
1665ddb68e47SBing Zhao 				"metadata mode %u is not supported in HWS eswitch mode",
1666ddb68e47SBing Zhao 				priv->sh->config.dv_xmeta_en);
1667ddb68e47SBing Zhao 				err = ENOTSUP;
1668ddb68e47SBing Zhao 				goto error;
1669ddb68e47SBing Zhao 		}
16701939eb6fSDariusz Sosnowski 		if (priv->sh->config.dv_esw_en &&
16711939eb6fSDariusz Sosnowski 		    flow_hw_create_vport_action(eth_dev)) {
16721939eb6fSDariusz Sosnowski 			DRV_LOG(ERR, "port %u failed to create vport action",
16731939eb6fSDariusz Sosnowski 				eth_dev->data->port_id);
16741939eb6fSDariusz Sosnowski 			err = EINVAL;
16751939eb6fSDariusz Sosnowski 			goto error;
16761939eb6fSDariusz Sosnowski 		}
1677042f52ddSDariusz Sosnowski 		/*
1678042f52ddSDariusz Sosnowski 		 * If representor matching is disabled, PMD cannot create default flow rules
1679042f52ddSDariusz Sosnowski 		 * to receive traffic for all ports, since implicit source port match is not added.
1680042f52ddSDariusz Sosnowski 		 * Isolated mode is forced.
1681042f52ddSDariusz Sosnowski 		 */
1682042f52ddSDariusz Sosnowski 		if (priv->sh->config.dv_esw_en && !priv->sh->config.repr_matching) {
1683042f52ddSDariusz Sosnowski 			err = mlx5_flow_isolate(eth_dev, 1, NULL);
1684042f52ddSDariusz Sosnowski 			if (err < 0) {
1685042f52ddSDariusz Sosnowski 				err = -err;
1686042f52ddSDariusz Sosnowski 				goto error;
1687042f52ddSDariusz Sosnowski 			}
1688042f52ddSDariusz Sosnowski 			DRV_LOG(WARNING, "port %u ingress traffic is restricted to defined "
1689042f52ddSDariusz Sosnowski 					 "flow rules (isolated mode) since representor "
1690042f52ddSDariusz Sosnowski 					 "matching is disabled",
1691042f52ddSDariusz Sosnowski 				eth_dev->data->port_id);
1692042f52ddSDariusz Sosnowski 		}
1693df26aa6eSDariusz Sosnowski 		eth_dev->data->dev_flags |= RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE;
1694d84c3cf7SSuanming Mou 		return eth_dev;
16955bd0e3e6SDariusz Sosnowski #else
16965bd0e3e6SDariusz Sosnowski 		DRV_LOG(ERR, "DV support is missing for HWS.");
16975bd0e3e6SDariusz Sosnowski 		goto error;
16985bd0e3e6SDariusz Sosnowski #endif
16995bd0e3e6SDariusz Sosnowski 	}
17003c4338a4SJiawei Wang 	if (!priv->sh->flow_priority_check_flag) {
17012eb4d010SOphir Munk 		/* Supported Verbs flow priority number detection. */
17022eb4d010SOphir Munk 		err = mlx5_flow_discover_priorities(eth_dev);
17033c4338a4SJiawei Wang 		priv->sh->flow_max_priority = err;
17043c4338a4SJiawei Wang 		priv->sh->flow_priority_check_flag = 1;
17053c4338a4SJiawei Wang 	} else {
17063c4338a4SJiawei Wang 		err = priv->sh->flow_max_priority;
17073c4338a4SJiawei Wang 	}
17082eb4d010SOphir Munk 	if (err < 0) {
17092eb4d010SOphir Munk 		err = -err;
17102eb4d010SOphir Munk 		goto error;
17112eb4d010SOphir Munk 	}
17122eb4d010SOphir Munk 	/* Query availability of metadata reg_c's. */
17133c4338a4SJiawei Wang 	if (!priv->sh->metadata_regc_check_flag) {
17142eb4d010SOphir Munk 		err = mlx5_flow_discover_mreg_c(eth_dev);
17152eb4d010SOphir Munk 		if (err < 0) {
17162eb4d010SOphir Munk 			err = -err;
17172eb4d010SOphir Munk 			goto error;
17182eb4d010SOphir Munk 		}
17193c4338a4SJiawei Wang 	}
17202eb4d010SOphir Munk 	if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
17212eb4d010SOphir Munk 		DRV_LOG(DEBUG,
17222eb4d010SOphir Munk 			"port %u extensive metadata register is not supported",
17232eb4d010SOphir Munk 			eth_dev->data->port_id);
1724a13ec19cSMichael Baum 		if (sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
17252eb4d010SOphir Munk 			DRV_LOG(ERR, "metadata mode %u is not supported "
17262eb4d010SOphir Munk 				     "(no metadata registers available)",
1727a13ec19cSMichael Baum 				     sh->config.dv_xmeta_en);
17282eb4d010SOphir Munk 			err = ENOTSUP;
17292eb4d010SOphir Munk 			goto error;
17302eb4d010SOphir Munk 		}
17312eb4d010SOphir Munk 	}
1732a13ec19cSMichael Baum 	if (sh->config.dv_flow_en &&
1733a13ec19cSMichael Baum 	    sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
17342eb4d010SOphir Munk 	    mlx5_flow_ext_mreg_supported(eth_dev) &&
17352eb4d010SOphir Munk 	    priv->sh->dv_regc0_mask) {
17362eb4d010SOphir Munk 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
1737e69a5922SXueming Li 						      MLX5_FLOW_MREG_HTABLE_SZ,
1738961b6774SMatan Azrad 						      false, true, eth_dev,
1739f7f73ac1SXueming Li 						      flow_dv_mreg_create_cb,
1740f5b0aed2SSuanming Mou 						      flow_dv_mreg_match_cb,
1741961b6774SMatan Azrad 						      flow_dv_mreg_remove_cb,
1742961b6774SMatan Azrad 						      flow_dv_mreg_clone_cb,
1743961b6774SMatan Azrad 						    flow_dv_mreg_clone_free_cb);
17442eb4d010SOphir Munk 		if (!priv->mreg_cp_tbl) {
17452eb4d010SOphir Munk 			err = ENOMEM;
17462eb4d010SOphir Munk 			goto error;
17472eb4d010SOphir Munk 		}
17482eb4d010SOphir Munk 	}
1749cc608e4dSSuanming Mou 	rte_spinlock_init(&priv->shared_act_sl);
1750994829e6SSuanming Mou 	mlx5_flow_counter_mode_config(eth_dev);
175145633c46SSuanming Mou 	mlx5_flow_drop_action_config(eth_dev);
1752a13ec19cSMichael Baum 	if (sh->config.dv_flow_en)
17539fbe97f0SXueming Li 		eth_dev->data->dev_flags |= RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE;
17542eb4d010SOphir Munk 	return eth_dev;
17552eb4d010SOphir Munk error:
17562eb4d010SOphir Munk 	if (priv) {
175713c5c093SMichael Baum 		priv->sh->port[priv->dev_port - 1].nl_ih_port_id =
175813c5c093SMichael Baum 							       RTE_MAX_ETHPORTS;
175913c5c093SMichael Baum 		rte_io_wmb();
17601939eb6fSDariusz Sosnowski #ifdef HAVE_MLX5_HWS_SUPPORT
17611939eb6fSDariusz Sosnowski 		if (eth_dev &&
17621939eb6fSDariusz Sosnowski 		    priv->sh &&
17631939eb6fSDariusz Sosnowski 		    priv->sh->config.dv_flow_en == 2 &&
17641939eb6fSDariusz Sosnowski 		    priv->sh->config.dv_esw_en)
17651939eb6fSDariusz Sosnowski 			flow_hw_destroy_vport_action(eth_dev);
17661939eb6fSDariusz Sosnowski #endif
17672eb4d010SOphir Munk 		if (priv->mreg_cp_tbl)
1768e69a5922SXueming Li 			mlx5_hlist_destroy(priv->mreg_cp_tbl);
17692eb4d010SOphir Munk 		if (priv->sh)
17702eb4d010SOphir Munk 			mlx5_os_free_shared_dr(priv);
17712eb4d010SOphir Munk 		if (priv->nl_socket_route >= 0)
17722eb4d010SOphir Munk 			close(priv->nl_socket_route);
17732eb4d010SOphir Munk 		if (priv->vmwa_context)
17742eb4d010SOphir Munk 			mlx5_vlan_vmwa_exit(priv->vmwa_context);
177565b3cd0dSSuanming Mou 		if (eth_dev && priv->drop_queue.hrxq)
177665b3cd0dSSuanming Mou 			mlx5_drop_action_destroy(eth_dev);
1777a295c69aSShun Hao 		if (priv->mtr_profile_tbl)
1778a295c69aSShun Hao 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
17792eb4d010SOphir Munk 		if (own_domain_id)
17802eb4d010SOphir Munk 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
1781679f46c7SMatan Azrad 		if (priv->hrxqs)
1782679f46c7SMatan Azrad 			mlx5_list_destroy(priv->hrxqs);
1783db25cadcSViacheslav Ovsiienko 		if (eth_dev && priv->flex_item_map)
1784db25cadcSViacheslav Ovsiienko 			mlx5_flex_item_port_cleanup(eth_dev);
178580f872eeSMichael Baum 		mlx5_free(priv->ext_rxqs);
17861944fbc3SSuanming Mou 		mlx5_free(priv->ext_txqs);
17872175c4dcSSuanming Mou 		mlx5_free(priv);
17882eb4d010SOphir Munk 		if (eth_dev != NULL)
17892eb4d010SOphir Munk 			eth_dev->data->dev_private = NULL;
17902eb4d010SOphir Munk 	}
17912eb4d010SOphir Munk 	if (eth_dev != NULL) {
17922eb4d010SOphir Munk 		/* mac_addrs must not be freed alone because part of
17932eb4d010SOphir Munk 		 * dev_private
17942eb4d010SOphir Munk 		 **/
17952eb4d010SOphir Munk 		eth_dev->data->mac_addrs = NULL;
17962eb4d010SOphir Munk 		rte_eth_dev_release_port(eth_dev);
17972eb4d010SOphir Munk 	}
17982eb4d010SOphir Munk 	if (sh)
179991389890SOphir Munk 		mlx5_free_shared_dev_ctx(sh);
18003fd2961eSXueming Li 	if (nl_rdma >= 0)
18013fd2961eSXueming Li 		close(nl_rdma);
18022eb4d010SOphir Munk 	MLX5_ASSERT(err > 0);
18032eb4d010SOphir Munk 	rte_errno = err;
18042eb4d010SOphir Munk 	return NULL;
18052eb4d010SOphir Munk }
18062eb4d010SOphir Munk 
18072eb4d010SOphir Munk /**
18082eb4d010SOphir Munk  * Comparison callback to sort device data.
18092eb4d010SOphir Munk  *
18102eb4d010SOphir Munk  * This is meant to be used with qsort().
18112eb4d010SOphir Munk  *
18122eb4d010SOphir Munk  * @param a[in]
18132eb4d010SOphir Munk  *   Pointer to pointer to first data object.
18142eb4d010SOphir Munk  * @param b[in]
18152eb4d010SOphir Munk  *   Pointer to pointer to second data object.
18162eb4d010SOphir Munk  *
18172eb4d010SOphir Munk  * @return
18182eb4d010SOphir Munk  *   0 if both objects are equal, less than 0 if the first argument is less
18192eb4d010SOphir Munk  *   than the second, greater than 0 otherwise.
18202eb4d010SOphir Munk  */
18212eb4d010SOphir Munk static int
18222eb4d010SOphir Munk mlx5_dev_spawn_data_cmp(const void *a, const void *b)
18232eb4d010SOphir Munk {
18242eb4d010SOphir Munk 	const struct mlx5_switch_info *si_a =
18252eb4d010SOphir Munk 		&((const struct mlx5_dev_spawn_data *)a)->info;
18262eb4d010SOphir Munk 	const struct mlx5_switch_info *si_b =
18272eb4d010SOphir Munk 		&((const struct mlx5_dev_spawn_data *)b)->info;
18282e163295SDariusz Sosnowski 	int uplink_a = si_a->name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK;
18292e163295SDariusz Sosnowski 	int uplink_b = si_b->name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK;
18302eb4d010SOphir Munk 	int ret;
18312eb4d010SOphir Munk 
18322e163295SDariusz Sosnowski 	/* Uplink ports first. */
18332e163295SDariusz Sosnowski 	ret = uplink_b - uplink_a;
18342e163295SDariusz Sosnowski 	if (ret)
18352e163295SDariusz Sosnowski 		return ret;
18362e163295SDariusz Sosnowski 	/* Then master devices. */
18372eb4d010SOphir Munk 	ret = si_b->master - si_a->master;
18382eb4d010SOphir Munk 	if (ret)
18392eb4d010SOphir Munk 		return ret;
18402eb4d010SOphir Munk 	/* Then representor devices. */
18412eb4d010SOphir Munk 	ret = si_b->representor - si_a->representor;
18422eb4d010SOphir Munk 	if (ret)
18432eb4d010SOphir Munk 		return ret;
18442eb4d010SOphir Munk 	/* Unidentified devices come last in no specific order. */
18452eb4d010SOphir Munk 	if (!si_a->representor)
18462eb4d010SOphir Munk 		return 0;
18472eb4d010SOphir Munk 	/* Order representors by name. */
18482eb4d010SOphir Munk 	return si_a->port_name - si_b->port_name;
18492eb4d010SOphir Munk }
18502eb4d010SOphir Munk 
18512eb4d010SOphir Munk /**
18522eb4d010SOphir Munk  * Match PCI information for possible slaves of bonding device.
18532eb4d010SOphir Munk  *
1854ca1418ceSMichael Baum  * @param[in] ibdev_name
1855ca1418ceSMichael Baum  *   Name of Infiniband device.
18562eb4d010SOphir Munk  * @param[in] pci_dev
1857f926cce3SXueming Li  *   Pointer to primary PCI address structure to match.
18582eb4d010SOphir Munk  * @param[in] nl_rdma
18592eb4d010SOphir Munk  *   Netlink RDMA group socket handle.
1860f926cce3SXueming Li  * @param[in] owner
1861ca1418ceSMichael Baum  *   Representor owner PF index.
1862f5f4c482SXueming Li  * @param[out] bond_info
1863f5f4c482SXueming Li  *   Pointer to bonding information.
18642eb4d010SOphir Munk  *
18652eb4d010SOphir Munk  * @return
18662eb4d010SOphir Munk  *   negative value if no bonding device found, otherwise
18672eb4d010SOphir Munk  *   positive index of slave PF in bonding.
18682eb4d010SOphir Munk  */
18692eb4d010SOphir Munk static int
1870ca1418ceSMichael Baum mlx5_device_bond_pci_match(const char *ibdev_name,
1871f926cce3SXueming Li 			   const struct rte_pci_addr *pci_dev,
1872f5f4c482SXueming Li 			   int nl_rdma, uint16_t owner,
1873f5f4c482SXueming Li 			   struct mlx5_bond_info *bond_info)
18742eb4d010SOphir Munk {
18752eb4d010SOphir Munk 	char ifname[IF_NAMESIZE + 1];
18762eb4d010SOphir Munk 	unsigned int ifindex;
18772eb4d010SOphir Munk 	unsigned int np, i;
1878f5f4c482SXueming Li 	FILE *bond_file = NULL, *file;
18792eb4d010SOphir Munk 	int pf = -1;
1880f5f4c482SXueming Li 	int ret;
18817299ab68SRongwei Liu 	uint8_t cur_guid[32] = {0};
18827299ab68SRongwei Liu 	uint8_t guid[32] = {0};
18832eb4d010SOphir Munk 
18842eb4d010SOphir Munk 	/*
1885ca1418ceSMichael Baum 	 * Try to get master device name. If something goes wrong suppose
1886ca1418ceSMichael Baum 	 * the lack of kernel support and no bonding devices.
18872eb4d010SOphir Munk 	 */
1888f5f4c482SXueming Li 	memset(bond_info, 0, sizeof(*bond_info));
18892eb4d010SOphir Munk 	if (nl_rdma < 0)
18902eb4d010SOphir Munk 		return -1;
1891ca1418ceSMichael Baum 	if (!strstr(ibdev_name, "bond"))
18922eb4d010SOphir Munk 		return -1;
1893ca1418ceSMichael Baum 	np = mlx5_nl_portnum(nl_rdma, ibdev_name);
18942eb4d010SOphir Munk 	if (!np)
18952eb4d010SOphir Munk 		return -1;
18967299ab68SRongwei Liu 	if (mlx5_get_device_guid(pci_dev, cur_guid, sizeof(cur_guid)) < 0)
18977299ab68SRongwei Liu 		return -1;
18982eb4d010SOphir Munk 	/*
1899ca1418ceSMichael Baum 	 * The master device might not be on the predefined port(not on port
1900ca1418ceSMichael Baum 	 * index 1, it is not guaranteed), we have to scan all Infiniband
1901ca1418ceSMichael Baum 	 * device ports and find master.
19022eb4d010SOphir Munk 	 */
19032eb4d010SOphir Munk 	for (i = 1; i <= np; ++i) {
19042eb4d010SOphir Munk 		/* Check whether Infiniband port is populated. */
1905ca1418ceSMichael Baum 		ifindex = mlx5_nl_ifindex(nl_rdma, ibdev_name, i);
19062eb4d010SOphir Munk 		if (!ifindex)
19072eb4d010SOphir Munk 			continue;
19082eb4d010SOphir Munk 		if (!if_indextoname(ifindex, ifname))
19092eb4d010SOphir Munk 			continue;
19102eb4d010SOphir Munk 		/* Try to read bonding slave names from sysfs. */
19112eb4d010SOphir Munk 		MKSTR(slaves,
19122eb4d010SOphir Munk 		      "/sys/class/net/%s/master/bonding/slaves", ifname);
1913f5f4c482SXueming Li 		bond_file = fopen(slaves, "r");
1914f5f4c482SXueming Li 		if (bond_file)
19152eb4d010SOphir Munk 			break;
19162eb4d010SOphir Munk 	}
1917f5f4c482SXueming Li 	if (!bond_file)
19182eb4d010SOphir Munk 		return -1;
19192eb4d010SOphir Munk 	/* Use safe format to check maximal buffer length. */
19202eb4d010SOphir Munk 	MLX5_ASSERT(atol(RTE_STR(IF_NAMESIZE)) == IF_NAMESIZE);
1921f5f4c482SXueming Li 	while (fscanf(bond_file, "%" RTE_STR(IF_NAMESIZE) "s", ifname) == 1) {
19222eb4d010SOphir Munk 		char tmp_str[IF_NAMESIZE + 32];
19232eb4d010SOphir Munk 		struct rte_pci_addr pci_addr;
19242eb4d010SOphir Munk 		struct mlx5_switch_info	info;
19257299ab68SRongwei Liu 		int ret;
19262eb4d010SOphir Munk 
19272eb4d010SOphir Munk 		/* Process slave interface names in the loop. */
19282eb4d010SOphir Munk 		snprintf(tmp_str, sizeof(tmp_str),
19292eb4d010SOphir Munk 			 "/sys/class/net/%s", ifname);
19304d567938SThomas Monjalon 		if (mlx5_get_pci_addr(tmp_str, &pci_addr)) {
1931ca1418ceSMichael Baum 			DRV_LOG(WARNING,
1932ca1418ceSMichael Baum 				"Cannot get PCI address for netdev \"%s\".",
1933ca1418ceSMichael Baum 				ifname);
19342eb4d010SOphir Munk 			continue;
19352eb4d010SOphir Munk 		}
19362eb4d010SOphir Munk 		/* Slave interface PCI address match found. */
19372eb4d010SOphir Munk 		snprintf(tmp_str, sizeof(tmp_str),
19382eb4d010SOphir Munk 			 "/sys/class/net/%s/phys_port_name", ifname);
19392eb4d010SOphir Munk 		file = fopen(tmp_str, "rb");
19402eb4d010SOphir Munk 		if (!file)
19412eb4d010SOphir Munk 			break;
19422eb4d010SOphir Munk 		info.name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET;
19432eb4d010SOphir Munk 		if (fscanf(file, "%32s", tmp_str) == 1)
19442eb4d010SOphir Munk 			mlx5_translate_port_name(tmp_str, &info);
1945f5f4c482SXueming Li 		fclose(file);
1946f5f4c482SXueming Li 		/* Only process PF ports. */
1947f5f4c482SXueming Li 		if (info.name_type != MLX5_PHYS_PORT_NAME_TYPE_LEGACY &&
1948f5f4c482SXueming Li 		    info.name_type != MLX5_PHYS_PORT_NAME_TYPE_UPLINK)
1949f5f4c482SXueming Li 			continue;
1950f5f4c482SXueming Li 		/* Check max bonding member. */
1951f5f4c482SXueming Li 		if (info.port_name >= MLX5_BOND_MAX_PORTS) {
1952f5f4c482SXueming Li 			DRV_LOG(WARNING, "bonding index out of range, "
1953f5f4c482SXueming Li 				"please increase MLX5_BOND_MAX_PORTS: %s",
1954f5f4c482SXueming Li 				tmp_str);
19552eb4d010SOphir Munk 			break;
19562eb4d010SOphir Munk 		}
1957f5f4c482SXueming Li 		/* Get ifindex. */
1958f5f4c482SXueming Li 		snprintf(tmp_str, sizeof(tmp_str),
1959f5f4c482SXueming Li 			 "/sys/class/net/%s/ifindex", ifname);
1960f5f4c482SXueming Li 		file = fopen(tmp_str, "rb");
1961f5f4c482SXueming Li 		if (!file)
1962f5f4c482SXueming Li 			break;
1963f5f4c482SXueming Li 		ret = fscanf(file, "%u", &ifindex);
19642eb4d010SOphir Munk 		fclose(file);
1965f5f4c482SXueming Li 		if (ret != 1)
1966f5f4c482SXueming Li 			break;
1967f5f4c482SXueming Li 		/* Save bonding info. */
1968f5f4c482SXueming Li 		strncpy(bond_info->ports[info.port_name].ifname, ifname,
1969f5f4c482SXueming Li 			sizeof(bond_info->ports[0].ifname));
1970f5f4c482SXueming Li 		bond_info->ports[info.port_name].pci_addr = pci_addr;
1971f5f4c482SXueming Li 		bond_info->ports[info.port_name].ifindex = ifindex;
1972f5f4c482SXueming Li 		bond_info->n_port++;
19737299ab68SRongwei Liu 		/*
19747299ab68SRongwei Liu 		 * Under socket direct mode, bonding will use
19757299ab68SRongwei Liu 		 * system_image_guid as identification.
19767299ab68SRongwei Liu 		 * After OFED 5.4, guid is readable (ret >= 0) under sysfs.
19777299ab68SRongwei Liu 		 * All bonding members should have the same guid even if driver
19787299ab68SRongwei Liu 		 * is using PCIe BDF.
19797299ab68SRongwei Liu 		 */
19807299ab68SRongwei Liu 		ret = mlx5_get_device_guid(&pci_addr, guid, sizeof(guid));
19817299ab68SRongwei Liu 		if (ret < 0)
19827299ab68SRongwei Liu 			break;
19837299ab68SRongwei Liu 		else if (ret > 0) {
19847299ab68SRongwei Liu 			if (!memcmp(guid, cur_guid, sizeof(guid)) &&
19857299ab68SRongwei Liu 			    owner == info.port_name &&
19867299ab68SRongwei Liu 			    (owner != 0 || (owner == 0 &&
19877299ab68SRongwei Liu 			    !rte_pci_addr_cmp(pci_dev, &pci_addr))))
19887299ab68SRongwei Liu 				pf = info.port_name;
19897299ab68SRongwei Liu 		} else if (pci_dev->domain == pci_addr.domain &&
19907299ab68SRongwei Liu 		    pci_dev->bus == pci_addr.bus &&
19917299ab68SRongwei Liu 		    pci_dev->devid == pci_addr.devid &&
19927299ab68SRongwei Liu 		    ((pci_dev->function == 0 &&
19937299ab68SRongwei Liu 		      pci_dev->function + owner == pci_addr.function) ||
19947299ab68SRongwei Liu 		     (pci_dev->function == owner &&
19957299ab68SRongwei Liu 		      pci_addr.function == owner)))
19967299ab68SRongwei Liu 			pf = info.port_name;
1997f5f4c482SXueming Li 	}
1998f5f4c482SXueming Li 	if (pf >= 0) {
1999f5f4c482SXueming Li 		/* Get bond interface info */
2000f5f4c482SXueming Li 		ret = mlx5_sysfs_bond_info(ifindex, &bond_info->ifindex,
2001f5f4c482SXueming Li 					   bond_info->ifname);
2002f5f4c482SXueming Li 		if (ret)
2003f5f4c482SXueming Li 			DRV_LOG(ERR, "unable to get bond info: %s",
2004f5f4c482SXueming Li 				strerror(rte_errno));
2005f5f4c482SXueming Li 		else
2006f5f4c482SXueming Li 			DRV_LOG(INFO, "PF device %u, bond device %u(%s)",
2007f5f4c482SXueming Li 				ifindex, bond_info->ifindex, bond_info->ifname);
2008f5f4c482SXueming Li 	}
20097299ab68SRongwei Liu 	if (owner == 0 && pf != 0) {
20102fc03b23SThomas Monjalon 		DRV_LOG(INFO, "PCIe instance " PCI_PRI_FMT " isn't bonding owner",
20117299ab68SRongwei Liu 				pci_dev->domain, pci_dev->bus, pci_dev->devid,
20127299ab68SRongwei Liu 				pci_dev->function);
20137299ab68SRongwei Liu 	}
20142eb4d010SOphir Munk 	return pf;
20152eb4d010SOphir Munk }
20162eb4d010SOphir Munk 
2017bb2fee72SDariusz Sosnowski static int
2018bb2fee72SDariusz Sosnowski mlx5_nl_esw_multiport_get(struct rte_pci_addr *pci_addr, int *enabled)
2019bb2fee72SDariusz Sosnowski {
2020bb2fee72SDariusz Sosnowski 	char pci_addr_str[PCI_PRI_STR_SIZE] = { 0 };
2021bb2fee72SDariusz Sosnowski 	int nlsk_fd;
2022bb2fee72SDariusz Sosnowski 	int devlink_id;
2023bb2fee72SDariusz Sosnowski 	int ret;
2024bb2fee72SDariusz Sosnowski 
2025bb2fee72SDariusz Sosnowski 	/* Provide correct value to have defined enabled state in case of an error. */
2026bb2fee72SDariusz Sosnowski 	*enabled = 0;
2027bb2fee72SDariusz Sosnowski 	rte_pci_device_name(pci_addr, pci_addr_str, sizeof(pci_addr_str));
2028bb2fee72SDariusz Sosnowski 	nlsk_fd = mlx5_nl_init(NETLINK_GENERIC, 0);
2029bb2fee72SDariusz Sosnowski 	if (nlsk_fd < 0)
2030bb2fee72SDariusz Sosnowski 		return nlsk_fd;
2031bb2fee72SDariusz Sosnowski 	devlink_id = mlx5_nl_devlink_family_id_get(nlsk_fd);
2032bb2fee72SDariusz Sosnowski 	if (devlink_id < 0) {
2033bb2fee72SDariusz Sosnowski 		ret = devlink_id;
2034bb2fee72SDariusz Sosnowski 		DRV_LOG(DEBUG, "Unable to get devlink family id for Multiport E-Switch checks "
2035bb2fee72SDariusz Sosnowski 			       "by netlink, for PCI device %s", pci_addr_str);
2036bb2fee72SDariusz Sosnowski 		goto close_nlsk_fd;
2037bb2fee72SDariusz Sosnowski 	}
2038bb2fee72SDariusz Sosnowski 	ret = mlx5_nl_devlink_esw_multiport_get(nlsk_fd, devlink_id, pci_addr_str, enabled);
2039bb2fee72SDariusz Sosnowski 	if (ret < 0)
2040bb2fee72SDariusz Sosnowski 		DRV_LOG(DEBUG, "Unable to get Multiport E-Switch state by Netlink.");
2041bb2fee72SDariusz Sosnowski close_nlsk_fd:
2042bb2fee72SDariusz Sosnowski 	close(nlsk_fd);
2043bb2fee72SDariusz Sosnowski 	return ret;
2044bb2fee72SDariusz Sosnowski }
2045bb2fee72SDariusz Sosnowski 
2046b62f0485SDariusz Sosnowski #define SYSFS_MPESW_PARAM_MAX_LEN 16
2047b62f0485SDariusz Sosnowski 
2048bb2fee72SDariusz Sosnowski static int
2049b62f0485SDariusz Sosnowski mlx5_sysfs_esw_multiport_get(struct ibv_device *ibv, struct rte_pci_addr *pci_addr, int *enabled)
2050b62f0485SDariusz Sosnowski {
2051b62f0485SDariusz Sosnowski 	int nl_rdma;
2052b62f0485SDariusz Sosnowski 	unsigned int n_ports;
2053b62f0485SDariusz Sosnowski 	unsigned int i;
2054b62f0485SDariusz Sosnowski 	int ret;
2055b62f0485SDariusz Sosnowski 
2056b62f0485SDariusz Sosnowski 	/* Provide correct value to have defined enabled state in case of an error. */
2057b62f0485SDariusz Sosnowski 	*enabled = 0;
2058b62f0485SDariusz Sosnowski 	nl_rdma = mlx5_nl_init(NETLINK_RDMA, 0);
2059b62f0485SDariusz Sosnowski 	if (nl_rdma < 0)
2060b62f0485SDariusz Sosnowski 		return nl_rdma;
2061b62f0485SDariusz Sosnowski 	n_ports = mlx5_nl_portnum(nl_rdma, ibv->name);
2062b62f0485SDariusz Sosnowski 	if (!n_ports) {
2063b62f0485SDariusz Sosnowski 		ret = -rte_errno;
2064b62f0485SDariusz Sosnowski 		goto close_nl_rdma;
2065b62f0485SDariusz Sosnowski 	}
2066b62f0485SDariusz Sosnowski 	for (i = 1; i <= n_ports; ++i) {
2067b62f0485SDariusz Sosnowski 		unsigned int ifindex;
2068b62f0485SDariusz Sosnowski 		char ifname[IF_NAMESIZE + 1];
2069b62f0485SDariusz Sosnowski 		struct rte_pci_addr if_pci_addr;
2070b62f0485SDariusz Sosnowski 		char mpesw[SYSFS_MPESW_PARAM_MAX_LEN + 1];
2071b62f0485SDariusz Sosnowski 		FILE *sysfs;
2072b62f0485SDariusz Sosnowski 		int n;
2073b62f0485SDariusz Sosnowski 
2074b62f0485SDariusz Sosnowski 		ifindex = mlx5_nl_ifindex(nl_rdma, ibv->name, i);
2075b62f0485SDariusz Sosnowski 		if (!ifindex)
2076b62f0485SDariusz Sosnowski 			continue;
2077b62f0485SDariusz Sosnowski 		if (!if_indextoname(ifindex, ifname))
2078b62f0485SDariusz Sosnowski 			continue;
2079b62f0485SDariusz Sosnowski 		MKSTR(sysfs_if_path, "/sys/class/net/%s", ifname);
2080b62f0485SDariusz Sosnowski 		if (mlx5_get_pci_addr(sysfs_if_path, &if_pci_addr))
2081b62f0485SDariusz Sosnowski 			continue;
2082b62f0485SDariusz Sosnowski 		if (pci_addr->domain != if_pci_addr.domain ||
2083b62f0485SDariusz Sosnowski 		    pci_addr->bus != if_pci_addr.bus ||
2084b62f0485SDariusz Sosnowski 		    pci_addr->devid != if_pci_addr.devid ||
2085b62f0485SDariusz Sosnowski 		    pci_addr->function != if_pci_addr.function)
2086b62f0485SDariusz Sosnowski 			continue;
2087b62f0485SDariusz Sosnowski 		MKSTR(sysfs_mpesw_path,
2088b62f0485SDariusz Sosnowski 		      "/sys/class/net/%s/compat/devlink/lag_port_select_mode", ifname);
2089b62f0485SDariusz Sosnowski 		sysfs = fopen(sysfs_mpesw_path, "r");
2090b62f0485SDariusz Sosnowski 		if (!sysfs)
2091b62f0485SDariusz Sosnowski 			continue;
2092b62f0485SDariusz Sosnowski 		n = fscanf(sysfs, "%" RTE_STR(SYSFS_MPESW_PARAM_MAX_LEN) "s", mpesw);
2093b62f0485SDariusz Sosnowski 		fclose(sysfs);
2094b62f0485SDariusz Sosnowski 		if (n != 1)
2095b62f0485SDariusz Sosnowski 			continue;
2096b62f0485SDariusz Sosnowski 		ret = 0;
2097b62f0485SDariusz Sosnowski 		if (strcmp(mpesw, "multiport_esw") == 0) {
2098b62f0485SDariusz Sosnowski 			*enabled = 1;
2099b62f0485SDariusz Sosnowski 			break;
2100b62f0485SDariusz Sosnowski 		}
2101b62f0485SDariusz Sosnowski 		*enabled = 0;
2102b62f0485SDariusz Sosnowski 		break;
2103b62f0485SDariusz Sosnowski 	}
2104b62f0485SDariusz Sosnowski 	if (i > n_ports) {
2105b62f0485SDariusz Sosnowski 		DRV_LOG(DEBUG, "Unable to get Multiport E-Switch state by sysfs.");
2106b62f0485SDariusz Sosnowski 		rte_errno = ENOENT;
2107b62f0485SDariusz Sosnowski 		ret = -rte_errno;
2108b62f0485SDariusz Sosnowski 	}
2109b62f0485SDariusz Sosnowski 
2110b62f0485SDariusz Sosnowski close_nl_rdma:
2111b62f0485SDariusz Sosnowski 	close(nl_rdma);
2112b62f0485SDariusz Sosnowski 	return ret;
2113b62f0485SDariusz Sosnowski }
2114b62f0485SDariusz Sosnowski 
211511c73de9SDariusz Sosnowski static int
2116bb2fee72SDariusz Sosnowski mlx5_is_mpesw_enabled(struct ibv_device *ibv, struct rte_pci_addr *ibv_pci_addr, int *enabled)
2117bb2fee72SDariusz Sosnowski {
2118bb2fee72SDariusz Sosnowski 	/*
2119bb2fee72SDariusz Sosnowski 	 * Try getting Multiport E-Switch state through netlink interface
2120bb2fee72SDariusz Sosnowski 	 * If unable, try sysfs interface. If that is unable as well,
2121bb2fee72SDariusz Sosnowski 	 * assume that Multiport E-Switch is disabled and return an error.
2122bb2fee72SDariusz Sosnowski 	 */
2123bb2fee72SDariusz Sosnowski 	if (mlx5_nl_esw_multiport_get(ibv_pci_addr, enabled) >= 0 ||
2124bb2fee72SDariusz Sosnowski 	    mlx5_sysfs_esw_multiport_get(ibv, ibv_pci_addr, enabled) >= 0)
2125bb2fee72SDariusz Sosnowski 		return 0;
2126bb2fee72SDariusz Sosnowski 	DRV_LOG(DEBUG, "Unable to check MPESW state for IB device %s "
2127bb2fee72SDariusz Sosnowski 		       "(PCI: " PCI_PRI_FMT ")",
2128bb2fee72SDariusz Sosnowski 		       ibv->name,
2129bb2fee72SDariusz Sosnowski 		       ibv_pci_addr->domain, ibv_pci_addr->bus,
2130bb2fee72SDariusz Sosnowski 		       ibv_pci_addr->devid, ibv_pci_addr->function);
2131bb2fee72SDariusz Sosnowski 	*enabled = 0;
2132bb2fee72SDariusz Sosnowski 	return -rte_errno;
2133bb2fee72SDariusz Sosnowski }
2134bb2fee72SDariusz Sosnowski 
213511c73de9SDariusz Sosnowski static int
213611c73de9SDariusz Sosnowski mlx5_device_mpesw_pci_match(struct ibv_device *ibv,
213711c73de9SDariusz Sosnowski 			    const struct rte_pci_addr *owner_pci,
213811c73de9SDariusz Sosnowski 			    int nl_rdma)
213911c73de9SDariusz Sosnowski {
214011c73de9SDariusz Sosnowski 	struct rte_pci_addr ibdev_pci_addr = { 0 };
214111c73de9SDariusz Sosnowski 	char ifname[IF_NAMESIZE + 1] = { 0 };
214211c73de9SDariusz Sosnowski 	unsigned int ifindex;
214311c73de9SDariusz Sosnowski 	unsigned int np;
214411c73de9SDariusz Sosnowski 	unsigned int i;
214511c73de9SDariusz Sosnowski 	int enabled = 0;
214611c73de9SDariusz Sosnowski 	int ret;
214711c73de9SDariusz Sosnowski 
214811c73de9SDariusz Sosnowski 	/* Check if IB device's PCI address matches the probed PCI address. */
214911c73de9SDariusz Sosnowski 	if (mlx5_get_pci_addr(ibv->ibdev_path, &ibdev_pci_addr)) {
215011c73de9SDariusz Sosnowski 		DRV_LOG(DEBUG, "Skipping MPESW check for IB device %s since "
215111c73de9SDariusz Sosnowski 			       "there is no underlying PCI device", ibv->name);
215211c73de9SDariusz Sosnowski 		rte_errno = ENOENT;
215311c73de9SDariusz Sosnowski 		return -rte_errno;
215411c73de9SDariusz Sosnowski 	}
215511c73de9SDariusz Sosnowski 	if (ibdev_pci_addr.domain != owner_pci->domain ||
215611c73de9SDariusz Sosnowski 	    ibdev_pci_addr.bus != owner_pci->bus ||
215711c73de9SDariusz Sosnowski 	    ibdev_pci_addr.devid != owner_pci->devid ||
215811c73de9SDariusz Sosnowski 	    ibdev_pci_addr.function != owner_pci->function) {
215911c73de9SDariusz Sosnowski 		return -1;
216011c73de9SDariusz Sosnowski 	}
216111c73de9SDariusz Sosnowski 	/* Check if IB device has MPESW enabled. */
216211c73de9SDariusz Sosnowski 	if (mlx5_is_mpesw_enabled(ibv, &ibdev_pci_addr, &enabled))
216311c73de9SDariusz Sosnowski 		return -1;
216411c73de9SDariusz Sosnowski 	if (!enabled)
216511c73de9SDariusz Sosnowski 		return -1;
216611c73de9SDariusz Sosnowski 	/* Iterate through IB ports to find MPESW master uplink port. */
216711c73de9SDariusz Sosnowski 	if (nl_rdma < 0)
216811c73de9SDariusz Sosnowski 		return -1;
216911c73de9SDariusz Sosnowski 	np = mlx5_nl_portnum(nl_rdma, ibv->name);
217011c73de9SDariusz Sosnowski 	if (!np)
217111c73de9SDariusz Sosnowski 		return -1;
217211c73de9SDariusz Sosnowski 	for (i = 1; i <= np; ++i) {
217311c73de9SDariusz Sosnowski 		struct rte_pci_addr pci_addr;
217411c73de9SDariusz Sosnowski 		FILE *file;
217511c73de9SDariusz Sosnowski 		char port_name[IF_NAMESIZE + 1];
217611c73de9SDariusz Sosnowski 		struct mlx5_switch_info	info;
217711c73de9SDariusz Sosnowski 
217811c73de9SDariusz Sosnowski 		/* Check whether IB port has a corresponding netdev. */
217911c73de9SDariusz Sosnowski 		ifindex = mlx5_nl_ifindex(nl_rdma, ibv->name, i);
218011c73de9SDariusz Sosnowski 		if (!ifindex)
218111c73de9SDariusz Sosnowski 			continue;
218211c73de9SDariusz Sosnowski 		if (!if_indextoname(ifindex, ifname))
218311c73de9SDariusz Sosnowski 			continue;
218411c73de9SDariusz Sosnowski 		/* Read port name and determine its type. */
218511c73de9SDariusz Sosnowski 		MKSTR(ifphysportname, "/sys/class/net/%s/phys_port_name", ifname);
218611c73de9SDariusz Sosnowski 		file = fopen(ifphysportname, "rb");
218711c73de9SDariusz Sosnowski 		if (!file)
218811c73de9SDariusz Sosnowski 			continue;
218911c73de9SDariusz Sosnowski 		ret = fscanf(file, "%16s", port_name);
219011c73de9SDariusz Sosnowski 		fclose(file);
219111c73de9SDariusz Sosnowski 		if (ret != 1)
219211c73de9SDariusz Sosnowski 			continue;
219311c73de9SDariusz Sosnowski 		memset(&info, 0, sizeof(info));
219411c73de9SDariusz Sosnowski 		mlx5_translate_port_name(port_name, &info);
219511c73de9SDariusz Sosnowski 		if (info.name_type != MLX5_PHYS_PORT_NAME_TYPE_UPLINK)
219611c73de9SDariusz Sosnowski 			continue;
219711c73de9SDariusz Sosnowski 		/* Fetch PCI address of the device to which the netdev is bound. */
219811c73de9SDariusz Sosnowski 		MKSTR(ifpath, "/sys/class/net/%s", ifname);
219911c73de9SDariusz Sosnowski 		if (mlx5_get_pci_addr(ifpath, &pci_addr))
220011c73de9SDariusz Sosnowski 			continue;
220111c73de9SDariusz Sosnowski 		if (pci_addr.domain == ibdev_pci_addr.domain &&
220211c73de9SDariusz Sosnowski 		    pci_addr.bus == ibdev_pci_addr.bus &&
220311c73de9SDariusz Sosnowski 		    pci_addr.devid == ibdev_pci_addr.devid &&
220411c73de9SDariusz Sosnowski 		    pci_addr.function == ibdev_pci_addr.function) {
220511c73de9SDariusz Sosnowski 			MLX5_ASSERT(info.port_name >= 0);
220611c73de9SDariusz Sosnowski 			return info.port_name;
220711c73de9SDariusz Sosnowski 		}
220811c73de9SDariusz Sosnowski 	}
220911c73de9SDariusz Sosnowski 	/* No matching MPESW uplink port was found. */
221011c73de9SDariusz Sosnowski 	return -1;
221111c73de9SDariusz Sosnowski }
221211c73de9SDariusz Sosnowski 
22132eb4d010SOphir Munk /**
221408c2772fSXueming Li  * Register a PCI device within bonding.
22152eb4d010SOphir Munk  *
221608c2772fSXueming Li  * This function spawns Ethernet devices out of a given PCI device and
221708c2772fSXueming Li  * bonding owner PF index.
22182eb4d010SOphir Munk  *
22197af08c8fSMichael Baum  * @param[in] cdev
22207af08c8fSMichael Baum  *   Pointer to common mlx5 device structure.
222108c2772fSXueming Li  * @param[in] req_eth_da
222208c2772fSXueming Li  *   Requested ethdev device argument.
222308c2772fSXueming Li  * @param[in] owner_id
222408c2772fSXueming Li  *   Requested owner PF port ID within bonding device, default to 0.
2225a729d2f0SMichael Baum  * @param[in, out] mkvlist
2226a729d2f0SMichael Baum  *   Pointer to mlx5 kvargs control, can be NULL if there is no devargs.
22272eb4d010SOphir Munk  *
22282eb4d010SOphir Munk  * @return
22292eb4d010SOphir Munk  *   0 on success, a negative errno value otherwise and rte_errno is set.
22302eb4d010SOphir Munk  */
223108c2772fSXueming Li static int
2232ca1418ceSMichael Baum mlx5_os_pci_probe_pf(struct mlx5_common_device *cdev,
223308c2772fSXueming Li 		     struct rte_eth_devargs *req_eth_da,
2234a729d2f0SMichael Baum 		     uint16_t owner_id, struct mlx5_kvargs_ctrl *mkvlist)
22352eb4d010SOphir Munk {
22362eb4d010SOphir Munk 	struct ibv_device **ibv_list;
22372eb4d010SOphir Munk 	/*
22382eb4d010SOphir Munk 	 * Number of found IB Devices matching with requested PCI BDF.
22392eb4d010SOphir Munk 	 * nd != 1 means there are multiple IB devices over the same
22402eb4d010SOphir Munk 	 * PCI device and we have representors and master.
22412eb4d010SOphir Munk 	 */
22422eb4d010SOphir Munk 	unsigned int nd = 0;
22432eb4d010SOphir Munk 	/*
22442eb4d010SOphir Munk 	 * Number of found IB device Ports. nd = 1 and np = 1..n means
22452eb4d010SOphir Munk 	 * we have the single multiport IB device, and there may be
22462eb4d010SOphir Munk 	 * representors attached to some of found ports.
22472eb4d010SOphir Munk 	 */
22482eb4d010SOphir Munk 	unsigned int np = 0;
22492eb4d010SOphir Munk 	/*
22502eb4d010SOphir Munk 	 * Number of DPDK ethernet devices to Spawn - either over
22512eb4d010SOphir Munk 	 * multiple IB devices or multiple ports of single IB device.
22522eb4d010SOphir Munk 	 * Actually this is the number of iterations to spawn.
22532eb4d010SOphir Munk 	 */
22542eb4d010SOphir Munk 	unsigned int ns = 0;
22552eb4d010SOphir Munk 	/*
22562eb4d010SOphir Munk 	 * Bonding device
22572eb4d010SOphir Munk 	 *   < 0 - no bonding device (single one)
22582eb4d010SOphir Munk 	 *  >= 0 - bonding device (value is slave PF index)
22592eb4d010SOphir Munk 	 */
22602eb4d010SOphir Munk 	int bd = -1;
226111c73de9SDariusz Sosnowski 	/*
226211c73de9SDariusz Sosnowski 	 * Multiport E-Switch (MPESW) device:
226311c73de9SDariusz Sosnowski 	 *   < 0 - no MPESW device or could not determine if it is MPESW device,
226411c73de9SDariusz Sosnowski 	 *  >= 0 - MPESW device. Value is the port index of the MPESW owner.
226511c73de9SDariusz Sosnowski 	 */
226611c73de9SDariusz Sosnowski 	int mpesw = MLX5_MPESW_PORT_INVALID;
22677af08c8fSMichael Baum 	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(cdev->dev);
22682eb4d010SOphir Munk 	struct mlx5_dev_spawn_data *list = NULL;
226908c2772fSXueming Li 	struct rte_eth_devargs eth_da = *req_eth_da;
2270f926cce3SXueming Li 	struct rte_pci_addr owner_pci = pci_dev->addr; /* Owner PF. */
2271f5f4c482SXueming Li 	struct mlx5_bond_info bond_info;
2272f926cce3SXueming Li 	int ret = -1;
22732eb4d010SOphir Munk 
22742eb4d010SOphir Munk 	errno = 0;
22752eb4d010SOphir Munk 	ibv_list = mlx5_glue->get_device_list(&ret);
22762eb4d010SOphir Munk 	if (!ibv_list) {
22772eb4d010SOphir Munk 		rte_errno = errno ? errno : ENOSYS;
2278887183efSMichael Baum 		DRV_LOG(ERR, "Cannot list devices, is ib_uverbs loaded?");
22792eb4d010SOphir Munk 		return -rte_errno;
22802eb4d010SOphir Munk 	}
22812eb4d010SOphir Munk 	/*
22822eb4d010SOphir Munk 	 * First scan the list of all Infiniband devices to find
22832eb4d010SOphir Munk 	 * matching ones, gathering into the list.
22842eb4d010SOphir Munk 	 */
22852eb4d010SOphir Munk 	struct ibv_device *ibv_match[ret + 1];
2286be66461cSDmitry Kozlyuk 	int nl_route = mlx5_nl_init(NETLINK_ROUTE, 0);
2287be66461cSDmitry Kozlyuk 	int nl_rdma = mlx5_nl_init(NETLINK_RDMA, 0);
22882eb4d010SOphir Munk 	unsigned int i;
22892eb4d010SOphir Munk 
22902eb4d010SOphir Munk 	while (ret-- > 0) {
22912eb4d010SOphir Munk 		struct rte_pci_addr pci_addr;
22922eb4d010SOphir Munk 
2293887183efSMichael Baum 		DRV_LOG(DEBUG, "Checking device \"%s\"", ibv_list[ret]->name);
2294ca1418ceSMichael Baum 		bd = mlx5_device_bond_pci_match(ibv_list[ret]->name, &owner_pci,
2295ca1418ceSMichael Baum 						nl_rdma, owner_id, &bond_info);
22962eb4d010SOphir Munk 		if (bd >= 0) {
22972eb4d010SOphir Munk 			/*
22982eb4d010SOphir Munk 			 * Bonding device detected. Only one match is allowed,
22992eb4d010SOphir Munk 			 * the bonding is supported over multi-port IB device,
23002eb4d010SOphir Munk 			 * there should be no matches on representor PCI
23012eb4d010SOphir Munk 			 * functions or non VF LAG bonding devices with
23022eb4d010SOphir Munk 			 * specified address.
23032eb4d010SOphir Munk 			 */
23042eb4d010SOphir Munk 			if (nd) {
23052eb4d010SOphir Munk 				DRV_LOG(ERR,
23062eb4d010SOphir Munk 					"multiple PCI match on bonding device"
23072eb4d010SOphir Munk 					"\"%s\" found", ibv_list[ret]->name);
23082eb4d010SOphir Munk 				rte_errno = ENOENT;
23092eb4d010SOphir Munk 				ret = -rte_errno;
23102eb4d010SOphir Munk 				goto exit;
23112eb4d010SOphir Munk 			}
2312f926cce3SXueming Li 			/* Amend owner pci address if owner PF ID specified. */
2313f926cce3SXueming Li 			if (eth_da.nb_representor_ports)
231408c2772fSXueming Li 				owner_pci.function += owner_id;
2315ca1418ceSMichael Baum 			DRV_LOG(INFO,
2316ca1418ceSMichael Baum 				"PCI information matches for slave %d bonding device \"%s\"",
23172eb4d010SOphir Munk 				bd, ibv_list[ret]->name);
23182eb4d010SOphir Munk 			ibv_match[nd++] = ibv_list[ret];
23192eb4d010SOphir Munk 			break;
232011c73de9SDariusz Sosnowski 		}
232111c73de9SDariusz Sosnowski 		mpesw = mlx5_device_mpesw_pci_match(ibv_list[ret], &owner_pci, nl_rdma);
232211c73de9SDariusz Sosnowski 		if (mpesw >= 0) {
232311c73de9SDariusz Sosnowski 			/*
232411c73de9SDariusz Sosnowski 			 * MPESW device detected. Only one matching IB device is allowed,
232511c73de9SDariusz Sosnowski 			 * so if any matches were found previously, fail gracefully.
232611c73de9SDariusz Sosnowski 			 */
232711c73de9SDariusz Sosnowski 			if (nd) {
232811c73de9SDariusz Sosnowski 				DRV_LOG(ERR,
232911c73de9SDariusz Sosnowski 					"PCI information matches MPESW device \"%s\", "
233011c73de9SDariusz Sosnowski 					"but multiple matching PCI devices were found. "
233111c73de9SDariusz Sosnowski 					"Probing failed.",
233211c73de9SDariusz Sosnowski 					ibv_list[ret]->name);
233311c73de9SDariusz Sosnowski 				rte_errno = ENOENT;
233411c73de9SDariusz Sosnowski 				ret = -rte_errno;
233511c73de9SDariusz Sosnowski 				goto exit;
233611c73de9SDariusz Sosnowski 			}
233711c73de9SDariusz Sosnowski 			DRV_LOG(INFO,
233811c73de9SDariusz Sosnowski 				"PCI information matches MPESW device \"%s\"",
233911c73de9SDariusz Sosnowski 				ibv_list[ret]->name);
234011c73de9SDariusz Sosnowski 			ibv_match[nd++] = ibv_list[ret];
234111c73de9SDariusz Sosnowski 			break;
234211c73de9SDariusz Sosnowski 		}
234311c73de9SDariusz Sosnowski 		/* Bonding or MPESW device was not found. */
23444d567938SThomas Monjalon 		if (mlx5_get_pci_addr(ibv_list[ret]->ibdev_path,
23454d567938SThomas Monjalon 					&pci_addr))
23462eb4d010SOphir Munk 			continue;
23478fa22e1fSThomas Monjalon 		if (rte_pci_addr_cmp(&owner_pci, &pci_addr) != 0)
23482eb4d010SOphir Munk 			continue;
23492eb4d010SOphir Munk 		DRV_LOG(INFO, "PCI information matches for device \"%s\"",
23502eb4d010SOphir Munk 			ibv_list[ret]->name);
23512eb4d010SOphir Munk 		ibv_match[nd++] = ibv_list[ret];
23522eb4d010SOphir Munk 	}
23532eb4d010SOphir Munk 	ibv_match[nd] = NULL;
23542eb4d010SOphir Munk 	if (!nd) {
23552eb4d010SOphir Munk 		/* No device matches, just complain and bail out. */
23562eb4d010SOphir Munk 		DRV_LOG(WARNING,
2357f956d3d4SRongwei Liu 			"PF %u doesn't have Verbs device matches PCI device " PCI_PRI_FMT ","
23582eb4d010SOphir Munk 			" are kernel drivers loaded?",
2359f956d3d4SRongwei Liu 			owner_id, owner_pci.domain, owner_pci.bus,
2360f926cce3SXueming Li 			owner_pci.devid, owner_pci.function);
23612eb4d010SOphir Munk 		rte_errno = ENOENT;
23622eb4d010SOphir Munk 		ret = -rte_errno;
23632eb4d010SOphir Munk 		goto exit;
23642eb4d010SOphir Munk 	}
23652eb4d010SOphir Munk 	if (nd == 1) {
23662eb4d010SOphir Munk 		/*
23672eb4d010SOphir Munk 		 * Found single matching device may have multiple ports.
23682eb4d010SOphir Munk 		 * Each port may be representor, we have to check the port
23692eb4d010SOphir Munk 		 * number and check the representors existence.
23702eb4d010SOphir Munk 		 */
23712eb4d010SOphir Munk 		if (nl_rdma >= 0)
23722eb4d010SOphir Munk 			np = mlx5_nl_portnum(nl_rdma, ibv_match[0]->name);
23732eb4d010SOphir Munk 		if (!np)
2374887183efSMichael Baum 			DRV_LOG(WARNING,
2375887183efSMichael Baum 				"Cannot get IB device \"%s\" ports number.",
2376887183efSMichael Baum 				ibv_match[0]->name);
23772eb4d010SOphir Munk 		if (bd >= 0 && !np) {
2378887183efSMichael Baum 			DRV_LOG(ERR, "Cannot get ports for bonding device.");
23792eb4d010SOphir Munk 			rte_errno = ENOENT;
23802eb4d010SOphir Munk 			ret = -rte_errno;
23812eb4d010SOphir Munk 			goto exit;
23822eb4d010SOphir Munk 		}
238311c73de9SDariusz Sosnowski 		if (mpesw >= 0 && !np) {
238411c73de9SDariusz Sosnowski 			DRV_LOG(ERR, "Cannot get ports for MPESW device.");
238511c73de9SDariusz Sosnowski 			rte_errno = ENOENT;
238611c73de9SDariusz Sosnowski 			ret = -rte_errno;
238711c73de9SDariusz Sosnowski 			goto exit;
238811c73de9SDariusz Sosnowski 		}
23892eb4d010SOphir Munk 	}
2390887183efSMichael Baum 	/* Now we can determine the maximal amount of devices to be spawned. */
23912175c4dcSSuanming Mou 	list = mlx5_malloc(MLX5_MEM_ZERO,
2392887183efSMichael Baum 			   sizeof(struct mlx5_dev_spawn_data) * (np ? np : nd),
23932175c4dcSSuanming Mou 			   RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
23942eb4d010SOphir Munk 	if (!list) {
2395887183efSMichael Baum 		DRV_LOG(ERR, "Spawn data array allocation failure.");
23962eb4d010SOphir Munk 		rte_errno = ENOMEM;
23972eb4d010SOphir Munk 		ret = -rte_errno;
23982eb4d010SOphir Munk 		goto exit;
23992eb4d010SOphir Munk 	}
240011c73de9SDariusz Sosnowski 	if (bd >= 0 || mpesw >= 0 || np > 1) {
24012eb4d010SOphir Munk 		/*
24022eb4d010SOphir Munk 		 * Single IB device with multiple ports found,
24032eb4d010SOphir Munk 		 * it may be E-Switch master device and representors.
24042eb4d010SOphir Munk 		 * We have to perform identification through the ports.
24052eb4d010SOphir Munk 		 */
24062eb4d010SOphir Munk 		MLX5_ASSERT(nl_rdma >= 0);
24072eb4d010SOphir Munk 		MLX5_ASSERT(ns == 0);
24082eb4d010SOphir Munk 		MLX5_ASSERT(nd == 1);
24092eb4d010SOphir Munk 		MLX5_ASSERT(np);
24102eb4d010SOphir Munk 		for (i = 1; i <= np; ++i) {
2411f5f4c482SXueming Li 			list[ns].bond_info = &bond_info;
24122eb4d010SOphir Munk 			list[ns].max_port = np;
2413834a9019SOphir Munk 			list[ns].phys_port = i;
2414887183efSMichael Baum 			list[ns].phys_dev_name = ibv_match[0]->name;
24152eb4d010SOphir Munk 			list[ns].eth_dev = NULL;
24162eb4d010SOphir Munk 			list[ns].pci_dev = pci_dev;
24177af08c8fSMichael Baum 			list[ns].cdev = cdev;
24182eb4d010SOphir Munk 			list[ns].pf_bond = bd;
241911c73de9SDariusz Sosnowski 			list[ns].mpesw_port = MLX5_MPESW_PORT_INVALID;
2420887183efSMichael Baum 			list[ns].ifindex = mlx5_nl_ifindex(nl_rdma,
2421887183efSMichael Baum 							   ibv_match[0]->name,
2422887183efSMichael Baum 							   i);
24232eb4d010SOphir Munk 			if (!list[ns].ifindex) {
24242eb4d010SOphir Munk 				/*
24252eb4d010SOphir Munk 				 * No network interface index found for the
24262eb4d010SOphir Munk 				 * specified port, it means there is no
24272eb4d010SOphir Munk 				 * representor on this port. It's OK,
24282eb4d010SOphir Munk 				 * there can be disabled ports, for example
24292eb4d010SOphir Munk 				 * if sriov_numvfs < sriov_totalvfs.
24302eb4d010SOphir Munk 				 */
24312eb4d010SOphir Munk 				continue;
24322eb4d010SOphir Munk 			}
24332eb4d010SOphir Munk 			ret = -1;
24342eb4d010SOphir Munk 			if (nl_route >= 0)
2435887183efSMichael Baum 				ret = mlx5_nl_switch_info(nl_route,
24362eb4d010SOphir Munk 							  list[ns].ifindex,
24372eb4d010SOphir Munk 							  &list[ns].info);
24382eb4d010SOphir Munk 			if (ret || (!list[ns].info.representor &&
24392eb4d010SOphir Munk 				    !list[ns].info.master)) {
24402eb4d010SOphir Munk 				/*
24412eb4d010SOphir Munk 				 * We failed to recognize representors with
24422eb4d010SOphir Munk 				 * Netlink, let's try to perform the task
24432eb4d010SOphir Munk 				 * with sysfs.
24442eb4d010SOphir Munk 				 */
2445887183efSMichael Baum 				ret = mlx5_sysfs_switch_info(list[ns].ifindex,
24462eb4d010SOphir Munk 							     &list[ns].info);
24472eb4d010SOphir Munk 			}
24482eb4d010SOphir Munk 			if (!ret && bd >= 0) {
24492eb4d010SOphir Munk 				switch (list[ns].info.name_type) {
24502eb4d010SOphir Munk 				case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
24519f430dd7SViacheslav Ovsiienko 					if (np == 1) {
24529f430dd7SViacheslav Ovsiienko 						/*
24539f430dd7SViacheslav Ovsiienko 						 * Force standalone bonding
24549f430dd7SViacheslav Ovsiienko 						 * device for ROCE LAG
24557be78d02SJosh Soref 						 * configurations.
24569f430dd7SViacheslav Ovsiienko 						 */
24579f430dd7SViacheslav Ovsiienko 						list[ns].info.master = 0;
24589f430dd7SViacheslav Ovsiienko 						list[ns].info.representor = 0;
24599f430dd7SViacheslav Ovsiienko 					}
24602eb4d010SOphir Munk 					if (list[ns].info.port_name == bd)
24612eb4d010SOphir Munk 						ns++;
24622eb4d010SOphir Munk 					break;
2463420bbdaeSViacheslav Ovsiienko 				case MLX5_PHYS_PORT_NAME_TYPE_PFHPF:
2464420bbdaeSViacheslav Ovsiienko 					/* Fallthrough */
24652eb4d010SOphir Munk 				case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
2466cb95feefSXueming Li 					/* Fallthrough */
2467cb95feefSXueming Li 				case MLX5_PHYS_PORT_NAME_TYPE_PFSF:
24682eb4d010SOphir Munk 					if (list[ns].info.pf_num == bd)
24692eb4d010SOphir Munk 						ns++;
24702eb4d010SOphir Munk 					break;
24712eb4d010SOphir Munk 				default:
24722eb4d010SOphir Munk 					break;
24732eb4d010SOphir Munk 				}
24742eb4d010SOphir Munk 				continue;
24752eb4d010SOphir Munk 			}
247611c73de9SDariusz Sosnowski 			if (!ret && mpesw >= 0) {
247711c73de9SDariusz Sosnowski 				switch (list[ns].info.name_type) {
247811c73de9SDariusz Sosnowski 				case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
247911c73de9SDariusz Sosnowski 					/* Owner port is treated as master port. */
248011c73de9SDariusz Sosnowski 					if (list[ns].info.port_name == mpesw) {
248111c73de9SDariusz Sosnowski 						list[ns].info.master = 1;
248211c73de9SDariusz Sosnowski 						list[ns].info.representor = 0;
248311c73de9SDariusz Sosnowski 					} else {
248411c73de9SDariusz Sosnowski 						list[ns].info.master = 0;
248511c73de9SDariusz Sosnowski 						list[ns].info.representor = 1;
248611c73de9SDariusz Sosnowski 					}
248711c73de9SDariusz Sosnowski 					/*
248811c73de9SDariusz Sosnowski 					 * Ports of this type have uplink port index
248911c73de9SDariusz Sosnowski 					 * encoded in the name. This index is also a PF index.
249011c73de9SDariusz Sosnowski 					 */
249111c73de9SDariusz Sosnowski 					list[ns].info.pf_num = list[ns].info.port_name;
249211c73de9SDariusz Sosnowski 					list[ns].mpesw_port = list[ns].info.port_name;
249311c73de9SDariusz Sosnowski 					list[ns].info.mpesw_owner = mpesw;
249411c73de9SDariusz Sosnowski 					ns++;
249511c73de9SDariusz Sosnowski 					break;
249611c73de9SDariusz Sosnowski 				case MLX5_PHYS_PORT_NAME_TYPE_PFHPF:
249711c73de9SDariusz Sosnowski 				case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
249811c73de9SDariusz Sosnowski 				case MLX5_PHYS_PORT_NAME_TYPE_PFSF:
249911c73de9SDariusz Sosnowski 					/* Only spawn representors related to the probed PF. */
250011c73de9SDariusz Sosnowski 					if (list[ns].info.pf_num == owner_id) {
250111c73de9SDariusz Sosnowski 						/*
250211c73de9SDariusz Sosnowski 						 * Ports of this type have PF index encoded in name,
250311c73de9SDariusz Sosnowski 						 * which translate to the related uplink port index.
250411c73de9SDariusz Sosnowski 						 */
250511c73de9SDariusz Sosnowski 						list[ns].mpesw_port = list[ns].info.pf_num;
250611c73de9SDariusz Sosnowski 						/* MPESW owner is also saved but not used now. */
250711c73de9SDariusz Sosnowski 						list[ns].info.mpesw_owner = mpesw;
250811c73de9SDariusz Sosnowski 						ns++;
250911c73de9SDariusz Sosnowski 					}
251011c73de9SDariusz Sosnowski 					break;
251111c73de9SDariusz Sosnowski 				default:
251211c73de9SDariusz Sosnowski 					break;
251311c73de9SDariusz Sosnowski 				}
251411c73de9SDariusz Sosnowski 				continue;
251511c73de9SDariusz Sosnowski 			}
25162eb4d010SOphir Munk 			if (!ret && (list[ns].info.representor ^
25172eb4d010SOphir Munk 				     list[ns].info.master))
25182eb4d010SOphir Munk 				ns++;
25192eb4d010SOphir Munk 		}
25202eb4d010SOphir Munk 		if (!ns) {
25212eb4d010SOphir Munk 			DRV_LOG(ERR,
2522887183efSMichael Baum 				"Unable to recognize master/representors on the IB device with multiple ports.");
25232eb4d010SOphir Munk 			rte_errno = ENOENT;
25242eb4d010SOphir Munk 			ret = -rte_errno;
25252eb4d010SOphir Munk 			goto exit;
25262eb4d010SOphir Munk 		}
25272eb4d010SOphir Munk 	} else {
25282eb4d010SOphir Munk 		/*
25292eb4d010SOphir Munk 		 * The existence of several matching entries (nd > 1) means
25302eb4d010SOphir Munk 		 * port representors have been instantiated. No existing Verbs
25312eb4d010SOphir Munk 		 * call nor sysfs entries can tell them apart, this can only
25322eb4d010SOphir Munk 		 * be done through Netlink calls assuming kernel drivers are
25332eb4d010SOphir Munk 		 * recent enough to support them.
25342eb4d010SOphir Munk 		 *
25352eb4d010SOphir Munk 		 * In the event of identification failure through Netlink,
25362eb4d010SOphir Munk 		 * try again through sysfs, then:
25372eb4d010SOphir Munk 		 *
25382eb4d010SOphir Munk 		 * 1. A single IB device matches (nd == 1) with single
25392eb4d010SOphir Munk 		 *    port (np=0/1) and is not a representor, assume
25402eb4d010SOphir Munk 		 *    no switch support.
25412eb4d010SOphir Munk 		 *
25422eb4d010SOphir Munk 		 * 2. Otherwise no safe assumptions can be made;
25432eb4d010SOphir Munk 		 *    complain louder and bail out.
25442eb4d010SOphir Munk 		 */
25452eb4d010SOphir Munk 		for (i = 0; i != nd; ++i) {
25462eb4d010SOphir Munk 			memset(&list[ns].info, 0, sizeof(list[ns].info));
2547f5f4c482SXueming Li 			list[ns].bond_info = NULL;
25482eb4d010SOphir Munk 			list[ns].max_port = 1;
2549834a9019SOphir Munk 			list[ns].phys_port = 1;
2550887183efSMichael Baum 			list[ns].phys_dev_name = ibv_match[i]->name;
25512eb4d010SOphir Munk 			list[ns].eth_dev = NULL;
25522eb4d010SOphir Munk 			list[ns].pci_dev = pci_dev;
25537af08c8fSMichael Baum 			list[ns].cdev = cdev;
25542eb4d010SOphir Munk 			list[ns].pf_bond = -1;
255511c73de9SDariusz Sosnowski 			list[ns].mpesw_port = MLX5_MPESW_PORT_INVALID;
25562eb4d010SOphir Munk 			list[ns].ifindex = 0;
25572eb4d010SOphir Munk 			if (nl_rdma >= 0)
25582eb4d010SOphir Munk 				list[ns].ifindex = mlx5_nl_ifindex
2559834a9019SOphir Munk 							    (nl_rdma,
2560887183efSMichael Baum 							     ibv_match[i]->name,
2561887183efSMichael Baum 							     1);
25622eb4d010SOphir Munk 			if (!list[ns].ifindex) {
25632eb4d010SOphir Munk 				char ifname[IF_NAMESIZE];
25642eb4d010SOphir Munk 
25652eb4d010SOphir Munk 				/*
25662eb4d010SOphir Munk 				 * Netlink failed, it may happen with old
25672eb4d010SOphir Munk 				 * ib_core kernel driver (before 4.16).
25682eb4d010SOphir Munk 				 * We can assume there is old driver because
25692eb4d010SOphir Munk 				 * here we are processing single ports IB
25702eb4d010SOphir Munk 				 * devices. Let's try sysfs to retrieve
25712eb4d010SOphir Munk 				 * the ifindex. The method works for
25722eb4d010SOphir Munk 				 * master device only.
25732eb4d010SOphir Munk 				 */
25742eb4d010SOphir Munk 				if (nd > 1) {
25752eb4d010SOphir Munk 					/*
25762eb4d010SOphir Munk 					 * Multiple devices found, assume
25772eb4d010SOphir Munk 					 * representors, can not distinguish
25782eb4d010SOphir Munk 					 * master/representor and retrieve
25792eb4d010SOphir Munk 					 * ifindex via sysfs.
25802eb4d010SOphir Munk 					 */
25812eb4d010SOphir Munk 					continue;
25822eb4d010SOphir Munk 				}
2583aec086c9SMatan Azrad 				ret = mlx5_get_ifname_sysfs
2584aec086c9SMatan Azrad 					(ibv_match[i]->ibdev_path, ifname);
25852eb4d010SOphir Munk 				if (!ret)
25862eb4d010SOphir Munk 					list[ns].ifindex =
25872eb4d010SOphir Munk 						if_nametoindex(ifname);
25882eb4d010SOphir Munk 				if (!list[ns].ifindex) {
25892eb4d010SOphir Munk 					/*
25902eb4d010SOphir Munk 					 * No network interface index found
25912eb4d010SOphir Munk 					 * for the specified device, it means
25922eb4d010SOphir Munk 					 * there it is neither representor
25932eb4d010SOphir Munk 					 * nor master.
25942eb4d010SOphir Munk 					 */
25952eb4d010SOphir Munk 					continue;
25962eb4d010SOphir Munk 				}
25972eb4d010SOphir Munk 			}
25982eb4d010SOphir Munk 			ret = -1;
25992eb4d010SOphir Munk 			if (nl_route >= 0)
2600ca1418ceSMichael Baum 				ret = mlx5_nl_switch_info(nl_route,
26012eb4d010SOphir Munk 							  list[ns].ifindex,
26022eb4d010SOphir Munk 							  &list[ns].info);
26032eb4d010SOphir Munk 			if (ret || (!list[ns].info.representor &&
26042eb4d010SOphir Munk 				    !list[ns].info.master)) {
26052eb4d010SOphir Munk 				/*
26062eb4d010SOphir Munk 				 * We failed to recognize representors with
26072eb4d010SOphir Munk 				 * Netlink, let's try to perform the task
26082eb4d010SOphir Munk 				 * with sysfs.
26092eb4d010SOphir Munk 				 */
2610887183efSMichael Baum 				ret = mlx5_sysfs_switch_info(list[ns].ifindex,
26112eb4d010SOphir Munk 							     &list[ns].info);
26122eb4d010SOphir Munk 			}
26132eb4d010SOphir Munk 			if (!ret && (list[ns].info.representor ^
26142eb4d010SOphir Munk 				     list[ns].info.master)) {
26152eb4d010SOphir Munk 				ns++;
26162eb4d010SOphir Munk 			} else if ((nd == 1) &&
26172eb4d010SOphir Munk 				   !list[ns].info.representor &&
26182eb4d010SOphir Munk 				   !list[ns].info.master) {
26192eb4d010SOphir Munk 				/*
2620887183efSMichael Baum 				 * Single IB device with one physical port and
26212eb4d010SOphir Munk 				 * attached network device.
2622887183efSMichael Baum 				 * May be SRIOV is not enabled or there is no
2623887183efSMichael Baum 				 * representors.
26242eb4d010SOphir Munk 				 */
2625887183efSMichael Baum 				DRV_LOG(INFO, "No E-Switch support detected.");
26262eb4d010SOphir Munk 				ns++;
26272eb4d010SOphir Munk 				break;
26282eb4d010SOphir Munk 			}
26292eb4d010SOphir Munk 		}
26302eb4d010SOphir Munk 		if (!ns) {
26312eb4d010SOphir Munk 			DRV_LOG(ERR,
2632887183efSMichael Baum 				"Unable to recognize master/representors on the multiple IB devices.");
26332eb4d010SOphir Munk 			rte_errno = ENOENT;
26342eb4d010SOphir Munk 			ret = -rte_errno;
26352eb4d010SOphir Munk 			goto exit;
26362eb4d010SOphir Munk 		}
26376b157f3bSViacheslav Ovsiienko 		/*
26386b157f3bSViacheslav Ovsiienko 		 * New kernels may add the switch_id attribute for the case
2639ca1418ceSMichael Baum 		 * there is no E-Switch and we wrongly recognized the only
2640ca1418ceSMichael Baum 		 * device as master. Override this if there is the single
2641ca1418ceSMichael Baum 		 * device with single port and new device name format present.
26426b157f3bSViacheslav Ovsiienko 		 */
26436b157f3bSViacheslav Ovsiienko 		if (nd == 1 &&
26446b157f3bSViacheslav Ovsiienko 		    list[0].info.name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK) {
26456b157f3bSViacheslav Ovsiienko 			list[0].info.master = 0;
26466b157f3bSViacheslav Ovsiienko 			list[0].info.representor = 0;
26476b157f3bSViacheslav Ovsiienko 		}
26482eb4d010SOphir Munk 	}
26492eb4d010SOphir Munk 	MLX5_ASSERT(ns);
26502eb4d010SOphir Munk 	/*
26512eb4d010SOphir Munk 	 * Sort list to probe devices in natural order for users convenience
26522eb4d010SOphir Munk 	 * (i.e. master first, then representors from lowest to highest ID).
26532eb4d010SOphir Munk 	 */
26542eb4d010SOphir Munk 	qsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp);
2655f926cce3SXueming Li 	if (eth_da.type != RTE_ETH_REPRESENTOR_NONE) {
2656f926cce3SXueming Li 		/* Set devargs default values. */
2657f926cce3SXueming Li 		if (eth_da.nb_mh_controllers == 0) {
2658f926cce3SXueming Li 			eth_da.nb_mh_controllers = 1;
2659f926cce3SXueming Li 			eth_da.mh_controllers[0] = 0;
2660f926cce3SXueming Li 		}
2661f926cce3SXueming Li 		if (eth_da.nb_ports == 0 && ns > 0) {
2662f926cce3SXueming Li 			if (list[0].pf_bond >= 0 && list[0].info.representor)
2663f926cce3SXueming Li 				DRV_LOG(WARNING, "Representor on Bonding device should use pf#vf# syntax: %s",
2664f926cce3SXueming Li 					pci_dev->device.devargs->args);
2665f926cce3SXueming Li 			eth_da.nb_ports = 1;
2666f926cce3SXueming Li 			eth_da.ports[0] = list[0].info.pf_num;
2667f926cce3SXueming Li 		}
2668f926cce3SXueming Li 		if (eth_da.nb_representor_ports == 0) {
2669f926cce3SXueming Li 			eth_da.nb_representor_ports = 1;
2670f926cce3SXueming Li 			eth_da.representor_ports[0] = 0;
2671f926cce3SXueming Li 		}
2672f926cce3SXueming Li 	}
26732eb4d010SOphir Munk 	for (i = 0; i != ns; ++i) {
26742eb4d010SOphir Munk 		uint32_t restore;
26752eb4d010SOphir Munk 
2676a729d2f0SMichael Baum 		list[i].eth_dev = mlx5_dev_spawn(cdev->dev, &list[i], &eth_da,
2677a729d2f0SMichael Baum 						 mkvlist);
26782eb4d010SOphir Munk 		if (!list[i].eth_dev) {
26792eb4d010SOphir Munk 			if (rte_errno != EBUSY && rte_errno != EEXIST)
26802eb4d010SOphir Munk 				break;
26812eb4d010SOphir Munk 			/* Device is disabled or already spawned. Ignore it. */
26822eb4d010SOphir Munk 			continue;
26832eb4d010SOphir Munk 		}
26842eb4d010SOphir Munk 		restore = list[i].eth_dev->data->dev_flags;
26852eb4d010SOphir Munk 		rte_eth_copy_pci_info(list[i].eth_dev, pci_dev);
2686494d6863SGregory Etelson 		/**
2687494d6863SGregory Etelson 		 * Each representor has a dedicated interrupts vector.
2688494d6863SGregory Etelson 		 * rte_eth_copy_pci_info() assigns PF interrupts handle to
2689494d6863SGregory Etelson 		 * representor eth_dev object because representor and PF
2690494d6863SGregory Etelson 		 * share the same PCI address.
2691494d6863SGregory Etelson 		 * Override representor device with a dedicated
2692494d6863SGregory Etelson 		 * interrupts handle here.
2693494d6863SGregory Etelson 		 * Representor interrupts handle is released in mlx5_dev_stop().
2694494d6863SGregory Etelson 		 */
2695494d6863SGregory Etelson 		if (list[i].info.representor) {
2696d61138d4SHarman Kalra 			struct rte_intr_handle *intr_handle =
2697d61138d4SHarman Kalra 				rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
2698d61138d4SHarman Kalra 			if (intr_handle == NULL) {
2699494d6863SGregory Etelson 				DRV_LOG(ERR,
2700494d6863SGregory Etelson 					"port %u failed to allocate memory for interrupt handler "
2701494d6863SGregory Etelson 					"Rx interrupts will not be supported",
2702494d6863SGregory Etelson 					i);
2703494d6863SGregory Etelson 				rte_errno = ENOMEM;
2704494d6863SGregory Etelson 				ret = -rte_errno;
2705494d6863SGregory Etelson 				goto exit;
2706494d6863SGregory Etelson 			}
2707494d6863SGregory Etelson 			list[i].eth_dev->intr_handle = intr_handle;
2708494d6863SGregory Etelson 		}
27092eb4d010SOphir Munk 		/* Restore non-PCI flags cleared by the above call. */
27102eb4d010SOphir Munk 		list[i].eth_dev->data->dev_flags |= restore;
27112eb4d010SOphir Munk 		rte_eth_dev_probing_finish(list[i].eth_dev);
27122eb4d010SOphir Munk 	}
27132eb4d010SOphir Munk 	if (i != ns) {
27142eb4d010SOphir Munk 		DRV_LOG(ERR,
27152eb4d010SOphir Munk 			"probe of PCI device " PCI_PRI_FMT " aborted after"
27162eb4d010SOphir Munk 			" encountering an error: %s",
2717f926cce3SXueming Li 			owner_pci.domain, owner_pci.bus,
2718f926cce3SXueming Li 			owner_pci.devid, owner_pci.function,
27192eb4d010SOphir Munk 			strerror(rte_errno));
27202eb4d010SOphir Munk 		ret = -rte_errno;
27212eb4d010SOphir Munk 		/* Roll back. */
27222eb4d010SOphir Munk 		while (i--) {
27232eb4d010SOphir Munk 			if (!list[i].eth_dev)
27242eb4d010SOphir Munk 				continue;
27252eb4d010SOphir Munk 			mlx5_dev_close(list[i].eth_dev);
27262eb4d010SOphir Munk 			/* mac_addrs must not be freed because in dev_private */
27272eb4d010SOphir Munk 			list[i].eth_dev->data->mac_addrs = NULL;
27282eb4d010SOphir Munk 			claim_zero(rte_eth_dev_release_port(list[i].eth_dev));
27292eb4d010SOphir Munk 		}
27302eb4d010SOphir Munk 		/* Restore original error. */
27312eb4d010SOphir Munk 		rte_errno = -ret;
27322eb4d010SOphir Munk 	} else {
27332eb4d010SOphir Munk 		ret = 0;
27342eb4d010SOphir Munk 	}
27352eb4d010SOphir Munk exit:
27362eb4d010SOphir Munk 	/*
27372eb4d010SOphir Munk 	 * Do the routine cleanup:
27382eb4d010SOphir Munk 	 * - close opened Netlink sockets
27392eb4d010SOphir Munk 	 * - free allocated spawn data array
27402eb4d010SOphir Munk 	 * - free the Infiniband device list
27412eb4d010SOphir Munk 	 */
27422eb4d010SOphir Munk 	if (nl_rdma >= 0)
27432eb4d010SOphir Munk 		close(nl_rdma);
27442eb4d010SOphir Munk 	if (nl_route >= 0)
27452eb4d010SOphir Munk 		close(nl_route);
27462eb4d010SOphir Munk 	if (list)
27472175c4dcSSuanming Mou 		mlx5_free(list);
27482eb4d010SOphir Munk 	MLX5_ASSERT(ibv_list);
27492eb4d010SOphir Munk 	mlx5_glue->free_device_list(ibv_list);
27502eb4d010SOphir Munk 	return ret;
27512eb4d010SOphir Munk }
27522eb4d010SOphir Munk 
2753919488fbSXueming Li static int
2754919488fbSXueming Li mlx5_os_parse_eth_devargs(struct rte_device *dev,
2755919488fbSXueming Li 			  struct rte_eth_devargs *eth_da)
2756919488fbSXueming Li {
2757919488fbSXueming Li 	int ret = 0;
2758919488fbSXueming Li 
2759919488fbSXueming Li 	if (dev->devargs == NULL)
2760919488fbSXueming Li 		return 0;
2761919488fbSXueming Li 	memset(eth_da, 0, sizeof(*eth_da));
2762919488fbSXueming Li 	/* Parse representor information first from class argument. */
2763919488fbSXueming Li 	if (dev->devargs->cls_str)
27649a9eb104SHarman Kalra 		ret = rte_eth_devargs_parse(dev->devargs->cls_str, eth_da, 1);
27659a9eb104SHarman Kalra 	if (ret < 0) {
2766919488fbSXueming Li 		DRV_LOG(ERR, "failed to parse device arguments: %s",
2767919488fbSXueming Li 			dev->devargs->cls_str);
2768919488fbSXueming Li 		return -rte_errno;
2769919488fbSXueming Li 	}
2770c2e3b84eSMichael Baum 	if (eth_da->type == RTE_ETH_REPRESENTOR_NONE && dev->devargs->args) {
2771919488fbSXueming Li 		/* Parse legacy device argument */
27729a9eb104SHarman Kalra 		ret = rte_eth_devargs_parse(dev->devargs->args, eth_da, 1);
27739a9eb104SHarman Kalra 		if (ret < 0) {
2774919488fbSXueming Li 			DRV_LOG(ERR, "failed to parse device arguments: %s",
2775919488fbSXueming Li 				dev->devargs->args);
2776919488fbSXueming Li 			return -rte_errno;
2777919488fbSXueming Li 		}
2778919488fbSXueming Li 	}
2779919488fbSXueming Li 	return 0;
2780919488fbSXueming Li }
2781919488fbSXueming Li 
278208c2772fSXueming Li /**
2783a7f34989SXueming Li  * Callback to register a PCI device.
278408c2772fSXueming Li  *
278508c2772fSXueming Li  * This function spawns Ethernet devices out of a given PCI device.
278608c2772fSXueming Li  *
27877af08c8fSMichael Baum  * @param[in] cdev
27887af08c8fSMichael Baum  *   Pointer to common mlx5 device structure.
2789a729d2f0SMichael Baum  * @param[in, out] mkvlist
2790a729d2f0SMichael Baum  *   Pointer to mlx5 kvargs control, can be NULL if there is no devargs.
279108c2772fSXueming Li  *
279208c2772fSXueming Li  * @return
279308c2772fSXueming Li  *   0 on success, a negative errno value otherwise and rte_errno is set.
279408c2772fSXueming Li  */
2795a7f34989SXueming Li static int
2796a729d2f0SMichael Baum mlx5_os_pci_probe(struct mlx5_common_device *cdev,
2797a729d2f0SMichael Baum 		  struct mlx5_kvargs_ctrl *mkvlist)
279808c2772fSXueming Li {
27997af08c8fSMichael Baum 	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(cdev->dev);
2800919488fbSXueming Li 	struct rte_eth_devargs eth_da = { .nb_ports = 0 };
280108c2772fSXueming Li 	int ret = 0;
280208c2772fSXueming Li 	uint16_t p;
280308c2772fSXueming Li 
28047af08c8fSMichael Baum 	ret = mlx5_os_parse_eth_devargs(cdev->dev, &eth_da);
2805919488fbSXueming Li 	if (ret != 0)
2806919488fbSXueming Li 		return ret;
280708c2772fSXueming Li 
280808c2772fSXueming Li 	if (eth_da.nb_ports > 0) {
280908c2772fSXueming Li 		/* Iterate all port if devargs pf is range: "pf[0-1]vf[...]". */
28106856efa5SMichael Baum 		for (p = 0; p < eth_da.nb_ports; p++) {
2811ca1418ceSMichael Baum 			ret = mlx5_os_pci_probe_pf(cdev, &eth_da,
2812a729d2f0SMichael Baum 						   eth_da.ports[p], mkvlist);
28136856efa5SMichael Baum 			if (ret) {
2814f956d3d4SRongwei Liu 				DRV_LOG(INFO, "Probe of PCI device " PCI_PRI_FMT " "
2815f956d3d4SRongwei Liu 					"aborted due to proding failure of PF %u",
28166856efa5SMichael Baum 					pci_dev->addr.domain, pci_dev->addr.bus,
28176856efa5SMichael Baum 					pci_dev->addr.devid, pci_dev->addr.function,
28186856efa5SMichael Baum 					eth_da.ports[p]);
28197af08c8fSMichael Baum 				mlx5_net_remove(cdev);
2820f956d3d4SRongwei Liu 				if (p != 0)
2821f956d3d4SRongwei Liu 					break;
2822f956d3d4SRongwei Liu 			}
28236856efa5SMichael Baum 		}
282408c2772fSXueming Li 	} else {
2825a729d2f0SMichael Baum 		ret = mlx5_os_pci_probe_pf(cdev, &eth_da, 0, mkvlist);
282608c2772fSXueming Li 	}
282708c2772fSXueming Li 	return ret;
282808c2772fSXueming Li }
282908c2772fSXueming Li 
2830919488fbSXueming Li /* Probe a single SF device on auxiliary bus, no representor support. */
2831919488fbSXueming Li static int
2832a729d2f0SMichael Baum mlx5_os_auxiliary_probe(struct mlx5_common_device *cdev,
2833a729d2f0SMichael Baum 			struct mlx5_kvargs_ctrl *mkvlist)
2834919488fbSXueming Li {
2835919488fbSXueming Li 	struct rte_eth_devargs eth_da = { .nb_ports = 0 };
283611c73de9SDariusz Sosnowski 	struct mlx5_dev_spawn_data spawn = {
283711c73de9SDariusz Sosnowski 		.pf_bond = -1,
283811c73de9SDariusz Sosnowski 		.mpesw_port = MLX5_MPESW_PORT_INVALID,
283911c73de9SDariusz Sosnowski 	};
28407af08c8fSMichael Baum 	struct rte_device *dev = cdev->dev;
2841919488fbSXueming Li 	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
2842919488fbSXueming Li 	struct rte_eth_dev *eth_dev;
2843919488fbSXueming Li 	int ret = 0;
2844919488fbSXueming Li 
2845919488fbSXueming Li 	/* Parse ethdev devargs. */
2846919488fbSXueming Li 	ret = mlx5_os_parse_eth_devargs(dev, &eth_da);
2847919488fbSXueming Li 	if (ret != 0)
2848919488fbSXueming Li 		return ret;
2849919488fbSXueming Li 	/* Init spawn data. */
2850919488fbSXueming Li 	spawn.max_port = 1;
2851919488fbSXueming Li 	spawn.phys_port = 1;
2852ca1418ceSMichael Baum 	spawn.phys_dev_name = mlx5_os_get_ctx_device_name(cdev->ctx);
2853919488fbSXueming Li 	ret = mlx5_auxiliary_get_ifindex(dev->name);
2854919488fbSXueming Li 	if (ret < 0) {
2855919488fbSXueming Li 		DRV_LOG(ERR, "failed to get ethdev ifindex: %s", dev->name);
2856919488fbSXueming Li 		return ret;
2857919488fbSXueming Li 	}
2858919488fbSXueming Li 	spawn.ifindex = ret;
28597af08c8fSMichael Baum 	spawn.cdev = cdev;
2860919488fbSXueming Li 	/* Spawn device. */
2861a729d2f0SMichael Baum 	eth_dev = mlx5_dev_spawn(dev, &spawn, &eth_da, mkvlist);
2862919488fbSXueming Li 	if (eth_dev == NULL)
2863919488fbSXueming Li 		return -rte_errno;
2864919488fbSXueming Li 	/* Post create. */
2865d61138d4SHarman Kalra 	eth_dev->intr_handle = adev->intr_handle;
2866919488fbSXueming Li 	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2867919488fbSXueming Li 		eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
2868919488fbSXueming Li 		eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_RMV;
2869919488fbSXueming Li 		eth_dev->data->numa_node = dev->numa_node;
2870919488fbSXueming Li 	}
2871919488fbSXueming Li 	rte_eth_dev_probing_finish(eth_dev);
2872919488fbSXueming Li 	return 0;
2873919488fbSXueming Li }
2874919488fbSXueming Li 
2875a7f34989SXueming Li /**
2876a7f34989SXueming Li  * Net class driver callback to probe a device.
2877a7f34989SXueming Li  *
2878919488fbSXueming Li  * This function probe PCI bus device(s) or a single SF on auxiliary bus.
2879a7f34989SXueming Li  *
28807af08c8fSMichael Baum  * @param[in] cdev
28817af08c8fSMichael Baum  *   Pointer to the common mlx5 device.
2882a729d2f0SMichael Baum  * @param[in, out] mkvlist
2883a729d2f0SMichael Baum  *   Pointer to mlx5 kvargs control, can be NULL if there is no devargs.
2884a7f34989SXueming Li  *
2885a7f34989SXueming Li  * @return
28867af08c8fSMichael Baum  *   0 on success, a negative errno value otherwise and rte_errno is set.
2887a7f34989SXueming Li  */
2888a7f34989SXueming Li int
2889a729d2f0SMichael Baum mlx5_os_net_probe(struct mlx5_common_device *cdev,
2890a729d2f0SMichael Baum 		  struct mlx5_kvargs_ctrl *mkvlist)
2891a7f34989SXueming Li {
2892a7f34989SXueming Li 	int ret;
2893a7f34989SXueming Li 
2894ca1418ceSMichael Baum 	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
2895a7f34989SXueming Li 		mlx5_pmd_socket_init();
2896a7f34989SXueming Li 	ret = mlx5_init_once();
2897a7f34989SXueming Li 	if (ret) {
28987af08c8fSMichael Baum 		DRV_LOG(ERR, "Unable to init PMD global data: %s",
2899a7f34989SXueming Li 			strerror(rte_errno));
2900a7f34989SXueming Li 		return -rte_errno;
2901a7f34989SXueming Li 	}
2902a729d2f0SMichael Baum 	ret = mlx5_probe_again_args_validate(cdev, mkvlist);
2903a13ec19cSMichael Baum 	if (ret) {
2904a13ec19cSMichael Baum 		DRV_LOG(ERR, "Probe again parameters are not compatible : %s",
2905a13ec19cSMichael Baum 			strerror(rte_errno));
2906a13ec19cSMichael Baum 		return -rte_errno;
2907a13ec19cSMichael Baum 	}
29087af08c8fSMichael Baum 	if (mlx5_dev_is_pci(cdev->dev))
2909a729d2f0SMichael Baum 		return mlx5_os_pci_probe(cdev, mkvlist);
2910919488fbSXueming Li 	else
2911a729d2f0SMichael Baum 		return mlx5_os_auxiliary_probe(cdev, mkvlist);
29122eb4d010SOphir Munk }
29132eb4d010SOphir Munk 
29142eb4d010SOphir Munk /**
2915ea823b2cSDmitry Kozlyuk  * Cleanup resources when the last device is closed.
2916ea823b2cSDmitry Kozlyuk  */
2917ea823b2cSDmitry Kozlyuk void
2918ea823b2cSDmitry Kozlyuk mlx5_os_net_cleanup(void)
2919ea823b2cSDmitry Kozlyuk {
2920ea823b2cSDmitry Kozlyuk 	mlx5_pmd_socket_uninit();
2921ea823b2cSDmitry Kozlyuk }
2922ea823b2cSDmitry Kozlyuk 
2923ea823b2cSDmitry Kozlyuk /**
29242eb4d010SOphir Munk  * Install shared asynchronous device events handler.
29252eb4d010SOphir Munk  * This function is implemented to support event sharing
29262eb4d010SOphir Munk  * between multiple ports of single IB device.
29272eb4d010SOphir Munk  *
29282eb4d010SOphir Munk  * @param sh
29292eb4d010SOphir Munk  *   Pointer to mlx5_dev_ctx_shared object.
29302eb4d010SOphir Munk  */
29312eb4d010SOphir Munk void
29322eb4d010SOphir Munk mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh)
29332eb4d010SOphir Munk {
2934ca1418ceSMichael Baum 	struct ibv_context *ctx = sh->cdev->ctx;
293572d7efe4SSpike Du 	int nlsk_fd;
29362eb4d010SOphir Munk 
293772d7efe4SSpike Du 	sh->intr_handle = mlx5_os_interrupt_handler_create
293872d7efe4SSpike Du 		(RTE_INTR_INSTANCE_F_SHARED, true,
293972d7efe4SSpike Du 		 ctx->async_fd, mlx5_dev_interrupt_handler, sh);
294072d7efe4SSpike Du 	if (!sh->intr_handle) {
294172d7efe4SSpike Du 		DRV_LOG(ERR, "Failed to allocate intr_handle.");
2942d61138d4SHarman Kalra 		return;
2943d61138d4SHarman Kalra 	}
294472d7efe4SSpike Du 	nlsk_fd = mlx5_nl_init(NETLINK_ROUTE, RTMGRP_LINK);
294572d7efe4SSpike Du 	if (nlsk_fd < 0) {
294672d7efe4SSpike Du 		DRV_LOG(ERR, "Failed to create a socket for Netlink events: %s",
294772d7efe4SSpike Du 			rte_strerror(rte_errno));
294872d7efe4SSpike Du 		return;
29492eb4d010SOphir Munk 	}
295072d7efe4SSpike Du 	sh->intr_handle_nl = mlx5_os_interrupt_handler_create
295172d7efe4SSpike Du 		(RTE_INTR_INSTANCE_F_SHARED, true,
295272d7efe4SSpike Du 		 nlsk_fd, mlx5_dev_interrupt_handler_nl, sh);
295317f95513SDmitry Kozlyuk 	if (sh->intr_handle_nl == NULL) {
295417f95513SDmitry Kozlyuk 		DRV_LOG(ERR, "Fail to allocate intr_handle");
295517f95513SDmitry Kozlyuk 		return;
295617f95513SDmitry Kozlyuk 	}
29576dc0cbc6SMichael Baum 	if (sh->cdev->config.devx) {
29582eb4d010SOphir Munk #ifdef HAVE_IBV_DEVX_ASYNC
295972d7efe4SSpike Du 		struct mlx5dv_devx_cmd_comp *devx_comp;
296072d7efe4SSpike Du 
2961ca1418ceSMichael Baum 		sh->devx_comp = (void *)mlx5_glue->devx_create_cmd_comp(ctx);
296272d7efe4SSpike Du 		devx_comp = sh->devx_comp;
296321b7c452SOphir Munk 		if (!devx_comp) {
29642eb4d010SOphir Munk 			DRV_LOG(INFO, "failed to allocate devx_comp.");
29652eb4d010SOphir Munk 			return;
29662eb4d010SOphir Munk 		}
296772d7efe4SSpike Du 		sh->intr_handle_devx = mlx5_os_interrupt_handler_create
296872d7efe4SSpike Du 			(RTE_INTR_INSTANCE_F_SHARED, true,
296972d7efe4SSpike Du 			 devx_comp->fd,
297072d7efe4SSpike Du 			 mlx5_dev_interrupt_handler_devx, sh);
297172d7efe4SSpike Du 		if (!sh->intr_handle_devx) {
297272d7efe4SSpike Du 			DRV_LOG(ERR, "Failed to allocate intr_handle.");
29732eb4d010SOphir Munk 			return;
29742eb4d010SOphir Munk 		}
29752eb4d010SOphir Munk #endif /* HAVE_IBV_DEVX_ASYNC */
29762eb4d010SOphir Munk 	}
29772eb4d010SOphir Munk }
29782eb4d010SOphir Munk 
29792eb4d010SOphir Munk /**
29802eb4d010SOphir Munk  * Uninstall shared asynchronous device events handler.
29812eb4d010SOphir Munk  * This function is implemented to support event sharing
29822eb4d010SOphir Munk  * between multiple ports of single IB device.
29832eb4d010SOphir Munk  *
29842eb4d010SOphir Munk  * @param dev
29852eb4d010SOphir Munk  *   Pointer to mlx5_dev_ctx_shared object.
29862eb4d010SOphir Munk  */
29872eb4d010SOphir Munk void
29882eb4d010SOphir Munk mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh)
29892eb4d010SOphir Munk {
299072d7efe4SSpike Du 	mlx5_os_interrupt_handler_destroy(sh->intr_handle,
29912eb4d010SOphir Munk 					  mlx5_dev_interrupt_handler, sh);
299272d7efe4SSpike Du 	mlx5_os_interrupt_handler_destroy(sh->intr_handle_nl,
299372d7efe4SSpike Du 					  mlx5_dev_interrupt_handler_nl, sh);
29942eb4d010SOphir Munk #ifdef HAVE_IBV_DEVX_ASYNC
299572d7efe4SSpike Du 	mlx5_os_interrupt_handler_destroy(sh->intr_handle_devx,
29962eb4d010SOphir Munk 					  mlx5_dev_interrupt_handler_devx, sh);
29972eb4d010SOphir Munk 	if (sh->devx_comp)
29982eb4d010SOphir Munk 		mlx5_glue->devx_destroy_cmd_comp(sh->devx_comp);
29992eb4d010SOphir Munk #endif
30002eb4d010SOphir Munk }
3001042f5c94SOphir Munk 
300273bf9235SOphir Munk /**
300373bf9235SOphir Munk  * Read statistics by a named counter.
300473bf9235SOphir Munk  *
300573bf9235SOphir Munk  * @param[in] priv
300673bf9235SOphir Munk  *   Pointer to the private device data structure.
300773bf9235SOphir Munk  * @param[in] ctr_name
300873bf9235SOphir Munk  *   Pointer to the name of the statistic counter to read
300973bf9235SOphir Munk  * @param[out] stat
301073bf9235SOphir Munk  *   Pointer to read statistic value.
301173bf9235SOphir Munk  * @return
301273bf9235SOphir Munk  *   0 on success and stat is valud, 1 if failed to read the value
301373bf9235SOphir Munk  *   rte_errno is set.
301473bf9235SOphir Munk  *
301573bf9235SOphir Munk  */
301673bf9235SOphir Munk int
301773bf9235SOphir Munk mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name,
301873bf9235SOphir Munk 		      uint64_t *stat)
301973bf9235SOphir Munk {
302073bf9235SOphir Munk 	int fd;
302173bf9235SOphir Munk 
302273bf9235SOphir Munk 	if (priv->sh) {
3023e6988afdSMatan Azrad 		if (priv->q_counters != NULL &&
3024d312cab5SRongwei Liu 		    strcmp(ctr_name, "out_of_buffer") == 0) {
3025d312cab5SRongwei Liu 			if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
3026d312cab5SRongwei Liu 				DRV_LOG(WARNING, "Devx out_of_buffer counter is not supported in the secondary process");
3027d312cab5SRongwei Liu 				rte_errno = ENOTSUP;
3028d312cab5SRongwei Liu 				return 1;
3029d312cab5SRongwei Liu 			}
3030978a0303SViacheslav Ovsiienko 			return mlx5_devx_cmd_queue_counter_query
3031978a0303SViacheslav Ovsiienko 					(priv->q_counters, 0, (uint32_t *)stat);
3032d312cab5SRongwei Liu 		}
303373bf9235SOphir Munk 		MKSTR(path, "%s/ports/%d/hw_counters/%s",
303473bf9235SOphir Munk 		      priv->sh->ibdev_path,
303573bf9235SOphir Munk 		      priv->dev_port,
303673bf9235SOphir Munk 		      ctr_name);
303773bf9235SOphir Munk 		fd = open(path, O_RDONLY);
3038038e7fc0SShy Shyman 		/*
3039038e7fc0SShy Shyman 		 * in switchdev the file location is not per port
3040038e7fc0SShy Shyman 		 * but rather in <ibdev_path>/hw_counters/<file_name>.
3041038e7fc0SShy Shyman 		 */
3042038e7fc0SShy Shyman 		if (fd == -1) {
3043038e7fc0SShy Shyman 			MKSTR(path1, "%s/hw_counters/%s",
3044038e7fc0SShy Shyman 			      priv->sh->ibdev_path,
3045038e7fc0SShy Shyman 			      ctr_name);
3046038e7fc0SShy Shyman 			fd = open(path1, O_RDONLY);
3047038e7fc0SShy Shyman 		}
304873bf9235SOphir Munk 		if (fd != -1) {
304973bf9235SOphir Munk 			char buf[21] = {'\0'};
305073bf9235SOphir Munk 			ssize_t n = read(fd, buf, sizeof(buf));
305173bf9235SOphir Munk 
305273bf9235SOphir Munk 			close(fd);
305373bf9235SOphir Munk 			if (n != -1) {
305473bf9235SOphir Munk 				*stat = strtoull(buf, NULL, 10);
305573bf9235SOphir Munk 				return 0;
305673bf9235SOphir Munk 			}
305773bf9235SOphir Munk 		}
305873bf9235SOphir Munk 	}
305973bf9235SOphir Munk 	*stat = 0;
306073bf9235SOphir Munk 	return 1;
306173bf9235SOphir Munk }
306273bf9235SOphir Munk 
306373bf9235SOphir Munk /**
3064ab27cdd9SOphir Munk  * Remove a MAC address from device
3065ab27cdd9SOphir Munk  *
3066ab27cdd9SOphir Munk  * @param dev
3067ab27cdd9SOphir Munk  *   Pointer to Ethernet device structure.
3068ab27cdd9SOphir Munk  * @param index
3069ab27cdd9SOphir Munk  *   MAC address index.
3070ab27cdd9SOphir Munk  */
3071ab27cdd9SOphir Munk void
3072ab27cdd9SOphir Munk mlx5_os_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
3073ab27cdd9SOphir Munk {
3074ab27cdd9SOphir Munk 	struct mlx5_priv *priv = dev->data->dev_private;
307587af0d1eSMichael Baum 	const int vf = priv->sh->dev_cap.vf;
3076ab27cdd9SOphir Munk 
3077ab27cdd9SOphir Munk 	if (vf)
3078ab27cdd9SOphir Munk 		mlx5_nl_mac_addr_remove(priv->nl_socket_route,
3079ab27cdd9SOphir Munk 					mlx5_ifindex(dev), priv->mac_own,
3080ab27cdd9SOphir Munk 					&dev->data->mac_addrs[index], index);
3081ab27cdd9SOphir Munk }
3082ab27cdd9SOphir Munk 
3083ab27cdd9SOphir Munk /**
3084ab27cdd9SOphir Munk  * Adds a MAC address to the device
3085ab27cdd9SOphir Munk  *
3086ab27cdd9SOphir Munk  * @param dev
3087ab27cdd9SOphir Munk  *   Pointer to Ethernet device structure.
3088ab27cdd9SOphir Munk  * @param mac_addr
3089ab27cdd9SOphir Munk  *   MAC address to register.
3090ab27cdd9SOphir Munk  * @param index
3091ab27cdd9SOphir Munk  *   MAC address index.
3092ab27cdd9SOphir Munk  *
3093ab27cdd9SOphir Munk  * @return
3094ab27cdd9SOphir Munk  *   0 on success, a negative errno value otherwise
3095ab27cdd9SOphir Munk  */
3096ab27cdd9SOphir Munk int
3097ab27cdd9SOphir Munk mlx5_os_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac,
3098ab27cdd9SOphir Munk 		     uint32_t index)
3099ab27cdd9SOphir Munk {
3100ab27cdd9SOphir Munk 	struct mlx5_priv *priv = dev->data->dev_private;
310187af0d1eSMichael Baum 	const int vf = priv->sh->dev_cap.vf;
3102ab27cdd9SOphir Munk 	int ret = 0;
3103ab27cdd9SOphir Munk 
3104ab27cdd9SOphir Munk 	if (vf)
3105ab27cdd9SOphir Munk 		ret = mlx5_nl_mac_addr_add(priv->nl_socket_route,
3106ab27cdd9SOphir Munk 					   mlx5_ifindex(dev), priv->mac_own,
3107ab27cdd9SOphir Munk 					   mac, index);
3108ab27cdd9SOphir Munk 	return ret;
3109ab27cdd9SOphir Munk }
3110ab27cdd9SOphir Munk 
3111ab27cdd9SOphir Munk /**
3112ab27cdd9SOphir Munk  * Modify a VF MAC address
3113ab27cdd9SOphir Munk  *
3114ab27cdd9SOphir Munk  * @param priv
3115ab27cdd9SOphir Munk  *   Pointer to device private data.
3116ab27cdd9SOphir Munk  * @param mac_addr
3117ab27cdd9SOphir Munk  *   MAC address to modify into.
3118ab27cdd9SOphir Munk  * @param iface_idx
3119ab27cdd9SOphir Munk  *   Net device interface index
3120ab27cdd9SOphir Munk  * @param vf_index
3121ab27cdd9SOphir Munk  *   VF index
3122ab27cdd9SOphir Munk  *
3123ab27cdd9SOphir Munk  * @return
3124ab27cdd9SOphir Munk  *   0 on success, a negative errno value otherwise
3125ab27cdd9SOphir Munk  */
3126ab27cdd9SOphir Munk int
3127ab27cdd9SOphir Munk mlx5_os_vf_mac_addr_modify(struct mlx5_priv *priv,
3128ab27cdd9SOphir Munk 			   unsigned int iface_idx,
3129ab27cdd9SOphir Munk 			   struct rte_ether_addr *mac_addr,
3130ab27cdd9SOphir Munk 			   int vf_index)
3131ab27cdd9SOphir Munk {
3132ab27cdd9SOphir Munk 	return mlx5_nl_vf_mac_addr_modify
3133ab27cdd9SOphir Munk 		(priv->nl_socket_route, iface_idx, mac_addr, vf_index);
3134ab27cdd9SOphir Munk }
3135ab27cdd9SOphir Munk 
31364d18abd1SOphir Munk /**
31374d18abd1SOphir Munk  * Set device promiscuous mode
31384d18abd1SOphir Munk  *
31394d18abd1SOphir Munk  * @param dev
31404d18abd1SOphir Munk  *   Pointer to Ethernet device structure.
31414d18abd1SOphir Munk  * @param enable
31424d18abd1SOphir Munk  *   0 - promiscuous is disabled, otherwise - enabled
31434d18abd1SOphir Munk  *
31444d18abd1SOphir Munk  * @return
31454d18abd1SOphir Munk  *   0 on success, a negative error value otherwise
31464d18abd1SOphir Munk  */
31474d18abd1SOphir Munk int
31484d18abd1SOphir Munk mlx5_os_set_promisc(struct rte_eth_dev *dev, int enable)
31494d18abd1SOphir Munk {
31504d18abd1SOphir Munk 	struct mlx5_priv *priv = dev->data->dev_private;
31514d18abd1SOphir Munk 
31524d18abd1SOphir Munk 	return mlx5_nl_promisc(priv->nl_socket_route,
31534d18abd1SOphir Munk 			       mlx5_ifindex(dev), !!enable);
31544d18abd1SOphir Munk }
31554d18abd1SOphir Munk 
31564d18abd1SOphir Munk /**
31574d18abd1SOphir Munk  * Set device promiscuous mode
31584d18abd1SOphir Munk  *
31594d18abd1SOphir Munk  * @param dev
31604d18abd1SOphir Munk  *   Pointer to Ethernet device structure.
31614d18abd1SOphir Munk  * @param enable
31624d18abd1SOphir Munk  *   0 - all multicase is disabled, otherwise - enabled
31634d18abd1SOphir Munk  *
31644d18abd1SOphir Munk  * @return
31654d18abd1SOphir Munk  *   0 on success, a negative error value otherwise
31664d18abd1SOphir Munk  */
31674d18abd1SOphir Munk int
31684d18abd1SOphir Munk mlx5_os_set_allmulti(struct rte_eth_dev *dev, int enable)
31694d18abd1SOphir Munk {
31704d18abd1SOphir Munk 	struct mlx5_priv *priv = dev->data->dev_private;
31714d18abd1SOphir Munk 
31724d18abd1SOphir Munk 	return mlx5_nl_allmulti(priv->nl_socket_route,
31734d18abd1SOphir Munk 				mlx5_ifindex(dev), !!enable);
31744d18abd1SOphir Munk }
31754d18abd1SOphir Munk 
3176f00f6562SOphir Munk /**
3177f00f6562SOphir Munk  * Flush device MAC addresses
3178f00f6562SOphir Munk  *
3179f00f6562SOphir Munk  * @param dev
3180f00f6562SOphir Munk  *   Pointer to Ethernet device structure.
3181f00f6562SOphir Munk  *
3182f00f6562SOphir Munk  */
3183f00f6562SOphir Munk void
3184f00f6562SOphir Munk mlx5_os_mac_addr_flush(struct rte_eth_dev *dev)
3185f00f6562SOphir Munk {
3186f00f6562SOphir Munk 	struct mlx5_priv *priv = dev->data->dev_private;
3187f00f6562SOphir Munk 
3188f00f6562SOphir Munk 	mlx5_nl_mac_addr_flush(priv->nl_socket_route, mlx5_ifindex(dev),
3189f00f6562SOphir Munk 			       dev->data->mac_addrs,
3190f00f6562SOphir Munk 			       MLX5_MAX_MAC_ADDRESSES, priv->mac_own);
3191f00f6562SOphir Munk }
3192