1f44b09f9SOphir Munk /* SPDX-License-Identifier: BSD-3-Clause 2f44b09f9SOphir Munk * Copyright 2015 6WIND S.A. 3f44b09f9SOphir Munk * Copyright 2020 Mellanox Technologies, Ltd 4f44b09f9SOphir Munk */ 5f44b09f9SOphir Munk 6f44b09f9SOphir Munk #include <stddef.h> 7f44b09f9SOphir Munk #include <unistd.h> 8f44b09f9SOphir Munk #include <string.h> 9f44b09f9SOphir Munk #include <stdint.h> 10f44b09f9SOphir Munk #include <stdlib.h> 11f44b09f9SOphir Munk #include <errno.h> 12f44b09f9SOphir Munk #include <net/if.h> 13f44b09f9SOphir Munk #include <sys/mman.h> 14f44b09f9SOphir Munk #include <linux/rtnetlink.h> 1573bf9235SOphir Munk #include <linux/sockios.h> 1673bf9235SOphir Munk #include <linux/ethtool.h> 17f44b09f9SOphir Munk #include <fcntl.h> 18f44b09f9SOphir Munk 19f44b09f9SOphir Munk /* Verbs header. */ 20f44b09f9SOphir Munk /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 21f44b09f9SOphir Munk #ifdef PEDANTIC 22f44b09f9SOphir Munk #pragma GCC diagnostic ignored "-Wpedantic" 23f44b09f9SOphir Munk #endif 24f44b09f9SOphir Munk #include <infiniband/verbs.h> 25f44b09f9SOphir Munk #ifdef PEDANTIC 26f44b09f9SOphir Munk #pragma GCC diagnostic error "-Wpedantic" 27f44b09f9SOphir Munk #endif 28f44b09f9SOphir Munk 29f44b09f9SOphir Munk #include <rte_malloc.h> 30f44b09f9SOphir Munk #include <rte_ethdev_driver.h> 31f44b09f9SOphir Munk #include <rte_ethdev_pci.h> 32f44b09f9SOphir Munk #include <rte_pci.h> 33f44b09f9SOphir Munk #include <rte_bus_pci.h> 34f44b09f9SOphir Munk #include <rte_common.h> 35f44b09f9SOphir Munk #include <rte_kvargs.h> 36f44b09f9SOphir Munk #include <rte_rwlock.h> 37f44b09f9SOphir Munk #include <rte_spinlock.h> 38f44b09f9SOphir Munk #include <rte_string_fns.h> 39f44b09f9SOphir Munk #include <rte_alarm.h> 40f44b09f9SOphir Munk 41f44b09f9SOphir Munk #include <mlx5_glue.h> 42f44b09f9SOphir Munk #include <mlx5_devx_cmds.h> 43f44b09f9SOphir Munk #include <mlx5_common.h> 442eb4d010SOphir Munk #include <mlx5_common_mp.h> 45d5ed8aa9SOphir Munk #include <mlx5_common_mr.h> 46f44b09f9SOphir Munk 47f44b09f9SOphir Munk #include "mlx5_defs.h" 48f44b09f9SOphir Munk #include "mlx5.h" 49*391b8bccSOphir Munk #include "mlx5_common_os.h" 50f44b09f9SOphir Munk #include "mlx5_utils.h" 51f44b09f9SOphir Munk #include "mlx5_rxtx.h" 52f44b09f9SOphir Munk #include "mlx5_autoconf.h" 53f44b09f9SOphir Munk #include "mlx5_mr.h" 54f44b09f9SOphir Munk #include "mlx5_flow.h" 55f44b09f9SOphir Munk #include "rte_pmd_mlx5.h" 564f96d913SOphir Munk #include "mlx5_verbs.h" 57f44b09f9SOphir Munk 582eb4d010SOphir Munk #define MLX5_TAGS_HLIST_ARRAY_SIZE 8192 592eb4d010SOphir Munk 602eb4d010SOphir Munk #ifndef HAVE_IBV_MLX5_MOD_MPW 612eb4d010SOphir Munk #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2) 622eb4d010SOphir Munk #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3) 632eb4d010SOphir Munk #endif 642eb4d010SOphir Munk 652eb4d010SOphir Munk #ifndef HAVE_IBV_MLX5_MOD_CQE_128B_COMP 662eb4d010SOphir Munk #define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4) 672eb4d010SOphir Munk #endif 682eb4d010SOphir Munk 69f44b09f9SOphir Munk /** 70e85f623eSOphir Munk * Get mlx5 device attributes. The glue function query_device_ex() is called 71e85f623eSOphir Munk * with out parameter of type 'struct ibv_device_attr_ex *'. Then fill in mlx5 72e85f623eSOphir Munk * device attributes from the glue out parameter. 73e85f623eSOphir Munk * 74e85f623eSOphir Munk * @param dev 75e85f623eSOphir Munk * Pointer to ibv context. 76e85f623eSOphir Munk * 77e85f623eSOphir Munk * @param device_attr 78e85f623eSOphir Munk * Pointer to mlx5 device attributes. 79e85f623eSOphir Munk * 80e85f623eSOphir Munk * @return 81e85f623eSOphir Munk * 0 on success, non zero error number otherwise 82e85f623eSOphir Munk */ 83e85f623eSOphir Munk int 84e85f623eSOphir Munk mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *device_attr) 85e85f623eSOphir Munk { 86e85f623eSOphir Munk int err; 87e85f623eSOphir Munk struct ibv_device_attr_ex attr_ex; 88e85f623eSOphir Munk memset(device_attr, 0, sizeof(*device_attr)); 89e85f623eSOphir Munk err = mlx5_glue->query_device_ex(ctx, NULL, &attr_ex); 90e85f623eSOphir Munk if (err) 91e85f623eSOphir Munk return err; 92e85f623eSOphir Munk 93e85f623eSOphir Munk device_attr->device_cap_flags_ex = attr_ex.device_cap_flags_ex; 94e85f623eSOphir Munk device_attr->max_qp_wr = attr_ex.orig_attr.max_qp_wr; 95e85f623eSOphir Munk device_attr->max_sge = attr_ex.orig_attr.max_sge; 96e85f623eSOphir Munk device_attr->max_cq = attr_ex.orig_attr.max_cq; 97e85f623eSOphir Munk device_attr->max_qp = attr_ex.orig_attr.max_qp; 98e85f623eSOphir Munk device_attr->raw_packet_caps = attr_ex.raw_packet_caps; 99e85f623eSOphir Munk device_attr->max_rwq_indirection_table_size = 100e85f623eSOphir Munk attr_ex.rss_caps.max_rwq_indirection_table_size; 101e85f623eSOphir Munk device_attr->max_tso = attr_ex.tso_caps.max_tso; 102e85f623eSOphir Munk device_attr->tso_supported_qpts = attr_ex.tso_caps.supported_qpts; 103e85f623eSOphir Munk 104e85f623eSOphir Munk struct mlx5dv_context dv_attr = { .comp_mask = 0 }; 105e85f623eSOphir Munk err = mlx5_glue->dv_query_device(ctx, &dv_attr); 106e85f623eSOphir Munk if (err) 107e85f623eSOphir Munk return err; 108e85f623eSOphir Munk 109e85f623eSOphir Munk device_attr->flags = dv_attr.flags; 110e85f623eSOphir Munk device_attr->comp_mask = dv_attr.comp_mask; 111e85f623eSOphir Munk #ifdef HAVE_IBV_MLX5_MOD_SWP 112e85f623eSOphir Munk device_attr->sw_parsing_offloads = 113e85f623eSOphir Munk dv_attr.sw_parsing_caps.sw_parsing_offloads; 114e85f623eSOphir Munk #endif 115e85f623eSOphir Munk device_attr->min_single_stride_log_num_of_bytes = 116e85f623eSOphir Munk dv_attr.striding_rq_caps.min_single_stride_log_num_of_bytes; 117e85f623eSOphir Munk device_attr->max_single_stride_log_num_of_bytes = 118e85f623eSOphir Munk dv_attr.striding_rq_caps.max_single_stride_log_num_of_bytes; 119e85f623eSOphir Munk device_attr->min_single_wqe_log_num_of_strides = 120e85f623eSOphir Munk dv_attr.striding_rq_caps.min_single_wqe_log_num_of_strides; 121e85f623eSOphir Munk device_attr->max_single_wqe_log_num_of_strides = 122e85f623eSOphir Munk dv_attr.striding_rq_caps.max_single_wqe_log_num_of_strides; 123e85f623eSOphir Munk device_attr->stride_supported_qpts = 124e85f623eSOphir Munk dv_attr.striding_rq_caps.supported_qpts; 125e85f623eSOphir Munk #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 126e85f623eSOphir Munk device_attr->tunnel_offloads_caps = dv_attr.tunnel_offloads_caps; 127e85f623eSOphir Munk #endif 128e85f623eSOphir Munk 129e85f623eSOphir Munk return err; 130e85f623eSOphir Munk } 1312eb4d010SOphir Munk 1322eb4d010SOphir Munk /** 1332eb4d010SOphir Munk * Verbs callback to allocate a memory. This function should allocate the space 1342eb4d010SOphir Munk * according to the size provided residing inside a huge page. 1352eb4d010SOphir Munk * Please note that all allocation must respect the alignment from libmlx5 1362eb4d010SOphir Munk * (i.e. currently sysconf(_SC_PAGESIZE)). 1372eb4d010SOphir Munk * 1382eb4d010SOphir Munk * @param[in] size 1392eb4d010SOphir Munk * The size in bytes of the memory to allocate. 1402eb4d010SOphir Munk * @param[in] data 1412eb4d010SOphir Munk * A pointer to the callback data. 1422eb4d010SOphir Munk * 1432eb4d010SOphir Munk * @return 1442eb4d010SOphir Munk * Allocated buffer, NULL otherwise and rte_errno is set. 1452eb4d010SOphir Munk */ 1462eb4d010SOphir Munk static void * 1472eb4d010SOphir Munk mlx5_alloc_verbs_buf(size_t size, void *data) 1482eb4d010SOphir Munk { 1492eb4d010SOphir Munk struct mlx5_priv *priv = data; 1502eb4d010SOphir Munk void *ret; 1512eb4d010SOphir Munk size_t alignment = sysconf(_SC_PAGESIZE); 1522eb4d010SOphir Munk unsigned int socket = SOCKET_ID_ANY; 1532eb4d010SOphir Munk 1542eb4d010SOphir Munk if (priv->verbs_alloc_ctx.type == MLX5_VERBS_ALLOC_TYPE_TX_QUEUE) { 1552eb4d010SOphir Munk const struct mlx5_txq_ctrl *ctrl = priv->verbs_alloc_ctx.obj; 1562eb4d010SOphir Munk 1572eb4d010SOphir Munk socket = ctrl->socket; 1582eb4d010SOphir Munk } else if (priv->verbs_alloc_ctx.type == 1592eb4d010SOphir Munk MLX5_VERBS_ALLOC_TYPE_RX_QUEUE) { 1602eb4d010SOphir Munk const struct mlx5_rxq_ctrl *ctrl = priv->verbs_alloc_ctx.obj; 1612eb4d010SOphir Munk 1622eb4d010SOphir Munk socket = ctrl->socket; 1632eb4d010SOphir Munk } 1642eb4d010SOphir Munk MLX5_ASSERT(data != NULL); 1652eb4d010SOphir Munk ret = rte_malloc_socket(__func__, size, alignment, socket); 1662eb4d010SOphir Munk if (!ret && size) 1672eb4d010SOphir Munk rte_errno = ENOMEM; 1682eb4d010SOphir Munk return ret; 1692eb4d010SOphir Munk } 1702eb4d010SOphir Munk 1712eb4d010SOphir Munk /** 1722eb4d010SOphir Munk * Verbs callback to free a memory. 1732eb4d010SOphir Munk * 1742eb4d010SOphir Munk * @param[in] ptr 1752eb4d010SOphir Munk * A pointer to the memory to free. 1762eb4d010SOphir Munk * @param[in] data 1772eb4d010SOphir Munk * A pointer to the callback data. 1782eb4d010SOphir Munk */ 1792eb4d010SOphir Munk static void 1802eb4d010SOphir Munk mlx5_free_verbs_buf(void *ptr, void *data __rte_unused) 1812eb4d010SOphir Munk { 1822eb4d010SOphir Munk MLX5_ASSERT(data != NULL); 1832eb4d010SOphir Munk rte_free(ptr); 1842eb4d010SOphir Munk } 1852eb4d010SOphir Munk 1862eb4d010SOphir Munk /** 1872eb4d010SOphir Munk * Initialize DR related data within private structure. 1882eb4d010SOphir Munk * Routine checks the reference counter and does actual 1892eb4d010SOphir Munk * resources creation/initialization only if counter is zero. 1902eb4d010SOphir Munk * 1912eb4d010SOphir Munk * @param[in] priv 1922eb4d010SOphir Munk * Pointer to the private device data structure. 1932eb4d010SOphir Munk * 1942eb4d010SOphir Munk * @return 1952eb4d010SOphir Munk * Zero on success, positive error code otherwise. 1962eb4d010SOphir Munk */ 1972eb4d010SOphir Munk static int 1982eb4d010SOphir Munk mlx5_alloc_shared_dr(struct mlx5_priv *priv) 1992eb4d010SOphir Munk { 2002eb4d010SOphir Munk struct mlx5_dev_ctx_shared *sh = priv->sh; 2012eb4d010SOphir Munk char s[MLX5_HLIST_NAMESIZE]; 2022eb4d010SOphir Munk int err = 0; 2032eb4d010SOphir Munk 2042eb4d010SOphir Munk if (!sh->flow_tbls) 2052eb4d010SOphir Munk err = mlx5_alloc_table_hash_list(priv); 2062eb4d010SOphir Munk else 2072eb4d010SOphir Munk DRV_LOG(DEBUG, "sh->flow_tbls[%p] already created, reuse\n", 2082eb4d010SOphir Munk (void *)sh->flow_tbls); 2092eb4d010SOphir Munk if (err) 2102eb4d010SOphir Munk return err; 2112eb4d010SOphir Munk /* Create tags hash list table. */ 2122eb4d010SOphir Munk snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name); 2132eb4d010SOphir Munk sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE); 2142eb4d010SOphir Munk if (!sh->tag_table) { 21563783b01SDavid Marchand DRV_LOG(ERR, "tags with hash creation failed."); 2162eb4d010SOphir Munk err = ENOMEM; 2172eb4d010SOphir Munk goto error; 2182eb4d010SOphir Munk } 2192eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR 2202eb4d010SOphir Munk void *domain; 2212eb4d010SOphir Munk 2222eb4d010SOphir Munk if (sh->dv_refcnt) { 2232eb4d010SOphir Munk /* Shared DV/DR structures is already initialized. */ 2242eb4d010SOphir Munk sh->dv_refcnt++; 2252eb4d010SOphir Munk priv->dr_shared = 1; 2262eb4d010SOphir Munk return 0; 2272eb4d010SOphir Munk } 2282eb4d010SOphir Munk /* Reference counter is zero, we should initialize structures. */ 2292eb4d010SOphir Munk domain = mlx5_glue->dr_create_domain(sh->ctx, 2302eb4d010SOphir Munk MLX5DV_DR_DOMAIN_TYPE_NIC_RX); 2312eb4d010SOphir Munk if (!domain) { 2322eb4d010SOphir Munk DRV_LOG(ERR, "ingress mlx5dv_dr_create_domain failed"); 2332eb4d010SOphir Munk err = errno; 2342eb4d010SOphir Munk goto error; 2352eb4d010SOphir Munk } 2362eb4d010SOphir Munk sh->rx_domain = domain; 2372eb4d010SOphir Munk domain = mlx5_glue->dr_create_domain(sh->ctx, 2382eb4d010SOphir Munk MLX5DV_DR_DOMAIN_TYPE_NIC_TX); 2392eb4d010SOphir Munk if (!domain) { 2402eb4d010SOphir Munk DRV_LOG(ERR, "egress mlx5dv_dr_create_domain failed"); 2412eb4d010SOphir Munk err = errno; 2422eb4d010SOphir Munk goto error; 2432eb4d010SOphir Munk } 2442eb4d010SOphir Munk pthread_mutex_init(&sh->dv_mutex, NULL); 2452eb4d010SOphir Munk sh->tx_domain = domain; 2462eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_ESWITCH 2472eb4d010SOphir Munk if (priv->config.dv_esw_en) { 2482eb4d010SOphir Munk domain = mlx5_glue->dr_create_domain 2492eb4d010SOphir Munk (sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB); 2502eb4d010SOphir Munk if (!domain) { 2512eb4d010SOphir Munk DRV_LOG(ERR, "FDB mlx5dv_dr_create_domain failed"); 2522eb4d010SOphir Munk err = errno; 2532eb4d010SOphir Munk goto error; 2542eb4d010SOphir Munk } 2552eb4d010SOphir Munk sh->fdb_domain = domain; 2562eb4d010SOphir Munk sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); 2572eb4d010SOphir Munk } 2582eb4d010SOphir Munk #endif 2592eb4d010SOphir Munk if (priv->config.reclaim_mode == MLX5_RCM_AGGR) { 2602eb4d010SOphir Munk mlx5_glue->dr_reclaim_domain_memory(sh->rx_domain, 1); 2612eb4d010SOphir Munk mlx5_glue->dr_reclaim_domain_memory(sh->tx_domain, 1); 2622eb4d010SOphir Munk if (sh->fdb_domain) 2632eb4d010SOphir Munk mlx5_glue->dr_reclaim_domain_memory(sh->fdb_domain, 1); 2642eb4d010SOphir Munk } 2652eb4d010SOphir Munk sh->pop_vlan_action = mlx5_glue->dr_create_flow_action_pop_vlan(); 2662eb4d010SOphir Munk #endif /* HAVE_MLX5DV_DR */ 2672eb4d010SOphir Munk sh->dv_refcnt++; 2682eb4d010SOphir Munk priv->dr_shared = 1; 2692eb4d010SOphir Munk return 0; 2702eb4d010SOphir Munk error: 2712eb4d010SOphir Munk /* Rollback the created objects. */ 2722eb4d010SOphir Munk if (sh->rx_domain) { 2732eb4d010SOphir Munk mlx5_glue->dr_destroy_domain(sh->rx_domain); 2742eb4d010SOphir Munk sh->rx_domain = NULL; 2752eb4d010SOphir Munk } 2762eb4d010SOphir Munk if (sh->tx_domain) { 2772eb4d010SOphir Munk mlx5_glue->dr_destroy_domain(sh->tx_domain); 2782eb4d010SOphir Munk sh->tx_domain = NULL; 2792eb4d010SOphir Munk } 2802eb4d010SOphir Munk if (sh->fdb_domain) { 2812eb4d010SOphir Munk mlx5_glue->dr_destroy_domain(sh->fdb_domain); 2822eb4d010SOphir Munk sh->fdb_domain = NULL; 2832eb4d010SOphir Munk } 2842eb4d010SOphir Munk if (sh->esw_drop_action) { 2852eb4d010SOphir Munk mlx5_glue->destroy_flow_action(sh->esw_drop_action); 2862eb4d010SOphir Munk sh->esw_drop_action = NULL; 2872eb4d010SOphir Munk } 2882eb4d010SOphir Munk if (sh->pop_vlan_action) { 2892eb4d010SOphir Munk mlx5_glue->destroy_flow_action(sh->pop_vlan_action); 2902eb4d010SOphir Munk sh->pop_vlan_action = NULL; 2912eb4d010SOphir Munk } 2922eb4d010SOphir Munk if (sh->tag_table) { 2932eb4d010SOphir Munk /* tags should be destroyed with flow before. */ 2942eb4d010SOphir Munk mlx5_hlist_destroy(sh->tag_table, NULL, NULL); 2952eb4d010SOphir Munk sh->tag_table = NULL; 2962eb4d010SOphir Munk } 2972eb4d010SOphir Munk mlx5_free_table_hash_list(priv); 2982eb4d010SOphir Munk return err; 2992eb4d010SOphir Munk } 3002eb4d010SOphir Munk 3012eb4d010SOphir Munk /** 3022eb4d010SOphir Munk * Destroy DR related data within private structure. 3032eb4d010SOphir Munk * 3042eb4d010SOphir Munk * @param[in] priv 3052eb4d010SOphir Munk * Pointer to the private device data structure. 3062eb4d010SOphir Munk */ 3072eb4d010SOphir Munk void 3082eb4d010SOphir Munk mlx5_os_free_shared_dr(struct mlx5_priv *priv) 3092eb4d010SOphir Munk { 3102eb4d010SOphir Munk struct mlx5_dev_ctx_shared *sh; 3112eb4d010SOphir Munk 3122eb4d010SOphir Munk if (!priv->dr_shared) 3132eb4d010SOphir Munk return; 3142eb4d010SOphir Munk priv->dr_shared = 0; 3152eb4d010SOphir Munk sh = priv->sh; 3162eb4d010SOphir Munk MLX5_ASSERT(sh); 3172eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR 3182eb4d010SOphir Munk MLX5_ASSERT(sh->dv_refcnt); 3192eb4d010SOphir Munk if (sh->dv_refcnt && --sh->dv_refcnt) 3202eb4d010SOphir Munk return; 3212eb4d010SOphir Munk if (sh->rx_domain) { 3222eb4d010SOphir Munk mlx5_glue->dr_destroy_domain(sh->rx_domain); 3232eb4d010SOphir Munk sh->rx_domain = NULL; 3242eb4d010SOphir Munk } 3252eb4d010SOphir Munk if (sh->tx_domain) { 3262eb4d010SOphir Munk mlx5_glue->dr_destroy_domain(sh->tx_domain); 3272eb4d010SOphir Munk sh->tx_domain = NULL; 3282eb4d010SOphir Munk } 3292eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_ESWITCH 3302eb4d010SOphir Munk if (sh->fdb_domain) { 3312eb4d010SOphir Munk mlx5_glue->dr_destroy_domain(sh->fdb_domain); 3322eb4d010SOphir Munk sh->fdb_domain = NULL; 3332eb4d010SOphir Munk } 3342eb4d010SOphir Munk if (sh->esw_drop_action) { 3352eb4d010SOphir Munk mlx5_glue->destroy_flow_action(sh->esw_drop_action); 3362eb4d010SOphir Munk sh->esw_drop_action = NULL; 3372eb4d010SOphir Munk } 3382eb4d010SOphir Munk #endif 3392eb4d010SOphir Munk if (sh->pop_vlan_action) { 3402eb4d010SOphir Munk mlx5_glue->destroy_flow_action(sh->pop_vlan_action); 3412eb4d010SOphir Munk sh->pop_vlan_action = NULL; 3422eb4d010SOphir Munk } 3432eb4d010SOphir Munk pthread_mutex_destroy(&sh->dv_mutex); 3442eb4d010SOphir Munk #endif /* HAVE_MLX5DV_DR */ 3452eb4d010SOphir Munk if (sh->tag_table) { 3462eb4d010SOphir Munk /* tags should be destroyed with flow before. */ 3472eb4d010SOphir Munk mlx5_hlist_destroy(sh->tag_table, NULL, NULL); 3482eb4d010SOphir Munk sh->tag_table = NULL; 3492eb4d010SOphir Munk } 3502eb4d010SOphir Munk mlx5_free_table_hash_list(priv); 3512eb4d010SOphir Munk } 3522eb4d010SOphir Munk 3532eb4d010SOphir Munk /** 3542eb4d010SOphir Munk * Spawn an Ethernet device from Verbs information. 3552eb4d010SOphir Munk * 3562eb4d010SOphir Munk * @param dpdk_dev 3572eb4d010SOphir Munk * Backing DPDK device. 3582eb4d010SOphir Munk * @param spawn 3592eb4d010SOphir Munk * Verbs device parameters (name, port, switch_info) to spawn. 3602eb4d010SOphir Munk * @param config 3612eb4d010SOphir Munk * Device configuration parameters. 3622eb4d010SOphir Munk * 3632eb4d010SOphir Munk * @return 3642eb4d010SOphir Munk * A valid Ethernet device object on success, NULL otherwise and rte_errno 3652eb4d010SOphir Munk * is set. The following errors are defined: 3662eb4d010SOphir Munk * 3672eb4d010SOphir Munk * EBUSY: device is not supposed to be spawned. 3682eb4d010SOphir Munk * EEXIST: device is already spawned 3692eb4d010SOphir Munk */ 3702eb4d010SOphir Munk static struct rte_eth_dev * 3712eb4d010SOphir Munk mlx5_dev_spawn(struct rte_device *dpdk_dev, 3722eb4d010SOphir Munk struct mlx5_dev_spawn_data *spawn, 3732eb4d010SOphir Munk struct mlx5_dev_config config) 3742eb4d010SOphir Munk { 3752eb4d010SOphir Munk const struct mlx5_switch_info *switch_info = &spawn->info; 3762eb4d010SOphir Munk struct mlx5_dev_ctx_shared *sh = NULL; 3772eb4d010SOphir Munk struct ibv_port_attr port_attr; 3782eb4d010SOphir Munk struct mlx5dv_context dv_attr = { .comp_mask = 0 }; 3792eb4d010SOphir Munk struct rte_eth_dev *eth_dev = NULL; 3802eb4d010SOphir Munk struct mlx5_priv *priv = NULL; 3812eb4d010SOphir Munk int err = 0; 3822eb4d010SOphir Munk unsigned int hw_padding = 0; 3832eb4d010SOphir Munk unsigned int mps; 3842eb4d010SOphir Munk unsigned int cqe_comp; 3852eb4d010SOphir Munk unsigned int cqe_pad = 0; 3862eb4d010SOphir Munk unsigned int tunnel_en = 0; 3872eb4d010SOphir Munk unsigned int mpls_en = 0; 3882eb4d010SOphir Munk unsigned int swp = 0; 3892eb4d010SOphir Munk unsigned int mprq = 0; 3902eb4d010SOphir Munk unsigned int mprq_min_stride_size_n = 0; 3912eb4d010SOphir Munk unsigned int mprq_max_stride_size_n = 0; 3922eb4d010SOphir Munk unsigned int mprq_min_stride_num_n = 0; 3932eb4d010SOphir Munk unsigned int mprq_max_stride_num_n = 0; 3942eb4d010SOphir Munk struct rte_ether_addr mac; 3952eb4d010SOphir Munk char name[RTE_ETH_NAME_MAX_LEN]; 3962eb4d010SOphir Munk int own_domain_id = 0; 3972eb4d010SOphir Munk uint16_t port_id; 3982eb4d010SOphir Munk unsigned int i; 3992eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_DEVX_PORT 4002eb4d010SOphir Munk struct mlx5dv_devx_port devx_port = { .comp_mask = 0 }; 4012eb4d010SOphir Munk #endif 4022eb4d010SOphir Munk 4032eb4d010SOphir Munk /* Determine if this port representor is supposed to be spawned. */ 4042eb4d010SOphir Munk if (switch_info->representor && dpdk_dev->devargs) { 4052eb4d010SOphir Munk struct rte_eth_devargs eth_da; 4062eb4d010SOphir Munk 4072eb4d010SOphir Munk err = rte_eth_devargs_parse(dpdk_dev->devargs->args, ð_da); 4082eb4d010SOphir Munk if (err) { 4092eb4d010SOphir Munk rte_errno = -err; 4102eb4d010SOphir Munk DRV_LOG(ERR, "failed to process device arguments: %s", 4112eb4d010SOphir Munk strerror(rte_errno)); 4122eb4d010SOphir Munk return NULL; 4132eb4d010SOphir Munk } 4142eb4d010SOphir Munk for (i = 0; i < eth_da.nb_representor_ports; ++i) 4152eb4d010SOphir Munk if (eth_da.representor_ports[i] == 4162eb4d010SOphir Munk (uint16_t)switch_info->port_name) 4172eb4d010SOphir Munk break; 4182eb4d010SOphir Munk if (i == eth_da.nb_representor_ports) { 4192eb4d010SOphir Munk rte_errno = EBUSY; 4202eb4d010SOphir Munk return NULL; 4212eb4d010SOphir Munk } 4222eb4d010SOphir Munk } 4232eb4d010SOphir Munk /* Build device name. */ 4242eb4d010SOphir Munk if (spawn->pf_bond < 0) { 4252eb4d010SOphir Munk /* Single device. */ 4262eb4d010SOphir Munk if (!switch_info->representor) 4272eb4d010SOphir Munk strlcpy(name, dpdk_dev->name, sizeof(name)); 4282eb4d010SOphir Munk else 4292eb4d010SOphir Munk snprintf(name, sizeof(name), "%s_representor_%u", 4302eb4d010SOphir Munk dpdk_dev->name, switch_info->port_name); 4312eb4d010SOphir Munk } else { 4322eb4d010SOphir Munk /* Bonding device. */ 4332eb4d010SOphir Munk if (!switch_info->representor) 4342eb4d010SOphir Munk snprintf(name, sizeof(name), "%s_%s", 435834a9019SOphir Munk dpdk_dev->name, 436834a9019SOphir Munk mlx5_os_get_dev_device_name(spawn->phys_dev)); 4372eb4d010SOphir Munk else 4382eb4d010SOphir Munk snprintf(name, sizeof(name), "%s_%s_representor_%u", 439834a9019SOphir Munk dpdk_dev->name, 440834a9019SOphir Munk mlx5_os_get_dev_device_name(spawn->phys_dev), 4412eb4d010SOphir Munk switch_info->port_name); 4422eb4d010SOphir Munk } 4432eb4d010SOphir Munk /* check if the device is already spawned */ 4442eb4d010SOphir Munk if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) { 4452eb4d010SOphir Munk rte_errno = EEXIST; 4462eb4d010SOphir Munk return NULL; 4472eb4d010SOphir Munk } 4482eb4d010SOphir Munk DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name); 4492eb4d010SOphir Munk if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 4502eb4d010SOphir Munk struct mlx5_mp_id mp_id; 4512eb4d010SOphir Munk 4522eb4d010SOphir Munk eth_dev = rte_eth_dev_attach_secondary(name); 4532eb4d010SOphir Munk if (eth_dev == NULL) { 4542eb4d010SOphir Munk DRV_LOG(ERR, "can not attach rte ethdev"); 4552eb4d010SOphir Munk rte_errno = ENOMEM; 4562eb4d010SOphir Munk return NULL; 4572eb4d010SOphir Munk } 4582eb4d010SOphir Munk eth_dev->device = dpdk_dev; 459042f5c94SOphir Munk eth_dev->dev_ops = &mlx5_os_dev_sec_ops; 4602eb4d010SOphir Munk err = mlx5_proc_priv_init(eth_dev); 4612eb4d010SOphir Munk if (err) 4622eb4d010SOphir Munk return NULL; 4632eb4d010SOphir Munk mp_id.port_id = eth_dev->data->port_id; 4642eb4d010SOphir Munk strlcpy(mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN); 4652eb4d010SOphir Munk /* Receive command fd from primary process */ 4662eb4d010SOphir Munk err = mlx5_mp_req_verbs_cmd_fd(&mp_id); 4672eb4d010SOphir Munk if (err < 0) 4682eb4d010SOphir Munk goto err_secondary; 4692eb4d010SOphir Munk /* Remap UAR for Tx queues. */ 4702eb4d010SOphir Munk err = mlx5_tx_uar_init_secondary(eth_dev, err); 4712eb4d010SOphir Munk if (err) 4722eb4d010SOphir Munk goto err_secondary; 4732eb4d010SOphir Munk /* 4742eb4d010SOphir Munk * Ethdev pointer is still required as input since 4752eb4d010SOphir Munk * the primary device is not accessible from the 4762eb4d010SOphir Munk * secondary process. 4772eb4d010SOphir Munk */ 4782eb4d010SOphir Munk eth_dev->rx_pkt_burst = mlx5_select_rx_function(eth_dev); 4792eb4d010SOphir Munk eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev); 4802eb4d010SOphir Munk return eth_dev; 4812eb4d010SOphir Munk err_secondary: 4822eb4d010SOphir Munk mlx5_dev_close(eth_dev); 4832eb4d010SOphir Munk return NULL; 4842eb4d010SOphir Munk } 4852eb4d010SOphir Munk /* 4862eb4d010SOphir Munk * Some parameters ("tx_db_nc" in particularly) are needed in 4872eb4d010SOphir Munk * advance to create dv/verbs device context. We proceed the 4882eb4d010SOphir Munk * devargs here to get ones, and later proceed devargs again 4892eb4d010SOphir Munk * to override some hardware settings. 4902eb4d010SOphir Munk */ 4912eb4d010SOphir Munk err = mlx5_args(&config, dpdk_dev->devargs); 4922eb4d010SOphir Munk if (err) { 4932eb4d010SOphir Munk err = rte_errno; 4942eb4d010SOphir Munk DRV_LOG(ERR, "failed to process device arguments: %s", 4952eb4d010SOphir Munk strerror(rte_errno)); 4962eb4d010SOphir Munk goto error; 4972eb4d010SOphir Munk } 49891389890SOphir Munk sh = mlx5_alloc_shared_dev_ctx(spawn, &config); 4992eb4d010SOphir Munk if (!sh) 5002eb4d010SOphir Munk return NULL; 5012eb4d010SOphir Munk config.devx = sh->devx; 5022eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR 5032eb4d010SOphir Munk config.dest_tir = 1; 5042eb4d010SOphir Munk #endif 5052eb4d010SOphir Munk #ifdef HAVE_IBV_MLX5_MOD_SWP 5062eb4d010SOphir Munk dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP; 5072eb4d010SOphir Munk #endif 5082eb4d010SOphir Munk /* 5092eb4d010SOphir Munk * Multi-packet send is supported by ConnectX-4 Lx PF as well 5102eb4d010SOphir Munk * as all ConnectX-5 devices. 5112eb4d010SOphir Munk */ 5122eb4d010SOphir Munk #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 5132eb4d010SOphir Munk dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS; 5142eb4d010SOphir Munk #endif 5152eb4d010SOphir Munk #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT 5162eb4d010SOphir Munk dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ; 5172eb4d010SOphir Munk #endif 5182eb4d010SOphir Munk mlx5_glue->dv_query_device(sh->ctx, &dv_attr); 5192eb4d010SOphir Munk if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) { 5202eb4d010SOphir Munk if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) { 5212eb4d010SOphir Munk DRV_LOG(DEBUG, "enhanced MPW is supported"); 5222eb4d010SOphir Munk mps = MLX5_MPW_ENHANCED; 5232eb4d010SOphir Munk } else { 5242eb4d010SOphir Munk DRV_LOG(DEBUG, "MPW is supported"); 5252eb4d010SOphir Munk mps = MLX5_MPW; 5262eb4d010SOphir Munk } 5272eb4d010SOphir Munk } else { 5282eb4d010SOphir Munk DRV_LOG(DEBUG, "MPW isn't supported"); 5292eb4d010SOphir Munk mps = MLX5_MPW_DISABLED; 5302eb4d010SOphir Munk } 5312eb4d010SOphir Munk #ifdef HAVE_IBV_MLX5_MOD_SWP 5322eb4d010SOphir Munk if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP) 5332eb4d010SOphir Munk swp = dv_attr.sw_parsing_caps.sw_parsing_offloads; 5342eb4d010SOphir Munk DRV_LOG(DEBUG, "SWP support: %u", swp); 5352eb4d010SOphir Munk #endif 5362eb4d010SOphir Munk config.swp = !!swp; 5372eb4d010SOphir Munk #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT 5382eb4d010SOphir Munk if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) { 5392eb4d010SOphir Munk struct mlx5dv_striding_rq_caps mprq_caps = 5402eb4d010SOphir Munk dv_attr.striding_rq_caps; 5412eb4d010SOphir Munk 5422eb4d010SOphir Munk DRV_LOG(DEBUG, "\tmin_single_stride_log_num_of_bytes: %d", 5432eb4d010SOphir Munk mprq_caps.min_single_stride_log_num_of_bytes); 5442eb4d010SOphir Munk DRV_LOG(DEBUG, "\tmax_single_stride_log_num_of_bytes: %d", 5452eb4d010SOphir Munk mprq_caps.max_single_stride_log_num_of_bytes); 5462eb4d010SOphir Munk DRV_LOG(DEBUG, "\tmin_single_wqe_log_num_of_strides: %d", 5472eb4d010SOphir Munk mprq_caps.min_single_wqe_log_num_of_strides); 5482eb4d010SOphir Munk DRV_LOG(DEBUG, "\tmax_single_wqe_log_num_of_strides: %d", 5492eb4d010SOphir Munk mprq_caps.max_single_wqe_log_num_of_strides); 5502eb4d010SOphir Munk DRV_LOG(DEBUG, "\tsupported_qpts: %d", 5512eb4d010SOphir Munk mprq_caps.supported_qpts); 5522eb4d010SOphir Munk DRV_LOG(DEBUG, "device supports Multi-Packet RQ"); 5532eb4d010SOphir Munk mprq = 1; 5542eb4d010SOphir Munk mprq_min_stride_size_n = 5552eb4d010SOphir Munk mprq_caps.min_single_stride_log_num_of_bytes; 5562eb4d010SOphir Munk mprq_max_stride_size_n = 5572eb4d010SOphir Munk mprq_caps.max_single_stride_log_num_of_bytes; 5582eb4d010SOphir Munk mprq_min_stride_num_n = 5592eb4d010SOphir Munk mprq_caps.min_single_wqe_log_num_of_strides; 5602eb4d010SOphir Munk mprq_max_stride_num_n = 5612eb4d010SOphir Munk mprq_caps.max_single_wqe_log_num_of_strides; 5622eb4d010SOphir Munk } 5632eb4d010SOphir Munk #endif 5642eb4d010SOphir Munk if (RTE_CACHE_LINE_SIZE == 128 && 5652eb4d010SOphir Munk !(dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP)) 5662eb4d010SOphir Munk cqe_comp = 0; 5672eb4d010SOphir Munk else 5682eb4d010SOphir Munk cqe_comp = 1; 5692eb4d010SOphir Munk config.cqe_comp = cqe_comp; 5702eb4d010SOphir Munk #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD 5712eb4d010SOphir Munk /* Whether device supports 128B Rx CQE padding. */ 5722eb4d010SOphir Munk cqe_pad = RTE_CACHE_LINE_SIZE == 128 && 5732eb4d010SOphir Munk (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD); 5742eb4d010SOphir Munk #endif 5752eb4d010SOphir Munk #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 5762eb4d010SOphir Munk if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) { 5772eb4d010SOphir Munk tunnel_en = ((dv_attr.tunnel_offloads_caps & 5782eb4d010SOphir Munk MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN) && 5792eb4d010SOphir Munk (dv_attr.tunnel_offloads_caps & 5802eb4d010SOphir Munk MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE) && 5812eb4d010SOphir Munk (dv_attr.tunnel_offloads_caps & 5822eb4d010SOphir Munk MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GENEVE)); 5832eb4d010SOphir Munk } 5842eb4d010SOphir Munk DRV_LOG(DEBUG, "tunnel offloading is %ssupported", 5852eb4d010SOphir Munk tunnel_en ? "" : "not "); 5862eb4d010SOphir Munk #else 5872eb4d010SOphir Munk DRV_LOG(WARNING, 5882eb4d010SOphir Munk "tunnel offloading disabled due to old OFED/rdma-core version"); 5892eb4d010SOphir Munk #endif 5902eb4d010SOphir Munk config.tunnel_en = tunnel_en; 5912eb4d010SOphir Munk #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 5922eb4d010SOphir Munk mpls_en = ((dv_attr.tunnel_offloads_caps & 5932eb4d010SOphir Munk MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) && 5942eb4d010SOphir Munk (dv_attr.tunnel_offloads_caps & 5952eb4d010SOphir Munk MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP)); 5962eb4d010SOphir Munk DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is %ssupported", 5972eb4d010SOphir Munk mpls_en ? "" : "not "); 5982eb4d010SOphir Munk #else 5992eb4d010SOphir Munk DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to" 6002eb4d010SOphir Munk " old OFED/rdma-core version or firmware configuration"); 6012eb4d010SOphir Munk #endif 6022eb4d010SOphir Munk config.mpls_en = mpls_en; 6032eb4d010SOphir Munk /* Check port status. */ 604834a9019SOphir Munk err = mlx5_glue->query_port(sh->ctx, spawn->phys_port, &port_attr); 6052eb4d010SOphir Munk if (err) { 6062eb4d010SOphir Munk DRV_LOG(ERR, "port query failed: %s", strerror(err)); 6072eb4d010SOphir Munk goto error; 6082eb4d010SOphir Munk } 6092eb4d010SOphir Munk if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { 6102eb4d010SOphir Munk DRV_LOG(ERR, "port is not configured in Ethernet mode"); 6112eb4d010SOphir Munk err = EINVAL; 6122eb4d010SOphir Munk goto error; 6132eb4d010SOphir Munk } 6142eb4d010SOphir Munk if (port_attr.state != IBV_PORT_ACTIVE) 6152eb4d010SOphir Munk DRV_LOG(DEBUG, "port is not active: \"%s\" (%d)", 6162eb4d010SOphir Munk mlx5_glue->port_state_str(port_attr.state), 6172eb4d010SOphir Munk port_attr.state); 6182eb4d010SOphir Munk /* Allocate private eth device data. */ 6192eb4d010SOphir Munk priv = rte_zmalloc("ethdev private structure", 6202eb4d010SOphir Munk sizeof(*priv), 6212eb4d010SOphir Munk RTE_CACHE_LINE_SIZE); 6222eb4d010SOphir Munk if (priv == NULL) { 6232eb4d010SOphir Munk DRV_LOG(ERR, "priv allocation failure"); 6242eb4d010SOphir Munk err = ENOMEM; 6252eb4d010SOphir Munk goto error; 6262eb4d010SOphir Munk } 6272eb4d010SOphir Munk priv->sh = sh; 62891389890SOphir Munk priv->dev_port = spawn->phys_port; 6292eb4d010SOphir Munk priv->pci_dev = spawn->pci_dev; 6302eb4d010SOphir Munk priv->mtu = RTE_ETHER_MTU; 6312eb4d010SOphir Munk priv->mp_id.port_id = port_id; 6322eb4d010SOphir Munk strlcpy(priv->mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN); 6332eb4d010SOphir Munk #ifndef RTE_ARCH_64 6342eb4d010SOphir Munk /* Initialize UAR access locks for 32bit implementations. */ 6352eb4d010SOphir Munk rte_spinlock_init(&priv->uar_lock_cq); 6362eb4d010SOphir Munk for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) 6372eb4d010SOphir Munk rte_spinlock_init(&priv->uar_lock[i]); 6382eb4d010SOphir Munk #endif 6392eb4d010SOphir Munk /* Some internal functions rely on Netlink sockets, open them now. */ 6402eb4d010SOphir Munk priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA); 6412eb4d010SOphir Munk priv->nl_socket_route = mlx5_nl_init(NETLINK_ROUTE); 6422eb4d010SOphir Munk priv->representor = !!switch_info->representor; 6432eb4d010SOphir Munk priv->master = !!switch_info->master; 6442eb4d010SOphir Munk priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 6452eb4d010SOphir Munk priv->vport_meta_tag = 0; 6462eb4d010SOphir Munk priv->vport_meta_mask = 0; 6472eb4d010SOphir Munk priv->pf_bond = spawn->pf_bond; 6482eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_DEVX_PORT 6492eb4d010SOphir Munk /* 6502eb4d010SOphir Munk * The DevX port query API is implemented. E-Switch may use 6512eb4d010SOphir Munk * either vport or reg_c[0] metadata register to match on 6522eb4d010SOphir Munk * vport index. The engaged part of metadata register is 6532eb4d010SOphir Munk * defined by mask. 6542eb4d010SOphir Munk */ 6552eb4d010SOphir Munk if (switch_info->representor || switch_info->master) { 6562eb4d010SOphir Munk devx_port.comp_mask = MLX5DV_DEVX_PORT_VPORT | 6572eb4d010SOphir Munk MLX5DV_DEVX_PORT_MATCH_REG_C_0; 658834a9019SOphir Munk err = mlx5_glue->devx_port_query(sh->ctx, spawn->phys_port, 6592eb4d010SOphir Munk &devx_port); 6602eb4d010SOphir Munk if (err) { 6612eb4d010SOphir Munk DRV_LOG(WARNING, 6622eb4d010SOphir Munk "can't query devx port %d on device %s", 663834a9019SOphir Munk spawn->phys_port, 664834a9019SOphir Munk mlx5_os_get_dev_device_name(spawn->phys_dev)); 6652eb4d010SOphir Munk devx_port.comp_mask = 0; 6662eb4d010SOphir Munk } 6672eb4d010SOphir Munk } 6682eb4d010SOphir Munk if (devx_port.comp_mask & MLX5DV_DEVX_PORT_MATCH_REG_C_0) { 6692eb4d010SOphir Munk priv->vport_meta_tag = devx_port.reg_c_0.value; 6702eb4d010SOphir Munk priv->vport_meta_mask = devx_port.reg_c_0.mask; 6712eb4d010SOphir Munk if (!priv->vport_meta_mask) { 6722eb4d010SOphir Munk DRV_LOG(ERR, "vport zero mask for port %d" 6732eb4d010SOphir Munk " on bonding device %s", 674834a9019SOphir Munk spawn->phys_port, 675834a9019SOphir Munk mlx5_os_get_dev_device_name 676834a9019SOphir Munk (spawn->phys_dev)); 6772eb4d010SOphir Munk err = ENOTSUP; 6782eb4d010SOphir Munk goto error; 6792eb4d010SOphir Munk } 6802eb4d010SOphir Munk if (priv->vport_meta_tag & ~priv->vport_meta_mask) { 6812eb4d010SOphir Munk DRV_LOG(ERR, "invalid vport tag for port %d" 6822eb4d010SOphir Munk " on bonding device %s", 683834a9019SOphir Munk spawn->phys_port, 684834a9019SOphir Munk mlx5_os_get_dev_device_name 685834a9019SOphir Munk (spawn->phys_dev)); 6862eb4d010SOphir Munk err = ENOTSUP; 6872eb4d010SOphir Munk goto error; 6882eb4d010SOphir Munk } 6892eb4d010SOphir Munk } 6902eb4d010SOphir Munk if (devx_port.comp_mask & MLX5DV_DEVX_PORT_VPORT) { 6912eb4d010SOphir Munk priv->vport_id = devx_port.vport_num; 6922eb4d010SOphir Munk } else if (spawn->pf_bond >= 0) { 6932eb4d010SOphir Munk DRV_LOG(ERR, "can't deduce vport index for port %d" 6942eb4d010SOphir Munk " on bonding device %s", 695834a9019SOphir Munk spawn->phys_port, 696834a9019SOphir Munk mlx5_os_get_dev_device_name(spawn->phys_dev)); 6972eb4d010SOphir Munk err = ENOTSUP; 6982eb4d010SOphir Munk goto error; 6992eb4d010SOphir Munk } else { 7002eb4d010SOphir Munk /* Suppose vport index in compatible way. */ 7012eb4d010SOphir Munk priv->vport_id = switch_info->representor ? 7022eb4d010SOphir Munk switch_info->port_name + 1 : -1; 7032eb4d010SOphir Munk } 7042eb4d010SOphir Munk #else 7052eb4d010SOphir Munk /* 7062eb4d010SOphir Munk * Kernel/rdma_core support single E-Switch per PF configurations 7072eb4d010SOphir Munk * only and vport_id field contains the vport index for 7082eb4d010SOphir Munk * associated VF, which is deduced from representor port name. 7092eb4d010SOphir Munk * For example, let's have the IB device port 10, it has 7102eb4d010SOphir Munk * attached network device eth0, which has port name attribute 7112eb4d010SOphir Munk * pf0vf2, we can deduce the VF number as 2, and set vport index 7122eb4d010SOphir Munk * as 3 (2+1). This assigning schema should be changed if the 7132eb4d010SOphir Munk * multiple E-Switch instances per PF configurations or/and PCI 7142eb4d010SOphir Munk * subfunctions are added. 7152eb4d010SOphir Munk */ 7162eb4d010SOphir Munk priv->vport_id = switch_info->representor ? 7172eb4d010SOphir Munk switch_info->port_name + 1 : -1; 7182eb4d010SOphir Munk #endif 7192eb4d010SOphir Munk /* representor_id field keeps the unmodified VF index. */ 7202eb4d010SOphir Munk priv->representor_id = switch_info->representor ? 7212eb4d010SOphir Munk switch_info->port_name : -1; 7222eb4d010SOphir Munk /* 7232eb4d010SOphir Munk * Look for sibling devices in order to reuse their switch domain 7242eb4d010SOphir Munk * if any, otherwise allocate one. 7252eb4d010SOphir Munk */ 7262eb4d010SOphir Munk MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) { 7272eb4d010SOphir Munk const struct mlx5_priv *opriv = 7282eb4d010SOphir Munk rte_eth_devices[port_id].data->dev_private; 7292eb4d010SOphir Munk 7302eb4d010SOphir Munk if (!opriv || 7312eb4d010SOphir Munk opriv->sh != priv->sh || 7322eb4d010SOphir Munk opriv->domain_id == 7332eb4d010SOphir Munk RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) 7342eb4d010SOphir Munk continue; 7352eb4d010SOphir Munk priv->domain_id = opriv->domain_id; 7362eb4d010SOphir Munk break; 7372eb4d010SOphir Munk } 7382eb4d010SOphir Munk if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 7392eb4d010SOphir Munk err = rte_eth_switch_domain_alloc(&priv->domain_id); 7402eb4d010SOphir Munk if (err) { 7412eb4d010SOphir Munk err = rte_errno; 7422eb4d010SOphir Munk DRV_LOG(ERR, "unable to allocate switch domain: %s", 7432eb4d010SOphir Munk strerror(rte_errno)); 7442eb4d010SOphir Munk goto error; 7452eb4d010SOphir Munk } 7462eb4d010SOphir Munk own_domain_id = 1; 7472eb4d010SOphir Munk } 7482eb4d010SOphir Munk /* Override some values set by hardware configuration. */ 7492eb4d010SOphir Munk mlx5_args(&config, dpdk_dev->devargs); 7502eb4d010SOphir Munk err = mlx5_dev_check_sibling_config(priv, &config); 7512eb4d010SOphir Munk if (err) 7522eb4d010SOphir Munk goto error; 7532eb4d010SOphir Munk config.hw_csum = !!(sh->device_attr.device_cap_flags_ex & 7542eb4d010SOphir Munk IBV_DEVICE_RAW_IP_CSUM); 7552eb4d010SOphir Munk DRV_LOG(DEBUG, "checksum offloading is %ssupported", 7562eb4d010SOphir Munk (config.hw_csum ? "" : "not ")); 7572eb4d010SOphir Munk #if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \ 7582eb4d010SOphir Munk !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 7592eb4d010SOphir Munk DRV_LOG(DEBUG, "counters are not supported"); 7602eb4d010SOphir Munk #endif 7612eb4d010SOphir Munk #if !defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_MLX5DV_DR) 7622eb4d010SOphir Munk if (config.dv_flow_en) { 7632eb4d010SOphir Munk DRV_LOG(WARNING, "DV flow is not supported"); 7642eb4d010SOphir Munk config.dv_flow_en = 0; 7652eb4d010SOphir Munk } 7662eb4d010SOphir Munk #endif 7672eb4d010SOphir Munk config.ind_table_max_size = 7682eb4d010SOphir Munk sh->device_attr.max_rwq_indirection_table_size; 7692eb4d010SOphir Munk /* 7702eb4d010SOphir Munk * Remove this check once DPDK supports larger/variable 7712eb4d010SOphir Munk * indirection tables. 7722eb4d010SOphir Munk */ 7732eb4d010SOphir Munk if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512) 7742eb4d010SOphir Munk config.ind_table_max_size = ETH_RSS_RETA_SIZE_512; 7752eb4d010SOphir Munk DRV_LOG(DEBUG, "maximum Rx indirection table size is %u", 7762eb4d010SOphir Munk config.ind_table_max_size); 7772eb4d010SOphir Munk config.hw_vlan_strip = !!(sh->device_attr.raw_packet_caps & 7782eb4d010SOphir Munk IBV_RAW_PACKET_CAP_CVLAN_STRIPPING); 7792eb4d010SOphir Munk DRV_LOG(DEBUG, "VLAN stripping is %ssupported", 7802eb4d010SOphir Munk (config.hw_vlan_strip ? "" : "not ")); 7812eb4d010SOphir Munk config.hw_fcs_strip = !!(sh->device_attr.raw_packet_caps & 7822eb4d010SOphir Munk IBV_RAW_PACKET_CAP_SCATTER_FCS); 7832eb4d010SOphir Munk DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported", 7842eb4d010SOphir Munk (config.hw_fcs_strip ? "" : "not ")); 7852eb4d010SOphir Munk #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING) 7862eb4d010SOphir Munk hw_padding = !!sh->device_attr.rx_pad_end_addr_align; 7872eb4d010SOphir Munk #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING) 7882eb4d010SOphir Munk hw_padding = !!(sh->device_attr.device_cap_flags_ex & 7892eb4d010SOphir Munk IBV_DEVICE_PCI_WRITE_END_PADDING); 7902eb4d010SOphir Munk #endif 7912eb4d010SOphir Munk if (config.hw_padding && !hw_padding) { 7922eb4d010SOphir Munk DRV_LOG(DEBUG, "Rx end alignment padding isn't supported"); 7932eb4d010SOphir Munk config.hw_padding = 0; 7942eb4d010SOphir Munk } else if (config.hw_padding) { 7952eb4d010SOphir Munk DRV_LOG(DEBUG, "Rx end alignment padding is enabled"); 7962eb4d010SOphir Munk } 7972eb4d010SOphir Munk config.tso = (sh->device_attr.max_tso > 0 && 7982eb4d010SOphir Munk (sh->device_attr.tso_supported_qpts & 7992eb4d010SOphir Munk (1 << IBV_QPT_RAW_PACKET))); 8002eb4d010SOphir Munk if (config.tso) 8012eb4d010SOphir Munk config.tso_max_payload_sz = sh->device_attr.max_tso; 8022eb4d010SOphir Munk /* 8032eb4d010SOphir Munk * MPW is disabled by default, while the Enhanced MPW is enabled 8042eb4d010SOphir Munk * by default. 8052eb4d010SOphir Munk */ 8062eb4d010SOphir Munk if (config.mps == MLX5_ARG_UNSET) 8072eb4d010SOphir Munk config.mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED : 8082eb4d010SOphir Munk MLX5_MPW_DISABLED; 8092eb4d010SOphir Munk else 8102eb4d010SOphir Munk config.mps = config.mps ? mps : MLX5_MPW_DISABLED; 8112eb4d010SOphir Munk DRV_LOG(INFO, "%sMPS is %s", 8122eb4d010SOphir Munk config.mps == MLX5_MPW_ENHANCED ? "enhanced " : 8132eb4d010SOphir Munk config.mps == MLX5_MPW ? "legacy " : "", 8142eb4d010SOphir Munk config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); 8152eb4d010SOphir Munk if (config.cqe_comp && !cqe_comp) { 8162eb4d010SOphir Munk DRV_LOG(WARNING, "Rx CQE compression isn't supported"); 8172eb4d010SOphir Munk config.cqe_comp = 0; 8182eb4d010SOphir Munk } 8192eb4d010SOphir Munk if (config.cqe_pad && !cqe_pad) { 8202eb4d010SOphir Munk DRV_LOG(WARNING, "Rx CQE padding isn't supported"); 8212eb4d010SOphir Munk config.cqe_pad = 0; 8222eb4d010SOphir Munk } else if (config.cqe_pad) { 8232eb4d010SOphir Munk DRV_LOG(INFO, "Rx CQE padding is enabled"); 8242eb4d010SOphir Munk } 8252eb4d010SOphir Munk if (config.devx) { 8262eb4d010SOphir Munk priv->counter_fallback = 0; 8272eb4d010SOphir Munk err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config.hca_attr); 8282eb4d010SOphir Munk if (err) { 8292eb4d010SOphir Munk err = -err; 8302eb4d010SOphir Munk goto error; 8312eb4d010SOphir Munk } 8322eb4d010SOphir Munk if (!config.hca_attr.flow_counters_dump) 8332eb4d010SOphir Munk priv->counter_fallback = 1; 8342eb4d010SOphir Munk #ifndef HAVE_IBV_DEVX_ASYNC 8352eb4d010SOphir Munk priv->counter_fallback = 1; 8362eb4d010SOphir Munk #endif 8372eb4d010SOphir Munk if (priv->counter_fallback) 8382eb4d010SOphir Munk DRV_LOG(INFO, "Use fall-back DV counter management"); 8392eb4d010SOphir Munk /* Check for LRO support. */ 8402eb4d010SOphir Munk if (config.dest_tir && config.hca_attr.lro_cap && 8412eb4d010SOphir Munk config.dv_flow_en) { 8422eb4d010SOphir Munk /* TBD check tunnel lro caps. */ 8432eb4d010SOphir Munk config.lro.supported = config.hca_attr.lro_cap; 8442eb4d010SOphir Munk DRV_LOG(DEBUG, "Device supports LRO"); 8452eb4d010SOphir Munk /* 8462eb4d010SOphir Munk * If LRO timeout is not configured by application, 8472eb4d010SOphir Munk * use the minimal supported value. 8482eb4d010SOphir Munk */ 8492eb4d010SOphir Munk if (!config.lro.timeout) 8502eb4d010SOphir Munk config.lro.timeout = 8512eb4d010SOphir Munk config.hca_attr.lro_timer_supported_periods[0]; 8522eb4d010SOphir Munk DRV_LOG(DEBUG, "LRO session timeout set to %d usec", 8532eb4d010SOphir Munk config.lro.timeout); 8542eb4d010SOphir Munk } 8552eb4d010SOphir Munk #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) 8562eb4d010SOphir Munk if (config.hca_attr.qos.sup && config.hca_attr.qos.srtcm_sup && 8572eb4d010SOphir Munk config.dv_flow_en) { 8582eb4d010SOphir Munk uint8_t reg_c_mask = 8592eb4d010SOphir Munk config.hca_attr.qos.flow_meter_reg_c_ids; 8602eb4d010SOphir Munk /* 8612eb4d010SOphir Munk * Meter needs two REG_C's for color match and pre-sfx 8622eb4d010SOphir Munk * flow match. Here get the REG_C for color match. 8632eb4d010SOphir Munk * REG_C_0 and REG_C_1 is reserved for metadata feature. 8642eb4d010SOphir Munk */ 8652eb4d010SOphir Munk reg_c_mask &= 0xfc; 8662eb4d010SOphir Munk if (__builtin_popcount(reg_c_mask) < 1) { 8672eb4d010SOphir Munk priv->mtr_en = 0; 8682eb4d010SOphir Munk DRV_LOG(WARNING, "No available register for" 8692eb4d010SOphir Munk " meter."); 8702eb4d010SOphir Munk } else { 8712eb4d010SOphir Munk priv->mtr_color_reg = ffs(reg_c_mask) - 1 + 8722eb4d010SOphir Munk REG_C_0; 8732eb4d010SOphir Munk priv->mtr_en = 1; 8742eb4d010SOphir Munk priv->mtr_reg_share = 8752eb4d010SOphir Munk config.hca_attr.qos.flow_meter_reg_share; 8762eb4d010SOphir Munk DRV_LOG(DEBUG, "The REG_C meter uses is %d", 8772eb4d010SOphir Munk priv->mtr_color_reg); 8782eb4d010SOphir Munk } 8792eb4d010SOphir Munk } 8802eb4d010SOphir Munk #endif 8812eb4d010SOphir Munk } 8822eb4d010SOphir Munk if (config.mprq.enabled && mprq) { 8832eb4d010SOphir Munk if (config.mprq.stride_num_n && 8842eb4d010SOphir Munk (config.mprq.stride_num_n > mprq_max_stride_num_n || 8852eb4d010SOphir Munk config.mprq.stride_num_n < mprq_min_stride_num_n)) { 8862eb4d010SOphir Munk config.mprq.stride_num_n = 8872eb4d010SOphir Munk RTE_MIN(RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N, 8882eb4d010SOphir Munk mprq_min_stride_num_n), 8892eb4d010SOphir Munk mprq_max_stride_num_n); 8902eb4d010SOphir Munk DRV_LOG(WARNING, 8912eb4d010SOphir Munk "the number of strides" 8922eb4d010SOphir Munk " for Multi-Packet RQ is out of range," 8932eb4d010SOphir Munk " setting default value (%u)", 8942eb4d010SOphir Munk 1 << config.mprq.stride_num_n); 8952eb4d010SOphir Munk } 8962eb4d010SOphir Munk if (config.mprq.stride_size_n && 8972eb4d010SOphir Munk (config.mprq.stride_size_n > mprq_max_stride_size_n || 8982eb4d010SOphir Munk config.mprq.stride_size_n < mprq_min_stride_size_n)) { 8992eb4d010SOphir Munk config.mprq.stride_size_n = 9002eb4d010SOphir Munk RTE_MIN(RTE_MAX(MLX5_MPRQ_STRIDE_SIZE_N, 9012eb4d010SOphir Munk mprq_min_stride_size_n), 9022eb4d010SOphir Munk mprq_max_stride_size_n); 9032eb4d010SOphir Munk DRV_LOG(WARNING, 9042eb4d010SOphir Munk "the size of a stride" 9052eb4d010SOphir Munk " for Multi-Packet RQ is out of range," 9062eb4d010SOphir Munk " setting default value (%u)", 9072eb4d010SOphir Munk 1 << config.mprq.stride_size_n); 9082eb4d010SOphir Munk } 9092eb4d010SOphir Munk config.mprq.min_stride_size_n = mprq_min_stride_size_n; 9102eb4d010SOphir Munk config.mprq.max_stride_size_n = mprq_max_stride_size_n; 9112eb4d010SOphir Munk } else if (config.mprq.enabled && !mprq) { 9122eb4d010SOphir Munk DRV_LOG(WARNING, "Multi-Packet RQ isn't supported"); 9132eb4d010SOphir Munk config.mprq.enabled = 0; 9142eb4d010SOphir Munk } 9152eb4d010SOphir Munk if (config.max_dump_files_num == 0) 9162eb4d010SOphir Munk config.max_dump_files_num = 128; 9172eb4d010SOphir Munk eth_dev = rte_eth_dev_allocate(name); 9182eb4d010SOphir Munk if (eth_dev == NULL) { 9192eb4d010SOphir Munk DRV_LOG(ERR, "can not allocate rte ethdev"); 9202eb4d010SOphir Munk err = ENOMEM; 9212eb4d010SOphir Munk goto error; 9222eb4d010SOphir Munk } 9232eb4d010SOphir Munk /* Flag to call rte_eth_dev_release_port() in rte_eth_dev_close(). */ 9242eb4d010SOphir Munk eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 9252eb4d010SOphir Munk if (priv->representor) { 9262eb4d010SOphir Munk eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR; 9272eb4d010SOphir Munk eth_dev->data->representor_id = priv->representor_id; 9282eb4d010SOphir Munk } 9292eb4d010SOphir Munk /* 9302eb4d010SOphir Munk * Store associated network device interface index. This index 9312eb4d010SOphir Munk * is permanent throughout the lifetime of device. So, we may store 9322eb4d010SOphir Munk * the ifindex here and use the cached value further. 9332eb4d010SOphir Munk */ 9342eb4d010SOphir Munk MLX5_ASSERT(spawn->ifindex); 9352eb4d010SOphir Munk priv->if_index = spawn->ifindex; 9362eb4d010SOphir Munk eth_dev->data->dev_private = priv; 9372eb4d010SOphir Munk priv->dev_data = eth_dev->data; 9382eb4d010SOphir Munk eth_dev->data->mac_addrs = priv->mac; 9392eb4d010SOphir Munk eth_dev->device = dpdk_dev; 9402eb4d010SOphir Munk /* Configure the first MAC address by default. */ 9412eb4d010SOphir Munk if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) { 9422eb4d010SOphir Munk DRV_LOG(ERR, 9432eb4d010SOphir Munk "port %u cannot get MAC address, is mlx5_en" 9442eb4d010SOphir Munk " loaded? (errno: %s)", 9452eb4d010SOphir Munk eth_dev->data->port_id, strerror(rte_errno)); 9462eb4d010SOphir Munk err = ENODEV; 9472eb4d010SOphir Munk goto error; 9482eb4d010SOphir Munk } 9492eb4d010SOphir Munk DRV_LOG(INFO, 9502eb4d010SOphir Munk "port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", 9512eb4d010SOphir Munk eth_dev->data->port_id, 9522eb4d010SOphir Munk mac.addr_bytes[0], mac.addr_bytes[1], 9532eb4d010SOphir Munk mac.addr_bytes[2], mac.addr_bytes[3], 9542eb4d010SOphir Munk mac.addr_bytes[4], mac.addr_bytes[5]); 9552eb4d010SOphir Munk #ifdef RTE_LIBRTE_MLX5_DEBUG 9562eb4d010SOphir Munk { 9572eb4d010SOphir Munk char ifname[IF_NAMESIZE]; 9582eb4d010SOphir Munk 9592eb4d010SOphir Munk if (mlx5_get_ifname(eth_dev, &ifname) == 0) 9602eb4d010SOphir Munk DRV_LOG(DEBUG, "port %u ifname is \"%s\"", 9612eb4d010SOphir Munk eth_dev->data->port_id, ifname); 9622eb4d010SOphir Munk else 9632eb4d010SOphir Munk DRV_LOG(DEBUG, "port %u ifname is unknown", 9642eb4d010SOphir Munk eth_dev->data->port_id); 9652eb4d010SOphir Munk } 9662eb4d010SOphir Munk #endif 9672eb4d010SOphir Munk /* Get actual MTU if possible. */ 9682eb4d010SOphir Munk err = mlx5_get_mtu(eth_dev, &priv->mtu); 9692eb4d010SOphir Munk if (err) { 9702eb4d010SOphir Munk err = rte_errno; 9712eb4d010SOphir Munk goto error; 9722eb4d010SOphir Munk } 9732eb4d010SOphir Munk DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id, 9742eb4d010SOphir Munk priv->mtu); 9752eb4d010SOphir Munk /* Initialize burst functions to prevent crashes before link-up. */ 9762eb4d010SOphir Munk eth_dev->rx_pkt_burst = removed_rx_burst; 9772eb4d010SOphir Munk eth_dev->tx_pkt_burst = removed_tx_burst; 978042f5c94SOphir Munk eth_dev->dev_ops = &mlx5_os_dev_ops; 9792eb4d010SOphir Munk /* Register MAC address. */ 9802eb4d010SOphir Munk claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0)); 9812eb4d010SOphir Munk if (config.vf && config.vf_nl_en) 9822eb4d010SOphir Munk mlx5_nl_mac_addr_sync(priv->nl_socket_route, 9832eb4d010SOphir Munk mlx5_ifindex(eth_dev), 9842eb4d010SOphir Munk eth_dev->data->mac_addrs, 9852eb4d010SOphir Munk MLX5_MAX_MAC_ADDRESSES); 9862eb4d010SOphir Munk priv->flows = 0; 9872eb4d010SOphir Munk priv->ctrl_flows = 0; 9882eb4d010SOphir Munk TAILQ_INIT(&priv->flow_meters); 9892eb4d010SOphir Munk TAILQ_INIT(&priv->flow_meter_profiles); 9902eb4d010SOphir Munk /* Hint libmlx5 to use PMD allocator for data plane resources */ 9912eb4d010SOphir Munk struct mlx5dv_ctx_allocators alctr = { 9922eb4d010SOphir Munk .alloc = &mlx5_alloc_verbs_buf, 9932eb4d010SOphir Munk .free = &mlx5_free_verbs_buf, 9942eb4d010SOphir Munk .data = priv, 9952eb4d010SOphir Munk }; 9962eb4d010SOphir Munk mlx5_glue->dv_set_context_attr(sh->ctx, 9972eb4d010SOphir Munk MLX5DV_CTX_ATTR_BUF_ALLOCATORS, 9982eb4d010SOphir Munk (void *)((uintptr_t)&alctr)); 9992eb4d010SOphir Munk /* Bring Ethernet device up. */ 10002eb4d010SOphir Munk DRV_LOG(DEBUG, "port %u forcing Ethernet interface up", 10012eb4d010SOphir Munk eth_dev->data->port_id); 10022eb4d010SOphir Munk mlx5_set_link_up(eth_dev); 10032eb4d010SOphir Munk /* 10042eb4d010SOphir Munk * Even though the interrupt handler is not installed yet, 10052eb4d010SOphir Munk * interrupts will still trigger on the async_fd from 10062eb4d010SOphir Munk * Verbs context returned by ibv_open_device(). 10072eb4d010SOphir Munk */ 10082eb4d010SOphir Munk mlx5_link_update(eth_dev, 0); 10092eb4d010SOphir Munk #ifdef HAVE_MLX5DV_DR_ESWITCH 10102eb4d010SOphir Munk if (!(config.hca_attr.eswitch_manager && config.dv_flow_en && 10112eb4d010SOphir Munk (switch_info->representor || switch_info->master))) 10122eb4d010SOphir Munk config.dv_esw_en = 0; 10132eb4d010SOphir Munk #else 10142eb4d010SOphir Munk config.dv_esw_en = 0; 10152eb4d010SOphir Munk #endif 10162eb4d010SOphir Munk /* Detect minimal data bytes to inline. */ 10172eb4d010SOphir Munk mlx5_set_min_inline(spawn, &config); 10182eb4d010SOphir Munk /* Store device configuration on private structure. */ 10192eb4d010SOphir Munk priv->config = config; 10202eb4d010SOphir Munk /* Create context for virtual machine VLAN workaround. */ 10212eb4d010SOphir Munk priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex); 10222eb4d010SOphir Munk if (config.dv_flow_en) { 10232eb4d010SOphir Munk err = mlx5_alloc_shared_dr(priv); 10242eb4d010SOphir Munk if (err) 10252eb4d010SOphir Munk goto error; 10262eb4d010SOphir Munk /* 10272eb4d010SOphir Munk * RSS id is shared with meter flow id. Meter flow id can only 10282eb4d010SOphir Munk * use the 24 MSB of the register. 10292eb4d010SOphir Munk */ 10302eb4d010SOphir Munk priv->qrss_id_pool = mlx5_flow_id_pool_alloc(UINT32_MAX >> 10312eb4d010SOphir Munk MLX5_MTR_COLOR_BITS); 10322eb4d010SOphir Munk if (!priv->qrss_id_pool) { 10332eb4d010SOphir Munk DRV_LOG(ERR, "can't create flow id pool"); 10342eb4d010SOphir Munk err = ENOMEM; 10352eb4d010SOphir Munk goto error; 10362eb4d010SOphir Munk } 10372eb4d010SOphir Munk } 10382eb4d010SOphir Munk /* Supported Verbs flow priority number detection. */ 10392eb4d010SOphir Munk err = mlx5_flow_discover_priorities(eth_dev); 10402eb4d010SOphir Munk if (err < 0) { 10412eb4d010SOphir Munk err = -err; 10422eb4d010SOphir Munk goto error; 10432eb4d010SOphir Munk } 10442eb4d010SOphir Munk priv->config.flow_prio = err; 10452eb4d010SOphir Munk if (!priv->config.dv_esw_en && 10462eb4d010SOphir Munk priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 10472eb4d010SOphir Munk DRV_LOG(WARNING, "metadata mode %u is not supported " 10482eb4d010SOphir Munk "(no E-Switch)", priv->config.dv_xmeta_en); 10492eb4d010SOphir Munk priv->config.dv_xmeta_en = MLX5_XMETA_MODE_LEGACY; 10502eb4d010SOphir Munk } 10512eb4d010SOphir Munk mlx5_set_metadata_mask(eth_dev); 10522eb4d010SOphir Munk if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 10532eb4d010SOphir Munk !priv->sh->dv_regc0_mask) { 10542eb4d010SOphir Munk DRV_LOG(ERR, "metadata mode %u is not supported " 10552eb4d010SOphir Munk "(no metadata reg_c[0] is available)", 10562eb4d010SOphir Munk priv->config.dv_xmeta_en); 10572eb4d010SOphir Munk err = ENOTSUP; 10582eb4d010SOphir Munk goto error; 10592eb4d010SOphir Munk } 10602eb4d010SOphir Munk /* 10612eb4d010SOphir Munk * Allocate the buffer for flow creating, just once. 10622eb4d010SOphir Munk * The allocation must be done before any flow creating. 10632eb4d010SOphir Munk */ 10642eb4d010SOphir Munk mlx5_flow_alloc_intermediate(eth_dev); 10652eb4d010SOphir Munk /* Query availability of metadata reg_c's. */ 10662eb4d010SOphir Munk err = mlx5_flow_discover_mreg_c(eth_dev); 10672eb4d010SOphir Munk if (err < 0) { 10682eb4d010SOphir Munk err = -err; 10692eb4d010SOphir Munk goto error; 10702eb4d010SOphir Munk } 10712eb4d010SOphir Munk if (!mlx5_flow_ext_mreg_supported(eth_dev)) { 10722eb4d010SOphir Munk DRV_LOG(DEBUG, 10732eb4d010SOphir Munk "port %u extensive metadata register is not supported", 10742eb4d010SOphir Munk eth_dev->data->port_id); 10752eb4d010SOphir Munk if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 10762eb4d010SOphir Munk DRV_LOG(ERR, "metadata mode %u is not supported " 10772eb4d010SOphir Munk "(no metadata registers available)", 10782eb4d010SOphir Munk priv->config.dv_xmeta_en); 10792eb4d010SOphir Munk err = ENOTSUP; 10802eb4d010SOphir Munk goto error; 10812eb4d010SOphir Munk } 10822eb4d010SOphir Munk } 10832eb4d010SOphir Munk if (priv->config.dv_flow_en && 10842eb4d010SOphir Munk priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 10852eb4d010SOphir Munk mlx5_flow_ext_mreg_supported(eth_dev) && 10862eb4d010SOphir Munk priv->sh->dv_regc0_mask) { 10872eb4d010SOphir Munk priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME, 10882eb4d010SOphir Munk MLX5_FLOW_MREG_HTABLE_SZ); 10892eb4d010SOphir Munk if (!priv->mreg_cp_tbl) { 10902eb4d010SOphir Munk err = ENOMEM; 10912eb4d010SOphir Munk goto error; 10922eb4d010SOphir Munk } 10932eb4d010SOphir Munk } 10942eb4d010SOphir Munk return eth_dev; 10952eb4d010SOphir Munk error: 10962eb4d010SOphir Munk if (priv) { 10972eb4d010SOphir Munk if (priv->mreg_cp_tbl) 10982eb4d010SOphir Munk mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL); 10992eb4d010SOphir Munk if (priv->sh) 11002eb4d010SOphir Munk mlx5_os_free_shared_dr(priv); 11012eb4d010SOphir Munk if (priv->nl_socket_route >= 0) 11022eb4d010SOphir Munk close(priv->nl_socket_route); 11032eb4d010SOphir Munk if (priv->nl_socket_rdma >= 0) 11042eb4d010SOphir Munk close(priv->nl_socket_rdma); 11052eb4d010SOphir Munk if (priv->vmwa_context) 11062eb4d010SOphir Munk mlx5_vlan_vmwa_exit(priv->vmwa_context); 11072eb4d010SOphir Munk if (priv->qrss_id_pool) 11082eb4d010SOphir Munk mlx5_flow_id_pool_release(priv->qrss_id_pool); 11092eb4d010SOphir Munk if (own_domain_id) 11102eb4d010SOphir Munk claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 11112eb4d010SOphir Munk rte_free(priv); 11122eb4d010SOphir Munk if (eth_dev != NULL) 11132eb4d010SOphir Munk eth_dev->data->dev_private = NULL; 11142eb4d010SOphir Munk } 11152eb4d010SOphir Munk if (eth_dev != NULL) { 11162eb4d010SOphir Munk /* mac_addrs must not be freed alone because part of 11172eb4d010SOphir Munk * dev_private 11182eb4d010SOphir Munk **/ 11192eb4d010SOphir Munk eth_dev->data->mac_addrs = NULL; 11202eb4d010SOphir Munk rte_eth_dev_release_port(eth_dev); 11212eb4d010SOphir Munk } 11222eb4d010SOphir Munk if (sh) 112391389890SOphir Munk mlx5_free_shared_dev_ctx(sh); 11242eb4d010SOphir Munk MLX5_ASSERT(err > 0); 11252eb4d010SOphir Munk rte_errno = err; 11262eb4d010SOphir Munk return NULL; 11272eb4d010SOphir Munk } 11282eb4d010SOphir Munk 11292eb4d010SOphir Munk /** 11302eb4d010SOphir Munk * Comparison callback to sort device data. 11312eb4d010SOphir Munk * 11322eb4d010SOphir Munk * This is meant to be used with qsort(). 11332eb4d010SOphir Munk * 11342eb4d010SOphir Munk * @param a[in] 11352eb4d010SOphir Munk * Pointer to pointer to first data object. 11362eb4d010SOphir Munk * @param b[in] 11372eb4d010SOphir Munk * Pointer to pointer to second data object. 11382eb4d010SOphir Munk * 11392eb4d010SOphir Munk * @return 11402eb4d010SOphir Munk * 0 if both objects are equal, less than 0 if the first argument is less 11412eb4d010SOphir Munk * than the second, greater than 0 otherwise. 11422eb4d010SOphir Munk */ 11432eb4d010SOphir Munk static int 11442eb4d010SOphir Munk mlx5_dev_spawn_data_cmp(const void *a, const void *b) 11452eb4d010SOphir Munk { 11462eb4d010SOphir Munk const struct mlx5_switch_info *si_a = 11472eb4d010SOphir Munk &((const struct mlx5_dev_spawn_data *)a)->info; 11482eb4d010SOphir Munk const struct mlx5_switch_info *si_b = 11492eb4d010SOphir Munk &((const struct mlx5_dev_spawn_data *)b)->info; 11502eb4d010SOphir Munk int ret; 11512eb4d010SOphir Munk 11522eb4d010SOphir Munk /* Master device first. */ 11532eb4d010SOphir Munk ret = si_b->master - si_a->master; 11542eb4d010SOphir Munk if (ret) 11552eb4d010SOphir Munk return ret; 11562eb4d010SOphir Munk /* Then representor devices. */ 11572eb4d010SOphir Munk ret = si_b->representor - si_a->representor; 11582eb4d010SOphir Munk if (ret) 11592eb4d010SOphir Munk return ret; 11602eb4d010SOphir Munk /* Unidentified devices come last in no specific order. */ 11612eb4d010SOphir Munk if (!si_a->representor) 11622eb4d010SOphir Munk return 0; 11632eb4d010SOphir Munk /* Order representors by name. */ 11642eb4d010SOphir Munk return si_a->port_name - si_b->port_name; 11652eb4d010SOphir Munk } 11662eb4d010SOphir Munk 11672eb4d010SOphir Munk /** 11682eb4d010SOphir Munk * Match PCI information for possible slaves of bonding device. 11692eb4d010SOphir Munk * 11702eb4d010SOphir Munk * @param[in] ibv_dev 11712eb4d010SOphir Munk * Pointer to Infiniband device structure. 11722eb4d010SOphir Munk * @param[in] pci_dev 11732eb4d010SOphir Munk * Pointer to PCI device structure to match PCI address. 11742eb4d010SOphir Munk * @param[in] nl_rdma 11752eb4d010SOphir Munk * Netlink RDMA group socket handle. 11762eb4d010SOphir Munk * 11772eb4d010SOphir Munk * @return 11782eb4d010SOphir Munk * negative value if no bonding device found, otherwise 11792eb4d010SOphir Munk * positive index of slave PF in bonding. 11802eb4d010SOphir Munk */ 11812eb4d010SOphir Munk static int 11822eb4d010SOphir Munk mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev, 11832eb4d010SOphir Munk const struct rte_pci_device *pci_dev, 11842eb4d010SOphir Munk int nl_rdma) 11852eb4d010SOphir Munk { 11862eb4d010SOphir Munk char ifname[IF_NAMESIZE + 1]; 11872eb4d010SOphir Munk unsigned int ifindex; 11882eb4d010SOphir Munk unsigned int np, i; 11892eb4d010SOphir Munk FILE *file = NULL; 11902eb4d010SOphir Munk int pf = -1; 11912eb4d010SOphir Munk 11922eb4d010SOphir Munk /* 11932eb4d010SOphir Munk * Try to get master device name. If something goes 11942eb4d010SOphir Munk * wrong suppose the lack of kernel support and no 11952eb4d010SOphir Munk * bonding devices. 11962eb4d010SOphir Munk */ 11972eb4d010SOphir Munk if (nl_rdma < 0) 11982eb4d010SOphir Munk return -1; 11992eb4d010SOphir Munk if (!strstr(ibv_dev->name, "bond")) 12002eb4d010SOphir Munk return -1; 12012eb4d010SOphir Munk np = mlx5_nl_portnum(nl_rdma, ibv_dev->name); 12022eb4d010SOphir Munk if (!np) 12032eb4d010SOphir Munk return -1; 12042eb4d010SOphir Munk /* 12052eb4d010SOphir Munk * The Master device might not be on the predefined 12062eb4d010SOphir Munk * port (not on port index 1, it is not garanted), 12072eb4d010SOphir Munk * we have to scan all Infiniband device port and 12082eb4d010SOphir Munk * find master. 12092eb4d010SOphir Munk */ 12102eb4d010SOphir Munk for (i = 1; i <= np; ++i) { 12112eb4d010SOphir Munk /* Check whether Infiniband port is populated. */ 12122eb4d010SOphir Munk ifindex = mlx5_nl_ifindex(nl_rdma, ibv_dev->name, i); 12132eb4d010SOphir Munk if (!ifindex) 12142eb4d010SOphir Munk continue; 12152eb4d010SOphir Munk if (!if_indextoname(ifindex, ifname)) 12162eb4d010SOphir Munk continue; 12172eb4d010SOphir Munk /* Try to read bonding slave names from sysfs. */ 12182eb4d010SOphir Munk MKSTR(slaves, 12192eb4d010SOphir Munk "/sys/class/net/%s/master/bonding/slaves", ifname); 12202eb4d010SOphir Munk file = fopen(slaves, "r"); 12212eb4d010SOphir Munk if (file) 12222eb4d010SOphir Munk break; 12232eb4d010SOphir Munk } 12242eb4d010SOphir Munk if (!file) 12252eb4d010SOphir Munk return -1; 12262eb4d010SOphir Munk /* Use safe format to check maximal buffer length. */ 12272eb4d010SOphir Munk MLX5_ASSERT(atol(RTE_STR(IF_NAMESIZE)) == IF_NAMESIZE); 12282eb4d010SOphir Munk while (fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", ifname) == 1) { 12292eb4d010SOphir Munk char tmp_str[IF_NAMESIZE + 32]; 12302eb4d010SOphir Munk struct rte_pci_addr pci_addr; 12312eb4d010SOphir Munk struct mlx5_switch_info info; 12322eb4d010SOphir Munk 12332eb4d010SOphir Munk /* Process slave interface names in the loop. */ 12342eb4d010SOphir Munk snprintf(tmp_str, sizeof(tmp_str), 12352eb4d010SOphir Munk "/sys/class/net/%s", ifname); 12362eb4d010SOphir Munk if (mlx5_dev_to_pci_addr(tmp_str, &pci_addr)) { 12372eb4d010SOphir Munk DRV_LOG(WARNING, "can not get PCI address" 12382eb4d010SOphir Munk " for netdev \"%s\"", ifname); 12392eb4d010SOphir Munk continue; 12402eb4d010SOphir Munk } 12412eb4d010SOphir Munk if (pci_dev->addr.domain != pci_addr.domain || 12422eb4d010SOphir Munk pci_dev->addr.bus != pci_addr.bus || 12432eb4d010SOphir Munk pci_dev->addr.devid != pci_addr.devid || 12442eb4d010SOphir Munk pci_dev->addr.function != pci_addr.function) 12452eb4d010SOphir Munk continue; 12462eb4d010SOphir Munk /* Slave interface PCI address match found. */ 12472eb4d010SOphir Munk fclose(file); 12482eb4d010SOphir Munk snprintf(tmp_str, sizeof(tmp_str), 12492eb4d010SOphir Munk "/sys/class/net/%s/phys_port_name", ifname); 12502eb4d010SOphir Munk file = fopen(tmp_str, "rb"); 12512eb4d010SOphir Munk if (!file) 12522eb4d010SOphir Munk break; 12532eb4d010SOphir Munk info.name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET; 12542eb4d010SOphir Munk if (fscanf(file, "%32s", tmp_str) == 1) 12552eb4d010SOphir Munk mlx5_translate_port_name(tmp_str, &info); 12562eb4d010SOphir Munk if (info.name_type == MLX5_PHYS_PORT_NAME_TYPE_LEGACY || 12572eb4d010SOphir Munk info.name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK) 12582eb4d010SOphir Munk pf = info.port_name; 12592eb4d010SOphir Munk break; 12602eb4d010SOphir Munk } 12612eb4d010SOphir Munk if (file) 12622eb4d010SOphir Munk fclose(file); 12632eb4d010SOphir Munk return pf; 12642eb4d010SOphir Munk } 12652eb4d010SOphir Munk 12662eb4d010SOphir Munk /** 12672eb4d010SOphir Munk * DPDK callback to register a PCI device. 12682eb4d010SOphir Munk * 12692eb4d010SOphir Munk * This function spawns Ethernet devices out of a given PCI device. 12702eb4d010SOphir Munk * 12712eb4d010SOphir Munk * @param[in] pci_drv 12722eb4d010SOphir Munk * PCI driver structure (mlx5_driver). 12732eb4d010SOphir Munk * @param[in] pci_dev 12742eb4d010SOphir Munk * PCI device information. 12752eb4d010SOphir Munk * 12762eb4d010SOphir Munk * @return 12772eb4d010SOphir Munk * 0 on success, a negative errno value otherwise and rte_errno is set. 12782eb4d010SOphir Munk */ 12792eb4d010SOphir Munk int 12802eb4d010SOphir Munk mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 12812eb4d010SOphir Munk struct rte_pci_device *pci_dev) 12822eb4d010SOphir Munk { 12832eb4d010SOphir Munk struct ibv_device **ibv_list; 12842eb4d010SOphir Munk /* 12852eb4d010SOphir Munk * Number of found IB Devices matching with requested PCI BDF. 12862eb4d010SOphir Munk * nd != 1 means there are multiple IB devices over the same 12872eb4d010SOphir Munk * PCI device and we have representors and master. 12882eb4d010SOphir Munk */ 12892eb4d010SOphir Munk unsigned int nd = 0; 12902eb4d010SOphir Munk /* 12912eb4d010SOphir Munk * Number of found IB device Ports. nd = 1 and np = 1..n means 12922eb4d010SOphir Munk * we have the single multiport IB device, and there may be 12932eb4d010SOphir Munk * representors attached to some of found ports. 12942eb4d010SOphir Munk */ 12952eb4d010SOphir Munk unsigned int np = 0; 12962eb4d010SOphir Munk /* 12972eb4d010SOphir Munk * Number of DPDK ethernet devices to Spawn - either over 12982eb4d010SOphir Munk * multiple IB devices or multiple ports of single IB device. 12992eb4d010SOphir Munk * Actually this is the number of iterations to spawn. 13002eb4d010SOphir Munk */ 13012eb4d010SOphir Munk unsigned int ns = 0; 13022eb4d010SOphir Munk /* 13032eb4d010SOphir Munk * Bonding device 13042eb4d010SOphir Munk * < 0 - no bonding device (single one) 13052eb4d010SOphir Munk * >= 0 - bonding device (value is slave PF index) 13062eb4d010SOphir Munk */ 13072eb4d010SOphir Munk int bd = -1; 13082eb4d010SOphir Munk struct mlx5_dev_spawn_data *list = NULL; 13092eb4d010SOphir Munk struct mlx5_dev_config dev_config; 13102eb4d010SOphir Munk int ret; 13112eb4d010SOphir Munk 13122eb4d010SOphir Munk if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_NET) { 13132eb4d010SOphir Munk DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5" 13142eb4d010SOphir Munk " driver."); 13152eb4d010SOphir Munk return 1; 13162eb4d010SOphir Munk } 13172eb4d010SOphir Munk if (rte_eal_process_type() == RTE_PROC_PRIMARY) 13182eb4d010SOphir Munk mlx5_pmd_socket_init(); 13192eb4d010SOphir Munk ret = mlx5_init_once(); 13202eb4d010SOphir Munk if (ret) { 13212eb4d010SOphir Munk DRV_LOG(ERR, "unable to init PMD global data: %s", 13222eb4d010SOphir Munk strerror(rte_errno)); 13232eb4d010SOphir Munk return -rte_errno; 13242eb4d010SOphir Munk } 13252eb4d010SOphir Munk MLX5_ASSERT(pci_drv == &mlx5_driver); 13262eb4d010SOphir Munk errno = 0; 13272eb4d010SOphir Munk ibv_list = mlx5_glue->get_device_list(&ret); 13282eb4d010SOphir Munk if (!ibv_list) { 13292eb4d010SOphir Munk rte_errno = errno ? errno : ENOSYS; 13302eb4d010SOphir Munk DRV_LOG(ERR, "cannot list devices, is ib_uverbs loaded?"); 13312eb4d010SOphir Munk return -rte_errno; 13322eb4d010SOphir Munk } 13332eb4d010SOphir Munk /* 13342eb4d010SOphir Munk * First scan the list of all Infiniband devices to find 13352eb4d010SOphir Munk * matching ones, gathering into the list. 13362eb4d010SOphir Munk */ 13372eb4d010SOphir Munk struct ibv_device *ibv_match[ret + 1]; 13382eb4d010SOphir Munk int nl_route = mlx5_nl_init(NETLINK_ROUTE); 13392eb4d010SOphir Munk int nl_rdma = mlx5_nl_init(NETLINK_RDMA); 13402eb4d010SOphir Munk unsigned int i; 13412eb4d010SOphir Munk 13422eb4d010SOphir Munk while (ret-- > 0) { 13432eb4d010SOphir Munk struct rte_pci_addr pci_addr; 13442eb4d010SOphir Munk 13452eb4d010SOphir Munk DRV_LOG(DEBUG, "checking device \"%s\"", ibv_list[ret]->name); 13462eb4d010SOphir Munk bd = mlx5_device_bond_pci_match 13472eb4d010SOphir Munk (ibv_list[ret], pci_dev, nl_rdma); 13482eb4d010SOphir Munk if (bd >= 0) { 13492eb4d010SOphir Munk /* 13502eb4d010SOphir Munk * Bonding device detected. Only one match is allowed, 13512eb4d010SOphir Munk * the bonding is supported over multi-port IB device, 13522eb4d010SOphir Munk * there should be no matches on representor PCI 13532eb4d010SOphir Munk * functions or non VF LAG bonding devices with 13542eb4d010SOphir Munk * specified address. 13552eb4d010SOphir Munk */ 13562eb4d010SOphir Munk if (nd) { 13572eb4d010SOphir Munk DRV_LOG(ERR, 13582eb4d010SOphir Munk "multiple PCI match on bonding device" 13592eb4d010SOphir Munk "\"%s\" found", ibv_list[ret]->name); 13602eb4d010SOphir Munk rte_errno = ENOENT; 13612eb4d010SOphir Munk ret = -rte_errno; 13622eb4d010SOphir Munk goto exit; 13632eb4d010SOphir Munk } 13642eb4d010SOphir Munk DRV_LOG(INFO, "PCI information matches for" 13652eb4d010SOphir Munk " slave %d bonding device \"%s\"", 13662eb4d010SOphir Munk bd, ibv_list[ret]->name); 13672eb4d010SOphir Munk ibv_match[nd++] = ibv_list[ret]; 13682eb4d010SOphir Munk break; 13692eb4d010SOphir Munk } 13702eb4d010SOphir Munk if (mlx5_dev_to_pci_addr 13712eb4d010SOphir Munk (ibv_list[ret]->ibdev_path, &pci_addr)) 13722eb4d010SOphir Munk continue; 13732eb4d010SOphir Munk if (pci_dev->addr.domain != pci_addr.domain || 13742eb4d010SOphir Munk pci_dev->addr.bus != pci_addr.bus || 13752eb4d010SOphir Munk pci_dev->addr.devid != pci_addr.devid || 13762eb4d010SOphir Munk pci_dev->addr.function != pci_addr.function) 13772eb4d010SOphir Munk continue; 13782eb4d010SOphir Munk DRV_LOG(INFO, "PCI information matches for device \"%s\"", 13792eb4d010SOphir Munk ibv_list[ret]->name); 13802eb4d010SOphir Munk ibv_match[nd++] = ibv_list[ret]; 13812eb4d010SOphir Munk } 13822eb4d010SOphir Munk ibv_match[nd] = NULL; 13832eb4d010SOphir Munk if (!nd) { 13842eb4d010SOphir Munk /* No device matches, just complain and bail out. */ 13852eb4d010SOphir Munk DRV_LOG(WARNING, 13862eb4d010SOphir Munk "no Verbs device matches PCI device " PCI_PRI_FMT "," 13872eb4d010SOphir Munk " are kernel drivers loaded?", 13882eb4d010SOphir Munk pci_dev->addr.domain, pci_dev->addr.bus, 13892eb4d010SOphir Munk pci_dev->addr.devid, pci_dev->addr.function); 13902eb4d010SOphir Munk rte_errno = ENOENT; 13912eb4d010SOphir Munk ret = -rte_errno; 13922eb4d010SOphir Munk goto exit; 13932eb4d010SOphir Munk } 13942eb4d010SOphir Munk if (nd == 1) { 13952eb4d010SOphir Munk /* 13962eb4d010SOphir Munk * Found single matching device may have multiple ports. 13972eb4d010SOphir Munk * Each port may be representor, we have to check the port 13982eb4d010SOphir Munk * number and check the representors existence. 13992eb4d010SOphir Munk */ 14002eb4d010SOphir Munk if (nl_rdma >= 0) 14012eb4d010SOphir Munk np = mlx5_nl_portnum(nl_rdma, ibv_match[0]->name); 14022eb4d010SOphir Munk if (!np) 14032eb4d010SOphir Munk DRV_LOG(WARNING, "can not get IB device \"%s\"" 14042eb4d010SOphir Munk " ports number", ibv_match[0]->name); 14052eb4d010SOphir Munk if (bd >= 0 && !np) { 14062eb4d010SOphir Munk DRV_LOG(ERR, "can not get ports" 14072eb4d010SOphir Munk " for bonding device"); 14082eb4d010SOphir Munk rte_errno = ENOENT; 14092eb4d010SOphir Munk ret = -rte_errno; 14102eb4d010SOphir Munk goto exit; 14112eb4d010SOphir Munk } 14122eb4d010SOphir Munk } 14132eb4d010SOphir Munk #ifndef HAVE_MLX5DV_DR_DEVX_PORT 14142eb4d010SOphir Munk if (bd >= 0) { 14152eb4d010SOphir Munk /* 14162eb4d010SOphir Munk * This may happen if there is VF LAG kernel support and 14172eb4d010SOphir Munk * application is compiled with older rdma_core library. 14182eb4d010SOphir Munk */ 14192eb4d010SOphir Munk DRV_LOG(ERR, 14202eb4d010SOphir Munk "No kernel/verbs support for VF LAG bonding found."); 14212eb4d010SOphir Munk rte_errno = ENOTSUP; 14222eb4d010SOphir Munk ret = -rte_errno; 14232eb4d010SOphir Munk goto exit; 14242eb4d010SOphir Munk } 14252eb4d010SOphir Munk #endif 14262eb4d010SOphir Munk /* 14272eb4d010SOphir Munk * Now we can determine the maximal 14282eb4d010SOphir Munk * amount of devices to be spawned. 14292eb4d010SOphir Munk */ 14302eb4d010SOphir Munk list = rte_zmalloc("device spawn data", 14312eb4d010SOphir Munk sizeof(struct mlx5_dev_spawn_data) * 14322eb4d010SOphir Munk (np ? np : nd), 14332eb4d010SOphir Munk RTE_CACHE_LINE_SIZE); 14342eb4d010SOphir Munk if (!list) { 14352eb4d010SOphir Munk DRV_LOG(ERR, "spawn data array allocation failure"); 14362eb4d010SOphir Munk rte_errno = ENOMEM; 14372eb4d010SOphir Munk ret = -rte_errno; 14382eb4d010SOphir Munk goto exit; 14392eb4d010SOphir Munk } 14402eb4d010SOphir Munk if (bd >= 0 || np > 1) { 14412eb4d010SOphir Munk /* 14422eb4d010SOphir Munk * Single IB device with multiple ports found, 14432eb4d010SOphir Munk * it may be E-Switch master device and representors. 14442eb4d010SOphir Munk * We have to perform identification through the ports. 14452eb4d010SOphir Munk */ 14462eb4d010SOphir Munk MLX5_ASSERT(nl_rdma >= 0); 14472eb4d010SOphir Munk MLX5_ASSERT(ns == 0); 14482eb4d010SOphir Munk MLX5_ASSERT(nd == 1); 14492eb4d010SOphir Munk MLX5_ASSERT(np); 14502eb4d010SOphir Munk for (i = 1; i <= np; ++i) { 14512eb4d010SOphir Munk list[ns].max_port = np; 1452834a9019SOphir Munk list[ns].phys_port = i; 1453834a9019SOphir Munk list[ns].phys_dev = ibv_match[0]; 14542eb4d010SOphir Munk list[ns].eth_dev = NULL; 14552eb4d010SOphir Munk list[ns].pci_dev = pci_dev; 14562eb4d010SOphir Munk list[ns].pf_bond = bd; 14572eb4d010SOphir Munk list[ns].ifindex = mlx5_nl_ifindex 1458834a9019SOphir Munk (nl_rdma, 1459834a9019SOphir Munk mlx5_os_get_dev_device_name 1460834a9019SOphir Munk (list[ns].phys_dev), i); 14612eb4d010SOphir Munk if (!list[ns].ifindex) { 14622eb4d010SOphir Munk /* 14632eb4d010SOphir Munk * No network interface index found for the 14642eb4d010SOphir Munk * specified port, it means there is no 14652eb4d010SOphir Munk * representor on this port. It's OK, 14662eb4d010SOphir Munk * there can be disabled ports, for example 14672eb4d010SOphir Munk * if sriov_numvfs < sriov_totalvfs. 14682eb4d010SOphir Munk */ 14692eb4d010SOphir Munk continue; 14702eb4d010SOphir Munk } 14712eb4d010SOphir Munk ret = -1; 14722eb4d010SOphir Munk if (nl_route >= 0) 14732eb4d010SOphir Munk ret = mlx5_nl_switch_info 14742eb4d010SOphir Munk (nl_route, 14752eb4d010SOphir Munk list[ns].ifindex, 14762eb4d010SOphir Munk &list[ns].info); 14772eb4d010SOphir Munk if (ret || (!list[ns].info.representor && 14782eb4d010SOphir Munk !list[ns].info.master)) { 14792eb4d010SOphir Munk /* 14802eb4d010SOphir Munk * We failed to recognize representors with 14812eb4d010SOphir Munk * Netlink, let's try to perform the task 14822eb4d010SOphir Munk * with sysfs. 14832eb4d010SOphir Munk */ 14842eb4d010SOphir Munk ret = mlx5_sysfs_switch_info 14852eb4d010SOphir Munk (list[ns].ifindex, 14862eb4d010SOphir Munk &list[ns].info); 14872eb4d010SOphir Munk } 14882eb4d010SOphir Munk if (!ret && bd >= 0) { 14892eb4d010SOphir Munk switch (list[ns].info.name_type) { 14902eb4d010SOphir Munk case MLX5_PHYS_PORT_NAME_TYPE_UPLINK: 14912eb4d010SOphir Munk if (list[ns].info.port_name == bd) 14922eb4d010SOphir Munk ns++; 14932eb4d010SOphir Munk break; 14942eb4d010SOphir Munk case MLX5_PHYS_PORT_NAME_TYPE_PFVF: 14952eb4d010SOphir Munk if (list[ns].info.pf_num == bd) 14962eb4d010SOphir Munk ns++; 14972eb4d010SOphir Munk break; 14982eb4d010SOphir Munk default: 14992eb4d010SOphir Munk break; 15002eb4d010SOphir Munk } 15012eb4d010SOphir Munk continue; 15022eb4d010SOphir Munk } 15032eb4d010SOphir Munk if (!ret && (list[ns].info.representor ^ 15042eb4d010SOphir Munk list[ns].info.master)) 15052eb4d010SOphir Munk ns++; 15062eb4d010SOphir Munk } 15072eb4d010SOphir Munk if (!ns) { 15082eb4d010SOphir Munk DRV_LOG(ERR, 15092eb4d010SOphir Munk "unable to recognize master/representors" 15102eb4d010SOphir Munk " on the IB device with multiple ports"); 15112eb4d010SOphir Munk rte_errno = ENOENT; 15122eb4d010SOphir Munk ret = -rte_errno; 15132eb4d010SOphir Munk goto exit; 15142eb4d010SOphir Munk } 15152eb4d010SOphir Munk } else { 15162eb4d010SOphir Munk /* 15172eb4d010SOphir Munk * The existence of several matching entries (nd > 1) means 15182eb4d010SOphir Munk * port representors have been instantiated. No existing Verbs 15192eb4d010SOphir Munk * call nor sysfs entries can tell them apart, this can only 15202eb4d010SOphir Munk * be done through Netlink calls assuming kernel drivers are 15212eb4d010SOphir Munk * recent enough to support them. 15222eb4d010SOphir Munk * 15232eb4d010SOphir Munk * In the event of identification failure through Netlink, 15242eb4d010SOphir Munk * try again through sysfs, then: 15252eb4d010SOphir Munk * 15262eb4d010SOphir Munk * 1. A single IB device matches (nd == 1) with single 15272eb4d010SOphir Munk * port (np=0/1) and is not a representor, assume 15282eb4d010SOphir Munk * no switch support. 15292eb4d010SOphir Munk * 15302eb4d010SOphir Munk * 2. Otherwise no safe assumptions can be made; 15312eb4d010SOphir Munk * complain louder and bail out. 15322eb4d010SOphir Munk */ 15332eb4d010SOphir Munk for (i = 0; i != nd; ++i) { 15342eb4d010SOphir Munk memset(&list[ns].info, 0, sizeof(list[ns].info)); 15352eb4d010SOphir Munk list[ns].max_port = 1; 1536834a9019SOphir Munk list[ns].phys_port = 1; 1537834a9019SOphir Munk list[ns].phys_dev = ibv_match[i]; 15382eb4d010SOphir Munk list[ns].eth_dev = NULL; 15392eb4d010SOphir Munk list[ns].pci_dev = pci_dev; 15402eb4d010SOphir Munk list[ns].pf_bond = -1; 15412eb4d010SOphir Munk list[ns].ifindex = 0; 15422eb4d010SOphir Munk if (nl_rdma >= 0) 15432eb4d010SOphir Munk list[ns].ifindex = mlx5_nl_ifindex 1544834a9019SOphir Munk (nl_rdma, 1545834a9019SOphir Munk mlx5_os_get_dev_device_name 1546834a9019SOphir Munk (list[ns].phys_dev), 1); 15472eb4d010SOphir Munk if (!list[ns].ifindex) { 15482eb4d010SOphir Munk char ifname[IF_NAMESIZE]; 15492eb4d010SOphir Munk 15502eb4d010SOphir Munk /* 15512eb4d010SOphir Munk * Netlink failed, it may happen with old 15522eb4d010SOphir Munk * ib_core kernel driver (before 4.16). 15532eb4d010SOphir Munk * We can assume there is old driver because 15542eb4d010SOphir Munk * here we are processing single ports IB 15552eb4d010SOphir Munk * devices. Let's try sysfs to retrieve 15562eb4d010SOphir Munk * the ifindex. The method works for 15572eb4d010SOphir Munk * master device only. 15582eb4d010SOphir Munk */ 15592eb4d010SOphir Munk if (nd > 1) { 15602eb4d010SOphir Munk /* 15612eb4d010SOphir Munk * Multiple devices found, assume 15622eb4d010SOphir Munk * representors, can not distinguish 15632eb4d010SOphir Munk * master/representor and retrieve 15642eb4d010SOphir Munk * ifindex via sysfs. 15652eb4d010SOphir Munk */ 15662eb4d010SOphir Munk continue; 15672eb4d010SOphir Munk } 1568aec086c9SMatan Azrad ret = mlx5_get_ifname_sysfs 1569aec086c9SMatan Azrad (ibv_match[i]->ibdev_path, ifname); 15702eb4d010SOphir Munk if (!ret) 15712eb4d010SOphir Munk list[ns].ifindex = 15722eb4d010SOphir Munk if_nametoindex(ifname); 15732eb4d010SOphir Munk if (!list[ns].ifindex) { 15742eb4d010SOphir Munk /* 15752eb4d010SOphir Munk * No network interface index found 15762eb4d010SOphir Munk * for the specified device, it means 15772eb4d010SOphir Munk * there it is neither representor 15782eb4d010SOphir Munk * nor master. 15792eb4d010SOphir Munk */ 15802eb4d010SOphir Munk continue; 15812eb4d010SOphir Munk } 15822eb4d010SOphir Munk } 15832eb4d010SOphir Munk ret = -1; 15842eb4d010SOphir Munk if (nl_route >= 0) 15852eb4d010SOphir Munk ret = mlx5_nl_switch_info 15862eb4d010SOphir Munk (nl_route, 15872eb4d010SOphir Munk list[ns].ifindex, 15882eb4d010SOphir Munk &list[ns].info); 15892eb4d010SOphir Munk if (ret || (!list[ns].info.representor && 15902eb4d010SOphir Munk !list[ns].info.master)) { 15912eb4d010SOphir Munk /* 15922eb4d010SOphir Munk * We failed to recognize representors with 15932eb4d010SOphir Munk * Netlink, let's try to perform the task 15942eb4d010SOphir Munk * with sysfs. 15952eb4d010SOphir Munk */ 15962eb4d010SOphir Munk ret = mlx5_sysfs_switch_info 15972eb4d010SOphir Munk (list[ns].ifindex, 15982eb4d010SOphir Munk &list[ns].info); 15992eb4d010SOphir Munk } 16002eb4d010SOphir Munk if (!ret && (list[ns].info.representor ^ 16012eb4d010SOphir Munk list[ns].info.master)) { 16022eb4d010SOphir Munk ns++; 16032eb4d010SOphir Munk } else if ((nd == 1) && 16042eb4d010SOphir Munk !list[ns].info.representor && 16052eb4d010SOphir Munk !list[ns].info.master) { 16062eb4d010SOphir Munk /* 16072eb4d010SOphir Munk * Single IB device with 16082eb4d010SOphir Munk * one physical port and 16092eb4d010SOphir Munk * attached network device. 16102eb4d010SOphir Munk * May be SRIOV is not enabled 16112eb4d010SOphir Munk * or there is no representors. 16122eb4d010SOphir Munk */ 16132eb4d010SOphir Munk DRV_LOG(INFO, "no E-Switch support detected"); 16142eb4d010SOphir Munk ns++; 16152eb4d010SOphir Munk break; 16162eb4d010SOphir Munk } 16172eb4d010SOphir Munk } 16182eb4d010SOphir Munk if (!ns) { 16192eb4d010SOphir Munk DRV_LOG(ERR, 16202eb4d010SOphir Munk "unable to recognize master/representors" 16212eb4d010SOphir Munk " on the multiple IB devices"); 16222eb4d010SOphir Munk rte_errno = ENOENT; 16232eb4d010SOphir Munk ret = -rte_errno; 16242eb4d010SOphir Munk goto exit; 16252eb4d010SOphir Munk } 16262eb4d010SOphir Munk } 16272eb4d010SOphir Munk MLX5_ASSERT(ns); 16282eb4d010SOphir Munk /* 16292eb4d010SOphir Munk * Sort list to probe devices in natural order for users convenience 16302eb4d010SOphir Munk * (i.e. master first, then representors from lowest to highest ID). 16312eb4d010SOphir Munk */ 16322eb4d010SOphir Munk qsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp); 16332eb4d010SOphir Munk /* Default configuration. */ 16342eb4d010SOphir Munk dev_config = (struct mlx5_dev_config){ 16352eb4d010SOphir Munk .hw_padding = 0, 16362eb4d010SOphir Munk .mps = MLX5_ARG_UNSET, 16372eb4d010SOphir Munk .dbnc = MLX5_ARG_UNSET, 16382eb4d010SOphir Munk .rx_vec_en = 1, 16392eb4d010SOphir Munk .txq_inline_max = MLX5_ARG_UNSET, 16402eb4d010SOphir Munk .txq_inline_min = MLX5_ARG_UNSET, 16412eb4d010SOphir Munk .txq_inline_mpw = MLX5_ARG_UNSET, 16422eb4d010SOphir Munk .txqs_inline = MLX5_ARG_UNSET, 16432eb4d010SOphir Munk .vf_nl_en = 1, 16442eb4d010SOphir Munk .mr_ext_memseg_en = 1, 16452eb4d010SOphir Munk .mprq = { 16462eb4d010SOphir Munk .enabled = 0, /* Disabled by default. */ 16472eb4d010SOphir Munk .stride_num_n = 0, 16482eb4d010SOphir Munk .stride_size_n = 0, 16492eb4d010SOphir Munk .max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN, 16502eb4d010SOphir Munk .min_rxqs_num = MLX5_MPRQ_MIN_RXQS, 16512eb4d010SOphir Munk }, 16522eb4d010SOphir Munk .dv_esw_en = 1, 16532eb4d010SOphir Munk .dv_flow_en = 1, 16542eb4d010SOphir Munk .log_hp_size = MLX5_ARG_UNSET, 16552eb4d010SOphir Munk }; 16562eb4d010SOphir Munk /* Device specific configuration. */ 16572eb4d010SOphir Munk switch (pci_dev->id.device_id) { 16582eb4d010SOphir Munk case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 16592eb4d010SOphir Munk case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 16602eb4d010SOphir Munk case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 16612eb4d010SOphir Munk case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 16622eb4d010SOphir Munk case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF: 16632eb4d010SOphir Munk case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF: 16642eb4d010SOphir Munk case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF: 16652eb4d010SOphir Munk dev_config.vf = 1; 16662eb4d010SOphir Munk break; 16672eb4d010SOphir Munk default: 16682eb4d010SOphir Munk break; 16692eb4d010SOphir Munk } 16702eb4d010SOphir Munk for (i = 0; i != ns; ++i) { 16712eb4d010SOphir Munk uint32_t restore; 16722eb4d010SOphir Munk 16732eb4d010SOphir Munk list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device, 16742eb4d010SOphir Munk &list[i], 16752eb4d010SOphir Munk dev_config); 16762eb4d010SOphir Munk if (!list[i].eth_dev) { 16772eb4d010SOphir Munk if (rte_errno != EBUSY && rte_errno != EEXIST) 16782eb4d010SOphir Munk break; 16792eb4d010SOphir Munk /* Device is disabled or already spawned. Ignore it. */ 16802eb4d010SOphir Munk continue; 16812eb4d010SOphir Munk } 16822eb4d010SOphir Munk restore = list[i].eth_dev->data->dev_flags; 16832eb4d010SOphir Munk rte_eth_copy_pci_info(list[i].eth_dev, pci_dev); 16842eb4d010SOphir Munk /* Restore non-PCI flags cleared by the above call. */ 16852eb4d010SOphir Munk list[i].eth_dev->data->dev_flags |= restore; 16862eb4d010SOphir Munk rte_eth_dev_probing_finish(list[i].eth_dev); 16872eb4d010SOphir Munk } 16882eb4d010SOphir Munk if (i != ns) { 16892eb4d010SOphir Munk DRV_LOG(ERR, 16902eb4d010SOphir Munk "probe of PCI device " PCI_PRI_FMT " aborted after" 16912eb4d010SOphir Munk " encountering an error: %s", 16922eb4d010SOphir Munk pci_dev->addr.domain, pci_dev->addr.bus, 16932eb4d010SOphir Munk pci_dev->addr.devid, pci_dev->addr.function, 16942eb4d010SOphir Munk strerror(rte_errno)); 16952eb4d010SOphir Munk ret = -rte_errno; 16962eb4d010SOphir Munk /* Roll back. */ 16972eb4d010SOphir Munk while (i--) { 16982eb4d010SOphir Munk if (!list[i].eth_dev) 16992eb4d010SOphir Munk continue; 17002eb4d010SOphir Munk mlx5_dev_close(list[i].eth_dev); 17012eb4d010SOphir Munk /* mac_addrs must not be freed because in dev_private */ 17022eb4d010SOphir Munk list[i].eth_dev->data->mac_addrs = NULL; 17032eb4d010SOphir Munk claim_zero(rte_eth_dev_release_port(list[i].eth_dev)); 17042eb4d010SOphir Munk } 17052eb4d010SOphir Munk /* Restore original error. */ 17062eb4d010SOphir Munk rte_errno = -ret; 17072eb4d010SOphir Munk } else { 17082eb4d010SOphir Munk ret = 0; 17092eb4d010SOphir Munk } 17102eb4d010SOphir Munk exit: 17112eb4d010SOphir Munk /* 17122eb4d010SOphir Munk * Do the routine cleanup: 17132eb4d010SOphir Munk * - close opened Netlink sockets 17142eb4d010SOphir Munk * - free allocated spawn data array 17152eb4d010SOphir Munk * - free the Infiniband device list 17162eb4d010SOphir Munk */ 17172eb4d010SOphir Munk if (nl_rdma >= 0) 17182eb4d010SOphir Munk close(nl_rdma); 17192eb4d010SOphir Munk if (nl_route >= 0) 17202eb4d010SOphir Munk close(nl_route); 17212eb4d010SOphir Munk if (list) 17222eb4d010SOphir Munk rte_free(list); 17232eb4d010SOphir Munk MLX5_ASSERT(ibv_list); 17242eb4d010SOphir Munk mlx5_glue->free_device_list(ibv_list); 17252eb4d010SOphir Munk return ret; 17262eb4d010SOphir Munk } 17272eb4d010SOphir Munk 17282eb4d010SOphir Munk static int 17292eb4d010SOphir Munk mlx5_config_doorbell_mapping_env(const struct mlx5_dev_config *config) 17302eb4d010SOphir Munk { 17312eb4d010SOphir Munk char *env; 17322eb4d010SOphir Munk int value; 17332eb4d010SOphir Munk 17342eb4d010SOphir Munk MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 17352eb4d010SOphir Munk /* Get environment variable to store. */ 17362eb4d010SOphir Munk env = getenv(MLX5_SHUT_UP_BF); 17372eb4d010SOphir Munk value = env ? !!strcmp(env, "0") : MLX5_ARG_UNSET; 17382eb4d010SOphir Munk if (config->dbnc == MLX5_ARG_UNSET) 17392eb4d010SOphir Munk setenv(MLX5_SHUT_UP_BF, MLX5_SHUT_UP_BF_DEFAULT, 1); 17402eb4d010SOphir Munk else 17412eb4d010SOphir Munk setenv(MLX5_SHUT_UP_BF, 17422eb4d010SOphir Munk config->dbnc == MLX5_TXDB_NCACHED ? "1" : "0", 1); 17432eb4d010SOphir Munk return value; 17442eb4d010SOphir Munk } 17452eb4d010SOphir Munk 17462eb4d010SOphir Munk static void 17472eb4d010SOphir Munk mlx5_restore_doorbell_mapping_env(int value) 17482eb4d010SOphir Munk { 17492eb4d010SOphir Munk MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 17502eb4d010SOphir Munk /* Restore the original environment variable state. */ 17512eb4d010SOphir Munk if (value == MLX5_ARG_UNSET) 17522eb4d010SOphir Munk unsetenv(MLX5_SHUT_UP_BF); 17532eb4d010SOphir Munk else 17542eb4d010SOphir Munk setenv(MLX5_SHUT_UP_BF, value ? "1" : "0", 1); 17552eb4d010SOphir Munk } 17562eb4d010SOphir Munk 17572eb4d010SOphir Munk /** 17582eb4d010SOphir Munk * Extract pdn of PD object using DV API. 17592eb4d010SOphir Munk * 17602eb4d010SOphir Munk * @param[in] pd 17612eb4d010SOphir Munk * Pointer to the verbs PD object. 17622eb4d010SOphir Munk * @param[out] pdn 17632eb4d010SOphir Munk * Pointer to the PD object number variable. 17642eb4d010SOphir Munk * 17652eb4d010SOphir Munk * @return 17662eb4d010SOphir Munk * 0 on success, error value otherwise. 17672eb4d010SOphir Munk */ 17682eb4d010SOphir Munk int 17692eb4d010SOphir Munk mlx5_os_get_pdn(void *pd, uint32_t *pdn) 17702eb4d010SOphir Munk { 17712eb4d010SOphir Munk #ifdef HAVE_IBV_FLOW_DV_SUPPORT 17722eb4d010SOphir Munk struct mlx5dv_obj obj; 17732eb4d010SOphir Munk struct mlx5dv_pd pd_info; 17742eb4d010SOphir Munk int ret = 0; 17752eb4d010SOphir Munk 17762eb4d010SOphir Munk obj.pd.in = pd; 17772eb4d010SOphir Munk obj.pd.out = &pd_info; 17782eb4d010SOphir Munk ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD); 17792eb4d010SOphir Munk if (ret) { 17802eb4d010SOphir Munk DRV_LOG(DEBUG, "Fail to get PD object info"); 17812eb4d010SOphir Munk return ret; 17822eb4d010SOphir Munk } 17832eb4d010SOphir Munk *pdn = pd_info.pdn; 17842eb4d010SOphir Munk return 0; 17852eb4d010SOphir Munk #else 17862eb4d010SOphir Munk (void)pd; 17872eb4d010SOphir Munk (void)pdn; 17882eb4d010SOphir Munk return -ENOTSUP; 17892eb4d010SOphir Munk #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 17902eb4d010SOphir Munk } 17912eb4d010SOphir Munk 17922eb4d010SOphir Munk /** 17932eb4d010SOphir Munk * Function API to open IB device. 17942eb4d010SOphir Munk * 17952eb4d010SOphir Munk * This function calls the Linux glue APIs to open a device. 17962eb4d010SOphir Munk * 17972eb4d010SOphir Munk * @param[in] spawn 17982eb4d010SOphir Munk * Pointer to the IB device attributes (name, port, etc). 17992eb4d010SOphir Munk * @param[out] config 18002eb4d010SOphir Munk * Pointer to device configuration structure. 18012eb4d010SOphir Munk * @param[out] sh 18022eb4d010SOphir Munk * Pointer to shared context structure. 18032eb4d010SOphir Munk * 18042eb4d010SOphir Munk * @return 18052eb4d010SOphir Munk * 0 on success, a positive error value otherwise. 18062eb4d010SOphir Munk */ 18072eb4d010SOphir Munk int 18082eb4d010SOphir Munk mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn, 18092eb4d010SOphir Munk const struct mlx5_dev_config *config, 18102eb4d010SOphir Munk struct mlx5_dev_ctx_shared *sh) 18112eb4d010SOphir Munk { 18122eb4d010SOphir Munk int dbmap_env; 18132eb4d010SOphir Munk int err = 0; 18142eb4d010SOphir Munk /* 18152eb4d010SOphir Munk * Configure environment variable "MLX5_BF_SHUT_UP" 18162eb4d010SOphir Munk * before the device creation. The rdma_core library 18172eb4d010SOphir Munk * checks the variable at device creation and 18182eb4d010SOphir Munk * stores the result internally. 18192eb4d010SOphir Munk */ 18202eb4d010SOphir Munk dbmap_env = mlx5_config_doorbell_mapping_env(config); 18212eb4d010SOphir Munk /* Try to open IB device with DV first, then usual Verbs. */ 18222eb4d010SOphir Munk errno = 0; 1823834a9019SOphir Munk sh->ctx = mlx5_glue->dv_open_device(spawn->phys_dev); 18242eb4d010SOphir Munk if (sh->ctx) { 18252eb4d010SOphir Munk sh->devx = 1; 18262eb4d010SOphir Munk DRV_LOG(DEBUG, "DevX is supported"); 18272eb4d010SOphir Munk /* The device is created, no need for environment. */ 18282eb4d010SOphir Munk mlx5_restore_doorbell_mapping_env(dbmap_env); 18292eb4d010SOphir Munk } else { 18302eb4d010SOphir Munk /* The environment variable is still configured. */ 1831834a9019SOphir Munk sh->ctx = mlx5_glue->open_device(spawn->phys_dev); 18322eb4d010SOphir Munk err = errno ? errno : ENODEV; 18332eb4d010SOphir Munk /* 18342eb4d010SOphir Munk * The environment variable is not needed anymore, 18352eb4d010SOphir Munk * all device creation attempts are completed. 18362eb4d010SOphir Munk */ 18372eb4d010SOphir Munk mlx5_restore_doorbell_mapping_env(dbmap_env); 18382eb4d010SOphir Munk if (!sh->ctx) 18392eb4d010SOphir Munk return err; 18402eb4d010SOphir Munk DRV_LOG(DEBUG, "DevX is NOT supported"); 18412eb4d010SOphir Munk err = 0; 18422eb4d010SOphir Munk } 18432eb4d010SOphir Munk return err; 18442eb4d010SOphir Munk } 18452eb4d010SOphir Munk 18462eb4d010SOphir Munk /** 18472eb4d010SOphir Munk * Install shared asynchronous device events handler. 18482eb4d010SOphir Munk * This function is implemented to support event sharing 18492eb4d010SOphir Munk * between multiple ports of single IB device. 18502eb4d010SOphir Munk * 18512eb4d010SOphir Munk * @param sh 18522eb4d010SOphir Munk * Pointer to mlx5_dev_ctx_shared object. 18532eb4d010SOphir Munk */ 18542eb4d010SOphir Munk void 18552eb4d010SOphir Munk mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh) 18562eb4d010SOphir Munk { 18572eb4d010SOphir Munk int ret; 18582eb4d010SOphir Munk int flags; 18592eb4d010SOphir Munk 18602eb4d010SOphir Munk sh->intr_handle.fd = -1; 18612eb4d010SOphir Munk flags = fcntl(((struct ibv_context *)sh->ctx)->async_fd, F_GETFL); 18622eb4d010SOphir Munk ret = fcntl(((struct ibv_context *)sh->ctx)->async_fd, 18632eb4d010SOphir Munk F_SETFL, flags | O_NONBLOCK); 18642eb4d010SOphir Munk if (ret) { 18652eb4d010SOphir Munk DRV_LOG(INFO, "failed to change file descriptor async event" 18662eb4d010SOphir Munk " queue"); 18672eb4d010SOphir Munk } else { 18682eb4d010SOphir Munk sh->intr_handle.fd = ((struct ibv_context *)sh->ctx)->async_fd; 18692eb4d010SOphir Munk sh->intr_handle.type = RTE_INTR_HANDLE_EXT; 18702eb4d010SOphir Munk if (rte_intr_callback_register(&sh->intr_handle, 18712eb4d010SOphir Munk mlx5_dev_interrupt_handler, sh)) { 18722eb4d010SOphir Munk DRV_LOG(INFO, "Fail to install the shared interrupt."); 18732eb4d010SOphir Munk sh->intr_handle.fd = -1; 18742eb4d010SOphir Munk } 18752eb4d010SOphir Munk } 18762eb4d010SOphir Munk if (sh->devx) { 18772eb4d010SOphir Munk #ifdef HAVE_IBV_DEVX_ASYNC 18782eb4d010SOphir Munk sh->intr_handle_devx.fd = -1; 187921b7c452SOphir Munk sh->devx_comp = 188021b7c452SOphir Munk (void *)mlx5_glue->devx_create_cmd_comp(sh->ctx); 188121b7c452SOphir Munk struct mlx5dv_devx_cmd_comp *devx_comp = sh->devx_comp; 188221b7c452SOphir Munk if (!devx_comp) { 18832eb4d010SOphir Munk DRV_LOG(INFO, "failed to allocate devx_comp."); 18842eb4d010SOphir Munk return; 18852eb4d010SOphir Munk } 188621b7c452SOphir Munk flags = fcntl(devx_comp->fd, F_GETFL); 188721b7c452SOphir Munk ret = fcntl(devx_comp->fd, F_SETFL, flags | O_NONBLOCK); 18882eb4d010SOphir Munk if (ret) { 18892eb4d010SOphir Munk DRV_LOG(INFO, "failed to change file descriptor" 18902eb4d010SOphir Munk " devx comp"); 18912eb4d010SOphir Munk return; 18922eb4d010SOphir Munk } 189321b7c452SOphir Munk sh->intr_handle_devx.fd = devx_comp->fd; 18942eb4d010SOphir Munk sh->intr_handle_devx.type = RTE_INTR_HANDLE_EXT; 18952eb4d010SOphir Munk if (rte_intr_callback_register(&sh->intr_handle_devx, 18962eb4d010SOphir Munk mlx5_dev_interrupt_handler_devx, sh)) { 18972eb4d010SOphir Munk DRV_LOG(INFO, "Fail to install the devx shared" 18982eb4d010SOphir Munk " interrupt."); 18992eb4d010SOphir Munk sh->intr_handle_devx.fd = -1; 19002eb4d010SOphir Munk } 19012eb4d010SOphir Munk #endif /* HAVE_IBV_DEVX_ASYNC */ 19022eb4d010SOphir Munk } 19032eb4d010SOphir Munk } 19042eb4d010SOphir Munk 19052eb4d010SOphir Munk /** 19062eb4d010SOphir Munk * Uninstall shared asynchronous device events handler. 19072eb4d010SOphir Munk * This function is implemented to support event sharing 19082eb4d010SOphir Munk * between multiple ports of single IB device. 19092eb4d010SOphir Munk * 19102eb4d010SOphir Munk * @param dev 19112eb4d010SOphir Munk * Pointer to mlx5_dev_ctx_shared object. 19122eb4d010SOphir Munk */ 19132eb4d010SOphir Munk void 19142eb4d010SOphir Munk mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh) 19152eb4d010SOphir Munk { 19162eb4d010SOphir Munk if (sh->intr_handle.fd >= 0) 19172eb4d010SOphir Munk mlx5_intr_callback_unregister(&sh->intr_handle, 19182eb4d010SOphir Munk mlx5_dev_interrupt_handler, sh); 19192eb4d010SOphir Munk #ifdef HAVE_IBV_DEVX_ASYNC 19202eb4d010SOphir Munk if (sh->intr_handle_devx.fd >= 0) 19212eb4d010SOphir Munk rte_intr_callback_unregister(&sh->intr_handle_devx, 19222eb4d010SOphir Munk mlx5_dev_interrupt_handler_devx, sh); 19232eb4d010SOphir Munk if (sh->devx_comp) 19242eb4d010SOphir Munk mlx5_glue->devx_destroy_cmd_comp(sh->devx_comp); 19252eb4d010SOphir Munk #endif 19262eb4d010SOphir Munk } 1927042f5c94SOphir Munk 192873bf9235SOphir Munk /** 192973bf9235SOphir Munk * Read statistics by a named counter. 193073bf9235SOphir Munk * 193173bf9235SOphir Munk * @param[in] priv 193273bf9235SOphir Munk * Pointer to the private device data structure. 193373bf9235SOphir Munk * @param[in] ctr_name 193473bf9235SOphir Munk * Pointer to the name of the statistic counter to read 193573bf9235SOphir Munk * @param[out] stat 193673bf9235SOphir Munk * Pointer to read statistic value. 193773bf9235SOphir Munk * @return 193873bf9235SOphir Munk * 0 on success and stat is valud, 1 if failed to read the value 193973bf9235SOphir Munk * rte_errno is set. 194073bf9235SOphir Munk * 194173bf9235SOphir Munk */ 194273bf9235SOphir Munk int 194373bf9235SOphir Munk mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name, 194473bf9235SOphir Munk uint64_t *stat) 194573bf9235SOphir Munk { 194673bf9235SOphir Munk int fd; 194773bf9235SOphir Munk 194873bf9235SOphir Munk if (priv->sh) { 194973bf9235SOphir Munk MKSTR(path, "%s/ports/%d/hw_counters/%s", 195073bf9235SOphir Munk priv->sh->ibdev_path, 195173bf9235SOphir Munk priv->dev_port, 195273bf9235SOphir Munk ctr_name); 195373bf9235SOphir Munk fd = open(path, O_RDONLY); 195473bf9235SOphir Munk if (fd != -1) { 195573bf9235SOphir Munk char buf[21] = {'\0'}; 195673bf9235SOphir Munk ssize_t n = read(fd, buf, sizeof(buf)); 195773bf9235SOphir Munk 195873bf9235SOphir Munk close(fd); 195973bf9235SOphir Munk if (n != -1) { 196073bf9235SOphir Munk *stat = strtoull(buf, NULL, 10); 196173bf9235SOphir Munk return 0; 196273bf9235SOphir Munk } 196373bf9235SOphir Munk } 196473bf9235SOphir Munk } 196573bf9235SOphir Munk *stat = 0; 196673bf9235SOphir Munk return 1; 196773bf9235SOphir Munk } 196873bf9235SOphir Munk 196973bf9235SOphir Munk /** 197073bf9235SOphir Munk * Read device counters table. 197173bf9235SOphir Munk * 197273bf9235SOphir Munk * @param dev 197373bf9235SOphir Munk * Pointer to Ethernet device. 197473bf9235SOphir Munk * @param[out] stats 197573bf9235SOphir Munk * Counters table output buffer. 197673bf9235SOphir Munk * 197773bf9235SOphir Munk * @return 197873bf9235SOphir Munk * 0 on success and stats is filled, negative errno value otherwise and 197973bf9235SOphir Munk * rte_errno is set. 198073bf9235SOphir Munk */ 198173bf9235SOphir Munk int 198273bf9235SOphir Munk mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) 198373bf9235SOphir Munk { 198473bf9235SOphir Munk struct mlx5_priv *priv = dev->data->dev_private; 198573bf9235SOphir Munk struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; 198673bf9235SOphir Munk unsigned int i; 198773bf9235SOphir Munk struct ifreq ifr; 198873bf9235SOphir Munk unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t); 198973bf9235SOphir Munk unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz]; 199073bf9235SOphir Munk struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf; 199173bf9235SOphir Munk int ret; 199273bf9235SOphir Munk 199373bf9235SOphir Munk et_stats->cmd = ETHTOOL_GSTATS; 199473bf9235SOphir Munk et_stats->n_stats = xstats_ctrl->stats_n; 199573bf9235SOphir Munk ifr.ifr_data = (caddr_t)et_stats; 199673bf9235SOphir Munk ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); 199773bf9235SOphir Munk if (ret) { 199873bf9235SOphir Munk DRV_LOG(WARNING, 199973bf9235SOphir Munk "port %u unable to read statistic values from device", 200073bf9235SOphir Munk dev->data->port_id); 200173bf9235SOphir Munk return ret; 200273bf9235SOphir Munk } 200373bf9235SOphir Munk for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) { 200473bf9235SOphir Munk if (xstats_ctrl->info[i].dev) { 200573bf9235SOphir Munk ret = mlx5_os_read_dev_stat(priv, 200673bf9235SOphir Munk xstats_ctrl->info[i].ctr_name, 200773bf9235SOphir Munk &stats[i]); 200873bf9235SOphir Munk /* return last xstats counter if fail to read. */ 200973bf9235SOphir Munk if (ret == 0) 201073bf9235SOphir Munk xstats_ctrl->xstats[i] = stats[i]; 201173bf9235SOphir Munk else 201273bf9235SOphir Munk stats[i] = xstats_ctrl->xstats[i]; 201373bf9235SOphir Munk } else { 201473bf9235SOphir Munk stats[i] = (uint64_t) 201573bf9235SOphir Munk et_stats->data[xstats_ctrl->dev_table_idx[i]]; 201673bf9235SOphir Munk } 201773bf9235SOphir Munk } 201873bf9235SOphir Munk return 0; 201973bf9235SOphir Munk } 202073bf9235SOphir Munk 202173bf9235SOphir Munk /** 202273bf9235SOphir Munk * Query the number of statistics provided by ETHTOOL. 202373bf9235SOphir Munk * 202473bf9235SOphir Munk * @param dev 202573bf9235SOphir Munk * Pointer to Ethernet device. 202673bf9235SOphir Munk * 202773bf9235SOphir Munk * @return 202873bf9235SOphir Munk * Number of statistics on success, negative errno value otherwise and 202973bf9235SOphir Munk * rte_errno is set. 203073bf9235SOphir Munk */ 203173bf9235SOphir Munk int 203273bf9235SOphir Munk mlx5_os_get_stats_n(struct rte_eth_dev *dev) 203373bf9235SOphir Munk { 203473bf9235SOphir Munk struct ethtool_drvinfo drvinfo; 203573bf9235SOphir Munk struct ifreq ifr; 203673bf9235SOphir Munk int ret; 203773bf9235SOphir Munk 203873bf9235SOphir Munk drvinfo.cmd = ETHTOOL_GDRVINFO; 203973bf9235SOphir Munk ifr.ifr_data = (caddr_t)&drvinfo; 204073bf9235SOphir Munk ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); 204173bf9235SOphir Munk if (ret) { 204273bf9235SOphir Munk DRV_LOG(WARNING, "port %u unable to query number of statistics", 204373bf9235SOphir Munk dev->data->port_id); 204473bf9235SOphir Munk return ret; 204573bf9235SOphir Munk } 204673bf9235SOphir Munk return drvinfo.n_stats; 204773bf9235SOphir Munk } 204873bf9235SOphir Munk 204973bf9235SOphir Munk static const struct mlx5_counter_ctrl mlx5_counters_init[] = { 205073bf9235SOphir Munk { 205173bf9235SOphir Munk .dpdk_name = "rx_port_unicast_bytes", 205273bf9235SOphir Munk .ctr_name = "rx_vport_unicast_bytes", 205373bf9235SOphir Munk }, 205473bf9235SOphir Munk { 205573bf9235SOphir Munk .dpdk_name = "rx_port_multicast_bytes", 205673bf9235SOphir Munk .ctr_name = "rx_vport_multicast_bytes", 205773bf9235SOphir Munk }, 205873bf9235SOphir Munk { 205973bf9235SOphir Munk .dpdk_name = "rx_port_broadcast_bytes", 206073bf9235SOphir Munk .ctr_name = "rx_vport_broadcast_bytes", 206173bf9235SOphir Munk }, 206273bf9235SOphir Munk { 206373bf9235SOphir Munk .dpdk_name = "rx_port_unicast_packets", 206473bf9235SOphir Munk .ctr_name = "rx_vport_unicast_packets", 206573bf9235SOphir Munk }, 206673bf9235SOphir Munk { 206773bf9235SOphir Munk .dpdk_name = "rx_port_multicast_packets", 206873bf9235SOphir Munk .ctr_name = "rx_vport_multicast_packets", 206973bf9235SOphir Munk }, 207073bf9235SOphir Munk { 207173bf9235SOphir Munk .dpdk_name = "rx_port_broadcast_packets", 207273bf9235SOphir Munk .ctr_name = "rx_vport_broadcast_packets", 207373bf9235SOphir Munk }, 207473bf9235SOphir Munk { 207573bf9235SOphir Munk .dpdk_name = "tx_port_unicast_bytes", 207673bf9235SOphir Munk .ctr_name = "tx_vport_unicast_bytes", 207773bf9235SOphir Munk }, 207873bf9235SOphir Munk { 207973bf9235SOphir Munk .dpdk_name = "tx_port_multicast_bytes", 208073bf9235SOphir Munk .ctr_name = "tx_vport_multicast_bytes", 208173bf9235SOphir Munk }, 208273bf9235SOphir Munk { 208373bf9235SOphir Munk .dpdk_name = "tx_port_broadcast_bytes", 208473bf9235SOphir Munk .ctr_name = "tx_vport_broadcast_bytes", 208573bf9235SOphir Munk }, 208673bf9235SOphir Munk { 208773bf9235SOphir Munk .dpdk_name = "tx_port_unicast_packets", 208873bf9235SOphir Munk .ctr_name = "tx_vport_unicast_packets", 208973bf9235SOphir Munk }, 209073bf9235SOphir Munk { 209173bf9235SOphir Munk .dpdk_name = "tx_port_multicast_packets", 209273bf9235SOphir Munk .ctr_name = "tx_vport_multicast_packets", 209373bf9235SOphir Munk }, 209473bf9235SOphir Munk { 209573bf9235SOphir Munk .dpdk_name = "tx_port_broadcast_packets", 209673bf9235SOphir Munk .ctr_name = "tx_vport_broadcast_packets", 209773bf9235SOphir Munk }, 209873bf9235SOphir Munk { 209973bf9235SOphir Munk .dpdk_name = "rx_wqe_err", 210073bf9235SOphir Munk .ctr_name = "rx_wqe_err", 210173bf9235SOphir Munk }, 210273bf9235SOphir Munk { 210373bf9235SOphir Munk .dpdk_name = "rx_crc_errors_phy", 210473bf9235SOphir Munk .ctr_name = "rx_crc_errors_phy", 210573bf9235SOphir Munk }, 210673bf9235SOphir Munk { 210773bf9235SOphir Munk .dpdk_name = "rx_in_range_len_errors_phy", 210873bf9235SOphir Munk .ctr_name = "rx_in_range_len_errors_phy", 210973bf9235SOphir Munk }, 211073bf9235SOphir Munk { 211173bf9235SOphir Munk .dpdk_name = "rx_symbol_err_phy", 211273bf9235SOphir Munk .ctr_name = "rx_symbol_err_phy", 211373bf9235SOphir Munk }, 211473bf9235SOphir Munk { 211573bf9235SOphir Munk .dpdk_name = "tx_errors_phy", 211673bf9235SOphir Munk .ctr_name = "tx_errors_phy", 211773bf9235SOphir Munk }, 211873bf9235SOphir Munk { 211973bf9235SOphir Munk .dpdk_name = "rx_out_of_buffer", 212073bf9235SOphir Munk .ctr_name = "out_of_buffer", 212173bf9235SOphir Munk .dev = 1, 212273bf9235SOphir Munk }, 212373bf9235SOphir Munk { 212473bf9235SOphir Munk .dpdk_name = "tx_packets_phy", 212573bf9235SOphir Munk .ctr_name = "tx_packets_phy", 212673bf9235SOphir Munk }, 212773bf9235SOphir Munk { 212873bf9235SOphir Munk .dpdk_name = "rx_packets_phy", 212973bf9235SOphir Munk .ctr_name = "rx_packets_phy", 213073bf9235SOphir Munk }, 213173bf9235SOphir Munk { 213273bf9235SOphir Munk .dpdk_name = "tx_discards_phy", 213373bf9235SOphir Munk .ctr_name = "tx_discards_phy", 213473bf9235SOphir Munk }, 213573bf9235SOphir Munk { 213673bf9235SOphir Munk .dpdk_name = "rx_discards_phy", 213773bf9235SOphir Munk .ctr_name = "rx_discards_phy", 213873bf9235SOphir Munk }, 213973bf9235SOphir Munk { 214073bf9235SOphir Munk .dpdk_name = "tx_bytes_phy", 214173bf9235SOphir Munk .ctr_name = "tx_bytes_phy", 214273bf9235SOphir Munk }, 214373bf9235SOphir Munk { 214473bf9235SOphir Munk .dpdk_name = "rx_bytes_phy", 214573bf9235SOphir Munk .ctr_name = "rx_bytes_phy", 214673bf9235SOphir Munk }, 214773bf9235SOphir Munk /* Representor only */ 214873bf9235SOphir Munk { 214973bf9235SOphir Munk .dpdk_name = "rx_packets", 215073bf9235SOphir Munk .ctr_name = "vport_rx_packets", 215173bf9235SOphir Munk }, 215273bf9235SOphir Munk { 215373bf9235SOphir Munk .dpdk_name = "rx_bytes", 215473bf9235SOphir Munk .ctr_name = "vport_rx_bytes", 215573bf9235SOphir Munk }, 215673bf9235SOphir Munk { 215773bf9235SOphir Munk .dpdk_name = "tx_packets", 215873bf9235SOphir Munk .ctr_name = "vport_tx_packets", 215973bf9235SOphir Munk }, 216073bf9235SOphir Munk { 216173bf9235SOphir Munk .dpdk_name = "tx_bytes", 216273bf9235SOphir Munk .ctr_name = "vport_tx_bytes", 216373bf9235SOphir Munk }, 216473bf9235SOphir Munk }; 216573bf9235SOphir Munk 216673bf9235SOphir Munk static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); 216773bf9235SOphir Munk 216873bf9235SOphir Munk /** 216973bf9235SOphir Munk * Init the structures to read device counters. 217073bf9235SOphir Munk * 217173bf9235SOphir Munk * @param dev 217273bf9235SOphir Munk * Pointer to Ethernet device. 217373bf9235SOphir Munk */ 217473bf9235SOphir Munk void 217573bf9235SOphir Munk mlx5_os_stats_init(struct rte_eth_dev *dev) 217673bf9235SOphir Munk { 217773bf9235SOphir Munk struct mlx5_priv *priv = dev->data->dev_private; 217873bf9235SOphir Munk struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; 217973bf9235SOphir Munk struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl; 218073bf9235SOphir Munk unsigned int i; 218173bf9235SOphir Munk unsigned int j; 218273bf9235SOphir Munk struct ifreq ifr; 218373bf9235SOphir Munk struct ethtool_gstrings *strings = NULL; 218473bf9235SOphir Munk unsigned int dev_stats_n; 218573bf9235SOphir Munk unsigned int str_sz; 218673bf9235SOphir Munk int ret; 218773bf9235SOphir Munk 218873bf9235SOphir Munk /* So that it won't aggregate for each init. */ 218973bf9235SOphir Munk xstats_ctrl->mlx5_stats_n = 0; 219073bf9235SOphir Munk ret = mlx5_os_get_stats_n(dev); 219173bf9235SOphir Munk if (ret < 0) { 219273bf9235SOphir Munk DRV_LOG(WARNING, "port %u no extended statistics available", 219373bf9235SOphir Munk dev->data->port_id); 219473bf9235SOphir Munk return; 219573bf9235SOphir Munk } 219673bf9235SOphir Munk dev_stats_n = ret; 219773bf9235SOphir Munk /* Allocate memory to grab stat names and values. */ 219873bf9235SOphir Munk str_sz = dev_stats_n * ETH_GSTRING_LEN; 219973bf9235SOphir Munk strings = (struct ethtool_gstrings *) 220073bf9235SOphir Munk rte_malloc("xstats_strings", 220173bf9235SOphir Munk str_sz + sizeof(struct ethtool_gstrings), 0); 220273bf9235SOphir Munk if (!strings) { 220373bf9235SOphir Munk DRV_LOG(WARNING, "port %u unable to allocate memory for xstats", 220473bf9235SOphir Munk dev->data->port_id); 220573bf9235SOphir Munk return; 220673bf9235SOphir Munk } 220773bf9235SOphir Munk strings->cmd = ETHTOOL_GSTRINGS; 220873bf9235SOphir Munk strings->string_set = ETH_SS_STATS; 220973bf9235SOphir Munk strings->len = dev_stats_n; 221073bf9235SOphir Munk ifr.ifr_data = (caddr_t)strings; 221173bf9235SOphir Munk ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); 221273bf9235SOphir Munk if (ret) { 221373bf9235SOphir Munk DRV_LOG(WARNING, "port %u unable to get statistic names", 221473bf9235SOphir Munk dev->data->port_id); 221573bf9235SOphir Munk goto free; 221673bf9235SOphir Munk } 221773bf9235SOphir Munk for (i = 0; i != dev_stats_n; ++i) { 221873bf9235SOphir Munk const char *curr_string = (const char *) 221973bf9235SOphir Munk &strings->data[i * ETH_GSTRING_LEN]; 222073bf9235SOphir Munk 222173bf9235SOphir Munk for (j = 0; j != xstats_n; ++j) { 222273bf9235SOphir Munk if (!strcmp(mlx5_counters_init[j].ctr_name, 222373bf9235SOphir Munk curr_string)) { 222473bf9235SOphir Munk unsigned int idx = xstats_ctrl->mlx5_stats_n++; 222573bf9235SOphir Munk 222673bf9235SOphir Munk xstats_ctrl->dev_table_idx[idx] = i; 222773bf9235SOphir Munk xstats_ctrl->info[idx] = mlx5_counters_init[j]; 222873bf9235SOphir Munk break; 222973bf9235SOphir Munk } 223073bf9235SOphir Munk } 223173bf9235SOphir Munk } 223273bf9235SOphir Munk /* Add dev counters. */ 223373bf9235SOphir Munk for (i = 0; i != xstats_n; ++i) { 223473bf9235SOphir Munk if (mlx5_counters_init[i].dev) { 223573bf9235SOphir Munk unsigned int idx = xstats_ctrl->mlx5_stats_n++; 223673bf9235SOphir Munk 223773bf9235SOphir Munk xstats_ctrl->info[idx] = mlx5_counters_init[i]; 223873bf9235SOphir Munk xstats_ctrl->hw_stats[idx] = 0; 223973bf9235SOphir Munk } 224073bf9235SOphir Munk } 224173bf9235SOphir Munk MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS); 224273bf9235SOphir Munk xstats_ctrl->stats_n = dev_stats_n; 224373bf9235SOphir Munk /* Copy to base at first time. */ 224473bf9235SOphir Munk ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base); 224573bf9235SOphir Munk if (ret) 224673bf9235SOphir Munk DRV_LOG(ERR, "port %u cannot read device counters: %s", 224773bf9235SOphir Munk dev->data->port_id, strerror(rte_errno)); 224873bf9235SOphir Munk mlx5_os_read_dev_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base); 224973bf9235SOphir Munk stats_ctrl->imissed = 0; 225073bf9235SOphir Munk free: 225173bf9235SOphir Munk rte_free(strings); 225273bf9235SOphir Munk } 225373bf9235SOphir Munk 2254d5ed8aa9SOphir Munk /** 2255d5ed8aa9SOphir Munk * Set the reg_mr and dereg_mr call backs 2256d5ed8aa9SOphir Munk * 2257d5ed8aa9SOphir Munk * @param reg_mr_cb[out] 2258d5ed8aa9SOphir Munk * Pointer to reg_mr func 2259d5ed8aa9SOphir Munk * @param dereg_mr_cb[out] 2260d5ed8aa9SOphir Munk * Pointer to dereg_mr func 2261d5ed8aa9SOphir Munk * 2262d5ed8aa9SOphir Munk */ 2263d5ed8aa9SOphir Munk void 2264d5ed8aa9SOphir Munk mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, 2265d5ed8aa9SOphir Munk mlx5_dereg_mr_t *dereg_mr_cb) 2266d5ed8aa9SOphir Munk { 22674f96d913SOphir Munk *reg_mr_cb = mlx5_verbs_ops.reg_mr; 22684f96d913SOphir Munk *dereg_mr_cb = mlx5_verbs_ops.dereg_mr; 2269d5ed8aa9SOphir Munk } 2270d5ed8aa9SOphir Munk 2271042f5c94SOphir Munk const struct eth_dev_ops mlx5_os_dev_ops = { 2272042f5c94SOphir Munk .dev_configure = mlx5_dev_configure, 2273042f5c94SOphir Munk .dev_start = mlx5_dev_start, 2274042f5c94SOphir Munk .dev_stop = mlx5_dev_stop, 2275042f5c94SOphir Munk .dev_set_link_down = mlx5_set_link_down, 2276042f5c94SOphir Munk .dev_set_link_up = mlx5_set_link_up, 2277042f5c94SOphir Munk .dev_close = mlx5_dev_close, 2278042f5c94SOphir Munk .promiscuous_enable = mlx5_promiscuous_enable, 2279042f5c94SOphir Munk .promiscuous_disable = mlx5_promiscuous_disable, 2280042f5c94SOphir Munk .allmulticast_enable = mlx5_allmulticast_enable, 2281042f5c94SOphir Munk .allmulticast_disable = mlx5_allmulticast_disable, 2282042f5c94SOphir Munk .link_update = mlx5_link_update, 2283042f5c94SOphir Munk .stats_get = mlx5_stats_get, 2284042f5c94SOphir Munk .stats_reset = mlx5_stats_reset, 2285042f5c94SOphir Munk .xstats_get = mlx5_xstats_get, 2286042f5c94SOphir Munk .xstats_reset = mlx5_xstats_reset, 2287042f5c94SOphir Munk .xstats_get_names = mlx5_xstats_get_names, 2288042f5c94SOphir Munk .fw_version_get = mlx5_fw_version_get, 2289042f5c94SOphir Munk .dev_infos_get = mlx5_dev_infos_get, 2290042f5c94SOphir Munk .read_clock = mlx5_read_clock, 2291042f5c94SOphir Munk .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 2292042f5c94SOphir Munk .vlan_filter_set = mlx5_vlan_filter_set, 2293042f5c94SOphir Munk .rx_queue_setup = mlx5_rx_queue_setup, 2294042f5c94SOphir Munk .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 2295042f5c94SOphir Munk .tx_queue_setup = mlx5_tx_queue_setup, 2296042f5c94SOphir Munk .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 2297042f5c94SOphir Munk .rx_queue_release = mlx5_rx_queue_release, 2298042f5c94SOphir Munk .tx_queue_release = mlx5_tx_queue_release, 2299042f5c94SOphir Munk .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 2300042f5c94SOphir Munk .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 2301042f5c94SOphir Munk .mac_addr_remove = mlx5_mac_addr_remove, 2302042f5c94SOphir Munk .mac_addr_add = mlx5_mac_addr_add, 2303042f5c94SOphir Munk .mac_addr_set = mlx5_mac_addr_set, 2304042f5c94SOphir Munk .set_mc_addr_list = mlx5_set_mc_addr_list, 2305042f5c94SOphir Munk .mtu_set = mlx5_dev_set_mtu, 2306042f5c94SOphir Munk .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 2307042f5c94SOphir Munk .vlan_offload_set = mlx5_vlan_offload_set, 2308042f5c94SOphir Munk .reta_update = mlx5_dev_rss_reta_update, 2309042f5c94SOphir Munk .reta_query = mlx5_dev_rss_reta_query, 2310042f5c94SOphir Munk .rss_hash_update = mlx5_rss_hash_update, 2311042f5c94SOphir Munk .rss_hash_conf_get = mlx5_rss_hash_conf_get, 2312042f5c94SOphir Munk .filter_ctrl = mlx5_dev_filter_ctrl, 2313042f5c94SOphir Munk .rx_descriptor_status = mlx5_rx_descriptor_status, 2314042f5c94SOphir Munk .tx_descriptor_status = mlx5_tx_descriptor_status, 2315042f5c94SOphir Munk .rxq_info_get = mlx5_rxq_info_get, 2316042f5c94SOphir Munk .txq_info_get = mlx5_txq_info_get, 2317042f5c94SOphir Munk .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2318042f5c94SOphir Munk .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2319042f5c94SOphir Munk .rx_queue_count = mlx5_rx_queue_count, 2320042f5c94SOphir Munk .rx_queue_intr_enable = mlx5_rx_intr_enable, 2321042f5c94SOphir Munk .rx_queue_intr_disable = mlx5_rx_intr_disable, 2322042f5c94SOphir Munk .is_removed = mlx5_is_removed, 2323042f5c94SOphir Munk .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, 2324042f5c94SOphir Munk .get_module_info = mlx5_get_module_info, 2325042f5c94SOphir Munk .get_module_eeprom = mlx5_get_module_eeprom, 2326042f5c94SOphir Munk .hairpin_cap_get = mlx5_hairpin_cap_get, 2327042f5c94SOphir Munk .mtr_ops_get = mlx5_flow_meter_ops_get, 2328042f5c94SOphir Munk }; 2329042f5c94SOphir Munk 2330042f5c94SOphir Munk /* Available operations from secondary process. */ 2331042f5c94SOphir Munk const struct eth_dev_ops mlx5_os_dev_sec_ops = { 2332042f5c94SOphir Munk .stats_get = mlx5_stats_get, 2333042f5c94SOphir Munk .stats_reset = mlx5_stats_reset, 2334042f5c94SOphir Munk .xstats_get = mlx5_xstats_get, 2335042f5c94SOphir Munk .xstats_reset = mlx5_xstats_reset, 2336042f5c94SOphir Munk .xstats_get_names = mlx5_xstats_get_names, 2337042f5c94SOphir Munk .fw_version_get = mlx5_fw_version_get, 2338042f5c94SOphir Munk .dev_infos_get = mlx5_dev_infos_get, 2339042f5c94SOphir Munk .rx_descriptor_status = mlx5_rx_descriptor_status, 2340042f5c94SOphir Munk .tx_descriptor_status = mlx5_tx_descriptor_status, 2341042f5c94SOphir Munk .rxq_info_get = mlx5_rxq_info_get, 2342042f5c94SOphir Munk .txq_info_get = mlx5_txq_info_get, 2343042f5c94SOphir Munk .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2344042f5c94SOphir Munk .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2345042f5c94SOphir Munk .get_module_info = mlx5_get_module_info, 2346042f5c94SOphir Munk .get_module_eeprom = mlx5_get_module_eeprom, 2347042f5c94SOphir Munk }; 2348042f5c94SOphir Munk 2349042f5c94SOphir Munk /* Available operations in flow isolated mode. */ 2350042f5c94SOphir Munk const struct eth_dev_ops mlx5_os_dev_ops_isolate = { 2351042f5c94SOphir Munk .dev_configure = mlx5_dev_configure, 2352042f5c94SOphir Munk .dev_start = mlx5_dev_start, 2353042f5c94SOphir Munk .dev_stop = mlx5_dev_stop, 2354042f5c94SOphir Munk .dev_set_link_down = mlx5_set_link_down, 2355042f5c94SOphir Munk .dev_set_link_up = mlx5_set_link_up, 2356042f5c94SOphir Munk .dev_close = mlx5_dev_close, 2357042f5c94SOphir Munk .promiscuous_enable = mlx5_promiscuous_enable, 2358042f5c94SOphir Munk .promiscuous_disable = mlx5_promiscuous_disable, 2359042f5c94SOphir Munk .allmulticast_enable = mlx5_allmulticast_enable, 2360042f5c94SOphir Munk .allmulticast_disable = mlx5_allmulticast_disable, 2361042f5c94SOphir Munk .link_update = mlx5_link_update, 2362042f5c94SOphir Munk .stats_get = mlx5_stats_get, 2363042f5c94SOphir Munk .stats_reset = mlx5_stats_reset, 2364042f5c94SOphir Munk .xstats_get = mlx5_xstats_get, 2365042f5c94SOphir Munk .xstats_reset = mlx5_xstats_reset, 2366042f5c94SOphir Munk .xstats_get_names = mlx5_xstats_get_names, 2367042f5c94SOphir Munk .fw_version_get = mlx5_fw_version_get, 2368042f5c94SOphir Munk .dev_infos_get = mlx5_dev_infos_get, 2369042f5c94SOphir Munk .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 2370042f5c94SOphir Munk .vlan_filter_set = mlx5_vlan_filter_set, 2371042f5c94SOphir Munk .rx_queue_setup = mlx5_rx_queue_setup, 2372042f5c94SOphir Munk .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 2373042f5c94SOphir Munk .tx_queue_setup = mlx5_tx_queue_setup, 2374042f5c94SOphir Munk .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 2375042f5c94SOphir Munk .rx_queue_release = mlx5_rx_queue_release, 2376042f5c94SOphir Munk .tx_queue_release = mlx5_tx_queue_release, 2377042f5c94SOphir Munk .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 2378042f5c94SOphir Munk .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 2379042f5c94SOphir Munk .mac_addr_remove = mlx5_mac_addr_remove, 2380042f5c94SOphir Munk .mac_addr_add = mlx5_mac_addr_add, 2381042f5c94SOphir Munk .mac_addr_set = mlx5_mac_addr_set, 2382042f5c94SOphir Munk .set_mc_addr_list = mlx5_set_mc_addr_list, 2383042f5c94SOphir Munk .mtu_set = mlx5_dev_set_mtu, 2384042f5c94SOphir Munk .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 2385042f5c94SOphir Munk .vlan_offload_set = mlx5_vlan_offload_set, 2386042f5c94SOphir Munk .filter_ctrl = mlx5_dev_filter_ctrl, 2387042f5c94SOphir Munk .rx_descriptor_status = mlx5_rx_descriptor_status, 2388042f5c94SOphir Munk .tx_descriptor_status = mlx5_tx_descriptor_status, 2389042f5c94SOphir Munk .rxq_info_get = mlx5_rxq_info_get, 2390042f5c94SOphir Munk .txq_info_get = mlx5_txq_info_get, 2391042f5c94SOphir Munk .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2392042f5c94SOphir Munk .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2393042f5c94SOphir Munk .rx_queue_intr_enable = mlx5_rx_intr_enable, 2394042f5c94SOphir Munk .rx_queue_intr_disable = mlx5_rx_intr_disable, 2395042f5c94SOphir Munk .is_removed = mlx5_is_removed, 2396042f5c94SOphir Munk .get_module_info = mlx5_get_module_info, 2397042f5c94SOphir Munk .get_module_eeprom = mlx5_get_module_eeprom, 2398042f5c94SOphir Munk .hairpin_cap_get = mlx5_hairpin_cap_get, 2399042f5c94SOphir Munk .mtr_ops_get = mlx5_flow_meter_ops_get, 2400042f5c94SOphir Munk }; 2401