1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2020 Mellanox Technologies, Ltd 4 */ 5 6 #include <stddef.h> 7 #include <unistd.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <errno.h> 12 #include <net/if.h> 13 #include <sys/mman.h> 14 #include <linux/rtnetlink.h> 15 #include <linux/sockios.h> 16 #include <linux/ethtool.h> 17 #include <fcntl.h> 18 19 /* Verbs header. */ 20 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 21 #ifdef PEDANTIC 22 #pragma GCC diagnostic ignored "-Wpedantic" 23 #endif 24 #include <infiniband/verbs.h> 25 #ifdef PEDANTIC 26 #pragma GCC diagnostic error "-Wpedantic" 27 #endif 28 29 #include <rte_malloc.h> 30 #include <rte_ethdev_driver.h> 31 #include <rte_ethdev_pci.h> 32 #include <rte_pci.h> 33 #include <rte_bus_pci.h> 34 #include <rte_common.h> 35 #include <rte_kvargs.h> 36 #include <rte_rwlock.h> 37 #include <rte_spinlock.h> 38 #include <rte_string_fns.h> 39 #include <rte_alarm.h> 40 41 #include <mlx5_glue.h> 42 #include <mlx5_devx_cmds.h> 43 #include <mlx5_common.h> 44 #include <mlx5_common_mp.h> 45 #include <mlx5_common_mr.h> 46 47 #include "mlx5_defs.h" 48 #include "mlx5.h" 49 #include "mlx5_common_os.h" 50 #include "mlx5_utils.h" 51 #include "mlx5_rxtx.h" 52 #include "mlx5_autoconf.h" 53 #include "mlx5_mr.h" 54 #include "mlx5_flow.h" 55 #include "rte_pmd_mlx5.h" 56 #include "mlx5_verbs.h" 57 58 #define MLX5_TAGS_HLIST_ARRAY_SIZE 8192 59 60 #ifndef HAVE_IBV_MLX5_MOD_MPW 61 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2) 62 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3) 63 #endif 64 65 #ifndef HAVE_IBV_MLX5_MOD_CQE_128B_COMP 66 #define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4) 67 #endif 68 69 /** 70 * Get mlx5 device attributes. The glue function query_device_ex() is called 71 * with out parameter of type 'struct ibv_device_attr_ex *'. Then fill in mlx5 72 * device attributes from the glue out parameter. 73 * 74 * @param dev 75 * Pointer to ibv context. 76 * 77 * @param device_attr 78 * Pointer to mlx5 device attributes. 79 * 80 * @return 81 * 0 on success, non zero error number otherwise 82 */ 83 int 84 mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *device_attr) 85 { 86 int err; 87 struct ibv_device_attr_ex attr_ex; 88 memset(device_attr, 0, sizeof(*device_attr)); 89 err = mlx5_glue->query_device_ex(ctx, NULL, &attr_ex); 90 if (err) 91 return err; 92 93 device_attr->device_cap_flags_ex = attr_ex.device_cap_flags_ex; 94 device_attr->max_qp_wr = attr_ex.orig_attr.max_qp_wr; 95 device_attr->max_sge = attr_ex.orig_attr.max_sge; 96 device_attr->max_cq = attr_ex.orig_attr.max_cq; 97 device_attr->max_qp = attr_ex.orig_attr.max_qp; 98 device_attr->raw_packet_caps = attr_ex.raw_packet_caps; 99 device_attr->max_rwq_indirection_table_size = 100 attr_ex.rss_caps.max_rwq_indirection_table_size; 101 device_attr->max_tso = attr_ex.tso_caps.max_tso; 102 device_attr->tso_supported_qpts = attr_ex.tso_caps.supported_qpts; 103 104 struct mlx5dv_context dv_attr = { .comp_mask = 0 }; 105 err = mlx5_glue->dv_query_device(ctx, &dv_attr); 106 if (err) 107 return err; 108 109 device_attr->flags = dv_attr.flags; 110 device_attr->comp_mask = dv_attr.comp_mask; 111 #ifdef HAVE_IBV_MLX5_MOD_SWP 112 device_attr->sw_parsing_offloads = 113 dv_attr.sw_parsing_caps.sw_parsing_offloads; 114 #endif 115 device_attr->min_single_stride_log_num_of_bytes = 116 dv_attr.striding_rq_caps.min_single_stride_log_num_of_bytes; 117 device_attr->max_single_stride_log_num_of_bytes = 118 dv_attr.striding_rq_caps.max_single_stride_log_num_of_bytes; 119 device_attr->min_single_wqe_log_num_of_strides = 120 dv_attr.striding_rq_caps.min_single_wqe_log_num_of_strides; 121 device_attr->max_single_wqe_log_num_of_strides = 122 dv_attr.striding_rq_caps.max_single_wqe_log_num_of_strides; 123 device_attr->stride_supported_qpts = 124 dv_attr.striding_rq_caps.supported_qpts; 125 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 126 device_attr->tunnel_offloads_caps = dv_attr.tunnel_offloads_caps; 127 #endif 128 129 return err; 130 } 131 132 /** 133 * Verbs callback to allocate a memory. This function should allocate the space 134 * according to the size provided residing inside a huge page. 135 * Please note that all allocation must respect the alignment from libmlx5 136 * (i.e. currently sysconf(_SC_PAGESIZE)). 137 * 138 * @param[in] size 139 * The size in bytes of the memory to allocate. 140 * @param[in] data 141 * A pointer to the callback data. 142 * 143 * @return 144 * Allocated buffer, NULL otherwise and rte_errno is set. 145 */ 146 static void * 147 mlx5_alloc_verbs_buf(size_t size, void *data) 148 { 149 struct mlx5_priv *priv = data; 150 void *ret; 151 size_t alignment = sysconf(_SC_PAGESIZE); 152 unsigned int socket = SOCKET_ID_ANY; 153 154 if (priv->verbs_alloc_ctx.type == MLX5_VERBS_ALLOC_TYPE_TX_QUEUE) { 155 const struct mlx5_txq_ctrl *ctrl = priv->verbs_alloc_ctx.obj; 156 157 socket = ctrl->socket; 158 } else if (priv->verbs_alloc_ctx.type == 159 MLX5_VERBS_ALLOC_TYPE_RX_QUEUE) { 160 const struct mlx5_rxq_ctrl *ctrl = priv->verbs_alloc_ctx.obj; 161 162 socket = ctrl->socket; 163 } 164 MLX5_ASSERT(data != NULL); 165 ret = rte_malloc_socket(__func__, size, alignment, socket); 166 if (!ret && size) 167 rte_errno = ENOMEM; 168 return ret; 169 } 170 171 /** 172 * Verbs callback to free a memory. 173 * 174 * @param[in] ptr 175 * A pointer to the memory to free. 176 * @param[in] data 177 * A pointer to the callback data. 178 */ 179 static void 180 mlx5_free_verbs_buf(void *ptr, void *data __rte_unused) 181 { 182 MLX5_ASSERT(data != NULL); 183 rte_free(ptr); 184 } 185 186 /** 187 * Initialize DR related data within private structure. 188 * Routine checks the reference counter and does actual 189 * resources creation/initialization only if counter is zero. 190 * 191 * @param[in] priv 192 * Pointer to the private device data structure. 193 * 194 * @return 195 * Zero on success, positive error code otherwise. 196 */ 197 static int 198 mlx5_alloc_shared_dr(struct mlx5_priv *priv) 199 { 200 struct mlx5_dev_ctx_shared *sh = priv->sh; 201 char s[MLX5_HLIST_NAMESIZE]; 202 int err = 0; 203 204 if (!sh->flow_tbls) 205 err = mlx5_alloc_table_hash_list(priv); 206 else 207 DRV_LOG(DEBUG, "sh->flow_tbls[%p] already created, reuse\n", 208 (void *)sh->flow_tbls); 209 if (err) 210 return err; 211 /* Create tags hash list table. */ 212 snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name); 213 sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE); 214 if (!sh->tag_table) { 215 DRV_LOG(ERR, "tags with hash creation failed."); 216 err = ENOMEM; 217 goto error; 218 } 219 #ifdef HAVE_MLX5DV_DR 220 void *domain; 221 222 if (sh->dv_refcnt) { 223 /* Shared DV/DR structures is already initialized. */ 224 sh->dv_refcnt++; 225 priv->dr_shared = 1; 226 return 0; 227 } 228 /* Reference counter is zero, we should initialize structures. */ 229 domain = mlx5_glue->dr_create_domain(sh->ctx, 230 MLX5DV_DR_DOMAIN_TYPE_NIC_RX); 231 if (!domain) { 232 DRV_LOG(ERR, "ingress mlx5dv_dr_create_domain failed"); 233 err = errno; 234 goto error; 235 } 236 sh->rx_domain = domain; 237 domain = mlx5_glue->dr_create_domain(sh->ctx, 238 MLX5DV_DR_DOMAIN_TYPE_NIC_TX); 239 if (!domain) { 240 DRV_LOG(ERR, "egress mlx5dv_dr_create_domain failed"); 241 err = errno; 242 goto error; 243 } 244 pthread_mutex_init(&sh->dv_mutex, NULL); 245 sh->tx_domain = domain; 246 #ifdef HAVE_MLX5DV_DR_ESWITCH 247 if (priv->config.dv_esw_en) { 248 domain = mlx5_glue->dr_create_domain 249 (sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB); 250 if (!domain) { 251 DRV_LOG(ERR, "FDB mlx5dv_dr_create_domain failed"); 252 err = errno; 253 goto error; 254 } 255 sh->fdb_domain = domain; 256 sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); 257 } 258 #endif 259 if (priv->config.reclaim_mode == MLX5_RCM_AGGR) { 260 mlx5_glue->dr_reclaim_domain_memory(sh->rx_domain, 1); 261 mlx5_glue->dr_reclaim_domain_memory(sh->tx_domain, 1); 262 if (sh->fdb_domain) 263 mlx5_glue->dr_reclaim_domain_memory(sh->fdb_domain, 1); 264 } 265 sh->pop_vlan_action = mlx5_glue->dr_create_flow_action_pop_vlan(); 266 #endif /* HAVE_MLX5DV_DR */ 267 sh->dv_refcnt++; 268 priv->dr_shared = 1; 269 return 0; 270 error: 271 /* Rollback the created objects. */ 272 if (sh->rx_domain) { 273 mlx5_glue->dr_destroy_domain(sh->rx_domain); 274 sh->rx_domain = NULL; 275 } 276 if (sh->tx_domain) { 277 mlx5_glue->dr_destroy_domain(sh->tx_domain); 278 sh->tx_domain = NULL; 279 } 280 if (sh->fdb_domain) { 281 mlx5_glue->dr_destroy_domain(sh->fdb_domain); 282 sh->fdb_domain = NULL; 283 } 284 if (sh->esw_drop_action) { 285 mlx5_glue->destroy_flow_action(sh->esw_drop_action); 286 sh->esw_drop_action = NULL; 287 } 288 if (sh->pop_vlan_action) { 289 mlx5_glue->destroy_flow_action(sh->pop_vlan_action); 290 sh->pop_vlan_action = NULL; 291 } 292 if (sh->tag_table) { 293 /* tags should be destroyed with flow before. */ 294 mlx5_hlist_destroy(sh->tag_table, NULL, NULL); 295 sh->tag_table = NULL; 296 } 297 mlx5_free_table_hash_list(priv); 298 return err; 299 } 300 301 /** 302 * Destroy DR related data within private structure. 303 * 304 * @param[in] priv 305 * Pointer to the private device data structure. 306 */ 307 void 308 mlx5_os_free_shared_dr(struct mlx5_priv *priv) 309 { 310 struct mlx5_dev_ctx_shared *sh; 311 312 if (!priv->dr_shared) 313 return; 314 priv->dr_shared = 0; 315 sh = priv->sh; 316 MLX5_ASSERT(sh); 317 #ifdef HAVE_MLX5DV_DR 318 MLX5_ASSERT(sh->dv_refcnt); 319 if (sh->dv_refcnt && --sh->dv_refcnt) 320 return; 321 if (sh->rx_domain) { 322 mlx5_glue->dr_destroy_domain(sh->rx_domain); 323 sh->rx_domain = NULL; 324 } 325 if (sh->tx_domain) { 326 mlx5_glue->dr_destroy_domain(sh->tx_domain); 327 sh->tx_domain = NULL; 328 } 329 #ifdef HAVE_MLX5DV_DR_ESWITCH 330 if (sh->fdb_domain) { 331 mlx5_glue->dr_destroy_domain(sh->fdb_domain); 332 sh->fdb_domain = NULL; 333 } 334 if (sh->esw_drop_action) { 335 mlx5_glue->destroy_flow_action(sh->esw_drop_action); 336 sh->esw_drop_action = NULL; 337 } 338 #endif 339 if (sh->pop_vlan_action) { 340 mlx5_glue->destroy_flow_action(sh->pop_vlan_action); 341 sh->pop_vlan_action = NULL; 342 } 343 pthread_mutex_destroy(&sh->dv_mutex); 344 #endif /* HAVE_MLX5DV_DR */ 345 if (sh->tag_table) { 346 /* tags should be destroyed with flow before. */ 347 mlx5_hlist_destroy(sh->tag_table, NULL, NULL); 348 sh->tag_table = NULL; 349 } 350 mlx5_free_table_hash_list(priv); 351 } 352 353 /** 354 * Spawn an Ethernet device from Verbs information. 355 * 356 * @param dpdk_dev 357 * Backing DPDK device. 358 * @param spawn 359 * Verbs device parameters (name, port, switch_info) to spawn. 360 * @param config 361 * Device configuration parameters. 362 * 363 * @return 364 * A valid Ethernet device object on success, NULL otherwise and rte_errno 365 * is set. The following errors are defined: 366 * 367 * EBUSY: device is not supposed to be spawned. 368 * EEXIST: device is already spawned 369 */ 370 static struct rte_eth_dev * 371 mlx5_dev_spawn(struct rte_device *dpdk_dev, 372 struct mlx5_dev_spawn_data *spawn, 373 struct mlx5_dev_config config) 374 { 375 const struct mlx5_switch_info *switch_info = &spawn->info; 376 struct mlx5_dev_ctx_shared *sh = NULL; 377 struct ibv_port_attr port_attr; 378 struct mlx5dv_context dv_attr = { .comp_mask = 0 }; 379 struct rte_eth_dev *eth_dev = NULL; 380 struct mlx5_priv *priv = NULL; 381 int err = 0; 382 unsigned int hw_padding = 0; 383 unsigned int mps; 384 unsigned int cqe_comp; 385 unsigned int cqe_pad = 0; 386 unsigned int tunnel_en = 0; 387 unsigned int mpls_en = 0; 388 unsigned int swp = 0; 389 unsigned int mprq = 0; 390 unsigned int mprq_min_stride_size_n = 0; 391 unsigned int mprq_max_stride_size_n = 0; 392 unsigned int mprq_min_stride_num_n = 0; 393 unsigned int mprq_max_stride_num_n = 0; 394 struct rte_ether_addr mac; 395 char name[RTE_ETH_NAME_MAX_LEN]; 396 int own_domain_id = 0; 397 uint16_t port_id; 398 unsigned int i; 399 #ifdef HAVE_MLX5DV_DR_DEVX_PORT 400 struct mlx5dv_devx_port devx_port = { .comp_mask = 0 }; 401 #endif 402 403 /* Determine if this port representor is supposed to be spawned. */ 404 if (switch_info->representor && dpdk_dev->devargs) { 405 struct rte_eth_devargs eth_da; 406 407 err = rte_eth_devargs_parse(dpdk_dev->devargs->args, ð_da); 408 if (err) { 409 rte_errno = -err; 410 DRV_LOG(ERR, "failed to process device arguments: %s", 411 strerror(rte_errno)); 412 return NULL; 413 } 414 for (i = 0; i < eth_da.nb_representor_ports; ++i) 415 if (eth_da.representor_ports[i] == 416 (uint16_t)switch_info->port_name) 417 break; 418 if (i == eth_da.nb_representor_ports) { 419 rte_errno = EBUSY; 420 return NULL; 421 } 422 } 423 /* Build device name. */ 424 if (spawn->pf_bond < 0) { 425 /* Single device. */ 426 if (!switch_info->representor) 427 strlcpy(name, dpdk_dev->name, sizeof(name)); 428 else 429 snprintf(name, sizeof(name), "%s_representor_%u", 430 dpdk_dev->name, switch_info->port_name); 431 } else { 432 /* Bonding device. */ 433 if (!switch_info->representor) 434 snprintf(name, sizeof(name), "%s_%s", 435 dpdk_dev->name, 436 mlx5_os_get_dev_device_name(spawn->phys_dev)); 437 else 438 snprintf(name, sizeof(name), "%s_%s_representor_%u", 439 dpdk_dev->name, 440 mlx5_os_get_dev_device_name(spawn->phys_dev), 441 switch_info->port_name); 442 } 443 /* check if the device is already spawned */ 444 if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) { 445 rte_errno = EEXIST; 446 return NULL; 447 } 448 DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name); 449 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 450 struct mlx5_mp_id mp_id; 451 452 eth_dev = rte_eth_dev_attach_secondary(name); 453 if (eth_dev == NULL) { 454 DRV_LOG(ERR, "can not attach rte ethdev"); 455 rte_errno = ENOMEM; 456 return NULL; 457 } 458 eth_dev->device = dpdk_dev; 459 eth_dev->dev_ops = &mlx5_os_dev_sec_ops; 460 err = mlx5_proc_priv_init(eth_dev); 461 if (err) 462 return NULL; 463 mp_id.port_id = eth_dev->data->port_id; 464 strlcpy(mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN); 465 /* Receive command fd from primary process */ 466 err = mlx5_mp_req_verbs_cmd_fd(&mp_id); 467 if (err < 0) 468 goto err_secondary; 469 /* Remap UAR for Tx queues. */ 470 err = mlx5_tx_uar_init_secondary(eth_dev, err); 471 if (err) 472 goto err_secondary; 473 /* 474 * Ethdev pointer is still required as input since 475 * the primary device is not accessible from the 476 * secondary process. 477 */ 478 eth_dev->rx_pkt_burst = mlx5_select_rx_function(eth_dev); 479 eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev); 480 return eth_dev; 481 err_secondary: 482 mlx5_dev_close(eth_dev); 483 return NULL; 484 } 485 /* 486 * Some parameters ("tx_db_nc" in particularly) are needed in 487 * advance to create dv/verbs device context. We proceed the 488 * devargs here to get ones, and later proceed devargs again 489 * to override some hardware settings. 490 */ 491 err = mlx5_args(&config, dpdk_dev->devargs); 492 if (err) { 493 err = rte_errno; 494 DRV_LOG(ERR, "failed to process device arguments: %s", 495 strerror(rte_errno)); 496 goto error; 497 } 498 sh = mlx5_alloc_shared_dev_ctx(spawn, &config); 499 if (!sh) 500 return NULL; 501 config.devx = sh->devx; 502 #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR 503 config.dest_tir = 1; 504 #endif 505 #ifdef HAVE_IBV_MLX5_MOD_SWP 506 dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP; 507 #endif 508 /* 509 * Multi-packet send is supported by ConnectX-4 Lx PF as well 510 * as all ConnectX-5 devices. 511 */ 512 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 513 dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS; 514 #endif 515 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT 516 dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ; 517 #endif 518 mlx5_glue->dv_query_device(sh->ctx, &dv_attr); 519 if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) { 520 if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) { 521 DRV_LOG(DEBUG, "enhanced MPW is supported"); 522 mps = MLX5_MPW_ENHANCED; 523 } else { 524 DRV_LOG(DEBUG, "MPW is supported"); 525 mps = MLX5_MPW; 526 } 527 } else { 528 DRV_LOG(DEBUG, "MPW isn't supported"); 529 mps = MLX5_MPW_DISABLED; 530 } 531 #ifdef HAVE_IBV_MLX5_MOD_SWP 532 if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP) 533 swp = dv_attr.sw_parsing_caps.sw_parsing_offloads; 534 DRV_LOG(DEBUG, "SWP support: %u", swp); 535 #endif 536 config.swp = !!swp; 537 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT 538 if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) { 539 struct mlx5dv_striding_rq_caps mprq_caps = 540 dv_attr.striding_rq_caps; 541 542 DRV_LOG(DEBUG, "\tmin_single_stride_log_num_of_bytes: %d", 543 mprq_caps.min_single_stride_log_num_of_bytes); 544 DRV_LOG(DEBUG, "\tmax_single_stride_log_num_of_bytes: %d", 545 mprq_caps.max_single_stride_log_num_of_bytes); 546 DRV_LOG(DEBUG, "\tmin_single_wqe_log_num_of_strides: %d", 547 mprq_caps.min_single_wqe_log_num_of_strides); 548 DRV_LOG(DEBUG, "\tmax_single_wqe_log_num_of_strides: %d", 549 mprq_caps.max_single_wqe_log_num_of_strides); 550 DRV_LOG(DEBUG, "\tsupported_qpts: %d", 551 mprq_caps.supported_qpts); 552 DRV_LOG(DEBUG, "device supports Multi-Packet RQ"); 553 mprq = 1; 554 mprq_min_stride_size_n = 555 mprq_caps.min_single_stride_log_num_of_bytes; 556 mprq_max_stride_size_n = 557 mprq_caps.max_single_stride_log_num_of_bytes; 558 mprq_min_stride_num_n = 559 mprq_caps.min_single_wqe_log_num_of_strides; 560 mprq_max_stride_num_n = 561 mprq_caps.max_single_wqe_log_num_of_strides; 562 } 563 #endif 564 if (RTE_CACHE_LINE_SIZE == 128 && 565 !(dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP)) 566 cqe_comp = 0; 567 else 568 cqe_comp = 1; 569 config.cqe_comp = cqe_comp; 570 #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD 571 /* Whether device supports 128B Rx CQE padding. */ 572 cqe_pad = RTE_CACHE_LINE_SIZE == 128 && 573 (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD); 574 #endif 575 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 576 if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) { 577 tunnel_en = ((dv_attr.tunnel_offloads_caps & 578 MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN) && 579 (dv_attr.tunnel_offloads_caps & 580 MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE) && 581 (dv_attr.tunnel_offloads_caps & 582 MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GENEVE)); 583 } 584 DRV_LOG(DEBUG, "tunnel offloading is %ssupported", 585 tunnel_en ? "" : "not "); 586 #else 587 DRV_LOG(WARNING, 588 "tunnel offloading disabled due to old OFED/rdma-core version"); 589 #endif 590 config.tunnel_en = tunnel_en; 591 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 592 mpls_en = ((dv_attr.tunnel_offloads_caps & 593 MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) && 594 (dv_attr.tunnel_offloads_caps & 595 MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP)); 596 DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is %ssupported", 597 mpls_en ? "" : "not "); 598 #else 599 DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to" 600 " old OFED/rdma-core version or firmware configuration"); 601 #endif 602 config.mpls_en = mpls_en; 603 /* Check port status. */ 604 err = mlx5_glue->query_port(sh->ctx, spawn->phys_port, &port_attr); 605 if (err) { 606 DRV_LOG(ERR, "port query failed: %s", strerror(err)); 607 goto error; 608 } 609 if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { 610 DRV_LOG(ERR, "port is not configured in Ethernet mode"); 611 err = EINVAL; 612 goto error; 613 } 614 if (port_attr.state != IBV_PORT_ACTIVE) 615 DRV_LOG(DEBUG, "port is not active: \"%s\" (%d)", 616 mlx5_glue->port_state_str(port_attr.state), 617 port_attr.state); 618 /* Allocate private eth device data. */ 619 priv = rte_zmalloc("ethdev private structure", 620 sizeof(*priv), 621 RTE_CACHE_LINE_SIZE); 622 if (priv == NULL) { 623 DRV_LOG(ERR, "priv allocation failure"); 624 err = ENOMEM; 625 goto error; 626 } 627 priv->sh = sh; 628 priv->dev_port = spawn->phys_port; 629 priv->pci_dev = spawn->pci_dev; 630 priv->mtu = RTE_ETHER_MTU; 631 priv->mp_id.port_id = port_id; 632 strlcpy(priv->mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN); 633 #ifndef RTE_ARCH_64 634 /* Initialize UAR access locks for 32bit implementations. */ 635 rte_spinlock_init(&priv->uar_lock_cq); 636 for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) 637 rte_spinlock_init(&priv->uar_lock[i]); 638 #endif 639 /* Some internal functions rely on Netlink sockets, open them now. */ 640 priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA); 641 priv->nl_socket_route = mlx5_nl_init(NETLINK_ROUTE); 642 priv->representor = !!switch_info->representor; 643 priv->master = !!switch_info->master; 644 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 645 priv->vport_meta_tag = 0; 646 priv->vport_meta_mask = 0; 647 priv->pf_bond = spawn->pf_bond; 648 #ifdef HAVE_MLX5DV_DR_DEVX_PORT 649 /* 650 * The DevX port query API is implemented. E-Switch may use 651 * either vport or reg_c[0] metadata register to match on 652 * vport index. The engaged part of metadata register is 653 * defined by mask. 654 */ 655 if (switch_info->representor || switch_info->master) { 656 devx_port.comp_mask = MLX5DV_DEVX_PORT_VPORT | 657 MLX5DV_DEVX_PORT_MATCH_REG_C_0; 658 err = mlx5_glue->devx_port_query(sh->ctx, spawn->phys_port, 659 &devx_port); 660 if (err) { 661 DRV_LOG(WARNING, 662 "can't query devx port %d on device %s", 663 spawn->phys_port, 664 mlx5_os_get_dev_device_name(spawn->phys_dev)); 665 devx_port.comp_mask = 0; 666 } 667 } 668 if (devx_port.comp_mask & MLX5DV_DEVX_PORT_MATCH_REG_C_0) { 669 priv->vport_meta_tag = devx_port.reg_c_0.value; 670 priv->vport_meta_mask = devx_port.reg_c_0.mask; 671 if (!priv->vport_meta_mask) { 672 DRV_LOG(ERR, "vport zero mask for port %d" 673 " on bonding device %s", 674 spawn->phys_port, 675 mlx5_os_get_dev_device_name 676 (spawn->phys_dev)); 677 err = ENOTSUP; 678 goto error; 679 } 680 if (priv->vport_meta_tag & ~priv->vport_meta_mask) { 681 DRV_LOG(ERR, "invalid vport tag for port %d" 682 " on bonding device %s", 683 spawn->phys_port, 684 mlx5_os_get_dev_device_name 685 (spawn->phys_dev)); 686 err = ENOTSUP; 687 goto error; 688 } 689 } 690 if (devx_port.comp_mask & MLX5DV_DEVX_PORT_VPORT) { 691 priv->vport_id = devx_port.vport_num; 692 } else if (spawn->pf_bond >= 0) { 693 DRV_LOG(ERR, "can't deduce vport index for port %d" 694 " on bonding device %s", 695 spawn->phys_port, 696 mlx5_os_get_dev_device_name(spawn->phys_dev)); 697 err = ENOTSUP; 698 goto error; 699 } else { 700 /* Suppose vport index in compatible way. */ 701 priv->vport_id = switch_info->representor ? 702 switch_info->port_name + 1 : -1; 703 } 704 #else 705 /* 706 * Kernel/rdma_core support single E-Switch per PF configurations 707 * only and vport_id field contains the vport index for 708 * associated VF, which is deduced from representor port name. 709 * For example, let's have the IB device port 10, it has 710 * attached network device eth0, which has port name attribute 711 * pf0vf2, we can deduce the VF number as 2, and set vport index 712 * as 3 (2+1). This assigning schema should be changed if the 713 * multiple E-Switch instances per PF configurations or/and PCI 714 * subfunctions are added. 715 */ 716 priv->vport_id = switch_info->representor ? 717 switch_info->port_name + 1 : -1; 718 #endif 719 /* representor_id field keeps the unmodified VF index. */ 720 priv->representor_id = switch_info->representor ? 721 switch_info->port_name : -1; 722 /* 723 * Look for sibling devices in order to reuse their switch domain 724 * if any, otherwise allocate one. 725 */ 726 MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) { 727 const struct mlx5_priv *opriv = 728 rte_eth_devices[port_id].data->dev_private; 729 730 if (!opriv || 731 opriv->sh != priv->sh || 732 opriv->domain_id == 733 RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) 734 continue; 735 priv->domain_id = opriv->domain_id; 736 break; 737 } 738 if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 739 err = rte_eth_switch_domain_alloc(&priv->domain_id); 740 if (err) { 741 err = rte_errno; 742 DRV_LOG(ERR, "unable to allocate switch domain: %s", 743 strerror(rte_errno)); 744 goto error; 745 } 746 own_domain_id = 1; 747 } 748 /* Override some values set by hardware configuration. */ 749 mlx5_args(&config, dpdk_dev->devargs); 750 err = mlx5_dev_check_sibling_config(priv, &config); 751 if (err) 752 goto error; 753 config.hw_csum = !!(sh->device_attr.device_cap_flags_ex & 754 IBV_DEVICE_RAW_IP_CSUM); 755 DRV_LOG(DEBUG, "checksum offloading is %ssupported", 756 (config.hw_csum ? "" : "not ")); 757 #if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \ 758 !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 759 DRV_LOG(DEBUG, "counters are not supported"); 760 #endif 761 #if !defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_MLX5DV_DR) 762 if (config.dv_flow_en) { 763 DRV_LOG(WARNING, "DV flow is not supported"); 764 config.dv_flow_en = 0; 765 } 766 #endif 767 config.ind_table_max_size = 768 sh->device_attr.max_rwq_indirection_table_size; 769 /* 770 * Remove this check once DPDK supports larger/variable 771 * indirection tables. 772 */ 773 if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512) 774 config.ind_table_max_size = ETH_RSS_RETA_SIZE_512; 775 DRV_LOG(DEBUG, "maximum Rx indirection table size is %u", 776 config.ind_table_max_size); 777 config.hw_vlan_strip = !!(sh->device_attr.raw_packet_caps & 778 IBV_RAW_PACKET_CAP_CVLAN_STRIPPING); 779 DRV_LOG(DEBUG, "VLAN stripping is %ssupported", 780 (config.hw_vlan_strip ? "" : "not ")); 781 config.hw_fcs_strip = !!(sh->device_attr.raw_packet_caps & 782 IBV_RAW_PACKET_CAP_SCATTER_FCS); 783 DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported", 784 (config.hw_fcs_strip ? "" : "not ")); 785 #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING) 786 hw_padding = !!sh->device_attr.rx_pad_end_addr_align; 787 #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING) 788 hw_padding = !!(sh->device_attr.device_cap_flags_ex & 789 IBV_DEVICE_PCI_WRITE_END_PADDING); 790 #endif 791 if (config.hw_padding && !hw_padding) { 792 DRV_LOG(DEBUG, "Rx end alignment padding isn't supported"); 793 config.hw_padding = 0; 794 } else if (config.hw_padding) { 795 DRV_LOG(DEBUG, "Rx end alignment padding is enabled"); 796 } 797 config.tso = (sh->device_attr.max_tso > 0 && 798 (sh->device_attr.tso_supported_qpts & 799 (1 << IBV_QPT_RAW_PACKET))); 800 if (config.tso) 801 config.tso_max_payload_sz = sh->device_attr.max_tso; 802 /* 803 * MPW is disabled by default, while the Enhanced MPW is enabled 804 * by default. 805 */ 806 if (config.mps == MLX5_ARG_UNSET) 807 config.mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED : 808 MLX5_MPW_DISABLED; 809 else 810 config.mps = config.mps ? mps : MLX5_MPW_DISABLED; 811 DRV_LOG(INFO, "%sMPS is %s", 812 config.mps == MLX5_MPW_ENHANCED ? "enhanced " : 813 config.mps == MLX5_MPW ? "legacy " : "", 814 config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); 815 if (config.cqe_comp && !cqe_comp) { 816 DRV_LOG(WARNING, "Rx CQE compression isn't supported"); 817 config.cqe_comp = 0; 818 } 819 if (config.cqe_pad && !cqe_pad) { 820 DRV_LOG(WARNING, "Rx CQE padding isn't supported"); 821 config.cqe_pad = 0; 822 } else if (config.cqe_pad) { 823 DRV_LOG(INFO, "Rx CQE padding is enabled"); 824 } 825 if (config.devx) { 826 priv->counter_fallback = 0; 827 err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config.hca_attr); 828 if (err) { 829 err = -err; 830 goto error; 831 } 832 if (!config.hca_attr.flow_counters_dump) 833 priv->counter_fallback = 1; 834 #ifndef HAVE_IBV_DEVX_ASYNC 835 priv->counter_fallback = 1; 836 #endif 837 if (priv->counter_fallback) 838 DRV_LOG(INFO, "Use fall-back DV counter management"); 839 /* Check for LRO support. */ 840 if (config.dest_tir && config.hca_attr.lro_cap && 841 config.dv_flow_en) { 842 /* TBD check tunnel lro caps. */ 843 config.lro.supported = config.hca_attr.lro_cap; 844 DRV_LOG(DEBUG, "Device supports LRO"); 845 /* 846 * If LRO timeout is not configured by application, 847 * use the minimal supported value. 848 */ 849 if (!config.lro.timeout) 850 config.lro.timeout = 851 config.hca_attr.lro_timer_supported_periods[0]; 852 DRV_LOG(DEBUG, "LRO session timeout set to %d usec", 853 config.lro.timeout); 854 } 855 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) 856 if (config.hca_attr.qos.sup && config.hca_attr.qos.srtcm_sup && 857 config.dv_flow_en) { 858 uint8_t reg_c_mask = 859 config.hca_attr.qos.flow_meter_reg_c_ids; 860 /* 861 * Meter needs two REG_C's for color match and pre-sfx 862 * flow match. Here get the REG_C for color match. 863 * REG_C_0 and REG_C_1 is reserved for metadata feature. 864 */ 865 reg_c_mask &= 0xfc; 866 if (__builtin_popcount(reg_c_mask) < 1) { 867 priv->mtr_en = 0; 868 DRV_LOG(WARNING, "No available register for" 869 " meter."); 870 } else { 871 priv->mtr_color_reg = ffs(reg_c_mask) - 1 + 872 REG_C_0; 873 priv->mtr_en = 1; 874 priv->mtr_reg_share = 875 config.hca_attr.qos.flow_meter_reg_share; 876 DRV_LOG(DEBUG, "The REG_C meter uses is %d", 877 priv->mtr_color_reg); 878 } 879 } 880 #endif 881 } 882 if (config.mprq.enabled && mprq) { 883 if (config.mprq.stride_num_n && 884 (config.mprq.stride_num_n > mprq_max_stride_num_n || 885 config.mprq.stride_num_n < mprq_min_stride_num_n)) { 886 config.mprq.stride_num_n = 887 RTE_MIN(RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N, 888 mprq_min_stride_num_n), 889 mprq_max_stride_num_n); 890 DRV_LOG(WARNING, 891 "the number of strides" 892 " for Multi-Packet RQ is out of range," 893 " setting default value (%u)", 894 1 << config.mprq.stride_num_n); 895 } 896 if (config.mprq.stride_size_n && 897 (config.mprq.stride_size_n > mprq_max_stride_size_n || 898 config.mprq.stride_size_n < mprq_min_stride_size_n)) { 899 config.mprq.stride_size_n = 900 RTE_MIN(RTE_MAX(MLX5_MPRQ_STRIDE_SIZE_N, 901 mprq_min_stride_size_n), 902 mprq_max_stride_size_n); 903 DRV_LOG(WARNING, 904 "the size of a stride" 905 " for Multi-Packet RQ is out of range," 906 " setting default value (%u)", 907 1 << config.mprq.stride_size_n); 908 } 909 config.mprq.min_stride_size_n = mprq_min_stride_size_n; 910 config.mprq.max_stride_size_n = mprq_max_stride_size_n; 911 } else if (config.mprq.enabled && !mprq) { 912 DRV_LOG(WARNING, "Multi-Packet RQ isn't supported"); 913 config.mprq.enabled = 0; 914 } 915 if (config.max_dump_files_num == 0) 916 config.max_dump_files_num = 128; 917 eth_dev = rte_eth_dev_allocate(name); 918 if (eth_dev == NULL) { 919 DRV_LOG(ERR, "can not allocate rte ethdev"); 920 err = ENOMEM; 921 goto error; 922 } 923 /* Flag to call rte_eth_dev_release_port() in rte_eth_dev_close(). */ 924 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 925 if (priv->representor) { 926 eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR; 927 eth_dev->data->representor_id = priv->representor_id; 928 } 929 /* 930 * Store associated network device interface index. This index 931 * is permanent throughout the lifetime of device. So, we may store 932 * the ifindex here and use the cached value further. 933 */ 934 MLX5_ASSERT(spawn->ifindex); 935 priv->if_index = spawn->ifindex; 936 eth_dev->data->dev_private = priv; 937 priv->dev_data = eth_dev->data; 938 eth_dev->data->mac_addrs = priv->mac; 939 eth_dev->device = dpdk_dev; 940 /* Configure the first MAC address by default. */ 941 if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) { 942 DRV_LOG(ERR, 943 "port %u cannot get MAC address, is mlx5_en" 944 " loaded? (errno: %s)", 945 eth_dev->data->port_id, strerror(rte_errno)); 946 err = ENODEV; 947 goto error; 948 } 949 DRV_LOG(INFO, 950 "port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", 951 eth_dev->data->port_id, 952 mac.addr_bytes[0], mac.addr_bytes[1], 953 mac.addr_bytes[2], mac.addr_bytes[3], 954 mac.addr_bytes[4], mac.addr_bytes[5]); 955 #ifdef RTE_LIBRTE_MLX5_DEBUG 956 { 957 char ifname[IF_NAMESIZE]; 958 959 if (mlx5_get_ifname(eth_dev, &ifname) == 0) 960 DRV_LOG(DEBUG, "port %u ifname is \"%s\"", 961 eth_dev->data->port_id, ifname); 962 else 963 DRV_LOG(DEBUG, "port %u ifname is unknown", 964 eth_dev->data->port_id); 965 } 966 #endif 967 /* Get actual MTU if possible. */ 968 err = mlx5_get_mtu(eth_dev, &priv->mtu); 969 if (err) { 970 err = rte_errno; 971 goto error; 972 } 973 DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id, 974 priv->mtu); 975 /* Initialize burst functions to prevent crashes before link-up. */ 976 eth_dev->rx_pkt_burst = removed_rx_burst; 977 eth_dev->tx_pkt_burst = removed_tx_burst; 978 eth_dev->dev_ops = &mlx5_os_dev_ops; 979 /* Register MAC address. */ 980 claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0)); 981 if (config.vf && config.vf_nl_en) 982 mlx5_nl_mac_addr_sync(priv->nl_socket_route, 983 mlx5_ifindex(eth_dev), 984 eth_dev->data->mac_addrs, 985 MLX5_MAX_MAC_ADDRESSES); 986 priv->flows = 0; 987 priv->ctrl_flows = 0; 988 TAILQ_INIT(&priv->flow_meters); 989 TAILQ_INIT(&priv->flow_meter_profiles); 990 /* Hint libmlx5 to use PMD allocator for data plane resources */ 991 struct mlx5dv_ctx_allocators alctr = { 992 .alloc = &mlx5_alloc_verbs_buf, 993 .free = &mlx5_free_verbs_buf, 994 .data = priv, 995 }; 996 mlx5_glue->dv_set_context_attr(sh->ctx, 997 MLX5DV_CTX_ATTR_BUF_ALLOCATORS, 998 (void *)((uintptr_t)&alctr)); 999 /* Bring Ethernet device up. */ 1000 DRV_LOG(DEBUG, "port %u forcing Ethernet interface up", 1001 eth_dev->data->port_id); 1002 mlx5_set_link_up(eth_dev); 1003 /* 1004 * Even though the interrupt handler is not installed yet, 1005 * interrupts will still trigger on the async_fd from 1006 * Verbs context returned by ibv_open_device(). 1007 */ 1008 mlx5_link_update(eth_dev, 0); 1009 #ifdef HAVE_MLX5DV_DR_ESWITCH 1010 if (!(config.hca_attr.eswitch_manager && config.dv_flow_en && 1011 (switch_info->representor || switch_info->master))) 1012 config.dv_esw_en = 0; 1013 #else 1014 config.dv_esw_en = 0; 1015 #endif 1016 /* Detect minimal data bytes to inline. */ 1017 mlx5_set_min_inline(spawn, &config); 1018 /* Store device configuration on private structure. */ 1019 priv->config = config; 1020 /* Create context for virtual machine VLAN workaround. */ 1021 priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex); 1022 if (config.dv_flow_en) { 1023 err = mlx5_alloc_shared_dr(priv); 1024 if (err) 1025 goto error; 1026 /* 1027 * RSS id is shared with meter flow id. Meter flow id can only 1028 * use the 24 MSB of the register. 1029 */ 1030 priv->qrss_id_pool = mlx5_flow_id_pool_alloc(UINT32_MAX >> 1031 MLX5_MTR_COLOR_BITS); 1032 if (!priv->qrss_id_pool) { 1033 DRV_LOG(ERR, "can't create flow id pool"); 1034 err = ENOMEM; 1035 goto error; 1036 } 1037 } 1038 /* Supported Verbs flow priority number detection. */ 1039 err = mlx5_flow_discover_priorities(eth_dev); 1040 if (err < 0) { 1041 err = -err; 1042 goto error; 1043 } 1044 priv->config.flow_prio = err; 1045 if (!priv->config.dv_esw_en && 1046 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 1047 DRV_LOG(WARNING, "metadata mode %u is not supported " 1048 "(no E-Switch)", priv->config.dv_xmeta_en); 1049 priv->config.dv_xmeta_en = MLX5_XMETA_MODE_LEGACY; 1050 } 1051 mlx5_set_metadata_mask(eth_dev); 1052 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1053 !priv->sh->dv_regc0_mask) { 1054 DRV_LOG(ERR, "metadata mode %u is not supported " 1055 "(no metadata reg_c[0] is available)", 1056 priv->config.dv_xmeta_en); 1057 err = ENOTSUP; 1058 goto error; 1059 } 1060 /* 1061 * Allocate the buffer for flow creating, just once. 1062 * The allocation must be done before any flow creating. 1063 */ 1064 mlx5_flow_alloc_intermediate(eth_dev); 1065 /* Query availability of metadata reg_c's. */ 1066 err = mlx5_flow_discover_mreg_c(eth_dev); 1067 if (err < 0) { 1068 err = -err; 1069 goto error; 1070 } 1071 if (!mlx5_flow_ext_mreg_supported(eth_dev)) { 1072 DRV_LOG(DEBUG, 1073 "port %u extensive metadata register is not supported", 1074 eth_dev->data->port_id); 1075 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 1076 DRV_LOG(ERR, "metadata mode %u is not supported " 1077 "(no metadata registers available)", 1078 priv->config.dv_xmeta_en); 1079 err = ENOTSUP; 1080 goto error; 1081 } 1082 } 1083 if (priv->config.dv_flow_en && 1084 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1085 mlx5_flow_ext_mreg_supported(eth_dev) && 1086 priv->sh->dv_regc0_mask) { 1087 priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME, 1088 MLX5_FLOW_MREG_HTABLE_SZ); 1089 if (!priv->mreg_cp_tbl) { 1090 err = ENOMEM; 1091 goto error; 1092 } 1093 } 1094 return eth_dev; 1095 error: 1096 if (priv) { 1097 if (priv->mreg_cp_tbl) 1098 mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL); 1099 if (priv->sh) 1100 mlx5_os_free_shared_dr(priv); 1101 if (priv->nl_socket_route >= 0) 1102 close(priv->nl_socket_route); 1103 if (priv->nl_socket_rdma >= 0) 1104 close(priv->nl_socket_rdma); 1105 if (priv->vmwa_context) 1106 mlx5_vlan_vmwa_exit(priv->vmwa_context); 1107 if (priv->qrss_id_pool) 1108 mlx5_flow_id_pool_release(priv->qrss_id_pool); 1109 if (own_domain_id) 1110 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 1111 rte_free(priv); 1112 if (eth_dev != NULL) 1113 eth_dev->data->dev_private = NULL; 1114 } 1115 if (eth_dev != NULL) { 1116 /* mac_addrs must not be freed alone because part of 1117 * dev_private 1118 **/ 1119 eth_dev->data->mac_addrs = NULL; 1120 rte_eth_dev_release_port(eth_dev); 1121 } 1122 if (sh) 1123 mlx5_free_shared_dev_ctx(sh); 1124 MLX5_ASSERT(err > 0); 1125 rte_errno = err; 1126 return NULL; 1127 } 1128 1129 /** 1130 * Comparison callback to sort device data. 1131 * 1132 * This is meant to be used with qsort(). 1133 * 1134 * @param a[in] 1135 * Pointer to pointer to first data object. 1136 * @param b[in] 1137 * Pointer to pointer to second data object. 1138 * 1139 * @return 1140 * 0 if both objects are equal, less than 0 if the first argument is less 1141 * than the second, greater than 0 otherwise. 1142 */ 1143 static int 1144 mlx5_dev_spawn_data_cmp(const void *a, const void *b) 1145 { 1146 const struct mlx5_switch_info *si_a = 1147 &((const struct mlx5_dev_spawn_data *)a)->info; 1148 const struct mlx5_switch_info *si_b = 1149 &((const struct mlx5_dev_spawn_data *)b)->info; 1150 int ret; 1151 1152 /* Master device first. */ 1153 ret = si_b->master - si_a->master; 1154 if (ret) 1155 return ret; 1156 /* Then representor devices. */ 1157 ret = si_b->representor - si_a->representor; 1158 if (ret) 1159 return ret; 1160 /* Unidentified devices come last in no specific order. */ 1161 if (!si_a->representor) 1162 return 0; 1163 /* Order representors by name. */ 1164 return si_a->port_name - si_b->port_name; 1165 } 1166 1167 /** 1168 * Match PCI information for possible slaves of bonding device. 1169 * 1170 * @param[in] ibv_dev 1171 * Pointer to Infiniband device structure. 1172 * @param[in] pci_dev 1173 * Pointer to PCI device structure to match PCI address. 1174 * @param[in] nl_rdma 1175 * Netlink RDMA group socket handle. 1176 * 1177 * @return 1178 * negative value if no bonding device found, otherwise 1179 * positive index of slave PF in bonding. 1180 */ 1181 static int 1182 mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev, 1183 const struct rte_pci_device *pci_dev, 1184 int nl_rdma) 1185 { 1186 char ifname[IF_NAMESIZE + 1]; 1187 unsigned int ifindex; 1188 unsigned int np, i; 1189 FILE *file = NULL; 1190 int pf = -1; 1191 1192 /* 1193 * Try to get master device name. If something goes 1194 * wrong suppose the lack of kernel support and no 1195 * bonding devices. 1196 */ 1197 if (nl_rdma < 0) 1198 return -1; 1199 if (!strstr(ibv_dev->name, "bond")) 1200 return -1; 1201 np = mlx5_nl_portnum(nl_rdma, ibv_dev->name); 1202 if (!np) 1203 return -1; 1204 /* 1205 * The Master device might not be on the predefined 1206 * port (not on port index 1, it is not garanted), 1207 * we have to scan all Infiniband device port and 1208 * find master. 1209 */ 1210 for (i = 1; i <= np; ++i) { 1211 /* Check whether Infiniband port is populated. */ 1212 ifindex = mlx5_nl_ifindex(nl_rdma, ibv_dev->name, i); 1213 if (!ifindex) 1214 continue; 1215 if (!if_indextoname(ifindex, ifname)) 1216 continue; 1217 /* Try to read bonding slave names from sysfs. */ 1218 MKSTR(slaves, 1219 "/sys/class/net/%s/master/bonding/slaves", ifname); 1220 file = fopen(slaves, "r"); 1221 if (file) 1222 break; 1223 } 1224 if (!file) 1225 return -1; 1226 /* Use safe format to check maximal buffer length. */ 1227 MLX5_ASSERT(atol(RTE_STR(IF_NAMESIZE)) == IF_NAMESIZE); 1228 while (fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", ifname) == 1) { 1229 char tmp_str[IF_NAMESIZE + 32]; 1230 struct rte_pci_addr pci_addr; 1231 struct mlx5_switch_info info; 1232 1233 /* Process slave interface names in the loop. */ 1234 snprintf(tmp_str, sizeof(tmp_str), 1235 "/sys/class/net/%s", ifname); 1236 if (mlx5_dev_to_pci_addr(tmp_str, &pci_addr)) { 1237 DRV_LOG(WARNING, "can not get PCI address" 1238 " for netdev \"%s\"", ifname); 1239 continue; 1240 } 1241 if (pci_dev->addr.domain != pci_addr.domain || 1242 pci_dev->addr.bus != pci_addr.bus || 1243 pci_dev->addr.devid != pci_addr.devid || 1244 pci_dev->addr.function != pci_addr.function) 1245 continue; 1246 /* Slave interface PCI address match found. */ 1247 fclose(file); 1248 snprintf(tmp_str, sizeof(tmp_str), 1249 "/sys/class/net/%s/phys_port_name", ifname); 1250 file = fopen(tmp_str, "rb"); 1251 if (!file) 1252 break; 1253 info.name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET; 1254 if (fscanf(file, "%32s", tmp_str) == 1) 1255 mlx5_translate_port_name(tmp_str, &info); 1256 if (info.name_type == MLX5_PHYS_PORT_NAME_TYPE_LEGACY || 1257 info.name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK) 1258 pf = info.port_name; 1259 break; 1260 } 1261 if (file) 1262 fclose(file); 1263 return pf; 1264 } 1265 1266 /** 1267 * DPDK callback to register a PCI device. 1268 * 1269 * This function spawns Ethernet devices out of a given PCI device. 1270 * 1271 * @param[in] pci_drv 1272 * PCI driver structure (mlx5_driver). 1273 * @param[in] pci_dev 1274 * PCI device information. 1275 * 1276 * @return 1277 * 0 on success, a negative errno value otherwise and rte_errno is set. 1278 */ 1279 int 1280 mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1281 struct rte_pci_device *pci_dev) 1282 { 1283 struct ibv_device **ibv_list; 1284 /* 1285 * Number of found IB Devices matching with requested PCI BDF. 1286 * nd != 1 means there are multiple IB devices over the same 1287 * PCI device and we have representors and master. 1288 */ 1289 unsigned int nd = 0; 1290 /* 1291 * Number of found IB device Ports. nd = 1 and np = 1..n means 1292 * we have the single multiport IB device, and there may be 1293 * representors attached to some of found ports. 1294 */ 1295 unsigned int np = 0; 1296 /* 1297 * Number of DPDK ethernet devices to Spawn - either over 1298 * multiple IB devices or multiple ports of single IB device. 1299 * Actually this is the number of iterations to spawn. 1300 */ 1301 unsigned int ns = 0; 1302 /* 1303 * Bonding device 1304 * < 0 - no bonding device (single one) 1305 * >= 0 - bonding device (value is slave PF index) 1306 */ 1307 int bd = -1; 1308 struct mlx5_dev_spawn_data *list = NULL; 1309 struct mlx5_dev_config dev_config; 1310 int ret; 1311 1312 if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_NET) { 1313 DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5" 1314 " driver."); 1315 return 1; 1316 } 1317 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 1318 mlx5_pmd_socket_init(); 1319 ret = mlx5_init_once(); 1320 if (ret) { 1321 DRV_LOG(ERR, "unable to init PMD global data: %s", 1322 strerror(rte_errno)); 1323 return -rte_errno; 1324 } 1325 MLX5_ASSERT(pci_drv == &mlx5_driver); 1326 errno = 0; 1327 ibv_list = mlx5_glue->get_device_list(&ret); 1328 if (!ibv_list) { 1329 rte_errno = errno ? errno : ENOSYS; 1330 DRV_LOG(ERR, "cannot list devices, is ib_uverbs loaded?"); 1331 return -rte_errno; 1332 } 1333 /* 1334 * First scan the list of all Infiniband devices to find 1335 * matching ones, gathering into the list. 1336 */ 1337 struct ibv_device *ibv_match[ret + 1]; 1338 int nl_route = mlx5_nl_init(NETLINK_ROUTE); 1339 int nl_rdma = mlx5_nl_init(NETLINK_RDMA); 1340 unsigned int i; 1341 1342 while (ret-- > 0) { 1343 struct rte_pci_addr pci_addr; 1344 1345 DRV_LOG(DEBUG, "checking device \"%s\"", ibv_list[ret]->name); 1346 bd = mlx5_device_bond_pci_match 1347 (ibv_list[ret], pci_dev, nl_rdma); 1348 if (bd >= 0) { 1349 /* 1350 * Bonding device detected. Only one match is allowed, 1351 * the bonding is supported over multi-port IB device, 1352 * there should be no matches on representor PCI 1353 * functions or non VF LAG bonding devices with 1354 * specified address. 1355 */ 1356 if (nd) { 1357 DRV_LOG(ERR, 1358 "multiple PCI match on bonding device" 1359 "\"%s\" found", ibv_list[ret]->name); 1360 rte_errno = ENOENT; 1361 ret = -rte_errno; 1362 goto exit; 1363 } 1364 DRV_LOG(INFO, "PCI information matches for" 1365 " slave %d bonding device \"%s\"", 1366 bd, ibv_list[ret]->name); 1367 ibv_match[nd++] = ibv_list[ret]; 1368 break; 1369 } 1370 if (mlx5_dev_to_pci_addr 1371 (ibv_list[ret]->ibdev_path, &pci_addr)) 1372 continue; 1373 if (pci_dev->addr.domain != pci_addr.domain || 1374 pci_dev->addr.bus != pci_addr.bus || 1375 pci_dev->addr.devid != pci_addr.devid || 1376 pci_dev->addr.function != pci_addr.function) 1377 continue; 1378 DRV_LOG(INFO, "PCI information matches for device \"%s\"", 1379 ibv_list[ret]->name); 1380 ibv_match[nd++] = ibv_list[ret]; 1381 } 1382 ibv_match[nd] = NULL; 1383 if (!nd) { 1384 /* No device matches, just complain and bail out. */ 1385 DRV_LOG(WARNING, 1386 "no Verbs device matches PCI device " PCI_PRI_FMT "," 1387 " are kernel drivers loaded?", 1388 pci_dev->addr.domain, pci_dev->addr.bus, 1389 pci_dev->addr.devid, pci_dev->addr.function); 1390 rte_errno = ENOENT; 1391 ret = -rte_errno; 1392 goto exit; 1393 } 1394 if (nd == 1) { 1395 /* 1396 * Found single matching device may have multiple ports. 1397 * Each port may be representor, we have to check the port 1398 * number and check the representors existence. 1399 */ 1400 if (nl_rdma >= 0) 1401 np = mlx5_nl_portnum(nl_rdma, ibv_match[0]->name); 1402 if (!np) 1403 DRV_LOG(WARNING, "can not get IB device \"%s\"" 1404 " ports number", ibv_match[0]->name); 1405 if (bd >= 0 && !np) { 1406 DRV_LOG(ERR, "can not get ports" 1407 " for bonding device"); 1408 rte_errno = ENOENT; 1409 ret = -rte_errno; 1410 goto exit; 1411 } 1412 } 1413 #ifndef HAVE_MLX5DV_DR_DEVX_PORT 1414 if (bd >= 0) { 1415 /* 1416 * This may happen if there is VF LAG kernel support and 1417 * application is compiled with older rdma_core library. 1418 */ 1419 DRV_LOG(ERR, 1420 "No kernel/verbs support for VF LAG bonding found."); 1421 rte_errno = ENOTSUP; 1422 ret = -rte_errno; 1423 goto exit; 1424 } 1425 #endif 1426 /* 1427 * Now we can determine the maximal 1428 * amount of devices to be spawned. 1429 */ 1430 list = rte_zmalloc("device spawn data", 1431 sizeof(struct mlx5_dev_spawn_data) * 1432 (np ? np : nd), 1433 RTE_CACHE_LINE_SIZE); 1434 if (!list) { 1435 DRV_LOG(ERR, "spawn data array allocation failure"); 1436 rte_errno = ENOMEM; 1437 ret = -rte_errno; 1438 goto exit; 1439 } 1440 if (bd >= 0 || np > 1) { 1441 /* 1442 * Single IB device with multiple ports found, 1443 * it may be E-Switch master device and representors. 1444 * We have to perform identification through the ports. 1445 */ 1446 MLX5_ASSERT(nl_rdma >= 0); 1447 MLX5_ASSERT(ns == 0); 1448 MLX5_ASSERT(nd == 1); 1449 MLX5_ASSERT(np); 1450 for (i = 1; i <= np; ++i) { 1451 list[ns].max_port = np; 1452 list[ns].phys_port = i; 1453 list[ns].phys_dev = ibv_match[0]; 1454 list[ns].eth_dev = NULL; 1455 list[ns].pci_dev = pci_dev; 1456 list[ns].pf_bond = bd; 1457 list[ns].ifindex = mlx5_nl_ifindex 1458 (nl_rdma, 1459 mlx5_os_get_dev_device_name 1460 (list[ns].phys_dev), i); 1461 if (!list[ns].ifindex) { 1462 /* 1463 * No network interface index found for the 1464 * specified port, it means there is no 1465 * representor on this port. It's OK, 1466 * there can be disabled ports, for example 1467 * if sriov_numvfs < sriov_totalvfs. 1468 */ 1469 continue; 1470 } 1471 ret = -1; 1472 if (nl_route >= 0) 1473 ret = mlx5_nl_switch_info 1474 (nl_route, 1475 list[ns].ifindex, 1476 &list[ns].info); 1477 if (ret || (!list[ns].info.representor && 1478 !list[ns].info.master)) { 1479 /* 1480 * We failed to recognize representors with 1481 * Netlink, let's try to perform the task 1482 * with sysfs. 1483 */ 1484 ret = mlx5_sysfs_switch_info 1485 (list[ns].ifindex, 1486 &list[ns].info); 1487 } 1488 if (!ret && bd >= 0) { 1489 switch (list[ns].info.name_type) { 1490 case MLX5_PHYS_PORT_NAME_TYPE_UPLINK: 1491 if (list[ns].info.port_name == bd) 1492 ns++; 1493 break; 1494 case MLX5_PHYS_PORT_NAME_TYPE_PFHPF: 1495 /* Fallthrough */ 1496 case MLX5_PHYS_PORT_NAME_TYPE_PFVF: 1497 if (list[ns].info.pf_num == bd) 1498 ns++; 1499 break; 1500 default: 1501 break; 1502 } 1503 continue; 1504 } 1505 if (!ret && (list[ns].info.representor ^ 1506 list[ns].info.master)) 1507 ns++; 1508 } 1509 if (!ns) { 1510 DRV_LOG(ERR, 1511 "unable to recognize master/representors" 1512 " on the IB device with multiple ports"); 1513 rte_errno = ENOENT; 1514 ret = -rte_errno; 1515 goto exit; 1516 } 1517 } else { 1518 /* 1519 * The existence of several matching entries (nd > 1) means 1520 * port representors have been instantiated. No existing Verbs 1521 * call nor sysfs entries can tell them apart, this can only 1522 * be done through Netlink calls assuming kernel drivers are 1523 * recent enough to support them. 1524 * 1525 * In the event of identification failure through Netlink, 1526 * try again through sysfs, then: 1527 * 1528 * 1. A single IB device matches (nd == 1) with single 1529 * port (np=0/1) and is not a representor, assume 1530 * no switch support. 1531 * 1532 * 2. Otherwise no safe assumptions can be made; 1533 * complain louder and bail out. 1534 */ 1535 for (i = 0; i != nd; ++i) { 1536 memset(&list[ns].info, 0, sizeof(list[ns].info)); 1537 list[ns].max_port = 1; 1538 list[ns].phys_port = 1; 1539 list[ns].phys_dev = ibv_match[i]; 1540 list[ns].eth_dev = NULL; 1541 list[ns].pci_dev = pci_dev; 1542 list[ns].pf_bond = -1; 1543 list[ns].ifindex = 0; 1544 if (nl_rdma >= 0) 1545 list[ns].ifindex = mlx5_nl_ifindex 1546 (nl_rdma, 1547 mlx5_os_get_dev_device_name 1548 (list[ns].phys_dev), 1); 1549 if (!list[ns].ifindex) { 1550 char ifname[IF_NAMESIZE]; 1551 1552 /* 1553 * Netlink failed, it may happen with old 1554 * ib_core kernel driver (before 4.16). 1555 * We can assume there is old driver because 1556 * here we are processing single ports IB 1557 * devices. Let's try sysfs to retrieve 1558 * the ifindex. The method works for 1559 * master device only. 1560 */ 1561 if (nd > 1) { 1562 /* 1563 * Multiple devices found, assume 1564 * representors, can not distinguish 1565 * master/representor and retrieve 1566 * ifindex via sysfs. 1567 */ 1568 continue; 1569 } 1570 ret = mlx5_get_ifname_sysfs 1571 (ibv_match[i]->ibdev_path, ifname); 1572 if (!ret) 1573 list[ns].ifindex = 1574 if_nametoindex(ifname); 1575 if (!list[ns].ifindex) { 1576 /* 1577 * No network interface index found 1578 * for the specified device, it means 1579 * there it is neither representor 1580 * nor master. 1581 */ 1582 continue; 1583 } 1584 } 1585 ret = -1; 1586 if (nl_route >= 0) 1587 ret = mlx5_nl_switch_info 1588 (nl_route, 1589 list[ns].ifindex, 1590 &list[ns].info); 1591 if (ret || (!list[ns].info.representor && 1592 !list[ns].info.master)) { 1593 /* 1594 * We failed to recognize representors with 1595 * Netlink, let's try to perform the task 1596 * with sysfs. 1597 */ 1598 ret = mlx5_sysfs_switch_info 1599 (list[ns].ifindex, 1600 &list[ns].info); 1601 } 1602 if (!ret && (list[ns].info.representor ^ 1603 list[ns].info.master)) { 1604 ns++; 1605 } else if ((nd == 1) && 1606 !list[ns].info.representor && 1607 !list[ns].info.master) { 1608 /* 1609 * Single IB device with 1610 * one physical port and 1611 * attached network device. 1612 * May be SRIOV is not enabled 1613 * or there is no representors. 1614 */ 1615 DRV_LOG(INFO, "no E-Switch support detected"); 1616 ns++; 1617 break; 1618 } 1619 } 1620 if (!ns) { 1621 DRV_LOG(ERR, 1622 "unable to recognize master/representors" 1623 " on the multiple IB devices"); 1624 rte_errno = ENOENT; 1625 ret = -rte_errno; 1626 goto exit; 1627 } 1628 } 1629 MLX5_ASSERT(ns); 1630 /* 1631 * Sort list to probe devices in natural order for users convenience 1632 * (i.e. master first, then representors from lowest to highest ID). 1633 */ 1634 qsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp); 1635 /* Default configuration. */ 1636 dev_config = (struct mlx5_dev_config){ 1637 .hw_padding = 0, 1638 .mps = MLX5_ARG_UNSET, 1639 .dbnc = MLX5_ARG_UNSET, 1640 .rx_vec_en = 1, 1641 .txq_inline_max = MLX5_ARG_UNSET, 1642 .txq_inline_min = MLX5_ARG_UNSET, 1643 .txq_inline_mpw = MLX5_ARG_UNSET, 1644 .txqs_inline = MLX5_ARG_UNSET, 1645 .vf_nl_en = 1, 1646 .mr_ext_memseg_en = 1, 1647 .mprq = { 1648 .enabled = 0, /* Disabled by default. */ 1649 .stride_num_n = 0, 1650 .stride_size_n = 0, 1651 .max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN, 1652 .min_rxqs_num = MLX5_MPRQ_MIN_RXQS, 1653 }, 1654 .dv_esw_en = 1, 1655 .dv_flow_en = 1, 1656 .log_hp_size = MLX5_ARG_UNSET, 1657 }; 1658 /* Device specific configuration. */ 1659 switch (pci_dev->id.device_id) { 1660 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 1661 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 1662 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 1663 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 1664 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF: 1665 case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF: 1666 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF: 1667 dev_config.vf = 1; 1668 break; 1669 default: 1670 break; 1671 } 1672 for (i = 0; i != ns; ++i) { 1673 uint32_t restore; 1674 1675 list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device, 1676 &list[i], 1677 dev_config); 1678 if (!list[i].eth_dev) { 1679 if (rte_errno != EBUSY && rte_errno != EEXIST) 1680 break; 1681 /* Device is disabled or already spawned. Ignore it. */ 1682 continue; 1683 } 1684 restore = list[i].eth_dev->data->dev_flags; 1685 rte_eth_copy_pci_info(list[i].eth_dev, pci_dev); 1686 /* Restore non-PCI flags cleared by the above call. */ 1687 list[i].eth_dev->data->dev_flags |= restore; 1688 rte_eth_dev_probing_finish(list[i].eth_dev); 1689 } 1690 if (i != ns) { 1691 DRV_LOG(ERR, 1692 "probe of PCI device " PCI_PRI_FMT " aborted after" 1693 " encountering an error: %s", 1694 pci_dev->addr.domain, pci_dev->addr.bus, 1695 pci_dev->addr.devid, pci_dev->addr.function, 1696 strerror(rte_errno)); 1697 ret = -rte_errno; 1698 /* Roll back. */ 1699 while (i--) { 1700 if (!list[i].eth_dev) 1701 continue; 1702 mlx5_dev_close(list[i].eth_dev); 1703 /* mac_addrs must not be freed because in dev_private */ 1704 list[i].eth_dev->data->mac_addrs = NULL; 1705 claim_zero(rte_eth_dev_release_port(list[i].eth_dev)); 1706 } 1707 /* Restore original error. */ 1708 rte_errno = -ret; 1709 } else { 1710 ret = 0; 1711 } 1712 exit: 1713 /* 1714 * Do the routine cleanup: 1715 * - close opened Netlink sockets 1716 * - free allocated spawn data array 1717 * - free the Infiniband device list 1718 */ 1719 if (nl_rdma >= 0) 1720 close(nl_rdma); 1721 if (nl_route >= 0) 1722 close(nl_route); 1723 if (list) 1724 rte_free(list); 1725 MLX5_ASSERT(ibv_list); 1726 mlx5_glue->free_device_list(ibv_list); 1727 return ret; 1728 } 1729 1730 static int 1731 mlx5_config_doorbell_mapping_env(const struct mlx5_dev_config *config) 1732 { 1733 char *env; 1734 int value; 1735 1736 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1737 /* Get environment variable to store. */ 1738 env = getenv(MLX5_SHUT_UP_BF); 1739 value = env ? !!strcmp(env, "0") : MLX5_ARG_UNSET; 1740 if (config->dbnc == MLX5_ARG_UNSET) 1741 setenv(MLX5_SHUT_UP_BF, MLX5_SHUT_UP_BF_DEFAULT, 1); 1742 else 1743 setenv(MLX5_SHUT_UP_BF, 1744 config->dbnc == MLX5_TXDB_NCACHED ? "1" : "0", 1); 1745 return value; 1746 } 1747 1748 static void 1749 mlx5_restore_doorbell_mapping_env(int value) 1750 { 1751 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1752 /* Restore the original environment variable state. */ 1753 if (value == MLX5_ARG_UNSET) 1754 unsetenv(MLX5_SHUT_UP_BF); 1755 else 1756 setenv(MLX5_SHUT_UP_BF, value ? "1" : "0", 1); 1757 } 1758 1759 /** 1760 * Extract pdn of PD object using DV API. 1761 * 1762 * @param[in] pd 1763 * Pointer to the verbs PD object. 1764 * @param[out] pdn 1765 * Pointer to the PD object number variable. 1766 * 1767 * @return 1768 * 0 on success, error value otherwise. 1769 */ 1770 int 1771 mlx5_os_get_pdn(void *pd, uint32_t *pdn) 1772 { 1773 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 1774 struct mlx5dv_obj obj; 1775 struct mlx5dv_pd pd_info; 1776 int ret = 0; 1777 1778 obj.pd.in = pd; 1779 obj.pd.out = &pd_info; 1780 ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD); 1781 if (ret) { 1782 DRV_LOG(DEBUG, "Fail to get PD object info"); 1783 return ret; 1784 } 1785 *pdn = pd_info.pdn; 1786 return 0; 1787 #else 1788 (void)pd; 1789 (void)pdn; 1790 return -ENOTSUP; 1791 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 1792 } 1793 1794 /** 1795 * Function API to open IB device. 1796 * 1797 * This function calls the Linux glue APIs to open a device. 1798 * 1799 * @param[in] spawn 1800 * Pointer to the IB device attributes (name, port, etc). 1801 * @param[out] config 1802 * Pointer to device configuration structure. 1803 * @param[out] sh 1804 * Pointer to shared context structure. 1805 * 1806 * @return 1807 * 0 on success, a positive error value otherwise. 1808 */ 1809 int 1810 mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn, 1811 const struct mlx5_dev_config *config, 1812 struct mlx5_dev_ctx_shared *sh) 1813 { 1814 int dbmap_env; 1815 int err = 0; 1816 /* 1817 * Configure environment variable "MLX5_BF_SHUT_UP" 1818 * before the device creation. The rdma_core library 1819 * checks the variable at device creation and 1820 * stores the result internally. 1821 */ 1822 dbmap_env = mlx5_config_doorbell_mapping_env(config); 1823 /* Try to open IB device with DV first, then usual Verbs. */ 1824 errno = 0; 1825 sh->ctx = mlx5_glue->dv_open_device(spawn->phys_dev); 1826 if (sh->ctx) { 1827 sh->devx = 1; 1828 DRV_LOG(DEBUG, "DevX is supported"); 1829 /* The device is created, no need for environment. */ 1830 mlx5_restore_doorbell_mapping_env(dbmap_env); 1831 } else { 1832 /* The environment variable is still configured. */ 1833 sh->ctx = mlx5_glue->open_device(spawn->phys_dev); 1834 err = errno ? errno : ENODEV; 1835 /* 1836 * The environment variable is not needed anymore, 1837 * all device creation attempts are completed. 1838 */ 1839 mlx5_restore_doorbell_mapping_env(dbmap_env); 1840 if (!sh->ctx) 1841 return err; 1842 DRV_LOG(DEBUG, "DevX is NOT supported"); 1843 err = 0; 1844 } 1845 return err; 1846 } 1847 1848 /** 1849 * Install shared asynchronous device events handler. 1850 * This function is implemented to support event sharing 1851 * between multiple ports of single IB device. 1852 * 1853 * @param sh 1854 * Pointer to mlx5_dev_ctx_shared object. 1855 */ 1856 void 1857 mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh) 1858 { 1859 int ret; 1860 int flags; 1861 1862 sh->intr_handle.fd = -1; 1863 flags = fcntl(((struct ibv_context *)sh->ctx)->async_fd, F_GETFL); 1864 ret = fcntl(((struct ibv_context *)sh->ctx)->async_fd, 1865 F_SETFL, flags | O_NONBLOCK); 1866 if (ret) { 1867 DRV_LOG(INFO, "failed to change file descriptor async event" 1868 " queue"); 1869 } else { 1870 sh->intr_handle.fd = ((struct ibv_context *)sh->ctx)->async_fd; 1871 sh->intr_handle.type = RTE_INTR_HANDLE_EXT; 1872 if (rte_intr_callback_register(&sh->intr_handle, 1873 mlx5_dev_interrupt_handler, sh)) { 1874 DRV_LOG(INFO, "Fail to install the shared interrupt."); 1875 sh->intr_handle.fd = -1; 1876 } 1877 } 1878 if (sh->devx) { 1879 #ifdef HAVE_IBV_DEVX_ASYNC 1880 sh->intr_handle_devx.fd = -1; 1881 sh->devx_comp = 1882 (void *)mlx5_glue->devx_create_cmd_comp(sh->ctx); 1883 struct mlx5dv_devx_cmd_comp *devx_comp = sh->devx_comp; 1884 if (!devx_comp) { 1885 DRV_LOG(INFO, "failed to allocate devx_comp."); 1886 return; 1887 } 1888 flags = fcntl(devx_comp->fd, F_GETFL); 1889 ret = fcntl(devx_comp->fd, F_SETFL, flags | O_NONBLOCK); 1890 if (ret) { 1891 DRV_LOG(INFO, "failed to change file descriptor" 1892 " devx comp"); 1893 return; 1894 } 1895 sh->intr_handle_devx.fd = devx_comp->fd; 1896 sh->intr_handle_devx.type = RTE_INTR_HANDLE_EXT; 1897 if (rte_intr_callback_register(&sh->intr_handle_devx, 1898 mlx5_dev_interrupt_handler_devx, sh)) { 1899 DRV_LOG(INFO, "Fail to install the devx shared" 1900 " interrupt."); 1901 sh->intr_handle_devx.fd = -1; 1902 } 1903 #endif /* HAVE_IBV_DEVX_ASYNC */ 1904 } 1905 } 1906 1907 /** 1908 * Uninstall shared asynchronous device events handler. 1909 * This function is implemented to support event sharing 1910 * between multiple ports of single IB device. 1911 * 1912 * @param dev 1913 * Pointer to mlx5_dev_ctx_shared object. 1914 */ 1915 void 1916 mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh) 1917 { 1918 if (sh->intr_handle.fd >= 0) 1919 mlx5_intr_callback_unregister(&sh->intr_handle, 1920 mlx5_dev_interrupt_handler, sh); 1921 #ifdef HAVE_IBV_DEVX_ASYNC 1922 if (sh->intr_handle_devx.fd >= 0) 1923 rte_intr_callback_unregister(&sh->intr_handle_devx, 1924 mlx5_dev_interrupt_handler_devx, sh); 1925 if (sh->devx_comp) 1926 mlx5_glue->devx_destroy_cmd_comp(sh->devx_comp); 1927 #endif 1928 } 1929 1930 /** 1931 * Read statistics by a named counter. 1932 * 1933 * @param[in] priv 1934 * Pointer to the private device data structure. 1935 * @param[in] ctr_name 1936 * Pointer to the name of the statistic counter to read 1937 * @param[out] stat 1938 * Pointer to read statistic value. 1939 * @return 1940 * 0 on success and stat is valud, 1 if failed to read the value 1941 * rte_errno is set. 1942 * 1943 */ 1944 int 1945 mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name, 1946 uint64_t *stat) 1947 { 1948 int fd; 1949 1950 if (priv->sh) { 1951 MKSTR(path, "%s/ports/%d/hw_counters/%s", 1952 priv->sh->ibdev_path, 1953 priv->dev_port, 1954 ctr_name); 1955 fd = open(path, O_RDONLY); 1956 if (fd != -1) { 1957 char buf[21] = {'\0'}; 1958 ssize_t n = read(fd, buf, sizeof(buf)); 1959 1960 close(fd); 1961 if (n != -1) { 1962 *stat = strtoull(buf, NULL, 10); 1963 return 0; 1964 } 1965 } 1966 } 1967 *stat = 0; 1968 return 1; 1969 } 1970 1971 /** 1972 * Read device counters table. 1973 * 1974 * @param dev 1975 * Pointer to Ethernet device. 1976 * @param[out] stats 1977 * Counters table output buffer. 1978 * 1979 * @return 1980 * 0 on success and stats is filled, negative errno value otherwise and 1981 * rte_errno is set. 1982 */ 1983 int 1984 mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) 1985 { 1986 struct mlx5_priv *priv = dev->data->dev_private; 1987 struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; 1988 unsigned int i; 1989 struct ifreq ifr; 1990 unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t); 1991 unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz]; 1992 struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf; 1993 int ret; 1994 1995 et_stats->cmd = ETHTOOL_GSTATS; 1996 et_stats->n_stats = xstats_ctrl->stats_n; 1997 ifr.ifr_data = (caddr_t)et_stats; 1998 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); 1999 if (ret) { 2000 DRV_LOG(WARNING, 2001 "port %u unable to read statistic values from device", 2002 dev->data->port_id); 2003 return ret; 2004 } 2005 for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) { 2006 if (xstats_ctrl->info[i].dev) { 2007 ret = mlx5_os_read_dev_stat(priv, 2008 xstats_ctrl->info[i].ctr_name, 2009 &stats[i]); 2010 /* return last xstats counter if fail to read. */ 2011 if (ret == 0) 2012 xstats_ctrl->xstats[i] = stats[i]; 2013 else 2014 stats[i] = xstats_ctrl->xstats[i]; 2015 } else { 2016 stats[i] = (uint64_t) 2017 et_stats->data[xstats_ctrl->dev_table_idx[i]]; 2018 } 2019 } 2020 return 0; 2021 } 2022 2023 /** 2024 * Query the number of statistics provided by ETHTOOL. 2025 * 2026 * @param dev 2027 * Pointer to Ethernet device. 2028 * 2029 * @return 2030 * Number of statistics on success, negative errno value otherwise and 2031 * rte_errno is set. 2032 */ 2033 int 2034 mlx5_os_get_stats_n(struct rte_eth_dev *dev) 2035 { 2036 struct ethtool_drvinfo drvinfo; 2037 struct ifreq ifr; 2038 int ret; 2039 2040 drvinfo.cmd = ETHTOOL_GDRVINFO; 2041 ifr.ifr_data = (caddr_t)&drvinfo; 2042 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); 2043 if (ret) { 2044 DRV_LOG(WARNING, "port %u unable to query number of statistics", 2045 dev->data->port_id); 2046 return ret; 2047 } 2048 return drvinfo.n_stats; 2049 } 2050 2051 static const struct mlx5_counter_ctrl mlx5_counters_init[] = { 2052 { 2053 .dpdk_name = "rx_port_unicast_bytes", 2054 .ctr_name = "rx_vport_unicast_bytes", 2055 }, 2056 { 2057 .dpdk_name = "rx_port_multicast_bytes", 2058 .ctr_name = "rx_vport_multicast_bytes", 2059 }, 2060 { 2061 .dpdk_name = "rx_port_broadcast_bytes", 2062 .ctr_name = "rx_vport_broadcast_bytes", 2063 }, 2064 { 2065 .dpdk_name = "rx_port_unicast_packets", 2066 .ctr_name = "rx_vport_unicast_packets", 2067 }, 2068 { 2069 .dpdk_name = "rx_port_multicast_packets", 2070 .ctr_name = "rx_vport_multicast_packets", 2071 }, 2072 { 2073 .dpdk_name = "rx_port_broadcast_packets", 2074 .ctr_name = "rx_vport_broadcast_packets", 2075 }, 2076 { 2077 .dpdk_name = "tx_port_unicast_bytes", 2078 .ctr_name = "tx_vport_unicast_bytes", 2079 }, 2080 { 2081 .dpdk_name = "tx_port_multicast_bytes", 2082 .ctr_name = "tx_vport_multicast_bytes", 2083 }, 2084 { 2085 .dpdk_name = "tx_port_broadcast_bytes", 2086 .ctr_name = "tx_vport_broadcast_bytes", 2087 }, 2088 { 2089 .dpdk_name = "tx_port_unicast_packets", 2090 .ctr_name = "tx_vport_unicast_packets", 2091 }, 2092 { 2093 .dpdk_name = "tx_port_multicast_packets", 2094 .ctr_name = "tx_vport_multicast_packets", 2095 }, 2096 { 2097 .dpdk_name = "tx_port_broadcast_packets", 2098 .ctr_name = "tx_vport_broadcast_packets", 2099 }, 2100 { 2101 .dpdk_name = "rx_wqe_err", 2102 .ctr_name = "rx_wqe_err", 2103 }, 2104 { 2105 .dpdk_name = "rx_crc_errors_phy", 2106 .ctr_name = "rx_crc_errors_phy", 2107 }, 2108 { 2109 .dpdk_name = "rx_in_range_len_errors_phy", 2110 .ctr_name = "rx_in_range_len_errors_phy", 2111 }, 2112 { 2113 .dpdk_name = "rx_symbol_err_phy", 2114 .ctr_name = "rx_symbol_err_phy", 2115 }, 2116 { 2117 .dpdk_name = "tx_errors_phy", 2118 .ctr_name = "tx_errors_phy", 2119 }, 2120 { 2121 .dpdk_name = "rx_out_of_buffer", 2122 .ctr_name = "out_of_buffer", 2123 .dev = 1, 2124 }, 2125 { 2126 .dpdk_name = "tx_packets_phy", 2127 .ctr_name = "tx_packets_phy", 2128 }, 2129 { 2130 .dpdk_name = "rx_packets_phy", 2131 .ctr_name = "rx_packets_phy", 2132 }, 2133 { 2134 .dpdk_name = "tx_discards_phy", 2135 .ctr_name = "tx_discards_phy", 2136 }, 2137 { 2138 .dpdk_name = "rx_discards_phy", 2139 .ctr_name = "rx_discards_phy", 2140 }, 2141 { 2142 .dpdk_name = "tx_bytes_phy", 2143 .ctr_name = "tx_bytes_phy", 2144 }, 2145 { 2146 .dpdk_name = "rx_bytes_phy", 2147 .ctr_name = "rx_bytes_phy", 2148 }, 2149 /* Representor only */ 2150 { 2151 .dpdk_name = "rx_packets", 2152 .ctr_name = "vport_rx_packets", 2153 }, 2154 { 2155 .dpdk_name = "rx_bytes", 2156 .ctr_name = "vport_rx_bytes", 2157 }, 2158 { 2159 .dpdk_name = "tx_packets", 2160 .ctr_name = "vport_tx_packets", 2161 }, 2162 { 2163 .dpdk_name = "tx_bytes", 2164 .ctr_name = "vport_tx_bytes", 2165 }, 2166 }; 2167 2168 static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); 2169 2170 /** 2171 * Init the structures to read device counters. 2172 * 2173 * @param dev 2174 * Pointer to Ethernet device. 2175 */ 2176 void 2177 mlx5_os_stats_init(struct rte_eth_dev *dev) 2178 { 2179 struct mlx5_priv *priv = dev->data->dev_private; 2180 struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; 2181 struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl; 2182 unsigned int i; 2183 unsigned int j; 2184 struct ifreq ifr; 2185 struct ethtool_gstrings *strings = NULL; 2186 unsigned int dev_stats_n; 2187 unsigned int str_sz; 2188 int ret; 2189 2190 /* So that it won't aggregate for each init. */ 2191 xstats_ctrl->mlx5_stats_n = 0; 2192 ret = mlx5_os_get_stats_n(dev); 2193 if (ret < 0) { 2194 DRV_LOG(WARNING, "port %u no extended statistics available", 2195 dev->data->port_id); 2196 return; 2197 } 2198 dev_stats_n = ret; 2199 /* Allocate memory to grab stat names and values. */ 2200 str_sz = dev_stats_n * ETH_GSTRING_LEN; 2201 strings = (struct ethtool_gstrings *) 2202 rte_malloc("xstats_strings", 2203 str_sz + sizeof(struct ethtool_gstrings), 0); 2204 if (!strings) { 2205 DRV_LOG(WARNING, "port %u unable to allocate memory for xstats", 2206 dev->data->port_id); 2207 return; 2208 } 2209 strings->cmd = ETHTOOL_GSTRINGS; 2210 strings->string_set = ETH_SS_STATS; 2211 strings->len = dev_stats_n; 2212 ifr.ifr_data = (caddr_t)strings; 2213 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); 2214 if (ret) { 2215 DRV_LOG(WARNING, "port %u unable to get statistic names", 2216 dev->data->port_id); 2217 goto free; 2218 } 2219 for (i = 0; i != dev_stats_n; ++i) { 2220 const char *curr_string = (const char *) 2221 &strings->data[i * ETH_GSTRING_LEN]; 2222 2223 for (j = 0; j != xstats_n; ++j) { 2224 if (!strcmp(mlx5_counters_init[j].ctr_name, 2225 curr_string)) { 2226 unsigned int idx = xstats_ctrl->mlx5_stats_n++; 2227 2228 xstats_ctrl->dev_table_idx[idx] = i; 2229 xstats_ctrl->info[idx] = mlx5_counters_init[j]; 2230 break; 2231 } 2232 } 2233 } 2234 /* Add dev counters. */ 2235 for (i = 0; i != xstats_n; ++i) { 2236 if (mlx5_counters_init[i].dev) { 2237 unsigned int idx = xstats_ctrl->mlx5_stats_n++; 2238 2239 xstats_ctrl->info[idx] = mlx5_counters_init[i]; 2240 xstats_ctrl->hw_stats[idx] = 0; 2241 } 2242 } 2243 MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS); 2244 xstats_ctrl->stats_n = dev_stats_n; 2245 /* Copy to base at first time. */ 2246 ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base); 2247 if (ret) 2248 DRV_LOG(ERR, "port %u cannot read device counters: %s", 2249 dev->data->port_id, strerror(rte_errno)); 2250 mlx5_os_read_dev_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base); 2251 stats_ctrl->imissed = 0; 2252 free: 2253 rte_free(strings); 2254 } 2255 2256 /** 2257 * Set the reg_mr and dereg_mr call backs 2258 * 2259 * @param reg_mr_cb[out] 2260 * Pointer to reg_mr func 2261 * @param dereg_mr_cb[out] 2262 * Pointer to dereg_mr func 2263 * 2264 */ 2265 void 2266 mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, 2267 mlx5_dereg_mr_t *dereg_mr_cb) 2268 { 2269 *reg_mr_cb = mlx5_verbs_ops.reg_mr; 2270 *dereg_mr_cb = mlx5_verbs_ops.dereg_mr; 2271 } 2272 2273 const struct eth_dev_ops mlx5_os_dev_ops = { 2274 .dev_configure = mlx5_dev_configure, 2275 .dev_start = mlx5_dev_start, 2276 .dev_stop = mlx5_dev_stop, 2277 .dev_set_link_down = mlx5_set_link_down, 2278 .dev_set_link_up = mlx5_set_link_up, 2279 .dev_close = mlx5_dev_close, 2280 .promiscuous_enable = mlx5_promiscuous_enable, 2281 .promiscuous_disable = mlx5_promiscuous_disable, 2282 .allmulticast_enable = mlx5_allmulticast_enable, 2283 .allmulticast_disable = mlx5_allmulticast_disable, 2284 .link_update = mlx5_link_update, 2285 .stats_get = mlx5_stats_get, 2286 .stats_reset = mlx5_stats_reset, 2287 .xstats_get = mlx5_xstats_get, 2288 .xstats_reset = mlx5_xstats_reset, 2289 .xstats_get_names = mlx5_xstats_get_names, 2290 .fw_version_get = mlx5_fw_version_get, 2291 .dev_infos_get = mlx5_dev_infos_get, 2292 .read_clock = mlx5_read_clock, 2293 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 2294 .vlan_filter_set = mlx5_vlan_filter_set, 2295 .rx_queue_setup = mlx5_rx_queue_setup, 2296 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 2297 .tx_queue_setup = mlx5_tx_queue_setup, 2298 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 2299 .rx_queue_release = mlx5_rx_queue_release, 2300 .tx_queue_release = mlx5_tx_queue_release, 2301 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 2302 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 2303 .mac_addr_remove = mlx5_mac_addr_remove, 2304 .mac_addr_add = mlx5_mac_addr_add, 2305 .mac_addr_set = mlx5_mac_addr_set, 2306 .set_mc_addr_list = mlx5_set_mc_addr_list, 2307 .mtu_set = mlx5_dev_set_mtu, 2308 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 2309 .vlan_offload_set = mlx5_vlan_offload_set, 2310 .reta_update = mlx5_dev_rss_reta_update, 2311 .reta_query = mlx5_dev_rss_reta_query, 2312 .rss_hash_update = mlx5_rss_hash_update, 2313 .rss_hash_conf_get = mlx5_rss_hash_conf_get, 2314 .filter_ctrl = mlx5_dev_filter_ctrl, 2315 .rx_descriptor_status = mlx5_rx_descriptor_status, 2316 .tx_descriptor_status = mlx5_tx_descriptor_status, 2317 .rxq_info_get = mlx5_rxq_info_get, 2318 .txq_info_get = mlx5_txq_info_get, 2319 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2320 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2321 .rx_queue_count = mlx5_rx_queue_count, 2322 .rx_queue_intr_enable = mlx5_rx_intr_enable, 2323 .rx_queue_intr_disable = mlx5_rx_intr_disable, 2324 .is_removed = mlx5_is_removed, 2325 .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, 2326 .get_module_info = mlx5_get_module_info, 2327 .get_module_eeprom = mlx5_get_module_eeprom, 2328 .hairpin_cap_get = mlx5_hairpin_cap_get, 2329 .mtr_ops_get = mlx5_flow_meter_ops_get, 2330 }; 2331 2332 /* Available operations from secondary process. */ 2333 const struct eth_dev_ops mlx5_os_dev_sec_ops = { 2334 .stats_get = mlx5_stats_get, 2335 .stats_reset = mlx5_stats_reset, 2336 .xstats_get = mlx5_xstats_get, 2337 .xstats_reset = mlx5_xstats_reset, 2338 .xstats_get_names = mlx5_xstats_get_names, 2339 .fw_version_get = mlx5_fw_version_get, 2340 .dev_infos_get = mlx5_dev_infos_get, 2341 .rx_descriptor_status = mlx5_rx_descriptor_status, 2342 .tx_descriptor_status = mlx5_tx_descriptor_status, 2343 .rxq_info_get = mlx5_rxq_info_get, 2344 .txq_info_get = mlx5_txq_info_get, 2345 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2346 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2347 .get_module_info = mlx5_get_module_info, 2348 .get_module_eeprom = mlx5_get_module_eeprom, 2349 }; 2350 2351 /* Available operations in flow isolated mode. */ 2352 const struct eth_dev_ops mlx5_os_dev_ops_isolate = { 2353 .dev_configure = mlx5_dev_configure, 2354 .dev_start = mlx5_dev_start, 2355 .dev_stop = mlx5_dev_stop, 2356 .dev_set_link_down = mlx5_set_link_down, 2357 .dev_set_link_up = mlx5_set_link_up, 2358 .dev_close = mlx5_dev_close, 2359 .promiscuous_enable = mlx5_promiscuous_enable, 2360 .promiscuous_disable = mlx5_promiscuous_disable, 2361 .allmulticast_enable = mlx5_allmulticast_enable, 2362 .allmulticast_disable = mlx5_allmulticast_disable, 2363 .link_update = mlx5_link_update, 2364 .stats_get = mlx5_stats_get, 2365 .stats_reset = mlx5_stats_reset, 2366 .xstats_get = mlx5_xstats_get, 2367 .xstats_reset = mlx5_xstats_reset, 2368 .xstats_get_names = mlx5_xstats_get_names, 2369 .fw_version_get = mlx5_fw_version_get, 2370 .dev_infos_get = mlx5_dev_infos_get, 2371 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 2372 .vlan_filter_set = mlx5_vlan_filter_set, 2373 .rx_queue_setup = mlx5_rx_queue_setup, 2374 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 2375 .tx_queue_setup = mlx5_tx_queue_setup, 2376 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 2377 .rx_queue_release = mlx5_rx_queue_release, 2378 .tx_queue_release = mlx5_tx_queue_release, 2379 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 2380 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 2381 .mac_addr_remove = mlx5_mac_addr_remove, 2382 .mac_addr_add = mlx5_mac_addr_add, 2383 .mac_addr_set = mlx5_mac_addr_set, 2384 .set_mc_addr_list = mlx5_set_mc_addr_list, 2385 .mtu_set = mlx5_dev_set_mtu, 2386 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 2387 .vlan_offload_set = mlx5_vlan_offload_set, 2388 .filter_ctrl = mlx5_dev_filter_ctrl, 2389 .rx_descriptor_status = mlx5_rx_descriptor_status, 2390 .tx_descriptor_status = mlx5_tx_descriptor_status, 2391 .rxq_info_get = mlx5_rxq_info_get, 2392 .txq_info_get = mlx5_txq_info_get, 2393 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2394 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2395 .rx_queue_intr_enable = mlx5_rx_intr_enable, 2396 .rx_queue_intr_disable = mlx5_rx_intr_disable, 2397 .is_removed = mlx5_is_removed, 2398 .get_module_info = mlx5_get_module_info, 2399 .get_module_eeprom = mlx5_get_module_eeprom, 2400 .hairpin_cap_get = mlx5_hairpin_cap_get, 2401 .mtr_ops_get = mlx5_flow_meter_ops_get, 2402 }; 2403