1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <stddef.h> 7 #include <unistd.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <errno.h> 12 #include <fcntl.h> 13 14 #include <rte_malloc.h> 15 #include <ethdev_driver.h> 16 #include <rte_pci.h> 17 #include <bus_pci_driver.h> 18 #include <rte_common.h> 19 #include <rte_kvargs.h> 20 #include <rte_rwlock.h> 21 #include <rte_spinlock.h> 22 #include <rte_string_fns.h> 23 #include <rte_eal_paging.h> 24 #include <rte_alarm.h> 25 #include <rte_cycles.h> 26 #include <rte_interrupts.h> 27 28 #include <mlx5_glue.h> 29 #include <mlx5_devx_cmds.h> 30 #include <mlx5_common.h> 31 #include <mlx5_common_os.h> 32 #include <mlx5_common_mp.h> 33 #include <mlx5_malloc.h> 34 35 #include "mlx5_defs.h" 36 #include "mlx5.h" 37 #include "mlx5_utils.h" 38 #include "mlx5_rxtx.h" 39 #include "mlx5_rx.h" 40 #include "mlx5_tx.h" 41 #include "mlx5_autoconf.h" 42 #include "mlx5_flow.h" 43 #include "mlx5_flow_os.h" 44 #include "rte_pmd_mlx5.h" 45 46 #define MLX5_ETH_DRIVER_NAME mlx5_eth 47 48 /* Device parameter to enable RX completion queue compression. */ 49 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en" 50 51 /* Device parameter to enable padding Rx packet to cacheline size. */ 52 #define MLX5_RXQ_PKT_PAD_EN "rxq_pkt_pad_en" 53 54 /* Device parameter to enable Multi-Packet Rx queue. */ 55 #define MLX5_RX_MPRQ_EN "mprq_en" 56 57 /* Device parameter to configure log 2 of the number of strides for MPRQ. */ 58 #define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num" 59 60 /* Device parameter to configure log 2 of the stride size for MPRQ. */ 61 #define MLX5_RX_MPRQ_LOG_STRIDE_SIZE "mprq_log_stride_size" 62 63 /* Device parameter to limit the size of memcpy'd packet for MPRQ. */ 64 #define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len" 65 66 /* Device parameter to set the minimum number of Rx queues to enable MPRQ. */ 67 #define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq" 68 69 /* Device parameter to configure inline send. Deprecated, ignored.*/ 70 #define MLX5_TXQ_INLINE "txq_inline" 71 72 /* Device parameter to limit packet size to inline with ordinary SEND. */ 73 #define MLX5_TXQ_INLINE_MAX "txq_inline_max" 74 75 /* Device parameter to configure minimal data size to inline. */ 76 #define MLX5_TXQ_INLINE_MIN "txq_inline_min" 77 78 /* Device parameter to limit packet size to inline with Enhanced MPW. */ 79 #define MLX5_TXQ_INLINE_MPW "txq_inline_mpw" 80 81 /* 82 * Device parameter to configure the number of TX queues threshold for 83 * enabling inline send. 84 */ 85 #define MLX5_TXQS_MIN_INLINE "txqs_min_inline" 86 87 /* 88 * Device parameter to configure the number of TX queues threshold for 89 * enabling vectorized Tx, deprecated, ignored (no vectorized Tx routines). 90 */ 91 #define MLX5_TXQS_MAX_VEC "txqs_max_vec" 92 93 /* Device parameter to enable multi-packet send WQEs. */ 94 #define MLX5_TXQ_MPW_EN "txq_mpw_en" 95 96 /* 97 * Device parameter to include 2 dsegs in the title WQEBB. 98 * Deprecated, ignored. 99 */ 100 #define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en" 101 102 /* 103 * Device parameter to limit the size of inlining packet. 104 * Deprecated, ignored. 105 */ 106 #define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len" 107 108 /* 109 * Device parameter to enable Tx scheduling on timestamps 110 * and specify the packet pacing granularity in nanoseconds. 111 */ 112 #define MLX5_TX_PP "tx_pp" 113 114 /* 115 * Device parameter to specify skew in nanoseconds on Tx datapath, 116 * it represents the time between SQ start WQE processing and 117 * appearing actual packet data on the wire. 118 */ 119 #define MLX5_TX_SKEW "tx_skew" 120 121 /* 122 * Device parameter to enable hardware Tx vector. 123 * Deprecated, ignored (no vectorized Tx routines anymore). 124 */ 125 #define MLX5_TX_VEC_EN "tx_vec_en" 126 127 /* Device parameter to enable hardware Rx vector. */ 128 #define MLX5_RX_VEC_EN "rx_vec_en" 129 130 /* Allow L3 VXLAN flow creation. */ 131 #define MLX5_L3_VXLAN_EN "l3_vxlan_en" 132 133 /* Activate DV E-Switch flow steering. */ 134 #define MLX5_DV_ESW_EN "dv_esw_en" 135 136 /* Activate DV flow steering. */ 137 #define MLX5_DV_FLOW_EN "dv_flow_en" 138 139 /* Enable extensive flow metadata support. */ 140 #define MLX5_DV_XMETA_EN "dv_xmeta_en" 141 142 /* Device parameter to let the user manage the lacp traffic of bonded device */ 143 #define MLX5_LACP_BY_USER "lacp_by_user" 144 145 /* Activate Netlink support in VF mode. */ 146 #define MLX5_VF_NL_EN "vf_nl_en" 147 148 /* Select port representors to instantiate. */ 149 #define MLX5_REPRESENTOR "representor" 150 151 /* Device parameter to configure the maximum number of dump files per queue. */ 152 #define MLX5_MAX_DUMP_FILES_NUM "max_dump_files_num" 153 154 /* Configure timeout of LRO session (in microseconds). */ 155 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec" 156 157 /* 158 * Device parameter to configure the total data buffer size for a single 159 * hairpin queue (logarithm value). 160 */ 161 #define MLX5_HP_BUF_SIZE "hp_buf_log_sz" 162 163 /* Flow memory reclaim mode. */ 164 #define MLX5_RECLAIM_MEM "reclaim_mem_mode" 165 166 /* Decap will be used or not. */ 167 #define MLX5_DECAP_EN "decap_en" 168 169 /* Device parameter to configure allow or prevent duplicate rules pattern. */ 170 #define MLX5_ALLOW_DUPLICATE_PATTERN "allow_duplicate_pattern" 171 172 /* Device parameter to configure the delay drop when creating Rxqs. */ 173 #define MLX5_DELAY_DROP "delay_drop" 174 175 /* Device parameter to create the fdb default rule in PMD */ 176 #define MLX5_FDB_DEFAULT_RULE_EN "fdb_def_rule_en" 177 178 /* HW steering counter configuration. */ 179 #define MLX5_HWS_CNT_SERVICE_CORE "service_core" 180 181 /* HW steering counter's query interval. */ 182 #define MLX5_HWS_CNT_CYCLE_TIME "svc_cycle_time" 183 184 /* Device parameter to control representor matching in ingress/egress flows with HWS. */ 185 #define MLX5_REPR_MATCHING_EN "repr_matching_en" 186 187 /* Shared memory between primary and secondary processes. */ 188 struct mlx5_shared_data *mlx5_shared_data; 189 190 /** Driver-specific log messages type. */ 191 int mlx5_logtype; 192 193 static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = 194 LIST_HEAD_INITIALIZER(); 195 static pthread_mutex_t mlx5_dev_ctx_list_mutex; 196 static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { 197 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 198 [MLX5_IPOOL_DECAP_ENCAP] = { 199 .size = sizeof(struct mlx5_flow_dv_encap_decap_resource), 200 .trunk_size = 64, 201 .grow_trunk = 3, 202 .grow_shift = 2, 203 .need_lock = 1, 204 .release_mem_en = 1, 205 .malloc = mlx5_malloc, 206 .free = mlx5_free, 207 .type = "mlx5_encap_decap_ipool", 208 }, 209 [MLX5_IPOOL_PUSH_VLAN] = { 210 .size = sizeof(struct mlx5_flow_dv_push_vlan_action_resource), 211 .trunk_size = 64, 212 .grow_trunk = 3, 213 .grow_shift = 2, 214 .need_lock = 1, 215 .release_mem_en = 1, 216 .malloc = mlx5_malloc, 217 .free = mlx5_free, 218 .type = "mlx5_push_vlan_ipool", 219 }, 220 [MLX5_IPOOL_TAG] = { 221 .size = sizeof(struct mlx5_flow_dv_tag_resource), 222 .trunk_size = 64, 223 .grow_trunk = 3, 224 .grow_shift = 2, 225 .need_lock = 1, 226 .release_mem_en = 0, 227 .per_core_cache = (1 << 16), 228 .malloc = mlx5_malloc, 229 .free = mlx5_free, 230 .type = "mlx5_tag_ipool", 231 }, 232 [MLX5_IPOOL_PORT_ID] = { 233 .size = sizeof(struct mlx5_flow_dv_port_id_action_resource), 234 .trunk_size = 64, 235 .grow_trunk = 3, 236 .grow_shift = 2, 237 .need_lock = 1, 238 .release_mem_en = 1, 239 .malloc = mlx5_malloc, 240 .free = mlx5_free, 241 .type = "mlx5_port_id_ipool", 242 }, 243 [MLX5_IPOOL_JUMP] = { 244 .size = sizeof(struct mlx5_flow_tbl_data_entry), 245 .trunk_size = 64, 246 .grow_trunk = 3, 247 .grow_shift = 2, 248 .need_lock = 1, 249 .release_mem_en = 1, 250 .malloc = mlx5_malloc, 251 .free = mlx5_free, 252 .type = "mlx5_jump_ipool", 253 }, 254 [MLX5_IPOOL_SAMPLE] = { 255 .size = sizeof(struct mlx5_flow_dv_sample_resource), 256 .trunk_size = 64, 257 .grow_trunk = 3, 258 .grow_shift = 2, 259 .need_lock = 1, 260 .release_mem_en = 1, 261 .malloc = mlx5_malloc, 262 .free = mlx5_free, 263 .type = "mlx5_sample_ipool", 264 }, 265 [MLX5_IPOOL_DEST_ARRAY] = { 266 .size = sizeof(struct mlx5_flow_dv_dest_array_resource), 267 .trunk_size = 64, 268 .grow_trunk = 3, 269 .grow_shift = 2, 270 .need_lock = 1, 271 .release_mem_en = 1, 272 .malloc = mlx5_malloc, 273 .free = mlx5_free, 274 .type = "mlx5_dest_array_ipool", 275 }, 276 [MLX5_IPOOL_TUNNEL_ID] = { 277 .size = sizeof(struct mlx5_flow_tunnel), 278 .trunk_size = MLX5_MAX_TUNNELS, 279 .need_lock = 1, 280 .release_mem_en = 1, 281 .type = "mlx5_tunnel_offload", 282 }, 283 [MLX5_IPOOL_TNL_TBL_ID] = { 284 .size = 0, 285 .need_lock = 1, 286 .type = "mlx5_flow_tnl_tbl_ipool", 287 }, 288 #endif 289 [MLX5_IPOOL_MTR] = { 290 /** 291 * The ipool index should grow continually from small to big, 292 * for meter idx, so not set grow_trunk to avoid meter index 293 * not jump continually. 294 */ 295 .size = sizeof(struct mlx5_legacy_flow_meter), 296 .trunk_size = 64, 297 .need_lock = 1, 298 .release_mem_en = 1, 299 .malloc = mlx5_malloc, 300 .free = mlx5_free, 301 .type = "mlx5_meter_ipool", 302 }, 303 [MLX5_IPOOL_MCP] = { 304 .size = sizeof(struct mlx5_flow_mreg_copy_resource), 305 .trunk_size = 64, 306 .grow_trunk = 3, 307 .grow_shift = 2, 308 .need_lock = 1, 309 .release_mem_en = 1, 310 .malloc = mlx5_malloc, 311 .free = mlx5_free, 312 .type = "mlx5_mcp_ipool", 313 }, 314 [MLX5_IPOOL_HRXQ] = { 315 .size = (sizeof(struct mlx5_hrxq) + MLX5_RSS_HASH_KEY_LEN), 316 .trunk_size = 64, 317 .grow_trunk = 3, 318 .grow_shift = 2, 319 .need_lock = 1, 320 .release_mem_en = 1, 321 .malloc = mlx5_malloc, 322 .free = mlx5_free, 323 .type = "mlx5_hrxq_ipool", 324 }, 325 [MLX5_IPOOL_MLX5_FLOW] = { 326 /* 327 * MLX5_IPOOL_MLX5_FLOW size varies for DV and VERBS flows. 328 * It set in run time according to PCI function configuration. 329 */ 330 .size = 0, 331 .trunk_size = 64, 332 .grow_trunk = 3, 333 .grow_shift = 2, 334 .need_lock = 1, 335 .release_mem_en = 0, 336 .per_core_cache = 1 << 19, 337 .malloc = mlx5_malloc, 338 .free = mlx5_free, 339 .type = "mlx5_flow_handle_ipool", 340 }, 341 [MLX5_IPOOL_RTE_FLOW] = { 342 .size = sizeof(struct rte_flow), 343 .trunk_size = 4096, 344 .need_lock = 1, 345 .release_mem_en = 1, 346 .malloc = mlx5_malloc, 347 .free = mlx5_free, 348 .type = "rte_flow_ipool", 349 }, 350 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID] = { 351 .size = 0, 352 .need_lock = 1, 353 .type = "mlx5_flow_rss_id_ipool", 354 }, 355 [MLX5_IPOOL_RSS_SHARED_ACTIONS] = { 356 .size = sizeof(struct mlx5_shared_action_rss), 357 .trunk_size = 64, 358 .grow_trunk = 3, 359 .grow_shift = 2, 360 .need_lock = 1, 361 .release_mem_en = 1, 362 .malloc = mlx5_malloc, 363 .free = mlx5_free, 364 .type = "mlx5_shared_action_rss", 365 }, 366 [MLX5_IPOOL_MTR_POLICY] = { 367 /** 368 * The ipool index should grow continually from small to big, 369 * for policy idx, so not set grow_trunk to avoid policy index 370 * not jump continually. 371 */ 372 .size = sizeof(struct mlx5_flow_meter_sub_policy), 373 .trunk_size = 64, 374 .need_lock = 1, 375 .release_mem_en = 1, 376 .malloc = mlx5_malloc, 377 .free = mlx5_free, 378 .type = "mlx5_meter_policy_ipool", 379 }, 380 }; 381 382 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512 383 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16 384 385 #define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024 386 387 #define MLX5_RXQ_ENH_CQE_COMP_MASK 0x80 388 389 /** 390 * Decide whether representor ID is a HPF(host PF) port on BF2. 391 * 392 * @param dev 393 * Pointer to Ethernet device structure. 394 * 395 * @return 396 * Non-zero if HPF, otherwise 0. 397 */ 398 bool 399 mlx5_is_hpf(struct rte_eth_dev *dev) 400 { 401 struct mlx5_priv *priv = dev->data->dev_private; 402 uint16_t repr = MLX5_REPRESENTOR_REPR(priv->representor_id); 403 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 404 405 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_VF && 406 MLX5_REPRESENTOR_REPR(-1) == repr; 407 } 408 409 /** 410 * Decide whether representor ID is a SF port representor. 411 * 412 * @param dev 413 * Pointer to Ethernet device structure. 414 * 415 * @return 416 * Non-zero if HPF, otherwise 0. 417 */ 418 bool 419 mlx5_is_sf_repr(struct rte_eth_dev *dev) 420 { 421 struct mlx5_priv *priv = dev->data->dev_private; 422 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 423 424 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_SF; 425 } 426 427 /** 428 * Initialize the ASO aging management structure. 429 * 430 * @param[in] sh 431 * Pointer to mlx5_dev_ctx_shared object to free 432 * 433 * @return 434 * 0 on success, a negative errno value otherwise and rte_errno is set. 435 */ 436 int 437 mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh) 438 { 439 int err; 440 441 if (sh->aso_age_mng) 442 return 0; 443 sh->aso_age_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->aso_age_mng), 444 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 445 if (!sh->aso_age_mng) { 446 DRV_LOG(ERR, "aso_age_mng allocation was failed."); 447 rte_errno = ENOMEM; 448 return -ENOMEM; 449 } 450 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_FLOW_HIT, 1); 451 if (err) { 452 mlx5_free(sh->aso_age_mng); 453 return -1; 454 } 455 rte_rwlock_init(&sh->aso_age_mng->resize_rwl); 456 rte_spinlock_init(&sh->aso_age_mng->free_sl); 457 LIST_INIT(&sh->aso_age_mng->free); 458 return 0; 459 } 460 461 /** 462 * Close and release all the resources of the ASO aging management structure. 463 * 464 * @param[in] sh 465 * Pointer to mlx5_dev_ctx_shared object to free. 466 */ 467 static void 468 mlx5_flow_aso_age_mng_close(struct mlx5_dev_ctx_shared *sh) 469 { 470 int i, j; 471 472 mlx5_aso_flow_hit_queue_poll_stop(sh); 473 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_FLOW_HIT); 474 if (sh->aso_age_mng->pools) { 475 struct mlx5_aso_age_pool *pool; 476 477 for (i = 0; i < sh->aso_age_mng->next; ++i) { 478 pool = sh->aso_age_mng->pools[i]; 479 claim_zero(mlx5_devx_cmd_destroy 480 (pool->flow_hit_aso_obj)); 481 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) 482 if (pool->actions[j].dr_action) 483 claim_zero 484 (mlx5_flow_os_destroy_flow_action 485 (pool->actions[j].dr_action)); 486 mlx5_free(pool); 487 } 488 mlx5_free(sh->aso_age_mng->pools); 489 } 490 mlx5_free(sh->aso_age_mng); 491 } 492 493 /** 494 * Initialize the shared aging list information per port. 495 * 496 * @param[in] sh 497 * Pointer to mlx5_dev_ctx_shared object. 498 */ 499 static void 500 mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh) 501 { 502 uint32_t i; 503 struct mlx5_age_info *age_info; 504 505 /* 506 * In HW steering, aging information structure is initialized later 507 * during configure function. 508 */ 509 if (sh->config.dv_flow_en == 2) 510 return; 511 for (i = 0; i < sh->max_port; i++) { 512 age_info = &sh->port[i].age_info; 513 age_info->flags = 0; 514 TAILQ_INIT(&age_info->aged_counters); 515 LIST_INIT(&age_info->aged_aso); 516 rte_spinlock_init(&age_info->aged_sl); 517 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 518 } 519 } 520 521 /** 522 * DV flow counter mode detect and config. 523 * 524 * @param dev 525 * Pointer to rte_eth_dev structure. 526 * 527 */ 528 void 529 mlx5_flow_counter_mode_config(struct rte_eth_dev *dev __rte_unused) 530 { 531 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 532 struct mlx5_priv *priv = dev->data->dev_private; 533 struct mlx5_dev_ctx_shared *sh = priv->sh; 534 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 535 bool fallback; 536 537 #ifndef HAVE_IBV_DEVX_ASYNC 538 fallback = true; 539 #else 540 fallback = false; 541 if (!sh->cdev->config.devx || !sh->config.dv_flow_en || 542 !hca_attr->flow_counters_dump || 543 !(hca_attr->flow_counter_bulk_alloc_bitmap & 0x4) || 544 (mlx5_flow_dv_discover_counter_offset_support(dev) == -ENOTSUP)) 545 fallback = true; 546 #endif 547 if (fallback) 548 DRV_LOG(INFO, "Use fall-back DV counter management. Flow " 549 "counter dump:%d, bulk_alloc_bitmap:0x%hhx.", 550 hca_attr->flow_counters_dump, 551 hca_attr->flow_counter_bulk_alloc_bitmap); 552 /* Initialize fallback mode only on the port initializes sh. */ 553 if (sh->refcnt == 1) 554 sh->sws_cmng.counter_fallback = fallback; 555 else if (fallback != sh->sws_cmng.counter_fallback) 556 DRV_LOG(WARNING, "Port %d in sh has different fallback mode " 557 "with others:%d.", PORT_ID(priv), fallback); 558 #endif 559 } 560 561 /** 562 * Initialize the counters management structure. 563 * 564 * @param[in] sh 565 * Pointer to mlx5_dev_ctx_shared object to free 566 * 567 * @return 568 * 0 on success, otherwise negative errno value and rte_errno is set. 569 */ 570 static int 571 mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh) 572 { 573 int i, j; 574 575 if (sh->config.dv_flow_en < 2) { 576 void *pools; 577 578 pools = mlx5_malloc(MLX5_MEM_ZERO, 579 sizeof(struct mlx5_flow_counter_pool *) * 580 MLX5_COUNTER_POOLS_MAX_NUM, 581 0, SOCKET_ID_ANY); 582 if (!pools) { 583 DRV_LOG(ERR, 584 "Counter management allocation was failed."); 585 rte_errno = ENOMEM; 586 return -rte_errno; 587 } 588 memset(&sh->sws_cmng, 0, sizeof(sh->sws_cmng)); 589 TAILQ_INIT(&sh->sws_cmng.flow_counters); 590 sh->sws_cmng.min_id = MLX5_CNT_BATCH_OFFSET; 591 sh->sws_cmng.max_id = -1; 592 sh->sws_cmng.last_pool_idx = POOL_IDX_INVALID; 593 sh->sws_cmng.pools = pools; 594 rte_spinlock_init(&sh->sws_cmng.pool_update_sl); 595 for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) { 596 TAILQ_INIT(&sh->sws_cmng.counters[i]); 597 rte_spinlock_init(&sh->sws_cmng.csl[i]); 598 } 599 } else { 600 struct mlx5_hca_attr *attr = &sh->cdev->config.hca_attr; 601 uint32_t fw_max_nb_cnts = attr->max_flow_counter; 602 uint8_t log_dcs = log2above(fw_max_nb_cnts) - 1; 603 uint32_t max_nb_cnts = 0; 604 605 for (i = 0, j = 0; j < MLX5_HWS_CNT_DCS_NUM; ++i) { 606 int log_dcs_i = log_dcs - i; 607 608 if (log_dcs_i < 0) 609 break; 610 if ((max_nb_cnts | RTE_BIT32(log_dcs_i)) > 611 fw_max_nb_cnts) 612 continue; 613 max_nb_cnts |= RTE_BIT32(log_dcs_i); 614 j++; 615 } 616 sh->hws_max_log_bulk_sz = log_dcs; 617 sh->hws_max_nb_counters = max_nb_cnts; 618 } 619 return 0; 620 } 621 622 /** 623 * Destroy all the resources allocated for a counter memory management. 624 * 625 * @param[in] mng 626 * Pointer to the memory management structure. 627 */ 628 static void 629 mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng) 630 { 631 uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data; 632 633 LIST_REMOVE(mng, next); 634 mlx5_os_wrapped_mkey_destroy(&mng->wm); 635 mlx5_free(mem); 636 } 637 638 /** 639 * Close and release all the resources of the counters management. 640 * 641 * @param[in] sh 642 * Pointer to mlx5_dev_ctx_shared object to free. 643 */ 644 static void 645 mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) 646 { 647 struct mlx5_counter_stats_mem_mng *mng; 648 int i, j; 649 int retries = 1024; 650 651 rte_errno = 0; 652 while (--retries) { 653 rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh); 654 if (rte_errno != EINPROGRESS) 655 break; 656 rte_pause(); 657 } 658 659 if (sh->sws_cmng.pools) { 660 struct mlx5_flow_counter_pool *pool; 661 uint16_t n_valid = sh->sws_cmng.n_valid; 662 bool fallback = sh->sws_cmng.counter_fallback; 663 664 for (i = 0; i < n_valid; ++i) { 665 pool = sh->sws_cmng.pools[i]; 666 if (!fallback && pool->min_dcs) 667 claim_zero(mlx5_devx_cmd_destroy 668 (pool->min_dcs)); 669 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) { 670 struct mlx5_flow_counter *cnt = 671 MLX5_POOL_GET_CNT(pool, j); 672 673 if (cnt->action) 674 claim_zero 675 (mlx5_flow_os_destroy_flow_action 676 (cnt->action)); 677 if (fallback && cnt->dcs_when_free) 678 claim_zero(mlx5_devx_cmd_destroy 679 (cnt->dcs_when_free)); 680 } 681 mlx5_free(pool); 682 } 683 mlx5_free(sh->sws_cmng.pools); 684 } 685 mng = LIST_FIRST(&sh->sws_cmng.mem_mngs); 686 while (mng) { 687 mlx5_flow_destroy_counter_stat_mem_mng(mng); 688 mng = LIST_FIRST(&sh->sws_cmng.mem_mngs); 689 } 690 memset(&sh->sws_cmng, 0, sizeof(sh->sws_cmng)); 691 } 692 693 /** 694 * Initialize the aso flow meters management structure. 695 * 696 * @param[in] sh 697 * Pointer to mlx5_dev_ctx_shared object to free 698 */ 699 int 700 mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh) 701 { 702 if (!sh->mtrmng) { 703 sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, 704 sizeof(*sh->mtrmng), 705 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 706 if (!sh->mtrmng) { 707 DRV_LOG(ERR, 708 "meter management allocation was failed."); 709 rte_errno = ENOMEM; 710 return -ENOMEM; 711 } 712 if (sh->meter_aso_en) { 713 rte_spinlock_init(&sh->mtrmng->pools_mng.mtrsl); 714 rte_rwlock_init(&sh->mtrmng->pools_mng.resize_mtrwl); 715 LIST_INIT(&sh->mtrmng->pools_mng.meters); 716 } 717 sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 718 } 719 return 0; 720 } 721 722 /** 723 * Close and release all the resources of 724 * the ASO flow meter management structure. 725 * 726 * @param[in] sh 727 * Pointer to mlx5_dev_ctx_shared object to free. 728 */ 729 static void 730 mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) 731 { 732 struct mlx5_aso_mtr_pool *mtr_pool; 733 struct mlx5_flow_mtr_mng *mtrmng = sh->mtrmng; 734 uint32_t idx; 735 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 736 struct mlx5_aso_mtr *aso_mtr; 737 int i; 738 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 739 740 if (sh->meter_aso_en) { 741 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); 742 idx = mtrmng->pools_mng.n_valid; 743 while (idx--) { 744 mtr_pool = mtrmng->pools_mng.pools[idx]; 745 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 746 for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { 747 aso_mtr = &mtr_pool->mtrs[i]; 748 if (aso_mtr->fm.meter_action_g) 749 claim_zero 750 (mlx5_glue->destroy_flow_action 751 (aso_mtr->fm.meter_action_g)); 752 if (aso_mtr->fm.meter_action_y) 753 claim_zero 754 (mlx5_glue->destroy_flow_action 755 (aso_mtr->fm.meter_action_y)); 756 } 757 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 758 claim_zero(mlx5_devx_cmd_destroy 759 (mtr_pool->devx_obj)); 760 mtrmng->pools_mng.n_valid--; 761 mlx5_free(mtr_pool); 762 } 763 mlx5_free(sh->mtrmng->pools_mng.pools); 764 } 765 mlx5_free(sh->mtrmng); 766 sh->mtrmng = NULL; 767 } 768 769 /* Send FLOW_AGED event if needed. */ 770 void 771 mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) 772 { 773 struct mlx5_age_info *age_info; 774 uint32_t i; 775 776 for (i = 0; i < sh->max_port; i++) { 777 age_info = &sh->port[i].age_info; 778 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 779 continue; 780 MLX5_AGE_UNSET(age_info, MLX5_AGE_EVENT_NEW); 781 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) { 782 MLX5_AGE_UNSET(age_info, MLX5_AGE_TRIGGER); 783 rte_eth_dev_callback_process 784 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 785 RTE_ETH_EVENT_FLOW_AGED, NULL); 786 } 787 } 788 } 789 790 /* 791 * Initialize the ASO connection tracking structure. 792 * 793 * @param[in] sh 794 * Pointer to mlx5_dev_ctx_shared object. 795 * 796 * @return 797 * 0 on success, a negative errno value otherwise and rte_errno is set. 798 */ 799 int 800 mlx5_flow_aso_ct_mng_init(struct mlx5_dev_ctx_shared *sh) 801 { 802 int err; 803 804 if (sh->ct_mng) 805 return 0; 806 sh->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->ct_mng) + 807 sizeof(struct mlx5_aso_sq) * MLX5_ASO_CT_SQ_NUM, 808 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 809 if (!sh->ct_mng) { 810 DRV_LOG(ERR, "ASO CT management allocation failed."); 811 rte_errno = ENOMEM; 812 return -rte_errno; 813 } 814 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_CONNECTION_TRACKING, MLX5_ASO_CT_SQ_NUM); 815 if (err) { 816 mlx5_free(sh->ct_mng); 817 /* rte_errno should be extracted from the failure. */ 818 rte_errno = EINVAL; 819 return -rte_errno; 820 } 821 rte_spinlock_init(&sh->ct_mng->ct_sl); 822 rte_rwlock_init(&sh->ct_mng->resize_rwl); 823 LIST_INIT(&sh->ct_mng->free_cts); 824 return 0; 825 } 826 827 /* 828 * Close and release all the resources of the 829 * ASO connection tracking management structure. 830 * 831 * @param[in] sh 832 * Pointer to mlx5_dev_ctx_shared object to free. 833 */ 834 static void 835 mlx5_flow_aso_ct_mng_close(struct mlx5_dev_ctx_shared *sh) 836 { 837 struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng; 838 struct mlx5_aso_ct_pool *ct_pool; 839 struct mlx5_aso_ct_action *ct; 840 uint32_t idx; 841 uint32_t val; 842 uint32_t cnt; 843 int i; 844 845 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_CONNECTION_TRACKING); 846 idx = mng->next; 847 while (idx--) { 848 cnt = 0; 849 ct_pool = mng->pools[idx]; 850 for (i = 0; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) { 851 ct = &ct_pool->actions[i]; 852 val = __atomic_fetch_sub(&ct->refcnt, 1, 853 __ATOMIC_RELAXED); 854 MLX5_ASSERT(val == 1); 855 if (val > 1) 856 cnt++; 857 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT 858 if (ct->dr_action_orig) 859 claim_zero(mlx5_glue->destroy_flow_action 860 (ct->dr_action_orig)); 861 if (ct->dr_action_rply) 862 claim_zero(mlx5_glue->destroy_flow_action 863 (ct->dr_action_rply)); 864 #endif 865 } 866 claim_zero(mlx5_devx_cmd_destroy(ct_pool->devx_obj)); 867 if (cnt) { 868 DRV_LOG(DEBUG, "%u ASO CT objects are being used in the pool %u", 869 cnt, i); 870 } 871 mlx5_free(ct_pool); 872 /* in case of failure. */ 873 mng->next--; 874 } 875 mlx5_free(mng->pools); 876 mlx5_free(mng); 877 /* Management structure must be cleared to 0s during allocation. */ 878 sh->ct_mng = NULL; 879 } 880 881 /** 882 * Initialize the flow resources' indexed mempool. 883 * 884 * @param[in] sh 885 * Pointer to mlx5_dev_ctx_shared object. 886 */ 887 static void 888 mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh) 889 { 890 uint8_t i; 891 struct mlx5_indexed_pool_config cfg; 892 893 for (i = 0; i < MLX5_IPOOL_MAX; ++i) { 894 cfg = mlx5_ipool_cfg[i]; 895 switch (i) { 896 default: 897 break; 898 /* 899 * Set MLX5_IPOOL_MLX5_FLOW ipool size 900 * according to PCI function flow configuration. 901 */ 902 case MLX5_IPOOL_MLX5_FLOW: 903 cfg.size = sh->config.dv_flow_en ? 904 sizeof(struct mlx5_flow_handle) : 905 MLX5_FLOW_HANDLE_VERBS_SIZE; 906 break; 907 } 908 if (sh->config.reclaim_mode) { 909 cfg.release_mem_en = 1; 910 cfg.per_core_cache = 0; 911 } else { 912 cfg.release_mem_en = 0; 913 } 914 sh->ipool[i] = mlx5_ipool_create(&cfg); 915 } 916 } 917 918 919 /** 920 * Release the flow resources' indexed mempool. 921 * 922 * @param[in] sh 923 * Pointer to mlx5_dev_ctx_shared object. 924 */ 925 static void 926 mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh) 927 { 928 uint8_t i; 929 930 for (i = 0; i < MLX5_IPOOL_MAX; ++i) 931 mlx5_ipool_destroy(sh->ipool[i]); 932 for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i) 933 if (sh->mdh_ipools[i]) 934 mlx5_ipool_destroy(sh->mdh_ipools[i]); 935 } 936 937 /* 938 * Check if dynamic flex parser for eCPRI already exists. 939 * 940 * @param dev 941 * Pointer to Ethernet device structure. 942 * 943 * @return 944 * true on exists, false on not. 945 */ 946 bool 947 mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev) 948 { 949 struct mlx5_priv *priv = dev->data->dev_private; 950 struct mlx5_ecpri_parser_profile *prf = &priv->sh->ecpri_parser; 951 952 return !!prf->obj; 953 } 954 955 /* 956 * Allocation of a flex parser for eCPRI. Once created, this parser related 957 * resources will be held until the device is closed. 958 * 959 * @param dev 960 * Pointer to Ethernet device structure. 961 * 962 * @return 963 * 0 on success, a negative errno value otherwise and rte_errno is set. 964 */ 965 int 966 mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev) 967 { 968 struct mlx5_priv *priv = dev->data->dev_private; 969 struct mlx5_hca_flex_attr *attr = &priv->sh->cdev->config.hca_attr.flex; 970 struct mlx5_ecpri_parser_profile *prf = &priv->sh->ecpri_parser; 971 struct mlx5_devx_graph_node_attr node = { 972 .modify_field_select = 0, 973 }; 974 struct mlx5_ext_sample_id ids[8]; 975 int ret; 976 977 if (!priv->sh->cdev->config.hca_attr.parse_graph_flex_node) { 978 DRV_LOG(ERR, "Dynamic flex parser is not supported " 979 "for device %s.", priv->dev_data->name); 980 return -ENOTSUP; 981 } 982 node.header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED; 983 /* 8 bytes now: 4B common header + 4B message body header. */ 984 node.header_length_base_value = 0x8; 985 /* After MAC layer: Ether / VLAN. */ 986 node.in[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_MAC; 987 /* Type of compared condition should be 0xAEFE in the L2 layer. */ 988 node.in[0].compare_condition_value = RTE_ETHER_TYPE_ECPRI; 989 /* Sample #0: type in common header. */ 990 node.sample[0].flow_match_sample_en = 1; 991 /* Fixed offset. */ 992 node.sample[0].flow_match_sample_offset_mode = 0x0; 993 /* Only the 2nd byte will be used. */ 994 node.sample[0].flow_match_sample_field_base_offset = 0x0; 995 /* Sample #1: message payload. */ 996 node.sample[1].flow_match_sample_en = 1; 997 /* Fixed offset. */ 998 node.sample[1].flow_match_sample_offset_mode = 0x0; 999 /* 1000 * Only the first two bytes will be used right now, and its offset will 1001 * start after the common header that with the length of a DW(u32). 1002 */ 1003 node.sample[1].flow_match_sample_field_base_offset = sizeof(uint32_t); 1004 prf->obj = mlx5_devx_cmd_create_flex_parser(priv->sh->cdev->ctx, &node); 1005 if (!prf->obj) { 1006 DRV_LOG(ERR, "Failed to create flex parser node object."); 1007 return (rte_errno == 0) ? -ENODEV : -rte_errno; 1008 } 1009 prf->num = 2; 1010 ret = mlx5_devx_cmd_query_parse_samples(prf->obj, ids, prf->num, NULL); 1011 if (ret) { 1012 DRV_LOG(ERR, "Failed to query sample IDs."); 1013 return (rte_errno == 0) ? -ENODEV : -rte_errno; 1014 } 1015 prf->offset[0] = 0x0; 1016 prf->offset[1] = sizeof(uint32_t); 1017 if (attr->ext_sample_id) { 1018 prf->ids[0] = ids[0].sample_id; 1019 prf->ids[1] = ids[1].sample_id; 1020 } else { 1021 prf->ids[0] = ids[0].id; 1022 prf->ids[1] = ids[1].id; 1023 } 1024 return 0; 1025 } 1026 1027 /* 1028 * Destroy the flex parser node, including the parser itself, input / output 1029 * arcs and DW samples. Resources could be reused then. 1030 * 1031 * @param dev 1032 * Pointer to Ethernet device structure. 1033 */ 1034 static void 1035 mlx5_flex_parser_ecpri_release(struct rte_eth_dev *dev) 1036 { 1037 struct mlx5_priv *priv = dev->data->dev_private; 1038 struct mlx5_ecpri_parser_profile *prf = &priv->sh->ecpri_parser; 1039 1040 if (prf->obj) 1041 mlx5_devx_cmd_destroy(prf->obj); 1042 prf->obj = NULL; 1043 } 1044 1045 /* 1046 * Allocation of a flex parser for srh. Once refcnt is zero, the resources held 1047 * by this parser will be freed. 1048 * @param dev 1049 * Pointer to Ethernet device structure. 1050 * 1051 * @return 1052 * 0 on success, a negative errno value otherwise and rte_errno is set. 1053 */ 1054 int 1055 mlx5_alloc_srh_flex_parser(struct rte_eth_dev *dev) 1056 { 1057 struct mlx5_devx_graph_node_attr node = { 1058 .modify_field_select = 0, 1059 }; 1060 struct mlx5_ext_sample_id ids[MLX5_GRAPH_NODE_SAMPLE_NUM]; 1061 struct mlx5_priv *priv = dev->data->dev_private; 1062 struct mlx5_common_dev_config *config = &priv->sh->cdev->config; 1063 void *ibv_ctx = priv->sh->cdev->ctx; 1064 int ret; 1065 1066 memset(ids, 0xff, sizeof(ids)); 1067 if (!config->hca_attr.parse_graph_flex_node) { 1068 DRV_LOG(ERR, "Dynamic flex parser is not supported"); 1069 return -ENOTSUP; 1070 } 1071 if (__atomic_add_fetch(&priv->sh->srh_flex_parser.refcnt, 1, __ATOMIC_RELAXED) > 1) 1072 return 0; 1073 1074 node.header_length_mode = MLX5_GRAPH_NODE_LEN_FIELD; 1075 /* Srv6 first two DW are not counted in. */ 1076 node.header_length_base_value = 0x8; 1077 /* The unit is uint64_t. */ 1078 node.header_length_field_shift = 0x3; 1079 /* Header length is the 2nd byte. */ 1080 node.header_length_field_offset = 0x8; 1081 node.header_length_field_mask = 0xF; 1082 /* One byte next header protocol. */ 1083 node.next_header_field_size = 0x8; 1084 node.in[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_IP; 1085 node.in[0].compare_condition_value = IPPROTO_ROUTING; 1086 node.sample[0].flow_match_sample_en = 1; 1087 /* First come first serve no matter inner or outer. */ 1088 node.sample[0].flow_match_sample_tunnel_mode = MLX5_GRAPH_SAMPLE_TUNNEL_FIRST; 1089 node.out[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_TCP; 1090 node.out[0].compare_condition_value = IPPROTO_TCP; 1091 node.out[1].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_UDP; 1092 node.out[1].compare_condition_value = IPPROTO_UDP; 1093 node.out[2].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_IPV6; 1094 node.out[2].compare_condition_value = IPPROTO_IPV6; 1095 priv->sh->srh_flex_parser.fp = mlx5_devx_cmd_create_flex_parser(ibv_ctx, &node); 1096 if (!priv->sh->srh_flex_parser.fp) { 1097 DRV_LOG(ERR, "Failed to create flex parser node object."); 1098 return (rte_errno == 0) ? -ENODEV : -rte_errno; 1099 } 1100 priv->sh->srh_flex_parser.num = 1; 1101 ret = mlx5_devx_cmd_query_parse_samples(priv->sh->srh_flex_parser.fp, ids, 1102 priv->sh->srh_flex_parser.num, 1103 &priv->sh->srh_flex_parser.anchor_id); 1104 if (ret) { 1105 DRV_LOG(ERR, "Failed to query sample IDs."); 1106 return (rte_errno == 0) ? -ENODEV : -rte_errno; 1107 } 1108 priv->sh->srh_flex_parser.offset[0] = 0x0; 1109 priv->sh->srh_flex_parser.ids[0].id = ids[0].id; 1110 return 0; 1111 } 1112 1113 /* 1114 * Destroy the flex parser node, including the parser itself, input / output 1115 * arcs and DW samples. Resources could be reused then. 1116 * 1117 * @param dev 1118 * Pointer to Ethernet device structure 1119 */ 1120 void 1121 mlx5_free_srh_flex_parser(struct rte_eth_dev *dev) 1122 { 1123 struct mlx5_priv *priv = dev->data->dev_private; 1124 struct mlx5_internal_flex_parser_profile *fp = &priv->sh->srh_flex_parser; 1125 1126 if (__atomic_sub_fetch(&fp->refcnt, 1, __ATOMIC_RELAXED)) 1127 return; 1128 if (fp->fp) 1129 mlx5_devx_cmd_destroy(fp->fp); 1130 fp->fp = NULL; 1131 fp->num = 0; 1132 } 1133 1134 uint32_t 1135 mlx5_get_supported_sw_parsing_offloads(const struct mlx5_hca_attr *attr) 1136 { 1137 uint32_t sw_parsing_offloads = 0; 1138 1139 if (attr->swp) { 1140 sw_parsing_offloads |= MLX5_SW_PARSING_CAP; 1141 if (attr->swp_csum) 1142 sw_parsing_offloads |= MLX5_SW_PARSING_CSUM_CAP; 1143 1144 if (attr->swp_lso) 1145 sw_parsing_offloads |= MLX5_SW_PARSING_TSO_CAP; 1146 } 1147 return sw_parsing_offloads; 1148 } 1149 1150 uint32_t 1151 mlx5_get_supported_tunneling_offloads(const struct mlx5_hca_attr *attr) 1152 { 1153 uint32_t tn_offloads = 0; 1154 1155 if (attr->tunnel_stateless_vxlan) 1156 tn_offloads |= MLX5_TUNNELED_OFFLOADS_VXLAN_CAP; 1157 if (attr->tunnel_stateless_gre) 1158 tn_offloads |= MLX5_TUNNELED_OFFLOADS_GRE_CAP; 1159 if (attr->tunnel_stateless_geneve_rx) 1160 tn_offloads |= MLX5_TUNNELED_OFFLOADS_GENEVE_CAP; 1161 return tn_offloads; 1162 } 1163 1164 /* Fill all fields of UAR structure. */ 1165 static int 1166 mlx5_rxtx_uars_prepare(struct mlx5_dev_ctx_shared *sh) 1167 { 1168 int ret; 1169 1170 ret = mlx5_devx_uar_prepare(sh->cdev, &sh->tx_uar); 1171 if (ret) { 1172 DRV_LOG(ERR, "Failed to prepare Tx DevX UAR."); 1173 return -rte_errno; 1174 } 1175 MLX5_ASSERT(sh->tx_uar.obj); 1176 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->tx_uar.obj)); 1177 ret = mlx5_devx_uar_prepare(sh->cdev, &sh->rx_uar); 1178 if (ret) { 1179 DRV_LOG(ERR, "Failed to prepare Rx DevX UAR."); 1180 mlx5_devx_uar_release(&sh->tx_uar); 1181 return -rte_errno; 1182 } 1183 MLX5_ASSERT(sh->rx_uar.obj); 1184 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->rx_uar.obj)); 1185 return 0; 1186 } 1187 1188 static void 1189 mlx5_rxtx_uars_release(struct mlx5_dev_ctx_shared *sh) 1190 { 1191 mlx5_devx_uar_release(&sh->rx_uar); 1192 mlx5_devx_uar_release(&sh->tx_uar); 1193 } 1194 1195 /** 1196 * rte_mempool_walk() callback to unregister Rx mempools. 1197 * It used when implicit mempool registration is disabled. 1198 * 1199 * @param mp 1200 * The mempool being walked. 1201 * @param arg 1202 * Pointer to the device shared context. 1203 */ 1204 static void 1205 mlx5_dev_ctx_shared_rx_mempool_unregister_cb(struct rte_mempool *mp, void *arg) 1206 { 1207 struct mlx5_dev_ctx_shared *sh = arg; 1208 1209 mlx5_dev_mempool_unregister(sh->cdev, mp); 1210 } 1211 1212 /** 1213 * Callback used when implicit mempool registration is disabled 1214 * in order to track Rx mempool destruction. 1215 * 1216 * @param event 1217 * Mempool life cycle event. 1218 * @param mp 1219 * An Rx mempool registered explicitly when the port is started. 1220 * @param arg 1221 * Pointer to a device shared context. 1222 */ 1223 static void 1224 mlx5_dev_ctx_shared_rx_mempool_event_cb(enum rte_mempool_event event, 1225 struct rte_mempool *mp, void *arg) 1226 { 1227 struct mlx5_dev_ctx_shared *sh = arg; 1228 1229 if (event == RTE_MEMPOOL_EVENT_DESTROY) 1230 mlx5_dev_mempool_unregister(sh->cdev, mp); 1231 } 1232 1233 int 1234 mlx5_dev_ctx_shared_mempool_subscribe(struct rte_eth_dev *dev) 1235 { 1236 struct mlx5_priv *priv = dev->data->dev_private; 1237 struct mlx5_dev_ctx_shared *sh = priv->sh; 1238 int ret; 1239 1240 /* Check if we only need to track Rx mempool destruction. */ 1241 if (!sh->cdev->config.mr_mempool_reg_en) { 1242 ret = rte_mempool_event_callback_register 1243 (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); 1244 return ret == 0 || rte_errno == EEXIST ? 0 : ret; 1245 } 1246 return mlx5_dev_mempool_subscribe(sh->cdev); 1247 } 1248 1249 /** 1250 * Set up multiple TISs with different affinities according to 1251 * number of bonding ports 1252 * 1253 * @param priv 1254 * Pointer of shared context. 1255 * 1256 * @return 1257 * Zero on success, -1 otherwise. 1258 */ 1259 static int 1260 mlx5_setup_tis(struct mlx5_dev_ctx_shared *sh) 1261 { 1262 struct mlx5_devx_lag_context lag_ctx = { 0 }; 1263 struct mlx5_devx_tis_attr tis_attr = { 0 }; 1264 int i; 1265 1266 tis_attr.transport_domain = sh->td->id; 1267 if (sh->bond.n_port) { 1268 if (!mlx5_devx_cmd_query_lag(sh->cdev->ctx, &lag_ctx)) { 1269 sh->lag.tx_remap_affinity[0] = 1270 lag_ctx.tx_remap_affinity_1; 1271 sh->lag.tx_remap_affinity[1] = 1272 lag_ctx.tx_remap_affinity_2; 1273 sh->lag.affinity_mode = lag_ctx.port_select_mode; 1274 } else { 1275 DRV_LOG(ERR, "Failed to query lag affinity."); 1276 return -1; 1277 } 1278 if (sh->lag.affinity_mode == MLX5_LAG_MODE_TIS) 1279 DRV_LOG(DEBUG, "LAG number of ports : %d, affinity_1 & 2 : pf%d & %d.\n", 1280 sh->bond.n_port, lag_ctx.tx_remap_affinity_1, 1281 lag_ctx.tx_remap_affinity_2); 1282 else if (sh->lag.affinity_mode == MLX5_LAG_MODE_HASH) 1283 DRV_LOG(INFO, "Device %s enabled HW hash based LAG.", 1284 sh->ibdev_name); 1285 } 1286 for (i = 0; i <= sh->bond.n_port; i++) { 1287 /* 1288 * lag_tx_port_affinity: 0 auto-selection, 1 PF1, 2 PF2 vice versa. 1289 * Each TIS binds to one PF by setting lag_tx_port_affinity (> 0). 1290 * Once LAG enabled, we create multiple TISs and bind each one to 1291 * different PFs, then TIS[i+1] gets affinity i+1 and goes to PF i+1. 1292 * TIS[0] is reserved for HW Hash mode. 1293 */ 1294 tis_attr.lag_tx_port_affinity = i; 1295 sh->tis[i] = mlx5_devx_cmd_create_tis(sh->cdev->ctx, &tis_attr); 1296 if (!sh->tis[i]) { 1297 DRV_LOG(ERR, "Failed to create TIS %d/%d for [bonding] device" 1298 " %s.", i, sh->bond.n_port, 1299 sh->ibdev_name); 1300 return -1; 1301 } 1302 } 1303 return 0; 1304 } 1305 1306 /** 1307 * Verify and store value for share device argument. 1308 * 1309 * @param[in] key 1310 * Key argument to verify. 1311 * @param[in] val 1312 * Value associated with key. 1313 * @param opaque 1314 * User data. 1315 * 1316 * @return 1317 * 0 on success, a negative errno value otherwise and rte_errno is set. 1318 */ 1319 static int 1320 mlx5_dev_args_check_handler(const char *key, const char *val, void *opaque) 1321 { 1322 struct mlx5_sh_config *config = opaque; 1323 signed long tmp; 1324 1325 errno = 0; 1326 tmp = strtol(val, NULL, 0); 1327 if (errno) { 1328 rte_errno = errno; 1329 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); 1330 return -rte_errno; 1331 } 1332 if (tmp < 0 && strcmp(MLX5_TX_PP, key) && strcmp(MLX5_TX_SKEW, key)) { 1333 /* Negative values are acceptable for some keys only. */ 1334 rte_errno = EINVAL; 1335 DRV_LOG(WARNING, "%s: invalid negative value \"%s\"", key, val); 1336 return -rte_errno; 1337 } 1338 if (strcmp(MLX5_TX_PP, key) == 0) { 1339 unsigned long mod = tmp >= 0 ? tmp : -tmp; 1340 1341 if (!mod) { 1342 DRV_LOG(ERR, "Zero Tx packet pacing parameter."); 1343 rte_errno = EINVAL; 1344 return -rte_errno; 1345 } 1346 config->tx_pp = tmp; 1347 } else if (strcmp(MLX5_TX_SKEW, key) == 0) { 1348 config->tx_skew = tmp; 1349 } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) { 1350 config->l3_vxlan_en = !!tmp; 1351 } else if (strcmp(MLX5_VF_NL_EN, key) == 0) { 1352 config->vf_nl_en = !!tmp; 1353 } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) { 1354 config->dv_esw_en = !!tmp; 1355 } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) { 1356 if (tmp > 2) { 1357 DRV_LOG(ERR, "Invalid %s parameter.", key); 1358 rte_errno = EINVAL; 1359 return -rte_errno; 1360 } 1361 config->dv_flow_en = tmp; 1362 } else if (strcmp(MLX5_DV_XMETA_EN, key) == 0) { 1363 if (tmp != MLX5_XMETA_MODE_LEGACY && 1364 tmp != MLX5_XMETA_MODE_META16 && 1365 tmp != MLX5_XMETA_MODE_META32 && 1366 tmp != MLX5_XMETA_MODE_MISS_INFO && 1367 tmp != MLX5_XMETA_MODE_META32_HWS) { 1368 DRV_LOG(ERR, "Invalid extensive metadata parameter."); 1369 rte_errno = EINVAL; 1370 return -rte_errno; 1371 } 1372 if (tmp != MLX5_XMETA_MODE_MISS_INFO) 1373 config->dv_xmeta_en = tmp; 1374 else 1375 config->dv_miss_info = 1; 1376 } else if (strcmp(MLX5_LACP_BY_USER, key) == 0) { 1377 config->lacp_by_user = !!tmp; 1378 } else if (strcmp(MLX5_RECLAIM_MEM, key) == 0) { 1379 if (tmp != MLX5_RCM_NONE && 1380 tmp != MLX5_RCM_LIGHT && 1381 tmp != MLX5_RCM_AGGR) { 1382 DRV_LOG(ERR, "Unrecognize %s: \"%s\"", key, val); 1383 rte_errno = EINVAL; 1384 return -rte_errno; 1385 } 1386 config->reclaim_mode = tmp; 1387 } else if (strcmp(MLX5_DECAP_EN, key) == 0) { 1388 config->decap_en = !!tmp; 1389 } else if (strcmp(MLX5_ALLOW_DUPLICATE_PATTERN, key) == 0) { 1390 config->allow_duplicate_pattern = !!tmp; 1391 } else if (strcmp(MLX5_FDB_DEFAULT_RULE_EN, key) == 0) { 1392 config->fdb_def_rule = !!tmp; 1393 } else if (strcmp(MLX5_HWS_CNT_SERVICE_CORE, key) == 0) { 1394 config->cnt_svc.service_core = tmp; 1395 } else if (strcmp(MLX5_HWS_CNT_CYCLE_TIME, key) == 0) { 1396 config->cnt_svc.cycle_time = tmp; 1397 } else if (strcmp(MLX5_REPR_MATCHING_EN, key) == 0) { 1398 config->repr_matching = !!tmp; 1399 } 1400 return 0; 1401 } 1402 1403 /** 1404 * Parse user device parameters and adjust them according to device 1405 * capabilities. 1406 * 1407 * @param sh 1408 * Pointer to shared device context. 1409 * @param mkvlist 1410 * Pointer to mlx5 kvargs control, can be NULL if there is no devargs. 1411 * @param config 1412 * Pointer to shared device configuration structure. 1413 * 1414 * @return 1415 * 0 on success, a negative errno value otherwise and rte_errno is set. 1416 */ 1417 static int 1418 mlx5_shared_dev_ctx_args_config(struct mlx5_dev_ctx_shared *sh, 1419 struct mlx5_kvargs_ctrl *mkvlist, 1420 struct mlx5_sh_config *config) 1421 { 1422 const char **params = (const char *[]){ 1423 MLX5_TX_PP, 1424 MLX5_TX_SKEW, 1425 MLX5_L3_VXLAN_EN, 1426 MLX5_VF_NL_EN, 1427 MLX5_DV_ESW_EN, 1428 MLX5_DV_FLOW_EN, 1429 MLX5_DV_XMETA_EN, 1430 MLX5_LACP_BY_USER, 1431 MLX5_RECLAIM_MEM, 1432 MLX5_DECAP_EN, 1433 MLX5_ALLOW_DUPLICATE_PATTERN, 1434 MLX5_FDB_DEFAULT_RULE_EN, 1435 MLX5_HWS_CNT_SERVICE_CORE, 1436 MLX5_HWS_CNT_CYCLE_TIME, 1437 MLX5_REPR_MATCHING_EN, 1438 NULL, 1439 }; 1440 int ret = 0; 1441 1442 /* Default configuration. */ 1443 memset(config, 0, sizeof(*config)); 1444 config->vf_nl_en = 1; 1445 config->dv_esw_en = 1; 1446 config->dv_flow_en = 1; 1447 config->decap_en = 1; 1448 config->allow_duplicate_pattern = 1; 1449 config->fdb_def_rule = 1; 1450 config->cnt_svc.cycle_time = MLX5_CNT_SVC_CYCLE_TIME_DEFAULT; 1451 config->cnt_svc.service_core = rte_get_main_lcore(); 1452 config->repr_matching = 1; 1453 if (mkvlist != NULL) { 1454 /* Process parameters. */ 1455 ret = mlx5_kvargs_process(mkvlist, params, 1456 mlx5_dev_args_check_handler, config); 1457 if (ret) { 1458 DRV_LOG(ERR, "Failed to process device arguments: %s", 1459 strerror(rte_errno)); 1460 return -rte_errno; 1461 } 1462 } 1463 /* Adjust parameters according to device capabilities. */ 1464 if (config->dv_flow_en && !sh->dev_cap.dv_flow_en) { 1465 DRV_LOG(WARNING, "DV flow is not supported."); 1466 config->dv_flow_en = 0; 1467 } 1468 if (config->dv_esw_en && !sh->dev_cap.dv_esw_en) { 1469 DRV_LOG(DEBUG, "E-Switch DV flow is not supported."); 1470 config->dv_esw_en = 0; 1471 } 1472 if (config->dv_esw_en && !config->dv_flow_en) { 1473 DRV_LOG(DEBUG, 1474 "E-Switch DV flow is supported only when DV flow is enabled."); 1475 config->dv_esw_en = 0; 1476 } 1477 if (config->dv_miss_info && config->dv_esw_en) 1478 config->dv_xmeta_en = MLX5_XMETA_MODE_META16; 1479 if (!config->dv_esw_en && 1480 config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 1481 DRV_LOG(WARNING, 1482 "Metadata mode %u is not supported (no E-Switch).", 1483 config->dv_xmeta_en); 1484 config->dv_xmeta_en = MLX5_XMETA_MODE_LEGACY; 1485 } 1486 if (config->dv_flow_en != 2 && !config->repr_matching) { 1487 DRV_LOG(DEBUG, "Disabling representor matching is valid only " 1488 "when HW Steering is enabled."); 1489 config->repr_matching = 1; 1490 } 1491 if (config->tx_pp && !sh->dev_cap.txpp_en) { 1492 DRV_LOG(ERR, "Packet pacing is not supported."); 1493 rte_errno = ENODEV; 1494 return -rte_errno; 1495 } 1496 if (!config->tx_pp && config->tx_skew && 1497 !sh->cdev->config.hca_attr.wait_on_time) { 1498 DRV_LOG(WARNING, 1499 "\"tx_skew\" doesn't affect without \"tx_pp\"."); 1500 } 1501 /* Check for LRO support. */ 1502 if (mlx5_devx_obj_ops_en(sh) && sh->cdev->config.hca_attr.lro_cap) { 1503 /* TBD check tunnel lro caps. */ 1504 config->lro_allowed = 1; 1505 DRV_LOG(DEBUG, "LRO is allowed."); 1506 DRV_LOG(DEBUG, 1507 "LRO minimal size of TCP segment required for coalescing is %d bytes.", 1508 sh->cdev->config.hca_attr.lro_min_mss_size); 1509 } 1510 /* 1511 * If HW has bug working with tunnel packet decapsulation and scatter 1512 * FCS, and decapsulation is needed, clear the hw_fcs_strip bit. 1513 * Then RTE_ETH_RX_OFFLOAD_KEEP_CRC bit will not be set anymore. 1514 */ 1515 if (sh->dev_cap.scatter_fcs_w_decap_disable && sh->config.decap_en) 1516 config->hw_fcs_strip = 0; 1517 else 1518 config->hw_fcs_strip = sh->dev_cap.hw_fcs_strip; 1519 DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported", 1520 (config->hw_fcs_strip ? "" : "not ")); 1521 DRV_LOG(DEBUG, "\"tx_pp\" is %d.", config->tx_pp); 1522 DRV_LOG(DEBUG, "\"tx_skew\" is %d.", config->tx_skew); 1523 DRV_LOG(DEBUG, "\"reclaim_mode\" is %u.", config->reclaim_mode); 1524 DRV_LOG(DEBUG, "\"dv_esw_en\" is %u.", config->dv_esw_en); 1525 DRV_LOG(DEBUG, "\"dv_flow_en\" is %u.", config->dv_flow_en); 1526 DRV_LOG(DEBUG, "\"dv_xmeta_en\" is %u.", config->dv_xmeta_en); 1527 DRV_LOG(DEBUG, "\"dv_miss_info\" is %u.", config->dv_miss_info); 1528 DRV_LOG(DEBUG, "\"l3_vxlan_en\" is %u.", config->l3_vxlan_en); 1529 DRV_LOG(DEBUG, "\"vf_nl_en\" is %u.", config->vf_nl_en); 1530 DRV_LOG(DEBUG, "\"lacp_by_user\" is %u.", config->lacp_by_user); 1531 DRV_LOG(DEBUG, "\"decap_en\" is %u.", config->decap_en); 1532 DRV_LOG(DEBUG, "\"allow_duplicate_pattern\" is %u.", 1533 config->allow_duplicate_pattern); 1534 DRV_LOG(DEBUG, "\"fdb_def_rule_en\" is %u.", config->fdb_def_rule); 1535 DRV_LOG(DEBUG, "\"repr_matching_en\" is %u.", config->repr_matching); 1536 return 0; 1537 } 1538 1539 /** 1540 * Configure realtime timestamp format. 1541 * 1542 * @param sh 1543 * Pointer to mlx5_dev_ctx_shared object. 1544 * @param hca_attr 1545 * Pointer to DevX HCA capabilities structure. 1546 */ 1547 void 1548 mlx5_rt_timestamp_config(struct mlx5_dev_ctx_shared *sh, 1549 struct mlx5_hca_attr *hca_attr) 1550 { 1551 uint32_t dw_cnt = MLX5_ST_SZ_DW(register_mtutc); 1552 uint32_t reg[dw_cnt]; 1553 int ret = ENOTSUP; 1554 1555 if (hca_attr->access_register_user) 1556 ret = mlx5_devx_cmd_register_read(sh->cdev->ctx, 1557 MLX5_REGISTER_ID_MTUTC, 0, 1558 reg, dw_cnt); 1559 if (!ret) { 1560 uint32_t ts_mode; 1561 1562 /* MTUTC register is read successfully. */ 1563 ts_mode = MLX5_GET(register_mtutc, reg, time_stamp_mode); 1564 if (ts_mode == MLX5_MTUTC_TIMESTAMP_MODE_REAL_TIME) 1565 sh->dev_cap.rt_timestamp = 1; 1566 } else { 1567 /* Kernel does not support register reading. */ 1568 if (hca_attr->dev_freq_khz == (NS_PER_S / MS_PER_S)) 1569 sh->dev_cap.rt_timestamp = 1; 1570 } 1571 } 1572 1573 /** 1574 * Allocate shared device context. If there is multiport device the 1575 * master and representors will share this context, if there is single 1576 * port dedicated device, the context will be used by only given 1577 * port due to unification. 1578 * 1579 * Routine first searches the context for the specified device name, 1580 * if found the shared context assumed and reference counter is incremented. 1581 * If no context found the new one is created and initialized with specified 1582 * device context and parameters. 1583 * 1584 * @param[in] spawn 1585 * Pointer to the device attributes (name, port, etc). 1586 * @param mkvlist 1587 * Pointer to mlx5 kvargs control, can be NULL if there is no devargs. 1588 * 1589 * @return 1590 * Pointer to mlx5_dev_ctx_shared object on success, 1591 * otherwise NULL and rte_errno is set. 1592 */ 1593 struct mlx5_dev_ctx_shared * 1594 mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, 1595 struct mlx5_kvargs_ctrl *mkvlist) 1596 { 1597 struct mlx5_dev_ctx_shared *sh; 1598 int err = 0; 1599 uint32_t i; 1600 1601 MLX5_ASSERT(spawn); 1602 /* Secondary process should not create the shared context. */ 1603 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1604 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1605 /* Search for IB context by device name. */ 1606 LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) { 1607 if (!strcmp(sh->ibdev_name, spawn->phys_dev_name)) { 1608 sh->refcnt++; 1609 goto exit; 1610 } 1611 } 1612 /* No device found, we have to create new shared context. */ 1613 MLX5_ASSERT(spawn->max_port); 1614 sh = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 1615 sizeof(struct mlx5_dev_ctx_shared) + 1616 spawn->max_port * sizeof(struct mlx5_dev_shared_port), 1617 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1618 if (!sh) { 1619 DRV_LOG(ERR, "Shared context allocation failure."); 1620 rte_errno = ENOMEM; 1621 goto exit; 1622 } 1623 pthread_mutex_init(&sh->txpp.mutex, NULL); 1624 sh->numa_node = spawn->cdev->dev->numa_node; 1625 sh->cdev = spawn->cdev; 1626 sh->esw_mode = !!(spawn->info.master || spawn->info.representor); 1627 if (spawn->bond_info) 1628 sh->bond = *spawn->bond_info; 1629 err = mlx5_os_capabilities_prepare(sh); 1630 if (err) { 1631 DRV_LOG(ERR, "Fail to configure device capabilities."); 1632 goto error; 1633 } 1634 err = mlx5_shared_dev_ctx_args_config(sh, mkvlist, &sh->config); 1635 if (err) { 1636 DRV_LOG(ERR, "Failed to process device configure: %s", 1637 strerror(rte_errno)); 1638 goto error; 1639 } 1640 sh->refcnt = 1; 1641 sh->max_port = spawn->max_port; 1642 strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->cdev->ctx), 1643 sizeof(sh->ibdev_name) - 1); 1644 strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->cdev->ctx), 1645 sizeof(sh->ibdev_path) - 1); 1646 /* 1647 * Setting port_id to max unallowed value means there is no interrupt 1648 * subhandler installed for the given port index i. 1649 */ 1650 for (i = 0; i < sh->max_port; i++) { 1651 sh->port[i].ih_port_id = RTE_MAX_ETHPORTS; 1652 sh->port[i].devx_ih_port_id = RTE_MAX_ETHPORTS; 1653 sh->port[i].nl_ih_port_id = RTE_MAX_ETHPORTS; 1654 } 1655 if (sh->cdev->config.devx) { 1656 sh->td = mlx5_devx_cmd_create_td(sh->cdev->ctx); 1657 if (!sh->td) { 1658 DRV_LOG(ERR, "TD allocation failure"); 1659 rte_errno = ENOMEM; 1660 goto error; 1661 } 1662 if (mlx5_setup_tis(sh)) { 1663 DRV_LOG(ERR, "TIS allocation failure"); 1664 rte_errno = ENOMEM; 1665 goto error; 1666 } 1667 err = mlx5_rxtx_uars_prepare(sh); 1668 if (err) 1669 goto error; 1670 #ifndef RTE_ARCH_64 1671 } else { 1672 /* Initialize UAR access locks for 32bit implementations. */ 1673 rte_spinlock_init(&sh->uar_lock_cq); 1674 for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) 1675 rte_spinlock_init(&sh->uar_lock[i]); 1676 #endif 1677 } 1678 mlx5_os_dev_shared_handler_install(sh); 1679 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1680 err = mlx5_flow_os_init_workspace_once(); 1681 if (err) 1682 goto error; 1683 } 1684 err = mlx5_flow_counters_mng_init(sh); 1685 if (err) { 1686 DRV_LOG(ERR, "Fail to initialize counters manage."); 1687 goto error; 1688 } 1689 mlx5_flow_aging_init(sh); 1690 mlx5_flow_ipool_create(sh); 1691 /* Add context to the global device list. */ 1692 LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next); 1693 rte_spinlock_init(&sh->geneve_tlv_opt_sl); 1694 exit: 1695 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1696 return sh; 1697 error: 1698 err = rte_errno; 1699 pthread_mutex_destroy(&sh->txpp.mutex); 1700 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1701 MLX5_ASSERT(sh); 1702 mlx5_rxtx_uars_release(sh); 1703 i = 0; 1704 do { 1705 if (sh->tis[i]) 1706 claim_zero(mlx5_devx_cmd_destroy(sh->tis[i])); 1707 } while (++i < (uint32_t)sh->bond.n_port); 1708 if (sh->td) 1709 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1710 mlx5_free(sh); 1711 rte_errno = err; 1712 return NULL; 1713 } 1714 1715 /** 1716 * Create LWM event_channel and interrupt handle for shared device 1717 * context. All rxqs sharing the device context share the event_channel. 1718 * A callback is registered in interrupt thread to receive the LWM event. 1719 * 1720 * @param[in] priv 1721 * Pointer to mlx5_priv instance. 1722 * 1723 * @return 1724 * 0 on success, negative with rte_errno set. 1725 */ 1726 int 1727 mlx5_lwm_setup(struct mlx5_priv *priv) 1728 { 1729 int fd_lwm; 1730 1731 pthread_mutex_init(&priv->sh->lwm_config_lock, NULL); 1732 priv->sh->devx_channel_lwm = mlx5_os_devx_create_event_channel 1733 (priv->sh->cdev->ctx, 1734 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA); 1735 if (!priv->sh->devx_channel_lwm) 1736 goto err; 1737 fd_lwm = mlx5_os_get_devx_channel_fd(priv->sh->devx_channel_lwm); 1738 priv->sh->intr_handle_lwm = mlx5_os_interrupt_handler_create 1739 (RTE_INTR_INSTANCE_F_SHARED, true, 1740 fd_lwm, mlx5_dev_interrupt_handler_lwm, priv); 1741 if (!priv->sh->intr_handle_lwm) 1742 goto err; 1743 return 0; 1744 err: 1745 if (priv->sh->devx_channel_lwm) { 1746 mlx5_os_devx_destroy_event_channel 1747 (priv->sh->devx_channel_lwm); 1748 priv->sh->devx_channel_lwm = NULL; 1749 } 1750 pthread_mutex_destroy(&priv->sh->lwm_config_lock); 1751 return -rte_errno; 1752 } 1753 1754 /** 1755 * Destroy LWM event_channel and interrupt handle for shared device 1756 * context before free this context. The interrupt handler is also 1757 * unregistered. 1758 * 1759 * @param[in] sh 1760 * Pointer to shared device context. 1761 */ 1762 void 1763 mlx5_lwm_unset(struct mlx5_dev_ctx_shared *sh) 1764 { 1765 if (sh->intr_handle_lwm) { 1766 mlx5_os_interrupt_handler_destroy(sh->intr_handle_lwm, 1767 mlx5_dev_interrupt_handler_lwm, (void *)-1); 1768 sh->intr_handle_lwm = NULL; 1769 } 1770 if (sh->devx_channel_lwm) { 1771 mlx5_os_devx_destroy_event_channel 1772 (sh->devx_channel_lwm); 1773 sh->devx_channel_lwm = NULL; 1774 } 1775 pthread_mutex_destroy(&sh->lwm_config_lock); 1776 } 1777 1778 /** 1779 * Free shared IB device context. Decrement counter and if zero free 1780 * all allocated resources and close handles. 1781 * 1782 * @param[in] sh 1783 * Pointer to mlx5_dev_ctx_shared object to free 1784 */ 1785 void 1786 mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) 1787 { 1788 int ret; 1789 int i = 0; 1790 1791 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1792 #ifdef RTE_LIBRTE_MLX5_DEBUG 1793 /* Check the object presence in the list. */ 1794 struct mlx5_dev_ctx_shared *lctx; 1795 1796 LIST_FOREACH(lctx, &mlx5_dev_ctx_list, next) 1797 if (lctx == sh) 1798 break; 1799 MLX5_ASSERT(lctx); 1800 if (lctx != sh) { 1801 DRV_LOG(ERR, "Freeing non-existing shared IB context"); 1802 goto exit; 1803 } 1804 #endif 1805 MLX5_ASSERT(sh); 1806 MLX5_ASSERT(sh->refcnt); 1807 /* Secondary process should not free the shared context. */ 1808 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1809 if (--sh->refcnt) 1810 goto exit; 1811 /* Stop watching for mempool events and unregister all mempools. */ 1812 if (!sh->cdev->config.mr_mempool_reg_en) { 1813 ret = rte_mempool_event_callback_unregister 1814 (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); 1815 if (ret == 0) 1816 rte_mempool_walk 1817 (mlx5_dev_ctx_shared_rx_mempool_unregister_cb, sh); 1818 } 1819 /* Remove context from the global device list. */ 1820 LIST_REMOVE(sh, next); 1821 /* Release resources on the last device removal. */ 1822 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1823 mlx5_os_net_cleanup(); 1824 mlx5_flow_os_release_workspace(); 1825 } 1826 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1827 if (sh->flex_parsers_dv) { 1828 mlx5_list_destroy(sh->flex_parsers_dv); 1829 sh->flex_parsers_dv = NULL; 1830 } 1831 /* 1832 * Ensure there is no async event handler installed. 1833 * Only primary process handles async device events. 1834 **/ 1835 mlx5_flow_counters_mng_close(sh); 1836 if (sh->ct_mng) 1837 mlx5_flow_aso_ct_mng_close(sh); 1838 if (sh->aso_age_mng) { 1839 mlx5_flow_aso_age_mng_close(sh); 1840 sh->aso_age_mng = NULL; 1841 } 1842 if (sh->mtrmng) 1843 mlx5_aso_flow_mtrs_mng_close(sh); 1844 mlx5_flow_ipool_destroy(sh); 1845 mlx5_os_dev_shared_handler_uninstall(sh); 1846 mlx5_rxtx_uars_release(sh); 1847 do { 1848 if (sh->tis[i]) 1849 claim_zero(mlx5_devx_cmd_destroy(sh->tis[i])); 1850 } while (++i < sh->bond.n_port); 1851 if (sh->td) 1852 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1853 #ifdef HAVE_MLX5_HWS_SUPPORT 1854 /* HWS manages geneve_tlv_option resource as global. */ 1855 if (sh->config.dv_flow_en == 2) 1856 flow_dev_geneve_tlv_option_resource_release(sh); 1857 else 1858 #endif 1859 MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL); 1860 pthread_mutex_destroy(&sh->txpp.mutex); 1861 mlx5_lwm_unset(sh); 1862 mlx5_free(sh); 1863 return; 1864 exit: 1865 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1866 } 1867 1868 /** 1869 * Destroy table hash list. 1870 * 1871 * @param[in] priv 1872 * Pointer to the private device data structure. 1873 */ 1874 void 1875 mlx5_free_table_hash_list(struct mlx5_priv *priv) 1876 { 1877 struct mlx5_dev_ctx_shared *sh = priv->sh; 1878 struct mlx5_hlist **tbls = (priv->sh->config.dv_flow_en == 2) ? 1879 &sh->groups : &sh->flow_tbls; 1880 if (*tbls == NULL) 1881 return; 1882 mlx5_hlist_destroy(*tbls); 1883 *tbls = NULL; 1884 } 1885 1886 #ifdef HAVE_MLX5_HWS_SUPPORT 1887 /** 1888 * Allocate HW steering group hash list. 1889 * 1890 * @param[in] priv 1891 * Pointer to the private device data structure. 1892 */ 1893 static int 1894 mlx5_alloc_hw_group_hash_list(struct mlx5_priv *priv) 1895 { 1896 int err = 0; 1897 struct mlx5_dev_ctx_shared *sh = priv->sh; 1898 char s[MLX5_NAME_SIZE]; 1899 1900 MLX5_ASSERT(sh); 1901 snprintf(s, sizeof(s), "%s_flow_groups", priv->sh->ibdev_name); 1902 sh->groups = mlx5_hlist_create 1903 (s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, 1904 false, true, sh, 1905 flow_hw_grp_create_cb, 1906 flow_hw_grp_match_cb, 1907 flow_hw_grp_remove_cb, 1908 flow_hw_grp_clone_cb, 1909 flow_hw_grp_clone_free_cb); 1910 if (!sh->groups) { 1911 DRV_LOG(ERR, "flow groups with hash creation failed."); 1912 err = ENOMEM; 1913 } 1914 return err; 1915 } 1916 #endif 1917 1918 1919 /** 1920 * Initialize flow table hash list and create the root tables entry 1921 * for each domain. 1922 * 1923 * @param[in] priv 1924 * Pointer to the private device data structure. 1925 * 1926 * @return 1927 * Zero on success, positive error code otherwise. 1928 */ 1929 int 1930 mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) 1931 { 1932 int err = 0; 1933 1934 /* Tables are only used in DV and DR modes. */ 1935 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 1936 struct mlx5_dev_ctx_shared *sh = priv->sh; 1937 char s[MLX5_NAME_SIZE]; 1938 1939 #ifdef HAVE_MLX5_HWS_SUPPORT 1940 if (priv->sh->config.dv_flow_en == 2) 1941 return mlx5_alloc_hw_group_hash_list(priv); 1942 #endif 1943 MLX5_ASSERT(sh); 1944 snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); 1945 sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, 1946 false, true, sh, 1947 flow_dv_tbl_create_cb, 1948 flow_dv_tbl_match_cb, 1949 flow_dv_tbl_remove_cb, 1950 flow_dv_tbl_clone_cb, 1951 flow_dv_tbl_clone_free_cb); 1952 if (!sh->flow_tbls) { 1953 DRV_LOG(ERR, "flow tables with hash creation failed."); 1954 err = ENOMEM; 1955 return err; 1956 } 1957 #ifndef HAVE_MLX5DV_DR 1958 struct rte_flow_error error; 1959 struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; 1960 1961 /* 1962 * In case we have not DR support, the zero tables should be created 1963 * because DV expect to see them even if they cannot be created by 1964 * RDMA-CORE. 1965 */ 1966 if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, 1967 NULL, 0, 1, 0, &error) || 1968 !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, 1969 NULL, 0, 1, 0, &error) || 1970 !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, 1971 NULL, 0, 1, 0, &error)) { 1972 err = ENOMEM; 1973 goto error; 1974 } 1975 return err; 1976 error: 1977 mlx5_free_table_hash_list(priv); 1978 #endif /* HAVE_MLX5DV_DR */ 1979 #endif 1980 return err; 1981 } 1982 1983 /** 1984 * Retrieve integer value from environment variable. 1985 * 1986 * @param[in] name 1987 * Environment variable name. 1988 * 1989 * @return 1990 * Integer value, 0 if the variable is not set. 1991 */ 1992 int 1993 mlx5_getenv_int(const char *name) 1994 { 1995 const char *val = getenv(name); 1996 1997 if (val == NULL) 1998 return 0; 1999 return atoi(val); 2000 } 2001 2002 /** 2003 * DPDK callback to add udp tunnel port 2004 * 2005 * @param[in] dev 2006 * A pointer to eth_dev 2007 * @param[in] udp_tunnel 2008 * A pointer to udp tunnel 2009 * 2010 * @return 2011 * 0 on valid udp ports and tunnels, -ENOTSUP otherwise. 2012 */ 2013 int 2014 mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev __rte_unused, 2015 struct rte_eth_udp_tunnel *udp_tunnel) 2016 { 2017 MLX5_ASSERT(udp_tunnel != NULL); 2018 if (udp_tunnel->prot_type == RTE_ETH_TUNNEL_TYPE_VXLAN && 2019 udp_tunnel->udp_port == 4789) 2020 return 0; 2021 if (udp_tunnel->prot_type == RTE_ETH_TUNNEL_TYPE_VXLAN_GPE && 2022 udp_tunnel->udp_port == 4790) 2023 return 0; 2024 return -ENOTSUP; 2025 } 2026 2027 /** 2028 * Initialize process private data structure. 2029 * 2030 * @param dev 2031 * Pointer to Ethernet device structure. 2032 * 2033 * @return 2034 * 0 on success, a negative errno value otherwise and rte_errno is set. 2035 */ 2036 int 2037 mlx5_proc_priv_init(struct rte_eth_dev *dev) 2038 { 2039 struct mlx5_priv *priv = dev->data->dev_private; 2040 struct mlx5_proc_priv *ppriv; 2041 size_t ppriv_size; 2042 2043 mlx5_proc_priv_uninit(dev); 2044 /* 2045 * UAR register table follows the process private structure. BlueFlame 2046 * registers for Tx queues are stored in the table. 2047 */ 2048 ppriv_size = sizeof(struct mlx5_proc_priv) + 2049 priv->txqs_n * sizeof(struct mlx5_uar_data); 2050 ppriv = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, ppriv_size, 2051 RTE_CACHE_LINE_SIZE, dev->device->numa_node); 2052 if (!ppriv) { 2053 rte_errno = ENOMEM; 2054 return -rte_errno; 2055 } 2056 ppriv->uar_table_sz = priv->txqs_n; 2057 dev->process_private = ppriv; 2058 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 2059 priv->sh->pppriv = ppriv; 2060 return 0; 2061 } 2062 2063 /** 2064 * Un-initialize process private data structure. 2065 * 2066 * @param dev 2067 * Pointer to Ethernet device structure. 2068 */ 2069 void 2070 mlx5_proc_priv_uninit(struct rte_eth_dev *dev) 2071 { 2072 struct mlx5_proc_priv *ppriv = dev->process_private; 2073 2074 if (!ppriv) 2075 return; 2076 if (ppriv->hca_bar) 2077 mlx5_txpp_unmap_hca_bar(dev); 2078 mlx5_free(dev->process_private); 2079 dev->process_private = NULL; 2080 } 2081 2082 /** 2083 * DPDK callback to close the device. 2084 * 2085 * Destroy all queues and objects, free memory. 2086 * 2087 * @param dev 2088 * Pointer to Ethernet device structure. 2089 */ 2090 int 2091 mlx5_dev_close(struct rte_eth_dev *dev) 2092 { 2093 struct mlx5_priv *priv = dev->data->dev_private; 2094 unsigned int i; 2095 int ret; 2096 2097 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 2098 /* Check if process_private released. */ 2099 if (!dev->process_private) 2100 return 0; 2101 mlx5_tx_uar_uninit_secondary(dev); 2102 mlx5_proc_priv_uninit(dev); 2103 rte_eth_dev_release_port(dev); 2104 return 0; 2105 } 2106 if (!priv->sh) 2107 return 0; 2108 if (priv->shared_refcnt) { 2109 DRV_LOG(ERR, "port %u is shared host in use (%u)", 2110 dev->data->port_id, priv->shared_refcnt); 2111 rte_errno = EBUSY; 2112 return -EBUSY; 2113 } 2114 DRV_LOG(DEBUG, "port %u closing device \"%s\"", 2115 dev->data->port_id, 2116 ((priv->sh->cdev->ctx != NULL) ? 2117 mlx5_os_get_ctx_device_name(priv->sh->cdev->ctx) : "")); 2118 /* 2119 * If default mreg copy action is removed at the stop stage, 2120 * the search will return none and nothing will be done anymore. 2121 */ 2122 if (priv->sh->config.dv_flow_en != 2) 2123 mlx5_flow_stop_default(dev); 2124 mlx5_traffic_disable(dev); 2125 /* 2126 * If all the flows are already flushed in the device stop stage, 2127 * then this will return directly without any action. 2128 */ 2129 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true); 2130 mlx5_action_handle_flush(dev); 2131 mlx5_flow_meter_flush(dev, NULL); 2132 /* Prevent crashes when queues are still in use. */ 2133 dev->rx_pkt_burst = rte_eth_pkt_burst_dummy; 2134 dev->tx_pkt_burst = rte_eth_pkt_burst_dummy; 2135 rte_wmb(); 2136 /* Disable datapath on secondary process. */ 2137 mlx5_mp_os_req_stop_rxtx(dev); 2138 /* Free the eCPRI flex parser resource. */ 2139 mlx5_flex_parser_ecpri_release(dev); 2140 mlx5_flex_item_port_cleanup(dev); 2141 #ifdef HAVE_MLX5_HWS_SUPPORT 2142 flow_hw_destroy_vport_action(dev); 2143 flow_hw_resource_release(dev); 2144 flow_hw_clear_port_info(dev); 2145 if (priv->sh->config.dv_flow_en == 2) { 2146 flow_hw_clear_flow_metadata_config(); 2147 flow_hw_clear_tags_set(dev); 2148 } 2149 #endif 2150 if (priv->rxq_privs != NULL) { 2151 /* XXX race condition if mlx5_rx_burst() is still running. */ 2152 rte_delay_us_sleep(1000); 2153 for (i = 0; (i != priv->rxqs_n); ++i) 2154 mlx5_rxq_release(dev, i); 2155 priv->rxqs_n = 0; 2156 mlx5_free(priv->rxq_privs); 2157 priv->rxq_privs = NULL; 2158 } 2159 if (priv->txqs != NULL) { 2160 /* XXX race condition if mlx5_tx_burst() is still running. */ 2161 rte_delay_us_sleep(1000); 2162 for (i = 0; (i != priv->txqs_n); ++i) 2163 mlx5_txq_release(dev, i); 2164 priv->txqs_n = 0; 2165 priv->txqs = NULL; 2166 } 2167 mlx5_proc_priv_uninit(dev); 2168 if (priv->q_counters) { 2169 mlx5_devx_cmd_destroy(priv->q_counters); 2170 priv->q_counters = NULL; 2171 } 2172 if (priv->drop_queue.hrxq) 2173 mlx5_drop_action_destroy(dev); 2174 if (priv->mreg_cp_tbl) 2175 mlx5_hlist_destroy(priv->mreg_cp_tbl); 2176 mlx5_mprq_free_mp(dev); 2177 mlx5_os_free_shared_dr(priv); 2178 if (priv->rss_conf.rss_key != NULL) 2179 mlx5_free(priv->rss_conf.rss_key); 2180 if (priv->reta_idx != NULL) 2181 mlx5_free(priv->reta_idx); 2182 if (priv->sh->dev_cap.vf) 2183 mlx5_os_mac_addr_flush(dev); 2184 if (priv->nl_socket_route >= 0) 2185 close(priv->nl_socket_route); 2186 if (priv->nl_socket_rdma >= 0) 2187 close(priv->nl_socket_rdma); 2188 if (priv->vmwa_context) 2189 mlx5_vlan_vmwa_exit(priv->vmwa_context); 2190 ret = mlx5_hrxq_verify(dev); 2191 if (ret) 2192 DRV_LOG(WARNING, "port %u some hash Rx queue still remain", 2193 dev->data->port_id); 2194 ret = mlx5_ind_table_obj_verify(dev); 2195 if (ret) 2196 DRV_LOG(WARNING, "port %u some indirection table still remain", 2197 dev->data->port_id); 2198 ret = mlx5_rxq_obj_verify(dev); 2199 if (ret) 2200 DRV_LOG(WARNING, "port %u some Rx queue objects still remain", 2201 dev->data->port_id); 2202 ret = mlx5_ext_rxq_verify(dev); 2203 if (ret) 2204 DRV_LOG(WARNING, "Port %u some external RxQ still remain.", 2205 dev->data->port_id); 2206 ret = mlx5_rxq_verify(dev); 2207 if (ret) 2208 DRV_LOG(WARNING, "port %u some Rx queues still remain", 2209 dev->data->port_id); 2210 ret = mlx5_txq_obj_verify(dev); 2211 if (ret) 2212 DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain", 2213 dev->data->port_id); 2214 ret = mlx5_txq_verify(dev); 2215 if (ret) 2216 DRV_LOG(WARNING, "port %u some Tx queues still remain", 2217 dev->data->port_id); 2218 ret = mlx5_flow_verify(dev); 2219 if (ret) 2220 DRV_LOG(WARNING, "port %u some flows still remain", 2221 dev->data->port_id); 2222 if (priv->hrxqs) 2223 mlx5_list_destroy(priv->hrxqs); 2224 mlx5_free(priv->ext_rxqs); 2225 priv->sh->port[priv->dev_port - 1].nl_ih_port_id = RTE_MAX_ETHPORTS; 2226 /* 2227 * The interrupt handler port id must be reset before priv is reset 2228 * since 'mlx5_dev_interrupt_nl_cb' uses priv. 2229 */ 2230 rte_io_wmb(); 2231 /* 2232 * Free the shared context in last turn, because the cleanup 2233 * routines above may use some shared fields, like 2234 * mlx5_os_mac_addr_flush() uses ibdev_path for retrieving 2235 * ifindex if Netlink fails. 2236 */ 2237 mlx5_free_shared_dev_ctx(priv->sh); 2238 if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 2239 unsigned int c = 0; 2240 uint16_t port_id; 2241 2242 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 2243 struct mlx5_priv *opriv = 2244 rte_eth_devices[port_id].data->dev_private; 2245 2246 if (!opriv || 2247 opriv->domain_id != priv->domain_id || 2248 &rte_eth_devices[port_id] == dev) 2249 continue; 2250 ++c; 2251 break; 2252 } 2253 if (!c) 2254 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 2255 } 2256 memset(priv, 0, sizeof(*priv)); 2257 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 2258 /* 2259 * Reset mac_addrs to NULL such that it is not freed as part of 2260 * rte_eth_dev_release_port(). mac_addrs is part of dev_private so 2261 * it is freed when dev_private is freed. 2262 */ 2263 dev->data->mac_addrs = NULL; 2264 return 0; 2265 } 2266 2267 const struct eth_dev_ops mlx5_dev_ops = { 2268 .dev_configure = mlx5_dev_configure, 2269 .dev_start = mlx5_dev_start, 2270 .dev_stop = mlx5_dev_stop, 2271 .dev_set_link_down = mlx5_set_link_down, 2272 .dev_set_link_up = mlx5_set_link_up, 2273 .dev_close = mlx5_dev_close, 2274 .promiscuous_enable = mlx5_promiscuous_enable, 2275 .promiscuous_disable = mlx5_promiscuous_disable, 2276 .allmulticast_enable = mlx5_allmulticast_enable, 2277 .allmulticast_disable = mlx5_allmulticast_disable, 2278 .link_update = mlx5_link_update, 2279 .stats_get = mlx5_stats_get, 2280 .stats_reset = mlx5_stats_reset, 2281 .xstats_get = mlx5_xstats_get, 2282 .xstats_reset = mlx5_xstats_reset, 2283 .xstats_get_names = mlx5_xstats_get_names, 2284 .fw_version_get = mlx5_fw_version_get, 2285 .dev_infos_get = mlx5_dev_infos_get, 2286 .representor_info_get = mlx5_representor_info_get, 2287 .read_clock = mlx5_txpp_read_clock, 2288 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 2289 .vlan_filter_set = mlx5_vlan_filter_set, 2290 .rx_queue_setup = mlx5_rx_queue_setup, 2291 .rx_queue_avail_thresh_set = mlx5_rx_queue_lwm_set, 2292 .rx_queue_avail_thresh_query = mlx5_rx_queue_lwm_query, 2293 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 2294 .tx_queue_setup = mlx5_tx_queue_setup, 2295 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 2296 .rx_queue_release = mlx5_rx_queue_release, 2297 .tx_queue_release = mlx5_tx_queue_release, 2298 .rx_queue_start = mlx5_rx_queue_start, 2299 .rx_queue_stop = mlx5_rx_queue_stop, 2300 .tx_queue_start = mlx5_tx_queue_start, 2301 .tx_queue_stop = mlx5_tx_queue_stop, 2302 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 2303 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 2304 .mac_addr_remove = mlx5_mac_addr_remove, 2305 .mac_addr_add = mlx5_mac_addr_add, 2306 .mac_addr_set = mlx5_mac_addr_set, 2307 .set_mc_addr_list = mlx5_set_mc_addr_list, 2308 .mtu_set = mlx5_dev_set_mtu, 2309 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 2310 .vlan_offload_set = mlx5_vlan_offload_set, 2311 .reta_update = mlx5_dev_rss_reta_update, 2312 .reta_query = mlx5_dev_rss_reta_query, 2313 .rss_hash_update = mlx5_rss_hash_update, 2314 .rss_hash_conf_get = mlx5_rss_hash_conf_get, 2315 .flow_ops_get = mlx5_flow_ops_get, 2316 .rxq_info_get = mlx5_rxq_info_get, 2317 .txq_info_get = mlx5_txq_info_get, 2318 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2319 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2320 .rx_queue_intr_enable = mlx5_rx_intr_enable, 2321 .rx_queue_intr_disable = mlx5_rx_intr_disable, 2322 .is_removed = mlx5_is_removed, 2323 .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, 2324 .get_module_info = mlx5_get_module_info, 2325 .get_module_eeprom = mlx5_get_module_eeprom, 2326 .hairpin_cap_get = mlx5_hairpin_cap_get, 2327 .mtr_ops_get = mlx5_flow_meter_ops_get, 2328 .hairpin_bind = mlx5_hairpin_bind, 2329 .hairpin_unbind = mlx5_hairpin_unbind, 2330 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 2331 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 2332 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 2333 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 2334 .get_monitor_addr = mlx5_get_monitor_addr, 2335 .count_aggr_ports = mlx5_count_aggr_ports, 2336 .map_aggr_tx_affinity = mlx5_map_aggr_tx_affinity, 2337 }; 2338 2339 /* Available operations from secondary process. */ 2340 const struct eth_dev_ops mlx5_dev_sec_ops = { 2341 .stats_get = mlx5_stats_get, 2342 .stats_reset = mlx5_stats_reset, 2343 .xstats_get = mlx5_xstats_get, 2344 .xstats_reset = mlx5_xstats_reset, 2345 .xstats_get_names = mlx5_xstats_get_names, 2346 .fw_version_get = mlx5_fw_version_get, 2347 .dev_infos_get = mlx5_dev_infos_get, 2348 .representor_info_get = mlx5_representor_info_get, 2349 .read_clock = mlx5_txpp_read_clock, 2350 .rx_queue_start = mlx5_rx_queue_start, 2351 .rx_queue_stop = mlx5_rx_queue_stop, 2352 .tx_queue_start = mlx5_tx_queue_start, 2353 .tx_queue_stop = mlx5_tx_queue_stop, 2354 .rxq_info_get = mlx5_rxq_info_get, 2355 .txq_info_get = mlx5_txq_info_get, 2356 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2357 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2358 .get_module_info = mlx5_get_module_info, 2359 .get_module_eeprom = mlx5_get_module_eeprom, 2360 .count_aggr_ports = mlx5_count_aggr_ports, 2361 .map_aggr_tx_affinity = mlx5_map_aggr_tx_affinity, 2362 }; 2363 2364 /* Available operations in flow isolated mode. */ 2365 const struct eth_dev_ops mlx5_dev_ops_isolate = { 2366 .dev_configure = mlx5_dev_configure, 2367 .dev_start = mlx5_dev_start, 2368 .dev_stop = mlx5_dev_stop, 2369 .dev_set_link_down = mlx5_set_link_down, 2370 .dev_set_link_up = mlx5_set_link_up, 2371 .dev_close = mlx5_dev_close, 2372 .promiscuous_enable = mlx5_promiscuous_enable, 2373 .promiscuous_disable = mlx5_promiscuous_disable, 2374 .allmulticast_enable = mlx5_allmulticast_enable, 2375 .allmulticast_disable = mlx5_allmulticast_disable, 2376 .link_update = mlx5_link_update, 2377 .stats_get = mlx5_stats_get, 2378 .stats_reset = mlx5_stats_reset, 2379 .xstats_get = mlx5_xstats_get, 2380 .xstats_reset = mlx5_xstats_reset, 2381 .xstats_get_names = mlx5_xstats_get_names, 2382 .fw_version_get = mlx5_fw_version_get, 2383 .dev_infos_get = mlx5_dev_infos_get, 2384 .representor_info_get = mlx5_representor_info_get, 2385 .read_clock = mlx5_txpp_read_clock, 2386 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 2387 .vlan_filter_set = mlx5_vlan_filter_set, 2388 .rx_queue_setup = mlx5_rx_queue_setup, 2389 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 2390 .tx_queue_setup = mlx5_tx_queue_setup, 2391 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 2392 .rx_queue_release = mlx5_rx_queue_release, 2393 .tx_queue_release = mlx5_tx_queue_release, 2394 .rx_queue_start = mlx5_rx_queue_start, 2395 .rx_queue_stop = mlx5_rx_queue_stop, 2396 .tx_queue_start = mlx5_tx_queue_start, 2397 .tx_queue_stop = mlx5_tx_queue_stop, 2398 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 2399 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 2400 .mac_addr_remove = mlx5_mac_addr_remove, 2401 .mac_addr_add = mlx5_mac_addr_add, 2402 .mac_addr_set = mlx5_mac_addr_set, 2403 .set_mc_addr_list = mlx5_set_mc_addr_list, 2404 .mtu_set = mlx5_dev_set_mtu, 2405 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 2406 .vlan_offload_set = mlx5_vlan_offload_set, 2407 .flow_ops_get = mlx5_flow_ops_get, 2408 .rxq_info_get = mlx5_rxq_info_get, 2409 .txq_info_get = mlx5_txq_info_get, 2410 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2411 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2412 .rx_queue_intr_enable = mlx5_rx_intr_enable, 2413 .rx_queue_intr_disable = mlx5_rx_intr_disable, 2414 .is_removed = mlx5_is_removed, 2415 .get_module_info = mlx5_get_module_info, 2416 .get_module_eeprom = mlx5_get_module_eeprom, 2417 .hairpin_cap_get = mlx5_hairpin_cap_get, 2418 .mtr_ops_get = mlx5_flow_meter_ops_get, 2419 .hairpin_bind = mlx5_hairpin_bind, 2420 .hairpin_unbind = mlx5_hairpin_unbind, 2421 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 2422 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 2423 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 2424 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 2425 .get_monitor_addr = mlx5_get_monitor_addr, 2426 .count_aggr_ports = mlx5_count_aggr_ports, 2427 .map_aggr_tx_affinity = mlx5_map_aggr_tx_affinity, 2428 }; 2429 2430 /** 2431 * Verify and store value for device argument. 2432 * 2433 * @param[in] key 2434 * Key argument to verify. 2435 * @param[in] val 2436 * Value associated with key. 2437 * @param opaque 2438 * User data. 2439 * 2440 * @return 2441 * 0 on success, a negative errno value otherwise and rte_errno is set. 2442 */ 2443 static int 2444 mlx5_port_args_check_handler(const char *key, const char *val, void *opaque) 2445 { 2446 struct mlx5_port_config *config = opaque; 2447 signed long tmp; 2448 2449 /* No-op, port representors are processed in mlx5_dev_spawn(). */ 2450 if (!strcmp(MLX5_REPRESENTOR, key)) 2451 return 0; 2452 errno = 0; 2453 tmp = strtol(val, NULL, 0); 2454 if (errno) { 2455 rte_errno = errno; 2456 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); 2457 return -rte_errno; 2458 } 2459 if (tmp < 0) { 2460 /* Negative values are acceptable for some keys only. */ 2461 rte_errno = EINVAL; 2462 DRV_LOG(WARNING, "%s: invalid negative value \"%s\"", key, val); 2463 return -rte_errno; 2464 } 2465 if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) { 2466 if ((tmp & ~MLX5_RXQ_ENH_CQE_COMP_MASK) > 2467 MLX5_CQE_RESP_FORMAT_L34H_STRIDX) { 2468 DRV_LOG(ERR, "invalid CQE compression " 2469 "format parameter"); 2470 rte_errno = EINVAL; 2471 return -rte_errno; 2472 } 2473 config->cqe_comp = !!tmp; 2474 config->cqe_comp_fmt = tmp & ~MLX5_RXQ_ENH_CQE_COMP_MASK; 2475 config->enh_cqe_comp = !!(tmp & MLX5_RXQ_ENH_CQE_COMP_MASK); 2476 } else if (strcmp(MLX5_RXQ_PKT_PAD_EN, key) == 0) { 2477 config->hw_padding = !!tmp; 2478 } else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) { 2479 config->mprq.enabled = !!tmp; 2480 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) { 2481 config->mprq.log_stride_num = tmp; 2482 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_SIZE, key) == 0) { 2483 config->mprq.log_stride_size = tmp; 2484 } else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) { 2485 config->mprq.max_memcpy_len = tmp; 2486 } else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) { 2487 config->mprq.min_rxqs_num = tmp; 2488 } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) { 2489 DRV_LOG(WARNING, "%s: deprecated parameter," 2490 " converted to txq_inline_max", key); 2491 config->txq_inline_max = tmp; 2492 } else if (strcmp(MLX5_TXQ_INLINE_MAX, key) == 0) { 2493 config->txq_inline_max = tmp; 2494 } else if (strcmp(MLX5_TXQ_INLINE_MIN, key) == 0) { 2495 config->txq_inline_min = tmp; 2496 } else if (strcmp(MLX5_TXQ_INLINE_MPW, key) == 0) { 2497 config->txq_inline_mpw = tmp; 2498 } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) { 2499 config->txqs_inline = tmp; 2500 } else if (strcmp(MLX5_TXQS_MAX_VEC, key) == 0) { 2501 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2502 } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { 2503 config->mps = !!tmp; 2504 } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { 2505 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2506 } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { 2507 DRV_LOG(WARNING, "%s: deprecated parameter," 2508 " converted to txq_inline_mpw", key); 2509 config->txq_inline_mpw = tmp; 2510 } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) { 2511 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2512 } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { 2513 config->rx_vec_en = !!tmp; 2514 } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) { 2515 config->max_dump_files_num = tmp; 2516 } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) { 2517 config->lro_timeout = tmp; 2518 } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) { 2519 config->log_hp_size = tmp; 2520 } else if (strcmp(MLX5_DELAY_DROP, key) == 0) { 2521 config->std_delay_drop = !!(tmp & MLX5_DELAY_DROP_STANDARD); 2522 config->hp_delay_drop = !!(tmp & MLX5_DELAY_DROP_HAIRPIN); 2523 } 2524 return 0; 2525 } 2526 2527 /** 2528 * Parse user port parameters and adjust them according to device capabilities. 2529 * 2530 * @param priv 2531 * Pointer to shared device context. 2532 * @param mkvlist 2533 * Pointer to mlx5 kvargs control, can be NULL if there is no devargs. 2534 * @param config 2535 * Pointer to port configuration structure. 2536 * 2537 * @return 2538 * 0 on success, a negative errno value otherwise and rte_errno is set. 2539 */ 2540 int 2541 mlx5_port_args_config(struct mlx5_priv *priv, struct mlx5_kvargs_ctrl *mkvlist, 2542 struct mlx5_port_config *config) 2543 { 2544 struct mlx5_hca_attr *hca_attr = &priv->sh->cdev->config.hca_attr; 2545 struct mlx5_dev_cap *dev_cap = &priv->sh->dev_cap; 2546 bool devx = priv->sh->cdev->config.devx; 2547 const char **params = (const char *[]){ 2548 MLX5_RXQ_CQE_COMP_EN, 2549 MLX5_RXQ_PKT_PAD_EN, 2550 MLX5_RX_MPRQ_EN, 2551 MLX5_RX_MPRQ_LOG_STRIDE_NUM, 2552 MLX5_RX_MPRQ_LOG_STRIDE_SIZE, 2553 MLX5_RX_MPRQ_MAX_MEMCPY_LEN, 2554 MLX5_RXQS_MIN_MPRQ, 2555 MLX5_TXQ_INLINE, 2556 MLX5_TXQ_INLINE_MIN, 2557 MLX5_TXQ_INLINE_MAX, 2558 MLX5_TXQ_INLINE_MPW, 2559 MLX5_TXQS_MIN_INLINE, 2560 MLX5_TXQS_MAX_VEC, 2561 MLX5_TXQ_MPW_EN, 2562 MLX5_TXQ_MPW_HDR_DSEG_EN, 2563 MLX5_TXQ_MAX_INLINE_LEN, 2564 MLX5_TX_VEC_EN, 2565 MLX5_RX_VEC_EN, 2566 MLX5_REPRESENTOR, 2567 MLX5_MAX_DUMP_FILES_NUM, 2568 MLX5_LRO_TIMEOUT_USEC, 2569 MLX5_HP_BUF_SIZE, 2570 MLX5_DELAY_DROP, 2571 NULL, 2572 }; 2573 int ret = 0; 2574 2575 /* Default configuration. */ 2576 memset(config, 0, sizeof(*config)); 2577 config->mps = MLX5_ARG_UNSET; 2578 config->cqe_comp = 1; 2579 config->rx_vec_en = 1; 2580 config->txq_inline_max = MLX5_ARG_UNSET; 2581 config->txq_inline_min = MLX5_ARG_UNSET; 2582 config->txq_inline_mpw = MLX5_ARG_UNSET; 2583 config->txqs_inline = MLX5_ARG_UNSET; 2584 config->mprq.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN; 2585 config->mprq.min_rxqs_num = MLX5_MPRQ_MIN_RXQS; 2586 config->mprq.log_stride_num = MLX5_MPRQ_DEFAULT_LOG_STRIDE_NUM; 2587 config->log_hp_size = MLX5_ARG_UNSET; 2588 config->std_delay_drop = 0; 2589 config->hp_delay_drop = 0; 2590 if (mkvlist != NULL) { 2591 /* Process parameters. */ 2592 ret = mlx5_kvargs_process(mkvlist, params, 2593 mlx5_port_args_check_handler, config); 2594 if (ret) { 2595 DRV_LOG(ERR, "Failed to process port arguments: %s", 2596 strerror(rte_errno)); 2597 return -rte_errno; 2598 } 2599 } 2600 /* Adjust parameters according to device capabilities. */ 2601 if (config->hw_padding && !dev_cap->hw_padding) { 2602 DRV_LOG(DEBUG, "Rx end alignment padding isn't supported."); 2603 config->hw_padding = 0; 2604 } else if (config->hw_padding) { 2605 DRV_LOG(DEBUG, "Rx end alignment padding is enabled."); 2606 } 2607 /* 2608 * MPW is disabled by default, while the Enhanced MPW is enabled 2609 * by default. 2610 */ 2611 if (config->mps == MLX5_ARG_UNSET) 2612 config->mps = (dev_cap->mps == MLX5_MPW_ENHANCED) ? 2613 MLX5_MPW_ENHANCED : MLX5_MPW_DISABLED; 2614 else 2615 config->mps = config->mps ? dev_cap->mps : MLX5_MPW_DISABLED; 2616 DRV_LOG(INFO, "%sMPS is %s", 2617 config->mps == MLX5_MPW_ENHANCED ? "enhanced " : 2618 config->mps == MLX5_MPW ? "legacy " : "", 2619 config->mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); 2620 if (priv->sh->config.lro_allowed) { 2621 /* 2622 * If LRO timeout is not configured by application, 2623 * use the minimal supported value. 2624 */ 2625 if (!config->lro_timeout) 2626 config->lro_timeout = 2627 hca_attr->lro_timer_supported_periods[0]; 2628 DRV_LOG(DEBUG, "LRO session timeout set to %d usec.", 2629 config->lro_timeout); 2630 } 2631 if (config->cqe_comp && !dev_cap->cqe_comp) { 2632 DRV_LOG(WARNING, "Rx CQE 128B compression is not supported."); 2633 config->cqe_comp = 0; 2634 } 2635 if (config->cqe_comp_fmt == MLX5_CQE_RESP_FORMAT_FTAG_STRIDX && 2636 (!devx || !hca_attr->mini_cqe_resp_flow_tag)) { 2637 DRV_LOG(WARNING, 2638 "Flow Tag CQE compression format isn't supported."); 2639 config->cqe_comp = 0; 2640 } 2641 if (config->cqe_comp_fmt == MLX5_CQE_RESP_FORMAT_L34H_STRIDX && 2642 (!devx || !hca_attr->mini_cqe_resp_l3_l4_tag)) { 2643 DRV_LOG(WARNING, 2644 "L3/L4 Header CQE compression format isn't supported."); 2645 config->cqe_comp = 0; 2646 } 2647 if (config->enh_cqe_comp && !hca_attr->enhanced_cqe_compression) { 2648 DRV_LOG(WARNING, 2649 "Enhanced CQE compression isn't supported."); 2650 config->enh_cqe_comp = 0; 2651 } 2652 DRV_LOG(DEBUG, "%sRx CQE compression is %ssupported.", 2653 config->enh_cqe_comp ? "Enhanced " : "", 2654 config->cqe_comp ? "" : "not "); 2655 if ((config->std_delay_drop || config->hp_delay_drop) && 2656 !dev_cap->rq_delay_drop_en) { 2657 config->std_delay_drop = 0; 2658 config->hp_delay_drop = 0; 2659 DRV_LOG(WARNING, "dev_port-%u: Rxq delay drop isn't supported.", 2660 priv->dev_port); 2661 } 2662 if (config->mprq.enabled && !priv->sh->dev_cap.mprq.enabled) { 2663 DRV_LOG(WARNING, "Multi-Packet RQ isn't supported."); 2664 config->mprq.enabled = 0; 2665 } 2666 if (config->max_dump_files_num == 0) 2667 config->max_dump_files_num = 128; 2668 /* Detect minimal data bytes to inline. */ 2669 mlx5_set_min_inline(priv); 2670 DRV_LOG(DEBUG, "VLAN insertion in WQE is %ssupported.", 2671 config->hw_vlan_insert ? "" : "not "); 2672 DRV_LOG(DEBUG, "\"rxq_pkt_pad_en\" is %u.", config->hw_padding); 2673 DRV_LOG(DEBUG, "\"rxq_cqe_comp_en\" is %u.", config->cqe_comp); 2674 DRV_LOG(DEBUG, "\"cqe_comp_fmt\" is %u.", config->cqe_comp_fmt); 2675 DRV_LOG(DEBUG, "\"enh_cqe_comp\" is %u.", config->enh_cqe_comp); 2676 DRV_LOG(DEBUG, "\"rx_vec_en\" is %u.", config->rx_vec_en); 2677 DRV_LOG(DEBUG, "Standard \"delay_drop\" is %u.", 2678 config->std_delay_drop); 2679 DRV_LOG(DEBUG, "Hairpin \"delay_drop\" is %u.", config->hp_delay_drop); 2680 DRV_LOG(DEBUG, "\"max_dump_files_num\" is %u.", 2681 config->max_dump_files_num); 2682 DRV_LOG(DEBUG, "\"log_hp_size\" is %u.", config->log_hp_size); 2683 DRV_LOG(DEBUG, "\"mprq_en\" is %u.", config->mprq.enabled); 2684 DRV_LOG(DEBUG, "\"mprq_log_stride_num\" is %u.", 2685 config->mprq.log_stride_num); 2686 DRV_LOG(DEBUG, "\"mprq_log_stride_size\" is %u.", 2687 config->mprq.log_stride_size); 2688 DRV_LOG(DEBUG, "\"mprq_max_memcpy_len\" is %u.", 2689 config->mprq.max_memcpy_len); 2690 DRV_LOG(DEBUG, "\"rxqs_min_mprq\" is %u.", config->mprq.min_rxqs_num); 2691 DRV_LOG(DEBUG, "\"lro_timeout_usec\" is %u.", config->lro_timeout); 2692 DRV_LOG(DEBUG, "\"txq_mpw_en\" is %d.", config->mps); 2693 DRV_LOG(DEBUG, "\"txqs_min_inline\" is %d.", config->txqs_inline); 2694 DRV_LOG(DEBUG, "\"txq_inline_min\" is %d.", config->txq_inline_min); 2695 DRV_LOG(DEBUG, "\"txq_inline_max\" is %d.", config->txq_inline_max); 2696 DRV_LOG(DEBUG, "\"txq_inline_mpw\" is %d.", config->txq_inline_mpw); 2697 return 0; 2698 } 2699 2700 /** 2701 * Print the key for device argument. 2702 * 2703 * It is "dummy" handler whose whole purpose is to enable using 2704 * mlx5_kvargs_process() function which set devargs as used. 2705 * 2706 * @param key 2707 * Key argument. 2708 * @param val 2709 * Value associated with key, unused. 2710 * @param opaque 2711 * Unused, can be NULL. 2712 * 2713 * @return 2714 * 0 on success, function cannot fail. 2715 */ 2716 static int 2717 mlx5_dummy_handler(const char *key, const char *val, void *opaque) 2718 { 2719 DRV_LOG(DEBUG, "\tKey: \"%s\" is set as used.", key); 2720 RTE_SET_USED(opaque); 2721 RTE_SET_USED(val); 2722 return 0; 2723 } 2724 2725 /** 2726 * Set requested devargs as used when device is already spawned. 2727 * 2728 * It is necessary since it is valid to ask probe again for existing device, 2729 * if its devargs don't assign as used, mlx5_kvargs_validate() will fail. 2730 * 2731 * @param name 2732 * Name of the existing device. 2733 * @param port_id 2734 * Port identifier of the device. 2735 * @param mkvlist 2736 * Pointer to mlx5 kvargs control to sign as used. 2737 */ 2738 void 2739 mlx5_port_args_set_used(const char *name, uint16_t port_id, 2740 struct mlx5_kvargs_ctrl *mkvlist) 2741 { 2742 const char **params = (const char *[]){ 2743 MLX5_RXQ_CQE_COMP_EN, 2744 MLX5_RXQ_PKT_PAD_EN, 2745 MLX5_RX_MPRQ_EN, 2746 MLX5_RX_MPRQ_LOG_STRIDE_NUM, 2747 MLX5_RX_MPRQ_LOG_STRIDE_SIZE, 2748 MLX5_RX_MPRQ_MAX_MEMCPY_LEN, 2749 MLX5_RXQS_MIN_MPRQ, 2750 MLX5_TXQ_INLINE, 2751 MLX5_TXQ_INLINE_MIN, 2752 MLX5_TXQ_INLINE_MAX, 2753 MLX5_TXQ_INLINE_MPW, 2754 MLX5_TXQS_MIN_INLINE, 2755 MLX5_TXQS_MAX_VEC, 2756 MLX5_TXQ_MPW_EN, 2757 MLX5_TXQ_MPW_HDR_DSEG_EN, 2758 MLX5_TXQ_MAX_INLINE_LEN, 2759 MLX5_TX_VEC_EN, 2760 MLX5_RX_VEC_EN, 2761 MLX5_REPRESENTOR, 2762 MLX5_MAX_DUMP_FILES_NUM, 2763 MLX5_LRO_TIMEOUT_USEC, 2764 MLX5_HP_BUF_SIZE, 2765 MLX5_DELAY_DROP, 2766 NULL, 2767 }; 2768 2769 /* Secondary process should not handle devargs. */ 2770 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2771 return; 2772 MLX5_ASSERT(mkvlist != NULL); 2773 DRV_LOG(DEBUG, "Ethernet device \"%s\" for port %u " 2774 "already exists, set devargs as used:", name, port_id); 2775 /* This function cannot fail with this handler. */ 2776 mlx5_kvargs_process(mkvlist, params, mlx5_dummy_handler, NULL); 2777 } 2778 2779 /** 2780 * Check sibling device configurations when probing again. 2781 * 2782 * Sibling devices sharing infiniband device context should have compatible 2783 * configurations. This regards representors and bonding device. 2784 * 2785 * @param cdev 2786 * Pointer to mlx5 device structure. 2787 * @param mkvlist 2788 * Pointer to mlx5 kvargs control, can be NULL if there is no devargs. 2789 * 2790 * @return 2791 * 0 on success, a negative errno value otherwise and rte_errno is set. 2792 */ 2793 int 2794 mlx5_probe_again_args_validate(struct mlx5_common_device *cdev, 2795 struct mlx5_kvargs_ctrl *mkvlist) 2796 { 2797 struct mlx5_dev_ctx_shared *sh = NULL; 2798 struct mlx5_sh_config *config; 2799 int ret; 2800 2801 /* Secondary process should not handle devargs. */ 2802 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2803 return 0; 2804 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 2805 /* Search for IB context by common device pointer. */ 2806 LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) 2807 if (sh->cdev == cdev) 2808 break; 2809 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 2810 /* There is sh for this device -> it isn't probe again. */ 2811 if (sh == NULL) 2812 return 0; 2813 config = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 2814 sizeof(struct mlx5_sh_config), 2815 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 2816 if (config == NULL) { 2817 rte_errno = -ENOMEM; 2818 return -rte_errno; 2819 } 2820 /* 2821 * Creates a temporary IB context configure structure according to new 2822 * devargs attached in probing again. 2823 */ 2824 ret = mlx5_shared_dev_ctx_args_config(sh, mkvlist, config); 2825 if (ret) { 2826 DRV_LOG(ERR, "Failed to process device configure: %s", 2827 strerror(rte_errno)); 2828 mlx5_free(config); 2829 return ret; 2830 } 2831 /* 2832 * Checks the match between the temporary structure and the existing 2833 * IB context structure. 2834 */ 2835 if (sh->config.dv_flow_en ^ config->dv_flow_en) { 2836 DRV_LOG(ERR, "\"dv_flow_en\" " 2837 "configuration mismatch for shared %s context.", 2838 sh->ibdev_name); 2839 goto error; 2840 } 2841 if ((sh->config.dv_xmeta_en ^ config->dv_xmeta_en) || 2842 (sh->config.dv_miss_info ^ config->dv_miss_info)) { 2843 DRV_LOG(ERR, "\"dv_xmeta_en\" " 2844 "configuration mismatch for shared %s context.", 2845 sh->ibdev_name); 2846 goto error; 2847 } 2848 if (sh->config.dv_esw_en ^ config->dv_esw_en) { 2849 DRV_LOG(ERR, "\"dv_esw_en\" " 2850 "configuration mismatch for shared %s context.", 2851 sh->ibdev_name); 2852 goto error; 2853 } 2854 if (sh->config.reclaim_mode ^ config->reclaim_mode) { 2855 DRV_LOG(ERR, "\"reclaim_mode\" " 2856 "configuration mismatch for shared %s context.", 2857 sh->ibdev_name); 2858 goto error; 2859 } 2860 if (sh->config.allow_duplicate_pattern ^ 2861 config->allow_duplicate_pattern) { 2862 DRV_LOG(ERR, "\"allow_duplicate_pattern\" " 2863 "configuration mismatch for shared %s context.", 2864 sh->ibdev_name); 2865 goto error; 2866 } 2867 if (sh->config.fdb_def_rule ^ config->fdb_def_rule) { 2868 DRV_LOG(ERR, "\"fdb_def_rule_en\" configuration mismatch for shared %s context.", 2869 sh->ibdev_name); 2870 goto error; 2871 } 2872 if (sh->config.l3_vxlan_en ^ config->l3_vxlan_en) { 2873 DRV_LOG(ERR, "\"l3_vxlan_en\" " 2874 "configuration mismatch for shared %s context.", 2875 sh->ibdev_name); 2876 goto error; 2877 } 2878 if (sh->config.decap_en ^ config->decap_en) { 2879 DRV_LOG(ERR, "\"decap_en\" " 2880 "configuration mismatch for shared %s context.", 2881 sh->ibdev_name); 2882 goto error; 2883 } 2884 if (sh->config.lacp_by_user ^ config->lacp_by_user) { 2885 DRV_LOG(ERR, "\"lacp_by_user\" " 2886 "configuration mismatch for shared %s context.", 2887 sh->ibdev_name); 2888 goto error; 2889 } 2890 if (sh->config.tx_pp ^ config->tx_pp) { 2891 DRV_LOG(ERR, "\"tx_pp\" " 2892 "configuration mismatch for shared %s context.", 2893 sh->ibdev_name); 2894 goto error; 2895 } 2896 if (sh->config.tx_skew ^ config->tx_skew) { 2897 DRV_LOG(ERR, "\"tx_skew\" " 2898 "configuration mismatch for shared %s context.", 2899 sh->ibdev_name); 2900 goto error; 2901 } 2902 mlx5_free(config); 2903 return 0; 2904 error: 2905 mlx5_free(config); 2906 rte_errno = EINVAL; 2907 return -rte_errno; 2908 } 2909 2910 /** 2911 * Configures the minimal amount of data to inline into WQE 2912 * while sending packets. 2913 * 2914 * - the txq_inline_min has the maximal priority, if this 2915 * key is specified in devargs 2916 * - if DevX is enabled the inline mode is queried from the 2917 * device (HCA attributes and NIC vport context if needed). 2918 * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4 Lx 2919 * and none (0 bytes) for other NICs 2920 * 2921 * @param priv 2922 * Pointer to the private device data structure. 2923 */ 2924 void 2925 mlx5_set_min_inline(struct mlx5_priv *priv) 2926 { 2927 struct mlx5_hca_attr *hca_attr = &priv->sh->cdev->config.hca_attr; 2928 struct mlx5_port_config *config = &priv->config; 2929 2930 if (config->txq_inline_min != MLX5_ARG_UNSET) { 2931 /* Application defines size of inlined data explicitly. */ 2932 if (priv->pci_dev != NULL) { 2933 switch (priv->pci_dev->id.device_id) { 2934 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2935 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2936 if (config->txq_inline_min < 2937 (int)MLX5_INLINE_HSIZE_L2) { 2938 DRV_LOG(DEBUG, 2939 "txq_inline_mix aligned to minimal ConnectX-4 required value %d", 2940 (int)MLX5_INLINE_HSIZE_L2); 2941 config->txq_inline_min = 2942 MLX5_INLINE_HSIZE_L2; 2943 } 2944 break; 2945 } 2946 } 2947 goto exit; 2948 } 2949 if (hca_attr->eth_net_offloads) { 2950 /* We have DevX enabled, inline mode queried successfully. */ 2951 switch (hca_attr->wqe_inline_mode) { 2952 case MLX5_CAP_INLINE_MODE_L2: 2953 /* outer L2 header must be inlined. */ 2954 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2955 goto exit; 2956 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2957 /* No inline data are required by NIC. */ 2958 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2959 config->hw_vlan_insert = 2960 hca_attr->wqe_vlan_insert; 2961 DRV_LOG(DEBUG, "Tx VLAN insertion is supported"); 2962 goto exit; 2963 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2964 /* inline mode is defined by NIC vport context. */ 2965 if (!hca_attr->eth_virt) 2966 break; 2967 switch (hca_attr->vport_inline_mode) { 2968 case MLX5_INLINE_MODE_NONE: 2969 config->txq_inline_min = 2970 MLX5_INLINE_HSIZE_NONE; 2971 goto exit; 2972 case MLX5_INLINE_MODE_L2: 2973 config->txq_inline_min = 2974 MLX5_INLINE_HSIZE_L2; 2975 goto exit; 2976 case MLX5_INLINE_MODE_IP: 2977 config->txq_inline_min = 2978 MLX5_INLINE_HSIZE_L3; 2979 goto exit; 2980 case MLX5_INLINE_MODE_TCP_UDP: 2981 config->txq_inline_min = 2982 MLX5_INLINE_HSIZE_L4; 2983 goto exit; 2984 case MLX5_INLINE_MODE_INNER_L2: 2985 config->txq_inline_min = 2986 MLX5_INLINE_HSIZE_INNER_L2; 2987 goto exit; 2988 case MLX5_INLINE_MODE_INNER_IP: 2989 config->txq_inline_min = 2990 MLX5_INLINE_HSIZE_INNER_L3; 2991 goto exit; 2992 case MLX5_INLINE_MODE_INNER_TCP_UDP: 2993 config->txq_inline_min = 2994 MLX5_INLINE_HSIZE_INNER_L4; 2995 goto exit; 2996 } 2997 } 2998 } 2999 if (priv->pci_dev == NULL) { 3000 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 3001 goto exit; 3002 } 3003 /* 3004 * We get here if we are unable to deduce 3005 * inline data size with DevX. Try PCI ID 3006 * to determine old NICs. 3007 */ 3008 switch (priv->pci_dev->id.device_id) { 3009 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 3010 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 3011 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: 3012 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 3013 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 3014 config->hw_vlan_insert = 0; 3015 break; 3016 case PCI_DEVICE_ID_MELLANOX_CONNECTX5: 3017 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 3018 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: 3019 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 3020 /* 3021 * These NICs support VLAN insertion from WQE and 3022 * report the wqe_vlan_insert flag. But there is the bug 3023 * and PFC control may be broken, so disable feature. 3024 */ 3025 config->hw_vlan_insert = 0; 3026 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 3027 break; 3028 default: 3029 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 3030 break; 3031 } 3032 exit: 3033 DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min); 3034 } 3035 3036 /** 3037 * Configures the metadata mask fields in the shared context. 3038 * 3039 * @param [in] dev 3040 * Pointer to Ethernet device. 3041 */ 3042 void 3043 mlx5_set_metadata_mask(struct rte_eth_dev *dev) 3044 { 3045 struct mlx5_priv *priv = dev->data->dev_private; 3046 struct mlx5_dev_ctx_shared *sh = priv->sh; 3047 uint32_t meta, mark, reg_c0; 3048 3049 reg_c0 = ~priv->vport_meta_mask; 3050 switch (sh->config.dv_xmeta_en) { 3051 case MLX5_XMETA_MODE_LEGACY: 3052 meta = UINT32_MAX; 3053 mark = MLX5_FLOW_MARK_MASK; 3054 break; 3055 case MLX5_XMETA_MODE_META16: 3056 meta = reg_c0 >> rte_bsf32(reg_c0); 3057 mark = MLX5_FLOW_MARK_MASK; 3058 break; 3059 case MLX5_XMETA_MODE_META32: 3060 meta = UINT32_MAX; 3061 mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK; 3062 break; 3063 case MLX5_XMETA_MODE_META32_HWS: 3064 meta = UINT32_MAX; 3065 mark = MLX5_FLOW_MARK_MASK; 3066 break; 3067 default: 3068 meta = 0; 3069 mark = 0; 3070 MLX5_ASSERT(false); 3071 break; 3072 } 3073 if (sh->dv_mark_mask && sh->dv_mark_mask != mark) 3074 DRV_LOG(WARNING, "metadata MARK mask mismatch %08X:%08X", 3075 sh->dv_mark_mask, mark); 3076 else 3077 sh->dv_mark_mask = mark; 3078 if (sh->dv_meta_mask && sh->dv_meta_mask != meta) 3079 DRV_LOG(WARNING, "metadata META mask mismatch %08X:%08X", 3080 sh->dv_meta_mask, meta); 3081 else 3082 sh->dv_meta_mask = meta; 3083 if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0) 3084 DRV_LOG(WARNING, "metadata reg_c0 mask mismatch %08X:%08X", 3085 sh->dv_meta_mask, reg_c0); 3086 else 3087 sh->dv_regc0_mask = reg_c0; 3088 DRV_LOG(DEBUG, "metadata mode %u", sh->config.dv_xmeta_en); 3089 DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask); 3090 DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask); 3091 DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask); 3092 } 3093 3094 int 3095 rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n) 3096 { 3097 static const char *const dynf_names[] = { 3098 RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, 3099 RTE_MBUF_DYNFLAG_METADATA_NAME, 3100 RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME 3101 }; 3102 unsigned int i; 3103 3104 if (n < RTE_DIM(dynf_names)) 3105 return -ENOMEM; 3106 for (i = 0; i < RTE_DIM(dynf_names); i++) { 3107 if (names[i] == NULL) 3108 return -EINVAL; 3109 strcpy(names[i], dynf_names[i]); 3110 } 3111 return RTE_DIM(dynf_names); 3112 } 3113 3114 /** 3115 * Look for the ethernet device belonging to mlx5 driver. 3116 * 3117 * @param[in] port_id 3118 * port_id to start looking for device. 3119 * @param[in] odev 3120 * Pointer to the hint device. When device is being probed 3121 * the its siblings (master and preceding representors might 3122 * not have assigned driver yet (because the mlx5_os_pci_probe() 3123 * is not completed yet, for this case match on hint 3124 * device may be used to detect sibling device. 3125 * 3126 * @return 3127 * port_id of found device, RTE_MAX_ETHPORT if not found. 3128 */ 3129 uint16_t 3130 mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev) 3131 { 3132 while (port_id < RTE_MAX_ETHPORTS) { 3133 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 3134 3135 if (dev->state != RTE_ETH_DEV_UNUSED && 3136 dev->device && 3137 (dev->device == odev || 3138 (dev->device->driver && 3139 dev->device->driver->name && 3140 ((strcmp(dev->device->driver->name, 3141 MLX5_PCI_DRIVER_NAME) == 0) || 3142 (strcmp(dev->device->driver->name, 3143 MLX5_AUXILIARY_DRIVER_NAME) == 0))))) 3144 break; 3145 port_id++; 3146 } 3147 if (port_id >= RTE_MAX_ETHPORTS) 3148 return RTE_MAX_ETHPORTS; 3149 return port_id; 3150 } 3151 3152 /** 3153 * Callback to remove a device. 3154 * 3155 * This function removes all Ethernet devices belong to a given device. 3156 * 3157 * @param[in] cdev 3158 * Pointer to the generic device. 3159 * 3160 * @return 3161 * 0 on success, the function cannot fail. 3162 */ 3163 int 3164 mlx5_net_remove(struct mlx5_common_device *cdev) 3165 { 3166 uint16_t port_id; 3167 int ret = 0; 3168 3169 RTE_ETH_FOREACH_DEV_OF(port_id, cdev->dev) { 3170 /* 3171 * mlx5_dev_close() is not registered to secondary process, 3172 * call the close function explicitly for secondary process. 3173 */ 3174 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3175 ret |= mlx5_dev_close(&rte_eth_devices[port_id]); 3176 else 3177 ret |= rte_eth_dev_close(port_id); 3178 } 3179 return ret == 0 ? 0 : -EIO; 3180 } 3181 3182 static const struct rte_pci_id mlx5_pci_id_map[] = { 3183 { 3184 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3185 PCI_DEVICE_ID_MELLANOX_CONNECTX4) 3186 }, 3187 { 3188 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3189 PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) 3190 }, 3191 { 3192 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3193 PCI_DEVICE_ID_MELLANOX_CONNECTX4LX) 3194 }, 3195 { 3196 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3197 PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) 3198 }, 3199 { 3200 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3201 PCI_DEVICE_ID_MELLANOX_CONNECTX5) 3202 }, 3203 { 3204 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3205 PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) 3206 }, 3207 { 3208 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3209 PCI_DEVICE_ID_MELLANOX_CONNECTX5EX) 3210 }, 3211 { 3212 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3213 PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF) 3214 }, 3215 { 3216 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3217 PCI_DEVICE_ID_MELLANOX_BLUEFIELD) 3218 }, 3219 { 3220 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3221 PCI_DEVICE_ID_MELLANOX_BLUEFIELDVF) 3222 }, 3223 { 3224 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3225 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 3226 }, 3227 { 3228 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3229 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF) 3230 }, 3231 { 3232 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3233 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 3234 }, 3235 { 3236 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3237 PCI_DEVICE_ID_MELLANOX_CONNECTXVF) 3238 }, 3239 { 3240 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3241 PCI_DEVICE_ID_MELLANOX_BLUEFIELD2) 3242 }, 3243 { 3244 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3245 PCI_DEVICE_ID_MELLANOX_CONNECTX6LX) 3246 }, 3247 { 3248 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3249 PCI_DEVICE_ID_MELLANOX_CONNECTX7) 3250 }, 3251 { 3252 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3253 PCI_DEVICE_ID_MELLANOX_BLUEFIELD3) 3254 }, 3255 { 3256 .vendor_id = 0 3257 } 3258 }; 3259 3260 static struct mlx5_class_driver mlx5_net_driver = { 3261 .drv_class = MLX5_CLASS_ETH, 3262 .name = RTE_STR(MLX5_ETH_DRIVER_NAME), 3263 .id_table = mlx5_pci_id_map, 3264 .probe = mlx5_os_net_probe, 3265 .remove = mlx5_net_remove, 3266 .probe_again = 1, 3267 .intr_lsc = 1, 3268 .intr_rmv = 1, 3269 }; 3270 3271 /* Initialize driver log type. */ 3272 RTE_LOG_REGISTER_DEFAULT(mlx5_logtype, NOTICE) 3273 3274 /** 3275 * Driver initialization routine. 3276 */ 3277 RTE_INIT(rte_mlx5_pmd_init) 3278 { 3279 pthread_mutex_init(&mlx5_dev_ctx_list_mutex, NULL); 3280 mlx5_common_init(); 3281 /* Build the static tables for Verbs conversion. */ 3282 mlx5_set_ptype_table(); 3283 mlx5_set_cksum_table(); 3284 mlx5_set_swp_types_table(); 3285 if (mlx5_glue) 3286 mlx5_class_driver_register(&mlx5_net_driver); 3287 } 3288 3289 RTE_PMD_EXPORT_NAME(MLX5_ETH_DRIVER_NAME, __COUNTER__); 3290 RTE_PMD_REGISTER_PCI_TABLE(MLX5_ETH_DRIVER_NAME, mlx5_pci_id_map); 3291 RTE_PMD_REGISTER_KMOD_DEP(MLX5_ETH_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 3292