1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <stddef.h> 7 #include <unistd.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <errno.h> 12 13 #include <rte_malloc.h> 14 #include <ethdev_driver.h> 15 #include <rte_pci.h> 16 #include <rte_bus_pci.h> 17 #include <rte_common.h> 18 #include <rte_kvargs.h> 19 #include <rte_rwlock.h> 20 #include <rte_spinlock.h> 21 #include <rte_string_fns.h> 22 #include <rte_alarm.h> 23 #include <rte_cycles.h> 24 25 #include <mlx5_glue.h> 26 #include <mlx5_devx_cmds.h> 27 #include <mlx5_common.h> 28 #include <mlx5_common_os.h> 29 #include <mlx5_common_mp.h> 30 #include <mlx5_malloc.h> 31 32 #include "mlx5_defs.h" 33 #include "mlx5.h" 34 #include "mlx5_utils.h" 35 #include "mlx5_rxtx.h" 36 #include "mlx5_rx.h" 37 #include "mlx5_tx.h" 38 #include "mlx5_autoconf.h" 39 #include "mlx5_flow.h" 40 #include "mlx5_flow_os.h" 41 #include "rte_pmd_mlx5.h" 42 43 #define MLX5_ETH_DRIVER_NAME mlx5_eth 44 45 /* Driver type key for new device global syntax. */ 46 #define MLX5_DRIVER_KEY "driver" 47 48 /* Device parameter to enable RX completion queue compression. */ 49 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en" 50 51 /* Device parameter to enable padding Rx packet to cacheline size. */ 52 #define MLX5_RXQ_PKT_PAD_EN "rxq_pkt_pad_en" 53 54 /* Device parameter to enable Multi-Packet Rx queue. */ 55 #define MLX5_RX_MPRQ_EN "mprq_en" 56 57 /* Device parameter to configure log 2 of the number of strides for MPRQ. */ 58 #define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num" 59 60 /* Device parameter to configure log 2 of the stride size for MPRQ. */ 61 #define MLX5_RX_MPRQ_LOG_STRIDE_SIZE "mprq_log_stride_size" 62 63 /* Device parameter to limit the size of memcpy'd packet for MPRQ. */ 64 #define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len" 65 66 /* Device parameter to set the minimum number of Rx queues to enable MPRQ. */ 67 #define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq" 68 69 /* Device parameter to configure inline send. Deprecated, ignored.*/ 70 #define MLX5_TXQ_INLINE "txq_inline" 71 72 /* Device parameter to limit packet size to inline with ordinary SEND. */ 73 #define MLX5_TXQ_INLINE_MAX "txq_inline_max" 74 75 /* Device parameter to configure minimal data size to inline. */ 76 #define MLX5_TXQ_INLINE_MIN "txq_inline_min" 77 78 /* Device parameter to limit packet size to inline with Enhanced MPW. */ 79 #define MLX5_TXQ_INLINE_MPW "txq_inline_mpw" 80 81 /* 82 * Device parameter to configure the number of TX queues threshold for 83 * enabling inline send. 84 */ 85 #define MLX5_TXQS_MIN_INLINE "txqs_min_inline" 86 87 /* 88 * Device parameter to configure the number of TX queues threshold for 89 * enabling vectorized Tx, deprecated, ignored (no vectorized Tx routines). 90 */ 91 #define MLX5_TXQS_MAX_VEC "txqs_max_vec" 92 93 /* Device parameter to enable multi-packet send WQEs. */ 94 #define MLX5_TXQ_MPW_EN "txq_mpw_en" 95 96 /* 97 * Device parameter to force doorbell register mapping 98 * to non-cahed region eliminating the extra write memory barrier. 99 */ 100 #define MLX5_TX_DB_NC "tx_db_nc" 101 102 /* 103 * Device parameter to include 2 dsegs in the title WQEBB. 104 * Deprecated, ignored. 105 */ 106 #define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en" 107 108 /* 109 * Device parameter to limit the size of inlining packet. 110 * Deprecated, ignored. 111 */ 112 #define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len" 113 114 /* 115 * Device parameter to enable Tx scheduling on timestamps 116 * and specify the packet pacing granularity in nanoseconds. 117 */ 118 #define MLX5_TX_PP "tx_pp" 119 120 /* 121 * Device parameter to specify skew in nanoseconds on Tx datapath, 122 * it represents the time between SQ start WQE processing and 123 * appearing actual packet data on the wire. 124 */ 125 #define MLX5_TX_SKEW "tx_skew" 126 127 /* 128 * Device parameter to enable hardware Tx vector. 129 * Deprecated, ignored (no vectorized Tx routines anymore). 130 */ 131 #define MLX5_TX_VEC_EN "tx_vec_en" 132 133 /* Device parameter to enable hardware Rx vector. */ 134 #define MLX5_RX_VEC_EN "rx_vec_en" 135 136 /* Allow L3 VXLAN flow creation. */ 137 #define MLX5_L3_VXLAN_EN "l3_vxlan_en" 138 139 /* Activate DV E-Switch flow steering. */ 140 #define MLX5_DV_ESW_EN "dv_esw_en" 141 142 /* Activate DV flow steering. */ 143 #define MLX5_DV_FLOW_EN "dv_flow_en" 144 145 /* Enable extensive flow metadata support. */ 146 #define MLX5_DV_XMETA_EN "dv_xmeta_en" 147 148 /* Device parameter to let the user manage the lacp traffic of bonded device */ 149 #define MLX5_LACP_BY_USER "lacp_by_user" 150 151 /* Activate Netlink support in VF mode. */ 152 #define MLX5_VF_NL_EN "vf_nl_en" 153 154 /* Enable extending memsegs when creating a MR. */ 155 #define MLX5_MR_EXT_MEMSEG_EN "mr_ext_memseg_en" 156 157 /* Select port representors to instantiate. */ 158 #define MLX5_REPRESENTOR "representor" 159 160 /* Device parameter to configure the maximum number of dump files per queue. */ 161 #define MLX5_MAX_DUMP_FILES_NUM "max_dump_files_num" 162 163 /* Configure timeout of LRO session (in microseconds). */ 164 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec" 165 166 /* 167 * Device parameter to configure the total data buffer size for a single 168 * hairpin queue (logarithm value). 169 */ 170 #define MLX5_HP_BUF_SIZE "hp_buf_log_sz" 171 172 /* Flow memory reclaim mode. */ 173 #define MLX5_RECLAIM_MEM "reclaim_mem_mode" 174 175 /* The default memory allocator used in PMD. */ 176 #define MLX5_SYS_MEM_EN "sys_mem_en" 177 /* Decap will be used or not. */ 178 #define MLX5_DECAP_EN "decap_en" 179 180 /* Device parameter to configure allow or prevent duplicate rules pattern. */ 181 #define MLX5_ALLOW_DUPLICATE_PATTERN "allow_duplicate_pattern" 182 183 /* Device parameter to configure implicit registration of mempool memory. */ 184 #define MLX5_MR_MEMPOOL_REG_EN "mr_mempool_reg_en" 185 186 /* Shared memory between primary and secondary processes. */ 187 struct mlx5_shared_data *mlx5_shared_data; 188 189 /** Driver-specific log messages type. */ 190 int mlx5_logtype; 191 192 static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = 193 LIST_HEAD_INITIALIZER(); 194 static pthread_mutex_t mlx5_dev_ctx_list_mutex; 195 static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { 196 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 197 [MLX5_IPOOL_DECAP_ENCAP] = { 198 .size = sizeof(struct mlx5_flow_dv_encap_decap_resource), 199 .trunk_size = 64, 200 .grow_trunk = 3, 201 .grow_shift = 2, 202 .need_lock = 1, 203 .release_mem_en = 1, 204 .malloc = mlx5_malloc, 205 .free = mlx5_free, 206 .type = "mlx5_encap_decap_ipool", 207 }, 208 [MLX5_IPOOL_PUSH_VLAN] = { 209 .size = sizeof(struct mlx5_flow_dv_push_vlan_action_resource), 210 .trunk_size = 64, 211 .grow_trunk = 3, 212 .grow_shift = 2, 213 .need_lock = 1, 214 .release_mem_en = 1, 215 .malloc = mlx5_malloc, 216 .free = mlx5_free, 217 .type = "mlx5_push_vlan_ipool", 218 }, 219 [MLX5_IPOOL_TAG] = { 220 .size = sizeof(struct mlx5_flow_dv_tag_resource), 221 .trunk_size = 64, 222 .grow_trunk = 3, 223 .grow_shift = 2, 224 .need_lock = 1, 225 .release_mem_en = 0, 226 .per_core_cache = (1 << 16), 227 .malloc = mlx5_malloc, 228 .free = mlx5_free, 229 .type = "mlx5_tag_ipool", 230 }, 231 [MLX5_IPOOL_PORT_ID] = { 232 .size = sizeof(struct mlx5_flow_dv_port_id_action_resource), 233 .trunk_size = 64, 234 .grow_trunk = 3, 235 .grow_shift = 2, 236 .need_lock = 1, 237 .release_mem_en = 1, 238 .malloc = mlx5_malloc, 239 .free = mlx5_free, 240 .type = "mlx5_port_id_ipool", 241 }, 242 [MLX5_IPOOL_JUMP] = { 243 .size = sizeof(struct mlx5_flow_tbl_data_entry), 244 .trunk_size = 64, 245 .grow_trunk = 3, 246 .grow_shift = 2, 247 .need_lock = 1, 248 .release_mem_en = 1, 249 .malloc = mlx5_malloc, 250 .free = mlx5_free, 251 .type = "mlx5_jump_ipool", 252 }, 253 [MLX5_IPOOL_SAMPLE] = { 254 .size = sizeof(struct mlx5_flow_dv_sample_resource), 255 .trunk_size = 64, 256 .grow_trunk = 3, 257 .grow_shift = 2, 258 .need_lock = 1, 259 .release_mem_en = 1, 260 .malloc = mlx5_malloc, 261 .free = mlx5_free, 262 .type = "mlx5_sample_ipool", 263 }, 264 [MLX5_IPOOL_DEST_ARRAY] = { 265 .size = sizeof(struct mlx5_flow_dv_dest_array_resource), 266 .trunk_size = 64, 267 .grow_trunk = 3, 268 .grow_shift = 2, 269 .need_lock = 1, 270 .release_mem_en = 1, 271 .malloc = mlx5_malloc, 272 .free = mlx5_free, 273 .type = "mlx5_dest_array_ipool", 274 }, 275 [MLX5_IPOOL_TUNNEL_ID] = { 276 .size = sizeof(struct mlx5_flow_tunnel), 277 .trunk_size = MLX5_MAX_TUNNELS, 278 .need_lock = 1, 279 .release_mem_en = 1, 280 .type = "mlx5_tunnel_offload", 281 }, 282 [MLX5_IPOOL_TNL_TBL_ID] = { 283 .size = 0, 284 .need_lock = 1, 285 .type = "mlx5_flow_tnl_tbl_ipool", 286 }, 287 #endif 288 [MLX5_IPOOL_MTR] = { 289 /** 290 * The ipool index should grow continually from small to big, 291 * for meter idx, so not set grow_trunk to avoid meter index 292 * not jump continually. 293 */ 294 .size = sizeof(struct mlx5_legacy_flow_meter), 295 .trunk_size = 64, 296 .need_lock = 1, 297 .release_mem_en = 1, 298 .malloc = mlx5_malloc, 299 .free = mlx5_free, 300 .type = "mlx5_meter_ipool", 301 }, 302 [MLX5_IPOOL_MCP] = { 303 .size = sizeof(struct mlx5_flow_mreg_copy_resource), 304 .trunk_size = 64, 305 .grow_trunk = 3, 306 .grow_shift = 2, 307 .need_lock = 1, 308 .release_mem_en = 1, 309 .malloc = mlx5_malloc, 310 .free = mlx5_free, 311 .type = "mlx5_mcp_ipool", 312 }, 313 [MLX5_IPOOL_HRXQ] = { 314 .size = (sizeof(struct mlx5_hrxq) + MLX5_RSS_HASH_KEY_LEN), 315 .trunk_size = 64, 316 .grow_trunk = 3, 317 .grow_shift = 2, 318 .need_lock = 1, 319 .release_mem_en = 1, 320 .malloc = mlx5_malloc, 321 .free = mlx5_free, 322 .type = "mlx5_hrxq_ipool", 323 }, 324 [MLX5_IPOOL_MLX5_FLOW] = { 325 /* 326 * MLX5_IPOOL_MLX5_FLOW size varies for DV and VERBS flows. 327 * It set in run time according to PCI function configuration. 328 */ 329 .size = 0, 330 .trunk_size = 64, 331 .grow_trunk = 3, 332 .grow_shift = 2, 333 .need_lock = 1, 334 .release_mem_en = 0, 335 .per_core_cache = 1 << 19, 336 .malloc = mlx5_malloc, 337 .free = mlx5_free, 338 .type = "mlx5_flow_handle_ipool", 339 }, 340 [MLX5_IPOOL_RTE_FLOW] = { 341 .size = sizeof(struct rte_flow), 342 .trunk_size = 4096, 343 .need_lock = 1, 344 .release_mem_en = 1, 345 .malloc = mlx5_malloc, 346 .free = mlx5_free, 347 .type = "rte_flow_ipool", 348 }, 349 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID] = { 350 .size = 0, 351 .need_lock = 1, 352 .type = "mlx5_flow_rss_id_ipool", 353 }, 354 [MLX5_IPOOL_RSS_SHARED_ACTIONS] = { 355 .size = sizeof(struct mlx5_shared_action_rss), 356 .trunk_size = 64, 357 .grow_trunk = 3, 358 .grow_shift = 2, 359 .need_lock = 1, 360 .release_mem_en = 1, 361 .malloc = mlx5_malloc, 362 .free = mlx5_free, 363 .type = "mlx5_shared_action_rss", 364 }, 365 [MLX5_IPOOL_MTR_POLICY] = { 366 /** 367 * The ipool index should grow continually from small to big, 368 * for policy idx, so not set grow_trunk to avoid policy index 369 * not jump continually. 370 */ 371 .size = sizeof(struct mlx5_flow_meter_sub_policy), 372 .trunk_size = 64, 373 .need_lock = 1, 374 .release_mem_en = 1, 375 .malloc = mlx5_malloc, 376 .free = mlx5_free, 377 .type = "mlx5_meter_policy_ipool", 378 }, 379 }; 380 381 382 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512 383 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16 384 385 #define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024 386 387 /** 388 * Decide whether representor ID is a HPF(host PF) port on BF2. 389 * 390 * @param dev 391 * Pointer to Ethernet device structure. 392 * 393 * @return 394 * Non-zero if HPF, otherwise 0. 395 */ 396 bool 397 mlx5_is_hpf(struct rte_eth_dev *dev) 398 { 399 struct mlx5_priv *priv = dev->data->dev_private; 400 uint16_t repr = MLX5_REPRESENTOR_REPR(priv->representor_id); 401 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 402 403 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_VF && 404 MLX5_REPRESENTOR_REPR(-1) == repr; 405 } 406 407 /** 408 * Decide whether representor ID is a SF port representor. 409 * 410 * @param dev 411 * Pointer to Ethernet device structure. 412 * 413 * @return 414 * Non-zero if HPF, otherwise 0. 415 */ 416 bool 417 mlx5_is_sf_repr(struct rte_eth_dev *dev) 418 { 419 struct mlx5_priv *priv = dev->data->dev_private; 420 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 421 422 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_SF; 423 } 424 425 /** 426 * Initialize the ASO aging management structure. 427 * 428 * @param[in] sh 429 * Pointer to mlx5_dev_ctx_shared object to free 430 * 431 * @return 432 * 0 on success, a negative errno value otherwise and rte_errno is set. 433 */ 434 int 435 mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh) 436 { 437 int err; 438 439 if (sh->aso_age_mng) 440 return 0; 441 sh->aso_age_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->aso_age_mng), 442 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 443 if (!sh->aso_age_mng) { 444 DRV_LOG(ERR, "aso_age_mng allocation was failed."); 445 rte_errno = ENOMEM; 446 return -ENOMEM; 447 } 448 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_FLOW_HIT); 449 if (err) { 450 mlx5_free(sh->aso_age_mng); 451 return -1; 452 } 453 rte_spinlock_init(&sh->aso_age_mng->resize_sl); 454 rte_spinlock_init(&sh->aso_age_mng->free_sl); 455 LIST_INIT(&sh->aso_age_mng->free); 456 return 0; 457 } 458 459 /** 460 * Close and release all the resources of the ASO aging management structure. 461 * 462 * @param[in] sh 463 * Pointer to mlx5_dev_ctx_shared object to free. 464 */ 465 static void 466 mlx5_flow_aso_age_mng_close(struct mlx5_dev_ctx_shared *sh) 467 { 468 int i, j; 469 470 mlx5_aso_flow_hit_queue_poll_stop(sh); 471 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_FLOW_HIT); 472 if (sh->aso_age_mng->pools) { 473 struct mlx5_aso_age_pool *pool; 474 475 for (i = 0; i < sh->aso_age_mng->next; ++i) { 476 pool = sh->aso_age_mng->pools[i]; 477 claim_zero(mlx5_devx_cmd_destroy 478 (pool->flow_hit_aso_obj)); 479 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) 480 if (pool->actions[j].dr_action) 481 claim_zero 482 (mlx5_flow_os_destroy_flow_action 483 (pool->actions[j].dr_action)); 484 mlx5_free(pool); 485 } 486 mlx5_free(sh->aso_age_mng->pools); 487 } 488 mlx5_free(sh->aso_age_mng); 489 } 490 491 /** 492 * Initialize the shared aging list information per port. 493 * 494 * @param[in] sh 495 * Pointer to mlx5_dev_ctx_shared object. 496 */ 497 static void 498 mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh) 499 { 500 uint32_t i; 501 struct mlx5_age_info *age_info; 502 503 for (i = 0; i < sh->max_port; i++) { 504 age_info = &sh->port[i].age_info; 505 age_info->flags = 0; 506 TAILQ_INIT(&age_info->aged_counters); 507 LIST_INIT(&age_info->aged_aso); 508 rte_spinlock_init(&age_info->aged_sl); 509 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 510 } 511 } 512 513 /** 514 * Initialize the counters management structure. 515 * 516 * @param[in] sh 517 * Pointer to mlx5_dev_ctx_shared object to free 518 */ 519 static void 520 mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh) 521 { 522 struct mlx5_hca_attr *attr = &sh->cdev->config.hca_attr; 523 int i; 524 525 memset(&sh->cmng, 0, sizeof(sh->cmng)); 526 TAILQ_INIT(&sh->cmng.flow_counters); 527 sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET; 528 sh->cmng.max_id = -1; 529 sh->cmng.last_pool_idx = POOL_IDX_INVALID; 530 rte_spinlock_init(&sh->cmng.pool_update_sl); 531 for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) { 532 TAILQ_INIT(&sh->cmng.counters[i]); 533 rte_spinlock_init(&sh->cmng.csl[i]); 534 } 535 if (sh->devx && !haswell_broadwell_cpu) { 536 sh->cmng.relaxed_ordering_write = attr->relaxed_ordering_write; 537 sh->cmng.relaxed_ordering_read = attr->relaxed_ordering_read; 538 } 539 } 540 541 /** 542 * Destroy all the resources allocated for a counter memory management. 543 * 544 * @param[in] mng 545 * Pointer to the memory management structure. 546 */ 547 static void 548 mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng) 549 { 550 uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data; 551 552 LIST_REMOVE(mng, next); 553 claim_zero(mlx5_devx_cmd_destroy(mng->dm)); 554 claim_zero(mlx5_os_umem_dereg(mng->umem)); 555 mlx5_free(mem); 556 } 557 558 /** 559 * Close and release all the resources of the counters management. 560 * 561 * @param[in] sh 562 * Pointer to mlx5_dev_ctx_shared object to free. 563 */ 564 static void 565 mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) 566 { 567 struct mlx5_counter_stats_mem_mng *mng; 568 int i, j; 569 int retries = 1024; 570 571 rte_errno = 0; 572 while (--retries) { 573 rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh); 574 if (rte_errno != EINPROGRESS) 575 break; 576 rte_pause(); 577 } 578 579 if (sh->cmng.pools) { 580 struct mlx5_flow_counter_pool *pool; 581 uint16_t n_valid = sh->cmng.n_valid; 582 bool fallback = sh->cmng.counter_fallback; 583 584 for (i = 0; i < n_valid; ++i) { 585 pool = sh->cmng.pools[i]; 586 if (!fallback && pool->min_dcs) 587 claim_zero(mlx5_devx_cmd_destroy 588 (pool->min_dcs)); 589 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) { 590 struct mlx5_flow_counter *cnt = 591 MLX5_POOL_GET_CNT(pool, j); 592 593 if (cnt->action) 594 claim_zero 595 (mlx5_flow_os_destroy_flow_action 596 (cnt->action)); 597 if (fallback && MLX5_POOL_GET_CNT 598 (pool, j)->dcs_when_free) 599 claim_zero(mlx5_devx_cmd_destroy 600 (cnt->dcs_when_free)); 601 } 602 mlx5_free(pool); 603 } 604 mlx5_free(sh->cmng.pools); 605 } 606 mng = LIST_FIRST(&sh->cmng.mem_mngs); 607 while (mng) { 608 mlx5_flow_destroy_counter_stat_mem_mng(mng); 609 mng = LIST_FIRST(&sh->cmng.mem_mngs); 610 } 611 memset(&sh->cmng, 0, sizeof(sh->cmng)); 612 } 613 614 /** 615 * Initialize the aso flow meters management structure. 616 * 617 * @param[in] sh 618 * Pointer to mlx5_dev_ctx_shared object to free 619 */ 620 int 621 mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh) 622 { 623 if (!sh->mtrmng) { 624 sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, 625 sizeof(*sh->mtrmng), 626 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 627 if (!sh->mtrmng) { 628 DRV_LOG(ERR, 629 "meter management allocation was failed."); 630 rte_errno = ENOMEM; 631 return -ENOMEM; 632 } 633 if (sh->meter_aso_en) { 634 rte_spinlock_init(&sh->mtrmng->pools_mng.mtrsl); 635 LIST_INIT(&sh->mtrmng->pools_mng.meters); 636 } 637 sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 638 } 639 return 0; 640 } 641 642 /** 643 * Close and release all the resources of 644 * the ASO flow meter management structure. 645 * 646 * @param[in] sh 647 * Pointer to mlx5_dev_ctx_shared object to free. 648 */ 649 static void 650 mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) 651 { 652 struct mlx5_aso_mtr_pool *mtr_pool; 653 struct mlx5_flow_mtr_mng *mtrmng = sh->mtrmng; 654 uint32_t idx; 655 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 656 struct mlx5_aso_mtr *aso_mtr; 657 int i; 658 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 659 660 if (sh->meter_aso_en) { 661 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); 662 idx = mtrmng->pools_mng.n_valid; 663 while (idx--) { 664 mtr_pool = mtrmng->pools_mng.pools[idx]; 665 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 666 for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { 667 aso_mtr = &mtr_pool->mtrs[i]; 668 if (aso_mtr->fm.meter_action) 669 claim_zero 670 (mlx5_glue->destroy_flow_action 671 (aso_mtr->fm.meter_action)); 672 } 673 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 674 claim_zero(mlx5_devx_cmd_destroy 675 (mtr_pool->devx_obj)); 676 mtrmng->pools_mng.n_valid--; 677 mlx5_free(mtr_pool); 678 } 679 mlx5_free(sh->mtrmng->pools_mng.pools); 680 } 681 mlx5_free(sh->mtrmng); 682 sh->mtrmng = NULL; 683 } 684 685 /* Send FLOW_AGED event if needed. */ 686 void 687 mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) 688 { 689 struct mlx5_age_info *age_info; 690 uint32_t i; 691 692 for (i = 0; i < sh->max_port; i++) { 693 age_info = &sh->port[i].age_info; 694 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 695 continue; 696 MLX5_AGE_UNSET(age_info, MLX5_AGE_EVENT_NEW); 697 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) { 698 MLX5_AGE_UNSET(age_info, MLX5_AGE_TRIGGER); 699 rte_eth_dev_callback_process 700 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 701 RTE_ETH_EVENT_FLOW_AGED, NULL); 702 } 703 } 704 } 705 706 /* 707 * Initialize the ASO connection tracking structure. 708 * 709 * @param[in] sh 710 * Pointer to mlx5_dev_ctx_shared object. 711 * 712 * @return 713 * 0 on success, a negative errno value otherwise and rte_errno is set. 714 */ 715 int 716 mlx5_flow_aso_ct_mng_init(struct mlx5_dev_ctx_shared *sh) 717 { 718 int err; 719 720 if (sh->ct_mng) 721 return 0; 722 sh->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->ct_mng), 723 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 724 if (!sh->ct_mng) { 725 DRV_LOG(ERR, "ASO CT management allocation failed."); 726 rte_errno = ENOMEM; 727 return -rte_errno; 728 } 729 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_CONNECTION_TRACKING); 730 if (err) { 731 mlx5_free(sh->ct_mng); 732 /* rte_errno should be extracted from the failure. */ 733 rte_errno = EINVAL; 734 return -rte_errno; 735 } 736 rte_spinlock_init(&sh->ct_mng->ct_sl); 737 rte_rwlock_init(&sh->ct_mng->resize_rwl); 738 LIST_INIT(&sh->ct_mng->free_cts); 739 return 0; 740 } 741 742 /* 743 * Close and release all the resources of the 744 * ASO connection tracking management structure. 745 * 746 * @param[in] sh 747 * Pointer to mlx5_dev_ctx_shared object to free. 748 */ 749 static void 750 mlx5_flow_aso_ct_mng_close(struct mlx5_dev_ctx_shared *sh) 751 { 752 struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng; 753 struct mlx5_aso_ct_pool *ct_pool; 754 struct mlx5_aso_ct_action *ct; 755 uint32_t idx; 756 uint32_t val; 757 uint32_t cnt; 758 int i; 759 760 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_CONNECTION_TRACKING); 761 idx = mng->next; 762 while (idx--) { 763 cnt = 0; 764 ct_pool = mng->pools[idx]; 765 for (i = 0; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) { 766 ct = &ct_pool->actions[i]; 767 val = __atomic_fetch_sub(&ct->refcnt, 1, 768 __ATOMIC_RELAXED); 769 MLX5_ASSERT(val == 1); 770 if (val > 1) 771 cnt++; 772 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT 773 if (ct->dr_action_orig) 774 claim_zero(mlx5_glue->destroy_flow_action 775 (ct->dr_action_orig)); 776 if (ct->dr_action_rply) 777 claim_zero(mlx5_glue->destroy_flow_action 778 (ct->dr_action_rply)); 779 #endif 780 } 781 claim_zero(mlx5_devx_cmd_destroy(ct_pool->devx_obj)); 782 if (cnt) { 783 DRV_LOG(DEBUG, "%u ASO CT objects are being used in the pool %u", 784 cnt, i); 785 } 786 mlx5_free(ct_pool); 787 /* in case of failure. */ 788 mng->next--; 789 } 790 mlx5_free(mng->pools); 791 mlx5_free(mng); 792 /* Management structure must be cleared to 0s during allocation. */ 793 sh->ct_mng = NULL; 794 } 795 796 /** 797 * Initialize the flow resources' indexed mempool. 798 * 799 * @param[in] sh 800 * Pointer to mlx5_dev_ctx_shared object. 801 * @param[in] config 802 * Pointer to user dev config. 803 */ 804 static void 805 mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh, 806 const struct mlx5_dev_config *config) 807 { 808 uint8_t i; 809 struct mlx5_indexed_pool_config cfg; 810 811 for (i = 0; i < MLX5_IPOOL_MAX; ++i) { 812 cfg = mlx5_ipool_cfg[i]; 813 switch (i) { 814 default: 815 break; 816 /* 817 * Set MLX5_IPOOL_MLX5_FLOW ipool size 818 * according to PCI function flow configuration. 819 */ 820 case MLX5_IPOOL_MLX5_FLOW: 821 cfg.size = config->dv_flow_en ? 822 sizeof(struct mlx5_flow_handle) : 823 MLX5_FLOW_HANDLE_VERBS_SIZE; 824 break; 825 } 826 if (config->reclaim_mode) { 827 cfg.release_mem_en = 1; 828 cfg.per_core_cache = 0; 829 } else { 830 cfg.release_mem_en = 0; 831 } 832 sh->ipool[i] = mlx5_ipool_create(&cfg); 833 } 834 } 835 836 837 /** 838 * Release the flow resources' indexed mempool. 839 * 840 * @param[in] sh 841 * Pointer to mlx5_dev_ctx_shared object. 842 */ 843 static void 844 mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh) 845 { 846 uint8_t i; 847 848 for (i = 0; i < MLX5_IPOOL_MAX; ++i) 849 mlx5_ipool_destroy(sh->ipool[i]); 850 for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i) 851 if (sh->mdh_ipools[i]) 852 mlx5_ipool_destroy(sh->mdh_ipools[i]); 853 } 854 855 /* 856 * Check if dynamic flex parser for eCPRI already exists. 857 * 858 * @param dev 859 * Pointer to Ethernet device structure. 860 * 861 * @return 862 * true on exists, false on not. 863 */ 864 bool 865 mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev) 866 { 867 struct mlx5_priv *priv = dev->data->dev_private; 868 struct mlx5_flex_parser_profiles *prf = 869 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 870 871 return !!prf->obj; 872 } 873 874 /* 875 * Allocation of a flex parser for eCPRI. Once created, this parser related 876 * resources will be held until the device is closed. 877 * 878 * @param dev 879 * Pointer to Ethernet device structure. 880 * 881 * @return 882 * 0 on success, a negative errno value otherwise and rte_errno is set. 883 */ 884 int 885 mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev) 886 { 887 struct mlx5_priv *priv = dev->data->dev_private; 888 struct mlx5_flex_parser_profiles *prf = 889 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 890 struct mlx5_devx_graph_node_attr node = { 891 .modify_field_select = 0, 892 }; 893 uint32_t ids[8]; 894 int ret; 895 896 if (!priv->config.hca_attr.parse_graph_flex_node) { 897 DRV_LOG(ERR, "Dynamic flex parser is not supported " 898 "for device %s.", priv->dev_data->name); 899 return -ENOTSUP; 900 } 901 node.header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED; 902 /* 8 bytes now: 4B common header + 4B message body header. */ 903 node.header_length_base_value = 0x8; 904 /* After MAC layer: Ether / VLAN. */ 905 node.in[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_MAC; 906 /* Type of compared condition should be 0xAEFE in the L2 layer. */ 907 node.in[0].compare_condition_value = RTE_ETHER_TYPE_ECPRI; 908 /* Sample #0: type in common header. */ 909 node.sample[0].flow_match_sample_en = 1; 910 /* Fixed offset. */ 911 node.sample[0].flow_match_sample_offset_mode = 0x0; 912 /* Only the 2nd byte will be used. */ 913 node.sample[0].flow_match_sample_field_base_offset = 0x0; 914 /* Sample #1: message payload. */ 915 node.sample[1].flow_match_sample_en = 1; 916 /* Fixed offset. */ 917 node.sample[1].flow_match_sample_offset_mode = 0x0; 918 /* 919 * Only the first two bytes will be used right now, and its offset will 920 * start after the common header that with the length of a DW(u32). 921 */ 922 node.sample[1].flow_match_sample_field_base_offset = sizeof(uint32_t); 923 prf->obj = mlx5_devx_cmd_create_flex_parser(priv->sh->cdev->ctx, &node); 924 if (!prf->obj) { 925 DRV_LOG(ERR, "Failed to create flex parser node object."); 926 return (rte_errno == 0) ? -ENODEV : -rte_errno; 927 } 928 prf->num = 2; 929 ret = mlx5_devx_cmd_query_parse_samples(prf->obj, ids, prf->num); 930 if (ret) { 931 DRV_LOG(ERR, "Failed to query sample IDs."); 932 return (rte_errno == 0) ? -ENODEV : -rte_errno; 933 } 934 prf->offset[0] = 0x0; 935 prf->offset[1] = sizeof(uint32_t); 936 prf->ids[0] = ids[0]; 937 prf->ids[1] = ids[1]; 938 return 0; 939 } 940 941 /* 942 * Destroy the flex parser node, including the parser itself, input / output 943 * arcs and DW samples. Resources could be reused then. 944 * 945 * @param dev 946 * Pointer to Ethernet device structure. 947 */ 948 static void 949 mlx5_flex_parser_ecpri_release(struct rte_eth_dev *dev) 950 { 951 struct mlx5_priv *priv = dev->data->dev_private; 952 struct mlx5_flex_parser_profiles *prf = 953 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 954 955 if (prf->obj) 956 mlx5_devx_cmd_destroy(prf->obj); 957 prf->obj = NULL; 958 } 959 960 uint32_t 961 mlx5_get_supported_sw_parsing_offloads(const struct mlx5_hca_attr *attr) 962 { 963 uint32_t sw_parsing_offloads = 0; 964 965 if (attr->swp) { 966 sw_parsing_offloads |= MLX5_SW_PARSING_CAP; 967 if (attr->swp_csum) 968 sw_parsing_offloads |= MLX5_SW_PARSING_CSUM_CAP; 969 970 if (attr->swp_lso) 971 sw_parsing_offloads |= MLX5_SW_PARSING_TSO_CAP; 972 } 973 return sw_parsing_offloads; 974 } 975 976 uint32_t 977 mlx5_get_supported_tunneling_offloads(const struct mlx5_hca_attr *attr) 978 { 979 uint32_t tn_offloads = 0; 980 981 if (attr->tunnel_stateless_vxlan) 982 tn_offloads |= MLX5_TUNNELED_OFFLOADS_VXLAN_CAP; 983 if (attr->tunnel_stateless_gre) 984 tn_offloads |= MLX5_TUNNELED_OFFLOADS_GRE_CAP; 985 if (attr->tunnel_stateless_geneve_rx) 986 tn_offloads |= MLX5_TUNNELED_OFFLOADS_GENEVE_CAP; 987 return tn_offloads; 988 } 989 990 /* 991 * Allocate Rx and Tx UARs in robust fashion. 992 * This routine handles the following UAR allocation issues: 993 * 994 * - tries to allocate the UAR with the most appropriate memory 995 * mapping type from the ones supported by the host 996 * 997 * - tries to allocate the UAR with non-NULL base address 998 * OFED 5.0.x and Upstream rdma_core before v29 returned the NULL as 999 * UAR base address if UAR was not the first object in the UAR page. 1000 * It caused the PMD failure and we should try to get another UAR 1001 * till we get the first one with non-NULL base address returned. 1002 */ 1003 static int 1004 mlx5_alloc_rxtx_uars(struct mlx5_dev_ctx_shared *sh, 1005 const struct mlx5_common_dev_config *config) 1006 { 1007 uint32_t uar_mapping, retry; 1008 int err = 0; 1009 void *base_addr; 1010 1011 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 1012 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 1013 /* Control the mapping type according to the settings. */ 1014 uar_mapping = (config->dbnc == MLX5_TXDB_NCACHED) ? 1015 MLX5DV_UAR_ALLOC_TYPE_NC : 1016 MLX5DV_UAR_ALLOC_TYPE_BF; 1017 #else 1018 RTE_SET_USED(config); 1019 /* 1020 * It seems we have no way to control the memory mapping type 1021 * for the UAR, the default "Write-Combining" type is supposed. 1022 * The UAR initialization on queue creation queries the 1023 * actual mapping type done by Verbs/kernel and setups the 1024 * PMD datapath accordingly. 1025 */ 1026 uar_mapping = 0; 1027 #endif 1028 sh->tx_uar = mlx5_glue->devx_alloc_uar(sh->cdev->ctx, 1029 uar_mapping); 1030 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 1031 if (!sh->tx_uar && 1032 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 1033 if (config->dbnc == MLX5_TXDB_CACHED || 1034 config->dbnc == MLX5_TXDB_HEURISTIC) 1035 DRV_LOG(WARNING, "Devarg tx_db_nc setting " 1036 "is not supported by DevX"); 1037 /* 1038 * In some environments like virtual machine 1039 * the Write Combining mapped might be not supported 1040 * and UAR allocation fails. We try "Non-Cached" 1041 * mapping for the case. The tx_burst routines take 1042 * the UAR mapping type into account on UAR setup 1043 * on queue creation. 1044 */ 1045 DRV_LOG(DEBUG, "Failed to allocate Tx DevX UAR (BF)"); 1046 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 1047 sh->tx_uar = mlx5_glue->devx_alloc_uar(sh->cdev->ctx, 1048 uar_mapping); 1049 } else if (!sh->tx_uar && 1050 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) { 1051 if (config->dbnc == MLX5_TXDB_NCACHED) 1052 DRV_LOG(WARNING, "Devarg tx_db_nc settings " 1053 "is not supported by DevX"); 1054 /* 1055 * If Verbs/kernel does not support "Non-Cached" 1056 * try the "Write-Combining". 1057 */ 1058 DRV_LOG(DEBUG, "Failed to allocate Tx DevX UAR (NC)"); 1059 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF; 1060 sh->tx_uar = mlx5_glue->devx_alloc_uar(sh->cdev->ctx, 1061 uar_mapping); 1062 } 1063 #endif 1064 if (!sh->tx_uar) { 1065 DRV_LOG(ERR, "Failed to allocate Tx DevX UAR (BF/NC)"); 1066 err = ENOMEM; 1067 goto exit; 1068 } 1069 base_addr = mlx5_os_get_devx_uar_base_addr(sh->tx_uar); 1070 if (base_addr) 1071 break; 1072 /* 1073 * The UARs are allocated by rdma_core within the 1074 * IB device context, on context closure all UARs 1075 * will be freed, should be no memory/object leakage. 1076 */ 1077 DRV_LOG(DEBUG, "Retrying to allocate Tx DevX UAR"); 1078 sh->tx_uar = NULL; 1079 } 1080 /* Check whether we finally succeeded with valid UAR allocation. */ 1081 if (!sh->tx_uar) { 1082 DRV_LOG(ERR, "Failed to allocate Tx DevX UAR (NULL base)"); 1083 err = ENOMEM; 1084 goto exit; 1085 } 1086 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 1087 uar_mapping = 0; 1088 sh->devx_rx_uar = mlx5_glue->devx_alloc_uar(sh->cdev->ctx, 1089 uar_mapping); 1090 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 1091 if (!sh->devx_rx_uar && 1092 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 1093 /* 1094 * Rx UAR is used to control interrupts only, 1095 * should be no datapath noticeable impact, 1096 * can try "Non-Cached" mapping safely. 1097 */ 1098 DRV_LOG(DEBUG, "Failed to allocate Rx DevX UAR (BF)"); 1099 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 1100 sh->devx_rx_uar = mlx5_glue->devx_alloc_uar 1101 (sh->cdev->ctx, uar_mapping); 1102 } 1103 #endif 1104 if (!sh->devx_rx_uar) { 1105 DRV_LOG(ERR, "Failed to allocate Rx DevX UAR (BF/NC)"); 1106 err = ENOMEM; 1107 goto exit; 1108 } 1109 base_addr = mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar); 1110 if (base_addr) 1111 break; 1112 /* 1113 * The UARs are allocated by rdma_core within the 1114 * IB device context, on context closure all UARs 1115 * will be freed, should be no memory/object leakage. 1116 */ 1117 DRV_LOG(DEBUG, "Retrying to allocate Rx DevX UAR"); 1118 sh->devx_rx_uar = NULL; 1119 } 1120 /* Check whether we finally succeeded with valid UAR allocation. */ 1121 if (!sh->devx_rx_uar) { 1122 DRV_LOG(ERR, "Failed to allocate Rx DevX UAR (NULL base)"); 1123 err = ENOMEM; 1124 } 1125 exit: 1126 return err; 1127 } 1128 1129 /** 1130 * rte_mempool_walk() callback to unregister Rx mempools. 1131 * It used when implicit mempool registration is disabled. 1132 * 1133 * @param mp 1134 * The mempool being walked. 1135 * @param arg 1136 * Pointer to the device shared context. 1137 */ 1138 static void 1139 mlx5_dev_ctx_shared_rx_mempool_unregister_cb(struct rte_mempool *mp, void *arg) 1140 { 1141 struct mlx5_dev_ctx_shared *sh = arg; 1142 1143 mlx5_dev_mempool_unregister(sh->cdev, mp); 1144 } 1145 1146 /** 1147 * Callback used when implicit mempool registration is disabled 1148 * in order to track Rx mempool destruction. 1149 * 1150 * @param event 1151 * Mempool life cycle event. 1152 * @param mp 1153 * An Rx mempool registered explicitly when the port is started. 1154 * @param arg 1155 * Pointer to a device shared context. 1156 */ 1157 static void 1158 mlx5_dev_ctx_shared_rx_mempool_event_cb(enum rte_mempool_event event, 1159 struct rte_mempool *mp, void *arg) 1160 { 1161 struct mlx5_dev_ctx_shared *sh = arg; 1162 1163 if (event == RTE_MEMPOOL_EVENT_DESTROY) 1164 mlx5_dev_mempool_unregister(sh->cdev, mp); 1165 } 1166 1167 int 1168 mlx5_dev_ctx_shared_mempool_subscribe(struct rte_eth_dev *dev) 1169 { 1170 struct mlx5_priv *priv = dev->data->dev_private; 1171 struct mlx5_dev_ctx_shared *sh = priv->sh; 1172 int ret; 1173 1174 /* Check if we only need to track Rx mempool destruction. */ 1175 if (!sh->cdev->config.mr_mempool_reg_en) { 1176 ret = rte_mempool_event_callback_register 1177 (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); 1178 return ret == 0 || rte_errno == EEXIST ? 0 : ret; 1179 } 1180 return mlx5_dev_mempool_subscribe(sh->cdev); 1181 } 1182 1183 /** 1184 * Set up multiple TISs with different affinities according to 1185 * number of bonding ports 1186 * 1187 * @param priv 1188 * Pointer of shared context. 1189 * 1190 * @return 1191 * Zero on success, -1 otherwise. 1192 */ 1193 static int 1194 mlx5_setup_tis(struct mlx5_dev_ctx_shared *sh) 1195 { 1196 int i; 1197 struct mlx5_devx_lag_context lag_ctx = { 0 }; 1198 struct mlx5_devx_tis_attr tis_attr = { 0 }; 1199 1200 tis_attr.transport_domain = sh->td->id; 1201 if (sh->bond.n_port) { 1202 if (!mlx5_devx_cmd_query_lag(sh->cdev->ctx, &lag_ctx)) { 1203 sh->lag.tx_remap_affinity[0] = 1204 lag_ctx.tx_remap_affinity_1; 1205 sh->lag.tx_remap_affinity[1] = 1206 lag_ctx.tx_remap_affinity_2; 1207 sh->lag.affinity_mode = lag_ctx.port_select_mode; 1208 } else { 1209 DRV_LOG(ERR, "Failed to query lag affinity."); 1210 return -1; 1211 } 1212 if (sh->lag.affinity_mode == MLX5_LAG_MODE_TIS) { 1213 for (i = 0; i < sh->bond.n_port; i++) { 1214 tis_attr.lag_tx_port_affinity = 1215 MLX5_IFC_LAG_MAP_TIS_AFFINITY(i, 1216 sh->bond.n_port); 1217 sh->tis[i] = mlx5_devx_cmd_create_tis(sh->cdev->ctx, 1218 &tis_attr); 1219 if (!sh->tis[i]) { 1220 DRV_LOG(ERR, "Failed to TIS %d/%d for bonding device" 1221 " %s.", i, sh->bond.n_port, 1222 sh->ibdev_name); 1223 return -1; 1224 } 1225 } 1226 DRV_LOG(DEBUG, "LAG number of ports : %d, affinity_1 & 2 : pf%d & %d.\n", 1227 sh->bond.n_port, lag_ctx.tx_remap_affinity_1, 1228 lag_ctx.tx_remap_affinity_2); 1229 return 0; 1230 } 1231 if (sh->lag.affinity_mode == MLX5_LAG_MODE_HASH) 1232 DRV_LOG(INFO, "Device %s enabled HW hash based LAG.", 1233 sh->ibdev_name); 1234 } 1235 tis_attr.lag_tx_port_affinity = 0; 1236 sh->tis[0] = mlx5_devx_cmd_create_tis(sh->cdev->ctx, &tis_attr); 1237 if (!sh->tis[0]) { 1238 DRV_LOG(ERR, "Failed to TIS 0 for bonding device" 1239 " %s.", sh->ibdev_name); 1240 return -1; 1241 } 1242 return 0; 1243 } 1244 1245 /** 1246 * Allocate shared device context. If there is multiport device the 1247 * master and representors will share this context, if there is single 1248 * port dedicated device, the context will be used by only given 1249 * port due to unification. 1250 * 1251 * Routine first searches the context for the specified device name, 1252 * if found the shared context assumed and reference counter is incremented. 1253 * If no context found the new one is created and initialized with specified 1254 * device context and parameters. 1255 * 1256 * @param[in] spawn 1257 * Pointer to the device attributes (name, port, etc). 1258 * @param[in] config 1259 * Pointer to device configuration structure. 1260 * 1261 * @return 1262 * Pointer to mlx5_dev_ctx_shared object on success, 1263 * otherwise NULL and rte_errno is set. 1264 */ 1265 struct mlx5_dev_ctx_shared * 1266 mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, 1267 const struct mlx5_dev_config *config) 1268 { 1269 struct mlx5_dev_ctx_shared *sh; 1270 int err = 0; 1271 uint32_t i; 1272 1273 MLX5_ASSERT(spawn); 1274 /* Secondary process should not create the shared context. */ 1275 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1276 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1277 /* Search for IB context by device name. */ 1278 LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) { 1279 if (!strcmp(sh->ibdev_name, spawn->phys_dev_name)) { 1280 sh->refcnt++; 1281 goto exit; 1282 } 1283 } 1284 /* No device found, we have to create new shared context. */ 1285 MLX5_ASSERT(spawn->max_port); 1286 sh = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 1287 sizeof(struct mlx5_dev_ctx_shared) + 1288 spawn->max_port * 1289 sizeof(struct mlx5_dev_shared_port), 1290 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1291 if (!sh) { 1292 DRV_LOG(ERR, "shared context allocation failure"); 1293 rte_errno = ENOMEM; 1294 goto exit; 1295 } 1296 pthread_mutex_init(&sh->txpp.mutex, NULL); 1297 sh->numa_node = spawn->cdev->dev->numa_node; 1298 sh->cdev = spawn->cdev; 1299 sh->devx = sh->cdev->config.devx; 1300 if (spawn->bond_info) 1301 sh->bond = *spawn->bond_info; 1302 err = mlx5_os_get_dev_attr(sh->cdev, &sh->device_attr); 1303 if (err) { 1304 DRV_LOG(DEBUG, "mlx5_os_get_dev_attr() failed"); 1305 goto error; 1306 } 1307 sh->refcnt = 1; 1308 sh->max_port = spawn->max_port; 1309 sh->reclaim_mode = config->reclaim_mode; 1310 strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->cdev->ctx), 1311 sizeof(sh->ibdev_name) - 1); 1312 strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->cdev->ctx), 1313 sizeof(sh->ibdev_path) - 1); 1314 /* 1315 * Setting port_id to max unallowed value means 1316 * there is no interrupt subhandler installed for 1317 * the given port index i. 1318 */ 1319 for (i = 0; i < sh->max_port; i++) { 1320 sh->port[i].ih_port_id = RTE_MAX_ETHPORTS; 1321 sh->port[i].devx_ih_port_id = RTE_MAX_ETHPORTS; 1322 } 1323 if (sh->devx) { 1324 sh->td = mlx5_devx_cmd_create_td(sh->cdev->ctx); 1325 if (!sh->td) { 1326 DRV_LOG(ERR, "TD allocation failure"); 1327 err = ENOMEM; 1328 goto error; 1329 } 1330 if (mlx5_setup_tis(sh)) { 1331 DRV_LOG(ERR, "TIS allocation failure"); 1332 err = ENOMEM; 1333 goto error; 1334 } 1335 err = mlx5_alloc_rxtx_uars(sh, &sh->cdev->config); 1336 if (err) 1337 goto error; 1338 MLX5_ASSERT(sh->tx_uar); 1339 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->tx_uar)); 1340 1341 MLX5_ASSERT(sh->devx_rx_uar); 1342 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar)); 1343 } 1344 #ifndef RTE_ARCH_64 1345 /* Initialize UAR access locks for 32bit implementations. */ 1346 rte_spinlock_init(&sh->uar_lock_cq); 1347 for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) 1348 rte_spinlock_init(&sh->uar_lock[i]); 1349 #endif 1350 mlx5_os_dev_shared_handler_install(sh); 1351 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1352 err = mlx5_flow_os_init_workspace_once(); 1353 if (err) 1354 goto error; 1355 } 1356 mlx5_flow_aging_init(sh); 1357 mlx5_flow_counters_mng_init(sh); 1358 mlx5_flow_ipool_create(sh, config); 1359 /* Add context to the global device list. */ 1360 LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next); 1361 rte_spinlock_init(&sh->geneve_tlv_opt_sl); 1362 exit: 1363 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1364 return sh; 1365 error: 1366 pthread_mutex_destroy(&sh->txpp.mutex); 1367 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1368 MLX5_ASSERT(sh); 1369 if (sh->td) 1370 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1371 i = 0; 1372 do { 1373 if (sh->tis[i]) 1374 claim_zero(mlx5_devx_cmd_destroy(sh->tis[i])); 1375 } while (++i < (uint32_t)sh->bond.n_port); 1376 if (sh->devx_rx_uar) 1377 mlx5_glue->devx_free_uar(sh->devx_rx_uar); 1378 if (sh->tx_uar) 1379 mlx5_glue->devx_free_uar(sh->tx_uar); 1380 mlx5_free(sh); 1381 MLX5_ASSERT(err > 0); 1382 rte_errno = err; 1383 return NULL; 1384 } 1385 1386 /** 1387 * Free shared IB device context. Decrement counter and if zero free 1388 * all allocated resources and close handles. 1389 * 1390 * @param[in] sh 1391 * Pointer to mlx5_dev_ctx_shared object to free 1392 */ 1393 void 1394 mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) 1395 { 1396 int ret; 1397 int i = 0; 1398 1399 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1400 #ifdef RTE_LIBRTE_MLX5_DEBUG 1401 /* Check the object presence in the list. */ 1402 struct mlx5_dev_ctx_shared *lctx; 1403 1404 LIST_FOREACH(lctx, &mlx5_dev_ctx_list, next) 1405 if (lctx == sh) 1406 break; 1407 MLX5_ASSERT(lctx); 1408 if (lctx != sh) { 1409 DRV_LOG(ERR, "Freeing non-existing shared IB context"); 1410 goto exit; 1411 } 1412 #endif 1413 MLX5_ASSERT(sh); 1414 MLX5_ASSERT(sh->refcnt); 1415 /* Secondary process should not free the shared context. */ 1416 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1417 if (--sh->refcnt) 1418 goto exit; 1419 /* Stop watching for mempool events and unregister all mempools. */ 1420 if (!sh->cdev->config.mr_mempool_reg_en) { 1421 ret = rte_mempool_event_callback_unregister 1422 (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); 1423 if (ret == 0) 1424 rte_mempool_walk 1425 (mlx5_dev_ctx_shared_rx_mempool_unregister_cb, sh); 1426 } 1427 /* Remove context from the global device list. */ 1428 LIST_REMOVE(sh, next); 1429 /* Release resources on the last device removal. */ 1430 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1431 mlx5_os_net_cleanup(); 1432 mlx5_flow_os_release_workspace(); 1433 } 1434 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1435 /* 1436 * Ensure there is no async event handler installed. 1437 * Only primary process handles async device events. 1438 **/ 1439 mlx5_flow_counters_mng_close(sh); 1440 if (sh->aso_age_mng) { 1441 mlx5_flow_aso_age_mng_close(sh); 1442 sh->aso_age_mng = NULL; 1443 } 1444 if (sh->mtrmng) 1445 mlx5_aso_flow_mtrs_mng_close(sh); 1446 mlx5_flow_ipool_destroy(sh); 1447 mlx5_os_dev_shared_handler_uninstall(sh); 1448 if (sh->tx_uar) { 1449 mlx5_glue->devx_free_uar(sh->tx_uar); 1450 sh->tx_uar = NULL; 1451 } 1452 do { 1453 if (sh->tis[i]) 1454 claim_zero(mlx5_devx_cmd_destroy(sh->tis[i])); 1455 } while (++i < sh->bond.n_port); 1456 if (sh->td) 1457 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1458 if (sh->devx_rx_uar) 1459 mlx5_glue->devx_free_uar(sh->devx_rx_uar); 1460 MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL); 1461 pthread_mutex_destroy(&sh->txpp.mutex); 1462 mlx5_free(sh); 1463 return; 1464 exit: 1465 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1466 } 1467 1468 /** 1469 * Destroy table hash list. 1470 * 1471 * @param[in] priv 1472 * Pointer to the private device data structure. 1473 */ 1474 void 1475 mlx5_free_table_hash_list(struct mlx5_priv *priv) 1476 { 1477 struct mlx5_dev_ctx_shared *sh = priv->sh; 1478 1479 if (!sh->flow_tbls) 1480 return; 1481 mlx5_hlist_destroy(sh->flow_tbls); 1482 sh->flow_tbls = NULL; 1483 } 1484 1485 /** 1486 * Initialize flow table hash list and create the root tables entry 1487 * for each domain. 1488 * 1489 * @param[in] priv 1490 * Pointer to the private device data structure. 1491 * 1492 * @return 1493 * Zero on success, positive error code otherwise. 1494 */ 1495 int 1496 mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) 1497 { 1498 int err = 0; 1499 /* Tables are only used in DV and DR modes. */ 1500 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 1501 struct mlx5_dev_ctx_shared *sh = priv->sh; 1502 char s[MLX5_NAME_SIZE]; 1503 1504 MLX5_ASSERT(sh); 1505 snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); 1506 sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, 1507 false, true, sh, 1508 flow_dv_tbl_create_cb, 1509 flow_dv_tbl_match_cb, 1510 flow_dv_tbl_remove_cb, 1511 flow_dv_tbl_clone_cb, 1512 flow_dv_tbl_clone_free_cb); 1513 if (!sh->flow_tbls) { 1514 DRV_LOG(ERR, "flow tables with hash creation failed."); 1515 err = ENOMEM; 1516 return err; 1517 } 1518 #ifndef HAVE_MLX5DV_DR 1519 struct rte_flow_error error; 1520 struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; 1521 1522 /* 1523 * In case we have not DR support, the zero tables should be created 1524 * because DV expect to see them even if they cannot be created by 1525 * RDMA-CORE. 1526 */ 1527 if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, 1528 NULL, 0, 1, 0, &error) || 1529 !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, 1530 NULL, 0, 1, 0, &error) || 1531 !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, 1532 NULL, 0, 1, 0, &error)) { 1533 err = ENOMEM; 1534 goto error; 1535 } 1536 return err; 1537 error: 1538 mlx5_free_table_hash_list(priv); 1539 #endif /* HAVE_MLX5DV_DR */ 1540 #endif 1541 return err; 1542 } 1543 1544 /** 1545 * Retrieve integer value from environment variable. 1546 * 1547 * @param[in] name 1548 * Environment variable name. 1549 * 1550 * @return 1551 * Integer value, 0 if the variable is not set. 1552 */ 1553 int 1554 mlx5_getenv_int(const char *name) 1555 { 1556 const char *val = getenv(name); 1557 1558 if (val == NULL) 1559 return 0; 1560 return atoi(val); 1561 } 1562 1563 /** 1564 * DPDK callback to add udp tunnel port 1565 * 1566 * @param[in] dev 1567 * A pointer to eth_dev 1568 * @param[in] udp_tunnel 1569 * A pointer to udp tunnel 1570 * 1571 * @return 1572 * 0 on valid udp ports and tunnels, -ENOTSUP otherwise. 1573 */ 1574 int 1575 mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev __rte_unused, 1576 struct rte_eth_udp_tunnel *udp_tunnel) 1577 { 1578 MLX5_ASSERT(udp_tunnel != NULL); 1579 if (udp_tunnel->prot_type == RTE_ETH_TUNNEL_TYPE_VXLAN && 1580 udp_tunnel->udp_port == 4789) 1581 return 0; 1582 if (udp_tunnel->prot_type == RTE_ETH_TUNNEL_TYPE_VXLAN_GPE && 1583 udp_tunnel->udp_port == 4790) 1584 return 0; 1585 return -ENOTSUP; 1586 } 1587 1588 /** 1589 * Initialize process private data structure. 1590 * 1591 * @param dev 1592 * Pointer to Ethernet device structure. 1593 * 1594 * @return 1595 * 0 on success, a negative errno value otherwise and rte_errno is set. 1596 */ 1597 int 1598 mlx5_proc_priv_init(struct rte_eth_dev *dev) 1599 { 1600 struct mlx5_priv *priv = dev->data->dev_private; 1601 struct mlx5_proc_priv *ppriv; 1602 size_t ppriv_size; 1603 1604 mlx5_proc_priv_uninit(dev); 1605 /* 1606 * UAR register table follows the process private structure. BlueFlame 1607 * registers for Tx queues are stored in the table. 1608 */ 1609 ppriv_size = 1610 sizeof(struct mlx5_proc_priv) + priv->txqs_n * sizeof(void *); 1611 ppriv = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, ppriv_size, 1612 RTE_CACHE_LINE_SIZE, dev->device->numa_node); 1613 if (!ppriv) { 1614 rte_errno = ENOMEM; 1615 return -rte_errno; 1616 } 1617 ppriv->uar_table_sz = priv->txqs_n; 1618 dev->process_private = ppriv; 1619 return 0; 1620 } 1621 1622 /** 1623 * Un-initialize process private data structure. 1624 * 1625 * @param dev 1626 * Pointer to Ethernet device structure. 1627 */ 1628 void 1629 mlx5_proc_priv_uninit(struct rte_eth_dev *dev) 1630 { 1631 if (!dev->process_private) 1632 return; 1633 mlx5_free(dev->process_private); 1634 dev->process_private = NULL; 1635 } 1636 1637 /** 1638 * DPDK callback to close the device. 1639 * 1640 * Destroy all queues and objects, free memory. 1641 * 1642 * @param dev 1643 * Pointer to Ethernet device structure. 1644 */ 1645 int 1646 mlx5_dev_close(struct rte_eth_dev *dev) 1647 { 1648 struct mlx5_priv *priv = dev->data->dev_private; 1649 unsigned int i; 1650 int ret; 1651 1652 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1653 /* Check if process_private released. */ 1654 if (!dev->process_private) 1655 return 0; 1656 mlx5_tx_uar_uninit_secondary(dev); 1657 mlx5_proc_priv_uninit(dev); 1658 rte_eth_dev_release_port(dev); 1659 return 0; 1660 } 1661 if (!priv->sh) 1662 return 0; 1663 DRV_LOG(DEBUG, "port %u closing device \"%s\"", 1664 dev->data->port_id, 1665 ((priv->sh->cdev->ctx != NULL) ? 1666 mlx5_os_get_ctx_device_name(priv->sh->cdev->ctx) : "")); 1667 /* 1668 * If default mreg copy action is removed at the stop stage, 1669 * the search will return none and nothing will be done anymore. 1670 */ 1671 mlx5_flow_stop_default(dev); 1672 mlx5_traffic_disable(dev); 1673 /* 1674 * If all the flows are already flushed in the device stop stage, 1675 * then this will return directly without any action. 1676 */ 1677 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true); 1678 mlx5_action_handle_flush(dev); 1679 mlx5_flow_meter_flush(dev, NULL); 1680 /* Prevent crashes when queues are still in use. */ 1681 dev->rx_pkt_burst = removed_rx_burst; 1682 dev->tx_pkt_burst = removed_tx_burst; 1683 rte_wmb(); 1684 /* Disable datapath on secondary process. */ 1685 mlx5_mp_os_req_stop_rxtx(dev); 1686 /* Free the eCPRI flex parser resource. */ 1687 mlx5_flex_parser_ecpri_release(dev); 1688 if (priv->rxqs != NULL) { 1689 /* XXX race condition if mlx5_rx_burst() is still running. */ 1690 rte_delay_us_sleep(1000); 1691 for (i = 0; (i != priv->rxqs_n); ++i) 1692 mlx5_rxq_release(dev, i); 1693 priv->rxqs_n = 0; 1694 priv->rxqs = NULL; 1695 } 1696 if (priv->representor) { 1697 /* Each representor has a dedicated interrupts handler */ 1698 mlx5_free(dev->intr_handle); 1699 dev->intr_handle = NULL; 1700 } 1701 if (priv->txqs != NULL) { 1702 /* XXX race condition if mlx5_tx_burst() is still running. */ 1703 rte_delay_us_sleep(1000); 1704 for (i = 0; (i != priv->txqs_n); ++i) 1705 mlx5_txq_release(dev, i); 1706 priv->txqs_n = 0; 1707 priv->txqs = NULL; 1708 } 1709 mlx5_proc_priv_uninit(dev); 1710 if (priv->q_counters) { 1711 mlx5_devx_cmd_destroy(priv->q_counters); 1712 priv->q_counters = NULL; 1713 } 1714 if (priv->drop_queue.hrxq) 1715 mlx5_drop_action_destroy(dev); 1716 if (priv->mreg_cp_tbl) 1717 mlx5_hlist_destroy(priv->mreg_cp_tbl); 1718 mlx5_mprq_free_mp(dev); 1719 if (priv->sh->ct_mng) 1720 mlx5_flow_aso_ct_mng_close(priv->sh); 1721 mlx5_os_free_shared_dr(priv); 1722 if (priv->rss_conf.rss_key != NULL) 1723 mlx5_free(priv->rss_conf.rss_key); 1724 if (priv->reta_idx != NULL) 1725 mlx5_free(priv->reta_idx); 1726 if (priv->config.vf) 1727 mlx5_os_mac_addr_flush(dev); 1728 if (priv->nl_socket_route >= 0) 1729 close(priv->nl_socket_route); 1730 if (priv->nl_socket_rdma >= 0) 1731 close(priv->nl_socket_rdma); 1732 if (priv->vmwa_context) 1733 mlx5_vlan_vmwa_exit(priv->vmwa_context); 1734 ret = mlx5_hrxq_verify(dev); 1735 if (ret) 1736 DRV_LOG(WARNING, "port %u some hash Rx queue still remain", 1737 dev->data->port_id); 1738 ret = mlx5_ind_table_obj_verify(dev); 1739 if (ret) 1740 DRV_LOG(WARNING, "port %u some indirection table still remain", 1741 dev->data->port_id); 1742 ret = mlx5_rxq_obj_verify(dev); 1743 if (ret) 1744 DRV_LOG(WARNING, "port %u some Rx queue objects still remain", 1745 dev->data->port_id); 1746 ret = mlx5_rxq_verify(dev); 1747 if (ret) 1748 DRV_LOG(WARNING, "port %u some Rx queues still remain", 1749 dev->data->port_id); 1750 ret = mlx5_txq_obj_verify(dev); 1751 if (ret) 1752 DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain", 1753 dev->data->port_id); 1754 ret = mlx5_txq_verify(dev); 1755 if (ret) 1756 DRV_LOG(WARNING, "port %u some Tx queues still remain", 1757 dev->data->port_id); 1758 ret = mlx5_flow_verify(dev); 1759 if (ret) 1760 DRV_LOG(WARNING, "port %u some flows still remain", 1761 dev->data->port_id); 1762 if (priv->hrxqs) 1763 mlx5_list_destroy(priv->hrxqs); 1764 /* 1765 * Free the shared context in last turn, because the cleanup 1766 * routines above may use some shared fields, like 1767 * mlx5_os_mac_addr_flush() uses ibdev_path for retrieveing 1768 * ifindex if Netlink fails. 1769 */ 1770 mlx5_free_shared_dev_ctx(priv->sh); 1771 if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 1772 unsigned int c = 0; 1773 uint16_t port_id; 1774 1775 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1776 struct mlx5_priv *opriv = 1777 rte_eth_devices[port_id].data->dev_private; 1778 1779 if (!opriv || 1780 opriv->domain_id != priv->domain_id || 1781 &rte_eth_devices[port_id] == dev) 1782 continue; 1783 ++c; 1784 break; 1785 } 1786 if (!c) 1787 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 1788 } 1789 memset(priv, 0, sizeof(*priv)); 1790 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 1791 /* 1792 * Reset mac_addrs to NULL such that it is not freed as part of 1793 * rte_eth_dev_release_port(). mac_addrs is part of dev_private so 1794 * it is freed when dev_private is freed. 1795 */ 1796 dev->data->mac_addrs = NULL; 1797 return 0; 1798 } 1799 1800 const struct eth_dev_ops mlx5_dev_ops = { 1801 .dev_configure = mlx5_dev_configure, 1802 .dev_start = mlx5_dev_start, 1803 .dev_stop = mlx5_dev_stop, 1804 .dev_set_link_down = mlx5_set_link_down, 1805 .dev_set_link_up = mlx5_set_link_up, 1806 .dev_close = mlx5_dev_close, 1807 .promiscuous_enable = mlx5_promiscuous_enable, 1808 .promiscuous_disable = mlx5_promiscuous_disable, 1809 .allmulticast_enable = mlx5_allmulticast_enable, 1810 .allmulticast_disable = mlx5_allmulticast_disable, 1811 .link_update = mlx5_link_update, 1812 .stats_get = mlx5_stats_get, 1813 .stats_reset = mlx5_stats_reset, 1814 .xstats_get = mlx5_xstats_get, 1815 .xstats_reset = mlx5_xstats_reset, 1816 .xstats_get_names = mlx5_xstats_get_names, 1817 .fw_version_get = mlx5_fw_version_get, 1818 .dev_infos_get = mlx5_dev_infos_get, 1819 .representor_info_get = mlx5_representor_info_get, 1820 .read_clock = mlx5_txpp_read_clock, 1821 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1822 .vlan_filter_set = mlx5_vlan_filter_set, 1823 .rx_queue_setup = mlx5_rx_queue_setup, 1824 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1825 .tx_queue_setup = mlx5_tx_queue_setup, 1826 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1827 .rx_queue_release = mlx5_rx_queue_release, 1828 .tx_queue_release = mlx5_tx_queue_release, 1829 .rx_queue_start = mlx5_rx_queue_start, 1830 .rx_queue_stop = mlx5_rx_queue_stop, 1831 .tx_queue_start = mlx5_tx_queue_start, 1832 .tx_queue_stop = mlx5_tx_queue_stop, 1833 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1834 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1835 .mac_addr_remove = mlx5_mac_addr_remove, 1836 .mac_addr_add = mlx5_mac_addr_add, 1837 .mac_addr_set = mlx5_mac_addr_set, 1838 .set_mc_addr_list = mlx5_set_mc_addr_list, 1839 .mtu_set = mlx5_dev_set_mtu, 1840 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1841 .vlan_offload_set = mlx5_vlan_offload_set, 1842 .reta_update = mlx5_dev_rss_reta_update, 1843 .reta_query = mlx5_dev_rss_reta_query, 1844 .rss_hash_update = mlx5_rss_hash_update, 1845 .rss_hash_conf_get = mlx5_rss_hash_conf_get, 1846 .flow_ops_get = mlx5_flow_ops_get, 1847 .rxq_info_get = mlx5_rxq_info_get, 1848 .txq_info_get = mlx5_txq_info_get, 1849 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1850 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1851 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1852 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1853 .is_removed = mlx5_is_removed, 1854 .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, 1855 .get_module_info = mlx5_get_module_info, 1856 .get_module_eeprom = mlx5_get_module_eeprom, 1857 .hairpin_cap_get = mlx5_hairpin_cap_get, 1858 .mtr_ops_get = mlx5_flow_meter_ops_get, 1859 .hairpin_bind = mlx5_hairpin_bind, 1860 .hairpin_unbind = mlx5_hairpin_unbind, 1861 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1862 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1863 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1864 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1865 .get_monitor_addr = mlx5_get_monitor_addr, 1866 }; 1867 1868 /* Available operations from secondary process. */ 1869 const struct eth_dev_ops mlx5_dev_sec_ops = { 1870 .stats_get = mlx5_stats_get, 1871 .stats_reset = mlx5_stats_reset, 1872 .xstats_get = mlx5_xstats_get, 1873 .xstats_reset = mlx5_xstats_reset, 1874 .xstats_get_names = mlx5_xstats_get_names, 1875 .fw_version_get = mlx5_fw_version_get, 1876 .dev_infos_get = mlx5_dev_infos_get, 1877 .representor_info_get = mlx5_representor_info_get, 1878 .read_clock = mlx5_txpp_read_clock, 1879 .rx_queue_start = mlx5_rx_queue_start, 1880 .rx_queue_stop = mlx5_rx_queue_stop, 1881 .tx_queue_start = mlx5_tx_queue_start, 1882 .tx_queue_stop = mlx5_tx_queue_stop, 1883 .rxq_info_get = mlx5_rxq_info_get, 1884 .txq_info_get = mlx5_txq_info_get, 1885 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1886 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1887 .get_module_info = mlx5_get_module_info, 1888 .get_module_eeprom = mlx5_get_module_eeprom, 1889 }; 1890 1891 /* Available operations in flow isolated mode. */ 1892 const struct eth_dev_ops mlx5_dev_ops_isolate = { 1893 .dev_configure = mlx5_dev_configure, 1894 .dev_start = mlx5_dev_start, 1895 .dev_stop = mlx5_dev_stop, 1896 .dev_set_link_down = mlx5_set_link_down, 1897 .dev_set_link_up = mlx5_set_link_up, 1898 .dev_close = mlx5_dev_close, 1899 .promiscuous_enable = mlx5_promiscuous_enable, 1900 .promiscuous_disable = mlx5_promiscuous_disable, 1901 .allmulticast_enable = mlx5_allmulticast_enable, 1902 .allmulticast_disable = mlx5_allmulticast_disable, 1903 .link_update = mlx5_link_update, 1904 .stats_get = mlx5_stats_get, 1905 .stats_reset = mlx5_stats_reset, 1906 .xstats_get = mlx5_xstats_get, 1907 .xstats_reset = mlx5_xstats_reset, 1908 .xstats_get_names = mlx5_xstats_get_names, 1909 .fw_version_get = mlx5_fw_version_get, 1910 .dev_infos_get = mlx5_dev_infos_get, 1911 .representor_info_get = mlx5_representor_info_get, 1912 .read_clock = mlx5_txpp_read_clock, 1913 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1914 .vlan_filter_set = mlx5_vlan_filter_set, 1915 .rx_queue_setup = mlx5_rx_queue_setup, 1916 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1917 .tx_queue_setup = mlx5_tx_queue_setup, 1918 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1919 .rx_queue_release = mlx5_rx_queue_release, 1920 .tx_queue_release = mlx5_tx_queue_release, 1921 .rx_queue_start = mlx5_rx_queue_start, 1922 .rx_queue_stop = mlx5_rx_queue_stop, 1923 .tx_queue_start = mlx5_tx_queue_start, 1924 .tx_queue_stop = mlx5_tx_queue_stop, 1925 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1926 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1927 .mac_addr_remove = mlx5_mac_addr_remove, 1928 .mac_addr_add = mlx5_mac_addr_add, 1929 .mac_addr_set = mlx5_mac_addr_set, 1930 .set_mc_addr_list = mlx5_set_mc_addr_list, 1931 .mtu_set = mlx5_dev_set_mtu, 1932 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1933 .vlan_offload_set = mlx5_vlan_offload_set, 1934 .flow_ops_get = mlx5_flow_ops_get, 1935 .rxq_info_get = mlx5_rxq_info_get, 1936 .txq_info_get = mlx5_txq_info_get, 1937 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1938 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1939 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1940 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1941 .is_removed = mlx5_is_removed, 1942 .get_module_info = mlx5_get_module_info, 1943 .get_module_eeprom = mlx5_get_module_eeprom, 1944 .hairpin_cap_get = mlx5_hairpin_cap_get, 1945 .mtr_ops_get = mlx5_flow_meter_ops_get, 1946 .hairpin_bind = mlx5_hairpin_bind, 1947 .hairpin_unbind = mlx5_hairpin_unbind, 1948 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1949 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1950 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1951 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1952 .get_monitor_addr = mlx5_get_monitor_addr, 1953 }; 1954 1955 /** 1956 * Verify and store value for device argument. 1957 * 1958 * @param[in] key 1959 * Key argument to verify. 1960 * @param[in] val 1961 * Value associated with key. 1962 * @param opaque 1963 * User data. 1964 * 1965 * @return 1966 * 0 on success, a negative errno value otherwise and rte_errno is set. 1967 */ 1968 static int 1969 mlx5_args_check(const char *key, const char *val, void *opaque) 1970 { 1971 struct mlx5_dev_config *config = opaque; 1972 unsigned long mod; 1973 signed long tmp; 1974 1975 /* No-op, port representors are processed in mlx5_dev_spawn(). */ 1976 if (!strcmp(MLX5_DRIVER_KEY, key) || !strcmp(MLX5_REPRESENTOR, key) || 1977 !strcmp(MLX5_SYS_MEM_EN, key) || !strcmp(MLX5_TX_DB_NC, key) || 1978 !strcmp(MLX5_MR_MEMPOOL_REG_EN, key) || 1979 !strcmp(MLX5_MR_EXT_MEMSEG_EN, key)) 1980 return 0; 1981 errno = 0; 1982 tmp = strtol(val, NULL, 0); 1983 if (errno) { 1984 rte_errno = errno; 1985 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); 1986 return -rte_errno; 1987 } 1988 if (tmp < 0 && strcmp(MLX5_TX_PP, key) && strcmp(MLX5_TX_SKEW, key)) { 1989 /* Negative values are acceptable for some keys only. */ 1990 rte_errno = EINVAL; 1991 DRV_LOG(WARNING, "%s: invalid negative value \"%s\"", key, val); 1992 return -rte_errno; 1993 } 1994 mod = tmp >= 0 ? tmp : -tmp; 1995 if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) { 1996 if (tmp > MLX5_CQE_RESP_FORMAT_L34H_STRIDX) { 1997 DRV_LOG(ERR, "invalid CQE compression " 1998 "format parameter"); 1999 rte_errno = EINVAL; 2000 return -rte_errno; 2001 } 2002 config->cqe_comp = !!tmp; 2003 config->cqe_comp_fmt = tmp; 2004 } else if (strcmp(MLX5_RXQ_PKT_PAD_EN, key) == 0) { 2005 config->hw_padding = !!tmp; 2006 } else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) { 2007 config->mprq.enabled = !!tmp; 2008 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) { 2009 config->mprq.stride_num_n = tmp; 2010 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_SIZE, key) == 0) { 2011 config->mprq.stride_size_n = tmp; 2012 } else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) { 2013 config->mprq.max_memcpy_len = tmp; 2014 } else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) { 2015 config->mprq.min_rxqs_num = tmp; 2016 } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) { 2017 DRV_LOG(WARNING, "%s: deprecated parameter," 2018 " converted to txq_inline_max", key); 2019 config->txq_inline_max = tmp; 2020 } else if (strcmp(MLX5_TXQ_INLINE_MAX, key) == 0) { 2021 config->txq_inline_max = tmp; 2022 } else if (strcmp(MLX5_TXQ_INLINE_MIN, key) == 0) { 2023 config->txq_inline_min = tmp; 2024 } else if (strcmp(MLX5_TXQ_INLINE_MPW, key) == 0) { 2025 config->txq_inline_mpw = tmp; 2026 } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) { 2027 config->txqs_inline = tmp; 2028 } else if (strcmp(MLX5_TXQS_MAX_VEC, key) == 0) { 2029 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2030 } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { 2031 config->mps = !!tmp; 2032 } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { 2033 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2034 } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { 2035 DRV_LOG(WARNING, "%s: deprecated parameter," 2036 " converted to txq_inline_mpw", key); 2037 config->txq_inline_mpw = tmp; 2038 } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) { 2039 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2040 } else if (strcmp(MLX5_TX_PP, key) == 0) { 2041 if (!mod) { 2042 DRV_LOG(ERR, "Zero Tx packet pacing parameter"); 2043 rte_errno = EINVAL; 2044 return -rte_errno; 2045 } 2046 config->tx_pp = tmp; 2047 } else if (strcmp(MLX5_TX_SKEW, key) == 0) { 2048 config->tx_skew = tmp; 2049 } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { 2050 config->rx_vec_en = !!tmp; 2051 } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) { 2052 config->l3_vxlan_en = !!tmp; 2053 } else if (strcmp(MLX5_VF_NL_EN, key) == 0) { 2054 config->vf_nl_en = !!tmp; 2055 } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) { 2056 config->dv_esw_en = !!tmp; 2057 } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) { 2058 config->dv_flow_en = !!tmp; 2059 } else if (strcmp(MLX5_DV_XMETA_EN, key) == 0) { 2060 if (tmp != MLX5_XMETA_MODE_LEGACY && 2061 tmp != MLX5_XMETA_MODE_META16 && 2062 tmp != MLX5_XMETA_MODE_META32 && 2063 tmp != MLX5_XMETA_MODE_MISS_INFO) { 2064 DRV_LOG(ERR, "invalid extensive " 2065 "metadata parameter"); 2066 rte_errno = EINVAL; 2067 return -rte_errno; 2068 } 2069 if (tmp != MLX5_XMETA_MODE_MISS_INFO) 2070 config->dv_xmeta_en = tmp; 2071 else 2072 config->dv_miss_info = 1; 2073 } else if (strcmp(MLX5_LACP_BY_USER, key) == 0) { 2074 config->lacp_by_user = !!tmp; 2075 } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) { 2076 config->max_dump_files_num = tmp; 2077 } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) { 2078 config->lro.timeout = tmp; 2079 } else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) { 2080 DRV_LOG(DEBUG, "class argument is %s.", val); 2081 } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) { 2082 config->log_hp_size = tmp; 2083 } else if (strcmp(MLX5_RECLAIM_MEM, key) == 0) { 2084 if (tmp != MLX5_RCM_NONE && 2085 tmp != MLX5_RCM_LIGHT && 2086 tmp != MLX5_RCM_AGGR) { 2087 DRV_LOG(ERR, "Unrecognize %s: \"%s\"", key, val); 2088 rte_errno = EINVAL; 2089 return -rte_errno; 2090 } 2091 config->reclaim_mode = tmp; 2092 } else if (strcmp(MLX5_DECAP_EN, key) == 0) { 2093 config->decap_en = !!tmp; 2094 } else if (strcmp(MLX5_ALLOW_DUPLICATE_PATTERN, key) == 0) { 2095 config->allow_duplicate_pattern = !!tmp; 2096 } else { 2097 DRV_LOG(WARNING, "%s: unknown parameter", key); 2098 rte_errno = EINVAL; 2099 return -rte_errno; 2100 } 2101 return 0; 2102 } 2103 2104 /** 2105 * Parse device parameters. 2106 * 2107 * @param config 2108 * Pointer to device configuration structure. 2109 * @param devargs 2110 * Device arguments structure. 2111 * 2112 * @return 2113 * 0 on success, a negative errno value otherwise and rte_errno is set. 2114 */ 2115 int 2116 mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs) 2117 { 2118 const char **params = (const char *[]){ 2119 MLX5_DRIVER_KEY, 2120 MLX5_RXQ_CQE_COMP_EN, 2121 MLX5_RXQ_PKT_PAD_EN, 2122 MLX5_RX_MPRQ_EN, 2123 MLX5_RX_MPRQ_LOG_STRIDE_NUM, 2124 MLX5_RX_MPRQ_LOG_STRIDE_SIZE, 2125 MLX5_RX_MPRQ_MAX_MEMCPY_LEN, 2126 MLX5_RXQS_MIN_MPRQ, 2127 MLX5_TXQ_INLINE, 2128 MLX5_TXQ_INLINE_MIN, 2129 MLX5_TXQ_INLINE_MAX, 2130 MLX5_TXQ_INLINE_MPW, 2131 MLX5_TXQS_MIN_INLINE, 2132 MLX5_TXQS_MAX_VEC, 2133 MLX5_TXQ_MPW_EN, 2134 MLX5_TXQ_MPW_HDR_DSEG_EN, 2135 MLX5_TXQ_MAX_INLINE_LEN, 2136 MLX5_TX_DB_NC, 2137 MLX5_TX_PP, 2138 MLX5_TX_SKEW, 2139 MLX5_TX_VEC_EN, 2140 MLX5_RX_VEC_EN, 2141 MLX5_L3_VXLAN_EN, 2142 MLX5_VF_NL_EN, 2143 MLX5_DV_ESW_EN, 2144 MLX5_DV_FLOW_EN, 2145 MLX5_DV_XMETA_EN, 2146 MLX5_LACP_BY_USER, 2147 MLX5_MR_EXT_MEMSEG_EN, 2148 MLX5_REPRESENTOR, 2149 MLX5_MAX_DUMP_FILES_NUM, 2150 MLX5_LRO_TIMEOUT_USEC, 2151 RTE_DEVARGS_KEY_CLASS, 2152 MLX5_HP_BUF_SIZE, 2153 MLX5_RECLAIM_MEM, 2154 MLX5_SYS_MEM_EN, 2155 MLX5_DECAP_EN, 2156 MLX5_ALLOW_DUPLICATE_PATTERN, 2157 MLX5_MR_MEMPOOL_REG_EN, 2158 NULL, 2159 }; 2160 struct rte_kvargs *kvlist; 2161 int ret = 0; 2162 int i; 2163 2164 if (devargs == NULL) 2165 return 0; 2166 /* Following UGLY cast is done to pass checkpatch. */ 2167 kvlist = rte_kvargs_parse(devargs->args, params); 2168 if (kvlist == NULL) { 2169 rte_errno = EINVAL; 2170 return -rte_errno; 2171 } 2172 /* Process parameters. */ 2173 for (i = 0; (params[i] != NULL); ++i) { 2174 if (rte_kvargs_count(kvlist, params[i])) { 2175 ret = rte_kvargs_process(kvlist, params[i], 2176 mlx5_args_check, config); 2177 if (ret) { 2178 rte_errno = EINVAL; 2179 rte_kvargs_free(kvlist); 2180 return -rte_errno; 2181 } 2182 } 2183 } 2184 rte_kvargs_free(kvlist); 2185 return 0; 2186 } 2187 2188 /** 2189 * Configures the minimal amount of data to inline into WQE 2190 * while sending packets. 2191 * 2192 * - the txq_inline_min has the maximal priority, if this 2193 * key is specified in devargs 2194 * - if DevX is enabled the inline mode is queried from the 2195 * device (HCA attributes and NIC vport context if needed). 2196 * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4 Lx 2197 * and none (0 bytes) for other NICs 2198 * 2199 * @param spawn 2200 * Verbs device parameters (name, port, switch_info) to spawn. 2201 * @param config 2202 * Device configuration parameters. 2203 */ 2204 void 2205 mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn, 2206 struct mlx5_dev_config *config) 2207 { 2208 if (config->txq_inline_min != MLX5_ARG_UNSET) { 2209 /* Application defines size of inlined data explicitly. */ 2210 if (spawn->pci_dev != NULL) { 2211 switch (spawn->pci_dev->id.device_id) { 2212 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2213 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2214 if (config->txq_inline_min < 2215 (int)MLX5_INLINE_HSIZE_L2) { 2216 DRV_LOG(DEBUG, 2217 "txq_inline_mix aligned to minimal ConnectX-4 required value %d", 2218 (int)MLX5_INLINE_HSIZE_L2); 2219 config->txq_inline_min = 2220 MLX5_INLINE_HSIZE_L2; 2221 } 2222 break; 2223 } 2224 } 2225 goto exit; 2226 } 2227 if (config->hca_attr.eth_net_offloads) { 2228 /* We have DevX enabled, inline mode queried successfully. */ 2229 switch (config->hca_attr.wqe_inline_mode) { 2230 case MLX5_CAP_INLINE_MODE_L2: 2231 /* outer L2 header must be inlined. */ 2232 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2233 goto exit; 2234 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2235 /* No inline data are required by NIC. */ 2236 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2237 config->hw_vlan_insert = 2238 config->hca_attr.wqe_vlan_insert; 2239 DRV_LOG(DEBUG, "Tx VLAN insertion is supported"); 2240 goto exit; 2241 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2242 /* inline mode is defined by NIC vport context. */ 2243 if (!config->hca_attr.eth_virt) 2244 break; 2245 switch (config->hca_attr.vport_inline_mode) { 2246 case MLX5_INLINE_MODE_NONE: 2247 config->txq_inline_min = 2248 MLX5_INLINE_HSIZE_NONE; 2249 goto exit; 2250 case MLX5_INLINE_MODE_L2: 2251 config->txq_inline_min = 2252 MLX5_INLINE_HSIZE_L2; 2253 goto exit; 2254 case MLX5_INLINE_MODE_IP: 2255 config->txq_inline_min = 2256 MLX5_INLINE_HSIZE_L3; 2257 goto exit; 2258 case MLX5_INLINE_MODE_TCP_UDP: 2259 config->txq_inline_min = 2260 MLX5_INLINE_HSIZE_L4; 2261 goto exit; 2262 case MLX5_INLINE_MODE_INNER_L2: 2263 config->txq_inline_min = 2264 MLX5_INLINE_HSIZE_INNER_L2; 2265 goto exit; 2266 case MLX5_INLINE_MODE_INNER_IP: 2267 config->txq_inline_min = 2268 MLX5_INLINE_HSIZE_INNER_L3; 2269 goto exit; 2270 case MLX5_INLINE_MODE_INNER_TCP_UDP: 2271 config->txq_inline_min = 2272 MLX5_INLINE_HSIZE_INNER_L4; 2273 goto exit; 2274 } 2275 } 2276 } 2277 if (spawn->pci_dev == NULL) { 2278 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2279 goto exit; 2280 } 2281 /* 2282 * We get here if we are unable to deduce 2283 * inline data size with DevX. Try PCI ID 2284 * to determine old NICs. 2285 */ 2286 switch (spawn->pci_dev->id.device_id) { 2287 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2288 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2289 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: 2290 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 2291 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2292 config->hw_vlan_insert = 0; 2293 break; 2294 case PCI_DEVICE_ID_MELLANOX_CONNECTX5: 2295 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 2296 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: 2297 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 2298 /* 2299 * These NICs support VLAN insertion from WQE and 2300 * report the wqe_vlan_insert flag. But there is the bug 2301 * and PFC control may be broken, so disable feature. 2302 */ 2303 config->hw_vlan_insert = 0; 2304 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2305 break; 2306 default: 2307 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2308 break; 2309 } 2310 exit: 2311 DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min); 2312 } 2313 2314 /** 2315 * Configures the metadata mask fields in the shared context. 2316 * 2317 * @param [in] dev 2318 * Pointer to Ethernet device. 2319 */ 2320 void 2321 mlx5_set_metadata_mask(struct rte_eth_dev *dev) 2322 { 2323 struct mlx5_priv *priv = dev->data->dev_private; 2324 struct mlx5_dev_ctx_shared *sh = priv->sh; 2325 uint32_t meta, mark, reg_c0; 2326 2327 reg_c0 = ~priv->vport_meta_mask; 2328 switch (priv->config.dv_xmeta_en) { 2329 case MLX5_XMETA_MODE_LEGACY: 2330 meta = UINT32_MAX; 2331 mark = MLX5_FLOW_MARK_MASK; 2332 break; 2333 case MLX5_XMETA_MODE_META16: 2334 meta = reg_c0 >> rte_bsf32(reg_c0); 2335 mark = MLX5_FLOW_MARK_MASK; 2336 break; 2337 case MLX5_XMETA_MODE_META32: 2338 meta = UINT32_MAX; 2339 mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK; 2340 break; 2341 default: 2342 meta = 0; 2343 mark = 0; 2344 MLX5_ASSERT(false); 2345 break; 2346 } 2347 if (sh->dv_mark_mask && sh->dv_mark_mask != mark) 2348 DRV_LOG(WARNING, "metadata MARK mask mismatche %08X:%08X", 2349 sh->dv_mark_mask, mark); 2350 else 2351 sh->dv_mark_mask = mark; 2352 if (sh->dv_meta_mask && sh->dv_meta_mask != meta) 2353 DRV_LOG(WARNING, "metadata META mask mismatche %08X:%08X", 2354 sh->dv_meta_mask, meta); 2355 else 2356 sh->dv_meta_mask = meta; 2357 if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0) 2358 DRV_LOG(WARNING, "metadata reg_c0 mask mismatche %08X:%08X", 2359 sh->dv_meta_mask, reg_c0); 2360 else 2361 sh->dv_regc0_mask = reg_c0; 2362 DRV_LOG(DEBUG, "metadata mode %u", priv->config.dv_xmeta_en); 2363 DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask); 2364 DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask); 2365 DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask); 2366 } 2367 2368 int 2369 rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n) 2370 { 2371 static const char *const dynf_names[] = { 2372 RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, 2373 RTE_MBUF_DYNFLAG_METADATA_NAME, 2374 RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME 2375 }; 2376 unsigned int i; 2377 2378 if (n < RTE_DIM(dynf_names)) 2379 return -ENOMEM; 2380 for (i = 0; i < RTE_DIM(dynf_names); i++) { 2381 if (names[i] == NULL) 2382 return -EINVAL; 2383 strcpy(names[i], dynf_names[i]); 2384 } 2385 return RTE_DIM(dynf_names); 2386 } 2387 2388 /** 2389 * Comparison callback to sort device data. 2390 * 2391 * This is meant to be used with qsort(). 2392 * 2393 * @param a[in] 2394 * Pointer to pointer to first data object. 2395 * @param b[in] 2396 * Pointer to pointer to second data object. 2397 * 2398 * @return 2399 * 0 if both objects are equal, less than 0 if the first argument is less 2400 * than the second, greater than 0 otherwise. 2401 */ 2402 int 2403 mlx5_dev_check_sibling_config(struct mlx5_priv *priv, 2404 struct mlx5_dev_config *config, 2405 struct rte_device *dpdk_dev) 2406 { 2407 struct mlx5_dev_ctx_shared *sh = priv->sh; 2408 struct mlx5_dev_config *sh_conf = NULL; 2409 uint16_t port_id; 2410 2411 MLX5_ASSERT(sh); 2412 /* Nothing to compare for the single/first device. */ 2413 if (sh->refcnt == 1) 2414 return 0; 2415 /* Find the device with shared context. */ 2416 MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) { 2417 struct mlx5_priv *opriv = 2418 rte_eth_devices[port_id].data->dev_private; 2419 2420 if (opriv && opriv != priv && opriv->sh == sh) { 2421 sh_conf = &opriv->config; 2422 break; 2423 } 2424 } 2425 if (!sh_conf) 2426 return 0; 2427 if (sh_conf->dv_flow_en ^ config->dv_flow_en) { 2428 DRV_LOG(ERR, "\"dv_flow_en\" configuration mismatch" 2429 " for shared %s context", sh->ibdev_name); 2430 rte_errno = EINVAL; 2431 return rte_errno; 2432 } 2433 if (sh_conf->dv_xmeta_en ^ config->dv_xmeta_en) { 2434 DRV_LOG(ERR, "\"dv_xmeta_en\" configuration mismatch" 2435 " for shared %s context", sh->ibdev_name); 2436 rte_errno = EINVAL; 2437 return rte_errno; 2438 } 2439 return 0; 2440 } 2441 2442 /** 2443 * Look for the ethernet device belonging to mlx5 driver. 2444 * 2445 * @param[in] port_id 2446 * port_id to start looking for device. 2447 * @param[in] odev 2448 * Pointer to the hint device. When device is being probed 2449 * the its siblings (master and preceding representors might 2450 * not have assigned driver yet (because the mlx5_os_pci_probe() 2451 * is not completed yet, for this case match on hint 2452 * device may be used to detect sibling device. 2453 * 2454 * @return 2455 * port_id of found device, RTE_MAX_ETHPORT if not found. 2456 */ 2457 uint16_t 2458 mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev) 2459 { 2460 while (port_id < RTE_MAX_ETHPORTS) { 2461 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 2462 2463 if (dev->state != RTE_ETH_DEV_UNUSED && 2464 dev->device && 2465 (dev->device == odev || 2466 (dev->device->driver && 2467 dev->device->driver->name && 2468 ((strcmp(dev->device->driver->name, 2469 MLX5_PCI_DRIVER_NAME) == 0) || 2470 (strcmp(dev->device->driver->name, 2471 MLX5_AUXILIARY_DRIVER_NAME) == 0))))) 2472 break; 2473 port_id++; 2474 } 2475 if (port_id >= RTE_MAX_ETHPORTS) 2476 return RTE_MAX_ETHPORTS; 2477 return port_id; 2478 } 2479 2480 /** 2481 * Callback to remove a device. 2482 * 2483 * This function removes all Ethernet devices belong to a given device. 2484 * 2485 * @param[in] cdev 2486 * Pointer to the generic device. 2487 * 2488 * @return 2489 * 0 on success, the function cannot fail. 2490 */ 2491 int 2492 mlx5_net_remove(struct mlx5_common_device *cdev) 2493 { 2494 uint16_t port_id; 2495 int ret = 0; 2496 2497 RTE_ETH_FOREACH_DEV_OF(port_id, cdev->dev) { 2498 /* 2499 * mlx5_dev_close() is not registered to secondary process, 2500 * call the close function explicitly for secondary process. 2501 */ 2502 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 2503 ret |= mlx5_dev_close(&rte_eth_devices[port_id]); 2504 else 2505 ret |= rte_eth_dev_close(port_id); 2506 } 2507 return ret == 0 ? 0 : -EIO; 2508 } 2509 2510 static const struct rte_pci_id mlx5_pci_id_map[] = { 2511 { 2512 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2513 PCI_DEVICE_ID_MELLANOX_CONNECTX4) 2514 }, 2515 { 2516 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2517 PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) 2518 }, 2519 { 2520 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2521 PCI_DEVICE_ID_MELLANOX_CONNECTX4LX) 2522 }, 2523 { 2524 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2525 PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) 2526 }, 2527 { 2528 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2529 PCI_DEVICE_ID_MELLANOX_CONNECTX5) 2530 }, 2531 { 2532 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2533 PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) 2534 }, 2535 { 2536 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2537 PCI_DEVICE_ID_MELLANOX_CONNECTX5EX) 2538 }, 2539 { 2540 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2541 PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF) 2542 }, 2543 { 2544 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2545 PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) 2546 }, 2547 { 2548 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2549 PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF) 2550 }, 2551 { 2552 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2553 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 2554 }, 2555 { 2556 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2557 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF) 2558 }, 2559 { 2560 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2561 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 2562 }, 2563 { 2564 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2565 PCI_DEVICE_ID_MELLANOX_CONNECTXVF) 2566 }, 2567 { 2568 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2569 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF) 2570 }, 2571 { 2572 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2573 PCI_DEVICE_ID_MELLANOX_CONNECTX6LX) 2574 }, 2575 { 2576 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2577 PCI_DEVICE_ID_MELLANOX_CONNECTX7) 2578 }, 2579 { 2580 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2581 PCI_DEVICE_ID_MELLANOX_CONNECTX7BF) 2582 }, 2583 { 2584 .vendor_id = 0 2585 } 2586 }; 2587 2588 static struct mlx5_class_driver mlx5_net_driver = { 2589 .drv_class = MLX5_CLASS_ETH, 2590 .name = RTE_STR(MLX5_ETH_DRIVER_NAME), 2591 .id_table = mlx5_pci_id_map, 2592 .probe = mlx5_os_net_probe, 2593 .remove = mlx5_net_remove, 2594 .probe_again = 1, 2595 .intr_lsc = 1, 2596 .intr_rmv = 1, 2597 }; 2598 2599 /* Initialize driver log type. */ 2600 RTE_LOG_REGISTER_DEFAULT(mlx5_logtype, NOTICE) 2601 2602 /** 2603 * Driver initialization routine. 2604 */ 2605 RTE_INIT(rte_mlx5_pmd_init) 2606 { 2607 pthread_mutex_init(&mlx5_dev_ctx_list_mutex, NULL); 2608 mlx5_common_init(); 2609 /* Build the static tables for Verbs conversion. */ 2610 mlx5_set_ptype_table(); 2611 mlx5_set_cksum_table(); 2612 mlx5_set_swp_types_table(); 2613 if (mlx5_glue) 2614 mlx5_class_driver_register(&mlx5_net_driver); 2615 } 2616 2617 RTE_PMD_EXPORT_NAME(MLX5_ETH_DRIVER_NAME, __COUNTER__); 2618 RTE_PMD_REGISTER_PCI_TABLE(MLX5_ETH_DRIVER_NAME, mlx5_pci_id_map); 2619 RTE_PMD_REGISTER_KMOD_DEP(MLX5_ETH_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 2620