1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <stddef.h> 7 #include <unistd.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <errno.h> 12 13 #include <rte_malloc.h> 14 #include <ethdev_driver.h> 15 #include <rte_pci.h> 16 #include <rte_bus_pci.h> 17 #include <rte_common.h> 18 #include <rte_kvargs.h> 19 #include <rte_rwlock.h> 20 #include <rte_spinlock.h> 21 #include <rte_string_fns.h> 22 #include <rte_alarm.h> 23 #include <rte_cycles.h> 24 25 #include <mlx5_glue.h> 26 #include <mlx5_devx_cmds.h> 27 #include <mlx5_common.h> 28 #include <mlx5_common_os.h> 29 #include <mlx5_common_mp.h> 30 #include <mlx5_malloc.h> 31 32 #include "mlx5_defs.h" 33 #include "mlx5.h" 34 #include "mlx5_utils.h" 35 #include "mlx5_rxtx.h" 36 #include "mlx5_rx.h" 37 #include "mlx5_tx.h" 38 #include "mlx5_autoconf.h" 39 #include "mlx5_flow.h" 40 #include "mlx5_flow_os.h" 41 #include "rte_pmd_mlx5.h" 42 43 #define MLX5_ETH_DRIVER_NAME mlx5_eth 44 45 /* Driver type key for new device global syntax. */ 46 #define MLX5_DRIVER_KEY "driver" 47 48 /* Device parameter to enable RX completion queue compression. */ 49 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en" 50 51 /* Device parameter to enable padding Rx packet to cacheline size. */ 52 #define MLX5_RXQ_PKT_PAD_EN "rxq_pkt_pad_en" 53 54 /* Device parameter to enable Multi-Packet Rx queue. */ 55 #define MLX5_RX_MPRQ_EN "mprq_en" 56 57 /* Device parameter to configure log 2 of the number of strides for MPRQ. */ 58 #define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num" 59 60 /* Device parameter to configure log 2 of the stride size for MPRQ. */ 61 #define MLX5_RX_MPRQ_LOG_STRIDE_SIZE "mprq_log_stride_size" 62 63 /* Device parameter to limit the size of memcpy'd packet for MPRQ. */ 64 #define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len" 65 66 /* Device parameter to set the minimum number of Rx queues to enable MPRQ. */ 67 #define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq" 68 69 /* Device parameter to configure inline send. Deprecated, ignored.*/ 70 #define MLX5_TXQ_INLINE "txq_inline" 71 72 /* Device parameter to limit packet size to inline with ordinary SEND. */ 73 #define MLX5_TXQ_INLINE_MAX "txq_inline_max" 74 75 /* Device parameter to configure minimal data size to inline. */ 76 #define MLX5_TXQ_INLINE_MIN "txq_inline_min" 77 78 /* Device parameter to limit packet size to inline with Enhanced MPW. */ 79 #define MLX5_TXQ_INLINE_MPW "txq_inline_mpw" 80 81 /* 82 * Device parameter to configure the number of TX queues threshold for 83 * enabling inline send. 84 */ 85 #define MLX5_TXQS_MIN_INLINE "txqs_min_inline" 86 87 /* 88 * Device parameter to configure the number of TX queues threshold for 89 * enabling vectorized Tx, deprecated, ignored (no vectorized Tx routines). 90 */ 91 #define MLX5_TXQS_MAX_VEC "txqs_max_vec" 92 93 /* Device parameter to enable multi-packet send WQEs. */ 94 #define MLX5_TXQ_MPW_EN "txq_mpw_en" 95 96 /* 97 * Device parameter to force doorbell register mapping 98 * to non-cahed region eliminating the extra write memory barrier. 99 */ 100 #define MLX5_TX_DB_NC "tx_db_nc" 101 102 /* 103 * Device parameter to include 2 dsegs in the title WQEBB. 104 * Deprecated, ignored. 105 */ 106 #define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en" 107 108 /* 109 * Device parameter to limit the size of inlining packet. 110 * Deprecated, ignored. 111 */ 112 #define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len" 113 114 /* 115 * Device parameter to enable Tx scheduling on timestamps 116 * and specify the packet pacing granularity in nanoseconds. 117 */ 118 #define MLX5_TX_PP "tx_pp" 119 120 /* 121 * Device parameter to specify skew in nanoseconds on Tx datapath, 122 * it represents the time between SQ start WQE processing and 123 * appearing actual packet data on the wire. 124 */ 125 #define MLX5_TX_SKEW "tx_skew" 126 127 /* 128 * Device parameter to enable hardware Tx vector. 129 * Deprecated, ignored (no vectorized Tx routines anymore). 130 */ 131 #define MLX5_TX_VEC_EN "tx_vec_en" 132 133 /* Device parameter to enable hardware Rx vector. */ 134 #define MLX5_RX_VEC_EN "rx_vec_en" 135 136 /* Allow L3 VXLAN flow creation. */ 137 #define MLX5_L3_VXLAN_EN "l3_vxlan_en" 138 139 /* Activate DV E-Switch flow steering. */ 140 #define MLX5_DV_ESW_EN "dv_esw_en" 141 142 /* Activate DV flow steering. */ 143 #define MLX5_DV_FLOW_EN "dv_flow_en" 144 145 /* Enable extensive flow metadata support. */ 146 #define MLX5_DV_XMETA_EN "dv_xmeta_en" 147 148 /* Device parameter to let the user manage the lacp traffic of bonded device */ 149 #define MLX5_LACP_BY_USER "lacp_by_user" 150 151 /* Activate Netlink support in VF mode. */ 152 #define MLX5_VF_NL_EN "vf_nl_en" 153 154 /* Enable extending memsegs when creating a MR. */ 155 #define MLX5_MR_EXT_MEMSEG_EN "mr_ext_memseg_en" 156 157 /* Select port representors to instantiate. */ 158 #define MLX5_REPRESENTOR "representor" 159 160 /* Device parameter to configure the maximum number of dump files per queue. */ 161 #define MLX5_MAX_DUMP_FILES_NUM "max_dump_files_num" 162 163 /* Configure timeout of LRO session (in microseconds). */ 164 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec" 165 166 /* 167 * Device parameter to configure the total data buffer size for a single 168 * hairpin queue (logarithm value). 169 */ 170 #define MLX5_HP_BUF_SIZE "hp_buf_log_sz" 171 172 /* Flow memory reclaim mode. */ 173 #define MLX5_RECLAIM_MEM "reclaim_mem_mode" 174 175 /* The default memory allocator used in PMD. */ 176 #define MLX5_SYS_MEM_EN "sys_mem_en" 177 /* Decap will be used or not. */ 178 #define MLX5_DECAP_EN "decap_en" 179 180 /* Device parameter to configure allow or prevent duplicate rules pattern. */ 181 #define MLX5_ALLOW_DUPLICATE_PATTERN "allow_duplicate_pattern" 182 183 /* Device parameter to configure implicit registration of mempool memory. */ 184 #define MLX5_MR_MEMPOOL_REG_EN "mr_mempool_reg_en" 185 186 /* Shared memory between primary and secondary processes. */ 187 struct mlx5_shared_data *mlx5_shared_data; 188 189 /** Driver-specific log messages type. */ 190 int mlx5_logtype; 191 192 static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = 193 LIST_HEAD_INITIALIZER(); 194 static pthread_mutex_t mlx5_dev_ctx_list_mutex; 195 static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { 196 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 197 [MLX5_IPOOL_DECAP_ENCAP] = { 198 .size = sizeof(struct mlx5_flow_dv_encap_decap_resource), 199 .trunk_size = 64, 200 .grow_trunk = 3, 201 .grow_shift = 2, 202 .need_lock = 1, 203 .release_mem_en = 1, 204 .malloc = mlx5_malloc, 205 .free = mlx5_free, 206 .type = "mlx5_encap_decap_ipool", 207 }, 208 [MLX5_IPOOL_PUSH_VLAN] = { 209 .size = sizeof(struct mlx5_flow_dv_push_vlan_action_resource), 210 .trunk_size = 64, 211 .grow_trunk = 3, 212 .grow_shift = 2, 213 .need_lock = 1, 214 .release_mem_en = 1, 215 .malloc = mlx5_malloc, 216 .free = mlx5_free, 217 .type = "mlx5_push_vlan_ipool", 218 }, 219 [MLX5_IPOOL_TAG] = { 220 .size = sizeof(struct mlx5_flow_dv_tag_resource), 221 .trunk_size = 64, 222 .grow_trunk = 3, 223 .grow_shift = 2, 224 .need_lock = 1, 225 .release_mem_en = 0, 226 .per_core_cache = (1 << 16), 227 .malloc = mlx5_malloc, 228 .free = mlx5_free, 229 .type = "mlx5_tag_ipool", 230 }, 231 [MLX5_IPOOL_PORT_ID] = { 232 .size = sizeof(struct mlx5_flow_dv_port_id_action_resource), 233 .trunk_size = 64, 234 .grow_trunk = 3, 235 .grow_shift = 2, 236 .need_lock = 1, 237 .release_mem_en = 1, 238 .malloc = mlx5_malloc, 239 .free = mlx5_free, 240 .type = "mlx5_port_id_ipool", 241 }, 242 [MLX5_IPOOL_JUMP] = { 243 .size = sizeof(struct mlx5_flow_tbl_data_entry), 244 .trunk_size = 64, 245 .grow_trunk = 3, 246 .grow_shift = 2, 247 .need_lock = 1, 248 .release_mem_en = 1, 249 .malloc = mlx5_malloc, 250 .free = mlx5_free, 251 .type = "mlx5_jump_ipool", 252 }, 253 [MLX5_IPOOL_SAMPLE] = { 254 .size = sizeof(struct mlx5_flow_dv_sample_resource), 255 .trunk_size = 64, 256 .grow_trunk = 3, 257 .grow_shift = 2, 258 .need_lock = 1, 259 .release_mem_en = 1, 260 .malloc = mlx5_malloc, 261 .free = mlx5_free, 262 .type = "mlx5_sample_ipool", 263 }, 264 [MLX5_IPOOL_DEST_ARRAY] = { 265 .size = sizeof(struct mlx5_flow_dv_dest_array_resource), 266 .trunk_size = 64, 267 .grow_trunk = 3, 268 .grow_shift = 2, 269 .need_lock = 1, 270 .release_mem_en = 1, 271 .malloc = mlx5_malloc, 272 .free = mlx5_free, 273 .type = "mlx5_dest_array_ipool", 274 }, 275 [MLX5_IPOOL_TUNNEL_ID] = { 276 .size = sizeof(struct mlx5_flow_tunnel), 277 .trunk_size = MLX5_MAX_TUNNELS, 278 .need_lock = 1, 279 .release_mem_en = 1, 280 .type = "mlx5_tunnel_offload", 281 }, 282 [MLX5_IPOOL_TNL_TBL_ID] = { 283 .size = 0, 284 .need_lock = 1, 285 .type = "mlx5_flow_tnl_tbl_ipool", 286 }, 287 #endif 288 [MLX5_IPOOL_MTR] = { 289 /** 290 * The ipool index should grow continually from small to big, 291 * for meter idx, so not set grow_trunk to avoid meter index 292 * not jump continually. 293 */ 294 .size = sizeof(struct mlx5_legacy_flow_meter), 295 .trunk_size = 64, 296 .need_lock = 1, 297 .release_mem_en = 1, 298 .malloc = mlx5_malloc, 299 .free = mlx5_free, 300 .type = "mlx5_meter_ipool", 301 }, 302 [MLX5_IPOOL_MCP] = { 303 .size = sizeof(struct mlx5_flow_mreg_copy_resource), 304 .trunk_size = 64, 305 .grow_trunk = 3, 306 .grow_shift = 2, 307 .need_lock = 1, 308 .release_mem_en = 1, 309 .malloc = mlx5_malloc, 310 .free = mlx5_free, 311 .type = "mlx5_mcp_ipool", 312 }, 313 [MLX5_IPOOL_HRXQ] = { 314 .size = (sizeof(struct mlx5_hrxq) + MLX5_RSS_HASH_KEY_LEN), 315 .trunk_size = 64, 316 .grow_trunk = 3, 317 .grow_shift = 2, 318 .need_lock = 1, 319 .release_mem_en = 1, 320 .malloc = mlx5_malloc, 321 .free = mlx5_free, 322 .type = "mlx5_hrxq_ipool", 323 }, 324 [MLX5_IPOOL_MLX5_FLOW] = { 325 /* 326 * MLX5_IPOOL_MLX5_FLOW size varies for DV and VERBS flows. 327 * It set in run time according to PCI function configuration. 328 */ 329 .size = 0, 330 .trunk_size = 64, 331 .grow_trunk = 3, 332 .grow_shift = 2, 333 .need_lock = 1, 334 .release_mem_en = 0, 335 .per_core_cache = 1 << 19, 336 .malloc = mlx5_malloc, 337 .free = mlx5_free, 338 .type = "mlx5_flow_handle_ipool", 339 }, 340 [MLX5_IPOOL_RTE_FLOW] = { 341 .size = sizeof(struct rte_flow), 342 .trunk_size = 4096, 343 .need_lock = 1, 344 .release_mem_en = 1, 345 .malloc = mlx5_malloc, 346 .free = mlx5_free, 347 .type = "rte_flow_ipool", 348 }, 349 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID] = { 350 .size = 0, 351 .need_lock = 1, 352 .type = "mlx5_flow_rss_id_ipool", 353 }, 354 [MLX5_IPOOL_RSS_SHARED_ACTIONS] = { 355 .size = sizeof(struct mlx5_shared_action_rss), 356 .trunk_size = 64, 357 .grow_trunk = 3, 358 .grow_shift = 2, 359 .need_lock = 1, 360 .release_mem_en = 1, 361 .malloc = mlx5_malloc, 362 .free = mlx5_free, 363 .type = "mlx5_shared_action_rss", 364 }, 365 [MLX5_IPOOL_MTR_POLICY] = { 366 /** 367 * The ipool index should grow continually from small to big, 368 * for policy idx, so not set grow_trunk to avoid policy index 369 * not jump continually. 370 */ 371 .size = sizeof(struct mlx5_flow_meter_sub_policy), 372 .trunk_size = 64, 373 .need_lock = 1, 374 .release_mem_en = 1, 375 .malloc = mlx5_malloc, 376 .free = mlx5_free, 377 .type = "mlx5_meter_policy_ipool", 378 }, 379 }; 380 381 382 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512 383 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16 384 385 #define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024 386 387 /** 388 * Decide whether representor ID is a HPF(host PF) port on BF2. 389 * 390 * @param dev 391 * Pointer to Ethernet device structure. 392 * 393 * @return 394 * Non-zero if HPF, otherwise 0. 395 */ 396 bool 397 mlx5_is_hpf(struct rte_eth_dev *dev) 398 { 399 struct mlx5_priv *priv = dev->data->dev_private; 400 uint16_t repr = MLX5_REPRESENTOR_REPR(priv->representor_id); 401 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 402 403 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_VF && 404 MLX5_REPRESENTOR_REPR(-1) == repr; 405 } 406 407 /** 408 * Decide whether representor ID is a SF port representor. 409 * 410 * @param dev 411 * Pointer to Ethernet device structure. 412 * 413 * @return 414 * Non-zero if HPF, otherwise 0. 415 */ 416 bool 417 mlx5_is_sf_repr(struct rte_eth_dev *dev) 418 { 419 struct mlx5_priv *priv = dev->data->dev_private; 420 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 421 422 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_SF; 423 } 424 425 /** 426 * Initialize the ASO aging management structure. 427 * 428 * @param[in] sh 429 * Pointer to mlx5_dev_ctx_shared object to free 430 * 431 * @return 432 * 0 on success, a negative errno value otherwise and rte_errno is set. 433 */ 434 int 435 mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh) 436 { 437 int err; 438 439 if (sh->aso_age_mng) 440 return 0; 441 sh->aso_age_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->aso_age_mng), 442 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 443 if (!sh->aso_age_mng) { 444 DRV_LOG(ERR, "aso_age_mng allocation was failed."); 445 rte_errno = ENOMEM; 446 return -ENOMEM; 447 } 448 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_FLOW_HIT); 449 if (err) { 450 mlx5_free(sh->aso_age_mng); 451 return -1; 452 } 453 rte_rwlock_init(&sh->aso_age_mng->resize_rwl); 454 rte_spinlock_init(&sh->aso_age_mng->free_sl); 455 LIST_INIT(&sh->aso_age_mng->free); 456 return 0; 457 } 458 459 /** 460 * Close and release all the resources of the ASO aging management structure. 461 * 462 * @param[in] sh 463 * Pointer to mlx5_dev_ctx_shared object to free. 464 */ 465 static void 466 mlx5_flow_aso_age_mng_close(struct mlx5_dev_ctx_shared *sh) 467 { 468 int i, j; 469 470 mlx5_aso_flow_hit_queue_poll_stop(sh); 471 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_FLOW_HIT); 472 if (sh->aso_age_mng->pools) { 473 struct mlx5_aso_age_pool *pool; 474 475 for (i = 0; i < sh->aso_age_mng->next; ++i) { 476 pool = sh->aso_age_mng->pools[i]; 477 claim_zero(mlx5_devx_cmd_destroy 478 (pool->flow_hit_aso_obj)); 479 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) 480 if (pool->actions[j].dr_action) 481 claim_zero 482 (mlx5_flow_os_destroy_flow_action 483 (pool->actions[j].dr_action)); 484 mlx5_free(pool); 485 } 486 mlx5_free(sh->aso_age_mng->pools); 487 } 488 mlx5_free(sh->aso_age_mng); 489 } 490 491 /** 492 * Initialize the shared aging list information per port. 493 * 494 * @param[in] sh 495 * Pointer to mlx5_dev_ctx_shared object. 496 */ 497 static void 498 mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh) 499 { 500 uint32_t i; 501 struct mlx5_age_info *age_info; 502 503 for (i = 0; i < sh->max_port; i++) { 504 age_info = &sh->port[i].age_info; 505 age_info->flags = 0; 506 TAILQ_INIT(&age_info->aged_counters); 507 LIST_INIT(&age_info->aged_aso); 508 rte_spinlock_init(&age_info->aged_sl); 509 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 510 } 511 } 512 513 /** 514 * Initialize the counters management structure. 515 * 516 * @param[in] sh 517 * Pointer to mlx5_dev_ctx_shared object to free 518 */ 519 static void 520 mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh) 521 { 522 struct mlx5_hca_attr *attr = &sh->cdev->config.hca_attr; 523 int i; 524 525 memset(&sh->cmng, 0, sizeof(sh->cmng)); 526 TAILQ_INIT(&sh->cmng.flow_counters); 527 sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET; 528 sh->cmng.max_id = -1; 529 sh->cmng.last_pool_idx = POOL_IDX_INVALID; 530 rte_spinlock_init(&sh->cmng.pool_update_sl); 531 for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) { 532 TAILQ_INIT(&sh->cmng.counters[i]); 533 rte_spinlock_init(&sh->cmng.csl[i]); 534 } 535 if (sh->devx && !haswell_broadwell_cpu) { 536 sh->cmng.relaxed_ordering_write = attr->relaxed_ordering_write; 537 sh->cmng.relaxed_ordering_read = attr->relaxed_ordering_read; 538 } 539 } 540 541 /** 542 * Destroy all the resources allocated for a counter memory management. 543 * 544 * @param[in] mng 545 * Pointer to the memory management structure. 546 */ 547 static void 548 mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng) 549 { 550 uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data; 551 552 LIST_REMOVE(mng, next); 553 claim_zero(mlx5_devx_cmd_destroy(mng->dm)); 554 claim_zero(mlx5_os_umem_dereg(mng->umem)); 555 mlx5_free(mem); 556 } 557 558 /** 559 * Close and release all the resources of the counters management. 560 * 561 * @param[in] sh 562 * Pointer to mlx5_dev_ctx_shared object to free. 563 */ 564 static void 565 mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) 566 { 567 struct mlx5_counter_stats_mem_mng *mng; 568 int i, j; 569 int retries = 1024; 570 571 rte_errno = 0; 572 while (--retries) { 573 rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh); 574 if (rte_errno != EINPROGRESS) 575 break; 576 rte_pause(); 577 } 578 579 if (sh->cmng.pools) { 580 struct mlx5_flow_counter_pool *pool; 581 uint16_t n_valid = sh->cmng.n_valid; 582 bool fallback = sh->cmng.counter_fallback; 583 584 for (i = 0; i < n_valid; ++i) { 585 pool = sh->cmng.pools[i]; 586 if (!fallback && pool->min_dcs) 587 claim_zero(mlx5_devx_cmd_destroy 588 (pool->min_dcs)); 589 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) { 590 struct mlx5_flow_counter *cnt = 591 MLX5_POOL_GET_CNT(pool, j); 592 593 if (cnt->action) 594 claim_zero 595 (mlx5_flow_os_destroy_flow_action 596 (cnt->action)); 597 if (fallback && MLX5_POOL_GET_CNT 598 (pool, j)->dcs_when_free) 599 claim_zero(mlx5_devx_cmd_destroy 600 (cnt->dcs_when_free)); 601 } 602 mlx5_free(pool); 603 } 604 mlx5_free(sh->cmng.pools); 605 } 606 mng = LIST_FIRST(&sh->cmng.mem_mngs); 607 while (mng) { 608 mlx5_flow_destroy_counter_stat_mem_mng(mng); 609 mng = LIST_FIRST(&sh->cmng.mem_mngs); 610 } 611 memset(&sh->cmng, 0, sizeof(sh->cmng)); 612 } 613 614 /** 615 * Initialize the aso flow meters management structure. 616 * 617 * @param[in] sh 618 * Pointer to mlx5_dev_ctx_shared object to free 619 */ 620 int 621 mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh) 622 { 623 if (!sh->mtrmng) { 624 sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, 625 sizeof(*sh->mtrmng), 626 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 627 if (!sh->mtrmng) { 628 DRV_LOG(ERR, 629 "meter management allocation was failed."); 630 rte_errno = ENOMEM; 631 return -ENOMEM; 632 } 633 if (sh->meter_aso_en) { 634 rte_spinlock_init(&sh->mtrmng->pools_mng.mtrsl); 635 rte_rwlock_init(&sh->mtrmng->pools_mng.resize_mtrwl); 636 LIST_INIT(&sh->mtrmng->pools_mng.meters); 637 } 638 sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 639 } 640 return 0; 641 } 642 643 /** 644 * Close and release all the resources of 645 * the ASO flow meter management structure. 646 * 647 * @param[in] sh 648 * Pointer to mlx5_dev_ctx_shared object to free. 649 */ 650 static void 651 mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) 652 { 653 struct mlx5_aso_mtr_pool *mtr_pool; 654 struct mlx5_flow_mtr_mng *mtrmng = sh->mtrmng; 655 uint32_t idx; 656 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 657 struct mlx5_aso_mtr *aso_mtr; 658 int i; 659 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 660 661 if (sh->meter_aso_en) { 662 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); 663 idx = mtrmng->pools_mng.n_valid; 664 while (idx--) { 665 mtr_pool = mtrmng->pools_mng.pools[idx]; 666 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 667 for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { 668 aso_mtr = &mtr_pool->mtrs[i]; 669 if (aso_mtr->fm.meter_action) 670 claim_zero 671 (mlx5_glue->destroy_flow_action 672 (aso_mtr->fm.meter_action)); 673 } 674 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 675 claim_zero(mlx5_devx_cmd_destroy 676 (mtr_pool->devx_obj)); 677 mtrmng->pools_mng.n_valid--; 678 mlx5_free(mtr_pool); 679 } 680 mlx5_free(sh->mtrmng->pools_mng.pools); 681 } 682 mlx5_free(sh->mtrmng); 683 sh->mtrmng = NULL; 684 } 685 686 /* Send FLOW_AGED event if needed. */ 687 void 688 mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) 689 { 690 struct mlx5_age_info *age_info; 691 uint32_t i; 692 693 for (i = 0; i < sh->max_port; i++) { 694 age_info = &sh->port[i].age_info; 695 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 696 continue; 697 MLX5_AGE_UNSET(age_info, MLX5_AGE_EVENT_NEW); 698 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) { 699 MLX5_AGE_UNSET(age_info, MLX5_AGE_TRIGGER); 700 rte_eth_dev_callback_process 701 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 702 RTE_ETH_EVENT_FLOW_AGED, NULL); 703 } 704 } 705 } 706 707 /* 708 * Initialize the ASO connection tracking structure. 709 * 710 * @param[in] sh 711 * Pointer to mlx5_dev_ctx_shared object. 712 * 713 * @return 714 * 0 on success, a negative errno value otherwise and rte_errno is set. 715 */ 716 int 717 mlx5_flow_aso_ct_mng_init(struct mlx5_dev_ctx_shared *sh) 718 { 719 int err; 720 721 if (sh->ct_mng) 722 return 0; 723 sh->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->ct_mng), 724 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 725 if (!sh->ct_mng) { 726 DRV_LOG(ERR, "ASO CT management allocation failed."); 727 rte_errno = ENOMEM; 728 return -rte_errno; 729 } 730 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_CONNECTION_TRACKING); 731 if (err) { 732 mlx5_free(sh->ct_mng); 733 /* rte_errno should be extracted from the failure. */ 734 rte_errno = EINVAL; 735 return -rte_errno; 736 } 737 rte_spinlock_init(&sh->ct_mng->ct_sl); 738 rte_rwlock_init(&sh->ct_mng->resize_rwl); 739 LIST_INIT(&sh->ct_mng->free_cts); 740 return 0; 741 } 742 743 /* 744 * Close and release all the resources of the 745 * ASO connection tracking management structure. 746 * 747 * @param[in] sh 748 * Pointer to mlx5_dev_ctx_shared object to free. 749 */ 750 static void 751 mlx5_flow_aso_ct_mng_close(struct mlx5_dev_ctx_shared *sh) 752 { 753 struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng; 754 struct mlx5_aso_ct_pool *ct_pool; 755 struct mlx5_aso_ct_action *ct; 756 uint32_t idx; 757 uint32_t val; 758 uint32_t cnt; 759 int i; 760 761 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_CONNECTION_TRACKING); 762 idx = mng->next; 763 while (idx--) { 764 cnt = 0; 765 ct_pool = mng->pools[idx]; 766 for (i = 0; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) { 767 ct = &ct_pool->actions[i]; 768 val = __atomic_fetch_sub(&ct->refcnt, 1, 769 __ATOMIC_RELAXED); 770 MLX5_ASSERT(val == 1); 771 if (val > 1) 772 cnt++; 773 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT 774 if (ct->dr_action_orig) 775 claim_zero(mlx5_glue->destroy_flow_action 776 (ct->dr_action_orig)); 777 if (ct->dr_action_rply) 778 claim_zero(mlx5_glue->destroy_flow_action 779 (ct->dr_action_rply)); 780 #endif 781 } 782 claim_zero(mlx5_devx_cmd_destroy(ct_pool->devx_obj)); 783 if (cnt) { 784 DRV_LOG(DEBUG, "%u ASO CT objects are being used in the pool %u", 785 cnt, i); 786 } 787 mlx5_free(ct_pool); 788 /* in case of failure. */ 789 mng->next--; 790 } 791 mlx5_free(mng->pools); 792 mlx5_free(mng); 793 /* Management structure must be cleared to 0s during allocation. */ 794 sh->ct_mng = NULL; 795 } 796 797 /** 798 * Initialize the flow resources' indexed mempool. 799 * 800 * @param[in] sh 801 * Pointer to mlx5_dev_ctx_shared object. 802 * @param[in] config 803 * Pointer to user dev config. 804 */ 805 static void 806 mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh, 807 const struct mlx5_dev_config *config) 808 { 809 uint8_t i; 810 struct mlx5_indexed_pool_config cfg; 811 812 for (i = 0; i < MLX5_IPOOL_MAX; ++i) { 813 cfg = mlx5_ipool_cfg[i]; 814 switch (i) { 815 default: 816 break; 817 /* 818 * Set MLX5_IPOOL_MLX5_FLOW ipool size 819 * according to PCI function flow configuration. 820 */ 821 case MLX5_IPOOL_MLX5_FLOW: 822 cfg.size = config->dv_flow_en ? 823 sizeof(struct mlx5_flow_handle) : 824 MLX5_FLOW_HANDLE_VERBS_SIZE; 825 break; 826 } 827 if (config->reclaim_mode) { 828 cfg.release_mem_en = 1; 829 cfg.per_core_cache = 0; 830 } else { 831 cfg.release_mem_en = 0; 832 } 833 sh->ipool[i] = mlx5_ipool_create(&cfg); 834 } 835 } 836 837 838 /** 839 * Release the flow resources' indexed mempool. 840 * 841 * @param[in] sh 842 * Pointer to mlx5_dev_ctx_shared object. 843 */ 844 static void 845 mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh) 846 { 847 uint8_t i; 848 849 for (i = 0; i < MLX5_IPOOL_MAX; ++i) 850 mlx5_ipool_destroy(sh->ipool[i]); 851 for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i) 852 if (sh->mdh_ipools[i]) 853 mlx5_ipool_destroy(sh->mdh_ipools[i]); 854 } 855 856 /* 857 * Check if dynamic flex parser for eCPRI already exists. 858 * 859 * @param dev 860 * Pointer to Ethernet device structure. 861 * 862 * @return 863 * true on exists, false on not. 864 */ 865 bool 866 mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev) 867 { 868 struct mlx5_priv *priv = dev->data->dev_private; 869 struct mlx5_flex_parser_profiles *prf = 870 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 871 872 return !!prf->obj; 873 } 874 875 /* 876 * Allocation of a flex parser for eCPRI. Once created, this parser related 877 * resources will be held until the device is closed. 878 * 879 * @param dev 880 * Pointer to Ethernet device structure. 881 * 882 * @return 883 * 0 on success, a negative errno value otherwise and rte_errno is set. 884 */ 885 int 886 mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev) 887 { 888 struct mlx5_priv *priv = dev->data->dev_private; 889 struct mlx5_flex_parser_profiles *prf = 890 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 891 struct mlx5_devx_graph_node_attr node = { 892 .modify_field_select = 0, 893 }; 894 uint32_t ids[8]; 895 int ret; 896 897 if (!priv->config.hca_attr.parse_graph_flex_node) { 898 DRV_LOG(ERR, "Dynamic flex parser is not supported " 899 "for device %s.", priv->dev_data->name); 900 return -ENOTSUP; 901 } 902 node.header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED; 903 /* 8 bytes now: 4B common header + 4B message body header. */ 904 node.header_length_base_value = 0x8; 905 /* After MAC layer: Ether / VLAN. */ 906 node.in[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_MAC; 907 /* Type of compared condition should be 0xAEFE in the L2 layer. */ 908 node.in[0].compare_condition_value = RTE_ETHER_TYPE_ECPRI; 909 /* Sample #0: type in common header. */ 910 node.sample[0].flow_match_sample_en = 1; 911 /* Fixed offset. */ 912 node.sample[0].flow_match_sample_offset_mode = 0x0; 913 /* Only the 2nd byte will be used. */ 914 node.sample[0].flow_match_sample_field_base_offset = 0x0; 915 /* Sample #1: message payload. */ 916 node.sample[1].flow_match_sample_en = 1; 917 /* Fixed offset. */ 918 node.sample[1].flow_match_sample_offset_mode = 0x0; 919 /* 920 * Only the first two bytes will be used right now, and its offset will 921 * start after the common header that with the length of a DW(u32). 922 */ 923 node.sample[1].flow_match_sample_field_base_offset = sizeof(uint32_t); 924 prf->obj = mlx5_devx_cmd_create_flex_parser(priv->sh->cdev->ctx, &node); 925 if (!prf->obj) { 926 DRV_LOG(ERR, "Failed to create flex parser node object."); 927 return (rte_errno == 0) ? -ENODEV : -rte_errno; 928 } 929 prf->num = 2; 930 ret = mlx5_devx_cmd_query_parse_samples(prf->obj, ids, prf->num); 931 if (ret) { 932 DRV_LOG(ERR, "Failed to query sample IDs."); 933 return (rte_errno == 0) ? -ENODEV : -rte_errno; 934 } 935 prf->offset[0] = 0x0; 936 prf->offset[1] = sizeof(uint32_t); 937 prf->ids[0] = ids[0]; 938 prf->ids[1] = ids[1]; 939 return 0; 940 } 941 942 /* 943 * Destroy the flex parser node, including the parser itself, input / output 944 * arcs and DW samples. Resources could be reused then. 945 * 946 * @param dev 947 * Pointer to Ethernet device structure. 948 */ 949 static void 950 mlx5_flex_parser_ecpri_release(struct rte_eth_dev *dev) 951 { 952 struct mlx5_priv *priv = dev->data->dev_private; 953 struct mlx5_flex_parser_profiles *prf = 954 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 955 956 if (prf->obj) 957 mlx5_devx_cmd_destroy(prf->obj); 958 prf->obj = NULL; 959 } 960 961 uint32_t 962 mlx5_get_supported_sw_parsing_offloads(const struct mlx5_hca_attr *attr) 963 { 964 uint32_t sw_parsing_offloads = 0; 965 966 if (attr->swp) { 967 sw_parsing_offloads |= MLX5_SW_PARSING_CAP; 968 if (attr->swp_csum) 969 sw_parsing_offloads |= MLX5_SW_PARSING_CSUM_CAP; 970 971 if (attr->swp_lso) 972 sw_parsing_offloads |= MLX5_SW_PARSING_TSO_CAP; 973 } 974 return sw_parsing_offloads; 975 } 976 977 uint32_t 978 mlx5_get_supported_tunneling_offloads(const struct mlx5_hca_attr *attr) 979 { 980 uint32_t tn_offloads = 0; 981 982 if (attr->tunnel_stateless_vxlan) 983 tn_offloads |= MLX5_TUNNELED_OFFLOADS_VXLAN_CAP; 984 if (attr->tunnel_stateless_gre) 985 tn_offloads |= MLX5_TUNNELED_OFFLOADS_GRE_CAP; 986 if (attr->tunnel_stateless_geneve_rx) 987 tn_offloads |= MLX5_TUNNELED_OFFLOADS_GENEVE_CAP; 988 return tn_offloads; 989 } 990 991 /* 992 * Allocate Rx and Tx UARs in robust fashion. 993 * This routine handles the following UAR allocation issues: 994 * 995 * - tries to allocate the UAR with the most appropriate memory 996 * mapping type from the ones supported by the host 997 * 998 * - tries to allocate the UAR with non-NULL base address 999 * OFED 5.0.x and Upstream rdma_core before v29 returned the NULL as 1000 * UAR base address if UAR was not the first object in the UAR page. 1001 * It caused the PMD failure and we should try to get another UAR 1002 * till we get the first one with non-NULL base address returned. 1003 */ 1004 static int 1005 mlx5_alloc_rxtx_uars(struct mlx5_dev_ctx_shared *sh, 1006 const struct mlx5_common_dev_config *config) 1007 { 1008 uint32_t uar_mapping, retry; 1009 int err = 0; 1010 void *base_addr; 1011 1012 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 1013 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 1014 /* Control the mapping type according to the settings. */ 1015 uar_mapping = (config->dbnc == MLX5_TXDB_NCACHED) ? 1016 MLX5DV_UAR_ALLOC_TYPE_NC : 1017 MLX5DV_UAR_ALLOC_TYPE_BF; 1018 #else 1019 RTE_SET_USED(config); 1020 /* 1021 * It seems we have no way to control the memory mapping type 1022 * for the UAR, the default "Write-Combining" type is supposed. 1023 * The UAR initialization on queue creation queries the 1024 * actual mapping type done by Verbs/kernel and setups the 1025 * PMD datapath accordingly. 1026 */ 1027 uar_mapping = 0; 1028 #endif 1029 sh->tx_uar = mlx5_glue->devx_alloc_uar(sh->cdev->ctx, 1030 uar_mapping); 1031 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 1032 if (!sh->tx_uar && 1033 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 1034 if (config->dbnc == MLX5_TXDB_CACHED || 1035 config->dbnc == MLX5_TXDB_HEURISTIC) 1036 DRV_LOG(WARNING, "Devarg tx_db_nc setting " 1037 "is not supported by DevX"); 1038 /* 1039 * In some environments like virtual machine 1040 * the Write Combining mapped might be not supported 1041 * and UAR allocation fails. We try "Non-Cached" 1042 * mapping for the case. The tx_burst routines take 1043 * the UAR mapping type into account on UAR setup 1044 * on queue creation. 1045 */ 1046 DRV_LOG(DEBUG, "Failed to allocate Tx DevX UAR (BF)"); 1047 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 1048 sh->tx_uar = mlx5_glue->devx_alloc_uar(sh->cdev->ctx, 1049 uar_mapping); 1050 } else if (!sh->tx_uar && 1051 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) { 1052 if (config->dbnc == MLX5_TXDB_NCACHED) 1053 DRV_LOG(WARNING, "Devarg tx_db_nc settings " 1054 "is not supported by DevX"); 1055 /* 1056 * If Verbs/kernel does not support "Non-Cached" 1057 * try the "Write-Combining". 1058 */ 1059 DRV_LOG(DEBUG, "Failed to allocate Tx DevX UAR (NC)"); 1060 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF; 1061 sh->tx_uar = mlx5_glue->devx_alloc_uar(sh->cdev->ctx, 1062 uar_mapping); 1063 } 1064 #endif 1065 if (!sh->tx_uar) { 1066 DRV_LOG(ERR, "Failed to allocate Tx DevX UAR (BF/NC)"); 1067 err = ENOMEM; 1068 goto exit; 1069 } 1070 base_addr = mlx5_os_get_devx_uar_base_addr(sh->tx_uar); 1071 if (base_addr) 1072 break; 1073 /* 1074 * The UARs are allocated by rdma_core within the 1075 * IB device context, on context closure all UARs 1076 * will be freed, should be no memory/object leakage. 1077 */ 1078 DRV_LOG(DEBUG, "Retrying to allocate Tx DevX UAR"); 1079 sh->tx_uar = NULL; 1080 } 1081 /* Check whether we finally succeeded with valid UAR allocation. */ 1082 if (!sh->tx_uar) { 1083 DRV_LOG(ERR, "Failed to allocate Tx DevX UAR (NULL base)"); 1084 err = ENOMEM; 1085 goto exit; 1086 } 1087 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 1088 uar_mapping = 0; 1089 sh->devx_rx_uar = mlx5_glue->devx_alloc_uar(sh->cdev->ctx, 1090 uar_mapping); 1091 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 1092 if (!sh->devx_rx_uar && 1093 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 1094 /* 1095 * Rx UAR is used to control interrupts only, 1096 * should be no datapath noticeable impact, 1097 * can try "Non-Cached" mapping safely. 1098 */ 1099 DRV_LOG(DEBUG, "Failed to allocate Rx DevX UAR (BF)"); 1100 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 1101 sh->devx_rx_uar = mlx5_glue->devx_alloc_uar 1102 (sh->cdev->ctx, uar_mapping); 1103 } 1104 #endif 1105 if (!sh->devx_rx_uar) { 1106 DRV_LOG(ERR, "Failed to allocate Rx DevX UAR (BF/NC)"); 1107 err = ENOMEM; 1108 goto exit; 1109 } 1110 base_addr = mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar); 1111 if (base_addr) 1112 break; 1113 /* 1114 * The UARs are allocated by rdma_core within the 1115 * IB device context, on context closure all UARs 1116 * will be freed, should be no memory/object leakage. 1117 */ 1118 DRV_LOG(DEBUG, "Retrying to allocate Rx DevX UAR"); 1119 sh->devx_rx_uar = NULL; 1120 } 1121 /* Check whether we finally succeeded with valid UAR allocation. */ 1122 if (!sh->devx_rx_uar) { 1123 DRV_LOG(ERR, "Failed to allocate Rx DevX UAR (NULL base)"); 1124 err = ENOMEM; 1125 } 1126 exit: 1127 return err; 1128 } 1129 1130 /** 1131 * rte_mempool_walk() callback to unregister Rx mempools. 1132 * It used when implicit mempool registration is disabled. 1133 * 1134 * @param mp 1135 * The mempool being walked. 1136 * @param arg 1137 * Pointer to the device shared context. 1138 */ 1139 static void 1140 mlx5_dev_ctx_shared_rx_mempool_unregister_cb(struct rte_mempool *mp, void *arg) 1141 { 1142 struct mlx5_dev_ctx_shared *sh = arg; 1143 1144 mlx5_dev_mempool_unregister(sh->cdev, mp); 1145 } 1146 1147 /** 1148 * Callback used when implicit mempool registration is disabled 1149 * in order to track Rx mempool destruction. 1150 * 1151 * @param event 1152 * Mempool life cycle event. 1153 * @param mp 1154 * An Rx mempool registered explicitly when the port is started. 1155 * @param arg 1156 * Pointer to a device shared context. 1157 */ 1158 static void 1159 mlx5_dev_ctx_shared_rx_mempool_event_cb(enum rte_mempool_event event, 1160 struct rte_mempool *mp, void *arg) 1161 { 1162 struct mlx5_dev_ctx_shared *sh = arg; 1163 1164 if (event == RTE_MEMPOOL_EVENT_DESTROY) 1165 mlx5_dev_mempool_unregister(sh->cdev, mp); 1166 } 1167 1168 int 1169 mlx5_dev_ctx_shared_mempool_subscribe(struct rte_eth_dev *dev) 1170 { 1171 struct mlx5_priv *priv = dev->data->dev_private; 1172 struct mlx5_dev_ctx_shared *sh = priv->sh; 1173 int ret; 1174 1175 /* Check if we only need to track Rx mempool destruction. */ 1176 if (!sh->cdev->config.mr_mempool_reg_en) { 1177 ret = rte_mempool_event_callback_register 1178 (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); 1179 return ret == 0 || rte_errno == EEXIST ? 0 : ret; 1180 } 1181 return mlx5_dev_mempool_subscribe(sh->cdev); 1182 } 1183 1184 /** 1185 * Set up multiple TISs with different affinities according to 1186 * number of bonding ports 1187 * 1188 * @param priv 1189 * Pointer of shared context. 1190 * 1191 * @return 1192 * Zero on success, -1 otherwise. 1193 */ 1194 static int 1195 mlx5_setup_tis(struct mlx5_dev_ctx_shared *sh) 1196 { 1197 int i; 1198 struct mlx5_devx_lag_context lag_ctx = { 0 }; 1199 struct mlx5_devx_tis_attr tis_attr = { 0 }; 1200 1201 tis_attr.transport_domain = sh->td->id; 1202 if (sh->bond.n_port) { 1203 if (!mlx5_devx_cmd_query_lag(sh->cdev->ctx, &lag_ctx)) { 1204 sh->lag.tx_remap_affinity[0] = 1205 lag_ctx.tx_remap_affinity_1; 1206 sh->lag.tx_remap_affinity[1] = 1207 lag_ctx.tx_remap_affinity_2; 1208 sh->lag.affinity_mode = lag_ctx.port_select_mode; 1209 } else { 1210 DRV_LOG(ERR, "Failed to query lag affinity."); 1211 return -1; 1212 } 1213 if (sh->lag.affinity_mode == MLX5_LAG_MODE_TIS) { 1214 for (i = 0; i < sh->bond.n_port; i++) { 1215 tis_attr.lag_tx_port_affinity = 1216 MLX5_IFC_LAG_MAP_TIS_AFFINITY(i, 1217 sh->bond.n_port); 1218 sh->tis[i] = mlx5_devx_cmd_create_tis(sh->cdev->ctx, 1219 &tis_attr); 1220 if (!sh->tis[i]) { 1221 DRV_LOG(ERR, "Failed to TIS %d/%d for bonding device" 1222 " %s.", i, sh->bond.n_port, 1223 sh->ibdev_name); 1224 return -1; 1225 } 1226 } 1227 DRV_LOG(DEBUG, "LAG number of ports : %d, affinity_1 & 2 : pf%d & %d.\n", 1228 sh->bond.n_port, lag_ctx.tx_remap_affinity_1, 1229 lag_ctx.tx_remap_affinity_2); 1230 return 0; 1231 } 1232 if (sh->lag.affinity_mode == MLX5_LAG_MODE_HASH) 1233 DRV_LOG(INFO, "Device %s enabled HW hash based LAG.", 1234 sh->ibdev_name); 1235 } 1236 tis_attr.lag_tx_port_affinity = 0; 1237 sh->tis[0] = mlx5_devx_cmd_create_tis(sh->cdev->ctx, &tis_attr); 1238 if (!sh->tis[0]) { 1239 DRV_LOG(ERR, "Failed to TIS 0 for bonding device" 1240 " %s.", sh->ibdev_name); 1241 return -1; 1242 } 1243 return 0; 1244 } 1245 1246 /** 1247 * Allocate shared device context. If there is multiport device the 1248 * master and representors will share this context, if there is single 1249 * port dedicated device, the context will be used by only given 1250 * port due to unification. 1251 * 1252 * Routine first searches the context for the specified device name, 1253 * if found the shared context assumed and reference counter is incremented. 1254 * If no context found the new one is created and initialized with specified 1255 * device context and parameters. 1256 * 1257 * @param[in] spawn 1258 * Pointer to the device attributes (name, port, etc). 1259 * @param[in] config 1260 * Pointer to device configuration structure. 1261 * 1262 * @return 1263 * Pointer to mlx5_dev_ctx_shared object on success, 1264 * otherwise NULL and rte_errno is set. 1265 */ 1266 struct mlx5_dev_ctx_shared * 1267 mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, 1268 const struct mlx5_dev_config *config) 1269 { 1270 struct mlx5_dev_ctx_shared *sh; 1271 int err = 0; 1272 uint32_t i; 1273 1274 MLX5_ASSERT(spawn); 1275 /* Secondary process should not create the shared context. */ 1276 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1277 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1278 /* Search for IB context by device name. */ 1279 LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) { 1280 if (!strcmp(sh->ibdev_name, spawn->phys_dev_name)) { 1281 sh->refcnt++; 1282 goto exit; 1283 } 1284 } 1285 /* No device found, we have to create new shared context. */ 1286 MLX5_ASSERT(spawn->max_port); 1287 sh = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 1288 sizeof(struct mlx5_dev_ctx_shared) + 1289 spawn->max_port * 1290 sizeof(struct mlx5_dev_shared_port), 1291 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1292 if (!sh) { 1293 DRV_LOG(ERR, "shared context allocation failure"); 1294 rte_errno = ENOMEM; 1295 goto exit; 1296 } 1297 pthread_mutex_init(&sh->txpp.mutex, NULL); 1298 sh->numa_node = spawn->cdev->dev->numa_node; 1299 sh->cdev = spawn->cdev; 1300 sh->devx = sh->cdev->config.devx; 1301 if (spawn->bond_info) 1302 sh->bond = *spawn->bond_info; 1303 err = mlx5_os_get_dev_attr(sh->cdev, &sh->device_attr); 1304 if (err) { 1305 DRV_LOG(DEBUG, "mlx5_os_get_dev_attr() failed"); 1306 goto error; 1307 } 1308 sh->refcnt = 1; 1309 sh->max_port = spawn->max_port; 1310 sh->reclaim_mode = config->reclaim_mode; 1311 strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->cdev->ctx), 1312 sizeof(sh->ibdev_name) - 1); 1313 strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->cdev->ctx), 1314 sizeof(sh->ibdev_path) - 1); 1315 /* 1316 * Setting port_id to max unallowed value means 1317 * there is no interrupt subhandler installed for 1318 * the given port index i. 1319 */ 1320 for (i = 0; i < sh->max_port; i++) { 1321 sh->port[i].ih_port_id = RTE_MAX_ETHPORTS; 1322 sh->port[i].devx_ih_port_id = RTE_MAX_ETHPORTS; 1323 } 1324 if (sh->devx) { 1325 sh->td = mlx5_devx_cmd_create_td(sh->cdev->ctx); 1326 if (!sh->td) { 1327 DRV_LOG(ERR, "TD allocation failure"); 1328 err = ENOMEM; 1329 goto error; 1330 } 1331 if (mlx5_setup_tis(sh)) { 1332 DRV_LOG(ERR, "TIS allocation failure"); 1333 err = ENOMEM; 1334 goto error; 1335 } 1336 err = mlx5_alloc_rxtx_uars(sh, &sh->cdev->config); 1337 if (err) 1338 goto error; 1339 MLX5_ASSERT(sh->tx_uar); 1340 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->tx_uar)); 1341 1342 MLX5_ASSERT(sh->devx_rx_uar); 1343 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar)); 1344 } 1345 #ifndef RTE_ARCH_64 1346 /* Initialize UAR access locks for 32bit implementations. */ 1347 rte_spinlock_init(&sh->uar_lock_cq); 1348 for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) 1349 rte_spinlock_init(&sh->uar_lock[i]); 1350 #endif 1351 mlx5_os_dev_shared_handler_install(sh); 1352 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1353 err = mlx5_flow_os_init_workspace_once(); 1354 if (err) 1355 goto error; 1356 } 1357 mlx5_flow_aging_init(sh); 1358 mlx5_flow_counters_mng_init(sh); 1359 mlx5_flow_ipool_create(sh, config); 1360 /* Add context to the global device list. */ 1361 LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next); 1362 rte_spinlock_init(&sh->geneve_tlv_opt_sl); 1363 exit: 1364 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1365 return sh; 1366 error: 1367 pthread_mutex_destroy(&sh->txpp.mutex); 1368 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1369 MLX5_ASSERT(sh); 1370 if (sh->td) 1371 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1372 i = 0; 1373 do { 1374 if (sh->tis[i]) 1375 claim_zero(mlx5_devx_cmd_destroy(sh->tis[i])); 1376 } while (++i < (uint32_t)sh->bond.n_port); 1377 if (sh->devx_rx_uar) 1378 mlx5_glue->devx_free_uar(sh->devx_rx_uar); 1379 if (sh->tx_uar) 1380 mlx5_glue->devx_free_uar(sh->tx_uar); 1381 mlx5_free(sh); 1382 MLX5_ASSERT(err > 0); 1383 rte_errno = err; 1384 return NULL; 1385 } 1386 1387 /** 1388 * Free shared IB device context. Decrement counter and if zero free 1389 * all allocated resources and close handles. 1390 * 1391 * @param[in] sh 1392 * Pointer to mlx5_dev_ctx_shared object to free 1393 */ 1394 void 1395 mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) 1396 { 1397 int ret; 1398 int i = 0; 1399 1400 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1401 #ifdef RTE_LIBRTE_MLX5_DEBUG 1402 /* Check the object presence in the list. */ 1403 struct mlx5_dev_ctx_shared *lctx; 1404 1405 LIST_FOREACH(lctx, &mlx5_dev_ctx_list, next) 1406 if (lctx == sh) 1407 break; 1408 MLX5_ASSERT(lctx); 1409 if (lctx != sh) { 1410 DRV_LOG(ERR, "Freeing non-existing shared IB context"); 1411 goto exit; 1412 } 1413 #endif 1414 MLX5_ASSERT(sh); 1415 MLX5_ASSERT(sh->refcnt); 1416 /* Secondary process should not free the shared context. */ 1417 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1418 if (--sh->refcnt) 1419 goto exit; 1420 /* Stop watching for mempool events and unregister all mempools. */ 1421 if (!sh->cdev->config.mr_mempool_reg_en) { 1422 ret = rte_mempool_event_callback_unregister 1423 (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); 1424 if (ret == 0) 1425 rte_mempool_walk 1426 (mlx5_dev_ctx_shared_rx_mempool_unregister_cb, sh); 1427 } 1428 /* Remove context from the global device list. */ 1429 LIST_REMOVE(sh, next); 1430 /* Release resources on the last device removal. */ 1431 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1432 mlx5_os_net_cleanup(); 1433 mlx5_flow_os_release_workspace(); 1434 } 1435 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1436 /* 1437 * Ensure there is no async event handler installed. 1438 * Only primary process handles async device events. 1439 **/ 1440 mlx5_flow_counters_mng_close(sh); 1441 if (sh->aso_age_mng) { 1442 mlx5_flow_aso_age_mng_close(sh); 1443 sh->aso_age_mng = NULL; 1444 } 1445 if (sh->mtrmng) 1446 mlx5_aso_flow_mtrs_mng_close(sh); 1447 mlx5_flow_ipool_destroy(sh); 1448 mlx5_os_dev_shared_handler_uninstall(sh); 1449 if (sh->tx_uar) { 1450 mlx5_glue->devx_free_uar(sh->tx_uar); 1451 sh->tx_uar = NULL; 1452 } 1453 do { 1454 if (sh->tis[i]) 1455 claim_zero(mlx5_devx_cmd_destroy(sh->tis[i])); 1456 } while (++i < sh->bond.n_port); 1457 if (sh->td) 1458 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1459 if (sh->devx_rx_uar) 1460 mlx5_glue->devx_free_uar(sh->devx_rx_uar); 1461 MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL); 1462 pthread_mutex_destroy(&sh->txpp.mutex); 1463 mlx5_free(sh); 1464 return; 1465 exit: 1466 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1467 } 1468 1469 /** 1470 * Destroy table hash list. 1471 * 1472 * @param[in] priv 1473 * Pointer to the private device data structure. 1474 */ 1475 void 1476 mlx5_free_table_hash_list(struct mlx5_priv *priv) 1477 { 1478 struct mlx5_dev_ctx_shared *sh = priv->sh; 1479 1480 if (!sh->flow_tbls) 1481 return; 1482 mlx5_hlist_destroy(sh->flow_tbls); 1483 sh->flow_tbls = NULL; 1484 } 1485 1486 /** 1487 * Initialize flow table hash list and create the root tables entry 1488 * for each domain. 1489 * 1490 * @param[in] priv 1491 * Pointer to the private device data structure. 1492 * 1493 * @return 1494 * Zero on success, positive error code otherwise. 1495 */ 1496 int 1497 mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) 1498 { 1499 int err = 0; 1500 /* Tables are only used in DV and DR modes. */ 1501 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 1502 struct mlx5_dev_ctx_shared *sh = priv->sh; 1503 char s[MLX5_NAME_SIZE]; 1504 1505 MLX5_ASSERT(sh); 1506 snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); 1507 sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, 1508 false, true, sh, 1509 flow_dv_tbl_create_cb, 1510 flow_dv_tbl_match_cb, 1511 flow_dv_tbl_remove_cb, 1512 flow_dv_tbl_clone_cb, 1513 flow_dv_tbl_clone_free_cb); 1514 if (!sh->flow_tbls) { 1515 DRV_LOG(ERR, "flow tables with hash creation failed."); 1516 err = ENOMEM; 1517 return err; 1518 } 1519 #ifndef HAVE_MLX5DV_DR 1520 struct rte_flow_error error; 1521 struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; 1522 1523 /* 1524 * In case we have not DR support, the zero tables should be created 1525 * because DV expect to see them even if they cannot be created by 1526 * RDMA-CORE. 1527 */ 1528 if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, 1529 NULL, 0, 1, 0, &error) || 1530 !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, 1531 NULL, 0, 1, 0, &error) || 1532 !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, 1533 NULL, 0, 1, 0, &error)) { 1534 err = ENOMEM; 1535 goto error; 1536 } 1537 return err; 1538 error: 1539 mlx5_free_table_hash_list(priv); 1540 #endif /* HAVE_MLX5DV_DR */ 1541 #endif 1542 return err; 1543 } 1544 1545 /** 1546 * Retrieve integer value from environment variable. 1547 * 1548 * @param[in] name 1549 * Environment variable name. 1550 * 1551 * @return 1552 * Integer value, 0 if the variable is not set. 1553 */ 1554 int 1555 mlx5_getenv_int(const char *name) 1556 { 1557 const char *val = getenv(name); 1558 1559 if (val == NULL) 1560 return 0; 1561 return atoi(val); 1562 } 1563 1564 /** 1565 * DPDK callback to add udp tunnel port 1566 * 1567 * @param[in] dev 1568 * A pointer to eth_dev 1569 * @param[in] udp_tunnel 1570 * A pointer to udp tunnel 1571 * 1572 * @return 1573 * 0 on valid udp ports and tunnels, -ENOTSUP otherwise. 1574 */ 1575 int 1576 mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev __rte_unused, 1577 struct rte_eth_udp_tunnel *udp_tunnel) 1578 { 1579 MLX5_ASSERT(udp_tunnel != NULL); 1580 if (udp_tunnel->prot_type == RTE_ETH_TUNNEL_TYPE_VXLAN && 1581 udp_tunnel->udp_port == 4789) 1582 return 0; 1583 if (udp_tunnel->prot_type == RTE_ETH_TUNNEL_TYPE_VXLAN_GPE && 1584 udp_tunnel->udp_port == 4790) 1585 return 0; 1586 return -ENOTSUP; 1587 } 1588 1589 /** 1590 * Initialize process private data structure. 1591 * 1592 * @param dev 1593 * Pointer to Ethernet device structure. 1594 * 1595 * @return 1596 * 0 on success, a negative errno value otherwise and rte_errno is set. 1597 */ 1598 int 1599 mlx5_proc_priv_init(struct rte_eth_dev *dev) 1600 { 1601 struct mlx5_priv *priv = dev->data->dev_private; 1602 struct mlx5_proc_priv *ppriv; 1603 size_t ppriv_size; 1604 1605 mlx5_proc_priv_uninit(dev); 1606 /* 1607 * UAR register table follows the process private structure. BlueFlame 1608 * registers for Tx queues are stored in the table. 1609 */ 1610 ppriv_size = 1611 sizeof(struct mlx5_proc_priv) + priv->txqs_n * sizeof(void *); 1612 ppriv = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, ppriv_size, 1613 RTE_CACHE_LINE_SIZE, dev->device->numa_node); 1614 if (!ppriv) { 1615 rte_errno = ENOMEM; 1616 return -rte_errno; 1617 } 1618 ppriv->uar_table_sz = priv->txqs_n; 1619 dev->process_private = ppriv; 1620 return 0; 1621 } 1622 1623 /** 1624 * Un-initialize process private data structure. 1625 * 1626 * @param dev 1627 * Pointer to Ethernet device structure. 1628 */ 1629 void 1630 mlx5_proc_priv_uninit(struct rte_eth_dev *dev) 1631 { 1632 if (!dev->process_private) 1633 return; 1634 mlx5_free(dev->process_private); 1635 dev->process_private = NULL; 1636 } 1637 1638 /** 1639 * DPDK callback to close the device. 1640 * 1641 * Destroy all queues and objects, free memory. 1642 * 1643 * @param dev 1644 * Pointer to Ethernet device structure. 1645 */ 1646 int 1647 mlx5_dev_close(struct rte_eth_dev *dev) 1648 { 1649 struct mlx5_priv *priv = dev->data->dev_private; 1650 unsigned int i; 1651 int ret; 1652 1653 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1654 /* Check if process_private released. */ 1655 if (!dev->process_private) 1656 return 0; 1657 mlx5_tx_uar_uninit_secondary(dev); 1658 mlx5_proc_priv_uninit(dev); 1659 rte_eth_dev_release_port(dev); 1660 return 0; 1661 } 1662 if (!priv->sh) 1663 return 0; 1664 DRV_LOG(DEBUG, "port %u closing device \"%s\"", 1665 dev->data->port_id, 1666 ((priv->sh->cdev->ctx != NULL) ? 1667 mlx5_os_get_ctx_device_name(priv->sh->cdev->ctx) : "")); 1668 /* 1669 * If default mreg copy action is removed at the stop stage, 1670 * the search will return none and nothing will be done anymore. 1671 */ 1672 mlx5_flow_stop_default(dev); 1673 mlx5_traffic_disable(dev); 1674 /* 1675 * If all the flows are already flushed in the device stop stage, 1676 * then this will return directly without any action. 1677 */ 1678 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true); 1679 mlx5_action_handle_flush(dev); 1680 mlx5_flow_meter_flush(dev, NULL); 1681 /* Prevent crashes when queues are still in use. */ 1682 dev->rx_pkt_burst = removed_rx_burst; 1683 dev->tx_pkt_burst = removed_tx_burst; 1684 rte_wmb(); 1685 /* Disable datapath on secondary process. */ 1686 mlx5_mp_os_req_stop_rxtx(dev); 1687 /* Free the eCPRI flex parser resource. */ 1688 mlx5_flex_parser_ecpri_release(dev); 1689 if (priv->rxqs != NULL) { 1690 /* XXX race condition if mlx5_rx_burst() is still running. */ 1691 rte_delay_us_sleep(1000); 1692 for (i = 0; (i != priv->rxqs_n); ++i) 1693 mlx5_rxq_release(dev, i); 1694 priv->rxqs_n = 0; 1695 priv->rxqs = NULL; 1696 } 1697 if (priv->representor) { 1698 /* Each representor has a dedicated interrupts handler */ 1699 mlx5_free(dev->intr_handle); 1700 dev->intr_handle = NULL; 1701 } 1702 if (priv->txqs != NULL) { 1703 /* XXX race condition if mlx5_tx_burst() is still running. */ 1704 rte_delay_us_sleep(1000); 1705 for (i = 0; (i != priv->txqs_n); ++i) 1706 mlx5_txq_release(dev, i); 1707 priv->txqs_n = 0; 1708 priv->txqs = NULL; 1709 } 1710 mlx5_proc_priv_uninit(dev); 1711 if (priv->q_counters) { 1712 mlx5_devx_cmd_destroy(priv->q_counters); 1713 priv->q_counters = NULL; 1714 } 1715 if (priv->drop_queue.hrxq) 1716 mlx5_drop_action_destroy(dev); 1717 if (priv->mreg_cp_tbl) 1718 mlx5_hlist_destroy(priv->mreg_cp_tbl); 1719 mlx5_mprq_free_mp(dev); 1720 if (priv->sh->ct_mng) 1721 mlx5_flow_aso_ct_mng_close(priv->sh); 1722 mlx5_os_free_shared_dr(priv); 1723 if (priv->rss_conf.rss_key != NULL) 1724 mlx5_free(priv->rss_conf.rss_key); 1725 if (priv->reta_idx != NULL) 1726 mlx5_free(priv->reta_idx); 1727 if (priv->config.vf) 1728 mlx5_os_mac_addr_flush(dev); 1729 if (priv->nl_socket_route >= 0) 1730 close(priv->nl_socket_route); 1731 if (priv->nl_socket_rdma >= 0) 1732 close(priv->nl_socket_rdma); 1733 if (priv->vmwa_context) 1734 mlx5_vlan_vmwa_exit(priv->vmwa_context); 1735 ret = mlx5_hrxq_verify(dev); 1736 if (ret) 1737 DRV_LOG(WARNING, "port %u some hash Rx queue still remain", 1738 dev->data->port_id); 1739 ret = mlx5_ind_table_obj_verify(dev); 1740 if (ret) 1741 DRV_LOG(WARNING, "port %u some indirection table still remain", 1742 dev->data->port_id); 1743 ret = mlx5_rxq_obj_verify(dev); 1744 if (ret) 1745 DRV_LOG(WARNING, "port %u some Rx queue objects still remain", 1746 dev->data->port_id); 1747 ret = mlx5_rxq_verify(dev); 1748 if (ret) 1749 DRV_LOG(WARNING, "port %u some Rx queues still remain", 1750 dev->data->port_id); 1751 ret = mlx5_txq_obj_verify(dev); 1752 if (ret) 1753 DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain", 1754 dev->data->port_id); 1755 ret = mlx5_txq_verify(dev); 1756 if (ret) 1757 DRV_LOG(WARNING, "port %u some Tx queues still remain", 1758 dev->data->port_id); 1759 ret = mlx5_flow_verify(dev); 1760 if (ret) 1761 DRV_LOG(WARNING, "port %u some flows still remain", 1762 dev->data->port_id); 1763 if (priv->hrxqs) 1764 mlx5_list_destroy(priv->hrxqs); 1765 /* 1766 * Free the shared context in last turn, because the cleanup 1767 * routines above may use some shared fields, like 1768 * mlx5_os_mac_addr_flush() uses ibdev_path for retrieveing 1769 * ifindex if Netlink fails. 1770 */ 1771 mlx5_free_shared_dev_ctx(priv->sh); 1772 if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 1773 unsigned int c = 0; 1774 uint16_t port_id; 1775 1776 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1777 struct mlx5_priv *opriv = 1778 rte_eth_devices[port_id].data->dev_private; 1779 1780 if (!opriv || 1781 opriv->domain_id != priv->domain_id || 1782 &rte_eth_devices[port_id] == dev) 1783 continue; 1784 ++c; 1785 break; 1786 } 1787 if (!c) 1788 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 1789 } 1790 memset(priv, 0, sizeof(*priv)); 1791 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 1792 /* 1793 * Reset mac_addrs to NULL such that it is not freed as part of 1794 * rte_eth_dev_release_port(). mac_addrs is part of dev_private so 1795 * it is freed when dev_private is freed. 1796 */ 1797 dev->data->mac_addrs = NULL; 1798 return 0; 1799 } 1800 1801 const struct eth_dev_ops mlx5_dev_ops = { 1802 .dev_configure = mlx5_dev_configure, 1803 .dev_start = mlx5_dev_start, 1804 .dev_stop = mlx5_dev_stop, 1805 .dev_set_link_down = mlx5_set_link_down, 1806 .dev_set_link_up = mlx5_set_link_up, 1807 .dev_close = mlx5_dev_close, 1808 .promiscuous_enable = mlx5_promiscuous_enable, 1809 .promiscuous_disable = mlx5_promiscuous_disable, 1810 .allmulticast_enable = mlx5_allmulticast_enable, 1811 .allmulticast_disable = mlx5_allmulticast_disable, 1812 .link_update = mlx5_link_update, 1813 .stats_get = mlx5_stats_get, 1814 .stats_reset = mlx5_stats_reset, 1815 .xstats_get = mlx5_xstats_get, 1816 .xstats_reset = mlx5_xstats_reset, 1817 .xstats_get_names = mlx5_xstats_get_names, 1818 .fw_version_get = mlx5_fw_version_get, 1819 .dev_infos_get = mlx5_dev_infos_get, 1820 .representor_info_get = mlx5_representor_info_get, 1821 .read_clock = mlx5_txpp_read_clock, 1822 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1823 .vlan_filter_set = mlx5_vlan_filter_set, 1824 .rx_queue_setup = mlx5_rx_queue_setup, 1825 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1826 .tx_queue_setup = mlx5_tx_queue_setup, 1827 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1828 .rx_queue_release = mlx5_rx_queue_release, 1829 .tx_queue_release = mlx5_tx_queue_release, 1830 .rx_queue_start = mlx5_rx_queue_start, 1831 .rx_queue_stop = mlx5_rx_queue_stop, 1832 .tx_queue_start = mlx5_tx_queue_start, 1833 .tx_queue_stop = mlx5_tx_queue_stop, 1834 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1835 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1836 .mac_addr_remove = mlx5_mac_addr_remove, 1837 .mac_addr_add = mlx5_mac_addr_add, 1838 .mac_addr_set = mlx5_mac_addr_set, 1839 .set_mc_addr_list = mlx5_set_mc_addr_list, 1840 .mtu_set = mlx5_dev_set_mtu, 1841 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1842 .vlan_offload_set = mlx5_vlan_offload_set, 1843 .reta_update = mlx5_dev_rss_reta_update, 1844 .reta_query = mlx5_dev_rss_reta_query, 1845 .rss_hash_update = mlx5_rss_hash_update, 1846 .rss_hash_conf_get = mlx5_rss_hash_conf_get, 1847 .flow_ops_get = mlx5_flow_ops_get, 1848 .rxq_info_get = mlx5_rxq_info_get, 1849 .txq_info_get = mlx5_txq_info_get, 1850 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1851 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1852 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1853 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1854 .is_removed = mlx5_is_removed, 1855 .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, 1856 .get_module_info = mlx5_get_module_info, 1857 .get_module_eeprom = mlx5_get_module_eeprom, 1858 .hairpin_cap_get = mlx5_hairpin_cap_get, 1859 .mtr_ops_get = mlx5_flow_meter_ops_get, 1860 .hairpin_bind = mlx5_hairpin_bind, 1861 .hairpin_unbind = mlx5_hairpin_unbind, 1862 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1863 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1864 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1865 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1866 .get_monitor_addr = mlx5_get_monitor_addr, 1867 }; 1868 1869 /* Available operations from secondary process. */ 1870 const struct eth_dev_ops mlx5_dev_sec_ops = { 1871 .stats_get = mlx5_stats_get, 1872 .stats_reset = mlx5_stats_reset, 1873 .xstats_get = mlx5_xstats_get, 1874 .xstats_reset = mlx5_xstats_reset, 1875 .xstats_get_names = mlx5_xstats_get_names, 1876 .fw_version_get = mlx5_fw_version_get, 1877 .dev_infos_get = mlx5_dev_infos_get, 1878 .representor_info_get = mlx5_representor_info_get, 1879 .read_clock = mlx5_txpp_read_clock, 1880 .rx_queue_start = mlx5_rx_queue_start, 1881 .rx_queue_stop = mlx5_rx_queue_stop, 1882 .tx_queue_start = mlx5_tx_queue_start, 1883 .tx_queue_stop = mlx5_tx_queue_stop, 1884 .rxq_info_get = mlx5_rxq_info_get, 1885 .txq_info_get = mlx5_txq_info_get, 1886 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1887 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1888 .get_module_info = mlx5_get_module_info, 1889 .get_module_eeprom = mlx5_get_module_eeprom, 1890 }; 1891 1892 /* Available operations in flow isolated mode. */ 1893 const struct eth_dev_ops mlx5_dev_ops_isolate = { 1894 .dev_configure = mlx5_dev_configure, 1895 .dev_start = mlx5_dev_start, 1896 .dev_stop = mlx5_dev_stop, 1897 .dev_set_link_down = mlx5_set_link_down, 1898 .dev_set_link_up = mlx5_set_link_up, 1899 .dev_close = mlx5_dev_close, 1900 .promiscuous_enable = mlx5_promiscuous_enable, 1901 .promiscuous_disable = mlx5_promiscuous_disable, 1902 .allmulticast_enable = mlx5_allmulticast_enable, 1903 .allmulticast_disable = mlx5_allmulticast_disable, 1904 .link_update = mlx5_link_update, 1905 .stats_get = mlx5_stats_get, 1906 .stats_reset = mlx5_stats_reset, 1907 .xstats_get = mlx5_xstats_get, 1908 .xstats_reset = mlx5_xstats_reset, 1909 .xstats_get_names = mlx5_xstats_get_names, 1910 .fw_version_get = mlx5_fw_version_get, 1911 .dev_infos_get = mlx5_dev_infos_get, 1912 .representor_info_get = mlx5_representor_info_get, 1913 .read_clock = mlx5_txpp_read_clock, 1914 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1915 .vlan_filter_set = mlx5_vlan_filter_set, 1916 .rx_queue_setup = mlx5_rx_queue_setup, 1917 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1918 .tx_queue_setup = mlx5_tx_queue_setup, 1919 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1920 .rx_queue_release = mlx5_rx_queue_release, 1921 .tx_queue_release = mlx5_tx_queue_release, 1922 .rx_queue_start = mlx5_rx_queue_start, 1923 .rx_queue_stop = mlx5_rx_queue_stop, 1924 .tx_queue_start = mlx5_tx_queue_start, 1925 .tx_queue_stop = mlx5_tx_queue_stop, 1926 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1927 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1928 .mac_addr_remove = mlx5_mac_addr_remove, 1929 .mac_addr_add = mlx5_mac_addr_add, 1930 .mac_addr_set = mlx5_mac_addr_set, 1931 .set_mc_addr_list = mlx5_set_mc_addr_list, 1932 .mtu_set = mlx5_dev_set_mtu, 1933 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1934 .vlan_offload_set = mlx5_vlan_offload_set, 1935 .flow_ops_get = mlx5_flow_ops_get, 1936 .rxq_info_get = mlx5_rxq_info_get, 1937 .txq_info_get = mlx5_txq_info_get, 1938 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1939 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1940 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1941 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1942 .is_removed = mlx5_is_removed, 1943 .get_module_info = mlx5_get_module_info, 1944 .get_module_eeprom = mlx5_get_module_eeprom, 1945 .hairpin_cap_get = mlx5_hairpin_cap_get, 1946 .mtr_ops_get = mlx5_flow_meter_ops_get, 1947 .hairpin_bind = mlx5_hairpin_bind, 1948 .hairpin_unbind = mlx5_hairpin_unbind, 1949 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1950 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1951 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1952 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1953 .get_monitor_addr = mlx5_get_monitor_addr, 1954 }; 1955 1956 /** 1957 * Verify and store value for device argument. 1958 * 1959 * @param[in] key 1960 * Key argument to verify. 1961 * @param[in] val 1962 * Value associated with key. 1963 * @param opaque 1964 * User data. 1965 * 1966 * @return 1967 * 0 on success, a negative errno value otherwise and rte_errno is set. 1968 */ 1969 static int 1970 mlx5_args_check(const char *key, const char *val, void *opaque) 1971 { 1972 struct mlx5_dev_config *config = opaque; 1973 unsigned long mod; 1974 signed long tmp; 1975 1976 /* No-op, port representors are processed in mlx5_dev_spawn(). */ 1977 if (!strcmp(MLX5_DRIVER_KEY, key) || !strcmp(MLX5_REPRESENTOR, key) || 1978 !strcmp(MLX5_SYS_MEM_EN, key) || !strcmp(MLX5_TX_DB_NC, key) || 1979 !strcmp(MLX5_MR_MEMPOOL_REG_EN, key) || 1980 !strcmp(MLX5_MR_EXT_MEMSEG_EN, key)) 1981 return 0; 1982 errno = 0; 1983 tmp = strtol(val, NULL, 0); 1984 if (errno) { 1985 rte_errno = errno; 1986 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); 1987 return -rte_errno; 1988 } 1989 if (tmp < 0 && strcmp(MLX5_TX_PP, key) && strcmp(MLX5_TX_SKEW, key)) { 1990 /* Negative values are acceptable for some keys only. */ 1991 rte_errno = EINVAL; 1992 DRV_LOG(WARNING, "%s: invalid negative value \"%s\"", key, val); 1993 return -rte_errno; 1994 } 1995 mod = tmp >= 0 ? tmp : -tmp; 1996 if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) { 1997 if (tmp > MLX5_CQE_RESP_FORMAT_L34H_STRIDX) { 1998 DRV_LOG(ERR, "invalid CQE compression " 1999 "format parameter"); 2000 rte_errno = EINVAL; 2001 return -rte_errno; 2002 } 2003 config->cqe_comp = !!tmp; 2004 config->cqe_comp_fmt = tmp; 2005 } else if (strcmp(MLX5_RXQ_PKT_PAD_EN, key) == 0) { 2006 config->hw_padding = !!tmp; 2007 } else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) { 2008 config->mprq.enabled = !!tmp; 2009 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) { 2010 config->mprq.stride_num_n = tmp; 2011 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_SIZE, key) == 0) { 2012 config->mprq.stride_size_n = tmp; 2013 } else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) { 2014 config->mprq.max_memcpy_len = tmp; 2015 } else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) { 2016 config->mprq.min_rxqs_num = tmp; 2017 } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) { 2018 DRV_LOG(WARNING, "%s: deprecated parameter," 2019 " converted to txq_inline_max", key); 2020 config->txq_inline_max = tmp; 2021 } else if (strcmp(MLX5_TXQ_INLINE_MAX, key) == 0) { 2022 config->txq_inline_max = tmp; 2023 } else if (strcmp(MLX5_TXQ_INLINE_MIN, key) == 0) { 2024 config->txq_inline_min = tmp; 2025 } else if (strcmp(MLX5_TXQ_INLINE_MPW, key) == 0) { 2026 config->txq_inline_mpw = tmp; 2027 } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) { 2028 config->txqs_inline = tmp; 2029 } else if (strcmp(MLX5_TXQS_MAX_VEC, key) == 0) { 2030 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2031 } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { 2032 config->mps = !!tmp; 2033 } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { 2034 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2035 } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { 2036 DRV_LOG(WARNING, "%s: deprecated parameter," 2037 " converted to txq_inline_mpw", key); 2038 config->txq_inline_mpw = tmp; 2039 } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) { 2040 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2041 } else if (strcmp(MLX5_TX_PP, key) == 0) { 2042 if (!mod) { 2043 DRV_LOG(ERR, "Zero Tx packet pacing parameter"); 2044 rte_errno = EINVAL; 2045 return -rte_errno; 2046 } 2047 config->tx_pp = tmp; 2048 } else if (strcmp(MLX5_TX_SKEW, key) == 0) { 2049 config->tx_skew = tmp; 2050 } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { 2051 config->rx_vec_en = !!tmp; 2052 } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) { 2053 config->l3_vxlan_en = !!tmp; 2054 } else if (strcmp(MLX5_VF_NL_EN, key) == 0) { 2055 config->vf_nl_en = !!tmp; 2056 } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) { 2057 config->dv_esw_en = !!tmp; 2058 } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) { 2059 config->dv_flow_en = !!tmp; 2060 } else if (strcmp(MLX5_DV_XMETA_EN, key) == 0) { 2061 if (tmp != MLX5_XMETA_MODE_LEGACY && 2062 tmp != MLX5_XMETA_MODE_META16 && 2063 tmp != MLX5_XMETA_MODE_META32 && 2064 tmp != MLX5_XMETA_MODE_MISS_INFO) { 2065 DRV_LOG(ERR, "invalid extensive " 2066 "metadata parameter"); 2067 rte_errno = EINVAL; 2068 return -rte_errno; 2069 } 2070 if (tmp != MLX5_XMETA_MODE_MISS_INFO) 2071 config->dv_xmeta_en = tmp; 2072 else 2073 config->dv_miss_info = 1; 2074 } else if (strcmp(MLX5_LACP_BY_USER, key) == 0) { 2075 config->lacp_by_user = !!tmp; 2076 } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) { 2077 config->max_dump_files_num = tmp; 2078 } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) { 2079 config->lro.timeout = tmp; 2080 } else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) { 2081 DRV_LOG(DEBUG, "class argument is %s.", val); 2082 } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) { 2083 config->log_hp_size = tmp; 2084 } else if (strcmp(MLX5_RECLAIM_MEM, key) == 0) { 2085 if (tmp != MLX5_RCM_NONE && 2086 tmp != MLX5_RCM_LIGHT && 2087 tmp != MLX5_RCM_AGGR) { 2088 DRV_LOG(ERR, "Unrecognize %s: \"%s\"", key, val); 2089 rte_errno = EINVAL; 2090 return -rte_errno; 2091 } 2092 config->reclaim_mode = tmp; 2093 } else if (strcmp(MLX5_DECAP_EN, key) == 0) { 2094 config->decap_en = !!tmp; 2095 } else if (strcmp(MLX5_ALLOW_DUPLICATE_PATTERN, key) == 0) { 2096 config->allow_duplicate_pattern = !!tmp; 2097 } else { 2098 DRV_LOG(WARNING, "%s: unknown parameter", key); 2099 rte_errno = EINVAL; 2100 return -rte_errno; 2101 } 2102 return 0; 2103 } 2104 2105 /** 2106 * Parse device parameters. 2107 * 2108 * @param config 2109 * Pointer to device configuration structure. 2110 * @param devargs 2111 * Device arguments structure. 2112 * 2113 * @return 2114 * 0 on success, a negative errno value otherwise and rte_errno is set. 2115 */ 2116 int 2117 mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs) 2118 { 2119 const char **params = (const char *[]){ 2120 MLX5_DRIVER_KEY, 2121 MLX5_RXQ_CQE_COMP_EN, 2122 MLX5_RXQ_PKT_PAD_EN, 2123 MLX5_RX_MPRQ_EN, 2124 MLX5_RX_MPRQ_LOG_STRIDE_NUM, 2125 MLX5_RX_MPRQ_LOG_STRIDE_SIZE, 2126 MLX5_RX_MPRQ_MAX_MEMCPY_LEN, 2127 MLX5_RXQS_MIN_MPRQ, 2128 MLX5_TXQ_INLINE, 2129 MLX5_TXQ_INLINE_MIN, 2130 MLX5_TXQ_INLINE_MAX, 2131 MLX5_TXQ_INLINE_MPW, 2132 MLX5_TXQS_MIN_INLINE, 2133 MLX5_TXQS_MAX_VEC, 2134 MLX5_TXQ_MPW_EN, 2135 MLX5_TXQ_MPW_HDR_DSEG_EN, 2136 MLX5_TXQ_MAX_INLINE_LEN, 2137 MLX5_TX_DB_NC, 2138 MLX5_TX_PP, 2139 MLX5_TX_SKEW, 2140 MLX5_TX_VEC_EN, 2141 MLX5_RX_VEC_EN, 2142 MLX5_L3_VXLAN_EN, 2143 MLX5_VF_NL_EN, 2144 MLX5_DV_ESW_EN, 2145 MLX5_DV_FLOW_EN, 2146 MLX5_DV_XMETA_EN, 2147 MLX5_LACP_BY_USER, 2148 MLX5_MR_EXT_MEMSEG_EN, 2149 MLX5_REPRESENTOR, 2150 MLX5_MAX_DUMP_FILES_NUM, 2151 MLX5_LRO_TIMEOUT_USEC, 2152 RTE_DEVARGS_KEY_CLASS, 2153 MLX5_HP_BUF_SIZE, 2154 MLX5_RECLAIM_MEM, 2155 MLX5_SYS_MEM_EN, 2156 MLX5_DECAP_EN, 2157 MLX5_ALLOW_DUPLICATE_PATTERN, 2158 MLX5_MR_MEMPOOL_REG_EN, 2159 NULL, 2160 }; 2161 struct rte_kvargs *kvlist; 2162 int ret = 0; 2163 int i; 2164 2165 if (devargs == NULL) 2166 return 0; 2167 /* Following UGLY cast is done to pass checkpatch. */ 2168 kvlist = rte_kvargs_parse(devargs->args, params); 2169 if (kvlist == NULL) { 2170 rte_errno = EINVAL; 2171 return -rte_errno; 2172 } 2173 /* Process parameters. */ 2174 for (i = 0; (params[i] != NULL); ++i) { 2175 if (rte_kvargs_count(kvlist, params[i])) { 2176 ret = rte_kvargs_process(kvlist, params[i], 2177 mlx5_args_check, config); 2178 if (ret) { 2179 rte_errno = EINVAL; 2180 rte_kvargs_free(kvlist); 2181 return -rte_errno; 2182 } 2183 } 2184 } 2185 rte_kvargs_free(kvlist); 2186 return 0; 2187 } 2188 2189 /** 2190 * Configures the minimal amount of data to inline into WQE 2191 * while sending packets. 2192 * 2193 * - the txq_inline_min has the maximal priority, if this 2194 * key is specified in devargs 2195 * - if DevX is enabled the inline mode is queried from the 2196 * device (HCA attributes and NIC vport context if needed). 2197 * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4 Lx 2198 * and none (0 bytes) for other NICs 2199 * 2200 * @param spawn 2201 * Verbs device parameters (name, port, switch_info) to spawn. 2202 * @param config 2203 * Device configuration parameters. 2204 */ 2205 void 2206 mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn, 2207 struct mlx5_dev_config *config) 2208 { 2209 if (config->txq_inline_min != MLX5_ARG_UNSET) { 2210 /* Application defines size of inlined data explicitly. */ 2211 if (spawn->pci_dev != NULL) { 2212 switch (spawn->pci_dev->id.device_id) { 2213 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2214 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2215 if (config->txq_inline_min < 2216 (int)MLX5_INLINE_HSIZE_L2) { 2217 DRV_LOG(DEBUG, 2218 "txq_inline_mix aligned to minimal ConnectX-4 required value %d", 2219 (int)MLX5_INLINE_HSIZE_L2); 2220 config->txq_inline_min = 2221 MLX5_INLINE_HSIZE_L2; 2222 } 2223 break; 2224 } 2225 } 2226 goto exit; 2227 } 2228 if (config->hca_attr.eth_net_offloads) { 2229 /* We have DevX enabled, inline mode queried successfully. */ 2230 switch (config->hca_attr.wqe_inline_mode) { 2231 case MLX5_CAP_INLINE_MODE_L2: 2232 /* outer L2 header must be inlined. */ 2233 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2234 goto exit; 2235 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2236 /* No inline data are required by NIC. */ 2237 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2238 config->hw_vlan_insert = 2239 config->hca_attr.wqe_vlan_insert; 2240 DRV_LOG(DEBUG, "Tx VLAN insertion is supported"); 2241 goto exit; 2242 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2243 /* inline mode is defined by NIC vport context. */ 2244 if (!config->hca_attr.eth_virt) 2245 break; 2246 switch (config->hca_attr.vport_inline_mode) { 2247 case MLX5_INLINE_MODE_NONE: 2248 config->txq_inline_min = 2249 MLX5_INLINE_HSIZE_NONE; 2250 goto exit; 2251 case MLX5_INLINE_MODE_L2: 2252 config->txq_inline_min = 2253 MLX5_INLINE_HSIZE_L2; 2254 goto exit; 2255 case MLX5_INLINE_MODE_IP: 2256 config->txq_inline_min = 2257 MLX5_INLINE_HSIZE_L3; 2258 goto exit; 2259 case MLX5_INLINE_MODE_TCP_UDP: 2260 config->txq_inline_min = 2261 MLX5_INLINE_HSIZE_L4; 2262 goto exit; 2263 case MLX5_INLINE_MODE_INNER_L2: 2264 config->txq_inline_min = 2265 MLX5_INLINE_HSIZE_INNER_L2; 2266 goto exit; 2267 case MLX5_INLINE_MODE_INNER_IP: 2268 config->txq_inline_min = 2269 MLX5_INLINE_HSIZE_INNER_L3; 2270 goto exit; 2271 case MLX5_INLINE_MODE_INNER_TCP_UDP: 2272 config->txq_inline_min = 2273 MLX5_INLINE_HSIZE_INNER_L4; 2274 goto exit; 2275 } 2276 } 2277 } 2278 if (spawn->pci_dev == NULL) { 2279 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2280 goto exit; 2281 } 2282 /* 2283 * We get here if we are unable to deduce 2284 * inline data size with DevX. Try PCI ID 2285 * to determine old NICs. 2286 */ 2287 switch (spawn->pci_dev->id.device_id) { 2288 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2289 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2290 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: 2291 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 2292 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2293 config->hw_vlan_insert = 0; 2294 break; 2295 case PCI_DEVICE_ID_MELLANOX_CONNECTX5: 2296 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 2297 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: 2298 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 2299 /* 2300 * These NICs support VLAN insertion from WQE and 2301 * report the wqe_vlan_insert flag. But there is the bug 2302 * and PFC control may be broken, so disable feature. 2303 */ 2304 config->hw_vlan_insert = 0; 2305 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2306 break; 2307 default: 2308 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2309 break; 2310 } 2311 exit: 2312 DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min); 2313 } 2314 2315 /** 2316 * Configures the metadata mask fields in the shared context. 2317 * 2318 * @param [in] dev 2319 * Pointer to Ethernet device. 2320 */ 2321 void 2322 mlx5_set_metadata_mask(struct rte_eth_dev *dev) 2323 { 2324 struct mlx5_priv *priv = dev->data->dev_private; 2325 struct mlx5_dev_ctx_shared *sh = priv->sh; 2326 uint32_t meta, mark, reg_c0; 2327 2328 reg_c0 = ~priv->vport_meta_mask; 2329 switch (priv->config.dv_xmeta_en) { 2330 case MLX5_XMETA_MODE_LEGACY: 2331 meta = UINT32_MAX; 2332 mark = MLX5_FLOW_MARK_MASK; 2333 break; 2334 case MLX5_XMETA_MODE_META16: 2335 meta = reg_c0 >> rte_bsf32(reg_c0); 2336 mark = MLX5_FLOW_MARK_MASK; 2337 break; 2338 case MLX5_XMETA_MODE_META32: 2339 meta = UINT32_MAX; 2340 mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK; 2341 break; 2342 default: 2343 meta = 0; 2344 mark = 0; 2345 MLX5_ASSERT(false); 2346 break; 2347 } 2348 if (sh->dv_mark_mask && sh->dv_mark_mask != mark) 2349 DRV_LOG(WARNING, "metadata MARK mask mismatche %08X:%08X", 2350 sh->dv_mark_mask, mark); 2351 else 2352 sh->dv_mark_mask = mark; 2353 if (sh->dv_meta_mask && sh->dv_meta_mask != meta) 2354 DRV_LOG(WARNING, "metadata META mask mismatche %08X:%08X", 2355 sh->dv_meta_mask, meta); 2356 else 2357 sh->dv_meta_mask = meta; 2358 if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0) 2359 DRV_LOG(WARNING, "metadata reg_c0 mask mismatche %08X:%08X", 2360 sh->dv_meta_mask, reg_c0); 2361 else 2362 sh->dv_regc0_mask = reg_c0; 2363 DRV_LOG(DEBUG, "metadata mode %u", priv->config.dv_xmeta_en); 2364 DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask); 2365 DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask); 2366 DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask); 2367 } 2368 2369 int 2370 rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n) 2371 { 2372 static const char *const dynf_names[] = { 2373 RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, 2374 RTE_MBUF_DYNFLAG_METADATA_NAME, 2375 RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME 2376 }; 2377 unsigned int i; 2378 2379 if (n < RTE_DIM(dynf_names)) 2380 return -ENOMEM; 2381 for (i = 0; i < RTE_DIM(dynf_names); i++) { 2382 if (names[i] == NULL) 2383 return -EINVAL; 2384 strcpy(names[i], dynf_names[i]); 2385 } 2386 return RTE_DIM(dynf_names); 2387 } 2388 2389 /** 2390 * Comparison callback to sort device data. 2391 * 2392 * This is meant to be used with qsort(). 2393 * 2394 * @param a[in] 2395 * Pointer to pointer to first data object. 2396 * @param b[in] 2397 * Pointer to pointer to second data object. 2398 * 2399 * @return 2400 * 0 if both objects are equal, less than 0 if the first argument is less 2401 * than the second, greater than 0 otherwise. 2402 */ 2403 int 2404 mlx5_dev_check_sibling_config(struct mlx5_priv *priv, 2405 struct mlx5_dev_config *config, 2406 struct rte_device *dpdk_dev) 2407 { 2408 struct mlx5_dev_ctx_shared *sh = priv->sh; 2409 struct mlx5_dev_config *sh_conf = NULL; 2410 uint16_t port_id; 2411 2412 MLX5_ASSERT(sh); 2413 /* Nothing to compare for the single/first device. */ 2414 if (sh->refcnt == 1) 2415 return 0; 2416 /* Find the device with shared context. */ 2417 MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) { 2418 struct mlx5_priv *opriv = 2419 rte_eth_devices[port_id].data->dev_private; 2420 2421 if (opriv && opriv != priv && opriv->sh == sh) { 2422 sh_conf = &opriv->config; 2423 break; 2424 } 2425 } 2426 if (!sh_conf) 2427 return 0; 2428 if (sh_conf->dv_flow_en ^ config->dv_flow_en) { 2429 DRV_LOG(ERR, "\"dv_flow_en\" configuration mismatch" 2430 " for shared %s context", sh->ibdev_name); 2431 rte_errno = EINVAL; 2432 return rte_errno; 2433 } 2434 if (sh_conf->dv_xmeta_en ^ config->dv_xmeta_en) { 2435 DRV_LOG(ERR, "\"dv_xmeta_en\" configuration mismatch" 2436 " for shared %s context", sh->ibdev_name); 2437 rte_errno = EINVAL; 2438 return rte_errno; 2439 } 2440 return 0; 2441 } 2442 2443 /** 2444 * Look for the ethernet device belonging to mlx5 driver. 2445 * 2446 * @param[in] port_id 2447 * port_id to start looking for device. 2448 * @param[in] odev 2449 * Pointer to the hint device. When device is being probed 2450 * the its siblings (master and preceding representors might 2451 * not have assigned driver yet (because the mlx5_os_pci_probe() 2452 * is not completed yet, for this case match on hint 2453 * device may be used to detect sibling device. 2454 * 2455 * @return 2456 * port_id of found device, RTE_MAX_ETHPORT if not found. 2457 */ 2458 uint16_t 2459 mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev) 2460 { 2461 while (port_id < RTE_MAX_ETHPORTS) { 2462 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 2463 2464 if (dev->state != RTE_ETH_DEV_UNUSED && 2465 dev->device && 2466 (dev->device == odev || 2467 (dev->device->driver && 2468 dev->device->driver->name && 2469 ((strcmp(dev->device->driver->name, 2470 MLX5_PCI_DRIVER_NAME) == 0) || 2471 (strcmp(dev->device->driver->name, 2472 MLX5_AUXILIARY_DRIVER_NAME) == 0))))) 2473 break; 2474 port_id++; 2475 } 2476 if (port_id >= RTE_MAX_ETHPORTS) 2477 return RTE_MAX_ETHPORTS; 2478 return port_id; 2479 } 2480 2481 /** 2482 * Callback to remove a device. 2483 * 2484 * This function removes all Ethernet devices belong to a given device. 2485 * 2486 * @param[in] cdev 2487 * Pointer to the generic device. 2488 * 2489 * @return 2490 * 0 on success, the function cannot fail. 2491 */ 2492 int 2493 mlx5_net_remove(struct mlx5_common_device *cdev) 2494 { 2495 uint16_t port_id; 2496 int ret = 0; 2497 2498 RTE_ETH_FOREACH_DEV_OF(port_id, cdev->dev) { 2499 /* 2500 * mlx5_dev_close() is not registered to secondary process, 2501 * call the close function explicitly for secondary process. 2502 */ 2503 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 2504 ret |= mlx5_dev_close(&rte_eth_devices[port_id]); 2505 else 2506 ret |= rte_eth_dev_close(port_id); 2507 } 2508 return ret == 0 ? 0 : -EIO; 2509 } 2510 2511 static const struct rte_pci_id mlx5_pci_id_map[] = { 2512 { 2513 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2514 PCI_DEVICE_ID_MELLANOX_CONNECTX4) 2515 }, 2516 { 2517 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2518 PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) 2519 }, 2520 { 2521 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2522 PCI_DEVICE_ID_MELLANOX_CONNECTX4LX) 2523 }, 2524 { 2525 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2526 PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) 2527 }, 2528 { 2529 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2530 PCI_DEVICE_ID_MELLANOX_CONNECTX5) 2531 }, 2532 { 2533 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2534 PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) 2535 }, 2536 { 2537 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2538 PCI_DEVICE_ID_MELLANOX_CONNECTX5EX) 2539 }, 2540 { 2541 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2542 PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF) 2543 }, 2544 { 2545 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2546 PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) 2547 }, 2548 { 2549 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2550 PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF) 2551 }, 2552 { 2553 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2554 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 2555 }, 2556 { 2557 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2558 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF) 2559 }, 2560 { 2561 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2562 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 2563 }, 2564 { 2565 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2566 PCI_DEVICE_ID_MELLANOX_CONNECTXVF) 2567 }, 2568 { 2569 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2570 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF) 2571 }, 2572 { 2573 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2574 PCI_DEVICE_ID_MELLANOX_CONNECTX6LX) 2575 }, 2576 { 2577 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2578 PCI_DEVICE_ID_MELLANOX_CONNECTX7) 2579 }, 2580 { 2581 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2582 PCI_DEVICE_ID_MELLANOX_CONNECTX7BF) 2583 }, 2584 { 2585 .vendor_id = 0 2586 } 2587 }; 2588 2589 static struct mlx5_class_driver mlx5_net_driver = { 2590 .drv_class = MLX5_CLASS_ETH, 2591 .name = RTE_STR(MLX5_ETH_DRIVER_NAME), 2592 .id_table = mlx5_pci_id_map, 2593 .probe = mlx5_os_net_probe, 2594 .remove = mlx5_net_remove, 2595 .probe_again = 1, 2596 .intr_lsc = 1, 2597 .intr_rmv = 1, 2598 }; 2599 2600 /* Initialize driver log type. */ 2601 RTE_LOG_REGISTER_DEFAULT(mlx5_logtype, NOTICE) 2602 2603 /** 2604 * Driver initialization routine. 2605 */ 2606 RTE_INIT(rte_mlx5_pmd_init) 2607 { 2608 pthread_mutex_init(&mlx5_dev_ctx_list_mutex, NULL); 2609 mlx5_common_init(); 2610 /* Build the static tables for Verbs conversion. */ 2611 mlx5_set_ptype_table(); 2612 mlx5_set_cksum_table(); 2613 mlx5_set_swp_types_table(); 2614 if (mlx5_glue) 2615 mlx5_class_driver_register(&mlx5_net_driver); 2616 } 2617 2618 RTE_PMD_EXPORT_NAME(MLX5_ETH_DRIVER_NAME, __COUNTER__); 2619 RTE_PMD_REGISTER_PCI_TABLE(MLX5_ETH_DRIVER_NAME, mlx5_pci_id_map); 2620 RTE_PMD_REGISTER_KMOD_DEP(MLX5_ETH_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 2621