1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <stddef.h> 7 #include <unistd.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <errno.h> 12 13 #include <rte_malloc.h> 14 #include <ethdev_driver.h> 15 #include <rte_pci.h> 16 #include <rte_bus_pci.h> 17 #include <rte_common.h> 18 #include <rte_kvargs.h> 19 #include <rte_rwlock.h> 20 #include <rte_spinlock.h> 21 #include <rte_string_fns.h> 22 #include <rte_alarm.h> 23 #include <rte_cycles.h> 24 25 #include <mlx5_glue.h> 26 #include <mlx5_devx_cmds.h> 27 #include <mlx5_common.h> 28 #include <mlx5_common_os.h> 29 #include <mlx5_common_mp.h> 30 #include <mlx5_malloc.h> 31 32 #include "mlx5_defs.h" 33 #include "mlx5.h" 34 #include "mlx5_utils.h" 35 #include "mlx5_rxtx.h" 36 #include "mlx5_rx.h" 37 #include "mlx5_tx.h" 38 #include "mlx5_autoconf.h" 39 #include "mlx5_mr.h" 40 #include "mlx5_flow.h" 41 #include "mlx5_flow_os.h" 42 #include "rte_pmd_mlx5.h" 43 44 #define MLX5_ETH_DRIVER_NAME mlx5_eth 45 46 /* Device parameter to enable RX completion queue compression. */ 47 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en" 48 49 /* Device parameter to enable padding Rx packet to cacheline size. */ 50 #define MLX5_RXQ_PKT_PAD_EN "rxq_pkt_pad_en" 51 52 /* Device parameter to enable Multi-Packet Rx queue. */ 53 #define MLX5_RX_MPRQ_EN "mprq_en" 54 55 /* Device parameter to configure log 2 of the number of strides for MPRQ. */ 56 #define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num" 57 58 /* Device parameter to configure log 2 of the stride size for MPRQ. */ 59 #define MLX5_RX_MPRQ_LOG_STRIDE_SIZE "mprq_log_stride_size" 60 61 /* Device parameter to limit the size of memcpy'd packet for MPRQ. */ 62 #define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len" 63 64 /* Device parameter to set the minimum number of Rx queues to enable MPRQ. */ 65 #define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq" 66 67 /* Device parameter to configure inline send. Deprecated, ignored.*/ 68 #define MLX5_TXQ_INLINE "txq_inline" 69 70 /* Device parameter to limit packet size to inline with ordinary SEND. */ 71 #define MLX5_TXQ_INLINE_MAX "txq_inline_max" 72 73 /* Device parameter to configure minimal data size to inline. */ 74 #define MLX5_TXQ_INLINE_MIN "txq_inline_min" 75 76 /* Device parameter to limit packet size to inline with Enhanced MPW. */ 77 #define MLX5_TXQ_INLINE_MPW "txq_inline_mpw" 78 79 /* 80 * Device parameter to configure the number of TX queues threshold for 81 * enabling inline send. 82 */ 83 #define MLX5_TXQS_MIN_INLINE "txqs_min_inline" 84 85 /* 86 * Device parameter to configure the number of TX queues threshold for 87 * enabling vectorized Tx, deprecated, ignored (no vectorized Tx routines). 88 */ 89 #define MLX5_TXQS_MAX_VEC "txqs_max_vec" 90 91 /* Device parameter to enable multi-packet send WQEs. */ 92 #define MLX5_TXQ_MPW_EN "txq_mpw_en" 93 94 /* 95 * Device parameter to force doorbell register mapping 96 * to non-cahed region eliminating the extra write memory barrier. 97 */ 98 #define MLX5_TX_DB_NC "tx_db_nc" 99 100 /* 101 * Device parameter to include 2 dsegs in the title WQEBB. 102 * Deprecated, ignored. 103 */ 104 #define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en" 105 106 /* 107 * Device parameter to limit the size of inlining packet. 108 * Deprecated, ignored. 109 */ 110 #define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len" 111 112 /* 113 * Device parameter to enable Tx scheduling on timestamps 114 * and specify the packet pacing granularity in nanoseconds. 115 */ 116 #define MLX5_TX_PP "tx_pp" 117 118 /* 119 * Device parameter to specify skew in nanoseconds on Tx datapath, 120 * it represents the time between SQ start WQE processing and 121 * appearing actual packet data on the wire. 122 */ 123 #define MLX5_TX_SKEW "tx_skew" 124 125 /* 126 * Device parameter to enable hardware Tx vector. 127 * Deprecated, ignored (no vectorized Tx routines anymore). 128 */ 129 #define MLX5_TX_VEC_EN "tx_vec_en" 130 131 /* Device parameter to enable hardware Rx vector. */ 132 #define MLX5_RX_VEC_EN "rx_vec_en" 133 134 /* Allow L3 VXLAN flow creation. */ 135 #define MLX5_L3_VXLAN_EN "l3_vxlan_en" 136 137 /* Activate DV E-Switch flow steering. */ 138 #define MLX5_DV_ESW_EN "dv_esw_en" 139 140 /* Activate DV flow steering. */ 141 #define MLX5_DV_FLOW_EN "dv_flow_en" 142 143 /* Enable extensive flow metadata support. */ 144 #define MLX5_DV_XMETA_EN "dv_xmeta_en" 145 146 /* Device parameter to let the user manage the lacp traffic of bonded device */ 147 #define MLX5_LACP_BY_USER "lacp_by_user" 148 149 /* Activate Netlink support in VF mode. */ 150 #define MLX5_VF_NL_EN "vf_nl_en" 151 152 /* Enable extending memsegs when creating a MR. */ 153 #define MLX5_MR_EXT_MEMSEG_EN "mr_ext_memseg_en" 154 155 /* Select port representors to instantiate. */ 156 #define MLX5_REPRESENTOR "representor" 157 158 /* Device parameter to configure the maximum number of dump files per queue. */ 159 #define MLX5_MAX_DUMP_FILES_NUM "max_dump_files_num" 160 161 /* Configure timeout of LRO session (in microseconds). */ 162 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec" 163 164 /* 165 * Device parameter to configure the total data buffer size for a single 166 * hairpin queue (logarithm value). 167 */ 168 #define MLX5_HP_BUF_SIZE "hp_buf_log_sz" 169 170 /* Flow memory reclaim mode. */ 171 #define MLX5_RECLAIM_MEM "reclaim_mem_mode" 172 173 /* The default memory allocator used in PMD. */ 174 #define MLX5_SYS_MEM_EN "sys_mem_en" 175 /* Decap will be used or not. */ 176 #define MLX5_DECAP_EN "decap_en" 177 178 /* Device parameter to configure allow or prevent duplicate rules pattern. */ 179 #define MLX5_ALLOW_DUPLICATE_PATTERN "allow_duplicate_pattern" 180 181 /* Shared memory between primary and secondary processes. */ 182 struct mlx5_shared_data *mlx5_shared_data; 183 184 /** Driver-specific log messages type. */ 185 int mlx5_logtype; 186 187 static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = 188 LIST_HEAD_INITIALIZER(); 189 static pthread_mutex_t mlx5_dev_ctx_list_mutex; 190 static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { 191 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 192 [MLX5_IPOOL_DECAP_ENCAP] = { 193 .size = sizeof(struct mlx5_flow_dv_encap_decap_resource), 194 .trunk_size = 64, 195 .grow_trunk = 3, 196 .grow_shift = 2, 197 .need_lock = 1, 198 .release_mem_en = 1, 199 .malloc = mlx5_malloc, 200 .free = mlx5_free, 201 .type = "mlx5_encap_decap_ipool", 202 }, 203 [MLX5_IPOOL_PUSH_VLAN] = { 204 .size = sizeof(struct mlx5_flow_dv_push_vlan_action_resource), 205 .trunk_size = 64, 206 .grow_trunk = 3, 207 .grow_shift = 2, 208 .need_lock = 1, 209 .release_mem_en = 1, 210 .malloc = mlx5_malloc, 211 .free = mlx5_free, 212 .type = "mlx5_push_vlan_ipool", 213 }, 214 [MLX5_IPOOL_TAG] = { 215 .size = sizeof(struct mlx5_flow_dv_tag_resource), 216 .trunk_size = 64, 217 .grow_trunk = 3, 218 .grow_shift = 2, 219 .need_lock = 1, 220 .release_mem_en = 0, 221 .per_core_cache = (1 << 16), 222 .malloc = mlx5_malloc, 223 .free = mlx5_free, 224 .type = "mlx5_tag_ipool", 225 }, 226 [MLX5_IPOOL_PORT_ID] = { 227 .size = sizeof(struct mlx5_flow_dv_port_id_action_resource), 228 .trunk_size = 64, 229 .grow_trunk = 3, 230 .grow_shift = 2, 231 .need_lock = 1, 232 .release_mem_en = 1, 233 .malloc = mlx5_malloc, 234 .free = mlx5_free, 235 .type = "mlx5_port_id_ipool", 236 }, 237 [MLX5_IPOOL_JUMP] = { 238 .size = sizeof(struct mlx5_flow_tbl_data_entry), 239 .trunk_size = 64, 240 .grow_trunk = 3, 241 .grow_shift = 2, 242 .need_lock = 1, 243 .release_mem_en = 1, 244 .malloc = mlx5_malloc, 245 .free = mlx5_free, 246 .type = "mlx5_jump_ipool", 247 }, 248 [MLX5_IPOOL_SAMPLE] = { 249 .size = sizeof(struct mlx5_flow_dv_sample_resource), 250 .trunk_size = 64, 251 .grow_trunk = 3, 252 .grow_shift = 2, 253 .need_lock = 1, 254 .release_mem_en = 1, 255 .malloc = mlx5_malloc, 256 .free = mlx5_free, 257 .type = "mlx5_sample_ipool", 258 }, 259 [MLX5_IPOOL_DEST_ARRAY] = { 260 .size = sizeof(struct mlx5_flow_dv_dest_array_resource), 261 .trunk_size = 64, 262 .grow_trunk = 3, 263 .grow_shift = 2, 264 .need_lock = 1, 265 .release_mem_en = 1, 266 .malloc = mlx5_malloc, 267 .free = mlx5_free, 268 .type = "mlx5_dest_array_ipool", 269 }, 270 [MLX5_IPOOL_TUNNEL_ID] = { 271 .size = sizeof(struct mlx5_flow_tunnel), 272 .trunk_size = MLX5_MAX_TUNNELS, 273 .need_lock = 1, 274 .release_mem_en = 1, 275 .type = "mlx5_tunnel_offload", 276 }, 277 [MLX5_IPOOL_TNL_TBL_ID] = { 278 .size = 0, 279 .need_lock = 1, 280 .type = "mlx5_flow_tnl_tbl_ipool", 281 }, 282 #endif 283 [MLX5_IPOOL_MTR] = { 284 /** 285 * The ipool index should grow continually from small to big, 286 * for meter idx, so not set grow_trunk to avoid meter index 287 * not jump continually. 288 */ 289 .size = sizeof(struct mlx5_legacy_flow_meter), 290 .trunk_size = 64, 291 .need_lock = 1, 292 .release_mem_en = 1, 293 .malloc = mlx5_malloc, 294 .free = mlx5_free, 295 .type = "mlx5_meter_ipool", 296 }, 297 [MLX5_IPOOL_MCP] = { 298 .size = sizeof(struct mlx5_flow_mreg_copy_resource), 299 .trunk_size = 64, 300 .grow_trunk = 3, 301 .grow_shift = 2, 302 .need_lock = 1, 303 .release_mem_en = 1, 304 .malloc = mlx5_malloc, 305 .free = mlx5_free, 306 .type = "mlx5_mcp_ipool", 307 }, 308 [MLX5_IPOOL_HRXQ] = { 309 .size = (sizeof(struct mlx5_hrxq) + MLX5_RSS_HASH_KEY_LEN), 310 .trunk_size = 64, 311 .grow_trunk = 3, 312 .grow_shift = 2, 313 .need_lock = 1, 314 .release_mem_en = 1, 315 .malloc = mlx5_malloc, 316 .free = mlx5_free, 317 .type = "mlx5_hrxq_ipool", 318 }, 319 [MLX5_IPOOL_MLX5_FLOW] = { 320 /* 321 * MLX5_IPOOL_MLX5_FLOW size varies for DV and VERBS flows. 322 * It set in run time according to PCI function configuration. 323 */ 324 .size = 0, 325 .trunk_size = 64, 326 .grow_trunk = 3, 327 .grow_shift = 2, 328 .need_lock = 1, 329 .release_mem_en = 0, 330 .per_core_cache = 1 << 19, 331 .malloc = mlx5_malloc, 332 .free = mlx5_free, 333 .type = "mlx5_flow_handle_ipool", 334 }, 335 [MLX5_IPOOL_RTE_FLOW] = { 336 .size = sizeof(struct rte_flow), 337 .trunk_size = 4096, 338 .need_lock = 1, 339 .release_mem_en = 1, 340 .malloc = mlx5_malloc, 341 .free = mlx5_free, 342 .type = "rte_flow_ipool", 343 }, 344 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID] = { 345 .size = 0, 346 .need_lock = 1, 347 .type = "mlx5_flow_rss_id_ipool", 348 }, 349 [MLX5_IPOOL_RSS_SHARED_ACTIONS] = { 350 .size = sizeof(struct mlx5_shared_action_rss), 351 .trunk_size = 64, 352 .grow_trunk = 3, 353 .grow_shift = 2, 354 .need_lock = 1, 355 .release_mem_en = 1, 356 .malloc = mlx5_malloc, 357 .free = mlx5_free, 358 .type = "mlx5_shared_action_rss", 359 }, 360 [MLX5_IPOOL_MTR_POLICY] = { 361 /** 362 * The ipool index should grow continually from small to big, 363 * for policy idx, so not set grow_trunk to avoid policy index 364 * not jump continually. 365 */ 366 .size = sizeof(struct mlx5_flow_meter_sub_policy), 367 .trunk_size = 64, 368 .need_lock = 1, 369 .release_mem_en = 1, 370 .malloc = mlx5_malloc, 371 .free = mlx5_free, 372 .type = "mlx5_meter_policy_ipool", 373 }, 374 }; 375 376 377 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512 378 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16 379 380 #define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024 381 382 /** 383 * Decide whether representor ID is a HPF(host PF) port on BF2. 384 * 385 * @param dev 386 * Pointer to Ethernet device structure. 387 * 388 * @return 389 * Non-zero if HPF, otherwise 0. 390 */ 391 bool 392 mlx5_is_hpf(struct rte_eth_dev *dev) 393 { 394 struct mlx5_priv *priv = dev->data->dev_private; 395 uint16_t repr = MLX5_REPRESENTOR_REPR(priv->representor_id); 396 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 397 398 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_VF && 399 MLX5_REPRESENTOR_REPR(-1) == repr; 400 } 401 402 /** 403 * Decide whether representor ID is a SF port representor. 404 * 405 * @param dev 406 * Pointer to Ethernet device structure. 407 * 408 * @return 409 * Non-zero if HPF, otherwise 0. 410 */ 411 bool 412 mlx5_is_sf_repr(struct rte_eth_dev *dev) 413 { 414 struct mlx5_priv *priv = dev->data->dev_private; 415 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 416 417 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_SF; 418 } 419 420 /** 421 * Initialize the ASO aging management structure. 422 * 423 * @param[in] sh 424 * Pointer to mlx5_dev_ctx_shared object to free 425 * 426 * @return 427 * 0 on success, a negative errno value otherwise and rte_errno is set. 428 */ 429 int 430 mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh) 431 { 432 int err; 433 434 if (sh->aso_age_mng) 435 return 0; 436 sh->aso_age_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->aso_age_mng), 437 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 438 if (!sh->aso_age_mng) { 439 DRV_LOG(ERR, "aso_age_mng allocation was failed."); 440 rte_errno = ENOMEM; 441 return -ENOMEM; 442 } 443 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_FLOW_HIT); 444 if (err) { 445 mlx5_free(sh->aso_age_mng); 446 return -1; 447 } 448 rte_spinlock_init(&sh->aso_age_mng->resize_sl); 449 rte_spinlock_init(&sh->aso_age_mng->free_sl); 450 LIST_INIT(&sh->aso_age_mng->free); 451 return 0; 452 } 453 454 /** 455 * Close and release all the resources of the ASO aging management structure. 456 * 457 * @param[in] sh 458 * Pointer to mlx5_dev_ctx_shared object to free. 459 */ 460 static void 461 mlx5_flow_aso_age_mng_close(struct mlx5_dev_ctx_shared *sh) 462 { 463 int i, j; 464 465 mlx5_aso_flow_hit_queue_poll_stop(sh); 466 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_FLOW_HIT); 467 if (sh->aso_age_mng->pools) { 468 struct mlx5_aso_age_pool *pool; 469 470 for (i = 0; i < sh->aso_age_mng->next; ++i) { 471 pool = sh->aso_age_mng->pools[i]; 472 claim_zero(mlx5_devx_cmd_destroy 473 (pool->flow_hit_aso_obj)); 474 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) 475 if (pool->actions[j].dr_action) 476 claim_zero 477 (mlx5_flow_os_destroy_flow_action 478 (pool->actions[j].dr_action)); 479 mlx5_free(pool); 480 } 481 mlx5_free(sh->aso_age_mng->pools); 482 } 483 mlx5_free(sh->aso_age_mng); 484 } 485 486 /** 487 * Initialize the shared aging list information per port. 488 * 489 * @param[in] sh 490 * Pointer to mlx5_dev_ctx_shared object. 491 */ 492 static void 493 mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh) 494 { 495 uint32_t i; 496 struct mlx5_age_info *age_info; 497 498 for (i = 0; i < sh->max_port; i++) { 499 age_info = &sh->port[i].age_info; 500 age_info->flags = 0; 501 TAILQ_INIT(&age_info->aged_counters); 502 LIST_INIT(&age_info->aged_aso); 503 rte_spinlock_init(&age_info->aged_sl); 504 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 505 } 506 } 507 508 /** 509 * Initialize the counters management structure. 510 * 511 * @param[in] sh 512 * Pointer to mlx5_dev_ctx_shared object to free 513 */ 514 static void 515 mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh) 516 { 517 int i; 518 519 memset(&sh->cmng, 0, sizeof(sh->cmng)); 520 TAILQ_INIT(&sh->cmng.flow_counters); 521 sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET; 522 sh->cmng.max_id = -1; 523 sh->cmng.last_pool_idx = POOL_IDX_INVALID; 524 rte_spinlock_init(&sh->cmng.pool_update_sl); 525 for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) { 526 TAILQ_INIT(&sh->cmng.counters[i]); 527 rte_spinlock_init(&sh->cmng.csl[i]); 528 } 529 } 530 531 /** 532 * Destroy all the resources allocated for a counter memory management. 533 * 534 * @param[in] mng 535 * Pointer to the memory management structure. 536 */ 537 static void 538 mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng) 539 { 540 uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data; 541 542 LIST_REMOVE(mng, next); 543 claim_zero(mlx5_devx_cmd_destroy(mng->dm)); 544 claim_zero(mlx5_os_umem_dereg(mng->umem)); 545 mlx5_free(mem); 546 } 547 548 /** 549 * Close and release all the resources of the counters management. 550 * 551 * @param[in] sh 552 * Pointer to mlx5_dev_ctx_shared object to free. 553 */ 554 static void 555 mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) 556 { 557 struct mlx5_counter_stats_mem_mng *mng; 558 int i, j; 559 int retries = 1024; 560 561 rte_errno = 0; 562 while (--retries) { 563 rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh); 564 if (rte_errno != EINPROGRESS) 565 break; 566 rte_pause(); 567 } 568 569 if (sh->cmng.pools) { 570 struct mlx5_flow_counter_pool *pool; 571 uint16_t n_valid = sh->cmng.n_valid; 572 bool fallback = sh->cmng.counter_fallback; 573 574 for (i = 0; i < n_valid; ++i) { 575 pool = sh->cmng.pools[i]; 576 if (!fallback && pool->min_dcs) 577 claim_zero(mlx5_devx_cmd_destroy 578 (pool->min_dcs)); 579 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) { 580 struct mlx5_flow_counter *cnt = 581 MLX5_POOL_GET_CNT(pool, j); 582 583 if (cnt->action) 584 claim_zero 585 (mlx5_flow_os_destroy_flow_action 586 (cnt->action)); 587 if (fallback && MLX5_POOL_GET_CNT 588 (pool, j)->dcs_when_free) 589 claim_zero(mlx5_devx_cmd_destroy 590 (cnt->dcs_when_free)); 591 } 592 mlx5_free(pool); 593 } 594 mlx5_free(sh->cmng.pools); 595 } 596 mng = LIST_FIRST(&sh->cmng.mem_mngs); 597 while (mng) { 598 mlx5_flow_destroy_counter_stat_mem_mng(mng); 599 mng = LIST_FIRST(&sh->cmng.mem_mngs); 600 } 601 memset(&sh->cmng, 0, sizeof(sh->cmng)); 602 } 603 604 /** 605 * Initialize the aso flow meters management structure. 606 * 607 * @param[in] sh 608 * Pointer to mlx5_dev_ctx_shared object to free 609 */ 610 int 611 mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh) 612 { 613 if (!sh->mtrmng) { 614 sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, 615 sizeof(*sh->mtrmng), 616 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 617 if (!sh->mtrmng) { 618 DRV_LOG(ERR, 619 "meter management allocation was failed."); 620 rte_errno = ENOMEM; 621 return -ENOMEM; 622 } 623 if (sh->meter_aso_en) { 624 rte_spinlock_init(&sh->mtrmng->pools_mng.mtrsl); 625 LIST_INIT(&sh->mtrmng->pools_mng.meters); 626 } 627 sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 628 } 629 return 0; 630 } 631 632 /** 633 * Close and release all the resources of 634 * the ASO flow meter management structure. 635 * 636 * @param[in] sh 637 * Pointer to mlx5_dev_ctx_shared object to free. 638 */ 639 static void 640 mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) 641 { 642 struct mlx5_aso_mtr_pool *mtr_pool; 643 struct mlx5_flow_mtr_mng *mtrmng = sh->mtrmng; 644 uint32_t idx; 645 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 646 struct mlx5_aso_mtr *aso_mtr; 647 int i; 648 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 649 650 if (sh->meter_aso_en) { 651 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); 652 idx = mtrmng->pools_mng.n_valid; 653 while (idx--) { 654 mtr_pool = mtrmng->pools_mng.pools[idx]; 655 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 656 for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { 657 aso_mtr = &mtr_pool->mtrs[i]; 658 if (aso_mtr->fm.meter_action) 659 claim_zero 660 (mlx5_glue->destroy_flow_action 661 (aso_mtr->fm.meter_action)); 662 } 663 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 664 claim_zero(mlx5_devx_cmd_destroy 665 (mtr_pool->devx_obj)); 666 mtrmng->pools_mng.n_valid--; 667 mlx5_free(mtr_pool); 668 } 669 mlx5_free(sh->mtrmng->pools_mng.pools); 670 } 671 mlx5_free(sh->mtrmng); 672 sh->mtrmng = NULL; 673 } 674 675 /* Send FLOW_AGED event if needed. */ 676 void 677 mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) 678 { 679 struct mlx5_age_info *age_info; 680 uint32_t i; 681 682 for (i = 0; i < sh->max_port; i++) { 683 age_info = &sh->port[i].age_info; 684 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 685 continue; 686 MLX5_AGE_UNSET(age_info, MLX5_AGE_EVENT_NEW); 687 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) { 688 MLX5_AGE_UNSET(age_info, MLX5_AGE_TRIGGER); 689 rte_eth_dev_callback_process 690 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 691 RTE_ETH_EVENT_FLOW_AGED, NULL); 692 } 693 } 694 } 695 696 /* 697 * Initialize the ASO connection tracking structure. 698 * 699 * @param[in] sh 700 * Pointer to mlx5_dev_ctx_shared object. 701 * 702 * @return 703 * 0 on success, a negative errno value otherwise and rte_errno is set. 704 */ 705 int 706 mlx5_flow_aso_ct_mng_init(struct mlx5_dev_ctx_shared *sh) 707 { 708 int err; 709 710 if (sh->ct_mng) 711 return 0; 712 sh->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->ct_mng), 713 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 714 if (!sh->ct_mng) { 715 DRV_LOG(ERR, "ASO CT management allocation failed."); 716 rte_errno = ENOMEM; 717 return -rte_errno; 718 } 719 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_CONNECTION_TRACKING); 720 if (err) { 721 mlx5_free(sh->ct_mng); 722 /* rte_errno should be extracted from the failure. */ 723 rte_errno = EINVAL; 724 return -rte_errno; 725 } 726 rte_spinlock_init(&sh->ct_mng->ct_sl); 727 rte_rwlock_init(&sh->ct_mng->resize_rwl); 728 LIST_INIT(&sh->ct_mng->free_cts); 729 return 0; 730 } 731 732 /* 733 * Close and release all the resources of the 734 * ASO connection tracking management structure. 735 * 736 * @param[in] sh 737 * Pointer to mlx5_dev_ctx_shared object to free. 738 */ 739 static void 740 mlx5_flow_aso_ct_mng_close(struct mlx5_dev_ctx_shared *sh) 741 { 742 struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng; 743 struct mlx5_aso_ct_pool *ct_pool; 744 struct mlx5_aso_ct_action *ct; 745 uint32_t idx; 746 uint32_t val; 747 uint32_t cnt; 748 int i; 749 750 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_CONNECTION_TRACKING); 751 idx = mng->next; 752 while (idx--) { 753 cnt = 0; 754 ct_pool = mng->pools[idx]; 755 for (i = 0; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) { 756 ct = &ct_pool->actions[i]; 757 val = __atomic_fetch_sub(&ct->refcnt, 1, 758 __ATOMIC_RELAXED); 759 MLX5_ASSERT(val == 1); 760 if (val > 1) 761 cnt++; 762 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT 763 if (ct->dr_action_orig) 764 claim_zero(mlx5_glue->destroy_flow_action 765 (ct->dr_action_orig)); 766 if (ct->dr_action_rply) 767 claim_zero(mlx5_glue->destroy_flow_action 768 (ct->dr_action_rply)); 769 #endif 770 } 771 claim_zero(mlx5_devx_cmd_destroy(ct_pool->devx_obj)); 772 if (cnt) { 773 DRV_LOG(DEBUG, "%u ASO CT objects are being used in the pool %u", 774 cnt, i); 775 } 776 mlx5_free(ct_pool); 777 /* in case of failure. */ 778 mng->next--; 779 } 780 mlx5_free(mng->pools); 781 mlx5_free(mng); 782 /* Management structure must be cleared to 0s during allocation. */ 783 sh->ct_mng = NULL; 784 } 785 786 /** 787 * Initialize the flow resources' indexed mempool. 788 * 789 * @param[in] sh 790 * Pointer to mlx5_dev_ctx_shared object. 791 * @param[in] config 792 * Pointer to user dev config. 793 */ 794 static void 795 mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh, 796 const struct mlx5_dev_config *config) 797 { 798 uint8_t i; 799 struct mlx5_indexed_pool_config cfg; 800 801 for (i = 0; i < MLX5_IPOOL_MAX; ++i) { 802 cfg = mlx5_ipool_cfg[i]; 803 switch (i) { 804 default: 805 break; 806 /* 807 * Set MLX5_IPOOL_MLX5_FLOW ipool size 808 * according to PCI function flow configuration. 809 */ 810 case MLX5_IPOOL_MLX5_FLOW: 811 cfg.size = config->dv_flow_en ? 812 sizeof(struct mlx5_flow_handle) : 813 MLX5_FLOW_HANDLE_VERBS_SIZE; 814 break; 815 } 816 if (config->reclaim_mode) { 817 cfg.release_mem_en = 1; 818 cfg.per_core_cache = 0; 819 } else { 820 cfg.release_mem_en = 0; 821 } 822 sh->ipool[i] = mlx5_ipool_create(&cfg); 823 } 824 } 825 826 827 /** 828 * Release the flow resources' indexed mempool. 829 * 830 * @param[in] sh 831 * Pointer to mlx5_dev_ctx_shared object. 832 */ 833 static void 834 mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh) 835 { 836 uint8_t i; 837 838 for (i = 0; i < MLX5_IPOOL_MAX; ++i) 839 mlx5_ipool_destroy(sh->ipool[i]); 840 for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i) 841 if (sh->mdh_ipools[i]) 842 mlx5_ipool_destroy(sh->mdh_ipools[i]); 843 } 844 845 /* 846 * Check if dynamic flex parser for eCPRI already exists. 847 * 848 * @param dev 849 * Pointer to Ethernet device structure. 850 * 851 * @return 852 * true on exists, false on not. 853 */ 854 bool 855 mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev) 856 { 857 struct mlx5_priv *priv = dev->data->dev_private; 858 struct mlx5_flex_parser_profiles *prf = 859 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 860 861 return !!prf->obj; 862 } 863 864 /* 865 * Allocation of a flex parser for eCPRI. Once created, this parser related 866 * resources will be held until the device is closed. 867 * 868 * @param dev 869 * Pointer to Ethernet device structure. 870 * 871 * @return 872 * 0 on success, a negative errno value otherwise and rte_errno is set. 873 */ 874 int 875 mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev) 876 { 877 struct mlx5_priv *priv = dev->data->dev_private; 878 struct mlx5_flex_parser_profiles *prf = 879 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 880 struct mlx5_devx_graph_node_attr node = { 881 .modify_field_select = 0, 882 }; 883 uint32_t ids[8]; 884 int ret; 885 886 if (!priv->config.hca_attr.parse_graph_flex_node) { 887 DRV_LOG(ERR, "Dynamic flex parser is not supported " 888 "for device %s.", priv->dev_data->name); 889 return -ENOTSUP; 890 } 891 node.header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED; 892 /* 8 bytes now: 4B common header + 4B message body header. */ 893 node.header_length_base_value = 0x8; 894 /* After MAC layer: Ether / VLAN. */ 895 node.in[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_MAC; 896 /* Type of compared condition should be 0xAEFE in the L2 layer. */ 897 node.in[0].compare_condition_value = RTE_ETHER_TYPE_ECPRI; 898 /* Sample #0: type in common header. */ 899 node.sample[0].flow_match_sample_en = 1; 900 /* Fixed offset. */ 901 node.sample[0].flow_match_sample_offset_mode = 0x0; 902 /* Only the 2nd byte will be used. */ 903 node.sample[0].flow_match_sample_field_base_offset = 0x0; 904 /* Sample #1: message payload. */ 905 node.sample[1].flow_match_sample_en = 1; 906 /* Fixed offset. */ 907 node.sample[1].flow_match_sample_offset_mode = 0x0; 908 /* 909 * Only the first two bytes will be used right now, and its offset will 910 * start after the common header that with the length of a DW(u32). 911 */ 912 node.sample[1].flow_match_sample_field_base_offset = sizeof(uint32_t); 913 prf->obj = mlx5_devx_cmd_create_flex_parser(priv->sh->ctx, &node); 914 if (!prf->obj) { 915 DRV_LOG(ERR, "Failed to create flex parser node object."); 916 return (rte_errno == 0) ? -ENODEV : -rte_errno; 917 } 918 prf->num = 2; 919 ret = mlx5_devx_cmd_query_parse_samples(prf->obj, ids, prf->num); 920 if (ret) { 921 DRV_LOG(ERR, "Failed to query sample IDs."); 922 return (rte_errno == 0) ? -ENODEV : -rte_errno; 923 } 924 prf->offset[0] = 0x0; 925 prf->offset[1] = sizeof(uint32_t); 926 prf->ids[0] = ids[0]; 927 prf->ids[1] = ids[1]; 928 return 0; 929 } 930 931 /* 932 * Destroy the flex parser node, including the parser itself, input / output 933 * arcs and DW samples. Resources could be reused then. 934 * 935 * @param dev 936 * Pointer to Ethernet device structure. 937 */ 938 static void 939 mlx5_flex_parser_ecpri_release(struct rte_eth_dev *dev) 940 { 941 struct mlx5_priv *priv = dev->data->dev_private; 942 struct mlx5_flex_parser_profiles *prf = 943 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 944 945 if (prf->obj) 946 mlx5_devx_cmd_destroy(prf->obj); 947 prf->obj = NULL; 948 } 949 950 /* 951 * Allocate Rx and Tx UARs in robust fashion. 952 * This routine handles the following UAR allocation issues: 953 * 954 * - tries to allocate the UAR with the most appropriate memory 955 * mapping type from the ones supported by the host 956 * 957 * - tries to allocate the UAR with non-NULL base address 958 * OFED 5.0.x and Upstream rdma_core before v29 returned the NULL as 959 * UAR base address if UAR was not the first object in the UAR page. 960 * It caused the PMD failure and we should try to get another UAR 961 * till we get the first one with non-NULL base address returned. 962 */ 963 static int 964 mlx5_alloc_rxtx_uars(struct mlx5_dev_ctx_shared *sh, 965 const struct mlx5_dev_config *config) 966 { 967 uint32_t uar_mapping, retry; 968 int err = 0; 969 void *base_addr; 970 971 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 972 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 973 /* Control the mapping type according to the settings. */ 974 uar_mapping = (config->dbnc == MLX5_TXDB_NCACHED) ? 975 MLX5DV_UAR_ALLOC_TYPE_NC : 976 MLX5DV_UAR_ALLOC_TYPE_BF; 977 #else 978 RTE_SET_USED(config); 979 /* 980 * It seems we have no way to control the memory mapping type 981 * for the UAR, the default "Write-Combining" type is supposed. 982 * The UAR initialization on queue creation queries the 983 * actual mapping type done by Verbs/kernel and setups the 984 * PMD datapath accordingly. 985 */ 986 uar_mapping = 0; 987 #endif 988 sh->tx_uar = mlx5_glue->devx_alloc_uar(sh->ctx, uar_mapping); 989 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 990 if (!sh->tx_uar && 991 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 992 if (config->dbnc == MLX5_TXDB_CACHED || 993 config->dbnc == MLX5_TXDB_HEURISTIC) 994 DRV_LOG(WARNING, "Devarg tx_db_nc setting " 995 "is not supported by DevX"); 996 /* 997 * In some environments like virtual machine 998 * the Write Combining mapped might be not supported 999 * and UAR allocation fails. We try "Non-Cached" 1000 * mapping for the case. The tx_burst routines take 1001 * the UAR mapping type into account on UAR setup 1002 * on queue creation. 1003 */ 1004 DRV_LOG(DEBUG, "Failed to allocate Tx DevX UAR (BF)"); 1005 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 1006 sh->tx_uar = mlx5_glue->devx_alloc_uar 1007 (sh->ctx, uar_mapping); 1008 } else if (!sh->tx_uar && 1009 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) { 1010 if (config->dbnc == MLX5_TXDB_NCACHED) 1011 DRV_LOG(WARNING, "Devarg tx_db_nc settings " 1012 "is not supported by DevX"); 1013 /* 1014 * If Verbs/kernel does not support "Non-Cached" 1015 * try the "Write-Combining". 1016 */ 1017 DRV_LOG(DEBUG, "Failed to allocate Tx DevX UAR (NC)"); 1018 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF; 1019 sh->tx_uar = mlx5_glue->devx_alloc_uar 1020 (sh->ctx, uar_mapping); 1021 } 1022 #endif 1023 if (!sh->tx_uar) { 1024 DRV_LOG(ERR, "Failed to allocate Tx DevX UAR (BF/NC)"); 1025 err = ENOMEM; 1026 goto exit; 1027 } 1028 base_addr = mlx5_os_get_devx_uar_base_addr(sh->tx_uar); 1029 if (base_addr) 1030 break; 1031 /* 1032 * The UARs are allocated by rdma_core within the 1033 * IB device context, on context closure all UARs 1034 * will be freed, should be no memory/object leakage. 1035 */ 1036 DRV_LOG(DEBUG, "Retrying to allocate Tx DevX UAR"); 1037 sh->tx_uar = NULL; 1038 } 1039 /* Check whether we finally succeeded with valid UAR allocation. */ 1040 if (!sh->tx_uar) { 1041 DRV_LOG(ERR, "Failed to allocate Tx DevX UAR (NULL base)"); 1042 err = ENOMEM; 1043 goto exit; 1044 } 1045 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 1046 uar_mapping = 0; 1047 sh->devx_rx_uar = mlx5_glue->devx_alloc_uar 1048 (sh->ctx, uar_mapping); 1049 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 1050 if (!sh->devx_rx_uar && 1051 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 1052 /* 1053 * Rx UAR is used to control interrupts only, 1054 * should be no datapath noticeable impact, 1055 * can try "Non-Cached" mapping safely. 1056 */ 1057 DRV_LOG(DEBUG, "Failed to allocate Rx DevX UAR (BF)"); 1058 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 1059 sh->devx_rx_uar = mlx5_glue->devx_alloc_uar 1060 (sh->ctx, uar_mapping); 1061 } 1062 #endif 1063 if (!sh->devx_rx_uar) { 1064 DRV_LOG(ERR, "Failed to allocate Rx DevX UAR (BF/NC)"); 1065 err = ENOMEM; 1066 goto exit; 1067 } 1068 base_addr = mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar); 1069 if (base_addr) 1070 break; 1071 /* 1072 * The UARs are allocated by rdma_core within the 1073 * IB device context, on context closure all UARs 1074 * will be freed, should be no memory/object leakage. 1075 */ 1076 DRV_LOG(DEBUG, "Retrying to allocate Rx DevX UAR"); 1077 sh->devx_rx_uar = NULL; 1078 } 1079 /* Check whether we finally succeeded with valid UAR allocation. */ 1080 if (!sh->devx_rx_uar) { 1081 DRV_LOG(ERR, "Failed to allocate Rx DevX UAR (NULL base)"); 1082 err = ENOMEM; 1083 } 1084 exit: 1085 return err; 1086 } 1087 1088 /** 1089 * Allocate shared device context. If there is multiport device the 1090 * master and representors will share this context, if there is single 1091 * port dedicated device, the context will be used by only given 1092 * port due to unification. 1093 * 1094 * Routine first searches the context for the specified device name, 1095 * if found the shared context assumed and reference counter is incremented. 1096 * If no context found the new one is created and initialized with specified 1097 * device context and parameters. 1098 * 1099 * @param[in] spawn 1100 * Pointer to the device attributes (name, port, etc). 1101 * @param[in] config 1102 * Pointer to device configuration structure. 1103 * 1104 * @return 1105 * Pointer to mlx5_dev_ctx_shared object on success, 1106 * otherwise NULL and rte_errno is set. 1107 */ 1108 struct mlx5_dev_ctx_shared * 1109 mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, 1110 const struct mlx5_dev_config *config) 1111 { 1112 struct mlx5_dev_ctx_shared *sh; 1113 int err = 0; 1114 uint32_t i; 1115 struct mlx5_devx_tis_attr tis_attr = { 0 }; 1116 1117 MLX5_ASSERT(spawn); 1118 /* Secondary process should not create the shared context. */ 1119 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1120 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1121 /* Search for IB context by device name. */ 1122 LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) { 1123 if (!strcmp(sh->ibdev_name, 1124 mlx5_os_get_dev_device_name(spawn->phys_dev))) { 1125 sh->refcnt++; 1126 goto exit; 1127 } 1128 } 1129 /* No device found, we have to create new shared context. */ 1130 MLX5_ASSERT(spawn->max_port); 1131 sh = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 1132 sizeof(struct mlx5_dev_ctx_shared) + 1133 spawn->max_port * 1134 sizeof(struct mlx5_dev_shared_port), 1135 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1136 if (!sh) { 1137 DRV_LOG(ERR, "shared context allocation failure"); 1138 rte_errno = ENOMEM; 1139 goto exit; 1140 } 1141 sh->numa_node = spawn->numa_node; 1142 if (spawn->bond_info) 1143 sh->bond = *spawn->bond_info; 1144 err = mlx5_os_open_device(spawn, config, sh); 1145 if (!sh->ctx) 1146 goto error; 1147 err = mlx5_os_get_dev_attr(sh->ctx, &sh->device_attr); 1148 if (err) { 1149 DRV_LOG(DEBUG, "mlx5_os_get_dev_attr() failed"); 1150 goto error; 1151 } 1152 sh->refcnt = 1; 1153 sh->max_port = spawn->max_port; 1154 sh->reclaim_mode = config->reclaim_mode; 1155 strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx), 1156 sizeof(sh->ibdev_name) - 1); 1157 strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx), 1158 sizeof(sh->ibdev_path) - 1); 1159 /* 1160 * Setting port_id to max unallowed value means 1161 * there is no interrupt subhandler installed for 1162 * the given port index i. 1163 */ 1164 for (i = 0; i < sh->max_port; i++) { 1165 sh->port[i].ih_port_id = RTE_MAX_ETHPORTS; 1166 sh->port[i].devx_ih_port_id = RTE_MAX_ETHPORTS; 1167 } 1168 sh->pd = mlx5_os_alloc_pd(sh->ctx); 1169 if (sh->pd == NULL) { 1170 DRV_LOG(ERR, "PD allocation failure"); 1171 err = ENOMEM; 1172 goto error; 1173 } 1174 if (sh->devx) { 1175 err = mlx5_os_get_pdn(sh->pd, &sh->pdn); 1176 if (err) { 1177 DRV_LOG(ERR, "Fail to extract pdn from PD"); 1178 goto error; 1179 } 1180 sh->td = mlx5_devx_cmd_create_td(sh->ctx); 1181 if (!sh->td) { 1182 DRV_LOG(ERR, "TD allocation failure"); 1183 err = ENOMEM; 1184 goto error; 1185 } 1186 tis_attr.transport_domain = sh->td->id; 1187 sh->tis = mlx5_devx_cmd_create_tis(sh->ctx, &tis_attr); 1188 if (!sh->tis) { 1189 DRV_LOG(ERR, "TIS allocation failure"); 1190 err = ENOMEM; 1191 goto error; 1192 } 1193 err = mlx5_alloc_rxtx_uars(sh, config); 1194 if (err) 1195 goto error; 1196 MLX5_ASSERT(sh->tx_uar); 1197 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->tx_uar)); 1198 1199 MLX5_ASSERT(sh->devx_rx_uar); 1200 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar)); 1201 } 1202 #ifndef RTE_ARCH_64 1203 /* Initialize UAR access locks for 32bit implementations. */ 1204 rte_spinlock_init(&sh->uar_lock_cq); 1205 for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) 1206 rte_spinlock_init(&sh->uar_lock[i]); 1207 #endif 1208 /* 1209 * Once the device is added to the list of memory event 1210 * callback, its global MR cache table cannot be expanded 1211 * on the fly because of deadlock. If it overflows, lookup 1212 * should be done by searching MR list linearly, which is slow. 1213 * 1214 * At this point the device is not added to the memory 1215 * event list yet, context is just being created. 1216 */ 1217 err = mlx5_mr_btree_init(&sh->share_cache.cache, 1218 MLX5_MR_BTREE_CACHE_N * 2, 1219 sh->numa_node); 1220 if (err) { 1221 err = rte_errno; 1222 goto error; 1223 } 1224 mlx5_os_set_reg_mr_cb(&sh->share_cache.reg_mr_cb, 1225 &sh->share_cache.dereg_mr_cb); 1226 mlx5_os_dev_shared_handler_install(sh); 1227 sh->cnt_id_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1228 if (!sh->cnt_id_tbl) { 1229 err = rte_errno; 1230 goto error; 1231 } 1232 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1233 err = mlx5_flow_os_init_workspace_once(); 1234 if (err) 1235 goto error; 1236 } 1237 mlx5_flow_aging_init(sh); 1238 mlx5_flow_counters_mng_init(sh); 1239 mlx5_flow_ipool_create(sh, config); 1240 /* Add device to memory callback list. */ 1241 rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock); 1242 LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list, 1243 sh, mem_event_cb); 1244 rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock); 1245 /* Add context to the global device list. */ 1246 LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next); 1247 rte_spinlock_init(&sh->geneve_tlv_opt_sl); 1248 exit: 1249 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1250 return sh; 1251 error: 1252 pthread_mutex_destroy(&sh->txpp.mutex); 1253 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1254 MLX5_ASSERT(sh); 1255 if (sh->cnt_id_tbl) 1256 mlx5_l3t_destroy(sh->cnt_id_tbl); 1257 if (sh->share_cache.cache.table) 1258 mlx5_mr_btree_free(&sh->share_cache.cache); 1259 if (sh->tis) 1260 claim_zero(mlx5_devx_cmd_destroy(sh->tis)); 1261 if (sh->td) 1262 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1263 if (sh->devx_rx_uar) 1264 mlx5_glue->devx_free_uar(sh->devx_rx_uar); 1265 if (sh->tx_uar) 1266 mlx5_glue->devx_free_uar(sh->tx_uar); 1267 if (sh->pd) 1268 claim_zero(mlx5_os_dealloc_pd(sh->pd)); 1269 if (sh->ctx) 1270 claim_zero(mlx5_glue->close_device(sh->ctx)); 1271 mlx5_free(sh); 1272 MLX5_ASSERT(err > 0); 1273 rte_errno = err; 1274 return NULL; 1275 } 1276 1277 /** 1278 * Free shared IB device context. Decrement counter and if zero free 1279 * all allocated resources and close handles. 1280 * 1281 * @param[in] sh 1282 * Pointer to mlx5_dev_ctx_shared object to free 1283 */ 1284 void 1285 mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) 1286 { 1287 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1288 #ifdef RTE_LIBRTE_MLX5_DEBUG 1289 /* Check the object presence in the list. */ 1290 struct mlx5_dev_ctx_shared *lctx; 1291 1292 LIST_FOREACH(lctx, &mlx5_dev_ctx_list, next) 1293 if (lctx == sh) 1294 break; 1295 MLX5_ASSERT(lctx); 1296 if (lctx != sh) { 1297 DRV_LOG(ERR, "Freeing non-existing shared IB context"); 1298 goto exit; 1299 } 1300 #endif 1301 MLX5_ASSERT(sh); 1302 MLX5_ASSERT(sh->refcnt); 1303 /* Secondary process should not free the shared context. */ 1304 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1305 if (--sh->refcnt) 1306 goto exit; 1307 /* Remove from memory callback device list. */ 1308 rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock); 1309 LIST_REMOVE(sh, mem_event_cb); 1310 rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock); 1311 /* Release created Memory Regions. */ 1312 mlx5_mr_release_cache(&sh->share_cache); 1313 /* Remove context from the global device list. */ 1314 LIST_REMOVE(sh, next); 1315 /* Release flow workspaces objects on the last device. */ 1316 if (LIST_EMPTY(&mlx5_dev_ctx_list)) 1317 mlx5_flow_os_release_workspace(); 1318 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1319 /* 1320 * Ensure there is no async event handler installed. 1321 * Only primary process handles async device events. 1322 **/ 1323 mlx5_flow_counters_mng_close(sh); 1324 if (sh->aso_age_mng) { 1325 mlx5_flow_aso_age_mng_close(sh); 1326 sh->aso_age_mng = NULL; 1327 } 1328 if (sh->mtrmng) 1329 mlx5_aso_flow_mtrs_mng_close(sh); 1330 mlx5_flow_ipool_destroy(sh); 1331 mlx5_os_dev_shared_handler_uninstall(sh); 1332 if (sh->cnt_id_tbl) { 1333 mlx5_l3t_destroy(sh->cnt_id_tbl); 1334 sh->cnt_id_tbl = NULL; 1335 } 1336 if (sh->tx_uar) { 1337 mlx5_glue->devx_free_uar(sh->tx_uar); 1338 sh->tx_uar = NULL; 1339 } 1340 if (sh->pd) 1341 claim_zero(mlx5_os_dealloc_pd(sh->pd)); 1342 if (sh->tis) 1343 claim_zero(mlx5_devx_cmd_destroy(sh->tis)); 1344 if (sh->td) 1345 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1346 if (sh->devx_rx_uar) 1347 mlx5_glue->devx_free_uar(sh->devx_rx_uar); 1348 if (sh->ctx) 1349 claim_zero(mlx5_glue->close_device(sh->ctx)); 1350 MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL); 1351 pthread_mutex_destroy(&sh->txpp.mutex); 1352 mlx5_free(sh); 1353 return; 1354 exit: 1355 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1356 } 1357 1358 /** 1359 * Destroy table hash list. 1360 * 1361 * @param[in] priv 1362 * Pointer to the private device data structure. 1363 */ 1364 void 1365 mlx5_free_table_hash_list(struct mlx5_priv *priv) 1366 { 1367 struct mlx5_dev_ctx_shared *sh = priv->sh; 1368 1369 if (!sh->flow_tbls) 1370 return; 1371 mlx5_hlist_destroy(sh->flow_tbls); 1372 } 1373 1374 /** 1375 * Initialize flow table hash list and create the root tables entry 1376 * for each domain. 1377 * 1378 * @param[in] priv 1379 * Pointer to the private device data structure. 1380 * 1381 * @return 1382 * Zero on success, positive error code otherwise. 1383 */ 1384 int 1385 mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) 1386 { 1387 int err = 0; 1388 /* Tables are only used in DV and DR modes. */ 1389 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 1390 struct mlx5_dev_ctx_shared *sh = priv->sh; 1391 char s[MLX5_NAME_SIZE]; 1392 1393 MLX5_ASSERT(sh); 1394 snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); 1395 sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, 1396 false, true, sh, 1397 flow_dv_tbl_create_cb, 1398 flow_dv_tbl_match_cb, 1399 flow_dv_tbl_remove_cb, 1400 flow_dv_tbl_clone_cb, 1401 flow_dv_tbl_clone_free_cb); 1402 if (!sh->flow_tbls) { 1403 DRV_LOG(ERR, "flow tables with hash creation failed."); 1404 err = ENOMEM; 1405 return err; 1406 } 1407 #ifndef HAVE_MLX5DV_DR 1408 struct rte_flow_error error; 1409 struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; 1410 1411 /* 1412 * In case we have not DR support, the zero tables should be created 1413 * because DV expect to see them even if they cannot be created by 1414 * RDMA-CORE. 1415 */ 1416 if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, 1417 NULL, 0, 1, 0, &error) || 1418 !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, 1419 NULL, 0, 1, 0, &error) || 1420 !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, 1421 NULL, 0, 1, 0, &error)) { 1422 err = ENOMEM; 1423 goto error; 1424 } 1425 return err; 1426 error: 1427 mlx5_free_table_hash_list(priv); 1428 #endif /* HAVE_MLX5DV_DR */ 1429 #endif 1430 return err; 1431 } 1432 1433 /** 1434 * Retrieve integer value from environment variable. 1435 * 1436 * @param[in] name 1437 * Environment variable name. 1438 * 1439 * @return 1440 * Integer value, 0 if the variable is not set. 1441 */ 1442 int 1443 mlx5_getenv_int(const char *name) 1444 { 1445 const char *val = getenv(name); 1446 1447 if (val == NULL) 1448 return 0; 1449 return atoi(val); 1450 } 1451 1452 /** 1453 * DPDK callback to add udp tunnel port 1454 * 1455 * @param[in] dev 1456 * A pointer to eth_dev 1457 * @param[in] udp_tunnel 1458 * A pointer to udp tunnel 1459 * 1460 * @return 1461 * 0 on valid udp ports and tunnels, -ENOTSUP otherwise. 1462 */ 1463 int 1464 mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev __rte_unused, 1465 struct rte_eth_udp_tunnel *udp_tunnel) 1466 { 1467 MLX5_ASSERT(udp_tunnel != NULL); 1468 if (udp_tunnel->prot_type == RTE_TUNNEL_TYPE_VXLAN && 1469 udp_tunnel->udp_port == 4789) 1470 return 0; 1471 if (udp_tunnel->prot_type == RTE_TUNNEL_TYPE_VXLAN_GPE && 1472 udp_tunnel->udp_port == 4790) 1473 return 0; 1474 return -ENOTSUP; 1475 } 1476 1477 /** 1478 * Initialize process private data structure. 1479 * 1480 * @param dev 1481 * Pointer to Ethernet device structure. 1482 * 1483 * @return 1484 * 0 on success, a negative errno value otherwise and rte_errno is set. 1485 */ 1486 int 1487 mlx5_proc_priv_init(struct rte_eth_dev *dev) 1488 { 1489 struct mlx5_priv *priv = dev->data->dev_private; 1490 struct mlx5_proc_priv *ppriv; 1491 size_t ppriv_size; 1492 1493 mlx5_proc_priv_uninit(dev); 1494 /* 1495 * UAR register table follows the process private structure. BlueFlame 1496 * registers for Tx queues are stored in the table. 1497 */ 1498 ppriv_size = 1499 sizeof(struct mlx5_proc_priv) + priv->txqs_n * sizeof(void *); 1500 ppriv = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, ppriv_size, 1501 RTE_CACHE_LINE_SIZE, dev->device->numa_node); 1502 if (!ppriv) { 1503 rte_errno = ENOMEM; 1504 return -rte_errno; 1505 } 1506 ppriv->uar_table_sz = priv->txqs_n; 1507 dev->process_private = ppriv; 1508 return 0; 1509 } 1510 1511 /** 1512 * Un-initialize process private data structure. 1513 * 1514 * @param dev 1515 * Pointer to Ethernet device structure. 1516 */ 1517 void 1518 mlx5_proc_priv_uninit(struct rte_eth_dev *dev) 1519 { 1520 if (!dev->process_private) 1521 return; 1522 mlx5_free(dev->process_private); 1523 dev->process_private = NULL; 1524 } 1525 1526 /** 1527 * DPDK callback to close the device. 1528 * 1529 * Destroy all queues and objects, free memory. 1530 * 1531 * @param dev 1532 * Pointer to Ethernet device structure. 1533 */ 1534 int 1535 mlx5_dev_close(struct rte_eth_dev *dev) 1536 { 1537 struct mlx5_priv *priv = dev->data->dev_private; 1538 unsigned int i; 1539 int ret; 1540 1541 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1542 /* Check if process_private released. */ 1543 if (!dev->process_private) 1544 return 0; 1545 mlx5_tx_uar_uninit_secondary(dev); 1546 mlx5_proc_priv_uninit(dev); 1547 rte_eth_dev_release_port(dev); 1548 return 0; 1549 } 1550 if (!priv->sh) 1551 return 0; 1552 DRV_LOG(DEBUG, "port %u closing device \"%s\"", 1553 dev->data->port_id, 1554 ((priv->sh->ctx != NULL) ? 1555 mlx5_os_get_ctx_device_name(priv->sh->ctx) : "")); 1556 /* 1557 * If default mreg copy action is removed at the stop stage, 1558 * the search will return none and nothing will be done anymore. 1559 */ 1560 mlx5_flow_stop_default(dev); 1561 mlx5_traffic_disable(dev); 1562 /* 1563 * If all the flows are already flushed in the device stop stage, 1564 * then this will return directly without any action. 1565 */ 1566 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true); 1567 mlx5_action_handle_flush(dev); 1568 mlx5_flow_meter_flush(dev, NULL); 1569 /* Prevent crashes when queues are still in use. */ 1570 dev->rx_pkt_burst = removed_rx_burst; 1571 dev->tx_pkt_burst = removed_tx_burst; 1572 rte_wmb(); 1573 /* Disable datapath on secondary process. */ 1574 mlx5_mp_os_req_stop_rxtx(dev); 1575 /* Free the eCPRI flex parser resource. */ 1576 mlx5_flex_parser_ecpri_release(dev); 1577 if (priv->rxqs != NULL) { 1578 /* XXX race condition if mlx5_rx_burst() is still running. */ 1579 rte_delay_us_sleep(1000); 1580 for (i = 0; (i != priv->rxqs_n); ++i) 1581 mlx5_rxq_release(dev, i); 1582 priv->rxqs_n = 0; 1583 priv->rxqs = NULL; 1584 } 1585 if (priv->representor) { 1586 /* Each representor has a dedicated interrupts handler */ 1587 mlx5_free(dev->intr_handle); 1588 dev->intr_handle = NULL; 1589 } 1590 if (priv->txqs != NULL) { 1591 /* XXX race condition if mlx5_tx_burst() is still running. */ 1592 rte_delay_us_sleep(1000); 1593 for (i = 0; (i != priv->txqs_n); ++i) 1594 mlx5_txq_release(dev, i); 1595 priv->txqs_n = 0; 1596 priv->txqs = NULL; 1597 } 1598 mlx5_proc_priv_uninit(dev); 1599 if (priv->q_counters) { 1600 mlx5_devx_cmd_destroy(priv->q_counters); 1601 priv->q_counters = NULL; 1602 } 1603 if (priv->drop_queue.hrxq) 1604 mlx5_drop_action_destroy(dev); 1605 if (priv->mreg_cp_tbl) 1606 mlx5_hlist_destroy(priv->mreg_cp_tbl); 1607 mlx5_mprq_free_mp(dev); 1608 if (priv->sh->ct_mng) 1609 mlx5_flow_aso_ct_mng_close(priv->sh); 1610 mlx5_os_free_shared_dr(priv); 1611 if (priv->rss_conf.rss_key != NULL) 1612 mlx5_free(priv->rss_conf.rss_key); 1613 if (priv->reta_idx != NULL) 1614 mlx5_free(priv->reta_idx); 1615 if (priv->config.vf) 1616 mlx5_os_mac_addr_flush(dev); 1617 if (priv->nl_socket_route >= 0) 1618 close(priv->nl_socket_route); 1619 if (priv->nl_socket_rdma >= 0) 1620 close(priv->nl_socket_rdma); 1621 if (priv->vmwa_context) 1622 mlx5_vlan_vmwa_exit(priv->vmwa_context); 1623 ret = mlx5_hrxq_verify(dev); 1624 if (ret) 1625 DRV_LOG(WARNING, "port %u some hash Rx queue still remain", 1626 dev->data->port_id); 1627 ret = mlx5_ind_table_obj_verify(dev); 1628 if (ret) 1629 DRV_LOG(WARNING, "port %u some indirection table still remain", 1630 dev->data->port_id); 1631 ret = mlx5_rxq_obj_verify(dev); 1632 if (ret) 1633 DRV_LOG(WARNING, "port %u some Rx queue objects still remain", 1634 dev->data->port_id); 1635 ret = mlx5_rxq_verify(dev); 1636 if (ret) 1637 DRV_LOG(WARNING, "port %u some Rx queues still remain", 1638 dev->data->port_id); 1639 ret = mlx5_txq_obj_verify(dev); 1640 if (ret) 1641 DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain", 1642 dev->data->port_id); 1643 ret = mlx5_txq_verify(dev); 1644 if (ret) 1645 DRV_LOG(WARNING, "port %u some Tx queues still remain", 1646 dev->data->port_id); 1647 ret = mlx5_flow_verify(dev); 1648 if (ret) 1649 DRV_LOG(WARNING, "port %u some flows still remain", 1650 dev->data->port_id); 1651 if (priv->hrxqs) 1652 mlx5_list_destroy(priv->hrxqs); 1653 /* 1654 * Free the shared context in last turn, because the cleanup 1655 * routines above may use some shared fields, like 1656 * mlx5_os_mac_addr_flush() uses ibdev_path for retrieveing 1657 * ifindex if Netlink fails. 1658 */ 1659 mlx5_free_shared_dev_ctx(priv->sh); 1660 if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 1661 unsigned int c = 0; 1662 uint16_t port_id; 1663 1664 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1665 struct mlx5_priv *opriv = 1666 rte_eth_devices[port_id].data->dev_private; 1667 1668 if (!opriv || 1669 opriv->domain_id != priv->domain_id || 1670 &rte_eth_devices[port_id] == dev) 1671 continue; 1672 ++c; 1673 break; 1674 } 1675 if (!c) 1676 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 1677 } 1678 memset(priv, 0, sizeof(*priv)); 1679 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 1680 /* 1681 * Reset mac_addrs to NULL such that it is not freed as part of 1682 * rte_eth_dev_release_port(). mac_addrs is part of dev_private so 1683 * it is freed when dev_private is freed. 1684 */ 1685 dev->data->mac_addrs = NULL; 1686 return 0; 1687 } 1688 1689 const struct eth_dev_ops mlx5_dev_ops = { 1690 .dev_configure = mlx5_dev_configure, 1691 .dev_start = mlx5_dev_start, 1692 .dev_stop = mlx5_dev_stop, 1693 .dev_set_link_down = mlx5_set_link_down, 1694 .dev_set_link_up = mlx5_set_link_up, 1695 .dev_close = mlx5_dev_close, 1696 .promiscuous_enable = mlx5_promiscuous_enable, 1697 .promiscuous_disable = mlx5_promiscuous_disable, 1698 .allmulticast_enable = mlx5_allmulticast_enable, 1699 .allmulticast_disable = mlx5_allmulticast_disable, 1700 .link_update = mlx5_link_update, 1701 .stats_get = mlx5_stats_get, 1702 .stats_reset = mlx5_stats_reset, 1703 .xstats_get = mlx5_xstats_get, 1704 .xstats_reset = mlx5_xstats_reset, 1705 .xstats_get_names = mlx5_xstats_get_names, 1706 .fw_version_get = mlx5_fw_version_get, 1707 .dev_infos_get = mlx5_dev_infos_get, 1708 .representor_info_get = mlx5_representor_info_get, 1709 .read_clock = mlx5_txpp_read_clock, 1710 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1711 .vlan_filter_set = mlx5_vlan_filter_set, 1712 .rx_queue_setup = mlx5_rx_queue_setup, 1713 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1714 .tx_queue_setup = mlx5_tx_queue_setup, 1715 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1716 .rx_queue_release = mlx5_rx_queue_release, 1717 .tx_queue_release = mlx5_tx_queue_release, 1718 .rx_queue_start = mlx5_rx_queue_start, 1719 .rx_queue_stop = mlx5_rx_queue_stop, 1720 .tx_queue_start = mlx5_tx_queue_start, 1721 .tx_queue_stop = mlx5_tx_queue_stop, 1722 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1723 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1724 .mac_addr_remove = mlx5_mac_addr_remove, 1725 .mac_addr_add = mlx5_mac_addr_add, 1726 .mac_addr_set = mlx5_mac_addr_set, 1727 .set_mc_addr_list = mlx5_set_mc_addr_list, 1728 .mtu_set = mlx5_dev_set_mtu, 1729 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1730 .vlan_offload_set = mlx5_vlan_offload_set, 1731 .reta_update = mlx5_dev_rss_reta_update, 1732 .reta_query = mlx5_dev_rss_reta_query, 1733 .rss_hash_update = mlx5_rss_hash_update, 1734 .rss_hash_conf_get = mlx5_rss_hash_conf_get, 1735 .flow_ops_get = mlx5_flow_ops_get, 1736 .rxq_info_get = mlx5_rxq_info_get, 1737 .txq_info_get = mlx5_txq_info_get, 1738 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1739 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1740 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1741 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1742 .is_removed = mlx5_is_removed, 1743 .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, 1744 .get_module_info = mlx5_get_module_info, 1745 .get_module_eeprom = mlx5_get_module_eeprom, 1746 .hairpin_cap_get = mlx5_hairpin_cap_get, 1747 .mtr_ops_get = mlx5_flow_meter_ops_get, 1748 .hairpin_bind = mlx5_hairpin_bind, 1749 .hairpin_unbind = mlx5_hairpin_unbind, 1750 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1751 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1752 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1753 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1754 .get_monitor_addr = mlx5_get_monitor_addr, 1755 }; 1756 1757 /* Available operations from secondary process. */ 1758 const struct eth_dev_ops mlx5_dev_sec_ops = { 1759 .stats_get = mlx5_stats_get, 1760 .stats_reset = mlx5_stats_reset, 1761 .xstats_get = mlx5_xstats_get, 1762 .xstats_reset = mlx5_xstats_reset, 1763 .xstats_get_names = mlx5_xstats_get_names, 1764 .fw_version_get = mlx5_fw_version_get, 1765 .dev_infos_get = mlx5_dev_infos_get, 1766 .representor_info_get = mlx5_representor_info_get, 1767 .read_clock = mlx5_txpp_read_clock, 1768 .rx_queue_start = mlx5_rx_queue_start, 1769 .rx_queue_stop = mlx5_rx_queue_stop, 1770 .tx_queue_start = mlx5_tx_queue_start, 1771 .tx_queue_stop = mlx5_tx_queue_stop, 1772 .rxq_info_get = mlx5_rxq_info_get, 1773 .txq_info_get = mlx5_txq_info_get, 1774 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1775 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1776 .get_module_info = mlx5_get_module_info, 1777 .get_module_eeprom = mlx5_get_module_eeprom, 1778 }; 1779 1780 /* Available operations in flow isolated mode. */ 1781 const struct eth_dev_ops mlx5_dev_ops_isolate = { 1782 .dev_configure = mlx5_dev_configure, 1783 .dev_start = mlx5_dev_start, 1784 .dev_stop = mlx5_dev_stop, 1785 .dev_set_link_down = mlx5_set_link_down, 1786 .dev_set_link_up = mlx5_set_link_up, 1787 .dev_close = mlx5_dev_close, 1788 .promiscuous_enable = mlx5_promiscuous_enable, 1789 .promiscuous_disable = mlx5_promiscuous_disable, 1790 .allmulticast_enable = mlx5_allmulticast_enable, 1791 .allmulticast_disable = mlx5_allmulticast_disable, 1792 .link_update = mlx5_link_update, 1793 .stats_get = mlx5_stats_get, 1794 .stats_reset = mlx5_stats_reset, 1795 .xstats_get = mlx5_xstats_get, 1796 .xstats_reset = mlx5_xstats_reset, 1797 .xstats_get_names = mlx5_xstats_get_names, 1798 .fw_version_get = mlx5_fw_version_get, 1799 .dev_infos_get = mlx5_dev_infos_get, 1800 .representor_info_get = mlx5_representor_info_get, 1801 .read_clock = mlx5_txpp_read_clock, 1802 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1803 .vlan_filter_set = mlx5_vlan_filter_set, 1804 .rx_queue_setup = mlx5_rx_queue_setup, 1805 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1806 .tx_queue_setup = mlx5_tx_queue_setup, 1807 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1808 .rx_queue_release = mlx5_rx_queue_release, 1809 .tx_queue_release = mlx5_tx_queue_release, 1810 .rx_queue_start = mlx5_rx_queue_start, 1811 .rx_queue_stop = mlx5_rx_queue_stop, 1812 .tx_queue_start = mlx5_tx_queue_start, 1813 .tx_queue_stop = mlx5_tx_queue_stop, 1814 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1815 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1816 .mac_addr_remove = mlx5_mac_addr_remove, 1817 .mac_addr_add = mlx5_mac_addr_add, 1818 .mac_addr_set = mlx5_mac_addr_set, 1819 .set_mc_addr_list = mlx5_set_mc_addr_list, 1820 .mtu_set = mlx5_dev_set_mtu, 1821 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1822 .vlan_offload_set = mlx5_vlan_offload_set, 1823 .flow_ops_get = mlx5_flow_ops_get, 1824 .rxq_info_get = mlx5_rxq_info_get, 1825 .txq_info_get = mlx5_txq_info_get, 1826 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1827 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1828 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1829 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1830 .is_removed = mlx5_is_removed, 1831 .get_module_info = mlx5_get_module_info, 1832 .get_module_eeprom = mlx5_get_module_eeprom, 1833 .hairpin_cap_get = mlx5_hairpin_cap_get, 1834 .mtr_ops_get = mlx5_flow_meter_ops_get, 1835 .hairpin_bind = mlx5_hairpin_bind, 1836 .hairpin_unbind = mlx5_hairpin_unbind, 1837 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1838 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1839 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1840 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1841 .get_monitor_addr = mlx5_get_monitor_addr, 1842 }; 1843 1844 /** 1845 * Verify and store value for device argument. 1846 * 1847 * @param[in] key 1848 * Key argument to verify. 1849 * @param[in] val 1850 * Value associated with key. 1851 * @param opaque 1852 * User data. 1853 * 1854 * @return 1855 * 0 on success, a negative errno value otherwise and rte_errno is set. 1856 */ 1857 static int 1858 mlx5_args_check(const char *key, const char *val, void *opaque) 1859 { 1860 struct mlx5_dev_config *config = opaque; 1861 unsigned long mod; 1862 signed long tmp; 1863 1864 /* No-op, port representors are processed in mlx5_dev_spawn(). */ 1865 if (!strcmp(MLX5_REPRESENTOR, key)) 1866 return 0; 1867 errno = 0; 1868 tmp = strtol(val, NULL, 0); 1869 if (errno) { 1870 rte_errno = errno; 1871 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); 1872 return -rte_errno; 1873 } 1874 if (tmp < 0 && strcmp(MLX5_TX_PP, key) && strcmp(MLX5_TX_SKEW, key)) { 1875 /* Negative values are acceptable for some keys only. */ 1876 rte_errno = EINVAL; 1877 DRV_LOG(WARNING, "%s: invalid negative value \"%s\"", key, val); 1878 return -rte_errno; 1879 } 1880 mod = tmp >= 0 ? tmp : -tmp; 1881 if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) { 1882 if (tmp > MLX5_CQE_RESP_FORMAT_L34H_STRIDX) { 1883 DRV_LOG(ERR, "invalid CQE compression " 1884 "format parameter"); 1885 rte_errno = EINVAL; 1886 return -rte_errno; 1887 } 1888 config->cqe_comp = !!tmp; 1889 config->cqe_comp_fmt = tmp; 1890 } else if (strcmp(MLX5_RXQ_PKT_PAD_EN, key) == 0) { 1891 config->hw_padding = !!tmp; 1892 } else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) { 1893 config->mprq.enabled = !!tmp; 1894 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) { 1895 config->mprq.stride_num_n = tmp; 1896 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_SIZE, key) == 0) { 1897 config->mprq.stride_size_n = tmp; 1898 } else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) { 1899 config->mprq.max_memcpy_len = tmp; 1900 } else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) { 1901 config->mprq.min_rxqs_num = tmp; 1902 } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) { 1903 DRV_LOG(WARNING, "%s: deprecated parameter," 1904 " converted to txq_inline_max", key); 1905 config->txq_inline_max = tmp; 1906 } else if (strcmp(MLX5_TXQ_INLINE_MAX, key) == 0) { 1907 config->txq_inline_max = tmp; 1908 } else if (strcmp(MLX5_TXQ_INLINE_MIN, key) == 0) { 1909 config->txq_inline_min = tmp; 1910 } else if (strcmp(MLX5_TXQ_INLINE_MPW, key) == 0) { 1911 config->txq_inline_mpw = tmp; 1912 } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) { 1913 config->txqs_inline = tmp; 1914 } else if (strcmp(MLX5_TXQS_MAX_VEC, key) == 0) { 1915 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1916 } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { 1917 config->mps = !!tmp; 1918 } else if (strcmp(MLX5_TX_DB_NC, key) == 0) { 1919 if (tmp != MLX5_TXDB_CACHED && 1920 tmp != MLX5_TXDB_NCACHED && 1921 tmp != MLX5_TXDB_HEURISTIC) { 1922 DRV_LOG(ERR, "invalid Tx doorbell " 1923 "mapping parameter"); 1924 rte_errno = EINVAL; 1925 return -rte_errno; 1926 } 1927 config->dbnc = tmp; 1928 } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { 1929 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1930 } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { 1931 DRV_LOG(WARNING, "%s: deprecated parameter," 1932 " converted to txq_inline_mpw", key); 1933 config->txq_inline_mpw = tmp; 1934 } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) { 1935 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1936 } else if (strcmp(MLX5_TX_PP, key) == 0) { 1937 if (!mod) { 1938 DRV_LOG(ERR, "Zero Tx packet pacing parameter"); 1939 rte_errno = EINVAL; 1940 return -rte_errno; 1941 } 1942 config->tx_pp = tmp; 1943 } else if (strcmp(MLX5_TX_SKEW, key) == 0) { 1944 config->tx_skew = tmp; 1945 } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { 1946 config->rx_vec_en = !!tmp; 1947 } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) { 1948 config->l3_vxlan_en = !!tmp; 1949 } else if (strcmp(MLX5_VF_NL_EN, key) == 0) { 1950 config->vf_nl_en = !!tmp; 1951 } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) { 1952 config->dv_esw_en = !!tmp; 1953 } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) { 1954 config->dv_flow_en = !!tmp; 1955 } else if (strcmp(MLX5_DV_XMETA_EN, key) == 0) { 1956 if (tmp != MLX5_XMETA_MODE_LEGACY && 1957 tmp != MLX5_XMETA_MODE_META16 && 1958 tmp != MLX5_XMETA_MODE_META32 && 1959 tmp != MLX5_XMETA_MODE_MISS_INFO) { 1960 DRV_LOG(ERR, "invalid extensive " 1961 "metadata parameter"); 1962 rte_errno = EINVAL; 1963 return -rte_errno; 1964 } 1965 if (tmp != MLX5_XMETA_MODE_MISS_INFO) 1966 config->dv_xmeta_en = tmp; 1967 else 1968 config->dv_miss_info = 1; 1969 } else if (strcmp(MLX5_LACP_BY_USER, key) == 0) { 1970 config->lacp_by_user = !!tmp; 1971 } else if (strcmp(MLX5_MR_EXT_MEMSEG_EN, key) == 0) { 1972 config->mr_ext_memseg_en = !!tmp; 1973 } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) { 1974 config->max_dump_files_num = tmp; 1975 } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) { 1976 config->lro.timeout = tmp; 1977 } else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) { 1978 DRV_LOG(DEBUG, "class argument is %s.", val); 1979 } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) { 1980 config->log_hp_size = tmp; 1981 } else if (strcmp(MLX5_RECLAIM_MEM, key) == 0) { 1982 if (tmp != MLX5_RCM_NONE && 1983 tmp != MLX5_RCM_LIGHT && 1984 tmp != MLX5_RCM_AGGR) { 1985 DRV_LOG(ERR, "Unrecognize %s: \"%s\"", key, val); 1986 rte_errno = EINVAL; 1987 return -rte_errno; 1988 } 1989 config->reclaim_mode = tmp; 1990 } else if (strcmp(MLX5_SYS_MEM_EN, key) == 0) { 1991 config->sys_mem_en = !!tmp; 1992 } else if (strcmp(MLX5_DECAP_EN, key) == 0) { 1993 config->decap_en = !!tmp; 1994 } else if (strcmp(MLX5_ALLOW_DUPLICATE_PATTERN, key) == 0) { 1995 config->allow_duplicate_pattern = !!tmp; 1996 } else { 1997 DRV_LOG(WARNING, "%s: unknown parameter", key); 1998 rte_errno = EINVAL; 1999 return -rte_errno; 2000 } 2001 return 0; 2002 } 2003 2004 /** 2005 * Parse device parameters. 2006 * 2007 * @param config 2008 * Pointer to device configuration structure. 2009 * @param devargs 2010 * Device arguments structure. 2011 * 2012 * @return 2013 * 0 on success, a negative errno value otherwise and rte_errno is set. 2014 */ 2015 int 2016 mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs) 2017 { 2018 const char **params = (const char *[]){ 2019 MLX5_RXQ_CQE_COMP_EN, 2020 MLX5_RXQ_PKT_PAD_EN, 2021 MLX5_RX_MPRQ_EN, 2022 MLX5_RX_MPRQ_LOG_STRIDE_NUM, 2023 MLX5_RX_MPRQ_LOG_STRIDE_SIZE, 2024 MLX5_RX_MPRQ_MAX_MEMCPY_LEN, 2025 MLX5_RXQS_MIN_MPRQ, 2026 MLX5_TXQ_INLINE, 2027 MLX5_TXQ_INLINE_MIN, 2028 MLX5_TXQ_INLINE_MAX, 2029 MLX5_TXQ_INLINE_MPW, 2030 MLX5_TXQS_MIN_INLINE, 2031 MLX5_TXQS_MAX_VEC, 2032 MLX5_TXQ_MPW_EN, 2033 MLX5_TXQ_MPW_HDR_DSEG_EN, 2034 MLX5_TXQ_MAX_INLINE_LEN, 2035 MLX5_TX_DB_NC, 2036 MLX5_TX_PP, 2037 MLX5_TX_SKEW, 2038 MLX5_TX_VEC_EN, 2039 MLX5_RX_VEC_EN, 2040 MLX5_L3_VXLAN_EN, 2041 MLX5_VF_NL_EN, 2042 MLX5_DV_ESW_EN, 2043 MLX5_DV_FLOW_EN, 2044 MLX5_DV_XMETA_EN, 2045 MLX5_LACP_BY_USER, 2046 MLX5_MR_EXT_MEMSEG_EN, 2047 MLX5_REPRESENTOR, 2048 MLX5_MAX_DUMP_FILES_NUM, 2049 MLX5_LRO_TIMEOUT_USEC, 2050 RTE_DEVARGS_KEY_CLASS, 2051 MLX5_HP_BUF_SIZE, 2052 MLX5_RECLAIM_MEM, 2053 MLX5_SYS_MEM_EN, 2054 MLX5_DECAP_EN, 2055 MLX5_ALLOW_DUPLICATE_PATTERN, 2056 NULL, 2057 }; 2058 struct rte_kvargs *kvlist; 2059 int ret = 0; 2060 int i; 2061 2062 if (devargs == NULL) 2063 return 0; 2064 /* Following UGLY cast is done to pass checkpatch. */ 2065 kvlist = rte_kvargs_parse(devargs->args, params); 2066 if (kvlist == NULL) { 2067 rte_errno = EINVAL; 2068 return -rte_errno; 2069 } 2070 /* Process parameters. */ 2071 for (i = 0; (params[i] != NULL); ++i) { 2072 if (rte_kvargs_count(kvlist, params[i])) { 2073 ret = rte_kvargs_process(kvlist, params[i], 2074 mlx5_args_check, config); 2075 if (ret) { 2076 rte_errno = EINVAL; 2077 rte_kvargs_free(kvlist); 2078 return -rte_errno; 2079 } 2080 } 2081 } 2082 rte_kvargs_free(kvlist); 2083 return 0; 2084 } 2085 2086 /** 2087 * Configures the minimal amount of data to inline into WQE 2088 * while sending packets. 2089 * 2090 * - the txq_inline_min has the maximal priority, if this 2091 * key is specified in devargs 2092 * - if DevX is enabled the inline mode is queried from the 2093 * device (HCA attributes and NIC vport context if needed). 2094 * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4 Lx 2095 * and none (0 bytes) for other NICs 2096 * 2097 * @param spawn 2098 * Verbs device parameters (name, port, switch_info) to spawn. 2099 * @param config 2100 * Device configuration parameters. 2101 */ 2102 void 2103 mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn, 2104 struct mlx5_dev_config *config) 2105 { 2106 if (config->txq_inline_min != MLX5_ARG_UNSET) { 2107 /* Application defines size of inlined data explicitly. */ 2108 if (spawn->pci_dev != NULL) { 2109 switch (spawn->pci_dev->id.device_id) { 2110 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2111 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2112 if (config->txq_inline_min < 2113 (int)MLX5_INLINE_HSIZE_L2) { 2114 DRV_LOG(DEBUG, 2115 "txq_inline_mix aligned to minimal ConnectX-4 required value %d", 2116 (int)MLX5_INLINE_HSIZE_L2); 2117 config->txq_inline_min = 2118 MLX5_INLINE_HSIZE_L2; 2119 } 2120 break; 2121 } 2122 } 2123 goto exit; 2124 } 2125 if (config->hca_attr.eth_net_offloads) { 2126 /* We have DevX enabled, inline mode queried successfully. */ 2127 switch (config->hca_attr.wqe_inline_mode) { 2128 case MLX5_CAP_INLINE_MODE_L2: 2129 /* outer L2 header must be inlined. */ 2130 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2131 goto exit; 2132 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2133 /* No inline data are required by NIC. */ 2134 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2135 config->hw_vlan_insert = 2136 config->hca_attr.wqe_vlan_insert; 2137 DRV_LOG(DEBUG, "Tx VLAN insertion is supported"); 2138 goto exit; 2139 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2140 /* inline mode is defined by NIC vport context. */ 2141 if (!config->hca_attr.eth_virt) 2142 break; 2143 switch (config->hca_attr.vport_inline_mode) { 2144 case MLX5_INLINE_MODE_NONE: 2145 config->txq_inline_min = 2146 MLX5_INLINE_HSIZE_NONE; 2147 goto exit; 2148 case MLX5_INLINE_MODE_L2: 2149 config->txq_inline_min = 2150 MLX5_INLINE_HSIZE_L2; 2151 goto exit; 2152 case MLX5_INLINE_MODE_IP: 2153 config->txq_inline_min = 2154 MLX5_INLINE_HSIZE_L3; 2155 goto exit; 2156 case MLX5_INLINE_MODE_TCP_UDP: 2157 config->txq_inline_min = 2158 MLX5_INLINE_HSIZE_L4; 2159 goto exit; 2160 case MLX5_INLINE_MODE_INNER_L2: 2161 config->txq_inline_min = 2162 MLX5_INLINE_HSIZE_INNER_L2; 2163 goto exit; 2164 case MLX5_INLINE_MODE_INNER_IP: 2165 config->txq_inline_min = 2166 MLX5_INLINE_HSIZE_INNER_L3; 2167 goto exit; 2168 case MLX5_INLINE_MODE_INNER_TCP_UDP: 2169 config->txq_inline_min = 2170 MLX5_INLINE_HSIZE_INNER_L4; 2171 goto exit; 2172 } 2173 } 2174 } 2175 if (spawn->pci_dev == NULL) { 2176 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2177 goto exit; 2178 } 2179 /* 2180 * We get here if we are unable to deduce 2181 * inline data size with DevX. Try PCI ID 2182 * to determine old NICs. 2183 */ 2184 switch (spawn->pci_dev->id.device_id) { 2185 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2186 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2187 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: 2188 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 2189 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2190 config->hw_vlan_insert = 0; 2191 break; 2192 case PCI_DEVICE_ID_MELLANOX_CONNECTX5: 2193 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 2194 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: 2195 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 2196 /* 2197 * These NICs support VLAN insertion from WQE and 2198 * report the wqe_vlan_insert flag. But there is the bug 2199 * and PFC control may be broken, so disable feature. 2200 */ 2201 config->hw_vlan_insert = 0; 2202 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2203 break; 2204 default: 2205 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2206 break; 2207 } 2208 exit: 2209 DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min); 2210 } 2211 2212 /** 2213 * Configures the metadata mask fields in the shared context. 2214 * 2215 * @param [in] dev 2216 * Pointer to Ethernet device. 2217 */ 2218 void 2219 mlx5_set_metadata_mask(struct rte_eth_dev *dev) 2220 { 2221 struct mlx5_priv *priv = dev->data->dev_private; 2222 struct mlx5_dev_ctx_shared *sh = priv->sh; 2223 uint32_t meta, mark, reg_c0; 2224 2225 reg_c0 = ~priv->vport_meta_mask; 2226 switch (priv->config.dv_xmeta_en) { 2227 case MLX5_XMETA_MODE_LEGACY: 2228 meta = UINT32_MAX; 2229 mark = MLX5_FLOW_MARK_MASK; 2230 break; 2231 case MLX5_XMETA_MODE_META16: 2232 meta = reg_c0 >> rte_bsf32(reg_c0); 2233 mark = MLX5_FLOW_MARK_MASK; 2234 break; 2235 case MLX5_XMETA_MODE_META32: 2236 meta = UINT32_MAX; 2237 mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK; 2238 break; 2239 default: 2240 meta = 0; 2241 mark = 0; 2242 MLX5_ASSERT(false); 2243 break; 2244 } 2245 if (sh->dv_mark_mask && sh->dv_mark_mask != mark) 2246 DRV_LOG(WARNING, "metadata MARK mask mismatche %08X:%08X", 2247 sh->dv_mark_mask, mark); 2248 else 2249 sh->dv_mark_mask = mark; 2250 if (sh->dv_meta_mask && sh->dv_meta_mask != meta) 2251 DRV_LOG(WARNING, "metadata META mask mismatche %08X:%08X", 2252 sh->dv_meta_mask, meta); 2253 else 2254 sh->dv_meta_mask = meta; 2255 if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0) 2256 DRV_LOG(WARNING, "metadata reg_c0 mask mismatche %08X:%08X", 2257 sh->dv_meta_mask, reg_c0); 2258 else 2259 sh->dv_regc0_mask = reg_c0; 2260 DRV_LOG(DEBUG, "metadata mode %u", priv->config.dv_xmeta_en); 2261 DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask); 2262 DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask); 2263 DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask); 2264 } 2265 2266 int 2267 rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n) 2268 { 2269 static const char *const dynf_names[] = { 2270 RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, 2271 RTE_MBUF_DYNFLAG_METADATA_NAME, 2272 RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME 2273 }; 2274 unsigned int i; 2275 2276 if (n < RTE_DIM(dynf_names)) 2277 return -ENOMEM; 2278 for (i = 0; i < RTE_DIM(dynf_names); i++) { 2279 if (names[i] == NULL) 2280 return -EINVAL; 2281 strcpy(names[i], dynf_names[i]); 2282 } 2283 return RTE_DIM(dynf_names); 2284 } 2285 2286 /** 2287 * Comparison callback to sort device data. 2288 * 2289 * This is meant to be used with qsort(). 2290 * 2291 * @param a[in] 2292 * Pointer to pointer to first data object. 2293 * @param b[in] 2294 * Pointer to pointer to second data object. 2295 * 2296 * @return 2297 * 0 if both objects are equal, less than 0 if the first argument is less 2298 * than the second, greater than 0 otherwise. 2299 */ 2300 int 2301 mlx5_dev_check_sibling_config(struct mlx5_priv *priv, 2302 struct mlx5_dev_config *config, 2303 struct rte_device *dpdk_dev) 2304 { 2305 struct mlx5_dev_ctx_shared *sh = priv->sh; 2306 struct mlx5_dev_config *sh_conf = NULL; 2307 uint16_t port_id; 2308 2309 MLX5_ASSERT(sh); 2310 /* Nothing to compare for the single/first device. */ 2311 if (sh->refcnt == 1) 2312 return 0; 2313 /* Find the device with shared context. */ 2314 MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) { 2315 struct mlx5_priv *opriv = 2316 rte_eth_devices[port_id].data->dev_private; 2317 2318 if (opriv && opriv != priv && opriv->sh == sh) { 2319 sh_conf = &opriv->config; 2320 break; 2321 } 2322 } 2323 if (!sh_conf) 2324 return 0; 2325 if (sh_conf->dv_flow_en ^ config->dv_flow_en) { 2326 DRV_LOG(ERR, "\"dv_flow_en\" configuration mismatch" 2327 " for shared %s context", sh->ibdev_name); 2328 rte_errno = EINVAL; 2329 return rte_errno; 2330 } 2331 if (sh_conf->dv_xmeta_en ^ config->dv_xmeta_en) { 2332 DRV_LOG(ERR, "\"dv_xmeta_en\" configuration mismatch" 2333 " for shared %s context", sh->ibdev_name); 2334 rte_errno = EINVAL; 2335 return rte_errno; 2336 } 2337 return 0; 2338 } 2339 2340 /** 2341 * Look for the ethernet device belonging to mlx5 driver. 2342 * 2343 * @param[in] port_id 2344 * port_id to start looking for device. 2345 * @param[in] odev 2346 * Pointer to the hint device. When device is being probed 2347 * the its siblings (master and preceding representors might 2348 * not have assigned driver yet (because the mlx5_os_pci_probe() 2349 * is not completed yet, for this case match on hint 2350 * device may be used to detect sibling device. 2351 * 2352 * @return 2353 * port_id of found device, RTE_MAX_ETHPORT if not found. 2354 */ 2355 uint16_t 2356 mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev) 2357 { 2358 while (port_id < RTE_MAX_ETHPORTS) { 2359 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 2360 2361 if (dev->state != RTE_ETH_DEV_UNUSED && 2362 dev->device && 2363 (dev->device == odev || 2364 (dev->device->driver && 2365 dev->device->driver->name && 2366 ((strcmp(dev->device->driver->name, 2367 MLX5_PCI_DRIVER_NAME) == 0) || 2368 (strcmp(dev->device->driver->name, 2369 MLX5_AUXILIARY_DRIVER_NAME) == 0))))) 2370 break; 2371 port_id++; 2372 } 2373 if (port_id >= RTE_MAX_ETHPORTS) 2374 return RTE_MAX_ETHPORTS; 2375 return port_id; 2376 } 2377 2378 /** 2379 * Callback to remove a device. 2380 * 2381 * This function removes all Ethernet devices belong to a given device. 2382 * 2383 * @param[in] dev 2384 * Pointer to the generic device. 2385 * 2386 * @return 2387 * 0 on success, the function cannot fail. 2388 */ 2389 int 2390 mlx5_net_remove(struct rte_device *dev) 2391 { 2392 uint16_t port_id; 2393 int ret = 0; 2394 2395 RTE_ETH_FOREACH_DEV_OF(port_id, dev) { 2396 /* 2397 * mlx5_dev_close() is not registered to secondary process, 2398 * call the close function explicitly for secondary process. 2399 */ 2400 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 2401 ret |= mlx5_dev_close(&rte_eth_devices[port_id]); 2402 else 2403 ret |= rte_eth_dev_close(port_id); 2404 } 2405 return ret == 0 ? 0 : -EIO; 2406 } 2407 2408 static const struct rte_pci_id mlx5_pci_id_map[] = { 2409 { 2410 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2411 PCI_DEVICE_ID_MELLANOX_CONNECTX4) 2412 }, 2413 { 2414 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2415 PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) 2416 }, 2417 { 2418 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2419 PCI_DEVICE_ID_MELLANOX_CONNECTX4LX) 2420 }, 2421 { 2422 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2423 PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) 2424 }, 2425 { 2426 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2427 PCI_DEVICE_ID_MELLANOX_CONNECTX5) 2428 }, 2429 { 2430 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2431 PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) 2432 }, 2433 { 2434 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2435 PCI_DEVICE_ID_MELLANOX_CONNECTX5EX) 2436 }, 2437 { 2438 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2439 PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF) 2440 }, 2441 { 2442 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2443 PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) 2444 }, 2445 { 2446 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2447 PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF) 2448 }, 2449 { 2450 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2451 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 2452 }, 2453 { 2454 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2455 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF) 2456 }, 2457 { 2458 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2459 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 2460 }, 2461 { 2462 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2463 PCI_DEVICE_ID_MELLANOX_CONNECTXVF) 2464 }, 2465 { 2466 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2467 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF) 2468 }, 2469 { 2470 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2471 PCI_DEVICE_ID_MELLANOX_CONNECTX6LX) 2472 }, 2473 { 2474 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2475 PCI_DEVICE_ID_MELLANOX_CONNECTX7) 2476 }, 2477 { 2478 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2479 PCI_DEVICE_ID_MELLANOX_CONNECTX7BF) 2480 }, 2481 { 2482 .vendor_id = 0 2483 } 2484 }; 2485 2486 static struct mlx5_class_driver mlx5_net_driver = { 2487 .drv_class = MLX5_CLASS_ETH, 2488 .name = RTE_STR(MLX5_ETH_DRIVER_NAME), 2489 .id_table = mlx5_pci_id_map, 2490 .probe = mlx5_os_net_probe, 2491 .remove = mlx5_net_remove, 2492 .dma_map = mlx5_net_dma_map, 2493 .dma_unmap = mlx5_net_dma_unmap, 2494 .probe_again = 1, 2495 .intr_lsc = 1, 2496 .intr_rmv = 1, 2497 }; 2498 2499 /* Initialize driver log type. */ 2500 RTE_LOG_REGISTER_DEFAULT(mlx5_logtype, NOTICE) 2501 2502 /** 2503 * Driver initialization routine. 2504 */ 2505 RTE_INIT(rte_mlx5_pmd_init) 2506 { 2507 pthread_mutex_init(&mlx5_dev_ctx_list_mutex, NULL); 2508 mlx5_common_init(); 2509 /* Build the static tables for Verbs conversion. */ 2510 mlx5_set_ptype_table(); 2511 mlx5_set_cksum_table(); 2512 mlx5_set_swp_types_table(); 2513 if (mlx5_glue) 2514 mlx5_class_driver_register(&mlx5_net_driver); 2515 } 2516 2517 RTE_PMD_EXPORT_NAME(MLX5_ETH_DRIVER_NAME, __COUNTER__); 2518 RTE_PMD_REGISTER_PCI_TABLE(MLX5_ETH_DRIVER_NAME, mlx5_pci_id_map); 2519 RTE_PMD_REGISTER_KMOD_DEP(MLX5_ETH_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 2520