1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <stddef.h> 7 #include <unistd.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <errno.h> 12 13 #include <rte_malloc.h> 14 #include <ethdev_driver.h> 15 #include <rte_pci.h> 16 #include <rte_bus_pci.h> 17 #include <rte_common.h> 18 #include <rte_kvargs.h> 19 #include <rte_rwlock.h> 20 #include <rte_spinlock.h> 21 #include <rte_string_fns.h> 22 #include <rte_eal_paging.h> 23 #include <rte_alarm.h> 24 #include <rte_cycles.h> 25 26 #include <mlx5_glue.h> 27 #include <mlx5_devx_cmds.h> 28 #include <mlx5_common.h> 29 #include <mlx5_common_os.h> 30 #include <mlx5_common_mp.h> 31 #include <mlx5_malloc.h> 32 33 #include "mlx5_defs.h" 34 #include "mlx5.h" 35 #include "mlx5_utils.h" 36 #include "mlx5_rxtx.h" 37 #include "mlx5_rx.h" 38 #include "mlx5_tx.h" 39 #include "mlx5_autoconf.h" 40 #include "mlx5_flow.h" 41 #include "mlx5_flow_os.h" 42 #include "rte_pmd_mlx5.h" 43 44 #define MLX5_ETH_DRIVER_NAME mlx5_eth 45 46 /* Driver type key for new device global syntax. */ 47 #define MLX5_DRIVER_KEY "driver" 48 49 /* Device parameter to enable RX completion queue compression. */ 50 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en" 51 52 /* Device parameter to enable padding Rx packet to cacheline size. */ 53 #define MLX5_RXQ_PKT_PAD_EN "rxq_pkt_pad_en" 54 55 /* Device parameter to enable Multi-Packet Rx queue. */ 56 #define MLX5_RX_MPRQ_EN "mprq_en" 57 58 /* Device parameter to configure log 2 of the number of strides for MPRQ. */ 59 #define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num" 60 61 /* Device parameter to configure log 2 of the stride size for MPRQ. */ 62 #define MLX5_RX_MPRQ_LOG_STRIDE_SIZE "mprq_log_stride_size" 63 64 /* Device parameter to limit the size of memcpy'd packet for MPRQ. */ 65 #define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len" 66 67 /* Device parameter to set the minimum number of Rx queues to enable MPRQ. */ 68 #define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq" 69 70 /* Device parameter to configure inline send. Deprecated, ignored.*/ 71 #define MLX5_TXQ_INLINE "txq_inline" 72 73 /* Device parameter to limit packet size to inline with ordinary SEND. */ 74 #define MLX5_TXQ_INLINE_MAX "txq_inline_max" 75 76 /* Device parameter to configure minimal data size to inline. */ 77 #define MLX5_TXQ_INLINE_MIN "txq_inline_min" 78 79 /* Device parameter to limit packet size to inline with Enhanced MPW. */ 80 #define MLX5_TXQ_INLINE_MPW "txq_inline_mpw" 81 82 /* 83 * Device parameter to configure the number of TX queues threshold for 84 * enabling inline send. 85 */ 86 #define MLX5_TXQS_MIN_INLINE "txqs_min_inline" 87 88 /* 89 * Device parameter to configure the number of TX queues threshold for 90 * enabling vectorized Tx, deprecated, ignored (no vectorized Tx routines). 91 */ 92 #define MLX5_TXQS_MAX_VEC "txqs_max_vec" 93 94 /* Device parameter to enable multi-packet send WQEs. */ 95 #define MLX5_TXQ_MPW_EN "txq_mpw_en" 96 97 /* 98 * Device parameter to force doorbell register mapping 99 * to non-cahed region eliminating the extra write memory barrier. 100 */ 101 #define MLX5_TX_DB_NC "tx_db_nc" 102 103 /* 104 * Device parameter to include 2 dsegs in the title WQEBB. 105 * Deprecated, ignored. 106 */ 107 #define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en" 108 109 /* 110 * Device parameter to limit the size of inlining packet. 111 * Deprecated, ignored. 112 */ 113 #define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len" 114 115 /* 116 * Device parameter to enable Tx scheduling on timestamps 117 * and specify the packet pacing granularity in nanoseconds. 118 */ 119 #define MLX5_TX_PP "tx_pp" 120 121 /* 122 * Device parameter to specify skew in nanoseconds on Tx datapath, 123 * it represents the time between SQ start WQE processing and 124 * appearing actual packet data on the wire. 125 */ 126 #define MLX5_TX_SKEW "tx_skew" 127 128 /* 129 * Device parameter to enable hardware Tx vector. 130 * Deprecated, ignored (no vectorized Tx routines anymore). 131 */ 132 #define MLX5_TX_VEC_EN "tx_vec_en" 133 134 /* Device parameter to enable hardware Rx vector. */ 135 #define MLX5_RX_VEC_EN "rx_vec_en" 136 137 /* Allow L3 VXLAN flow creation. */ 138 #define MLX5_L3_VXLAN_EN "l3_vxlan_en" 139 140 /* Activate DV E-Switch flow steering. */ 141 #define MLX5_DV_ESW_EN "dv_esw_en" 142 143 /* Activate DV flow steering. */ 144 #define MLX5_DV_FLOW_EN "dv_flow_en" 145 146 /* Enable extensive flow metadata support. */ 147 #define MLX5_DV_XMETA_EN "dv_xmeta_en" 148 149 /* Device parameter to let the user manage the lacp traffic of bonded device */ 150 #define MLX5_LACP_BY_USER "lacp_by_user" 151 152 /* Activate Netlink support in VF mode. */ 153 #define MLX5_VF_NL_EN "vf_nl_en" 154 155 /* Enable extending memsegs when creating a MR. */ 156 #define MLX5_MR_EXT_MEMSEG_EN "mr_ext_memseg_en" 157 158 /* Select port representors to instantiate. */ 159 #define MLX5_REPRESENTOR "representor" 160 161 /* Device parameter to configure the maximum number of dump files per queue. */ 162 #define MLX5_MAX_DUMP_FILES_NUM "max_dump_files_num" 163 164 /* Configure timeout of LRO session (in microseconds). */ 165 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec" 166 167 /* 168 * Device parameter to configure the total data buffer size for a single 169 * hairpin queue (logarithm value). 170 */ 171 #define MLX5_HP_BUF_SIZE "hp_buf_log_sz" 172 173 /* Flow memory reclaim mode. */ 174 #define MLX5_RECLAIM_MEM "reclaim_mem_mode" 175 176 /* The default memory allocator used in PMD. */ 177 #define MLX5_SYS_MEM_EN "sys_mem_en" 178 /* Decap will be used or not. */ 179 #define MLX5_DECAP_EN "decap_en" 180 181 /* Device parameter to configure allow or prevent duplicate rules pattern. */ 182 #define MLX5_ALLOW_DUPLICATE_PATTERN "allow_duplicate_pattern" 183 184 /* Device parameter to configure implicit registration of mempool memory. */ 185 #define MLX5_MR_MEMPOOL_REG_EN "mr_mempool_reg_en" 186 187 /* Device parameter to configure the delay drop when creating Rxqs. */ 188 #define MLX5_DELAY_DROP "delay_drop" 189 190 /* Shared memory between primary and secondary processes. */ 191 struct mlx5_shared_data *mlx5_shared_data; 192 193 /** Driver-specific log messages type. */ 194 int mlx5_logtype; 195 196 static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = 197 LIST_HEAD_INITIALIZER(); 198 static pthread_mutex_t mlx5_dev_ctx_list_mutex; 199 static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { 200 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 201 [MLX5_IPOOL_DECAP_ENCAP] = { 202 .size = sizeof(struct mlx5_flow_dv_encap_decap_resource), 203 .trunk_size = 64, 204 .grow_trunk = 3, 205 .grow_shift = 2, 206 .need_lock = 1, 207 .release_mem_en = 1, 208 .malloc = mlx5_malloc, 209 .free = mlx5_free, 210 .type = "mlx5_encap_decap_ipool", 211 }, 212 [MLX5_IPOOL_PUSH_VLAN] = { 213 .size = sizeof(struct mlx5_flow_dv_push_vlan_action_resource), 214 .trunk_size = 64, 215 .grow_trunk = 3, 216 .grow_shift = 2, 217 .need_lock = 1, 218 .release_mem_en = 1, 219 .malloc = mlx5_malloc, 220 .free = mlx5_free, 221 .type = "mlx5_push_vlan_ipool", 222 }, 223 [MLX5_IPOOL_TAG] = { 224 .size = sizeof(struct mlx5_flow_dv_tag_resource), 225 .trunk_size = 64, 226 .grow_trunk = 3, 227 .grow_shift = 2, 228 .need_lock = 1, 229 .release_mem_en = 0, 230 .per_core_cache = (1 << 16), 231 .malloc = mlx5_malloc, 232 .free = mlx5_free, 233 .type = "mlx5_tag_ipool", 234 }, 235 [MLX5_IPOOL_PORT_ID] = { 236 .size = sizeof(struct mlx5_flow_dv_port_id_action_resource), 237 .trunk_size = 64, 238 .grow_trunk = 3, 239 .grow_shift = 2, 240 .need_lock = 1, 241 .release_mem_en = 1, 242 .malloc = mlx5_malloc, 243 .free = mlx5_free, 244 .type = "mlx5_port_id_ipool", 245 }, 246 [MLX5_IPOOL_JUMP] = { 247 .size = sizeof(struct mlx5_flow_tbl_data_entry), 248 .trunk_size = 64, 249 .grow_trunk = 3, 250 .grow_shift = 2, 251 .need_lock = 1, 252 .release_mem_en = 1, 253 .malloc = mlx5_malloc, 254 .free = mlx5_free, 255 .type = "mlx5_jump_ipool", 256 }, 257 [MLX5_IPOOL_SAMPLE] = { 258 .size = sizeof(struct mlx5_flow_dv_sample_resource), 259 .trunk_size = 64, 260 .grow_trunk = 3, 261 .grow_shift = 2, 262 .need_lock = 1, 263 .release_mem_en = 1, 264 .malloc = mlx5_malloc, 265 .free = mlx5_free, 266 .type = "mlx5_sample_ipool", 267 }, 268 [MLX5_IPOOL_DEST_ARRAY] = { 269 .size = sizeof(struct mlx5_flow_dv_dest_array_resource), 270 .trunk_size = 64, 271 .grow_trunk = 3, 272 .grow_shift = 2, 273 .need_lock = 1, 274 .release_mem_en = 1, 275 .malloc = mlx5_malloc, 276 .free = mlx5_free, 277 .type = "mlx5_dest_array_ipool", 278 }, 279 [MLX5_IPOOL_TUNNEL_ID] = { 280 .size = sizeof(struct mlx5_flow_tunnel), 281 .trunk_size = MLX5_MAX_TUNNELS, 282 .need_lock = 1, 283 .release_mem_en = 1, 284 .type = "mlx5_tunnel_offload", 285 }, 286 [MLX5_IPOOL_TNL_TBL_ID] = { 287 .size = 0, 288 .need_lock = 1, 289 .type = "mlx5_flow_tnl_tbl_ipool", 290 }, 291 #endif 292 [MLX5_IPOOL_MTR] = { 293 /** 294 * The ipool index should grow continually from small to big, 295 * for meter idx, so not set grow_trunk to avoid meter index 296 * not jump continually. 297 */ 298 .size = sizeof(struct mlx5_legacy_flow_meter), 299 .trunk_size = 64, 300 .need_lock = 1, 301 .release_mem_en = 1, 302 .malloc = mlx5_malloc, 303 .free = mlx5_free, 304 .type = "mlx5_meter_ipool", 305 }, 306 [MLX5_IPOOL_MCP] = { 307 .size = sizeof(struct mlx5_flow_mreg_copy_resource), 308 .trunk_size = 64, 309 .grow_trunk = 3, 310 .grow_shift = 2, 311 .need_lock = 1, 312 .release_mem_en = 1, 313 .malloc = mlx5_malloc, 314 .free = mlx5_free, 315 .type = "mlx5_mcp_ipool", 316 }, 317 [MLX5_IPOOL_HRXQ] = { 318 .size = (sizeof(struct mlx5_hrxq) + MLX5_RSS_HASH_KEY_LEN), 319 .trunk_size = 64, 320 .grow_trunk = 3, 321 .grow_shift = 2, 322 .need_lock = 1, 323 .release_mem_en = 1, 324 .malloc = mlx5_malloc, 325 .free = mlx5_free, 326 .type = "mlx5_hrxq_ipool", 327 }, 328 [MLX5_IPOOL_MLX5_FLOW] = { 329 /* 330 * MLX5_IPOOL_MLX5_FLOW size varies for DV and VERBS flows. 331 * It set in run time according to PCI function configuration. 332 */ 333 .size = 0, 334 .trunk_size = 64, 335 .grow_trunk = 3, 336 .grow_shift = 2, 337 .need_lock = 1, 338 .release_mem_en = 0, 339 .per_core_cache = 1 << 19, 340 .malloc = mlx5_malloc, 341 .free = mlx5_free, 342 .type = "mlx5_flow_handle_ipool", 343 }, 344 [MLX5_IPOOL_RTE_FLOW] = { 345 .size = sizeof(struct rte_flow), 346 .trunk_size = 4096, 347 .need_lock = 1, 348 .release_mem_en = 1, 349 .malloc = mlx5_malloc, 350 .free = mlx5_free, 351 .type = "rte_flow_ipool", 352 }, 353 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID] = { 354 .size = 0, 355 .need_lock = 1, 356 .type = "mlx5_flow_rss_id_ipool", 357 }, 358 [MLX5_IPOOL_RSS_SHARED_ACTIONS] = { 359 .size = sizeof(struct mlx5_shared_action_rss), 360 .trunk_size = 64, 361 .grow_trunk = 3, 362 .grow_shift = 2, 363 .need_lock = 1, 364 .release_mem_en = 1, 365 .malloc = mlx5_malloc, 366 .free = mlx5_free, 367 .type = "mlx5_shared_action_rss", 368 }, 369 [MLX5_IPOOL_MTR_POLICY] = { 370 /** 371 * The ipool index should grow continually from small to big, 372 * for policy idx, so not set grow_trunk to avoid policy index 373 * not jump continually. 374 */ 375 .size = sizeof(struct mlx5_flow_meter_sub_policy), 376 .trunk_size = 64, 377 .need_lock = 1, 378 .release_mem_en = 1, 379 .malloc = mlx5_malloc, 380 .free = mlx5_free, 381 .type = "mlx5_meter_policy_ipool", 382 }, 383 }; 384 385 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512 386 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16 387 388 #define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024 389 390 /** 391 * Decide whether representor ID is a HPF(host PF) port on BF2. 392 * 393 * @param dev 394 * Pointer to Ethernet device structure. 395 * 396 * @return 397 * Non-zero if HPF, otherwise 0. 398 */ 399 bool 400 mlx5_is_hpf(struct rte_eth_dev *dev) 401 { 402 struct mlx5_priv *priv = dev->data->dev_private; 403 uint16_t repr = MLX5_REPRESENTOR_REPR(priv->representor_id); 404 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 405 406 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_VF && 407 MLX5_REPRESENTOR_REPR(-1) == repr; 408 } 409 410 /** 411 * Decide whether representor ID is a SF port representor. 412 * 413 * @param dev 414 * Pointer to Ethernet device structure. 415 * 416 * @return 417 * Non-zero if HPF, otherwise 0. 418 */ 419 bool 420 mlx5_is_sf_repr(struct rte_eth_dev *dev) 421 { 422 struct mlx5_priv *priv = dev->data->dev_private; 423 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 424 425 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_SF; 426 } 427 428 /** 429 * Initialize the ASO aging management structure. 430 * 431 * @param[in] sh 432 * Pointer to mlx5_dev_ctx_shared object to free 433 * 434 * @return 435 * 0 on success, a negative errno value otherwise and rte_errno is set. 436 */ 437 int 438 mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh) 439 { 440 int err; 441 442 if (sh->aso_age_mng) 443 return 0; 444 sh->aso_age_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->aso_age_mng), 445 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 446 if (!sh->aso_age_mng) { 447 DRV_LOG(ERR, "aso_age_mng allocation was failed."); 448 rte_errno = ENOMEM; 449 return -ENOMEM; 450 } 451 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_FLOW_HIT); 452 if (err) { 453 mlx5_free(sh->aso_age_mng); 454 return -1; 455 } 456 rte_rwlock_init(&sh->aso_age_mng->resize_rwl); 457 rte_spinlock_init(&sh->aso_age_mng->free_sl); 458 LIST_INIT(&sh->aso_age_mng->free); 459 return 0; 460 } 461 462 /** 463 * Close and release all the resources of the ASO aging management structure. 464 * 465 * @param[in] sh 466 * Pointer to mlx5_dev_ctx_shared object to free. 467 */ 468 static void 469 mlx5_flow_aso_age_mng_close(struct mlx5_dev_ctx_shared *sh) 470 { 471 int i, j; 472 473 mlx5_aso_flow_hit_queue_poll_stop(sh); 474 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_FLOW_HIT); 475 if (sh->aso_age_mng->pools) { 476 struct mlx5_aso_age_pool *pool; 477 478 for (i = 0; i < sh->aso_age_mng->next; ++i) { 479 pool = sh->aso_age_mng->pools[i]; 480 claim_zero(mlx5_devx_cmd_destroy 481 (pool->flow_hit_aso_obj)); 482 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) 483 if (pool->actions[j].dr_action) 484 claim_zero 485 (mlx5_flow_os_destroy_flow_action 486 (pool->actions[j].dr_action)); 487 mlx5_free(pool); 488 } 489 mlx5_free(sh->aso_age_mng->pools); 490 } 491 mlx5_free(sh->aso_age_mng); 492 } 493 494 /** 495 * Initialize the shared aging list information per port. 496 * 497 * @param[in] sh 498 * Pointer to mlx5_dev_ctx_shared object. 499 */ 500 static void 501 mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh) 502 { 503 uint32_t i; 504 struct mlx5_age_info *age_info; 505 506 for (i = 0; i < sh->max_port; i++) { 507 age_info = &sh->port[i].age_info; 508 age_info->flags = 0; 509 TAILQ_INIT(&age_info->aged_counters); 510 LIST_INIT(&age_info->aged_aso); 511 rte_spinlock_init(&age_info->aged_sl); 512 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 513 } 514 } 515 516 /** 517 * Initialize the counters management structure. 518 * 519 * @param[in] sh 520 * Pointer to mlx5_dev_ctx_shared object to free 521 */ 522 static void 523 mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh) 524 { 525 struct mlx5_hca_attr *attr = &sh->cdev->config.hca_attr; 526 int i; 527 528 memset(&sh->cmng, 0, sizeof(sh->cmng)); 529 TAILQ_INIT(&sh->cmng.flow_counters); 530 sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET; 531 sh->cmng.max_id = -1; 532 sh->cmng.last_pool_idx = POOL_IDX_INVALID; 533 rte_spinlock_init(&sh->cmng.pool_update_sl); 534 for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) { 535 TAILQ_INIT(&sh->cmng.counters[i]); 536 rte_spinlock_init(&sh->cmng.csl[i]); 537 } 538 if (sh->devx && !haswell_broadwell_cpu) { 539 sh->cmng.relaxed_ordering_write = attr->relaxed_ordering_write; 540 sh->cmng.relaxed_ordering_read = attr->relaxed_ordering_read; 541 } 542 } 543 544 /** 545 * Destroy all the resources allocated for a counter memory management. 546 * 547 * @param[in] mng 548 * Pointer to the memory management structure. 549 */ 550 static void 551 mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng) 552 { 553 uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data; 554 555 LIST_REMOVE(mng, next); 556 claim_zero(mlx5_devx_cmd_destroy(mng->dm)); 557 claim_zero(mlx5_os_umem_dereg(mng->umem)); 558 mlx5_free(mem); 559 } 560 561 /** 562 * Close and release all the resources of the counters management. 563 * 564 * @param[in] sh 565 * Pointer to mlx5_dev_ctx_shared object to free. 566 */ 567 static void 568 mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) 569 { 570 struct mlx5_counter_stats_mem_mng *mng; 571 int i, j; 572 int retries = 1024; 573 574 rte_errno = 0; 575 while (--retries) { 576 rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh); 577 if (rte_errno != EINPROGRESS) 578 break; 579 rte_pause(); 580 } 581 582 if (sh->cmng.pools) { 583 struct mlx5_flow_counter_pool *pool; 584 uint16_t n_valid = sh->cmng.n_valid; 585 bool fallback = sh->cmng.counter_fallback; 586 587 for (i = 0; i < n_valid; ++i) { 588 pool = sh->cmng.pools[i]; 589 if (!fallback && pool->min_dcs) 590 claim_zero(mlx5_devx_cmd_destroy 591 (pool->min_dcs)); 592 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) { 593 struct mlx5_flow_counter *cnt = 594 MLX5_POOL_GET_CNT(pool, j); 595 596 if (cnt->action) 597 claim_zero 598 (mlx5_flow_os_destroy_flow_action 599 (cnt->action)); 600 if (fallback && MLX5_POOL_GET_CNT 601 (pool, j)->dcs_when_free) 602 claim_zero(mlx5_devx_cmd_destroy 603 (cnt->dcs_when_free)); 604 } 605 mlx5_free(pool); 606 } 607 mlx5_free(sh->cmng.pools); 608 } 609 mng = LIST_FIRST(&sh->cmng.mem_mngs); 610 while (mng) { 611 mlx5_flow_destroy_counter_stat_mem_mng(mng); 612 mng = LIST_FIRST(&sh->cmng.mem_mngs); 613 } 614 memset(&sh->cmng, 0, sizeof(sh->cmng)); 615 } 616 617 /** 618 * Initialize the aso flow meters management structure. 619 * 620 * @param[in] sh 621 * Pointer to mlx5_dev_ctx_shared object to free 622 */ 623 int 624 mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh) 625 { 626 if (!sh->mtrmng) { 627 sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, 628 sizeof(*sh->mtrmng), 629 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 630 if (!sh->mtrmng) { 631 DRV_LOG(ERR, 632 "meter management allocation was failed."); 633 rte_errno = ENOMEM; 634 return -ENOMEM; 635 } 636 if (sh->meter_aso_en) { 637 rte_spinlock_init(&sh->mtrmng->pools_mng.mtrsl); 638 rte_rwlock_init(&sh->mtrmng->pools_mng.resize_mtrwl); 639 LIST_INIT(&sh->mtrmng->pools_mng.meters); 640 } 641 sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 642 } 643 return 0; 644 } 645 646 /** 647 * Close and release all the resources of 648 * the ASO flow meter management structure. 649 * 650 * @param[in] sh 651 * Pointer to mlx5_dev_ctx_shared object to free. 652 */ 653 static void 654 mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) 655 { 656 struct mlx5_aso_mtr_pool *mtr_pool; 657 struct mlx5_flow_mtr_mng *mtrmng = sh->mtrmng; 658 uint32_t idx; 659 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 660 struct mlx5_aso_mtr *aso_mtr; 661 int i; 662 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 663 664 if (sh->meter_aso_en) { 665 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); 666 idx = mtrmng->pools_mng.n_valid; 667 while (idx--) { 668 mtr_pool = mtrmng->pools_mng.pools[idx]; 669 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 670 for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { 671 aso_mtr = &mtr_pool->mtrs[i]; 672 if (aso_mtr->fm.meter_action) 673 claim_zero 674 (mlx5_glue->destroy_flow_action 675 (aso_mtr->fm.meter_action)); 676 } 677 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 678 claim_zero(mlx5_devx_cmd_destroy 679 (mtr_pool->devx_obj)); 680 mtrmng->pools_mng.n_valid--; 681 mlx5_free(mtr_pool); 682 } 683 mlx5_free(sh->mtrmng->pools_mng.pools); 684 } 685 mlx5_free(sh->mtrmng); 686 sh->mtrmng = NULL; 687 } 688 689 /* Send FLOW_AGED event if needed. */ 690 void 691 mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) 692 { 693 struct mlx5_age_info *age_info; 694 uint32_t i; 695 696 for (i = 0; i < sh->max_port; i++) { 697 age_info = &sh->port[i].age_info; 698 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 699 continue; 700 MLX5_AGE_UNSET(age_info, MLX5_AGE_EVENT_NEW); 701 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) { 702 MLX5_AGE_UNSET(age_info, MLX5_AGE_TRIGGER); 703 rte_eth_dev_callback_process 704 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 705 RTE_ETH_EVENT_FLOW_AGED, NULL); 706 } 707 } 708 } 709 710 /* 711 * Initialize the ASO connection tracking structure. 712 * 713 * @param[in] sh 714 * Pointer to mlx5_dev_ctx_shared object. 715 * 716 * @return 717 * 0 on success, a negative errno value otherwise and rte_errno is set. 718 */ 719 int 720 mlx5_flow_aso_ct_mng_init(struct mlx5_dev_ctx_shared *sh) 721 { 722 int err; 723 724 if (sh->ct_mng) 725 return 0; 726 sh->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->ct_mng), 727 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 728 if (!sh->ct_mng) { 729 DRV_LOG(ERR, "ASO CT management allocation failed."); 730 rte_errno = ENOMEM; 731 return -rte_errno; 732 } 733 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_CONNECTION_TRACKING); 734 if (err) { 735 mlx5_free(sh->ct_mng); 736 /* rte_errno should be extracted from the failure. */ 737 rte_errno = EINVAL; 738 return -rte_errno; 739 } 740 rte_spinlock_init(&sh->ct_mng->ct_sl); 741 rte_rwlock_init(&sh->ct_mng->resize_rwl); 742 LIST_INIT(&sh->ct_mng->free_cts); 743 return 0; 744 } 745 746 /* 747 * Close and release all the resources of the 748 * ASO connection tracking management structure. 749 * 750 * @param[in] sh 751 * Pointer to mlx5_dev_ctx_shared object to free. 752 */ 753 static void 754 mlx5_flow_aso_ct_mng_close(struct mlx5_dev_ctx_shared *sh) 755 { 756 struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng; 757 struct mlx5_aso_ct_pool *ct_pool; 758 struct mlx5_aso_ct_action *ct; 759 uint32_t idx; 760 uint32_t val; 761 uint32_t cnt; 762 int i; 763 764 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_CONNECTION_TRACKING); 765 idx = mng->next; 766 while (idx--) { 767 cnt = 0; 768 ct_pool = mng->pools[idx]; 769 for (i = 0; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) { 770 ct = &ct_pool->actions[i]; 771 val = __atomic_fetch_sub(&ct->refcnt, 1, 772 __ATOMIC_RELAXED); 773 MLX5_ASSERT(val == 1); 774 if (val > 1) 775 cnt++; 776 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT 777 if (ct->dr_action_orig) 778 claim_zero(mlx5_glue->destroy_flow_action 779 (ct->dr_action_orig)); 780 if (ct->dr_action_rply) 781 claim_zero(mlx5_glue->destroy_flow_action 782 (ct->dr_action_rply)); 783 #endif 784 } 785 claim_zero(mlx5_devx_cmd_destroy(ct_pool->devx_obj)); 786 if (cnt) { 787 DRV_LOG(DEBUG, "%u ASO CT objects are being used in the pool %u", 788 cnt, i); 789 } 790 mlx5_free(ct_pool); 791 /* in case of failure. */ 792 mng->next--; 793 } 794 mlx5_free(mng->pools); 795 mlx5_free(mng); 796 /* Management structure must be cleared to 0s during allocation. */ 797 sh->ct_mng = NULL; 798 } 799 800 /** 801 * Initialize the flow resources' indexed mempool. 802 * 803 * @param[in] sh 804 * Pointer to mlx5_dev_ctx_shared object. 805 * @param[in] config 806 * Pointer to user dev config. 807 */ 808 static void 809 mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh, 810 const struct mlx5_dev_config *config) 811 { 812 uint8_t i; 813 struct mlx5_indexed_pool_config cfg; 814 815 for (i = 0; i < MLX5_IPOOL_MAX; ++i) { 816 cfg = mlx5_ipool_cfg[i]; 817 switch (i) { 818 default: 819 break; 820 /* 821 * Set MLX5_IPOOL_MLX5_FLOW ipool size 822 * according to PCI function flow configuration. 823 */ 824 case MLX5_IPOOL_MLX5_FLOW: 825 cfg.size = config->dv_flow_en ? 826 sizeof(struct mlx5_flow_handle) : 827 MLX5_FLOW_HANDLE_VERBS_SIZE; 828 break; 829 } 830 if (config->reclaim_mode) { 831 cfg.release_mem_en = 1; 832 cfg.per_core_cache = 0; 833 } else { 834 cfg.release_mem_en = 0; 835 } 836 sh->ipool[i] = mlx5_ipool_create(&cfg); 837 } 838 } 839 840 841 /** 842 * Release the flow resources' indexed mempool. 843 * 844 * @param[in] sh 845 * Pointer to mlx5_dev_ctx_shared object. 846 */ 847 static void 848 mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh) 849 { 850 uint8_t i; 851 852 for (i = 0; i < MLX5_IPOOL_MAX; ++i) 853 mlx5_ipool_destroy(sh->ipool[i]); 854 for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i) 855 if (sh->mdh_ipools[i]) 856 mlx5_ipool_destroy(sh->mdh_ipools[i]); 857 } 858 859 /* 860 * Check if dynamic flex parser for eCPRI already exists. 861 * 862 * @param dev 863 * Pointer to Ethernet device structure. 864 * 865 * @return 866 * true on exists, false on not. 867 */ 868 bool 869 mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev) 870 { 871 struct mlx5_priv *priv = dev->data->dev_private; 872 struct mlx5_ecpri_parser_profile *prf = &priv->sh->ecpri_parser; 873 874 return !!prf->obj; 875 } 876 877 /* 878 * Allocation of a flex parser for eCPRI. Once created, this parser related 879 * resources will be held until the device is closed. 880 * 881 * @param dev 882 * Pointer to Ethernet device structure. 883 * 884 * @return 885 * 0 on success, a negative errno value otherwise and rte_errno is set. 886 */ 887 int 888 mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev) 889 { 890 struct mlx5_priv *priv = dev->data->dev_private; 891 struct mlx5_ecpri_parser_profile *prf = &priv->sh->ecpri_parser; 892 struct mlx5_devx_graph_node_attr node = { 893 .modify_field_select = 0, 894 }; 895 uint32_t ids[8]; 896 int ret; 897 898 if (!priv->config.hca_attr.parse_graph_flex_node) { 899 DRV_LOG(ERR, "Dynamic flex parser is not supported " 900 "for device %s.", priv->dev_data->name); 901 return -ENOTSUP; 902 } 903 node.header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED; 904 /* 8 bytes now: 4B common header + 4B message body header. */ 905 node.header_length_base_value = 0x8; 906 /* After MAC layer: Ether / VLAN. */ 907 node.in[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_MAC; 908 /* Type of compared condition should be 0xAEFE in the L2 layer. */ 909 node.in[0].compare_condition_value = RTE_ETHER_TYPE_ECPRI; 910 /* Sample #0: type in common header. */ 911 node.sample[0].flow_match_sample_en = 1; 912 /* Fixed offset. */ 913 node.sample[0].flow_match_sample_offset_mode = 0x0; 914 /* Only the 2nd byte will be used. */ 915 node.sample[0].flow_match_sample_field_base_offset = 0x0; 916 /* Sample #1: message payload. */ 917 node.sample[1].flow_match_sample_en = 1; 918 /* Fixed offset. */ 919 node.sample[1].flow_match_sample_offset_mode = 0x0; 920 /* 921 * Only the first two bytes will be used right now, and its offset will 922 * start after the common header that with the length of a DW(u32). 923 */ 924 node.sample[1].flow_match_sample_field_base_offset = sizeof(uint32_t); 925 prf->obj = mlx5_devx_cmd_create_flex_parser(priv->sh->cdev->ctx, &node); 926 if (!prf->obj) { 927 DRV_LOG(ERR, "Failed to create flex parser node object."); 928 return (rte_errno == 0) ? -ENODEV : -rte_errno; 929 } 930 prf->num = 2; 931 ret = mlx5_devx_cmd_query_parse_samples(prf->obj, ids, prf->num); 932 if (ret) { 933 DRV_LOG(ERR, "Failed to query sample IDs."); 934 return (rte_errno == 0) ? -ENODEV : -rte_errno; 935 } 936 prf->offset[0] = 0x0; 937 prf->offset[1] = sizeof(uint32_t); 938 prf->ids[0] = ids[0]; 939 prf->ids[1] = ids[1]; 940 return 0; 941 } 942 943 /* 944 * Destroy the flex parser node, including the parser itself, input / output 945 * arcs and DW samples. Resources could be reused then. 946 * 947 * @param dev 948 * Pointer to Ethernet device structure. 949 */ 950 static void 951 mlx5_flex_parser_ecpri_release(struct rte_eth_dev *dev) 952 { 953 struct mlx5_priv *priv = dev->data->dev_private; 954 struct mlx5_ecpri_parser_profile *prf = &priv->sh->ecpri_parser; 955 956 if (prf->obj) 957 mlx5_devx_cmd_destroy(prf->obj); 958 prf->obj = NULL; 959 } 960 961 uint32_t 962 mlx5_get_supported_sw_parsing_offloads(const struct mlx5_hca_attr *attr) 963 { 964 uint32_t sw_parsing_offloads = 0; 965 966 if (attr->swp) { 967 sw_parsing_offloads |= MLX5_SW_PARSING_CAP; 968 if (attr->swp_csum) 969 sw_parsing_offloads |= MLX5_SW_PARSING_CSUM_CAP; 970 971 if (attr->swp_lso) 972 sw_parsing_offloads |= MLX5_SW_PARSING_TSO_CAP; 973 } 974 return sw_parsing_offloads; 975 } 976 977 uint32_t 978 mlx5_get_supported_tunneling_offloads(const struct mlx5_hca_attr *attr) 979 { 980 uint32_t tn_offloads = 0; 981 982 if (attr->tunnel_stateless_vxlan) 983 tn_offloads |= MLX5_TUNNELED_OFFLOADS_VXLAN_CAP; 984 if (attr->tunnel_stateless_gre) 985 tn_offloads |= MLX5_TUNNELED_OFFLOADS_GRE_CAP; 986 if (attr->tunnel_stateless_geneve_rx) 987 tn_offloads |= MLX5_TUNNELED_OFFLOADS_GENEVE_CAP; 988 return tn_offloads; 989 } 990 991 /* Fill all fields of UAR structure. */ 992 static int 993 mlx5_rxtx_uars_prepare(struct mlx5_dev_ctx_shared *sh) 994 { 995 int ret; 996 997 ret = mlx5_devx_uar_prepare(sh->cdev, &sh->tx_uar); 998 if (ret) { 999 DRV_LOG(ERR, "Failed to prepare Tx DevX UAR."); 1000 return -rte_errno; 1001 } 1002 MLX5_ASSERT(sh->tx_uar.obj); 1003 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->tx_uar.obj)); 1004 ret = mlx5_devx_uar_prepare(sh->cdev, &sh->rx_uar); 1005 if (ret) { 1006 DRV_LOG(ERR, "Failed to prepare Rx DevX UAR."); 1007 mlx5_devx_uar_release(&sh->tx_uar); 1008 return -rte_errno; 1009 } 1010 MLX5_ASSERT(sh->rx_uar.obj); 1011 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->rx_uar.obj)); 1012 return 0; 1013 } 1014 1015 static void 1016 mlx5_rxtx_uars_release(struct mlx5_dev_ctx_shared *sh) 1017 { 1018 mlx5_devx_uar_release(&sh->rx_uar); 1019 mlx5_devx_uar_release(&sh->tx_uar); 1020 } 1021 1022 /** 1023 * rte_mempool_walk() callback to unregister Rx mempools. 1024 * It used when implicit mempool registration is disabled. 1025 * 1026 * @param mp 1027 * The mempool being walked. 1028 * @param arg 1029 * Pointer to the device shared context. 1030 */ 1031 static void 1032 mlx5_dev_ctx_shared_rx_mempool_unregister_cb(struct rte_mempool *mp, void *arg) 1033 { 1034 struct mlx5_dev_ctx_shared *sh = arg; 1035 1036 mlx5_dev_mempool_unregister(sh->cdev, mp); 1037 } 1038 1039 /** 1040 * Callback used when implicit mempool registration is disabled 1041 * in order to track Rx mempool destruction. 1042 * 1043 * @param event 1044 * Mempool life cycle event. 1045 * @param mp 1046 * An Rx mempool registered explicitly when the port is started. 1047 * @param arg 1048 * Pointer to a device shared context. 1049 */ 1050 static void 1051 mlx5_dev_ctx_shared_rx_mempool_event_cb(enum rte_mempool_event event, 1052 struct rte_mempool *mp, void *arg) 1053 { 1054 struct mlx5_dev_ctx_shared *sh = arg; 1055 1056 if (event == RTE_MEMPOOL_EVENT_DESTROY) 1057 mlx5_dev_mempool_unregister(sh->cdev, mp); 1058 } 1059 1060 int 1061 mlx5_dev_ctx_shared_mempool_subscribe(struct rte_eth_dev *dev) 1062 { 1063 struct mlx5_priv *priv = dev->data->dev_private; 1064 struct mlx5_dev_ctx_shared *sh = priv->sh; 1065 int ret; 1066 1067 /* Check if we only need to track Rx mempool destruction. */ 1068 if (!sh->cdev->config.mr_mempool_reg_en) { 1069 ret = rte_mempool_event_callback_register 1070 (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); 1071 return ret == 0 || rte_errno == EEXIST ? 0 : ret; 1072 } 1073 return mlx5_dev_mempool_subscribe(sh->cdev); 1074 } 1075 1076 /** 1077 * Set up multiple TISs with different affinities according to 1078 * number of bonding ports 1079 * 1080 * @param priv 1081 * Pointer of shared context. 1082 * 1083 * @return 1084 * Zero on success, -1 otherwise. 1085 */ 1086 static int 1087 mlx5_setup_tis(struct mlx5_dev_ctx_shared *sh) 1088 { 1089 int i; 1090 struct mlx5_devx_lag_context lag_ctx = { 0 }; 1091 struct mlx5_devx_tis_attr tis_attr = { 0 }; 1092 1093 tis_attr.transport_domain = sh->td->id; 1094 if (sh->bond.n_port) { 1095 if (!mlx5_devx_cmd_query_lag(sh->cdev->ctx, &lag_ctx)) { 1096 sh->lag.tx_remap_affinity[0] = 1097 lag_ctx.tx_remap_affinity_1; 1098 sh->lag.tx_remap_affinity[1] = 1099 lag_ctx.tx_remap_affinity_2; 1100 sh->lag.affinity_mode = lag_ctx.port_select_mode; 1101 } else { 1102 DRV_LOG(ERR, "Failed to query lag affinity."); 1103 return -1; 1104 } 1105 if (sh->lag.affinity_mode == MLX5_LAG_MODE_TIS) { 1106 for (i = 0; i < sh->bond.n_port; i++) { 1107 tis_attr.lag_tx_port_affinity = 1108 MLX5_IFC_LAG_MAP_TIS_AFFINITY(i, 1109 sh->bond.n_port); 1110 sh->tis[i] = mlx5_devx_cmd_create_tis(sh->cdev->ctx, 1111 &tis_attr); 1112 if (!sh->tis[i]) { 1113 DRV_LOG(ERR, "Failed to TIS %d/%d for bonding device" 1114 " %s.", i, sh->bond.n_port, 1115 sh->ibdev_name); 1116 return -1; 1117 } 1118 } 1119 DRV_LOG(DEBUG, "LAG number of ports : %d, affinity_1 & 2 : pf%d & %d.\n", 1120 sh->bond.n_port, lag_ctx.tx_remap_affinity_1, 1121 lag_ctx.tx_remap_affinity_2); 1122 return 0; 1123 } 1124 if (sh->lag.affinity_mode == MLX5_LAG_MODE_HASH) 1125 DRV_LOG(INFO, "Device %s enabled HW hash based LAG.", 1126 sh->ibdev_name); 1127 } 1128 tis_attr.lag_tx_port_affinity = 0; 1129 sh->tis[0] = mlx5_devx_cmd_create_tis(sh->cdev->ctx, &tis_attr); 1130 if (!sh->tis[0]) { 1131 DRV_LOG(ERR, "Failed to TIS 0 for bonding device" 1132 " %s.", sh->ibdev_name); 1133 return -1; 1134 } 1135 return 0; 1136 } 1137 1138 /** 1139 * Allocate shared device context. If there is multiport device the 1140 * master and representors will share this context, if there is single 1141 * port dedicated device, the context will be used by only given 1142 * port due to unification. 1143 * 1144 * Routine first searches the context for the specified device name, 1145 * if found the shared context assumed and reference counter is incremented. 1146 * If no context found the new one is created and initialized with specified 1147 * device context and parameters. 1148 * 1149 * @param[in] spawn 1150 * Pointer to the device attributes (name, port, etc). 1151 * @param[in] config 1152 * Pointer to device configuration structure. 1153 * 1154 * @return 1155 * Pointer to mlx5_dev_ctx_shared object on success, 1156 * otherwise NULL and rte_errno is set. 1157 */ 1158 struct mlx5_dev_ctx_shared * 1159 mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, 1160 const struct mlx5_dev_config *config) 1161 { 1162 struct mlx5_dev_ctx_shared *sh; 1163 int err = 0; 1164 uint32_t i; 1165 1166 MLX5_ASSERT(spawn); 1167 /* Secondary process should not create the shared context. */ 1168 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1169 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1170 /* Search for IB context by device name. */ 1171 LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) { 1172 if (!strcmp(sh->ibdev_name, spawn->phys_dev_name)) { 1173 sh->refcnt++; 1174 goto exit; 1175 } 1176 } 1177 /* No device found, we have to create new shared context. */ 1178 MLX5_ASSERT(spawn->max_port); 1179 sh = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 1180 sizeof(struct mlx5_dev_ctx_shared) + 1181 spawn->max_port * 1182 sizeof(struct mlx5_dev_shared_port), 1183 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1184 if (!sh) { 1185 DRV_LOG(ERR, "shared context allocation failure"); 1186 rte_errno = ENOMEM; 1187 goto exit; 1188 } 1189 pthread_mutex_init(&sh->txpp.mutex, NULL); 1190 sh->numa_node = spawn->cdev->dev->numa_node; 1191 sh->cdev = spawn->cdev; 1192 sh->devx = sh->cdev->config.devx; 1193 if (spawn->bond_info) 1194 sh->bond = *spawn->bond_info; 1195 err = mlx5_os_get_dev_attr(sh->cdev, &sh->device_attr); 1196 if (err) { 1197 DRV_LOG(DEBUG, "mlx5_os_get_dev_attr() failed"); 1198 goto error; 1199 } 1200 sh->refcnt = 1; 1201 sh->max_port = spawn->max_port; 1202 sh->reclaim_mode = config->reclaim_mode; 1203 strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->cdev->ctx), 1204 sizeof(sh->ibdev_name) - 1); 1205 strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->cdev->ctx), 1206 sizeof(sh->ibdev_path) - 1); 1207 /* 1208 * Setting port_id to max unallowed value means 1209 * there is no interrupt subhandler installed for 1210 * the given port index i. 1211 */ 1212 for (i = 0; i < sh->max_port; i++) { 1213 sh->port[i].ih_port_id = RTE_MAX_ETHPORTS; 1214 sh->port[i].devx_ih_port_id = RTE_MAX_ETHPORTS; 1215 } 1216 if (sh->devx) { 1217 sh->td = mlx5_devx_cmd_create_td(sh->cdev->ctx); 1218 if (!sh->td) { 1219 DRV_LOG(ERR, "TD allocation failure"); 1220 err = ENOMEM; 1221 goto error; 1222 } 1223 if (mlx5_setup_tis(sh)) { 1224 DRV_LOG(ERR, "TIS allocation failure"); 1225 err = ENOMEM; 1226 goto error; 1227 } 1228 err = mlx5_rxtx_uars_prepare(sh); 1229 if (err) 1230 goto error; 1231 #ifndef RTE_ARCH_64 1232 } else { 1233 /* Initialize UAR access locks for 32bit implementations. */ 1234 rte_spinlock_init(&sh->uar_lock_cq); 1235 for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) 1236 rte_spinlock_init(&sh->uar_lock[i]); 1237 #endif 1238 } 1239 mlx5_os_dev_shared_handler_install(sh); 1240 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1241 err = mlx5_flow_os_init_workspace_once(); 1242 if (err) 1243 goto error; 1244 } 1245 mlx5_flow_aging_init(sh); 1246 mlx5_flow_counters_mng_init(sh); 1247 mlx5_flow_ipool_create(sh, config); 1248 /* Add context to the global device list. */ 1249 LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next); 1250 rte_spinlock_init(&sh->geneve_tlv_opt_sl); 1251 exit: 1252 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1253 return sh; 1254 error: 1255 pthread_mutex_destroy(&sh->txpp.mutex); 1256 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1257 MLX5_ASSERT(sh); 1258 if (sh->td) 1259 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1260 i = 0; 1261 do { 1262 if (sh->tis[i]) 1263 claim_zero(mlx5_devx_cmd_destroy(sh->tis[i])); 1264 } while (++i < (uint32_t)sh->bond.n_port); 1265 mlx5_rxtx_uars_release(sh); 1266 mlx5_free(sh); 1267 MLX5_ASSERT(err > 0); 1268 rte_errno = err; 1269 return NULL; 1270 } 1271 1272 /** 1273 * Free shared IB device context. Decrement counter and if zero free 1274 * all allocated resources and close handles. 1275 * 1276 * @param[in] sh 1277 * Pointer to mlx5_dev_ctx_shared object to free 1278 */ 1279 void 1280 mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) 1281 { 1282 int ret; 1283 int i = 0; 1284 1285 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1286 #ifdef RTE_LIBRTE_MLX5_DEBUG 1287 /* Check the object presence in the list. */ 1288 struct mlx5_dev_ctx_shared *lctx; 1289 1290 LIST_FOREACH(lctx, &mlx5_dev_ctx_list, next) 1291 if (lctx == sh) 1292 break; 1293 MLX5_ASSERT(lctx); 1294 if (lctx != sh) { 1295 DRV_LOG(ERR, "Freeing non-existing shared IB context"); 1296 goto exit; 1297 } 1298 #endif 1299 MLX5_ASSERT(sh); 1300 MLX5_ASSERT(sh->refcnt); 1301 /* Secondary process should not free the shared context. */ 1302 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1303 if (--sh->refcnt) 1304 goto exit; 1305 /* Stop watching for mempool events and unregister all mempools. */ 1306 if (!sh->cdev->config.mr_mempool_reg_en) { 1307 ret = rte_mempool_event_callback_unregister 1308 (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); 1309 if (ret == 0) 1310 rte_mempool_walk 1311 (mlx5_dev_ctx_shared_rx_mempool_unregister_cb, sh); 1312 } 1313 /* Remove context from the global device list. */ 1314 LIST_REMOVE(sh, next); 1315 /* Release resources on the last device removal. */ 1316 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1317 mlx5_os_net_cleanup(); 1318 mlx5_flow_os_release_workspace(); 1319 } 1320 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1321 if (sh->flex_parsers_dv) { 1322 mlx5_list_destroy(sh->flex_parsers_dv); 1323 sh->flex_parsers_dv = NULL; 1324 } 1325 /* 1326 * Ensure there is no async event handler installed. 1327 * Only primary process handles async device events. 1328 **/ 1329 mlx5_flow_counters_mng_close(sh); 1330 if (sh->aso_age_mng) { 1331 mlx5_flow_aso_age_mng_close(sh); 1332 sh->aso_age_mng = NULL; 1333 } 1334 if (sh->mtrmng) 1335 mlx5_aso_flow_mtrs_mng_close(sh); 1336 mlx5_flow_ipool_destroy(sh); 1337 mlx5_os_dev_shared_handler_uninstall(sh); 1338 mlx5_rxtx_uars_release(sh); 1339 do { 1340 if (sh->tis[i]) 1341 claim_zero(mlx5_devx_cmd_destroy(sh->tis[i])); 1342 } while (++i < sh->bond.n_port); 1343 if (sh->td) 1344 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1345 MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL); 1346 pthread_mutex_destroy(&sh->txpp.mutex); 1347 mlx5_free(sh); 1348 return; 1349 exit: 1350 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1351 } 1352 1353 /** 1354 * Destroy table hash list. 1355 * 1356 * @param[in] priv 1357 * Pointer to the private device data structure. 1358 */ 1359 void 1360 mlx5_free_table_hash_list(struct mlx5_priv *priv) 1361 { 1362 struct mlx5_dev_ctx_shared *sh = priv->sh; 1363 1364 if (!sh->flow_tbls) 1365 return; 1366 mlx5_hlist_destroy(sh->flow_tbls); 1367 sh->flow_tbls = NULL; 1368 } 1369 1370 /** 1371 * Initialize flow table hash list and create the root tables entry 1372 * for each domain. 1373 * 1374 * @param[in] priv 1375 * Pointer to the private device data structure. 1376 * 1377 * @return 1378 * Zero on success, positive error code otherwise. 1379 */ 1380 int 1381 mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) 1382 { 1383 int err = 0; 1384 /* Tables are only used in DV and DR modes. */ 1385 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 1386 struct mlx5_dev_ctx_shared *sh = priv->sh; 1387 char s[MLX5_NAME_SIZE]; 1388 1389 MLX5_ASSERT(sh); 1390 snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); 1391 sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, 1392 false, true, sh, 1393 flow_dv_tbl_create_cb, 1394 flow_dv_tbl_match_cb, 1395 flow_dv_tbl_remove_cb, 1396 flow_dv_tbl_clone_cb, 1397 flow_dv_tbl_clone_free_cb); 1398 if (!sh->flow_tbls) { 1399 DRV_LOG(ERR, "flow tables with hash creation failed."); 1400 err = ENOMEM; 1401 return err; 1402 } 1403 #ifndef HAVE_MLX5DV_DR 1404 struct rte_flow_error error; 1405 struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; 1406 1407 /* 1408 * In case we have not DR support, the zero tables should be created 1409 * because DV expect to see them even if they cannot be created by 1410 * RDMA-CORE. 1411 */ 1412 if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, 1413 NULL, 0, 1, 0, &error) || 1414 !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, 1415 NULL, 0, 1, 0, &error) || 1416 !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, 1417 NULL, 0, 1, 0, &error)) { 1418 err = ENOMEM; 1419 goto error; 1420 } 1421 return err; 1422 error: 1423 mlx5_free_table_hash_list(priv); 1424 #endif /* HAVE_MLX5DV_DR */ 1425 #endif 1426 return err; 1427 } 1428 1429 /** 1430 * Retrieve integer value from environment variable. 1431 * 1432 * @param[in] name 1433 * Environment variable name. 1434 * 1435 * @return 1436 * Integer value, 0 if the variable is not set. 1437 */ 1438 int 1439 mlx5_getenv_int(const char *name) 1440 { 1441 const char *val = getenv(name); 1442 1443 if (val == NULL) 1444 return 0; 1445 return atoi(val); 1446 } 1447 1448 /** 1449 * DPDK callback to add udp tunnel port 1450 * 1451 * @param[in] dev 1452 * A pointer to eth_dev 1453 * @param[in] udp_tunnel 1454 * A pointer to udp tunnel 1455 * 1456 * @return 1457 * 0 on valid udp ports and tunnels, -ENOTSUP otherwise. 1458 */ 1459 int 1460 mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev __rte_unused, 1461 struct rte_eth_udp_tunnel *udp_tunnel) 1462 { 1463 MLX5_ASSERT(udp_tunnel != NULL); 1464 if (udp_tunnel->prot_type == RTE_ETH_TUNNEL_TYPE_VXLAN && 1465 udp_tunnel->udp_port == 4789) 1466 return 0; 1467 if (udp_tunnel->prot_type == RTE_ETH_TUNNEL_TYPE_VXLAN_GPE && 1468 udp_tunnel->udp_port == 4790) 1469 return 0; 1470 return -ENOTSUP; 1471 } 1472 1473 /** 1474 * Initialize process private data structure. 1475 * 1476 * @param dev 1477 * Pointer to Ethernet device structure. 1478 * 1479 * @return 1480 * 0 on success, a negative errno value otherwise and rte_errno is set. 1481 */ 1482 int 1483 mlx5_proc_priv_init(struct rte_eth_dev *dev) 1484 { 1485 struct mlx5_priv *priv = dev->data->dev_private; 1486 struct mlx5_proc_priv *ppriv; 1487 size_t ppriv_size; 1488 1489 mlx5_proc_priv_uninit(dev); 1490 /* 1491 * UAR register table follows the process private structure. BlueFlame 1492 * registers for Tx queues are stored in the table. 1493 */ 1494 ppriv_size = sizeof(struct mlx5_proc_priv) + 1495 priv->txqs_n * sizeof(struct mlx5_uar_data); 1496 ppriv = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, ppriv_size, 1497 RTE_CACHE_LINE_SIZE, dev->device->numa_node); 1498 if (!ppriv) { 1499 rte_errno = ENOMEM; 1500 return -rte_errno; 1501 } 1502 ppriv->uar_table_sz = priv->txqs_n; 1503 dev->process_private = ppriv; 1504 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 1505 priv->sh->pppriv = ppriv; 1506 return 0; 1507 } 1508 1509 /** 1510 * Un-initialize process private data structure. 1511 * 1512 * @param dev 1513 * Pointer to Ethernet device structure. 1514 */ 1515 void 1516 mlx5_proc_priv_uninit(struct rte_eth_dev *dev) 1517 { 1518 if (!dev->process_private) 1519 return; 1520 mlx5_free(dev->process_private); 1521 dev->process_private = NULL; 1522 } 1523 1524 /** 1525 * DPDK callback to close the device. 1526 * 1527 * Destroy all queues and objects, free memory. 1528 * 1529 * @param dev 1530 * Pointer to Ethernet device structure. 1531 */ 1532 int 1533 mlx5_dev_close(struct rte_eth_dev *dev) 1534 { 1535 struct mlx5_priv *priv = dev->data->dev_private; 1536 unsigned int i; 1537 int ret; 1538 1539 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1540 /* Check if process_private released. */ 1541 if (!dev->process_private) 1542 return 0; 1543 mlx5_tx_uar_uninit_secondary(dev); 1544 mlx5_proc_priv_uninit(dev); 1545 rte_eth_dev_release_port(dev); 1546 return 0; 1547 } 1548 if (!priv->sh) 1549 return 0; 1550 DRV_LOG(DEBUG, "port %u closing device \"%s\"", 1551 dev->data->port_id, 1552 ((priv->sh->cdev->ctx != NULL) ? 1553 mlx5_os_get_ctx_device_name(priv->sh->cdev->ctx) : "")); 1554 /* 1555 * If default mreg copy action is removed at the stop stage, 1556 * the search will return none and nothing will be done anymore. 1557 */ 1558 mlx5_flow_stop_default(dev); 1559 mlx5_traffic_disable(dev); 1560 /* 1561 * If all the flows are already flushed in the device stop stage, 1562 * then this will return directly without any action. 1563 */ 1564 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true); 1565 mlx5_action_handle_flush(dev); 1566 mlx5_flow_meter_flush(dev, NULL); 1567 /* Prevent crashes when queues are still in use. */ 1568 dev->rx_pkt_burst = removed_rx_burst; 1569 dev->tx_pkt_burst = removed_tx_burst; 1570 rte_wmb(); 1571 /* Disable datapath on secondary process. */ 1572 mlx5_mp_os_req_stop_rxtx(dev); 1573 /* Free the eCPRI flex parser resource. */ 1574 mlx5_flex_parser_ecpri_release(dev); 1575 mlx5_flex_item_port_cleanup(dev); 1576 if (priv->rxq_privs != NULL) { 1577 /* XXX race condition if mlx5_rx_burst() is still running. */ 1578 rte_delay_us_sleep(1000); 1579 for (i = 0; (i != priv->rxqs_n); ++i) 1580 mlx5_rxq_release(dev, i); 1581 priv->rxqs_n = 0; 1582 mlx5_free(priv->rxq_privs); 1583 priv->rxq_privs = NULL; 1584 } 1585 if (priv->txqs != NULL) { 1586 /* XXX race condition if mlx5_tx_burst() is still running. */ 1587 rte_delay_us_sleep(1000); 1588 for (i = 0; (i != priv->txqs_n); ++i) 1589 mlx5_txq_release(dev, i); 1590 priv->txqs_n = 0; 1591 priv->txqs = NULL; 1592 } 1593 mlx5_proc_priv_uninit(dev); 1594 if (priv->q_counters) { 1595 mlx5_devx_cmd_destroy(priv->q_counters); 1596 priv->q_counters = NULL; 1597 } 1598 if (priv->drop_queue.hrxq) 1599 mlx5_drop_action_destroy(dev); 1600 if (priv->mreg_cp_tbl) 1601 mlx5_hlist_destroy(priv->mreg_cp_tbl); 1602 mlx5_mprq_free_mp(dev); 1603 if (priv->sh->ct_mng) 1604 mlx5_flow_aso_ct_mng_close(priv->sh); 1605 mlx5_os_free_shared_dr(priv); 1606 if (priv->rss_conf.rss_key != NULL) 1607 mlx5_free(priv->rss_conf.rss_key); 1608 if (priv->reta_idx != NULL) 1609 mlx5_free(priv->reta_idx); 1610 if (priv->config.vf) 1611 mlx5_os_mac_addr_flush(dev); 1612 if (priv->nl_socket_route >= 0) 1613 close(priv->nl_socket_route); 1614 if (priv->nl_socket_rdma >= 0) 1615 close(priv->nl_socket_rdma); 1616 if (priv->vmwa_context) 1617 mlx5_vlan_vmwa_exit(priv->vmwa_context); 1618 ret = mlx5_hrxq_verify(dev); 1619 if (ret) 1620 DRV_LOG(WARNING, "port %u some hash Rx queue still remain", 1621 dev->data->port_id); 1622 ret = mlx5_ind_table_obj_verify(dev); 1623 if (ret) 1624 DRV_LOG(WARNING, "port %u some indirection table still remain", 1625 dev->data->port_id); 1626 ret = mlx5_rxq_obj_verify(dev); 1627 if (ret) 1628 DRV_LOG(WARNING, "port %u some Rx queue objects still remain", 1629 dev->data->port_id); 1630 ret = mlx5_rxq_verify(dev); 1631 if (ret) 1632 DRV_LOG(WARNING, "port %u some Rx queues still remain", 1633 dev->data->port_id); 1634 ret = mlx5_txq_obj_verify(dev); 1635 if (ret) 1636 DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain", 1637 dev->data->port_id); 1638 ret = mlx5_txq_verify(dev); 1639 if (ret) 1640 DRV_LOG(WARNING, "port %u some Tx queues still remain", 1641 dev->data->port_id); 1642 ret = mlx5_flow_verify(dev); 1643 if (ret) 1644 DRV_LOG(WARNING, "port %u some flows still remain", 1645 dev->data->port_id); 1646 if (priv->hrxqs) 1647 mlx5_list_destroy(priv->hrxqs); 1648 /* 1649 * Free the shared context in last turn, because the cleanup 1650 * routines above may use some shared fields, like 1651 * mlx5_os_mac_addr_flush() uses ibdev_path for retrieveing 1652 * ifindex if Netlink fails. 1653 */ 1654 mlx5_free_shared_dev_ctx(priv->sh); 1655 if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 1656 unsigned int c = 0; 1657 uint16_t port_id; 1658 1659 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1660 struct mlx5_priv *opriv = 1661 rte_eth_devices[port_id].data->dev_private; 1662 1663 if (!opriv || 1664 opriv->domain_id != priv->domain_id || 1665 &rte_eth_devices[port_id] == dev) 1666 continue; 1667 ++c; 1668 break; 1669 } 1670 if (!c) 1671 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 1672 } 1673 memset(priv, 0, sizeof(*priv)); 1674 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 1675 /* 1676 * Reset mac_addrs to NULL such that it is not freed as part of 1677 * rte_eth_dev_release_port(). mac_addrs is part of dev_private so 1678 * it is freed when dev_private is freed. 1679 */ 1680 dev->data->mac_addrs = NULL; 1681 return 0; 1682 } 1683 1684 const struct eth_dev_ops mlx5_dev_ops = { 1685 .dev_configure = mlx5_dev_configure, 1686 .dev_start = mlx5_dev_start, 1687 .dev_stop = mlx5_dev_stop, 1688 .dev_set_link_down = mlx5_set_link_down, 1689 .dev_set_link_up = mlx5_set_link_up, 1690 .dev_close = mlx5_dev_close, 1691 .promiscuous_enable = mlx5_promiscuous_enable, 1692 .promiscuous_disable = mlx5_promiscuous_disable, 1693 .allmulticast_enable = mlx5_allmulticast_enable, 1694 .allmulticast_disable = mlx5_allmulticast_disable, 1695 .link_update = mlx5_link_update, 1696 .stats_get = mlx5_stats_get, 1697 .stats_reset = mlx5_stats_reset, 1698 .xstats_get = mlx5_xstats_get, 1699 .xstats_reset = mlx5_xstats_reset, 1700 .xstats_get_names = mlx5_xstats_get_names, 1701 .fw_version_get = mlx5_fw_version_get, 1702 .dev_infos_get = mlx5_dev_infos_get, 1703 .representor_info_get = mlx5_representor_info_get, 1704 .read_clock = mlx5_txpp_read_clock, 1705 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1706 .vlan_filter_set = mlx5_vlan_filter_set, 1707 .rx_queue_setup = mlx5_rx_queue_setup, 1708 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1709 .tx_queue_setup = mlx5_tx_queue_setup, 1710 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1711 .rx_queue_release = mlx5_rx_queue_release, 1712 .tx_queue_release = mlx5_tx_queue_release, 1713 .rx_queue_start = mlx5_rx_queue_start, 1714 .rx_queue_stop = mlx5_rx_queue_stop, 1715 .tx_queue_start = mlx5_tx_queue_start, 1716 .tx_queue_stop = mlx5_tx_queue_stop, 1717 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1718 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1719 .mac_addr_remove = mlx5_mac_addr_remove, 1720 .mac_addr_add = mlx5_mac_addr_add, 1721 .mac_addr_set = mlx5_mac_addr_set, 1722 .set_mc_addr_list = mlx5_set_mc_addr_list, 1723 .mtu_set = mlx5_dev_set_mtu, 1724 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1725 .vlan_offload_set = mlx5_vlan_offload_set, 1726 .reta_update = mlx5_dev_rss_reta_update, 1727 .reta_query = mlx5_dev_rss_reta_query, 1728 .rss_hash_update = mlx5_rss_hash_update, 1729 .rss_hash_conf_get = mlx5_rss_hash_conf_get, 1730 .flow_ops_get = mlx5_flow_ops_get, 1731 .rxq_info_get = mlx5_rxq_info_get, 1732 .txq_info_get = mlx5_txq_info_get, 1733 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1734 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1735 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1736 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1737 .is_removed = mlx5_is_removed, 1738 .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, 1739 .get_module_info = mlx5_get_module_info, 1740 .get_module_eeprom = mlx5_get_module_eeprom, 1741 .hairpin_cap_get = mlx5_hairpin_cap_get, 1742 .mtr_ops_get = mlx5_flow_meter_ops_get, 1743 .hairpin_bind = mlx5_hairpin_bind, 1744 .hairpin_unbind = mlx5_hairpin_unbind, 1745 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1746 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1747 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1748 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1749 .get_monitor_addr = mlx5_get_monitor_addr, 1750 }; 1751 1752 /* Available operations from secondary process. */ 1753 const struct eth_dev_ops mlx5_dev_sec_ops = { 1754 .stats_get = mlx5_stats_get, 1755 .stats_reset = mlx5_stats_reset, 1756 .xstats_get = mlx5_xstats_get, 1757 .xstats_reset = mlx5_xstats_reset, 1758 .xstats_get_names = mlx5_xstats_get_names, 1759 .fw_version_get = mlx5_fw_version_get, 1760 .dev_infos_get = mlx5_dev_infos_get, 1761 .representor_info_get = mlx5_representor_info_get, 1762 .read_clock = mlx5_txpp_read_clock, 1763 .rx_queue_start = mlx5_rx_queue_start, 1764 .rx_queue_stop = mlx5_rx_queue_stop, 1765 .tx_queue_start = mlx5_tx_queue_start, 1766 .tx_queue_stop = mlx5_tx_queue_stop, 1767 .rxq_info_get = mlx5_rxq_info_get, 1768 .txq_info_get = mlx5_txq_info_get, 1769 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1770 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1771 .get_module_info = mlx5_get_module_info, 1772 .get_module_eeprom = mlx5_get_module_eeprom, 1773 }; 1774 1775 /* Available operations in flow isolated mode. */ 1776 const struct eth_dev_ops mlx5_dev_ops_isolate = { 1777 .dev_configure = mlx5_dev_configure, 1778 .dev_start = mlx5_dev_start, 1779 .dev_stop = mlx5_dev_stop, 1780 .dev_set_link_down = mlx5_set_link_down, 1781 .dev_set_link_up = mlx5_set_link_up, 1782 .dev_close = mlx5_dev_close, 1783 .promiscuous_enable = mlx5_promiscuous_enable, 1784 .promiscuous_disable = mlx5_promiscuous_disable, 1785 .allmulticast_enable = mlx5_allmulticast_enable, 1786 .allmulticast_disable = mlx5_allmulticast_disable, 1787 .link_update = mlx5_link_update, 1788 .stats_get = mlx5_stats_get, 1789 .stats_reset = mlx5_stats_reset, 1790 .xstats_get = mlx5_xstats_get, 1791 .xstats_reset = mlx5_xstats_reset, 1792 .xstats_get_names = mlx5_xstats_get_names, 1793 .fw_version_get = mlx5_fw_version_get, 1794 .dev_infos_get = mlx5_dev_infos_get, 1795 .representor_info_get = mlx5_representor_info_get, 1796 .read_clock = mlx5_txpp_read_clock, 1797 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1798 .vlan_filter_set = mlx5_vlan_filter_set, 1799 .rx_queue_setup = mlx5_rx_queue_setup, 1800 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1801 .tx_queue_setup = mlx5_tx_queue_setup, 1802 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1803 .rx_queue_release = mlx5_rx_queue_release, 1804 .tx_queue_release = mlx5_tx_queue_release, 1805 .rx_queue_start = mlx5_rx_queue_start, 1806 .rx_queue_stop = mlx5_rx_queue_stop, 1807 .tx_queue_start = mlx5_tx_queue_start, 1808 .tx_queue_stop = mlx5_tx_queue_stop, 1809 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1810 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1811 .mac_addr_remove = mlx5_mac_addr_remove, 1812 .mac_addr_add = mlx5_mac_addr_add, 1813 .mac_addr_set = mlx5_mac_addr_set, 1814 .set_mc_addr_list = mlx5_set_mc_addr_list, 1815 .mtu_set = mlx5_dev_set_mtu, 1816 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1817 .vlan_offload_set = mlx5_vlan_offload_set, 1818 .flow_ops_get = mlx5_flow_ops_get, 1819 .rxq_info_get = mlx5_rxq_info_get, 1820 .txq_info_get = mlx5_txq_info_get, 1821 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1822 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1823 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1824 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1825 .is_removed = mlx5_is_removed, 1826 .get_module_info = mlx5_get_module_info, 1827 .get_module_eeprom = mlx5_get_module_eeprom, 1828 .hairpin_cap_get = mlx5_hairpin_cap_get, 1829 .mtr_ops_get = mlx5_flow_meter_ops_get, 1830 .hairpin_bind = mlx5_hairpin_bind, 1831 .hairpin_unbind = mlx5_hairpin_unbind, 1832 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1833 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1834 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1835 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1836 .get_monitor_addr = mlx5_get_monitor_addr, 1837 }; 1838 1839 /** 1840 * Verify and store value for device argument. 1841 * 1842 * @param[in] key 1843 * Key argument to verify. 1844 * @param[in] val 1845 * Value associated with key. 1846 * @param opaque 1847 * User data. 1848 * 1849 * @return 1850 * 0 on success, a negative errno value otherwise and rte_errno is set. 1851 */ 1852 static int 1853 mlx5_args_check(const char *key, const char *val, void *opaque) 1854 { 1855 struct mlx5_dev_config *config = opaque; 1856 unsigned long mod; 1857 signed long tmp; 1858 1859 /* No-op, port representors are processed in mlx5_dev_spawn(). */ 1860 if (!strcmp(MLX5_DRIVER_KEY, key) || !strcmp(MLX5_REPRESENTOR, key) || 1861 !strcmp(MLX5_SYS_MEM_EN, key) || !strcmp(MLX5_TX_DB_NC, key) || 1862 !strcmp(MLX5_MR_MEMPOOL_REG_EN, key) || 1863 !strcmp(MLX5_MR_EXT_MEMSEG_EN, key)) 1864 return 0; 1865 errno = 0; 1866 tmp = strtol(val, NULL, 0); 1867 if (errno) { 1868 rte_errno = errno; 1869 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); 1870 return -rte_errno; 1871 } 1872 if (tmp < 0 && strcmp(MLX5_TX_PP, key) && strcmp(MLX5_TX_SKEW, key)) { 1873 /* Negative values are acceptable for some keys only. */ 1874 rte_errno = EINVAL; 1875 DRV_LOG(WARNING, "%s: invalid negative value \"%s\"", key, val); 1876 return -rte_errno; 1877 } 1878 mod = tmp >= 0 ? tmp : -tmp; 1879 if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) { 1880 if (tmp > MLX5_CQE_RESP_FORMAT_L34H_STRIDX) { 1881 DRV_LOG(ERR, "invalid CQE compression " 1882 "format parameter"); 1883 rte_errno = EINVAL; 1884 return -rte_errno; 1885 } 1886 config->cqe_comp = !!tmp; 1887 config->cqe_comp_fmt = tmp; 1888 } else if (strcmp(MLX5_RXQ_PKT_PAD_EN, key) == 0) { 1889 config->hw_padding = !!tmp; 1890 } else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) { 1891 config->mprq.enabled = !!tmp; 1892 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) { 1893 config->mprq.stride_num_n = tmp; 1894 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_SIZE, key) == 0) { 1895 config->mprq.stride_size_n = tmp; 1896 } else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) { 1897 config->mprq.max_memcpy_len = tmp; 1898 } else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) { 1899 config->mprq.min_rxqs_num = tmp; 1900 } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) { 1901 DRV_LOG(WARNING, "%s: deprecated parameter," 1902 " converted to txq_inline_max", key); 1903 config->txq_inline_max = tmp; 1904 } else if (strcmp(MLX5_TXQ_INLINE_MAX, key) == 0) { 1905 config->txq_inline_max = tmp; 1906 } else if (strcmp(MLX5_TXQ_INLINE_MIN, key) == 0) { 1907 config->txq_inline_min = tmp; 1908 } else if (strcmp(MLX5_TXQ_INLINE_MPW, key) == 0) { 1909 config->txq_inline_mpw = tmp; 1910 } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) { 1911 config->txqs_inline = tmp; 1912 } else if (strcmp(MLX5_TXQS_MAX_VEC, key) == 0) { 1913 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1914 } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { 1915 config->mps = !!tmp; 1916 } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { 1917 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1918 } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { 1919 DRV_LOG(WARNING, "%s: deprecated parameter," 1920 " converted to txq_inline_mpw", key); 1921 config->txq_inline_mpw = tmp; 1922 } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) { 1923 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1924 } else if (strcmp(MLX5_TX_PP, key) == 0) { 1925 if (!mod) { 1926 DRV_LOG(ERR, "Zero Tx packet pacing parameter"); 1927 rte_errno = EINVAL; 1928 return -rte_errno; 1929 } 1930 config->tx_pp = tmp; 1931 } else if (strcmp(MLX5_TX_SKEW, key) == 0) { 1932 config->tx_skew = tmp; 1933 } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { 1934 config->rx_vec_en = !!tmp; 1935 } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) { 1936 config->l3_vxlan_en = !!tmp; 1937 } else if (strcmp(MLX5_VF_NL_EN, key) == 0) { 1938 config->vf_nl_en = !!tmp; 1939 } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) { 1940 config->dv_esw_en = !!tmp; 1941 } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) { 1942 config->dv_flow_en = !!tmp; 1943 } else if (strcmp(MLX5_DV_XMETA_EN, key) == 0) { 1944 if (tmp != MLX5_XMETA_MODE_LEGACY && 1945 tmp != MLX5_XMETA_MODE_META16 && 1946 tmp != MLX5_XMETA_MODE_META32 && 1947 tmp != MLX5_XMETA_MODE_MISS_INFO) { 1948 DRV_LOG(ERR, "invalid extensive " 1949 "metadata parameter"); 1950 rte_errno = EINVAL; 1951 return -rte_errno; 1952 } 1953 if (tmp != MLX5_XMETA_MODE_MISS_INFO) 1954 config->dv_xmeta_en = tmp; 1955 else 1956 config->dv_miss_info = 1; 1957 } else if (strcmp(MLX5_LACP_BY_USER, key) == 0) { 1958 config->lacp_by_user = !!tmp; 1959 } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) { 1960 config->max_dump_files_num = tmp; 1961 } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) { 1962 config->lro.timeout = tmp; 1963 } else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) { 1964 DRV_LOG(DEBUG, "class argument is %s.", val); 1965 } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) { 1966 config->log_hp_size = tmp; 1967 } else if (strcmp(MLX5_RECLAIM_MEM, key) == 0) { 1968 if (tmp != MLX5_RCM_NONE && 1969 tmp != MLX5_RCM_LIGHT && 1970 tmp != MLX5_RCM_AGGR) { 1971 DRV_LOG(ERR, "Unrecognize %s: \"%s\"", key, val); 1972 rte_errno = EINVAL; 1973 return -rte_errno; 1974 } 1975 config->reclaim_mode = tmp; 1976 } else if (strcmp(MLX5_DECAP_EN, key) == 0) { 1977 config->decap_en = !!tmp; 1978 } else if (strcmp(MLX5_ALLOW_DUPLICATE_PATTERN, key) == 0) { 1979 config->allow_duplicate_pattern = !!tmp; 1980 } else if (strcmp(MLX5_DELAY_DROP, key) == 0) { 1981 config->std_delay_drop = tmp & MLX5_DELAY_DROP_STANDARD; 1982 config->hp_delay_drop = tmp & MLX5_DELAY_DROP_HAIRPIN; 1983 } else { 1984 DRV_LOG(WARNING, "%s: unknown parameter", key); 1985 rte_errno = EINVAL; 1986 return -rte_errno; 1987 } 1988 return 0; 1989 } 1990 1991 /** 1992 * Parse device parameters. 1993 * 1994 * @param config 1995 * Pointer to device configuration structure. 1996 * @param devargs 1997 * Device arguments structure. 1998 * 1999 * @return 2000 * 0 on success, a negative errno value otherwise and rte_errno is set. 2001 */ 2002 int 2003 mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs) 2004 { 2005 const char **params = (const char *[]){ 2006 MLX5_DRIVER_KEY, 2007 MLX5_RXQ_CQE_COMP_EN, 2008 MLX5_RXQ_PKT_PAD_EN, 2009 MLX5_RX_MPRQ_EN, 2010 MLX5_RX_MPRQ_LOG_STRIDE_NUM, 2011 MLX5_RX_MPRQ_LOG_STRIDE_SIZE, 2012 MLX5_RX_MPRQ_MAX_MEMCPY_LEN, 2013 MLX5_RXQS_MIN_MPRQ, 2014 MLX5_TXQ_INLINE, 2015 MLX5_TXQ_INLINE_MIN, 2016 MLX5_TXQ_INLINE_MAX, 2017 MLX5_TXQ_INLINE_MPW, 2018 MLX5_TXQS_MIN_INLINE, 2019 MLX5_TXQS_MAX_VEC, 2020 MLX5_TXQ_MPW_EN, 2021 MLX5_TXQ_MPW_HDR_DSEG_EN, 2022 MLX5_TXQ_MAX_INLINE_LEN, 2023 MLX5_TX_DB_NC, 2024 MLX5_TX_PP, 2025 MLX5_TX_SKEW, 2026 MLX5_TX_VEC_EN, 2027 MLX5_RX_VEC_EN, 2028 MLX5_L3_VXLAN_EN, 2029 MLX5_VF_NL_EN, 2030 MLX5_DV_ESW_EN, 2031 MLX5_DV_FLOW_EN, 2032 MLX5_DV_XMETA_EN, 2033 MLX5_LACP_BY_USER, 2034 MLX5_MR_EXT_MEMSEG_EN, 2035 MLX5_REPRESENTOR, 2036 MLX5_MAX_DUMP_FILES_NUM, 2037 MLX5_LRO_TIMEOUT_USEC, 2038 RTE_DEVARGS_KEY_CLASS, 2039 MLX5_HP_BUF_SIZE, 2040 MLX5_RECLAIM_MEM, 2041 MLX5_SYS_MEM_EN, 2042 MLX5_DECAP_EN, 2043 MLX5_ALLOW_DUPLICATE_PATTERN, 2044 MLX5_MR_MEMPOOL_REG_EN, 2045 MLX5_DELAY_DROP, 2046 NULL, 2047 }; 2048 struct rte_kvargs *kvlist; 2049 int ret = 0; 2050 int i; 2051 2052 if (devargs == NULL) 2053 return 0; 2054 /* Following UGLY cast is done to pass checkpatch. */ 2055 kvlist = rte_kvargs_parse(devargs->args, params); 2056 if (kvlist == NULL) { 2057 rte_errno = EINVAL; 2058 return -rte_errno; 2059 } 2060 /* Process parameters. */ 2061 for (i = 0; (params[i] != NULL); ++i) { 2062 if (rte_kvargs_count(kvlist, params[i])) { 2063 ret = rte_kvargs_process(kvlist, params[i], 2064 mlx5_args_check, config); 2065 if (ret) { 2066 rte_errno = EINVAL; 2067 rte_kvargs_free(kvlist); 2068 return -rte_errno; 2069 } 2070 } 2071 } 2072 rte_kvargs_free(kvlist); 2073 return 0; 2074 } 2075 2076 /** 2077 * Configures the minimal amount of data to inline into WQE 2078 * while sending packets. 2079 * 2080 * - the txq_inline_min has the maximal priority, if this 2081 * key is specified in devargs 2082 * - if DevX is enabled the inline mode is queried from the 2083 * device (HCA attributes and NIC vport context if needed). 2084 * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4 Lx 2085 * and none (0 bytes) for other NICs 2086 * 2087 * @param spawn 2088 * Verbs device parameters (name, port, switch_info) to spawn. 2089 * @param config 2090 * Device configuration parameters. 2091 */ 2092 void 2093 mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn, 2094 struct mlx5_dev_config *config) 2095 { 2096 if (config->txq_inline_min != MLX5_ARG_UNSET) { 2097 /* Application defines size of inlined data explicitly. */ 2098 if (spawn->pci_dev != NULL) { 2099 switch (spawn->pci_dev->id.device_id) { 2100 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2101 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2102 if (config->txq_inline_min < 2103 (int)MLX5_INLINE_HSIZE_L2) { 2104 DRV_LOG(DEBUG, 2105 "txq_inline_mix aligned to minimal ConnectX-4 required value %d", 2106 (int)MLX5_INLINE_HSIZE_L2); 2107 config->txq_inline_min = 2108 MLX5_INLINE_HSIZE_L2; 2109 } 2110 break; 2111 } 2112 } 2113 goto exit; 2114 } 2115 if (config->hca_attr.eth_net_offloads) { 2116 /* We have DevX enabled, inline mode queried successfully. */ 2117 switch (config->hca_attr.wqe_inline_mode) { 2118 case MLX5_CAP_INLINE_MODE_L2: 2119 /* outer L2 header must be inlined. */ 2120 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2121 goto exit; 2122 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2123 /* No inline data are required by NIC. */ 2124 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2125 config->hw_vlan_insert = 2126 config->hca_attr.wqe_vlan_insert; 2127 DRV_LOG(DEBUG, "Tx VLAN insertion is supported"); 2128 goto exit; 2129 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2130 /* inline mode is defined by NIC vport context. */ 2131 if (!config->hca_attr.eth_virt) 2132 break; 2133 switch (config->hca_attr.vport_inline_mode) { 2134 case MLX5_INLINE_MODE_NONE: 2135 config->txq_inline_min = 2136 MLX5_INLINE_HSIZE_NONE; 2137 goto exit; 2138 case MLX5_INLINE_MODE_L2: 2139 config->txq_inline_min = 2140 MLX5_INLINE_HSIZE_L2; 2141 goto exit; 2142 case MLX5_INLINE_MODE_IP: 2143 config->txq_inline_min = 2144 MLX5_INLINE_HSIZE_L3; 2145 goto exit; 2146 case MLX5_INLINE_MODE_TCP_UDP: 2147 config->txq_inline_min = 2148 MLX5_INLINE_HSIZE_L4; 2149 goto exit; 2150 case MLX5_INLINE_MODE_INNER_L2: 2151 config->txq_inline_min = 2152 MLX5_INLINE_HSIZE_INNER_L2; 2153 goto exit; 2154 case MLX5_INLINE_MODE_INNER_IP: 2155 config->txq_inline_min = 2156 MLX5_INLINE_HSIZE_INNER_L3; 2157 goto exit; 2158 case MLX5_INLINE_MODE_INNER_TCP_UDP: 2159 config->txq_inline_min = 2160 MLX5_INLINE_HSIZE_INNER_L4; 2161 goto exit; 2162 } 2163 } 2164 } 2165 if (spawn->pci_dev == NULL) { 2166 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2167 goto exit; 2168 } 2169 /* 2170 * We get here if we are unable to deduce 2171 * inline data size with DevX. Try PCI ID 2172 * to determine old NICs. 2173 */ 2174 switch (spawn->pci_dev->id.device_id) { 2175 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2176 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2177 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: 2178 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 2179 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2180 config->hw_vlan_insert = 0; 2181 break; 2182 case PCI_DEVICE_ID_MELLANOX_CONNECTX5: 2183 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 2184 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: 2185 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 2186 /* 2187 * These NICs support VLAN insertion from WQE and 2188 * report the wqe_vlan_insert flag. But there is the bug 2189 * and PFC control may be broken, so disable feature. 2190 */ 2191 config->hw_vlan_insert = 0; 2192 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2193 break; 2194 default: 2195 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2196 break; 2197 } 2198 exit: 2199 DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min); 2200 } 2201 2202 /** 2203 * Configures the metadata mask fields in the shared context. 2204 * 2205 * @param [in] dev 2206 * Pointer to Ethernet device. 2207 */ 2208 void 2209 mlx5_set_metadata_mask(struct rte_eth_dev *dev) 2210 { 2211 struct mlx5_priv *priv = dev->data->dev_private; 2212 struct mlx5_dev_ctx_shared *sh = priv->sh; 2213 uint32_t meta, mark, reg_c0; 2214 2215 reg_c0 = ~priv->vport_meta_mask; 2216 switch (priv->config.dv_xmeta_en) { 2217 case MLX5_XMETA_MODE_LEGACY: 2218 meta = UINT32_MAX; 2219 mark = MLX5_FLOW_MARK_MASK; 2220 break; 2221 case MLX5_XMETA_MODE_META16: 2222 meta = reg_c0 >> rte_bsf32(reg_c0); 2223 mark = MLX5_FLOW_MARK_MASK; 2224 break; 2225 case MLX5_XMETA_MODE_META32: 2226 meta = UINT32_MAX; 2227 mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK; 2228 break; 2229 default: 2230 meta = 0; 2231 mark = 0; 2232 MLX5_ASSERT(false); 2233 break; 2234 } 2235 if (sh->dv_mark_mask && sh->dv_mark_mask != mark) 2236 DRV_LOG(WARNING, "metadata MARK mask mismatche %08X:%08X", 2237 sh->dv_mark_mask, mark); 2238 else 2239 sh->dv_mark_mask = mark; 2240 if (sh->dv_meta_mask && sh->dv_meta_mask != meta) 2241 DRV_LOG(WARNING, "metadata META mask mismatche %08X:%08X", 2242 sh->dv_meta_mask, meta); 2243 else 2244 sh->dv_meta_mask = meta; 2245 if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0) 2246 DRV_LOG(WARNING, "metadata reg_c0 mask mismatche %08X:%08X", 2247 sh->dv_meta_mask, reg_c0); 2248 else 2249 sh->dv_regc0_mask = reg_c0; 2250 DRV_LOG(DEBUG, "metadata mode %u", priv->config.dv_xmeta_en); 2251 DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask); 2252 DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask); 2253 DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask); 2254 } 2255 2256 int 2257 rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n) 2258 { 2259 static const char *const dynf_names[] = { 2260 RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, 2261 RTE_MBUF_DYNFLAG_METADATA_NAME, 2262 RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME 2263 }; 2264 unsigned int i; 2265 2266 if (n < RTE_DIM(dynf_names)) 2267 return -ENOMEM; 2268 for (i = 0; i < RTE_DIM(dynf_names); i++) { 2269 if (names[i] == NULL) 2270 return -EINVAL; 2271 strcpy(names[i], dynf_names[i]); 2272 } 2273 return RTE_DIM(dynf_names); 2274 } 2275 2276 /** 2277 * Comparison callback to sort device data. 2278 * 2279 * This is meant to be used with qsort(). 2280 * 2281 * @param a[in] 2282 * Pointer to pointer to first data object. 2283 * @param b[in] 2284 * Pointer to pointer to second data object. 2285 * 2286 * @return 2287 * 0 if both objects are equal, less than 0 if the first argument is less 2288 * than the second, greater than 0 otherwise. 2289 */ 2290 int 2291 mlx5_dev_check_sibling_config(struct mlx5_priv *priv, 2292 struct mlx5_dev_config *config, 2293 struct rte_device *dpdk_dev) 2294 { 2295 struct mlx5_dev_ctx_shared *sh = priv->sh; 2296 struct mlx5_dev_config *sh_conf = NULL; 2297 uint16_t port_id; 2298 2299 MLX5_ASSERT(sh); 2300 /* Nothing to compare for the single/first device. */ 2301 if (sh->refcnt == 1) 2302 return 0; 2303 /* Find the device with shared context. */ 2304 MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) { 2305 struct mlx5_priv *opriv = 2306 rte_eth_devices[port_id].data->dev_private; 2307 2308 if (opriv && opriv != priv && opriv->sh == sh) { 2309 sh_conf = &opriv->config; 2310 break; 2311 } 2312 } 2313 if (!sh_conf) 2314 return 0; 2315 if (sh_conf->dv_flow_en ^ config->dv_flow_en) { 2316 DRV_LOG(ERR, "\"dv_flow_en\" configuration mismatch" 2317 " for shared %s context", sh->ibdev_name); 2318 rte_errno = EINVAL; 2319 return rte_errno; 2320 } 2321 if (sh_conf->dv_xmeta_en ^ config->dv_xmeta_en) { 2322 DRV_LOG(ERR, "\"dv_xmeta_en\" configuration mismatch" 2323 " for shared %s context", sh->ibdev_name); 2324 rte_errno = EINVAL; 2325 return rte_errno; 2326 } 2327 return 0; 2328 } 2329 2330 /** 2331 * Look for the ethernet device belonging to mlx5 driver. 2332 * 2333 * @param[in] port_id 2334 * port_id to start looking for device. 2335 * @param[in] odev 2336 * Pointer to the hint device. When device is being probed 2337 * the its siblings (master and preceding representors might 2338 * not have assigned driver yet (because the mlx5_os_pci_probe() 2339 * is not completed yet, for this case match on hint 2340 * device may be used to detect sibling device. 2341 * 2342 * @return 2343 * port_id of found device, RTE_MAX_ETHPORT if not found. 2344 */ 2345 uint16_t 2346 mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev) 2347 { 2348 while (port_id < RTE_MAX_ETHPORTS) { 2349 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 2350 2351 if (dev->state != RTE_ETH_DEV_UNUSED && 2352 dev->device && 2353 (dev->device == odev || 2354 (dev->device->driver && 2355 dev->device->driver->name && 2356 ((strcmp(dev->device->driver->name, 2357 MLX5_PCI_DRIVER_NAME) == 0) || 2358 (strcmp(dev->device->driver->name, 2359 MLX5_AUXILIARY_DRIVER_NAME) == 0))))) 2360 break; 2361 port_id++; 2362 } 2363 if (port_id >= RTE_MAX_ETHPORTS) 2364 return RTE_MAX_ETHPORTS; 2365 return port_id; 2366 } 2367 2368 /** 2369 * Callback to remove a device. 2370 * 2371 * This function removes all Ethernet devices belong to a given device. 2372 * 2373 * @param[in] cdev 2374 * Pointer to the generic device. 2375 * 2376 * @return 2377 * 0 on success, the function cannot fail. 2378 */ 2379 int 2380 mlx5_net_remove(struct mlx5_common_device *cdev) 2381 { 2382 uint16_t port_id; 2383 int ret = 0; 2384 2385 RTE_ETH_FOREACH_DEV_OF(port_id, cdev->dev) { 2386 /* 2387 * mlx5_dev_close() is not registered to secondary process, 2388 * call the close function explicitly for secondary process. 2389 */ 2390 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 2391 ret |= mlx5_dev_close(&rte_eth_devices[port_id]); 2392 else 2393 ret |= rte_eth_dev_close(port_id); 2394 } 2395 return ret == 0 ? 0 : -EIO; 2396 } 2397 2398 static const struct rte_pci_id mlx5_pci_id_map[] = { 2399 { 2400 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2401 PCI_DEVICE_ID_MELLANOX_CONNECTX4) 2402 }, 2403 { 2404 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2405 PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) 2406 }, 2407 { 2408 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2409 PCI_DEVICE_ID_MELLANOX_CONNECTX4LX) 2410 }, 2411 { 2412 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2413 PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) 2414 }, 2415 { 2416 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2417 PCI_DEVICE_ID_MELLANOX_CONNECTX5) 2418 }, 2419 { 2420 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2421 PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) 2422 }, 2423 { 2424 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2425 PCI_DEVICE_ID_MELLANOX_CONNECTX5EX) 2426 }, 2427 { 2428 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2429 PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF) 2430 }, 2431 { 2432 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2433 PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) 2434 }, 2435 { 2436 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2437 PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF) 2438 }, 2439 { 2440 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2441 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 2442 }, 2443 { 2444 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2445 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF) 2446 }, 2447 { 2448 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2449 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 2450 }, 2451 { 2452 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2453 PCI_DEVICE_ID_MELLANOX_CONNECTXVF) 2454 }, 2455 { 2456 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2457 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF) 2458 }, 2459 { 2460 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2461 PCI_DEVICE_ID_MELLANOX_CONNECTX6LX) 2462 }, 2463 { 2464 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2465 PCI_DEVICE_ID_MELLANOX_CONNECTX7) 2466 }, 2467 { 2468 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2469 PCI_DEVICE_ID_MELLANOX_CONNECTX7BF) 2470 }, 2471 { 2472 .vendor_id = 0 2473 } 2474 }; 2475 2476 static struct mlx5_class_driver mlx5_net_driver = { 2477 .drv_class = MLX5_CLASS_ETH, 2478 .name = RTE_STR(MLX5_ETH_DRIVER_NAME), 2479 .id_table = mlx5_pci_id_map, 2480 .probe = mlx5_os_net_probe, 2481 .remove = mlx5_net_remove, 2482 .probe_again = 1, 2483 .intr_lsc = 1, 2484 .intr_rmv = 1, 2485 }; 2486 2487 /* Initialize driver log type. */ 2488 RTE_LOG_REGISTER_DEFAULT(mlx5_logtype, NOTICE) 2489 2490 /** 2491 * Driver initialization routine. 2492 */ 2493 RTE_INIT(rte_mlx5_pmd_init) 2494 { 2495 pthread_mutex_init(&mlx5_dev_ctx_list_mutex, NULL); 2496 mlx5_common_init(); 2497 /* Build the static tables for Verbs conversion. */ 2498 mlx5_set_ptype_table(); 2499 mlx5_set_cksum_table(); 2500 mlx5_set_swp_types_table(); 2501 if (mlx5_glue) 2502 mlx5_class_driver_register(&mlx5_net_driver); 2503 } 2504 2505 RTE_PMD_EXPORT_NAME(MLX5_ETH_DRIVER_NAME, __COUNTER__); 2506 RTE_PMD_REGISTER_PCI_TABLE(MLX5_ETH_DRIVER_NAME, mlx5_pci_id_map); 2507 RTE_PMD_REGISTER_KMOD_DEP(MLX5_ETH_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 2508