1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <stddef.h> 7 #include <unistd.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <errno.h> 12 #include <fcntl.h> 13 14 #include <rte_malloc.h> 15 #include <ethdev_driver.h> 16 #include <rte_pci.h> 17 #include <bus_pci_driver.h> 18 #include <rte_common.h> 19 #include <rte_kvargs.h> 20 #include <rte_rwlock.h> 21 #include <rte_spinlock.h> 22 #include <rte_string_fns.h> 23 #include <rte_eal_paging.h> 24 #include <rte_alarm.h> 25 #include <rte_cycles.h> 26 #include <rte_interrupts.h> 27 28 #include <mlx5_glue.h> 29 #include <mlx5_devx_cmds.h> 30 #include <mlx5_common.h> 31 #include <mlx5_common_os.h> 32 #include <mlx5_common_mp.h> 33 #include <mlx5_malloc.h> 34 35 #include "mlx5_defs.h" 36 #include "mlx5.h" 37 #include "mlx5_utils.h" 38 #include "mlx5_rxtx.h" 39 #include "mlx5_rx.h" 40 #include "mlx5_tx.h" 41 #include "mlx5_autoconf.h" 42 #include "mlx5_flow.h" 43 #include "mlx5_flow_os.h" 44 #include "rte_pmd_mlx5.h" 45 46 #define MLX5_ETH_DRIVER_NAME mlx5_eth 47 48 /* Device parameter to enable RX completion queue compression. */ 49 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en" 50 51 /* Device parameter to enable padding Rx packet to cacheline size. */ 52 #define MLX5_RXQ_PKT_PAD_EN "rxq_pkt_pad_en" 53 54 /* Device parameter to enable Multi-Packet Rx queue. */ 55 #define MLX5_RX_MPRQ_EN "mprq_en" 56 57 /* Device parameter to configure log 2 of the number of strides for MPRQ. */ 58 #define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num" 59 60 /* Device parameter to configure log 2 of the stride size for MPRQ. */ 61 #define MLX5_RX_MPRQ_LOG_STRIDE_SIZE "mprq_log_stride_size" 62 63 /* Device parameter to limit the size of memcpy'd packet for MPRQ. */ 64 #define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len" 65 66 /* Device parameter to set the minimum number of Rx queues to enable MPRQ. */ 67 #define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq" 68 69 /* Device parameter to configure inline send. Deprecated, ignored.*/ 70 #define MLX5_TXQ_INLINE "txq_inline" 71 72 /* Device parameter to limit packet size to inline with ordinary SEND. */ 73 #define MLX5_TXQ_INLINE_MAX "txq_inline_max" 74 75 /* Device parameter to configure minimal data size to inline. */ 76 #define MLX5_TXQ_INLINE_MIN "txq_inline_min" 77 78 /* Device parameter to limit packet size to inline with Enhanced MPW. */ 79 #define MLX5_TXQ_INLINE_MPW "txq_inline_mpw" 80 81 /* 82 * Device parameter to configure the number of TX queues threshold for 83 * enabling inline send. 84 */ 85 #define MLX5_TXQS_MIN_INLINE "txqs_min_inline" 86 87 /* 88 * Device parameter to configure the number of TX queues threshold for 89 * enabling vectorized Tx, deprecated, ignored (no vectorized Tx routines). 90 */ 91 #define MLX5_TXQS_MAX_VEC "txqs_max_vec" 92 93 /* Device parameter to enable multi-packet send WQEs. */ 94 #define MLX5_TXQ_MPW_EN "txq_mpw_en" 95 96 /* 97 * Device parameter to include 2 dsegs in the title WQEBB. 98 * Deprecated, ignored. 99 */ 100 #define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en" 101 102 /* 103 * Device parameter to limit the size of inlining packet. 104 * Deprecated, ignored. 105 */ 106 #define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len" 107 108 /* 109 * Device parameter to enable Tx scheduling on timestamps 110 * and specify the packet pacing granularity in nanoseconds. 111 */ 112 #define MLX5_TX_PP "tx_pp" 113 114 /* 115 * Device parameter to specify skew in nanoseconds on Tx datapath, 116 * it represents the time between SQ start WQE processing and 117 * appearing actual packet data on the wire. 118 */ 119 #define MLX5_TX_SKEW "tx_skew" 120 121 /* 122 * Device parameter to enable hardware Tx vector. 123 * Deprecated, ignored (no vectorized Tx routines anymore). 124 */ 125 #define MLX5_TX_VEC_EN "tx_vec_en" 126 127 /* Device parameter to enable hardware Rx vector. */ 128 #define MLX5_RX_VEC_EN "rx_vec_en" 129 130 /* Allow L3 VXLAN flow creation. */ 131 #define MLX5_L3_VXLAN_EN "l3_vxlan_en" 132 133 /* Activate DV E-Switch flow steering. */ 134 #define MLX5_DV_ESW_EN "dv_esw_en" 135 136 /* Activate DV flow steering. */ 137 #define MLX5_DV_FLOW_EN "dv_flow_en" 138 139 /* Enable extensive flow metadata support. */ 140 #define MLX5_DV_XMETA_EN "dv_xmeta_en" 141 142 /* Device parameter to let the user manage the lacp traffic of bonding device */ 143 #define MLX5_LACP_BY_USER "lacp_by_user" 144 145 /* Activate Netlink support in VF mode. */ 146 #define MLX5_VF_NL_EN "vf_nl_en" 147 148 /* Select port representors to instantiate. */ 149 #define MLX5_REPRESENTOR "representor" 150 151 /* Device parameter to configure the maximum number of dump files per queue. */ 152 #define MLX5_MAX_DUMP_FILES_NUM "max_dump_files_num" 153 154 /* Configure timeout of LRO session (in microseconds). */ 155 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec" 156 157 /* 158 * Device parameter to configure the total data buffer size for a single 159 * hairpin queue (logarithm value). 160 */ 161 #define MLX5_HP_BUF_SIZE "hp_buf_log_sz" 162 163 /* Flow memory reclaim mode. */ 164 #define MLX5_RECLAIM_MEM "reclaim_mem_mode" 165 166 /* Decap will be used or not. */ 167 #define MLX5_DECAP_EN "decap_en" 168 169 /* Device parameter to configure allow or prevent duplicate rules pattern. */ 170 #define MLX5_ALLOW_DUPLICATE_PATTERN "allow_duplicate_pattern" 171 172 /* Device parameter to configure the delay drop when creating Rxqs. */ 173 #define MLX5_DELAY_DROP "delay_drop" 174 175 /* Device parameter to create the fdb default rule in PMD */ 176 #define MLX5_FDB_DEFAULT_RULE_EN "fdb_def_rule_en" 177 178 /* HW steering counter configuration. */ 179 #define MLX5_HWS_CNT_SERVICE_CORE "service_core" 180 181 /* HW steering counter's query interval. */ 182 #define MLX5_HWS_CNT_CYCLE_TIME "svc_cycle_time" 183 184 /* Device parameter to control representor matching in ingress/egress flows with HWS. */ 185 #define MLX5_REPR_MATCHING_EN "repr_matching_en" 186 187 /* Shared memory between primary and secondary processes. */ 188 struct mlx5_shared_data *mlx5_shared_data; 189 190 /** Driver-specific log messages type. */ 191 int mlx5_logtype; 192 193 static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = 194 LIST_HEAD_INITIALIZER(); 195 static pthread_mutex_t mlx5_dev_ctx_list_mutex; 196 static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { 197 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 198 [MLX5_IPOOL_DECAP_ENCAP] = { 199 .size = sizeof(struct mlx5_flow_dv_encap_decap_resource), 200 .trunk_size = 64, 201 .grow_trunk = 3, 202 .grow_shift = 2, 203 .need_lock = 1, 204 .release_mem_en = 1, 205 .malloc = mlx5_malloc, 206 .free = mlx5_free, 207 .type = "mlx5_encap_decap_ipool", 208 }, 209 [MLX5_IPOOL_PUSH_VLAN] = { 210 .size = sizeof(struct mlx5_flow_dv_push_vlan_action_resource), 211 .trunk_size = 64, 212 .grow_trunk = 3, 213 .grow_shift = 2, 214 .need_lock = 1, 215 .release_mem_en = 1, 216 .malloc = mlx5_malloc, 217 .free = mlx5_free, 218 .type = "mlx5_push_vlan_ipool", 219 }, 220 [MLX5_IPOOL_TAG] = { 221 .size = sizeof(struct mlx5_flow_dv_tag_resource), 222 .trunk_size = 64, 223 .grow_trunk = 3, 224 .grow_shift = 2, 225 .need_lock = 1, 226 .release_mem_en = 0, 227 .per_core_cache = (1 << 16), 228 .malloc = mlx5_malloc, 229 .free = mlx5_free, 230 .type = "mlx5_tag_ipool", 231 }, 232 [MLX5_IPOOL_PORT_ID] = { 233 .size = sizeof(struct mlx5_flow_dv_port_id_action_resource), 234 .trunk_size = 64, 235 .grow_trunk = 3, 236 .grow_shift = 2, 237 .need_lock = 1, 238 .release_mem_en = 1, 239 .malloc = mlx5_malloc, 240 .free = mlx5_free, 241 .type = "mlx5_port_id_ipool", 242 }, 243 [MLX5_IPOOL_JUMP] = { 244 .size = sizeof(struct mlx5_flow_tbl_data_entry), 245 .trunk_size = 64, 246 .grow_trunk = 3, 247 .grow_shift = 2, 248 .need_lock = 1, 249 .release_mem_en = 1, 250 .malloc = mlx5_malloc, 251 .free = mlx5_free, 252 .type = "mlx5_jump_ipool", 253 }, 254 [MLX5_IPOOL_SAMPLE] = { 255 .size = sizeof(struct mlx5_flow_dv_sample_resource), 256 .trunk_size = 64, 257 .grow_trunk = 3, 258 .grow_shift = 2, 259 .need_lock = 1, 260 .release_mem_en = 1, 261 .malloc = mlx5_malloc, 262 .free = mlx5_free, 263 .type = "mlx5_sample_ipool", 264 }, 265 [MLX5_IPOOL_DEST_ARRAY] = { 266 .size = sizeof(struct mlx5_flow_dv_dest_array_resource), 267 .trunk_size = 64, 268 .grow_trunk = 3, 269 .grow_shift = 2, 270 .need_lock = 1, 271 .release_mem_en = 1, 272 .malloc = mlx5_malloc, 273 .free = mlx5_free, 274 .type = "mlx5_dest_array_ipool", 275 }, 276 [MLX5_IPOOL_TUNNEL_ID] = { 277 .size = sizeof(struct mlx5_flow_tunnel), 278 .trunk_size = MLX5_MAX_TUNNELS, 279 .need_lock = 1, 280 .release_mem_en = 1, 281 .type = "mlx5_tunnel_offload", 282 }, 283 [MLX5_IPOOL_TNL_TBL_ID] = { 284 .size = 0, 285 .need_lock = 1, 286 .type = "mlx5_flow_tnl_tbl_ipool", 287 }, 288 #endif 289 [MLX5_IPOOL_MTR] = { 290 /** 291 * The ipool index should grow continually from small to big, 292 * for meter idx, so not set grow_trunk to avoid meter index 293 * not jump continually. 294 */ 295 .size = sizeof(struct mlx5_legacy_flow_meter), 296 .trunk_size = 64, 297 .need_lock = 1, 298 .release_mem_en = 1, 299 .malloc = mlx5_malloc, 300 .free = mlx5_free, 301 .type = "mlx5_meter_ipool", 302 }, 303 [MLX5_IPOOL_MCP] = { 304 .size = sizeof(struct mlx5_flow_mreg_copy_resource), 305 .trunk_size = 64, 306 .grow_trunk = 3, 307 .grow_shift = 2, 308 .need_lock = 1, 309 .release_mem_en = 1, 310 .malloc = mlx5_malloc, 311 .free = mlx5_free, 312 .type = "mlx5_mcp_ipool", 313 }, 314 [MLX5_IPOOL_HRXQ] = { 315 .size = (sizeof(struct mlx5_hrxq) + MLX5_RSS_HASH_KEY_LEN), 316 .trunk_size = 64, 317 .grow_trunk = 3, 318 .grow_shift = 2, 319 .need_lock = 1, 320 .release_mem_en = 1, 321 .malloc = mlx5_malloc, 322 .free = mlx5_free, 323 .type = "mlx5_hrxq_ipool", 324 }, 325 [MLX5_IPOOL_MLX5_FLOW] = { 326 /* 327 * MLX5_IPOOL_MLX5_FLOW size varies for DV and VERBS flows. 328 * It set in run time according to PCI function configuration. 329 */ 330 .size = 0, 331 .trunk_size = 64, 332 .grow_trunk = 3, 333 .grow_shift = 2, 334 .need_lock = 1, 335 .release_mem_en = 0, 336 .per_core_cache = 1 << 19, 337 .malloc = mlx5_malloc, 338 .free = mlx5_free, 339 .type = "mlx5_flow_handle_ipool", 340 }, 341 [MLX5_IPOOL_RTE_FLOW] = { 342 .size = sizeof(struct rte_flow), 343 .trunk_size = 4096, 344 .need_lock = 1, 345 .release_mem_en = 1, 346 .malloc = mlx5_malloc, 347 .free = mlx5_free, 348 .type = "rte_flow_ipool", 349 }, 350 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID] = { 351 .size = 0, 352 .need_lock = 1, 353 .type = "mlx5_flow_rss_id_ipool", 354 }, 355 [MLX5_IPOOL_RSS_SHARED_ACTIONS] = { 356 .size = sizeof(struct mlx5_shared_action_rss), 357 .trunk_size = 64, 358 .grow_trunk = 3, 359 .grow_shift = 2, 360 .need_lock = 1, 361 .release_mem_en = 1, 362 .malloc = mlx5_malloc, 363 .free = mlx5_free, 364 .type = "mlx5_shared_action_rss", 365 }, 366 [MLX5_IPOOL_MTR_POLICY] = { 367 /** 368 * The ipool index should grow continually from small to big, 369 * for policy idx, so not set grow_trunk to avoid policy index 370 * not jump continually. 371 */ 372 .size = sizeof(struct mlx5_flow_meter_sub_policy), 373 .trunk_size = 64, 374 .need_lock = 1, 375 .release_mem_en = 1, 376 .malloc = mlx5_malloc, 377 .free = mlx5_free, 378 .type = "mlx5_meter_policy_ipool", 379 }, 380 }; 381 382 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512 383 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16 384 385 #define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024 386 387 #define MLX5_RXQ_ENH_CQE_COMP_MASK 0x80 388 389 /** 390 * Decide whether representor ID is a HPF(host PF) port on BF2. 391 * 392 * @param dev 393 * Pointer to Ethernet device structure. 394 * 395 * @return 396 * Non-zero if HPF, otherwise 0. 397 */ 398 bool 399 mlx5_is_hpf(struct rte_eth_dev *dev) 400 { 401 struct mlx5_priv *priv = dev->data->dev_private; 402 uint16_t repr = MLX5_REPRESENTOR_REPR(priv->representor_id); 403 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 404 405 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_VF && 406 MLX5_REPRESENTOR_REPR(-1) == repr; 407 } 408 409 /** 410 * Decide whether representor ID is a SF port representor. 411 * 412 * @param dev 413 * Pointer to Ethernet device structure. 414 * 415 * @return 416 * Non-zero if HPF, otherwise 0. 417 */ 418 bool 419 mlx5_is_sf_repr(struct rte_eth_dev *dev) 420 { 421 struct mlx5_priv *priv = dev->data->dev_private; 422 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 423 424 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_SF; 425 } 426 427 /** 428 * Initialize the ASO aging management structure. 429 * 430 * @param[in] sh 431 * Pointer to mlx5_dev_ctx_shared object to free 432 * 433 * @return 434 * 0 on success, a negative errno value otherwise and rte_errno is set. 435 */ 436 int 437 mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh) 438 { 439 int err; 440 441 if (sh->aso_age_mng) 442 return 0; 443 sh->aso_age_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->aso_age_mng), 444 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 445 if (!sh->aso_age_mng) { 446 DRV_LOG(ERR, "aso_age_mng allocation was failed."); 447 rte_errno = ENOMEM; 448 return -ENOMEM; 449 } 450 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_FLOW_HIT, 1); 451 if (err) { 452 mlx5_free(sh->aso_age_mng); 453 return -1; 454 } 455 rte_rwlock_init(&sh->aso_age_mng->resize_rwl); 456 rte_spinlock_init(&sh->aso_age_mng->free_sl); 457 LIST_INIT(&sh->aso_age_mng->free); 458 return 0; 459 } 460 461 /** 462 * Close and release all the resources of the ASO aging management structure. 463 * 464 * @param[in] sh 465 * Pointer to mlx5_dev_ctx_shared object to free. 466 */ 467 static void 468 mlx5_flow_aso_age_mng_close(struct mlx5_dev_ctx_shared *sh) 469 { 470 int i, j; 471 472 mlx5_aso_flow_hit_queue_poll_stop(sh); 473 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_FLOW_HIT); 474 if (sh->aso_age_mng->pools) { 475 struct mlx5_aso_age_pool *pool; 476 477 for (i = 0; i < sh->aso_age_mng->next; ++i) { 478 pool = sh->aso_age_mng->pools[i]; 479 claim_zero(mlx5_devx_cmd_destroy 480 (pool->flow_hit_aso_obj)); 481 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) 482 if (pool->actions[j].dr_action) 483 claim_zero 484 (mlx5_flow_os_destroy_flow_action 485 (pool->actions[j].dr_action)); 486 mlx5_free(pool); 487 } 488 mlx5_free(sh->aso_age_mng->pools); 489 } 490 mlx5_free(sh->aso_age_mng); 491 } 492 493 /** 494 * Initialize the shared aging list information per port. 495 * 496 * @param[in] sh 497 * Pointer to mlx5_dev_ctx_shared object. 498 */ 499 static void 500 mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh) 501 { 502 uint32_t i; 503 struct mlx5_age_info *age_info; 504 505 /* 506 * In HW steering, aging information structure is initialized later 507 * during configure function. 508 */ 509 if (sh->config.dv_flow_en == 2) 510 return; 511 for (i = 0; i < sh->max_port; i++) { 512 age_info = &sh->port[i].age_info; 513 age_info->flags = 0; 514 TAILQ_INIT(&age_info->aged_counters); 515 LIST_INIT(&age_info->aged_aso); 516 rte_spinlock_init(&age_info->aged_sl); 517 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 518 } 519 } 520 521 /** 522 * DV flow counter mode detect and config. 523 * 524 * @param dev 525 * Pointer to rte_eth_dev structure. 526 * 527 */ 528 void 529 mlx5_flow_counter_mode_config(struct rte_eth_dev *dev __rte_unused) 530 { 531 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 532 struct mlx5_priv *priv = dev->data->dev_private; 533 struct mlx5_dev_ctx_shared *sh = priv->sh; 534 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 535 bool fallback; 536 537 #ifndef HAVE_IBV_DEVX_ASYNC 538 fallback = true; 539 #else 540 fallback = false; 541 if (!sh->cdev->config.devx || !sh->config.dv_flow_en || 542 !hca_attr->flow_counters_dump || 543 !(hca_attr->flow_counter_bulk_alloc_bitmap & 0x4) || 544 (mlx5_flow_dv_discover_counter_offset_support(dev) == -ENOTSUP)) 545 fallback = true; 546 #endif 547 if (fallback) 548 DRV_LOG(INFO, "Use fall-back DV counter management. Flow " 549 "counter dump:%d, bulk_alloc_bitmap:0x%hhx.", 550 hca_attr->flow_counters_dump, 551 hca_attr->flow_counter_bulk_alloc_bitmap); 552 /* Initialize fallback mode only on the port initializes sh. */ 553 if (sh->refcnt == 1) 554 sh->sws_cmng.counter_fallback = fallback; 555 else if (fallback != sh->sws_cmng.counter_fallback) 556 DRV_LOG(WARNING, "Port %d in sh has different fallback mode " 557 "with others:%d.", PORT_ID(priv), fallback); 558 #endif 559 } 560 561 /** 562 * Initialize the counters management structure. 563 * 564 * @param[in] sh 565 * Pointer to mlx5_dev_ctx_shared object to free 566 * 567 * @return 568 * 0 on success, otherwise negative errno value and rte_errno is set. 569 */ 570 static int 571 mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh) 572 { 573 int i, j; 574 575 if (sh->config.dv_flow_en < 2) { 576 void *pools; 577 578 pools = mlx5_malloc(MLX5_MEM_ZERO, 579 sizeof(struct mlx5_flow_counter_pool *) * 580 MLX5_COUNTER_POOLS_MAX_NUM, 581 0, SOCKET_ID_ANY); 582 if (!pools) { 583 DRV_LOG(ERR, 584 "Counter management allocation was failed."); 585 rte_errno = ENOMEM; 586 return -rte_errno; 587 } 588 memset(&sh->sws_cmng, 0, sizeof(sh->sws_cmng)); 589 TAILQ_INIT(&sh->sws_cmng.flow_counters); 590 sh->sws_cmng.min_id = MLX5_CNT_BATCH_OFFSET; 591 sh->sws_cmng.max_id = -1; 592 sh->sws_cmng.last_pool_idx = POOL_IDX_INVALID; 593 sh->sws_cmng.pools = pools; 594 rte_spinlock_init(&sh->sws_cmng.pool_update_sl); 595 for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) { 596 TAILQ_INIT(&sh->sws_cmng.counters[i]); 597 rte_spinlock_init(&sh->sws_cmng.csl[i]); 598 } 599 } else { 600 struct mlx5_hca_attr *attr = &sh->cdev->config.hca_attr; 601 uint32_t fw_max_nb_cnts = attr->max_flow_counter; 602 uint8_t log_dcs = log2above(fw_max_nb_cnts) - 1; 603 uint32_t max_nb_cnts = 0; 604 605 for (i = 0, j = 0; j < MLX5_HWS_CNT_DCS_NUM; ++i) { 606 int log_dcs_i = log_dcs - i; 607 608 if (log_dcs_i < 0) 609 break; 610 if ((max_nb_cnts | RTE_BIT32(log_dcs_i)) > 611 fw_max_nb_cnts) 612 continue; 613 max_nb_cnts |= RTE_BIT32(log_dcs_i); 614 j++; 615 } 616 sh->hws_max_log_bulk_sz = log_dcs; 617 sh->hws_max_nb_counters = max_nb_cnts; 618 } 619 return 0; 620 } 621 622 /** 623 * Destroy all the resources allocated for a counter memory management. 624 * 625 * @param[in] mng 626 * Pointer to the memory management structure. 627 */ 628 static void 629 mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng) 630 { 631 uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data; 632 633 LIST_REMOVE(mng, next); 634 mlx5_os_wrapped_mkey_destroy(&mng->wm); 635 mlx5_free(mem); 636 } 637 638 /** 639 * Close and release all the resources of the counters management. 640 * 641 * @param[in] sh 642 * Pointer to mlx5_dev_ctx_shared object to free. 643 */ 644 static void 645 mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) 646 { 647 struct mlx5_counter_stats_mem_mng *mng; 648 int i, j; 649 int retries = 1024; 650 651 rte_errno = 0; 652 while (--retries) { 653 rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh); 654 if (rte_errno != EINPROGRESS) 655 break; 656 rte_pause(); 657 } 658 659 if (sh->sws_cmng.pools) { 660 struct mlx5_flow_counter_pool *pool; 661 uint16_t n_valid = sh->sws_cmng.n_valid; 662 bool fallback = sh->sws_cmng.counter_fallback; 663 664 for (i = 0; i < n_valid; ++i) { 665 pool = sh->sws_cmng.pools[i]; 666 if (!fallback && pool->min_dcs) 667 claim_zero(mlx5_devx_cmd_destroy 668 (pool->min_dcs)); 669 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) { 670 struct mlx5_flow_counter *cnt = 671 MLX5_POOL_GET_CNT(pool, j); 672 673 if (cnt->action) 674 claim_zero 675 (mlx5_flow_os_destroy_flow_action 676 (cnt->action)); 677 if (fallback && cnt->dcs_when_free) 678 claim_zero(mlx5_devx_cmd_destroy 679 (cnt->dcs_when_free)); 680 } 681 mlx5_free(pool); 682 } 683 mlx5_free(sh->sws_cmng.pools); 684 } 685 mng = LIST_FIRST(&sh->sws_cmng.mem_mngs); 686 while (mng) { 687 mlx5_flow_destroy_counter_stat_mem_mng(mng); 688 mng = LIST_FIRST(&sh->sws_cmng.mem_mngs); 689 } 690 memset(&sh->sws_cmng, 0, sizeof(sh->sws_cmng)); 691 } 692 693 /** 694 * Initialize the aso flow meters management structure. 695 * 696 * @param[in] sh 697 * Pointer to mlx5_dev_ctx_shared object to free 698 */ 699 int 700 mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh) 701 { 702 if (!sh->mtrmng) { 703 sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, 704 sizeof(*sh->mtrmng), 705 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 706 if (!sh->mtrmng) { 707 DRV_LOG(ERR, 708 "meter management allocation was failed."); 709 rte_errno = ENOMEM; 710 return -ENOMEM; 711 } 712 if (sh->meter_aso_en) { 713 rte_spinlock_init(&sh->mtrmng->pools_mng.mtrsl); 714 rte_rwlock_init(&sh->mtrmng->pools_mng.resize_mtrwl); 715 LIST_INIT(&sh->mtrmng->pools_mng.meters); 716 } 717 sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 718 } 719 return 0; 720 } 721 722 /** 723 * Close and release all the resources of 724 * the ASO flow meter management structure. 725 * 726 * @param[in] sh 727 * Pointer to mlx5_dev_ctx_shared object to free. 728 */ 729 static void 730 mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) 731 { 732 struct mlx5_aso_mtr_pool *mtr_pool; 733 struct mlx5_flow_mtr_mng *mtrmng = sh->mtrmng; 734 uint32_t idx; 735 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 736 struct mlx5_aso_mtr *aso_mtr; 737 int i; 738 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 739 740 if (sh->meter_aso_en) { 741 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); 742 idx = mtrmng->pools_mng.n_valid; 743 while (idx--) { 744 mtr_pool = mtrmng->pools_mng.pools[idx]; 745 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 746 for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { 747 aso_mtr = &mtr_pool->mtrs[i]; 748 if (aso_mtr->fm.meter_action_g) 749 claim_zero 750 (mlx5_glue->destroy_flow_action 751 (aso_mtr->fm.meter_action_g)); 752 if (aso_mtr->fm.meter_action_y) 753 claim_zero 754 (mlx5_glue->destroy_flow_action 755 (aso_mtr->fm.meter_action_y)); 756 } 757 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 758 claim_zero(mlx5_devx_cmd_destroy 759 (mtr_pool->devx_obj)); 760 mtrmng->pools_mng.n_valid--; 761 mlx5_free(mtr_pool); 762 } 763 mlx5_free(sh->mtrmng->pools_mng.pools); 764 } 765 mlx5_free(sh->mtrmng); 766 sh->mtrmng = NULL; 767 } 768 769 /* Send FLOW_AGED event if needed. */ 770 void 771 mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) 772 { 773 struct mlx5_age_info *age_info; 774 uint32_t i; 775 776 for (i = 0; i < sh->max_port; i++) { 777 age_info = &sh->port[i].age_info; 778 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 779 continue; 780 MLX5_AGE_UNSET(age_info, MLX5_AGE_EVENT_NEW); 781 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) { 782 MLX5_AGE_UNSET(age_info, MLX5_AGE_TRIGGER); 783 rte_eth_dev_callback_process 784 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 785 RTE_ETH_EVENT_FLOW_AGED, NULL); 786 } 787 } 788 } 789 790 /* 791 * Initialize the ASO connection tracking structure. 792 * 793 * @param[in] sh 794 * Pointer to mlx5_dev_ctx_shared object. 795 * 796 * @return 797 * 0 on success, a negative errno value otherwise and rte_errno is set. 798 */ 799 int 800 mlx5_flow_aso_ct_mng_init(struct mlx5_dev_ctx_shared *sh) 801 { 802 int err; 803 804 if (sh->ct_mng) 805 return 0; 806 sh->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->ct_mng) + 807 sizeof(struct mlx5_aso_sq) * MLX5_ASO_CT_SQ_NUM, 808 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 809 if (!sh->ct_mng) { 810 DRV_LOG(ERR, "ASO CT management allocation failed."); 811 rte_errno = ENOMEM; 812 return -rte_errno; 813 } 814 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_CONNECTION_TRACKING, MLX5_ASO_CT_SQ_NUM); 815 if (err) { 816 mlx5_free(sh->ct_mng); 817 /* rte_errno should be extracted from the failure. */ 818 rte_errno = EINVAL; 819 return -rte_errno; 820 } 821 rte_spinlock_init(&sh->ct_mng->ct_sl); 822 rte_rwlock_init(&sh->ct_mng->resize_rwl); 823 LIST_INIT(&sh->ct_mng->free_cts); 824 return 0; 825 } 826 827 /* 828 * Close and release all the resources of the 829 * ASO connection tracking management structure. 830 * 831 * @param[in] sh 832 * Pointer to mlx5_dev_ctx_shared object to free. 833 */ 834 static void 835 mlx5_flow_aso_ct_mng_close(struct mlx5_dev_ctx_shared *sh) 836 { 837 struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng; 838 struct mlx5_aso_ct_pool *ct_pool; 839 struct mlx5_aso_ct_action *ct; 840 uint32_t idx; 841 uint32_t val; 842 uint32_t cnt; 843 int i; 844 845 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_CONNECTION_TRACKING); 846 idx = mng->next; 847 while (idx--) { 848 cnt = 0; 849 ct_pool = mng->pools[idx]; 850 for (i = 0; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) { 851 ct = &ct_pool->actions[i]; 852 val = __atomic_fetch_sub(&ct->refcnt, 1, 853 __ATOMIC_RELAXED); 854 MLX5_ASSERT(val == 1); 855 if (val > 1) 856 cnt++; 857 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT 858 if (ct->dr_action_orig) 859 claim_zero(mlx5_glue->destroy_flow_action 860 (ct->dr_action_orig)); 861 if (ct->dr_action_rply) 862 claim_zero(mlx5_glue->destroy_flow_action 863 (ct->dr_action_rply)); 864 #endif 865 } 866 claim_zero(mlx5_devx_cmd_destroy(ct_pool->devx_obj)); 867 if (cnt) { 868 DRV_LOG(DEBUG, "%u ASO CT objects are being used in the pool %u", 869 cnt, i); 870 } 871 mlx5_free(ct_pool); 872 /* in case of failure. */ 873 mng->next--; 874 } 875 mlx5_free(mng->pools); 876 mlx5_free(mng); 877 /* Management structure must be cleared to 0s during allocation. */ 878 sh->ct_mng = NULL; 879 } 880 881 /** 882 * Initialize the flow resources' indexed mempool. 883 * 884 * @param[in] sh 885 * Pointer to mlx5_dev_ctx_shared object. 886 */ 887 static void 888 mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh) 889 { 890 uint8_t i; 891 struct mlx5_indexed_pool_config cfg; 892 893 for (i = 0; i < MLX5_IPOOL_MAX; ++i) { 894 cfg = mlx5_ipool_cfg[i]; 895 switch (i) { 896 default: 897 break; 898 /* 899 * Set MLX5_IPOOL_MLX5_FLOW ipool size 900 * according to PCI function flow configuration. 901 */ 902 case MLX5_IPOOL_MLX5_FLOW: 903 cfg.size = sh->config.dv_flow_en ? 904 sizeof(struct mlx5_flow_handle) : 905 MLX5_FLOW_HANDLE_VERBS_SIZE; 906 break; 907 } 908 if (sh->config.reclaim_mode) { 909 cfg.release_mem_en = 1; 910 cfg.per_core_cache = 0; 911 } else { 912 cfg.release_mem_en = 0; 913 } 914 sh->ipool[i] = mlx5_ipool_create(&cfg); 915 } 916 } 917 918 919 /** 920 * Release the flow resources' indexed mempool. 921 * 922 * @param[in] sh 923 * Pointer to mlx5_dev_ctx_shared object. 924 */ 925 static void 926 mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh) 927 { 928 uint8_t i; 929 930 for (i = 0; i < MLX5_IPOOL_MAX; ++i) 931 mlx5_ipool_destroy(sh->ipool[i]); 932 for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i) 933 if (sh->mdh_ipools[i]) 934 mlx5_ipool_destroy(sh->mdh_ipools[i]); 935 } 936 937 /* 938 * Check if dynamic flex parser for eCPRI already exists. 939 * 940 * @param dev 941 * Pointer to Ethernet device structure. 942 * 943 * @return 944 * true on exists, false on not. 945 */ 946 bool 947 mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev) 948 { 949 struct mlx5_priv *priv = dev->data->dev_private; 950 struct mlx5_ecpri_parser_profile *prf = &priv->sh->ecpri_parser; 951 952 return !!prf->obj; 953 } 954 955 /* 956 * Allocation of a flex parser for eCPRI. Once created, this parser related 957 * resources will be held until the device is closed. 958 * 959 * @param dev 960 * Pointer to Ethernet device structure. 961 * 962 * @return 963 * 0 on success, a negative errno value otherwise and rte_errno is set. 964 */ 965 int 966 mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev) 967 { 968 struct mlx5_priv *priv = dev->data->dev_private; 969 struct mlx5_ecpri_parser_profile *prf = &priv->sh->ecpri_parser; 970 struct mlx5_devx_graph_node_attr node = { 971 .modify_field_select = 0, 972 }; 973 uint32_t ids[8]; 974 int ret; 975 976 if (!priv->sh->cdev->config.hca_attr.parse_graph_flex_node) { 977 DRV_LOG(ERR, "Dynamic flex parser is not supported " 978 "for device %s.", priv->dev_data->name); 979 return -ENOTSUP; 980 } 981 node.header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED; 982 /* 8 bytes now: 4B common header + 4B message body header. */ 983 node.header_length_base_value = 0x8; 984 /* After MAC layer: Ether / VLAN. */ 985 node.in[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_MAC; 986 /* Type of compared condition should be 0xAEFE in the L2 layer. */ 987 node.in[0].compare_condition_value = RTE_ETHER_TYPE_ECPRI; 988 /* Sample #0: type in common header. */ 989 node.sample[0].flow_match_sample_en = 1; 990 /* Fixed offset. */ 991 node.sample[0].flow_match_sample_offset_mode = 0x0; 992 /* Only the 2nd byte will be used. */ 993 node.sample[0].flow_match_sample_field_base_offset = 0x0; 994 /* Sample #1: message payload. */ 995 node.sample[1].flow_match_sample_en = 1; 996 /* Fixed offset. */ 997 node.sample[1].flow_match_sample_offset_mode = 0x0; 998 /* 999 * Only the first two bytes will be used right now, and its offset will 1000 * start after the common header that with the length of a DW(u32). 1001 */ 1002 node.sample[1].flow_match_sample_field_base_offset = sizeof(uint32_t); 1003 prf->obj = mlx5_devx_cmd_create_flex_parser(priv->sh->cdev->ctx, &node); 1004 if (!prf->obj) { 1005 DRV_LOG(ERR, "Failed to create flex parser node object."); 1006 return (rte_errno == 0) ? -ENODEV : -rte_errno; 1007 } 1008 prf->num = 2; 1009 ret = mlx5_devx_cmd_query_parse_samples(prf->obj, ids, prf->num, NULL); 1010 if (ret) { 1011 DRV_LOG(ERR, "Failed to query sample IDs."); 1012 goto error; 1013 } 1014 prf->offset[0] = 0x0; 1015 prf->offset[1] = sizeof(uint32_t); 1016 prf->ids[0] = ids[0]; 1017 prf->ids[1] = ids[1]; 1018 return 0; 1019 error: 1020 mlx5_devx_cmd_destroy(prf->obj); 1021 return (rte_errno == 0) ? -ENODEV : -rte_errno; 1022 } 1023 1024 /* 1025 * Destroy the flex parser node, including the parser itself, input / output 1026 * arcs and DW samples. Resources could be reused then. 1027 * 1028 * @param dev 1029 * Pointer to Ethernet device structure. 1030 */ 1031 static void 1032 mlx5_flex_parser_ecpri_release(struct rte_eth_dev *dev) 1033 { 1034 struct mlx5_priv *priv = dev->data->dev_private; 1035 struct mlx5_ecpri_parser_profile *prf = &priv->sh->ecpri_parser; 1036 1037 if (prf->obj) 1038 mlx5_devx_cmd_destroy(prf->obj); 1039 prf->obj = NULL; 1040 } 1041 1042 /* 1043 * Allocation of a flex parser for srh. Once refcnt is zero, the resources held 1044 * by this parser will be freed. 1045 * @param dev 1046 * Pointer to Ethernet device structure. 1047 * 1048 * @return 1049 * 0 on success, a negative errno value otherwise and rte_errno is set. 1050 */ 1051 int 1052 mlx5_alloc_srh_flex_parser(struct rte_eth_dev *dev) 1053 { 1054 struct mlx5_devx_graph_node_attr node = { 1055 .modify_field_select = 0, 1056 }; 1057 uint32_t ids[MLX5_GRAPH_NODE_SAMPLE_NUM]; 1058 struct mlx5_priv *priv = dev->data->dev_private; 1059 struct mlx5_common_dev_config *config = &priv->sh->cdev->config; 1060 struct mlx5_hca_flex_attr *attr = &priv->sh->cdev->config.hca_attr.flex; 1061 void *fp = NULL, *ibv_ctx = priv->sh->cdev->ctx; 1062 int ret; 1063 1064 memset(ids, 0xff, sizeof(ids)); 1065 if (!config->hca_attr.parse_graph_flex_node || 1066 !config->hca_attr.flex.query_match_sample_info) { 1067 DRV_LOG(ERR, "Dynamic flex parser is not supported on HWS"); 1068 return -ENOTSUP; 1069 } 1070 if (__atomic_fetch_add(&priv->sh->srh_flex_parser.refcnt, 1, __ATOMIC_RELAXED) + 1 > 1) 1071 return 0; 1072 priv->sh->srh_flex_parser.flex.devx_fp = mlx5_malloc(MLX5_MEM_ZERO, 1073 sizeof(struct mlx5_flex_parser_devx), 0, SOCKET_ID_ANY); 1074 if (!priv->sh->srh_flex_parser.flex.devx_fp) 1075 return -ENOMEM; 1076 node.header_length_mode = MLX5_GRAPH_NODE_LEN_FIELD; 1077 /* Srv6 first two DW are not counted in. */ 1078 node.header_length_base_value = 0x8; 1079 /* The unit is uint64_t. */ 1080 node.header_length_field_shift = 0x3; 1081 /* Header length is the 2nd byte. */ 1082 node.header_length_field_offset = 0x8; 1083 if (attr->header_length_mask_width < 8) 1084 node.header_length_field_offset += 8 - attr->header_length_mask_width; 1085 node.header_length_field_mask = 0xF; 1086 /* One byte next header protocol. */ 1087 node.next_header_field_size = 0x8; 1088 node.in[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_IP; 1089 node.in[0].compare_condition_value = IPPROTO_ROUTING; 1090 node.sample[0].flow_match_sample_en = 1; 1091 /* First come first serve no matter inner or outer. */ 1092 node.sample[0].flow_match_sample_tunnel_mode = MLX5_GRAPH_SAMPLE_TUNNEL_FIRST; 1093 node.sample[0].flow_match_sample_offset_mode = MLX5_GRAPH_SAMPLE_OFFSET_FIXED; 1094 node.out[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_TCP; 1095 node.out[0].compare_condition_value = IPPROTO_TCP; 1096 node.out[1].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_UDP; 1097 node.out[1].compare_condition_value = IPPROTO_UDP; 1098 node.out[2].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_IPV6; 1099 node.out[2].compare_condition_value = IPPROTO_IPV6; 1100 fp = mlx5_devx_cmd_create_flex_parser(ibv_ctx, &node); 1101 if (!fp) { 1102 DRV_LOG(ERR, "Failed to create flex parser node object."); 1103 goto error; 1104 } 1105 priv->sh->srh_flex_parser.flex.devx_fp->devx_obj = fp; 1106 priv->sh->srh_flex_parser.flex.mapnum = 1; 1107 priv->sh->srh_flex_parser.flex.devx_fp->num_samples = 1; 1108 1109 ret = mlx5_devx_cmd_query_parse_samples(fp, ids, priv->sh->srh_flex_parser.flex.mapnum, 1110 &priv->sh->srh_flex_parser.flex.devx_fp->anchor_id); 1111 if (ret) { 1112 DRV_LOG(ERR, "Failed to query sample IDs."); 1113 goto error; 1114 } 1115 ret = mlx5_devx_cmd_match_sample_info_query(ibv_ctx, ids[0], 1116 &priv->sh->srh_flex_parser.flex.devx_fp->sample_info[0]); 1117 if (ret) { 1118 DRV_LOG(ERR, "Failed to query sample id information."); 1119 goto error; 1120 } 1121 return 0; 1122 error: 1123 if (fp) 1124 mlx5_devx_cmd_destroy(fp); 1125 if (priv->sh->srh_flex_parser.flex.devx_fp) 1126 mlx5_free(priv->sh->srh_flex_parser.flex.devx_fp); 1127 return (rte_errno == 0) ? -ENODEV : -rte_errno; 1128 } 1129 1130 /* 1131 * Destroy the flex parser node, including the parser itself, input / output 1132 * arcs and DW samples. Resources could be reused then. 1133 * 1134 * @param dev 1135 * Pointer to Ethernet device structure 1136 */ 1137 void 1138 mlx5_free_srh_flex_parser(struct rte_eth_dev *dev) 1139 { 1140 struct mlx5_priv *priv = dev->data->dev_private; 1141 struct mlx5_internal_flex_parser_profile *fp = &priv->sh->srh_flex_parser; 1142 1143 if (__atomic_fetch_sub(&fp->refcnt, 1, __ATOMIC_RELAXED) - 1) 1144 return; 1145 mlx5_devx_cmd_destroy(fp->flex.devx_fp->devx_obj); 1146 mlx5_free(fp->flex.devx_fp); 1147 fp->flex.devx_fp = NULL; 1148 } 1149 1150 uint32_t 1151 mlx5_get_supported_sw_parsing_offloads(const struct mlx5_hca_attr *attr) 1152 { 1153 uint32_t sw_parsing_offloads = 0; 1154 1155 if (attr->swp) { 1156 sw_parsing_offloads |= MLX5_SW_PARSING_CAP; 1157 if (attr->swp_csum) 1158 sw_parsing_offloads |= MLX5_SW_PARSING_CSUM_CAP; 1159 1160 if (attr->swp_lso) 1161 sw_parsing_offloads |= MLX5_SW_PARSING_TSO_CAP; 1162 } 1163 return sw_parsing_offloads; 1164 } 1165 1166 uint32_t 1167 mlx5_get_supported_tunneling_offloads(const struct mlx5_hca_attr *attr) 1168 { 1169 uint32_t tn_offloads = 0; 1170 1171 if (attr->tunnel_stateless_vxlan) 1172 tn_offloads |= MLX5_TUNNELED_OFFLOADS_VXLAN_CAP; 1173 if (attr->tunnel_stateless_gre) 1174 tn_offloads |= MLX5_TUNNELED_OFFLOADS_GRE_CAP; 1175 if (attr->tunnel_stateless_geneve_rx) 1176 tn_offloads |= MLX5_TUNNELED_OFFLOADS_GENEVE_CAP; 1177 return tn_offloads; 1178 } 1179 1180 /* Fill all fields of UAR structure. */ 1181 static int 1182 mlx5_rxtx_uars_prepare(struct mlx5_dev_ctx_shared *sh) 1183 { 1184 int ret; 1185 1186 ret = mlx5_devx_uar_prepare(sh->cdev, &sh->tx_uar); 1187 if (ret) { 1188 DRV_LOG(ERR, "Failed to prepare Tx DevX UAR."); 1189 return -rte_errno; 1190 } 1191 MLX5_ASSERT(sh->tx_uar.obj); 1192 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->tx_uar.obj)); 1193 ret = mlx5_devx_uar_prepare(sh->cdev, &sh->rx_uar); 1194 if (ret) { 1195 DRV_LOG(ERR, "Failed to prepare Rx DevX UAR."); 1196 mlx5_devx_uar_release(&sh->tx_uar); 1197 return -rte_errno; 1198 } 1199 MLX5_ASSERT(sh->rx_uar.obj); 1200 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->rx_uar.obj)); 1201 return 0; 1202 } 1203 1204 static void 1205 mlx5_rxtx_uars_release(struct mlx5_dev_ctx_shared *sh) 1206 { 1207 mlx5_devx_uar_release(&sh->rx_uar); 1208 mlx5_devx_uar_release(&sh->tx_uar); 1209 } 1210 1211 /** 1212 * rte_mempool_walk() callback to unregister Rx mempools. 1213 * It used when implicit mempool registration is disabled. 1214 * 1215 * @param mp 1216 * The mempool being walked. 1217 * @param arg 1218 * Pointer to the device shared context. 1219 */ 1220 static void 1221 mlx5_dev_ctx_shared_rx_mempool_unregister_cb(struct rte_mempool *mp, void *arg) 1222 { 1223 struct mlx5_dev_ctx_shared *sh = arg; 1224 1225 mlx5_dev_mempool_unregister(sh->cdev, mp); 1226 } 1227 1228 /** 1229 * Callback used when implicit mempool registration is disabled 1230 * in order to track Rx mempool destruction. 1231 * 1232 * @param event 1233 * Mempool life cycle event. 1234 * @param mp 1235 * An Rx mempool registered explicitly when the port is started. 1236 * @param arg 1237 * Pointer to a device shared context. 1238 */ 1239 static void 1240 mlx5_dev_ctx_shared_rx_mempool_event_cb(enum rte_mempool_event event, 1241 struct rte_mempool *mp, void *arg) 1242 { 1243 struct mlx5_dev_ctx_shared *sh = arg; 1244 1245 if (event == RTE_MEMPOOL_EVENT_DESTROY) 1246 mlx5_dev_mempool_unregister(sh->cdev, mp); 1247 } 1248 1249 int 1250 mlx5_dev_ctx_shared_mempool_subscribe(struct rte_eth_dev *dev) 1251 { 1252 struct mlx5_priv *priv = dev->data->dev_private; 1253 struct mlx5_dev_ctx_shared *sh = priv->sh; 1254 int ret; 1255 1256 /* Check if we only need to track Rx mempool destruction. */ 1257 if (!sh->cdev->config.mr_mempool_reg_en) { 1258 ret = rte_mempool_event_callback_register 1259 (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); 1260 return ret == 0 || rte_errno == EEXIST ? 0 : ret; 1261 } 1262 return mlx5_dev_mempool_subscribe(sh->cdev); 1263 } 1264 1265 /** 1266 * Set up multiple TISs with different affinities according to 1267 * number of bonding ports 1268 * 1269 * @param priv 1270 * Pointer of shared context. 1271 * 1272 * @return 1273 * Zero on success, -1 otherwise. 1274 */ 1275 static int 1276 mlx5_setup_tis(struct mlx5_dev_ctx_shared *sh) 1277 { 1278 struct mlx5_devx_lag_context lag_ctx = { 0 }; 1279 struct mlx5_devx_tis_attr tis_attr = { 0 }; 1280 int i; 1281 1282 tis_attr.transport_domain = sh->td->id; 1283 if (sh->bond.n_port) { 1284 if (!mlx5_devx_cmd_query_lag(sh->cdev->ctx, &lag_ctx)) { 1285 sh->lag.tx_remap_affinity[0] = 1286 lag_ctx.tx_remap_affinity_1; 1287 sh->lag.tx_remap_affinity[1] = 1288 lag_ctx.tx_remap_affinity_2; 1289 sh->lag.affinity_mode = lag_ctx.port_select_mode; 1290 } else { 1291 DRV_LOG(ERR, "Failed to query lag affinity."); 1292 return -1; 1293 } 1294 if (sh->lag.affinity_mode == MLX5_LAG_MODE_TIS) 1295 DRV_LOG(DEBUG, "LAG number of ports : %d, affinity_1 & 2 : pf%d & %d.\n", 1296 sh->bond.n_port, lag_ctx.tx_remap_affinity_1, 1297 lag_ctx.tx_remap_affinity_2); 1298 else if (sh->lag.affinity_mode == MLX5_LAG_MODE_HASH) 1299 DRV_LOG(INFO, "Device %s enabled HW hash based LAG.", 1300 sh->ibdev_name); 1301 } 1302 for (i = 0; i <= sh->bond.n_port; i++) { 1303 /* 1304 * lag_tx_port_affinity: 0 auto-selection, 1 PF1, 2 PF2 vice versa. 1305 * Each TIS binds to one PF by setting lag_tx_port_affinity (> 0). 1306 * Once LAG enabled, we create multiple TISs and bind each one to 1307 * different PFs, then TIS[i+1] gets affinity i+1 and goes to PF i+1. 1308 * TIS[0] is reserved for HW Hash mode. 1309 */ 1310 tis_attr.lag_tx_port_affinity = i; 1311 sh->tis[i] = mlx5_devx_cmd_create_tis(sh->cdev->ctx, &tis_attr); 1312 if (!sh->tis[i]) { 1313 DRV_LOG(ERR, "Failed to create TIS %d/%d for [bonding] device" 1314 " %s.", i, sh->bond.n_port, 1315 sh->ibdev_name); 1316 return -1; 1317 } 1318 } 1319 return 0; 1320 } 1321 1322 /** 1323 * Verify and store value for share device argument. 1324 * 1325 * @param[in] key 1326 * Key argument to verify. 1327 * @param[in] val 1328 * Value associated with key. 1329 * @param opaque 1330 * User data. 1331 * 1332 * @return 1333 * 0 on success, a negative errno value otherwise and rte_errno is set. 1334 */ 1335 static int 1336 mlx5_dev_args_check_handler(const char *key, const char *val, void *opaque) 1337 { 1338 struct mlx5_sh_config *config = opaque; 1339 signed long tmp; 1340 1341 errno = 0; 1342 tmp = strtol(val, NULL, 0); 1343 if (errno) { 1344 rte_errno = errno; 1345 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); 1346 return -rte_errno; 1347 } 1348 if (tmp < 0 && strcmp(MLX5_TX_PP, key) && strcmp(MLX5_TX_SKEW, key)) { 1349 /* Negative values are acceptable for some keys only. */ 1350 rte_errno = EINVAL; 1351 DRV_LOG(WARNING, "%s: invalid negative value \"%s\"", key, val); 1352 return -rte_errno; 1353 } 1354 if (strcmp(MLX5_TX_PP, key) == 0) { 1355 unsigned long mod = tmp >= 0 ? tmp : -tmp; 1356 1357 if (!mod) { 1358 DRV_LOG(ERR, "Zero Tx packet pacing parameter."); 1359 rte_errno = EINVAL; 1360 return -rte_errno; 1361 } 1362 config->tx_pp = tmp; 1363 } else if (strcmp(MLX5_TX_SKEW, key) == 0) { 1364 config->tx_skew = tmp; 1365 } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) { 1366 config->l3_vxlan_en = !!tmp; 1367 } else if (strcmp(MLX5_VF_NL_EN, key) == 0) { 1368 config->vf_nl_en = !!tmp; 1369 } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) { 1370 config->dv_esw_en = !!tmp; 1371 } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) { 1372 if (tmp > 2) { 1373 DRV_LOG(ERR, "Invalid %s parameter.", key); 1374 rte_errno = EINVAL; 1375 return -rte_errno; 1376 } 1377 config->dv_flow_en = tmp; 1378 } else if (strcmp(MLX5_DV_XMETA_EN, key) == 0) { 1379 if (tmp != MLX5_XMETA_MODE_LEGACY && 1380 tmp != MLX5_XMETA_MODE_META16 && 1381 tmp != MLX5_XMETA_MODE_META32 && 1382 tmp != MLX5_XMETA_MODE_MISS_INFO && 1383 tmp != MLX5_XMETA_MODE_META32_HWS) { 1384 DRV_LOG(ERR, "Invalid extensive metadata parameter."); 1385 rte_errno = EINVAL; 1386 return -rte_errno; 1387 } 1388 if (tmp != MLX5_XMETA_MODE_MISS_INFO) 1389 config->dv_xmeta_en = tmp; 1390 else 1391 config->dv_miss_info = 1; 1392 } else if (strcmp(MLX5_LACP_BY_USER, key) == 0) { 1393 config->lacp_by_user = !!tmp; 1394 } else if (strcmp(MLX5_RECLAIM_MEM, key) == 0) { 1395 if (tmp != MLX5_RCM_NONE && 1396 tmp != MLX5_RCM_LIGHT && 1397 tmp != MLX5_RCM_AGGR) { 1398 DRV_LOG(ERR, "Unrecognize %s: \"%s\"", key, val); 1399 rte_errno = EINVAL; 1400 return -rte_errno; 1401 } 1402 config->reclaim_mode = tmp; 1403 } else if (strcmp(MLX5_DECAP_EN, key) == 0) { 1404 config->decap_en = !!tmp; 1405 } else if (strcmp(MLX5_ALLOW_DUPLICATE_PATTERN, key) == 0) { 1406 config->allow_duplicate_pattern = !!tmp; 1407 } else if (strcmp(MLX5_FDB_DEFAULT_RULE_EN, key) == 0) { 1408 config->fdb_def_rule = !!tmp; 1409 } else if (strcmp(MLX5_HWS_CNT_SERVICE_CORE, key) == 0) { 1410 config->cnt_svc.service_core = tmp; 1411 } else if (strcmp(MLX5_HWS_CNT_CYCLE_TIME, key) == 0) { 1412 config->cnt_svc.cycle_time = tmp; 1413 } else if (strcmp(MLX5_REPR_MATCHING_EN, key) == 0) { 1414 config->repr_matching = !!tmp; 1415 } 1416 return 0; 1417 } 1418 1419 /** 1420 * Parse user device parameters and adjust them according to device 1421 * capabilities. 1422 * 1423 * @param sh 1424 * Pointer to shared device context. 1425 * @param mkvlist 1426 * Pointer to mlx5 kvargs control, can be NULL if there is no devargs. 1427 * @param config 1428 * Pointer to shared device configuration structure. 1429 * 1430 * @return 1431 * 0 on success, a negative errno value otherwise and rte_errno is set. 1432 */ 1433 static int 1434 mlx5_shared_dev_ctx_args_config(struct mlx5_dev_ctx_shared *sh, 1435 struct mlx5_kvargs_ctrl *mkvlist, 1436 struct mlx5_sh_config *config) 1437 { 1438 const char **params = (const char *[]){ 1439 MLX5_TX_PP, 1440 MLX5_TX_SKEW, 1441 MLX5_L3_VXLAN_EN, 1442 MLX5_VF_NL_EN, 1443 MLX5_DV_ESW_EN, 1444 MLX5_DV_FLOW_EN, 1445 MLX5_DV_XMETA_EN, 1446 MLX5_LACP_BY_USER, 1447 MLX5_RECLAIM_MEM, 1448 MLX5_DECAP_EN, 1449 MLX5_ALLOW_DUPLICATE_PATTERN, 1450 MLX5_FDB_DEFAULT_RULE_EN, 1451 MLX5_HWS_CNT_SERVICE_CORE, 1452 MLX5_HWS_CNT_CYCLE_TIME, 1453 MLX5_REPR_MATCHING_EN, 1454 NULL, 1455 }; 1456 int ret = 0; 1457 1458 /* Default configuration. */ 1459 memset(config, 0, sizeof(*config)); 1460 config->vf_nl_en = 1; 1461 config->dv_esw_en = 1; 1462 config->dv_flow_en = 1; 1463 config->decap_en = 1; 1464 config->allow_duplicate_pattern = 1; 1465 config->fdb_def_rule = 1; 1466 config->cnt_svc.cycle_time = MLX5_CNT_SVC_CYCLE_TIME_DEFAULT; 1467 config->cnt_svc.service_core = rte_get_main_lcore(); 1468 config->repr_matching = 1; 1469 if (mkvlist != NULL) { 1470 /* Process parameters. */ 1471 ret = mlx5_kvargs_process(mkvlist, params, 1472 mlx5_dev_args_check_handler, config); 1473 if (ret) { 1474 DRV_LOG(ERR, "Failed to process device arguments: %s", 1475 strerror(rte_errno)); 1476 return -rte_errno; 1477 } 1478 } 1479 /* Adjust parameters according to device capabilities. */ 1480 if (config->dv_flow_en && !sh->dev_cap.dv_flow_en) { 1481 DRV_LOG(WARNING, "DV flow is not supported."); 1482 config->dv_flow_en = 0; 1483 } 1484 if (config->dv_esw_en && !sh->dev_cap.dv_esw_en) { 1485 DRV_LOG(DEBUG, "E-Switch DV flow is not supported."); 1486 config->dv_esw_en = 0; 1487 } 1488 if (config->dv_esw_en && !config->dv_flow_en) { 1489 DRV_LOG(DEBUG, 1490 "E-Switch DV flow is supported only when DV flow is enabled."); 1491 config->dv_esw_en = 0; 1492 } 1493 if (config->dv_miss_info && config->dv_esw_en) 1494 config->dv_xmeta_en = MLX5_XMETA_MODE_META16; 1495 if (!config->dv_esw_en && 1496 config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 1497 DRV_LOG(WARNING, 1498 "Metadata mode %u is not supported (no E-Switch).", 1499 config->dv_xmeta_en); 1500 config->dv_xmeta_en = MLX5_XMETA_MODE_LEGACY; 1501 } 1502 if (config->dv_flow_en != 2 && !config->repr_matching) { 1503 DRV_LOG(DEBUG, "Disabling representor matching is valid only " 1504 "when HW Steering is enabled."); 1505 config->repr_matching = 1; 1506 } 1507 if (config->tx_pp && !sh->dev_cap.txpp_en) { 1508 DRV_LOG(ERR, "Packet pacing is not supported."); 1509 rte_errno = ENODEV; 1510 return -rte_errno; 1511 } 1512 if (!config->tx_pp && config->tx_skew && 1513 !sh->cdev->config.hca_attr.wait_on_time) { 1514 DRV_LOG(WARNING, 1515 "\"tx_skew\" doesn't affect without \"tx_pp\"."); 1516 } 1517 /* Check for LRO support. */ 1518 if (mlx5_devx_obj_ops_en(sh) && sh->cdev->config.hca_attr.lro_cap) { 1519 /* TBD check tunnel lro caps. */ 1520 config->lro_allowed = 1; 1521 DRV_LOG(DEBUG, "LRO is allowed."); 1522 DRV_LOG(DEBUG, 1523 "LRO minimal size of TCP segment required for coalescing is %d bytes.", 1524 sh->cdev->config.hca_attr.lro_min_mss_size); 1525 } 1526 /* 1527 * If HW has bug working with tunnel packet decapsulation and scatter 1528 * FCS, and decapsulation is needed, clear the hw_fcs_strip bit. 1529 * Then RTE_ETH_RX_OFFLOAD_KEEP_CRC bit will not be set anymore. 1530 */ 1531 if (sh->dev_cap.scatter_fcs_w_decap_disable && sh->config.decap_en) 1532 config->hw_fcs_strip = 0; 1533 else 1534 config->hw_fcs_strip = sh->dev_cap.hw_fcs_strip; 1535 DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported", 1536 (config->hw_fcs_strip ? "" : "not ")); 1537 DRV_LOG(DEBUG, "\"tx_pp\" is %d.", config->tx_pp); 1538 DRV_LOG(DEBUG, "\"tx_skew\" is %d.", config->tx_skew); 1539 DRV_LOG(DEBUG, "\"reclaim_mode\" is %u.", config->reclaim_mode); 1540 DRV_LOG(DEBUG, "\"dv_esw_en\" is %u.", config->dv_esw_en); 1541 DRV_LOG(DEBUG, "\"dv_flow_en\" is %u.", config->dv_flow_en); 1542 DRV_LOG(DEBUG, "\"dv_xmeta_en\" is %u.", config->dv_xmeta_en); 1543 DRV_LOG(DEBUG, "\"dv_miss_info\" is %u.", config->dv_miss_info); 1544 DRV_LOG(DEBUG, "\"l3_vxlan_en\" is %u.", config->l3_vxlan_en); 1545 DRV_LOG(DEBUG, "\"vf_nl_en\" is %u.", config->vf_nl_en); 1546 DRV_LOG(DEBUG, "\"lacp_by_user\" is %u.", config->lacp_by_user); 1547 DRV_LOG(DEBUG, "\"decap_en\" is %u.", config->decap_en); 1548 DRV_LOG(DEBUG, "\"allow_duplicate_pattern\" is %u.", 1549 config->allow_duplicate_pattern); 1550 DRV_LOG(DEBUG, "\"fdb_def_rule_en\" is %u.", config->fdb_def_rule); 1551 DRV_LOG(DEBUG, "\"repr_matching_en\" is %u.", config->repr_matching); 1552 return 0; 1553 } 1554 1555 /** 1556 * Configure realtime timestamp format. 1557 * 1558 * @param sh 1559 * Pointer to mlx5_dev_ctx_shared object. 1560 * @param hca_attr 1561 * Pointer to DevX HCA capabilities structure. 1562 */ 1563 void 1564 mlx5_rt_timestamp_config(struct mlx5_dev_ctx_shared *sh, 1565 struct mlx5_hca_attr *hca_attr) 1566 { 1567 uint32_t dw_cnt = MLX5_ST_SZ_DW(register_mtutc); 1568 uint32_t reg[dw_cnt]; 1569 int ret = ENOTSUP; 1570 1571 if (hca_attr->access_register_user) 1572 ret = mlx5_devx_cmd_register_read(sh->cdev->ctx, 1573 MLX5_REGISTER_ID_MTUTC, 0, 1574 reg, dw_cnt); 1575 if (!ret) { 1576 uint32_t ts_mode; 1577 1578 /* MTUTC register is read successfully. */ 1579 ts_mode = MLX5_GET(register_mtutc, reg, time_stamp_mode); 1580 if (ts_mode == MLX5_MTUTC_TIMESTAMP_MODE_REAL_TIME) 1581 sh->dev_cap.rt_timestamp = 1; 1582 } else { 1583 /* Kernel does not support register reading. */ 1584 if (hca_attr->dev_freq_khz == (NS_PER_S / MS_PER_S)) 1585 sh->dev_cap.rt_timestamp = 1; 1586 } 1587 } 1588 1589 /** 1590 * Allocate shared device context. If there is multiport device the 1591 * master and representors will share this context, if there is single 1592 * port dedicated device, the context will be used by only given 1593 * port due to unification. 1594 * 1595 * Routine first searches the context for the specified device name, 1596 * if found the shared context assumed and reference counter is incremented. 1597 * If no context found the new one is created and initialized with specified 1598 * device context and parameters. 1599 * 1600 * @param[in] spawn 1601 * Pointer to the device attributes (name, port, etc). 1602 * @param mkvlist 1603 * Pointer to mlx5 kvargs control, can be NULL if there is no devargs. 1604 * 1605 * @return 1606 * Pointer to mlx5_dev_ctx_shared object on success, 1607 * otherwise NULL and rte_errno is set. 1608 */ 1609 struct mlx5_dev_ctx_shared * 1610 mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, 1611 struct mlx5_kvargs_ctrl *mkvlist) 1612 { 1613 struct mlx5_dev_ctx_shared *sh; 1614 int err = 0; 1615 uint32_t i; 1616 1617 MLX5_ASSERT(spawn); 1618 /* Secondary process should not create the shared context. */ 1619 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1620 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1621 /* Search for IB context by device name. */ 1622 LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) { 1623 if (!strcmp(sh->ibdev_name, spawn->phys_dev_name)) { 1624 sh->refcnt++; 1625 goto exit; 1626 } 1627 } 1628 /* No device found, we have to create new shared context. */ 1629 MLX5_ASSERT(spawn->max_port); 1630 sh = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 1631 sizeof(struct mlx5_dev_ctx_shared) + 1632 spawn->max_port * sizeof(struct mlx5_dev_shared_port), 1633 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1634 if (!sh) { 1635 DRV_LOG(ERR, "Shared context allocation failure."); 1636 rte_errno = ENOMEM; 1637 goto exit; 1638 } 1639 pthread_mutex_init(&sh->txpp.mutex, NULL); 1640 sh->numa_node = spawn->cdev->dev->numa_node; 1641 sh->cdev = spawn->cdev; 1642 sh->esw_mode = !!(spawn->info.master || spawn->info.representor); 1643 if (spawn->bond_info) 1644 sh->bond = *spawn->bond_info; 1645 err = mlx5_os_capabilities_prepare(sh); 1646 if (err) { 1647 DRV_LOG(ERR, "Fail to configure device capabilities."); 1648 goto error; 1649 } 1650 err = mlx5_shared_dev_ctx_args_config(sh, mkvlist, &sh->config); 1651 if (err) { 1652 DRV_LOG(ERR, "Failed to process device configure: %s", 1653 strerror(rte_errno)); 1654 goto error; 1655 } 1656 sh->refcnt = 1; 1657 sh->max_port = spawn->max_port; 1658 strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->cdev->ctx), 1659 sizeof(sh->ibdev_name) - 1); 1660 strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->cdev->ctx), 1661 sizeof(sh->ibdev_path) - 1); 1662 /* 1663 * Setting port_id to max unallowed value means there is no interrupt 1664 * subhandler installed for the given port index i. 1665 */ 1666 for (i = 0; i < sh->max_port; i++) { 1667 sh->port[i].ih_port_id = RTE_MAX_ETHPORTS; 1668 sh->port[i].devx_ih_port_id = RTE_MAX_ETHPORTS; 1669 sh->port[i].nl_ih_port_id = RTE_MAX_ETHPORTS; 1670 } 1671 if (sh->cdev->config.devx) { 1672 sh->td = mlx5_devx_cmd_create_td(sh->cdev->ctx); 1673 if (!sh->td) { 1674 DRV_LOG(ERR, "TD allocation failure"); 1675 rte_errno = ENOMEM; 1676 goto error; 1677 } 1678 if (mlx5_setup_tis(sh)) { 1679 DRV_LOG(ERR, "TIS allocation failure"); 1680 rte_errno = ENOMEM; 1681 goto error; 1682 } 1683 err = mlx5_rxtx_uars_prepare(sh); 1684 if (err) 1685 goto error; 1686 #ifndef RTE_ARCH_64 1687 } else { 1688 /* Initialize UAR access locks for 32bit implementations. */ 1689 rte_spinlock_init(&sh->uar_lock_cq); 1690 for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) 1691 rte_spinlock_init(&sh->uar_lock[i]); 1692 #endif 1693 } 1694 mlx5_os_dev_shared_handler_install(sh); 1695 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1696 err = mlx5_flow_os_init_workspace_once(); 1697 if (err) 1698 goto error; 1699 } 1700 err = mlx5_flow_counters_mng_init(sh); 1701 if (err) { 1702 DRV_LOG(ERR, "Fail to initialize counters manage."); 1703 goto error; 1704 } 1705 mlx5_flow_aging_init(sh); 1706 mlx5_flow_ipool_create(sh); 1707 /* Add context to the global device list. */ 1708 LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next); 1709 rte_spinlock_init(&sh->geneve_tlv_opt_sl); 1710 exit: 1711 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1712 return sh; 1713 error: 1714 err = rte_errno; 1715 pthread_mutex_destroy(&sh->txpp.mutex); 1716 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1717 MLX5_ASSERT(sh); 1718 mlx5_rxtx_uars_release(sh); 1719 i = 0; 1720 do { 1721 if (sh->tis[i]) 1722 claim_zero(mlx5_devx_cmd_destroy(sh->tis[i])); 1723 } while (++i < (uint32_t)sh->bond.n_port); 1724 if (sh->td) 1725 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1726 mlx5_free(sh); 1727 rte_errno = err; 1728 return NULL; 1729 } 1730 1731 /** 1732 * Create LWM event_channel and interrupt handle for shared device 1733 * context. All rxqs sharing the device context share the event_channel. 1734 * A callback is registered in interrupt thread to receive the LWM event. 1735 * 1736 * @param[in] priv 1737 * Pointer to mlx5_priv instance. 1738 * 1739 * @return 1740 * 0 on success, negative with rte_errno set. 1741 */ 1742 int 1743 mlx5_lwm_setup(struct mlx5_priv *priv) 1744 { 1745 int fd_lwm; 1746 1747 pthread_mutex_init(&priv->sh->lwm_config_lock, NULL); 1748 priv->sh->devx_channel_lwm = mlx5_os_devx_create_event_channel 1749 (priv->sh->cdev->ctx, 1750 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA); 1751 if (!priv->sh->devx_channel_lwm) 1752 goto err; 1753 fd_lwm = mlx5_os_get_devx_channel_fd(priv->sh->devx_channel_lwm); 1754 priv->sh->intr_handle_lwm = mlx5_os_interrupt_handler_create 1755 (RTE_INTR_INSTANCE_F_SHARED, true, 1756 fd_lwm, mlx5_dev_interrupt_handler_lwm, priv); 1757 if (!priv->sh->intr_handle_lwm) 1758 goto err; 1759 return 0; 1760 err: 1761 if (priv->sh->devx_channel_lwm) { 1762 mlx5_os_devx_destroy_event_channel 1763 (priv->sh->devx_channel_lwm); 1764 priv->sh->devx_channel_lwm = NULL; 1765 } 1766 pthread_mutex_destroy(&priv->sh->lwm_config_lock); 1767 return -rte_errno; 1768 } 1769 1770 /** 1771 * Destroy LWM event_channel and interrupt handle for shared device 1772 * context before free this context. The interrupt handler is also 1773 * unregistered. 1774 * 1775 * @param[in] sh 1776 * Pointer to shared device context. 1777 */ 1778 void 1779 mlx5_lwm_unset(struct mlx5_dev_ctx_shared *sh) 1780 { 1781 if (sh->intr_handle_lwm) { 1782 mlx5_os_interrupt_handler_destroy(sh->intr_handle_lwm, 1783 mlx5_dev_interrupt_handler_lwm, (void *)-1); 1784 sh->intr_handle_lwm = NULL; 1785 } 1786 if (sh->devx_channel_lwm) { 1787 mlx5_os_devx_destroy_event_channel 1788 (sh->devx_channel_lwm); 1789 sh->devx_channel_lwm = NULL; 1790 } 1791 pthread_mutex_destroy(&sh->lwm_config_lock); 1792 } 1793 1794 /** 1795 * Free shared IB device context. Decrement counter and if zero free 1796 * all allocated resources and close handles. 1797 * 1798 * @param[in] sh 1799 * Pointer to mlx5_dev_ctx_shared object to free 1800 */ 1801 void 1802 mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) 1803 { 1804 int ret; 1805 int i = 0; 1806 1807 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1808 #ifdef RTE_LIBRTE_MLX5_DEBUG 1809 /* Check the object presence in the list. */ 1810 struct mlx5_dev_ctx_shared *lctx; 1811 1812 LIST_FOREACH(lctx, &mlx5_dev_ctx_list, next) 1813 if (lctx == sh) 1814 break; 1815 MLX5_ASSERT(lctx); 1816 if (lctx != sh) { 1817 DRV_LOG(ERR, "Freeing non-existing shared IB context"); 1818 goto exit; 1819 } 1820 #endif 1821 MLX5_ASSERT(sh); 1822 MLX5_ASSERT(sh->refcnt); 1823 /* Secondary process should not free the shared context. */ 1824 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1825 if (--sh->refcnt) 1826 goto exit; 1827 /* Stop watching for mempool events and unregister all mempools. */ 1828 if (!sh->cdev->config.mr_mempool_reg_en) { 1829 ret = rte_mempool_event_callback_unregister 1830 (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); 1831 if (ret == 0) 1832 rte_mempool_walk 1833 (mlx5_dev_ctx_shared_rx_mempool_unregister_cb, sh); 1834 } 1835 /* Remove context from the global device list. */ 1836 LIST_REMOVE(sh, next); 1837 /* Release resources on the last device removal. */ 1838 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1839 mlx5_os_net_cleanup(); 1840 mlx5_flow_os_release_workspace(); 1841 mlx5_flow_workspace_gc_release(); 1842 } 1843 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1844 if (sh->flex_parsers_dv) { 1845 mlx5_list_destroy(sh->flex_parsers_dv); 1846 sh->flex_parsers_dv = NULL; 1847 } 1848 /* 1849 * Ensure there is no async event handler installed. 1850 * Only primary process handles async device events. 1851 **/ 1852 mlx5_flow_counters_mng_close(sh); 1853 if (sh->ct_mng) 1854 mlx5_flow_aso_ct_mng_close(sh); 1855 if (sh->aso_age_mng) { 1856 mlx5_flow_aso_age_mng_close(sh); 1857 sh->aso_age_mng = NULL; 1858 } 1859 if (sh->mtrmng) 1860 mlx5_aso_flow_mtrs_mng_close(sh); 1861 mlx5_flow_ipool_destroy(sh); 1862 mlx5_os_dev_shared_handler_uninstall(sh); 1863 mlx5_rxtx_uars_release(sh); 1864 do { 1865 if (sh->tis[i]) 1866 claim_zero(mlx5_devx_cmd_destroy(sh->tis[i])); 1867 } while (++i < sh->bond.n_port); 1868 if (sh->td) 1869 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1870 #ifdef HAVE_MLX5_HWS_SUPPORT 1871 /* HWS manages geneve_tlv_option resource as global. */ 1872 if (sh->config.dv_flow_en == 2) 1873 flow_dev_geneve_tlv_option_resource_release(sh); 1874 else 1875 #endif 1876 MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL); 1877 pthread_mutex_destroy(&sh->txpp.mutex); 1878 mlx5_lwm_unset(sh); 1879 mlx5_free(sh); 1880 return; 1881 exit: 1882 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1883 } 1884 1885 /** 1886 * Destroy table hash list. 1887 * 1888 * @param[in] priv 1889 * Pointer to the private device data structure. 1890 */ 1891 void 1892 mlx5_free_table_hash_list(struct mlx5_priv *priv) 1893 { 1894 struct mlx5_dev_ctx_shared *sh = priv->sh; 1895 struct mlx5_hlist **tbls = (priv->sh->config.dv_flow_en == 2) ? 1896 &sh->groups : &sh->flow_tbls; 1897 if (*tbls == NULL) 1898 return; 1899 mlx5_hlist_destroy(*tbls); 1900 *tbls = NULL; 1901 } 1902 1903 #ifdef HAVE_MLX5_HWS_SUPPORT 1904 /** 1905 * Allocate HW steering group hash list. 1906 * 1907 * @param[in] priv 1908 * Pointer to the private device data structure. 1909 */ 1910 static int 1911 mlx5_alloc_hw_group_hash_list(struct mlx5_priv *priv) 1912 { 1913 int err = 0; 1914 struct mlx5_dev_ctx_shared *sh = priv->sh; 1915 char s[MLX5_NAME_SIZE]; 1916 1917 MLX5_ASSERT(sh); 1918 snprintf(s, sizeof(s), "%s_flow_groups", priv->sh->ibdev_name); 1919 sh->groups = mlx5_hlist_create 1920 (s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, 1921 false, true, sh, 1922 flow_hw_grp_create_cb, 1923 flow_hw_grp_match_cb, 1924 flow_hw_grp_remove_cb, 1925 flow_hw_grp_clone_cb, 1926 flow_hw_grp_clone_free_cb); 1927 if (!sh->groups) { 1928 DRV_LOG(ERR, "flow groups with hash creation failed."); 1929 err = ENOMEM; 1930 } 1931 return err; 1932 } 1933 #endif 1934 1935 1936 /** 1937 * Initialize flow table hash list and create the root tables entry 1938 * for each domain. 1939 * 1940 * @param[in] priv 1941 * Pointer to the private device data structure. 1942 * 1943 * @return 1944 * Zero on success, positive error code otherwise. 1945 */ 1946 int 1947 mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) 1948 { 1949 int err = 0; 1950 1951 /* Tables are only used in DV and DR modes. */ 1952 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 1953 struct mlx5_dev_ctx_shared *sh = priv->sh; 1954 char s[MLX5_NAME_SIZE]; 1955 1956 #ifdef HAVE_MLX5_HWS_SUPPORT 1957 if (priv->sh->config.dv_flow_en == 2) 1958 return mlx5_alloc_hw_group_hash_list(priv); 1959 #endif 1960 MLX5_ASSERT(sh); 1961 snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); 1962 sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, 1963 false, true, sh, 1964 flow_dv_tbl_create_cb, 1965 flow_dv_tbl_match_cb, 1966 flow_dv_tbl_remove_cb, 1967 flow_dv_tbl_clone_cb, 1968 flow_dv_tbl_clone_free_cb); 1969 if (!sh->flow_tbls) { 1970 DRV_LOG(ERR, "flow tables with hash creation failed."); 1971 err = ENOMEM; 1972 return err; 1973 } 1974 #ifndef HAVE_MLX5DV_DR 1975 struct rte_flow_error error; 1976 struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; 1977 1978 /* 1979 * In case we have not DR support, the zero tables should be created 1980 * because DV expect to see them even if they cannot be created by 1981 * RDMA-CORE. 1982 */ 1983 if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, 1984 NULL, 0, 1, 0, &error) || 1985 !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, 1986 NULL, 0, 1, 0, &error) || 1987 !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, 1988 NULL, 0, 1, 0, &error)) { 1989 err = ENOMEM; 1990 goto error; 1991 } 1992 return err; 1993 error: 1994 mlx5_free_table_hash_list(priv); 1995 #endif /* HAVE_MLX5DV_DR */ 1996 #endif 1997 return err; 1998 } 1999 2000 /** 2001 * Retrieve integer value from environment variable. 2002 * 2003 * @param[in] name 2004 * Environment variable name. 2005 * 2006 * @return 2007 * Integer value, 0 if the variable is not set. 2008 */ 2009 int 2010 mlx5_getenv_int(const char *name) 2011 { 2012 const char *val = getenv(name); 2013 2014 if (val == NULL) 2015 return 0; 2016 return atoi(val); 2017 } 2018 2019 /** 2020 * DPDK callback to add udp tunnel port 2021 * 2022 * @param[in] dev 2023 * A pointer to eth_dev 2024 * @param[in] udp_tunnel 2025 * A pointer to udp tunnel 2026 * 2027 * @return 2028 * 0 on valid udp ports and tunnels, -ENOTSUP otherwise. 2029 */ 2030 int 2031 mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev __rte_unused, 2032 struct rte_eth_udp_tunnel *udp_tunnel) 2033 { 2034 MLX5_ASSERT(udp_tunnel != NULL); 2035 if (udp_tunnel->prot_type == RTE_ETH_TUNNEL_TYPE_VXLAN && 2036 udp_tunnel->udp_port == 4789) 2037 return 0; 2038 if (udp_tunnel->prot_type == RTE_ETH_TUNNEL_TYPE_VXLAN_GPE && 2039 udp_tunnel->udp_port == 4790) 2040 return 0; 2041 return -ENOTSUP; 2042 } 2043 2044 /** 2045 * Initialize process private data structure. 2046 * 2047 * @param dev 2048 * Pointer to Ethernet device structure. 2049 * 2050 * @return 2051 * 0 on success, a negative errno value otherwise and rte_errno is set. 2052 */ 2053 int 2054 mlx5_proc_priv_init(struct rte_eth_dev *dev) 2055 { 2056 struct mlx5_priv *priv = dev->data->dev_private; 2057 struct mlx5_proc_priv *ppriv; 2058 size_t ppriv_size; 2059 2060 mlx5_proc_priv_uninit(dev); 2061 /* 2062 * UAR register table follows the process private structure. BlueFlame 2063 * registers for Tx queues are stored in the table. 2064 */ 2065 ppriv_size = sizeof(struct mlx5_proc_priv) + 2066 priv->txqs_n * sizeof(struct mlx5_uar_data); 2067 ppriv = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, ppriv_size, 2068 RTE_CACHE_LINE_SIZE, dev->device->numa_node); 2069 if (!ppriv) { 2070 rte_errno = ENOMEM; 2071 return -rte_errno; 2072 } 2073 ppriv->uar_table_sz = priv->txqs_n; 2074 dev->process_private = ppriv; 2075 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 2076 priv->sh->pppriv = ppriv; 2077 return 0; 2078 } 2079 2080 /** 2081 * Un-initialize process private data structure. 2082 * 2083 * @param dev 2084 * Pointer to Ethernet device structure. 2085 */ 2086 void 2087 mlx5_proc_priv_uninit(struct rte_eth_dev *dev) 2088 { 2089 struct mlx5_proc_priv *ppriv = dev->process_private; 2090 2091 if (!ppriv) 2092 return; 2093 if (ppriv->hca_bar) 2094 mlx5_txpp_unmap_hca_bar(dev); 2095 mlx5_free(dev->process_private); 2096 dev->process_private = NULL; 2097 } 2098 2099 /** 2100 * DPDK callback to close the device. 2101 * 2102 * Destroy all queues and objects, free memory. 2103 * 2104 * @param dev 2105 * Pointer to Ethernet device structure. 2106 */ 2107 int 2108 mlx5_dev_close(struct rte_eth_dev *dev) 2109 { 2110 struct mlx5_priv *priv = dev->data->dev_private; 2111 unsigned int i; 2112 int ret; 2113 2114 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 2115 /* Check if process_private released. */ 2116 if (!dev->process_private) 2117 return 0; 2118 mlx5_tx_uar_uninit_secondary(dev); 2119 mlx5_proc_priv_uninit(dev); 2120 rte_eth_dev_release_port(dev); 2121 return 0; 2122 } 2123 if (!priv->sh) 2124 return 0; 2125 if (priv->shared_refcnt) { 2126 DRV_LOG(ERR, "port %u is shared host in use (%u)", 2127 dev->data->port_id, priv->shared_refcnt); 2128 rte_errno = EBUSY; 2129 return -EBUSY; 2130 } 2131 DRV_LOG(DEBUG, "port %u closing device \"%s\"", 2132 dev->data->port_id, 2133 ((priv->sh->cdev->ctx != NULL) ? 2134 mlx5_os_get_ctx_device_name(priv->sh->cdev->ctx) : "")); 2135 /* 2136 * If default mreg copy action is removed at the stop stage, 2137 * the search will return none and nothing will be done anymore. 2138 */ 2139 if (priv->sh->config.dv_flow_en != 2) 2140 mlx5_flow_stop_default(dev); 2141 mlx5_traffic_disable(dev); 2142 /* 2143 * If all the flows are already flushed in the device stop stage, 2144 * then this will return directly without any action. 2145 */ 2146 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true); 2147 mlx5_action_handle_flush(dev); 2148 mlx5_flow_meter_flush(dev, NULL); 2149 /* Prevent crashes when queues are still in use. */ 2150 dev->rx_pkt_burst = rte_eth_pkt_burst_dummy; 2151 dev->tx_pkt_burst = rte_eth_pkt_burst_dummy; 2152 rte_wmb(); 2153 /* Disable datapath on secondary process. */ 2154 mlx5_mp_os_req_stop_rxtx(dev); 2155 /* Free the eCPRI flex parser resource. */ 2156 mlx5_flex_parser_ecpri_release(dev); 2157 mlx5_flex_item_port_cleanup(dev); 2158 #ifdef HAVE_MLX5_HWS_SUPPORT 2159 flow_hw_destroy_vport_action(dev); 2160 flow_hw_resource_release(dev); 2161 flow_hw_clear_port_info(dev); 2162 if (priv->sh->config.dv_flow_en == 2) { 2163 flow_hw_clear_flow_metadata_config(); 2164 flow_hw_clear_tags_set(dev); 2165 } 2166 #endif 2167 if (priv->rxq_privs != NULL) { 2168 /* XXX race condition if mlx5_rx_burst() is still running. */ 2169 rte_delay_us_sleep(1000); 2170 for (i = 0; (i != priv->rxqs_n); ++i) 2171 mlx5_rxq_release(dev, i); 2172 priv->rxqs_n = 0; 2173 mlx5_free(priv->rxq_privs); 2174 priv->rxq_privs = NULL; 2175 } 2176 if (priv->txqs != NULL) { 2177 /* XXX race condition if mlx5_tx_burst() is still running. */ 2178 rte_delay_us_sleep(1000); 2179 for (i = 0; (i != priv->txqs_n); ++i) 2180 mlx5_txq_release(dev, i); 2181 priv->txqs_n = 0; 2182 priv->txqs = NULL; 2183 } 2184 mlx5_proc_priv_uninit(dev); 2185 if (priv->q_counters) { 2186 mlx5_devx_cmd_destroy(priv->q_counters); 2187 priv->q_counters = NULL; 2188 } 2189 if (priv->drop_queue.hrxq) 2190 mlx5_drop_action_destroy(dev); 2191 if (priv->mreg_cp_tbl) 2192 mlx5_hlist_destroy(priv->mreg_cp_tbl); 2193 mlx5_mprq_free_mp(dev); 2194 mlx5_os_free_shared_dr(priv); 2195 if (priv->rss_conf.rss_key != NULL) 2196 mlx5_free(priv->rss_conf.rss_key); 2197 if (priv->reta_idx != NULL) 2198 mlx5_free(priv->reta_idx); 2199 if (priv->sh->dev_cap.vf) 2200 mlx5_os_mac_addr_flush(dev); 2201 if (priv->nl_socket_route >= 0) 2202 close(priv->nl_socket_route); 2203 if (priv->nl_socket_rdma >= 0) 2204 close(priv->nl_socket_rdma); 2205 if (priv->vmwa_context) 2206 mlx5_vlan_vmwa_exit(priv->vmwa_context); 2207 ret = mlx5_hrxq_verify(dev); 2208 if (ret) 2209 DRV_LOG(WARNING, "port %u some hash Rx queue still remain", 2210 dev->data->port_id); 2211 ret = mlx5_ind_table_obj_verify(dev); 2212 if (ret) 2213 DRV_LOG(WARNING, "port %u some indirection table still remain", 2214 dev->data->port_id); 2215 ret = mlx5_rxq_obj_verify(dev); 2216 if (ret) 2217 DRV_LOG(WARNING, "port %u some Rx queue objects still remain", 2218 dev->data->port_id); 2219 ret = mlx5_ext_rxq_verify(dev); 2220 if (ret) 2221 DRV_LOG(WARNING, "Port %u some external RxQ still remain.", 2222 dev->data->port_id); 2223 ret = mlx5_rxq_verify(dev); 2224 if (ret) 2225 DRV_LOG(WARNING, "port %u some Rx queues still remain", 2226 dev->data->port_id); 2227 ret = mlx5_txq_obj_verify(dev); 2228 if (ret) 2229 DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain", 2230 dev->data->port_id); 2231 ret = mlx5_txq_verify(dev); 2232 if (ret) 2233 DRV_LOG(WARNING, "port %u some Tx queues still remain", 2234 dev->data->port_id); 2235 ret = mlx5_flow_verify(dev); 2236 if (ret) 2237 DRV_LOG(WARNING, "port %u some flows still remain", 2238 dev->data->port_id); 2239 if (priv->hrxqs) 2240 mlx5_list_destroy(priv->hrxqs); 2241 mlx5_free(priv->ext_rxqs); 2242 priv->sh->port[priv->dev_port - 1].nl_ih_port_id = RTE_MAX_ETHPORTS; 2243 /* 2244 * The interrupt handler port id must be reset before priv is reset 2245 * since 'mlx5_dev_interrupt_nl_cb' uses priv. 2246 */ 2247 rte_io_wmb(); 2248 /* 2249 * Free the shared context in last turn, because the cleanup 2250 * routines above may use some shared fields, like 2251 * mlx5_os_mac_addr_flush() uses ibdev_path for retrieving 2252 * ifindex if Netlink fails. 2253 */ 2254 mlx5_free_shared_dev_ctx(priv->sh); 2255 if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 2256 unsigned int c = 0; 2257 uint16_t port_id; 2258 2259 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 2260 struct mlx5_priv *opriv = 2261 rte_eth_devices[port_id].data->dev_private; 2262 2263 if (!opriv || 2264 opriv->domain_id != priv->domain_id || 2265 &rte_eth_devices[port_id] == dev) 2266 continue; 2267 ++c; 2268 break; 2269 } 2270 if (!c) 2271 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 2272 } 2273 memset(priv, 0, sizeof(*priv)); 2274 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 2275 /* 2276 * Reset mac_addrs to NULL such that it is not freed as part of 2277 * rte_eth_dev_release_port(). mac_addrs is part of dev_private so 2278 * it is freed when dev_private is freed. 2279 */ 2280 dev->data->mac_addrs = NULL; 2281 return 0; 2282 } 2283 2284 const struct eth_dev_ops mlx5_dev_ops = { 2285 .dev_configure = mlx5_dev_configure, 2286 .dev_start = mlx5_dev_start, 2287 .dev_stop = mlx5_dev_stop, 2288 .dev_set_link_down = mlx5_set_link_down, 2289 .dev_set_link_up = mlx5_set_link_up, 2290 .dev_close = mlx5_dev_close, 2291 .promiscuous_enable = mlx5_promiscuous_enable, 2292 .promiscuous_disable = mlx5_promiscuous_disable, 2293 .allmulticast_enable = mlx5_allmulticast_enable, 2294 .allmulticast_disable = mlx5_allmulticast_disable, 2295 .link_update = mlx5_link_update, 2296 .stats_get = mlx5_stats_get, 2297 .stats_reset = mlx5_stats_reset, 2298 .xstats_get = mlx5_xstats_get, 2299 .xstats_reset = mlx5_xstats_reset, 2300 .xstats_get_names = mlx5_xstats_get_names, 2301 .fw_version_get = mlx5_fw_version_get, 2302 .dev_infos_get = mlx5_dev_infos_get, 2303 .representor_info_get = mlx5_representor_info_get, 2304 .read_clock = mlx5_txpp_read_clock, 2305 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 2306 .vlan_filter_set = mlx5_vlan_filter_set, 2307 .rx_queue_setup = mlx5_rx_queue_setup, 2308 .rx_queue_avail_thresh_set = mlx5_rx_queue_lwm_set, 2309 .rx_queue_avail_thresh_query = mlx5_rx_queue_lwm_query, 2310 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 2311 .tx_queue_setup = mlx5_tx_queue_setup, 2312 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 2313 .rx_queue_release = mlx5_rx_queue_release, 2314 .tx_queue_release = mlx5_tx_queue_release, 2315 .rx_queue_start = mlx5_rx_queue_start, 2316 .rx_queue_stop = mlx5_rx_queue_stop, 2317 .tx_queue_start = mlx5_tx_queue_start, 2318 .tx_queue_stop = mlx5_tx_queue_stop, 2319 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 2320 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 2321 .mac_addr_remove = mlx5_mac_addr_remove, 2322 .mac_addr_add = mlx5_mac_addr_add, 2323 .mac_addr_set = mlx5_mac_addr_set, 2324 .set_mc_addr_list = mlx5_set_mc_addr_list, 2325 .mtu_set = mlx5_dev_set_mtu, 2326 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 2327 .vlan_offload_set = mlx5_vlan_offload_set, 2328 .reta_update = mlx5_dev_rss_reta_update, 2329 .reta_query = mlx5_dev_rss_reta_query, 2330 .rss_hash_update = mlx5_rss_hash_update, 2331 .rss_hash_conf_get = mlx5_rss_hash_conf_get, 2332 .flow_ops_get = mlx5_flow_ops_get, 2333 .rxq_info_get = mlx5_rxq_info_get, 2334 .txq_info_get = mlx5_txq_info_get, 2335 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2336 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2337 .rx_queue_intr_enable = mlx5_rx_intr_enable, 2338 .rx_queue_intr_disable = mlx5_rx_intr_disable, 2339 .is_removed = mlx5_is_removed, 2340 .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, 2341 .get_module_info = mlx5_get_module_info, 2342 .get_module_eeprom = mlx5_get_module_eeprom, 2343 .hairpin_cap_get = mlx5_hairpin_cap_get, 2344 .mtr_ops_get = mlx5_flow_meter_ops_get, 2345 .hairpin_bind = mlx5_hairpin_bind, 2346 .hairpin_unbind = mlx5_hairpin_unbind, 2347 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 2348 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 2349 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 2350 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 2351 .get_monitor_addr = mlx5_get_monitor_addr, 2352 .count_aggr_ports = mlx5_count_aggr_ports, 2353 .map_aggr_tx_affinity = mlx5_map_aggr_tx_affinity, 2354 .rx_metadata_negotiate = mlx5_flow_rx_metadata_negotiate, 2355 }; 2356 2357 /* Available operations from secondary process. */ 2358 const struct eth_dev_ops mlx5_dev_sec_ops = { 2359 .stats_get = mlx5_stats_get, 2360 .stats_reset = mlx5_stats_reset, 2361 .xstats_get = mlx5_xstats_get, 2362 .xstats_reset = mlx5_xstats_reset, 2363 .xstats_get_names = mlx5_xstats_get_names, 2364 .fw_version_get = mlx5_fw_version_get, 2365 .dev_infos_get = mlx5_dev_infos_get, 2366 .representor_info_get = mlx5_representor_info_get, 2367 .read_clock = mlx5_txpp_read_clock, 2368 .rx_queue_start = mlx5_rx_queue_start, 2369 .rx_queue_stop = mlx5_rx_queue_stop, 2370 .tx_queue_start = mlx5_tx_queue_start, 2371 .tx_queue_stop = mlx5_tx_queue_stop, 2372 .rxq_info_get = mlx5_rxq_info_get, 2373 .txq_info_get = mlx5_txq_info_get, 2374 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2375 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2376 .get_module_info = mlx5_get_module_info, 2377 .get_module_eeprom = mlx5_get_module_eeprom, 2378 .count_aggr_ports = mlx5_count_aggr_ports, 2379 .map_aggr_tx_affinity = mlx5_map_aggr_tx_affinity, 2380 .rx_metadata_negotiate = mlx5_flow_rx_metadata_negotiate, 2381 }; 2382 2383 /* Available operations in flow isolated mode. */ 2384 const struct eth_dev_ops mlx5_dev_ops_isolate = { 2385 .dev_configure = mlx5_dev_configure, 2386 .dev_start = mlx5_dev_start, 2387 .dev_stop = mlx5_dev_stop, 2388 .dev_set_link_down = mlx5_set_link_down, 2389 .dev_set_link_up = mlx5_set_link_up, 2390 .dev_close = mlx5_dev_close, 2391 .promiscuous_enable = mlx5_promiscuous_enable, 2392 .promiscuous_disable = mlx5_promiscuous_disable, 2393 .allmulticast_enable = mlx5_allmulticast_enable, 2394 .allmulticast_disable = mlx5_allmulticast_disable, 2395 .link_update = mlx5_link_update, 2396 .stats_get = mlx5_stats_get, 2397 .stats_reset = mlx5_stats_reset, 2398 .xstats_get = mlx5_xstats_get, 2399 .xstats_reset = mlx5_xstats_reset, 2400 .xstats_get_names = mlx5_xstats_get_names, 2401 .fw_version_get = mlx5_fw_version_get, 2402 .dev_infos_get = mlx5_dev_infos_get, 2403 .representor_info_get = mlx5_representor_info_get, 2404 .read_clock = mlx5_txpp_read_clock, 2405 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 2406 .vlan_filter_set = mlx5_vlan_filter_set, 2407 .rx_queue_setup = mlx5_rx_queue_setup, 2408 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 2409 .tx_queue_setup = mlx5_tx_queue_setup, 2410 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 2411 .rx_queue_release = mlx5_rx_queue_release, 2412 .tx_queue_release = mlx5_tx_queue_release, 2413 .rx_queue_start = mlx5_rx_queue_start, 2414 .rx_queue_stop = mlx5_rx_queue_stop, 2415 .tx_queue_start = mlx5_tx_queue_start, 2416 .tx_queue_stop = mlx5_tx_queue_stop, 2417 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 2418 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 2419 .mac_addr_remove = mlx5_mac_addr_remove, 2420 .mac_addr_add = mlx5_mac_addr_add, 2421 .mac_addr_set = mlx5_mac_addr_set, 2422 .set_mc_addr_list = mlx5_set_mc_addr_list, 2423 .mtu_set = mlx5_dev_set_mtu, 2424 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 2425 .vlan_offload_set = mlx5_vlan_offload_set, 2426 .flow_ops_get = mlx5_flow_ops_get, 2427 .rxq_info_get = mlx5_rxq_info_get, 2428 .txq_info_get = mlx5_txq_info_get, 2429 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 2430 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 2431 .rx_queue_intr_enable = mlx5_rx_intr_enable, 2432 .rx_queue_intr_disable = mlx5_rx_intr_disable, 2433 .is_removed = mlx5_is_removed, 2434 .get_module_info = mlx5_get_module_info, 2435 .get_module_eeprom = mlx5_get_module_eeprom, 2436 .hairpin_cap_get = mlx5_hairpin_cap_get, 2437 .mtr_ops_get = mlx5_flow_meter_ops_get, 2438 .hairpin_bind = mlx5_hairpin_bind, 2439 .hairpin_unbind = mlx5_hairpin_unbind, 2440 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 2441 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 2442 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 2443 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 2444 .get_monitor_addr = mlx5_get_monitor_addr, 2445 .count_aggr_ports = mlx5_count_aggr_ports, 2446 .map_aggr_tx_affinity = mlx5_map_aggr_tx_affinity, 2447 }; 2448 2449 /** 2450 * Verify and store value for device argument. 2451 * 2452 * @param[in] key 2453 * Key argument to verify. 2454 * @param[in] val 2455 * Value associated with key. 2456 * @param opaque 2457 * User data. 2458 * 2459 * @return 2460 * 0 on success, a negative errno value otherwise and rte_errno is set. 2461 */ 2462 static int 2463 mlx5_port_args_check_handler(const char *key, const char *val, void *opaque) 2464 { 2465 struct mlx5_port_config *config = opaque; 2466 signed long tmp; 2467 2468 /* No-op, port representors are processed in mlx5_dev_spawn(). */ 2469 if (!strcmp(MLX5_REPRESENTOR, key)) 2470 return 0; 2471 errno = 0; 2472 tmp = strtol(val, NULL, 0); 2473 if (errno) { 2474 rte_errno = errno; 2475 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); 2476 return -rte_errno; 2477 } 2478 if (tmp < 0) { 2479 /* Negative values are acceptable for some keys only. */ 2480 rte_errno = EINVAL; 2481 DRV_LOG(WARNING, "%s: invalid negative value \"%s\"", key, val); 2482 return -rte_errno; 2483 } 2484 if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) { 2485 if ((tmp & ~MLX5_RXQ_ENH_CQE_COMP_MASK) > 2486 MLX5_CQE_RESP_FORMAT_L34H_STRIDX) { 2487 DRV_LOG(ERR, "invalid CQE compression " 2488 "format parameter"); 2489 rte_errno = EINVAL; 2490 return -rte_errno; 2491 } 2492 config->cqe_comp = !!tmp; 2493 config->cqe_comp_fmt = tmp & ~MLX5_RXQ_ENH_CQE_COMP_MASK; 2494 config->enh_cqe_comp = !!(tmp & MLX5_RXQ_ENH_CQE_COMP_MASK); 2495 } else if (strcmp(MLX5_RXQ_PKT_PAD_EN, key) == 0) { 2496 config->hw_padding = !!tmp; 2497 } else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) { 2498 config->mprq.enabled = !!tmp; 2499 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) { 2500 config->mprq.log_stride_num = tmp; 2501 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_SIZE, key) == 0) { 2502 config->mprq.log_stride_size = tmp; 2503 } else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) { 2504 config->mprq.max_memcpy_len = tmp; 2505 } else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) { 2506 config->mprq.min_rxqs_num = tmp; 2507 } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) { 2508 DRV_LOG(WARNING, "%s: deprecated parameter," 2509 " converted to txq_inline_max", key); 2510 config->txq_inline_max = tmp; 2511 } else if (strcmp(MLX5_TXQ_INLINE_MAX, key) == 0) { 2512 config->txq_inline_max = tmp; 2513 } else if (strcmp(MLX5_TXQ_INLINE_MIN, key) == 0) { 2514 config->txq_inline_min = tmp; 2515 } else if (strcmp(MLX5_TXQ_INLINE_MPW, key) == 0) { 2516 config->txq_inline_mpw = tmp; 2517 } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) { 2518 config->txqs_inline = tmp; 2519 } else if (strcmp(MLX5_TXQS_MAX_VEC, key) == 0) { 2520 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2521 } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { 2522 config->mps = !!tmp; 2523 } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { 2524 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2525 } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { 2526 DRV_LOG(WARNING, "%s: deprecated parameter," 2527 " converted to txq_inline_mpw", key); 2528 config->txq_inline_mpw = tmp; 2529 } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) { 2530 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 2531 } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { 2532 config->rx_vec_en = !!tmp; 2533 } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) { 2534 config->max_dump_files_num = tmp; 2535 } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) { 2536 config->lro_timeout = tmp; 2537 } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) { 2538 config->log_hp_size = tmp; 2539 } else if (strcmp(MLX5_DELAY_DROP, key) == 0) { 2540 config->std_delay_drop = !!(tmp & MLX5_DELAY_DROP_STANDARD); 2541 config->hp_delay_drop = !!(tmp & MLX5_DELAY_DROP_HAIRPIN); 2542 } 2543 return 0; 2544 } 2545 2546 /** 2547 * Parse user port parameters and adjust them according to device capabilities. 2548 * 2549 * @param priv 2550 * Pointer to shared device context. 2551 * @param mkvlist 2552 * Pointer to mlx5 kvargs control, can be NULL if there is no devargs. 2553 * @param config 2554 * Pointer to port configuration structure. 2555 * 2556 * @return 2557 * 0 on success, a negative errno value otherwise and rte_errno is set. 2558 */ 2559 int 2560 mlx5_port_args_config(struct mlx5_priv *priv, struct mlx5_kvargs_ctrl *mkvlist, 2561 struct mlx5_port_config *config) 2562 { 2563 struct mlx5_hca_attr *hca_attr = &priv->sh->cdev->config.hca_attr; 2564 struct mlx5_dev_cap *dev_cap = &priv->sh->dev_cap; 2565 bool devx = priv->sh->cdev->config.devx; 2566 const char **params = (const char *[]){ 2567 MLX5_RXQ_CQE_COMP_EN, 2568 MLX5_RXQ_PKT_PAD_EN, 2569 MLX5_RX_MPRQ_EN, 2570 MLX5_RX_MPRQ_LOG_STRIDE_NUM, 2571 MLX5_RX_MPRQ_LOG_STRIDE_SIZE, 2572 MLX5_RX_MPRQ_MAX_MEMCPY_LEN, 2573 MLX5_RXQS_MIN_MPRQ, 2574 MLX5_TXQ_INLINE, 2575 MLX5_TXQ_INLINE_MIN, 2576 MLX5_TXQ_INLINE_MAX, 2577 MLX5_TXQ_INLINE_MPW, 2578 MLX5_TXQS_MIN_INLINE, 2579 MLX5_TXQS_MAX_VEC, 2580 MLX5_TXQ_MPW_EN, 2581 MLX5_TXQ_MPW_HDR_DSEG_EN, 2582 MLX5_TXQ_MAX_INLINE_LEN, 2583 MLX5_TX_VEC_EN, 2584 MLX5_RX_VEC_EN, 2585 MLX5_REPRESENTOR, 2586 MLX5_MAX_DUMP_FILES_NUM, 2587 MLX5_LRO_TIMEOUT_USEC, 2588 MLX5_HP_BUF_SIZE, 2589 MLX5_DELAY_DROP, 2590 NULL, 2591 }; 2592 int ret = 0; 2593 2594 /* Default configuration. */ 2595 memset(config, 0, sizeof(*config)); 2596 config->mps = MLX5_ARG_UNSET; 2597 config->cqe_comp = 1; 2598 config->rx_vec_en = 1; 2599 config->txq_inline_max = MLX5_ARG_UNSET; 2600 config->txq_inline_min = MLX5_ARG_UNSET; 2601 config->txq_inline_mpw = MLX5_ARG_UNSET; 2602 config->txqs_inline = MLX5_ARG_UNSET; 2603 config->mprq.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN; 2604 config->mprq.min_rxqs_num = MLX5_MPRQ_MIN_RXQS; 2605 config->mprq.log_stride_num = MLX5_MPRQ_DEFAULT_LOG_STRIDE_NUM; 2606 config->mprq.log_stride_size = MLX5_MPRQ_DEFAULT_LOG_STRIDE_SIZE; 2607 config->log_hp_size = MLX5_ARG_UNSET; 2608 config->std_delay_drop = 0; 2609 config->hp_delay_drop = 0; 2610 if (mkvlist != NULL) { 2611 /* Process parameters. */ 2612 ret = mlx5_kvargs_process(mkvlist, params, 2613 mlx5_port_args_check_handler, config); 2614 if (ret) { 2615 DRV_LOG(ERR, "Failed to process port arguments: %s", 2616 strerror(rte_errno)); 2617 return -rte_errno; 2618 } 2619 } 2620 /* Adjust parameters according to device capabilities. */ 2621 if (config->hw_padding && !dev_cap->hw_padding) { 2622 DRV_LOG(DEBUG, "Rx end alignment padding isn't supported."); 2623 config->hw_padding = 0; 2624 } else if (config->hw_padding) { 2625 DRV_LOG(DEBUG, "Rx end alignment padding is enabled."); 2626 } 2627 /* 2628 * MPW is disabled by default, while the Enhanced MPW is enabled 2629 * by default. 2630 */ 2631 if (config->mps == MLX5_ARG_UNSET) 2632 config->mps = (dev_cap->mps == MLX5_MPW_ENHANCED) ? 2633 MLX5_MPW_ENHANCED : MLX5_MPW_DISABLED; 2634 else 2635 config->mps = config->mps ? dev_cap->mps : MLX5_MPW_DISABLED; 2636 DRV_LOG(INFO, "%sMPS is %s", 2637 config->mps == MLX5_MPW_ENHANCED ? "enhanced " : 2638 config->mps == MLX5_MPW ? "legacy " : "", 2639 config->mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); 2640 if (priv->sh->config.lro_allowed) { 2641 /* 2642 * If LRO timeout is not configured by application, 2643 * use the minimal supported value. 2644 */ 2645 if (!config->lro_timeout) 2646 config->lro_timeout = 2647 hca_attr->lro_timer_supported_periods[0]; 2648 DRV_LOG(DEBUG, "LRO session timeout set to %d usec.", 2649 config->lro_timeout); 2650 } 2651 if (config->cqe_comp && !dev_cap->cqe_comp) { 2652 DRV_LOG(WARNING, "Rx CQE 128B compression is not supported."); 2653 config->cqe_comp = 0; 2654 } 2655 if (config->cqe_comp_fmt == MLX5_CQE_RESP_FORMAT_FTAG_STRIDX && 2656 (!devx || !hca_attr->mini_cqe_resp_flow_tag)) { 2657 DRV_LOG(WARNING, 2658 "Flow Tag CQE compression format isn't supported."); 2659 config->cqe_comp = 0; 2660 } 2661 if (config->cqe_comp_fmt == MLX5_CQE_RESP_FORMAT_L34H_STRIDX && 2662 (!devx || !hca_attr->mini_cqe_resp_l3_l4_tag)) { 2663 DRV_LOG(WARNING, 2664 "L3/L4 Header CQE compression format isn't supported."); 2665 config->cqe_comp = 0; 2666 } 2667 if (config->enh_cqe_comp && !hca_attr->enhanced_cqe_compression) { 2668 DRV_LOG(WARNING, 2669 "Enhanced CQE compression isn't supported."); 2670 config->enh_cqe_comp = 0; 2671 } 2672 DRV_LOG(DEBUG, "%sRx CQE compression is %ssupported.", 2673 config->enh_cqe_comp ? "Enhanced " : "", 2674 config->cqe_comp ? "" : "not "); 2675 if ((config->std_delay_drop || config->hp_delay_drop) && 2676 !dev_cap->rq_delay_drop_en) { 2677 config->std_delay_drop = 0; 2678 config->hp_delay_drop = 0; 2679 DRV_LOG(WARNING, "dev_port-%u: Rxq delay drop isn't supported.", 2680 priv->dev_port); 2681 } 2682 if (config->mprq.enabled && !priv->sh->dev_cap.mprq.enabled) { 2683 DRV_LOG(WARNING, "Multi-Packet RQ isn't supported."); 2684 config->mprq.enabled = 0; 2685 } 2686 if (config->max_dump_files_num == 0) 2687 config->max_dump_files_num = 128; 2688 /* Detect minimal data bytes to inline. */ 2689 mlx5_set_min_inline(priv); 2690 DRV_LOG(DEBUG, "VLAN insertion in WQE is %ssupported.", 2691 config->hw_vlan_insert ? "" : "not "); 2692 DRV_LOG(DEBUG, "\"rxq_pkt_pad_en\" is %u.", config->hw_padding); 2693 DRV_LOG(DEBUG, "\"rxq_cqe_comp_en\" is %u.", config->cqe_comp); 2694 DRV_LOG(DEBUG, "\"cqe_comp_fmt\" is %u.", config->cqe_comp_fmt); 2695 DRV_LOG(DEBUG, "\"enh_cqe_comp\" is %u.", config->enh_cqe_comp); 2696 DRV_LOG(DEBUG, "\"rx_vec_en\" is %u.", config->rx_vec_en); 2697 DRV_LOG(DEBUG, "Standard \"delay_drop\" is %u.", 2698 config->std_delay_drop); 2699 DRV_LOG(DEBUG, "Hairpin \"delay_drop\" is %u.", config->hp_delay_drop); 2700 DRV_LOG(DEBUG, "\"max_dump_files_num\" is %u.", 2701 config->max_dump_files_num); 2702 DRV_LOG(DEBUG, "\"log_hp_size\" is %u.", config->log_hp_size); 2703 DRV_LOG(DEBUG, "\"mprq_en\" is %u.", config->mprq.enabled); 2704 DRV_LOG(DEBUG, "\"mprq_log_stride_num\" is %u.", 2705 config->mprq.log_stride_num); 2706 DRV_LOG(DEBUG, "\"mprq_log_stride_size\" is %u.", 2707 config->mprq.log_stride_size); 2708 DRV_LOG(DEBUG, "\"mprq_max_memcpy_len\" is %u.", 2709 config->mprq.max_memcpy_len); 2710 DRV_LOG(DEBUG, "\"rxqs_min_mprq\" is %u.", config->mprq.min_rxqs_num); 2711 DRV_LOG(DEBUG, "\"lro_timeout_usec\" is %u.", config->lro_timeout); 2712 DRV_LOG(DEBUG, "\"txq_mpw_en\" is %d.", config->mps); 2713 DRV_LOG(DEBUG, "\"txqs_min_inline\" is %d.", config->txqs_inline); 2714 DRV_LOG(DEBUG, "\"txq_inline_min\" is %d.", config->txq_inline_min); 2715 DRV_LOG(DEBUG, "\"txq_inline_max\" is %d.", config->txq_inline_max); 2716 DRV_LOG(DEBUG, "\"txq_inline_mpw\" is %d.", config->txq_inline_mpw); 2717 return 0; 2718 } 2719 2720 /** 2721 * Print the key for device argument. 2722 * 2723 * It is "dummy" handler whose whole purpose is to enable using 2724 * mlx5_kvargs_process() function which set devargs as used. 2725 * 2726 * @param key 2727 * Key argument. 2728 * @param val 2729 * Value associated with key, unused. 2730 * @param opaque 2731 * Unused, can be NULL. 2732 * 2733 * @return 2734 * 0 on success, function cannot fail. 2735 */ 2736 static int 2737 mlx5_dummy_handler(const char *key, const char *val, void *opaque) 2738 { 2739 DRV_LOG(DEBUG, "\tKey: \"%s\" is set as used.", key); 2740 RTE_SET_USED(opaque); 2741 RTE_SET_USED(val); 2742 return 0; 2743 } 2744 2745 /** 2746 * Set requested devargs as used when device is already spawned. 2747 * 2748 * It is necessary since it is valid to ask probe again for existing device, 2749 * if its devargs don't assign as used, mlx5_kvargs_validate() will fail. 2750 * 2751 * @param name 2752 * Name of the existing device. 2753 * @param port_id 2754 * Port identifier of the device. 2755 * @param mkvlist 2756 * Pointer to mlx5 kvargs control to sign as used. 2757 */ 2758 void 2759 mlx5_port_args_set_used(const char *name, uint16_t port_id, 2760 struct mlx5_kvargs_ctrl *mkvlist) 2761 { 2762 const char **params = (const char *[]){ 2763 MLX5_RXQ_CQE_COMP_EN, 2764 MLX5_RXQ_PKT_PAD_EN, 2765 MLX5_RX_MPRQ_EN, 2766 MLX5_RX_MPRQ_LOG_STRIDE_NUM, 2767 MLX5_RX_MPRQ_LOG_STRIDE_SIZE, 2768 MLX5_RX_MPRQ_MAX_MEMCPY_LEN, 2769 MLX5_RXQS_MIN_MPRQ, 2770 MLX5_TXQ_INLINE, 2771 MLX5_TXQ_INLINE_MIN, 2772 MLX5_TXQ_INLINE_MAX, 2773 MLX5_TXQ_INLINE_MPW, 2774 MLX5_TXQS_MIN_INLINE, 2775 MLX5_TXQS_MAX_VEC, 2776 MLX5_TXQ_MPW_EN, 2777 MLX5_TXQ_MPW_HDR_DSEG_EN, 2778 MLX5_TXQ_MAX_INLINE_LEN, 2779 MLX5_TX_VEC_EN, 2780 MLX5_RX_VEC_EN, 2781 MLX5_REPRESENTOR, 2782 MLX5_MAX_DUMP_FILES_NUM, 2783 MLX5_LRO_TIMEOUT_USEC, 2784 MLX5_HP_BUF_SIZE, 2785 MLX5_DELAY_DROP, 2786 NULL, 2787 }; 2788 2789 /* Secondary process should not handle devargs. */ 2790 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2791 return; 2792 MLX5_ASSERT(mkvlist != NULL); 2793 DRV_LOG(DEBUG, "Ethernet device \"%s\" for port %u " 2794 "already exists, set devargs as used:", name, port_id); 2795 /* This function cannot fail with this handler. */ 2796 mlx5_kvargs_process(mkvlist, params, mlx5_dummy_handler, NULL); 2797 } 2798 2799 /** 2800 * Check sibling device configurations when probing again. 2801 * 2802 * Sibling devices sharing infiniband device context should have compatible 2803 * configurations. This regards representors and bonding device. 2804 * 2805 * @param cdev 2806 * Pointer to mlx5 device structure. 2807 * @param mkvlist 2808 * Pointer to mlx5 kvargs control, can be NULL if there is no devargs. 2809 * 2810 * @return 2811 * 0 on success, a negative errno value otherwise and rte_errno is set. 2812 */ 2813 int 2814 mlx5_probe_again_args_validate(struct mlx5_common_device *cdev, 2815 struct mlx5_kvargs_ctrl *mkvlist) 2816 { 2817 struct mlx5_dev_ctx_shared *sh = NULL; 2818 struct mlx5_sh_config *config; 2819 int ret; 2820 2821 /* Secondary process should not handle devargs. */ 2822 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2823 return 0; 2824 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 2825 /* Search for IB context by common device pointer. */ 2826 LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) 2827 if (sh->cdev == cdev) 2828 break; 2829 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 2830 /* There is sh for this device -> it isn't probe again. */ 2831 if (sh == NULL) 2832 return 0; 2833 config = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 2834 sizeof(struct mlx5_sh_config), 2835 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 2836 if (config == NULL) { 2837 rte_errno = -ENOMEM; 2838 return -rte_errno; 2839 } 2840 /* 2841 * Creates a temporary IB context configure structure according to new 2842 * devargs attached in probing again. 2843 */ 2844 ret = mlx5_shared_dev_ctx_args_config(sh, mkvlist, config); 2845 if (ret) { 2846 DRV_LOG(ERR, "Failed to process device configure: %s", 2847 strerror(rte_errno)); 2848 mlx5_free(config); 2849 return ret; 2850 } 2851 /* 2852 * Checks the match between the temporary structure and the existing 2853 * IB context structure. 2854 */ 2855 if (sh->config.dv_flow_en ^ config->dv_flow_en) { 2856 DRV_LOG(ERR, "\"dv_flow_en\" " 2857 "configuration mismatch for shared %s context.", 2858 sh->ibdev_name); 2859 goto error; 2860 } 2861 if ((sh->config.dv_xmeta_en ^ config->dv_xmeta_en) || 2862 (sh->config.dv_miss_info ^ config->dv_miss_info)) { 2863 DRV_LOG(ERR, "\"dv_xmeta_en\" " 2864 "configuration mismatch for shared %s context.", 2865 sh->ibdev_name); 2866 goto error; 2867 } 2868 if (sh->config.dv_esw_en ^ config->dv_esw_en) { 2869 DRV_LOG(ERR, "\"dv_esw_en\" " 2870 "configuration mismatch for shared %s context.", 2871 sh->ibdev_name); 2872 goto error; 2873 } 2874 if (sh->config.reclaim_mode ^ config->reclaim_mode) { 2875 DRV_LOG(ERR, "\"reclaim_mode\" " 2876 "configuration mismatch for shared %s context.", 2877 sh->ibdev_name); 2878 goto error; 2879 } 2880 if (sh->config.allow_duplicate_pattern ^ 2881 config->allow_duplicate_pattern) { 2882 DRV_LOG(ERR, "\"allow_duplicate_pattern\" " 2883 "configuration mismatch for shared %s context.", 2884 sh->ibdev_name); 2885 goto error; 2886 } 2887 if (sh->config.fdb_def_rule ^ config->fdb_def_rule) { 2888 DRV_LOG(ERR, "\"fdb_def_rule_en\" configuration mismatch for shared %s context.", 2889 sh->ibdev_name); 2890 goto error; 2891 } 2892 if (sh->config.l3_vxlan_en ^ config->l3_vxlan_en) { 2893 DRV_LOG(ERR, "\"l3_vxlan_en\" " 2894 "configuration mismatch for shared %s context.", 2895 sh->ibdev_name); 2896 goto error; 2897 } 2898 if (sh->config.decap_en ^ config->decap_en) { 2899 DRV_LOG(ERR, "\"decap_en\" " 2900 "configuration mismatch for shared %s context.", 2901 sh->ibdev_name); 2902 goto error; 2903 } 2904 if (sh->config.lacp_by_user ^ config->lacp_by_user) { 2905 DRV_LOG(ERR, "\"lacp_by_user\" " 2906 "configuration mismatch for shared %s context.", 2907 sh->ibdev_name); 2908 goto error; 2909 } 2910 if (sh->config.tx_pp ^ config->tx_pp) { 2911 DRV_LOG(ERR, "\"tx_pp\" " 2912 "configuration mismatch for shared %s context.", 2913 sh->ibdev_name); 2914 goto error; 2915 } 2916 if (sh->config.tx_skew ^ config->tx_skew) { 2917 DRV_LOG(ERR, "\"tx_skew\" " 2918 "configuration mismatch for shared %s context.", 2919 sh->ibdev_name); 2920 goto error; 2921 } 2922 mlx5_free(config); 2923 return 0; 2924 error: 2925 mlx5_free(config); 2926 rte_errno = EINVAL; 2927 return -rte_errno; 2928 } 2929 2930 /** 2931 * Configures the minimal amount of data to inline into WQE 2932 * while sending packets. 2933 * 2934 * - the txq_inline_min has the maximal priority, if this 2935 * key is specified in devargs 2936 * - if DevX is enabled the inline mode is queried from the 2937 * device (HCA attributes and NIC vport context if needed). 2938 * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4 Lx 2939 * and none (0 bytes) for other NICs 2940 * 2941 * @param priv 2942 * Pointer to the private device data structure. 2943 */ 2944 void 2945 mlx5_set_min_inline(struct mlx5_priv *priv) 2946 { 2947 struct mlx5_hca_attr *hca_attr = &priv->sh->cdev->config.hca_attr; 2948 struct mlx5_port_config *config = &priv->config; 2949 2950 if (config->txq_inline_min != MLX5_ARG_UNSET) { 2951 /* Application defines size of inlined data explicitly. */ 2952 if (priv->pci_dev != NULL) { 2953 switch (priv->pci_dev->id.device_id) { 2954 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2955 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2956 if (config->txq_inline_min < 2957 (int)MLX5_INLINE_HSIZE_L2) { 2958 DRV_LOG(DEBUG, 2959 "txq_inline_mix aligned to minimal ConnectX-4 required value %d", 2960 (int)MLX5_INLINE_HSIZE_L2); 2961 config->txq_inline_min = 2962 MLX5_INLINE_HSIZE_L2; 2963 } 2964 break; 2965 } 2966 } 2967 goto exit; 2968 } 2969 if (hca_attr->eth_net_offloads) { 2970 /* We have DevX enabled, inline mode queried successfully. */ 2971 switch (hca_attr->wqe_inline_mode) { 2972 case MLX5_CAP_INLINE_MODE_L2: 2973 /* outer L2 header must be inlined. */ 2974 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2975 goto exit; 2976 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2977 /* No inline data are required by NIC. */ 2978 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2979 config->hw_vlan_insert = 2980 hca_attr->wqe_vlan_insert; 2981 DRV_LOG(DEBUG, "Tx VLAN insertion is supported"); 2982 goto exit; 2983 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2984 /* inline mode is defined by NIC vport context. */ 2985 if (!hca_attr->eth_virt) 2986 break; 2987 switch (hca_attr->vport_inline_mode) { 2988 case MLX5_INLINE_MODE_NONE: 2989 config->txq_inline_min = 2990 MLX5_INLINE_HSIZE_NONE; 2991 goto exit; 2992 case MLX5_INLINE_MODE_L2: 2993 config->txq_inline_min = 2994 MLX5_INLINE_HSIZE_L2; 2995 goto exit; 2996 case MLX5_INLINE_MODE_IP: 2997 config->txq_inline_min = 2998 MLX5_INLINE_HSIZE_L3; 2999 goto exit; 3000 case MLX5_INLINE_MODE_TCP_UDP: 3001 config->txq_inline_min = 3002 MLX5_INLINE_HSIZE_L4; 3003 goto exit; 3004 case MLX5_INLINE_MODE_INNER_L2: 3005 config->txq_inline_min = 3006 MLX5_INLINE_HSIZE_INNER_L2; 3007 goto exit; 3008 case MLX5_INLINE_MODE_INNER_IP: 3009 config->txq_inline_min = 3010 MLX5_INLINE_HSIZE_INNER_L3; 3011 goto exit; 3012 case MLX5_INLINE_MODE_INNER_TCP_UDP: 3013 config->txq_inline_min = 3014 MLX5_INLINE_HSIZE_INNER_L4; 3015 goto exit; 3016 } 3017 } 3018 } 3019 if (priv->pci_dev == NULL) { 3020 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 3021 goto exit; 3022 } 3023 /* 3024 * We get here if we are unable to deduce 3025 * inline data size with DevX. Try PCI ID 3026 * to determine old NICs. 3027 */ 3028 switch (priv->pci_dev->id.device_id) { 3029 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 3030 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 3031 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: 3032 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 3033 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 3034 config->hw_vlan_insert = 0; 3035 break; 3036 case PCI_DEVICE_ID_MELLANOX_CONNECTX5: 3037 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 3038 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: 3039 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 3040 /* 3041 * These NICs support VLAN insertion from WQE and 3042 * report the wqe_vlan_insert flag. But there is the bug 3043 * and PFC control may be broken, so disable feature. 3044 */ 3045 config->hw_vlan_insert = 0; 3046 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 3047 break; 3048 default: 3049 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 3050 break; 3051 } 3052 exit: 3053 DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min); 3054 } 3055 3056 /** 3057 * Configures the metadata mask fields in the shared context. 3058 * 3059 * @param [in] dev 3060 * Pointer to Ethernet device. 3061 */ 3062 void 3063 mlx5_set_metadata_mask(struct rte_eth_dev *dev) 3064 { 3065 struct mlx5_priv *priv = dev->data->dev_private; 3066 struct mlx5_dev_ctx_shared *sh = priv->sh; 3067 uint32_t meta, mark, reg_c0; 3068 3069 reg_c0 = ~priv->vport_meta_mask; 3070 switch (sh->config.dv_xmeta_en) { 3071 case MLX5_XMETA_MODE_LEGACY: 3072 meta = UINT32_MAX; 3073 mark = MLX5_FLOW_MARK_MASK; 3074 break; 3075 case MLX5_XMETA_MODE_META16: 3076 meta = reg_c0 >> rte_bsf32(reg_c0); 3077 mark = MLX5_FLOW_MARK_MASK; 3078 break; 3079 case MLX5_XMETA_MODE_META32: 3080 meta = UINT32_MAX; 3081 mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK; 3082 break; 3083 case MLX5_XMETA_MODE_META32_HWS: 3084 meta = UINT32_MAX; 3085 mark = MLX5_FLOW_MARK_MASK; 3086 break; 3087 default: 3088 meta = 0; 3089 mark = 0; 3090 MLX5_ASSERT(false); 3091 break; 3092 } 3093 if (sh->dv_mark_mask && sh->dv_mark_mask != mark) 3094 DRV_LOG(WARNING, "metadata MARK mask mismatch %08X:%08X", 3095 sh->dv_mark_mask, mark); 3096 else 3097 sh->dv_mark_mask = mark; 3098 if (sh->dv_meta_mask && sh->dv_meta_mask != meta) 3099 DRV_LOG(WARNING, "metadata META mask mismatch %08X:%08X", 3100 sh->dv_meta_mask, meta); 3101 else 3102 sh->dv_meta_mask = meta; 3103 if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0) 3104 DRV_LOG(WARNING, "metadata reg_c0 mask mismatch %08X:%08X", 3105 sh->dv_meta_mask, reg_c0); 3106 else 3107 sh->dv_regc0_mask = reg_c0; 3108 DRV_LOG(DEBUG, "metadata mode %u", sh->config.dv_xmeta_en); 3109 DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask); 3110 DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask); 3111 DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask); 3112 } 3113 3114 int 3115 rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n) 3116 { 3117 static const char *const dynf_names[] = { 3118 RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, 3119 RTE_MBUF_DYNFLAG_METADATA_NAME, 3120 RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME 3121 }; 3122 unsigned int i; 3123 3124 if (n < RTE_DIM(dynf_names)) 3125 return -ENOMEM; 3126 for (i = 0; i < RTE_DIM(dynf_names); i++) { 3127 if (names[i] == NULL) 3128 return -EINVAL; 3129 strcpy(names[i], dynf_names[i]); 3130 } 3131 return RTE_DIM(dynf_names); 3132 } 3133 3134 /** 3135 * Look for the ethernet device belonging to mlx5 driver. 3136 * 3137 * @param[in] port_id 3138 * port_id to start looking for device. 3139 * @param[in] odev 3140 * Pointer to the hint device. When device is being probed 3141 * the its siblings (master and preceding representors might 3142 * not have assigned driver yet (because the mlx5_os_pci_probe() 3143 * is not completed yet, for this case match on hint 3144 * device may be used to detect sibling device. 3145 * 3146 * @return 3147 * port_id of found device, RTE_MAX_ETHPORT if not found. 3148 */ 3149 uint16_t 3150 mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev) 3151 { 3152 while (port_id < RTE_MAX_ETHPORTS) { 3153 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 3154 3155 if (dev->state != RTE_ETH_DEV_UNUSED && 3156 dev->device && 3157 (dev->device == odev || 3158 (dev->device->driver && 3159 dev->device->driver->name && 3160 ((strcmp(dev->device->driver->name, 3161 MLX5_PCI_DRIVER_NAME) == 0) || 3162 (strcmp(dev->device->driver->name, 3163 MLX5_AUXILIARY_DRIVER_NAME) == 0))))) 3164 break; 3165 port_id++; 3166 } 3167 if (port_id >= RTE_MAX_ETHPORTS) 3168 return RTE_MAX_ETHPORTS; 3169 return port_id; 3170 } 3171 3172 /** 3173 * Callback to remove a device. 3174 * 3175 * This function removes all Ethernet devices belong to a given device. 3176 * 3177 * @param[in] cdev 3178 * Pointer to the generic device. 3179 * 3180 * @return 3181 * 0 on success, the function cannot fail. 3182 */ 3183 int 3184 mlx5_net_remove(struct mlx5_common_device *cdev) 3185 { 3186 uint16_t port_id; 3187 int ret = 0; 3188 3189 RTE_ETH_FOREACH_DEV_OF(port_id, cdev->dev) { 3190 /* 3191 * mlx5_dev_close() is not registered to secondary process, 3192 * call the close function explicitly for secondary process. 3193 */ 3194 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3195 ret |= mlx5_dev_close(&rte_eth_devices[port_id]); 3196 else 3197 ret |= rte_eth_dev_close(port_id); 3198 } 3199 return ret == 0 ? 0 : -EIO; 3200 } 3201 3202 static const struct rte_pci_id mlx5_pci_id_map[] = { 3203 { 3204 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3205 PCI_DEVICE_ID_MELLANOX_CONNECTX4) 3206 }, 3207 { 3208 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3209 PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) 3210 }, 3211 { 3212 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3213 PCI_DEVICE_ID_MELLANOX_CONNECTX4LX) 3214 }, 3215 { 3216 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3217 PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) 3218 }, 3219 { 3220 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3221 PCI_DEVICE_ID_MELLANOX_CONNECTX5) 3222 }, 3223 { 3224 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3225 PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) 3226 }, 3227 { 3228 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3229 PCI_DEVICE_ID_MELLANOX_CONNECTX5EX) 3230 }, 3231 { 3232 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3233 PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF) 3234 }, 3235 { 3236 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3237 PCI_DEVICE_ID_MELLANOX_BLUEFIELD) 3238 }, 3239 { 3240 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3241 PCI_DEVICE_ID_MELLANOX_BLUEFIELDVF) 3242 }, 3243 { 3244 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3245 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 3246 }, 3247 { 3248 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3249 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF) 3250 }, 3251 { 3252 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3253 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 3254 }, 3255 { 3256 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3257 PCI_DEVICE_ID_MELLANOX_CONNECTXVF) 3258 }, 3259 { 3260 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3261 PCI_DEVICE_ID_MELLANOX_BLUEFIELD2) 3262 }, 3263 { 3264 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3265 PCI_DEVICE_ID_MELLANOX_CONNECTX6LX) 3266 }, 3267 { 3268 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3269 PCI_DEVICE_ID_MELLANOX_CONNECTX7) 3270 }, 3271 { 3272 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 3273 PCI_DEVICE_ID_MELLANOX_BLUEFIELD3) 3274 }, 3275 { 3276 .vendor_id = 0 3277 } 3278 }; 3279 3280 static struct mlx5_class_driver mlx5_net_driver = { 3281 .drv_class = MLX5_CLASS_ETH, 3282 .name = RTE_STR(MLX5_ETH_DRIVER_NAME), 3283 .id_table = mlx5_pci_id_map, 3284 .probe = mlx5_os_net_probe, 3285 .remove = mlx5_net_remove, 3286 .probe_again = 1, 3287 .intr_lsc = 1, 3288 .intr_rmv = 1, 3289 }; 3290 3291 /* Initialize driver log type. */ 3292 RTE_LOG_REGISTER_DEFAULT(mlx5_logtype, NOTICE) 3293 3294 /** 3295 * Driver initialization routine. 3296 */ 3297 RTE_INIT(rte_mlx5_pmd_init) 3298 { 3299 pthread_mutex_init(&mlx5_dev_ctx_list_mutex, NULL); 3300 mlx5_common_init(); 3301 /* Build the static tables for Verbs conversion. */ 3302 mlx5_set_ptype_table(); 3303 mlx5_set_cksum_table(); 3304 mlx5_set_swp_types_table(); 3305 if (mlx5_glue) 3306 mlx5_class_driver_register(&mlx5_net_driver); 3307 } 3308 3309 RTE_PMD_EXPORT_NAME(MLX5_ETH_DRIVER_NAME, __COUNTER__); 3310 RTE_PMD_REGISTER_PCI_TABLE(MLX5_ETH_DRIVER_NAME, mlx5_pci_id_map); 3311 RTE_PMD_REGISTER_KMOD_DEP(MLX5_ETH_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 3312