1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <stddef.h> 7 #include <unistd.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <errno.h> 12 13 #include <rte_malloc.h> 14 #include <ethdev_driver.h> 15 #include <ethdev_pci.h> 16 #include <rte_pci.h> 17 #include <rte_bus_pci.h> 18 #include <rte_common.h> 19 #include <rte_kvargs.h> 20 #include <rte_rwlock.h> 21 #include <rte_spinlock.h> 22 #include <rte_string_fns.h> 23 #include <rte_alarm.h> 24 #include <rte_cycles.h> 25 26 #include <mlx5_glue.h> 27 #include <mlx5_devx_cmds.h> 28 #include <mlx5_common.h> 29 #include <mlx5_common_os.h> 30 #include <mlx5_common_mp.h> 31 #include <mlx5_common_pci.h> 32 #include <mlx5_malloc.h> 33 34 #include "mlx5_defs.h" 35 #include "mlx5.h" 36 #include "mlx5_utils.h" 37 #include "mlx5_rxtx.h" 38 #include "mlx5_rx.h" 39 #include "mlx5_tx.h" 40 #include "mlx5_autoconf.h" 41 #include "mlx5_mr.h" 42 #include "mlx5_flow.h" 43 #include "mlx5_flow_os.h" 44 #include "rte_pmd_mlx5.h" 45 46 /* Device parameter to enable RX completion queue compression. */ 47 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en" 48 49 /* Device parameter to enable padding Rx packet to cacheline size. */ 50 #define MLX5_RXQ_PKT_PAD_EN "rxq_pkt_pad_en" 51 52 /* Device parameter to enable Multi-Packet Rx queue. */ 53 #define MLX5_RX_MPRQ_EN "mprq_en" 54 55 /* Device parameter to configure log 2 of the number of strides for MPRQ. */ 56 #define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num" 57 58 /* Device parameter to configure log 2 of the stride size for MPRQ. */ 59 #define MLX5_RX_MPRQ_LOG_STRIDE_SIZE "mprq_log_stride_size" 60 61 /* Device parameter to limit the size of memcpy'd packet for MPRQ. */ 62 #define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len" 63 64 /* Device parameter to set the minimum number of Rx queues to enable MPRQ. */ 65 #define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq" 66 67 /* Device parameter to configure inline send. Deprecated, ignored.*/ 68 #define MLX5_TXQ_INLINE "txq_inline" 69 70 /* Device parameter to limit packet size to inline with ordinary SEND. */ 71 #define MLX5_TXQ_INLINE_MAX "txq_inline_max" 72 73 /* Device parameter to configure minimal data size to inline. */ 74 #define MLX5_TXQ_INLINE_MIN "txq_inline_min" 75 76 /* Device parameter to limit packet size to inline with Enhanced MPW. */ 77 #define MLX5_TXQ_INLINE_MPW "txq_inline_mpw" 78 79 /* 80 * Device parameter to configure the number of TX queues threshold for 81 * enabling inline send. 82 */ 83 #define MLX5_TXQS_MIN_INLINE "txqs_min_inline" 84 85 /* 86 * Device parameter to configure the number of TX queues threshold for 87 * enabling vectorized Tx, deprecated, ignored (no vectorized Tx routines). 88 */ 89 #define MLX5_TXQS_MAX_VEC "txqs_max_vec" 90 91 /* Device parameter to enable multi-packet send WQEs. */ 92 #define MLX5_TXQ_MPW_EN "txq_mpw_en" 93 94 /* 95 * Device parameter to force doorbell register mapping 96 * to non-cahed region eliminating the extra write memory barrier. 97 */ 98 #define MLX5_TX_DB_NC "tx_db_nc" 99 100 /* 101 * Device parameter to include 2 dsegs in the title WQEBB. 102 * Deprecated, ignored. 103 */ 104 #define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en" 105 106 /* 107 * Device parameter to limit the size of inlining packet. 108 * Deprecated, ignored. 109 */ 110 #define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len" 111 112 /* 113 * Device parameter to enable Tx scheduling on timestamps 114 * and specify the packet pacing granularity in nanoseconds. 115 */ 116 #define MLX5_TX_PP "tx_pp" 117 118 /* 119 * Device parameter to specify skew in nanoseconds on Tx datapath, 120 * it represents the time between SQ start WQE processing and 121 * appearing actual packet data on the wire. 122 */ 123 #define MLX5_TX_SKEW "tx_skew" 124 125 /* 126 * Device parameter to enable hardware Tx vector. 127 * Deprecated, ignored (no vectorized Tx routines anymore). 128 */ 129 #define MLX5_TX_VEC_EN "tx_vec_en" 130 131 /* Device parameter to enable hardware Rx vector. */ 132 #define MLX5_RX_VEC_EN "rx_vec_en" 133 134 /* Allow L3 VXLAN flow creation. */ 135 #define MLX5_L3_VXLAN_EN "l3_vxlan_en" 136 137 /* Activate DV E-Switch flow steering. */ 138 #define MLX5_DV_ESW_EN "dv_esw_en" 139 140 /* Activate DV flow steering. */ 141 #define MLX5_DV_FLOW_EN "dv_flow_en" 142 143 /* Enable extensive flow metadata support. */ 144 #define MLX5_DV_XMETA_EN "dv_xmeta_en" 145 146 /* Device parameter to let the user manage the lacp traffic of bonded device */ 147 #define MLX5_LACP_BY_USER "lacp_by_user" 148 149 /* Activate Netlink support in VF mode. */ 150 #define MLX5_VF_NL_EN "vf_nl_en" 151 152 /* Enable extending memsegs when creating a MR. */ 153 #define MLX5_MR_EXT_MEMSEG_EN "mr_ext_memseg_en" 154 155 /* Select port representors to instantiate. */ 156 #define MLX5_REPRESENTOR "representor" 157 158 /* Device parameter to configure the maximum number of dump files per queue. */ 159 #define MLX5_MAX_DUMP_FILES_NUM "max_dump_files_num" 160 161 /* Configure timeout of LRO session (in microseconds). */ 162 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec" 163 164 /* 165 * Device parameter to configure the total data buffer size for a single 166 * hairpin queue (logarithm value). 167 */ 168 #define MLX5_HP_BUF_SIZE "hp_buf_log_sz" 169 170 /* Flow memory reclaim mode. */ 171 #define MLX5_RECLAIM_MEM "reclaim_mem_mode" 172 173 /* The default memory allocator used in PMD. */ 174 #define MLX5_SYS_MEM_EN "sys_mem_en" 175 /* Decap will be used or not. */ 176 #define MLX5_DECAP_EN "decap_en" 177 178 /* Shared memory between primary and secondary processes. */ 179 struct mlx5_shared_data *mlx5_shared_data; 180 181 /** Driver-specific log messages type. */ 182 int mlx5_logtype; 183 184 static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = 185 LIST_HEAD_INITIALIZER(); 186 static pthread_mutex_t mlx5_dev_ctx_list_mutex; 187 static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { 188 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 189 [MLX5_IPOOL_DECAP_ENCAP] = { 190 .size = sizeof(struct mlx5_flow_dv_encap_decap_resource), 191 .trunk_size = 64, 192 .grow_trunk = 3, 193 .grow_shift = 2, 194 .need_lock = 1, 195 .release_mem_en = 1, 196 .malloc = mlx5_malloc, 197 .free = mlx5_free, 198 .type = "mlx5_encap_decap_ipool", 199 }, 200 [MLX5_IPOOL_PUSH_VLAN] = { 201 .size = sizeof(struct mlx5_flow_dv_push_vlan_action_resource), 202 .trunk_size = 64, 203 .grow_trunk = 3, 204 .grow_shift = 2, 205 .need_lock = 1, 206 .release_mem_en = 1, 207 .malloc = mlx5_malloc, 208 .free = mlx5_free, 209 .type = "mlx5_push_vlan_ipool", 210 }, 211 [MLX5_IPOOL_TAG] = { 212 .size = sizeof(struct mlx5_flow_dv_tag_resource), 213 .trunk_size = 64, 214 .grow_trunk = 3, 215 .grow_shift = 2, 216 .need_lock = 1, 217 .release_mem_en = 1, 218 .malloc = mlx5_malloc, 219 .free = mlx5_free, 220 .type = "mlx5_tag_ipool", 221 }, 222 [MLX5_IPOOL_PORT_ID] = { 223 .size = sizeof(struct mlx5_flow_dv_port_id_action_resource), 224 .trunk_size = 64, 225 .grow_trunk = 3, 226 .grow_shift = 2, 227 .need_lock = 1, 228 .release_mem_en = 1, 229 .malloc = mlx5_malloc, 230 .free = mlx5_free, 231 .type = "mlx5_port_id_ipool", 232 }, 233 [MLX5_IPOOL_JUMP] = { 234 .size = sizeof(struct mlx5_flow_tbl_data_entry), 235 .trunk_size = 64, 236 .grow_trunk = 3, 237 .grow_shift = 2, 238 .need_lock = 1, 239 .release_mem_en = 1, 240 .malloc = mlx5_malloc, 241 .free = mlx5_free, 242 .type = "mlx5_jump_ipool", 243 }, 244 [MLX5_IPOOL_SAMPLE] = { 245 .size = sizeof(struct mlx5_flow_dv_sample_resource), 246 .trunk_size = 64, 247 .grow_trunk = 3, 248 .grow_shift = 2, 249 .need_lock = 1, 250 .release_mem_en = 1, 251 .malloc = mlx5_malloc, 252 .free = mlx5_free, 253 .type = "mlx5_sample_ipool", 254 }, 255 [MLX5_IPOOL_DEST_ARRAY] = { 256 .size = sizeof(struct mlx5_flow_dv_dest_array_resource), 257 .trunk_size = 64, 258 .grow_trunk = 3, 259 .grow_shift = 2, 260 .need_lock = 1, 261 .release_mem_en = 1, 262 .malloc = mlx5_malloc, 263 .free = mlx5_free, 264 .type = "mlx5_dest_array_ipool", 265 }, 266 [MLX5_IPOOL_TUNNEL_ID] = { 267 .size = sizeof(struct mlx5_flow_tunnel), 268 .trunk_size = MLX5_MAX_TUNNELS, 269 .need_lock = 1, 270 .release_mem_en = 1, 271 .type = "mlx5_tunnel_offload", 272 }, 273 [MLX5_IPOOL_TNL_TBL_ID] = { 274 .size = 0, 275 .need_lock = 1, 276 .type = "mlx5_flow_tnl_tbl_ipool", 277 }, 278 #endif 279 [MLX5_IPOOL_MTR] = { 280 /** 281 * The ipool index should grow continually from small to big, 282 * for meter idx, so not set grow_trunk to avoid meter index 283 * not jump continually. 284 */ 285 .size = sizeof(struct mlx5_legacy_flow_meter), 286 .trunk_size = 64, 287 .need_lock = 1, 288 .release_mem_en = 1, 289 .malloc = mlx5_malloc, 290 .free = mlx5_free, 291 .type = "mlx5_meter_ipool", 292 }, 293 [MLX5_IPOOL_MCP] = { 294 .size = sizeof(struct mlx5_flow_mreg_copy_resource), 295 .trunk_size = 64, 296 .grow_trunk = 3, 297 .grow_shift = 2, 298 .need_lock = 1, 299 .release_mem_en = 1, 300 .malloc = mlx5_malloc, 301 .free = mlx5_free, 302 .type = "mlx5_mcp_ipool", 303 }, 304 [MLX5_IPOOL_HRXQ] = { 305 .size = (sizeof(struct mlx5_hrxq) + MLX5_RSS_HASH_KEY_LEN), 306 .trunk_size = 64, 307 .grow_trunk = 3, 308 .grow_shift = 2, 309 .need_lock = 1, 310 .release_mem_en = 1, 311 .malloc = mlx5_malloc, 312 .free = mlx5_free, 313 .type = "mlx5_hrxq_ipool", 314 }, 315 [MLX5_IPOOL_MLX5_FLOW] = { 316 /* 317 * MLX5_IPOOL_MLX5_FLOW size varies for DV and VERBS flows. 318 * It set in run time according to PCI function configuration. 319 */ 320 .size = 0, 321 .trunk_size = 64, 322 .grow_trunk = 3, 323 .grow_shift = 2, 324 .need_lock = 1, 325 .release_mem_en = 1, 326 .malloc = mlx5_malloc, 327 .free = mlx5_free, 328 .type = "mlx5_flow_handle_ipool", 329 }, 330 [MLX5_IPOOL_RTE_FLOW] = { 331 .size = sizeof(struct rte_flow), 332 .trunk_size = 4096, 333 .need_lock = 1, 334 .release_mem_en = 1, 335 .malloc = mlx5_malloc, 336 .free = mlx5_free, 337 .type = "rte_flow_ipool", 338 }, 339 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID] = { 340 .size = 0, 341 .need_lock = 1, 342 .type = "mlx5_flow_rss_id_ipool", 343 }, 344 [MLX5_IPOOL_RSS_SHARED_ACTIONS] = { 345 .size = sizeof(struct mlx5_shared_action_rss), 346 .trunk_size = 64, 347 .grow_trunk = 3, 348 .grow_shift = 2, 349 .need_lock = 1, 350 .release_mem_en = 1, 351 .malloc = mlx5_malloc, 352 .free = mlx5_free, 353 .type = "mlx5_shared_action_rss", 354 }, 355 [MLX5_IPOOL_MTR_POLICY] = { 356 /** 357 * The ipool index should grow continually from small to big, 358 * for policy idx, so not set grow_trunk to avoid policy index 359 * not jump continually. 360 */ 361 .size = sizeof(struct mlx5_flow_meter_sub_policy), 362 .trunk_size = 64, 363 .need_lock = 1, 364 .release_mem_en = 1, 365 .malloc = mlx5_malloc, 366 .free = mlx5_free, 367 .type = "mlx5_meter_policy_ipool", 368 }, 369 }; 370 371 372 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512 373 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16 374 375 #define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096 376 377 /** 378 * Decide whether representor ID is a HPF(host PF) port on BF2. 379 * 380 * @param dev 381 * Pointer to Ethernet device structure. 382 * 383 * @return 384 * Non-zero if HPF, otherwise 0. 385 */ 386 bool 387 mlx5_is_hpf(struct rte_eth_dev *dev) 388 { 389 struct mlx5_priv *priv = dev->data->dev_private; 390 uint16_t repr = MLX5_REPRESENTOR_REPR(priv->representor_id); 391 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 392 393 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_VF && 394 MLX5_REPRESENTOR_REPR(-1) == repr; 395 } 396 397 /** 398 * Initialize the ASO aging management structure. 399 * 400 * @param[in] sh 401 * Pointer to mlx5_dev_ctx_shared object to free 402 * 403 * @return 404 * 0 on success, a negative errno value otherwise and rte_errno is set. 405 */ 406 int 407 mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh) 408 { 409 int err; 410 411 if (sh->aso_age_mng) 412 return 0; 413 sh->aso_age_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->aso_age_mng), 414 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 415 if (!sh->aso_age_mng) { 416 DRV_LOG(ERR, "aso_age_mng allocation was failed."); 417 rte_errno = ENOMEM; 418 return -ENOMEM; 419 } 420 err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_FLOW_HIT); 421 if (err) { 422 mlx5_free(sh->aso_age_mng); 423 return -1; 424 } 425 rte_spinlock_init(&sh->aso_age_mng->resize_sl); 426 rte_spinlock_init(&sh->aso_age_mng->free_sl); 427 LIST_INIT(&sh->aso_age_mng->free); 428 return 0; 429 } 430 431 /** 432 * Close and release all the resources of the ASO aging management structure. 433 * 434 * @param[in] sh 435 * Pointer to mlx5_dev_ctx_shared object to free. 436 */ 437 static void 438 mlx5_flow_aso_age_mng_close(struct mlx5_dev_ctx_shared *sh) 439 { 440 int i, j; 441 442 mlx5_aso_flow_hit_queue_poll_stop(sh); 443 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_FLOW_HIT); 444 if (sh->aso_age_mng->pools) { 445 struct mlx5_aso_age_pool *pool; 446 447 for (i = 0; i < sh->aso_age_mng->next; ++i) { 448 pool = sh->aso_age_mng->pools[i]; 449 claim_zero(mlx5_devx_cmd_destroy 450 (pool->flow_hit_aso_obj)); 451 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) 452 if (pool->actions[j].dr_action) 453 claim_zero 454 (mlx5_flow_os_destroy_flow_action 455 (pool->actions[j].dr_action)); 456 mlx5_free(pool); 457 } 458 mlx5_free(sh->aso_age_mng->pools); 459 } 460 mlx5_free(sh->aso_age_mng); 461 } 462 463 /** 464 * Initialize the shared aging list information per port. 465 * 466 * @param[in] sh 467 * Pointer to mlx5_dev_ctx_shared object. 468 */ 469 static void 470 mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh) 471 { 472 uint32_t i; 473 struct mlx5_age_info *age_info; 474 475 for (i = 0; i < sh->max_port; i++) { 476 age_info = &sh->port[i].age_info; 477 age_info->flags = 0; 478 TAILQ_INIT(&age_info->aged_counters); 479 LIST_INIT(&age_info->aged_aso); 480 rte_spinlock_init(&age_info->aged_sl); 481 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 482 } 483 } 484 485 /** 486 * Initialize the counters management structure. 487 * 488 * @param[in] sh 489 * Pointer to mlx5_dev_ctx_shared object to free 490 */ 491 static void 492 mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh) 493 { 494 int i; 495 496 memset(&sh->cmng, 0, sizeof(sh->cmng)); 497 TAILQ_INIT(&sh->cmng.flow_counters); 498 sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET; 499 sh->cmng.max_id = -1; 500 sh->cmng.last_pool_idx = POOL_IDX_INVALID; 501 rte_spinlock_init(&sh->cmng.pool_update_sl); 502 for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) { 503 TAILQ_INIT(&sh->cmng.counters[i]); 504 rte_spinlock_init(&sh->cmng.csl[i]); 505 } 506 } 507 508 /** 509 * Destroy all the resources allocated for a counter memory management. 510 * 511 * @param[in] mng 512 * Pointer to the memory management structure. 513 */ 514 static void 515 mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng) 516 { 517 uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data; 518 519 LIST_REMOVE(mng, next); 520 claim_zero(mlx5_devx_cmd_destroy(mng->dm)); 521 claim_zero(mlx5_os_umem_dereg(mng->umem)); 522 mlx5_free(mem); 523 } 524 525 /** 526 * Close and release all the resources of the counters management. 527 * 528 * @param[in] sh 529 * Pointer to mlx5_dev_ctx_shared object to free. 530 */ 531 static void 532 mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) 533 { 534 struct mlx5_counter_stats_mem_mng *mng; 535 int i, j; 536 int retries = 1024; 537 538 rte_errno = 0; 539 while (--retries) { 540 rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh); 541 if (rte_errno != EINPROGRESS) 542 break; 543 rte_pause(); 544 } 545 546 if (sh->cmng.pools) { 547 struct mlx5_flow_counter_pool *pool; 548 uint16_t n_valid = sh->cmng.n_valid; 549 bool fallback = sh->cmng.counter_fallback; 550 551 for (i = 0; i < n_valid; ++i) { 552 pool = sh->cmng.pools[i]; 553 if (!fallback && pool->min_dcs) 554 claim_zero(mlx5_devx_cmd_destroy 555 (pool->min_dcs)); 556 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) { 557 struct mlx5_flow_counter *cnt = 558 MLX5_POOL_GET_CNT(pool, j); 559 560 if (cnt->action) 561 claim_zero 562 (mlx5_flow_os_destroy_flow_action 563 (cnt->action)); 564 if (fallback && MLX5_POOL_GET_CNT 565 (pool, j)->dcs_when_free) 566 claim_zero(mlx5_devx_cmd_destroy 567 (cnt->dcs_when_free)); 568 } 569 mlx5_free(pool); 570 } 571 mlx5_free(sh->cmng.pools); 572 } 573 mng = LIST_FIRST(&sh->cmng.mem_mngs); 574 while (mng) { 575 mlx5_flow_destroy_counter_stat_mem_mng(mng); 576 mng = LIST_FIRST(&sh->cmng.mem_mngs); 577 } 578 memset(&sh->cmng, 0, sizeof(sh->cmng)); 579 } 580 581 /** 582 * Initialize the aso flow meters management structure. 583 * 584 * @param[in] sh 585 * Pointer to mlx5_dev_ctx_shared object to free 586 */ 587 int 588 mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh) 589 { 590 if (!sh->mtrmng) { 591 sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, 592 sizeof(*sh->mtrmng), 593 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 594 if (!sh->mtrmng) { 595 DRV_LOG(ERR, 596 "meter management allocation was failed."); 597 rte_errno = ENOMEM; 598 return -ENOMEM; 599 } 600 if (sh->meter_aso_en) { 601 rte_spinlock_init(&sh->mtrmng->pools_mng.mtrsl); 602 LIST_INIT(&sh->mtrmng->pools_mng.meters); 603 sh->mtrmng->policy_idx_tbl = 604 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 605 } 606 sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 607 } 608 return 0; 609 } 610 611 /** 612 * Close and release all the resources of 613 * the ASO flow meter management structure. 614 * 615 * @param[in] sh 616 * Pointer to mlx5_dev_ctx_shared object to free. 617 */ 618 static void 619 mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) 620 { 621 struct mlx5_aso_mtr_pool *mtr_pool; 622 struct mlx5_flow_mtr_mng *mtrmng = sh->mtrmng; 623 uint32_t idx; 624 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 625 struct mlx5_aso_mtr *aso_mtr; 626 int i; 627 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 628 629 if (sh->meter_aso_en) { 630 mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); 631 idx = mtrmng->pools_mng.n_valid; 632 while (idx--) { 633 mtr_pool = mtrmng->pools_mng.pools[idx]; 634 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 635 for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { 636 aso_mtr = &mtr_pool->mtrs[i]; 637 if (aso_mtr->fm.meter_action) 638 claim_zero 639 (mlx5_glue->destroy_flow_action 640 (aso_mtr->fm.meter_action)); 641 } 642 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 643 claim_zero(mlx5_devx_cmd_destroy 644 (mtr_pool->devx_obj)); 645 mtrmng->pools_mng.n_valid--; 646 mlx5_free(mtr_pool); 647 } 648 mlx5_free(sh->mtrmng->pools_mng.pools); 649 } 650 mlx5_free(sh->mtrmng); 651 sh->mtrmng = NULL; 652 } 653 654 /* Send FLOW_AGED event if needed. */ 655 void 656 mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) 657 { 658 struct mlx5_age_info *age_info; 659 uint32_t i; 660 661 for (i = 0; i < sh->max_port; i++) { 662 age_info = &sh->port[i].age_info; 663 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 664 continue; 665 MLX5_AGE_UNSET(age_info, MLX5_AGE_EVENT_NEW); 666 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) { 667 MLX5_AGE_UNSET(age_info, MLX5_AGE_TRIGGER); 668 rte_eth_dev_callback_process 669 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 670 RTE_ETH_EVENT_FLOW_AGED, NULL); 671 } 672 } 673 } 674 675 /** 676 * Initialize the flow resources' indexed mempool. 677 * 678 * @param[in] sh 679 * Pointer to mlx5_dev_ctx_shared object. 680 * @param[in] config 681 * Pointer to user dev config. 682 */ 683 static void 684 mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh, 685 const struct mlx5_dev_config *config) 686 { 687 uint8_t i; 688 struct mlx5_indexed_pool_config cfg; 689 690 for (i = 0; i < MLX5_IPOOL_MAX; ++i) { 691 cfg = mlx5_ipool_cfg[i]; 692 switch (i) { 693 default: 694 break; 695 /* 696 * Set MLX5_IPOOL_MLX5_FLOW ipool size 697 * according to PCI function flow configuration. 698 */ 699 case MLX5_IPOOL_MLX5_FLOW: 700 cfg.size = config->dv_flow_en ? 701 sizeof(struct mlx5_flow_handle) : 702 MLX5_FLOW_HANDLE_VERBS_SIZE; 703 break; 704 } 705 if (config->reclaim_mode) 706 cfg.release_mem_en = 1; 707 sh->ipool[i] = mlx5_ipool_create(&cfg); 708 } 709 } 710 711 /** 712 * Release the flow resources' indexed mempool. 713 * 714 * @param[in] sh 715 * Pointer to mlx5_dev_ctx_shared object. 716 */ 717 static void 718 mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh) 719 { 720 uint8_t i; 721 722 for (i = 0; i < MLX5_IPOOL_MAX; ++i) 723 mlx5_ipool_destroy(sh->ipool[i]); 724 } 725 726 /* 727 * Check if dynamic flex parser for eCPRI already exists. 728 * 729 * @param dev 730 * Pointer to Ethernet device structure. 731 * 732 * @return 733 * true on exists, false on not. 734 */ 735 bool 736 mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev) 737 { 738 struct mlx5_priv *priv = dev->data->dev_private; 739 struct mlx5_flex_parser_profiles *prf = 740 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 741 742 return !!prf->obj; 743 } 744 745 /* 746 * Allocation of a flex parser for eCPRI. Once created, this parser related 747 * resources will be held until the device is closed. 748 * 749 * @param dev 750 * Pointer to Ethernet device structure. 751 * 752 * @return 753 * 0 on success, a negative errno value otherwise and rte_errno is set. 754 */ 755 int 756 mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev) 757 { 758 struct mlx5_priv *priv = dev->data->dev_private; 759 struct mlx5_flex_parser_profiles *prf = 760 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 761 struct mlx5_devx_graph_node_attr node = { 762 .modify_field_select = 0, 763 }; 764 uint32_t ids[8]; 765 int ret; 766 767 if (!priv->config.hca_attr.parse_graph_flex_node) { 768 DRV_LOG(ERR, "Dynamic flex parser is not supported " 769 "for device %s.", priv->dev_data->name); 770 return -ENOTSUP; 771 } 772 node.header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED; 773 /* 8 bytes now: 4B common header + 4B message body header. */ 774 node.header_length_base_value = 0x8; 775 /* After MAC layer: Ether / VLAN. */ 776 node.in[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_MAC; 777 /* Type of compared condition should be 0xAEFE in the L2 layer. */ 778 node.in[0].compare_condition_value = RTE_ETHER_TYPE_ECPRI; 779 /* Sample #0: type in common header. */ 780 node.sample[0].flow_match_sample_en = 1; 781 /* Fixed offset. */ 782 node.sample[0].flow_match_sample_offset_mode = 0x0; 783 /* Only the 2nd byte will be used. */ 784 node.sample[0].flow_match_sample_field_base_offset = 0x0; 785 /* Sample #1: message payload. */ 786 node.sample[1].flow_match_sample_en = 1; 787 /* Fixed offset. */ 788 node.sample[1].flow_match_sample_offset_mode = 0x0; 789 /* 790 * Only the first two bytes will be used right now, and its offset will 791 * start after the common header that with the length of a DW(u32). 792 */ 793 node.sample[1].flow_match_sample_field_base_offset = sizeof(uint32_t); 794 prf->obj = mlx5_devx_cmd_create_flex_parser(priv->sh->ctx, &node); 795 if (!prf->obj) { 796 DRV_LOG(ERR, "Failed to create flex parser node object."); 797 return (rte_errno == 0) ? -ENODEV : -rte_errno; 798 } 799 prf->num = 2; 800 ret = mlx5_devx_cmd_query_parse_samples(prf->obj, ids, prf->num); 801 if (ret) { 802 DRV_LOG(ERR, "Failed to query sample IDs."); 803 return (rte_errno == 0) ? -ENODEV : -rte_errno; 804 } 805 prf->offset[0] = 0x0; 806 prf->offset[1] = sizeof(uint32_t); 807 prf->ids[0] = ids[0]; 808 prf->ids[1] = ids[1]; 809 return 0; 810 } 811 812 /* 813 * Destroy the flex parser node, including the parser itself, input / output 814 * arcs and DW samples. Resources could be reused then. 815 * 816 * @param dev 817 * Pointer to Ethernet device structure. 818 */ 819 static void 820 mlx5_flex_parser_ecpri_release(struct rte_eth_dev *dev) 821 { 822 struct mlx5_priv *priv = dev->data->dev_private; 823 struct mlx5_flex_parser_profiles *prf = 824 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 825 826 if (prf->obj) 827 mlx5_devx_cmd_destroy(prf->obj); 828 prf->obj = NULL; 829 } 830 831 /* 832 * Allocate Rx and Tx UARs in robust fashion. 833 * This routine handles the following UAR allocation issues: 834 * 835 * - tries to allocate the UAR with the most appropriate memory 836 * mapping type from the ones supported by the host 837 * 838 * - tries to allocate the UAR with non-NULL base address 839 * OFED 5.0.x and Upstream rdma_core before v29 returned the NULL as 840 * UAR base address if UAR was not the first object in the UAR page. 841 * It caused the PMD failure and we should try to get another UAR 842 * till we get the first one with non-NULL base address returned. 843 */ 844 static int 845 mlx5_alloc_rxtx_uars(struct mlx5_dev_ctx_shared *sh, 846 const struct mlx5_dev_config *config) 847 { 848 uint32_t uar_mapping, retry; 849 int err = 0; 850 void *base_addr; 851 852 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 853 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 854 /* Control the mapping type according to the settings. */ 855 uar_mapping = (config->dbnc == MLX5_TXDB_NCACHED) ? 856 MLX5DV_UAR_ALLOC_TYPE_NC : 857 MLX5DV_UAR_ALLOC_TYPE_BF; 858 #else 859 RTE_SET_USED(config); 860 /* 861 * It seems we have no way to control the memory mapping type 862 * for the UAR, the default "Write-Combining" type is supposed. 863 * The UAR initialization on queue creation queries the 864 * actual mapping type done by Verbs/kernel and setups the 865 * PMD datapath accordingly. 866 */ 867 uar_mapping = 0; 868 #endif 869 sh->tx_uar = mlx5_glue->devx_alloc_uar(sh->ctx, uar_mapping); 870 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 871 if (!sh->tx_uar && 872 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 873 if (config->dbnc == MLX5_TXDB_CACHED || 874 config->dbnc == MLX5_TXDB_HEURISTIC) 875 DRV_LOG(WARNING, "Devarg tx_db_nc setting " 876 "is not supported by DevX"); 877 /* 878 * In some environments like virtual machine 879 * the Write Combining mapped might be not supported 880 * and UAR allocation fails. We try "Non-Cached" 881 * mapping for the case. The tx_burst routines take 882 * the UAR mapping type into account on UAR setup 883 * on queue creation. 884 */ 885 DRV_LOG(DEBUG, "Failed to allocate Tx DevX UAR (BF)"); 886 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 887 sh->tx_uar = mlx5_glue->devx_alloc_uar 888 (sh->ctx, uar_mapping); 889 } else if (!sh->tx_uar && 890 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) { 891 if (config->dbnc == MLX5_TXDB_NCACHED) 892 DRV_LOG(WARNING, "Devarg tx_db_nc settings " 893 "is not supported by DevX"); 894 /* 895 * If Verbs/kernel does not support "Non-Cached" 896 * try the "Write-Combining". 897 */ 898 DRV_LOG(DEBUG, "Failed to allocate Tx DevX UAR (NC)"); 899 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF; 900 sh->tx_uar = mlx5_glue->devx_alloc_uar 901 (sh->ctx, uar_mapping); 902 } 903 #endif 904 if (!sh->tx_uar) { 905 DRV_LOG(ERR, "Failed to allocate Tx DevX UAR (BF/NC)"); 906 err = ENOMEM; 907 goto exit; 908 } 909 base_addr = mlx5_os_get_devx_uar_base_addr(sh->tx_uar); 910 if (base_addr) 911 break; 912 /* 913 * The UARs are allocated by rdma_core within the 914 * IB device context, on context closure all UARs 915 * will be freed, should be no memory/object leakage. 916 */ 917 DRV_LOG(DEBUG, "Retrying to allocate Tx DevX UAR"); 918 sh->tx_uar = NULL; 919 } 920 /* Check whether we finally succeeded with valid UAR allocation. */ 921 if (!sh->tx_uar) { 922 DRV_LOG(ERR, "Failed to allocate Tx DevX UAR (NULL base)"); 923 err = ENOMEM; 924 goto exit; 925 } 926 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 927 uar_mapping = 0; 928 sh->devx_rx_uar = mlx5_glue->devx_alloc_uar 929 (sh->ctx, uar_mapping); 930 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 931 if (!sh->devx_rx_uar && 932 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 933 /* 934 * Rx UAR is used to control interrupts only, 935 * should be no datapath noticeable impact, 936 * can try "Non-Cached" mapping safely. 937 */ 938 DRV_LOG(DEBUG, "Failed to allocate Rx DevX UAR (BF)"); 939 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 940 sh->devx_rx_uar = mlx5_glue->devx_alloc_uar 941 (sh->ctx, uar_mapping); 942 } 943 #endif 944 if (!sh->devx_rx_uar) { 945 DRV_LOG(ERR, "Failed to allocate Rx DevX UAR (BF/NC)"); 946 err = ENOMEM; 947 goto exit; 948 } 949 base_addr = mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar); 950 if (base_addr) 951 break; 952 /* 953 * The UARs are allocated by rdma_core within the 954 * IB device context, on context closure all UARs 955 * will be freed, should be no memory/object leakage. 956 */ 957 DRV_LOG(DEBUG, "Retrying to allocate Rx DevX UAR"); 958 sh->devx_rx_uar = NULL; 959 } 960 /* Check whether we finally succeeded with valid UAR allocation. */ 961 if (!sh->devx_rx_uar) { 962 DRV_LOG(ERR, "Failed to allocate Rx DevX UAR (NULL base)"); 963 err = ENOMEM; 964 } 965 exit: 966 return err; 967 } 968 969 /** 970 * Allocate shared device context. If there is multiport device the 971 * master and representors will share this context, if there is single 972 * port dedicated device, the context will be used by only given 973 * port due to unification. 974 * 975 * Routine first searches the context for the specified device name, 976 * if found the shared context assumed and reference counter is incremented. 977 * If no context found the new one is created and initialized with specified 978 * device context and parameters. 979 * 980 * @param[in] spawn 981 * Pointer to the device attributes (name, port, etc). 982 * @param[in] config 983 * Pointer to device configuration structure. 984 * 985 * @return 986 * Pointer to mlx5_dev_ctx_shared object on success, 987 * otherwise NULL and rte_errno is set. 988 */ 989 struct mlx5_dev_ctx_shared * 990 mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, 991 const struct mlx5_dev_config *config) 992 { 993 struct mlx5_dev_ctx_shared *sh; 994 int err = 0; 995 uint32_t i; 996 struct mlx5_devx_tis_attr tis_attr = { 0 }; 997 998 MLX5_ASSERT(spawn); 999 /* Secondary process should not create the shared context. */ 1000 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1001 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1002 /* Search for IB context by device name. */ 1003 LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) { 1004 if (!strcmp(sh->ibdev_name, 1005 mlx5_os_get_dev_device_name(spawn->phys_dev))) { 1006 sh->refcnt++; 1007 goto exit; 1008 } 1009 } 1010 /* No device found, we have to create new shared context. */ 1011 MLX5_ASSERT(spawn->max_port); 1012 sh = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 1013 sizeof(struct mlx5_dev_ctx_shared) + 1014 spawn->max_port * 1015 sizeof(struct mlx5_dev_shared_port), 1016 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1017 if (!sh) { 1018 DRV_LOG(ERR, "shared context allocation failure"); 1019 rte_errno = ENOMEM; 1020 goto exit; 1021 } 1022 if (spawn->bond_info) 1023 sh->bond = *spawn->bond_info; 1024 err = mlx5_os_open_device(spawn, config, sh); 1025 if (!sh->ctx) 1026 goto error; 1027 err = mlx5_os_get_dev_attr(sh->ctx, &sh->device_attr); 1028 if (err) { 1029 DRV_LOG(DEBUG, "mlx5_os_get_dev_attr() failed"); 1030 goto error; 1031 } 1032 sh->refcnt = 1; 1033 sh->max_port = spawn->max_port; 1034 strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx), 1035 sizeof(sh->ibdev_name) - 1); 1036 strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx), 1037 sizeof(sh->ibdev_path) - 1); 1038 /* 1039 * Setting port_id to max unallowed value means 1040 * there is no interrupt subhandler installed for 1041 * the given port index i. 1042 */ 1043 for (i = 0; i < sh->max_port; i++) { 1044 sh->port[i].ih_port_id = RTE_MAX_ETHPORTS; 1045 sh->port[i].devx_ih_port_id = RTE_MAX_ETHPORTS; 1046 } 1047 sh->pd = mlx5_os_alloc_pd(sh->ctx); 1048 if (sh->pd == NULL) { 1049 DRV_LOG(ERR, "PD allocation failure"); 1050 err = ENOMEM; 1051 goto error; 1052 } 1053 if (sh->devx) { 1054 err = mlx5_os_get_pdn(sh->pd, &sh->pdn); 1055 if (err) { 1056 DRV_LOG(ERR, "Fail to extract pdn from PD"); 1057 goto error; 1058 } 1059 sh->td = mlx5_devx_cmd_create_td(sh->ctx); 1060 if (!sh->td) { 1061 DRV_LOG(ERR, "TD allocation failure"); 1062 err = ENOMEM; 1063 goto error; 1064 } 1065 tis_attr.transport_domain = sh->td->id; 1066 sh->tis = mlx5_devx_cmd_create_tis(sh->ctx, &tis_attr); 1067 if (!sh->tis) { 1068 DRV_LOG(ERR, "TIS allocation failure"); 1069 err = ENOMEM; 1070 goto error; 1071 } 1072 err = mlx5_alloc_rxtx_uars(sh, config); 1073 if (err) 1074 goto error; 1075 MLX5_ASSERT(sh->tx_uar); 1076 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->tx_uar)); 1077 1078 MLX5_ASSERT(sh->devx_rx_uar); 1079 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar)); 1080 } 1081 #ifndef RTE_ARCH_64 1082 /* Initialize UAR access locks for 32bit implementations. */ 1083 rte_spinlock_init(&sh->uar_lock_cq); 1084 for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) 1085 rte_spinlock_init(&sh->uar_lock[i]); 1086 #endif 1087 /* 1088 * Once the device is added to the list of memory event 1089 * callback, its global MR cache table cannot be expanded 1090 * on the fly because of deadlock. If it overflows, lookup 1091 * should be done by searching MR list linearly, which is slow. 1092 * 1093 * At this point the device is not added to the memory 1094 * event list yet, context is just being created. 1095 */ 1096 err = mlx5_mr_btree_init(&sh->share_cache.cache, 1097 MLX5_MR_BTREE_CACHE_N * 2, 1098 spawn->pci_dev->device.numa_node); 1099 if (err) { 1100 err = rte_errno; 1101 goto error; 1102 } 1103 mlx5_os_set_reg_mr_cb(&sh->share_cache.reg_mr_cb, 1104 &sh->share_cache.dereg_mr_cb); 1105 mlx5_os_dev_shared_handler_install(sh); 1106 sh->cnt_id_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1107 if (!sh->cnt_id_tbl) { 1108 err = rte_errno; 1109 goto error; 1110 } 1111 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1112 err = mlx5_flow_os_init_workspace_once(); 1113 if (err) 1114 goto error; 1115 } 1116 mlx5_flow_aging_init(sh); 1117 mlx5_flow_counters_mng_init(sh); 1118 mlx5_flow_ipool_create(sh, config); 1119 /* Add device to memory callback list. */ 1120 rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock); 1121 LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list, 1122 sh, mem_event_cb); 1123 rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock); 1124 /* Add context to the global device list. */ 1125 LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next); 1126 rte_spinlock_init(&sh->geneve_tlv_opt_sl); 1127 exit: 1128 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1129 return sh; 1130 error: 1131 pthread_mutex_destroy(&sh->txpp.mutex); 1132 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1133 MLX5_ASSERT(sh); 1134 if (sh->cnt_id_tbl) 1135 mlx5_l3t_destroy(sh->cnt_id_tbl); 1136 if (sh->tis) 1137 claim_zero(mlx5_devx_cmd_destroy(sh->tis)); 1138 if (sh->td) 1139 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1140 if (sh->devx_rx_uar) 1141 mlx5_glue->devx_free_uar(sh->devx_rx_uar); 1142 if (sh->tx_uar) 1143 mlx5_glue->devx_free_uar(sh->tx_uar); 1144 if (sh->pd) 1145 claim_zero(mlx5_os_dealloc_pd(sh->pd)); 1146 if (sh->ctx) 1147 claim_zero(mlx5_glue->close_device(sh->ctx)); 1148 mlx5_free(sh); 1149 MLX5_ASSERT(err > 0); 1150 rte_errno = err; 1151 return NULL; 1152 } 1153 1154 /** 1155 * Free shared IB device context. Decrement counter and if zero free 1156 * all allocated resources and close handles. 1157 * 1158 * @param[in] sh 1159 * Pointer to mlx5_dev_ctx_shared object to free 1160 */ 1161 void 1162 mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) 1163 { 1164 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1165 #ifdef RTE_LIBRTE_MLX5_DEBUG 1166 /* Check the object presence in the list. */ 1167 struct mlx5_dev_ctx_shared *lctx; 1168 1169 LIST_FOREACH(lctx, &mlx5_dev_ctx_list, next) 1170 if (lctx == sh) 1171 break; 1172 MLX5_ASSERT(lctx); 1173 if (lctx != sh) { 1174 DRV_LOG(ERR, "Freeing non-existing shared IB context"); 1175 goto exit; 1176 } 1177 #endif 1178 MLX5_ASSERT(sh); 1179 MLX5_ASSERT(sh->refcnt); 1180 /* Secondary process should not free the shared context. */ 1181 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1182 if (--sh->refcnt) 1183 goto exit; 1184 /* Remove from memory callback device list. */ 1185 rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock); 1186 LIST_REMOVE(sh, mem_event_cb); 1187 rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock); 1188 /* Release created Memory Regions. */ 1189 mlx5_mr_release_cache(&sh->share_cache); 1190 /* Remove context from the global device list. */ 1191 LIST_REMOVE(sh, next); 1192 /* Release flow workspaces objects on the last device. */ 1193 if (LIST_EMPTY(&mlx5_dev_ctx_list)) 1194 mlx5_flow_os_release_workspace(); 1195 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1196 /* 1197 * Ensure there is no async event handler installed. 1198 * Only primary process handles async device events. 1199 **/ 1200 mlx5_flow_counters_mng_close(sh); 1201 if (sh->aso_age_mng) { 1202 mlx5_flow_aso_age_mng_close(sh); 1203 sh->aso_age_mng = NULL; 1204 } 1205 if (sh->mtrmng) 1206 mlx5_aso_flow_mtrs_mng_close(sh); 1207 mlx5_flow_ipool_destroy(sh); 1208 mlx5_os_dev_shared_handler_uninstall(sh); 1209 if (sh->cnt_id_tbl) { 1210 mlx5_l3t_destroy(sh->cnt_id_tbl); 1211 sh->cnt_id_tbl = NULL; 1212 } 1213 if (sh->tx_uar) { 1214 mlx5_glue->devx_free_uar(sh->tx_uar); 1215 sh->tx_uar = NULL; 1216 } 1217 if (sh->pd) 1218 claim_zero(mlx5_os_dealloc_pd(sh->pd)); 1219 if (sh->tis) 1220 claim_zero(mlx5_devx_cmd_destroy(sh->tis)); 1221 if (sh->td) 1222 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1223 if (sh->devx_rx_uar) 1224 mlx5_glue->devx_free_uar(sh->devx_rx_uar); 1225 if (sh->ctx) 1226 claim_zero(mlx5_glue->close_device(sh->ctx)); 1227 MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL); 1228 pthread_mutex_destroy(&sh->txpp.mutex); 1229 mlx5_free(sh); 1230 return; 1231 exit: 1232 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1233 } 1234 1235 /** 1236 * Destroy table hash list. 1237 * 1238 * @param[in] priv 1239 * Pointer to the private device data structure. 1240 */ 1241 void 1242 mlx5_free_table_hash_list(struct mlx5_priv *priv) 1243 { 1244 struct mlx5_dev_ctx_shared *sh = priv->sh; 1245 1246 if (!sh->flow_tbls) 1247 return; 1248 mlx5_hlist_destroy(sh->flow_tbls); 1249 } 1250 1251 /** 1252 * Initialize flow table hash list and create the root tables entry 1253 * for each domain. 1254 * 1255 * @param[in] priv 1256 * Pointer to the private device data structure. 1257 * 1258 * @return 1259 * Zero on success, positive error code otherwise. 1260 */ 1261 int 1262 mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) 1263 { 1264 int err = 0; 1265 /* Tables are only used in DV and DR modes. */ 1266 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 1267 struct mlx5_dev_ctx_shared *sh = priv->sh; 1268 char s[MLX5_HLIST_NAMESIZE]; 1269 1270 MLX5_ASSERT(sh); 1271 snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); 1272 sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, 1273 0, 0, flow_dv_tbl_create_cb, 1274 flow_dv_tbl_match_cb, 1275 flow_dv_tbl_remove_cb); 1276 if (!sh->flow_tbls) { 1277 DRV_LOG(ERR, "flow tables with hash creation failed."); 1278 err = ENOMEM; 1279 return err; 1280 } 1281 sh->flow_tbls->ctx = sh; 1282 #ifndef HAVE_MLX5DV_DR 1283 struct rte_flow_error error; 1284 struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; 1285 1286 /* 1287 * In case we have not DR support, the zero tables should be created 1288 * because DV expect to see them even if they cannot be created by 1289 * RDMA-CORE. 1290 */ 1291 if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, 1292 NULL, 0, 1, 0, &error) || 1293 !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, 1294 NULL, 0, 1, 0, &error) || 1295 !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, 1296 NULL, 0, 1, 0, &error)) { 1297 err = ENOMEM; 1298 goto error; 1299 } 1300 return err; 1301 error: 1302 mlx5_free_table_hash_list(priv); 1303 #endif /* HAVE_MLX5DV_DR */ 1304 #endif 1305 return err; 1306 } 1307 1308 /** 1309 * Retrieve integer value from environment variable. 1310 * 1311 * @param[in] name 1312 * Environment variable name. 1313 * 1314 * @return 1315 * Integer value, 0 if the variable is not set. 1316 */ 1317 int 1318 mlx5_getenv_int(const char *name) 1319 { 1320 const char *val = getenv(name); 1321 1322 if (val == NULL) 1323 return 0; 1324 return atoi(val); 1325 } 1326 1327 /** 1328 * DPDK callback to add udp tunnel port 1329 * 1330 * @param[in] dev 1331 * A pointer to eth_dev 1332 * @param[in] udp_tunnel 1333 * A pointer to udp tunnel 1334 * 1335 * @return 1336 * 0 on valid udp ports and tunnels, -ENOTSUP otherwise. 1337 */ 1338 int 1339 mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev __rte_unused, 1340 struct rte_eth_udp_tunnel *udp_tunnel) 1341 { 1342 MLX5_ASSERT(udp_tunnel != NULL); 1343 if (udp_tunnel->prot_type == RTE_TUNNEL_TYPE_VXLAN && 1344 udp_tunnel->udp_port == 4789) 1345 return 0; 1346 if (udp_tunnel->prot_type == RTE_TUNNEL_TYPE_VXLAN_GPE && 1347 udp_tunnel->udp_port == 4790) 1348 return 0; 1349 return -ENOTSUP; 1350 } 1351 1352 /** 1353 * Initialize process private data structure. 1354 * 1355 * @param dev 1356 * Pointer to Ethernet device structure. 1357 * 1358 * @return 1359 * 0 on success, a negative errno value otherwise and rte_errno is set. 1360 */ 1361 int 1362 mlx5_proc_priv_init(struct rte_eth_dev *dev) 1363 { 1364 struct mlx5_priv *priv = dev->data->dev_private; 1365 struct mlx5_proc_priv *ppriv; 1366 size_t ppriv_size; 1367 1368 /* 1369 * UAR register table follows the process private structure. BlueFlame 1370 * registers for Tx queues are stored in the table. 1371 */ 1372 ppriv_size = 1373 sizeof(struct mlx5_proc_priv) + priv->txqs_n * sizeof(void *); 1374 ppriv = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, ppriv_size, 1375 RTE_CACHE_LINE_SIZE, dev->device->numa_node); 1376 if (!ppriv) { 1377 rte_errno = ENOMEM; 1378 return -rte_errno; 1379 } 1380 ppriv->uar_table_sz = priv->txqs_n; 1381 dev->process_private = ppriv; 1382 return 0; 1383 } 1384 1385 /** 1386 * Un-initialize process private data structure. 1387 * 1388 * @param dev 1389 * Pointer to Ethernet device structure. 1390 */ 1391 void 1392 mlx5_proc_priv_uninit(struct rte_eth_dev *dev) 1393 { 1394 if (!dev->process_private) 1395 return; 1396 mlx5_free(dev->process_private); 1397 dev->process_private = NULL; 1398 } 1399 1400 /** 1401 * DPDK callback to close the device. 1402 * 1403 * Destroy all queues and objects, free memory. 1404 * 1405 * @param dev 1406 * Pointer to Ethernet device structure. 1407 */ 1408 int 1409 mlx5_dev_close(struct rte_eth_dev *dev) 1410 { 1411 struct mlx5_priv *priv = dev->data->dev_private; 1412 unsigned int i; 1413 int ret; 1414 1415 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1416 /* Check if process_private released. */ 1417 if (!dev->process_private) 1418 return 0; 1419 mlx5_tx_uar_uninit_secondary(dev); 1420 mlx5_proc_priv_uninit(dev); 1421 rte_eth_dev_release_port(dev); 1422 return 0; 1423 } 1424 if (!priv->sh) 1425 return 0; 1426 DRV_LOG(DEBUG, "port %u closing device \"%s\"", 1427 dev->data->port_id, 1428 ((priv->sh->ctx != NULL) ? 1429 mlx5_os_get_ctx_device_name(priv->sh->ctx) : "")); 1430 /* 1431 * If default mreg copy action is removed at the stop stage, 1432 * the search will return none and nothing will be done anymore. 1433 */ 1434 mlx5_flow_stop_default(dev); 1435 mlx5_traffic_disable(dev); 1436 /* 1437 * If all the flows are already flushed in the device stop stage, 1438 * then this will return directly without any action. 1439 */ 1440 mlx5_flow_list_flush(dev, &priv->flows, true); 1441 mlx5_action_handle_flush(dev); 1442 mlx5_flow_meter_flush(dev, NULL); 1443 /* Prevent crashes when queues are still in use. */ 1444 dev->rx_pkt_burst = removed_rx_burst; 1445 dev->tx_pkt_burst = removed_tx_burst; 1446 rte_wmb(); 1447 /* Disable datapath on secondary process. */ 1448 mlx5_mp_os_req_stop_rxtx(dev); 1449 /* Free the eCPRI flex parser resource. */ 1450 mlx5_flex_parser_ecpri_release(dev); 1451 if (priv->rxqs != NULL) { 1452 /* XXX race condition if mlx5_rx_burst() is still running. */ 1453 rte_delay_us_sleep(1000); 1454 for (i = 0; (i != priv->rxqs_n); ++i) 1455 mlx5_rxq_release(dev, i); 1456 priv->rxqs_n = 0; 1457 priv->rxqs = NULL; 1458 } 1459 if (priv->txqs != NULL) { 1460 /* XXX race condition if mlx5_tx_burst() is still running. */ 1461 rte_delay_us_sleep(1000); 1462 for (i = 0; (i != priv->txqs_n); ++i) 1463 mlx5_txq_release(dev, i); 1464 priv->txqs_n = 0; 1465 priv->txqs = NULL; 1466 } 1467 mlx5_proc_priv_uninit(dev); 1468 if (priv->q_counters) { 1469 mlx5_devx_cmd_destroy(priv->q_counters); 1470 priv->q_counters = NULL; 1471 } 1472 if (priv->drop_queue.hrxq) 1473 mlx5_drop_action_destroy(dev); 1474 if (priv->mreg_cp_tbl) 1475 mlx5_hlist_destroy(priv->mreg_cp_tbl); 1476 mlx5_mprq_free_mp(dev); 1477 mlx5_os_free_shared_dr(priv); 1478 if (priv->rss_conf.rss_key != NULL) 1479 mlx5_free(priv->rss_conf.rss_key); 1480 if (priv->reta_idx != NULL) 1481 mlx5_free(priv->reta_idx); 1482 if (priv->config.vf) 1483 mlx5_os_mac_addr_flush(dev); 1484 if (priv->nl_socket_route >= 0) 1485 close(priv->nl_socket_route); 1486 if (priv->nl_socket_rdma >= 0) 1487 close(priv->nl_socket_rdma); 1488 if (priv->vmwa_context) 1489 mlx5_vlan_vmwa_exit(priv->vmwa_context); 1490 ret = mlx5_hrxq_verify(dev); 1491 if (ret) 1492 DRV_LOG(WARNING, "port %u some hash Rx queue still remain", 1493 dev->data->port_id); 1494 ret = mlx5_ind_table_obj_verify(dev); 1495 if (ret) 1496 DRV_LOG(WARNING, "port %u some indirection table still remain", 1497 dev->data->port_id); 1498 ret = mlx5_rxq_obj_verify(dev); 1499 if (ret) 1500 DRV_LOG(WARNING, "port %u some Rx queue objects still remain", 1501 dev->data->port_id); 1502 ret = mlx5_rxq_verify(dev); 1503 if (ret) 1504 DRV_LOG(WARNING, "port %u some Rx queues still remain", 1505 dev->data->port_id); 1506 ret = mlx5_txq_obj_verify(dev); 1507 if (ret) 1508 DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain", 1509 dev->data->port_id); 1510 ret = mlx5_txq_verify(dev); 1511 if (ret) 1512 DRV_LOG(WARNING, "port %u some Tx queues still remain", 1513 dev->data->port_id); 1514 ret = mlx5_flow_verify(dev); 1515 if (ret) 1516 DRV_LOG(WARNING, "port %u some flows still remain", 1517 dev->data->port_id); 1518 mlx5_cache_list_destroy(&priv->hrxqs); 1519 /* 1520 * Free the shared context in last turn, because the cleanup 1521 * routines above may use some shared fields, like 1522 * mlx5_os_mac_addr_flush() uses ibdev_path for retrieveing 1523 * ifindex if Netlink fails. 1524 */ 1525 mlx5_free_shared_dev_ctx(priv->sh); 1526 if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 1527 unsigned int c = 0; 1528 uint16_t port_id; 1529 1530 MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) { 1531 struct mlx5_priv *opriv = 1532 rte_eth_devices[port_id].data->dev_private; 1533 1534 if (!opriv || 1535 opriv->domain_id != priv->domain_id || 1536 &rte_eth_devices[port_id] == dev) 1537 continue; 1538 ++c; 1539 break; 1540 } 1541 if (!c) 1542 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 1543 } 1544 memset(priv, 0, sizeof(*priv)); 1545 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 1546 /* 1547 * Reset mac_addrs to NULL such that it is not freed as part of 1548 * rte_eth_dev_release_port(). mac_addrs is part of dev_private so 1549 * it is freed when dev_private is freed. 1550 */ 1551 dev->data->mac_addrs = NULL; 1552 return 0; 1553 } 1554 1555 const struct eth_dev_ops mlx5_dev_ops = { 1556 .dev_configure = mlx5_dev_configure, 1557 .dev_start = mlx5_dev_start, 1558 .dev_stop = mlx5_dev_stop, 1559 .dev_set_link_down = mlx5_set_link_down, 1560 .dev_set_link_up = mlx5_set_link_up, 1561 .dev_close = mlx5_dev_close, 1562 .promiscuous_enable = mlx5_promiscuous_enable, 1563 .promiscuous_disable = mlx5_promiscuous_disable, 1564 .allmulticast_enable = mlx5_allmulticast_enable, 1565 .allmulticast_disable = mlx5_allmulticast_disable, 1566 .link_update = mlx5_link_update, 1567 .stats_get = mlx5_stats_get, 1568 .stats_reset = mlx5_stats_reset, 1569 .xstats_get = mlx5_xstats_get, 1570 .xstats_reset = mlx5_xstats_reset, 1571 .xstats_get_names = mlx5_xstats_get_names, 1572 .fw_version_get = mlx5_fw_version_get, 1573 .dev_infos_get = mlx5_dev_infos_get, 1574 .representor_info_get = mlx5_representor_info_get, 1575 .read_clock = mlx5_txpp_read_clock, 1576 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1577 .vlan_filter_set = mlx5_vlan_filter_set, 1578 .rx_queue_setup = mlx5_rx_queue_setup, 1579 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1580 .tx_queue_setup = mlx5_tx_queue_setup, 1581 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1582 .rx_queue_release = mlx5_rx_queue_release, 1583 .tx_queue_release = mlx5_tx_queue_release, 1584 .rx_queue_start = mlx5_rx_queue_start, 1585 .rx_queue_stop = mlx5_rx_queue_stop, 1586 .tx_queue_start = mlx5_tx_queue_start, 1587 .tx_queue_stop = mlx5_tx_queue_stop, 1588 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1589 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1590 .mac_addr_remove = mlx5_mac_addr_remove, 1591 .mac_addr_add = mlx5_mac_addr_add, 1592 .mac_addr_set = mlx5_mac_addr_set, 1593 .set_mc_addr_list = mlx5_set_mc_addr_list, 1594 .mtu_set = mlx5_dev_set_mtu, 1595 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1596 .vlan_offload_set = mlx5_vlan_offload_set, 1597 .reta_update = mlx5_dev_rss_reta_update, 1598 .reta_query = mlx5_dev_rss_reta_query, 1599 .rss_hash_update = mlx5_rss_hash_update, 1600 .rss_hash_conf_get = mlx5_rss_hash_conf_get, 1601 .flow_ops_get = mlx5_flow_ops_get, 1602 .rxq_info_get = mlx5_rxq_info_get, 1603 .txq_info_get = mlx5_txq_info_get, 1604 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1605 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1606 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1607 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1608 .is_removed = mlx5_is_removed, 1609 .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, 1610 .get_module_info = mlx5_get_module_info, 1611 .get_module_eeprom = mlx5_get_module_eeprom, 1612 .hairpin_cap_get = mlx5_hairpin_cap_get, 1613 .mtr_ops_get = mlx5_flow_meter_ops_get, 1614 .hairpin_bind = mlx5_hairpin_bind, 1615 .hairpin_unbind = mlx5_hairpin_unbind, 1616 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1617 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1618 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1619 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1620 }; 1621 1622 /* Available operations from secondary process. */ 1623 const struct eth_dev_ops mlx5_dev_sec_ops = { 1624 .stats_get = mlx5_stats_get, 1625 .stats_reset = mlx5_stats_reset, 1626 .xstats_get = mlx5_xstats_get, 1627 .xstats_reset = mlx5_xstats_reset, 1628 .xstats_get_names = mlx5_xstats_get_names, 1629 .fw_version_get = mlx5_fw_version_get, 1630 .dev_infos_get = mlx5_dev_infos_get, 1631 .read_clock = mlx5_txpp_read_clock, 1632 .rx_queue_start = mlx5_rx_queue_start, 1633 .rx_queue_stop = mlx5_rx_queue_stop, 1634 .tx_queue_start = mlx5_tx_queue_start, 1635 .tx_queue_stop = mlx5_tx_queue_stop, 1636 .rxq_info_get = mlx5_rxq_info_get, 1637 .txq_info_get = mlx5_txq_info_get, 1638 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1639 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1640 .get_module_info = mlx5_get_module_info, 1641 .get_module_eeprom = mlx5_get_module_eeprom, 1642 }; 1643 1644 /* Available operations in flow isolated mode. */ 1645 const struct eth_dev_ops mlx5_dev_ops_isolate = { 1646 .dev_configure = mlx5_dev_configure, 1647 .dev_start = mlx5_dev_start, 1648 .dev_stop = mlx5_dev_stop, 1649 .dev_set_link_down = mlx5_set_link_down, 1650 .dev_set_link_up = mlx5_set_link_up, 1651 .dev_close = mlx5_dev_close, 1652 .promiscuous_enable = mlx5_promiscuous_enable, 1653 .promiscuous_disable = mlx5_promiscuous_disable, 1654 .allmulticast_enable = mlx5_allmulticast_enable, 1655 .allmulticast_disable = mlx5_allmulticast_disable, 1656 .link_update = mlx5_link_update, 1657 .stats_get = mlx5_stats_get, 1658 .stats_reset = mlx5_stats_reset, 1659 .xstats_get = mlx5_xstats_get, 1660 .xstats_reset = mlx5_xstats_reset, 1661 .xstats_get_names = mlx5_xstats_get_names, 1662 .fw_version_get = mlx5_fw_version_get, 1663 .dev_infos_get = mlx5_dev_infos_get, 1664 .read_clock = mlx5_txpp_read_clock, 1665 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1666 .vlan_filter_set = mlx5_vlan_filter_set, 1667 .rx_queue_setup = mlx5_rx_queue_setup, 1668 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1669 .tx_queue_setup = mlx5_tx_queue_setup, 1670 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1671 .rx_queue_release = mlx5_rx_queue_release, 1672 .tx_queue_release = mlx5_tx_queue_release, 1673 .rx_queue_start = mlx5_rx_queue_start, 1674 .rx_queue_stop = mlx5_rx_queue_stop, 1675 .tx_queue_start = mlx5_tx_queue_start, 1676 .tx_queue_stop = mlx5_tx_queue_stop, 1677 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1678 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1679 .mac_addr_remove = mlx5_mac_addr_remove, 1680 .mac_addr_add = mlx5_mac_addr_add, 1681 .mac_addr_set = mlx5_mac_addr_set, 1682 .set_mc_addr_list = mlx5_set_mc_addr_list, 1683 .mtu_set = mlx5_dev_set_mtu, 1684 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1685 .vlan_offload_set = mlx5_vlan_offload_set, 1686 .flow_ops_get = mlx5_flow_ops_get, 1687 .rxq_info_get = mlx5_rxq_info_get, 1688 .txq_info_get = mlx5_txq_info_get, 1689 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1690 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1691 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1692 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1693 .is_removed = mlx5_is_removed, 1694 .get_module_info = mlx5_get_module_info, 1695 .get_module_eeprom = mlx5_get_module_eeprom, 1696 .hairpin_cap_get = mlx5_hairpin_cap_get, 1697 .mtr_ops_get = mlx5_flow_meter_ops_get, 1698 .hairpin_bind = mlx5_hairpin_bind, 1699 .hairpin_unbind = mlx5_hairpin_unbind, 1700 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1701 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1702 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1703 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1704 }; 1705 1706 /** 1707 * Verify and store value for device argument. 1708 * 1709 * @param[in] key 1710 * Key argument to verify. 1711 * @param[in] val 1712 * Value associated with key. 1713 * @param opaque 1714 * User data. 1715 * 1716 * @return 1717 * 0 on success, a negative errno value otherwise and rte_errno is set. 1718 */ 1719 static int 1720 mlx5_args_check(const char *key, const char *val, void *opaque) 1721 { 1722 struct mlx5_dev_config *config = opaque; 1723 unsigned long mod; 1724 signed long tmp; 1725 1726 /* No-op, port representors are processed in mlx5_dev_spawn(). */ 1727 if (!strcmp(MLX5_REPRESENTOR, key)) 1728 return 0; 1729 errno = 0; 1730 tmp = strtol(val, NULL, 0); 1731 if (errno) { 1732 rte_errno = errno; 1733 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); 1734 return -rte_errno; 1735 } 1736 if (tmp < 0 && strcmp(MLX5_TX_PP, key) && strcmp(MLX5_TX_SKEW, key)) { 1737 /* Negative values are acceptable for some keys only. */ 1738 rte_errno = EINVAL; 1739 DRV_LOG(WARNING, "%s: invalid negative value \"%s\"", key, val); 1740 return -rte_errno; 1741 } 1742 mod = tmp >= 0 ? tmp : -tmp; 1743 if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) { 1744 if (tmp > MLX5_CQE_RESP_FORMAT_L34H_STRIDX) { 1745 DRV_LOG(ERR, "invalid CQE compression " 1746 "format parameter"); 1747 rte_errno = EINVAL; 1748 return -rte_errno; 1749 } 1750 config->cqe_comp = !!tmp; 1751 config->cqe_comp_fmt = tmp; 1752 } else if (strcmp(MLX5_RXQ_PKT_PAD_EN, key) == 0) { 1753 config->hw_padding = !!tmp; 1754 } else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) { 1755 config->mprq.enabled = !!tmp; 1756 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) { 1757 config->mprq.stride_num_n = tmp; 1758 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_SIZE, key) == 0) { 1759 config->mprq.stride_size_n = tmp; 1760 } else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) { 1761 config->mprq.max_memcpy_len = tmp; 1762 } else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) { 1763 config->mprq.min_rxqs_num = tmp; 1764 } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) { 1765 DRV_LOG(WARNING, "%s: deprecated parameter," 1766 " converted to txq_inline_max", key); 1767 config->txq_inline_max = tmp; 1768 } else if (strcmp(MLX5_TXQ_INLINE_MAX, key) == 0) { 1769 config->txq_inline_max = tmp; 1770 } else if (strcmp(MLX5_TXQ_INLINE_MIN, key) == 0) { 1771 config->txq_inline_min = tmp; 1772 } else if (strcmp(MLX5_TXQ_INLINE_MPW, key) == 0) { 1773 config->txq_inline_mpw = tmp; 1774 } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) { 1775 config->txqs_inline = tmp; 1776 } else if (strcmp(MLX5_TXQS_MAX_VEC, key) == 0) { 1777 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1778 } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { 1779 config->mps = !!tmp; 1780 } else if (strcmp(MLX5_TX_DB_NC, key) == 0) { 1781 if (tmp != MLX5_TXDB_CACHED && 1782 tmp != MLX5_TXDB_NCACHED && 1783 tmp != MLX5_TXDB_HEURISTIC) { 1784 DRV_LOG(ERR, "invalid Tx doorbell " 1785 "mapping parameter"); 1786 rte_errno = EINVAL; 1787 return -rte_errno; 1788 } 1789 config->dbnc = tmp; 1790 } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { 1791 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1792 } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { 1793 DRV_LOG(WARNING, "%s: deprecated parameter," 1794 " converted to txq_inline_mpw", key); 1795 config->txq_inline_mpw = tmp; 1796 } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) { 1797 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1798 } else if (strcmp(MLX5_TX_PP, key) == 0) { 1799 if (!mod) { 1800 DRV_LOG(ERR, "Zero Tx packet pacing parameter"); 1801 rte_errno = EINVAL; 1802 return -rte_errno; 1803 } 1804 config->tx_pp = tmp; 1805 } else if (strcmp(MLX5_TX_SKEW, key) == 0) { 1806 config->tx_skew = tmp; 1807 } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { 1808 config->rx_vec_en = !!tmp; 1809 } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) { 1810 config->l3_vxlan_en = !!tmp; 1811 } else if (strcmp(MLX5_VF_NL_EN, key) == 0) { 1812 config->vf_nl_en = !!tmp; 1813 } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) { 1814 config->dv_esw_en = !!tmp; 1815 } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) { 1816 config->dv_flow_en = !!tmp; 1817 } else if (strcmp(MLX5_DV_XMETA_EN, key) == 0) { 1818 if (tmp != MLX5_XMETA_MODE_LEGACY && 1819 tmp != MLX5_XMETA_MODE_META16 && 1820 tmp != MLX5_XMETA_MODE_META32 && 1821 tmp != MLX5_XMETA_MODE_MISS_INFO) { 1822 DRV_LOG(ERR, "invalid extensive " 1823 "metadata parameter"); 1824 rte_errno = EINVAL; 1825 return -rte_errno; 1826 } 1827 if (tmp != MLX5_XMETA_MODE_MISS_INFO) 1828 config->dv_xmeta_en = tmp; 1829 else 1830 config->dv_miss_info = 1; 1831 } else if (strcmp(MLX5_LACP_BY_USER, key) == 0) { 1832 config->lacp_by_user = !!tmp; 1833 } else if (strcmp(MLX5_MR_EXT_MEMSEG_EN, key) == 0) { 1834 config->mr_ext_memseg_en = !!tmp; 1835 } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) { 1836 config->max_dump_files_num = tmp; 1837 } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) { 1838 config->lro.timeout = tmp; 1839 } else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) { 1840 DRV_LOG(DEBUG, "class argument is %s.", val); 1841 } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) { 1842 config->log_hp_size = tmp; 1843 } else if (strcmp(MLX5_RECLAIM_MEM, key) == 0) { 1844 if (tmp != MLX5_RCM_NONE && 1845 tmp != MLX5_RCM_LIGHT && 1846 tmp != MLX5_RCM_AGGR) { 1847 DRV_LOG(ERR, "Unrecognize %s: \"%s\"", key, val); 1848 rte_errno = EINVAL; 1849 return -rte_errno; 1850 } 1851 config->reclaim_mode = tmp; 1852 } else if (strcmp(MLX5_SYS_MEM_EN, key) == 0) { 1853 config->sys_mem_en = !!tmp; 1854 } else if (strcmp(MLX5_DECAP_EN, key) == 0) { 1855 config->decap_en = !!tmp; 1856 } else { 1857 DRV_LOG(WARNING, "%s: unknown parameter", key); 1858 rte_errno = EINVAL; 1859 return -rte_errno; 1860 } 1861 return 0; 1862 } 1863 1864 /** 1865 * Parse device parameters. 1866 * 1867 * @param config 1868 * Pointer to device configuration structure. 1869 * @param devargs 1870 * Device arguments structure. 1871 * 1872 * @return 1873 * 0 on success, a negative errno value otherwise and rte_errno is set. 1874 */ 1875 int 1876 mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs) 1877 { 1878 const char **params = (const char *[]){ 1879 MLX5_RXQ_CQE_COMP_EN, 1880 MLX5_RXQ_PKT_PAD_EN, 1881 MLX5_RX_MPRQ_EN, 1882 MLX5_RX_MPRQ_LOG_STRIDE_NUM, 1883 MLX5_RX_MPRQ_LOG_STRIDE_SIZE, 1884 MLX5_RX_MPRQ_MAX_MEMCPY_LEN, 1885 MLX5_RXQS_MIN_MPRQ, 1886 MLX5_TXQ_INLINE, 1887 MLX5_TXQ_INLINE_MIN, 1888 MLX5_TXQ_INLINE_MAX, 1889 MLX5_TXQ_INLINE_MPW, 1890 MLX5_TXQS_MIN_INLINE, 1891 MLX5_TXQS_MAX_VEC, 1892 MLX5_TXQ_MPW_EN, 1893 MLX5_TXQ_MPW_HDR_DSEG_EN, 1894 MLX5_TXQ_MAX_INLINE_LEN, 1895 MLX5_TX_DB_NC, 1896 MLX5_TX_PP, 1897 MLX5_TX_SKEW, 1898 MLX5_TX_VEC_EN, 1899 MLX5_RX_VEC_EN, 1900 MLX5_L3_VXLAN_EN, 1901 MLX5_VF_NL_EN, 1902 MLX5_DV_ESW_EN, 1903 MLX5_DV_FLOW_EN, 1904 MLX5_DV_XMETA_EN, 1905 MLX5_LACP_BY_USER, 1906 MLX5_MR_EXT_MEMSEG_EN, 1907 MLX5_REPRESENTOR, 1908 MLX5_MAX_DUMP_FILES_NUM, 1909 MLX5_LRO_TIMEOUT_USEC, 1910 MLX5_CLASS_ARG_NAME, 1911 MLX5_HP_BUF_SIZE, 1912 MLX5_RECLAIM_MEM, 1913 MLX5_SYS_MEM_EN, 1914 MLX5_DECAP_EN, 1915 NULL, 1916 }; 1917 struct rte_kvargs *kvlist; 1918 int ret = 0; 1919 int i; 1920 1921 if (devargs == NULL) 1922 return 0; 1923 /* Following UGLY cast is done to pass checkpatch. */ 1924 kvlist = rte_kvargs_parse(devargs->args, params); 1925 if (kvlist == NULL) { 1926 rte_errno = EINVAL; 1927 return -rte_errno; 1928 } 1929 /* Process parameters. */ 1930 for (i = 0; (params[i] != NULL); ++i) { 1931 if (rte_kvargs_count(kvlist, params[i])) { 1932 ret = rte_kvargs_process(kvlist, params[i], 1933 mlx5_args_check, config); 1934 if (ret) { 1935 rte_errno = EINVAL; 1936 rte_kvargs_free(kvlist); 1937 return -rte_errno; 1938 } 1939 } 1940 } 1941 rte_kvargs_free(kvlist); 1942 return 0; 1943 } 1944 1945 /** 1946 * Configures the minimal amount of data to inline into WQE 1947 * while sending packets. 1948 * 1949 * - the txq_inline_min has the maximal priority, if this 1950 * key is specified in devargs 1951 * - if DevX is enabled the inline mode is queried from the 1952 * device (HCA attributes and NIC vport context if needed). 1953 * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4 Lx 1954 * and none (0 bytes) for other NICs 1955 * 1956 * @param spawn 1957 * Verbs device parameters (name, port, switch_info) to spawn. 1958 * @param config 1959 * Device configuration parameters. 1960 */ 1961 void 1962 mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn, 1963 struct mlx5_dev_config *config) 1964 { 1965 if (config->txq_inline_min != MLX5_ARG_UNSET) { 1966 /* Application defines size of inlined data explicitly. */ 1967 switch (spawn->pci_dev->id.device_id) { 1968 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 1969 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 1970 if (config->txq_inline_min < 1971 (int)MLX5_INLINE_HSIZE_L2) { 1972 DRV_LOG(DEBUG, 1973 "txq_inline_mix aligned to minimal" 1974 " ConnectX-4 required value %d", 1975 (int)MLX5_INLINE_HSIZE_L2); 1976 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 1977 } 1978 break; 1979 } 1980 goto exit; 1981 } 1982 if (config->hca_attr.eth_net_offloads) { 1983 /* We have DevX enabled, inline mode queried successfully. */ 1984 switch (config->hca_attr.wqe_inline_mode) { 1985 case MLX5_CAP_INLINE_MODE_L2: 1986 /* outer L2 header must be inlined. */ 1987 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 1988 goto exit; 1989 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 1990 /* No inline data are required by NIC. */ 1991 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 1992 config->hw_vlan_insert = 1993 config->hca_attr.wqe_vlan_insert; 1994 DRV_LOG(DEBUG, "Tx VLAN insertion is supported"); 1995 goto exit; 1996 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 1997 /* inline mode is defined by NIC vport context. */ 1998 if (!config->hca_attr.eth_virt) 1999 break; 2000 switch (config->hca_attr.vport_inline_mode) { 2001 case MLX5_INLINE_MODE_NONE: 2002 config->txq_inline_min = 2003 MLX5_INLINE_HSIZE_NONE; 2004 goto exit; 2005 case MLX5_INLINE_MODE_L2: 2006 config->txq_inline_min = 2007 MLX5_INLINE_HSIZE_L2; 2008 goto exit; 2009 case MLX5_INLINE_MODE_IP: 2010 config->txq_inline_min = 2011 MLX5_INLINE_HSIZE_L3; 2012 goto exit; 2013 case MLX5_INLINE_MODE_TCP_UDP: 2014 config->txq_inline_min = 2015 MLX5_INLINE_HSIZE_L4; 2016 goto exit; 2017 case MLX5_INLINE_MODE_INNER_L2: 2018 config->txq_inline_min = 2019 MLX5_INLINE_HSIZE_INNER_L2; 2020 goto exit; 2021 case MLX5_INLINE_MODE_INNER_IP: 2022 config->txq_inline_min = 2023 MLX5_INLINE_HSIZE_INNER_L3; 2024 goto exit; 2025 case MLX5_INLINE_MODE_INNER_TCP_UDP: 2026 config->txq_inline_min = 2027 MLX5_INLINE_HSIZE_INNER_L4; 2028 goto exit; 2029 } 2030 } 2031 } 2032 /* 2033 * We get here if we are unable to deduce 2034 * inline data size with DevX. Try PCI ID 2035 * to determine old NICs. 2036 */ 2037 switch (spawn->pci_dev->id.device_id) { 2038 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 2039 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 2040 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: 2041 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 2042 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 2043 config->hw_vlan_insert = 0; 2044 break; 2045 case PCI_DEVICE_ID_MELLANOX_CONNECTX5: 2046 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 2047 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: 2048 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 2049 /* 2050 * These NICs support VLAN insertion from WQE and 2051 * report the wqe_vlan_insert flag. But there is the bug 2052 * and PFC control may be broken, so disable feature. 2053 */ 2054 config->hw_vlan_insert = 0; 2055 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2056 break; 2057 default: 2058 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 2059 break; 2060 } 2061 exit: 2062 DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min); 2063 } 2064 2065 /** 2066 * Configures the metadata mask fields in the shared context. 2067 * 2068 * @param [in] dev 2069 * Pointer to Ethernet device. 2070 */ 2071 void 2072 mlx5_set_metadata_mask(struct rte_eth_dev *dev) 2073 { 2074 struct mlx5_priv *priv = dev->data->dev_private; 2075 struct mlx5_dev_ctx_shared *sh = priv->sh; 2076 uint32_t meta, mark, reg_c0; 2077 2078 reg_c0 = ~priv->vport_meta_mask; 2079 switch (priv->config.dv_xmeta_en) { 2080 case MLX5_XMETA_MODE_LEGACY: 2081 meta = UINT32_MAX; 2082 mark = MLX5_FLOW_MARK_MASK; 2083 break; 2084 case MLX5_XMETA_MODE_META16: 2085 meta = reg_c0 >> rte_bsf32(reg_c0); 2086 mark = MLX5_FLOW_MARK_MASK; 2087 break; 2088 case MLX5_XMETA_MODE_META32: 2089 meta = UINT32_MAX; 2090 mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK; 2091 break; 2092 default: 2093 meta = 0; 2094 mark = 0; 2095 MLX5_ASSERT(false); 2096 break; 2097 } 2098 if (sh->dv_mark_mask && sh->dv_mark_mask != mark) 2099 DRV_LOG(WARNING, "metadata MARK mask mismatche %08X:%08X", 2100 sh->dv_mark_mask, mark); 2101 else 2102 sh->dv_mark_mask = mark; 2103 if (sh->dv_meta_mask && sh->dv_meta_mask != meta) 2104 DRV_LOG(WARNING, "metadata META mask mismatche %08X:%08X", 2105 sh->dv_meta_mask, meta); 2106 else 2107 sh->dv_meta_mask = meta; 2108 if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0) 2109 DRV_LOG(WARNING, "metadata reg_c0 mask mismatche %08X:%08X", 2110 sh->dv_meta_mask, reg_c0); 2111 else 2112 sh->dv_regc0_mask = reg_c0; 2113 DRV_LOG(DEBUG, "metadata mode %u", priv->config.dv_xmeta_en); 2114 DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask); 2115 DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask); 2116 DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask); 2117 } 2118 2119 int 2120 rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n) 2121 { 2122 static const char *const dynf_names[] = { 2123 RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, 2124 RTE_MBUF_DYNFLAG_METADATA_NAME, 2125 RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME 2126 }; 2127 unsigned int i; 2128 2129 if (n < RTE_DIM(dynf_names)) 2130 return -ENOMEM; 2131 for (i = 0; i < RTE_DIM(dynf_names); i++) { 2132 if (names[i] == NULL) 2133 return -EINVAL; 2134 strcpy(names[i], dynf_names[i]); 2135 } 2136 return RTE_DIM(dynf_names); 2137 } 2138 2139 /** 2140 * Comparison callback to sort device data. 2141 * 2142 * This is meant to be used with qsort(). 2143 * 2144 * @param a[in] 2145 * Pointer to pointer to first data object. 2146 * @param b[in] 2147 * Pointer to pointer to second data object. 2148 * 2149 * @return 2150 * 0 if both objects are equal, less than 0 if the first argument is less 2151 * than the second, greater than 0 otherwise. 2152 */ 2153 int 2154 mlx5_dev_check_sibling_config(struct mlx5_priv *priv, 2155 struct mlx5_dev_config *config) 2156 { 2157 struct mlx5_dev_ctx_shared *sh = priv->sh; 2158 struct mlx5_dev_config *sh_conf = NULL; 2159 uint16_t port_id; 2160 2161 MLX5_ASSERT(sh); 2162 /* Nothing to compare for the single/first device. */ 2163 if (sh->refcnt == 1) 2164 return 0; 2165 /* Find the device with shared context. */ 2166 MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) { 2167 struct mlx5_priv *opriv = 2168 rte_eth_devices[port_id].data->dev_private; 2169 2170 if (opriv && opriv != priv && opriv->sh == sh) { 2171 sh_conf = &opriv->config; 2172 break; 2173 } 2174 } 2175 if (!sh_conf) 2176 return 0; 2177 if (sh_conf->dv_flow_en ^ config->dv_flow_en) { 2178 DRV_LOG(ERR, "\"dv_flow_en\" configuration mismatch" 2179 " for shared %s context", sh->ibdev_name); 2180 rte_errno = EINVAL; 2181 return rte_errno; 2182 } 2183 if (sh_conf->dv_xmeta_en ^ config->dv_xmeta_en) { 2184 DRV_LOG(ERR, "\"dv_xmeta_en\" configuration mismatch" 2185 " for shared %s context", sh->ibdev_name); 2186 rte_errno = EINVAL; 2187 return rte_errno; 2188 } 2189 return 0; 2190 } 2191 2192 /** 2193 * Look for the ethernet device belonging to mlx5 driver. 2194 * 2195 * @param[in] port_id 2196 * port_id to start looking for device. 2197 * @param[in] pci_dev 2198 * Pointer to the hint PCI device. When device is being probed 2199 * the its siblings (master and preceding representors might 2200 * not have assigned driver yet (because the mlx5_os_pci_probe() 2201 * is not completed yet, for this case match on hint PCI 2202 * device may be used to detect sibling device. 2203 * 2204 * @return 2205 * port_id of found device, RTE_MAX_ETHPORT if not found. 2206 */ 2207 uint16_t 2208 mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev) 2209 { 2210 while (port_id < RTE_MAX_ETHPORTS) { 2211 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 2212 2213 if (dev->state != RTE_ETH_DEV_UNUSED && 2214 dev->device && 2215 (dev->device == &pci_dev->device || 2216 (dev->device->driver && 2217 dev->device->driver->name && 2218 !strcmp(dev->device->driver->name, MLX5_PCI_DRIVER_NAME)))) 2219 break; 2220 port_id++; 2221 } 2222 if (port_id >= RTE_MAX_ETHPORTS) 2223 return RTE_MAX_ETHPORTS; 2224 return port_id; 2225 } 2226 2227 /** 2228 * DPDK callback to remove a PCI device. 2229 * 2230 * This function removes all Ethernet devices belong to a given PCI device. 2231 * 2232 * @param[in] pci_dev 2233 * Pointer to the PCI device. 2234 * 2235 * @return 2236 * 0 on success, the function cannot fail. 2237 */ 2238 static int 2239 mlx5_pci_remove(struct rte_pci_device *pci_dev) 2240 { 2241 uint16_t port_id; 2242 int ret = 0; 2243 2244 RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device) { 2245 /* 2246 * mlx5_dev_close() is not registered to secondary process, 2247 * call the close function explicitly for secondary process. 2248 */ 2249 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 2250 ret |= mlx5_dev_close(&rte_eth_devices[port_id]); 2251 else 2252 ret |= rte_eth_dev_close(port_id); 2253 } 2254 return ret == 0 ? 0 : -EIO; 2255 } 2256 2257 static const struct rte_pci_id mlx5_pci_id_map[] = { 2258 { 2259 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2260 PCI_DEVICE_ID_MELLANOX_CONNECTX4) 2261 }, 2262 { 2263 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2264 PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) 2265 }, 2266 { 2267 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2268 PCI_DEVICE_ID_MELLANOX_CONNECTX4LX) 2269 }, 2270 { 2271 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2272 PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) 2273 }, 2274 { 2275 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2276 PCI_DEVICE_ID_MELLANOX_CONNECTX5) 2277 }, 2278 { 2279 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2280 PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) 2281 }, 2282 { 2283 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2284 PCI_DEVICE_ID_MELLANOX_CONNECTX5EX) 2285 }, 2286 { 2287 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2288 PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF) 2289 }, 2290 { 2291 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2292 PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) 2293 }, 2294 { 2295 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2296 PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF) 2297 }, 2298 { 2299 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2300 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 2301 }, 2302 { 2303 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2304 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF) 2305 }, 2306 { 2307 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2308 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 2309 }, 2310 { 2311 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2312 PCI_DEVICE_ID_MELLANOX_CONNECTXVF) 2313 }, 2314 { 2315 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2316 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF) 2317 }, 2318 { 2319 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2320 PCI_DEVICE_ID_MELLANOX_CONNECTX6LX) 2321 }, 2322 { 2323 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2324 PCI_DEVICE_ID_MELLANOX_CONNECTX7) 2325 }, 2326 { 2327 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2328 PCI_DEVICE_ID_MELLANOX_CONNECTX7BF) 2329 }, 2330 { 2331 .vendor_id = 0 2332 } 2333 }; 2334 2335 static struct mlx5_pci_driver mlx5_driver = { 2336 .driver_class = MLX5_CLASS_NET, 2337 .pci_driver = { 2338 .driver = { 2339 .name = MLX5_PCI_DRIVER_NAME, 2340 }, 2341 .id_table = mlx5_pci_id_map, 2342 .probe = mlx5_os_pci_probe, 2343 .remove = mlx5_pci_remove, 2344 .dma_map = mlx5_dma_map, 2345 .dma_unmap = mlx5_dma_unmap, 2346 .drv_flags = PCI_DRV_FLAGS, 2347 }, 2348 }; 2349 2350 /* Initialize driver log type. */ 2351 RTE_LOG_REGISTER(mlx5_logtype, pmd.net.mlx5, NOTICE) 2352 2353 /** 2354 * Driver initialization routine. 2355 */ 2356 RTE_INIT(rte_mlx5_pmd_init) 2357 { 2358 pthread_mutex_init(&mlx5_dev_ctx_list_mutex, NULL); 2359 mlx5_common_init(); 2360 /* Build the static tables for Verbs conversion. */ 2361 mlx5_set_ptype_table(); 2362 mlx5_set_cksum_table(); 2363 mlx5_set_swp_types_table(); 2364 if (mlx5_glue) 2365 mlx5_pci_driver_register(&mlx5_driver); 2366 } 2367 2368 RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__); 2369 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5, mlx5_pci_id_map); 2370 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5, "* ib_uverbs & mlx5_core & mlx5_ib"); 2371