1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <stddef.h> 7 #include <unistd.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <errno.h> 12 13 #include <rte_malloc.h> 14 #include <ethdev_driver.h> 15 #include <ethdev_pci.h> 16 #include <rte_pci.h> 17 #include <rte_bus_pci.h> 18 #include <rte_common.h> 19 #include <rte_kvargs.h> 20 #include <rte_rwlock.h> 21 #include <rte_spinlock.h> 22 #include <rte_string_fns.h> 23 #include <rte_alarm.h> 24 #include <rte_cycles.h> 25 26 #include <mlx5_glue.h> 27 #include <mlx5_devx_cmds.h> 28 #include <mlx5_common.h> 29 #include <mlx5_common_os.h> 30 #include <mlx5_common_mp.h> 31 #include <mlx5_common_pci.h> 32 #include <mlx5_malloc.h> 33 34 #include "mlx5_defs.h" 35 #include "mlx5.h" 36 #include "mlx5_utils.h" 37 #include "mlx5_rxtx.h" 38 #include "mlx5_autoconf.h" 39 #include "mlx5_mr.h" 40 #include "mlx5_flow.h" 41 #include "mlx5_flow_os.h" 42 #include "rte_pmd_mlx5.h" 43 44 /* Device parameter to enable RX completion queue compression. */ 45 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en" 46 47 /* Device parameter to enable padding Rx packet to cacheline size. */ 48 #define MLX5_RXQ_PKT_PAD_EN "rxq_pkt_pad_en" 49 50 /* Device parameter to enable Multi-Packet Rx queue. */ 51 #define MLX5_RX_MPRQ_EN "mprq_en" 52 53 /* Device parameter to configure log 2 of the number of strides for MPRQ. */ 54 #define MLX5_RX_MPRQ_LOG_STRIDE_NUM "mprq_log_stride_num" 55 56 /* Device parameter to configure log 2 of the stride size for MPRQ. */ 57 #define MLX5_RX_MPRQ_LOG_STRIDE_SIZE "mprq_log_stride_size" 58 59 /* Device parameter to limit the size of memcpy'd packet for MPRQ. */ 60 #define MLX5_RX_MPRQ_MAX_MEMCPY_LEN "mprq_max_memcpy_len" 61 62 /* Device parameter to set the minimum number of Rx queues to enable MPRQ. */ 63 #define MLX5_RXQS_MIN_MPRQ "rxqs_min_mprq" 64 65 /* Device parameter to configure inline send. Deprecated, ignored.*/ 66 #define MLX5_TXQ_INLINE "txq_inline" 67 68 /* Device parameter to limit packet size to inline with ordinary SEND. */ 69 #define MLX5_TXQ_INLINE_MAX "txq_inline_max" 70 71 /* Device parameter to configure minimal data size to inline. */ 72 #define MLX5_TXQ_INLINE_MIN "txq_inline_min" 73 74 /* Device parameter to limit packet size to inline with Enhanced MPW. */ 75 #define MLX5_TXQ_INLINE_MPW "txq_inline_mpw" 76 77 /* 78 * Device parameter to configure the number of TX queues threshold for 79 * enabling inline send. 80 */ 81 #define MLX5_TXQS_MIN_INLINE "txqs_min_inline" 82 83 /* 84 * Device parameter to configure the number of TX queues threshold for 85 * enabling vectorized Tx, deprecated, ignored (no vectorized Tx routines). 86 */ 87 #define MLX5_TXQS_MAX_VEC "txqs_max_vec" 88 89 /* Device parameter to enable multi-packet send WQEs. */ 90 #define MLX5_TXQ_MPW_EN "txq_mpw_en" 91 92 /* 93 * Device parameter to force doorbell register mapping 94 * to non-cahed region eliminating the extra write memory barrier. 95 */ 96 #define MLX5_TX_DB_NC "tx_db_nc" 97 98 /* 99 * Device parameter to include 2 dsegs in the title WQEBB. 100 * Deprecated, ignored. 101 */ 102 #define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en" 103 104 /* 105 * Device parameter to limit the size of inlining packet. 106 * Deprecated, ignored. 107 */ 108 #define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len" 109 110 /* 111 * Device parameter to enable Tx scheduling on timestamps 112 * and specify the packet pacing granularity in nanoseconds. 113 */ 114 #define MLX5_TX_PP "tx_pp" 115 116 /* 117 * Device parameter to specify skew in nanoseconds on Tx datapath, 118 * it represents the time between SQ start WQE processing and 119 * appearing actual packet data on the wire. 120 */ 121 #define MLX5_TX_SKEW "tx_skew" 122 123 /* 124 * Device parameter to enable hardware Tx vector. 125 * Deprecated, ignored (no vectorized Tx routines anymore). 126 */ 127 #define MLX5_TX_VEC_EN "tx_vec_en" 128 129 /* Device parameter to enable hardware Rx vector. */ 130 #define MLX5_RX_VEC_EN "rx_vec_en" 131 132 /* Allow L3 VXLAN flow creation. */ 133 #define MLX5_L3_VXLAN_EN "l3_vxlan_en" 134 135 /* Activate DV E-Switch flow steering. */ 136 #define MLX5_DV_ESW_EN "dv_esw_en" 137 138 /* Activate DV flow steering. */ 139 #define MLX5_DV_FLOW_EN "dv_flow_en" 140 141 /* Enable extensive flow metadata support. */ 142 #define MLX5_DV_XMETA_EN "dv_xmeta_en" 143 144 /* Device parameter to let the user manage the lacp traffic of bonded device */ 145 #define MLX5_LACP_BY_USER "lacp_by_user" 146 147 /* Activate Netlink support in VF mode. */ 148 #define MLX5_VF_NL_EN "vf_nl_en" 149 150 /* Enable extending memsegs when creating a MR. */ 151 #define MLX5_MR_EXT_MEMSEG_EN "mr_ext_memseg_en" 152 153 /* Select port representors to instantiate. */ 154 #define MLX5_REPRESENTOR "representor" 155 156 /* Device parameter to configure the maximum number of dump files per queue. */ 157 #define MLX5_MAX_DUMP_FILES_NUM "max_dump_files_num" 158 159 /* Configure timeout of LRO session (in microseconds). */ 160 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec" 161 162 /* 163 * Device parameter to configure the total data buffer size for a single 164 * hairpin queue (logarithm value). 165 */ 166 #define MLX5_HP_BUF_SIZE "hp_buf_log_sz" 167 168 /* Flow memory reclaim mode. */ 169 #define MLX5_RECLAIM_MEM "reclaim_mem_mode" 170 171 /* The default memory allocator used in PMD. */ 172 #define MLX5_SYS_MEM_EN "sys_mem_en" 173 /* Decap will be used or not. */ 174 #define MLX5_DECAP_EN "decap_en" 175 176 /* Shared memory between primary and secondary processes. */ 177 struct mlx5_shared_data *mlx5_shared_data; 178 179 /** Driver-specific log messages type. */ 180 int mlx5_logtype; 181 182 static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = 183 LIST_HEAD_INITIALIZER(); 184 static pthread_mutex_t mlx5_dev_ctx_list_mutex; 185 static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { 186 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 187 [MLX5_IPOOL_DECAP_ENCAP] = { 188 .size = sizeof(struct mlx5_flow_dv_encap_decap_resource), 189 .trunk_size = 64, 190 .grow_trunk = 3, 191 .grow_shift = 2, 192 .need_lock = 1, 193 .release_mem_en = 1, 194 .malloc = mlx5_malloc, 195 .free = mlx5_free, 196 .type = "mlx5_encap_decap_ipool", 197 }, 198 [MLX5_IPOOL_PUSH_VLAN] = { 199 .size = sizeof(struct mlx5_flow_dv_push_vlan_action_resource), 200 .trunk_size = 64, 201 .grow_trunk = 3, 202 .grow_shift = 2, 203 .need_lock = 1, 204 .release_mem_en = 1, 205 .malloc = mlx5_malloc, 206 .free = mlx5_free, 207 .type = "mlx5_push_vlan_ipool", 208 }, 209 [MLX5_IPOOL_TAG] = { 210 .size = sizeof(struct mlx5_flow_dv_tag_resource), 211 .trunk_size = 64, 212 .grow_trunk = 3, 213 .grow_shift = 2, 214 .need_lock = 1, 215 .release_mem_en = 1, 216 .malloc = mlx5_malloc, 217 .free = mlx5_free, 218 .type = "mlx5_tag_ipool", 219 }, 220 [MLX5_IPOOL_PORT_ID] = { 221 .size = sizeof(struct mlx5_flow_dv_port_id_action_resource), 222 .trunk_size = 64, 223 .grow_trunk = 3, 224 .grow_shift = 2, 225 .need_lock = 1, 226 .release_mem_en = 1, 227 .malloc = mlx5_malloc, 228 .free = mlx5_free, 229 .type = "mlx5_port_id_ipool", 230 }, 231 [MLX5_IPOOL_JUMP] = { 232 .size = sizeof(struct mlx5_flow_tbl_data_entry), 233 .trunk_size = 64, 234 .grow_trunk = 3, 235 .grow_shift = 2, 236 .need_lock = 1, 237 .release_mem_en = 1, 238 .malloc = mlx5_malloc, 239 .free = mlx5_free, 240 .type = "mlx5_jump_ipool", 241 }, 242 [MLX5_IPOOL_SAMPLE] = { 243 .size = sizeof(struct mlx5_flow_dv_sample_resource), 244 .trunk_size = 64, 245 .grow_trunk = 3, 246 .grow_shift = 2, 247 .need_lock = 1, 248 .release_mem_en = 1, 249 .malloc = mlx5_malloc, 250 .free = mlx5_free, 251 .type = "mlx5_sample_ipool", 252 }, 253 [MLX5_IPOOL_DEST_ARRAY] = { 254 .size = sizeof(struct mlx5_flow_dv_dest_array_resource), 255 .trunk_size = 64, 256 .grow_trunk = 3, 257 .grow_shift = 2, 258 .need_lock = 1, 259 .release_mem_en = 1, 260 .malloc = mlx5_malloc, 261 .free = mlx5_free, 262 .type = "mlx5_dest_array_ipool", 263 }, 264 [MLX5_IPOOL_TUNNEL_ID] = { 265 .size = sizeof(struct mlx5_flow_tunnel), 266 .trunk_size = MLX5_MAX_TUNNELS, 267 .need_lock = 1, 268 .release_mem_en = 1, 269 .type = "mlx5_tunnel_offload", 270 }, 271 [MLX5_IPOOL_TNL_TBL_ID] = { 272 .size = 0, 273 .need_lock = 1, 274 .type = "mlx5_flow_tnl_tbl_ipool", 275 }, 276 #endif 277 [MLX5_IPOOL_MTR] = { 278 .size = sizeof(struct mlx5_flow_meter), 279 .trunk_size = 64, 280 .grow_trunk = 3, 281 .grow_shift = 2, 282 .need_lock = 1, 283 .release_mem_en = 1, 284 .malloc = mlx5_malloc, 285 .free = mlx5_free, 286 .type = "mlx5_meter_ipool", 287 }, 288 [MLX5_IPOOL_MCP] = { 289 .size = sizeof(struct mlx5_flow_mreg_copy_resource), 290 .trunk_size = 64, 291 .grow_trunk = 3, 292 .grow_shift = 2, 293 .need_lock = 1, 294 .release_mem_en = 1, 295 .malloc = mlx5_malloc, 296 .free = mlx5_free, 297 .type = "mlx5_mcp_ipool", 298 }, 299 [MLX5_IPOOL_HRXQ] = { 300 .size = (sizeof(struct mlx5_hrxq) + MLX5_RSS_HASH_KEY_LEN), 301 .trunk_size = 64, 302 .grow_trunk = 3, 303 .grow_shift = 2, 304 .need_lock = 1, 305 .release_mem_en = 1, 306 .malloc = mlx5_malloc, 307 .free = mlx5_free, 308 .type = "mlx5_hrxq_ipool", 309 }, 310 [MLX5_IPOOL_MLX5_FLOW] = { 311 /* 312 * MLX5_IPOOL_MLX5_FLOW size varies for DV and VERBS flows. 313 * It set in run time according to PCI function configuration. 314 */ 315 .size = 0, 316 .trunk_size = 64, 317 .grow_trunk = 3, 318 .grow_shift = 2, 319 .need_lock = 1, 320 .release_mem_en = 1, 321 .malloc = mlx5_malloc, 322 .free = mlx5_free, 323 .type = "mlx5_flow_handle_ipool", 324 }, 325 [MLX5_IPOOL_RTE_FLOW] = { 326 .size = sizeof(struct rte_flow), 327 .trunk_size = 4096, 328 .need_lock = 1, 329 .release_mem_en = 1, 330 .malloc = mlx5_malloc, 331 .free = mlx5_free, 332 .type = "rte_flow_ipool", 333 }, 334 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID] = { 335 .size = 0, 336 .need_lock = 1, 337 .type = "mlx5_flow_rss_id_ipool", 338 }, 339 [MLX5_IPOOL_RSS_SHARED_ACTIONS] = { 340 .size = sizeof(struct mlx5_shared_action_rss), 341 .trunk_size = 64, 342 .grow_trunk = 3, 343 .grow_shift = 2, 344 .need_lock = 1, 345 .release_mem_en = 1, 346 .malloc = mlx5_malloc, 347 .free = mlx5_free, 348 .type = "mlx5_shared_action_rss", 349 }, 350 }; 351 352 353 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512 354 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16 355 356 #define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096 357 358 /** 359 * Decide whether representor ID is a HPF(host PF) port on BF2. 360 * 361 * @param dev 362 * Pointer to Ethernet device structure. 363 * 364 * @return 365 * Non-zero if HPF, otherwise 0. 366 */ 367 bool 368 mlx5_is_hpf(struct rte_eth_dev *dev) 369 { 370 struct mlx5_priv *priv = dev->data->dev_private; 371 uint16_t repr = MLX5_REPRESENTOR_REPR(priv->representor_id); 372 int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); 373 374 return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_VF && 375 MLX5_REPRESENTOR_REPR(-1) == repr; 376 } 377 378 /** 379 * Initialize the ASO aging management structure. 380 * 381 * @param[in] sh 382 * Pointer to mlx5_dev_ctx_shared object to free 383 * 384 * @return 385 * 0 on success, a negative errno value otherwise and rte_errno is set. 386 */ 387 int 388 mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh) 389 { 390 int err; 391 392 if (sh->aso_age_mng) 393 return 0; 394 sh->aso_age_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->aso_age_mng), 395 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 396 if (!sh->aso_age_mng) { 397 DRV_LOG(ERR, "aso_age_mng allocation was failed."); 398 rte_errno = ENOMEM; 399 return -ENOMEM; 400 } 401 err = mlx5_aso_queue_init(sh); 402 if (err) { 403 mlx5_free(sh->aso_age_mng); 404 return -1; 405 } 406 rte_spinlock_init(&sh->aso_age_mng->resize_sl); 407 rte_spinlock_init(&sh->aso_age_mng->free_sl); 408 LIST_INIT(&sh->aso_age_mng->free); 409 return 0; 410 } 411 412 /** 413 * Close and release all the resources of the ASO aging management structure. 414 * 415 * @param[in] sh 416 * Pointer to mlx5_dev_ctx_shared object to free. 417 */ 418 static void 419 mlx5_flow_aso_age_mng_close(struct mlx5_dev_ctx_shared *sh) 420 { 421 int i, j; 422 423 mlx5_aso_queue_stop(sh); 424 mlx5_aso_queue_uninit(sh); 425 if (sh->aso_age_mng->pools) { 426 struct mlx5_aso_age_pool *pool; 427 428 for (i = 0; i < sh->aso_age_mng->next; ++i) { 429 pool = sh->aso_age_mng->pools[i]; 430 claim_zero(mlx5_devx_cmd_destroy 431 (pool->flow_hit_aso_obj)); 432 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) 433 if (pool->actions[j].dr_action) 434 claim_zero 435 (mlx5_flow_os_destroy_flow_action 436 (pool->actions[j].dr_action)); 437 mlx5_free(pool); 438 } 439 mlx5_free(sh->aso_age_mng->pools); 440 } 441 mlx5_free(sh->aso_age_mng); 442 } 443 444 /** 445 * Initialize the shared aging list information per port. 446 * 447 * @param[in] sh 448 * Pointer to mlx5_dev_ctx_shared object. 449 */ 450 static void 451 mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh) 452 { 453 uint32_t i; 454 struct mlx5_age_info *age_info; 455 456 for (i = 0; i < sh->max_port; i++) { 457 age_info = &sh->port[i].age_info; 458 age_info->flags = 0; 459 TAILQ_INIT(&age_info->aged_counters); 460 LIST_INIT(&age_info->aged_aso); 461 rte_spinlock_init(&age_info->aged_sl); 462 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 463 } 464 } 465 466 /** 467 * Initialize the counters management structure. 468 * 469 * @param[in] sh 470 * Pointer to mlx5_dev_ctx_shared object to free 471 */ 472 static void 473 mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh) 474 { 475 int i; 476 477 memset(&sh->cmng, 0, sizeof(sh->cmng)); 478 TAILQ_INIT(&sh->cmng.flow_counters); 479 sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET; 480 sh->cmng.max_id = -1; 481 sh->cmng.last_pool_idx = POOL_IDX_INVALID; 482 rte_spinlock_init(&sh->cmng.pool_update_sl); 483 for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) { 484 TAILQ_INIT(&sh->cmng.counters[i]); 485 rte_spinlock_init(&sh->cmng.csl[i]); 486 } 487 } 488 489 /** 490 * Destroy all the resources allocated for a counter memory management. 491 * 492 * @param[in] mng 493 * Pointer to the memory management structure. 494 */ 495 static void 496 mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng) 497 { 498 uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data; 499 500 LIST_REMOVE(mng, next); 501 claim_zero(mlx5_devx_cmd_destroy(mng->dm)); 502 claim_zero(mlx5_os_umem_dereg(mng->umem)); 503 mlx5_free(mem); 504 } 505 506 /** 507 * Close and release all the resources of the counters management. 508 * 509 * @param[in] sh 510 * Pointer to mlx5_dev_ctx_shared object to free. 511 */ 512 static void 513 mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) 514 { 515 struct mlx5_counter_stats_mem_mng *mng; 516 int i, j; 517 int retries = 1024; 518 519 rte_errno = 0; 520 while (--retries) { 521 rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh); 522 if (rte_errno != EINPROGRESS) 523 break; 524 rte_pause(); 525 } 526 527 if (sh->cmng.pools) { 528 struct mlx5_flow_counter_pool *pool; 529 uint16_t n_valid = sh->cmng.n_valid; 530 bool fallback = sh->cmng.counter_fallback; 531 532 for (i = 0; i < n_valid; ++i) { 533 pool = sh->cmng.pools[i]; 534 if (!fallback && pool->min_dcs) 535 claim_zero(mlx5_devx_cmd_destroy 536 (pool->min_dcs)); 537 for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) { 538 struct mlx5_flow_counter *cnt = 539 MLX5_POOL_GET_CNT(pool, j); 540 541 if (cnt->action) 542 claim_zero 543 (mlx5_flow_os_destroy_flow_action 544 (cnt->action)); 545 if (fallback && MLX5_POOL_GET_CNT 546 (pool, j)->dcs_when_free) 547 claim_zero(mlx5_devx_cmd_destroy 548 (cnt->dcs_when_free)); 549 } 550 mlx5_free(pool); 551 } 552 mlx5_free(sh->cmng.pools); 553 } 554 mng = LIST_FIRST(&sh->cmng.mem_mngs); 555 while (mng) { 556 mlx5_flow_destroy_counter_stat_mem_mng(mng); 557 mng = LIST_FIRST(&sh->cmng.mem_mngs); 558 } 559 memset(&sh->cmng, 0, sizeof(sh->cmng)); 560 } 561 562 /* Send FLOW_AGED event if needed. */ 563 void 564 mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) 565 { 566 struct mlx5_age_info *age_info; 567 uint32_t i; 568 569 for (i = 0; i < sh->max_port; i++) { 570 age_info = &sh->port[i].age_info; 571 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 572 continue; 573 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) 574 rte_eth_dev_callback_process 575 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 576 RTE_ETH_EVENT_FLOW_AGED, NULL); 577 age_info->flags = 0; 578 } 579 } 580 581 /** 582 * Initialize the flow resources' indexed mempool. 583 * 584 * @param[in] sh 585 * Pointer to mlx5_dev_ctx_shared object. 586 * @param[in] sh 587 * Pointer to user dev config. 588 */ 589 static void 590 mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh, 591 const struct mlx5_dev_config *config) 592 { 593 uint8_t i; 594 struct mlx5_indexed_pool_config cfg; 595 596 for (i = 0; i < MLX5_IPOOL_MAX; ++i) { 597 cfg = mlx5_ipool_cfg[i]; 598 switch (i) { 599 default: 600 break; 601 /* 602 * Set MLX5_IPOOL_MLX5_FLOW ipool size 603 * according to PCI function flow configuration. 604 */ 605 case MLX5_IPOOL_MLX5_FLOW: 606 cfg.size = config->dv_flow_en ? 607 sizeof(struct mlx5_flow_handle) : 608 MLX5_FLOW_HANDLE_VERBS_SIZE; 609 break; 610 } 611 if (config->reclaim_mode) 612 cfg.release_mem_en = 1; 613 sh->ipool[i] = mlx5_ipool_create(&cfg); 614 } 615 } 616 617 /** 618 * Release the flow resources' indexed mempool. 619 * 620 * @param[in] sh 621 * Pointer to mlx5_dev_ctx_shared object. 622 */ 623 static void 624 mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh) 625 { 626 uint8_t i; 627 628 for (i = 0; i < MLX5_IPOOL_MAX; ++i) 629 mlx5_ipool_destroy(sh->ipool[i]); 630 } 631 632 /* 633 * Check if dynamic flex parser for eCPRI already exists. 634 * 635 * @param dev 636 * Pointer to Ethernet device structure. 637 * 638 * @return 639 * true on exists, false on not. 640 */ 641 bool 642 mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev) 643 { 644 struct mlx5_priv *priv = dev->data->dev_private; 645 struct mlx5_flex_parser_profiles *prf = 646 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 647 648 return !!prf->obj; 649 } 650 651 /* 652 * Allocation of a flex parser for eCPRI. Once created, this parser related 653 * resources will be held until the device is closed. 654 * 655 * @param dev 656 * Pointer to Ethernet device structure. 657 * 658 * @return 659 * 0 on success, a negative errno value otherwise and rte_errno is set. 660 */ 661 int 662 mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev) 663 { 664 struct mlx5_priv *priv = dev->data->dev_private; 665 struct mlx5_flex_parser_profiles *prf = 666 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 667 struct mlx5_devx_graph_node_attr node = { 668 .modify_field_select = 0, 669 }; 670 uint32_t ids[8]; 671 int ret; 672 673 if (!priv->config.hca_attr.parse_graph_flex_node) { 674 DRV_LOG(ERR, "Dynamic flex parser is not supported " 675 "for device %s.", priv->dev_data->name); 676 return -ENOTSUP; 677 } 678 node.header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED; 679 /* 8 bytes now: 4B common header + 4B message body header. */ 680 node.header_length_base_value = 0x8; 681 /* After MAC layer: Ether / VLAN. */ 682 node.in[0].arc_parse_graph_node = MLX5_GRAPH_ARC_NODE_MAC; 683 /* Type of compared condition should be 0xAEFE in the L2 layer. */ 684 node.in[0].compare_condition_value = RTE_ETHER_TYPE_ECPRI; 685 /* Sample #0: type in common header. */ 686 node.sample[0].flow_match_sample_en = 1; 687 /* Fixed offset. */ 688 node.sample[0].flow_match_sample_offset_mode = 0x0; 689 /* Only the 2nd byte will be used. */ 690 node.sample[0].flow_match_sample_field_base_offset = 0x0; 691 /* Sample #1: message payload. */ 692 node.sample[1].flow_match_sample_en = 1; 693 /* Fixed offset. */ 694 node.sample[1].flow_match_sample_offset_mode = 0x0; 695 /* 696 * Only the first two bytes will be used right now, and its offset will 697 * start after the common header that with the length of a DW(u32). 698 */ 699 node.sample[1].flow_match_sample_field_base_offset = sizeof(uint32_t); 700 prf->obj = mlx5_devx_cmd_create_flex_parser(priv->sh->ctx, &node); 701 if (!prf->obj) { 702 DRV_LOG(ERR, "Failed to create flex parser node object."); 703 return (rte_errno == 0) ? -ENODEV : -rte_errno; 704 } 705 prf->num = 2; 706 ret = mlx5_devx_cmd_query_parse_samples(prf->obj, ids, prf->num); 707 if (ret) { 708 DRV_LOG(ERR, "Failed to query sample IDs."); 709 return (rte_errno == 0) ? -ENODEV : -rte_errno; 710 } 711 prf->offset[0] = 0x0; 712 prf->offset[1] = sizeof(uint32_t); 713 prf->ids[0] = ids[0]; 714 prf->ids[1] = ids[1]; 715 return 0; 716 } 717 718 /* 719 * Destroy the flex parser node, including the parser itself, input / output 720 * arcs and DW samples. Resources could be reused then. 721 * 722 * @param dev 723 * Pointer to Ethernet device structure. 724 */ 725 static void 726 mlx5_flex_parser_ecpri_release(struct rte_eth_dev *dev) 727 { 728 struct mlx5_priv *priv = dev->data->dev_private; 729 struct mlx5_flex_parser_profiles *prf = 730 &priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0]; 731 732 if (prf->obj) 733 mlx5_devx_cmd_destroy(prf->obj); 734 prf->obj = NULL; 735 } 736 737 /* 738 * Allocate Rx and Tx UARs in robust fashion. 739 * This routine handles the following UAR allocation issues: 740 * 741 * - tries to allocate the UAR with the most appropriate memory 742 * mapping type from the ones supported by the host 743 * 744 * - tries to allocate the UAR with non-NULL base address 745 * OFED 5.0.x and Upstream rdma_core before v29 returned the NULL as 746 * UAR base address if UAR was not the first object in the UAR page. 747 * It caused the PMD failure and we should try to get another UAR 748 * till we get the first one with non-NULL base address returned. 749 */ 750 static int 751 mlx5_alloc_rxtx_uars(struct mlx5_dev_ctx_shared *sh, 752 const struct mlx5_dev_config *config) 753 { 754 uint32_t uar_mapping, retry; 755 int err = 0; 756 void *base_addr; 757 758 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 759 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 760 /* Control the mapping type according to the settings. */ 761 uar_mapping = (config->dbnc == MLX5_TXDB_NCACHED) ? 762 MLX5DV_UAR_ALLOC_TYPE_NC : 763 MLX5DV_UAR_ALLOC_TYPE_BF; 764 #else 765 RTE_SET_USED(config); 766 /* 767 * It seems we have no way to control the memory mapping type 768 * for the UAR, the default "Write-Combining" type is supposed. 769 * The UAR initialization on queue creation queries the 770 * actual mapping type done by Verbs/kernel and setups the 771 * PMD datapath accordingly. 772 */ 773 uar_mapping = 0; 774 #endif 775 sh->tx_uar = mlx5_glue->devx_alloc_uar(sh->ctx, uar_mapping); 776 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 777 if (!sh->tx_uar && 778 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 779 if (config->dbnc == MLX5_TXDB_CACHED || 780 config->dbnc == MLX5_TXDB_HEURISTIC) 781 DRV_LOG(WARNING, "Devarg tx_db_nc setting " 782 "is not supported by DevX"); 783 /* 784 * In some environments like virtual machine 785 * the Write Combining mapped might be not supported 786 * and UAR allocation fails. We try "Non-Cached" 787 * mapping for the case. The tx_burst routines take 788 * the UAR mapping type into account on UAR setup 789 * on queue creation. 790 */ 791 DRV_LOG(DEBUG, "Failed to allocate Tx DevX UAR (BF)"); 792 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 793 sh->tx_uar = mlx5_glue->devx_alloc_uar 794 (sh->ctx, uar_mapping); 795 } else if (!sh->tx_uar && 796 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) { 797 if (config->dbnc == MLX5_TXDB_NCACHED) 798 DRV_LOG(WARNING, "Devarg tx_db_nc settings " 799 "is not supported by DevX"); 800 /* 801 * If Verbs/kernel does not support "Non-Cached" 802 * try the "Write-Combining". 803 */ 804 DRV_LOG(DEBUG, "Failed to allocate Tx DevX UAR (NC)"); 805 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF; 806 sh->tx_uar = mlx5_glue->devx_alloc_uar 807 (sh->ctx, uar_mapping); 808 } 809 #endif 810 if (!sh->tx_uar) { 811 DRV_LOG(ERR, "Failed to allocate Tx DevX UAR (BF/NC)"); 812 err = ENOMEM; 813 goto exit; 814 } 815 base_addr = mlx5_os_get_devx_uar_base_addr(sh->tx_uar); 816 if (base_addr) 817 break; 818 /* 819 * The UARs are allocated by rdma_core within the 820 * IB device context, on context closure all UARs 821 * will be freed, should be no memory/object leakage. 822 */ 823 DRV_LOG(DEBUG, "Retrying to allocate Tx DevX UAR"); 824 sh->tx_uar = NULL; 825 } 826 /* Check whether we finally succeeded with valid UAR allocation. */ 827 if (!sh->tx_uar) { 828 DRV_LOG(ERR, "Failed to allocate Tx DevX UAR (NULL base)"); 829 err = ENOMEM; 830 goto exit; 831 } 832 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) { 833 uar_mapping = 0; 834 sh->devx_rx_uar = mlx5_glue->devx_alloc_uar 835 (sh->ctx, uar_mapping); 836 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 837 if (!sh->devx_rx_uar && 838 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) { 839 /* 840 * Rx UAR is used to control interrupts only, 841 * should be no datapath noticeable impact, 842 * can try "Non-Cached" mapping safely. 843 */ 844 DRV_LOG(DEBUG, "Failed to allocate Rx DevX UAR (BF)"); 845 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC; 846 sh->devx_rx_uar = mlx5_glue->devx_alloc_uar 847 (sh->ctx, uar_mapping); 848 } 849 #endif 850 if (!sh->devx_rx_uar) { 851 DRV_LOG(ERR, "Failed to allocate Rx DevX UAR (BF/NC)"); 852 err = ENOMEM; 853 goto exit; 854 } 855 base_addr = mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar); 856 if (base_addr) 857 break; 858 /* 859 * The UARs are allocated by rdma_core within the 860 * IB device context, on context closure all UARs 861 * will be freed, should be no memory/object leakage. 862 */ 863 DRV_LOG(DEBUG, "Retrying to allocate Rx DevX UAR"); 864 sh->devx_rx_uar = NULL; 865 } 866 /* Check whether we finally succeeded with valid UAR allocation. */ 867 if (!sh->devx_rx_uar) { 868 DRV_LOG(ERR, "Failed to allocate Rx DevX UAR (NULL base)"); 869 err = ENOMEM; 870 } 871 exit: 872 return err; 873 } 874 875 /** 876 * Allocate shared device context. If there is multiport device the 877 * master and representors will share this context, if there is single 878 * port dedicated device, the context will be used by only given 879 * port due to unification. 880 * 881 * Routine first searches the context for the specified device name, 882 * if found the shared context assumed and reference counter is incremented. 883 * If no context found the new one is created and initialized with specified 884 * device context and parameters. 885 * 886 * @param[in] spawn 887 * Pointer to the device attributes (name, port, etc). 888 * @param[in] config 889 * Pointer to device configuration structure. 890 * 891 * @return 892 * Pointer to mlx5_dev_ctx_shared object on success, 893 * otherwise NULL and rte_errno is set. 894 */ 895 struct mlx5_dev_ctx_shared * 896 mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, 897 const struct mlx5_dev_config *config) 898 { 899 struct mlx5_dev_ctx_shared *sh; 900 int err = 0; 901 uint32_t i; 902 struct mlx5_devx_tis_attr tis_attr = { 0 }; 903 904 MLX5_ASSERT(spawn); 905 /* Secondary process should not create the shared context. */ 906 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 907 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 908 /* Search for IB context by device name. */ 909 LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) { 910 if (!strcmp(sh->ibdev_name, 911 mlx5_os_get_dev_device_name(spawn->phys_dev))) { 912 sh->refcnt++; 913 goto exit; 914 } 915 } 916 /* No device found, we have to create new shared context. */ 917 MLX5_ASSERT(spawn->max_port); 918 sh = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 919 sizeof(struct mlx5_dev_ctx_shared) + 920 spawn->max_port * 921 sizeof(struct mlx5_dev_shared_port), 922 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 923 if (!sh) { 924 DRV_LOG(ERR, "shared context allocation failure"); 925 rte_errno = ENOMEM; 926 goto exit; 927 } 928 if (spawn->bond_info) 929 sh->bond = *spawn->bond_info; 930 err = mlx5_os_open_device(spawn, config, sh); 931 if (!sh->ctx) 932 goto error; 933 err = mlx5_os_get_dev_attr(sh->ctx, &sh->device_attr); 934 if (err) { 935 DRV_LOG(DEBUG, "mlx5_os_get_dev_attr() failed"); 936 goto error; 937 } 938 sh->refcnt = 1; 939 sh->max_port = spawn->max_port; 940 strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx), 941 sizeof(sh->ibdev_name) - 1); 942 strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx), 943 sizeof(sh->ibdev_path) - 1); 944 /* 945 * Setting port_id to max unallowed value means 946 * there is no interrupt subhandler installed for 947 * the given port index i. 948 */ 949 for (i = 0; i < sh->max_port; i++) { 950 sh->port[i].ih_port_id = RTE_MAX_ETHPORTS; 951 sh->port[i].devx_ih_port_id = RTE_MAX_ETHPORTS; 952 } 953 sh->pd = mlx5_os_alloc_pd(sh->ctx); 954 if (sh->pd == NULL) { 955 DRV_LOG(ERR, "PD allocation failure"); 956 err = ENOMEM; 957 goto error; 958 } 959 if (sh->devx) { 960 err = mlx5_os_get_pdn(sh->pd, &sh->pdn); 961 if (err) { 962 DRV_LOG(ERR, "Fail to extract pdn from PD"); 963 goto error; 964 } 965 sh->td = mlx5_devx_cmd_create_td(sh->ctx); 966 if (!sh->td) { 967 DRV_LOG(ERR, "TD allocation failure"); 968 err = ENOMEM; 969 goto error; 970 } 971 tis_attr.transport_domain = sh->td->id; 972 sh->tis = mlx5_devx_cmd_create_tis(sh->ctx, &tis_attr); 973 if (!sh->tis) { 974 DRV_LOG(ERR, "TIS allocation failure"); 975 err = ENOMEM; 976 goto error; 977 } 978 err = mlx5_alloc_rxtx_uars(sh, config); 979 if (err) 980 goto error; 981 MLX5_ASSERT(sh->tx_uar); 982 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->tx_uar)); 983 984 MLX5_ASSERT(sh->devx_rx_uar); 985 MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar)); 986 } 987 #ifndef RTE_ARCH_64 988 /* Initialize UAR access locks for 32bit implementations. */ 989 rte_spinlock_init(&sh->uar_lock_cq); 990 for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) 991 rte_spinlock_init(&sh->uar_lock[i]); 992 #endif 993 /* 994 * Once the device is added to the list of memory event 995 * callback, its global MR cache table cannot be expanded 996 * on the fly because of deadlock. If it overflows, lookup 997 * should be done by searching MR list linearly, which is slow. 998 * 999 * At this point the device is not added to the memory 1000 * event list yet, context is just being created. 1001 */ 1002 err = mlx5_mr_btree_init(&sh->share_cache.cache, 1003 MLX5_MR_BTREE_CACHE_N * 2, 1004 spawn->pci_dev->device.numa_node); 1005 if (err) { 1006 err = rte_errno; 1007 goto error; 1008 } 1009 mlx5_os_set_reg_mr_cb(&sh->share_cache.reg_mr_cb, 1010 &sh->share_cache.dereg_mr_cb); 1011 mlx5_os_dev_shared_handler_install(sh); 1012 sh->cnt_id_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1013 if (!sh->cnt_id_tbl) { 1014 err = rte_errno; 1015 goto error; 1016 } 1017 if (LIST_EMPTY(&mlx5_dev_ctx_list)) { 1018 err = mlx5_flow_os_init_workspace_once(); 1019 if (err) 1020 goto error; 1021 } 1022 mlx5_flow_aging_init(sh); 1023 mlx5_flow_counters_mng_init(sh); 1024 mlx5_flow_ipool_create(sh, config); 1025 /* Add device to memory callback list. */ 1026 rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock); 1027 LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list, 1028 sh, mem_event_cb); 1029 rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock); 1030 /* Add context to the global device list. */ 1031 LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next); 1032 rte_spinlock_init(&sh->geneve_tlv_opt_sl); 1033 exit: 1034 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1035 return sh; 1036 error: 1037 pthread_mutex_destroy(&sh->txpp.mutex); 1038 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1039 MLX5_ASSERT(sh); 1040 if (sh->cnt_id_tbl) 1041 mlx5_l3t_destroy(sh->cnt_id_tbl); 1042 if (sh->tis) 1043 claim_zero(mlx5_devx_cmd_destroy(sh->tis)); 1044 if (sh->td) 1045 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1046 if (sh->devx_rx_uar) 1047 mlx5_glue->devx_free_uar(sh->devx_rx_uar); 1048 if (sh->tx_uar) 1049 mlx5_glue->devx_free_uar(sh->tx_uar); 1050 if (sh->pd) 1051 claim_zero(mlx5_os_dealloc_pd(sh->pd)); 1052 if (sh->ctx) 1053 claim_zero(mlx5_glue->close_device(sh->ctx)); 1054 mlx5_free(sh); 1055 MLX5_ASSERT(err > 0); 1056 rte_errno = err; 1057 return NULL; 1058 } 1059 1060 /** 1061 * Free shared IB device context. Decrement counter and if zero free 1062 * all allocated resources and close handles. 1063 * 1064 * @param[in] sh 1065 * Pointer to mlx5_dev_ctx_shared object to free 1066 */ 1067 void 1068 mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) 1069 { 1070 pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); 1071 #ifdef RTE_LIBRTE_MLX5_DEBUG 1072 /* Check the object presence in the list. */ 1073 struct mlx5_dev_ctx_shared *lctx; 1074 1075 LIST_FOREACH(lctx, &mlx5_dev_ctx_list, next) 1076 if (lctx == sh) 1077 break; 1078 MLX5_ASSERT(lctx); 1079 if (lctx != sh) { 1080 DRV_LOG(ERR, "Freeing non-existing shared IB context"); 1081 goto exit; 1082 } 1083 #endif 1084 MLX5_ASSERT(sh); 1085 MLX5_ASSERT(sh->refcnt); 1086 /* Secondary process should not free the shared context. */ 1087 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1088 if (--sh->refcnt) 1089 goto exit; 1090 /* Remove from memory callback device list. */ 1091 rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock); 1092 LIST_REMOVE(sh, mem_event_cb); 1093 rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock); 1094 /* Release created Memory Regions. */ 1095 mlx5_mr_release_cache(&sh->share_cache); 1096 /* Remove context from the global device list. */ 1097 LIST_REMOVE(sh, next); 1098 /* Release flow workspaces objects on the last device. */ 1099 if (LIST_EMPTY(&mlx5_dev_ctx_list)) 1100 mlx5_flow_os_release_workspace(); 1101 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1102 /* 1103 * Ensure there is no async event handler installed. 1104 * Only primary process handles async device events. 1105 **/ 1106 mlx5_flow_counters_mng_close(sh); 1107 if (sh->aso_age_mng) { 1108 mlx5_flow_aso_age_mng_close(sh); 1109 sh->aso_age_mng = NULL; 1110 } 1111 mlx5_flow_ipool_destroy(sh); 1112 mlx5_os_dev_shared_handler_uninstall(sh); 1113 if (sh->cnt_id_tbl) { 1114 mlx5_l3t_destroy(sh->cnt_id_tbl); 1115 sh->cnt_id_tbl = NULL; 1116 } 1117 if (sh->tx_uar) { 1118 mlx5_glue->devx_free_uar(sh->tx_uar); 1119 sh->tx_uar = NULL; 1120 } 1121 if (sh->pd) 1122 claim_zero(mlx5_os_dealloc_pd(sh->pd)); 1123 if (sh->tis) 1124 claim_zero(mlx5_devx_cmd_destroy(sh->tis)); 1125 if (sh->td) 1126 claim_zero(mlx5_devx_cmd_destroy(sh->td)); 1127 if (sh->devx_rx_uar) 1128 mlx5_glue->devx_free_uar(sh->devx_rx_uar); 1129 if (sh->ctx) 1130 claim_zero(mlx5_glue->close_device(sh->ctx)); 1131 MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL); 1132 pthread_mutex_destroy(&sh->txpp.mutex); 1133 mlx5_free(sh); 1134 return; 1135 exit: 1136 pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); 1137 } 1138 1139 /** 1140 * Destroy table hash list. 1141 * 1142 * @param[in] priv 1143 * Pointer to the private device data structure. 1144 */ 1145 void 1146 mlx5_free_table_hash_list(struct mlx5_priv *priv) 1147 { 1148 struct mlx5_dev_ctx_shared *sh = priv->sh; 1149 1150 if (!sh->flow_tbls) 1151 return; 1152 mlx5_hlist_destroy(sh->flow_tbls); 1153 } 1154 1155 /** 1156 * Initialize flow table hash list and create the root tables entry 1157 * for each domain. 1158 * 1159 * @param[in] priv 1160 * Pointer to the private device data structure. 1161 * 1162 * @return 1163 * Zero on success, positive error code otherwise. 1164 */ 1165 int 1166 mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) 1167 { 1168 int err = 0; 1169 /* Tables are only used in DV and DR modes. */ 1170 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 1171 struct mlx5_dev_ctx_shared *sh = priv->sh; 1172 char s[MLX5_HLIST_NAMESIZE]; 1173 1174 MLX5_ASSERT(sh); 1175 snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); 1176 sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, 1177 0, 0, flow_dv_tbl_create_cb, 1178 flow_dv_tbl_match_cb, 1179 flow_dv_tbl_remove_cb); 1180 if (!sh->flow_tbls) { 1181 DRV_LOG(ERR, "flow tables with hash creation failed."); 1182 err = ENOMEM; 1183 return err; 1184 } 1185 sh->flow_tbls->ctx = sh; 1186 #ifndef HAVE_MLX5DV_DR 1187 struct rte_flow_error error; 1188 struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; 1189 1190 /* 1191 * In case we have not DR support, the zero tables should be created 1192 * because DV expect to see them even if they cannot be created by 1193 * RDMA-CORE. 1194 */ 1195 if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, NULL, 0, 1, &error) || 1196 !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, NULL, 0, 1, &error) || 1197 !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, NULL, 0, 1, &error)) { 1198 err = ENOMEM; 1199 goto error; 1200 } 1201 return err; 1202 error: 1203 mlx5_free_table_hash_list(priv); 1204 #endif /* HAVE_MLX5DV_DR */ 1205 #endif 1206 return err; 1207 } 1208 1209 /** 1210 * Retrieve integer value from environment variable. 1211 * 1212 * @param[in] name 1213 * Environment variable name. 1214 * 1215 * @return 1216 * Integer value, 0 if the variable is not set. 1217 */ 1218 int 1219 mlx5_getenv_int(const char *name) 1220 { 1221 const char *val = getenv(name); 1222 1223 if (val == NULL) 1224 return 0; 1225 return atoi(val); 1226 } 1227 1228 /** 1229 * DPDK callback to add udp tunnel port 1230 * 1231 * @param[in] dev 1232 * A pointer to eth_dev 1233 * @param[in] udp_tunnel 1234 * A pointer to udp tunnel 1235 * 1236 * @return 1237 * 0 on valid udp ports and tunnels, -ENOTSUP otherwise. 1238 */ 1239 int 1240 mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev __rte_unused, 1241 struct rte_eth_udp_tunnel *udp_tunnel) 1242 { 1243 MLX5_ASSERT(udp_tunnel != NULL); 1244 if (udp_tunnel->prot_type == RTE_TUNNEL_TYPE_VXLAN && 1245 udp_tunnel->udp_port == 4789) 1246 return 0; 1247 if (udp_tunnel->prot_type == RTE_TUNNEL_TYPE_VXLAN_GPE && 1248 udp_tunnel->udp_port == 4790) 1249 return 0; 1250 return -ENOTSUP; 1251 } 1252 1253 /** 1254 * Initialize process private data structure. 1255 * 1256 * @param dev 1257 * Pointer to Ethernet device structure. 1258 * 1259 * @return 1260 * 0 on success, a negative errno value otherwise and rte_errno is set. 1261 */ 1262 int 1263 mlx5_proc_priv_init(struct rte_eth_dev *dev) 1264 { 1265 struct mlx5_priv *priv = dev->data->dev_private; 1266 struct mlx5_proc_priv *ppriv; 1267 size_t ppriv_size; 1268 1269 /* 1270 * UAR register table follows the process private structure. BlueFlame 1271 * registers for Tx queues are stored in the table. 1272 */ 1273 ppriv_size = 1274 sizeof(struct mlx5_proc_priv) + priv->txqs_n * sizeof(void *); 1275 ppriv = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, ppriv_size, 1276 RTE_CACHE_LINE_SIZE, dev->device->numa_node); 1277 if (!ppriv) { 1278 rte_errno = ENOMEM; 1279 return -rte_errno; 1280 } 1281 ppriv->uar_table_sz = priv->txqs_n; 1282 dev->process_private = ppriv; 1283 return 0; 1284 } 1285 1286 /** 1287 * Un-initialize process private data structure. 1288 * 1289 * @param dev 1290 * Pointer to Ethernet device structure. 1291 */ 1292 void 1293 mlx5_proc_priv_uninit(struct rte_eth_dev *dev) 1294 { 1295 if (!dev->process_private) 1296 return; 1297 mlx5_free(dev->process_private); 1298 dev->process_private = NULL; 1299 } 1300 1301 /** 1302 * DPDK callback to close the device. 1303 * 1304 * Destroy all queues and objects, free memory. 1305 * 1306 * @param dev 1307 * Pointer to Ethernet device structure. 1308 */ 1309 int 1310 mlx5_dev_close(struct rte_eth_dev *dev) 1311 { 1312 struct mlx5_priv *priv = dev->data->dev_private; 1313 unsigned int i; 1314 int ret; 1315 1316 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1317 /* Check if process_private released. */ 1318 if (!dev->process_private) 1319 return 0; 1320 mlx5_tx_uar_uninit_secondary(dev); 1321 mlx5_proc_priv_uninit(dev); 1322 rte_eth_dev_release_port(dev); 1323 return 0; 1324 } 1325 if (!priv->sh) 1326 return 0; 1327 DRV_LOG(DEBUG, "port %u closing device \"%s\"", 1328 dev->data->port_id, 1329 ((priv->sh->ctx != NULL) ? 1330 mlx5_os_get_ctx_device_name(priv->sh->ctx) : "")); 1331 /* 1332 * If default mreg copy action is removed at the stop stage, 1333 * the search will return none and nothing will be done anymore. 1334 */ 1335 mlx5_flow_stop_default(dev); 1336 mlx5_traffic_disable(dev); 1337 /* 1338 * If all the flows are already flushed in the device stop stage, 1339 * then this will return directly without any action. 1340 */ 1341 mlx5_flow_list_flush(dev, &priv->flows, true); 1342 mlx5_shared_action_flush(dev); 1343 mlx5_flow_meter_flush(dev, NULL); 1344 /* Prevent crashes when queues are still in use. */ 1345 dev->rx_pkt_burst = removed_rx_burst; 1346 dev->tx_pkt_burst = removed_tx_burst; 1347 rte_wmb(); 1348 /* Disable datapath on secondary process. */ 1349 mlx5_mp_os_req_stop_rxtx(dev); 1350 /* Free the eCPRI flex parser resource. */ 1351 mlx5_flex_parser_ecpri_release(dev); 1352 if (priv->rxqs != NULL) { 1353 /* XXX race condition if mlx5_rx_burst() is still running. */ 1354 rte_delay_us_sleep(1000); 1355 for (i = 0; (i != priv->rxqs_n); ++i) 1356 mlx5_rxq_release(dev, i); 1357 priv->rxqs_n = 0; 1358 priv->rxqs = NULL; 1359 } 1360 if (priv->txqs != NULL) { 1361 /* XXX race condition if mlx5_tx_burst() is still running. */ 1362 rte_delay_us_sleep(1000); 1363 for (i = 0; (i != priv->txqs_n); ++i) 1364 mlx5_txq_release(dev, i); 1365 priv->txqs_n = 0; 1366 priv->txqs = NULL; 1367 } 1368 mlx5_proc_priv_uninit(dev); 1369 if (priv->q_counters) { 1370 mlx5_devx_cmd_destroy(priv->q_counters); 1371 priv->q_counters = NULL; 1372 } 1373 if (priv->drop_queue.hrxq) 1374 mlx5_drop_action_destroy(dev); 1375 if (priv->mreg_cp_tbl) 1376 mlx5_hlist_destroy(priv->mreg_cp_tbl); 1377 mlx5_mprq_free_mp(dev); 1378 mlx5_os_free_shared_dr(priv); 1379 if (priv->rss_conf.rss_key != NULL) 1380 mlx5_free(priv->rss_conf.rss_key); 1381 if (priv->reta_idx != NULL) 1382 mlx5_free(priv->reta_idx); 1383 if (priv->config.vf) 1384 mlx5_os_mac_addr_flush(dev); 1385 if (priv->nl_socket_route >= 0) 1386 close(priv->nl_socket_route); 1387 if (priv->nl_socket_rdma >= 0) 1388 close(priv->nl_socket_rdma); 1389 if (priv->vmwa_context) 1390 mlx5_vlan_vmwa_exit(priv->vmwa_context); 1391 ret = mlx5_hrxq_verify(dev); 1392 if (ret) 1393 DRV_LOG(WARNING, "port %u some hash Rx queue still remain", 1394 dev->data->port_id); 1395 ret = mlx5_ind_table_obj_verify(dev); 1396 if (ret) 1397 DRV_LOG(WARNING, "port %u some indirection table still remain", 1398 dev->data->port_id); 1399 ret = mlx5_rxq_obj_verify(dev); 1400 if (ret) 1401 DRV_LOG(WARNING, "port %u some Rx queue objects still remain", 1402 dev->data->port_id); 1403 ret = mlx5_rxq_verify(dev); 1404 if (ret) 1405 DRV_LOG(WARNING, "port %u some Rx queues still remain", 1406 dev->data->port_id); 1407 ret = mlx5_txq_obj_verify(dev); 1408 if (ret) 1409 DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain", 1410 dev->data->port_id); 1411 ret = mlx5_txq_verify(dev); 1412 if (ret) 1413 DRV_LOG(WARNING, "port %u some Tx queues still remain", 1414 dev->data->port_id); 1415 ret = mlx5_flow_verify(dev); 1416 if (ret) 1417 DRV_LOG(WARNING, "port %u some flows still remain", 1418 dev->data->port_id); 1419 mlx5_cache_list_destroy(&priv->hrxqs); 1420 /* 1421 * Free the shared context in last turn, because the cleanup 1422 * routines above may use some shared fields, like 1423 * mlx5_os_mac_addr_flush() uses ibdev_path for retrieveing 1424 * ifindex if Netlink fails. 1425 */ 1426 mlx5_free_shared_dev_ctx(priv->sh); 1427 if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 1428 unsigned int c = 0; 1429 uint16_t port_id; 1430 1431 MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) { 1432 struct mlx5_priv *opriv = 1433 rte_eth_devices[port_id].data->dev_private; 1434 1435 if (!opriv || 1436 opriv->domain_id != priv->domain_id || 1437 &rte_eth_devices[port_id] == dev) 1438 continue; 1439 ++c; 1440 break; 1441 } 1442 if (!c) 1443 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 1444 } 1445 memset(priv, 0, sizeof(*priv)); 1446 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 1447 /* 1448 * Reset mac_addrs to NULL such that it is not freed as part of 1449 * rte_eth_dev_release_port(). mac_addrs is part of dev_private so 1450 * it is freed when dev_private is freed. 1451 */ 1452 dev->data->mac_addrs = NULL; 1453 return 0; 1454 } 1455 1456 const struct eth_dev_ops mlx5_dev_ops = { 1457 .dev_configure = mlx5_dev_configure, 1458 .dev_start = mlx5_dev_start, 1459 .dev_stop = mlx5_dev_stop, 1460 .dev_set_link_down = mlx5_set_link_down, 1461 .dev_set_link_up = mlx5_set_link_up, 1462 .dev_close = mlx5_dev_close, 1463 .promiscuous_enable = mlx5_promiscuous_enable, 1464 .promiscuous_disable = mlx5_promiscuous_disable, 1465 .allmulticast_enable = mlx5_allmulticast_enable, 1466 .allmulticast_disable = mlx5_allmulticast_disable, 1467 .link_update = mlx5_link_update, 1468 .stats_get = mlx5_stats_get, 1469 .stats_reset = mlx5_stats_reset, 1470 .xstats_get = mlx5_xstats_get, 1471 .xstats_reset = mlx5_xstats_reset, 1472 .xstats_get_names = mlx5_xstats_get_names, 1473 .fw_version_get = mlx5_fw_version_get, 1474 .dev_infos_get = mlx5_dev_infos_get, 1475 .representor_info_get = mlx5_representor_info_get, 1476 .read_clock = mlx5_txpp_read_clock, 1477 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1478 .vlan_filter_set = mlx5_vlan_filter_set, 1479 .rx_queue_setup = mlx5_rx_queue_setup, 1480 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1481 .tx_queue_setup = mlx5_tx_queue_setup, 1482 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1483 .rx_queue_release = mlx5_rx_queue_release, 1484 .tx_queue_release = mlx5_tx_queue_release, 1485 .rx_queue_start = mlx5_rx_queue_start, 1486 .rx_queue_stop = mlx5_rx_queue_stop, 1487 .tx_queue_start = mlx5_tx_queue_start, 1488 .tx_queue_stop = mlx5_tx_queue_stop, 1489 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1490 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1491 .mac_addr_remove = mlx5_mac_addr_remove, 1492 .mac_addr_add = mlx5_mac_addr_add, 1493 .mac_addr_set = mlx5_mac_addr_set, 1494 .set_mc_addr_list = mlx5_set_mc_addr_list, 1495 .mtu_set = mlx5_dev_set_mtu, 1496 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1497 .vlan_offload_set = mlx5_vlan_offload_set, 1498 .reta_update = mlx5_dev_rss_reta_update, 1499 .reta_query = mlx5_dev_rss_reta_query, 1500 .rss_hash_update = mlx5_rss_hash_update, 1501 .rss_hash_conf_get = mlx5_rss_hash_conf_get, 1502 .flow_ops_get = mlx5_flow_ops_get, 1503 .rxq_info_get = mlx5_rxq_info_get, 1504 .txq_info_get = mlx5_txq_info_get, 1505 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1506 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1507 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1508 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1509 .is_removed = mlx5_is_removed, 1510 .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, 1511 .get_module_info = mlx5_get_module_info, 1512 .get_module_eeprom = mlx5_get_module_eeprom, 1513 .hairpin_cap_get = mlx5_hairpin_cap_get, 1514 .mtr_ops_get = mlx5_flow_meter_ops_get, 1515 .hairpin_bind = mlx5_hairpin_bind, 1516 .hairpin_unbind = mlx5_hairpin_unbind, 1517 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1518 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1519 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1520 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1521 }; 1522 1523 /* Available operations from secondary process. */ 1524 const struct eth_dev_ops mlx5_dev_sec_ops = { 1525 .stats_get = mlx5_stats_get, 1526 .stats_reset = mlx5_stats_reset, 1527 .xstats_get = mlx5_xstats_get, 1528 .xstats_reset = mlx5_xstats_reset, 1529 .xstats_get_names = mlx5_xstats_get_names, 1530 .fw_version_get = mlx5_fw_version_get, 1531 .dev_infos_get = mlx5_dev_infos_get, 1532 .read_clock = mlx5_txpp_read_clock, 1533 .rx_queue_start = mlx5_rx_queue_start, 1534 .rx_queue_stop = mlx5_rx_queue_stop, 1535 .tx_queue_start = mlx5_tx_queue_start, 1536 .tx_queue_stop = mlx5_tx_queue_stop, 1537 .rxq_info_get = mlx5_rxq_info_get, 1538 .txq_info_get = mlx5_txq_info_get, 1539 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1540 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1541 .get_module_info = mlx5_get_module_info, 1542 .get_module_eeprom = mlx5_get_module_eeprom, 1543 }; 1544 1545 /* Available operations in flow isolated mode. */ 1546 const struct eth_dev_ops mlx5_dev_ops_isolate = { 1547 .dev_configure = mlx5_dev_configure, 1548 .dev_start = mlx5_dev_start, 1549 .dev_stop = mlx5_dev_stop, 1550 .dev_set_link_down = mlx5_set_link_down, 1551 .dev_set_link_up = mlx5_set_link_up, 1552 .dev_close = mlx5_dev_close, 1553 .promiscuous_enable = mlx5_promiscuous_enable, 1554 .promiscuous_disable = mlx5_promiscuous_disable, 1555 .allmulticast_enable = mlx5_allmulticast_enable, 1556 .allmulticast_disable = mlx5_allmulticast_disable, 1557 .link_update = mlx5_link_update, 1558 .stats_get = mlx5_stats_get, 1559 .stats_reset = mlx5_stats_reset, 1560 .xstats_get = mlx5_xstats_get, 1561 .xstats_reset = mlx5_xstats_reset, 1562 .xstats_get_names = mlx5_xstats_get_names, 1563 .fw_version_get = mlx5_fw_version_get, 1564 .dev_infos_get = mlx5_dev_infos_get, 1565 .read_clock = mlx5_txpp_read_clock, 1566 .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, 1567 .vlan_filter_set = mlx5_vlan_filter_set, 1568 .rx_queue_setup = mlx5_rx_queue_setup, 1569 .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, 1570 .tx_queue_setup = mlx5_tx_queue_setup, 1571 .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, 1572 .rx_queue_release = mlx5_rx_queue_release, 1573 .tx_queue_release = mlx5_tx_queue_release, 1574 .rx_queue_start = mlx5_rx_queue_start, 1575 .rx_queue_stop = mlx5_rx_queue_stop, 1576 .tx_queue_start = mlx5_tx_queue_start, 1577 .tx_queue_stop = mlx5_tx_queue_stop, 1578 .flow_ctrl_get = mlx5_dev_get_flow_ctrl, 1579 .flow_ctrl_set = mlx5_dev_set_flow_ctrl, 1580 .mac_addr_remove = mlx5_mac_addr_remove, 1581 .mac_addr_add = mlx5_mac_addr_add, 1582 .mac_addr_set = mlx5_mac_addr_set, 1583 .set_mc_addr_list = mlx5_set_mc_addr_list, 1584 .mtu_set = mlx5_dev_set_mtu, 1585 .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, 1586 .vlan_offload_set = mlx5_vlan_offload_set, 1587 .flow_ops_get = mlx5_flow_ops_get, 1588 .rxq_info_get = mlx5_rxq_info_get, 1589 .txq_info_get = mlx5_txq_info_get, 1590 .rx_burst_mode_get = mlx5_rx_burst_mode_get, 1591 .tx_burst_mode_get = mlx5_tx_burst_mode_get, 1592 .rx_queue_intr_enable = mlx5_rx_intr_enable, 1593 .rx_queue_intr_disable = mlx5_rx_intr_disable, 1594 .is_removed = mlx5_is_removed, 1595 .get_module_info = mlx5_get_module_info, 1596 .get_module_eeprom = mlx5_get_module_eeprom, 1597 .hairpin_cap_get = mlx5_hairpin_cap_get, 1598 .mtr_ops_get = mlx5_flow_meter_ops_get, 1599 .hairpin_bind = mlx5_hairpin_bind, 1600 .hairpin_unbind = mlx5_hairpin_unbind, 1601 .hairpin_get_peer_ports = mlx5_hairpin_get_peer_ports, 1602 .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, 1603 .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, 1604 .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, 1605 }; 1606 1607 /** 1608 * Verify and store value for device argument. 1609 * 1610 * @param[in] key 1611 * Key argument to verify. 1612 * @param[in] val 1613 * Value associated with key. 1614 * @param opaque 1615 * User data. 1616 * 1617 * @return 1618 * 0 on success, a negative errno value otherwise and rte_errno is set. 1619 */ 1620 static int 1621 mlx5_args_check(const char *key, const char *val, void *opaque) 1622 { 1623 struct mlx5_dev_config *config = opaque; 1624 unsigned long mod; 1625 signed long tmp; 1626 1627 /* No-op, port representors are processed in mlx5_dev_spawn(). */ 1628 if (!strcmp(MLX5_REPRESENTOR, key)) 1629 return 0; 1630 errno = 0; 1631 tmp = strtol(val, NULL, 0); 1632 if (errno) { 1633 rte_errno = errno; 1634 DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); 1635 return -rte_errno; 1636 } 1637 if (tmp < 0 && strcmp(MLX5_TX_PP, key) && strcmp(MLX5_TX_SKEW, key)) { 1638 /* Negative values are acceptable for some keys only. */ 1639 rte_errno = EINVAL; 1640 DRV_LOG(WARNING, "%s: invalid negative value \"%s\"", key, val); 1641 return -rte_errno; 1642 } 1643 mod = tmp >= 0 ? tmp : -tmp; 1644 if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) { 1645 if (tmp > MLX5_CQE_RESP_FORMAT_L34H_STRIDX) { 1646 DRV_LOG(ERR, "invalid CQE compression " 1647 "format parameter"); 1648 rte_errno = EINVAL; 1649 return -rte_errno; 1650 } 1651 config->cqe_comp = !!tmp; 1652 config->cqe_comp_fmt = tmp; 1653 } else if (strcmp(MLX5_RXQ_PKT_PAD_EN, key) == 0) { 1654 config->hw_padding = !!tmp; 1655 } else if (strcmp(MLX5_RX_MPRQ_EN, key) == 0) { 1656 config->mprq.enabled = !!tmp; 1657 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_NUM, key) == 0) { 1658 config->mprq.stride_num_n = tmp; 1659 } else if (strcmp(MLX5_RX_MPRQ_LOG_STRIDE_SIZE, key) == 0) { 1660 config->mprq.stride_size_n = tmp; 1661 } else if (strcmp(MLX5_RX_MPRQ_MAX_MEMCPY_LEN, key) == 0) { 1662 config->mprq.max_memcpy_len = tmp; 1663 } else if (strcmp(MLX5_RXQS_MIN_MPRQ, key) == 0) { 1664 config->mprq.min_rxqs_num = tmp; 1665 } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) { 1666 DRV_LOG(WARNING, "%s: deprecated parameter," 1667 " converted to txq_inline_max", key); 1668 config->txq_inline_max = tmp; 1669 } else if (strcmp(MLX5_TXQ_INLINE_MAX, key) == 0) { 1670 config->txq_inline_max = tmp; 1671 } else if (strcmp(MLX5_TXQ_INLINE_MIN, key) == 0) { 1672 config->txq_inline_min = tmp; 1673 } else if (strcmp(MLX5_TXQ_INLINE_MPW, key) == 0) { 1674 config->txq_inline_mpw = tmp; 1675 } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) { 1676 config->txqs_inline = tmp; 1677 } else if (strcmp(MLX5_TXQS_MAX_VEC, key) == 0) { 1678 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1679 } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { 1680 config->mps = !!tmp; 1681 } else if (strcmp(MLX5_TX_DB_NC, key) == 0) { 1682 if (tmp != MLX5_TXDB_CACHED && 1683 tmp != MLX5_TXDB_NCACHED && 1684 tmp != MLX5_TXDB_HEURISTIC) { 1685 DRV_LOG(ERR, "invalid Tx doorbell " 1686 "mapping parameter"); 1687 rte_errno = EINVAL; 1688 return -rte_errno; 1689 } 1690 config->dbnc = tmp; 1691 } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { 1692 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1693 } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { 1694 DRV_LOG(WARNING, "%s: deprecated parameter," 1695 " converted to txq_inline_mpw", key); 1696 config->txq_inline_mpw = tmp; 1697 } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) { 1698 DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); 1699 } else if (strcmp(MLX5_TX_PP, key) == 0) { 1700 if (!mod) { 1701 DRV_LOG(ERR, "Zero Tx packet pacing parameter"); 1702 rte_errno = EINVAL; 1703 return -rte_errno; 1704 } 1705 config->tx_pp = tmp; 1706 } else if (strcmp(MLX5_TX_SKEW, key) == 0) { 1707 config->tx_skew = tmp; 1708 } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { 1709 config->rx_vec_en = !!tmp; 1710 } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) { 1711 config->l3_vxlan_en = !!tmp; 1712 } else if (strcmp(MLX5_VF_NL_EN, key) == 0) { 1713 config->vf_nl_en = !!tmp; 1714 } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) { 1715 config->dv_esw_en = !!tmp; 1716 } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) { 1717 config->dv_flow_en = !!tmp; 1718 } else if (strcmp(MLX5_DV_XMETA_EN, key) == 0) { 1719 if (tmp != MLX5_XMETA_MODE_LEGACY && 1720 tmp != MLX5_XMETA_MODE_META16 && 1721 tmp != MLX5_XMETA_MODE_META32 && 1722 tmp != MLX5_XMETA_MODE_MISS_INFO) { 1723 DRV_LOG(ERR, "invalid extensive " 1724 "metadata parameter"); 1725 rte_errno = EINVAL; 1726 return -rte_errno; 1727 } 1728 if (tmp != MLX5_XMETA_MODE_MISS_INFO) 1729 config->dv_xmeta_en = tmp; 1730 else 1731 config->dv_miss_info = 1; 1732 } else if (strcmp(MLX5_LACP_BY_USER, key) == 0) { 1733 config->lacp_by_user = !!tmp; 1734 } else if (strcmp(MLX5_MR_EXT_MEMSEG_EN, key) == 0) { 1735 config->mr_ext_memseg_en = !!tmp; 1736 } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) { 1737 config->max_dump_files_num = tmp; 1738 } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) { 1739 config->lro.timeout = tmp; 1740 } else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) { 1741 DRV_LOG(DEBUG, "class argument is %s.", val); 1742 } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) { 1743 config->log_hp_size = tmp; 1744 } else if (strcmp(MLX5_RECLAIM_MEM, key) == 0) { 1745 if (tmp != MLX5_RCM_NONE && 1746 tmp != MLX5_RCM_LIGHT && 1747 tmp != MLX5_RCM_AGGR) { 1748 DRV_LOG(ERR, "Unrecognize %s: \"%s\"", key, val); 1749 rte_errno = EINVAL; 1750 return -rte_errno; 1751 } 1752 config->reclaim_mode = tmp; 1753 } else if (strcmp(MLX5_SYS_MEM_EN, key) == 0) { 1754 config->sys_mem_en = !!tmp; 1755 } else if (strcmp(MLX5_DECAP_EN, key) == 0) { 1756 config->decap_en = !!tmp; 1757 } else { 1758 DRV_LOG(WARNING, "%s: unknown parameter", key); 1759 rte_errno = EINVAL; 1760 return -rte_errno; 1761 } 1762 return 0; 1763 } 1764 1765 /** 1766 * Parse device parameters. 1767 * 1768 * @param config 1769 * Pointer to device configuration structure. 1770 * @param devargs 1771 * Device arguments structure. 1772 * 1773 * @return 1774 * 0 on success, a negative errno value otherwise and rte_errno is set. 1775 */ 1776 int 1777 mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs) 1778 { 1779 const char **params = (const char *[]){ 1780 MLX5_RXQ_CQE_COMP_EN, 1781 MLX5_RXQ_PKT_PAD_EN, 1782 MLX5_RX_MPRQ_EN, 1783 MLX5_RX_MPRQ_LOG_STRIDE_NUM, 1784 MLX5_RX_MPRQ_LOG_STRIDE_SIZE, 1785 MLX5_RX_MPRQ_MAX_MEMCPY_LEN, 1786 MLX5_RXQS_MIN_MPRQ, 1787 MLX5_TXQ_INLINE, 1788 MLX5_TXQ_INLINE_MIN, 1789 MLX5_TXQ_INLINE_MAX, 1790 MLX5_TXQ_INLINE_MPW, 1791 MLX5_TXQS_MIN_INLINE, 1792 MLX5_TXQS_MAX_VEC, 1793 MLX5_TXQ_MPW_EN, 1794 MLX5_TXQ_MPW_HDR_DSEG_EN, 1795 MLX5_TXQ_MAX_INLINE_LEN, 1796 MLX5_TX_DB_NC, 1797 MLX5_TX_PP, 1798 MLX5_TX_SKEW, 1799 MLX5_TX_VEC_EN, 1800 MLX5_RX_VEC_EN, 1801 MLX5_L3_VXLAN_EN, 1802 MLX5_VF_NL_EN, 1803 MLX5_DV_ESW_EN, 1804 MLX5_DV_FLOW_EN, 1805 MLX5_DV_XMETA_EN, 1806 MLX5_LACP_BY_USER, 1807 MLX5_MR_EXT_MEMSEG_EN, 1808 MLX5_REPRESENTOR, 1809 MLX5_MAX_DUMP_FILES_NUM, 1810 MLX5_LRO_TIMEOUT_USEC, 1811 MLX5_CLASS_ARG_NAME, 1812 MLX5_HP_BUF_SIZE, 1813 MLX5_RECLAIM_MEM, 1814 MLX5_SYS_MEM_EN, 1815 MLX5_DECAP_EN, 1816 NULL, 1817 }; 1818 struct rte_kvargs *kvlist; 1819 int ret = 0; 1820 int i; 1821 1822 if (devargs == NULL) 1823 return 0; 1824 /* Following UGLY cast is done to pass checkpatch. */ 1825 kvlist = rte_kvargs_parse(devargs->args, params); 1826 if (kvlist == NULL) { 1827 rte_errno = EINVAL; 1828 return -rte_errno; 1829 } 1830 /* Process parameters. */ 1831 for (i = 0; (params[i] != NULL); ++i) { 1832 if (rte_kvargs_count(kvlist, params[i])) { 1833 ret = rte_kvargs_process(kvlist, params[i], 1834 mlx5_args_check, config); 1835 if (ret) { 1836 rte_errno = EINVAL; 1837 rte_kvargs_free(kvlist); 1838 return -rte_errno; 1839 } 1840 } 1841 } 1842 rte_kvargs_free(kvlist); 1843 return 0; 1844 } 1845 1846 /** 1847 * Configures the minimal amount of data to inline into WQE 1848 * while sending packets. 1849 * 1850 * - the txq_inline_min has the maximal priority, if this 1851 * key is specified in devargs 1852 * - if DevX is enabled the inline mode is queried from the 1853 * device (HCA attributes and NIC vport context if needed). 1854 * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4 Lx 1855 * and none (0 bytes) for other NICs 1856 * 1857 * @param spawn 1858 * Verbs device parameters (name, port, switch_info) to spawn. 1859 * @param config 1860 * Device configuration parameters. 1861 */ 1862 void 1863 mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn, 1864 struct mlx5_dev_config *config) 1865 { 1866 if (config->txq_inline_min != MLX5_ARG_UNSET) { 1867 /* Application defines size of inlined data explicitly. */ 1868 switch (spawn->pci_dev->id.device_id) { 1869 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 1870 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 1871 if (config->txq_inline_min < 1872 (int)MLX5_INLINE_HSIZE_L2) { 1873 DRV_LOG(DEBUG, 1874 "txq_inline_mix aligned to minimal" 1875 " ConnectX-4 required value %d", 1876 (int)MLX5_INLINE_HSIZE_L2); 1877 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 1878 } 1879 break; 1880 } 1881 goto exit; 1882 } 1883 if (config->hca_attr.eth_net_offloads) { 1884 /* We have DevX enabled, inline mode queried successfully. */ 1885 switch (config->hca_attr.wqe_inline_mode) { 1886 case MLX5_CAP_INLINE_MODE_L2: 1887 /* outer L2 header must be inlined. */ 1888 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 1889 goto exit; 1890 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 1891 /* No inline data are required by NIC. */ 1892 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 1893 config->hw_vlan_insert = 1894 config->hca_attr.wqe_vlan_insert; 1895 DRV_LOG(DEBUG, "Tx VLAN insertion is supported"); 1896 goto exit; 1897 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 1898 /* inline mode is defined by NIC vport context. */ 1899 if (!config->hca_attr.eth_virt) 1900 break; 1901 switch (config->hca_attr.vport_inline_mode) { 1902 case MLX5_INLINE_MODE_NONE: 1903 config->txq_inline_min = 1904 MLX5_INLINE_HSIZE_NONE; 1905 goto exit; 1906 case MLX5_INLINE_MODE_L2: 1907 config->txq_inline_min = 1908 MLX5_INLINE_HSIZE_L2; 1909 goto exit; 1910 case MLX5_INLINE_MODE_IP: 1911 config->txq_inline_min = 1912 MLX5_INLINE_HSIZE_L3; 1913 goto exit; 1914 case MLX5_INLINE_MODE_TCP_UDP: 1915 config->txq_inline_min = 1916 MLX5_INLINE_HSIZE_L4; 1917 goto exit; 1918 case MLX5_INLINE_MODE_INNER_L2: 1919 config->txq_inline_min = 1920 MLX5_INLINE_HSIZE_INNER_L2; 1921 goto exit; 1922 case MLX5_INLINE_MODE_INNER_IP: 1923 config->txq_inline_min = 1924 MLX5_INLINE_HSIZE_INNER_L3; 1925 goto exit; 1926 case MLX5_INLINE_MODE_INNER_TCP_UDP: 1927 config->txq_inline_min = 1928 MLX5_INLINE_HSIZE_INNER_L4; 1929 goto exit; 1930 } 1931 } 1932 } 1933 /* 1934 * We get here if we are unable to deduce 1935 * inline data size with DevX. Try PCI ID 1936 * to determine old NICs. 1937 */ 1938 switch (spawn->pci_dev->id.device_id) { 1939 case PCI_DEVICE_ID_MELLANOX_CONNECTX4: 1940 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 1941 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: 1942 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 1943 config->txq_inline_min = MLX5_INLINE_HSIZE_L2; 1944 config->hw_vlan_insert = 0; 1945 break; 1946 case PCI_DEVICE_ID_MELLANOX_CONNECTX5: 1947 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 1948 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: 1949 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 1950 /* 1951 * These NICs support VLAN insertion from WQE and 1952 * report the wqe_vlan_insert flag. But there is the bug 1953 * and PFC control may be broken, so disable feature. 1954 */ 1955 config->hw_vlan_insert = 0; 1956 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 1957 break; 1958 default: 1959 config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; 1960 break; 1961 } 1962 exit: 1963 DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min); 1964 } 1965 1966 /** 1967 * Configures the metadata mask fields in the shared context. 1968 * 1969 * @param [in] dev 1970 * Pointer to Ethernet device. 1971 */ 1972 void 1973 mlx5_set_metadata_mask(struct rte_eth_dev *dev) 1974 { 1975 struct mlx5_priv *priv = dev->data->dev_private; 1976 struct mlx5_dev_ctx_shared *sh = priv->sh; 1977 uint32_t meta, mark, reg_c0; 1978 1979 reg_c0 = ~priv->vport_meta_mask; 1980 switch (priv->config.dv_xmeta_en) { 1981 case MLX5_XMETA_MODE_LEGACY: 1982 meta = UINT32_MAX; 1983 mark = MLX5_FLOW_MARK_MASK; 1984 break; 1985 case MLX5_XMETA_MODE_META16: 1986 meta = reg_c0 >> rte_bsf32(reg_c0); 1987 mark = MLX5_FLOW_MARK_MASK; 1988 break; 1989 case MLX5_XMETA_MODE_META32: 1990 meta = UINT32_MAX; 1991 mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK; 1992 break; 1993 default: 1994 meta = 0; 1995 mark = 0; 1996 MLX5_ASSERT(false); 1997 break; 1998 } 1999 if (sh->dv_mark_mask && sh->dv_mark_mask != mark) 2000 DRV_LOG(WARNING, "metadata MARK mask mismatche %08X:%08X", 2001 sh->dv_mark_mask, mark); 2002 else 2003 sh->dv_mark_mask = mark; 2004 if (sh->dv_meta_mask && sh->dv_meta_mask != meta) 2005 DRV_LOG(WARNING, "metadata META mask mismatche %08X:%08X", 2006 sh->dv_meta_mask, meta); 2007 else 2008 sh->dv_meta_mask = meta; 2009 if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0) 2010 DRV_LOG(WARNING, "metadata reg_c0 mask mismatche %08X:%08X", 2011 sh->dv_meta_mask, reg_c0); 2012 else 2013 sh->dv_regc0_mask = reg_c0; 2014 DRV_LOG(DEBUG, "metadata mode %u", priv->config.dv_xmeta_en); 2015 DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask); 2016 DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask); 2017 DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask); 2018 } 2019 2020 int 2021 rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n) 2022 { 2023 static const char *const dynf_names[] = { 2024 RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, 2025 RTE_MBUF_DYNFLAG_METADATA_NAME, 2026 RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME 2027 }; 2028 unsigned int i; 2029 2030 if (n < RTE_DIM(dynf_names)) 2031 return -ENOMEM; 2032 for (i = 0; i < RTE_DIM(dynf_names); i++) { 2033 if (names[i] == NULL) 2034 return -EINVAL; 2035 strcpy(names[i], dynf_names[i]); 2036 } 2037 return RTE_DIM(dynf_names); 2038 } 2039 2040 /** 2041 * Comparison callback to sort device data. 2042 * 2043 * This is meant to be used with qsort(). 2044 * 2045 * @param a[in] 2046 * Pointer to pointer to first data object. 2047 * @param b[in] 2048 * Pointer to pointer to second data object. 2049 * 2050 * @return 2051 * 0 if both objects are equal, less than 0 if the first argument is less 2052 * than the second, greater than 0 otherwise. 2053 */ 2054 int 2055 mlx5_dev_check_sibling_config(struct mlx5_priv *priv, 2056 struct mlx5_dev_config *config) 2057 { 2058 struct mlx5_dev_ctx_shared *sh = priv->sh; 2059 struct mlx5_dev_config *sh_conf = NULL; 2060 uint16_t port_id; 2061 2062 MLX5_ASSERT(sh); 2063 /* Nothing to compare for the single/first device. */ 2064 if (sh->refcnt == 1) 2065 return 0; 2066 /* Find the device with shared context. */ 2067 MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) { 2068 struct mlx5_priv *opriv = 2069 rte_eth_devices[port_id].data->dev_private; 2070 2071 if (opriv && opriv != priv && opriv->sh == sh) { 2072 sh_conf = &opriv->config; 2073 break; 2074 } 2075 } 2076 if (!sh_conf) 2077 return 0; 2078 if (sh_conf->dv_flow_en ^ config->dv_flow_en) { 2079 DRV_LOG(ERR, "\"dv_flow_en\" configuration mismatch" 2080 " for shared %s context", sh->ibdev_name); 2081 rte_errno = EINVAL; 2082 return rte_errno; 2083 } 2084 if (sh_conf->dv_xmeta_en ^ config->dv_xmeta_en) { 2085 DRV_LOG(ERR, "\"dv_xmeta_en\" configuration mismatch" 2086 " for shared %s context", sh->ibdev_name); 2087 rte_errno = EINVAL; 2088 return rte_errno; 2089 } 2090 return 0; 2091 } 2092 2093 /** 2094 * Look for the ethernet device belonging to mlx5 driver. 2095 * 2096 * @param[in] port_id 2097 * port_id to start looking for device. 2098 * @param[in] pci_dev 2099 * Pointer to the hint PCI device. When device is being probed 2100 * the its siblings (master and preceding representors might 2101 * not have assigned driver yet (because the mlx5_os_pci_probe() 2102 * is not completed yet, for this case match on hint PCI 2103 * device may be used to detect sibling device. 2104 * 2105 * @return 2106 * port_id of found device, RTE_MAX_ETHPORT if not found. 2107 */ 2108 uint16_t 2109 mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev) 2110 { 2111 while (port_id < RTE_MAX_ETHPORTS) { 2112 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 2113 2114 if (dev->state != RTE_ETH_DEV_UNUSED && 2115 dev->device && 2116 (dev->device == &pci_dev->device || 2117 (dev->device->driver && 2118 dev->device->driver->name && 2119 !strcmp(dev->device->driver->name, MLX5_DRIVER_NAME)))) 2120 break; 2121 port_id++; 2122 } 2123 if (port_id >= RTE_MAX_ETHPORTS) 2124 return RTE_MAX_ETHPORTS; 2125 return port_id; 2126 } 2127 2128 /** 2129 * DPDK callback to remove a PCI device. 2130 * 2131 * This function removes all Ethernet devices belong to a given PCI device. 2132 * 2133 * @param[in] pci_dev 2134 * Pointer to the PCI device. 2135 * 2136 * @return 2137 * 0 on success, the function cannot fail. 2138 */ 2139 static int 2140 mlx5_pci_remove(struct rte_pci_device *pci_dev) 2141 { 2142 uint16_t port_id; 2143 int ret = 0; 2144 2145 RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device) { 2146 /* 2147 * mlx5_dev_close() is not registered to secondary process, 2148 * call the close function explicitly for secondary process. 2149 */ 2150 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 2151 ret |= mlx5_dev_close(&rte_eth_devices[port_id]); 2152 else 2153 ret |= rte_eth_dev_close(port_id); 2154 } 2155 return ret == 0 ? 0 : -EIO; 2156 } 2157 2158 static const struct rte_pci_id mlx5_pci_id_map[] = { 2159 { 2160 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2161 PCI_DEVICE_ID_MELLANOX_CONNECTX4) 2162 }, 2163 { 2164 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2165 PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) 2166 }, 2167 { 2168 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2169 PCI_DEVICE_ID_MELLANOX_CONNECTX4LX) 2170 }, 2171 { 2172 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2173 PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) 2174 }, 2175 { 2176 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2177 PCI_DEVICE_ID_MELLANOX_CONNECTX5) 2178 }, 2179 { 2180 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2181 PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) 2182 }, 2183 { 2184 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2185 PCI_DEVICE_ID_MELLANOX_CONNECTX5EX) 2186 }, 2187 { 2188 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2189 PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF) 2190 }, 2191 { 2192 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2193 PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) 2194 }, 2195 { 2196 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2197 PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF) 2198 }, 2199 { 2200 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2201 PCI_DEVICE_ID_MELLANOX_CONNECTX6) 2202 }, 2203 { 2204 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2205 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF) 2206 }, 2207 { 2208 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2209 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX) 2210 }, 2211 { 2212 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2213 PCI_DEVICE_ID_MELLANOX_CONNECTXVF) 2214 }, 2215 { 2216 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2217 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF) 2218 }, 2219 { 2220 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2221 PCI_DEVICE_ID_MELLANOX_CONNECTX6LX) 2222 }, 2223 { 2224 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2225 PCI_DEVICE_ID_MELLANOX_CONNECTX7) 2226 }, 2227 { 2228 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 2229 PCI_DEVICE_ID_MELLANOX_CONNECTX7BF) 2230 }, 2231 { 2232 .vendor_id = 0 2233 } 2234 }; 2235 2236 static struct mlx5_pci_driver mlx5_driver = { 2237 .driver_class = MLX5_CLASS_NET, 2238 .pci_driver = { 2239 .driver = { 2240 .name = MLX5_DRIVER_NAME, 2241 }, 2242 .id_table = mlx5_pci_id_map, 2243 .probe = mlx5_os_pci_probe, 2244 .remove = mlx5_pci_remove, 2245 .dma_map = mlx5_dma_map, 2246 .dma_unmap = mlx5_dma_unmap, 2247 .drv_flags = PCI_DRV_FLAGS, 2248 }, 2249 }; 2250 2251 /* Initialize driver log type. */ 2252 RTE_LOG_REGISTER(mlx5_logtype, pmd.net.mlx5, NOTICE) 2253 2254 /** 2255 * Driver initialization routine. 2256 */ 2257 RTE_INIT(rte_mlx5_pmd_init) 2258 { 2259 pthread_mutex_init(&mlx5_dev_ctx_list_mutex, NULL); 2260 mlx5_common_init(); 2261 /* Build the static tables for Verbs conversion. */ 2262 mlx5_set_ptype_table(); 2263 mlx5_set_cksum_table(); 2264 mlx5_set_swp_types_table(); 2265 if (mlx5_glue) 2266 mlx5_pci_driver_register(&mlx5_driver); 2267 } 2268 2269 RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__); 2270 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5, mlx5_pci_id_map); 2271 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5, "* ib_uverbs & mlx5_core & mlx5_ib"); 2272