1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <errno.h> 6 #include <stdalign.h> 7 #include <stddef.h> 8 #include <stdint.h> 9 #include <stdlib.h> 10 11 #include <rte_windows.h> 12 #include <ethdev_pci.h> 13 14 #include <mlx5_glue.h> 15 #include <mlx5_devx_cmds.h> 16 #include <mlx5_common.h> 17 #include <mlx5_common_mp.h> 18 #include <mlx5_common_mr.h> 19 #include <mlx5_malloc.h> 20 21 #include "mlx5_defs.h" 22 #include "mlx5.h" 23 #include "mlx5_common_os.h" 24 #include "mlx5_utils.h" 25 #include "mlx5_rxtx.h" 26 #include "mlx5_rx.h" 27 #include "mlx5_tx.h" 28 #include "mlx5_autoconf.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_devx.h" 31 32 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data"; 33 34 /* Spinlock for mlx5_shared_data allocation. */ 35 static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER; 36 37 /* rte flow indexed pool configuration. */ 38 static struct mlx5_indexed_pool_config icfg[] = { 39 { 40 .size = sizeof(struct rte_flow), 41 .trunk_size = 64, 42 .need_lock = 1, 43 .release_mem_en = 0, 44 .malloc = mlx5_malloc, 45 .free = mlx5_free, 46 .per_core_cache = 0, 47 .type = "ctl_flow_ipool", 48 }, 49 { 50 .size = sizeof(struct rte_flow), 51 .trunk_size = 64, 52 .grow_trunk = 3, 53 .grow_shift = 2, 54 .need_lock = 1, 55 .release_mem_en = 0, 56 .malloc = mlx5_malloc, 57 .free = mlx5_free, 58 .per_core_cache = 1 << 14, 59 .type = "rte_flow_ipool", 60 }, 61 { 62 .size = sizeof(struct rte_flow), 63 .trunk_size = 64, 64 .grow_trunk = 3, 65 .grow_shift = 2, 66 .need_lock = 1, 67 .release_mem_en = 0, 68 .malloc = mlx5_malloc, 69 .free = mlx5_free, 70 .per_core_cache = 0, 71 .type = "mcp_flow_ipool", 72 }, 73 }; 74 75 static void 76 mlx5_queue_counter_id_prepare(struct rte_eth_dev *dev) 77 { 78 struct mlx5_priv *priv = dev->data->dev_private; 79 void *ctx = priv->sh->cdev->ctx; 80 81 priv->q_counters = mlx5_devx_cmd_queue_counter_alloc(ctx); 82 if (!priv->q_counters) { 83 DRV_LOG(ERR, "Port %d queue counter object cannot be created " 84 "by DevX - imissed counter will be unavailable", 85 dev->data->port_id); 86 return; 87 } 88 priv->counter_set_id = priv->q_counters->id; 89 } 90 91 /** 92 * Initialize shared data between primary and secondary process. 93 * 94 * A memzone is reserved by primary process and secondary processes attach to 95 * the memzone. 96 * 97 * @return 98 * 0 on success, a negative errno value otherwise and rte_errno is set. 99 */ 100 static int 101 mlx5_init_shared_data(void) 102 { 103 const struct rte_memzone *mz; 104 int ret = 0; 105 106 rte_spinlock_lock(&mlx5_shared_data_lock); 107 if (mlx5_shared_data == NULL) { 108 /* Allocate shared memory. */ 109 mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA, 110 sizeof(*mlx5_shared_data), 111 SOCKET_ID_ANY, 0); 112 if (mz == NULL) { 113 DRV_LOG(ERR, 114 "Cannot allocate mlx5 shared data"); 115 ret = -rte_errno; 116 goto error; 117 } 118 mlx5_shared_data = mz->addr; 119 memset(mlx5_shared_data, 0, sizeof(*mlx5_shared_data)); 120 rte_spinlock_init(&mlx5_shared_data->lock); 121 } 122 error: 123 rte_spinlock_unlock(&mlx5_shared_data_lock); 124 return ret; 125 } 126 127 /** 128 * PMD global initialization. 129 * 130 * Independent from individual device, this function initializes global 131 * per-PMD data structures distinguishing primary and secondary processes. 132 * Hence, each initialization is called once per a process. 133 * 134 * @return 135 * 0 on success, a negative errno value otherwise and rte_errno is set. 136 */ 137 static int 138 mlx5_init_once(void) 139 { 140 if (mlx5_init_shared_data()) 141 return -rte_errno; 142 return 0; 143 } 144 145 /** 146 * Get mlx5 device attributes. 147 * 148 * @param cdev 149 * Pointer to mlx5 device. 150 * 151 * @param device_attr 152 * Pointer to mlx5 device attributes. 153 * 154 * @return 155 * 0 on success, non zero error number otherwise. 156 */ 157 int 158 mlx5_os_get_dev_attr(struct mlx5_common_device *cdev, 159 struct mlx5_dev_attr *device_attr) 160 { 161 struct mlx5_context *mlx5_ctx; 162 void *pv_iseg = NULL; 163 u32 cb_iseg = 0; 164 int err = 0; 165 166 if (!cdev || !cdev->ctx) 167 return -EINVAL; 168 mlx5_ctx = (struct mlx5_context *)cdev->ctx; 169 memset(device_attr, 0, sizeof(*device_attr)); 170 device_attr->max_cq = 1 << cdev->config.hca_attr.log_max_cq; 171 device_attr->max_qp = 1 << cdev->config.hca_attr.log_max_qp; 172 device_attr->max_qp_wr = 1 << cdev->config.hca_attr.log_max_qp_sz; 173 device_attr->max_cqe = 1 << cdev->config.hca_attr.log_max_cq_sz; 174 device_attr->max_mr = 1 << cdev->config.hca_attr.log_max_mrw_sz; 175 device_attr->max_pd = 1 << cdev->config.hca_attr.log_max_pd; 176 device_attr->max_srq = 1 << cdev->config.hca_attr.log_max_srq; 177 device_attr->max_srq_wr = 1 << cdev->config.hca_attr.log_max_srq_sz; 178 device_attr->max_tso = 1 << cdev->config.hca_attr.max_lso_cap; 179 if (cdev->config.hca_attr.rss_ind_tbl_cap) { 180 device_attr->max_rwq_indirection_table_size = 181 1 << cdev->config.hca_attr.rss_ind_tbl_cap; 182 } 183 device_attr->sw_parsing_offloads = 184 mlx5_get_supported_sw_parsing_offloads(&cdev->config.hca_attr); 185 device_attr->tunnel_offloads_caps = 186 mlx5_get_supported_tunneling_offloads(&cdev->config.hca_attr); 187 pv_iseg = mlx5_glue->query_hca_iseg(mlx5_ctx, &cb_iseg); 188 if (pv_iseg == NULL) { 189 DRV_LOG(ERR, "Failed to get device hca_iseg"); 190 return errno; 191 } 192 if (!err) { 193 snprintf(device_attr->fw_ver, 64, "%x.%x.%04x", 194 MLX5_GET(initial_seg, pv_iseg, fw_rev_major), 195 MLX5_GET(initial_seg, pv_iseg, fw_rev_minor), 196 MLX5_GET(initial_seg, pv_iseg, fw_rev_subminor)); 197 } 198 return err; 199 } 200 201 /** 202 * Initialize DR related data within private structure. 203 * Routine checks the reference counter and does actual 204 * resources creation/initialization only if counter is zero. 205 * 206 * @param[in] priv 207 * Pointer to the private device data structure. 208 * 209 * @return 210 * Zero on success, positive error code otherwise. 211 */ 212 static int 213 mlx5_alloc_shared_dr(struct mlx5_priv *priv) 214 { 215 struct mlx5_dev_ctx_shared *sh = priv->sh; 216 int err = 0; 217 218 if (!sh->flow_tbls) 219 err = mlx5_alloc_table_hash_list(priv); 220 else 221 DRV_LOG(DEBUG, "sh->flow_tbls[%p] already created, reuse", 222 (void *)sh->flow_tbls); 223 return err; 224 } 225 /** 226 * Destroy DR related data within private structure. 227 * 228 * @param[in] priv 229 * Pointer to the private device data structure. 230 */ 231 void 232 mlx5_os_free_shared_dr(struct mlx5_priv *priv) 233 { 234 mlx5_free_table_hash_list(priv); 235 } 236 237 /** 238 * Set the completion channel file descriptor interrupt as non-blocking. 239 * Currently it has no support under Windows. 240 * 241 * @param[in] rxq_obj 242 * Pointer to RQ channel object, which includes the channel fd 243 * 244 * @param[out] fd 245 * The file descriptor (representing the interrupt) used in this channel. 246 * 247 * @return 248 * 0 on successfully setting the fd to non-blocking, non-zero otherwise. 249 */ 250 int 251 mlx5_os_set_nonblock_channel_fd(int fd) 252 { 253 (void)fd; 254 DRV_LOG(WARNING, "%s: is not supported", __func__); 255 return -ENOTSUP; 256 } 257 258 /** 259 * DV flow counter mode detect and config. 260 * 261 * @param dev 262 * Pointer to rte_eth_dev structure. 263 * 264 */ 265 static void 266 mlx5_flow_counter_mode_config(struct rte_eth_dev *dev __rte_unused) 267 { 268 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 269 struct mlx5_priv *priv = dev->data->dev_private; 270 struct mlx5_dev_ctx_shared *sh = priv->sh; 271 bool fallback; 272 273 #ifndef HAVE_IBV_DEVX_ASYNC 274 fallback = true; 275 #else 276 fallback = false; 277 if (!sh->devx || !priv->config.dv_flow_en || 278 !priv->config.hca_attr.flow_counters_dump || 279 !(priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4) || 280 (mlx5_flow_dv_discover_counter_offset_support(dev) == -ENOTSUP)) 281 fallback = true; 282 #endif 283 if (fallback) 284 DRV_LOG(INFO, "Use fall-back DV counter management. Flow " 285 "counter dump:%d, bulk_alloc_bitmap:0x%hhx.", 286 priv->config.hca_attr.flow_counters_dump, 287 priv->config.hca_attr.flow_counter_bulk_alloc_bitmap); 288 /* Initialize fallback mode only on the port initializes sh. */ 289 if (sh->refcnt == 1) 290 sh->cmng.counter_fallback = fallback; 291 else if (fallback != sh->cmng.counter_fallback) 292 DRV_LOG(WARNING, "Port %d in sh has different fallback mode " 293 "with others:%d.", PORT_ID(priv), fallback); 294 #endif 295 } 296 297 /** 298 * Spawn an Ethernet device from DevX information. 299 * 300 * @param dpdk_dev 301 * Backing DPDK device. 302 * @param spawn 303 * Verbs device parameters (name, port, switch_info) to spawn. 304 * @param config 305 * Device configuration parameters. 306 * 307 * @return 308 * A valid Ethernet device object on success, NULL otherwise and rte_errno 309 * is set. The following errors are defined: 310 * 311 * EEXIST: device is already spawned 312 */ 313 static struct rte_eth_dev * 314 mlx5_dev_spawn(struct rte_device *dpdk_dev, 315 struct mlx5_dev_spawn_data *spawn, 316 struct mlx5_dev_config *config) 317 { 318 const struct mlx5_switch_info *switch_info = &spawn->info; 319 struct mlx5_dev_ctx_shared *sh = NULL; 320 struct mlx5_dev_attr device_attr; 321 struct rte_eth_dev *eth_dev = NULL; 322 struct mlx5_priv *priv = NULL; 323 int err = 0; 324 unsigned int cqe_comp; 325 struct rte_ether_addr mac; 326 char name[RTE_ETH_NAME_MAX_LEN]; 327 int own_domain_id = 0; 328 uint16_t port_id; 329 int i; 330 331 /* Build device name. */ 332 strlcpy(name, dpdk_dev->name, sizeof(name)); 333 /* check if the device is already spawned */ 334 if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) { 335 rte_errno = EEXIST; 336 return NULL; 337 } 338 DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name); 339 /* 340 * Some parameters are needed in advance to create device context. We 341 * process the devargs here to get ones, and later process devargs 342 * again to override some hardware settings. 343 */ 344 err = mlx5_args(config, dpdk_dev->devargs); 345 if (err) { 346 err = rte_errno; 347 DRV_LOG(ERR, "failed to process device arguments: %s", 348 strerror(rte_errno)); 349 goto error; 350 } 351 sh = mlx5_alloc_shared_dev_ctx(spawn, config); 352 if (!sh) 353 return NULL; 354 /* Initialize the shutdown event in mlx5_dev_spawn to 355 * support mlx5_is_removed for Windows. 356 */ 357 err = mlx5_glue->devx_init_showdown_event(sh->cdev->ctx); 358 if (err) { 359 DRV_LOG(ERR, "failed to init showdown event: %s", 360 strerror(errno)); 361 goto error; 362 } 363 DRV_LOG(DEBUG, "MPW isn't supported"); 364 mlx5_os_get_dev_attr(sh->cdev, &device_attr); 365 config->swp = device_attr.sw_parsing_offloads & 366 (MLX5_SW_PARSING_CAP | MLX5_SW_PARSING_CSUM_CAP | 367 MLX5_SW_PARSING_TSO_CAP); 368 config->ind_table_max_size = 369 sh->device_attr.max_rwq_indirection_table_size; 370 cqe_comp = 0; 371 config->cqe_comp = cqe_comp; 372 config->tunnel_en = device_attr.tunnel_offloads_caps & 373 (MLX5_TUNNELED_OFFLOADS_VXLAN_CAP | 374 MLX5_TUNNELED_OFFLOADS_GRE_CAP | 375 MLX5_TUNNELED_OFFLOADS_GENEVE_CAP); 376 if (config->tunnel_en) { 377 DRV_LOG(DEBUG, "tunnel offloading is supported for %s%s%s", 378 config->tunnel_en & 379 MLX5_TUNNELED_OFFLOADS_VXLAN_CAP ? "[VXLAN]" : "", 380 config->tunnel_en & 381 MLX5_TUNNELED_OFFLOADS_GRE_CAP ? "[GRE]" : "", 382 config->tunnel_en & 383 MLX5_TUNNELED_OFFLOADS_GENEVE_CAP ? "[GENEVE]" : "" 384 ); 385 } else { 386 DRV_LOG(DEBUG, "tunnel offloading is not supported"); 387 } 388 DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is no supported"); 389 config->mpls_en = 0; 390 /* Allocate private eth device data. */ 391 priv = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 392 sizeof(*priv), 393 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 394 if (priv == NULL) { 395 DRV_LOG(ERR, "priv allocation failure"); 396 err = ENOMEM; 397 goto error; 398 } 399 priv->sh = sh; 400 priv->dev_port = spawn->phys_port; 401 priv->pci_dev = spawn->pci_dev; 402 priv->mtu = RTE_ETHER_MTU; 403 priv->mp_id.port_id = port_id; 404 strlcpy(priv->mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN); 405 priv->representor = !!switch_info->representor; 406 priv->master = !!switch_info->master; 407 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 408 priv->vport_meta_tag = 0; 409 priv->vport_meta_mask = 0; 410 priv->pf_bond = spawn->pf_bond; 411 priv->vport_id = -1; 412 /* representor_id field keeps the unmodified VF index. */ 413 priv->representor_id = -1; 414 /* 415 * Look for sibling devices in order to reuse their switch domain 416 * if any, otherwise allocate one. 417 */ 418 MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) { 419 const struct mlx5_priv *opriv = 420 rte_eth_devices[port_id].data->dev_private; 421 422 if (!opriv || 423 opriv->sh != priv->sh || 424 opriv->domain_id == 425 RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) 426 continue; 427 priv->domain_id = opriv->domain_id; 428 break; 429 } 430 if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 431 err = rte_eth_switch_domain_alloc(&priv->domain_id); 432 if (err) { 433 err = rte_errno; 434 DRV_LOG(ERR, "unable to allocate switch domain: %s", 435 strerror(rte_errno)); 436 goto error; 437 } 438 own_domain_id = 1; 439 } 440 /* Override some values set by hardware configuration. */ 441 mlx5_args(config, dpdk_dev->devargs); 442 err = mlx5_dev_check_sibling_config(priv, config, dpdk_dev); 443 if (err) 444 goto error; 445 DRV_LOG(DEBUG, "counters are not supported"); 446 config->ind_table_max_size = 447 sh->device_attr.max_rwq_indirection_table_size; 448 /* 449 * Remove this check once DPDK supports larger/variable 450 * indirection tables. 451 */ 452 if (config->ind_table_max_size > (unsigned int)RTE_ETH_RSS_RETA_SIZE_512) 453 config->ind_table_max_size = RTE_ETH_RSS_RETA_SIZE_512; 454 DRV_LOG(DEBUG, "maximum Rx indirection table size is %u", 455 config->ind_table_max_size); 456 if (config->hw_padding) { 457 DRV_LOG(DEBUG, "Rx end alignment padding isn't supported"); 458 config->hw_padding = 0; 459 } 460 config->tso = (sh->device_attr.max_tso > 0); 461 if (config->tso) 462 config->tso_max_payload_sz = sh->device_attr.max_tso; 463 DRV_LOG(DEBUG, "%sMPS is %s.", 464 config->mps == MLX5_MPW_ENHANCED ? "enhanced " : 465 config->mps == MLX5_MPW ? "legacy " : "", 466 config->mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); 467 if (config->cqe_comp && !cqe_comp) { 468 DRV_LOG(WARNING, "Rx CQE compression isn't supported."); 469 config->cqe_comp = 0; 470 } 471 if (sh->devx) { 472 config->hca_attr = sh->cdev->config.hca_attr; 473 config->hw_csum = config->hca_attr.csum_cap; 474 DRV_LOG(DEBUG, "checksum offloading is %ssupported", 475 (config->hw_csum ? "" : "not ")); 476 config->hw_vlan_strip = config->hca_attr.vlan_cap; 477 DRV_LOG(DEBUG, "VLAN stripping is %ssupported", 478 (config->hw_vlan_strip ? "" : "not ")); 479 config->hw_fcs_strip = config->hca_attr.scatter_fcs; 480 } 481 if (sh->devx) { 482 uint32_t reg[MLX5_ST_SZ_DW(register_mtutc)]; 483 484 err = config->hca_attr.access_register_user ? 485 mlx5_devx_cmd_register_read 486 (sh->cdev->ctx, MLX5_REGISTER_ID_MTUTC, 0, 487 reg, MLX5_ST_SZ_DW(register_mtutc)) : ENOTSUP; 488 if (!err) { 489 uint32_t ts_mode; 490 491 /* MTUTC register is read successfully. */ 492 ts_mode = MLX5_GET(register_mtutc, reg, 493 time_stamp_mode); 494 if (ts_mode == MLX5_MTUTC_TIMESTAMP_MODE_REAL_TIME) 495 config->rt_timestamp = 1; 496 } else { 497 /* Kernel does not support register reading. */ 498 if (config->hca_attr.dev_freq_khz == 499 (NS_PER_S / MS_PER_S)) 500 config->rt_timestamp = 1; 501 } 502 } 503 if (config->mprq.enabled) { 504 DRV_LOG(WARNING, "Multi-Packet RQ isn't supported"); 505 config->mprq.enabled = 0; 506 } 507 if (config->max_dump_files_num == 0) 508 config->max_dump_files_num = 128; 509 eth_dev = rte_eth_dev_allocate(name); 510 if (eth_dev == NULL) { 511 DRV_LOG(ERR, "can not allocate rte ethdev"); 512 err = ENOMEM; 513 goto error; 514 } 515 if (priv->representor) { 516 eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR; 517 eth_dev->data->representor_id = priv->representor_id; 518 MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) { 519 struct mlx5_priv *opriv = 520 rte_eth_devices[port_id].data->dev_private; 521 if (opriv && 522 opriv->master && 523 opriv->domain_id == priv->domain_id && 524 opriv->sh == priv->sh) { 525 eth_dev->data->backer_port_id = port_id; 526 break; 527 } 528 } 529 if (port_id >= RTE_MAX_ETHPORTS) 530 eth_dev->data->backer_port_id = eth_dev->data->port_id; 531 } 532 /* 533 * Store associated network device interface index. This index 534 * is permanent throughout the lifetime of device. So, we may store 535 * the ifindex here and use the cached value further. 536 */ 537 MLX5_ASSERT(spawn->ifindex); 538 priv->if_index = spawn->ifindex; 539 eth_dev->data->dev_private = priv; 540 priv->dev_data = eth_dev->data; 541 eth_dev->data->mac_addrs = priv->mac; 542 eth_dev->device = dpdk_dev; 543 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 544 /* Configure the first MAC address by default. */ 545 if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) { 546 DRV_LOG(ERR, 547 "port %u cannot get MAC address, is mlx5_en" 548 " loaded? (errno: %s).", 549 eth_dev->data->port_id, strerror(rte_errno)); 550 err = ENODEV; 551 goto error; 552 } 553 DRV_LOG(INFO, 554 "port %u MAC address is " RTE_ETHER_ADDR_PRT_FMT, 555 eth_dev->data->port_id, RTE_ETHER_ADDR_BYTES(&mac)); 556 #ifdef RTE_LIBRTE_MLX5_DEBUG 557 { 558 char ifname[MLX5_NAMESIZE]; 559 560 if (mlx5_get_ifname(eth_dev, &ifname) == 0) 561 DRV_LOG(DEBUG, "port %u ifname is \"%s\"", 562 eth_dev->data->port_id, ifname); 563 else 564 DRV_LOG(DEBUG, "port %u ifname is unknown.", 565 eth_dev->data->port_id); 566 } 567 #endif 568 /* Get actual MTU if possible. */ 569 err = mlx5_get_mtu(eth_dev, &priv->mtu); 570 if (err) { 571 err = rte_errno; 572 goto error; 573 } 574 DRV_LOG(DEBUG, "port %u MTU is %u.", eth_dev->data->port_id, 575 priv->mtu); 576 /* Initialize burst functions to prevent crashes before link-up. */ 577 eth_dev->rx_pkt_burst = removed_rx_burst; 578 eth_dev->tx_pkt_burst = removed_tx_burst; 579 eth_dev->dev_ops = &mlx5_dev_ops; 580 eth_dev->rx_descriptor_status = mlx5_rx_descriptor_status; 581 eth_dev->tx_descriptor_status = mlx5_tx_descriptor_status; 582 eth_dev->rx_queue_count = mlx5_rx_queue_count; 583 /* Register MAC address. */ 584 claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0)); 585 priv->ctrl_flows = 0; 586 TAILQ_INIT(&priv->flow_meters); 587 priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR); 588 if (!priv->mtr_profile_tbl) 589 goto error; 590 /* Bring Ethernet device up. */ 591 DRV_LOG(DEBUG, "port %u forcing Ethernet interface up.", 592 eth_dev->data->port_id); 593 /* nl calls are unsupported - set to -1 not to fail on release */ 594 priv->nl_socket_rdma = -1; 595 priv->nl_socket_route = -1; 596 mlx5_set_link_up(eth_dev); 597 /* 598 * Even though the interrupt handler is not installed yet, 599 * interrupts will still trigger on the async_fd from 600 * Verbs context returned by ibv_open_device(). 601 */ 602 mlx5_link_update(eth_dev, 0); 603 config->dv_esw_en = 0; 604 /* Detect minimal data bytes to inline. */ 605 mlx5_set_min_inline(spawn, config); 606 /* Store device configuration on private structure. */ 607 priv->config = *config; 608 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 609 icfg[i].release_mem_en = !!config->reclaim_mode; 610 if (config->reclaim_mode) 611 icfg[i].per_core_cache = 0; 612 priv->flows[i] = mlx5_ipool_create(&icfg[i]); 613 if (!priv->flows[i]) 614 goto error; 615 } 616 /* Create context for virtual machine VLAN workaround. */ 617 priv->vmwa_context = NULL; 618 if (config->dv_flow_en) { 619 err = mlx5_alloc_shared_dr(priv); 620 if (err) 621 goto error; 622 } 623 /* No supported flow priority number detection. */ 624 priv->sh->flow_max_priority = -1; 625 if (!priv->config.dv_esw_en && 626 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 627 DRV_LOG(WARNING, "metadata mode %u is not supported " 628 "(no E-Switch)", priv->config.dv_xmeta_en); 629 priv->config.dv_xmeta_en = MLX5_XMETA_MODE_LEGACY; 630 } 631 mlx5_set_metadata_mask(eth_dev); 632 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 633 !priv->sh->dv_regc0_mask) { 634 DRV_LOG(ERR, "metadata mode %u is not supported " 635 "(no metadata reg_c[0] is available).", 636 priv->config.dv_xmeta_en); 637 err = ENOTSUP; 638 goto error; 639 } 640 priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true, 641 mlx5_hrxq_create_cb, mlx5_hrxq_match_cb, 642 mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb, 643 mlx5_hrxq_clone_free_cb); 644 /* Query availability of metadata reg_c's. */ 645 if (!priv->sh->metadata_regc_check_flag) { 646 err = mlx5_flow_discover_mreg_c(eth_dev); 647 if (err < 0) { 648 err = -err; 649 goto error; 650 } 651 } 652 if (!mlx5_flow_ext_mreg_supported(eth_dev)) { 653 DRV_LOG(DEBUG, 654 "port %u extensive metadata register is not supported.", 655 eth_dev->data->port_id); 656 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 657 DRV_LOG(ERR, "metadata mode %u is not supported " 658 "(no metadata registers available).", 659 priv->config.dv_xmeta_en); 660 err = ENOTSUP; 661 goto error; 662 } 663 } 664 if (sh->devx && config->dv_flow_en) { 665 priv->obj_ops = devx_obj_ops; 666 } else { 667 DRV_LOG(ERR, "Flow mode %u is not supported " 668 "(Windows flow must be DevX with DV flow enabled).", 669 priv->config.dv_flow_en); 670 err = ENOTSUP; 671 goto error; 672 } 673 mlx5_flow_counter_mode_config(eth_dev); 674 mlx5_queue_counter_id_prepare(eth_dev); 675 return eth_dev; 676 error: 677 if (priv) { 678 if (priv->mtr_profile_tbl) 679 mlx5_l3t_destroy(priv->mtr_profile_tbl); 680 if (own_domain_id) 681 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 682 mlx5_free(priv); 683 if (eth_dev != NULL) 684 eth_dev->data->dev_private = NULL; 685 } 686 if (eth_dev != NULL) { 687 /* mac_addrs must not be freed alone because part of 688 * dev_private 689 **/ 690 eth_dev->data->mac_addrs = NULL; 691 rte_eth_dev_release_port(eth_dev); 692 } 693 if (sh) 694 mlx5_free_shared_dev_ctx(sh); 695 MLX5_ASSERT(err > 0); 696 rte_errno = err; 697 return NULL; 698 } 699 700 /** 701 * This function should share events between multiple ports of single IB 702 * device. Currently it has no support under Windows. 703 * 704 * @param sh 705 * Pointer to mlx5_dev_ctx_shared object. 706 */ 707 void 708 mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh) 709 { 710 (void)sh; 711 DRV_LOG(WARNING, "%s: is not supported", __func__); 712 } 713 714 /** 715 * This function should share events between multiple ports of single IB 716 * device. Currently it has no support under Windows. 717 * 718 * @param dev 719 * Pointer to mlx5_dev_ctx_shared object. 720 */ 721 void 722 mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh) 723 { 724 (void)sh; 725 DRV_LOG(WARNING, "%s: is not supported", __func__); 726 } 727 728 /** 729 * Read statistics by a named counter. 730 * 731 * @param[in] priv 732 * Pointer to the private device data structure. 733 * @param[in] ctr_name 734 * Pointer to the name of the statistic counter to read 735 * @param[out] stat 736 * Pointer to read statistic value. 737 * @return 738 * 0 on success and stat is valid, non-zero if failed to read the value 739 * or counter is not supported. 740 * rte_errno is set. 741 * 742 */ 743 int 744 mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name, 745 uint64_t *stat) 746 { 747 if (priv->q_counters != NULL && strcmp(ctr_name, "out_of_buffer") == 0) 748 return mlx5_devx_cmd_queue_counter_query 749 (priv->q_counters, 0, (uint32_t *)stat); 750 DRV_LOG(WARNING, "%s: is not supported for the %s counter", 751 __func__, ctr_name); 752 return -ENOTSUP; 753 } 754 755 /** 756 * Flush device MAC addresses 757 * Currently it has no support under Windows. 758 * 759 * @param dev 760 * Pointer to Ethernet device structure. 761 * 762 */ 763 void 764 mlx5_os_mac_addr_flush(struct rte_eth_dev *dev) 765 { 766 (void)dev; 767 DRV_LOG(WARNING, "%s: is not supported", __func__); 768 } 769 770 /** 771 * Remove a MAC address from device 772 * Currently it has no support under Windows. 773 * 774 * @param dev 775 * Pointer to Ethernet device structure. 776 * @param index 777 * MAC address index. 778 */ 779 void 780 mlx5_os_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) 781 { 782 (void)dev; 783 (void)(index); 784 DRV_LOG(WARNING, "%s: is not supported", __func__); 785 } 786 787 /** 788 * Adds a MAC address to the device 789 * Currently it has no support under Windows. 790 * 791 * @param dev 792 * Pointer to Ethernet device structure. 793 * @param mac_addr 794 * MAC address to register. 795 * @param index 796 * MAC address index. 797 * 798 * @return 799 * 0 on success, a negative errno value otherwise 800 */ 801 int 802 mlx5_os_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac, 803 uint32_t index) 804 { 805 (void)index; 806 struct rte_ether_addr lmac; 807 808 if (mlx5_get_mac(dev, &lmac.addr_bytes)) { 809 DRV_LOG(ERR, 810 "port %u cannot get MAC address, is mlx5_en" 811 " loaded? (errno: %s)", 812 dev->data->port_id, strerror(rte_errno)); 813 return rte_errno; 814 } 815 if (!rte_is_same_ether_addr(&lmac, mac)) { 816 DRV_LOG(ERR, 817 "adding new mac address to device is unsupported"); 818 return -ENOTSUP; 819 } 820 return 0; 821 } 822 823 /** 824 * Modify a VF MAC address 825 * Currently it has no support under Windows. 826 * 827 * @param priv 828 * Pointer to device private data. 829 * @param mac_addr 830 * MAC address to modify into. 831 * @param iface_idx 832 * Net device interface index 833 * @param vf_index 834 * VF index 835 * 836 * @return 837 * 0 on success, a negative errno value otherwise 838 */ 839 int 840 mlx5_os_vf_mac_addr_modify(struct mlx5_priv *priv, 841 unsigned int iface_idx, 842 struct rte_ether_addr *mac_addr, 843 int vf_index) 844 { 845 (void)priv; 846 (void)iface_idx; 847 (void)mac_addr; 848 (void)vf_index; 849 DRV_LOG(WARNING, "%s: is not supported", __func__); 850 return -ENOTSUP; 851 } 852 853 /** 854 * Set device promiscuous mode 855 * Currently it has no support under Windows. 856 * 857 * @param dev 858 * Pointer to Ethernet device structure. 859 * @param enable 860 * 0 - promiscuous is disabled, otherwise - enabled 861 * 862 * @return 863 * 0 on success, a negative error value otherwise 864 */ 865 int 866 mlx5_os_set_promisc(struct rte_eth_dev *dev, int enable) 867 { 868 (void)dev; 869 (void)enable; 870 DRV_LOG(WARNING, "%s: is not supported", __func__); 871 return -ENOTSUP; 872 } 873 874 /** 875 * Set device allmulti mode 876 * 877 * @param dev 878 * Pointer to Ethernet device structure. 879 * @param enable 880 * 0 - all multicase is disabled, otherwise - enabled 881 * 882 * @return 883 * 0 on success, a negative error value otherwise 884 */ 885 int 886 mlx5_os_set_allmulti(struct rte_eth_dev *dev, int enable) 887 { 888 (void)dev; 889 (void)enable; 890 DRV_LOG(WARNING, "%s: is not supported", __func__); 891 return -ENOTSUP; 892 } 893 894 /** 895 * DPDK callback to register a PCI device. 896 * 897 * This function spawns Ethernet devices out of a given device. 898 * 899 * @param[in] dev 900 * Pointer to the common device. 901 * 902 * @return 903 * 0 on success, a negative errno value otherwise and rte_errno is set. 904 */ 905 int 906 mlx5_os_net_probe(struct mlx5_common_device *cdev) 907 { 908 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(cdev->dev); 909 struct mlx5_dev_spawn_data spawn = { 910 .pf_bond = -1, 911 .max_port = 1, 912 .phys_port = 1, 913 .phys_dev_name = mlx5_os_get_ctx_device_name(cdev->ctx), 914 .pci_dev = pci_dev, 915 .cdev = cdev, 916 .ifindex = -1, /* Spawn will assign */ 917 .info = (struct mlx5_switch_info){ 918 .name_type = MLX5_PHYS_PORT_NAME_TYPE_UPLINK, 919 }, 920 }; 921 struct mlx5_dev_config dev_config = { 922 .rx_vec_en = 1, 923 .txq_inline_max = MLX5_ARG_UNSET, 924 .txq_inline_min = MLX5_ARG_UNSET, 925 .txq_inline_mpw = MLX5_ARG_UNSET, 926 .txqs_inline = MLX5_ARG_UNSET, 927 .mprq = { 928 .max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN, 929 .min_rxqs_num = MLX5_MPRQ_MIN_RXQS, 930 }, 931 .dv_flow_en = 1, 932 .log_hp_size = MLX5_ARG_UNSET, 933 }; 934 int ret; 935 uint32_t restore; 936 937 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 938 DRV_LOG(ERR, "Secondary process is not supported on Windows."); 939 return -ENOTSUP; 940 } 941 ret = mlx5_init_once(); 942 if (ret) { 943 DRV_LOG(ERR, "unable to init PMD global data: %s", 944 strerror(rte_errno)); 945 return -rte_errno; 946 } 947 /* Device specific configuration. */ 948 switch (pci_dev->id.device_id) { 949 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 950 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 951 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 952 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 953 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF: 954 case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF: 955 case PCI_DEVICE_ID_MELLANOX_CONNECTXVF: 956 dev_config.vf = 1; 957 break; 958 default: 959 dev_config.vf = 0; 960 break; 961 } 962 spawn.eth_dev = mlx5_dev_spawn(cdev->dev, &spawn, &dev_config); 963 if (!spawn.eth_dev) 964 return -rte_errno; 965 restore = spawn.eth_dev->data->dev_flags; 966 rte_eth_copy_pci_info(spawn.eth_dev, pci_dev); 967 /* Restore non-PCI flags cleared by the above call. */ 968 spawn.eth_dev->data->dev_flags |= restore; 969 rte_eth_dev_probing_finish(spawn.eth_dev); 970 return 0; 971 } 972 973 /** 974 * Cleanup resources when the last device is closed. 975 */ 976 void 977 mlx5_os_net_cleanup(void) 978 { 979 } 980 981 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {0}; 982