1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <errno.h> 6 #include <stdalign.h> 7 #include <stddef.h> 8 #include <stdint.h> 9 #include <stdlib.h> 10 11 #include <rte_windows.h> 12 #include <ethdev_pci.h> 13 14 #include <mlx5_glue.h> 15 #include <mlx5_devx_cmds.h> 16 #include <mlx5_common.h> 17 #include <mlx5_common_mp.h> 18 #include <mlx5_common_mr.h> 19 #include <mlx5_malloc.h> 20 21 #include "mlx5_defs.h" 22 #include "mlx5.h" 23 #include "mlx5_common_os.h" 24 #include "mlx5_utils.h" 25 #include "mlx5_rxtx.h" 26 #include "mlx5_rx.h" 27 #include "mlx5_tx.h" 28 #include "mlx5_autoconf.h" 29 #include "mlx5_mr.h" 30 #include "mlx5_flow.h" 31 #include "mlx5_devx.h" 32 33 #define MLX5_TAGS_HLIST_ARRAY_SIZE 8192 34 35 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data"; 36 37 /* Spinlock for mlx5_shared_data allocation. */ 38 static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER; 39 40 /** 41 * Initialize shared data between primary and secondary process. 42 * 43 * A memzone is reserved by primary process and secondary processes attach to 44 * the memzone. 45 * 46 * @return 47 * 0 on success, a negative errno value otherwise and rte_errno is set. 48 */ 49 static int 50 mlx5_init_shared_data(void) 51 { 52 const struct rte_memzone *mz; 53 int ret = 0; 54 55 rte_spinlock_lock(&mlx5_shared_data_lock); 56 if (mlx5_shared_data == NULL) { 57 /* Allocate shared memory. */ 58 mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA, 59 sizeof(*mlx5_shared_data), 60 SOCKET_ID_ANY, 0); 61 if (mz == NULL) { 62 DRV_LOG(ERR, 63 "Cannot allocate mlx5 shared data"); 64 ret = -rte_errno; 65 goto error; 66 } 67 mlx5_shared_data = mz->addr; 68 memset(mlx5_shared_data, 0, sizeof(*mlx5_shared_data)); 69 rte_spinlock_init(&mlx5_shared_data->lock); 70 } 71 error: 72 rte_spinlock_unlock(&mlx5_shared_data_lock); 73 return ret; 74 } 75 76 /** 77 * PMD global initialization. 78 * 79 * Independent from individual device, this function initializes global 80 * per-PMD data structures distinguishing primary and secondary processes. 81 * Hence, each initialization is called once per a process. 82 * 83 * @return 84 * 0 on success, a negative errno value otherwise and rte_errno is set. 85 */ 86 static int 87 mlx5_init_once(void) 88 { 89 if (mlx5_init_shared_data()) 90 return -rte_errno; 91 return 0; 92 } 93 94 /** 95 * Get mlx5 device attributes. 96 * 97 * @param ctx 98 * Pointer to device context. 99 * 100 * @param device_attr 101 * Pointer to mlx5 device attributes. 102 * 103 * @return 104 * 0 on success, non zero error number otherwise 105 */ 106 int 107 mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *device_attr) 108 { 109 struct mlx5_context *mlx5_ctx; 110 struct mlx5_hca_attr hca_attr; 111 void *pv_iseg = NULL; 112 u32 cb_iseg = 0; 113 int err = 0; 114 115 if (!ctx) 116 return -EINVAL; 117 mlx5_ctx = (struct mlx5_context *)ctx; 118 memset(device_attr, 0, sizeof(*device_attr)); 119 err = mlx5_devx_cmd_query_hca_attr(mlx5_ctx, &hca_attr); 120 if (err) { 121 DRV_LOG(ERR, "Failed to get device hca_cap"); 122 return err; 123 } 124 device_attr->max_cq = 1 << hca_attr.log_max_cq; 125 device_attr->max_qp = 1 << hca_attr.log_max_qp; 126 device_attr->max_qp_wr = 1 << hca_attr.log_max_qp_sz; 127 device_attr->max_cqe = 1 << hca_attr.log_max_cq_sz; 128 device_attr->max_mr = 1 << hca_attr.log_max_mrw_sz; 129 device_attr->max_pd = 1 << hca_attr.log_max_pd; 130 device_attr->max_srq = 1 << hca_attr.log_max_srq; 131 device_attr->max_srq_wr = 1 << hca_attr.log_max_srq_sz; 132 if (hca_attr.rss_ind_tbl_cap) { 133 device_attr->max_rwq_indirection_table_size = 134 1 << hca_attr.rss_ind_tbl_cap; 135 } 136 pv_iseg = mlx5_glue->query_hca_iseg(mlx5_ctx, &cb_iseg); 137 if (pv_iseg == NULL) { 138 DRV_LOG(ERR, "Failed to get device hca_iseg"); 139 return errno; 140 } 141 if (!err) { 142 snprintf(device_attr->fw_ver, 64, "%x.%x.%04x", 143 MLX5_GET(initial_seg, pv_iseg, fw_rev_major), 144 MLX5_GET(initial_seg, pv_iseg, fw_rev_minor), 145 MLX5_GET(initial_seg, pv_iseg, fw_rev_subminor)); 146 } 147 return err; 148 } 149 150 /** 151 * Initialize DR related data within private structure. 152 * Routine checks the reference counter and does actual 153 * resources creation/initialization only if counter is zero. 154 * 155 * @param[in] priv 156 * Pointer to the private device data structure. 157 * 158 * @return 159 * Zero on success, positive error code otherwise. 160 */ 161 static int 162 mlx5_alloc_shared_dr(struct mlx5_priv *priv) 163 { 164 struct mlx5_dev_ctx_shared *sh = priv->sh; 165 int err = 0; 166 167 if (!sh->flow_tbls) 168 err = mlx5_alloc_table_hash_list(priv); 169 else 170 DRV_LOG(DEBUG, "sh->flow_tbls[%p] already created, reuse", 171 (void *)sh->flow_tbls); 172 return err; 173 } 174 /** 175 * Destroy DR related data within private structure. 176 * 177 * @param[in] priv 178 * Pointer to the private device data structure. 179 */ 180 void 181 mlx5_os_free_shared_dr(struct mlx5_priv *priv) 182 { 183 mlx5_free_table_hash_list(priv); 184 } 185 186 /** 187 * Set the completion channel file descriptor interrupt as non-blocking. 188 * Currently it has no support under Windows. 189 * 190 * @param[in] rxq_obj 191 * Pointer to RQ channel object, which includes the channel fd 192 * 193 * @param[out] fd 194 * The file descriptor (representing the intetrrupt) used in this channel. 195 * 196 * @return 197 * 0 on successfully setting the fd to non-blocking, non-zero otherwise. 198 */ 199 int 200 mlx5_os_set_nonblock_channel_fd(int fd) 201 { 202 (void)fd; 203 DRV_LOG(WARNING, "%s: is not supported", __func__); 204 return -ENOTSUP; 205 } 206 207 /** 208 * Function API open device under Windows 209 * 210 * This function calls the Windows glue APIs to open a device. 211 * 212 * @param[in] spawn 213 * Pointer to the device attributes (name, port, etc). 214 * @param[out] config 215 * Pointer to device configuration structure. 216 * @param[out] sh 217 * Pointer to shared context structure. 218 * 219 * @return 220 * 0 on success, a positive error value otherwise. 221 */ 222 int 223 mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn, 224 const struct mlx5_dev_config *config, 225 struct mlx5_dev_ctx_shared *sh) 226 { 227 RTE_SET_USED(config); 228 int err = 0; 229 struct mlx5_context *mlx5_ctx; 230 231 pthread_mutex_init(&sh->txpp.mutex, NULL); 232 /* Set numa node from pci probe */ 233 sh->numa_node = spawn->pci_dev->device.numa_node; 234 235 /* Try to open device with DevX */ 236 rte_errno = 0; 237 sh->ctx = mlx5_glue->open_device(spawn->phys_dev); 238 if (!sh->ctx) { 239 DRV_LOG(ERR, "open_device failed"); 240 err = errno; 241 return err; 242 } 243 sh->devx = 1; 244 mlx5_ctx = (struct mlx5_context *)sh->ctx; 245 err = mlx5_glue->query_device(spawn->phys_dev, &mlx5_ctx->mlx5_dev); 246 if (err) 247 DRV_LOG(ERR, "Failed to query device context fields."); 248 return err; 249 } 250 251 /** 252 * DV flow counter mode detect and config. 253 * 254 * @param dev 255 * Pointer to rte_eth_dev structure. 256 * 257 */ 258 static void 259 mlx5_flow_counter_mode_config(struct rte_eth_dev *dev __rte_unused) 260 { 261 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 262 struct mlx5_priv *priv = dev->data->dev_private; 263 struct mlx5_dev_ctx_shared *sh = priv->sh; 264 bool fallback; 265 266 #ifndef HAVE_IBV_DEVX_ASYNC 267 fallback = true; 268 #else 269 fallback = false; 270 if (!priv->config.devx || !priv->config.dv_flow_en || 271 !priv->config.hca_attr.flow_counters_dump || 272 !(priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4) || 273 (mlx5_flow_dv_discover_counter_offset_support(dev) == -ENOTSUP)) 274 fallback = true; 275 #endif 276 if (fallback) 277 DRV_LOG(INFO, "Use fall-back DV counter management. Flow " 278 "counter dump:%d, bulk_alloc_bitmap:0x%hhx.", 279 priv->config.hca_attr.flow_counters_dump, 280 priv->config.hca_attr.flow_counter_bulk_alloc_bitmap); 281 /* Initialize fallback mode only on the port initializes sh. */ 282 if (sh->refcnt == 1) 283 sh->cmng.counter_fallback = fallback; 284 else if (fallback != sh->cmng.counter_fallback) 285 DRV_LOG(WARNING, "Port %d in sh has different fallback mode " 286 "with others:%d.", PORT_ID(priv), fallback); 287 #endif 288 } 289 290 /** 291 * Spawn an Ethernet device from Verbs information. 292 * 293 * @param dpdk_dev 294 * Backing DPDK device. 295 * @param spawn 296 * Verbs device parameters (name, port, switch_info) to spawn. 297 * @param config 298 * Device configuration parameters. 299 * 300 * @return 301 * A valid Ethernet device object on success, NULL otherwise and rte_errno 302 * is set. The following errors are defined: 303 * 304 * EEXIST: device is already spawned 305 */ 306 static struct rte_eth_dev * 307 mlx5_dev_spawn(struct rte_device *dpdk_dev, 308 struct mlx5_dev_spawn_data *spawn, 309 struct mlx5_dev_config *config) 310 { 311 const struct mlx5_switch_info *switch_info = &spawn->info; 312 struct mlx5_dev_ctx_shared *sh = NULL; 313 struct mlx5_dev_attr device_attr; 314 struct rte_eth_dev *eth_dev = NULL; 315 struct mlx5_priv *priv = NULL; 316 int err = 0; 317 unsigned int cqe_comp; 318 struct rte_ether_addr mac; 319 char name[RTE_ETH_NAME_MAX_LEN]; 320 int own_domain_id = 0; 321 uint16_t port_id; 322 323 /* Build device name. */ 324 strlcpy(name, dpdk_dev->name, sizeof(name)); 325 /* check if the device is already spawned */ 326 if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) { 327 rte_errno = EEXIST; 328 return NULL; 329 } 330 DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name); 331 /* 332 * Some parameters are needed in advance to create device context. We 333 * process the devargs here to get ones, and later process devargs 334 * again to override some hardware settings. 335 */ 336 err = mlx5_args(config, dpdk_dev->devargs); 337 if (err) { 338 err = rte_errno; 339 DRV_LOG(ERR, "failed to process device arguments: %s", 340 strerror(rte_errno)); 341 goto error; 342 } 343 mlx5_malloc_mem_select(config->sys_mem_en); 344 sh = mlx5_alloc_shared_dev_ctx(spawn, config); 345 if (!sh) 346 return NULL; 347 config->devx = sh->devx; 348 /* Initialize the shutdown event in mlx5_dev_spawn to 349 * support mlx5_is_removed for Windows. 350 */ 351 err = mlx5_glue->devx_init_showdown_event(sh->ctx); 352 if (err) { 353 DRV_LOG(ERR, "failed to init showdown event: %s", 354 strerror(errno)); 355 goto error; 356 } 357 DRV_LOG(DEBUG, "MPW isn't supported"); 358 mlx5_os_get_dev_attr(sh->ctx, &device_attr); 359 config->swp = 0; 360 config->ind_table_max_size = 361 sh->device_attr.max_rwq_indirection_table_size; 362 cqe_comp = 0; 363 config->cqe_comp = cqe_comp; 364 DRV_LOG(DEBUG, "tunnel offloading is not supported"); 365 config->tunnel_en = 0; 366 DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is no supported"); 367 config->mpls_en = 0; 368 /* Allocate private eth device data. */ 369 priv = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, 370 sizeof(*priv), 371 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 372 if (priv == NULL) { 373 DRV_LOG(ERR, "priv allocation failure"); 374 err = ENOMEM; 375 goto error; 376 } 377 priv->sh = sh; 378 priv->dev_port = spawn->phys_port; 379 priv->pci_dev = spawn->pci_dev; 380 priv->mtu = RTE_ETHER_MTU; 381 priv->mp_id.port_id = port_id; 382 strlcpy(priv->mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN); 383 priv->representor = !!switch_info->representor; 384 priv->master = !!switch_info->master; 385 priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID; 386 priv->vport_meta_tag = 0; 387 priv->vport_meta_mask = 0; 388 priv->pf_bond = spawn->pf_bond; 389 priv->vport_id = -1; 390 /* representor_id field keeps the unmodified VF index. */ 391 priv->representor_id = -1; 392 /* 393 * Look for sibling devices in order to reuse their switch domain 394 * if any, otherwise allocate one. 395 */ 396 MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) { 397 const struct mlx5_priv *opriv = 398 rte_eth_devices[port_id].data->dev_private; 399 400 if (!opriv || 401 opriv->sh != priv->sh || 402 opriv->domain_id == 403 RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) 404 continue; 405 priv->domain_id = opriv->domain_id; 406 break; 407 } 408 if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { 409 err = rte_eth_switch_domain_alloc(&priv->domain_id); 410 if (err) { 411 err = rte_errno; 412 DRV_LOG(ERR, "unable to allocate switch domain: %s", 413 strerror(rte_errno)); 414 goto error; 415 } 416 own_domain_id = 1; 417 } 418 /* Override some values set by hardware configuration. */ 419 mlx5_args(config, dpdk_dev->devargs); 420 err = mlx5_dev_check_sibling_config(priv, config); 421 if (err) 422 goto error; 423 DRV_LOG(DEBUG, "counters are not supported"); 424 config->ind_table_max_size = 425 sh->device_attr.max_rwq_indirection_table_size; 426 /* 427 * Remove this check once DPDK supports larger/variable 428 * indirection tables. 429 */ 430 if (config->ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512) 431 config->ind_table_max_size = ETH_RSS_RETA_SIZE_512; 432 DRV_LOG(DEBUG, "maximum Rx indirection table size is %u", 433 config->ind_table_max_size); 434 DRV_LOG(DEBUG, "VLAN stripping is %ssupported", 435 (config->hw_vlan_strip ? "" : "not ")); 436 if (config->hw_padding) { 437 DRV_LOG(DEBUG, "Rx end alignment padding isn't supported"); 438 config->hw_padding = 0; 439 } 440 if (config->tso) 441 config->tso_max_payload_sz = sh->device_attr.max_tso; 442 DRV_LOG(DEBUG, "%sMPS is %s.", 443 config->mps == MLX5_MPW_ENHANCED ? "enhanced " : 444 config->mps == MLX5_MPW ? "legacy " : "", 445 config->mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); 446 if (config->cqe_comp && !cqe_comp) { 447 DRV_LOG(WARNING, "Rx CQE compression isn't supported."); 448 config->cqe_comp = 0; 449 } 450 if (config->devx) { 451 err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config->hca_attr); 452 if (err) { 453 err = -err; 454 goto error; 455 } 456 /* Check relax ordering support. */ 457 sh->cmng.relaxed_ordering_read = 0; 458 sh->cmng.relaxed_ordering_write = 0; 459 if (!haswell_broadwell_cpu) { 460 sh->cmng.relaxed_ordering_write = 461 config->hca_attr.relaxed_ordering_write; 462 sh->cmng.relaxed_ordering_read = 463 config->hca_attr.relaxed_ordering_read; 464 } 465 config->hw_csum = config->hca_attr.csum_cap; 466 DRV_LOG(DEBUG, "checksum offloading is %ssupported", 467 (config->hw_csum ? "" : "not ")); 468 } 469 if (config->devx) { 470 uint32_t reg[MLX5_ST_SZ_DW(register_mtutc)]; 471 472 err = config->hca_attr.access_register_user ? 473 mlx5_devx_cmd_register_read 474 (sh->ctx, MLX5_REGISTER_ID_MTUTC, 0, 475 reg, MLX5_ST_SZ_DW(register_mtutc)) : ENOTSUP; 476 if (!err) { 477 uint32_t ts_mode; 478 479 /* MTUTC register is read successfully. */ 480 ts_mode = MLX5_GET(register_mtutc, reg, 481 time_stamp_mode); 482 if (ts_mode == MLX5_MTUTC_TIMESTAMP_MODE_REAL_TIME) 483 config->rt_timestamp = 1; 484 } else { 485 /* Kernel does not support register reading. */ 486 if (config->hca_attr.dev_freq_khz == 487 (NS_PER_S / MS_PER_S)) 488 config->rt_timestamp = 1; 489 } 490 sh->rq_ts_format = config->hca_attr.rq_ts_format; 491 sh->sq_ts_format = config->hca_attr.sq_ts_format; 492 sh->qp_ts_format = config->hca_attr.qp_ts_format; 493 } 494 if (config->mprq.enabled) { 495 DRV_LOG(WARNING, "Multi-Packet RQ isn't supported"); 496 config->mprq.enabled = 0; 497 } 498 if (config->max_dump_files_num == 0) 499 config->max_dump_files_num = 128; 500 eth_dev = rte_eth_dev_allocate(name); 501 if (eth_dev == NULL) { 502 DRV_LOG(ERR, "can not allocate rte ethdev"); 503 err = ENOMEM; 504 goto error; 505 } 506 if (priv->representor) { 507 eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR; 508 eth_dev->data->representor_id = priv->representor_id; 509 } 510 /* 511 * Store associated network device interface index. This index 512 * is permanent throughout the lifetime of device. So, we may store 513 * the ifindex here and use the cached value further. 514 */ 515 MLX5_ASSERT(spawn->ifindex); 516 priv->if_index = spawn->ifindex; 517 eth_dev->data->dev_private = priv; 518 priv->dev_data = eth_dev->data; 519 eth_dev->data->mac_addrs = priv->mac; 520 eth_dev->device = dpdk_dev; 521 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 522 /* Configure the first MAC address by default. */ 523 if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) { 524 DRV_LOG(ERR, 525 "port %u cannot get MAC address, is mlx5_en" 526 " loaded? (errno: %s).", 527 eth_dev->data->port_id, strerror(rte_errno)); 528 err = ENODEV; 529 goto error; 530 } 531 DRV_LOG(INFO, 532 "port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", 533 eth_dev->data->port_id, 534 mac.addr_bytes[0], mac.addr_bytes[1], 535 mac.addr_bytes[2], mac.addr_bytes[3], 536 mac.addr_bytes[4], mac.addr_bytes[5]); 537 #ifdef RTE_LIBRTE_MLX5_DEBUG 538 { 539 char ifname[MLX5_NAMESIZE]; 540 541 if (mlx5_get_ifname(eth_dev, &ifname) == 0) 542 DRV_LOG(DEBUG, "port %u ifname is \"%s\"", 543 eth_dev->data->port_id, ifname); 544 else 545 DRV_LOG(DEBUG, "port %u ifname is unknown.", 546 eth_dev->data->port_id); 547 } 548 #endif 549 /* Get actual MTU if possible. */ 550 err = mlx5_get_mtu(eth_dev, &priv->mtu); 551 if (err) { 552 err = rte_errno; 553 goto error; 554 } 555 DRV_LOG(DEBUG, "port %u MTU is %u.", eth_dev->data->port_id, 556 priv->mtu); 557 /* Initialize burst functions to prevent crashes before link-up. */ 558 eth_dev->rx_pkt_burst = removed_rx_burst; 559 eth_dev->tx_pkt_burst = removed_tx_burst; 560 eth_dev->dev_ops = &mlx5_dev_ops; 561 eth_dev->rx_descriptor_status = mlx5_rx_descriptor_status; 562 eth_dev->tx_descriptor_status = mlx5_tx_descriptor_status; 563 eth_dev->rx_queue_count = mlx5_rx_queue_count; 564 /* Register MAC address. */ 565 claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0)); 566 priv->flows = 0; 567 priv->ctrl_flows = 0; 568 TAILQ_INIT(&priv->flow_meters); 569 TAILQ_INIT(&priv->flow_meter_profiles); 570 /* Bring Ethernet device up. */ 571 DRV_LOG(DEBUG, "port %u forcing Ethernet interface up.", 572 eth_dev->data->port_id); 573 /* nl calls are unsupported - set to -1 not to fail on release */ 574 priv->nl_socket_rdma = -1; 575 priv->nl_socket_route = -1; 576 mlx5_set_link_up(eth_dev); 577 /* 578 * Even though the interrupt handler is not installed yet, 579 * interrupts will still trigger on the async_fd from 580 * Verbs context returned by ibv_open_device(). 581 */ 582 mlx5_link_update(eth_dev, 0); 583 config->dv_esw_en = 0; 584 /* Detect minimal data bytes to inline. */ 585 mlx5_set_min_inline(spawn, config); 586 /* Store device configuration on private structure. */ 587 priv->config = *config; 588 /* Create context for virtual machine VLAN workaround. */ 589 priv->vmwa_context = NULL; 590 if (config->dv_flow_en) { 591 err = mlx5_alloc_shared_dr(priv); 592 if (err) 593 goto error; 594 } 595 /* No supported flow priority number detection. */ 596 priv->config.flow_prio = -1; 597 if (!priv->config.dv_esw_en && 598 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 599 DRV_LOG(WARNING, "metadata mode %u is not supported " 600 "(no E-Switch)", priv->config.dv_xmeta_en); 601 priv->config.dv_xmeta_en = MLX5_XMETA_MODE_LEGACY; 602 } 603 mlx5_set_metadata_mask(eth_dev); 604 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 605 !priv->sh->dv_regc0_mask) { 606 DRV_LOG(ERR, "metadata mode %u is not supported " 607 "(no metadata reg_c[0] is available).", 608 priv->config.dv_xmeta_en); 609 err = ENOTSUP; 610 goto error; 611 } 612 mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev, 613 mlx5_hrxq_create_cb, 614 mlx5_hrxq_match_cb, 615 mlx5_hrxq_remove_cb); 616 /* Query availability of metadata reg_c's. */ 617 err = mlx5_flow_discover_mreg_c(eth_dev); 618 if (err < 0) { 619 err = -err; 620 goto error; 621 } 622 if (!mlx5_flow_ext_mreg_supported(eth_dev)) { 623 DRV_LOG(DEBUG, 624 "port %u extensive metadata register is not supported.", 625 eth_dev->data->port_id); 626 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 627 DRV_LOG(ERR, "metadata mode %u is not supported " 628 "(no metadata registers available).", 629 priv->config.dv_xmeta_en); 630 err = ENOTSUP; 631 goto error; 632 } 633 } 634 if (config->devx && config->dv_flow_en) { 635 priv->obj_ops = devx_obj_ops; 636 } else { 637 DRV_LOG(ERR, "Flow mode %u is not supported " 638 "(Windows flow must be DevX with DV flow enabled).", 639 priv->config.dv_flow_en); 640 err = ENOTSUP; 641 goto error; 642 } 643 mlx5_flow_counter_mode_config(eth_dev); 644 return eth_dev; 645 error: 646 if (priv) { 647 if (own_domain_id) 648 claim_zero(rte_eth_switch_domain_free(priv->domain_id)); 649 mlx5_free(priv); 650 if (eth_dev != NULL) 651 eth_dev->data->dev_private = NULL; 652 } 653 if (eth_dev != NULL) { 654 /* mac_addrs must not be freed alone because part of 655 * dev_private 656 **/ 657 eth_dev->data->mac_addrs = NULL; 658 rte_eth_dev_release_port(eth_dev); 659 } 660 if (sh) 661 mlx5_free_shared_dev_ctx(sh); 662 MLX5_ASSERT(err > 0); 663 rte_errno = err; 664 return NULL; 665 } 666 667 /** 668 * This function should share events between multiple ports of single IB 669 * device. Currently it has no support under Windows. 670 * 671 * @param sh 672 * Pointer to mlx5_dev_ctx_shared object. 673 */ 674 void 675 mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh) 676 { 677 (void)sh; 678 DRV_LOG(WARNING, "%s: is not supported", __func__); 679 } 680 681 /** 682 * This function should share events between multiple ports of single IB 683 * device. Currently it has no support under Windows. 684 * 685 * @param dev 686 * Pointer to mlx5_dev_ctx_shared object. 687 */ 688 void 689 mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh) 690 { 691 (void)sh; 692 DRV_LOG(WARNING, "%s: is not supported", __func__); 693 } 694 695 /** 696 * Read statistics by a named counter. 697 * 698 * @param[in] priv 699 * Pointer to the private device data structure. 700 * @param[in] ctr_name 701 * Pointer to the name of the statistic counter to read 702 * @param[out] stat 703 * Pointer to read statistic value. 704 * @return 705 * 0 on success and stat is valud, 1 if failed to read the value 706 * rte_errno is set. 707 * 708 */ 709 int 710 mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name, 711 uint64_t *stat) 712 { 713 RTE_SET_USED(priv); 714 RTE_SET_USED(ctr_name); 715 RTE_SET_USED(stat); 716 DRV_LOG(WARNING, "%s: is not supported", __func__); 717 return -ENOTSUP; 718 } 719 720 /** 721 * Flush device MAC addresses 722 * Currently it has no support under Windows. 723 * 724 * @param dev 725 * Pointer to Ethernet device structure. 726 * 727 */ 728 void 729 mlx5_os_mac_addr_flush(struct rte_eth_dev *dev) 730 { 731 (void)dev; 732 DRV_LOG(WARNING, "%s: is not supported", __func__); 733 } 734 735 /** 736 * Remove a MAC address from device 737 * Currently it has no support under Windows. 738 * 739 * @param dev 740 * Pointer to Ethernet device structure. 741 * @param index 742 * MAC address index. 743 */ 744 void 745 mlx5_os_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) 746 { 747 (void)dev; 748 (void)(index); 749 DRV_LOG(WARNING, "%s: is not supported", __func__); 750 } 751 752 /** 753 * Adds a MAC address to the device 754 * Currently it has no support under Windows. 755 * 756 * @param dev 757 * Pointer to Ethernet device structure. 758 * @param mac_addr 759 * MAC address to register. 760 * @param index 761 * MAC address index. 762 * 763 * @return 764 * 0 on success, a negative errno value otherwise 765 */ 766 int 767 mlx5_os_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac, 768 uint32_t index) 769 { 770 (void)index; 771 struct rte_ether_addr lmac; 772 773 if (mlx5_get_mac(dev, &lmac.addr_bytes)) { 774 DRV_LOG(ERR, 775 "port %u cannot get MAC address, is mlx5_en" 776 " loaded? (errno: %s)", 777 dev->data->port_id, strerror(rte_errno)); 778 return rte_errno; 779 } 780 if (!rte_is_same_ether_addr(&lmac, mac)) { 781 DRV_LOG(ERR, 782 "adding new mac address to device is unsupported"); 783 return -ENOTSUP; 784 } 785 return 0; 786 } 787 788 /** 789 * Modify a VF MAC address 790 * Currently it has no support under Windows. 791 * 792 * @param priv 793 * Pointer to device private data. 794 * @param mac_addr 795 * MAC address to modify into. 796 * @param iface_idx 797 * Net device interface index 798 * @param vf_index 799 * VF index 800 * 801 * @return 802 * 0 on success, a negative errno value otherwise 803 */ 804 int 805 mlx5_os_vf_mac_addr_modify(struct mlx5_priv *priv, 806 unsigned int iface_idx, 807 struct rte_ether_addr *mac_addr, 808 int vf_index) 809 { 810 (void)priv; 811 (void)iface_idx; 812 (void)mac_addr; 813 (void)vf_index; 814 DRV_LOG(WARNING, "%s: is not supported", __func__); 815 return -ENOTSUP; 816 } 817 818 /** 819 * Set device promiscuous mode 820 * Currently it has no support under Windows. 821 * 822 * @param dev 823 * Pointer to Ethernet device structure. 824 * @param enable 825 * 0 - promiscuous is disabled, otherwise - enabled 826 * 827 * @return 828 * 0 on success, a negative error value otherwise 829 */ 830 int 831 mlx5_os_set_promisc(struct rte_eth_dev *dev, int enable) 832 { 833 (void)dev; 834 (void)enable; 835 DRV_LOG(WARNING, "%s: is not supported", __func__); 836 return -ENOTSUP; 837 } 838 839 /** 840 * Set device allmulti mode 841 * 842 * @param dev 843 * Pointer to Ethernet device structure. 844 * @param enable 845 * 0 - all multicase is disabled, otherwise - enabled 846 * 847 * @return 848 * 0 on success, a negative error value otherwise 849 */ 850 int 851 mlx5_os_set_allmulti(struct rte_eth_dev *dev, int enable) 852 { 853 (void)dev; 854 (void)enable; 855 DRV_LOG(WARNING, "%s: is not supported", __func__); 856 return -ENOTSUP; 857 } 858 859 /** 860 * Detect if a devx_device_bdf object has identical DBDF values to the 861 * rte_pci_addr found in bus/pci probing 862 * 863 * @param[in] devx_bdf 864 * Pointer to the devx_device_bdf structure. 865 * @param[in] addr 866 * Pointer to the rte_pci_addr structure. 867 * 868 * @return 869 * 1 on Device match, 0 on mismatch. 870 */ 871 static int 872 mlx5_match_devx_bdf_to_addr(struct devx_device_bdf *devx_bdf, 873 struct rte_pci_addr *addr) 874 { 875 if (addr->domain != (devx_bdf->bus_id >> 8) || 876 addr->bus != (devx_bdf->bus_id & 0xff) || 877 addr->devid != devx_bdf->dev_id || 878 addr->function != devx_bdf->fnc_id) { 879 return 0; 880 } 881 return 1; 882 } 883 884 /** 885 * Detect if a devx_device_bdf object matches the rte_pci_addr 886 * found in bus/pci probing 887 * Compare both the Native/PF BDF and the raw_bdf representing a VF BDF. 888 * 889 * @param[in] devx_bdf 890 * Pointer to the devx_device_bdf structure. 891 * @param[in] addr 892 * Pointer to the rte_pci_addr structure. 893 * 894 * @return 895 * 1 on Device match, 0 on mismatch, rte_errno code on failure. 896 */ 897 static int 898 mlx5_match_devx_devices_to_addr(struct devx_device_bdf *devx_bdf, 899 struct rte_pci_addr *addr) 900 { 901 int err; 902 struct devx_device mlx5_dev; 903 904 if (mlx5_match_devx_bdf_to_addr(devx_bdf, addr)) 905 return 1; 906 /** 907 * Didn't match on Native/PF BDF, could still 908 * Match a VF BDF, check it next 909 */ 910 err = mlx5_glue->query_device(devx_bdf, &mlx5_dev); 911 if (err) { 912 DRV_LOG(ERR, "query_device failed"); 913 rte_errno = err; 914 return rte_errno; 915 } 916 if (mlx5_match_devx_bdf_to_addr(&mlx5_dev.raw_bdf, addr)) 917 return 1; 918 return 0; 919 } 920 921 /** 922 * DPDK callback to register a PCI device. 923 * 924 * This function spawns Ethernet devices out of a given PCI device. 925 * 926 * @param[in] pci_drv 927 * PCI driver structure (mlx5_driver). 928 * @param[in] pci_dev 929 * PCI device information. 930 * 931 * @return 932 * 0 on success, a negative errno value otherwise and rte_errno is set. 933 */ 934 int 935 mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 936 struct rte_pci_device *pci_dev) 937 { 938 struct devx_device_bdf *devx_bdf_devs, *orig_devx_bdf_devs; 939 /* 940 * Number of found IB Devices matching with requested PCI BDF. 941 * nd != 1 means there are multiple IB devices over the same 942 * PCI device and we have representors and master. 943 */ 944 unsigned int nd = 0; 945 /* 946 * Number of found IB device Ports. nd = 1 and np = 1..n means 947 * we have the single multiport IB device, and there may be 948 * representors attached to some of found ports. 949 * Currently not supported. 950 * unsigned int np = 0; 951 */ 952 953 /* 954 * Number of DPDK ethernet devices to Spawn - either over 955 * multiple IB devices or multiple ports of single IB device. 956 * Actually this is the number of iterations to spawn. 957 */ 958 unsigned int ns = 0; 959 /* 960 * Bonding device 961 * < 0 - no bonding device (single one) 962 * >= 0 - bonding device (value is slave PF index) 963 */ 964 int bd = -1; 965 struct mlx5_dev_spawn_data *list = NULL; 966 struct mlx5_dev_config dev_config; 967 unsigned int dev_config_vf; 968 int ret, err; 969 uint32_t restore; 970 971 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 972 DRV_LOG(ERR, "Secondary process is not supported on Windows."); 973 return -ENOTSUP; 974 } 975 ret = mlx5_init_once(); 976 if (ret) { 977 DRV_LOG(ERR, "unable to init PMD global data: %s", 978 strerror(rte_errno)); 979 return -rte_errno; 980 } 981 errno = 0; 982 devx_bdf_devs = mlx5_glue->get_device_list(&ret); 983 orig_devx_bdf_devs = devx_bdf_devs; 984 if (!devx_bdf_devs) { 985 rte_errno = errno ? errno : ENOSYS; 986 DRV_LOG(ERR, "cannot list devices, is ib_uverbs loaded?"); 987 return -rte_errno; 988 } 989 /* 990 * First scan the list of all Infiniband devices to find 991 * matching ones, gathering into the list. 992 */ 993 struct devx_device_bdf *devx_bdf_match[ret + 1]; 994 995 while (ret-- > 0) { 996 err = mlx5_match_devx_devices_to_addr(devx_bdf_devs, 997 &pci_dev->addr); 998 if (!err) { 999 devx_bdf_devs++; 1000 continue; 1001 } 1002 if (err != 1) { 1003 ret = -err; 1004 goto exit; 1005 } 1006 devx_bdf_match[nd++] = devx_bdf_devs; 1007 } 1008 devx_bdf_match[nd] = NULL; 1009 if (!nd) { 1010 /* No device matches, just complain and bail out. */ 1011 DRV_LOG(WARNING, 1012 "no DevX device matches PCI device " PCI_PRI_FMT "," 1013 " is DevX Configured?", 1014 pci_dev->addr.domain, pci_dev->addr.bus, 1015 pci_dev->addr.devid, pci_dev->addr.function); 1016 rte_errno = ENOENT; 1017 ret = -rte_errno; 1018 goto exit; 1019 } 1020 /* 1021 * Now we can determine the maximal 1022 * amount of devices to be spawned. 1023 */ 1024 list = mlx5_malloc(MLX5_MEM_ZERO, 1025 sizeof(struct mlx5_dev_spawn_data), 1026 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1027 if (!list) { 1028 DRV_LOG(ERR, "spawn data array allocation failure"); 1029 rte_errno = ENOMEM; 1030 ret = -rte_errno; 1031 goto exit; 1032 } 1033 memset(&list[ns].info, 0, sizeof(list[ns].info)); 1034 list[ns].max_port = 1; 1035 list[ns].phys_port = 1; 1036 list[ns].phys_dev = devx_bdf_match[ns]; 1037 list[ns].eth_dev = NULL; 1038 list[ns].pci_dev = pci_dev; 1039 list[ns].pf_bond = bd; 1040 list[ns].ifindex = -1; /* Spawn will assign */ 1041 list[ns].info = 1042 (struct mlx5_switch_info){ 1043 .master = 0, 1044 .representor = 0, 1045 .name_type = MLX5_PHYS_PORT_NAME_TYPE_UPLINK, 1046 .port_name = 0, 1047 .switch_id = 0, 1048 }; 1049 /* Device specific configuration. */ 1050 switch (pci_dev->id.device_id) { 1051 case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: 1052 case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF: 1053 case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: 1054 case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: 1055 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF: 1056 case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF: 1057 case PCI_DEVICE_ID_MELLANOX_CONNECTXVF: 1058 dev_config_vf = 1; 1059 break; 1060 default: 1061 dev_config_vf = 0; 1062 break; 1063 } 1064 /* Default configuration. */ 1065 memset(&dev_config, 0, sizeof(struct mlx5_dev_config)); 1066 dev_config.vf = dev_config_vf; 1067 dev_config.mps = 0; 1068 dev_config.dbnc = MLX5_ARG_UNSET; 1069 dev_config.rx_vec_en = 1; 1070 dev_config.txq_inline_max = MLX5_ARG_UNSET; 1071 dev_config.txq_inline_min = MLX5_ARG_UNSET; 1072 dev_config.txq_inline_mpw = MLX5_ARG_UNSET; 1073 dev_config.txqs_inline = MLX5_ARG_UNSET; 1074 dev_config.vf_nl_en = 0; 1075 dev_config.mr_ext_memseg_en = 1; 1076 dev_config.mprq.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN; 1077 dev_config.mprq.min_rxqs_num = MLX5_MPRQ_MIN_RXQS; 1078 dev_config.dv_esw_en = 0; 1079 dev_config.dv_flow_en = 1; 1080 dev_config.decap_en = 0; 1081 dev_config.log_hp_size = MLX5_ARG_UNSET; 1082 list[ns].eth_dev = mlx5_dev_spawn(&pci_dev->device, 1083 &list[ns], 1084 &dev_config); 1085 if (!list[ns].eth_dev) 1086 goto exit; 1087 restore = list[ns].eth_dev->data->dev_flags; 1088 rte_eth_copy_pci_info(list[ns].eth_dev, pci_dev); 1089 /* Restore non-PCI flags cleared by the above call. */ 1090 list[ns].eth_dev->data->dev_flags |= restore; 1091 rte_eth_dev_probing_finish(list[ns].eth_dev); 1092 ret = 0; 1093 exit: 1094 /* 1095 * Do the routine cleanup: 1096 * - free allocated spawn data array 1097 * - free the device list 1098 */ 1099 if (list) 1100 mlx5_free(list); 1101 MLX5_ASSERT(orig_devx_bdf_devs); 1102 mlx5_glue->free_device_list(orig_devx_bdf_devs); 1103 return ret; 1104 } 1105 1106 /** 1107 * Set the reg_mr and dereg_mr call backs 1108 * 1109 * @param reg_mr_cb[out] 1110 * Pointer to reg_mr func 1111 * @param dereg_mr_cb[out] 1112 * Pointer to dereg_mr func 1113 * 1114 */ 1115 void 1116 mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, 1117 mlx5_dereg_mr_t *dereg_mr_cb) 1118 { 1119 *reg_mr_cb = mlx5_os_reg_mr; 1120 *dereg_mr_cb = mlx5_os_dereg_mr; 1121 } 1122 1123 /** 1124 * Extract pdn of PD object using DevX 1125 * 1126 * @param[in] pd 1127 * Pointer to the DevX PD object. 1128 * @param[out] pdn 1129 * Pointer to the PD object number variable. 1130 * 1131 * @return 1132 * 0 on success, error value otherwise. 1133 */ 1134 int 1135 mlx5_os_get_pdn(void *pd, uint32_t *pdn) 1136 { 1137 if (!pd) 1138 return -EINVAL; 1139 1140 *pdn = ((struct mlx5_pd *)pd)->pdn; 1141 return 0; 1142 } 1143 1144 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {0}; 1145