1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2015 6WIND S.A. 5 * Copyright 2015 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stddef.h> 35 #include <assert.h> 36 #include <unistd.h> 37 #include <stdint.h> 38 #include <stdio.h> 39 #include <string.h> 40 #include <stdlib.h> 41 #include <errno.h> 42 #include <dirent.h> 43 #include <net/if.h> 44 #include <sys/ioctl.h> 45 #include <sys/socket.h> 46 #include <sys/utsname.h> 47 #include <netinet/in.h> 48 #include <linux/ethtool.h> 49 #include <linux/sockios.h> 50 #include <linux/version.h> 51 #include <fcntl.h> 52 #include <stdalign.h> 53 54 #include <rte_atomic.h> 55 #include <rte_ethdev.h> 56 #include <rte_mbuf.h> 57 #include <rte_common.h> 58 #include <rte_interrupts.h> 59 #include <rte_alarm.h> 60 #include <rte_malloc.h> 61 62 #include "mlx5.h" 63 #include "mlx5_rxtx.h" 64 #include "mlx5_utils.h" 65 66 /* Add defines in case the running kernel is not the same as user headers. */ 67 #ifndef ETHTOOL_GLINKSETTINGS 68 struct ethtool_link_settings { 69 uint32_t cmd; 70 uint32_t speed; 71 uint8_t duplex; 72 uint8_t port; 73 uint8_t phy_address; 74 uint8_t autoneg; 75 uint8_t mdio_support; 76 uint8_t eth_to_mdix; 77 uint8_t eth_tp_mdix_ctrl; 78 int8_t link_mode_masks_nwords; 79 uint32_t reserved[8]; 80 uint32_t link_mode_masks[]; 81 }; 82 83 #define ETHTOOL_GLINKSETTINGS 0x0000004c 84 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5 85 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6 86 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17 87 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18 88 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19 89 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20 90 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21 91 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22 92 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23 93 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24 94 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25 95 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26 96 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27 97 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28 98 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29 99 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30 100 #endif 101 #ifndef HAVE_ETHTOOL_LINK_MODE_25G 102 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31 103 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32 104 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33 105 #endif 106 #ifndef HAVE_ETHTOOL_LINK_MODE_50G 107 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34 108 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35 109 #endif 110 #ifndef HAVE_ETHTOOL_LINK_MODE_100G 111 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36 112 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37 113 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38 114 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39 115 #endif 116 117 /** 118 * Return private structure associated with an Ethernet device. 119 * 120 * @param dev 121 * Pointer to Ethernet device structure. 122 * 123 * @return 124 * Pointer to private structure. 125 */ 126 struct priv * 127 mlx5_get_priv(struct rte_eth_dev *dev) 128 { 129 return dev->data->dev_private; 130 } 131 132 /** 133 * Check if running as a secondary process. 134 * 135 * @return 136 * Nonzero if running as a secondary process. 137 */ 138 inline int 139 mlx5_is_secondary(void) 140 { 141 return rte_eal_process_type() == RTE_PROC_SECONDARY; 142 } 143 144 /** 145 * Get interface name from private structure. 146 * 147 * @param[in] priv 148 * Pointer to private structure. 149 * @param[out] ifname 150 * Interface name output buffer. 151 * 152 * @return 153 * 0 on success, -1 on failure and errno is set. 154 */ 155 int 156 priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) 157 { 158 DIR *dir; 159 struct dirent *dent; 160 unsigned int dev_type = 0; 161 unsigned int dev_port_prev = ~0u; 162 char match[IF_NAMESIZE] = ""; 163 164 { 165 MKSTR(path, "%s/device/net", priv->ctx->device->ibdev_path); 166 167 dir = opendir(path); 168 if (dir == NULL) 169 return -1; 170 } 171 while ((dent = readdir(dir)) != NULL) { 172 char *name = dent->d_name; 173 FILE *file; 174 unsigned int dev_port; 175 int r; 176 177 if ((name[0] == '.') && 178 ((name[1] == '\0') || 179 ((name[1] == '.') && (name[2] == '\0')))) 180 continue; 181 182 MKSTR(path, "%s/device/net/%s/%s", 183 priv->ctx->device->ibdev_path, name, 184 (dev_type ? "dev_id" : "dev_port")); 185 186 file = fopen(path, "rb"); 187 if (file == NULL) { 188 if (errno != ENOENT) 189 continue; 190 /* 191 * Switch to dev_id when dev_port does not exist as 192 * is the case with Linux kernel versions < 3.15. 193 */ 194 try_dev_id: 195 match[0] = '\0'; 196 if (dev_type) 197 break; 198 dev_type = 1; 199 dev_port_prev = ~0u; 200 rewinddir(dir); 201 continue; 202 } 203 r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port); 204 fclose(file); 205 if (r != 1) 206 continue; 207 /* 208 * Switch to dev_id when dev_port returns the same value for 209 * all ports. May happen when using a MOFED release older than 210 * 3.0 with a Linux kernel >= 3.15. 211 */ 212 if (dev_port == dev_port_prev) 213 goto try_dev_id; 214 dev_port_prev = dev_port; 215 if (dev_port == (priv->port - 1u)) 216 snprintf(match, sizeof(match), "%s", name); 217 } 218 closedir(dir); 219 if (match[0] == '\0') 220 return -1; 221 strncpy(*ifname, match, sizeof(*ifname)); 222 return 0; 223 } 224 225 /** 226 * Check if the counter is located on ib counters file. 227 * 228 * @param[in] cntr 229 * Counter name. 230 * 231 * @return 232 * 1 if counter is located on ib counters file , 0 otherwise. 233 */ 234 int 235 priv_is_ib_cntr(const char *cntr) 236 { 237 if (!strcmp(cntr, "out_of_buffer")) 238 return 1; 239 return 0; 240 } 241 242 /** 243 * Read from sysfs entry. 244 * 245 * @param[in] priv 246 * Pointer to private structure. 247 * @param[in] entry 248 * Entry name relative to sysfs path. 249 * @param[out] buf 250 * Data output buffer. 251 * @param size 252 * Buffer size. 253 * 254 * @return 255 * 0 on success, -1 on failure and errno is set. 256 */ 257 static int 258 priv_sysfs_read(const struct priv *priv, const char *entry, 259 char *buf, size_t size) 260 { 261 char ifname[IF_NAMESIZE]; 262 FILE *file; 263 int ret; 264 int err; 265 266 if (priv_get_ifname(priv, &ifname)) 267 return -1; 268 269 if (priv_is_ib_cntr(entry)) { 270 MKSTR(path, "%s/ports/1/hw_counters/%s", 271 priv->ctx->device->ibdev_path, entry); 272 file = fopen(path, "rb"); 273 } else { 274 MKSTR(path, "%s/device/net/%s/%s", 275 priv->ctx->device->ibdev_path, ifname, entry); 276 file = fopen(path, "rb"); 277 } 278 if (file == NULL) 279 return -1; 280 ret = fread(buf, 1, size, file); 281 err = errno; 282 if (((size_t)ret < size) && (ferror(file))) 283 ret = -1; 284 else 285 ret = size; 286 fclose(file); 287 errno = err; 288 return ret; 289 } 290 291 /** 292 * Write to sysfs entry. 293 * 294 * @param[in] priv 295 * Pointer to private structure. 296 * @param[in] entry 297 * Entry name relative to sysfs path. 298 * @param[in] buf 299 * Data buffer. 300 * @param size 301 * Buffer size. 302 * 303 * @return 304 * 0 on success, -1 on failure and errno is set. 305 */ 306 static int 307 priv_sysfs_write(const struct priv *priv, const char *entry, 308 char *buf, size_t size) 309 { 310 char ifname[IF_NAMESIZE]; 311 FILE *file; 312 int ret; 313 int err; 314 315 if (priv_get_ifname(priv, &ifname)) 316 return -1; 317 318 MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, 319 ifname, entry); 320 321 file = fopen(path, "wb"); 322 if (file == NULL) 323 return -1; 324 ret = fwrite(buf, 1, size, file); 325 err = errno; 326 if (((size_t)ret < size) || (ferror(file))) 327 ret = -1; 328 else 329 ret = size; 330 fclose(file); 331 errno = err; 332 return ret; 333 } 334 335 /** 336 * Get unsigned long sysfs property. 337 * 338 * @param priv 339 * Pointer to private structure. 340 * @param[in] name 341 * Entry name relative to sysfs path. 342 * @param[out] value 343 * Value output buffer. 344 * 345 * @return 346 * 0 on success, -1 on failure and errno is set. 347 */ 348 static int 349 priv_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) 350 { 351 int ret; 352 unsigned long value_ret; 353 char value_str[32]; 354 355 ret = priv_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); 356 if (ret == -1) { 357 DEBUG("cannot read %s value from sysfs: %s", 358 name, strerror(errno)); 359 return -1; 360 } 361 value_str[ret] = '\0'; 362 errno = 0; 363 value_ret = strtoul(value_str, NULL, 0); 364 if (errno) { 365 DEBUG("invalid %s value `%s': %s", name, value_str, 366 strerror(errno)); 367 return -1; 368 } 369 *value = value_ret; 370 return 0; 371 } 372 373 /** 374 * Set unsigned long sysfs property. 375 * 376 * @param priv 377 * Pointer to private structure. 378 * @param[in] name 379 * Entry name relative to sysfs path. 380 * @param value 381 * Value to set. 382 * 383 * @return 384 * 0 on success, -1 on failure and errno is set. 385 */ 386 static int 387 priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) 388 { 389 int ret; 390 MKSTR(value_str, "%lu", value); 391 392 ret = priv_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); 393 if (ret == -1) { 394 DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", 395 name, value_str, value, strerror(errno)); 396 return -1; 397 } 398 return 0; 399 } 400 401 /** 402 * Perform ifreq ioctl() on associated Ethernet device. 403 * 404 * @param[in] priv 405 * Pointer to private structure. 406 * @param req 407 * Request number to pass to ioctl(). 408 * @param[out] ifr 409 * Interface request structure output buffer. 410 * 411 * @return 412 * 0 on success, -1 on failure and errno is set. 413 */ 414 int 415 priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) 416 { 417 int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); 418 int ret = -1; 419 420 if (sock == -1) 421 return ret; 422 if (priv_get_ifname(priv, &ifr->ifr_name) == 0) 423 ret = ioctl(sock, req, ifr); 424 close(sock); 425 return ret; 426 } 427 428 /** 429 * Return the number of active VFs for the current device. 430 * 431 * @param[in] priv 432 * Pointer to private structure. 433 * @param[out] num_vfs 434 * Number of active VFs. 435 * 436 * @return 437 * 0 on success, -1 on failure and errno is set. 438 */ 439 int 440 priv_get_num_vfs(struct priv *priv, uint16_t *num_vfs) 441 { 442 /* The sysfs entry name depends on the operating system. */ 443 const char **name = (const char *[]){ 444 "device/sriov_numvfs", 445 "device/mlx5_num_vfs", 446 NULL, 447 }; 448 int ret; 449 450 do { 451 unsigned long ulong_num_vfs; 452 453 ret = priv_get_sysfs_ulong(priv, *name, &ulong_num_vfs); 454 if (!ret) 455 *num_vfs = ulong_num_vfs; 456 } while (*(++name) && ret); 457 return ret; 458 } 459 460 /** 461 * Get device MTU. 462 * 463 * @param priv 464 * Pointer to private structure. 465 * @param[out] mtu 466 * MTU value output buffer. 467 * 468 * @return 469 * 0 on success, -1 on failure and errno is set. 470 */ 471 int 472 priv_get_mtu(struct priv *priv, uint16_t *mtu) 473 { 474 unsigned long ulong_mtu; 475 476 if (priv_get_sysfs_ulong(priv, "mtu", &ulong_mtu) == -1) 477 return -1; 478 *mtu = ulong_mtu; 479 return 0; 480 } 481 482 /** 483 * Read device counter from sysfs. 484 * 485 * @param priv 486 * Pointer to private structure. 487 * @param name 488 * Counter name. 489 * @param[out] cntr 490 * Counter output buffer. 491 * 492 * @return 493 * 0 on success, -1 on failure and errno is set. 494 */ 495 int 496 priv_get_cntr_sysfs(struct priv *priv, const char *name, uint64_t *cntr) 497 { 498 unsigned long ulong_ctr; 499 500 if (priv_get_sysfs_ulong(priv, name, &ulong_ctr) == -1) 501 return -1; 502 *cntr = ulong_ctr; 503 return 0; 504 } 505 506 /** 507 * Set device MTU. 508 * 509 * @param priv 510 * Pointer to private structure. 511 * @param mtu 512 * MTU value to set. 513 * 514 * @return 515 * 0 on success, -1 on failure and errno is set. 516 */ 517 static int 518 priv_set_mtu(struct priv *priv, uint16_t mtu) 519 { 520 uint16_t new_mtu; 521 522 if (priv_set_sysfs_ulong(priv, "mtu", mtu) || 523 priv_get_mtu(priv, &new_mtu)) 524 return -1; 525 if (new_mtu == mtu) 526 return 0; 527 errno = EINVAL; 528 return -1; 529 } 530 531 /** 532 * Set device flags. 533 * 534 * @param priv 535 * Pointer to private structure. 536 * @param keep 537 * Bitmask for flags that must remain untouched. 538 * @param flags 539 * Bitmask for flags to modify. 540 * 541 * @return 542 * 0 on success, -1 on failure and errno is set. 543 */ 544 int 545 priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) 546 { 547 unsigned long tmp; 548 549 if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1) 550 return -1; 551 tmp &= keep; 552 tmp |= (flags & (~keep)); 553 return priv_set_sysfs_ulong(priv, "flags", tmp); 554 } 555 556 /** 557 * Ethernet device configuration. 558 * 559 * Prepare the driver for a given number of TX and RX queues. 560 * 561 * @param dev 562 * Pointer to Ethernet device structure. 563 * 564 * @return 565 * 0 on success, errno value on failure. 566 */ 567 static int 568 dev_configure(struct rte_eth_dev *dev) 569 { 570 struct priv *priv = dev->data->dev_private; 571 unsigned int rxqs_n = dev->data->nb_rx_queues; 572 unsigned int txqs_n = dev->data->nb_tx_queues; 573 unsigned int i; 574 unsigned int j; 575 unsigned int reta_idx_n; 576 577 priv->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; 578 priv->rxqs = (void *)dev->data->rx_queues; 579 priv->txqs = (void *)dev->data->tx_queues; 580 if (txqs_n != priv->txqs_n) { 581 INFO("%p: TX queues number update: %u -> %u", 582 (void *)dev, priv->txqs_n, txqs_n); 583 priv->txqs_n = txqs_n; 584 } 585 if (rxqs_n > priv->ind_table_max_size) { 586 ERROR("cannot handle this many RX queues (%u)", rxqs_n); 587 return EINVAL; 588 } 589 if (rxqs_n == priv->rxqs_n) 590 return 0; 591 INFO("%p: RX queues number update: %u -> %u", 592 (void *)dev, priv->rxqs_n, rxqs_n); 593 priv->rxqs_n = rxqs_n; 594 /* If the requested number of RX queues is not a power of two, use the 595 * maximum indirection table size for better balancing. 596 * The result is always rounded to the next power of two. */ 597 reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ? 598 priv->ind_table_max_size : 599 rxqs_n)); 600 if (priv_rss_reta_index_resize(priv, reta_idx_n)) 601 return ENOMEM; 602 /* When the number of RX queues is not a power of two, the remaining 603 * table entries are padded with reused WQs and hashes are not spread 604 * uniformly. */ 605 for (i = 0, j = 0; (i != reta_idx_n); ++i) { 606 (*priv->reta_idx)[i] = j; 607 if (++j == rxqs_n) 608 j = 0; 609 } 610 return 0; 611 } 612 613 /** 614 * DPDK callback for Ethernet device configuration. 615 * 616 * @param dev 617 * Pointer to Ethernet device structure. 618 * 619 * @return 620 * 0 on success, negative errno value on failure. 621 */ 622 int 623 mlx5_dev_configure(struct rte_eth_dev *dev) 624 { 625 struct priv *priv = dev->data->dev_private; 626 int ret; 627 628 if (mlx5_is_secondary()) 629 return -E_RTE_SECONDARY; 630 631 priv_lock(priv); 632 ret = dev_configure(dev); 633 assert(ret >= 0); 634 priv_unlock(priv); 635 return -ret; 636 } 637 638 /** 639 * DPDK callback to get information about the device. 640 * 641 * @param dev 642 * Pointer to Ethernet device structure. 643 * @param[out] info 644 * Info structure output buffer. 645 */ 646 void 647 mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) 648 { 649 struct priv *priv = mlx5_get_priv(dev); 650 unsigned int max; 651 char ifname[IF_NAMESIZE]; 652 653 info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 654 655 priv_lock(priv); 656 /* FIXME: we should ask the device for these values. */ 657 info->min_rx_bufsize = 32; 658 info->max_rx_pktlen = 65536; 659 /* 660 * Since we need one CQ per QP, the limit is the minimum number 661 * between the two values. 662 */ 663 max = ((priv->device_attr.max_cq > priv->device_attr.max_qp) ? 664 priv->device_attr.max_qp : priv->device_attr.max_cq); 665 /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */ 666 if (max >= 65535) 667 max = 65535; 668 info->max_rx_queues = max; 669 info->max_tx_queues = max; 670 info->max_mac_addrs = RTE_DIM(priv->mac); 671 info->rx_offload_capa = 672 (priv->hw_csum ? 673 (DEV_RX_OFFLOAD_IPV4_CKSUM | 674 DEV_RX_OFFLOAD_UDP_CKSUM | 675 DEV_RX_OFFLOAD_TCP_CKSUM) : 676 0) | 677 (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0); 678 if (!priv->mps) 679 info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; 680 if (priv->hw_csum) 681 info->tx_offload_capa |= 682 (DEV_TX_OFFLOAD_IPV4_CKSUM | 683 DEV_TX_OFFLOAD_UDP_CKSUM | 684 DEV_TX_OFFLOAD_TCP_CKSUM); 685 if (priv->tso) 686 info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 687 if (priv->tunnel_en) 688 info->tx_offload_capa |= (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 689 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 690 DEV_TX_OFFLOAD_GRE_TNL_TSO); 691 if (priv_get_ifname(priv, &ifname) == 0) 692 info->if_index = if_nametoindex(ifname); 693 info->reta_size = priv->reta_idx_n ? 694 priv->reta_idx_n : priv->ind_table_max_size; 695 info->hash_key_size = ((*priv->rss_conf) ? 696 (*priv->rss_conf)[0]->rss_key_len : 697 0); 698 info->speed_capa = priv->link_speed_capa; 699 priv_unlock(priv); 700 } 701 702 const uint32_t * 703 mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) 704 { 705 static const uint32_t ptypes[] = { 706 /* refers to rxq_cq_to_pkt_type() */ 707 RTE_PTYPE_L2_ETHER, 708 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 709 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 710 RTE_PTYPE_L4_NONFRAG, 711 RTE_PTYPE_L4_FRAG, 712 RTE_PTYPE_L4_TCP, 713 RTE_PTYPE_L4_UDP, 714 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 715 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 716 RTE_PTYPE_INNER_L4_NONFRAG, 717 RTE_PTYPE_INNER_L4_FRAG, 718 RTE_PTYPE_INNER_L4_TCP, 719 RTE_PTYPE_INNER_L4_UDP, 720 RTE_PTYPE_UNKNOWN 721 }; 722 723 if (dev->rx_pkt_burst == mlx5_rx_burst || 724 dev->rx_pkt_burst == mlx5_rx_burst_vec) 725 return ptypes; 726 return NULL; 727 } 728 729 /** 730 * DPDK callback to retrieve physical link information. 731 * 732 * @param dev 733 * Pointer to Ethernet device structure. 734 * @param wait_to_complete 735 * Wait for request completion (ignored). 736 */ 737 static int 738 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) 739 { 740 struct priv *priv = mlx5_get_priv(dev); 741 struct ethtool_cmd edata = { 742 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */ 743 }; 744 struct ifreq ifr; 745 struct rte_eth_link dev_link; 746 int link_speed = 0; 747 748 /* priv_lock() is not taken to allow concurrent calls. */ 749 750 (void)wait_to_complete; 751 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 752 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 753 return -1; 754 } 755 memset(&dev_link, 0, sizeof(dev_link)); 756 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 757 (ifr.ifr_flags & IFF_RUNNING)); 758 ifr.ifr_data = (void *)&edata; 759 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 760 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", 761 strerror(errno)); 762 return -1; 763 } 764 link_speed = ethtool_cmd_speed(&edata); 765 if (link_speed == -1) 766 dev_link.link_speed = 0; 767 else 768 dev_link.link_speed = link_speed; 769 priv->link_speed_capa = 0; 770 if (edata.supported & SUPPORTED_Autoneg) 771 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; 772 if (edata.supported & (SUPPORTED_1000baseT_Full | 773 SUPPORTED_1000baseKX_Full)) 774 priv->link_speed_capa |= ETH_LINK_SPEED_1G; 775 if (edata.supported & SUPPORTED_10000baseKR_Full) 776 priv->link_speed_capa |= ETH_LINK_SPEED_10G; 777 if (edata.supported & (SUPPORTED_40000baseKR4_Full | 778 SUPPORTED_40000baseCR4_Full | 779 SUPPORTED_40000baseSR4_Full | 780 SUPPORTED_40000baseLR4_Full)) 781 priv->link_speed_capa |= ETH_LINK_SPEED_40G; 782 dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? 783 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 784 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 785 ETH_LINK_SPEED_FIXED); 786 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 787 /* Link status changed. */ 788 dev->data->dev_link = dev_link; 789 return 0; 790 } 791 /* Link status is still the same. */ 792 return -1; 793 } 794 795 /** 796 * Retrieve physical link information (unlocked version using new ioctl). 797 * 798 * @param dev 799 * Pointer to Ethernet device structure. 800 * @param wait_to_complete 801 * Wait for request completion (ignored). 802 */ 803 static int 804 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) 805 { 806 struct priv *priv = mlx5_get_priv(dev); 807 struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS }; 808 struct ifreq ifr; 809 struct rte_eth_link dev_link; 810 uint64_t sc; 811 812 (void)wait_to_complete; 813 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 814 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 815 return -1; 816 } 817 memset(&dev_link, 0, sizeof(dev_link)); 818 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 819 (ifr.ifr_flags & IFF_RUNNING)); 820 ifr.ifr_data = (void *)&gcmd; 821 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 822 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", 823 strerror(errno)); 824 return -1; 825 } 826 gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords; 827 828 alignas(struct ethtool_link_settings) 829 uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) + 830 sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3]; 831 struct ethtool_link_settings *ecmd = (void *)data; 832 833 *ecmd = gcmd; 834 ifr.ifr_data = (void *)ecmd; 835 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 836 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", 837 strerror(errno)); 838 return -1; 839 } 840 dev_link.link_speed = ecmd->speed; 841 sc = ecmd->link_mode_masks[0] | 842 ((uint64_t)ecmd->link_mode_masks[1] << 32); 843 priv->link_speed_capa = 0; 844 if (sc & ETHTOOL_LINK_MODE_Autoneg_BIT) 845 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; 846 if (sc & (ETHTOOL_LINK_MODE_1000baseT_Full_BIT | 847 ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)) 848 priv->link_speed_capa |= ETH_LINK_SPEED_1G; 849 if (sc & (ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT | 850 ETHTOOL_LINK_MODE_10000baseKR_Full_BIT | 851 ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)) 852 priv->link_speed_capa |= ETH_LINK_SPEED_10G; 853 if (sc & (ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT | 854 ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)) 855 priv->link_speed_capa |= ETH_LINK_SPEED_20G; 856 if (sc & (ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT | 857 ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT | 858 ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT | 859 ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)) 860 priv->link_speed_capa |= ETH_LINK_SPEED_40G; 861 if (sc & (ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT | 862 ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT | 863 ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT | 864 ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)) 865 priv->link_speed_capa |= ETH_LINK_SPEED_56G; 866 if (sc & (ETHTOOL_LINK_MODE_25000baseCR_Full_BIT | 867 ETHTOOL_LINK_MODE_25000baseKR_Full_BIT | 868 ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)) 869 priv->link_speed_capa |= ETH_LINK_SPEED_25G; 870 if (sc & (ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT | 871 ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)) 872 priv->link_speed_capa |= ETH_LINK_SPEED_50G; 873 if (sc & (ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT | 874 ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT | 875 ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT | 876 ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)) 877 priv->link_speed_capa |= ETH_LINK_SPEED_100G; 878 dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ? 879 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 880 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 881 ETH_LINK_SPEED_FIXED); 882 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 883 /* Link status changed. */ 884 dev->data->dev_link = dev_link; 885 return 0; 886 } 887 /* Link status is still the same. */ 888 return -1; 889 } 890 891 /** 892 * DPDK callback to retrieve physical link information. 893 * 894 * @param dev 895 * Pointer to Ethernet device structure. 896 * @param wait_to_complete 897 * Wait for request completion (ignored). 898 */ 899 int 900 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) 901 { 902 struct utsname utsname; 903 int ver[3]; 904 905 if (uname(&utsname) == -1 || 906 sscanf(utsname.release, "%d.%d.%d", 907 &ver[0], &ver[1], &ver[2]) != 3 || 908 KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0)) 909 return mlx5_link_update_unlocked_gset(dev, wait_to_complete); 910 return mlx5_link_update_unlocked_gs(dev, wait_to_complete); 911 } 912 913 /** 914 * DPDK callback to change the MTU. 915 * 916 * @param dev 917 * Pointer to Ethernet device structure. 918 * @param in_mtu 919 * New MTU. 920 * 921 * @return 922 * 0 on success, negative errno value on failure. 923 */ 924 int 925 mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 926 { 927 struct priv *priv = dev->data->dev_private; 928 uint16_t kern_mtu; 929 int ret = 0; 930 931 if (mlx5_is_secondary()) 932 return -E_RTE_SECONDARY; 933 934 priv_lock(priv); 935 ret = priv_get_mtu(priv, &kern_mtu); 936 if (ret) 937 goto out; 938 /* Set kernel interface MTU first. */ 939 ret = priv_set_mtu(priv, mtu); 940 if (ret) 941 goto out; 942 ret = priv_get_mtu(priv, &kern_mtu); 943 if (ret) 944 goto out; 945 if (kern_mtu == mtu) { 946 priv->mtu = mtu; 947 DEBUG("adapter port %u MTU set to %u", priv->port, mtu); 948 } 949 priv_unlock(priv); 950 return 0; 951 out: 952 ret = errno; 953 WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, 954 strerror(ret)); 955 priv_unlock(priv); 956 assert(ret >= 0); 957 return -ret; 958 } 959 960 /** 961 * DPDK callback to get flow control status. 962 * 963 * @param dev 964 * Pointer to Ethernet device structure. 965 * @param[out] fc_conf 966 * Flow control output buffer. 967 * 968 * @return 969 * 0 on success, negative errno value on failure. 970 */ 971 int 972 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 973 { 974 struct priv *priv = dev->data->dev_private; 975 struct ifreq ifr; 976 struct ethtool_pauseparam ethpause = { 977 .cmd = ETHTOOL_GPAUSEPARAM 978 }; 979 int ret; 980 981 if (mlx5_is_secondary()) 982 return -E_RTE_SECONDARY; 983 984 ifr.ifr_data = (void *)ðpause; 985 priv_lock(priv); 986 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 987 ret = errno; 988 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" 989 " failed: %s", 990 strerror(ret)); 991 goto out; 992 } 993 994 fc_conf->autoneg = ethpause.autoneg; 995 if (ethpause.rx_pause && ethpause.tx_pause) 996 fc_conf->mode = RTE_FC_FULL; 997 else if (ethpause.rx_pause) 998 fc_conf->mode = RTE_FC_RX_PAUSE; 999 else if (ethpause.tx_pause) 1000 fc_conf->mode = RTE_FC_TX_PAUSE; 1001 else 1002 fc_conf->mode = RTE_FC_NONE; 1003 ret = 0; 1004 1005 out: 1006 priv_unlock(priv); 1007 assert(ret >= 0); 1008 return -ret; 1009 } 1010 1011 /** 1012 * DPDK callback to modify flow control parameters. 1013 * 1014 * @param dev 1015 * Pointer to Ethernet device structure. 1016 * @param[in] fc_conf 1017 * Flow control parameters. 1018 * 1019 * @return 1020 * 0 on success, negative errno value on failure. 1021 */ 1022 int 1023 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 1024 { 1025 struct priv *priv = dev->data->dev_private; 1026 struct ifreq ifr; 1027 struct ethtool_pauseparam ethpause = { 1028 .cmd = ETHTOOL_SPAUSEPARAM 1029 }; 1030 int ret; 1031 1032 if (mlx5_is_secondary()) 1033 return -E_RTE_SECONDARY; 1034 1035 ifr.ifr_data = (void *)ðpause; 1036 ethpause.autoneg = fc_conf->autoneg; 1037 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 1038 (fc_conf->mode & RTE_FC_RX_PAUSE)) 1039 ethpause.rx_pause = 1; 1040 else 1041 ethpause.rx_pause = 0; 1042 1043 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 1044 (fc_conf->mode & RTE_FC_TX_PAUSE)) 1045 ethpause.tx_pause = 1; 1046 else 1047 ethpause.tx_pause = 0; 1048 1049 priv_lock(priv); 1050 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 1051 ret = errno; 1052 WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" 1053 " failed: %s", 1054 strerror(ret)); 1055 goto out; 1056 } 1057 ret = 0; 1058 1059 out: 1060 priv_unlock(priv); 1061 assert(ret >= 0); 1062 return -ret; 1063 } 1064 1065 /** 1066 * Get PCI information from struct ibv_device. 1067 * 1068 * @param device 1069 * Pointer to Ethernet device structure. 1070 * @param[out] pci_addr 1071 * PCI bus address output buffer. 1072 * 1073 * @return 1074 * 0 on success, -1 on failure and errno is set. 1075 */ 1076 int 1077 mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, 1078 struct rte_pci_addr *pci_addr) 1079 { 1080 FILE *file; 1081 char line[32]; 1082 MKSTR(path, "%s/device/uevent", device->ibdev_path); 1083 1084 file = fopen(path, "rb"); 1085 if (file == NULL) 1086 return -1; 1087 while (fgets(line, sizeof(line), file) == line) { 1088 size_t len = strlen(line); 1089 int ret; 1090 1091 /* Truncate long lines. */ 1092 if (len == (sizeof(line) - 1)) 1093 while (line[(len - 1)] != '\n') { 1094 ret = fgetc(file); 1095 if (ret == EOF) 1096 break; 1097 line[(len - 1)] = ret; 1098 } 1099 /* Extract information. */ 1100 if (sscanf(line, 1101 "PCI_SLOT_NAME=" 1102 "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", 1103 &pci_addr->domain, 1104 &pci_addr->bus, 1105 &pci_addr->devid, 1106 &pci_addr->function) == 4) { 1107 ret = 0; 1108 break; 1109 } 1110 } 1111 fclose(file); 1112 return 0; 1113 } 1114 1115 /** 1116 * Link status handler. 1117 * 1118 * @param priv 1119 * Pointer to private structure. 1120 * @param dev 1121 * Pointer to the rte_eth_dev structure. 1122 * 1123 * @return 1124 * Nonzero if the callback process can be called immediately. 1125 */ 1126 static int 1127 priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) 1128 { 1129 struct ibv_async_event event; 1130 struct rte_eth_link *link = &dev->data->dev_link; 1131 int ret = 0; 1132 1133 /* Read all message and acknowledge them. */ 1134 for (;;) { 1135 if (ibv_get_async_event(priv->ctx, &event)) 1136 break; 1137 1138 if (event.event_type != IBV_EVENT_PORT_ACTIVE && 1139 event.event_type != IBV_EVENT_PORT_ERR) 1140 DEBUG("event type %d on port %d not handled", 1141 event.event_type, event.element.port_num); 1142 ibv_ack_async_event(&event); 1143 } 1144 mlx5_link_update(dev, 0); 1145 if (((link->link_speed == 0) && link->link_status) || 1146 ((link->link_speed != 0) && !link->link_status)) { 1147 if (!priv->pending_alarm) { 1148 /* Inconsistent status, check again later. */ 1149 priv->pending_alarm = 1; 1150 rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US, 1151 mlx5_dev_link_status_handler, 1152 dev); 1153 } 1154 } else { 1155 ret = 1; 1156 } 1157 return ret; 1158 } 1159 1160 /** 1161 * Handle delayed link status event. 1162 * 1163 * @param arg 1164 * Registered argument. 1165 */ 1166 void 1167 mlx5_dev_link_status_handler(void *arg) 1168 { 1169 struct rte_eth_dev *dev = arg; 1170 struct priv *priv = dev->data->dev_private; 1171 int ret; 1172 1173 priv_lock(priv); 1174 assert(priv->pending_alarm == 1); 1175 priv->pending_alarm = 0; 1176 ret = priv_dev_link_status_handler(priv, dev); 1177 priv_unlock(priv); 1178 if (ret) 1179 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, 1180 NULL); 1181 } 1182 1183 /** 1184 * Handle interrupts from the NIC. 1185 * 1186 * @param[in] intr_handle 1187 * Interrupt handler. 1188 * @param cb_arg 1189 * Callback argument. 1190 */ 1191 void 1192 mlx5_dev_interrupt_handler(void *cb_arg) 1193 { 1194 struct rte_eth_dev *dev = cb_arg; 1195 struct priv *priv = dev->data->dev_private; 1196 int ret; 1197 1198 priv_lock(priv); 1199 ret = priv_dev_link_status_handler(priv, dev); 1200 priv_unlock(priv); 1201 if (ret) 1202 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, 1203 NULL); 1204 } 1205 1206 /** 1207 * Uninstall interrupt handler. 1208 * 1209 * @param priv 1210 * Pointer to private structure. 1211 * @param dev 1212 * Pointer to the rte_eth_dev structure. 1213 */ 1214 void 1215 priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) 1216 { 1217 if (!dev->data->dev_conf.intr_conf.lsc) 1218 return; 1219 rte_intr_callback_unregister(&priv->intr_handle, 1220 mlx5_dev_interrupt_handler, 1221 dev); 1222 if (priv->pending_alarm) 1223 rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev); 1224 priv->pending_alarm = 0; 1225 priv->intr_handle.fd = 0; 1226 priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; 1227 } 1228 1229 /** 1230 * Install interrupt handler. 1231 * 1232 * @param priv 1233 * Pointer to private structure. 1234 * @param dev 1235 * Pointer to the rte_eth_dev structure. 1236 */ 1237 void 1238 priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) 1239 { 1240 int rc, flags; 1241 1242 if (!dev->data->dev_conf.intr_conf.lsc) 1243 return; 1244 assert(priv->ctx->async_fd > 0); 1245 flags = fcntl(priv->ctx->async_fd, F_GETFL); 1246 rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); 1247 if (rc < 0) { 1248 INFO("failed to change file descriptor async event queue"); 1249 dev->data->dev_conf.intr_conf.lsc = 0; 1250 } else { 1251 priv->intr_handle.fd = priv->ctx->async_fd; 1252 priv->intr_handle.type = RTE_INTR_HANDLE_EXT; 1253 rte_intr_callback_register(&priv->intr_handle, 1254 mlx5_dev_interrupt_handler, 1255 dev); 1256 } 1257 } 1258 1259 /** 1260 * Change the link state (UP / DOWN). 1261 * 1262 * @param priv 1263 * Pointer to Ethernet device structure. 1264 * @param up 1265 * Nonzero for link up, otherwise link down. 1266 * 1267 * @return 1268 * 0 on success, errno value on failure. 1269 */ 1270 static int 1271 priv_set_link(struct priv *priv, int up) 1272 { 1273 struct rte_eth_dev *dev = priv->dev; 1274 int err; 1275 1276 if (up) { 1277 err = priv_set_flags(priv, ~IFF_UP, IFF_UP); 1278 if (err) 1279 return err; 1280 priv_select_tx_function(priv); 1281 priv_select_rx_function(priv); 1282 } else { 1283 err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP); 1284 if (err) 1285 return err; 1286 dev->rx_pkt_burst = removed_rx_burst; 1287 dev->tx_pkt_burst = removed_tx_burst; 1288 } 1289 return 0; 1290 } 1291 1292 /** 1293 * DPDK callback to bring the link DOWN. 1294 * 1295 * @param dev 1296 * Pointer to Ethernet device structure. 1297 * 1298 * @return 1299 * 0 on success, errno value on failure. 1300 */ 1301 int 1302 mlx5_set_link_down(struct rte_eth_dev *dev) 1303 { 1304 struct priv *priv = dev->data->dev_private; 1305 int err; 1306 1307 priv_lock(priv); 1308 err = priv_set_link(priv, 0); 1309 priv_unlock(priv); 1310 return err; 1311 } 1312 1313 /** 1314 * DPDK callback to bring the link UP. 1315 * 1316 * @param dev 1317 * Pointer to Ethernet device structure. 1318 * 1319 * @return 1320 * 0 on success, errno value on failure. 1321 */ 1322 int 1323 mlx5_set_link_up(struct rte_eth_dev *dev) 1324 { 1325 struct priv *priv = dev->data->dev_private; 1326 int err; 1327 1328 priv_lock(priv); 1329 err = priv_set_link(priv, 1); 1330 priv_unlock(priv); 1331 return err; 1332 } 1333 1334 /** 1335 * Configure the TX function to use. 1336 * 1337 * @param priv 1338 * Pointer to private structure. 1339 */ 1340 void 1341 priv_select_tx_function(struct priv *priv) 1342 { 1343 priv->dev->tx_pkt_burst = mlx5_tx_burst; 1344 /* Select appropriate TX function. */ 1345 if (priv->mps == MLX5_MPW_ENHANCED) { 1346 if (priv_check_vec_tx_support(priv) > 0) { 1347 if (priv_check_raw_vec_tx_support(priv) > 0) 1348 priv->dev->tx_pkt_burst = mlx5_tx_burst_raw_vec; 1349 else 1350 priv->dev->tx_pkt_burst = mlx5_tx_burst_vec; 1351 DEBUG("selected Enhanced MPW TX vectorized function"); 1352 } else { 1353 priv->dev->tx_pkt_burst = mlx5_tx_burst_empw; 1354 DEBUG("selected Enhanced MPW TX function"); 1355 } 1356 } else if (priv->mps && priv->txq_inline) { 1357 priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; 1358 DEBUG("selected MPW inline TX function"); 1359 } else if (priv->mps) { 1360 priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw; 1361 DEBUG("selected MPW TX function"); 1362 } 1363 } 1364 1365 /** 1366 * Configure the RX function to use. 1367 * 1368 * @param priv 1369 * Pointer to private structure. 1370 */ 1371 void 1372 priv_select_rx_function(struct priv *priv) 1373 { 1374 if (priv_check_vec_rx_support(priv) > 0) { 1375 priv->dev->rx_pkt_burst = mlx5_rx_burst_vec; 1376 DEBUG("selected RX vectorized function"); 1377 } else { 1378 priv->dev->rx_pkt_burst = mlx5_rx_burst; 1379 } 1380 } 1381