1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2015 6WIND S.A. 5 * Copyright 2015 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stddef.h> 35 #include <assert.h> 36 #include <unistd.h> 37 #include <stdint.h> 38 #include <stdio.h> 39 #include <string.h> 40 #include <stdlib.h> 41 #include <errno.h> 42 #include <dirent.h> 43 #include <net/if.h> 44 #include <sys/ioctl.h> 45 #include <sys/socket.h> 46 #include <sys/utsname.h> 47 #include <netinet/in.h> 48 #include <linux/ethtool.h> 49 #include <linux/sockios.h> 50 #include <linux/version.h> 51 #include <fcntl.h> 52 #include <stdalign.h> 53 54 #include <rte_atomic.h> 55 #include <rte_ethdev.h> 56 #include <rte_mbuf.h> 57 #include <rte_common.h> 58 #include <rte_interrupts.h> 59 #include <rte_alarm.h> 60 #include <rte_malloc.h> 61 62 #include "mlx5.h" 63 #include "mlx5_rxtx.h" 64 #include "mlx5_utils.h" 65 66 /* Add defines in case the running kernel is not the same as user headers. */ 67 #ifndef ETHTOOL_GLINKSETTINGS 68 struct ethtool_link_settings { 69 uint32_t cmd; 70 uint32_t speed; 71 uint8_t duplex; 72 uint8_t port; 73 uint8_t phy_address; 74 uint8_t autoneg; 75 uint8_t mdio_support; 76 uint8_t eth_to_mdix; 77 uint8_t eth_tp_mdix_ctrl; 78 int8_t link_mode_masks_nwords; 79 uint32_t reserved[8]; 80 uint32_t link_mode_masks[]; 81 }; 82 83 #define ETHTOOL_GLINKSETTINGS 0x0000004c 84 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5 85 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6 86 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17 87 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18 88 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19 89 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20 90 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21 91 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22 92 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23 93 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24 94 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25 95 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26 96 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27 97 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28 98 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29 99 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30 100 #endif 101 #ifndef HAVE_ETHTOOL_LINK_MODE_25G 102 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31 103 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32 104 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33 105 #endif 106 #ifndef HAVE_ETHTOOL_LINK_MODE_50G 107 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34 108 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35 109 #endif 110 #ifndef HAVE_ETHTOOL_LINK_MODE_100G 111 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36 112 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37 113 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38 114 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39 115 #endif 116 #define ETHTOOL_LINK_MODE_MASK_MAX_KERNEL_NU32 (SCHAR_MAX) 117 118 /** 119 * Return private structure associated with an Ethernet device. 120 * 121 * @param dev 122 * Pointer to Ethernet device structure. 123 * 124 * @return 125 * Pointer to private structure. 126 */ 127 struct priv * 128 mlx5_get_priv(struct rte_eth_dev *dev) 129 { 130 return dev->data->dev_private; 131 } 132 133 /** 134 * Check if running as a secondary process. 135 * 136 * @return 137 * Nonzero if running as a secondary process. 138 */ 139 inline int 140 mlx5_is_secondary(void) 141 { 142 return rte_eal_process_type() == RTE_PROC_SECONDARY; 143 } 144 145 /** 146 * Get interface name from private structure. 147 * 148 * @param[in] priv 149 * Pointer to private structure. 150 * @param[out] ifname 151 * Interface name output buffer. 152 * 153 * @return 154 * 0 on success, -1 on failure and errno is set. 155 */ 156 int 157 priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) 158 { 159 DIR *dir; 160 struct dirent *dent; 161 unsigned int dev_type = 0; 162 unsigned int dev_port_prev = ~0u; 163 char match[IF_NAMESIZE] = ""; 164 165 { 166 MKSTR(path, "%s/device/net", priv->ctx->device->ibdev_path); 167 168 dir = opendir(path); 169 if (dir == NULL) 170 return -1; 171 } 172 while ((dent = readdir(dir)) != NULL) { 173 char *name = dent->d_name; 174 FILE *file; 175 unsigned int dev_port; 176 int r; 177 178 if ((name[0] == '.') && 179 ((name[1] == '\0') || 180 ((name[1] == '.') && (name[2] == '\0')))) 181 continue; 182 183 MKSTR(path, "%s/device/net/%s/%s", 184 priv->ctx->device->ibdev_path, name, 185 (dev_type ? "dev_id" : "dev_port")); 186 187 file = fopen(path, "rb"); 188 if (file == NULL) { 189 if (errno != ENOENT) 190 continue; 191 /* 192 * Switch to dev_id when dev_port does not exist as 193 * is the case with Linux kernel versions < 3.15. 194 */ 195 try_dev_id: 196 match[0] = '\0'; 197 if (dev_type) 198 break; 199 dev_type = 1; 200 dev_port_prev = ~0u; 201 rewinddir(dir); 202 continue; 203 } 204 r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port); 205 fclose(file); 206 if (r != 1) 207 continue; 208 /* 209 * Switch to dev_id when dev_port returns the same value for 210 * all ports. May happen when using a MOFED release older than 211 * 3.0 with a Linux kernel >= 3.15. 212 */ 213 if (dev_port == dev_port_prev) 214 goto try_dev_id; 215 dev_port_prev = dev_port; 216 if (dev_port == (priv->port - 1u)) 217 snprintf(match, sizeof(match), "%s", name); 218 } 219 closedir(dir); 220 if (match[0] == '\0') 221 return -1; 222 strncpy(*ifname, match, sizeof(*ifname)); 223 return 0; 224 } 225 226 /** 227 * Check if the counter is located on ib counters file. 228 * 229 * @param[in] cntr 230 * Counter name. 231 * 232 * @return 233 * 1 if counter is located on ib counters file , 0 otherwise. 234 */ 235 int 236 priv_is_ib_cntr(const char *cntr) 237 { 238 if (!strcmp(cntr, "out_of_buffer")) 239 return 1; 240 return 0; 241 } 242 243 /** 244 * Read from sysfs entry. 245 * 246 * @param[in] priv 247 * Pointer to private structure. 248 * @param[in] entry 249 * Entry name relative to sysfs path. 250 * @param[out] buf 251 * Data output buffer. 252 * @param size 253 * Buffer size. 254 * 255 * @return 256 * 0 on success, -1 on failure and errno is set. 257 */ 258 static int 259 priv_sysfs_read(const struct priv *priv, const char *entry, 260 char *buf, size_t size) 261 { 262 char ifname[IF_NAMESIZE]; 263 FILE *file; 264 int ret; 265 int err; 266 267 if (priv_get_ifname(priv, &ifname)) 268 return -1; 269 270 if (priv_is_ib_cntr(entry)) { 271 MKSTR(path, "%s/ports/1/hw_counters/%s", 272 priv->ctx->device->ibdev_path, entry); 273 file = fopen(path, "rb"); 274 } else { 275 MKSTR(path, "%s/device/net/%s/%s", 276 priv->ctx->device->ibdev_path, ifname, entry); 277 file = fopen(path, "rb"); 278 } 279 if (file == NULL) 280 return -1; 281 ret = fread(buf, 1, size, file); 282 err = errno; 283 if (((size_t)ret < size) && (ferror(file))) 284 ret = -1; 285 else 286 ret = size; 287 fclose(file); 288 errno = err; 289 return ret; 290 } 291 292 /** 293 * Write to sysfs entry. 294 * 295 * @param[in] priv 296 * Pointer to private structure. 297 * @param[in] entry 298 * Entry name relative to sysfs path. 299 * @param[in] buf 300 * Data buffer. 301 * @param size 302 * Buffer size. 303 * 304 * @return 305 * 0 on success, -1 on failure and errno is set. 306 */ 307 static int 308 priv_sysfs_write(const struct priv *priv, const char *entry, 309 char *buf, size_t size) 310 { 311 char ifname[IF_NAMESIZE]; 312 FILE *file; 313 int ret; 314 int err; 315 316 if (priv_get_ifname(priv, &ifname)) 317 return -1; 318 319 MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, 320 ifname, entry); 321 322 file = fopen(path, "wb"); 323 if (file == NULL) 324 return -1; 325 ret = fwrite(buf, 1, size, file); 326 err = errno; 327 if (((size_t)ret < size) || (ferror(file))) 328 ret = -1; 329 else 330 ret = size; 331 fclose(file); 332 errno = err; 333 return ret; 334 } 335 336 /** 337 * Get unsigned long sysfs property. 338 * 339 * @param priv 340 * Pointer to private structure. 341 * @param[in] name 342 * Entry name relative to sysfs path. 343 * @param[out] value 344 * Value output buffer. 345 * 346 * @return 347 * 0 on success, -1 on failure and errno is set. 348 */ 349 static int 350 priv_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) 351 { 352 int ret; 353 unsigned long value_ret; 354 char value_str[32]; 355 356 ret = priv_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); 357 if (ret == -1) { 358 DEBUG("cannot read %s value from sysfs: %s", 359 name, strerror(errno)); 360 return -1; 361 } 362 value_str[ret] = '\0'; 363 errno = 0; 364 value_ret = strtoul(value_str, NULL, 0); 365 if (errno) { 366 DEBUG("invalid %s value `%s': %s", name, value_str, 367 strerror(errno)); 368 return -1; 369 } 370 *value = value_ret; 371 return 0; 372 } 373 374 /** 375 * Set unsigned long sysfs property. 376 * 377 * @param priv 378 * Pointer to private structure. 379 * @param[in] name 380 * Entry name relative to sysfs path. 381 * @param value 382 * Value to set. 383 * 384 * @return 385 * 0 on success, -1 on failure and errno is set. 386 */ 387 static int 388 priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) 389 { 390 int ret; 391 MKSTR(value_str, "%lu", value); 392 393 ret = priv_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); 394 if (ret == -1) { 395 DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", 396 name, value_str, value, strerror(errno)); 397 return -1; 398 } 399 return 0; 400 } 401 402 /** 403 * Perform ifreq ioctl() on associated Ethernet device. 404 * 405 * @param[in] priv 406 * Pointer to private structure. 407 * @param req 408 * Request number to pass to ioctl(). 409 * @param[out] ifr 410 * Interface request structure output buffer. 411 * 412 * @return 413 * 0 on success, -1 on failure and errno is set. 414 */ 415 int 416 priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) 417 { 418 int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); 419 int ret = -1; 420 421 if (sock == -1) 422 return ret; 423 if (priv_get_ifname(priv, &ifr->ifr_name) == 0) 424 ret = ioctl(sock, req, ifr); 425 close(sock); 426 return ret; 427 } 428 429 /** 430 * Return the number of active VFs for the current device. 431 * 432 * @param[in] priv 433 * Pointer to private structure. 434 * @param[out] num_vfs 435 * Number of active VFs. 436 * 437 * @return 438 * 0 on success, -1 on failure and errno is set. 439 */ 440 int 441 priv_get_num_vfs(struct priv *priv, uint16_t *num_vfs) 442 { 443 /* The sysfs entry name depends on the operating system. */ 444 const char **name = (const char *[]){ 445 "device/sriov_numvfs", 446 "device/mlx5_num_vfs", 447 NULL, 448 }; 449 int ret; 450 451 do { 452 unsigned long ulong_num_vfs; 453 454 ret = priv_get_sysfs_ulong(priv, *name, &ulong_num_vfs); 455 if (!ret) 456 *num_vfs = ulong_num_vfs; 457 } while (*(++name) && ret); 458 return ret; 459 } 460 461 /** 462 * Get device MTU. 463 * 464 * @param priv 465 * Pointer to private structure. 466 * @param[out] mtu 467 * MTU value output buffer. 468 * 469 * @return 470 * 0 on success, -1 on failure and errno is set. 471 */ 472 int 473 priv_get_mtu(struct priv *priv, uint16_t *mtu) 474 { 475 unsigned long ulong_mtu; 476 477 if (priv_get_sysfs_ulong(priv, "mtu", &ulong_mtu) == -1) 478 return -1; 479 *mtu = ulong_mtu; 480 return 0; 481 } 482 483 /** 484 * Read device counter from sysfs. 485 * 486 * @param priv 487 * Pointer to private structure. 488 * @param name 489 * Counter name. 490 * @param[out] cntr 491 * Counter output buffer. 492 * 493 * @return 494 * 0 on success, -1 on failure and errno is set. 495 */ 496 int 497 priv_get_cntr_sysfs(struct priv *priv, const char *name, uint64_t *cntr) 498 { 499 unsigned long ulong_ctr; 500 501 if (priv_get_sysfs_ulong(priv, name, &ulong_ctr) == -1) 502 return -1; 503 *cntr = ulong_ctr; 504 return 0; 505 } 506 507 /** 508 * Set device MTU. 509 * 510 * @param priv 511 * Pointer to private structure. 512 * @param mtu 513 * MTU value to set. 514 * 515 * @return 516 * 0 on success, -1 on failure and errno is set. 517 */ 518 static int 519 priv_set_mtu(struct priv *priv, uint16_t mtu) 520 { 521 uint16_t new_mtu; 522 523 if (priv_set_sysfs_ulong(priv, "mtu", mtu) || 524 priv_get_mtu(priv, &new_mtu)) 525 return -1; 526 if (new_mtu == mtu) 527 return 0; 528 errno = EINVAL; 529 return -1; 530 } 531 532 /** 533 * Set device flags. 534 * 535 * @param priv 536 * Pointer to private structure. 537 * @param keep 538 * Bitmask for flags that must remain untouched. 539 * @param flags 540 * Bitmask for flags to modify. 541 * 542 * @return 543 * 0 on success, -1 on failure and errno is set. 544 */ 545 int 546 priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) 547 { 548 unsigned long tmp; 549 550 if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1) 551 return -1; 552 tmp &= keep; 553 tmp |= (flags & (~keep)); 554 return priv_set_sysfs_ulong(priv, "flags", tmp); 555 } 556 557 /** 558 * Ethernet device configuration. 559 * 560 * Prepare the driver for a given number of TX and RX queues. 561 * 562 * @param dev 563 * Pointer to Ethernet device structure. 564 * 565 * @return 566 * 0 on success, errno value on failure. 567 */ 568 static int 569 dev_configure(struct rte_eth_dev *dev) 570 { 571 struct priv *priv = dev->data->dev_private; 572 unsigned int rxqs_n = dev->data->nb_rx_queues; 573 unsigned int txqs_n = dev->data->nb_tx_queues; 574 unsigned int i; 575 unsigned int j; 576 unsigned int reta_idx_n; 577 578 priv->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; 579 priv->rxqs = (void *)dev->data->rx_queues; 580 priv->txqs = (void *)dev->data->tx_queues; 581 if (txqs_n != priv->txqs_n) { 582 INFO("%p: TX queues number update: %u -> %u", 583 (void *)dev, priv->txqs_n, txqs_n); 584 priv->txqs_n = txqs_n; 585 } 586 if (rxqs_n > priv->ind_table_max_size) { 587 ERROR("cannot handle this many RX queues (%u)", rxqs_n); 588 return EINVAL; 589 } 590 if (rxqs_n == priv->rxqs_n) 591 return 0; 592 INFO("%p: RX queues number update: %u -> %u", 593 (void *)dev, priv->rxqs_n, rxqs_n); 594 priv->rxqs_n = rxqs_n; 595 /* If the requested number of RX queues is not a power of two, use the 596 * maximum indirection table size for better balancing. 597 * The result is always rounded to the next power of two. */ 598 reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ? 599 priv->ind_table_max_size : 600 rxqs_n)); 601 if (priv_rss_reta_index_resize(priv, reta_idx_n)) 602 return ENOMEM; 603 /* When the number of RX queues is not a power of two, the remaining 604 * table entries are padded with reused WQs and hashes are not spread 605 * uniformly. */ 606 for (i = 0, j = 0; (i != reta_idx_n); ++i) { 607 (*priv->reta_idx)[i] = j; 608 if (++j == rxqs_n) 609 j = 0; 610 } 611 return 0; 612 } 613 614 /** 615 * DPDK callback for Ethernet device configuration. 616 * 617 * @param dev 618 * Pointer to Ethernet device structure. 619 * 620 * @return 621 * 0 on success, negative errno value on failure. 622 */ 623 int 624 mlx5_dev_configure(struct rte_eth_dev *dev) 625 { 626 struct priv *priv = dev->data->dev_private; 627 int ret; 628 629 if (mlx5_is_secondary()) 630 return -E_RTE_SECONDARY; 631 632 priv_lock(priv); 633 ret = dev_configure(dev); 634 assert(ret >= 0); 635 priv_unlock(priv); 636 return -ret; 637 } 638 639 /** 640 * DPDK callback to get information about the device. 641 * 642 * @param dev 643 * Pointer to Ethernet device structure. 644 * @param[out] info 645 * Info structure output buffer. 646 */ 647 void 648 mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) 649 { 650 struct priv *priv = mlx5_get_priv(dev); 651 unsigned int max; 652 char ifname[IF_NAMESIZE]; 653 654 info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 655 656 priv_lock(priv); 657 /* FIXME: we should ask the device for these values. */ 658 info->min_rx_bufsize = 32; 659 info->max_rx_pktlen = 65536; 660 /* 661 * Since we need one CQ per QP, the limit is the minimum number 662 * between the two values. 663 */ 664 max = ((priv->device_attr.max_cq > priv->device_attr.max_qp) ? 665 priv->device_attr.max_qp : priv->device_attr.max_cq); 666 /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */ 667 if (max >= 65535) 668 max = 65535; 669 info->max_rx_queues = max; 670 info->max_tx_queues = max; 671 info->max_mac_addrs = RTE_DIM(priv->mac); 672 info->rx_offload_capa = 673 (priv->hw_csum ? 674 (DEV_RX_OFFLOAD_IPV4_CKSUM | 675 DEV_RX_OFFLOAD_UDP_CKSUM | 676 DEV_RX_OFFLOAD_TCP_CKSUM) : 677 0) | 678 (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0); 679 if (!priv->mps) 680 info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; 681 if (priv->hw_csum) 682 info->tx_offload_capa |= 683 (DEV_TX_OFFLOAD_IPV4_CKSUM | 684 DEV_TX_OFFLOAD_UDP_CKSUM | 685 DEV_TX_OFFLOAD_TCP_CKSUM); 686 if (priv->tso) 687 info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 688 if (priv->tunnel_en) 689 info->tx_offload_capa |= (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 690 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 691 DEV_TX_OFFLOAD_GRE_TNL_TSO); 692 if (priv_get_ifname(priv, &ifname) == 0) 693 info->if_index = if_nametoindex(ifname); 694 info->reta_size = priv->reta_idx_n ? 695 priv->reta_idx_n : priv->ind_table_max_size; 696 info->hash_key_size = ((*priv->rss_conf) ? 697 (*priv->rss_conf)[0]->rss_key_len : 698 0); 699 info->speed_capa = priv->link_speed_capa; 700 priv_unlock(priv); 701 } 702 703 const uint32_t * 704 mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) 705 { 706 static const uint32_t ptypes[] = { 707 /* refers to rxq_cq_to_pkt_type() */ 708 RTE_PTYPE_L2_ETHER, 709 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 710 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 711 RTE_PTYPE_L4_NONFRAG, 712 RTE_PTYPE_L4_FRAG, 713 RTE_PTYPE_L4_TCP, 714 RTE_PTYPE_L4_UDP, 715 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 716 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 717 RTE_PTYPE_INNER_L4_NONFRAG, 718 RTE_PTYPE_INNER_L4_FRAG, 719 RTE_PTYPE_INNER_L4_TCP, 720 RTE_PTYPE_INNER_L4_UDP, 721 RTE_PTYPE_UNKNOWN 722 }; 723 724 if (dev->rx_pkt_burst == mlx5_rx_burst || 725 dev->rx_pkt_burst == mlx5_rx_burst_vec) 726 return ptypes; 727 return NULL; 728 } 729 730 /** 731 * DPDK callback to retrieve physical link information. 732 * 733 * @param dev 734 * Pointer to Ethernet device structure. 735 * @param wait_to_complete 736 * Wait for request completion (ignored). 737 */ 738 static int 739 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) 740 { 741 struct priv *priv = mlx5_get_priv(dev); 742 struct ethtool_cmd edata = { 743 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */ 744 }; 745 struct ifreq ifr; 746 struct rte_eth_link dev_link; 747 int link_speed = 0; 748 749 /* priv_lock() is not taken to allow concurrent calls. */ 750 751 (void)wait_to_complete; 752 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 753 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 754 return -1; 755 } 756 memset(&dev_link, 0, sizeof(dev_link)); 757 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 758 (ifr.ifr_flags & IFF_RUNNING)); 759 ifr.ifr_data = (void *)&edata; 760 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 761 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", 762 strerror(errno)); 763 return -1; 764 } 765 link_speed = ethtool_cmd_speed(&edata); 766 if (link_speed == -1) 767 dev_link.link_speed = 0; 768 else 769 dev_link.link_speed = link_speed; 770 priv->link_speed_capa = 0; 771 if (edata.supported & SUPPORTED_Autoneg) 772 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; 773 if (edata.supported & (SUPPORTED_1000baseT_Full | 774 SUPPORTED_1000baseKX_Full)) 775 priv->link_speed_capa |= ETH_LINK_SPEED_1G; 776 if (edata.supported & SUPPORTED_10000baseKR_Full) 777 priv->link_speed_capa |= ETH_LINK_SPEED_10G; 778 if (edata.supported & (SUPPORTED_40000baseKR4_Full | 779 SUPPORTED_40000baseCR4_Full | 780 SUPPORTED_40000baseSR4_Full | 781 SUPPORTED_40000baseLR4_Full)) 782 priv->link_speed_capa |= ETH_LINK_SPEED_40G; 783 dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? 784 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 785 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 786 ETH_LINK_SPEED_FIXED); 787 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 788 /* Link status changed. */ 789 dev->data->dev_link = dev_link; 790 return 0; 791 } 792 /* Link status is still the same. */ 793 return -1; 794 } 795 796 /** 797 * Retrieve physical link information (unlocked version using new ioctl). 798 * 799 * @param dev 800 * Pointer to Ethernet device structure. 801 * @param wait_to_complete 802 * Wait for request completion (ignored). 803 */ 804 static int 805 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) 806 { 807 struct priv *priv = mlx5_get_priv(dev); 808 alignas(struct ethtool_link_settings) 809 uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) + 810 sizeof(uint32_t) * 811 ETHTOOL_LINK_MODE_MASK_MAX_KERNEL_NU32 * 3]; 812 struct ethtool_link_settings *ecmd = (void *)data; 813 struct ifreq ifr; 814 struct rte_eth_link dev_link; 815 uint64_t sc; 816 817 (void)wait_to_complete; 818 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 819 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 820 return -1; 821 } 822 memset(&dev_link, 0, sizeof(dev_link)); 823 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 824 (ifr.ifr_flags & IFF_RUNNING)); 825 memset(ecmd, 0, sizeof(*ecmd)); 826 ecmd->cmd = ETHTOOL_GLINKSETTINGS; 827 ifr.ifr_data = (void *)ecmd; 828 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 829 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", 830 strerror(errno)); 831 return -1; 832 } 833 ecmd->link_mode_masks_nwords = -ecmd->link_mode_masks_nwords; 834 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 835 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", 836 strerror(errno)); 837 return -1; 838 } 839 dev_link.link_speed = ecmd->speed; 840 sc = ecmd->link_mode_masks[0] | 841 ((uint64_t)ecmd->link_mode_masks[1] << 32); 842 priv->link_speed_capa = 0; 843 if (sc & ETHTOOL_LINK_MODE_Autoneg_BIT) 844 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; 845 if (sc & (ETHTOOL_LINK_MODE_1000baseT_Full_BIT | 846 ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)) 847 priv->link_speed_capa |= ETH_LINK_SPEED_1G; 848 if (sc & (ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT | 849 ETHTOOL_LINK_MODE_10000baseKR_Full_BIT | 850 ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)) 851 priv->link_speed_capa |= ETH_LINK_SPEED_10G; 852 if (sc & (ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT | 853 ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)) 854 priv->link_speed_capa |= ETH_LINK_SPEED_20G; 855 if (sc & (ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT | 856 ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT | 857 ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT | 858 ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)) 859 priv->link_speed_capa |= ETH_LINK_SPEED_40G; 860 if (sc & (ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT | 861 ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT | 862 ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT | 863 ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)) 864 priv->link_speed_capa |= ETH_LINK_SPEED_56G; 865 if (sc & (ETHTOOL_LINK_MODE_25000baseCR_Full_BIT | 866 ETHTOOL_LINK_MODE_25000baseKR_Full_BIT | 867 ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)) 868 priv->link_speed_capa |= ETH_LINK_SPEED_25G; 869 if (sc & (ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT | 870 ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)) 871 priv->link_speed_capa |= ETH_LINK_SPEED_50G; 872 if (sc & (ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT | 873 ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT | 874 ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT | 875 ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)) 876 priv->link_speed_capa |= ETH_LINK_SPEED_100G; 877 dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ? 878 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 879 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 880 ETH_LINK_SPEED_FIXED); 881 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 882 /* Link status changed. */ 883 dev->data->dev_link = dev_link; 884 return 0; 885 } 886 /* Link status is still the same. */ 887 return -1; 888 } 889 890 /** 891 * DPDK callback to retrieve physical link information. 892 * 893 * @param dev 894 * Pointer to Ethernet device structure. 895 * @param wait_to_complete 896 * Wait for request completion (ignored). 897 */ 898 int 899 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) 900 { 901 struct utsname utsname; 902 int ver[3]; 903 904 if (uname(&utsname) == -1 || 905 sscanf(utsname.release, "%d.%d.%d", 906 &ver[0], &ver[1], &ver[2]) != 3 || 907 KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0)) 908 return mlx5_link_update_unlocked_gset(dev, wait_to_complete); 909 return mlx5_link_update_unlocked_gs(dev, wait_to_complete); 910 } 911 912 /** 913 * DPDK callback to change the MTU. 914 * 915 * @param dev 916 * Pointer to Ethernet device structure. 917 * @param in_mtu 918 * New MTU. 919 * 920 * @return 921 * 0 on success, negative errno value on failure. 922 */ 923 int 924 mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 925 { 926 struct priv *priv = dev->data->dev_private; 927 uint16_t kern_mtu; 928 int ret = 0; 929 930 if (mlx5_is_secondary()) 931 return -E_RTE_SECONDARY; 932 933 priv_lock(priv); 934 ret = priv_get_mtu(priv, &kern_mtu); 935 if (ret) 936 goto out; 937 /* Set kernel interface MTU first. */ 938 ret = priv_set_mtu(priv, mtu); 939 if (ret) 940 goto out; 941 ret = priv_get_mtu(priv, &kern_mtu); 942 if (ret) 943 goto out; 944 if (kern_mtu == mtu) { 945 priv->mtu = mtu; 946 DEBUG("adapter port %u MTU set to %u", priv->port, mtu); 947 } 948 priv_unlock(priv); 949 return 0; 950 out: 951 ret = errno; 952 WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, 953 strerror(ret)); 954 priv_unlock(priv); 955 assert(ret >= 0); 956 return -ret; 957 } 958 959 /** 960 * DPDK callback to get flow control status. 961 * 962 * @param dev 963 * Pointer to Ethernet device structure. 964 * @param[out] fc_conf 965 * Flow control output buffer. 966 * 967 * @return 968 * 0 on success, negative errno value on failure. 969 */ 970 int 971 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 972 { 973 struct priv *priv = dev->data->dev_private; 974 struct ifreq ifr; 975 struct ethtool_pauseparam ethpause = { 976 .cmd = ETHTOOL_GPAUSEPARAM 977 }; 978 int ret; 979 980 if (mlx5_is_secondary()) 981 return -E_RTE_SECONDARY; 982 983 ifr.ifr_data = (void *)ðpause; 984 priv_lock(priv); 985 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 986 ret = errno; 987 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" 988 " failed: %s", 989 strerror(ret)); 990 goto out; 991 } 992 993 fc_conf->autoneg = ethpause.autoneg; 994 if (ethpause.rx_pause && ethpause.tx_pause) 995 fc_conf->mode = RTE_FC_FULL; 996 else if (ethpause.rx_pause) 997 fc_conf->mode = RTE_FC_RX_PAUSE; 998 else if (ethpause.tx_pause) 999 fc_conf->mode = RTE_FC_TX_PAUSE; 1000 else 1001 fc_conf->mode = RTE_FC_NONE; 1002 ret = 0; 1003 1004 out: 1005 priv_unlock(priv); 1006 assert(ret >= 0); 1007 return -ret; 1008 } 1009 1010 /** 1011 * DPDK callback to modify flow control parameters. 1012 * 1013 * @param dev 1014 * Pointer to Ethernet device structure. 1015 * @param[in] fc_conf 1016 * Flow control parameters. 1017 * 1018 * @return 1019 * 0 on success, negative errno value on failure. 1020 */ 1021 int 1022 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 1023 { 1024 struct priv *priv = dev->data->dev_private; 1025 struct ifreq ifr; 1026 struct ethtool_pauseparam ethpause = { 1027 .cmd = ETHTOOL_SPAUSEPARAM 1028 }; 1029 int ret; 1030 1031 if (mlx5_is_secondary()) 1032 return -E_RTE_SECONDARY; 1033 1034 ifr.ifr_data = (void *)ðpause; 1035 ethpause.autoneg = fc_conf->autoneg; 1036 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 1037 (fc_conf->mode & RTE_FC_RX_PAUSE)) 1038 ethpause.rx_pause = 1; 1039 else 1040 ethpause.rx_pause = 0; 1041 1042 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 1043 (fc_conf->mode & RTE_FC_TX_PAUSE)) 1044 ethpause.tx_pause = 1; 1045 else 1046 ethpause.tx_pause = 0; 1047 1048 priv_lock(priv); 1049 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 1050 ret = errno; 1051 WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" 1052 " failed: %s", 1053 strerror(ret)); 1054 goto out; 1055 } 1056 ret = 0; 1057 1058 out: 1059 priv_unlock(priv); 1060 assert(ret >= 0); 1061 return -ret; 1062 } 1063 1064 /** 1065 * Get PCI information from struct ibv_device. 1066 * 1067 * @param device 1068 * Pointer to Ethernet device structure. 1069 * @param[out] pci_addr 1070 * PCI bus address output buffer. 1071 * 1072 * @return 1073 * 0 on success, -1 on failure and errno is set. 1074 */ 1075 int 1076 mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, 1077 struct rte_pci_addr *pci_addr) 1078 { 1079 FILE *file; 1080 char line[32]; 1081 MKSTR(path, "%s/device/uevent", device->ibdev_path); 1082 1083 file = fopen(path, "rb"); 1084 if (file == NULL) 1085 return -1; 1086 while (fgets(line, sizeof(line), file) == line) { 1087 size_t len = strlen(line); 1088 int ret; 1089 1090 /* Truncate long lines. */ 1091 if (len == (sizeof(line) - 1)) 1092 while (line[(len - 1)] != '\n') { 1093 ret = fgetc(file); 1094 if (ret == EOF) 1095 break; 1096 line[(len - 1)] = ret; 1097 } 1098 /* Extract information. */ 1099 if (sscanf(line, 1100 "PCI_SLOT_NAME=" 1101 "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", 1102 &pci_addr->domain, 1103 &pci_addr->bus, 1104 &pci_addr->devid, 1105 &pci_addr->function) == 4) { 1106 ret = 0; 1107 break; 1108 } 1109 } 1110 fclose(file); 1111 return 0; 1112 } 1113 1114 /** 1115 * Link status handler. 1116 * 1117 * @param priv 1118 * Pointer to private structure. 1119 * @param dev 1120 * Pointer to the rte_eth_dev structure. 1121 * 1122 * @return 1123 * Nonzero if the callback process can be called immediately. 1124 */ 1125 static int 1126 priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) 1127 { 1128 struct ibv_async_event event; 1129 struct rte_eth_link *link = &dev->data->dev_link; 1130 int ret = 0; 1131 1132 /* Read all message and acknowledge them. */ 1133 for (;;) { 1134 if (ibv_get_async_event(priv->ctx, &event)) 1135 break; 1136 1137 if (event.event_type != IBV_EVENT_PORT_ACTIVE && 1138 event.event_type != IBV_EVENT_PORT_ERR) 1139 DEBUG("event type %d on port %d not handled", 1140 event.event_type, event.element.port_num); 1141 ibv_ack_async_event(&event); 1142 } 1143 mlx5_link_update(dev, 0); 1144 if (((link->link_speed == 0) && link->link_status) || 1145 ((link->link_speed != 0) && !link->link_status)) { 1146 if (!priv->pending_alarm) { 1147 /* Inconsistent status, check again later. */ 1148 priv->pending_alarm = 1; 1149 rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US, 1150 mlx5_dev_link_status_handler, 1151 dev); 1152 } 1153 } else { 1154 ret = 1; 1155 } 1156 return ret; 1157 } 1158 1159 /** 1160 * Handle delayed link status event. 1161 * 1162 * @param arg 1163 * Registered argument. 1164 */ 1165 void 1166 mlx5_dev_link_status_handler(void *arg) 1167 { 1168 struct rte_eth_dev *dev = arg; 1169 struct priv *priv = dev->data->dev_private; 1170 int ret; 1171 1172 priv_lock(priv); 1173 assert(priv->pending_alarm == 1); 1174 priv->pending_alarm = 0; 1175 ret = priv_dev_link_status_handler(priv, dev); 1176 priv_unlock(priv); 1177 if (ret) 1178 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, 1179 NULL); 1180 } 1181 1182 /** 1183 * Handle interrupts from the NIC. 1184 * 1185 * @param[in] intr_handle 1186 * Interrupt handler. 1187 * @param cb_arg 1188 * Callback argument. 1189 */ 1190 void 1191 mlx5_dev_interrupt_handler(void *cb_arg) 1192 { 1193 struct rte_eth_dev *dev = cb_arg; 1194 struct priv *priv = dev->data->dev_private; 1195 int ret; 1196 1197 priv_lock(priv); 1198 ret = priv_dev_link_status_handler(priv, dev); 1199 priv_unlock(priv); 1200 if (ret) 1201 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, 1202 NULL); 1203 } 1204 1205 /** 1206 * Uninstall interrupt handler. 1207 * 1208 * @param priv 1209 * Pointer to private structure. 1210 * @param dev 1211 * Pointer to the rte_eth_dev structure. 1212 */ 1213 void 1214 priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) 1215 { 1216 if (!dev->data->dev_conf.intr_conf.lsc) 1217 return; 1218 rte_intr_callback_unregister(&priv->intr_handle, 1219 mlx5_dev_interrupt_handler, 1220 dev); 1221 if (priv->pending_alarm) 1222 rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev); 1223 priv->pending_alarm = 0; 1224 priv->intr_handle.fd = 0; 1225 priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; 1226 } 1227 1228 /** 1229 * Install interrupt handler. 1230 * 1231 * @param priv 1232 * Pointer to private structure. 1233 * @param dev 1234 * Pointer to the rte_eth_dev structure. 1235 */ 1236 void 1237 priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) 1238 { 1239 int rc, flags; 1240 1241 if (!dev->data->dev_conf.intr_conf.lsc) 1242 return; 1243 assert(priv->ctx->async_fd > 0); 1244 flags = fcntl(priv->ctx->async_fd, F_GETFL); 1245 rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); 1246 if (rc < 0) { 1247 INFO("failed to change file descriptor async event queue"); 1248 dev->data->dev_conf.intr_conf.lsc = 0; 1249 } else { 1250 priv->intr_handle.fd = priv->ctx->async_fd; 1251 priv->intr_handle.type = RTE_INTR_HANDLE_EXT; 1252 rte_intr_callback_register(&priv->intr_handle, 1253 mlx5_dev_interrupt_handler, 1254 dev); 1255 } 1256 } 1257 1258 /** 1259 * Change the link state (UP / DOWN). 1260 * 1261 * @param priv 1262 * Pointer to Ethernet device structure. 1263 * @param up 1264 * Nonzero for link up, otherwise link down. 1265 * 1266 * @return 1267 * 0 on success, errno value on failure. 1268 */ 1269 static int 1270 priv_set_link(struct priv *priv, int up) 1271 { 1272 struct rte_eth_dev *dev = priv->dev; 1273 int err; 1274 1275 if (up) { 1276 err = priv_set_flags(priv, ~IFF_UP, IFF_UP); 1277 if (err) 1278 return err; 1279 priv_select_tx_function(priv); 1280 priv_select_rx_function(priv); 1281 } else { 1282 err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP); 1283 if (err) 1284 return err; 1285 dev->rx_pkt_burst = removed_rx_burst; 1286 dev->tx_pkt_burst = removed_tx_burst; 1287 } 1288 return 0; 1289 } 1290 1291 /** 1292 * DPDK callback to bring the link DOWN. 1293 * 1294 * @param dev 1295 * Pointer to Ethernet device structure. 1296 * 1297 * @return 1298 * 0 on success, errno value on failure. 1299 */ 1300 int 1301 mlx5_set_link_down(struct rte_eth_dev *dev) 1302 { 1303 struct priv *priv = dev->data->dev_private; 1304 int err; 1305 1306 priv_lock(priv); 1307 err = priv_set_link(priv, 0); 1308 priv_unlock(priv); 1309 return err; 1310 } 1311 1312 /** 1313 * DPDK callback to bring the link UP. 1314 * 1315 * @param dev 1316 * Pointer to Ethernet device structure. 1317 * 1318 * @return 1319 * 0 on success, errno value on failure. 1320 */ 1321 int 1322 mlx5_set_link_up(struct rte_eth_dev *dev) 1323 { 1324 struct priv *priv = dev->data->dev_private; 1325 int err; 1326 1327 priv_lock(priv); 1328 err = priv_set_link(priv, 1); 1329 priv_unlock(priv); 1330 return err; 1331 } 1332 1333 /** 1334 * Configure the TX function to use. 1335 * 1336 * @param priv 1337 * Pointer to private structure. 1338 */ 1339 void 1340 priv_select_tx_function(struct priv *priv) 1341 { 1342 priv->dev->tx_pkt_burst = mlx5_tx_burst; 1343 /* Select appropriate TX function. */ 1344 if (priv->mps == MLX5_MPW_ENHANCED) { 1345 if (priv_check_vec_tx_support(priv) > 0) { 1346 if (priv_check_raw_vec_tx_support(priv) > 0) 1347 priv->dev->tx_pkt_burst = mlx5_tx_burst_raw_vec; 1348 else 1349 priv->dev->tx_pkt_burst = mlx5_tx_burst_vec; 1350 DEBUG("selected Enhanced MPW TX vectorized function"); 1351 } else { 1352 priv->dev->tx_pkt_burst = mlx5_tx_burst_empw; 1353 DEBUG("selected Enhanced MPW TX function"); 1354 } 1355 } else if (priv->mps && priv->txq_inline) { 1356 priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; 1357 DEBUG("selected MPW inline TX function"); 1358 } else if (priv->mps) { 1359 priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw; 1360 DEBUG("selected MPW TX function"); 1361 } 1362 } 1363 1364 /** 1365 * Configure the RX function to use. 1366 * 1367 * @param priv 1368 * Pointer to private structure. 1369 */ 1370 void 1371 priv_select_rx_function(struct priv *priv) 1372 { 1373 if (priv_check_vec_rx_support(priv) > 0) { 1374 priv->dev->rx_pkt_burst = mlx5_rx_burst_vec; 1375 DEBUG("selected RX vectorized function"); 1376 } else { 1377 priv->dev->rx_pkt_burst = mlx5_rx_burst; 1378 } 1379 } 1380