1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2015 6WIND S.A. 5 * Copyright 2015 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #define _GNU_SOURCE 35 36 #include <stddef.h> 37 #include <assert.h> 38 #include <unistd.h> 39 #include <stdint.h> 40 #include <stdio.h> 41 #include <string.h> 42 #include <stdlib.h> 43 #include <errno.h> 44 #include <dirent.h> 45 #include <net/if.h> 46 #include <sys/ioctl.h> 47 #include <sys/socket.h> 48 #include <sys/utsname.h> 49 #include <netinet/in.h> 50 #include <linux/ethtool.h> 51 #include <linux/sockios.h> 52 #include <linux/version.h> 53 #include <fcntl.h> 54 #include <stdalign.h> 55 #include <sys/un.h> 56 57 #include <rte_atomic.h> 58 #include <rte_ethdev.h> 59 #include <rte_mbuf.h> 60 #include <rte_common.h> 61 #include <rte_interrupts.h> 62 #include <rte_alarm.h> 63 #include <rte_malloc.h> 64 65 #include "mlx5.h" 66 #include "mlx5_rxtx.h" 67 #include "mlx5_utils.h" 68 69 /* Add defines in case the running kernel is not the same as user headers. */ 70 #ifndef ETHTOOL_GLINKSETTINGS 71 struct ethtool_link_settings { 72 uint32_t cmd; 73 uint32_t speed; 74 uint8_t duplex; 75 uint8_t port; 76 uint8_t phy_address; 77 uint8_t autoneg; 78 uint8_t mdio_support; 79 uint8_t eth_to_mdix; 80 uint8_t eth_tp_mdix_ctrl; 81 int8_t link_mode_masks_nwords; 82 uint32_t reserved[8]; 83 uint32_t link_mode_masks[]; 84 }; 85 86 #define ETHTOOL_GLINKSETTINGS 0x0000004c 87 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5 88 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6 89 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17 90 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18 91 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19 92 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20 93 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21 94 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22 95 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23 96 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24 97 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25 98 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26 99 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27 100 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28 101 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29 102 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30 103 #endif 104 #ifndef HAVE_ETHTOOL_LINK_MODE_25G 105 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31 106 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32 107 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33 108 #endif 109 #ifndef HAVE_ETHTOOL_LINK_MODE_50G 110 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34 111 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35 112 #endif 113 #ifndef HAVE_ETHTOOL_LINK_MODE_100G 114 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36 115 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37 116 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38 117 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39 118 #endif 119 120 /** 121 * Return private structure associated with an Ethernet device. 122 * 123 * @param dev 124 * Pointer to Ethernet device structure. 125 * 126 * @return 127 * Pointer to private structure. 128 */ 129 struct priv * 130 mlx5_get_priv(struct rte_eth_dev *dev) 131 { 132 return dev->data->dev_private; 133 } 134 135 /** 136 * Check if running as a secondary process. 137 * 138 * @return 139 * Nonzero if running as a secondary process. 140 */ 141 inline int 142 mlx5_is_secondary(void) 143 { 144 return rte_eal_process_type() == RTE_PROC_SECONDARY; 145 } 146 147 /** 148 * Get interface name from private structure. 149 * 150 * @param[in] priv 151 * Pointer to private structure. 152 * @param[out] ifname 153 * Interface name output buffer. 154 * 155 * @return 156 * 0 on success, -1 on failure and errno is set. 157 */ 158 int 159 priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) 160 { 161 DIR *dir; 162 struct dirent *dent; 163 unsigned int dev_type = 0; 164 unsigned int dev_port_prev = ~0u; 165 char match[IF_NAMESIZE] = ""; 166 167 { 168 MKSTR(path, "%s/device/net", priv->ctx->device->ibdev_path); 169 170 dir = opendir(path); 171 if (dir == NULL) 172 return -1; 173 } 174 while ((dent = readdir(dir)) != NULL) { 175 char *name = dent->d_name; 176 FILE *file; 177 unsigned int dev_port; 178 int r; 179 180 if ((name[0] == '.') && 181 ((name[1] == '\0') || 182 ((name[1] == '.') && (name[2] == '\0')))) 183 continue; 184 185 MKSTR(path, "%s/device/net/%s/%s", 186 priv->ctx->device->ibdev_path, name, 187 (dev_type ? "dev_id" : "dev_port")); 188 189 file = fopen(path, "rb"); 190 if (file == NULL) { 191 if (errno != ENOENT) 192 continue; 193 /* 194 * Switch to dev_id when dev_port does not exist as 195 * is the case with Linux kernel versions < 3.15. 196 */ 197 try_dev_id: 198 match[0] = '\0'; 199 if (dev_type) 200 break; 201 dev_type = 1; 202 dev_port_prev = ~0u; 203 rewinddir(dir); 204 continue; 205 } 206 r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port); 207 fclose(file); 208 if (r != 1) 209 continue; 210 /* 211 * Switch to dev_id when dev_port returns the same value for 212 * all ports. May happen when using a MOFED release older than 213 * 3.0 with a Linux kernel >= 3.15. 214 */ 215 if (dev_port == dev_port_prev) 216 goto try_dev_id; 217 dev_port_prev = dev_port; 218 if (dev_port == (priv->port - 1u)) 219 snprintf(match, sizeof(match), "%s", name); 220 } 221 closedir(dir); 222 if (match[0] == '\0') 223 return -1; 224 strncpy(*ifname, match, sizeof(*ifname)); 225 return 0; 226 } 227 228 /** 229 * Check if the counter is located on ib counters file. 230 * 231 * @param[in] cntr 232 * Counter name. 233 * 234 * @return 235 * 1 if counter is located on ib counters file , 0 otherwise. 236 */ 237 int 238 priv_is_ib_cntr(const char *cntr) 239 { 240 if (!strcmp(cntr, "out_of_buffer")) 241 return 1; 242 return 0; 243 } 244 245 /** 246 * Read from sysfs entry. 247 * 248 * @param[in] priv 249 * Pointer to private structure. 250 * @param[in] entry 251 * Entry name relative to sysfs path. 252 * @param[out] buf 253 * Data output buffer. 254 * @param size 255 * Buffer size. 256 * 257 * @return 258 * 0 on success, -1 on failure and errno is set. 259 */ 260 static int 261 priv_sysfs_read(const struct priv *priv, const char *entry, 262 char *buf, size_t size) 263 { 264 char ifname[IF_NAMESIZE]; 265 FILE *file; 266 int ret; 267 int err; 268 269 if (priv_get_ifname(priv, &ifname)) 270 return -1; 271 272 if (priv_is_ib_cntr(entry)) { 273 MKSTR(path, "%s/ports/1/hw_counters/%s", 274 priv->ctx->device->ibdev_path, entry); 275 file = fopen(path, "rb"); 276 } else { 277 MKSTR(path, "%s/device/net/%s/%s", 278 priv->ctx->device->ibdev_path, ifname, entry); 279 file = fopen(path, "rb"); 280 } 281 if (file == NULL) 282 return -1; 283 ret = fread(buf, 1, size, file); 284 err = errno; 285 if (((size_t)ret < size) && (ferror(file))) 286 ret = -1; 287 else 288 ret = size; 289 fclose(file); 290 errno = err; 291 return ret; 292 } 293 294 /** 295 * Write to sysfs entry. 296 * 297 * @param[in] priv 298 * Pointer to private structure. 299 * @param[in] entry 300 * Entry name relative to sysfs path. 301 * @param[in] buf 302 * Data buffer. 303 * @param size 304 * Buffer size. 305 * 306 * @return 307 * 0 on success, -1 on failure and errno is set. 308 */ 309 static int 310 priv_sysfs_write(const struct priv *priv, const char *entry, 311 char *buf, size_t size) 312 { 313 char ifname[IF_NAMESIZE]; 314 FILE *file; 315 int ret; 316 int err; 317 318 if (priv_get_ifname(priv, &ifname)) 319 return -1; 320 321 MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, 322 ifname, entry); 323 324 file = fopen(path, "wb"); 325 if (file == NULL) 326 return -1; 327 ret = fwrite(buf, 1, size, file); 328 err = errno; 329 if (((size_t)ret < size) || (ferror(file))) 330 ret = -1; 331 else 332 ret = size; 333 fclose(file); 334 errno = err; 335 return ret; 336 } 337 338 /** 339 * Get unsigned long sysfs property. 340 * 341 * @param priv 342 * Pointer to private structure. 343 * @param[in] name 344 * Entry name relative to sysfs path. 345 * @param[out] value 346 * Value output buffer. 347 * 348 * @return 349 * 0 on success, -1 on failure and errno is set. 350 */ 351 static int 352 priv_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) 353 { 354 int ret; 355 unsigned long value_ret; 356 char value_str[32]; 357 358 ret = priv_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); 359 if (ret == -1) { 360 DEBUG("cannot read %s value from sysfs: %s", 361 name, strerror(errno)); 362 return -1; 363 } 364 value_str[ret] = '\0'; 365 errno = 0; 366 value_ret = strtoul(value_str, NULL, 0); 367 if (errno) { 368 DEBUG("invalid %s value `%s': %s", name, value_str, 369 strerror(errno)); 370 return -1; 371 } 372 *value = value_ret; 373 return 0; 374 } 375 376 /** 377 * Set unsigned long sysfs property. 378 * 379 * @param priv 380 * Pointer to private structure. 381 * @param[in] name 382 * Entry name relative to sysfs path. 383 * @param value 384 * Value to set. 385 * 386 * @return 387 * 0 on success, -1 on failure and errno is set. 388 */ 389 static int 390 priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) 391 { 392 int ret; 393 MKSTR(value_str, "%lu", value); 394 395 ret = priv_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); 396 if (ret == -1) { 397 DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", 398 name, value_str, value, strerror(errno)); 399 return -1; 400 } 401 return 0; 402 } 403 404 /** 405 * Perform ifreq ioctl() on associated Ethernet device. 406 * 407 * @param[in] priv 408 * Pointer to private structure. 409 * @param req 410 * Request number to pass to ioctl(). 411 * @param[out] ifr 412 * Interface request structure output buffer. 413 * 414 * @return 415 * 0 on success, -1 on failure and errno is set. 416 */ 417 int 418 priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) 419 { 420 int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); 421 int ret = -1; 422 423 if (sock == -1) 424 return ret; 425 if (priv_get_ifname(priv, &ifr->ifr_name) == 0) 426 ret = ioctl(sock, req, ifr); 427 close(sock); 428 return ret; 429 } 430 431 /** 432 * Return the number of active VFs for the current device. 433 * 434 * @param[in] priv 435 * Pointer to private structure. 436 * @param[out] num_vfs 437 * Number of active VFs. 438 * 439 * @return 440 * 0 on success, -1 on failure and errno is set. 441 */ 442 int 443 priv_get_num_vfs(struct priv *priv, uint16_t *num_vfs) 444 { 445 /* The sysfs entry name depends on the operating system. */ 446 const char **name = (const char *[]){ 447 "device/sriov_numvfs", 448 "device/mlx5_num_vfs", 449 NULL, 450 }; 451 int ret; 452 453 do { 454 unsigned long ulong_num_vfs; 455 456 ret = priv_get_sysfs_ulong(priv, *name, &ulong_num_vfs); 457 if (!ret) 458 *num_vfs = ulong_num_vfs; 459 } while (*(++name) && ret); 460 return ret; 461 } 462 463 /** 464 * Get device MTU. 465 * 466 * @param priv 467 * Pointer to private structure. 468 * @param[out] mtu 469 * MTU value output buffer. 470 * 471 * @return 472 * 0 on success, -1 on failure and errno is set. 473 */ 474 int 475 priv_get_mtu(struct priv *priv, uint16_t *mtu) 476 { 477 unsigned long ulong_mtu; 478 479 if (priv_get_sysfs_ulong(priv, "mtu", &ulong_mtu) == -1) 480 return -1; 481 *mtu = ulong_mtu; 482 return 0; 483 } 484 485 /** 486 * Read device counter from sysfs. 487 * 488 * @param priv 489 * Pointer to private structure. 490 * @param name 491 * Counter name. 492 * @param[out] cntr 493 * Counter output buffer. 494 * 495 * @return 496 * 0 on success, -1 on failure and errno is set. 497 */ 498 int 499 priv_get_cntr_sysfs(struct priv *priv, const char *name, uint64_t *cntr) 500 { 501 unsigned long ulong_ctr; 502 503 if (priv_get_sysfs_ulong(priv, name, &ulong_ctr) == -1) 504 return -1; 505 *cntr = ulong_ctr; 506 return 0; 507 } 508 509 /** 510 * Set device MTU. 511 * 512 * @param priv 513 * Pointer to private structure. 514 * @param mtu 515 * MTU value to set. 516 * 517 * @return 518 * 0 on success, -1 on failure and errno is set. 519 */ 520 static int 521 priv_set_mtu(struct priv *priv, uint16_t mtu) 522 { 523 uint16_t new_mtu; 524 525 if (priv_set_sysfs_ulong(priv, "mtu", mtu) || 526 priv_get_mtu(priv, &new_mtu)) 527 return -1; 528 if (new_mtu == mtu) 529 return 0; 530 errno = EINVAL; 531 return -1; 532 } 533 534 /** 535 * Set device flags. 536 * 537 * @param priv 538 * Pointer to private structure. 539 * @param keep 540 * Bitmask for flags that must remain untouched. 541 * @param flags 542 * Bitmask for flags to modify. 543 * 544 * @return 545 * 0 on success, -1 on failure and errno is set. 546 */ 547 int 548 priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) 549 { 550 unsigned long tmp; 551 552 if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1) 553 return -1; 554 tmp &= keep; 555 tmp |= (flags & (~keep)); 556 return priv_set_sysfs_ulong(priv, "flags", tmp); 557 } 558 559 /** 560 * Ethernet device configuration. 561 * 562 * Prepare the driver for a given number of TX and RX queues. 563 * 564 * @param dev 565 * Pointer to Ethernet device structure. 566 * 567 * @return 568 * 0 on success, errno value on failure. 569 */ 570 static int 571 dev_configure(struct rte_eth_dev *dev) 572 { 573 struct priv *priv = dev->data->dev_private; 574 unsigned int rxqs_n = dev->data->nb_rx_queues; 575 unsigned int txqs_n = dev->data->nb_tx_queues; 576 unsigned int i; 577 unsigned int j; 578 unsigned int reta_idx_n; 579 580 priv->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; 581 priv->rxqs = (void *)dev->data->rx_queues; 582 priv->txqs = (void *)dev->data->tx_queues; 583 if (txqs_n != priv->txqs_n) { 584 INFO("%p: TX queues number update: %u -> %u", 585 (void *)dev, priv->txqs_n, txqs_n); 586 priv->txqs_n = txqs_n; 587 } 588 if (rxqs_n > priv->ind_table_max_size) { 589 ERROR("cannot handle this many RX queues (%u)", rxqs_n); 590 return EINVAL; 591 } 592 if (rxqs_n == priv->rxqs_n) 593 return 0; 594 INFO("%p: RX queues number update: %u -> %u", 595 (void *)dev, priv->rxqs_n, rxqs_n); 596 priv->rxqs_n = rxqs_n; 597 /* If the requested number of RX queues is not a power of two, use the 598 * maximum indirection table size for better balancing. 599 * The result is always rounded to the next power of two. */ 600 reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ? 601 priv->ind_table_max_size : 602 rxqs_n)); 603 if (priv_rss_reta_index_resize(priv, reta_idx_n)) 604 return ENOMEM; 605 /* When the number of RX queues is not a power of two, the remaining 606 * table entries are padded with reused WQs and hashes are not spread 607 * uniformly. */ 608 for (i = 0, j = 0; (i != reta_idx_n); ++i) { 609 (*priv->reta_idx)[i] = j; 610 if (++j == rxqs_n) 611 j = 0; 612 } 613 return 0; 614 } 615 616 /** 617 * DPDK callback for Ethernet device configuration. 618 * 619 * @param dev 620 * Pointer to Ethernet device structure. 621 * 622 * @return 623 * 0 on success, negative errno value on failure. 624 */ 625 int 626 mlx5_dev_configure(struct rte_eth_dev *dev) 627 { 628 struct priv *priv = dev->data->dev_private; 629 int ret; 630 631 if (mlx5_is_secondary()) 632 return -E_RTE_SECONDARY; 633 634 priv_lock(priv); 635 ret = dev_configure(dev); 636 assert(ret >= 0); 637 priv_unlock(priv); 638 return -ret; 639 } 640 641 /** 642 * DPDK callback to get information about the device. 643 * 644 * @param dev 645 * Pointer to Ethernet device structure. 646 * @param[out] info 647 * Info structure output buffer. 648 */ 649 void 650 mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) 651 { 652 struct priv *priv = mlx5_get_priv(dev); 653 unsigned int max; 654 char ifname[IF_NAMESIZE]; 655 656 info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 657 658 priv_lock(priv); 659 /* FIXME: we should ask the device for these values. */ 660 info->min_rx_bufsize = 32; 661 info->max_rx_pktlen = 65536; 662 /* 663 * Since we need one CQ per QP, the limit is the minimum number 664 * between the two values. 665 */ 666 max = RTE_MIN(priv->device_attr.orig_attr.max_cq, 667 priv->device_attr.orig_attr.max_qp); 668 /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */ 669 if (max >= 65535) 670 max = 65535; 671 info->max_rx_queues = max; 672 info->max_tx_queues = max; 673 info->max_mac_addrs = RTE_DIM(priv->mac); 674 info->rx_offload_capa = 675 (priv->hw_csum ? 676 (DEV_RX_OFFLOAD_IPV4_CKSUM | 677 DEV_RX_OFFLOAD_UDP_CKSUM | 678 DEV_RX_OFFLOAD_TCP_CKSUM) : 679 0) | 680 (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0); 681 if (!priv->mps) 682 info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; 683 if (priv->hw_csum) 684 info->tx_offload_capa |= 685 (DEV_TX_OFFLOAD_IPV4_CKSUM | 686 DEV_TX_OFFLOAD_UDP_CKSUM | 687 DEV_TX_OFFLOAD_TCP_CKSUM); 688 if (priv->tso) 689 info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 690 if (priv->tunnel_en) 691 info->tx_offload_capa |= (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 692 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 693 DEV_TX_OFFLOAD_GRE_TNL_TSO); 694 if (priv_get_ifname(priv, &ifname) == 0) 695 info->if_index = if_nametoindex(ifname); 696 info->reta_size = priv->reta_idx_n ? 697 priv->reta_idx_n : priv->ind_table_max_size; 698 info->hash_key_size = ((*priv->rss_conf) ? 699 (*priv->rss_conf)[0]->rss_key_len : 700 0); 701 info->speed_capa = priv->link_speed_capa; 702 priv_unlock(priv); 703 } 704 705 const uint32_t * 706 mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) 707 { 708 static const uint32_t ptypes[] = { 709 /* refers to rxq_cq_to_pkt_type() */ 710 RTE_PTYPE_L2_ETHER, 711 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 712 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 713 RTE_PTYPE_L4_NONFRAG, 714 RTE_PTYPE_L4_FRAG, 715 RTE_PTYPE_L4_TCP, 716 RTE_PTYPE_L4_UDP, 717 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 718 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 719 RTE_PTYPE_INNER_L4_NONFRAG, 720 RTE_PTYPE_INNER_L4_FRAG, 721 RTE_PTYPE_INNER_L4_TCP, 722 RTE_PTYPE_INNER_L4_UDP, 723 RTE_PTYPE_UNKNOWN 724 }; 725 726 if (dev->rx_pkt_burst == mlx5_rx_burst || 727 dev->rx_pkt_burst == mlx5_rx_burst_vec) 728 return ptypes; 729 return NULL; 730 } 731 732 /** 733 * DPDK callback to retrieve physical link information. 734 * 735 * @param dev 736 * Pointer to Ethernet device structure. 737 * @param wait_to_complete 738 * Wait for request completion (ignored). 739 */ 740 static int 741 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) 742 { 743 struct priv *priv = mlx5_get_priv(dev); 744 struct ethtool_cmd edata = { 745 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */ 746 }; 747 struct ifreq ifr; 748 struct rte_eth_link dev_link; 749 int link_speed = 0; 750 751 /* priv_lock() is not taken to allow concurrent calls. */ 752 753 (void)wait_to_complete; 754 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 755 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 756 return -1; 757 } 758 memset(&dev_link, 0, sizeof(dev_link)); 759 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 760 (ifr.ifr_flags & IFF_RUNNING)); 761 ifr.ifr_data = (void *)&edata; 762 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 763 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", 764 strerror(errno)); 765 return -1; 766 } 767 link_speed = ethtool_cmd_speed(&edata); 768 if (link_speed == -1) 769 dev_link.link_speed = 0; 770 else 771 dev_link.link_speed = link_speed; 772 priv->link_speed_capa = 0; 773 if (edata.supported & SUPPORTED_Autoneg) 774 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; 775 if (edata.supported & (SUPPORTED_1000baseT_Full | 776 SUPPORTED_1000baseKX_Full)) 777 priv->link_speed_capa |= ETH_LINK_SPEED_1G; 778 if (edata.supported & SUPPORTED_10000baseKR_Full) 779 priv->link_speed_capa |= ETH_LINK_SPEED_10G; 780 if (edata.supported & (SUPPORTED_40000baseKR4_Full | 781 SUPPORTED_40000baseCR4_Full | 782 SUPPORTED_40000baseSR4_Full | 783 SUPPORTED_40000baseLR4_Full)) 784 priv->link_speed_capa |= ETH_LINK_SPEED_40G; 785 dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? 786 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 787 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 788 ETH_LINK_SPEED_FIXED); 789 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 790 /* Link status changed. */ 791 dev->data->dev_link = dev_link; 792 return 0; 793 } 794 /* Link status is still the same. */ 795 return -1; 796 } 797 798 /** 799 * Retrieve physical link information (unlocked version using new ioctl). 800 * 801 * @param dev 802 * Pointer to Ethernet device structure. 803 * @param wait_to_complete 804 * Wait for request completion (ignored). 805 */ 806 static int 807 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) 808 { 809 struct priv *priv = mlx5_get_priv(dev); 810 struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS }; 811 struct ifreq ifr; 812 struct rte_eth_link dev_link; 813 uint64_t sc; 814 815 (void)wait_to_complete; 816 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 817 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 818 return -1; 819 } 820 memset(&dev_link, 0, sizeof(dev_link)); 821 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 822 (ifr.ifr_flags & IFF_RUNNING)); 823 ifr.ifr_data = (void *)&gcmd; 824 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 825 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", 826 strerror(errno)); 827 return -1; 828 } 829 gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords; 830 831 alignas(struct ethtool_link_settings) 832 uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) + 833 sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3]; 834 struct ethtool_link_settings *ecmd = (void *)data; 835 836 *ecmd = gcmd; 837 ifr.ifr_data = (void *)ecmd; 838 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 839 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", 840 strerror(errno)); 841 return -1; 842 } 843 dev_link.link_speed = ecmd->speed; 844 sc = ecmd->link_mode_masks[0] | 845 ((uint64_t)ecmd->link_mode_masks[1] << 32); 846 priv->link_speed_capa = 0; 847 if (sc & ETHTOOL_LINK_MODE_Autoneg_BIT) 848 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; 849 if (sc & (ETHTOOL_LINK_MODE_1000baseT_Full_BIT | 850 ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)) 851 priv->link_speed_capa |= ETH_LINK_SPEED_1G; 852 if (sc & (ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT | 853 ETHTOOL_LINK_MODE_10000baseKR_Full_BIT | 854 ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)) 855 priv->link_speed_capa |= ETH_LINK_SPEED_10G; 856 if (sc & (ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT | 857 ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)) 858 priv->link_speed_capa |= ETH_LINK_SPEED_20G; 859 if (sc & (ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT | 860 ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT | 861 ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT | 862 ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)) 863 priv->link_speed_capa |= ETH_LINK_SPEED_40G; 864 if (sc & (ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT | 865 ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT | 866 ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT | 867 ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)) 868 priv->link_speed_capa |= ETH_LINK_SPEED_56G; 869 if (sc & (ETHTOOL_LINK_MODE_25000baseCR_Full_BIT | 870 ETHTOOL_LINK_MODE_25000baseKR_Full_BIT | 871 ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)) 872 priv->link_speed_capa |= ETH_LINK_SPEED_25G; 873 if (sc & (ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT | 874 ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)) 875 priv->link_speed_capa |= ETH_LINK_SPEED_50G; 876 if (sc & (ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT | 877 ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT | 878 ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT | 879 ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)) 880 priv->link_speed_capa |= ETH_LINK_SPEED_100G; 881 dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ? 882 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 883 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 884 ETH_LINK_SPEED_FIXED); 885 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 886 /* Link status changed. */ 887 dev->data->dev_link = dev_link; 888 return 0; 889 } 890 /* Link status is still the same. */ 891 return -1; 892 } 893 894 /** 895 * DPDK callback to retrieve physical link information. 896 * 897 * @param dev 898 * Pointer to Ethernet device structure. 899 * @param wait_to_complete 900 * Wait for request completion (ignored). 901 */ 902 int 903 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) 904 { 905 struct utsname utsname; 906 int ver[3]; 907 908 if (uname(&utsname) == -1 || 909 sscanf(utsname.release, "%d.%d.%d", 910 &ver[0], &ver[1], &ver[2]) != 3 || 911 KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0)) 912 return mlx5_link_update_unlocked_gset(dev, wait_to_complete); 913 return mlx5_link_update_unlocked_gs(dev, wait_to_complete); 914 } 915 916 /** 917 * DPDK callback to change the MTU. 918 * 919 * @param dev 920 * Pointer to Ethernet device structure. 921 * @param in_mtu 922 * New MTU. 923 * 924 * @return 925 * 0 on success, negative errno value on failure. 926 */ 927 int 928 mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 929 { 930 struct priv *priv = dev->data->dev_private; 931 uint16_t kern_mtu; 932 int ret = 0; 933 934 if (mlx5_is_secondary()) 935 return -E_RTE_SECONDARY; 936 937 priv_lock(priv); 938 ret = priv_get_mtu(priv, &kern_mtu); 939 if (ret) 940 goto out; 941 /* Set kernel interface MTU first. */ 942 ret = priv_set_mtu(priv, mtu); 943 if (ret) 944 goto out; 945 ret = priv_get_mtu(priv, &kern_mtu); 946 if (ret) 947 goto out; 948 if (kern_mtu == mtu) { 949 priv->mtu = mtu; 950 DEBUG("adapter port %u MTU set to %u", priv->port, mtu); 951 } 952 priv_unlock(priv); 953 return 0; 954 out: 955 ret = errno; 956 WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, 957 strerror(ret)); 958 priv_unlock(priv); 959 assert(ret >= 0); 960 return -ret; 961 } 962 963 /** 964 * DPDK callback to get flow control status. 965 * 966 * @param dev 967 * Pointer to Ethernet device structure. 968 * @param[out] fc_conf 969 * Flow control output buffer. 970 * 971 * @return 972 * 0 on success, negative errno value on failure. 973 */ 974 int 975 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 976 { 977 struct priv *priv = dev->data->dev_private; 978 struct ifreq ifr; 979 struct ethtool_pauseparam ethpause = { 980 .cmd = ETHTOOL_GPAUSEPARAM 981 }; 982 int ret; 983 984 if (mlx5_is_secondary()) 985 return -E_RTE_SECONDARY; 986 987 ifr.ifr_data = (void *)ðpause; 988 priv_lock(priv); 989 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 990 ret = errno; 991 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" 992 " failed: %s", 993 strerror(ret)); 994 goto out; 995 } 996 997 fc_conf->autoneg = ethpause.autoneg; 998 if (ethpause.rx_pause && ethpause.tx_pause) 999 fc_conf->mode = RTE_FC_FULL; 1000 else if (ethpause.rx_pause) 1001 fc_conf->mode = RTE_FC_RX_PAUSE; 1002 else if (ethpause.tx_pause) 1003 fc_conf->mode = RTE_FC_TX_PAUSE; 1004 else 1005 fc_conf->mode = RTE_FC_NONE; 1006 ret = 0; 1007 1008 out: 1009 priv_unlock(priv); 1010 assert(ret >= 0); 1011 return -ret; 1012 } 1013 1014 /** 1015 * DPDK callback to modify flow control parameters. 1016 * 1017 * @param dev 1018 * Pointer to Ethernet device structure. 1019 * @param[in] fc_conf 1020 * Flow control parameters. 1021 * 1022 * @return 1023 * 0 on success, negative errno value on failure. 1024 */ 1025 int 1026 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 1027 { 1028 struct priv *priv = dev->data->dev_private; 1029 struct ifreq ifr; 1030 struct ethtool_pauseparam ethpause = { 1031 .cmd = ETHTOOL_SPAUSEPARAM 1032 }; 1033 int ret; 1034 1035 if (mlx5_is_secondary()) 1036 return -E_RTE_SECONDARY; 1037 1038 ifr.ifr_data = (void *)ðpause; 1039 ethpause.autoneg = fc_conf->autoneg; 1040 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 1041 (fc_conf->mode & RTE_FC_RX_PAUSE)) 1042 ethpause.rx_pause = 1; 1043 else 1044 ethpause.rx_pause = 0; 1045 1046 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 1047 (fc_conf->mode & RTE_FC_TX_PAUSE)) 1048 ethpause.tx_pause = 1; 1049 else 1050 ethpause.tx_pause = 0; 1051 1052 priv_lock(priv); 1053 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 1054 ret = errno; 1055 WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" 1056 " failed: %s", 1057 strerror(ret)); 1058 goto out; 1059 } 1060 ret = 0; 1061 1062 out: 1063 priv_unlock(priv); 1064 assert(ret >= 0); 1065 return -ret; 1066 } 1067 1068 /** 1069 * Get PCI information from struct ibv_device. 1070 * 1071 * @param device 1072 * Pointer to Ethernet device structure. 1073 * @param[out] pci_addr 1074 * PCI bus address output buffer. 1075 * 1076 * @return 1077 * 0 on success, -1 on failure and errno is set. 1078 */ 1079 int 1080 mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, 1081 struct rte_pci_addr *pci_addr) 1082 { 1083 FILE *file; 1084 char line[32]; 1085 MKSTR(path, "%s/device/uevent", device->ibdev_path); 1086 1087 file = fopen(path, "rb"); 1088 if (file == NULL) 1089 return -1; 1090 while (fgets(line, sizeof(line), file) == line) { 1091 size_t len = strlen(line); 1092 int ret; 1093 1094 /* Truncate long lines. */ 1095 if (len == (sizeof(line) - 1)) 1096 while (line[(len - 1)] != '\n') { 1097 ret = fgetc(file); 1098 if (ret == EOF) 1099 break; 1100 line[(len - 1)] = ret; 1101 } 1102 /* Extract information. */ 1103 if (sscanf(line, 1104 "PCI_SLOT_NAME=" 1105 "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", 1106 &pci_addr->domain, 1107 &pci_addr->bus, 1108 &pci_addr->devid, 1109 &pci_addr->function) == 4) { 1110 ret = 0; 1111 break; 1112 } 1113 } 1114 fclose(file); 1115 return 0; 1116 } 1117 1118 /** 1119 * Update the link status. 1120 * 1121 * @param priv 1122 * Pointer to private structure. 1123 * 1124 * @return 1125 * Zero if the callback process can be called immediately. 1126 */ 1127 static int 1128 priv_link_status_update(struct priv *priv) 1129 { 1130 struct rte_eth_link *link = &priv->dev->data->dev_link; 1131 1132 mlx5_link_update(priv->dev, 0); 1133 if (((link->link_speed == 0) && link->link_status) || 1134 ((link->link_speed != 0) && !link->link_status)) { 1135 /* 1136 * Inconsistent status. Event likely occurred before the 1137 * kernel netdevice exposes the new status. 1138 */ 1139 if (!priv->pending_alarm) { 1140 priv->pending_alarm = 1; 1141 rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US, 1142 mlx5_dev_link_status_handler, 1143 priv->dev); 1144 } 1145 return 1; 1146 } else if (unlikely(priv->pending_alarm)) { 1147 /* Link interrupt occurred while alarm is already scheduled. */ 1148 priv->pending_alarm = 0; 1149 rte_eal_alarm_cancel(mlx5_dev_link_status_handler, priv->dev); 1150 } 1151 return 0; 1152 } 1153 1154 /** 1155 * Device status handler. 1156 * 1157 * @param priv 1158 * Pointer to private structure. 1159 * @param events 1160 * Pointer to event flags holder. 1161 * 1162 * @return 1163 * Events bitmap of callback process which can be called immediately. 1164 */ 1165 static uint32_t 1166 priv_dev_status_handler(struct priv *priv) 1167 { 1168 struct ibv_async_event event; 1169 uint32_t ret = 0; 1170 1171 /* Read all message and acknowledge them. */ 1172 for (;;) { 1173 if (ibv_get_async_event(priv->ctx, &event)) 1174 break; 1175 if ((event.event_type == IBV_EVENT_PORT_ACTIVE || 1176 event.event_type == IBV_EVENT_PORT_ERR) && 1177 (priv->dev->data->dev_conf.intr_conf.lsc == 1)) 1178 ret |= (1 << RTE_ETH_EVENT_INTR_LSC); 1179 else if (event.event_type == IBV_EVENT_DEVICE_FATAL && 1180 priv->dev->data->dev_conf.intr_conf.rmv == 1) 1181 ret |= (1 << RTE_ETH_EVENT_INTR_RMV); 1182 else 1183 DEBUG("event type %d on port %d not handled", 1184 event.event_type, event.element.port_num); 1185 ibv_ack_async_event(&event); 1186 } 1187 if (ret & (1 << RTE_ETH_EVENT_INTR_LSC)) 1188 if (priv_link_status_update(priv)) 1189 ret &= ~(1 << RTE_ETH_EVENT_INTR_LSC); 1190 return ret; 1191 } 1192 1193 /** 1194 * Handle delayed link status event. 1195 * 1196 * @param arg 1197 * Registered argument. 1198 */ 1199 void 1200 mlx5_dev_link_status_handler(void *arg) 1201 { 1202 struct rte_eth_dev *dev = arg; 1203 struct priv *priv = dev->data->dev_private; 1204 int ret; 1205 1206 priv_lock(priv); 1207 assert(priv->pending_alarm == 1); 1208 priv->pending_alarm = 0; 1209 ret = priv_link_status_update(priv); 1210 priv_unlock(priv); 1211 if (!ret) 1212 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, 1213 NULL); 1214 } 1215 1216 /** 1217 * Handle interrupts from the NIC. 1218 * 1219 * @param[in] intr_handle 1220 * Interrupt handler. 1221 * @param cb_arg 1222 * Callback argument. 1223 */ 1224 void 1225 mlx5_dev_interrupt_handler(void *cb_arg) 1226 { 1227 struct rte_eth_dev *dev = cb_arg; 1228 struct priv *priv = dev->data->dev_private; 1229 uint32_t events; 1230 1231 priv_lock(priv); 1232 events = priv_dev_status_handler(priv); 1233 priv_unlock(priv); 1234 if (events & (1 << RTE_ETH_EVENT_INTR_LSC)) 1235 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, 1236 NULL); 1237 if (events & (1 << RTE_ETH_EVENT_INTR_RMV)) 1238 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL, 1239 NULL); 1240 } 1241 1242 /** 1243 * Handle interrupts from the socket. 1244 * 1245 * @param cb_arg 1246 * Callback argument. 1247 */ 1248 static void 1249 mlx5_dev_handler_socket(void *cb_arg) 1250 { 1251 struct rte_eth_dev *dev = cb_arg; 1252 struct priv *priv = dev->data->dev_private; 1253 1254 priv_lock(priv); 1255 priv_socket_handle(priv); 1256 priv_unlock(priv); 1257 } 1258 1259 /** 1260 * Uninstall interrupt handler. 1261 * 1262 * @param priv 1263 * Pointer to private structure. 1264 * @param dev 1265 * Pointer to the rte_eth_dev structure. 1266 */ 1267 void 1268 priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) 1269 { 1270 if (dev->data->dev_conf.intr_conf.lsc || 1271 dev->data->dev_conf.intr_conf.rmv) 1272 rte_intr_callback_unregister(&priv->intr_handle, 1273 mlx5_dev_interrupt_handler, dev); 1274 if (priv->primary_socket) 1275 rte_intr_callback_unregister(&priv->intr_handle_socket, 1276 mlx5_dev_handler_socket, dev); 1277 if (priv->pending_alarm) 1278 rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev); 1279 priv->pending_alarm = 0; 1280 priv->intr_handle.fd = 0; 1281 priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; 1282 priv->intr_handle_socket.fd = 0; 1283 priv->intr_handle_socket.type = RTE_INTR_HANDLE_UNKNOWN; 1284 } 1285 1286 /** 1287 * Install interrupt handler. 1288 * 1289 * @param priv 1290 * Pointer to private structure. 1291 * @param dev 1292 * Pointer to the rte_eth_dev structure. 1293 */ 1294 void 1295 priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) 1296 { 1297 int rc, flags; 1298 1299 assert(!mlx5_is_secondary()); 1300 assert(priv->ctx->async_fd > 0); 1301 flags = fcntl(priv->ctx->async_fd, F_GETFL); 1302 rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); 1303 if (rc < 0) { 1304 INFO("failed to change file descriptor async event queue"); 1305 dev->data->dev_conf.intr_conf.lsc = 0; 1306 dev->data->dev_conf.intr_conf.rmv = 0; 1307 } 1308 if (dev->data->dev_conf.intr_conf.lsc || 1309 dev->data->dev_conf.intr_conf.rmv) { 1310 priv->intr_handle.fd = priv->ctx->async_fd; 1311 priv->intr_handle.type = RTE_INTR_HANDLE_EXT; 1312 rte_intr_callback_register(&priv->intr_handle, 1313 mlx5_dev_interrupt_handler, dev); 1314 } 1315 1316 rc = priv_socket_init(priv); 1317 if (!rc && priv->primary_socket) { 1318 priv->intr_handle_socket.fd = priv->primary_socket; 1319 priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT; 1320 rte_intr_callback_register(&priv->intr_handle_socket, 1321 mlx5_dev_handler_socket, dev); 1322 } 1323 } 1324 1325 /** 1326 * Change the link state (UP / DOWN). 1327 * 1328 * @param priv 1329 * Pointer to private data structure. 1330 * @param dev 1331 * Pointer to rte_eth_dev structure. 1332 * @param up 1333 * Nonzero for link up, otherwise link down. 1334 * 1335 * @return 1336 * 0 on success, errno value on failure. 1337 */ 1338 static int 1339 priv_dev_set_link(struct priv *priv, struct rte_eth_dev *dev, int up) 1340 { 1341 int err; 1342 1343 if (up) { 1344 err = priv_set_flags(priv, ~IFF_UP, IFF_UP); 1345 if (err) 1346 return err; 1347 priv_dev_select_tx_function(priv, dev); 1348 priv_dev_select_rx_function(priv, dev); 1349 } else { 1350 err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP); 1351 if (err) 1352 return err; 1353 dev->rx_pkt_burst = removed_rx_burst; 1354 dev->tx_pkt_burst = removed_tx_burst; 1355 } 1356 return 0; 1357 } 1358 1359 /** 1360 * DPDK callback to bring the link DOWN. 1361 * 1362 * @param dev 1363 * Pointer to Ethernet device structure. 1364 * 1365 * @return 1366 * 0 on success, errno value on failure. 1367 */ 1368 int 1369 mlx5_set_link_down(struct rte_eth_dev *dev) 1370 { 1371 struct priv *priv = dev->data->dev_private; 1372 int err; 1373 1374 priv_lock(priv); 1375 err = priv_dev_set_link(priv, dev, 0); 1376 priv_unlock(priv); 1377 return err; 1378 } 1379 1380 /** 1381 * DPDK callback to bring the link UP. 1382 * 1383 * @param dev 1384 * Pointer to Ethernet device structure. 1385 * 1386 * @return 1387 * 0 on success, errno value on failure. 1388 */ 1389 int 1390 mlx5_set_link_up(struct rte_eth_dev *dev) 1391 { 1392 struct priv *priv = dev->data->dev_private; 1393 int err; 1394 1395 priv_lock(priv); 1396 err = priv_dev_set_link(priv, dev, 1); 1397 priv_unlock(priv); 1398 return err; 1399 } 1400 1401 /** 1402 * Configure the TX function to use. 1403 * 1404 * @param priv 1405 * Pointer to private data structure. 1406 * @param dev 1407 * Pointer to rte_eth_dev structure. 1408 */ 1409 void 1410 priv_dev_select_tx_function(struct priv *priv, struct rte_eth_dev *dev) 1411 { 1412 assert(priv != NULL); 1413 assert(dev != NULL); 1414 dev->tx_pkt_burst = mlx5_tx_burst; 1415 /* Select appropriate TX function. */ 1416 if (priv->mps == MLX5_MPW_ENHANCED) { 1417 if (priv_check_vec_tx_support(priv) > 0) { 1418 if (priv_check_raw_vec_tx_support(priv) > 0) 1419 dev->tx_pkt_burst = mlx5_tx_burst_raw_vec; 1420 else 1421 dev->tx_pkt_burst = mlx5_tx_burst_vec; 1422 DEBUG("selected Enhanced MPW TX vectorized function"); 1423 } else { 1424 dev->tx_pkt_burst = mlx5_tx_burst_empw; 1425 DEBUG("selected Enhanced MPW TX function"); 1426 } 1427 } else if (priv->mps && priv->txq_inline) { 1428 dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; 1429 DEBUG("selected MPW inline TX function"); 1430 } else if (priv->mps) { 1431 dev->tx_pkt_burst = mlx5_tx_burst_mpw; 1432 DEBUG("selected MPW TX function"); 1433 } 1434 } 1435 1436 /** 1437 * Configure the RX function to use. 1438 * 1439 * @param priv 1440 * Pointer to private data structure. 1441 * @param dev 1442 * Pointer to rte_eth_dev structure. 1443 */ 1444 void 1445 priv_dev_select_rx_function(struct priv *priv, struct rte_eth_dev *dev) 1446 { 1447 assert(priv != NULL); 1448 assert(dev != NULL); 1449 if (priv_check_vec_rx_support(priv) > 0) { 1450 dev->rx_pkt_burst = mlx5_rx_burst_vec; 1451 DEBUG("selected RX vectorized function"); 1452 } else { 1453 dev->rx_pkt_burst = mlx5_rx_burst; 1454 } 1455 } 1456