1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2015 6WIND S.A. 5 * Copyright 2015 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #define _GNU_SOURCE 35 36 #include <stddef.h> 37 #include <assert.h> 38 #include <unistd.h> 39 #include <stdint.h> 40 #include <stdio.h> 41 #include <string.h> 42 #include <stdlib.h> 43 #include <errno.h> 44 #include <dirent.h> 45 #include <net/if.h> 46 #include <sys/ioctl.h> 47 #include <sys/socket.h> 48 #include <sys/utsname.h> 49 #include <netinet/in.h> 50 #include <linux/ethtool.h> 51 #include <linux/sockios.h> 52 #include <linux/version.h> 53 #include <fcntl.h> 54 #include <stdalign.h> 55 #include <sys/un.h> 56 57 #include <rte_atomic.h> 58 #include <rte_ethdev.h> 59 #include <rte_bus_pci.h> 60 #include <rte_mbuf.h> 61 #include <rte_common.h> 62 #include <rte_interrupts.h> 63 #include <rte_alarm.h> 64 #include <rte_malloc.h> 65 66 #include "mlx5.h" 67 #include "mlx5_rxtx.h" 68 #include "mlx5_utils.h" 69 70 /* Add defines in case the running kernel is not the same as user headers. */ 71 #ifndef ETHTOOL_GLINKSETTINGS 72 struct ethtool_link_settings { 73 uint32_t cmd; 74 uint32_t speed; 75 uint8_t duplex; 76 uint8_t port; 77 uint8_t phy_address; 78 uint8_t autoneg; 79 uint8_t mdio_support; 80 uint8_t eth_to_mdix; 81 uint8_t eth_tp_mdix_ctrl; 82 int8_t link_mode_masks_nwords; 83 uint32_t reserved[8]; 84 uint32_t link_mode_masks[]; 85 }; 86 87 #define ETHTOOL_GLINKSETTINGS 0x0000004c 88 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5 89 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6 90 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17 91 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18 92 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19 93 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20 94 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21 95 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22 96 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23 97 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24 98 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25 99 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26 100 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27 101 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28 102 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29 103 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30 104 #endif 105 #ifndef HAVE_ETHTOOL_LINK_MODE_25G 106 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31 107 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32 108 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33 109 #endif 110 #ifndef HAVE_ETHTOOL_LINK_MODE_50G 111 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34 112 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35 113 #endif 114 #ifndef HAVE_ETHTOOL_LINK_MODE_100G 115 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36 116 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37 117 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38 118 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39 119 #endif 120 121 /** 122 * Return private structure associated with an Ethernet device. 123 * 124 * @param dev 125 * Pointer to Ethernet device structure. 126 * 127 * @return 128 * Pointer to private structure. 129 */ 130 struct priv * 131 mlx5_get_priv(struct rte_eth_dev *dev) 132 { 133 return dev->data->dev_private; 134 } 135 136 /** 137 * Check if running as a secondary process. 138 * 139 * @return 140 * Nonzero if running as a secondary process. 141 */ 142 inline int 143 mlx5_is_secondary(void) 144 { 145 return rte_eal_process_type() == RTE_PROC_SECONDARY; 146 } 147 148 /** 149 * Get interface name from private structure. 150 * 151 * @param[in] priv 152 * Pointer to private structure. 153 * @param[out] ifname 154 * Interface name output buffer. 155 * 156 * @return 157 * 0 on success, -1 on failure and errno is set. 158 */ 159 int 160 priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) 161 { 162 DIR *dir; 163 struct dirent *dent; 164 unsigned int dev_type = 0; 165 unsigned int dev_port_prev = ~0u; 166 char match[IF_NAMESIZE] = ""; 167 168 { 169 MKSTR(path, "%s/device/net", priv->ibdev_path); 170 171 dir = opendir(path); 172 if (dir == NULL) 173 return -1; 174 } 175 while ((dent = readdir(dir)) != NULL) { 176 char *name = dent->d_name; 177 FILE *file; 178 unsigned int dev_port; 179 int r; 180 181 if ((name[0] == '.') && 182 ((name[1] == '\0') || 183 ((name[1] == '.') && (name[2] == '\0')))) 184 continue; 185 186 MKSTR(path, "%s/device/net/%s/%s", 187 priv->ibdev_path, name, 188 (dev_type ? "dev_id" : "dev_port")); 189 190 file = fopen(path, "rb"); 191 if (file == NULL) { 192 if (errno != ENOENT) 193 continue; 194 /* 195 * Switch to dev_id when dev_port does not exist as 196 * is the case with Linux kernel versions < 3.15. 197 */ 198 try_dev_id: 199 match[0] = '\0'; 200 if (dev_type) 201 break; 202 dev_type = 1; 203 dev_port_prev = ~0u; 204 rewinddir(dir); 205 continue; 206 } 207 r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port); 208 fclose(file); 209 if (r != 1) 210 continue; 211 /* 212 * Switch to dev_id when dev_port returns the same value for 213 * all ports. May happen when using a MOFED release older than 214 * 3.0 with a Linux kernel >= 3.15. 215 */ 216 if (dev_port == dev_port_prev) 217 goto try_dev_id; 218 dev_port_prev = dev_port; 219 if (dev_port == (priv->port - 1u)) 220 snprintf(match, sizeof(match), "%s", name); 221 } 222 closedir(dir); 223 if (match[0] == '\0') 224 return -1; 225 strncpy(*ifname, match, sizeof(*ifname)); 226 return 0; 227 } 228 229 /** 230 * Check if the counter is located on ib counters file. 231 * 232 * @param[in] cntr 233 * Counter name. 234 * 235 * @return 236 * 1 if counter is located on ib counters file , 0 otherwise. 237 */ 238 int 239 priv_is_ib_cntr(const char *cntr) 240 { 241 if (!strcmp(cntr, "out_of_buffer")) 242 return 1; 243 return 0; 244 } 245 246 /** 247 * Read from sysfs entry. 248 * 249 * @param[in] priv 250 * Pointer to private structure. 251 * @param[in] entry 252 * Entry name relative to sysfs path. 253 * @param[out] buf 254 * Data output buffer. 255 * @param size 256 * Buffer size. 257 * 258 * @return 259 * 0 on success, -1 on failure and errno is set. 260 */ 261 static int 262 priv_sysfs_read(const struct priv *priv, const char *entry, 263 char *buf, size_t size) 264 { 265 char ifname[IF_NAMESIZE]; 266 FILE *file; 267 int ret; 268 int err; 269 270 if (priv_get_ifname(priv, &ifname)) 271 return -1; 272 273 if (priv_is_ib_cntr(entry)) { 274 MKSTR(path, "%s/ports/1/hw_counters/%s", 275 priv->ibdev_path, entry); 276 file = fopen(path, "rb"); 277 } else { 278 MKSTR(path, "%s/device/net/%s/%s", 279 priv->ibdev_path, ifname, entry); 280 file = fopen(path, "rb"); 281 } 282 if (file == NULL) 283 return -1; 284 ret = fread(buf, 1, size, file); 285 err = errno; 286 if (((size_t)ret < size) && (ferror(file))) 287 ret = -1; 288 else 289 ret = size; 290 fclose(file); 291 errno = err; 292 return ret; 293 } 294 295 /** 296 * Write to sysfs entry. 297 * 298 * @param[in] priv 299 * Pointer to private structure. 300 * @param[in] entry 301 * Entry name relative to sysfs path. 302 * @param[in] buf 303 * Data buffer. 304 * @param size 305 * Buffer size. 306 * 307 * @return 308 * 0 on success, -1 on failure and errno is set. 309 */ 310 static int 311 priv_sysfs_write(const struct priv *priv, const char *entry, 312 char *buf, size_t size) 313 { 314 char ifname[IF_NAMESIZE]; 315 FILE *file; 316 int ret; 317 int err; 318 319 if (priv_get_ifname(priv, &ifname)) 320 return -1; 321 322 MKSTR(path, "%s/device/net/%s/%s", priv->ibdev_path, ifname, entry); 323 324 file = fopen(path, "wb"); 325 if (file == NULL) 326 return -1; 327 ret = fwrite(buf, 1, size, file); 328 err = errno; 329 if (((size_t)ret < size) || (ferror(file))) 330 ret = -1; 331 else 332 ret = size; 333 fclose(file); 334 errno = err; 335 return ret; 336 } 337 338 /** 339 * Get unsigned long sysfs property. 340 * 341 * @param priv 342 * Pointer to private structure. 343 * @param[in] name 344 * Entry name relative to sysfs path. 345 * @param[out] value 346 * Value output buffer. 347 * 348 * @return 349 * 0 on success, -1 on failure and errno is set. 350 */ 351 static int 352 priv_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) 353 { 354 int ret; 355 unsigned long value_ret; 356 char value_str[32]; 357 358 ret = priv_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); 359 if (ret == -1) { 360 DEBUG("cannot read %s value from sysfs: %s", 361 name, strerror(errno)); 362 return -1; 363 } 364 value_str[ret] = '\0'; 365 errno = 0; 366 value_ret = strtoul(value_str, NULL, 0); 367 if (errno) { 368 DEBUG("invalid %s value `%s': %s", name, value_str, 369 strerror(errno)); 370 return -1; 371 } 372 *value = value_ret; 373 return 0; 374 } 375 376 /** 377 * Set unsigned long sysfs property. 378 * 379 * @param priv 380 * Pointer to private structure. 381 * @param[in] name 382 * Entry name relative to sysfs path. 383 * @param value 384 * Value to set. 385 * 386 * @return 387 * 0 on success, -1 on failure and errno is set. 388 */ 389 static int 390 priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) 391 { 392 int ret; 393 MKSTR(value_str, "%lu", value); 394 395 ret = priv_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); 396 if (ret == -1) { 397 DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", 398 name, value_str, value, strerror(errno)); 399 return -1; 400 } 401 return 0; 402 } 403 404 /** 405 * Perform ifreq ioctl() on associated Ethernet device. 406 * 407 * @param[in] priv 408 * Pointer to private structure. 409 * @param req 410 * Request number to pass to ioctl(). 411 * @param[out] ifr 412 * Interface request structure output buffer. 413 * 414 * @return 415 * 0 on success, -1 on failure and errno is set. 416 */ 417 int 418 priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) 419 { 420 int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); 421 int ret = -1; 422 423 if (sock == -1) 424 return ret; 425 if (priv_get_ifname(priv, &ifr->ifr_name) == 0) 426 ret = ioctl(sock, req, ifr); 427 close(sock); 428 return ret; 429 } 430 431 /** 432 * Return the number of active VFs for the current device. 433 * 434 * @param[in] priv 435 * Pointer to private structure. 436 * @param[out] num_vfs 437 * Number of active VFs. 438 * 439 * @return 440 * 0 on success, -1 on failure and errno is set. 441 */ 442 int 443 priv_get_num_vfs(struct priv *priv, uint16_t *num_vfs) 444 { 445 /* The sysfs entry name depends on the operating system. */ 446 const char **name = (const char *[]){ 447 "device/sriov_numvfs", 448 "device/mlx5_num_vfs", 449 NULL, 450 }; 451 int ret; 452 453 do { 454 unsigned long ulong_num_vfs; 455 456 ret = priv_get_sysfs_ulong(priv, *name, &ulong_num_vfs); 457 if (!ret) 458 *num_vfs = ulong_num_vfs; 459 } while (*(++name) && ret); 460 return ret; 461 } 462 463 /** 464 * Get device MTU. 465 * 466 * @param priv 467 * Pointer to private structure. 468 * @param[out] mtu 469 * MTU value output buffer. 470 * 471 * @return 472 * 0 on success, -1 on failure and errno is set. 473 */ 474 int 475 priv_get_mtu(struct priv *priv, uint16_t *mtu) 476 { 477 unsigned long ulong_mtu; 478 479 if (priv_get_sysfs_ulong(priv, "mtu", &ulong_mtu) == -1) 480 return -1; 481 *mtu = ulong_mtu; 482 return 0; 483 } 484 485 /** 486 * Read device counter from sysfs. 487 * 488 * @param priv 489 * Pointer to private structure. 490 * @param name 491 * Counter name. 492 * @param[out] cntr 493 * Counter output buffer. 494 * 495 * @return 496 * 0 on success, -1 on failure and errno is set. 497 */ 498 int 499 priv_get_cntr_sysfs(struct priv *priv, const char *name, uint64_t *cntr) 500 { 501 unsigned long ulong_ctr; 502 503 if (priv_get_sysfs_ulong(priv, name, &ulong_ctr) == -1) 504 return -1; 505 *cntr = ulong_ctr; 506 return 0; 507 } 508 509 /** 510 * Set device MTU. 511 * 512 * @param priv 513 * Pointer to private structure. 514 * @param mtu 515 * MTU value to set. 516 * 517 * @return 518 * 0 on success, -1 on failure and errno is set. 519 */ 520 static int 521 priv_set_mtu(struct priv *priv, uint16_t mtu) 522 { 523 uint16_t new_mtu; 524 525 if (priv_set_sysfs_ulong(priv, "mtu", mtu) || 526 priv_get_mtu(priv, &new_mtu)) 527 return -1; 528 if (new_mtu == mtu) 529 return 0; 530 errno = EINVAL; 531 return -1; 532 } 533 534 /** 535 * Set device flags. 536 * 537 * @param priv 538 * Pointer to private structure. 539 * @param keep 540 * Bitmask for flags that must remain untouched. 541 * @param flags 542 * Bitmask for flags to modify. 543 * 544 * @return 545 * 0 on success, -1 on failure and errno is set. 546 */ 547 int 548 priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) 549 { 550 unsigned long tmp; 551 552 if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1) 553 return -1; 554 tmp &= keep; 555 tmp |= (flags & (~keep)); 556 return priv_set_sysfs_ulong(priv, "flags", tmp); 557 } 558 559 /** 560 * Ethernet device configuration. 561 * 562 * Prepare the driver for a given number of TX and RX queues. 563 * 564 * @param dev 565 * Pointer to Ethernet device structure. 566 * 567 * @return 568 * 0 on success, errno value on failure. 569 */ 570 static int 571 dev_configure(struct rte_eth_dev *dev) 572 { 573 struct priv *priv = dev->data->dev_private; 574 unsigned int rxqs_n = dev->data->nb_rx_queues; 575 unsigned int txqs_n = dev->data->nb_tx_queues; 576 unsigned int i; 577 unsigned int j; 578 unsigned int reta_idx_n; 579 const uint8_t use_app_rss_key = 580 !!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len; 581 582 if (use_app_rss_key && 583 (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len != 584 rss_hash_default_key_len)) { 585 /* MLX5 RSS only support 40bytes key. */ 586 return EINVAL; 587 } 588 priv->rss_conf.rss_key = 589 rte_realloc(priv->rss_conf.rss_key, 590 rss_hash_default_key_len, 0); 591 if (!priv->rss_conf.rss_key) { 592 ERROR("cannot allocate RSS hash key memory (%u)", rxqs_n); 593 return ENOMEM; 594 } 595 memcpy(priv->rss_conf.rss_key, 596 use_app_rss_key ? 597 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key : 598 rss_hash_default_key, 599 rss_hash_default_key_len); 600 priv->rss_conf.rss_key_len = rss_hash_default_key_len; 601 priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; 602 priv->rxqs = (void *)dev->data->rx_queues; 603 priv->txqs = (void *)dev->data->tx_queues; 604 if (txqs_n != priv->txqs_n) { 605 INFO("%p: TX queues number update: %u -> %u", 606 (void *)dev, priv->txqs_n, txqs_n); 607 priv->txqs_n = txqs_n; 608 } 609 if (rxqs_n > priv->ind_table_max_size) { 610 ERROR("cannot handle this many RX queues (%u)", rxqs_n); 611 return EINVAL; 612 } 613 if (rxqs_n == priv->rxqs_n) 614 return 0; 615 INFO("%p: RX queues number update: %u -> %u", 616 (void *)dev, priv->rxqs_n, rxqs_n); 617 priv->rxqs_n = rxqs_n; 618 /* If the requested number of RX queues is not a power of two, use the 619 * maximum indirection table size for better balancing. 620 * The result is always rounded to the next power of two. */ 621 reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ? 622 priv->ind_table_max_size : 623 rxqs_n)); 624 if (priv_rss_reta_index_resize(priv, reta_idx_n)) 625 return ENOMEM; 626 /* When the number of RX queues is not a power of two, the remaining 627 * table entries are padded with reused WQs and hashes are not spread 628 * uniformly. */ 629 for (i = 0, j = 0; (i != reta_idx_n); ++i) { 630 (*priv->reta_idx)[i] = j; 631 if (++j == rxqs_n) 632 j = 0; 633 } 634 return 0; 635 } 636 637 /** 638 * DPDK callback for Ethernet device configuration. 639 * 640 * @param dev 641 * Pointer to Ethernet device structure. 642 * 643 * @return 644 * 0 on success, negative errno value on failure. 645 */ 646 int 647 mlx5_dev_configure(struct rte_eth_dev *dev) 648 { 649 struct priv *priv = dev->data->dev_private; 650 int ret; 651 652 if (mlx5_is_secondary()) 653 return -E_RTE_SECONDARY; 654 655 priv_lock(priv); 656 ret = dev_configure(dev); 657 assert(ret >= 0); 658 priv_unlock(priv); 659 return -ret; 660 } 661 662 /** 663 * DPDK callback to get information about the device. 664 * 665 * @param dev 666 * Pointer to Ethernet device structure. 667 * @param[out] info 668 * Info structure output buffer. 669 */ 670 void 671 mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) 672 { 673 struct priv *priv = mlx5_get_priv(dev); 674 unsigned int max; 675 char ifname[IF_NAMESIZE]; 676 677 info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 678 679 priv_lock(priv); 680 /* FIXME: we should ask the device for these values. */ 681 info->min_rx_bufsize = 32; 682 info->max_rx_pktlen = 65536; 683 /* 684 * Since we need one CQ per QP, the limit is the minimum number 685 * between the two values. 686 */ 687 max = RTE_MIN(priv->device_attr.orig_attr.max_cq, 688 priv->device_attr.orig_attr.max_qp); 689 /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */ 690 if (max >= 65535) 691 max = 65535; 692 info->max_rx_queues = max; 693 info->max_tx_queues = max; 694 info->max_mac_addrs = RTE_DIM(priv->mac); 695 info->rx_offload_capa = 696 (priv->hw_csum ? 697 (DEV_RX_OFFLOAD_IPV4_CKSUM | 698 DEV_RX_OFFLOAD_UDP_CKSUM | 699 DEV_RX_OFFLOAD_TCP_CKSUM) : 700 0) | 701 (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0) | 702 DEV_RX_OFFLOAD_TIMESTAMP; 703 704 if (!priv->mps) 705 info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; 706 if (priv->hw_csum) 707 info->tx_offload_capa |= 708 (DEV_TX_OFFLOAD_IPV4_CKSUM | 709 DEV_TX_OFFLOAD_UDP_CKSUM | 710 DEV_TX_OFFLOAD_TCP_CKSUM); 711 if (priv->tso) 712 info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 713 if (priv->tunnel_en) 714 info->tx_offload_capa |= (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 715 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 716 DEV_TX_OFFLOAD_GRE_TNL_TSO); 717 if (priv_get_ifname(priv, &ifname) == 0) 718 info->if_index = if_nametoindex(ifname); 719 info->reta_size = priv->reta_idx_n ? 720 priv->reta_idx_n : priv->ind_table_max_size; 721 info->hash_key_size = priv->rss_conf.rss_key_len; 722 info->speed_capa = priv->link_speed_capa; 723 priv_unlock(priv); 724 } 725 726 const uint32_t * 727 mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) 728 { 729 static const uint32_t ptypes[] = { 730 /* refers to rxq_cq_to_pkt_type() */ 731 RTE_PTYPE_L2_ETHER, 732 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 733 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 734 RTE_PTYPE_L4_NONFRAG, 735 RTE_PTYPE_L4_FRAG, 736 RTE_PTYPE_L4_TCP, 737 RTE_PTYPE_L4_UDP, 738 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 739 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 740 RTE_PTYPE_INNER_L4_NONFRAG, 741 RTE_PTYPE_INNER_L4_FRAG, 742 RTE_PTYPE_INNER_L4_TCP, 743 RTE_PTYPE_INNER_L4_UDP, 744 RTE_PTYPE_UNKNOWN 745 }; 746 747 if (dev->rx_pkt_burst == mlx5_rx_burst || 748 dev->rx_pkt_burst == mlx5_rx_burst_vec) 749 return ptypes; 750 return NULL; 751 } 752 753 /** 754 * DPDK callback to retrieve physical link information. 755 * 756 * @param dev 757 * Pointer to Ethernet device structure. 758 * @param wait_to_complete 759 * Wait for request completion (ignored). 760 */ 761 static int 762 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) 763 { 764 struct priv *priv = mlx5_get_priv(dev); 765 struct ethtool_cmd edata = { 766 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */ 767 }; 768 struct ifreq ifr; 769 struct rte_eth_link dev_link; 770 int link_speed = 0; 771 772 /* priv_lock() is not taken to allow concurrent calls. */ 773 774 (void)wait_to_complete; 775 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 776 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 777 return -1; 778 } 779 memset(&dev_link, 0, sizeof(dev_link)); 780 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 781 (ifr.ifr_flags & IFF_RUNNING)); 782 ifr.ifr_data = (void *)&edata; 783 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 784 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", 785 strerror(errno)); 786 return -1; 787 } 788 link_speed = ethtool_cmd_speed(&edata); 789 if (link_speed == -1) 790 dev_link.link_speed = 0; 791 else 792 dev_link.link_speed = link_speed; 793 priv->link_speed_capa = 0; 794 if (edata.supported & SUPPORTED_Autoneg) 795 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; 796 if (edata.supported & (SUPPORTED_1000baseT_Full | 797 SUPPORTED_1000baseKX_Full)) 798 priv->link_speed_capa |= ETH_LINK_SPEED_1G; 799 if (edata.supported & SUPPORTED_10000baseKR_Full) 800 priv->link_speed_capa |= ETH_LINK_SPEED_10G; 801 if (edata.supported & (SUPPORTED_40000baseKR4_Full | 802 SUPPORTED_40000baseCR4_Full | 803 SUPPORTED_40000baseSR4_Full | 804 SUPPORTED_40000baseLR4_Full)) 805 priv->link_speed_capa |= ETH_LINK_SPEED_40G; 806 dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? 807 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 808 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 809 ETH_LINK_SPEED_FIXED); 810 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 811 /* Link status changed. */ 812 dev->data->dev_link = dev_link; 813 return 0; 814 } 815 /* Link status is still the same. */ 816 return -1; 817 } 818 819 /** 820 * Retrieve physical link information (unlocked version using new ioctl). 821 * 822 * @param dev 823 * Pointer to Ethernet device structure. 824 * @param wait_to_complete 825 * Wait for request completion (ignored). 826 */ 827 static int 828 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) 829 { 830 struct priv *priv = mlx5_get_priv(dev); 831 struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS }; 832 struct ifreq ifr; 833 struct rte_eth_link dev_link; 834 uint64_t sc; 835 836 (void)wait_to_complete; 837 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 838 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 839 return -1; 840 } 841 memset(&dev_link, 0, sizeof(dev_link)); 842 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 843 (ifr.ifr_flags & IFF_RUNNING)); 844 ifr.ifr_data = (void *)&gcmd; 845 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 846 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", 847 strerror(errno)); 848 return -1; 849 } 850 gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords; 851 852 alignas(struct ethtool_link_settings) 853 uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) + 854 sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3]; 855 struct ethtool_link_settings *ecmd = (void *)data; 856 857 *ecmd = gcmd; 858 ifr.ifr_data = (void *)ecmd; 859 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 860 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", 861 strerror(errno)); 862 return -1; 863 } 864 dev_link.link_speed = ecmd->speed; 865 sc = ecmd->link_mode_masks[0] | 866 ((uint64_t)ecmd->link_mode_masks[1] << 32); 867 priv->link_speed_capa = 0; 868 if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT)) 869 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; 870 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) | 871 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT))) 872 priv->link_speed_capa |= ETH_LINK_SPEED_1G; 873 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) | 874 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) | 875 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT))) 876 priv->link_speed_capa |= ETH_LINK_SPEED_10G; 877 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) | 878 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT))) 879 priv->link_speed_capa |= ETH_LINK_SPEED_20G; 880 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) | 881 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) | 882 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) | 883 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT))) 884 priv->link_speed_capa |= ETH_LINK_SPEED_40G; 885 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) | 886 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) | 887 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) | 888 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT))) 889 priv->link_speed_capa |= ETH_LINK_SPEED_56G; 890 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) | 891 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) | 892 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT))) 893 priv->link_speed_capa |= ETH_LINK_SPEED_25G; 894 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) | 895 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT))) 896 priv->link_speed_capa |= ETH_LINK_SPEED_50G; 897 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) | 898 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) | 899 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) | 900 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT))) 901 priv->link_speed_capa |= ETH_LINK_SPEED_100G; 902 dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ? 903 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 904 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 905 ETH_LINK_SPEED_FIXED); 906 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 907 /* Link status changed. */ 908 dev->data->dev_link = dev_link; 909 return 0; 910 } 911 /* Link status is still the same. */ 912 return -1; 913 } 914 915 /** 916 * DPDK callback to retrieve physical link information. 917 * 918 * @param dev 919 * Pointer to Ethernet device structure. 920 * @param wait_to_complete 921 * Wait for request completion (ignored). 922 */ 923 int 924 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) 925 { 926 struct utsname utsname; 927 int ver[3]; 928 929 if (uname(&utsname) == -1 || 930 sscanf(utsname.release, "%d.%d.%d", 931 &ver[0], &ver[1], &ver[2]) != 3 || 932 KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0)) 933 return mlx5_link_update_unlocked_gset(dev, wait_to_complete); 934 return mlx5_link_update_unlocked_gs(dev, wait_to_complete); 935 } 936 937 /** 938 * DPDK callback to change the MTU. 939 * 940 * @param dev 941 * Pointer to Ethernet device structure. 942 * @param in_mtu 943 * New MTU. 944 * 945 * @return 946 * 0 on success, negative errno value on failure. 947 */ 948 int 949 mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 950 { 951 struct priv *priv = dev->data->dev_private; 952 uint16_t kern_mtu; 953 int ret = 0; 954 955 if (mlx5_is_secondary()) 956 return -E_RTE_SECONDARY; 957 958 priv_lock(priv); 959 ret = priv_get_mtu(priv, &kern_mtu); 960 if (ret) 961 goto out; 962 /* Set kernel interface MTU first. */ 963 ret = priv_set_mtu(priv, mtu); 964 if (ret) 965 goto out; 966 ret = priv_get_mtu(priv, &kern_mtu); 967 if (ret) 968 goto out; 969 if (kern_mtu == mtu) { 970 priv->mtu = mtu; 971 DEBUG("adapter port %u MTU set to %u", priv->port, mtu); 972 } 973 priv_unlock(priv); 974 return 0; 975 out: 976 ret = errno; 977 WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, 978 strerror(ret)); 979 priv_unlock(priv); 980 assert(ret >= 0); 981 return -ret; 982 } 983 984 /** 985 * DPDK callback to get flow control status. 986 * 987 * @param dev 988 * Pointer to Ethernet device structure. 989 * @param[out] fc_conf 990 * Flow control output buffer. 991 * 992 * @return 993 * 0 on success, negative errno value on failure. 994 */ 995 int 996 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 997 { 998 struct priv *priv = dev->data->dev_private; 999 struct ifreq ifr; 1000 struct ethtool_pauseparam ethpause = { 1001 .cmd = ETHTOOL_GPAUSEPARAM 1002 }; 1003 int ret; 1004 1005 if (mlx5_is_secondary()) 1006 return -E_RTE_SECONDARY; 1007 1008 ifr.ifr_data = (void *)ðpause; 1009 priv_lock(priv); 1010 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 1011 ret = errno; 1012 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" 1013 " failed: %s", 1014 strerror(ret)); 1015 goto out; 1016 } 1017 1018 fc_conf->autoneg = ethpause.autoneg; 1019 if (ethpause.rx_pause && ethpause.tx_pause) 1020 fc_conf->mode = RTE_FC_FULL; 1021 else if (ethpause.rx_pause) 1022 fc_conf->mode = RTE_FC_RX_PAUSE; 1023 else if (ethpause.tx_pause) 1024 fc_conf->mode = RTE_FC_TX_PAUSE; 1025 else 1026 fc_conf->mode = RTE_FC_NONE; 1027 ret = 0; 1028 1029 out: 1030 priv_unlock(priv); 1031 assert(ret >= 0); 1032 return -ret; 1033 } 1034 1035 /** 1036 * DPDK callback to modify flow control parameters. 1037 * 1038 * @param dev 1039 * Pointer to Ethernet device structure. 1040 * @param[in] fc_conf 1041 * Flow control parameters. 1042 * 1043 * @return 1044 * 0 on success, negative errno value on failure. 1045 */ 1046 int 1047 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 1048 { 1049 struct priv *priv = dev->data->dev_private; 1050 struct ifreq ifr; 1051 struct ethtool_pauseparam ethpause = { 1052 .cmd = ETHTOOL_SPAUSEPARAM 1053 }; 1054 int ret; 1055 1056 if (mlx5_is_secondary()) 1057 return -E_RTE_SECONDARY; 1058 1059 ifr.ifr_data = (void *)ðpause; 1060 ethpause.autoneg = fc_conf->autoneg; 1061 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 1062 (fc_conf->mode & RTE_FC_RX_PAUSE)) 1063 ethpause.rx_pause = 1; 1064 else 1065 ethpause.rx_pause = 0; 1066 1067 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 1068 (fc_conf->mode & RTE_FC_TX_PAUSE)) 1069 ethpause.tx_pause = 1; 1070 else 1071 ethpause.tx_pause = 0; 1072 1073 priv_lock(priv); 1074 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 1075 ret = errno; 1076 WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" 1077 " failed: %s", 1078 strerror(ret)); 1079 goto out; 1080 } 1081 ret = 0; 1082 1083 out: 1084 priv_unlock(priv); 1085 assert(ret >= 0); 1086 return -ret; 1087 } 1088 1089 /** 1090 * Get PCI information from struct ibv_device. 1091 * 1092 * @param device 1093 * Pointer to Ethernet device structure. 1094 * @param[out] pci_addr 1095 * PCI bus address output buffer. 1096 * 1097 * @return 1098 * 0 on success, -1 on failure and errno is set. 1099 */ 1100 int 1101 mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, 1102 struct rte_pci_addr *pci_addr) 1103 { 1104 FILE *file; 1105 char line[32]; 1106 MKSTR(path, "%s/device/uevent", device->ibdev_path); 1107 1108 file = fopen(path, "rb"); 1109 if (file == NULL) 1110 return -1; 1111 while (fgets(line, sizeof(line), file) == line) { 1112 size_t len = strlen(line); 1113 int ret; 1114 1115 /* Truncate long lines. */ 1116 if (len == (sizeof(line) - 1)) 1117 while (line[(len - 1)] != '\n') { 1118 ret = fgetc(file); 1119 if (ret == EOF) 1120 break; 1121 line[(len - 1)] = ret; 1122 } 1123 /* Extract information. */ 1124 if (sscanf(line, 1125 "PCI_SLOT_NAME=" 1126 "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", 1127 &pci_addr->domain, 1128 &pci_addr->bus, 1129 &pci_addr->devid, 1130 &pci_addr->function) == 4) { 1131 ret = 0; 1132 break; 1133 } 1134 } 1135 fclose(file); 1136 return 0; 1137 } 1138 1139 /** 1140 * Update the link status. 1141 * 1142 * @param priv 1143 * Pointer to private structure. 1144 * 1145 * @return 1146 * Zero if the callback process can be called immediately. 1147 */ 1148 static int 1149 priv_link_status_update(struct priv *priv) 1150 { 1151 struct rte_eth_link *link = &priv->dev->data->dev_link; 1152 1153 mlx5_link_update(priv->dev, 0); 1154 if (((link->link_speed == 0) && link->link_status) || 1155 ((link->link_speed != 0) && !link->link_status)) { 1156 /* 1157 * Inconsistent status. Event likely occurred before the 1158 * kernel netdevice exposes the new status. 1159 */ 1160 if (!priv->pending_alarm) { 1161 priv->pending_alarm = 1; 1162 rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US, 1163 mlx5_dev_link_status_handler, 1164 priv->dev); 1165 } 1166 return 1; 1167 } else if (unlikely(priv->pending_alarm)) { 1168 /* Link interrupt occurred while alarm is already scheduled. */ 1169 priv->pending_alarm = 0; 1170 rte_eal_alarm_cancel(mlx5_dev_link_status_handler, priv->dev); 1171 } 1172 return 0; 1173 } 1174 1175 /** 1176 * Device status handler. 1177 * 1178 * @param priv 1179 * Pointer to private structure. 1180 * @param events 1181 * Pointer to event flags holder. 1182 * 1183 * @return 1184 * Events bitmap of callback process which can be called immediately. 1185 */ 1186 static uint32_t 1187 priv_dev_status_handler(struct priv *priv) 1188 { 1189 struct ibv_async_event event; 1190 uint32_t ret = 0; 1191 1192 /* Read all message and acknowledge them. */ 1193 for (;;) { 1194 if (ibv_get_async_event(priv->ctx, &event)) 1195 break; 1196 if ((event.event_type == IBV_EVENT_PORT_ACTIVE || 1197 event.event_type == IBV_EVENT_PORT_ERR) && 1198 (priv->dev->data->dev_conf.intr_conf.lsc == 1)) 1199 ret |= (1 << RTE_ETH_EVENT_INTR_LSC); 1200 else if (event.event_type == IBV_EVENT_DEVICE_FATAL && 1201 priv->dev->data->dev_conf.intr_conf.rmv == 1) 1202 ret |= (1 << RTE_ETH_EVENT_INTR_RMV); 1203 else 1204 DEBUG("event type %d on port %d not handled", 1205 event.event_type, event.element.port_num); 1206 ibv_ack_async_event(&event); 1207 } 1208 if (ret & (1 << RTE_ETH_EVENT_INTR_LSC)) 1209 if (priv_link_status_update(priv)) 1210 ret &= ~(1 << RTE_ETH_EVENT_INTR_LSC); 1211 return ret; 1212 } 1213 1214 /** 1215 * Handle delayed link status event. 1216 * 1217 * @param arg 1218 * Registered argument. 1219 */ 1220 void 1221 mlx5_dev_link_status_handler(void *arg) 1222 { 1223 struct rte_eth_dev *dev = arg; 1224 struct priv *priv = dev->data->dev_private; 1225 int ret; 1226 1227 priv_lock(priv); 1228 assert(priv->pending_alarm == 1); 1229 priv->pending_alarm = 0; 1230 ret = priv_link_status_update(priv); 1231 priv_unlock(priv); 1232 if (!ret) 1233 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, 1234 NULL); 1235 } 1236 1237 /** 1238 * Handle interrupts from the NIC. 1239 * 1240 * @param[in] intr_handle 1241 * Interrupt handler. 1242 * @param cb_arg 1243 * Callback argument. 1244 */ 1245 void 1246 mlx5_dev_interrupt_handler(void *cb_arg) 1247 { 1248 struct rte_eth_dev *dev = cb_arg; 1249 struct priv *priv = dev->data->dev_private; 1250 uint32_t events; 1251 1252 priv_lock(priv); 1253 events = priv_dev_status_handler(priv); 1254 priv_unlock(priv); 1255 if (events & (1 << RTE_ETH_EVENT_INTR_LSC)) 1256 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, 1257 NULL); 1258 if (events & (1 << RTE_ETH_EVENT_INTR_RMV)) 1259 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL, 1260 NULL); 1261 } 1262 1263 /** 1264 * Handle interrupts from the socket. 1265 * 1266 * @param cb_arg 1267 * Callback argument. 1268 */ 1269 static void 1270 mlx5_dev_handler_socket(void *cb_arg) 1271 { 1272 struct rte_eth_dev *dev = cb_arg; 1273 struct priv *priv = dev->data->dev_private; 1274 1275 priv_lock(priv); 1276 priv_socket_handle(priv); 1277 priv_unlock(priv); 1278 } 1279 1280 /** 1281 * Uninstall interrupt handler. 1282 * 1283 * @param priv 1284 * Pointer to private structure. 1285 * @param dev 1286 * Pointer to the rte_eth_dev structure. 1287 */ 1288 void 1289 priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) 1290 { 1291 if (dev->data->dev_conf.intr_conf.lsc || 1292 dev->data->dev_conf.intr_conf.rmv) 1293 rte_intr_callback_unregister(&priv->intr_handle, 1294 mlx5_dev_interrupt_handler, dev); 1295 if (priv->primary_socket) 1296 rte_intr_callback_unregister(&priv->intr_handle_socket, 1297 mlx5_dev_handler_socket, dev); 1298 if (priv->pending_alarm) 1299 rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev); 1300 priv->pending_alarm = 0; 1301 priv->intr_handle.fd = 0; 1302 priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; 1303 priv->intr_handle_socket.fd = 0; 1304 priv->intr_handle_socket.type = RTE_INTR_HANDLE_UNKNOWN; 1305 } 1306 1307 /** 1308 * Install interrupt handler. 1309 * 1310 * @param priv 1311 * Pointer to private structure. 1312 * @param dev 1313 * Pointer to the rte_eth_dev structure. 1314 */ 1315 void 1316 priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) 1317 { 1318 int rc, flags; 1319 1320 assert(!mlx5_is_secondary()); 1321 assert(priv->ctx->async_fd > 0); 1322 flags = fcntl(priv->ctx->async_fd, F_GETFL); 1323 rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); 1324 if (rc < 0) { 1325 INFO("failed to change file descriptor async event queue"); 1326 dev->data->dev_conf.intr_conf.lsc = 0; 1327 dev->data->dev_conf.intr_conf.rmv = 0; 1328 } 1329 if (dev->data->dev_conf.intr_conf.lsc || 1330 dev->data->dev_conf.intr_conf.rmv) { 1331 priv->intr_handle.fd = priv->ctx->async_fd; 1332 priv->intr_handle.type = RTE_INTR_HANDLE_EXT; 1333 rte_intr_callback_register(&priv->intr_handle, 1334 mlx5_dev_interrupt_handler, dev); 1335 } 1336 1337 rc = priv_socket_init(priv); 1338 if (!rc && priv->primary_socket) { 1339 priv->intr_handle_socket.fd = priv->primary_socket; 1340 priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT; 1341 rte_intr_callback_register(&priv->intr_handle_socket, 1342 mlx5_dev_handler_socket, dev); 1343 } 1344 } 1345 1346 /** 1347 * Change the link state (UP / DOWN). 1348 * 1349 * @param priv 1350 * Pointer to private data structure. 1351 * @param dev 1352 * Pointer to rte_eth_dev structure. 1353 * @param up 1354 * Nonzero for link up, otherwise link down. 1355 * 1356 * @return 1357 * 0 on success, errno value on failure. 1358 */ 1359 static int 1360 priv_dev_set_link(struct priv *priv, struct rte_eth_dev *dev, int up) 1361 { 1362 int err; 1363 1364 if (up) { 1365 err = priv_set_flags(priv, ~IFF_UP, IFF_UP); 1366 if (err) 1367 return err; 1368 priv_dev_select_tx_function(priv, dev); 1369 priv_dev_select_rx_function(priv, dev); 1370 } else { 1371 err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP); 1372 if (err) 1373 return err; 1374 dev->rx_pkt_burst = removed_rx_burst; 1375 dev->tx_pkt_burst = removed_tx_burst; 1376 } 1377 return 0; 1378 } 1379 1380 /** 1381 * DPDK callback to bring the link DOWN. 1382 * 1383 * @param dev 1384 * Pointer to Ethernet device structure. 1385 * 1386 * @return 1387 * 0 on success, errno value on failure. 1388 */ 1389 int 1390 mlx5_set_link_down(struct rte_eth_dev *dev) 1391 { 1392 struct priv *priv = dev->data->dev_private; 1393 int err; 1394 1395 priv_lock(priv); 1396 err = priv_dev_set_link(priv, dev, 0); 1397 priv_unlock(priv); 1398 return err; 1399 } 1400 1401 /** 1402 * DPDK callback to bring the link UP. 1403 * 1404 * @param dev 1405 * Pointer to Ethernet device structure. 1406 * 1407 * @return 1408 * 0 on success, errno value on failure. 1409 */ 1410 int 1411 mlx5_set_link_up(struct rte_eth_dev *dev) 1412 { 1413 struct priv *priv = dev->data->dev_private; 1414 int err; 1415 1416 priv_lock(priv); 1417 err = priv_dev_set_link(priv, dev, 1); 1418 priv_unlock(priv); 1419 return err; 1420 } 1421 1422 /** 1423 * Configure the TX function to use. 1424 * 1425 * @param priv 1426 * Pointer to private data structure. 1427 * @param dev 1428 * Pointer to rte_eth_dev structure. 1429 */ 1430 void 1431 priv_dev_select_tx_function(struct priv *priv, struct rte_eth_dev *dev) 1432 { 1433 assert(priv != NULL); 1434 assert(dev != NULL); 1435 dev->tx_pkt_burst = mlx5_tx_burst; 1436 /* Select appropriate TX function. */ 1437 if (priv->mps == MLX5_MPW_ENHANCED) { 1438 if (priv_check_vec_tx_support(priv) > 0) { 1439 if (priv_check_raw_vec_tx_support(priv) > 0) 1440 dev->tx_pkt_burst = mlx5_tx_burst_raw_vec; 1441 else 1442 dev->tx_pkt_burst = mlx5_tx_burst_vec; 1443 DEBUG("selected Enhanced MPW TX vectorized function"); 1444 } else { 1445 dev->tx_pkt_burst = mlx5_tx_burst_empw; 1446 DEBUG("selected Enhanced MPW TX function"); 1447 } 1448 } else if (priv->mps && priv->txq_inline) { 1449 dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; 1450 DEBUG("selected MPW inline TX function"); 1451 } else if (priv->mps) { 1452 dev->tx_pkt_burst = mlx5_tx_burst_mpw; 1453 DEBUG("selected MPW TX function"); 1454 } 1455 } 1456 1457 /** 1458 * Configure the RX function to use. 1459 * 1460 * @param priv 1461 * Pointer to private data structure. 1462 * @param dev 1463 * Pointer to rte_eth_dev structure. 1464 */ 1465 void 1466 priv_dev_select_rx_function(struct priv *priv, struct rte_eth_dev *dev) 1467 { 1468 assert(priv != NULL); 1469 assert(dev != NULL); 1470 if (priv_check_vec_rx_support(priv) > 0) { 1471 dev->rx_pkt_burst = mlx5_rx_burst_vec; 1472 DEBUG("selected RX vectorized function"); 1473 } else { 1474 dev->rx_pkt_burst = mlx5_rx_burst; 1475 } 1476 } 1477