1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2015 6WIND S.A. 5 * Copyright 2015 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #define _GNU_SOURCE 35 36 #include <stddef.h> 37 #include <assert.h> 38 #include <unistd.h> 39 #include <stdint.h> 40 #include <stdio.h> 41 #include <string.h> 42 #include <stdlib.h> 43 #include <errno.h> 44 #include <dirent.h> 45 #include <net/if.h> 46 #include <sys/ioctl.h> 47 #include <sys/socket.h> 48 #include <sys/utsname.h> 49 #include <netinet/in.h> 50 #include <linux/ethtool.h> 51 #include <linux/sockios.h> 52 #include <linux/version.h> 53 #include <fcntl.h> 54 #include <stdalign.h> 55 #include <sys/un.h> 56 57 #include <rte_atomic.h> 58 #include <rte_ethdev_driver.h> 59 #include <rte_bus_pci.h> 60 #include <rte_mbuf.h> 61 #include <rte_common.h> 62 #include <rte_interrupts.h> 63 #include <rte_alarm.h> 64 #include <rte_malloc.h> 65 66 #include "mlx5.h" 67 #include "mlx5_rxtx.h" 68 #include "mlx5_utils.h" 69 70 /* Add defines in case the running kernel is not the same as user headers. */ 71 #ifndef ETHTOOL_GLINKSETTINGS 72 struct ethtool_link_settings { 73 uint32_t cmd; 74 uint32_t speed; 75 uint8_t duplex; 76 uint8_t port; 77 uint8_t phy_address; 78 uint8_t autoneg; 79 uint8_t mdio_support; 80 uint8_t eth_to_mdix; 81 uint8_t eth_tp_mdix_ctrl; 82 int8_t link_mode_masks_nwords; 83 uint32_t reserved[8]; 84 uint32_t link_mode_masks[]; 85 }; 86 87 #define ETHTOOL_GLINKSETTINGS 0x0000004c 88 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5 89 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6 90 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17 91 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18 92 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19 93 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20 94 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21 95 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22 96 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23 97 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24 98 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25 99 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26 100 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27 101 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28 102 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29 103 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30 104 #endif 105 #ifndef HAVE_ETHTOOL_LINK_MODE_25G 106 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31 107 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32 108 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33 109 #endif 110 #ifndef HAVE_ETHTOOL_LINK_MODE_50G 111 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34 112 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35 113 #endif 114 #ifndef HAVE_ETHTOOL_LINK_MODE_100G 115 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36 116 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37 117 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38 118 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39 119 #endif 120 121 /** 122 * Get interface name from private structure. 123 * 124 * @param[in] priv 125 * Pointer to private structure. 126 * @param[out] ifname 127 * Interface name output buffer. 128 * 129 * @return 130 * 0 on success, -1 on failure and errno is set. 131 */ 132 int 133 priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) 134 { 135 DIR *dir; 136 struct dirent *dent; 137 unsigned int dev_type = 0; 138 unsigned int dev_port_prev = ~0u; 139 char match[IF_NAMESIZE] = ""; 140 141 { 142 MKSTR(path, "%s/device/net", priv->ibdev_path); 143 144 dir = opendir(path); 145 if (dir == NULL) 146 return -1; 147 } 148 while ((dent = readdir(dir)) != NULL) { 149 char *name = dent->d_name; 150 FILE *file; 151 unsigned int dev_port; 152 int r; 153 154 if ((name[0] == '.') && 155 ((name[1] == '\0') || 156 ((name[1] == '.') && (name[2] == '\0')))) 157 continue; 158 159 MKSTR(path, "%s/device/net/%s/%s", 160 priv->ibdev_path, name, 161 (dev_type ? "dev_id" : "dev_port")); 162 163 file = fopen(path, "rb"); 164 if (file == NULL) { 165 if (errno != ENOENT) 166 continue; 167 /* 168 * Switch to dev_id when dev_port does not exist as 169 * is the case with Linux kernel versions < 3.15. 170 */ 171 try_dev_id: 172 match[0] = '\0'; 173 if (dev_type) 174 break; 175 dev_type = 1; 176 dev_port_prev = ~0u; 177 rewinddir(dir); 178 continue; 179 } 180 r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port); 181 fclose(file); 182 if (r != 1) 183 continue; 184 /* 185 * Switch to dev_id when dev_port returns the same value for 186 * all ports. May happen when using a MOFED release older than 187 * 3.0 with a Linux kernel >= 3.15. 188 */ 189 if (dev_port == dev_port_prev) 190 goto try_dev_id; 191 dev_port_prev = dev_port; 192 if (dev_port == (priv->port - 1u)) 193 snprintf(match, sizeof(match), "%s", name); 194 } 195 closedir(dir); 196 if (match[0] == '\0') 197 return -1; 198 strncpy(*ifname, match, sizeof(*ifname)); 199 return 0; 200 } 201 202 /** 203 * Check if the counter is located on ib counters file. 204 * 205 * @param[in] cntr 206 * Counter name. 207 * 208 * @return 209 * 1 if counter is located on ib counters file , 0 otherwise. 210 */ 211 int 212 priv_is_ib_cntr(const char *cntr) 213 { 214 if (!strcmp(cntr, "out_of_buffer")) 215 return 1; 216 return 0; 217 } 218 219 /** 220 * Read from sysfs entry. 221 * 222 * @param[in] priv 223 * Pointer to private structure. 224 * @param[in] entry 225 * Entry name relative to sysfs path. 226 * @param[out] buf 227 * Data output buffer. 228 * @param size 229 * Buffer size. 230 * 231 * @return 232 * 0 on success, -1 on failure and errno is set. 233 */ 234 static int 235 priv_sysfs_read(const struct priv *priv, const char *entry, 236 char *buf, size_t size) 237 { 238 char ifname[IF_NAMESIZE]; 239 FILE *file; 240 int ret; 241 int err; 242 243 if (priv_get_ifname(priv, &ifname)) 244 return -1; 245 246 if (priv_is_ib_cntr(entry)) { 247 MKSTR(path, "%s/ports/1/hw_counters/%s", 248 priv->ibdev_path, entry); 249 file = fopen(path, "rb"); 250 } else { 251 MKSTR(path, "%s/device/net/%s/%s", 252 priv->ibdev_path, ifname, entry); 253 file = fopen(path, "rb"); 254 } 255 if (file == NULL) 256 return -1; 257 ret = fread(buf, 1, size, file); 258 err = errno; 259 if (((size_t)ret < size) && (ferror(file))) 260 ret = -1; 261 else 262 ret = size; 263 fclose(file); 264 errno = err; 265 return ret; 266 } 267 268 /** 269 * Write to sysfs entry. 270 * 271 * @param[in] priv 272 * Pointer to private structure. 273 * @param[in] entry 274 * Entry name relative to sysfs path. 275 * @param[in] buf 276 * Data buffer. 277 * @param size 278 * Buffer size. 279 * 280 * @return 281 * 0 on success, -1 on failure and errno is set. 282 */ 283 static int 284 priv_sysfs_write(const struct priv *priv, const char *entry, 285 char *buf, size_t size) 286 { 287 char ifname[IF_NAMESIZE]; 288 FILE *file; 289 int ret; 290 int err; 291 292 if (priv_get_ifname(priv, &ifname)) 293 return -1; 294 295 MKSTR(path, "%s/device/net/%s/%s", priv->ibdev_path, ifname, entry); 296 297 file = fopen(path, "wb"); 298 if (file == NULL) 299 return -1; 300 ret = fwrite(buf, 1, size, file); 301 err = errno; 302 if (((size_t)ret < size) || (ferror(file))) 303 ret = -1; 304 else 305 ret = size; 306 fclose(file); 307 errno = err; 308 return ret; 309 } 310 311 /** 312 * Get unsigned long sysfs property. 313 * 314 * @param priv 315 * Pointer to private structure. 316 * @param[in] name 317 * Entry name relative to sysfs path. 318 * @param[out] value 319 * Value output buffer. 320 * 321 * @return 322 * 0 on success, -1 on failure and errno is set. 323 */ 324 static int 325 priv_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) 326 { 327 int ret; 328 unsigned long value_ret; 329 char value_str[32]; 330 331 ret = priv_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); 332 if (ret == -1) { 333 DEBUG("cannot read %s value from sysfs: %s", 334 name, strerror(errno)); 335 return -1; 336 } 337 value_str[ret] = '\0'; 338 errno = 0; 339 value_ret = strtoul(value_str, NULL, 0); 340 if (errno) { 341 DEBUG("invalid %s value `%s': %s", name, value_str, 342 strerror(errno)); 343 return -1; 344 } 345 *value = value_ret; 346 return 0; 347 } 348 349 /** 350 * Set unsigned long sysfs property. 351 * 352 * @param priv 353 * Pointer to private structure. 354 * @param[in] name 355 * Entry name relative to sysfs path. 356 * @param value 357 * Value to set. 358 * 359 * @return 360 * 0 on success, -1 on failure and errno is set. 361 */ 362 static int 363 priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) 364 { 365 int ret; 366 MKSTR(value_str, "%lu", value); 367 368 ret = priv_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); 369 if (ret == -1) { 370 DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", 371 name, value_str, value, strerror(errno)); 372 return -1; 373 } 374 return 0; 375 } 376 377 /** 378 * Perform ifreq ioctl() on associated Ethernet device. 379 * 380 * @param[in] priv 381 * Pointer to private structure. 382 * @param req 383 * Request number to pass to ioctl(). 384 * @param[out] ifr 385 * Interface request structure output buffer. 386 * 387 * @return 388 * 0 on success, -1 on failure and errno is set. 389 */ 390 int 391 priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) 392 { 393 int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); 394 int ret = -1; 395 396 if (sock == -1) 397 return ret; 398 if (priv_get_ifname(priv, &ifr->ifr_name) == 0) 399 ret = ioctl(sock, req, ifr); 400 close(sock); 401 return ret; 402 } 403 404 /** 405 * Return the number of active VFs for the current device. 406 * 407 * @param[in] priv 408 * Pointer to private structure. 409 * @param[out] num_vfs 410 * Number of active VFs. 411 * 412 * @return 413 * 0 on success, -1 on failure and errno is set. 414 */ 415 int 416 priv_get_num_vfs(struct priv *priv, uint16_t *num_vfs) 417 { 418 /* The sysfs entry name depends on the operating system. */ 419 const char **name = (const char *[]){ 420 "device/sriov_numvfs", 421 "device/mlx5_num_vfs", 422 NULL, 423 }; 424 int ret; 425 426 do { 427 unsigned long ulong_num_vfs; 428 429 ret = priv_get_sysfs_ulong(priv, *name, &ulong_num_vfs); 430 if (!ret) 431 *num_vfs = ulong_num_vfs; 432 } while (*(++name) && ret); 433 return ret; 434 } 435 436 /** 437 * Get device MTU. 438 * 439 * @param priv 440 * Pointer to private structure. 441 * @param[out] mtu 442 * MTU value output buffer. 443 * 444 * @return 445 * 0 on success, -1 on failure and errno is set. 446 */ 447 int 448 priv_get_mtu(struct priv *priv, uint16_t *mtu) 449 { 450 unsigned long ulong_mtu; 451 452 if (priv_get_sysfs_ulong(priv, "mtu", &ulong_mtu) == -1) 453 return -1; 454 *mtu = ulong_mtu; 455 return 0; 456 } 457 458 /** 459 * Read device counter from sysfs. 460 * 461 * @param priv 462 * Pointer to private structure. 463 * @param name 464 * Counter name. 465 * @param[out] cntr 466 * Counter output buffer. 467 * 468 * @return 469 * 0 on success, -1 on failure and errno is set. 470 */ 471 int 472 priv_get_cntr_sysfs(struct priv *priv, const char *name, uint64_t *cntr) 473 { 474 unsigned long ulong_ctr; 475 476 if (priv_get_sysfs_ulong(priv, name, &ulong_ctr) == -1) 477 return -1; 478 *cntr = ulong_ctr; 479 return 0; 480 } 481 482 /** 483 * Set device MTU. 484 * 485 * @param priv 486 * Pointer to private structure. 487 * @param mtu 488 * MTU value to set. 489 * 490 * @return 491 * 0 on success, -1 on failure and errno is set. 492 */ 493 static int 494 priv_set_mtu(struct priv *priv, uint16_t mtu) 495 { 496 uint16_t new_mtu; 497 498 if (priv_set_sysfs_ulong(priv, "mtu", mtu) || 499 priv_get_mtu(priv, &new_mtu)) 500 return -1; 501 if (new_mtu == mtu) 502 return 0; 503 errno = EINVAL; 504 return -1; 505 } 506 507 /** 508 * Set device flags. 509 * 510 * @param priv 511 * Pointer to private structure. 512 * @param keep 513 * Bitmask for flags that must remain untouched. 514 * @param flags 515 * Bitmask for flags to modify. 516 * 517 * @return 518 * 0 on success, -1 on failure and errno is set. 519 */ 520 int 521 priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) 522 { 523 unsigned long tmp; 524 525 if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1) 526 return -1; 527 tmp &= keep; 528 tmp |= (flags & (~keep)); 529 return priv_set_sysfs_ulong(priv, "flags", tmp); 530 } 531 532 /** 533 * Ethernet device configuration. 534 * 535 * Prepare the driver for a given number of TX and RX queues. 536 * 537 * @param dev 538 * Pointer to Ethernet device structure. 539 * 540 * @return 541 * 0 on success, errno value on failure. 542 */ 543 static int 544 dev_configure(struct rte_eth_dev *dev) 545 { 546 struct priv *priv = dev->data->dev_private; 547 unsigned int rxqs_n = dev->data->nb_rx_queues; 548 unsigned int txqs_n = dev->data->nb_tx_queues; 549 unsigned int i; 550 unsigned int j; 551 unsigned int reta_idx_n; 552 const uint8_t use_app_rss_key = 553 !!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key; 554 uint64_t supp_tx_offloads = mlx5_priv_get_tx_port_offloads(priv); 555 uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; 556 uint64_t supp_rx_offloads = 557 (mlx5_priv_get_rx_port_offloads(priv) | 558 mlx5_priv_get_rx_queue_offloads(priv)); 559 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; 560 561 if ((tx_offloads & supp_tx_offloads) != tx_offloads) { 562 ERROR("Some Tx offloads are not supported " 563 "requested 0x%" PRIx64 " supported 0x%" PRIx64, 564 tx_offloads, supp_tx_offloads); 565 return ENOTSUP; 566 } 567 if ((rx_offloads & supp_rx_offloads) != rx_offloads) { 568 ERROR("Some Rx offloads are not supported " 569 "requested 0x%" PRIx64 " supported 0x%" PRIx64, 570 rx_offloads, supp_rx_offloads); 571 return ENOTSUP; 572 } 573 if (use_app_rss_key && 574 (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len != 575 rss_hash_default_key_len)) { 576 /* MLX5 RSS only support 40bytes key. */ 577 return EINVAL; 578 } 579 priv->rss_conf.rss_key = 580 rte_realloc(priv->rss_conf.rss_key, 581 rss_hash_default_key_len, 0); 582 if (!priv->rss_conf.rss_key) { 583 ERROR("cannot allocate RSS hash key memory (%u)", rxqs_n); 584 return ENOMEM; 585 } 586 memcpy(priv->rss_conf.rss_key, 587 use_app_rss_key ? 588 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key : 589 rss_hash_default_key, 590 rss_hash_default_key_len); 591 priv->rss_conf.rss_key_len = rss_hash_default_key_len; 592 priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; 593 priv->rxqs = (void *)dev->data->rx_queues; 594 priv->txqs = (void *)dev->data->tx_queues; 595 if (txqs_n != priv->txqs_n) { 596 INFO("%p: TX queues number update: %u -> %u", 597 (void *)dev, priv->txqs_n, txqs_n); 598 priv->txqs_n = txqs_n; 599 } 600 if (rxqs_n > priv->config.ind_table_max_size) { 601 ERROR("cannot handle this many RX queues (%u)", rxqs_n); 602 return EINVAL; 603 } 604 if (rxqs_n == priv->rxqs_n) 605 return 0; 606 INFO("%p: RX queues number update: %u -> %u", 607 (void *)dev, priv->rxqs_n, rxqs_n); 608 priv->rxqs_n = rxqs_n; 609 /* If the requested number of RX queues is not a power of two, use the 610 * maximum indirection table size for better balancing. 611 * The result is always rounded to the next power of two. */ 612 reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ? 613 priv->config.ind_table_max_size : 614 rxqs_n)); 615 if (priv_rss_reta_index_resize(priv, reta_idx_n)) 616 return ENOMEM; 617 /* When the number of RX queues is not a power of two, the remaining 618 * table entries are padded with reused WQs and hashes are not spread 619 * uniformly. */ 620 for (i = 0, j = 0; (i != reta_idx_n); ++i) { 621 (*priv->reta_idx)[i] = j; 622 if (++j == rxqs_n) 623 j = 0; 624 } 625 return 0; 626 } 627 628 /** 629 * DPDK callback for Ethernet device configuration. 630 * 631 * @param dev 632 * Pointer to Ethernet device structure. 633 * 634 * @return 635 * 0 on success, negative errno value on failure. 636 */ 637 int 638 mlx5_dev_configure(struct rte_eth_dev *dev) 639 { 640 struct priv *priv = dev->data->dev_private; 641 int ret; 642 643 priv_lock(priv); 644 ret = dev_configure(dev); 645 assert(ret >= 0); 646 priv_unlock(priv); 647 return -ret; 648 } 649 650 /** 651 * DPDK callback to get information about the device. 652 * 653 * @param dev 654 * Pointer to Ethernet device structure. 655 * @param[out] info 656 * Info structure output buffer. 657 */ 658 void 659 mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) 660 { 661 struct priv *priv = dev->data->dev_private; 662 struct mlx5_dev_config *config = &priv->config; 663 unsigned int max; 664 char ifname[IF_NAMESIZE]; 665 666 info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 667 668 priv_lock(priv); 669 /* FIXME: we should ask the device for these values. */ 670 info->min_rx_bufsize = 32; 671 info->max_rx_pktlen = 65536; 672 /* 673 * Since we need one CQ per QP, the limit is the minimum number 674 * between the two values. 675 */ 676 max = RTE_MIN(priv->device_attr.orig_attr.max_cq, 677 priv->device_attr.orig_attr.max_qp); 678 /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */ 679 if (max >= 65535) 680 max = 65535; 681 info->max_rx_queues = max; 682 info->max_tx_queues = max; 683 info->max_mac_addrs = RTE_DIM(priv->mac); 684 info->rx_queue_offload_capa = 685 mlx5_priv_get_rx_queue_offloads(priv); 686 info->rx_offload_capa = (mlx5_priv_get_rx_port_offloads(priv) | 687 info->rx_queue_offload_capa); 688 info->tx_offload_capa = mlx5_priv_get_tx_port_offloads(priv); 689 if (priv_get_ifname(priv, &ifname) == 0) 690 info->if_index = if_nametoindex(ifname); 691 info->reta_size = priv->reta_idx_n ? 692 priv->reta_idx_n : config->ind_table_max_size; 693 info->hash_key_size = priv->rss_conf.rss_key_len; 694 info->speed_capa = priv->link_speed_capa; 695 info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK; 696 priv_unlock(priv); 697 } 698 699 const uint32_t * 700 mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) 701 { 702 static const uint32_t ptypes[] = { 703 /* refers to rxq_cq_to_pkt_type() */ 704 RTE_PTYPE_L2_ETHER, 705 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 706 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 707 RTE_PTYPE_L4_NONFRAG, 708 RTE_PTYPE_L4_FRAG, 709 RTE_PTYPE_L4_TCP, 710 RTE_PTYPE_L4_UDP, 711 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 712 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 713 RTE_PTYPE_INNER_L4_NONFRAG, 714 RTE_PTYPE_INNER_L4_FRAG, 715 RTE_PTYPE_INNER_L4_TCP, 716 RTE_PTYPE_INNER_L4_UDP, 717 RTE_PTYPE_UNKNOWN 718 }; 719 720 if (dev->rx_pkt_burst == mlx5_rx_burst || 721 dev->rx_pkt_burst == mlx5_rx_burst_vec) 722 return ptypes; 723 return NULL; 724 } 725 726 /** 727 * DPDK callback to retrieve physical link information. 728 * 729 * @param dev 730 * Pointer to Ethernet device structure. 731 * @param wait_to_complete 732 * Wait for request completion (ignored). 733 */ 734 static int 735 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) 736 { 737 struct priv *priv = dev->data->dev_private; 738 struct ethtool_cmd edata = { 739 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */ 740 }; 741 struct ifreq ifr; 742 struct rte_eth_link dev_link; 743 int link_speed = 0; 744 745 /* priv_lock() is not taken to allow concurrent calls. */ 746 747 (void)wait_to_complete; 748 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 749 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 750 return -1; 751 } 752 memset(&dev_link, 0, sizeof(dev_link)); 753 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 754 (ifr.ifr_flags & IFF_RUNNING)); 755 ifr.ifr_data = (void *)&edata; 756 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 757 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", 758 strerror(errno)); 759 return -1; 760 } 761 link_speed = ethtool_cmd_speed(&edata); 762 if (link_speed == -1) 763 dev_link.link_speed = 0; 764 else 765 dev_link.link_speed = link_speed; 766 priv->link_speed_capa = 0; 767 if (edata.supported & SUPPORTED_Autoneg) 768 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; 769 if (edata.supported & (SUPPORTED_1000baseT_Full | 770 SUPPORTED_1000baseKX_Full)) 771 priv->link_speed_capa |= ETH_LINK_SPEED_1G; 772 if (edata.supported & SUPPORTED_10000baseKR_Full) 773 priv->link_speed_capa |= ETH_LINK_SPEED_10G; 774 if (edata.supported & (SUPPORTED_40000baseKR4_Full | 775 SUPPORTED_40000baseCR4_Full | 776 SUPPORTED_40000baseSR4_Full | 777 SUPPORTED_40000baseLR4_Full)) 778 priv->link_speed_capa |= ETH_LINK_SPEED_40G; 779 dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? 780 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 781 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 782 ETH_LINK_SPEED_FIXED); 783 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 784 /* Link status changed. */ 785 dev->data->dev_link = dev_link; 786 return 0; 787 } 788 /* Link status is still the same. */ 789 return -1; 790 } 791 792 /** 793 * Retrieve physical link information (unlocked version using new ioctl). 794 * 795 * @param dev 796 * Pointer to Ethernet device structure. 797 * @param wait_to_complete 798 * Wait for request completion (ignored). 799 */ 800 static int 801 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) 802 { 803 struct priv *priv = dev->data->dev_private; 804 struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS }; 805 struct ifreq ifr; 806 struct rte_eth_link dev_link; 807 uint64_t sc; 808 809 (void)wait_to_complete; 810 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 811 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 812 return -1; 813 } 814 memset(&dev_link, 0, sizeof(dev_link)); 815 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 816 (ifr.ifr_flags & IFF_RUNNING)); 817 ifr.ifr_data = (void *)&gcmd; 818 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 819 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", 820 strerror(errno)); 821 return -1; 822 } 823 gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords; 824 825 alignas(struct ethtool_link_settings) 826 uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) + 827 sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3]; 828 struct ethtool_link_settings *ecmd = (void *)data; 829 830 *ecmd = gcmd; 831 ifr.ifr_data = (void *)ecmd; 832 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 833 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", 834 strerror(errno)); 835 return -1; 836 } 837 dev_link.link_speed = ecmd->speed; 838 sc = ecmd->link_mode_masks[0] | 839 ((uint64_t)ecmd->link_mode_masks[1] << 32); 840 priv->link_speed_capa = 0; 841 if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT)) 842 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; 843 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) | 844 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT))) 845 priv->link_speed_capa |= ETH_LINK_SPEED_1G; 846 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) | 847 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) | 848 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT))) 849 priv->link_speed_capa |= ETH_LINK_SPEED_10G; 850 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) | 851 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT))) 852 priv->link_speed_capa |= ETH_LINK_SPEED_20G; 853 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) | 854 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) | 855 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) | 856 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT))) 857 priv->link_speed_capa |= ETH_LINK_SPEED_40G; 858 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) | 859 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) | 860 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) | 861 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT))) 862 priv->link_speed_capa |= ETH_LINK_SPEED_56G; 863 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) | 864 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) | 865 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT))) 866 priv->link_speed_capa |= ETH_LINK_SPEED_25G; 867 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) | 868 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT))) 869 priv->link_speed_capa |= ETH_LINK_SPEED_50G; 870 if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) | 871 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) | 872 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) | 873 MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT))) 874 priv->link_speed_capa |= ETH_LINK_SPEED_100G; 875 dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ? 876 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 877 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 878 ETH_LINK_SPEED_FIXED); 879 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 880 /* Link status changed. */ 881 dev->data->dev_link = dev_link; 882 return 0; 883 } 884 /* Link status is still the same. */ 885 return -1; 886 } 887 888 /** 889 * Enable receiving and transmitting traffic. 890 * 891 * @param priv 892 * Pointer to private structure. 893 */ 894 static void 895 priv_link_start(struct priv *priv) 896 { 897 struct rte_eth_dev *dev = priv->dev; 898 int err; 899 900 dev->tx_pkt_burst = priv_select_tx_function(priv, dev); 901 dev->rx_pkt_burst = priv_select_rx_function(priv, dev); 902 err = priv_dev_traffic_enable(priv, dev); 903 if (err) 904 ERROR("%p: error occurred while configuring control flows: %s", 905 (void *)priv, strerror(err)); 906 err = priv_flow_start(priv, &priv->flows); 907 if (err) 908 ERROR("%p: error occurred while configuring flows: %s", 909 (void *)priv, strerror(err)); 910 } 911 912 /** 913 * Disable receiving and transmitting traffic. 914 * 915 * @param priv 916 * Pointer to private structure. 917 */ 918 static void 919 priv_link_stop(struct priv *priv) 920 { 921 struct rte_eth_dev *dev = priv->dev; 922 923 priv_flow_stop(priv, &priv->flows); 924 priv_dev_traffic_disable(priv, dev); 925 dev->rx_pkt_burst = removed_rx_burst; 926 dev->tx_pkt_burst = removed_tx_burst; 927 } 928 929 /** 930 * Retrieve physical link information and update rx/tx_pkt_burst callbacks 931 * accordingly. 932 * 933 * @param priv 934 * Pointer to private structure. 935 * @param wait_to_complete 936 * Wait for request completion (ignored). 937 */ 938 int 939 priv_link_update(struct priv *priv, int wait_to_complete) 940 { 941 struct rte_eth_dev *dev = priv->dev; 942 struct utsname utsname; 943 int ver[3]; 944 int ret; 945 struct rte_eth_link dev_link = dev->data->dev_link; 946 947 if (uname(&utsname) == -1 || 948 sscanf(utsname.release, "%d.%d.%d", 949 &ver[0], &ver[1], &ver[2]) != 3 || 950 KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0)) 951 ret = mlx5_link_update_unlocked_gset(dev, wait_to_complete); 952 else 953 ret = mlx5_link_update_unlocked_gs(dev, wait_to_complete); 954 /* If lsc interrupt is disabled, should always be ready for traffic. */ 955 if (!dev->data->dev_conf.intr_conf.lsc) { 956 priv_link_start(priv); 957 return ret; 958 } 959 /* Re-select burst callbacks only if link status has been changed. */ 960 if (!ret && dev_link.link_status != dev->data->dev_link.link_status) { 961 if (dev->data->dev_link.link_status == ETH_LINK_UP) 962 priv_link_start(priv); 963 else 964 priv_link_stop(priv); 965 } 966 return ret; 967 } 968 969 /** 970 * Querying the link status till it changes to the desired state. 971 * Number of query attempts is bounded by MLX5_MAX_LINK_QUERY_ATTEMPTS. 972 * 973 * @param priv 974 * Pointer to private structure. 975 * @param status 976 * Link desired status. 977 * 978 * @return 979 * 0 on success, negative errno value on failure. 980 */ 981 int 982 priv_force_link_status_change(struct priv *priv, int status) 983 { 984 int try = 0; 985 986 while (try < MLX5_MAX_LINK_QUERY_ATTEMPTS) { 987 priv_link_update(priv, 0); 988 if (priv->dev->data->dev_link.link_status == status) 989 return 0; 990 try++; 991 sleep(1); 992 } 993 return -EAGAIN; 994 } 995 996 /** 997 * DPDK callback to retrieve physical link information. 998 * 999 * @param dev 1000 * Pointer to Ethernet device structure. 1001 * @param wait_to_complete 1002 * Wait for request completion (ignored). 1003 */ 1004 int 1005 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) 1006 { 1007 struct priv *priv = dev->data->dev_private; 1008 int ret; 1009 1010 priv_lock(priv); 1011 ret = priv_link_update(priv, wait_to_complete); 1012 priv_unlock(priv); 1013 return ret; 1014 } 1015 1016 /** 1017 * DPDK callback to change the MTU. 1018 * 1019 * @param dev 1020 * Pointer to Ethernet device structure. 1021 * @param in_mtu 1022 * New MTU. 1023 * 1024 * @return 1025 * 0 on success, negative errno value on failure. 1026 */ 1027 int 1028 mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 1029 { 1030 struct priv *priv = dev->data->dev_private; 1031 uint16_t kern_mtu; 1032 int ret = 0; 1033 1034 priv_lock(priv); 1035 ret = priv_get_mtu(priv, &kern_mtu); 1036 if (ret) 1037 goto out; 1038 /* Set kernel interface MTU first. */ 1039 ret = priv_set_mtu(priv, mtu); 1040 if (ret) 1041 goto out; 1042 ret = priv_get_mtu(priv, &kern_mtu); 1043 if (ret) 1044 goto out; 1045 if (kern_mtu == mtu) { 1046 priv->mtu = mtu; 1047 DEBUG("adapter port %u MTU set to %u", priv->port, mtu); 1048 } 1049 priv_unlock(priv); 1050 return 0; 1051 out: 1052 ret = errno; 1053 WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, 1054 strerror(ret)); 1055 priv_unlock(priv); 1056 assert(ret >= 0); 1057 return -ret; 1058 } 1059 1060 /** 1061 * DPDK callback to get flow control status. 1062 * 1063 * @param dev 1064 * Pointer to Ethernet device structure. 1065 * @param[out] fc_conf 1066 * Flow control output buffer. 1067 * 1068 * @return 1069 * 0 on success, negative errno value on failure. 1070 */ 1071 int 1072 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 1073 { 1074 struct priv *priv = dev->data->dev_private; 1075 struct ifreq ifr; 1076 struct ethtool_pauseparam ethpause = { 1077 .cmd = ETHTOOL_GPAUSEPARAM 1078 }; 1079 int ret; 1080 1081 ifr.ifr_data = (void *)ðpause; 1082 priv_lock(priv); 1083 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 1084 ret = errno; 1085 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" 1086 " failed: %s", 1087 strerror(ret)); 1088 goto out; 1089 } 1090 1091 fc_conf->autoneg = ethpause.autoneg; 1092 if (ethpause.rx_pause && ethpause.tx_pause) 1093 fc_conf->mode = RTE_FC_FULL; 1094 else if (ethpause.rx_pause) 1095 fc_conf->mode = RTE_FC_RX_PAUSE; 1096 else if (ethpause.tx_pause) 1097 fc_conf->mode = RTE_FC_TX_PAUSE; 1098 else 1099 fc_conf->mode = RTE_FC_NONE; 1100 ret = 0; 1101 1102 out: 1103 priv_unlock(priv); 1104 assert(ret >= 0); 1105 return -ret; 1106 } 1107 1108 /** 1109 * DPDK callback to modify flow control parameters. 1110 * 1111 * @param dev 1112 * Pointer to Ethernet device structure. 1113 * @param[in] fc_conf 1114 * Flow control parameters. 1115 * 1116 * @return 1117 * 0 on success, negative errno value on failure. 1118 */ 1119 int 1120 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 1121 { 1122 struct priv *priv = dev->data->dev_private; 1123 struct ifreq ifr; 1124 struct ethtool_pauseparam ethpause = { 1125 .cmd = ETHTOOL_SPAUSEPARAM 1126 }; 1127 int ret; 1128 1129 ifr.ifr_data = (void *)ðpause; 1130 ethpause.autoneg = fc_conf->autoneg; 1131 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 1132 (fc_conf->mode & RTE_FC_RX_PAUSE)) 1133 ethpause.rx_pause = 1; 1134 else 1135 ethpause.rx_pause = 0; 1136 1137 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 1138 (fc_conf->mode & RTE_FC_TX_PAUSE)) 1139 ethpause.tx_pause = 1; 1140 else 1141 ethpause.tx_pause = 0; 1142 1143 priv_lock(priv); 1144 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 1145 ret = errno; 1146 WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" 1147 " failed: %s", 1148 strerror(ret)); 1149 goto out; 1150 } 1151 ret = 0; 1152 1153 out: 1154 priv_unlock(priv); 1155 assert(ret >= 0); 1156 return -ret; 1157 } 1158 1159 /** 1160 * Get PCI information from struct ibv_device. 1161 * 1162 * @param device 1163 * Pointer to Ethernet device structure. 1164 * @param[out] pci_addr 1165 * PCI bus address output buffer. 1166 * 1167 * @return 1168 * 0 on success, -1 on failure and errno is set. 1169 */ 1170 int 1171 mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, 1172 struct rte_pci_addr *pci_addr) 1173 { 1174 FILE *file; 1175 char line[32]; 1176 MKSTR(path, "%s/device/uevent", device->ibdev_path); 1177 1178 file = fopen(path, "rb"); 1179 if (file == NULL) 1180 return -1; 1181 while (fgets(line, sizeof(line), file) == line) { 1182 size_t len = strlen(line); 1183 int ret; 1184 1185 /* Truncate long lines. */ 1186 if (len == (sizeof(line) - 1)) 1187 while (line[(len - 1)] != '\n') { 1188 ret = fgetc(file); 1189 if (ret == EOF) 1190 break; 1191 line[(len - 1)] = ret; 1192 } 1193 /* Extract information. */ 1194 if (sscanf(line, 1195 "PCI_SLOT_NAME=" 1196 "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", 1197 &pci_addr->domain, 1198 &pci_addr->bus, 1199 &pci_addr->devid, 1200 &pci_addr->function) == 4) { 1201 ret = 0; 1202 break; 1203 } 1204 } 1205 fclose(file); 1206 return 0; 1207 } 1208 1209 /** 1210 * Update the link status. 1211 * 1212 * @param priv 1213 * Pointer to private structure. 1214 * 1215 * @return 1216 * Zero if the callback process can be called immediately. 1217 */ 1218 static int 1219 priv_link_status_update(struct priv *priv) 1220 { 1221 struct rte_eth_link *link = &priv->dev->data->dev_link; 1222 1223 priv_link_update(priv, 0); 1224 if (((link->link_speed == 0) && link->link_status) || 1225 ((link->link_speed != 0) && !link->link_status)) { 1226 /* 1227 * Inconsistent status. Event likely occurred before the 1228 * kernel netdevice exposes the new status. 1229 */ 1230 if (!priv->pending_alarm) { 1231 priv->pending_alarm = 1; 1232 rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US, 1233 mlx5_dev_link_status_handler, 1234 priv->dev); 1235 } 1236 return 1; 1237 } else if (unlikely(priv->pending_alarm)) { 1238 /* Link interrupt occurred while alarm is already scheduled. */ 1239 priv->pending_alarm = 0; 1240 rte_eal_alarm_cancel(mlx5_dev_link_status_handler, priv->dev); 1241 } 1242 return 0; 1243 } 1244 1245 /** 1246 * Device status handler. 1247 * 1248 * @param priv 1249 * Pointer to private structure. 1250 * @param events 1251 * Pointer to event flags holder. 1252 * 1253 * @return 1254 * Events bitmap of callback process which can be called immediately. 1255 */ 1256 static uint32_t 1257 priv_dev_status_handler(struct priv *priv) 1258 { 1259 struct ibv_async_event event; 1260 uint32_t ret = 0; 1261 1262 /* Read all message and acknowledge them. */ 1263 for (;;) { 1264 if (ibv_get_async_event(priv->ctx, &event)) 1265 break; 1266 if ((event.event_type == IBV_EVENT_PORT_ACTIVE || 1267 event.event_type == IBV_EVENT_PORT_ERR) && 1268 (priv->dev->data->dev_conf.intr_conf.lsc == 1)) 1269 ret |= (1 << RTE_ETH_EVENT_INTR_LSC); 1270 else if (event.event_type == IBV_EVENT_DEVICE_FATAL && 1271 priv->dev->data->dev_conf.intr_conf.rmv == 1) 1272 ret |= (1 << RTE_ETH_EVENT_INTR_RMV); 1273 else 1274 DEBUG("event type %d on port %d not handled", 1275 event.event_type, event.element.port_num); 1276 ibv_ack_async_event(&event); 1277 } 1278 if (ret & (1 << RTE_ETH_EVENT_INTR_LSC)) 1279 if (priv_link_status_update(priv)) 1280 ret &= ~(1 << RTE_ETH_EVENT_INTR_LSC); 1281 return ret; 1282 } 1283 1284 /** 1285 * Handle delayed link status event. 1286 * 1287 * @param arg 1288 * Registered argument. 1289 */ 1290 void 1291 mlx5_dev_link_status_handler(void *arg) 1292 { 1293 struct rte_eth_dev *dev = arg; 1294 struct priv *priv = dev->data->dev_private; 1295 int ret; 1296 1297 while (!priv_trylock(priv)) { 1298 /* Alarm is being canceled. */ 1299 if (priv->pending_alarm == 0) 1300 return; 1301 rte_pause(); 1302 } 1303 priv->pending_alarm = 0; 1304 ret = priv_link_status_update(priv); 1305 priv_unlock(priv); 1306 if (!ret) 1307 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 1308 } 1309 1310 /** 1311 * Handle interrupts from the NIC. 1312 * 1313 * @param[in] intr_handle 1314 * Interrupt handler. 1315 * @param cb_arg 1316 * Callback argument. 1317 */ 1318 void 1319 mlx5_dev_interrupt_handler(void *cb_arg) 1320 { 1321 struct rte_eth_dev *dev = cb_arg; 1322 struct priv *priv = dev->data->dev_private; 1323 uint32_t events; 1324 1325 priv_lock(priv); 1326 events = priv_dev_status_handler(priv); 1327 priv_unlock(priv); 1328 if (events & (1 << RTE_ETH_EVENT_INTR_LSC)) 1329 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 1330 if (events & (1 << RTE_ETH_EVENT_INTR_RMV)) 1331 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL); 1332 } 1333 1334 /** 1335 * Handle interrupts from the socket. 1336 * 1337 * @param cb_arg 1338 * Callback argument. 1339 */ 1340 static void 1341 mlx5_dev_handler_socket(void *cb_arg) 1342 { 1343 struct rte_eth_dev *dev = cb_arg; 1344 struct priv *priv = dev->data->dev_private; 1345 1346 priv_lock(priv); 1347 priv_socket_handle(priv); 1348 priv_unlock(priv); 1349 } 1350 1351 /** 1352 * Uninstall interrupt handler. 1353 * 1354 * @param priv 1355 * Pointer to private structure. 1356 * @param dev 1357 * Pointer to the rte_eth_dev structure. 1358 */ 1359 void 1360 priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) 1361 { 1362 if (dev->data->dev_conf.intr_conf.lsc || 1363 dev->data->dev_conf.intr_conf.rmv) 1364 rte_intr_callback_unregister(&priv->intr_handle, 1365 mlx5_dev_interrupt_handler, dev); 1366 if (priv->primary_socket) 1367 rte_intr_callback_unregister(&priv->intr_handle_socket, 1368 mlx5_dev_handler_socket, dev); 1369 if (priv->pending_alarm) { 1370 priv->pending_alarm = 0; 1371 rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev); 1372 } 1373 priv->intr_handle.fd = 0; 1374 priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; 1375 priv->intr_handle_socket.fd = 0; 1376 priv->intr_handle_socket.type = RTE_INTR_HANDLE_UNKNOWN; 1377 } 1378 1379 /** 1380 * Install interrupt handler. 1381 * 1382 * @param priv 1383 * Pointer to private structure. 1384 * @param dev 1385 * Pointer to the rte_eth_dev structure. 1386 */ 1387 void 1388 priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) 1389 { 1390 int rc, flags; 1391 1392 assert(priv->ctx->async_fd > 0); 1393 flags = fcntl(priv->ctx->async_fd, F_GETFL); 1394 rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); 1395 if (rc < 0) { 1396 INFO("failed to change file descriptor async event queue"); 1397 dev->data->dev_conf.intr_conf.lsc = 0; 1398 dev->data->dev_conf.intr_conf.rmv = 0; 1399 } 1400 if (dev->data->dev_conf.intr_conf.lsc || 1401 dev->data->dev_conf.intr_conf.rmv) { 1402 priv->intr_handle.fd = priv->ctx->async_fd; 1403 priv->intr_handle.type = RTE_INTR_HANDLE_EXT; 1404 rte_intr_callback_register(&priv->intr_handle, 1405 mlx5_dev_interrupt_handler, dev); 1406 } 1407 1408 rc = priv_socket_init(priv); 1409 if (!rc && priv->primary_socket) { 1410 priv->intr_handle_socket.fd = priv->primary_socket; 1411 priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT; 1412 rte_intr_callback_register(&priv->intr_handle_socket, 1413 mlx5_dev_handler_socket, dev); 1414 } 1415 } 1416 1417 /** 1418 * Change the link state (UP / DOWN). 1419 * 1420 * @param priv 1421 * Pointer to private data structure. 1422 * @param up 1423 * Nonzero for link up, otherwise link down. 1424 * 1425 * @return 1426 * 0 on success, errno value on failure. 1427 */ 1428 static int 1429 priv_dev_set_link(struct priv *priv, int up) 1430 { 1431 return priv_set_flags(priv, ~IFF_UP, up ? IFF_UP : ~IFF_UP); 1432 } 1433 1434 /** 1435 * DPDK callback to bring the link DOWN. 1436 * 1437 * @param dev 1438 * Pointer to Ethernet device structure. 1439 * 1440 * @return 1441 * 0 on success, errno value on failure. 1442 */ 1443 int 1444 mlx5_set_link_down(struct rte_eth_dev *dev) 1445 { 1446 struct priv *priv = dev->data->dev_private; 1447 int err; 1448 1449 priv_lock(priv); 1450 err = priv_dev_set_link(priv, 0); 1451 priv_unlock(priv); 1452 return err; 1453 } 1454 1455 /** 1456 * DPDK callback to bring the link UP. 1457 * 1458 * @param dev 1459 * Pointer to Ethernet device structure. 1460 * 1461 * @return 1462 * 0 on success, errno value on failure. 1463 */ 1464 int 1465 mlx5_set_link_up(struct rte_eth_dev *dev) 1466 { 1467 struct priv *priv = dev->data->dev_private; 1468 int err; 1469 1470 priv_lock(priv); 1471 err = priv_dev_set_link(priv, 1); 1472 priv_unlock(priv); 1473 return err; 1474 } 1475 1476 /** 1477 * Configure the TX function to use. 1478 * 1479 * @param priv 1480 * Pointer to private data structure. 1481 * @param dev 1482 * Pointer to rte_eth_dev structure. 1483 * 1484 * @return 1485 * Pointer to selected Tx burst function. 1486 */ 1487 eth_tx_burst_t 1488 priv_select_tx_function(struct priv *priv, struct rte_eth_dev *dev) 1489 { 1490 eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst; 1491 struct mlx5_dev_config *config = &priv->config; 1492 uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; 1493 int tso = !!(tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO | 1494 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1495 DEV_TX_OFFLOAD_GRE_TNL_TSO)); 1496 int vlan_insert = !!(tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT); 1497 1498 assert(priv != NULL); 1499 /* Select appropriate TX function. */ 1500 if (vlan_insert || tso) 1501 return tx_pkt_burst; 1502 if (config->mps == MLX5_MPW_ENHANCED) { 1503 if (priv_check_vec_tx_support(priv, dev) > 0) { 1504 if (priv_check_raw_vec_tx_support(priv, dev) > 0) 1505 tx_pkt_burst = mlx5_tx_burst_raw_vec; 1506 else 1507 tx_pkt_burst = mlx5_tx_burst_vec; 1508 DEBUG("selected Enhanced MPW TX vectorized function"); 1509 } else { 1510 tx_pkt_burst = mlx5_tx_burst_empw; 1511 DEBUG("selected Enhanced MPW TX function"); 1512 } 1513 } else if (config->mps && (config->txq_inline > 0)) { 1514 tx_pkt_burst = mlx5_tx_burst_mpw_inline; 1515 DEBUG("selected MPW inline TX function"); 1516 } else if (config->mps) { 1517 tx_pkt_burst = mlx5_tx_burst_mpw; 1518 DEBUG("selected MPW TX function"); 1519 } 1520 return tx_pkt_burst; 1521 } 1522 1523 /** 1524 * Configure the RX function to use. 1525 * 1526 * @param priv 1527 * Pointer to private data structure. 1528 * @param dev 1529 * Pointer to rte_eth_dev structure. 1530 * 1531 * @return 1532 * Pointer to selected Rx burst function. 1533 */ 1534 eth_rx_burst_t 1535 priv_select_rx_function(struct priv *priv, __rte_unused struct rte_eth_dev *dev) 1536 { 1537 eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst; 1538 1539 assert(priv != NULL); 1540 if (priv_check_vec_rx_support(priv) > 0) { 1541 rx_pkt_burst = mlx5_rx_burst_vec; 1542 DEBUG("selected RX vectorized function"); 1543 } 1544 return rx_pkt_burst; 1545 } 1546 1547 /** 1548 * Check if mlx5 device was removed. 1549 * 1550 * @param dev 1551 * Pointer to Ethernet device structure. 1552 * 1553 * @return 1554 * 1 when device is removed, otherwise 0. 1555 */ 1556 int 1557 mlx5_is_removed(struct rte_eth_dev *dev) 1558 { 1559 struct ibv_device_attr device_attr; 1560 struct priv *priv = dev->data->dev_private; 1561 1562 if (ibv_query_device(priv->ctx, &device_attr) == EIO) 1563 return 1; 1564 return 0; 1565 } 1566