1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2015 6WIND S.A. 5 * Copyright 2015 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stddef.h> 35 #include <assert.h> 36 #include <unistd.h> 37 #include <stdint.h> 38 #include <stdio.h> 39 #include <string.h> 40 #include <stdlib.h> 41 #include <errno.h> 42 #include <dirent.h> 43 #include <net/if.h> 44 #include <sys/ioctl.h> 45 #include <sys/socket.h> 46 #include <netinet/in.h> 47 #include <linux/ethtool.h> 48 #include <linux/sockios.h> 49 #include <fcntl.h> 50 51 /* DPDK headers don't like -pedantic. */ 52 #ifdef PEDANTIC 53 #pragma GCC diagnostic ignored "-pedantic" 54 #endif 55 #include <rte_atomic.h> 56 #include <rte_ethdev.h> 57 #include <rte_mbuf.h> 58 #include <rte_common.h> 59 #include <rte_interrupts.h> 60 #include <rte_alarm.h> 61 #include <rte_malloc.h> 62 #ifdef PEDANTIC 63 #pragma GCC diagnostic error "-pedantic" 64 #endif 65 66 #include "mlx5.h" 67 #include "mlx5_rxtx.h" 68 #include "mlx5_utils.h" 69 70 /** 71 * Return private structure associated with an Ethernet device. 72 * 73 * @param dev 74 * Pointer to Ethernet device structure. 75 * 76 * @return 77 * Pointer to private structure. 78 */ 79 struct priv * 80 mlx5_get_priv(struct rte_eth_dev *dev) 81 { 82 struct mlx5_secondary_data *sd; 83 84 if (!mlx5_is_secondary()) 85 return dev->data->dev_private; 86 sd = &mlx5_secondary_data[dev->data->port_id]; 87 return sd->data.dev_private; 88 } 89 90 /** 91 * Check if running as a secondary process. 92 * 93 * @return 94 * Nonzero if running as a secondary process. 95 */ 96 inline int 97 mlx5_is_secondary(void) 98 { 99 return rte_eal_process_type() != RTE_PROC_PRIMARY; 100 } 101 102 /** 103 * Get interface name from private structure. 104 * 105 * @param[in] priv 106 * Pointer to private structure. 107 * @param[out] ifname 108 * Interface name output buffer. 109 * 110 * @return 111 * 0 on success, -1 on failure and errno is set. 112 */ 113 int 114 priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) 115 { 116 DIR *dir; 117 struct dirent *dent; 118 unsigned int dev_type = 0; 119 unsigned int dev_port_prev = ~0u; 120 char match[IF_NAMESIZE] = ""; 121 122 { 123 MKSTR(path, "%s/device/net", priv->ctx->device->ibdev_path); 124 125 dir = opendir(path); 126 if (dir == NULL) 127 return -1; 128 } 129 while ((dent = readdir(dir)) != NULL) { 130 char *name = dent->d_name; 131 FILE *file; 132 unsigned int dev_port; 133 int r; 134 135 if ((name[0] == '.') && 136 ((name[1] == '\0') || 137 ((name[1] == '.') && (name[2] == '\0')))) 138 continue; 139 140 MKSTR(path, "%s/device/net/%s/%s", 141 priv->ctx->device->ibdev_path, name, 142 (dev_type ? "dev_id" : "dev_port")); 143 144 file = fopen(path, "rb"); 145 if (file == NULL) { 146 if (errno != ENOENT) 147 continue; 148 /* 149 * Switch to dev_id when dev_port does not exist as 150 * is the case with Linux kernel versions < 3.15. 151 */ 152 try_dev_id: 153 match[0] = '\0'; 154 if (dev_type) 155 break; 156 dev_type = 1; 157 dev_port_prev = ~0u; 158 rewinddir(dir); 159 continue; 160 } 161 r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port); 162 fclose(file); 163 if (r != 1) 164 continue; 165 /* 166 * Switch to dev_id when dev_port returns the same value for 167 * all ports. May happen when using a MOFED release older than 168 * 3.0 with a Linux kernel >= 3.15. 169 */ 170 if (dev_port == dev_port_prev) 171 goto try_dev_id; 172 dev_port_prev = dev_port; 173 if (dev_port == (priv->port - 1u)) 174 snprintf(match, sizeof(match), "%s", name); 175 } 176 closedir(dir); 177 if (match[0] == '\0') 178 return -1; 179 strncpy(*ifname, match, sizeof(*ifname)); 180 return 0; 181 } 182 183 /** 184 * Read from sysfs entry. 185 * 186 * @param[in] priv 187 * Pointer to private structure. 188 * @param[in] entry 189 * Entry name relative to sysfs path. 190 * @param[out] buf 191 * Data output buffer. 192 * @param size 193 * Buffer size. 194 * 195 * @return 196 * 0 on success, -1 on failure and errno is set. 197 */ 198 static int 199 priv_sysfs_read(const struct priv *priv, const char *entry, 200 char *buf, size_t size) 201 { 202 char ifname[IF_NAMESIZE]; 203 FILE *file; 204 int ret; 205 int err; 206 207 if (priv_get_ifname(priv, &ifname)) 208 return -1; 209 210 MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, 211 ifname, entry); 212 213 file = fopen(path, "rb"); 214 if (file == NULL) 215 return -1; 216 ret = fread(buf, 1, size, file); 217 err = errno; 218 if (((size_t)ret < size) && (ferror(file))) 219 ret = -1; 220 else 221 ret = size; 222 fclose(file); 223 errno = err; 224 return ret; 225 } 226 227 /** 228 * Write to sysfs entry. 229 * 230 * @param[in] priv 231 * Pointer to private structure. 232 * @param[in] entry 233 * Entry name relative to sysfs path. 234 * @param[in] buf 235 * Data buffer. 236 * @param size 237 * Buffer size. 238 * 239 * @return 240 * 0 on success, -1 on failure and errno is set. 241 */ 242 static int 243 priv_sysfs_write(const struct priv *priv, const char *entry, 244 char *buf, size_t size) 245 { 246 char ifname[IF_NAMESIZE]; 247 FILE *file; 248 int ret; 249 int err; 250 251 if (priv_get_ifname(priv, &ifname)) 252 return -1; 253 254 MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, 255 ifname, entry); 256 257 file = fopen(path, "wb"); 258 if (file == NULL) 259 return -1; 260 ret = fwrite(buf, 1, size, file); 261 err = errno; 262 if (((size_t)ret < size) || (ferror(file))) 263 ret = -1; 264 else 265 ret = size; 266 fclose(file); 267 errno = err; 268 return ret; 269 } 270 271 /** 272 * Get unsigned long sysfs property. 273 * 274 * @param priv 275 * Pointer to private structure. 276 * @param[in] name 277 * Entry name relative to sysfs path. 278 * @param[out] value 279 * Value output buffer. 280 * 281 * @return 282 * 0 on success, -1 on failure and errno is set. 283 */ 284 static int 285 priv_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) 286 { 287 int ret; 288 unsigned long value_ret; 289 char value_str[32]; 290 291 ret = priv_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); 292 if (ret == -1) { 293 DEBUG("cannot read %s value from sysfs: %s", 294 name, strerror(errno)); 295 return -1; 296 } 297 value_str[ret] = '\0'; 298 errno = 0; 299 value_ret = strtoul(value_str, NULL, 0); 300 if (errno) { 301 DEBUG("invalid %s value `%s': %s", name, value_str, 302 strerror(errno)); 303 return -1; 304 } 305 *value = value_ret; 306 return 0; 307 } 308 309 /** 310 * Set unsigned long sysfs property. 311 * 312 * @param priv 313 * Pointer to private structure. 314 * @param[in] name 315 * Entry name relative to sysfs path. 316 * @param value 317 * Value to set. 318 * 319 * @return 320 * 0 on success, -1 on failure and errno is set. 321 */ 322 static int 323 priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) 324 { 325 int ret; 326 MKSTR(value_str, "%lu", value); 327 328 ret = priv_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); 329 if (ret == -1) { 330 DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", 331 name, value_str, value, strerror(errno)); 332 return -1; 333 } 334 return 0; 335 } 336 337 /** 338 * Perform ifreq ioctl() on associated Ethernet device. 339 * 340 * @param[in] priv 341 * Pointer to private structure. 342 * @param req 343 * Request number to pass to ioctl(). 344 * @param[out] ifr 345 * Interface request structure output buffer. 346 * 347 * @return 348 * 0 on success, -1 on failure and errno is set. 349 */ 350 int 351 priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) 352 { 353 int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); 354 int ret = -1; 355 356 if (sock == -1) 357 return ret; 358 if (priv_get_ifname(priv, &ifr->ifr_name) == 0) 359 ret = ioctl(sock, req, ifr); 360 close(sock); 361 return ret; 362 } 363 364 /** 365 * Return the number of active VFs for the current device. 366 * 367 * @param[in] priv 368 * Pointer to private structure. 369 * @param[out] num_vfs 370 * Number of active VFs. 371 * 372 * @return 373 * 0 on success, -1 on failure and errno is set. 374 */ 375 int 376 priv_get_num_vfs(struct priv *priv, uint16_t *num_vfs) 377 { 378 /* The sysfs entry name depends on the operating system. */ 379 const char **name = (const char *[]){ 380 "device/sriov_numvfs", 381 "device/mlx5_num_vfs", 382 NULL, 383 }; 384 int ret; 385 386 do { 387 unsigned long ulong_num_vfs; 388 389 ret = priv_get_sysfs_ulong(priv, *name, &ulong_num_vfs); 390 if (!ret) 391 *num_vfs = ulong_num_vfs; 392 } while (*(++name) && ret); 393 return ret; 394 } 395 396 /** 397 * Get device MTU. 398 * 399 * @param priv 400 * Pointer to private structure. 401 * @param[out] mtu 402 * MTU value output buffer. 403 * 404 * @return 405 * 0 on success, -1 on failure and errno is set. 406 */ 407 int 408 priv_get_mtu(struct priv *priv, uint16_t *mtu) 409 { 410 unsigned long ulong_mtu; 411 412 if (priv_get_sysfs_ulong(priv, "mtu", &ulong_mtu) == -1) 413 return -1; 414 *mtu = ulong_mtu; 415 return 0; 416 } 417 418 /** 419 * Set device MTU. 420 * 421 * @param priv 422 * Pointer to private structure. 423 * @param mtu 424 * MTU value to set. 425 * 426 * @return 427 * 0 on success, -1 on failure and errno is set. 428 */ 429 static int 430 priv_set_mtu(struct priv *priv, uint16_t mtu) 431 { 432 uint16_t new_mtu; 433 434 if (priv_set_sysfs_ulong(priv, "mtu", mtu) || 435 priv_get_mtu(priv, &new_mtu)) 436 return -1; 437 if (new_mtu == mtu) 438 return 0; 439 errno = EINVAL; 440 return -1; 441 } 442 443 /** 444 * Set device flags. 445 * 446 * @param priv 447 * Pointer to private structure. 448 * @param keep 449 * Bitmask for flags that must remain untouched. 450 * @param flags 451 * Bitmask for flags to modify. 452 * 453 * @return 454 * 0 on success, -1 on failure and errno is set. 455 */ 456 int 457 priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) 458 { 459 unsigned long tmp; 460 461 if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1) 462 return -1; 463 tmp &= keep; 464 tmp |= (flags & (~keep)); 465 return priv_set_sysfs_ulong(priv, "flags", tmp); 466 } 467 468 /** 469 * Ethernet device configuration. 470 * 471 * Prepare the driver for a given number of TX and RX queues. 472 * 473 * @param dev 474 * Pointer to Ethernet device structure. 475 * 476 * @return 477 * 0 on success, errno value on failure. 478 */ 479 static int 480 dev_configure(struct rte_eth_dev *dev) 481 { 482 struct priv *priv = dev->data->dev_private; 483 unsigned int rxqs_n = dev->data->nb_rx_queues; 484 unsigned int txqs_n = dev->data->nb_tx_queues; 485 unsigned int i; 486 unsigned int j; 487 unsigned int reta_idx_n; 488 489 priv->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; 490 priv->rxqs = (void *)dev->data->rx_queues; 491 priv->txqs = (void *)dev->data->tx_queues; 492 if (txqs_n != priv->txqs_n) { 493 INFO("%p: TX queues number update: %u -> %u", 494 (void *)dev, priv->txqs_n, txqs_n); 495 priv->txqs_n = txqs_n; 496 } 497 if (rxqs_n > priv->ind_table_max_size) { 498 ERROR("cannot handle this many RX queues (%u)", rxqs_n); 499 return EINVAL; 500 } 501 if (rxqs_n == priv->rxqs_n) 502 return 0; 503 INFO("%p: RX queues number update: %u -> %u", 504 (void *)dev, priv->rxqs_n, rxqs_n); 505 priv->rxqs_n = rxqs_n; 506 /* If the requested number of RX queues is not a power of two, use the 507 * maximum indirection table size for better balancing. 508 * The result is always rounded to the next power of two. */ 509 reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ? 510 priv->ind_table_max_size : 511 rxqs_n)); 512 if (priv_rss_reta_index_resize(priv, reta_idx_n)) 513 return ENOMEM; 514 /* When the number of RX queues is not a power of two, the remaining 515 * table entries are padded with reused WQs and hashes are not spread 516 * uniformly. */ 517 for (i = 0, j = 0; (i != reta_idx_n); ++i) { 518 (*priv->reta_idx)[i] = j; 519 if (++j == rxqs_n) 520 j = 0; 521 } 522 return 0; 523 } 524 525 /** 526 * DPDK callback for Ethernet device configuration. 527 * 528 * @param dev 529 * Pointer to Ethernet device structure. 530 * 531 * @return 532 * 0 on success, negative errno value on failure. 533 */ 534 int 535 mlx5_dev_configure(struct rte_eth_dev *dev) 536 { 537 struct priv *priv = dev->data->dev_private; 538 int ret; 539 540 if (mlx5_is_secondary()) 541 return -E_RTE_SECONDARY; 542 543 priv_lock(priv); 544 ret = dev_configure(dev); 545 assert(ret >= 0); 546 priv_unlock(priv); 547 return -ret; 548 } 549 550 /** 551 * DPDK callback to get information about the device. 552 * 553 * @param dev 554 * Pointer to Ethernet device structure. 555 * @param[out] info 556 * Info structure output buffer. 557 */ 558 void 559 mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) 560 { 561 struct priv *priv = mlx5_get_priv(dev); 562 unsigned int max; 563 char ifname[IF_NAMESIZE]; 564 565 priv_lock(priv); 566 /* FIXME: we should ask the device for these values. */ 567 info->min_rx_bufsize = 32; 568 info->max_rx_pktlen = 65536; 569 /* 570 * Since we need one CQ per QP, the limit is the minimum number 571 * between the two values. 572 */ 573 max = ((priv->device_attr.max_cq > priv->device_attr.max_qp) ? 574 priv->device_attr.max_qp : priv->device_attr.max_cq); 575 /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */ 576 if (max >= 65535) 577 max = 65535; 578 info->max_rx_queues = max; 579 info->max_tx_queues = max; 580 info->max_mac_addrs = RTE_DIM(priv->mac); 581 info->rx_offload_capa = 582 (priv->hw_csum ? 583 (DEV_RX_OFFLOAD_IPV4_CKSUM | 584 DEV_RX_OFFLOAD_UDP_CKSUM | 585 DEV_RX_OFFLOAD_TCP_CKSUM) : 586 0); 587 if (!priv->mps) 588 info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; 589 if (priv->hw_csum) 590 info->tx_offload_capa |= 591 (DEV_TX_OFFLOAD_IPV4_CKSUM | 592 DEV_TX_OFFLOAD_UDP_CKSUM | 593 DEV_TX_OFFLOAD_TCP_CKSUM); 594 if (priv_get_ifname(priv, &ifname) == 0) 595 info->if_index = if_nametoindex(ifname); 596 /* FIXME: RETA update/query API expects the callee to know the size of 597 * the indirection table, for this PMD the size varies depending on 598 * the number of RX queues, it becomes impossible to find the correct 599 * size if it is not fixed. 600 * The API should be updated to solve this problem. */ 601 info->reta_size = priv->ind_table_max_size; 602 info->speed_capa = 603 ETH_LINK_SPEED_1G | 604 ETH_LINK_SPEED_10G | 605 ETH_LINK_SPEED_20G | 606 ETH_LINK_SPEED_25G | 607 ETH_LINK_SPEED_40G | 608 ETH_LINK_SPEED_50G | 609 ETH_LINK_SPEED_56G | 610 ETH_LINK_SPEED_100G; 611 priv_unlock(priv); 612 } 613 614 const uint32_t * 615 mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) 616 { 617 static const uint32_t ptypes[] = { 618 /* refers to rxq_cq_to_pkt_type() */ 619 RTE_PTYPE_L3_IPV4, 620 RTE_PTYPE_L3_IPV6, 621 RTE_PTYPE_INNER_L3_IPV4, 622 RTE_PTYPE_INNER_L3_IPV6, 623 RTE_PTYPE_UNKNOWN 624 625 }; 626 627 if (dev->rx_pkt_burst == mlx5_rx_burst) 628 return ptypes; 629 return NULL; 630 } 631 632 /** 633 * DPDK callback to retrieve physical link information (unlocked version). 634 * 635 * @param dev 636 * Pointer to Ethernet device structure. 637 * @param wait_to_complete 638 * Wait for request completion (ignored). 639 */ 640 static int 641 mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) 642 { 643 struct priv *priv = mlx5_get_priv(dev); 644 struct ethtool_cmd edata = { 645 .cmd = ETHTOOL_GSET 646 }; 647 struct ifreq ifr; 648 struct rte_eth_link dev_link; 649 int link_speed = 0; 650 651 (void)wait_to_complete; 652 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { 653 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); 654 return -1; 655 } 656 memset(&dev_link, 0, sizeof(dev_link)); 657 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && 658 (ifr.ifr_flags & IFF_RUNNING)); 659 ifr.ifr_data = (void *)&edata; 660 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 661 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", 662 strerror(errno)); 663 return -1; 664 } 665 link_speed = ethtool_cmd_speed(&edata); 666 if (link_speed == -1) 667 dev_link.link_speed = 0; 668 else 669 dev_link.link_speed = link_speed; 670 dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? 671 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); 672 dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 673 ETH_LINK_SPEED_FIXED); 674 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { 675 /* Link status changed. */ 676 dev->data->dev_link = dev_link; 677 return 0; 678 } 679 /* Link status is still the same. */ 680 return -1; 681 } 682 683 /** 684 * DPDK callback to retrieve physical link information. 685 * 686 * @param dev 687 * Pointer to Ethernet device structure. 688 * @param wait_to_complete 689 * Wait for request completion (ignored). 690 */ 691 int 692 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) 693 { 694 struct priv *priv = mlx5_get_priv(dev); 695 int ret; 696 697 priv_lock(priv); 698 ret = mlx5_link_update_unlocked(dev, wait_to_complete); 699 priv_unlock(priv); 700 return ret; 701 } 702 703 /** 704 * DPDK callback to change the MTU. 705 * 706 * Setting the MTU affects hardware MRU (packets larger than the MTU cannot be 707 * received). Use this as a hint to enable/disable scattered packets support 708 * and improve performance when not needed. 709 * Since failure is not an option, reconfiguring queues on the fly is not 710 * recommended. 711 * 712 * @param dev 713 * Pointer to Ethernet device structure. 714 * @param in_mtu 715 * New MTU. 716 * 717 * @return 718 * 0 on success, negative errno value on failure. 719 */ 720 int 721 mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 722 { 723 struct priv *priv = dev->data->dev_private; 724 int ret = 0; 725 unsigned int i; 726 uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) = 727 mlx5_rx_burst; 728 unsigned int max_frame_len; 729 int rehash; 730 int restart = priv->started; 731 732 if (mlx5_is_secondary()) 733 return -E_RTE_SECONDARY; 734 735 priv_lock(priv); 736 /* Set kernel interface MTU first. */ 737 if (priv_set_mtu(priv, mtu)) { 738 ret = errno; 739 WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, 740 strerror(ret)); 741 goto out; 742 } else 743 DEBUG("adapter port %u MTU set to %u", priv->port, mtu); 744 /* Temporarily replace RX handler with a fake one, assuming it has not 745 * been copied elsewhere. */ 746 dev->rx_pkt_burst = removed_rx_burst; 747 /* Make sure everyone has left mlx5_rx_burst() and uses 748 * removed_rx_burst() instead. */ 749 rte_wmb(); 750 usleep(1000); 751 /* MTU does not include header and CRC. */ 752 max_frame_len = ETHER_HDR_LEN + mtu + ETHER_CRC_LEN; 753 /* Check if at least one queue is going to need a SGE update. */ 754 for (i = 0; i != priv->rxqs_n; ++i) { 755 struct rxq *rxq = (*priv->rxqs)[i]; 756 unsigned int mb_len; 757 unsigned int size = RTE_PKTMBUF_HEADROOM + max_frame_len; 758 unsigned int sges_n; 759 760 if (rxq == NULL) 761 continue; 762 mb_len = rte_pktmbuf_data_room_size(rxq->mp); 763 assert(mb_len >= RTE_PKTMBUF_HEADROOM); 764 /* 765 * Determine the number of SGEs needed for a full packet 766 * and round it to the next power of two. 767 */ 768 sges_n = log2above((size / mb_len) + !!(size % mb_len)); 769 if (sges_n != rxq->sges_n) 770 break; 771 } 772 /* 773 * If all queues have the right number of SGEs, a simple rehash 774 * of their buffers is enough, otherwise SGE information can only 775 * be updated in a queue by recreating it. All resources that depend 776 * on queues (flows, indirection tables) must be recreated as well in 777 * that case. 778 */ 779 rehash = (i == priv->rxqs_n); 780 if (!rehash) { 781 /* Clean up everything as with mlx5_dev_stop(). */ 782 priv_special_flow_disable_all(priv); 783 priv_mac_addrs_disable(priv); 784 priv_destroy_hash_rxqs(priv); 785 priv_fdir_disable(priv); 786 priv_dev_interrupt_handler_uninstall(priv, dev); 787 } 788 recover: 789 /* Reconfigure each RX queue. */ 790 for (i = 0; (i != priv->rxqs_n); ++i) { 791 struct rxq *rxq = (*priv->rxqs)[i]; 792 struct rxq_ctrl *rxq_ctrl = 793 container_of(rxq, struct rxq_ctrl, rxq); 794 int sp; 795 unsigned int mb_len; 796 unsigned int tmp; 797 798 if (rxq == NULL) 799 continue; 800 mb_len = rte_pktmbuf_data_room_size(rxq->mp); 801 assert(mb_len >= RTE_PKTMBUF_HEADROOM); 802 /* Toggle scattered support (sp) if necessary. */ 803 sp = (max_frame_len > (mb_len - RTE_PKTMBUF_HEADROOM)); 804 /* Provide new values to rxq_setup(). */ 805 dev->data->dev_conf.rxmode.jumbo_frame = sp; 806 dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame_len; 807 if (rehash) 808 ret = rxq_rehash(dev, rxq_ctrl); 809 else 810 ret = rxq_ctrl_setup(dev, rxq_ctrl, rxq->elts_n, 811 rxq_ctrl->socket, NULL, rxq->mp); 812 if (!ret) 813 continue; 814 /* Attempt to roll back in case of error. */ 815 tmp = (mb_len << rxq->sges_n) - RTE_PKTMBUF_HEADROOM; 816 if (max_frame_len != tmp) { 817 max_frame_len = tmp; 818 goto recover; 819 } 820 /* Double fault, disable RX. */ 821 break; 822 } 823 /* 824 * Use a safe RX burst function in case of error, otherwise mimic 825 * mlx5_dev_start(). 826 */ 827 if (ret) { 828 ERROR("unable to reconfigure RX queues, RX disabled"); 829 rx_func = removed_rx_burst; 830 } else if (restart && 831 !rehash && 832 !priv_create_hash_rxqs(priv) && 833 !priv_rehash_flows(priv)) { 834 if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_NONE) 835 priv_fdir_enable(priv); 836 priv_dev_interrupt_handler_install(priv, dev); 837 } 838 priv->mtu = mtu; 839 /* Burst functions can now be called again. */ 840 rte_wmb(); 841 dev->rx_pkt_burst = rx_func; 842 out: 843 priv_unlock(priv); 844 assert(ret >= 0); 845 return -ret; 846 } 847 848 /** 849 * DPDK callback to get flow control status. 850 * 851 * @param dev 852 * Pointer to Ethernet device structure. 853 * @param[out] fc_conf 854 * Flow control output buffer. 855 * 856 * @return 857 * 0 on success, negative errno value on failure. 858 */ 859 int 860 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 861 { 862 struct priv *priv = dev->data->dev_private; 863 struct ifreq ifr; 864 struct ethtool_pauseparam ethpause = { 865 .cmd = ETHTOOL_GPAUSEPARAM 866 }; 867 int ret; 868 869 if (mlx5_is_secondary()) 870 return -E_RTE_SECONDARY; 871 872 ifr.ifr_data = (void *)ðpause; 873 priv_lock(priv); 874 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 875 ret = errno; 876 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" 877 " failed: %s", 878 strerror(ret)); 879 goto out; 880 } 881 882 fc_conf->autoneg = ethpause.autoneg; 883 if (ethpause.rx_pause && ethpause.tx_pause) 884 fc_conf->mode = RTE_FC_FULL; 885 else if (ethpause.rx_pause) 886 fc_conf->mode = RTE_FC_RX_PAUSE; 887 else if (ethpause.tx_pause) 888 fc_conf->mode = RTE_FC_TX_PAUSE; 889 else 890 fc_conf->mode = RTE_FC_NONE; 891 ret = 0; 892 893 out: 894 priv_unlock(priv); 895 assert(ret >= 0); 896 return -ret; 897 } 898 899 /** 900 * DPDK callback to modify flow control parameters. 901 * 902 * @param dev 903 * Pointer to Ethernet device structure. 904 * @param[in] fc_conf 905 * Flow control parameters. 906 * 907 * @return 908 * 0 on success, negative errno value on failure. 909 */ 910 int 911 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 912 { 913 struct priv *priv = dev->data->dev_private; 914 struct ifreq ifr; 915 struct ethtool_pauseparam ethpause = { 916 .cmd = ETHTOOL_SPAUSEPARAM 917 }; 918 int ret; 919 920 if (mlx5_is_secondary()) 921 return -E_RTE_SECONDARY; 922 923 ifr.ifr_data = (void *)ðpause; 924 ethpause.autoneg = fc_conf->autoneg; 925 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 926 (fc_conf->mode & RTE_FC_RX_PAUSE)) 927 ethpause.rx_pause = 1; 928 else 929 ethpause.rx_pause = 0; 930 931 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || 932 (fc_conf->mode & RTE_FC_TX_PAUSE)) 933 ethpause.tx_pause = 1; 934 else 935 ethpause.tx_pause = 0; 936 937 priv_lock(priv); 938 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { 939 ret = errno; 940 WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" 941 " failed: %s", 942 strerror(ret)); 943 goto out; 944 } 945 ret = 0; 946 947 out: 948 priv_unlock(priv); 949 assert(ret >= 0); 950 return -ret; 951 } 952 953 /** 954 * Get PCI information from struct ibv_device. 955 * 956 * @param device 957 * Pointer to Ethernet device structure. 958 * @param[out] pci_addr 959 * PCI bus address output buffer. 960 * 961 * @return 962 * 0 on success, -1 on failure and errno is set. 963 */ 964 int 965 mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, 966 struct rte_pci_addr *pci_addr) 967 { 968 FILE *file; 969 char line[32]; 970 MKSTR(path, "%s/device/uevent", device->ibdev_path); 971 972 file = fopen(path, "rb"); 973 if (file == NULL) 974 return -1; 975 while (fgets(line, sizeof(line), file) == line) { 976 size_t len = strlen(line); 977 int ret; 978 979 /* Truncate long lines. */ 980 if (len == (sizeof(line) - 1)) 981 while (line[(len - 1)] != '\n') { 982 ret = fgetc(file); 983 if (ret == EOF) 984 break; 985 line[(len - 1)] = ret; 986 } 987 /* Extract information. */ 988 if (sscanf(line, 989 "PCI_SLOT_NAME=" 990 "%" SCNx16 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", 991 &pci_addr->domain, 992 &pci_addr->bus, 993 &pci_addr->devid, 994 &pci_addr->function) == 4) { 995 ret = 0; 996 break; 997 } 998 } 999 fclose(file); 1000 return 0; 1001 } 1002 1003 /** 1004 * Link status handler. 1005 * 1006 * @param priv 1007 * Pointer to private structure. 1008 * @param dev 1009 * Pointer to the rte_eth_dev structure. 1010 * 1011 * @return 1012 * Nonzero if the callback process can be called immediately. 1013 */ 1014 static int 1015 priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) 1016 { 1017 struct ibv_async_event event; 1018 int port_change = 0; 1019 int ret = 0; 1020 1021 /* Read all message and acknowledge them. */ 1022 for (;;) { 1023 if (ibv_get_async_event(priv->ctx, &event)) 1024 break; 1025 1026 if (event.event_type == IBV_EVENT_PORT_ACTIVE || 1027 event.event_type == IBV_EVENT_PORT_ERR) 1028 port_change = 1; 1029 else 1030 DEBUG("event type %d on port %d not handled", 1031 event.event_type, event.element.port_num); 1032 ibv_ack_async_event(&event); 1033 } 1034 1035 if (port_change ^ priv->pending_alarm) { 1036 struct rte_eth_link *link = &dev->data->dev_link; 1037 1038 priv->pending_alarm = 0; 1039 mlx5_link_update_unlocked(dev, 0); 1040 if (((link->link_speed == 0) && link->link_status) || 1041 ((link->link_speed != 0) && !link->link_status)) { 1042 /* Inconsistent status, check again later. */ 1043 priv->pending_alarm = 1; 1044 rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US, 1045 mlx5_dev_link_status_handler, 1046 dev); 1047 } else 1048 ret = 1; 1049 } 1050 return ret; 1051 } 1052 1053 /** 1054 * Handle delayed link status event. 1055 * 1056 * @param arg 1057 * Registered argument. 1058 */ 1059 void 1060 mlx5_dev_link_status_handler(void *arg) 1061 { 1062 struct rte_eth_dev *dev = arg; 1063 struct priv *priv = dev->data->dev_private; 1064 int ret; 1065 1066 priv_lock(priv); 1067 assert(priv->pending_alarm == 1); 1068 ret = priv_dev_link_status_handler(priv, dev); 1069 priv_unlock(priv); 1070 if (ret) 1071 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); 1072 } 1073 1074 /** 1075 * Handle interrupts from the NIC. 1076 * 1077 * @param[in] intr_handle 1078 * Interrupt handler. 1079 * @param cb_arg 1080 * Callback argument. 1081 */ 1082 void 1083 mlx5_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg) 1084 { 1085 struct rte_eth_dev *dev = cb_arg; 1086 struct priv *priv = dev->data->dev_private; 1087 int ret; 1088 1089 (void)intr_handle; 1090 priv_lock(priv); 1091 ret = priv_dev_link_status_handler(priv, dev); 1092 priv_unlock(priv); 1093 if (ret) 1094 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); 1095 } 1096 1097 /** 1098 * Uninstall interrupt handler. 1099 * 1100 * @param priv 1101 * Pointer to private structure. 1102 * @param dev 1103 * Pointer to the rte_eth_dev structure. 1104 */ 1105 void 1106 priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) 1107 { 1108 if (!dev->data->dev_conf.intr_conf.lsc) 1109 return; 1110 rte_intr_callback_unregister(&priv->intr_handle, 1111 mlx5_dev_interrupt_handler, 1112 dev); 1113 if (priv->pending_alarm) 1114 rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev); 1115 priv->pending_alarm = 0; 1116 priv->intr_handle.fd = 0; 1117 priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; 1118 } 1119 1120 /** 1121 * Install interrupt handler. 1122 * 1123 * @param priv 1124 * Pointer to private structure. 1125 * @param dev 1126 * Pointer to the rte_eth_dev structure. 1127 */ 1128 void 1129 priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) 1130 { 1131 int rc, flags; 1132 1133 if (!dev->data->dev_conf.intr_conf.lsc) 1134 return; 1135 assert(priv->ctx->async_fd > 0); 1136 flags = fcntl(priv->ctx->async_fd, F_GETFL); 1137 rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); 1138 if (rc < 0) { 1139 INFO("failed to change file descriptor async event queue"); 1140 dev->data->dev_conf.intr_conf.lsc = 0; 1141 } else { 1142 priv->intr_handle.fd = priv->ctx->async_fd; 1143 priv->intr_handle.type = RTE_INTR_HANDLE_EXT; 1144 rte_intr_callback_register(&priv->intr_handle, 1145 mlx5_dev_interrupt_handler, 1146 dev); 1147 } 1148 } 1149 1150 /** 1151 * Change the link state (UP / DOWN). 1152 * 1153 * @param priv 1154 * Pointer to Ethernet device structure. 1155 * @param up 1156 * Nonzero for link up, otherwise link down. 1157 * 1158 * @return 1159 * 0 on success, errno value on failure. 1160 */ 1161 static int 1162 priv_set_link(struct priv *priv, int up) 1163 { 1164 struct rte_eth_dev *dev = priv->dev; 1165 int err; 1166 1167 if (up) { 1168 err = priv_set_flags(priv, ~IFF_UP, IFF_UP); 1169 if (err) 1170 return err; 1171 priv_select_tx_function(priv); 1172 priv_select_rx_function(priv); 1173 } else { 1174 err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP); 1175 if (err) 1176 return err; 1177 dev->rx_pkt_burst = removed_rx_burst; 1178 dev->tx_pkt_burst = removed_tx_burst; 1179 } 1180 return 0; 1181 } 1182 1183 /** 1184 * DPDK callback to bring the link DOWN. 1185 * 1186 * @param dev 1187 * Pointer to Ethernet device structure. 1188 * 1189 * @return 1190 * 0 on success, errno value on failure. 1191 */ 1192 int 1193 mlx5_set_link_down(struct rte_eth_dev *dev) 1194 { 1195 struct priv *priv = dev->data->dev_private; 1196 int err; 1197 1198 priv_lock(priv); 1199 err = priv_set_link(priv, 0); 1200 priv_unlock(priv); 1201 return err; 1202 } 1203 1204 /** 1205 * DPDK callback to bring the link UP. 1206 * 1207 * @param dev 1208 * Pointer to Ethernet device structure. 1209 * 1210 * @return 1211 * 0 on success, errno value on failure. 1212 */ 1213 int 1214 mlx5_set_link_up(struct rte_eth_dev *dev) 1215 { 1216 struct priv *priv = dev->data->dev_private; 1217 int err; 1218 1219 priv_lock(priv); 1220 err = priv_set_link(priv, 1); 1221 priv_unlock(priv); 1222 return err; 1223 } 1224 1225 /** 1226 * Configure secondary process queues from a private data pointer (primary 1227 * or secondary) and update burst callbacks. Can take place only once. 1228 * 1229 * All queues must have been previously created by the primary process to 1230 * avoid undefined behavior. 1231 * 1232 * @param priv 1233 * Private data pointer from either primary or secondary process. 1234 * 1235 * @return 1236 * Private data pointer from secondary process, NULL in case of error. 1237 */ 1238 struct priv * 1239 mlx5_secondary_data_setup(struct priv *priv) 1240 { 1241 unsigned int port_id = 0; 1242 struct mlx5_secondary_data *sd; 1243 void **tx_queues; 1244 void **rx_queues; 1245 unsigned int nb_tx_queues; 1246 unsigned int nb_rx_queues; 1247 unsigned int i; 1248 1249 /* priv must be valid at this point. */ 1250 assert(priv != NULL); 1251 /* priv->dev must also be valid but may point to local memory from 1252 * another process, possibly with the same address and must not 1253 * be dereferenced yet. */ 1254 assert(priv->dev != NULL); 1255 /* Determine port ID by finding out where priv comes from. */ 1256 while (1) { 1257 sd = &mlx5_secondary_data[port_id]; 1258 rte_spinlock_lock(&sd->lock); 1259 /* Primary process? */ 1260 if (sd->primary_priv == priv) 1261 break; 1262 /* Secondary process? */ 1263 if (sd->data.dev_private == priv) 1264 break; 1265 rte_spinlock_unlock(&sd->lock); 1266 if (++port_id == RTE_DIM(mlx5_secondary_data)) 1267 port_id = 0; 1268 } 1269 /* Switch to secondary private structure. If private data has already 1270 * been updated by another thread, there is nothing else to do. */ 1271 priv = sd->data.dev_private; 1272 if (priv->dev->data == &sd->data) 1273 goto end; 1274 /* Sanity checks. Secondary private structure is supposed to point 1275 * to local eth_dev, itself still pointing to the shared device data 1276 * structure allocated by the primary process. */ 1277 assert(sd->shared_dev_data != &sd->data); 1278 assert(sd->data.nb_tx_queues == 0); 1279 assert(sd->data.tx_queues == NULL); 1280 assert(sd->data.nb_rx_queues == 0); 1281 assert(sd->data.rx_queues == NULL); 1282 assert(priv != sd->primary_priv); 1283 assert(priv->dev->data == sd->shared_dev_data); 1284 assert(priv->txqs_n == 0); 1285 assert(priv->txqs == NULL); 1286 assert(priv->rxqs_n == 0); 1287 assert(priv->rxqs == NULL); 1288 nb_tx_queues = sd->shared_dev_data->nb_tx_queues; 1289 nb_rx_queues = sd->shared_dev_data->nb_rx_queues; 1290 /* Allocate local storage for queues. */ 1291 tx_queues = rte_zmalloc("secondary ethdev->tx_queues", 1292 sizeof(sd->data.tx_queues[0]) * nb_tx_queues, 1293 RTE_CACHE_LINE_SIZE); 1294 rx_queues = rte_zmalloc("secondary ethdev->rx_queues", 1295 sizeof(sd->data.rx_queues[0]) * nb_rx_queues, 1296 RTE_CACHE_LINE_SIZE); 1297 if (tx_queues == NULL || rx_queues == NULL) 1298 goto error; 1299 /* Lock to prevent control operations during setup. */ 1300 priv_lock(priv); 1301 /* TX queues. */ 1302 for (i = 0; i != nb_tx_queues; ++i) { 1303 struct txq *primary_txq = (*sd->primary_priv->txqs)[i]; 1304 struct txq_ctrl *primary_txq_ctrl; 1305 struct txq_ctrl *txq_ctrl; 1306 1307 if (primary_txq == NULL) 1308 continue; 1309 primary_txq_ctrl = container_of(primary_txq, 1310 struct txq_ctrl, txq); 1311 txq_ctrl = rte_calloc_socket("TXQ", 1, sizeof(*txq_ctrl), 0, 1312 primary_txq_ctrl->socket); 1313 if (txq_ctrl != NULL) { 1314 if (txq_ctrl_setup(priv->dev, 1315 primary_txq_ctrl, 1316 primary_txq->elts_n, 1317 primary_txq_ctrl->socket, 1318 NULL) == 0) { 1319 txq_ctrl->txq.stats.idx = 1320 primary_txq->stats.idx; 1321 tx_queues[i] = &txq_ctrl->txq; 1322 continue; 1323 } 1324 rte_free(txq_ctrl); 1325 } 1326 while (i) { 1327 txq_ctrl = tx_queues[--i]; 1328 txq_cleanup(txq_ctrl); 1329 rte_free(txq_ctrl); 1330 } 1331 goto error; 1332 } 1333 /* RX queues. */ 1334 for (i = 0; i != nb_rx_queues; ++i) { 1335 struct rxq_ctrl *primary_rxq = 1336 container_of((*sd->primary_priv->rxqs)[i], 1337 struct rxq_ctrl, rxq); 1338 1339 if (primary_rxq == NULL) 1340 continue; 1341 /* Not supported yet. */ 1342 rx_queues[i] = NULL; 1343 } 1344 /* Update everything. */ 1345 priv->txqs = (void *)tx_queues; 1346 priv->txqs_n = nb_tx_queues; 1347 priv->rxqs = (void *)rx_queues; 1348 priv->rxqs_n = nb_rx_queues; 1349 sd->data.rx_queues = rx_queues; 1350 sd->data.tx_queues = tx_queues; 1351 sd->data.nb_rx_queues = nb_rx_queues; 1352 sd->data.nb_tx_queues = nb_tx_queues; 1353 sd->data.dev_link = sd->shared_dev_data->dev_link; 1354 sd->data.mtu = sd->shared_dev_data->mtu; 1355 memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state, 1356 sizeof(sd->data.rx_queue_state)); 1357 memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state, 1358 sizeof(sd->data.tx_queue_state)); 1359 sd->data.dev_flags = sd->shared_dev_data->dev_flags; 1360 /* Use local data from now on. */ 1361 rte_mb(); 1362 priv->dev->data = &sd->data; 1363 rte_mb(); 1364 priv_select_tx_function(priv); 1365 priv_select_rx_function(priv); 1366 priv_unlock(priv); 1367 end: 1368 /* More sanity checks. */ 1369 assert(priv->dev->data == &sd->data); 1370 rte_spinlock_unlock(&sd->lock); 1371 return priv; 1372 error: 1373 priv_unlock(priv); 1374 rte_free(tx_queues); 1375 rte_free(rx_queues); 1376 rte_spinlock_unlock(&sd->lock); 1377 return NULL; 1378 } 1379 1380 /** 1381 * Configure the TX function to use. 1382 * 1383 * @param priv 1384 * Pointer to private structure. 1385 */ 1386 void 1387 priv_select_tx_function(struct priv *priv) 1388 { 1389 priv->dev->tx_pkt_burst = mlx5_tx_burst; 1390 /* Display warning for unsupported configurations. */ 1391 if (priv->sriov && priv->mps) 1392 WARN("multi-packet send WQE cannot be used on a SR-IOV setup"); 1393 /* Select appropriate TX function. */ 1394 if ((priv->sriov == 0) && priv->mps && priv->txq_inline) { 1395 priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; 1396 DEBUG("selected MPW inline TX function"); 1397 } else if ((priv->sriov == 0) && priv->mps) { 1398 priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw; 1399 DEBUG("selected MPW TX function"); 1400 } else if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) { 1401 priv->dev->tx_pkt_burst = mlx5_tx_burst_inline; 1402 DEBUG("selected inline TX function (%u >= %u queues)", 1403 priv->txqs_n, priv->txqs_inline); 1404 } 1405 } 1406 1407 /** 1408 * Configure the RX function to use. 1409 * 1410 * @param priv 1411 * Pointer to private structure. 1412 */ 1413 void 1414 priv_select_rx_function(struct priv *priv) 1415 { 1416 priv->dev->rx_pkt_burst = mlx5_rx_burst; 1417 } 1418