1 /* $OpenBSD: if_ix.c,v 1.8 2008/09/10 14:01:22 blambert Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2008, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.5 2008/05/16 18:46:30 jfv Exp $*/ 36 37 #include <dev/pci/if_ix.h> 38 39 /********************************************************************* 40 * Driver version 41 *********************************************************************/ 42 43 #define IXGBE_DRIVER_VERSION "1.4.4" 44 45 /********************************************************************* 46 * PCI Device ID Table 47 * 48 * Used by probe to select devices to load on 49 *********************************************************************/ 50 51 const struct pci_matchid ixgbe_devices[] = { 52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT } 59 }; 60 61 /********************************************************************* 62 * Function prototypes 63 *********************************************************************/ 64 int ixgbe_probe(struct device *, void *, void *); 65 void ixgbe_attach(struct device *, struct device *, void *); 66 int ixgbe_detach(struct device *, int); 67 void ixgbe_power(int, void *); 68 void ixgbe_shutdown(void *); 69 void ixgbe_start(struct ifnet *); 70 void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 71 int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 72 void ixgbe_watchdog(struct ifnet *); 73 void ixgbe_init(void *); 74 void ixgbe_stop(void *); 75 void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 76 int ixgbe_media_change(struct ifnet *); 77 void ixgbe_identify_hardware(struct ix_softc *); 78 int ixgbe_allocate_pci_resources(struct ix_softc *); 79 int ixgbe_allocate_legacy(struct ix_softc *); 80 int ixgbe_allocate_queues(struct ix_softc *); 81 void ixgbe_free_pci_resources(struct ix_softc *); 82 void ixgbe_local_timer(void *); 83 int ixgbe_hardware_init(struct ix_softc *); 84 void ixgbe_setup_interface(struct ix_softc *); 85 86 int ixgbe_allocate_transmit_buffers(struct tx_ring *); 87 int ixgbe_setup_transmit_structures(struct ix_softc *); 88 void ixgbe_setup_transmit_ring(struct tx_ring *); 89 void ixgbe_initialize_transmit_units(struct ix_softc *); 90 void ixgbe_free_transmit_structures(struct ix_softc *); 91 void ixgbe_free_transmit_buffers(struct tx_ring *); 92 93 int ixgbe_allocate_receive_buffers(struct rx_ring *); 94 int ixgbe_setup_receive_structures(struct ix_softc *); 95 int ixgbe_setup_receive_ring(struct rx_ring *); 96 void ixgbe_initialize_receive_units(struct ix_softc *); 97 void ixgbe_free_receive_structures(struct ix_softc *); 98 void ixgbe_free_receive_buffers(struct rx_ring *); 99 100 void ixgbe_enable_intr(struct ix_softc *); 101 void ixgbe_disable_intr(struct ix_softc *); 102 void ixgbe_update_stats_counters(struct ix_softc *); 103 int ixgbe_txeof(struct tx_ring *); 104 int ixgbe_rxeof(struct rx_ring *, int); 105 void ixgbe_rx_checksum(struct ix_softc *, uint32_t, struct mbuf *); 106 void ixgbe_set_promisc(struct ix_softc *); 107 void ixgbe_disable_promisc(struct ix_softc *); 108 void ixgbe_set_multi(struct ix_softc *); 109 #ifdef IX_DEBUG 110 void ixgbe_print_hw_stats(struct ix_softc *); 111 #endif 112 void ixgbe_update_link_status(struct ix_softc *); 113 int ixgbe_get_buf(struct rx_ring *, int, struct mbuf *); 114 int ixgbe_encap(struct tx_ring *, struct mbuf *); 115 #if NVLAN > 0 116 void ixgbe_enable_hw_vlans(struct ix_softc * sc); 117 #endif 118 int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 119 struct ixgbe_dma_alloc *, int); 120 void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 121 #ifdef IX_CSUM_OFFLOAD 122 int ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 123 int ixgbe_tso_setup(struct tx_ring *, struct mbuf *, uint32_t *); 124 #endif 125 void ixgbe_set_ivar(struct ix_softc *, uint16_t, uint8_t); 126 void ixgbe_configure_ivars(struct ix_softc *); 127 uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 128 129 /* Legacy (single vector interrupt handler */ 130 int ixgbe_legacy_irq(void *); 131 132 #ifndef NO_82598_A0_SUPPORT 133 void desc_flip(void *); 134 #endif 135 136 /********************************************************************* 137 * OpenBSD Device Interface Entry Points 138 *********************************************************************/ 139 140 struct cfdriver ix_cd = { 141 0, "ix", DV_IFNET 142 }; 143 144 struct cfattach ix_ca = { 145 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach 146 }; 147 148 /* Total number of Interfaces - need for config sanity check */ 149 static int ixgbe_total_ports; 150 151 /********************************************************************* 152 * Device identification routine 153 * 154 * ixgbe_probe determines if the driver should be loaded on 155 * sc based on PCI vendor/device id of the sc. 156 * 157 * return 0 on success, positive on failure 158 *********************************************************************/ 159 160 int 161 ixgbe_probe(struct device *parent, void *match, void *aux) 162 { 163 INIT_DEBUGOUT("ixgbe_probe: begin"); 164 165 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 166 sizeof(ixgbe_devices)/sizeof(ixgbe_devices[0]))); 167 } 168 169 /********************************************************************* 170 * Device initialization routine 171 * 172 * The attach entry point is called when the driver is being loaded. 173 * This routine identifies the type of hardware, allocates all resources 174 * and initializes the hardware. 175 * 176 * return 0 on success, positive on failure 177 *********************************************************************/ 178 179 void 180 ixgbe_attach(struct device *parent, struct device *self, void *aux) 181 { 182 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 183 struct ix_softc *sc = (struct ix_softc *)self; 184 int error = 0; 185 uint32_t ctrl_ext; 186 187 INIT_DEBUGOUT("ixgbe_attach: begin"); 188 189 sc->osdep.os_sc = sc; 190 sc->osdep.os_pa = pa; 191 192 /* Core Lock Init*/ 193 mtx_init(&sc->core_mtx, IPL_NET); 194 195 /* Set up the timer callout */ 196 timeout_set(&sc->timer, ixgbe_local_timer, sc); 197 198 /* Determine hardware revision */ 199 ixgbe_identify_hardware(sc); 200 201 /* Indicate to RX setup to use Jumbo Clusters */ 202 sc->bigbufs = FALSE; 203 sc->num_tx_desc = DEFAULT_TXD; 204 sc->num_rx_desc = DEFAULT_RXD; 205 sc->rx_process_limit = 100; // XXX 206 207 /* Do base PCI setup - map BAR0 */ 208 if (ixgbe_allocate_pci_resources(sc)) { 209 printf(": allocation of PCI resources failed\n"); 210 goto err_out; 211 } 212 213 /* Allocate our TX/RX Queues */ 214 if (ixgbe_allocate_queues(sc)) 215 goto err_out; 216 217 /* Initialize the shared code */ 218 sc->hw.mac.type = ixgbe_mac_82598EB; 219 if (ixgbe_init_ops_82598(&sc->hw) != 0) { 220 printf(": failed to init the 82598EB\n"); 221 goto err_late; 222 } 223 224 /* Initialize the hardware */ 225 if (ixgbe_hardware_init(sc)) { 226 printf(": unable to initialize the hardware\n"); 227 goto err_late; 228 } 229 230 /* XXX sc->msix > 1 && ixgbe_allocate_msix() */ 231 error = ixgbe_allocate_legacy(sc); 232 if (error) 233 goto err_late; 234 235 /* Setup OS specific network interface */ 236 ixgbe_setup_interface(sc); 237 238 /* Initialize statistics */ 239 ixgbe_update_stats_counters(sc); 240 241 /* let hardware know driver is loaded */ 242 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 243 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 244 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 245 246 sc->powerhook = powerhook_establish(ixgbe_power, sc); 247 sc->shutdownhook = shutdownhook_establish(ixgbe_shutdown, sc); 248 249 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 250 251 INIT_DEBUGOUT("ixgbe_attach: end"); 252 return; 253 254 err_late: 255 ixgbe_free_transmit_structures(sc); 256 ixgbe_free_receive_structures(sc); 257 err_out: 258 ixgbe_free_pci_resources(sc); 259 } 260 261 /********************************************************************* 262 * Device removal routine 263 * 264 * The detach entry point is called when the driver is being removed. 265 * This routine stops the sc and deallocates all the resources 266 * that were allocated for driver operation. 267 * 268 * return 0 on success, positive on failure 269 *********************************************************************/ 270 271 int 272 ixgbe_detach(struct device *self, int flags) 273 { 274 struct ix_softc *sc = (struct ix_softc *)self; 275 struct ifnet *ifp = &sc->arpcom.ac_if; 276 uint32_t ctrl_ext; 277 278 INIT_DEBUGOUT("ixgbe_detach: begin"); 279 280 ixgbe_stop(sc); 281 282 /* let hardware know driver is unloading */ 283 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 284 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 285 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 286 287 ether_ifdetach(ifp); 288 if_detach(ifp); 289 290 timeout_del(&sc->timer); 291 ixgbe_free_pci_resources(sc); 292 293 ixgbe_free_transmit_structures(sc); 294 ixgbe_free_receive_structures(sc); 295 296 return (0); 297 } 298 299 void 300 ixgbe_power(int why, void *arg) 301 { 302 struct ix_softc *sc = (struct ix_softc *)arg; 303 struct ifnet *ifp; 304 305 if (why == PWR_RESUME) { 306 ifp = &sc->arpcom.ac_if; 307 if (ifp->if_flags & IFF_UP) 308 ixgbe_init(sc); 309 } 310 } 311 312 /********************************************************************* 313 * 314 * Shutdown entry point 315 * 316 **********************************************************************/ 317 318 void 319 ixgbe_shutdown(void *arg) 320 { 321 struct ix_softc *sc = (struct ix_softc *)arg; 322 323 ixgbe_stop(sc); 324 } 325 326 /********************************************************************* 327 * Transmit entry point 328 * 329 * ixgbe_start is called by the stack to initiate a transmit. 330 * The driver will remain in this routine as long as there are 331 * packets to transmit and transmit resources are available. 332 * In case resources are not available stack is notified and 333 * the packet is requeued. 334 **********************************************************************/ 335 336 void 337 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 338 { 339 struct mbuf *m_head; 340 struct ix_softc *sc = txr->sc; 341 int post = 0; 342 343 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING) 344 return; 345 346 if (!sc->link_active) 347 return; 348 349 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 350 txr->txdma.dma_map->dm_mapsize, 351 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 352 353 for (;;) { 354 IFQ_POLL(&ifp->if_snd, m_head); 355 if (m_head == NULL) 356 break; 357 358 if (ixgbe_encap(txr, m_head)) { 359 ifp->if_flags |= IFF_OACTIVE; 360 break; 361 } 362 363 IFQ_DEQUEUE(&ifp->if_snd, m_head); 364 365 #if NBPFILTER > 0 366 if (ifp->if_bpf) 367 bpf_mtap(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 368 #endif 369 370 /* Set timeout in case hardware has problems transmitting */ 371 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 372 ifp->if_timer = IXGBE_TX_TIMEOUT; 373 374 post = 1; 375 } 376 377 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 378 0, txr->txdma.dma_map->dm_mapsize, 379 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 380 381 /* 382 * Advance the Transmit Descriptor Tail (Tdt), this tells the 383 * hardware that this frame is available to transmit. 384 */ 385 if (post) 386 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 387 txr->next_avail_tx_desc); 388 } 389 390 391 void 392 ixgbe_start(struct ifnet *ifp) 393 { 394 struct ix_softc *sc = ifp->if_softc; 395 struct tx_ring *txr = sc->tx_rings; 396 uint32_t queue = 0; 397 398 #if 0 399 /* 400 * This is really just here for testing 401 * TX multiqueue, ultimately what is 402 * needed is the flow support in the stack 403 * and appropriate logic here to deal with 404 * it. -jfv 405 */ 406 if (sc->num_tx_queues > 1) 407 queue = (curcpu % sc->num_tx_queues); 408 #endif 409 410 txr = &sc->tx_rings[queue]; 411 412 if (ifp->if_flags & IFF_RUNNING) 413 ixgbe_start_locked(txr, ifp); 414 415 return; 416 } 417 418 /********************************************************************* 419 * Ioctl entry point 420 * 421 * ixgbe_ioctl is called when the user wants to configure the 422 * interface. 423 * 424 * return 0 on success, positive on failure 425 **********************************************************************/ 426 427 int 428 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 429 { 430 int s, error = 0; 431 struct ifreq *ifr = (struct ifreq *) data; 432 struct ifaddr *ifa = (struct ifaddr *) data; 433 struct ix_softc *sc = ifp->if_softc; 434 435 s = splnet(); 436 437 if ((error = ether_ioctl(ifp, &sc->arpcom, command, data)) > 0) { 438 splx(s); 439 return (error); 440 } 441 442 switch (command) { 443 case SIOCSIFADDR: 444 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 445 ifp->if_flags |= IFF_UP; 446 if (!(ifp->if_flags & IFF_RUNNING)) 447 ixgbe_init(sc); 448 #ifdef INET 449 if (ifa->ifa_addr->sa_family == AF_INET) 450 arp_ifinit(&sc->arpcom, ifa); 451 #endif 452 break; 453 case SIOCSIFMTU: 454 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 455 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > ifp->if_hardmtu) 456 error = EINVAL; 457 else if (ifp->if_mtu != ifr->ifr_mtu) { 458 ifp->if_mtu = ifr->ifr_mtu; 459 sc->max_frame_size = 460 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 461 ixgbe_init(sc); 462 } 463 break; 464 case SIOCSIFFLAGS: 465 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 466 if (ifp->if_flags & IFF_UP) { 467 if ((ifp->if_flags & IFF_RUNNING)) { 468 if ((ifp->if_flags ^ sc->if_flags) & 469 (IFF_PROMISC | IFF_ALLMULTI)) { 470 ixgbe_disable_promisc(sc); 471 ixgbe_set_promisc(sc); 472 } 473 } else 474 ixgbe_init(sc); 475 } else 476 if (ifp->if_flags & IFF_RUNNING) 477 ixgbe_stop(sc); 478 sc->if_flags = ifp->if_flags; 479 break; 480 case SIOCADDMULTI: 481 case SIOCDELMULTI: 482 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 483 error = (command == SIOCADDMULTI) ? 484 ether_addmulti(ifr, &sc->arpcom) : 485 ether_delmulti(ifr, &sc->arpcom); 486 487 if (error == ENETRESET) { 488 if (ifp->if_flags & IFF_RUNNING) { 489 ixgbe_disable_intr(sc); 490 ixgbe_set_multi(sc); 491 ixgbe_enable_intr(sc); 492 } 493 error = 0; 494 } 495 break; 496 case SIOCSIFMEDIA: 497 case SIOCGIFMEDIA: 498 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 499 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 500 break; 501 default: 502 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 503 error = ENOTTY; 504 break; 505 } 506 507 splx(s); 508 return (error); 509 } 510 511 /********************************************************************* 512 * Watchdog entry point 513 * 514 * This routine is called by the local timer 515 * to detect hardware hangs . 516 * 517 **********************************************************************/ 518 519 void 520 ixgbe_watchdog(struct ifnet * ifp) 521 { 522 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 523 struct tx_ring *txr = sc->tx_rings; 524 struct ixgbe_hw *hw = &sc->hw; 525 int tx_hang = FALSE; 526 int i; 527 528 /* 529 * The timer is set to 5 every time ixgbe_start() queues a packet. 530 * Then ixgbe_txeof() keeps resetting to 5 as long as it cleans at 531 * least one descriptor. 532 * Finally, anytime all descriptors are clean the timer is 533 * set to 0. 534 */ 535 for (i = 0; i < sc->num_tx_queues; i++, txr++) { 536 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 537 continue; 538 else { 539 tx_hang = TRUE; 540 break; 541 } 542 } 543 if (tx_hang == FALSE) 544 return; 545 546 /* 547 * If we are in this routine because of pause frames, then don't 548 * reset the hardware. 549 */ 550 if (IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) { 551 for (i = 0; i < sc->num_tx_queues; i++, txr++) 552 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 553 ifp->if_timer = IXGBE_TX_TIMEOUT; 554 return; 555 } 556 557 558 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 559 for (i = 0; i < sc->num_tx_queues; i++, txr++) { 560 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 561 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 562 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 563 printf("%s: TX(%d) desc avail = %d, Next TX to Clean = %d\n", ifp->if_xname, 564 i, txr->tx_avail, txr->next_tx_to_clean); 565 } 566 ifp->if_flags &= ~IFF_RUNNING; 567 sc->watchdog_events++; 568 569 ixgbe_init(sc); 570 return; 571 } 572 573 /********************************************************************* 574 * Init entry point 575 * 576 * This routine is used in two ways. It is used by the stack as 577 * init entry point in network interface structure. It is also used 578 * by the driver as a hw/sw initialization routine to get to a 579 * consistent state. 580 * 581 * return 0 on success, positive on failure 582 **********************************************************************/ 583 #define IXGBE_MHADD_MFS_SHIFT 16 584 585 void 586 ixgbe_init(void *arg) 587 { 588 struct ix_softc *sc = (struct ix_softc *)arg; 589 struct ifnet *ifp = &sc->arpcom.ac_if; 590 uint32_t txdctl, rxdctl, mhadd, gpie; 591 int i, s; 592 593 INIT_DEBUGOUT("ixgbe_init: begin"); 594 595 s = splnet(); 596 597 ixgbe_stop(sc); 598 599 /* Get the latest mac address, User can use a LAA */ 600 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 601 IXGBE_ETH_LENGTH_OF_ADDRESS); 602 ixgbe_hw(&sc->hw, set_rar, 0, sc->hw.mac.addr, 0, 1); 603 sc->hw.addr_ctrl.rar_used_count = 1; 604 605 /* Initialize the hardware */ 606 if (ixgbe_hardware_init(sc)) { 607 printf("%s: Unable to initialize the hardware\n", 608 ifp->if_xname); 609 splx(s); 610 return; 611 } 612 613 #if NVLAN > 0 614 if (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) 615 ixgbe_enable_hw_vlans(sc); 616 #endif 617 618 /* Prepare transmit descriptors and buffers */ 619 if (ixgbe_setup_transmit_structures(sc)) { 620 printf("%s: Could not setup transmit structures\n", 621 ifp->if_xname); 622 ixgbe_stop(sc); 623 splx(s); 624 return; 625 } 626 627 ixgbe_initialize_transmit_units(sc); 628 629 /* Setup Multicast table */ 630 ixgbe_set_multi(sc); 631 632 /* 633 * If we are resetting MTU smaller than 2K 634 * drop to small RX buffers 635 */ 636 if (sc->max_frame_size <= MCLBYTES) 637 sc->bigbufs = FALSE; 638 639 /* Prepare receive descriptors and buffers */ 640 if (ixgbe_setup_receive_structures(sc)) { 641 printf("%s: Could not setup receive structures\n", ifp->if_xname); 642 ixgbe_stop(sc); 643 splx(s); 644 return; 645 } 646 647 /* Configure RX settings */ 648 ixgbe_initialize_receive_units(sc); 649 650 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 651 /* Enable Fan Failure Interrupt */ 652 if (sc->hw.phy.media_type == ixgbe_media_type_copper) 653 gpie |= IXGBE_SDP1_GPIEN; 654 if (sc->msix) { 655 /* Enable Enhanced MSIX mode */ 656 gpie |= IXGBE_GPIE_MSIX_MODE; 657 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 658 IXGBE_GPIE_OCD; 659 } 660 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 661 662 /* Set MTU size */ 663 if (ifp->if_mtu > ETHERMTU) { 664 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 665 mhadd &= ~IXGBE_MHADD_MFS_MASK; 666 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 667 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 668 } 669 670 /* Now enable all the queues */ 671 672 for (i = 0; i < sc->num_tx_queues; i++) { 673 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 674 txdctl |= IXGBE_TXDCTL_ENABLE; 675 /* Set WTHRESH to 8, burst writeback */ 676 txdctl |= (8 << 16); 677 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 678 } 679 680 for (i = 0; i < sc->num_rx_queues; i++) { 681 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 682 /* PTHRESH set to 32 */ 683 rxdctl |= 0x0020; 684 rxdctl |= IXGBE_RXDCTL_ENABLE; 685 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 686 } 687 688 timeout_add_sec(&sc->timer, 1); 689 690 /* Set up MSI/X routing */ 691 ixgbe_configure_ivars(sc); 692 693 ixgbe_enable_intr(sc); 694 695 /* Now inform the stack we're ready */ 696 ifp->if_flags |= IFF_RUNNING; 697 ifp->if_flags &= ~IFF_OACTIVE; 698 699 splx(s); 700 } 701 702 /********************************************************************* 703 * 704 * Legacy Interrupt Service routine 705 * 706 **********************************************************************/ 707 708 int 709 ixgbe_legacy_irq(void *arg) 710 { 711 struct ix_softc *sc = (struct ix_softc *)arg; 712 struct ifnet *ifp = &sc->arpcom.ac_if; 713 uint32_t reg_eicr; 714 struct tx_ring *txr = sc->tx_rings; 715 struct rx_ring *rxr = sc->rx_rings; 716 struct ixgbe_hw *hw = &sc->hw; 717 int claimed = 0; 718 719 for (;;) { 720 reg_eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR); 721 if (reg_eicr == 0) 722 break; 723 724 claimed = 1; 725 726 if (ifp->if_flags & IFF_RUNNING) { 727 ixgbe_rxeof(rxr, -1); 728 ixgbe_txeof(txr); 729 } 730 731 /* Check for fan failure */ 732 if ((hw->phy.media_type == ixgbe_media_type_copper) && 733 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 734 printf("%s: \nCRITICAL: FAN FAILURE!! " 735 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 736 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, 737 IXGBE_EICR_GPI_SDP1); 738 } 739 740 /* Link status change */ 741 if (reg_eicr & IXGBE_EICR_LSC) { 742 timeout_del(&sc->timer); 743 ixgbe_update_link_status(sc); 744 timeout_add_sec(&sc->timer, 1); 745 } 746 } 747 748 if (ifp->if_flags & IFF_RUNNING && !IFQ_IS_EMPTY(&ifp->if_snd)) 749 ixgbe_start_locked(txr, ifp); 750 751 return (claimed); 752 } 753 754 /********************************************************************* 755 * 756 * Media Ioctl callback 757 * 758 * This routine is called whenever the user queries the status of 759 * the interface using ifconfig. 760 * 761 **********************************************************************/ 762 void 763 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 764 { 765 struct ix_softc *sc = ifp->if_softc; 766 767 INIT_DEBUGOUT("ixgbe_media_status: begin"); 768 ixgbe_update_link_status(sc); 769 770 ifmr->ifm_status = IFM_AVALID; 771 ifmr->ifm_active = IFM_ETHER; 772 773 if (!sc->link_active) { 774 ifmr->ifm_status |= IFM_NONE; 775 return; 776 } 777 778 ifmr->ifm_status |= IFM_ACTIVE; 779 780 switch (sc->link_speed) { 781 case IXGBE_LINK_SPEED_1GB_FULL: 782 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 783 break; 784 case IXGBE_LINK_SPEED_10GB_FULL: 785 ifmr->ifm_active |= sc->optics | IFM_FDX; 786 break; 787 } 788 } 789 790 /********************************************************************* 791 * 792 * Media Ioctl callback 793 * 794 * This routine is called when the user changes speed/duplex using 795 * media/mediopt option with ifconfig. 796 * 797 **********************************************************************/ 798 int 799 ixgbe_media_change(struct ifnet * ifp) 800 { 801 struct ix_softc *sc = ifp->if_softc; 802 struct ifmedia *ifm = &sc->media; 803 804 INIT_DEBUGOUT("ixgbe_media_change: begin"); 805 806 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 807 return (EINVAL); 808 809 switch (IFM_SUBTYPE(ifm->ifm_media)) { 810 case IFM_AUTO: 811 sc->hw.mac.autoneg = TRUE; 812 sc->hw.phy.autoneg_advertised = 813 IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL; 814 break; 815 default: 816 printf("%s: Only auto media type\n", ifp->if_xname); 817 return (EINVAL); 818 } 819 820 return (0); 821 } 822 823 /********************************************************************* 824 * 825 * This routine maps the mbufs to tx descriptors. 826 * WARNING: while this code is using an MQ style infrastructure, 827 * it would NOT work as is with more than 1 queue. 828 * 829 * return 0 on success, positive on failure 830 **********************************************************************/ 831 832 int 833 ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head) 834 { 835 struct ix_softc *sc = txr->sc; 836 uint32_t olinfo_status = 0, cmd_type_len = 0; 837 int i, j, error; 838 int first, last = 0; 839 bus_dmamap_t map; 840 struct ixgbe_tx_buf *txbuf, *txbuf_mapped; 841 union ixgbe_adv_tx_desc *txd = NULL; 842 #ifdef IX_CSUM_OFFLOAD 843 uint32_t paylen = 0; 844 #endif 845 846 /* Basic descriptor defines */ 847 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 848 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 849 850 #if NVLAN > 0 851 /* VLAN tagging? */ 852 if ((m_head->m_flags & (M_PROTO1|M_PKTHDR)) == (M_PROTO1|M_PKTHDR) && 853 m_head->m_pkthdr.rcvif != NULL) 854 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 855 #endif 856 857 /* 858 * Force a cleanup if number of TX descriptors 859 * available is below the threshold. If it fails 860 * to get above, then abort transmit. 861 */ 862 if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) { 863 ixgbe_txeof(txr); 864 /* Make sure things have improved */ 865 if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) { 866 txr->no_tx_desc_avail++; 867 return (ENOBUFS); 868 } 869 } 870 871 /* 872 * Important to capture the first descriptor 873 * used because it will contain the index of 874 * the one we tell the hardware to report back 875 */ 876 first = txr->next_avail_tx_desc; 877 txbuf = &txr->tx_buffers[first]; 878 txbuf_mapped = txbuf; 879 map = txbuf->map; 880 881 /* 882 * Map the packet for DMA. 883 */ 884 error = bus_dmamap_load_mbuf(txr->txtag, map, 885 m_head, BUS_DMA_NOWAIT); 886 887 if (error == ENOMEM) { 888 sc->no_tx_dma_setup++; 889 return (error); 890 } else if (error != 0) { 891 sc->no_tx_dma_setup++; 892 return (error); 893 } 894 895 /* Make certain there are enough descriptors */ 896 if (map->dm_nsegs > txr->tx_avail - 2) { 897 txr->no_tx_desc_avail++; 898 error = ENOBUFS; 899 goto xmit_fail; 900 } 901 902 #ifdef IX_CSUM_OFFLOAD 903 /* 904 * Set the appropriate offload context 905 * this becomes the first descriptor of 906 * a packet. 907 */ 908 if (ixgbe_tso_setup(txr, m_head, &paylen)) { 909 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 910 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 911 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 912 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 913 ++sc->tso_tx; 914 } else if (ixgbe_tx_ctx_setup(txr, m_head)) 915 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 916 #endif 917 918 i = txr->next_avail_tx_desc; 919 for (j = 0; j < map->dm_nsegs; j++) { 920 txbuf = &txr->tx_buffers[i]; 921 txd = &txr->tx_base[i]; 922 923 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 924 txd->read.cmd_type_len = htole32(txr->txd_cmd | 925 cmd_type_len | map->dm_segs[j].ds_len); 926 txd->read.olinfo_status = htole32(olinfo_status); 927 last = i; /* Next descriptor that will get completed */ 928 929 if (++i == sc->num_tx_desc) 930 i = 0; 931 932 txbuf->m_head = NULL; 933 934 /* 935 * we have to do this inside the loop right now 936 * because of the hardware workaround. 937 */ 938 if (j == (map->dm_nsegs -1)) /* Last descriptor gets EOP and RS */ 939 txd->read.cmd_type_len |= 940 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 941 #ifndef NO_82598_A0_SUPPORT 942 if (sc->hw.revision_id == 0) 943 desc_flip(txd); 944 #endif 945 } 946 947 txr->tx_avail -= map->dm_nsegs; 948 txr->next_avail_tx_desc = i; 949 950 txbuf->m_head = m_head; 951 txbuf->map = map; 952 bus_dmamap_sync(txr->txtag, map, 0, map->dm_mapsize, 953 BUS_DMASYNC_PREWRITE); 954 955 /* Set the index of the descriptor that will be marked done */ 956 txbuf = &txr->tx_buffers[first]; 957 958 ++txr->tx_packets; 959 return (0); 960 961 xmit_fail: 962 bus_dmamap_unload(txr->txtag, txbuf->map); 963 return (error); 964 965 } 966 967 void 968 ixgbe_set_promisc(struct ix_softc *sc) 969 { 970 971 uint32_t reg_rctl; 972 struct ifnet *ifp = &sc->arpcom.ac_if; 973 974 reg_rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 975 976 if (ifp->if_flags & IFF_PROMISC) { 977 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 978 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 979 } else if (ifp->if_flags & IFF_ALLMULTI) { 980 reg_rctl |= IXGBE_FCTRL_MPE; 981 reg_rctl &= ~IXGBE_FCTRL_UPE; 982 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 983 } 984 return; 985 } 986 987 void 988 ixgbe_disable_promisc(struct ix_softc * sc) 989 { 990 uint32_t reg_rctl; 991 992 reg_rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 993 994 reg_rctl &= (~IXGBE_FCTRL_UPE); 995 reg_rctl &= (~IXGBE_FCTRL_MPE); 996 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 997 998 return; 999 } 1000 1001 1002 /********************************************************************* 1003 * Multicast Update 1004 * 1005 * This routine is called whenever multicast address list is updated. 1006 * 1007 **********************************************************************/ 1008 #define IXGBE_RAR_ENTRIES 16 1009 1010 void 1011 ixgbe_set_multi(struct ix_softc *sc) 1012 { 1013 uint32_t fctrl; 1014 uint8_t mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1015 uint8_t *update_ptr; 1016 struct ether_multi *enm; 1017 struct ether_multistep step; 1018 int mcnt = 0; 1019 struct ifnet *ifp = &sc->arpcom.ac_if; 1020 1021 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 1022 1023 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1024 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1025 if (ifp->if_flags & IFF_PROMISC) 1026 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1027 else if (ifp->if_flags & IFF_ALLMULTI) { 1028 fctrl |= IXGBE_FCTRL_MPE; 1029 fctrl &= ~IXGBE_FCTRL_UPE; 1030 } else 1031 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1032 1033 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1034 1035 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1036 while (enm != NULL) { 1037 if (bcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { 1038 ifp->if_flags |= IFF_ALLMULTI; 1039 mcnt = MAX_NUM_MULTICAST_ADDRESSES; 1040 } 1041 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 1042 break; 1043 bcopy(enm->enm_addrlo, 1044 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1045 IXGBE_ETH_LENGTH_OF_ADDRESS); 1046 mcnt++; 1047 ETHER_NEXT_MULTI(step, enm); 1048 } 1049 1050 update_ptr = mta; 1051 ixgbe_hw(&sc->hw, update_mc_addr_list, 1052 update_ptr, mcnt, ixgbe_mc_array_itr); 1053 1054 return; 1055 } 1056 1057 /* 1058 * This is an iterator function now needed by the multicast 1059 * shared code. It simply feeds the shared code routine the 1060 * addresses in the array of ixgbe_set_multi() one by one. 1061 */ 1062 uint8_t * 1063 ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1064 { 1065 uint8_t *addr = *update_ptr; 1066 uint8_t *newptr; 1067 *vmdq = 0; 1068 1069 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1070 *update_ptr = newptr; 1071 return addr; 1072 } 1073 1074 1075 /********************************************************************* 1076 * Timer routine 1077 * 1078 * This routine checks for link status,updates statistics, 1079 * and runs the watchdog timer. 1080 * 1081 **********************************************************************/ 1082 1083 void 1084 ixgbe_local_timer(void *arg) 1085 { 1086 struct ix_softc *sc = arg; 1087 #ifdef IX_DEBUG 1088 struct ifnet *ifp = &sc->arpcom.ac_if; 1089 #endif 1090 int s; 1091 1092 s = splnet(); 1093 1094 ixgbe_update_link_status(sc); 1095 ixgbe_update_stats_counters(sc); 1096 1097 #ifdef IX_DEBUG 1098 if ((ifp->if_flags & (IFF_RUNNING|IFF_DEBUG)) == 1099 (IFF_RUNNING|IFF_DEBUG)) 1100 ixgbe_print_hw_stats(sc); 1101 #endif 1102 1103 timeout_add_sec(&sc->timer, 1); 1104 1105 splx(s); 1106 } 1107 1108 void 1109 ixgbe_update_link_status(struct ix_softc *sc) 1110 { 1111 int link_up = FALSE; 1112 struct ifnet *ifp = &sc->arpcom.ac_if; 1113 struct tx_ring *txr = sc->tx_rings; 1114 int i; 1115 1116 ixgbe_hw(&sc->hw, check_link, &sc->link_speed, &link_up, 0); 1117 1118 switch (sc->link_speed) { 1119 case IXGBE_LINK_SPEED_UNKNOWN: 1120 ifp->if_baudrate = 0; 1121 break; 1122 case IXGBE_LINK_SPEED_100_FULL: 1123 ifp->if_baudrate = IF_Mbps(100); 1124 break; 1125 case IXGBE_LINK_SPEED_1GB_FULL: 1126 ifp->if_baudrate = IF_Gbps(1); 1127 break; 1128 case IXGBE_LINK_SPEED_10GB_FULL: 1129 ifp->if_baudrate = IF_Gbps(10); 1130 break; 1131 } 1132 1133 if (link_up){ 1134 if (sc->link_active == FALSE) { 1135 sc->link_active = TRUE; 1136 ifp->if_link_state = LINK_STATE_FULL_DUPLEX; 1137 if_link_state_change(ifp); 1138 } 1139 } else { /* Link down */ 1140 if (sc->link_active == TRUE) { 1141 ifp->if_baudrate = 0; 1142 ifp->if_link_state = LINK_STATE_DOWN; 1143 if_link_state_change(ifp); 1144 sc->link_active = FALSE; 1145 for (i = 0; i < sc->num_tx_queues; 1146 i++, txr++) 1147 txr->watchdog_timer = FALSE; 1148 ifp->if_timer = 0; 1149 } 1150 } 1151 1152 return; 1153 } 1154 1155 1156 1157 /********************************************************************* 1158 * 1159 * This routine disables all traffic on the sc by issuing a 1160 * global reset on the MAC and deallocates TX/RX buffers. 1161 * 1162 **********************************************************************/ 1163 1164 void 1165 ixgbe_stop(void *arg) 1166 { 1167 struct ix_softc *sc = arg; 1168 struct ifnet *ifp = &sc->arpcom.ac_if; 1169 1170 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1171 ixgbe_disable_intr(sc); 1172 1173 /* Tell the stack that the interface is no longer active */ 1174 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1175 1176 ixgbe_hw(&sc->hw, reset_hw); 1177 sc->hw.adapter_stopped = FALSE; 1178 ixgbe_hw(&sc->hw, stop_adapter); 1179 timeout_del(&sc->timer); 1180 1181 /* reprogram the RAR[0] in case user changed it. */ 1182 ixgbe_hw(&sc->hw, set_rar, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1183 } 1184 1185 1186 /********************************************************************* 1187 * 1188 * Determine hardware revision. 1189 * 1190 **********************************************************************/ 1191 void 1192 ixgbe_identify_hardware(struct ix_softc *sc) 1193 { 1194 struct ixgbe_osdep *os = &sc->osdep; 1195 struct pci_attach_args *pa = os->os_pa; 1196 uint32_t reg; 1197 1198 /* Save off the information about this board */ 1199 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1200 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1201 1202 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1203 sc->hw.revision_id = PCI_REVISION(reg); 1204 1205 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1206 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1207 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1208 1209 ixgbe_total_ports++; 1210 switch (sc->hw.device_id) { 1211 case PCI_PRODUCT_INTEL_82598AF_DUAL: 1212 case PCI_PRODUCT_INTEL_82598EB_CX4_DUAL: 1213 case PCI_PRODUCT_INTEL_82598AT_DUAL: 1214 ixgbe_total_ports++; 1215 break; 1216 } 1217 1218 switch (sc->hw.device_id) { 1219 case PCI_PRODUCT_INTEL_82598AF_DUAL: 1220 case PCI_PRODUCT_INTEL_82598AF: 1221 sc->optics = IFM_10G_SR; 1222 break; 1223 case PCI_PRODUCT_INTEL_82598EB_CX4_DUAL: 1224 case PCI_PRODUCT_INTEL_82598EB_CX4: 1225 sc->optics = IFM_10G_CX4; 1226 break; 1227 case PCI_PRODUCT_INTEL_82598EB_XF_LR: 1228 sc->optics = IFM_10G_LR; 1229 break; 1230 case PCI_PRODUCT_INTEL_82598AT_DUAL: 1231 case PCI_PRODUCT_INTEL_82598AT: 1232 sc->optics = IFM_10G_T; 1233 break; 1234 default: 1235 sc->optics = IFM_AUTO; 1236 break; 1237 } 1238 } 1239 1240 /********************************************************************* 1241 * 1242 * Setup the Legacy or MSI Interrupt handler 1243 * 1244 **********************************************************************/ 1245 int 1246 ixgbe_allocate_legacy(struct ix_softc *sc) 1247 { 1248 struct ifnet *ifp = &sc->arpcom.ac_if; 1249 struct ixgbe_osdep *os = &sc->osdep; 1250 struct pci_attach_args *pa = os->os_pa; 1251 const char *intrstr = NULL; 1252 pci_chipset_tag_t pc = pa->pa_pc; 1253 pci_intr_handle_t ih; 1254 1255 /* Legacy RID at 0 */ 1256 if (sc->msix == 0) 1257 sc->rid[0] = 0; 1258 1259 /* We allocate a single interrupt resource */ 1260 if (pci_intr_map(pa, &ih)) { 1261 printf(": couldn't map interrupt\n"); 1262 return (ENXIO); 1263 } 1264 1265 intrstr = pci_intr_string(pc, ih); 1266 sc->tag[0] = pci_intr_establish(pc, ih, IPL_NET, 1267 ixgbe_legacy_irq, sc, ifp->if_xname); 1268 if (sc->tag[0] == NULL) { 1269 printf(": couldn't establish interrupt"); 1270 if (intrstr != NULL) 1271 printf(" at %s", intrstr); 1272 printf("\n"); 1273 return (ENXIO); 1274 } 1275 printf(": %s", intrstr); 1276 1277 return (0); 1278 } 1279 1280 int 1281 ixgbe_allocate_pci_resources(struct ix_softc *sc) 1282 { 1283 struct ixgbe_osdep *os = &sc->osdep; 1284 struct pci_attach_args *pa = os->os_pa; 1285 int val, i; 1286 1287 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1288 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM && 1289 PCI_MAPREG_TYPE(val) != PCI_MAPREG_MEM_TYPE_64BIT) { 1290 printf(": mmba is not mem space\n"); 1291 return (ENXIO); 1292 } 1293 1294 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1295 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1296 printf(": cannot find mem space\n"); 1297 return (ENXIO); 1298 } 1299 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1300 1301 /* 1302 * Init the resource arrays 1303 */ 1304 for (i = 0; i < IXGBE_MSGS; i++) { 1305 sc->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 1306 sc->tag[i] = NULL; 1307 sc->res[i] = NULL; 1308 } 1309 1310 /* Legacy defaults */ 1311 sc->num_tx_queues = 1; 1312 sc->num_rx_queues = 1; 1313 1314 #ifdef notyet 1315 /* Now setup MSI or MSI/X */ 1316 sc->msix = ixgbe_setup_msix(sc); 1317 #endif 1318 sc->hw.back = os; 1319 1320 return (0); 1321 } 1322 1323 void 1324 ixgbe_free_pci_resources(struct ix_softc * sc) 1325 { 1326 struct ixgbe_osdep *os = &sc->osdep; 1327 struct pci_attach_args *pa = os->os_pa; 1328 1329 pci_intr_disestablish(pa->pa_pc, sc->tag[0]); 1330 sc->tag[0] = NULL; 1331 if (os->os_membase != NULL) 1332 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1333 os->os_membase = NULL; 1334 1335 return; 1336 } 1337 1338 /********************************************************************* 1339 * 1340 * Initialize the hardware to a configuration as specified by the 1341 * sc structure. The controller is reset, the EEPROM is 1342 * verified, the MAC address is set, then the shared initialization 1343 * routines are called. 1344 * 1345 **********************************************************************/ 1346 int 1347 ixgbe_hardware_init(struct ix_softc *sc) 1348 { 1349 struct ifnet *ifp = &sc->arpcom.ac_if; 1350 uint16_t csum; 1351 1352 csum = 0; 1353 /* Issue a global reset */ 1354 sc->hw.adapter_stopped = FALSE; 1355 ixgbe_hw(&sc->hw, stop_adapter); 1356 1357 /* Make sure we have a good EEPROM before we read from it */ 1358 if (ixgbe_ee(&sc->hw, validate_checksum, &csum) < 0) { 1359 printf("%s: The EEPROM Checksum Is Not Valid\n", ifp->if_xname); 1360 return (EIO); 1361 } 1362 1363 /* Get Hardware Flow Control setting */ 1364 sc->hw.fc.type = ixgbe_fc_full; 1365 sc->hw.fc.pause_time = IXGBE_FC_PAUSE; 1366 sc->hw.fc.low_water = IXGBE_FC_LO; 1367 sc->hw.fc.high_water = IXGBE_FC_HI; 1368 sc->hw.fc.send_xon = TRUE; 1369 1370 if (ixgbe_hw(&sc->hw, init_hw) != 0) { 1371 printf("%s: Hardware Initialization Failed"); 1372 return (EIO); 1373 } 1374 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 1375 IXGBE_ETH_LENGTH_OF_ADDRESS); 1376 1377 return (0); 1378 } 1379 1380 /********************************************************************* 1381 * 1382 * Setup networking device structure and register an interface. 1383 * 1384 **********************************************************************/ 1385 void 1386 ixgbe_setup_interface(struct ix_softc *sc) 1387 { 1388 struct ixgbe_hw *hw = &sc->hw; 1389 struct ifnet *ifp = &sc->arpcom.ac_if; 1390 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 1391 1392 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1393 ifp->if_baudrate = IF_Gbps(10); 1394 ifp->if_softc = sc; 1395 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1396 ifp->if_ioctl = ixgbe_ioctl; 1397 ifp->if_start = ixgbe_start; 1398 ifp->if_timer = 0; 1399 ifp->if_watchdog = ixgbe_watchdog; 1400 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1401 ETHER_HDR_LEN - ETHER_CRC_LEN; 1402 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1403 IFQ_SET_READY(&ifp->if_snd); 1404 1405 ifp->if_capabilities |= IFCAP_VLAN_MTU; 1406 #ifdef IX_CSUM_OFFLOAD 1407 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1408 ifp->if_capabilities |= IFCAP_CSUM_IPv4; 1409 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 1410 #endif 1411 1412 sc->max_frame_size = 1413 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1414 1415 if ((hw->device_id == PCI_PRODUCT_INTEL_82598AT) || 1416 (hw->device_id == PCI_PRODUCT_INTEL_82598AT_DUAL)) 1417 ixgbe_hw(hw, setup_link_speed, 1418 IXGBE_LINK_SPEED_10GB_FULL | 1419 IXGBE_LINK_SPEED_1GB_FULL, TRUE, TRUE); 1420 else 1421 ixgbe_hw(hw, setup_link_speed, 1422 IXGBE_LINK_SPEED_10GB_FULL, 1423 TRUE, FALSE); 1424 1425 /* 1426 * Specify the media types supported by this sc and register 1427 * callbacks to update media and link information 1428 */ 1429 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1430 ixgbe_media_status); 1431 ifmedia_add(&sc->media, IFM_ETHER | sc->optics | 1432 IFM_FDX, 0, NULL); 1433 if ((hw->device_id == PCI_PRODUCT_INTEL_82598AT) || 1434 (hw->device_id == PCI_PRODUCT_INTEL_82598AT_DUAL)) { 1435 ifmedia_add(&sc->media, 1436 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1437 ifmedia_add(&sc->media, 1438 IFM_ETHER | IFM_1000_T, 0, NULL); 1439 } 1440 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1441 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1442 1443 if_attach(ifp); 1444 ether_ifattach(ifp); 1445 1446 1447 return; 1448 } 1449 1450 int 1451 ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 1452 struct ixgbe_dma_alloc *dma, int mapflags) 1453 { 1454 struct ifnet *ifp = &sc->arpcom.ac_if; 1455 struct ixgbe_osdep *os = &sc->osdep; 1456 int r; 1457 1458 dma->dma_tag = os->os_pa->pa_dmat; 1459 r = bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1460 BUS_DMA_NOWAIT, &dma->dma_map); 1461 if (r != 0) { 1462 printf("%s: ixgbe_dma_malloc: bus_dma_tag_create failed; " 1463 "error %u\n", ifp->if_xname, r); 1464 goto fail_0; 1465 } 1466 1467 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, 1468 &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 1469 if (r != 0) { 1470 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 1471 "error %u\n", ifp->if_xname, r); 1472 goto fail_1; 1473 } 1474 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1475 &dma->dma_vaddr, BUS_DMA_NOWAIT); 1476 if (r != 0) { 1477 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 1478 "error %u\n", ifp->if_xname, r); 1479 goto fail_2; 1480 } 1481 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, 1482 dma->dma_vaddr, size, NULL, 1483 mapflags | BUS_DMA_NOWAIT); 1484 if (r != 0) { 1485 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 1486 "error %u\n", ifp->if_xname, r); 1487 goto fail_3; 1488 } 1489 dma->dma_size = size; 1490 return (0); 1491 fail_3: 1492 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1493 fail_2: 1494 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1495 fail_1: 1496 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1497 fail_0: 1498 dma->dma_map = NULL; 1499 dma->dma_tag = NULL; 1500 return (r); 1501 } 1502 1503 void 1504 ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 1505 { 1506 if (dma->dma_tag == NULL) 1507 return; 1508 1509 if (dma->dma_map != NULL) { 1510 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1511 dma->dma_map->dm_mapsize, 1512 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1513 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1514 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1515 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1516 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1517 } 1518 } 1519 1520 1521 /********************************************************************* 1522 * 1523 * Allocate memory for the transmit and receive rings, and then 1524 * the descriptors associated with each, called only once at attach. 1525 * 1526 **********************************************************************/ 1527 int 1528 ixgbe_allocate_queues(struct ix_softc *sc) 1529 { 1530 struct ifnet *ifp = &sc->arpcom.ac_if; 1531 struct tx_ring *txr; 1532 struct rx_ring *rxr; 1533 int rsize, tsize, error = IXGBE_SUCCESS; 1534 int txconf = 0, rxconf = 0, i; 1535 1536 /* First allocate the TX ring struct memory */ 1537 if (!(sc->tx_rings = 1538 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 1539 sc->num_tx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1540 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 1541 error = ENOMEM; 1542 goto fail; 1543 } 1544 txr = sc->tx_rings; 1545 1546 /* Next allocate the RX */ 1547 if (!(sc->rx_rings = 1548 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 1549 sc->num_rx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1550 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 1551 error = ENOMEM; 1552 goto rx_fail; 1553 } 1554 rxr = sc->rx_rings; 1555 1556 /* For the ring itself */ 1557 tsize = roundup2(sc->num_tx_desc * 1558 sizeof(union ixgbe_adv_tx_desc), 4096); 1559 1560 /* 1561 * Now set up the TX queues, txconf is needed to handle the 1562 * possibility that things fail midcourse and we need to 1563 * undo memory gracefully 1564 */ 1565 for (i = 0; i < sc->num_tx_queues; i++, txconf++) { 1566 /* Set up some basics */ 1567 txr = &sc->tx_rings[i]; 1568 txr->sc = sc; 1569 txr->me = i; 1570 1571 /* Initialize the TX side lock */ 1572 mtx_init(&txr->tx_mtx, IPL_NET); 1573 1574 if (ixgbe_dma_malloc(sc, tsize, 1575 &txr->txdma, BUS_DMA_NOWAIT)) { 1576 printf("%s: Unable to allocate TX Descriptor memory\n", 1577 ifp->if_xname); 1578 error = ENOMEM; 1579 goto err_tx_desc; 1580 } 1581 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 1582 bzero((void *)txr->tx_base, tsize); 1583 1584 if (ixgbe_dma_malloc(sc, sizeof(uint32_t), 1585 &txr->txwbdma, BUS_DMA_NOWAIT)) { 1586 printf("%s: Unable to allocate TX Write Back memory\n", 1587 ifp->if_xname); 1588 error = ENOMEM; 1589 goto err_tx_desc; 1590 } 1591 txr->tx_hwb = (uint32_t *)txr->txwbdma.dma_vaddr; 1592 *txr->tx_hwb = 0; 1593 1594 /* Now allocate transmit buffers for the ring */ 1595 if (ixgbe_allocate_transmit_buffers(txr)) { 1596 printf("%s: Critical Failure setting up transmit buffers\n", 1597 ifp->if_xname); 1598 error = ENOMEM; 1599 goto err_tx_desc; 1600 } 1601 1602 } 1603 1604 /* 1605 * Next the RX queues... 1606 */ 1607 rsize = roundup2(sc->num_rx_desc * 1608 sizeof(union ixgbe_adv_rx_desc), 4096); 1609 for (i = 0; i < sc->num_rx_queues; i++, rxconf++) { 1610 rxr = &sc->rx_rings[i]; 1611 /* Set up some basics */ 1612 rxr->sc = sc; 1613 rxr->me = i; 1614 1615 /* Initialize the TX side lock */ 1616 mtx_init(&rxr->rx_mtx, IPL_NET); 1617 1618 if (ixgbe_dma_malloc(sc, rsize, 1619 &rxr->rxdma, BUS_DMA_NOWAIT)) { 1620 printf("%s: Unable to allocate RxDescriptor memory\n", 1621 ifp->if_xname); 1622 error = ENOMEM; 1623 goto err_rx_desc; 1624 } 1625 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 1626 bzero((void *)rxr->rx_base, rsize); 1627 1628 /* Allocate receive buffers for the ring*/ 1629 if (ixgbe_allocate_receive_buffers(rxr)) { 1630 printf("%s: Critical Failure setting up receive buffers\n", 1631 ifp->if_xname); 1632 error = ENOMEM; 1633 goto err_rx_desc; 1634 } 1635 } 1636 1637 return (0); 1638 1639 err_rx_desc: 1640 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 1641 ixgbe_dma_free(sc, &rxr->rxdma); 1642 err_tx_desc: 1643 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) { 1644 ixgbe_dma_free(sc, &txr->txdma); 1645 ixgbe_dma_free(sc, &txr->txwbdma); 1646 } 1647 free(sc->rx_rings, M_DEVBUF); 1648 rx_fail: 1649 free(sc->tx_rings, M_DEVBUF); 1650 fail: 1651 return (error); 1652 } 1653 1654 /********************************************************************* 1655 * 1656 * Allocate memory for tx_buffer structures. The tx_buffer stores all 1657 * the information needed to transmit a packet on the wire. This is 1658 * called only once at attach, setup is done every reset. 1659 * 1660 **********************************************************************/ 1661 int 1662 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 1663 { 1664 struct ix_softc *sc = txr->sc; 1665 struct ixgbe_osdep *os = &sc->osdep; 1666 struct ifnet *ifp = &sc->arpcom.ac_if; 1667 struct ixgbe_tx_buf *txbuf; 1668 int error, i; 1669 1670 txr->txtag = os->os_pa->pa_dmat; 1671 if (!(txr->tx_buffers = 1672 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 1673 sc->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1674 printf("%s: Unable to allocate tx_buffer memory\n", ifp->if_xname); 1675 error = ENOMEM; 1676 goto fail; 1677 } 1678 1679 /* Create the descriptor buffer dma maps */ 1680 txbuf = txr->tx_buffers; 1681 for (i = 0; i < sc->num_tx_desc; i++, txbuf++) { 1682 error = bus_dmamap_create(txr->txtag, IXGBE_TSO_SIZE, 1683 IXGBE_MAX_SCATTER, PAGE_SIZE, 0, 1684 BUS_DMA_NOWAIT, &txbuf->map); 1685 if (error != 0) { 1686 printf("%s: Unable to create TX DMA map\n", ifp->if_xname); 1687 goto fail; 1688 } 1689 } 1690 1691 return 0; 1692 fail: 1693 /* We free all, it handles case where we are in the middle */ 1694 ixgbe_free_transmit_structures(sc); 1695 return (error); 1696 } 1697 1698 /********************************************************************* 1699 * 1700 * Initialize a transmit ring. 1701 * 1702 **********************************************************************/ 1703 void 1704 ixgbe_setup_transmit_ring(struct tx_ring *txr) 1705 { 1706 struct ix_softc *sc = txr->sc; 1707 struct ixgbe_tx_buf *txbuf; 1708 int i; 1709 1710 /* Clear the old ring contents */ 1711 bzero((void *)txr->tx_base, 1712 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 1713 /* Reset indices */ 1714 txr->next_avail_tx_desc = 0; 1715 txr->next_tx_to_clean = 0; 1716 1717 /* Free any existing tx buffers. */ 1718 txbuf = txr->tx_buffers; 1719 for (i = 0; i < sc->num_tx_desc; i++, txbuf++) { 1720 if (txbuf->m_head != NULL) { 1721 bus_dmamap_sync(txr->txtag, txbuf->map, 1722 0, txbuf->map->dm_mapsize, 1723 BUS_DMASYNC_POSTWRITE); 1724 bus_dmamap_unload(txr->txtag, txbuf->map); 1725 m_freem(txbuf->m_head); 1726 } 1727 txbuf->m_head = NULL; 1728 } 1729 1730 /* Set number of descriptors available */ 1731 txr->tx_avail = sc->num_tx_desc; 1732 1733 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1734 0, txr->txdma.dma_map->dm_mapsize, 1735 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1736 1737 } 1738 1739 /********************************************************************* 1740 * 1741 * Initialize all transmit rings. 1742 * 1743 **********************************************************************/ 1744 int 1745 ixgbe_setup_transmit_structures(struct ix_softc *sc) 1746 { 1747 struct tx_ring *txr = sc->tx_rings; 1748 int i; 1749 1750 for (i = 0; i < sc->num_tx_queues; i++, txr++) 1751 ixgbe_setup_transmit_ring(txr); 1752 1753 return (0); 1754 } 1755 1756 /********************************************************************* 1757 * 1758 * Enable transmit unit. 1759 * 1760 **********************************************************************/ 1761 void 1762 ixgbe_initialize_transmit_units(struct ix_softc *sc) 1763 { 1764 struct ifnet *ifp = &sc->arpcom.ac_if; 1765 struct tx_ring *txr; 1766 struct ixgbe_hw *hw = &sc->hw; 1767 int i; 1768 uint64_t tdba, txhwb; 1769 uint32_t txctrl; 1770 1771 /* Setup the Base and Length of the Tx Descriptor Ring */ 1772 1773 for (i = 0; i < sc->num_tx_queues; i++) { 1774 txr = &sc->tx_rings[i]; 1775 1776 /* Setup descriptor base address */ 1777 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 1778 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 1779 (tdba & 0x00000000ffffffffULL)); 1780 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 1781 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 1782 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1783 1784 /* Setup for Head WriteBack */ 1785 txhwb = txr->txwbdma.dma_map->dm_segs[0].ds_addr; 1786 txhwb |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 1787 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(i), 1788 (txhwb & 0x00000000ffffffffULL)); 1789 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(i), 1790 (txhwb >> 32)); 1791 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 1792 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 1793 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 1794 1795 /* Setup the HW Tx Head and Tail descriptor pointers */ 1796 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 1797 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 1798 1799 /* Setup Transmit Descriptor Cmd Settings */ 1800 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 1801 1802 txr->watchdog_timer = 0; 1803 } 1804 ifp->if_timer = 0; 1805 1806 return; 1807 } 1808 1809 /********************************************************************* 1810 * 1811 * Free all transmit rings. 1812 * 1813 **********************************************************************/ 1814 void 1815 ixgbe_free_transmit_structures(struct ix_softc *sc) 1816 { 1817 struct tx_ring *txr = sc->tx_rings; 1818 int i; 1819 1820 for (i = 0; i < sc->num_tx_queues; i++, txr++) { 1821 ixgbe_free_transmit_buffers(txr); 1822 ixgbe_dma_free(sc, &txr->txdma); 1823 ixgbe_dma_free(sc, &txr->txwbdma); 1824 } 1825 free(sc->tx_rings, M_DEVBUF); 1826 } 1827 1828 /********************************************************************* 1829 * 1830 * Free transmit ring related data structures. 1831 * 1832 **********************************************************************/ 1833 void 1834 ixgbe_free_transmit_buffers(struct tx_ring *txr) 1835 { 1836 struct ix_softc *sc = txr->sc; 1837 struct ixgbe_tx_buf *tx_buffer; 1838 int i; 1839 1840 INIT_DEBUGOUT("free_transmit_ring: begin"); 1841 1842 if (txr->tx_buffers == NULL) 1843 return; 1844 1845 tx_buffer = txr->tx_buffers; 1846 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 1847 if (tx_buffer->m_head != NULL) { 1848 bus_dmamap_sync(txr->txtag, tx_buffer->map, 1849 0, tx_buffer->map->dm_mapsize, 1850 BUS_DMASYNC_POSTWRITE); 1851 bus_dmamap_unload(txr->txtag, 1852 tx_buffer->map); 1853 m_freem(tx_buffer->m_head); 1854 if (tx_buffer->map != NULL) { 1855 bus_dmamap_destroy(txr->txtag, 1856 tx_buffer->map); 1857 } 1858 } else if (tx_buffer->map != NULL) { 1859 bus_dmamap_unload(txr->txtag, 1860 tx_buffer->map); 1861 bus_dmamap_destroy(txr->txtag, 1862 tx_buffer->map); 1863 } 1864 tx_buffer->m_head = NULL; 1865 tx_buffer->map = NULL; 1866 } 1867 1868 if (txr->tx_buffers != NULL) 1869 free(txr->tx_buffers, M_DEVBUF); 1870 txr->tx_buffers = NULL; 1871 txr->txtag = NULL; 1872 } 1873 1874 #ifdef IX_CSUM_OFFLOAD 1875 /********************************************************************* 1876 * 1877 * Advanced Context Descriptor setup for VLAN or CSUM 1878 * 1879 **********************************************************************/ 1880 1881 int 1882 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 1883 { 1884 struct ix_softc *sc = txr->sc; 1885 struct ifnet *ifp = &sc->arpcom.ac_if; 1886 struct ixgbe_adv_tx_context_desc *TXD; 1887 struct ixgbe_tx_buf *tx_buffer; 1888 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 1889 struct ip *ip; 1890 struct ip6_hdr *ip6; 1891 int ehdrlen, ip_hlen = 0; 1892 uint16_t etype; 1893 uint8_t ipproto = 0; 1894 int offload = TRUE; 1895 int ctxd = txr->next_avail_tx_desc; 1896 #if NVLAN > 0 1897 struct ether_vlan_header *eh; 1898 struct ifvlan *ifv = NULL; 1899 1900 if ((mp->m_flags & (M_PROTO1|M_PKTHDR)) == (M_PROTO1|M_PKTHDR) && 1901 mp->m_pkthdr.rcvif != NULL) 1902 ifv = mp->m_pkthdr.rcvif->if_softc; 1903 #else 1904 struct ether_header *eh; 1905 #endif 1906 1907 if ((ifp->if_capabilities & IFCAP_CSUM_IPv4) == 0) 1908 offload = FALSE; 1909 1910 tx_buffer = &txr->tx_buffers[ctxd]; 1911 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 1912 1913 /* 1914 * In advanced descriptors the vlan tag must 1915 * be placed into the descriptor itself. 1916 */ 1917 #if NVLAN > 0 1918 if (ifv != NULL) { 1919 vlan_macip_lens |= 1920 htole16(ifv->ifv_tag) << IXGBE_ADVTXD_VLAN_SHIFT; 1921 } else 1922 #endif 1923 if (offload == FALSE) 1924 return FALSE; /* No need for CTX */ 1925 1926 /* 1927 * Determine where frame payload starts. 1928 * Jump over vlan headers if already present, 1929 * helpful for QinQ too. 1930 */ 1931 #if NVLAN > 0 1932 eh = mtod(mp, struct ether_vlan_header *); 1933 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1934 etype = ntohs(eh->evl_proto); 1935 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1936 } else { 1937 etype = ntohs(eh->evl_encap_proto); 1938 ehdrlen = ETHER_HDR_LEN; 1939 } 1940 #else 1941 eh = mtod(mp, struct ether_header *); 1942 etype = ntohs(eh->ether_type); 1943 ehdrlen = ETHER_HDR_LEN; 1944 #endif 1945 1946 /* Set the ether header length */ 1947 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 1948 1949 switch (etype) { 1950 case ETHERTYPE_IP: 1951 ip = (struct ip *)(mp->m_data + ehdrlen); 1952 ip_hlen = ip->ip_hl << 2; 1953 if (mp->m_len < ehdrlen + ip_hlen) 1954 return FALSE; /* failure */ 1955 ipproto = ip->ip_p; 1956 if (mp->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT) 1957 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 1958 break; 1959 case ETHERTYPE_IPV6: 1960 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 1961 ip_hlen = sizeof(struct ip6_hdr); 1962 if (mp->m_len < ehdrlen + ip_hlen) 1963 return FALSE; /* failure */ 1964 ipproto = ip6->ip6_nxt; 1965 if (mp->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT) 1966 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 1967 break; 1968 default: 1969 offload = FALSE; 1970 break; 1971 } 1972 1973 vlan_macip_lens |= ip_hlen; 1974 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 1975 1976 switch (ipproto) { 1977 case IPPROTO_TCP: 1978 if (mp->m_pkthdr.csum_flags & M_TCPV4_CSUM_OUT) 1979 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 1980 break; 1981 case IPPROTO_UDP: 1982 if (mp->m_pkthdr.csum_flags & M_UDPV4_CSUM_OUT) 1983 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 1984 break; 1985 default: 1986 offload = FALSE; 1987 break; 1988 } 1989 1990 /* Now copy bits into descriptor */ 1991 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 1992 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 1993 TXD->seqnum_seed = htole32(0); 1994 TXD->mss_l4len_idx = htole32(0); 1995 1996 #ifndef NO_82598_A0_SUPPORT 1997 if (sc->hw.revision_id == 0) 1998 desc_flip(TXD); 1999 #endif 2000 2001 tx_buffer->m_head = NULL; 2002 2003 /* We've consumed the first desc, adjust counters */ 2004 if (++ctxd == sc->num_tx_desc) 2005 ctxd = 0; 2006 txr->next_avail_tx_desc = ctxd; 2007 --txr->tx_avail; 2008 2009 return (offload); 2010 } 2011 2012 #ifdef notyet 2013 /********************************************************************** 2014 * 2015 * Setup work for hardware segmentation offload (TSO) on 2016 * scs using advanced tx descriptors 2017 * 2018 **********************************************************************/ 2019 int 2020 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, uint32_t *paylen) 2021 { 2022 struct ix_softc *sc = txr->sc; 2023 struct ixgbe_adv_tx_context_desc *TXD; 2024 struct ixgbe_tx_buf *tx_buffer; 2025 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2026 uint32_t mss_l4len_idx = 0; 2027 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2028 #if NVLAN > 0 2029 uint16_t vtag = 0; 2030 struct ether_vlan_header *eh; 2031 2032 struct ifvlan *ifv = NULL; 2033 2034 if ((mp->m_flags & (M_PROTO1|M_PKTHDR)) == (M_PROTO1|M_PKTHDR) && 2035 mp->m_pkthdr.rcvif != NULL) 2036 ifv = mp->m_pkthdr.rcvif->if_softc; 2037 #else 2038 struct ether_header *eh; 2039 #endif 2040 struct ip *ip; 2041 struct tcphdr *th; 2042 2043 if (((mp->m_pkthdr.csum_flags & CSUM_TSO) == 0) || 2044 (mp->m_pkthdr.len <= IXGBE_TX_BUFFER_SIZE)) 2045 return FALSE; 2046 2047 /* 2048 * Determine where frame payload starts. 2049 * Jump over vlan headers if already present 2050 */ 2051 #if NVLAN > 0 2052 eh = mtod(mp, struct ether_vlan_header *); 2053 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2054 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2055 else 2056 ehdrlen = ETHER_HDR_LEN; 2057 #else 2058 eh = mtod(mp, struct ether_header *); 2059 ehdrlen = ETHER_HDR_LEN; 2060 #endif 2061 2062 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2063 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2064 return FALSE; 2065 2066 ctxd = txr->next_avail_tx_desc; 2067 tx_buffer = &txr->tx_buffers[ctxd]; 2068 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2069 2070 ip = (struct ip *)(mp->m_data + ehdrlen); 2071 if (ip->ip_p != IPPROTO_TCP) 2072 return FALSE; /* 0 */ 2073 ip->ip_len = 0; 2074 ip->ip_sum = 0; 2075 ip_hlen = ip->ip_hl << 2; 2076 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2077 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2078 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2079 tcp_hlen = th->th_off << 2; 2080 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2081 /* This is used in the transmit desc in encap */ 2082 *paylen = mp->m_pkthdr.len - hdrlen; 2083 2084 #if NVLAN > 0 2085 /* VLAN MACLEN IPLEN */ 2086 if (ifv != NULL) { 2087 vtag = htole16(mp->m_pkthdr.ether_vtag); 2088 vlan_macip_lens |= (ifv->ifv_tag << IXGBE_ADVTXD_VLAN_SHIFT); 2089 } 2090 #endif 2091 2092 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2093 vlan_macip_lens |= ip_hlen; 2094 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2095 2096 /* ADV DTYPE TUCMD */ 2097 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2098 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2099 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2100 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2101 2102 2103 /* MSS L4LEN IDX */ 2104 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2105 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2106 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2107 2108 TXD->seqnum_seed = htole32(0); 2109 tx_buffer->m_head = NULL; 2110 2111 #ifndef NO_82598_A0_SUPPORT 2112 if (sc->hw.revision_id == 0) 2113 desc_flip(TXD); 2114 #endif 2115 2116 if (++ctxd == sc->num_tx_desc) 2117 ctxd = 0; 2118 2119 txr->tx_avail--; 2120 txr->next_avail_tx_desc = ctxd; 2121 return TRUE; 2122 } 2123 2124 #else /* For 6.2 RELEASE */ 2125 /* This makes it easy to keep the code common */ 2126 int 2127 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, uint32_t *paylen) 2128 { 2129 return (FALSE); 2130 } 2131 #endif 2132 #endif 2133 2134 /********************************************************************** 2135 * 2136 * Examine each tx_buffer in the used queue. If the hardware is done 2137 * processing the packet then free associated resources. The 2138 * tx_buffer is put back on the free queue. 2139 * 2140 **********************************************************************/ 2141 int 2142 ixgbe_txeof(struct tx_ring *txr) 2143 { 2144 struct ix_softc *sc = txr->sc; 2145 struct ifnet *ifp = &sc->arpcom.ac_if; 2146 uint first, last, done, num_avail; 2147 struct ixgbe_tx_buf *tx_buffer; 2148 struct ixgbe_legacy_tx_desc *tx_desc; 2149 2150 if (txr->tx_avail == sc->num_tx_desc) 2151 return FALSE; 2152 2153 num_avail = txr->tx_avail; 2154 first = txr->next_tx_to_clean; 2155 2156 tx_buffer = &txr->tx_buffers[first]; 2157 2158 /* For cleanup we just use legacy struct */ 2159 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2160 2161 /* Get the HWB */ 2162 bus_dmamap_sync(txr->txwbdma.dma_tag, txr->txwbdma.dma_map, 2163 0, txr->txwbdma.dma_map->dm_mapsize, 2164 BUS_DMASYNC_POSTREAD); 2165 done = *txr->tx_hwb; 2166 2167 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2168 0, txr->txdma.dma_map->dm_mapsize, 2169 BUS_DMASYNC_POSTREAD); 2170 2171 while (TRUE) { 2172 /* We clean the range til last head write back */ 2173 while (first != done) { 2174 tx_desc->upper.data = 0; 2175 tx_desc->lower.data = 0; 2176 tx_desc->buffer_addr = 0; 2177 num_avail++; 2178 2179 if (tx_buffer->m_head) { 2180 ifp->if_opackets++; 2181 bus_dmamap_sync(txr->txtag, 2182 tx_buffer->map, 2183 0, tx_buffer->map->dm_mapsize, 2184 BUS_DMASYNC_POSTWRITE); 2185 bus_dmamap_unload(txr->txtag, 2186 tx_buffer->map); 2187 m_freem(tx_buffer->m_head); 2188 tx_buffer->m_head = NULL; 2189 } 2190 2191 if (++first == sc->num_tx_desc) 2192 first = 0; 2193 2194 tx_buffer = &txr->tx_buffers[first]; 2195 tx_desc = (struct ixgbe_legacy_tx_desc *) 2196 &txr->tx_base[first]; 2197 } 2198 /* See if there is more work now */ 2199 last = done; 2200 bus_dmamap_sync(txr->txwbdma.dma_tag, txr->txwbdma.dma_map, 2201 0, txr->txwbdma.dma_map->dm_mapsize, 2202 BUS_DMASYNC_POSTREAD); 2203 done = *txr->tx_hwb; 2204 if (last == done) 2205 break; 2206 } 2207 2208 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2209 0, txr->txdma.dma_map->dm_mapsize, 2210 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2211 2212 txr->next_tx_to_clean = first; 2213 2214 /* 2215 * If we have enough room, clear IFF_OACTIVE to tell the stack that 2216 * it is OK to send packets. If there are no pending descriptors, 2217 * clear the timeout. Otherwise, if some descriptors have been freed, 2218 * restart the timeout. 2219 */ 2220 if (num_avail > IXGBE_TX_CLEANUP_THRESHOLD) { 2221 ifp->if_flags &= ~IFF_OACTIVE; 2222 2223 /* If all are clean turn off the timer */ 2224 if (num_avail == sc->num_tx_desc) { 2225 ifp->if_timer = 0; 2226 txr->watchdog_timer = 0; 2227 txr->tx_avail = num_avail; 2228 return FALSE; 2229 } 2230 /* Some were cleaned, so reset timer */ 2231 else if (num_avail != txr->tx_avail) { 2232 ifp->if_timer = IXGBE_TX_TIMEOUT; 2233 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 2234 } 2235 } 2236 2237 txr->tx_avail = num_avail; 2238 2239 return TRUE; 2240 } 2241 2242 /********************************************************************* 2243 * 2244 * Get a buffer from system mbuf buffer pool. 2245 * 2246 **********************************************************************/ 2247 int 2248 ixgbe_get_buf(struct rx_ring *rxr, int i, struct mbuf *nmp) 2249 { 2250 struct ix_softc *sc = rxr->sc; 2251 struct mbuf *mp = nmp; 2252 bus_dmamap_t map; 2253 int error, old, s = 0; 2254 int size = MCLBYTES; 2255 struct ixgbe_rx_buf *rxbuf; 2256 2257 #ifdef notyet 2258 /* Are we going to Jumbo clusters? */ 2259 if (sc->bigbufs) { 2260 size = MJUMPAGESIZE; 2261 s = 1; 2262 }; 2263 2264 mp = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, size); 2265 if (mp == NULL) { 2266 sc->mbuf_alloc_failed++; 2267 return (ENOBUFS); 2268 } 2269 #endif 2270 2271 if (mp == NULL) { 2272 MGETHDR(mp, M_DONTWAIT, MT_DATA); 2273 if (mp == NULL) { 2274 sc->mbuf_alloc_failed++; 2275 return (ENOBUFS); 2276 } 2277 MCLGET(mp, M_DONTWAIT); 2278 if ((mp->m_flags & M_EXT) == 0) { 2279 m_freem(mp); 2280 sc->mbuf_cluster_failed++; 2281 return (ENOBUFS); 2282 } 2283 mp->m_len = mp->m_pkthdr.len = size; 2284 } else { 2285 mp->m_len = mp->m_pkthdr.len = size; 2286 mp->m_data = mp->m_ext.ext_buf; 2287 mp->m_next = NULL; 2288 } 2289 2290 if (sc->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 2291 m_adj(mp, ETHER_ALIGN); 2292 2293 /* 2294 * Using memory from the mbuf cluster pool, invoke the bus_dma 2295 * machinery to arrange the memory mapping. 2296 */ 2297 error = bus_dmamap_load_mbuf(rxr->rxtag[s], rxr->spare_map[s], 2298 mp, BUS_DMA_NOWAIT); 2299 if (error) { 2300 m_freem(mp); 2301 return (error); 2302 } 2303 2304 /* Now check our target buffer for existing mapping */ 2305 rxbuf = &rxr->rx_buffers[i]; 2306 old = rxbuf->bigbuf; 2307 if (rxbuf->m_head != NULL) 2308 bus_dmamap_unload(rxr->rxtag[old], rxbuf->map[old]); 2309 2310 map = rxbuf->map[old]; 2311 rxbuf->map[s] = rxr->spare_map[s]; 2312 rxr->spare_map[old] = map; 2313 rxbuf->m_head = mp; 2314 rxbuf->bigbuf = s; 2315 2316 rxr->rx_base[i].read.pkt_addr = 2317 htole64(rxbuf->map[s]->dm_segs[0].ds_addr); 2318 2319 bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s], 2320 0, rxbuf->map[s]->dm_mapsize, BUS_DMASYNC_PREREAD); 2321 2322 #ifndef NO_82598_A0_SUPPORT 2323 /* A0 needs to One's Compliment descriptors */ 2324 if (sc->hw.revision_id == 0) { 2325 struct dhack {uint32_t a1; uint32_t a2; uint32_t b1; uint32_t b2;}; 2326 struct dhack *d; 2327 2328 d = (struct dhack *)&rxr->rx_base[i]; 2329 d->a1 = ~(d->a1); 2330 d->a2 = ~(d->a2); 2331 } 2332 #endif 2333 2334 return (0); 2335 } 2336 2337 /********************************************************************* 2338 * 2339 * Allocate memory for rx_buffer structures. Since we use one 2340 * rx_buffer per received packet, the maximum number of rx_buffer's 2341 * that we'll need is equal to the number of receive descriptors 2342 * that we've allocated. 2343 * 2344 **********************************************************************/ 2345 int 2346 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2347 { 2348 struct ix_softc *sc = rxr->sc; 2349 struct ifnet *ifp = &sc->arpcom.ac_if; 2350 struct ixgbe_osdep *os = &sc->osdep; 2351 struct ixgbe_rx_buf *rxbuf; 2352 int i, bsize, error; 2353 2354 bsize = sizeof(struct ixgbe_rx_buf) * sc->num_rx_desc; 2355 if (!(rxr->rx_buffers = 2356 (struct ixgbe_rx_buf *) malloc(bsize, 2357 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2358 printf("%s: Unable to allocate rx_buffer memory\n", ifp->if_xname); 2359 error = ENOMEM; 2360 goto fail; 2361 } 2362 rxr->rxtag[0] = rxr->rxtag[1] = os->os_pa->pa_dmat; 2363 2364 /* Create the spare maps (used by getbuf) */ 2365 error = bus_dmamap_create(rxr->rxtag[0], 2366 MCLBYTES, 1, MCLBYTES, 0, BUS_DMA_NOWAIT, &rxr->spare_map[0]); 2367 if (error) { 2368 printf("%s: %s: bus_dmamap_create failed: %d\n", ifp->if_xname, 2369 __func__, error); 2370 goto fail; 2371 } 2372 error = bus_dmamap_create(rxr->rxtag[1], 2373 MJUMPAGESIZE, 1, MJUMPAGESIZE, 0, BUS_DMA_NOWAIT, &rxr->spare_map[1]); 2374 if (error) { 2375 printf("%s: %s: bus_dmamap_create failed: %d\n", ifp->if_xname, 2376 __func__, error); 2377 goto fail; 2378 } 2379 2380 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2381 rxbuf = &rxr->rx_buffers[i]; 2382 error = bus_dmamap_create(rxr->rxtag[0], MCLBYTES, 1, MCLBYTES, 2383 0, BUS_DMA_NOWAIT, &rxbuf->map[0]); 2384 if (error) { 2385 printf("%s: Unable to create Small RX DMA map\n", ifp->if_xname); 2386 goto fail; 2387 } 2388 error = bus_dmamap_create(rxr->rxtag[1], MJUMPAGESIZE, 1, MJUMPAGESIZE, 2389 0, BUS_DMA_NOWAIT, &rxbuf->map[1]); 2390 if (error) { 2391 printf("%s: Unable to create Large RX DMA map\n", ifp->if_xname); 2392 goto fail; 2393 } 2394 } 2395 2396 return (0); 2397 2398 fail: 2399 /* Frees all, but can handle partial completion */ 2400 ixgbe_free_receive_structures(sc); 2401 return (error); 2402 } 2403 2404 /********************************************************************* 2405 * 2406 * Initialize a receive ring and its buffers. 2407 * 2408 **********************************************************************/ 2409 int 2410 ixgbe_setup_receive_ring(struct rx_ring *rxr) 2411 { 2412 struct ix_softc *sc = rxr->sc; 2413 struct ixgbe_rx_buf *rxbuf; 2414 int j, rsize, s = 0, i; 2415 2416 rsize = roundup2(sc->num_rx_desc * 2417 sizeof(union ixgbe_adv_rx_desc), 4096); 2418 /* Clear the ring contents */ 2419 bzero((void *)rxr->rx_base, rsize); 2420 2421 /* 2422 ** Free current RX buffers: the size buffer 2423 ** that is loaded is indicated by the buffer 2424 ** bigbuf value. 2425 */ 2426 for (i = 0; i < sc->num_rx_desc; i++) { 2427 rxbuf = &rxr->rx_buffers[i]; 2428 s = rxbuf->bigbuf; 2429 if (rxbuf->m_head != NULL) { 2430 bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s], 2431 0, rxbuf->map[s]->dm_mapsize, 2432 BUS_DMASYNC_POSTREAD); 2433 bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]); 2434 m_freem(rxbuf->m_head); 2435 rxbuf->m_head = NULL; 2436 } 2437 } 2438 2439 for (j = 0; j < sc->num_rx_desc; j++) { 2440 if (ixgbe_get_buf(rxr, j, NULL) == ENOBUFS) { 2441 rxr->rx_buffers[j].m_head = NULL; 2442 rxr->rx_base[j].read.pkt_addr = 0; 2443 /* If we fail some may have change size */ 2444 s = sc->bigbufs; 2445 goto fail; 2446 } 2447 } 2448 2449 /* Setup our descriptor indices */ 2450 rxr->next_to_check = 0; 2451 rxr->last_cleaned = 0; 2452 2453 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2454 0, rxr->rxdma.dma_map->dm_mapsize, 2455 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2456 2457 return (0); 2458 fail: 2459 /* 2460 * We need to clean up any buffers allocated so far 2461 * 'j' is the failing index, decrement it to get the 2462 * last success. 2463 */ 2464 for (--j; j < 0; j--) { 2465 rxbuf = &rxr->rx_buffers[j]; 2466 if (rxbuf->m_head != NULL) { 2467 bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s], 2468 0, rxbuf->map[s]->dm_mapsize, 2469 BUS_DMASYNC_POSTREAD); 2470 bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]); 2471 m_freem(rxbuf->m_head); 2472 rxbuf->m_head = NULL; 2473 } 2474 } 2475 return (ENOBUFS); 2476 } 2477 2478 /********************************************************************* 2479 * 2480 * Initialize all receive rings. 2481 * 2482 **********************************************************************/ 2483 int 2484 ixgbe_setup_receive_structures(struct ix_softc *sc) 2485 { 2486 struct rx_ring *rxr = sc->rx_rings; 2487 int i, j, s; 2488 2489 for (i = 0; i < sc->num_rx_queues; i++, rxr++) 2490 if (ixgbe_setup_receive_ring(rxr)) 2491 goto fail; 2492 2493 return (0); 2494 fail: 2495 /* 2496 * Free RX buffers allocated so far, we will only handle 2497 * the rings that completed, the failing case will have 2498 * cleaned up for itself. The value of 'i' will be the 2499 * failed ring so we must pre-decrement it. 2500 */ 2501 rxr = sc->rx_rings; 2502 for (--i; i > 0; i--, rxr++) { 2503 for (j = 0; j < sc->num_rx_desc; j++) { 2504 struct ixgbe_rx_buf *rxbuf; 2505 rxbuf = &rxr->rx_buffers[j]; 2506 s = rxbuf->bigbuf; 2507 if (rxbuf->m_head != NULL) { 2508 bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s], 2509 0, rxbuf->map[s]->dm_mapsize, BUS_DMASYNC_POSTREAD); 2510 bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]); 2511 m_freem(rxbuf->m_head); 2512 rxbuf->m_head = NULL; 2513 } 2514 } 2515 } 2516 2517 return (ENOBUFS); 2518 } 2519 2520 /********************************************************************* 2521 * 2522 * Enable receive unit. 2523 * 2524 **********************************************************************/ 2525 void 2526 ixgbe_initialize_receive_units(struct ix_softc *sc) 2527 { 2528 struct rx_ring *rxr = sc->rx_rings; 2529 struct ifnet *ifp = &sc->arpcom.ac_if; 2530 uint32_t rxctrl, fctrl, srrctl, rxcsum; 2531 uint32_t reta, mrqc, hlreg, linkvec; 2532 uint32_t random[10]; 2533 int i; 2534 2535 /* 2536 * Make sure receives are disabled while 2537 * setting up the descriptor ring 2538 */ 2539 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 2540 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXCTRL, 2541 rxctrl & ~IXGBE_RXCTRL_RXEN); 2542 2543 /* Enable broadcasts */ 2544 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 2545 fctrl |= IXGBE_FCTRL_BAM; 2546 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 2547 2548 hlreg = IXGBE_READ_REG(&sc->hw, IXGBE_HLREG0); 2549 if (ifp->if_mtu > ETHERMTU) 2550 hlreg |= IXGBE_HLREG0_JUMBOEN; 2551 else 2552 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 2553 IXGBE_WRITE_REG(&sc->hw, IXGBE_HLREG0, hlreg); 2554 2555 srrctl = IXGBE_READ_REG(&sc->hw, IXGBE_SRRCTL(0)); 2556 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 2557 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 2558 if (sc->bigbufs) 2559 srrctl |= 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2560 else 2561 srrctl |= 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2562 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2563 IXGBE_WRITE_REG(&sc->hw, IXGBE_SRRCTL(0), srrctl); 2564 2565 /* Set Queue moderation rate */ 2566 for (i = 0; i < IXGBE_MSGS; i++) 2567 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(i), DEFAULT_ITR); 2568 2569 /* Set Link moderation lower */ 2570 linkvec = sc->num_tx_queues + sc->num_rx_queues; 2571 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(linkvec), LINK_ITR); 2572 2573 for (i = 0; i < sc->num_rx_queues; i++, rxr++) { 2574 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2575 /* Setup the Base and Length of the Rx Descriptor Ring */ 2576 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDBAL(i), 2577 (rdba & 0x00000000ffffffffULL)); 2578 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDBAH(i), (rdba >> 32)); 2579 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDLEN(i), 2580 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2581 2582 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2583 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDH(i), 0); 2584 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), 2585 sc->num_rx_desc - 1); 2586 } 2587 2588 rxcsum = IXGBE_READ_REG(&sc->hw, IXGBE_RXCSUM); 2589 2590 if (sc->num_rx_queues > 1) { 2591 /* set up random bits */ 2592 arc4random_buf(&random, sizeof(random)); 2593 switch (sc->num_rx_queues) { 2594 case 8: 2595 case 4: 2596 reta = 0x00010203; 2597 break; 2598 case 2: 2599 reta = 0x00010001; 2600 break; 2601 default: 2602 reta = 0x00000000; 2603 } 2604 2605 /* Set up the redirection table */ 2606 for (i = 0; i < 32; i++) { 2607 IXGBE_WRITE_REG(&sc->hw, IXGBE_RETA(i), reta); 2608 if (sc->num_rx_queues > 4) { 2609 ++i; 2610 IXGBE_WRITE_REG(&sc->hw, 2611 IXGBE_RETA(i), 0x04050607); 2612 } 2613 } 2614 2615 /* Now fill our hash function seeds */ 2616 for (i = 0; i < 10; i++) 2617 IXGBE_WRITE_REG_ARRAY(&sc->hw, 2618 IXGBE_RSSRK(0), i, random[i]); 2619 2620 mrqc = IXGBE_MRQC_RSSEN 2621 /* Perform hash on these packet types */ 2622 | IXGBE_MRQC_RSS_FIELD_IPV4 2623 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 2624 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 2625 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 2626 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 2627 | IXGBE_MRQC_RSS_FIELD_IPV6 2628 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 2629 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 2630 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2631 IXGBE_WRITE_REG(&sc->hw, IXGBE_MRQC, mrqc); 2632 2633 /* RSS and RX IPP Checksum are mutually exclusive */ 2634 rxcsum |= IXGBE_RXCSUM_PCSD; 2635 } 2636 2637 #if defined(IX_CSUM_OFFLOAD) 2638 rxcsum |= IXGBE_RXCSUM_PCSD; 2639 #endif 2640 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2641 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2642 2643 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXCSUM, rxcsum); 2644 2645 /* Enable Receive engine */ 2646 rxctrl |= (IXGBE_RXCTRL_RXEN | IXGBE_RXCTRL_DMBYPS); 2647 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXCTRL, rxctrl); 2648 2649 return; 2650 } 2651 2652 /********************************************************************* 2653 * 2654 * Free all receive rings. 2655 * 2656 **********************************************************************/ 2657 void 2658 ixgbe_free_receive_structures(struct ix_softc *sc) 2659 { 2660 struct rx_ring *rxr = sc->rx_rings; 2661 int i; 2662 2663 for (i = 0; i < sc->num_rx_queues; i++, rxr++) { 2664 ixgbe_free_receive_buffers(rxr); 2665 /* Free the ring memory as well */ 2666 ixgbe_dma_free(sc, &rxr->rxdma); 2667 } 2668 2669 free(sc->rx_rings, M_DEVBUF); 2670 } 2671 2672 /********************************************************************* 2673 * 2674 * Free receive ring data structures 2675 * 2676 **********************************************************************/ 2677 void 2678 ixgbe_free_receive_buffers(struct rx_ring *rxr) 2679 { 2680 struct ix_softc *sc = NULL; 2681 struct ixgbe_rx_buf *rxbuf = NULL; 2682 int i, s; 2683 2684 INIT_DEBUGOUT("free_receive_buffers: begin"); 2685 sc = rxr->sc; 2686 if (rxr->rx_buffers != NULL) { 2687 rxbuf = &rxr->rx_buffers[0]; 2688 for (i = 0; i < sc->num_rx_desc; i++) { 2689 int s = rxbuf->bigbuf; 2690 if (rxbuf->map != NULL) { 2691 bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]); 2692 bus_dmamap_destroy(rxr->rxtag[s], rxbuf->map[s]); 2693 } 2694 if (rxbuf->m_head != NULL) { 2695 m_freem(rxbuf->m_head); 2696 } 2697 rxbuf->m_head = NULL; 2698 ++rxbuf; 2699 } 2700 } 2701 if (rxr->rx_buffers != NULL) { 2702 free(rxr->rx_buffers, M_DEVBUF); 2703 rxr->rx_buffers = NULL; 2704 } 2705 for (s = 0; s < 2; s++) { 2706 if (rxr->rxtag[s] != NULL) 2707 rxr->rxtag[s] = NULL; 2708 } 2709 return; 2710 } 2711 2712 /********************************************************************* 2713 * 2714 * This routine executes in interrupt context. It replenishes 2715 * the mbufs in the descriptor and sends data which has been 2716 * dma'ed into host memory to upper layer. 2717 * 2718 * We loop at most count times if count is > 0, or until done if 2719 * count < 0. 2720 * 2721 *********************************************************************/ 2722 int 2723 ixgbe_rxeof(struct rx_ring *rxr, int count) 2724 { 2725 struct ix_softc *sc = rxr->sc; 2726 struct ifnet *ifp = &sc->arpcom.ac_if; 2727 struct mbuf *mp; 2728 int len, i, eop = 0; 2729 uint8_t accept_frame = 0; 2730 uint32_t staterr; 2731 union ixgbe_adv_rx_desc *cur; 2732 2733 i = rxr->next_to_check; 2734 cur = &rxr->rx_base[i]; 2735 2736 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2737 rxr->rxdma.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2738 2739 staterr = cur->wb.upper.status_error; 2740 if (!(staterr & IXGBE_RXD_STAT_DD)) 2741 return FALSE; 2742 2743 while ((staterr & IXGBE_RXD_STAT_DD) && (count != 0) && 2744 (ifp->if_flags & IFF_RUNNING)) { 2745 struct mbuf *m = NULL; 2746 int s; 2747 2748 mp = rxr->rx_buffers[i].m_head; 2749 s = rxr->rx_buffers[i].bigbuf; 2750 bus_dmamap_sync(rxr->rxtag[s], rxr->rx_buffers[i].map[s], 2751 0, rxr->rx_buffers[i].map[s]->dm_mapsize, 2752 BUS_DMASYNC_POSTREAD); 2753 bus_dmamap_unload(rxr->rxtag[s], 2754 rxr->rx_buffers[i].map[s]); 2755 2756 accept_frame = 1; 2757 if (staterr & IXGBE_RXD_STAT_EOP) { 2758 count--; 2759 eop = 1; 2760 } else { 2761 eop = 0; 2762 } 2763 len = cur->wb.upper.length; 2764 2765 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) 2766 accept_frame = 0; 2767 2768 if (accept_frame) { 2769 /* Get a fresh buffer */ 2770 if (ixgbe_get_buf(rxr, i, NULL) != 0) { 2771 ifp->if_iqdrops++; 2772 goto discard; 2773 } 2774 2775 /* Assign correct length to the current fragment */ 2776 mp->m_len = len; 2777 2778 if (rxr->fmp == NULL) { 2779 mp->m_pkthdr.len = len; 2780 rxr->fmp = mp; /* Store the first mbuf */ 2781 rxr->lmp = mp; 2782 } else { 2783 /* Chain mbuf's together */ 2784 mp->m_flags &= ~M_PKTHDR; 2785 rxr->lmp->m_next = mp; 2786 rxr->lmp = rxr->lmp->m_next; 2787 rxr->fmp->m_pkthdr.len += len; 2788 } 2789 2790 if (eop) { 2791 rxr->fmp->m_pkthdr.rcvif = ifp; 2792 ifp->if_ipackets++; 2793 rxr->packet_count++; 2794 rxr->byte_count += rxr->fmp->m_pkthdr.len; 2795 m = rxr->fmp; 2796 2797 ixgbe_rx_checksum(sc, staterr, rxr->fmp); 2798 2799 #if NVLAN > 0 && defined(IX_CSUM_OFFLOAD) 2800 if (staterr & IXGBE_RXD_STAT_VP) { 2801 struct ether_vlan_header vh; 2802 2803 if (m->m_pkthdr.len < ETHER_HDR_LEN) 2804 goto discard; 2805 m_copydata(m, 0, 2806 ETHER_HDR_LEN, (caddr_t)&vh); 2807 vh.evl_proto = vh.evl_encap_proto; 2808 vh.evl_tag = 2809 letoh16(cur->wb.upper.vlan); 2810 vh.evl_encap_proto = 2811 htons(ETHERTYPE_VLAN); 2812 m_adj(m, ETHER_HDR_LEN); 2813 M_PREPEND(m, sizeof(vh), M_DONTWAIT); 2814 if (m == NULL) 2815 goto discard; 2816 m_copyback(m, 0, sizeof(vh), &vh); 2817 } 2818 #endif 2819 rxr->fmp = NULL; 2820 rxr->lmp = NULL; 2821 } 2822 } else { 2823 discard: 2824 ixgbe_get_buf(rxr, i, mp); 2825 if (rxr->fmp != NULL) { 2826 m_freem(rxr->fmp); 2827 rxr->fmp = NULL; 2828 rxr->lmp = NULL; 2829 } 2830 m = NULL; 2831 } 2832 2833 /* Zero out the receive descriptors status */ 2834 cur->wb.upper.status_error = 0; 2835 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2836 0, rxr->rxdma.dma_map->dm_mapsize, 2837 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2838 2839 rxr->last_cleaned = i; /* for updating tail */ 2840 2841 if (++i == sc->num_rx_desc) 2842 i = 0; 2843 2844 /* Now send up to the stack */ 2845 if (m != NULL) { 2846 rxr->next_to_check = i; 2847 #if NBPFILTER > 0 2848 if (ifp->if_bpf) 2849 bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_IN); 2850 #endif 2851 ether_input_mbuf(ifp, m); 2852 i = rxr->next_to_check; 2853 } 2854 /* Get next descriptor */ 2855 cur = &rxr->rx_base[i]; 2856 staterr = cur->wb.upper.status_error; 2857 } 2858 rxr->next_to_check = i; 2859 2860 /* Advance the IXGB's Receive Queue "Tail Pointer" */ 2861 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me), rxr->last_cleaned); 2862 2863 if (!(staterr & IXGBE_RXD_STAT_DD)) 2864 return FALSE; 2865 2866 return TRUE; 2867 } 2868 2869 /********************************************************************* 2870 * 2871 * Verify that the hardware indicated that the checksum is valid. 2872 * Inform the stack about the status of checksum so that stack 2873 * doesn't spend time verifying the checksum. 2874 * 2875 *********************************************************************/ 2876 void 2877 ixgbe_rx_checksum(struct ix_softc *sc, 2878 uint32_t staterr, struct mbuf * mp) 2879 { 2880 struct ifnet *ifp = &sc->arpcom.ac_if; 2881 uint16_t status = (uint16_t) staterr; 2882 uint8_t errors = (uint8_t) (staterr >> 24); 2883 2884 /* Not offloading */ 2885 if ((ifp->if_capabilities & IFCAP_CSUM_IPv4) == 0) { 2886 mp->m_pkthdr.csum_flags = 0; 2887 return; 2888 } 2889 2890 // XXX 2891 printf("%s: status 0x%04x errors 0x%02x\n", ifp->if_xname, 2892 status, errors); 2893 2894 mp->m_pkthdr.csum_flags = 0; 2895 if (status & IXGBE_RXD_STAT_IPCS) { 2896 /* Did it pass? */ 2897 if (!(errors & IXGBE_RXD_ERR_IPE)) 2898 /* IP Checksum Good */ 2899 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 2900 } 2901 /* Did it pass? */ 2902 if (errors & IXGBE_RXD_ERR_TCPE) 2903 return; 2904 if (status & IXGBE_RXD_STAT_L4CS) 2905 mp->m_pkthdr.csum_flags |= M_TCP_CSUM_IN_OK; 2906 if (status & IXGBE_RXD_STAT_UDPCS) 2907 mp->m_pkthdr.csum_flags |= M_UDP_CSUM_IN_OK; 2908 } 2909 2910 #if NVLAN > 0 2911 void 2912 ixgbe_enable_hw_vlans(struct ix_softc *sc) 2913 { 2914 uint32_t ctrl; 2915 2916 ixgbe_disable_intr(sc); 2917 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 2918 ctrl |= IXGBE_VLNCTRL_VME; 2919 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 2920 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 2921 ixgbe_enable_intr(sc); 2922 } 2923 #endif 2924 2925 void 2926 ixgbe_enable_intr(struct ix_softc *sc) 2927 { 2928 struct ixgbe_hw *hw = &sc->hw; 2929 uint32_t mask = IXGBE_EIMS_ENABLE_MASK; 2930 2931 /* Enable Fan Failure detection */ 2932 if (hw->phy.media_type == ixgbe_media_type_copper) 2933 mask |= IXGBE_EIMS_GPI_SDP1; 2934 /* With RSS we use auto clear */ 2935 if (sc->msix_mem) { 2936 /* Dont autoclear Link */ 2937 mask &= ~IXGBE_EIMS_OTHER; 2938 mask &= ~IXGBE_EIMS_LSC; 2939 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 2940 sc->eims_mask | mask); 2941 } 2942 2943 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2944 IXGBE_WRITE_FLUSH(hw); 2945 2946 return; 2947 } 2948 2949 void 2950 ixgbe_disable_intr(struct ix_softc *sc) 2951 { 2952 if (sc->msix_mem) 2953 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 2954 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 2955 IXGBE_WRITE_FLUSH(&sc->hw); 2956 return; 2957 } 2958 2959 uint16_t 2960 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 2961 { 2962 struct pci_attach_args *pa; 2963 uint16_t value; 2964 2965 pa = ((struct ixgbe_osdep *)hw->back)->os_pa; 2966 2967 /* Should we do read/mask/write...? 16 vs 32 bit!!! */ 2968 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg) & 0xffff; 2969 2970 return (value); 2971 } 2972 2973 void 2974 ixgbe_set_ivar(struct ix_softc *sc, uint16_t entry, uint8_t vector) 2975 { 2976 uint32_t ivar, index; 2977 2978 vector |= IXGBE_IVAR_ALLOC_VAL; 2979 index = (entry >> 2) & 0x1F; 2980 ivar = IXGBE_READ_REG(&sc->hw, IXGBE_IVAR(index)); 2981 ivar &= ~(0xFF << (8 * (entry & 0x3))); 2982 ivar |= (vector << (8 * (entry & 0x3))); 2983 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 2984 } 2985 2986 void 2987 ixgbe_configure_ivars(struct ix_softc *sc) 2988 { 2989 struct tx_ring *txr = sc->tx_rings; 2990 struct rx_ring *rxr = sc->rx_rings; 2991 int i; 2992 2993 for (i = 0; i < sc->num_rx_queues; i++, rxr++) { 2994 ixgbe_set_ivar(sc, IXGBE_IVAR_RX_QUEUE(i), rxr->msix); 2995 sc->eims_mask |= rxr->eims; 2996 } 2997 2998 for (i = 0; i < sc->num_tx_queues; i++, txr++) { 2999 ixgbe_set_ivar(sc, IXGBE_IVAR_TX_QUEUE(i), txr->msix); 3000 sc->eims_mask |= txr->eims; 3001 } 3002 3003 /* For the Link interrupt */ 3004 ixgbe_set_ivar(sc, IXGBE_IVAR_OTHER_CAUSES_INDEX, 3005 sc->linkvec); 3006 sc->eims_mask |= IXGBE_IVAR_OTHER_CAUSES_INDEX; 3007 } 3008 3009 /********************************************************************** 3010 * 3011 * Update the board statistics counters. 3012 * 3013 **********************************************************************/ 3014 void 3015 ixgbe_update_stats_counters(struct ix_softc *sc) 3016 { 3017 struct ifnet *ifp = &sc->arpcom.ac_if;; 3018 struct ixgbe_hw *hw = &sc->hw; 3019 uint32_t missed_rx = 0, bprc, lxon, lxoff, total; 3020 int i; 3021 3022 sc->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3023 3024 for (i = 0; i < 8; i++) { 3025 int mp; 3026 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3027 missed_rx += mp; 3028 sc->stats.mpc[i] += mp; 3029 sc->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3030 } 3031 3032 /* Hardware workaround, gprc counts missed packets */ 3033 sc->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 3034 sc->stats.gprc -= missed_rx; 3035 3036 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3037 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3038 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3039 3040 /* 3041 * Workaround: mprc hardware is incorrectly counting 3042 * broadcasts, so for now we subtract those. 3043 */ 3044 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3045 sc->stats.bprc += bprc; 3046 sc->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3047 sc->stats.mprc -= bprc; 3048 3049 sc->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3050 sc->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3051 sc->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3052 sc->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3053 sc->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3054 sc->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3055 sc->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3056 sc->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3057 3058 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3059 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3060 3061 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3062 sc->stats.lxontxc += lxon; 3063 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3064 sc->stats.lxofftxc += lxoff; 3065 total = lxon + lxoff; 3066 3067 sc->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3068 sc->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3069 sc->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3070 sc->stats.gptc -= total; 3071 sc->stats.mptc -= total; 3072 sc->stats.ptc64 -= total; 3073 sc->stats.gotc -= total * ETHER_MIN_LEN; 3074 3075 sc->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3076 sc->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3077 sc->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3078 sc->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3079 sc->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3080 sc->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3081 sc->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3082 sc->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3083 sc->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3084 sc->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3085 3086 #if 0 3087 /* Fill out the OS statistics structure */ 3088 ifp->if_ipackets = sc->stats.gprc; 3089 ifp->if_opackets = sc->stats.gptc; 3090 ifp->if_ibytes = sc->stats.gorc; 3091 ifp->if_obytes = sc->stats.gotc; 3092 ifp->if_imcasts = sc->stats.mprc; 3093 #endif 3094 ifp->if_collisions = 0; 3095 ifp->if_oerrors = sc->watchdog_events; 3096 ifp->if_ierrors = missed_rx + sc->stats.crcerrs + sc->stats.rlec; 3097 } 3098 3099 #ifdef IX_DEBUG 3100 /********************************************************************** 3101 * 3102 * This routine is called only when ixgbe_display_debug_stats is enabled. 3103 * This routine provides a way to take a look at important statistics 3104 * maintained by the driver and hardware. 3105 * 3106 **********************************************************************/ 3107 void 3108 ixgbe_print_hw_stats(struct ix_softc * sc) 3109 { 3110 struct ifnet *ifp = &sc->arpcom.ac_if;; 3111 3112 printf("%s: mbuf alloc failed %lu, mbuf cluster failed %lu, " 3113 "missed pkts %llu, rx len errs %llu, crc errs %llu, " 3114 "dropped pkts %lu, watchdog timeouts %ld, " 3115 "XON rx %llu, XON tx %llu, XOFF rx %llu, XOFF tx %llu, " 3116 "total pkts rx %llu, good pkts rx %llu, good pkts tx %llu, " 3117 "tso tx %lu\n", 3118 ifp->if_xname, 3119 sc->mbuf_alloc_failed, 3120 sc->mbuf_cluster_failed, 3121 (long long)sc->stats.mpc[0], 3122 (long long)sc->stats.roc + (long long)sc->stats.ruc, 3123 (long long)sc->stats.crcerrs, 3124 sc->dropped_pkts, 3125 sc->watchdog_events, 3126 (long long)sc->stats.lxonrxc, 3127 (long long)sc->stats.lxontxc, 3128 (long long)sc->stats.lxoffrxc, 3129 (long long)sc->stats.lxofftxc, 3130 (long long)sc->stats.tpr, 3131 (long long)sc->stats.gprc, 3132 (long long)sc->stats.gptc, 3133 sc->tso_tx); 3134 } 3135 #endif 3136 3137 #ifndef NO_82598_A0_SUPPORT 3138 /* 3139 * A0 Workaround: invert descriptor for hardware 3140 */ 3141 void 3142 desc_flip(void *desc) 3143 { 3144 struct dhack {uint32_t a1; uint32_t a2; uint32_t b1; uint32_t b2;}; 3145 struct dhack *d; 3146 3147 d = (struct dhack *)desc; 3148 d->a1 = ~(d->a1); 3149 d->a2 = ~(d->a2); 3150 d->b1 = ~(d->b1); 3151 d->b2 = ~(d->b2); 3152 d->b2 &= 0xFFFFFFF0; 3153 d->b1 &= ~IXGBE_ADVTXD_DCMD_RS; 3154 } 3155 #endif 3156 3157 3158 3159