1 /* $OpenBSD: if_ix.c,v 1.132 2016/04/13 10:34:32 mpi Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2013, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */ 36 37 #include <dev/pci/if_ix.h> 38 #include <dev/pci/ixgbe_type.h> 39 40 /********************************************************************* 41 * Driver version 42 *********************************************************************/ 43 /* char ixgbe_driver_version[] = "2.5.13"; */ 44 45 /********************************************************************* 46 * PCI Device ID Table 47 * 48 * Used by probe to select devices to load on 49 *********************************************************************/ 50 51 const struct pci_matchid ixgbe_devices[] = { 52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 }, 53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX }, 54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT }, 57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT2 }, 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP }, 63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM }, 64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL }, 65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 }, 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BP }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_T3_LOM }, 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF2 }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_FCOE }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599EN_SFP }, 77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T }, 78 }; 79 80 /********************************************************************* 81 * Function prototypes 82 *********************************************************************/ 83 int ixgbe_probe(struct device *, void *, void *); 84 void ixgbe_attach(struct device *, struct device *, void *); 85 int ixgbe_detach(struct device *, int); 86 void ixgbe_start(struct ifnet *); 87 int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 88 int ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *); 89 void ixgbe_watchdog(struct ifnet *); 90 void ixgbe_init(void *); 91 void ixgbe_stop(void *); 92 void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 93 int ixgbe_media_change(struct ifnet *); 94 void ixgbe_identify_hardware(struct ix_softc *); 95 int ixgbe_allocate_pci_resources(struct ix_softc *); 96 int ixgbe_allocate_legacy(struct ix_softc *); 97 int ixgbe_allocate_queues(struct ix_softc *); 98 void ixgbe_free_pci_resources(struct ix_softc *); 99 void ixgbe_local_timer(void *); 100 void ixgbe_setup_interface(struct ix_softc *); 101 void ixgbe_config_link(struct ix_softc *sc); 102 103 int ixgbe_allocate_transmit_buffers(struct tx_ring *); 104 int ixgbe_setup_transmit_structures(struct ix_softc *); 105 int ixgbe_setup_transmit_ring(struct tx_ring *); 106 void ixgbe_initialize_transmit_units(struct ix_softc *); 107 void ixgbe_free_transmit_structures(struct ix_softc *); 108 void ixgbe_free_transmit_buffers(struct tx_ring *); 109 110 int ixgbe_allocate_receive_buffers(struct rx_ring *); 111 int ixgbe_setup_receive_structures(struct ix_softc *); 112 int ixgbe_setup_receive_ring(struct rx_ring *); 113 void ixgbe_initialize_receive_units(struct ix_softc *); 114 void ixgbe_free_receive_structures(struct ix_softc *); 115 void ixgbe_free_receive_buffers(struct rx_ring *); 116 int ixgbe_rxfill(struct rx_ring *); 117 void ixgbe_rxrefill(void *); 118 119 void ixgbe_enable_intr(struct ix_softc *); 120 void ixgbe_disable_intr(struct ix_softc *); 121 void ixgbe_update_stats_counters(struct ix_softc *); 122 int ixgbe_txeof(struct tx_ring *); 123 int ixgbe_rxeof(struct ix_queue *); 124 void ixgbe_rx_checksum(uint32_t, struct mbuf *, uint32_t); 125 void ixgbe_iff(struct ix_softc *); 126 #ifdef IX_DEBUG 127 void ixgbe_print_hw_stats(struct ix_softc *); 128 #endif 129 void ixgbe_update_link_status(struct ix_softc *); 130 int ixgbe_get_buf(struct rx_ring *, int); 131 int ixgbe_encap(struct tx_ring *, struct mbuf *); 132 int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 133 struct ixgbe_dma_alloc *, int); 134 void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 135 int ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *, uint32_t *, 136 uint32_t *); 137 int ixgbe_tso_setup(struct tx_ring *, struct mbuf *, uint32_t *, 138 uint32_t *); 139 void ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 140 void ixgbe_configure_ivars(struct ix_softc *); 141 uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 142 143 void ixgbe_setup_vlan_hw_support(struct ix_softc *); 144 145 /* Support for pluggable optic modules */ 146 void ixgbe_setup_optics(struct ix_softc *); 147 void ixgbe_handle_mod(struct ix_softc *); 148 void ixgbe_handle_msf(struct ix_softc *); 149 150 /* Legacy (single vector interrupt handler */ 151 int ixgbe_intr(void *); 152 void ixgbe_enable_queue(struct ix_softc *, uint32_t); 153 void ixgbe_disable_queue(struct ix_softc *, uint32_t); 154 void ixgbe_rearm_queue(struct ix_softc *, uint32_t); 155 156 /********************************************************************* 157 * OpenBSD Device Interface Entry Points 158 *********************************************************************/ 159 160 struct cfdriver ix_cd = { 161 NULL, "ix", DV_IFNET 162 }; 163 164 struct cfattach ix_ca = { 165 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach 166 }; 167 168 int ixgbe_smart_speed = ixgbe_smart_speed_on; 169 170 /********************************************************************* 171 * Device identification routine 172 * 173 * ixgbe_probe determines if the driver should be loaded on 174 * adapter based on PCI vendor/device id of the adapter. 175 * 176 * return 0 on success, positive on failure 177 *********************************************************************/ 178 179 int 180 ixgbe_probe(struct device *parent, void *match, void *aux) 181 { 182 INIT_DEBUGOUT("ixgbe_probe: begin"); 183 184 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 185 nitems(ixgbe_devices))); 186 } 187 188 /********************************************************************* 189 * Device initialization routine 190 * 191 * The attach entry point is called when the driver is being loaded. 192 * This routine identifies the type of hardware, allocates all resources 193 * and initializes the hardware. 194 * 195 * return 0 on success, positive on failure 196 *********************************************************************/ 197 198 void 199 ixgbe_attach(struct device *parent, struct device *self, void *aux) 200 { 201 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 202 struct ix_softc *sc = (struct ix_softc *)self; 203 int error = 0; 204 uint16_t csum; 205 uint32_t ctrl_ext; 206 struct ixgbe_hw *hw = &sc->hw; 207 208 INIT_DEBUGOUT("ixgbe_attach: begin"); 209 210 sc->osdep.os_sc = sc; 211 sc->osdep.os_pa = *pa; 212 213 /* Set up the timer callout */ 214 timeout_set(&sc->timer, ixgbe_local_timer, sc); 215 timeout_set(&sc->rx_refill, ixgbe_rxrefill, sc); 216 217 /* Determine hardware revision */ 218 ixgbe_identify_hardware(sc); 219 220 /* Indicate to RX setup to use Jumbo Clusters */ 221 sc->num_tx_desc = DEFAULT_TXD; 222 sc->num_rx_desc = DEFAULT_RXD; 223 224 /* Do base PCI setup - map BAR0 */ 225 if (ixgbe_allocate_pci_resources(sc)) 226 goto err_out; 227 228 /* Allocate our TX/RX Queues */ 229 if (ixgbe_allocate_queues(sc)) 230 goto err_out; 231 232 /* Allocate multicast array memory. */ 233 sc->mta = mallocarray(IXGBE_ETH_LENGTH_OF_ADDRESS, 234 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 235 if (sc->mta == NULL) { 236 printf(": Can not allocate multicast setup array\n"); 237 goto err_late; 238 } 239 240 /* Initialize the shared code */ 241 error = ixgbe_init_shared_code(hw); 242 if (error) { 243 printf(": Unable to initialize the shared code\n"); 244 goto err_late; 245 } 246 247 /* Make sure we have a good EEPROM before we read from it */ 248 if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) { 249 printf(": The EEPROM Checksum Is Not Valid\n"); 250 goto err_late; 251 } 252 253 error = ixgbe_init_hw(hw); 254 if (error == IXGBE_ERR_EEPROM_VERSION) { 255 printf(": This device is a pre-production adapter/" 256 "LOM. Please be aware there may be issues associated " 257 "with your hardware.\n If you are experiencing problems " 258 "please contact your Intel or hardware representative " 259 "who provided you with this hardware.\n"); 260 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 261 printf(": Unsupported SFP+ Module\n"); 262 } 263 264 if (error) { 265 printf(": Hardware Initialization Failure\n"); 266 goto err_late; 267 } 268 269 /* Detect and set physical type */ 270 ixgbe_setup_optics(sc); 271 272 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 273 IXGBE_ETH_LENGTH_OF_ADDRESS); 274 275 error = ixgbe_allocate_legacy(sc); 276 if (error) 277 goto err_late; 278 279 /* Setup OS specific network interface */ 280 ixgbe_setup_interface(sc); 281 282 /* Initialize statistics */ 283 ixgbe_update_stats_counters(sc); 284 285 /* Get the PCI-E bus info and determine LAN ID */ 286 hw->mac.ops.get_bus_info(hw); 287 288 /* Set an initial default flow control value */ 289 sc->fc = ixgbe_fc_full; 290 291 /* let hardware know driver is loaded */ 292 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 293 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 294 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 295 296 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 297 298 INIT_DEBUGOUT("ixgbe_attach: end"); 299 return; 300 301 err_late: 302 ixgbe_free_transmit_structures(sc); 303 ixgbe_free_receive_structures(sc); 304 err_out: 305 ixgbe_free_pci_resources(sc); 306 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 307 MAX_NUM_MULTICAST_ADDRESSES); 308 } 309 310 /********************************************************************* 311 * Device removal routine 312 * 313 * The detach entry point is called when the driver is being removed. 314 * This routine stops the adapter and deallocates all the resources 315 * that were allocated for driver operation. 316 * 317 * return 0 on success, positive on failure 318 *********************************************************************/ 319 320 int 321 ixgbe_detach(struct device *self, int flags) 322 { 323 struct ix_softc *sc = (struct ix_softc *)self; 324 struct ifnet *ifp = &sc->arpcom.ac_if; 325 uint32_t ctrl_ext; 326 327 INIT_DEBUGOUT("ixgbe_detach: begin"); 328 329 ixgbe_stop(sc); 330 331 /* let hardware know driver is unloading */ 332 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 333 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 334 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 335 336 ether_ifdetach(ifp); 337 if_detach(ifp); 338 339 timeout_del(&sc->timer); 340 timeout_del(&sc->rx_refill); 341 ixgbe_free_pci_resources(sc); 342 343 ixgbe_free_transmit_structures(sc); 344 ixgbe_free_receive_structures(sc); 345 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 346 MAX_NUM_MULTICAST_ADDRESSES); 347 348 return (0); 349 } 350 351 /********************************************************************* 352 * Transmit entry point 353 * 354 * ixgbe_start is called by the stack to initiate a transmit. 355 * The driver will remain in this routine as long as there are 356 * packets to transmit and transmit resources are available. 357 * In case resources are not available stack is notified and 358 * the packet is requeued. 359 **********************************************************************/ 360 361 void 362 ixgbe_start(struct ifnet * ifp) 363 { 364 struct ix_softc *sc = ifp->if_softc; 365 struct tx_ring *txr = sc->tx_rings; 366 struct mbuf *m_head; 367 int post = 0; 368 369 if (!(ifp->if_flags & IFF_RUNNING) || ifq_is_oactive(&ifp->if_snd)) 370 return; 371 if (!sc->link_up) 372 return; 373 374 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 375 txr->txdma.dma_map->dm_mapsize, 376 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 377 378 for (;;) { 379 /* Check that we have the minimal number of TX descriptors. */ 380 if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) { 381 ifq_set_oactive(&ifp->if_snd); 382 break; 383 } 384 385 m_head = ifq_dequeue(&ifp->if_snd); 386 if (m_head == NULL) 387 break; 388 389 if (ixgbe_encap(txr, m_head)) { 390 m_freem(m_head); 391 continue; 392 } 393 394 #if NBPFILTER > 0 395 if (ifp->if_bpf) 396 bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 397 #endif 398 399 /* Set timeout in case hardware has problems transmitting */ 400 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 401 ifp->if_timer = IXGBE_TX_TIMEOUT; 402 403 post = 1; 404 } 405 406 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 407 0, txr->txdma.dma_map->dm_mapsize, 408 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 409 410 /* 411 * Advance the Transmit Descriptor Tail (Tdt), this tells the 412 * hardware that this frame is available to transmit. 413 */ 414 if (post) 415 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 416 txr->next_avail_desc); 417 } 418 419 /********************************************************************* 420 * Ioctl entry point 421 * 422 * ixgbe_ioctl is called when the user wants to configure the 423 * interface. 424 * 425 * return 0 on success, positive on failure 426 **********************************************************************/ 427 428 int 429 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 430 { 431 struct ix_softc *sc = ifp->if_softc; 432 struct ifreq *ifr = (struct ifreq *) data; 433 int s, error = 0; 434 435 s = splnet(); 436 437 switch (command) { 438 case SIOCSIFADDR: 439 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 440 ifp->if_flags |= IFF_UP; 441 if (!(ifp->if_flags & IFF_RUNNING)) 442 ixgbe_init(sc); 443 break; 444 445 case SIOCSIFFLAGS: 446 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 447 if (ifp->if_flags & IFF_UP) { 448 if (ifp->if_flags & IFF_RUNNING) 449 error = ENETRESET; 450 else 451 ixgbe_init(sc); 452 } else { 453 if (ifp->if_flags & IFF_RUNNING) 454 ixgbe_stop(sc); 455 } 456 break; 457 458 case SIOCSIFMEDIA: 459 case SIOCGIFMEDIA: 460 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 461 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 462 break; 463 464 case SIOCGIFRXR: 465 error = ixgbe_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 466 break; 467 468 default: 469 error = ether_ioctl(ifp, &sc->arpcom, command, data); 470 } 471 472 if (error == ENETRESET) { 473 if (ifp->if_flags & IFF_RUNNING) { 474 ixgbe_disable_intr(sc); 475 ixgbe_iff(sc); 476 ixgbe_enable_intr(sc); 477 } 478 error = 0; 479 } 480 481 splx(s); 482 return (error); 483 } 484 485 int 486 ixgbe_rxrinfo(struct ix_softc *sc, struct if_rxrinfo *ifri) 487 { 488 struct if_rxring_info *ifr, ifr1; 489 struct rx_ring *rxr; 490 int error, i; 491 u_int n = 0; 492 493 if (sc->num_queues > 1) { 494 if ((ifr = mallocarray(sc->num_queues, sizeof(*ifr), M_DEVBUF, 495 M_WAITOK | M_ZERO)) == NULL) 496 return (ENOMEM); 497 } else 498 ifr = &ifr1; 499 500 for (i = 0; i < sc->num_queues; i++) { 501 rxr = &sc->rx_rings[i]; 502 ifr[n].ifr_size = sc->rx_mbuf_sz; 503 snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "/%d", i); 504 ifr[n].ifr_info = rxr->rx_ring; 505 n++; 506 } 507 508 error = if_rxr_info_ioctl(ifri, sc->num_queues, ifr); 509 510 if (sc->num_queues > 1) 511 free(ifr, M_DEVBUF, sc->num_queues * sizeof(*ifr)); 512 return (error); 513 } 514 515 /********************************************************************* 516 * Watchdog entry point 517 * 518 **********************************************************************/ 519 520 void 521 ixgbe_watchdog(struct ifnet * ifp) 522 { 523 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 524 struct tx_ring *txr = sc->tx_rings; 525 struct ixgbe_hw *hw = &sc->hw; 526 int tx_hang = FALSE; 527 int i; 528 529 /* 530 * The timer is set to 5 every time ixgbe_start() queues a packet. 531 * Anytime all descriptors are clean the timer is set to 0. 532 */ 533 for (i = 0; i < sc->num_queues; i++, txr++) { 534 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 535 continue; 536 else { 537 tx_hang = TRUE; 538 break; 539 } 540 } 541 if (tx_hang == FALSE) 542 return; 543 544 /* 545 * If we are in this routine because of pause frames, then don't 546 * reset the hardware. 547 */ 548 if (!(IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXON)) { 549 for (i = 0; i < sc->num_queues; i++, txr++) 550 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 551 ifp->if_timer = IXGBE_TX_TIMEOUT; 552 return; 553 } 554 555 556 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 557 for (i = 0; i < sc->num_queues; i++, txr++) { 558 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 559 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 560 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 561 printf("%s: TX(%d) desc avail = %d, Next TX to Clean = %d\n", ifp->if_xname, 562 i, txr->tx_avail, txr->next_to_clean); 563 } 564 ifp->if_flags &= ~IFF_RUNNING; 565 sc->watchdog_events++; 566 567 ixgbe_init(sc); 568 } 569 570 /********************************************************************* 571 * Init entry point 572 * 573 * This routine is used in two ways. It is used by the stack as 574 * init entry point in network interface structure. It is also used 575 * by the driver as a hw/sw initialization routine to get to a 576 * consistent state. 577 * 578 * return 0 on success, positive on failure 579 **********************************************************************/ 580 #define IXGBE_MHADD_MFS_SHIFT 16 581 582 void 583 ixgbe_init(void *arg) 584 { 585 struct ix_softc *sc = (struct ix_softc *)arg; 586 struct ifnet *ifp = &sc->arpcom.ac_if; 587 struct rx_ring *rxr = sc->rx_rings; 588 uint32_t k, txdctl, rxdctl, rxctrl, mhadd, gpie, itr; 589 int i, s, err; 590 591 INIT_DEBUGOUT("ixgbe_init: begin"); 592 593 s = splnet(); 594 595 ixgbe_stop(sc); 596 597 /* reprogram the RAR[0] in case user changed it. */ 598 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 599 600 /* Get the latest mac address, User can use a LAA */ 601 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 602 IXGBE_ETH_LENGTH_OF_ADDRESS); 603 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1); 604 sc->hw.addr_ctrl.rar_used_count = 1; 605 606 /* Prepare transmit descriptors and buffers */ 607 if (ixgbe_setup_transmit_structures(sc)) { 608 printf("%s: Could not setup transmit structures\n", 609 ifp->if_xname); 610 ixgbe_stop(sc); 611 splx(s); 612 return; 613 } 614 615 ixgbe_init_hw(&sc->hw); 616 ixgbe_initialize_transmit_units(sc); 617 618 /* Use 2k clusters, even for jumbo frames */ 619 #ifdef __STRICT_ALIGNMENT 620 sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN; 621 #else 622 sc->rx_mbuf_sz = MCLBYTES; 623 #endif 624 625 /* Prepare receive descriptors and buffers */ 626 if (ixgbe_setup_receive_structures(sc)) { 627 printf("%s: Could not setup receive structures\n", 628 ifp->if_xname); 629 ixgbe_stop(sc); 630 splx(s); 631 return; 632 } 633 634 /* Configure RX settings */ 635 ixgbe_initialize_receive_units(sc); 636 637 /* Program promiscuous mode and multicast filters. */ 638 ixgbe_iff(sc); 639 640 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 641 642 /* Enable Fan Failure Interrupt */ 643 gpie |= IXGBE_SDP1_GPIEN; 644 645 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 646 /* Add for Module detection */ 647 gpie |= IXGBE_SDP2_GPIEN; 648 649 /* 650 * Set LL interval to max to reduce the number of low latency 651 * interrupts hitting the card when the ring is getting full. 652 */ 653 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 654 } 655 656 if (sc->hw.mac.type == ixgbe_mac_X540) { 657 /* Thermal Failure Detection */ 658 gpie |= IXGBE_SDP0_GPIEN; 659 660 /* 661 * Set LL interval to max to reduce the number of low latency 662 * interrupts hitting the card when the ring is getting full. 663 */ 664 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 665 } 666 667 if (sc->msix > 1) { 668 /* Enable Enhanced MSIX mode */ 669 gpie |= IXGBE_GPIE_MSIX_MODE; 670 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 671 IXGBE_GPIE_OCD; 672 } 673 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 674 675 /* Set MRU size */ 676 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 677 mhadd &= ~IXGBE_MHADD_MFS_MASK; 678 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 679 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 680 681 /* Now enable all the queues */ 682 683 for (i = 0; i < sc->num_queues; i++) { 684 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 685 txdctl |= IXGBE_TXDCTL_ENABLE; 686 /* Set WTHRESH to 8, burst writeback */ 687 txdctl |= (8 << 16); 688 /* 689 * When the internal queue falls below PTHRESH (16), 690 * start prefetching as long as there are at least 691 * HTHRESH (1) buffers ready. 692 */ 693 txdctl |= (16 << 0) | (1 << 8); 694 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 695 } 696 697 for (i = 0; i < sc->num_queues; i++) { 698 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 699 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 700 /* 701 * PTHRESH = 21 702 * HTHRESH = 4 703 * WTHRESH = 8 704 */ 705 rxdctl &= ~0x3FFFFF; 706 rxdctl |= 0x080420; 707 } 708 rxdctl |= IXGBE_RXDCTL_ENABLE; 709 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 710 for (k = 0; k < 10; k++) { 711 if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) & 712 IXGBE_RXDCTL_ENABLE) 713 break; 714 else 715 msec_delay(1); 716 } 717 IXGBE_WRITE_FLUSH(&sc->hw); 718 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled); 719 } 720 721 /* Set up VLAN support and filter */ 722 ixgbe_setup_vlan_hw_support(sc); 723 724 /* Enable Receive engine */ 725 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 726 if (sc->hw.mac.type == ixgbe_mac_82598EB) 727 rxctrl |= IXGBE_RXCTRL_DMBYPS; 728 rxctrl |= IXGBE_RXCTRL_RXEN; 729 sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl); 730 731 timeout_add_sec(&sc->timer, 1); 732 733 /* Set up MSI/X routing */ 734 if (sc->msix > 1) { 735 ixgbe_configure_ivars(sc); 736 /* Set up auto-mask */ 737 if (sc->hw.mac.type == ixgbe_mac_82598EB) 738 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 739 else { 740 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 741 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 742 } 743 } else { /* Simple settings for Legacy/MSI */ 744 ixgbe_set_ivar(sc, 0, 0, 0); 745 ixgbe_set_ivar(sc, 0, 0, 1); 746 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 747 } 748 749 /* Check on any SFP devices that need to be kick-started */ 750 if (sc->hw.phy.type == ixgbe_phy_none) { 751 err = sc->hw.phy.ops.identify(&sc->hw); 752 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 753 printf("Unsupported SFP+ module type was detected.\n"); 754 splx(s); 755 return; 756 } 757 } 758 759 /* Setup interrupt moderation */ 760 itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8; 761 if (sc->hw.mac.type != ixgbe_mac_82598EB) 762 itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS; 763 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr); 764 765 /* Config/Enable Link */ 766 ixgbe_config_link(sc); 767 768 /* Hardware Packet Buffer & Flow Control setup */ 769 { 770 uint32_t rxpb, frame, size, tmp; 771 772 frame = sc->max_frame_size; 773 774 /* Calculate High Water */ 775 if (sc->hw.mac.type == ixgbe_mac_X540) 776 tmp = IXGBE_DV_X540(frame, frame); 777 else 778 tmp = IXGBE_DV(frame, frame); 779 size = IXGBE_BT2KB(tmp); 780 rxpb = IXGBE_READ_REG(&sc->hw, IXGBE_RXPBSIZE(0)) >> 10; 781 sc->hw.fc.high_water[0] = rxpb - size; 782 783 /* Now calculate Low Water */ 784 if (sc->hw.mac.type == ixgbe_mac_X540) 785 tmp = IXGBE_LOW_DV_X540(frame); 786 else 787 tmp = IXGBE_LOW_DV(frame); 788 sc->hw.fc.low_water[0] = IXGBE_BT2KB(tmp); 789 790 sc->hw.fc.requested_mode = sc->fc; 791 sc->hw.fc.pause_time = IXGBE_FC_PAUSE; 792 sc->hw.fc.send_xon = TRUE; 793 } 794 /* Initialize the FC settings */ 795 sc->hw.mac.ops.start_hw(&sc->hw); 796 797 /* And now turn on interrupts */ 798 ixgbe_enable_intr(sc); 799 800 /* Now inform the stack we're ready */ 801 ifp->if_flags |= IFF_RUNNING; 802 ifq_clr_oactive(&ifp->if_snd); 803 804 splx(s); 805 } 806 807 /* 808 * MSIX Interrupt Handlers 809 */ 810 void 811 ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector) 812 { 813 uint64_t queue = 1ULL << vector; 814 uint32_t mask; 815 816 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 817 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 818 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask); 819 } else { 820 mask = (queue & 0xFFFFFFFF); 821 if (mask) 822 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask); 823 mask = (queue >> 32); 824 if (mask) 825 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask); 826 } 827 } 828 829 void 830 ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector) 831 { 832 uint64_t queue = 1ULL << vector; 833 uint32_t mask; 834 835 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 836 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 837 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask); 838 } else { 839 mask = (queue & 0xFFFFFFFF); 840 if (mask) 841 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask); 842 mask = (queue >> 32); 843 if (mask) 844 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask); 845 } 846 } 847 848 /********************************************************************* 849 * 850 * Legacy Interrupt Service routine 851 * 852 **********************************************************************/ 853 854 int 855 ixgbe_intr(void *arg) 856 { 857 struct ix_softc *sc = (struct ix_softc *)arg; 858 struct ix_queue *que = sc->queues; 859 struct ifnet *ifp = &sc->arpcom.ac_if; 860 struct tx_ring *txr = sc->tx_rings; 861 struct ixgbe_hw *hw = &sc->hw; 862 uint32_t reg_eicr; 863 int i, refill = 0; 864 865 reg_eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR); 866 if (reg_eicr == 0) { 867 ixgbe_enable_intr(sc); 868 return (0); 869 } 870 871 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 872 ixgbe_rxeof(que); 873 ixgbe_txeof(txr); 874 refill = 1; 875 } 876 877 if (refill) { 878 if (ixgbe_rxfill(que->rxr)) { 879 /* Advance the Rx Queue "Tail Pointer" */ 880 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(que->rxr->me), 881 que->rxr->last_desc_filled); 882 } else 883 timeout_add(&sc->rx_refill, 1); 884 } 885 886 /* Link status change */ 887 if (reg_eicr & IXGBE_EICR_LSC) { 888 KERNEL_LOCK(); 889 ixgbe_update_link_status(sc); 890 KERNEL_UNLOCK(); 891 ifq_start(&ifp->if_snd); 892 } 893 894 /* ... more link status change */ 895 if (hw->mac.type != ixgbe_mac_82598EB) { 896 if (reg_eicr & IXGBE_EICR_GPI_SDP2) { 897 /* Clear the interrupt */ 898 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 899 KERNEL_LOCK(); 900 ixgbe_handle_mod(sc); 901 KERNEL_UNLOCK(); 902 } else if ((hw->phy.media_type != ixgbe_media_type_copper) && 903 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 904 /* Clear the interrupt */ 905 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 906 KERNEL_LOCK(); 907 ixgbe_handle_msf(sc); 908 KERNEL_UNLOCK(); 909 } 910 } 911 912 /* Check for fan failure */ 913 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 914 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 915 printf("%s: CRITICAL: FAN FAILURE!! " 916 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 917 IXGBE_WRITE_REG(&sc->hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 918 } 919 920 /* Check for over temp condition */ 921 if ((hw->mac.type == ixgbe_mac_X540) && 922 (reg_eicr & IXGBE_EICR_TS)) { 923 printf("%s: CRITICAL: OVER TEMP!! " 924 "PHY IS SHUT DOWN!!\n", ifp->if_xname); 925 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS); 926 } 927 928 for (i = 0; i < sc->num_queues; i++, que++) 929 ixgbe_enable_queue(sc, que->msix); 930 931 return (1); 932 } 933 934 /********************************************************************* 935 * 936 * Media Ioctl callback 937 * 938 * This routine is called whenever the user queries the status of 939 * the interface using ifconfig. 940 * 941 **********************************************************************/ 942 void 943 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 944 { 945 struct ix_softc *sc = ifp->if_softc; 946 947 ifmr->ifm_active = IFM_ETHER; 948 ifmr->ifm_status = IFM_AVALID; 949 950 INIT_DEBUGOUT("ixgbe_media_status: begin"); 951 ixgbe_update_link_status(sc); 952 953 if (LINK_STATE_IS_UP(ifp->if_link_state)) { 954 ifmr->ifm_status |= IFM_ACTIVE; 955 956 switch (sc->link_speed) { 957 case IXGBE_LINK_SPEED_100_FULL: 958 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 959 break; 960 case IXGBE_LINK_SPEED_1GB_FULL: 961 switch (sc->optics) { 962 case IFM_1000_SX: 963 case IFM_1000_LX: 964 ifmr->ifm_active |= sc->optics | IFM_FDX; 965 break; 966 default: 967 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 968 break; 969 } 970 break; 971 case IXGBE_LINK_SPEED_10GB_FULL: 972 ifmr->ifm_active |= sc->optics | IFM_FDX; 973 break; 974 } 975 976 switch (sc->hw.fc.current_mode) { 977 case ixgbe_fc_tx_pause: 978 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 979 break; 980 case ixgbe_fc_rx_pause: 981 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 982 break; 983 case ixgbe_fc_full: 984 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE | 985 IFM_ETH_TXPAUSE; 986 break; 987 default: 988 ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE | 989 IFM_ETH_TXPAUSE); 990 break; 991 } 992 } 993 } 994 995 /********************************************************************* 996 * 997 * Media Ioctl callback 998 * 999 * This routine is called when the user changes speed/duplex using 1000 * media/mediopt option with ifconfig. 1001 * 1002 **********************************************************************/ 1003 int 1004 ixgbe_media_change(struct ifnet *ifp) 1005 { 1006 struct ix_softc *sc = ifp->if_softc; 1007 struct ifmedia *ifm = &sc->media; 1008 1009 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1010 return (EINVAL); 1011 1012 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1013 case IFM_AUTO: 1014 sc->hw.phy.autoneg_advertised = 1015 IXGBE_LINK_SPEED_100_FULL | 1016 IXGBE_LINK_SPEED_1GB_FULL | 1017 IXGBE_LINK_SPEED_10GB_FULL; 1018 break; 1019 default: 1020 return (EINVAL); 1021 } 1022 1023 return (0); 1024 } 1025 1026 /********************************************************************* 1027 * 1028 * This routine maps the mbufs to tx descriptors, allowing the 1029 * TX engine to transmit the packets. 1030 * - return 0 on success, positive on failure 1031 * 1032 **********************************************************************/ 1033 1034 int 1035 ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head) 1036 { 1037 struct ix_softc *sc = txr->sc; 1038 uint32_t olinfo_status = 0, cmd_type_len; 1039 int i, j, error; 1040 int first, last = 0; 1041 bus_dmamap_t map; 1042 struct ixgbe_tx_buf *txbuf; 1043 union ixgbe_adv_tx_desc *txd = NULL; 1044 1045 /* Basic descriptor defines */ 1046 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1047 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1048 1049 #if NVLAN > 0 1050 if (m_head->m_flags & M_VLANTAG) 1051 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1052 #endif 1053 1054 /* 1055 * Important to capture the first descriptor 1056 * used because it will contain the index of 1057 * the one we tell the hardware to report back 1058 */ 1059 first = txr->next_avail_desc; 1060 txbuf = &txr->tx_buffers[first]; 1061 map = txbuf->map; 1062 1063 /* 1064 * Map the packet for DMA. 1065 */ 1066 error = bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, m_head, 1067 BUS_DMA_NOWAIT); 1068 switch (error) { 1069 case 0: 1070 break; 1071 case EFBIG: 1072 if (m_defrag(m_head, M_NOWAIT) == 0 && 1073 (error = bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1074 m_head, BUS_DMA_NOWAIT)) == 0) 1075 break; 1076 /* FALLTHROUGH */ 1077 default: 1078 sc->no_tx_dma_setup++; 1079 return (error); 1080 } 1081 1082 /* Make certain there are enough descriptors */ 1083 KASSERT(map->dm_nsegs <= txr->tx_avail - 2); 1084 1085 /* 1086 * Set the appropriate offload context 1087 * this will becomes the first descriptor. 1088 */ 1089 error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 1090 if (error) 1091 goto xmit_fail; 1092 1093 i = txr->next_avail_desc; 1094 for (j = 0; j < map->dm_nsegs; j++) { 1095 txbuf = &txr->tx_buffers[i]; 1096 txd = &txr->tx_base[i]; 1097 1098 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 1099 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1100 cmd_type_len | map->dm_segs[j].ds_len); 1101 txd->read.olinfo_status = htole32(olinfo_status); 1102 last = i; /* descriptor that will get completion IRQ */ 1103 1104 if (++i == sc->num_tx_desc) 1105 i = 0; 1106 1107 txbuf->m_head = NULL; 1108 txbuf->eop_index = -1; 1109 } 1110 1111 txd->read.cmd_type_len |= 1112 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1113 1114 txbuf->m_head = m_head; 1115 /* 1116 * Here we swap the map so the last descriptor, 1117 * which gets the completion interrupt has the 1118 * real map, and the first descriptor gets the 1119 * unused map from this descriptor. 1120 */ 1121 txr->tx_buffers[first].map = txbuf->map; 1122 txbuf->map = map; 1123 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1124 BUS_DMASYNC_PREWRITE); 1125 1126 /* Set the index of the descriptor that will be marked done */ 1127 txbuf = &txr->tx_buffers[first]; 1128 txbuf->eop_index = last; 1129 1130 membar_producer(); 1131 1132 atomic_sub_int(&txr->tx_avail, map->dm_nsegs); 1133 txr->next_avail_desc = i; 1134 1135 ++txr->tx_packets; 1136 return (0); 1137 1138 xmit_fail: 1139 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 1140 return (error); 1141 } 1142 1143 void 1144 ixgbe_iff(struct ix_softc *sc) 1145 { 1146 struct ifnet *ifp = &sc->arpcom.ac_if; 1147 struct arpcom *ac = &sc->arpcom; 1148 uint32_t fctrl; 1149 uint8_t *mta; 1150 uint8_t *update_ptr; 1151 struct ether_multi *enm; 1152 struct ether_multistep step; 1153 int mcnt = 0; 1154 1155 IOCTL_DEBUGOUT("ixgbe_iff: begin"); 1156 1157 mta = sc->mta; 1158 bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1159 MAX_NUM_MULTICAST_ADDRESSES); 1160 1161 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1162 fctrl &= ~(IXGBE_FCTRL_MPE | IXGBE_FCTRL_UPE); 1163 ifp->if_flags &= ~IFF_ALLMULTI; 1164 1165 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1166 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1167 ifp->if_flags |= IFF_ALLMULTI; 1168 fctrl |= IXGBE_FCTRL_MPE; 1169 if (ifp->if_flags & IFF_PROMISC) 1170 fctrl |= IXGBE_FCTRL_UPE; 1171 } else { 1172 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1173 while (enm != NULL) { 1174 bcopy(enm->enm_addrlo, 1175 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1176 IXGBE_ETH_LENGTH_OF_ADDRESS); 1177 mcnt++; 1178 1179 ETHER_NEXT_MULTI(step, enm); 1180 } 1181 1182 update_ptr = mta; 1183 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1184 ixgbe_mc_array_itr, TRUE); 1185 } 1186 1187 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1188 } 1189 1190 /* 1191 * This is an iterator function now needed by the multicast 1192 * shared code. It simply feeds the shared code routine the 1193 * addresses in the array of ixgbe_iff() one by one. 1194 */ 1195 uint8_t * 1196 ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1197 { 1198 uint8_t *addr = *update_ptr; 1199 uint8_t *newptr; 1200 *vmdq = 0; 1201 1202 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1203 *update_ptr = newptr; 1204 return addr; 1205 } 1206 1207 void 1208 ixgbe_local_timer(void *arg) 1209 { 1210 struct ix_softc *sc = arg; 1211 #ifdef IX_DEBUG 1212 struct ifnet *ifp = &sc->arpcom.ac_if; 1213 #endif 1214 int s; 1215 1216 s = splnet(); 1217 1218 ixgbe_update_stats_counters(sc); 1219 1220 #ifdef IX_DEBUG 1221 if ((ifp->if_flags & (IFF_RUNNING|IFF_DEBUG)) == 1222 (IFF_RUNNING|IFF_DEBUG)) 1223 ixgbe_print_hw_stats(sc); 1224 #endif 1225 1226 timeout_add_sec(&sc->timer, 1); 1227 1228 splx(s); 1229 } 1230 1231 void 1232 ixgbe_update_link_status(struct ix_softc *sc) 1233 { 1234 struct ifnet *ifp = &sc->arpcom.ac_if; 1235 int link_state = LINK_STATE_DOWN; 1236 1237 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 1238 1239 ifp->if_baudrate = 0; 1240 if (sc->link_up) { 1241 link_state = LINK_STATE_FULL_DUPLEX; 1242 1243 switch (sc->link_speed) { 1244 case IXGBE_LINK_SPEED_UNKNOWN: 1245 ifp->if_baudrate = 0; 1246 break; 1247 case IXGBE_LINK_SPEED_100_FULL: 1248 ifp->if_baudrate = IF_Mbps(100); 1249 break; 1250 case IXGBE_LINK_SPEED_1GB_FULL: 1251 ifp->if_baudrate = IF_Gbps(1); 1252 break; 1253 case IXGBE_LINK_SPEED_10GB_FULL: 1254 ifp->if_baudrate = IF_Gbps(10); 1255 break; 1256 } 1257 1258 /* Update any Flow Control changes */ 1259 sc->hw.mac.ops.fc_enable(&sc->hw); 1260 } 1261 if (ifp->if_link_state != link_state) { 1262 ifp->if_link_state = link_state; 1263 if_link_state_change(ifp); 1264 } 1265 } 1266 1267 1268 /********************************************************************* 1269 * 1270 * This routine disables all traffic on the adapter by issuing a 1271 * global reset on the MAC and deallocates TX/RX buffers. 1272 * 1273 **********************************************************************/ 1274 1275 void 1276 ixgbe_stop(void *arg) 1277 { 1278 struct ix_softc *sc = arg; 1279 struct ifnet *ifp = &sc->arpcom.ac_if; 1280 1281 /* Tell the stack that the interface is no longer active */ 1282 ifp->if_flags &= ~IFF_RUNNING; 1283 1284 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1285 ixgbe_disable_intr(sc); 1286 1287 sc->hw.mac.ops.reset_hw(&sc->hw); 1288 sc->hw.adapter_stopped = FALSE; 1289 sc->hw.mac.ops.stop_adapter(&sc->hw); 1290 if (sc->hw.mac.type == ixgbe_mac_82599EB) 1291 sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw); 1292 /* Turn off the laser */ 1293 if (sc->hw.phy.multispeed_fiber) 1294 sc->hw.mac.ops.disable_tx_laser(&sc->hw); 1295 timeout_del(&sc->timer); 1296 timeout_del(&sc->rx_refill); 1297 1298 /* reprogram the RAR[0] in case user changed it. */ 1299 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1300 1301 ifq_barrier(&ifp->if_snd); 1302 intr_barrier(sc->tag); 1303 1304 KASSERT((ifp->if_flags & IFF_RUNNING) == 0); 1305 1306 ifq_clr_oactive(&ifp->if_snd); 1307 1308 /* Should we really clear all structures on stop? */ 1309 ixgbe_free_transmit_structures(sc); 1310 ixgbe_free_receive_structures(sc); 1311 } 1312 1313 1314 /********************************************************************* 1315 * 1316 * Determine hardware revision. 1317 * 1318 **********************************************************************/ 1319 void 1320 ixgbe_identify_hardware(struct ix_softc *sc) 1321 { 1322 struct ixgbe_osdep *os = &sc->osdep; 1323 struct pci_attach_args *pa = &os->os_pa; 1324 uint32_t reg; 1325 1326 /* Save off the information about this board */ 1327 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1328 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1329 1330 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1331 sc->hw.revision_id = PCI_REVISION(reg); 1332 1333 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1334 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1335 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1336 1337 switch (sc->hw.device_id) { 1338 case PCI_PRODUCT_INTEL_82598: 1339 case PCI_PRODUCT_INTEL_82598AF_DUAL: 1340 case PCI_PRODUCT_INTEL_82598_DA_DUAL: 1341 case PCI_PRODUCT_INTEL_82598AF: 1342 case PCI_PRODUCT_INTEL_82598_SR_DUAL_EM: 1343 case PCI_PRODUCT_INTEL_82598EB_SFP: 1344 case PCI_PRODUCT_INTEL_82598EB_CX4_DUAL: 1345 case PCI_PRODUCT_INTEL_82598EB_CX4: 1346 case PCI_PRODUCT_INTEL_82598EB_XF_LR: 1347 case PCI_PRODUCT_INTEL_82598AT: 1348 case PCI_PRODUCT_INTEL_82598AT2: 1349 case PCI_PRODUCT_INTEL_82598AT_DUAL: 1350 case PCI_PRODUCT_INTEL_82598_BX: 1351 sc->hw.mac.type = ixgbe_mac_82598EB; 1352 break; 1353 case PCI_PRODUCT_INTEL_82599EN_SFP: 1354 case PCI_PRODUCT_INTEL_82599_SFP: 1355 case PCI_PRODUCT_INTEL_82599_SFP_EM: 1356 case PCI_PRODUCT_INTEL_82599_SFP_FCOE: 1357 case PCI_PRODUCT_INTEL_82599_SFP_SF2: 1358 case PCI_PRODUCT_INTEL_82599_KX4: 1359 case PCI_PRODUCT_INTEL_82599_KX4_MEZZ: 1360 case PCI_PRODUCT_INTEL_82599_CX4: 1361 case PCI_PRODUCT_INTEL_82599_T3_LOM: 1362 case PCI_PRODUCT_INTEL_82599_XAUI: 1363 case PCI_PRODUCT_INTEL_82599_COMBO_BP: 1364 case PCI_PRODUCT_INTEL_82599_BPLANE_FCOE: 1365 sc->hw.mac.type = ixgbe_mac_82599EB; 1366 break; 1367 case PCI_PRODUCT_INTEL_82599VF: 1368 sc->hw.mac.type = ixgbe_mac_82599_vf; 1369 break; 1370 case PCI_PRODUCT_INTEL_X540T: 1371 sc->hw.mac.type = ixgbe_mac_X540; 1372 break; 1373 default: 1374 break; 1375 } 1376 1377 /* Pick up the 82599 and VF settings */ 1378 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1379 sc->hw.phy.smart_speed = ixgbe_smart_speed; 1380 sc->num_segs = IXGBE_82599_SCATTER; 1381 } 1382 1383 /********************************************************************* 1384 * 1385 * Determine optic type 1386 * 1387 **********************************************************************/ 1388 void 1389 ixgbe_setup_optics(struct ix_softc *sc) 1390 { 1391 struct ixgbe_hw *hw = &sc->hw; 1392 int layer; 1393 1394 layer = hw->mac.ops.get_supported_physical_layer(hw); 1395 1396 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1397 sc->optics = IFM_10G_T; 1398 else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1399 sc->optics = IFM_1000_T; 1400 else if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1401 sc->optics = IFM_100_TX; 1402 else if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) 1403 sc->optics = IFM_10G_SFP_CU; 1404 else if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR | 1405 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) 1406 sc->optics = IFM_10G_LR; 1407 else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) 1408 sc->optics = IFM_10G_SR; 1409 else if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 | 1410 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) 1411 sc->optics = IFM_10G_CX4; 1412 else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 1413 sc->optics = IFM_1000_SX; 1414 else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_LX) 1415 sc->optics = IFM_1000_LX; 1416 } 1417 1418 /********************************************************************* 1419 * 1420 * Setup the Legacy or MSI Interrupt handler 1421 * 1422 **********************************************************************/ 1423 int 1424 ixgbe_allocate_legacy(struct ix_softc *sc) 1425 { 1426 struct ixgbe_osdep *os = &sc->osdep; 1427 struct pci_attach_args *pa = &os->os_pa; 1428 const char *intrstr = NULL; 1429 pci_chipset_tag_t pc = pa->pa_pc; 1430 pci_intr_handle_t ih; 1431 1432 /* We allocate a single interrupt resource */ 1433 if (pci_intr_map_msi(pa, &ih) != 0 && 1434 pci_intr_map(pa, &ih) != 0) { 1435 printf(": couldn't map interrupt\n"); 1436 return (ENXIO); 1437 } 1438 1439 #if 0 1440 /* XXX */ 1441 /* Tasklets for Link, SFP and Multispeed Fiber */ 1442 TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc); 1443 TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc); 1444 TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc); 1445 #endif 1446 1447 intrstr = pci_intr_string(pc, ih); 1448 sc->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE, 1449 ixgbe_intr, sc, sc->dev.dv_xname); 1450 if (sc->tag == NULL) { 1451 printf(": couldn't establish interrupt"); 1452 if (intrstr != NULL) 1453 printf(" at %s", intrstr); 1454 printf("\n"); 1455 return (ENXIO); 1456 } 1457 printf(": %s", intrstr); 1458 1459 /* For simplicity in the handlers */ 1460 sc->que_mask = IXGBE_EIMS_ENABLE_MASK; 1461 1462 return (0); 1463 } 1464 1465 int 1466 ixgbe_allocate_pci_resources(struct ix_softc *sc) 1467 { 1468 struct ixgbe_osdep *os = &sc->osdep; 1469 struct pci_attach_args *pa = &os->os_pa; 1470 int val; 1471 1472 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1473 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM && 1474 PCI_MAPREG_TYPE(val) != PCI_MAPREG_MEM_TYPE_64BIT) { 1475 printf(": mmba is not mem space\n"); 1476 return (ENXIO); 1477 } 1478 1479 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1480 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1481 printf(": cannot find mem space\n"); 1482 return (ENXIO); 1483 } 1484 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1485 1486 /* Legacy defaults */ 1487 sc->num_queues = 1; 1488 sc->hw.back = os; 1489 1490 #ifdef notyet 1491 /* Now setup MSI or MSI/X, return us the number of supported vectors. */ 1492 sc->msix = ixgbe_setup_msix(sc); 1493 #endif 1494 1495 return (0); 1496 } 1497 1498 void 1499 ixgbe_free_pci_resources(struct ix_softc * sc) 1500 { 1501 struct ixgbe_osdep *os = &sc->osdep; 1502 struct pci_attach_args *pa = &os->os_pa; 1503 struct ix_queue *que = sc->queues; 1504 int i; 1505 1506 /* Release all msix queue resources: */ 1507 for (i = 0; i < sc->num_queues; i++, que++) { 1508 if (que->tag) 1509 pci_intr_disestablish(pa->pa_pc, que->tag); 1510 que->tag = NULL; 1511 } 1512 1513 if (sc->tag) 1514 pci_intr_disestablish(pa->pa_pc, sc->tag); 1515 sc->tag = NULL; 1516 if (os->os_membase != 0) 1517 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1518 os->os_membase = 0; 1519 } 1520 1521 /********************************************************************* 1522 * 1523 * Setup networking device structure and register an interface. 1524 * 1525 **********************************************************************/ 1526 void 1527 ixgbe_setup_interface(struct ix_softc *sc) 1528 { 1529 struct ifnet *ifp = &sc->arpcom.ac_if; 1530 1531 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1532 ifp->if_softc = sc; 1533 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1534 ifp->if_xflags = IFXF_MPSAFE; 1535 ifp->if_ioctl = ixgbe_ioctl; 1536 ifp->if_start = ixgbe_start; 1537 ifp->if_timer = 0; 1538 ifp->if_watchdog = ixgbe_watchdog; 1539 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1540 ETHER_HDR_LEN - ETHER_CRC_LEN; 1541 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1542 1543 ifp->if_capabilities = IFCAP_VLAN_MTU; 1544 1545 #if NVLAN > 0 1546 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1547 #endif 1548 1549 #ifdef IX_CSUM_OFFLOAD 1550 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 1551 #endif 1552 1553 /* 1554 * Specify the media types supported by this sc and register 1555 * callbacks to update media and link information 1556 */ 1557 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1558 ixgbe_media_status); 1559 if (sc->optics) 1560 ifmedia_add(&sc->media, IFM_ETHER | sc->optics | 1561 IFM_FDX, 0, NULL); 1562 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1563 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1564 1565 if_attach(ifp); 1566 ether_ifattach(ifp); 1567 1568 sc->max_frame_size = IXGBE_MAX_FRAME_SIZE; 1569 } 1570 1571 void 1572 ixgbe_config_link(struct ix_softc *sc) 1573 { 1574 uint32_t autoneg, err = 0; 1575 bool sfp, negotiate; 1576 1577 switch (sc->hw.phy.type) { 1578 case ixgbe_phy_sfp_avago: 1579 case ixgbe_phy_sfp_ftl: 1580 case ixgbe_phy_sfp_intel: 1581 case ixgbe_phy_sfp_unknown: 1582 case ixgbe_phy_sfp_passive_tyco: 1583 case ixgbe_phy_sfp_passive_unknown: 1584 sfp = TRUE; 1585 break; 1586 default: 1587 sfp = FALSE; 1588 break; 1589 } 1590 1591 if (sfp) { 1592 if (sc->hw.phy.multispeed_fiber) { 1593 sc->hw.mac.ops.setup_sfp(&sc->hw); 1594 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 1595 ixgbe_handle_msf(sc); 1596 } else 1597 ixgbe_handle_mod(sc); 1598 } else { 1599 if (sc->hw.mac.ops.check_link) 1600 err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg, 1601 &sc->link_up, FALSE); 1602 if (err) 1603 return; 1604 autoneg = sc->hw.phy.autoneg_advertised; 1605 if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities)) 1606 err = sc->hw.mac.ops.get_link_capabilities(&sc->hw, 1607 &autoneg, &negotiate); 1608 if (err) 1609 return; 1610 if (sc->hw.mac.ops.setup_link) 1611 err = sc->hw.mac.ops.setup_link(&sc->hw, 1612 autoneg, sc->link_up); 1613 } 1614 } 1615 1616 /******************************************************************** 1617 * Manage DMA'able memory. 1618 *******************************************************************/ 1619 int 1620 ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 1621 struct ixgbe_dma_alloc *dma, int mapflags) 1622 { 1623 struct ifnet *ifp = &sc->arpcom.ac_if; 1624 struct ixgbe_osdep *os = &sc->osdep; 1625 int r; 1626 1627 dma->dma_tag = os->os_pa.pa_dmat; 1628 r = bus_dmamap_create(dma->dma_tag, size, 1, 1629 size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 1630 if (r != 0) { 1631 printf("%s: ixgbe_dma_malloc: bus_dma_tag_create failed; " 1632 "error %u\n", ifp->if_xname, r); 1633 goto fail_0; 1634 } 1635 1636 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1637 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 1638 if (r != 0) { 1639 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 1640 "error %u\n", ifp->if_xname, r); 1641 goto fail_1; 1642 } 1643 1644 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1645 &dma->dma_vaddr, BUS_DMA_NOWAIT); 1646 if (r != 0) { 1647 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 1648 "error %u\n", ifp->if_xname, r); 1649 goto fail_2; 1650 } 1651 1652 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1653 size, NULL, mapflags | BUS_DMA_NOWAIT); 1654 if (r != 0) { 1655 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 1656 "error %u\n", ifp->if_xname, r); 1657 goto fail_3; 1658 } 1659 1660 dma->dma_size = size; 1661 return (0); 1662 fail_3: 1663 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1664 fail_2: 1665 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1666 fail_1: 1667 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1668 fail_0: 1669 dma->dma_map = NULL; 1670 dma->dma_tag = NULL; 1671 return (r); 1672 } 1673 1674 void 1675 ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 1676 { 1677 if (dma->dma_tag == NULL) 1678 return; 1679 1680 if (dma->dma_map != NULL) { 1681 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1682 dma->dma_map->dm_mapsize, 1683 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1684 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1685 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1686 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1687 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1688 dma->dma_map = NULL; 1689 } 1690 } 1691 1692 1693 /********************************************************************* 1694 * 1695 * Allocate memory for the transmit and receive rings, and then 1696 * the descriptors associated with each, called only once at attach. 1697 * 1698 **********************************************************************/ 1699 int 1700 ixgbe_allocate_queues(struct ix_softc *sc) 1701 { 1702 struct ifnet *ifp = &sc->arpcom.ac_if; 1703 struct ix_queue *que; 1704 struct tx_ring *txr; 1705 struct rx_ring *rxr; 1706 int rsize, tsize; 1707 int txconf = 0, rxconf = 0, i; 1708 1709 /* First allocate the top level queue structs */ 1710 if (!(sc->queues = mallocarray(sc->num_queues, 1711 sizeof(struct ix_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) { 1712 printf("%s: Unable to allocate queue memory\n", ifp->if_xname); 1713 goto fail; 1714 } 1715 1716 /* Then allocate the TX ring struct memory */ 1717 if (!(sc->tx_rings = mallocarray(sc->num_queues, 1718 sizeof(struct tx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 1719 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 1720 goto fail; 1721 } 1722 1723 /* Next allocate the RX */ 1724 if (!(sc->rx_rings = mallocarray(sc->num_queues, 1725 sizeof(struct rx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 1726 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 1727 goto rx_fail; 1728 } 1729 1730 /* For the ring itself */ 1731 tsize = roundup2(sc->num_tx_desc * 1732 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 1733 1734 /* 1735 * Now set up the TX queues, txconf is needed to handle the 1736 * possibility that things fail midcourse and we need to 1737 * undo memory gracefully 1738 */ 1739 for (i = 0; i < sc->num_queues; i++, txconf++) { 1740 /* Set up some basics */ 1741 txr = &sc->tx_rings[i]; 1742 txr->sc = sc; 1743 txr->me = i; 1744 1745 if (ixgbe_dma_malloc(sc, tsize, 1746 &txr->txdma, BUS_DMA_NOWAIT)) { 1747 printf("%s: Unable to allocate TX Descriptor memory\n", 1748 ifp->if_xname); 1749 goto err_tx_desc; 1750 } 1751 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 1752 bzero((void *)txr->tx_base, tsize); 1753 } 1754 1755 /* 1756 * Next the RX queues... 1757 */ 1758 rsize = roundup2(sc->num_rx_desc * 1759 sizeof(union ixgbe_adv_rx_desc), 4096); 1760 for (i = 0; i < sc->num_queues; i++, rxconf++) { 1761 rxr = &sc->rx_rings[i]; 1762 /* Set up some basics */ 1763 rxr->sc = sc; 1764 rxr->me = i; 1765 1766 if (ixgbe_dma_malloc(sc, rsize, 1767 &rxr->rxdma, BUS_DMA_NOWAIT)) { 1768 printf("%s: Unable to allocate RxDescriptor memory\n", 1769 ifp->if_xname); 1770 goto err_rx_desc; 1771 } 1772 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 1773 bzero((void *)rxr->rx_base, rsize); 1774 } 1775 1776 /* 1777 * Finally set up the queue holding structs 1778 */ 1779 for (i = 0; i < sc->num_queues; i++) { 1780 que = &sc->queues[i]; 1781 que->sc = sc; 1782 que->txr = &sc->tx_rings[i]; 1783 que->rxr = &sc->rx_rings[i]; 1784 } 1785 1786 return (0); 1787 1788 err_rx_desc: 1789 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 1790 ixgbe_dma_free(sc, &rxr->rxdma); 1791 err_tx_desc: 1792 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 1793 ixgbe_dma_free(sc, &txr->txdma); 1794 free(sc->rx_rings, M_DEVBUF, sc->num_queues * sizeof(struct rx_ring)); 1795 sc->rx_rings = NULL; 1796 rx_fail: 1797 free(sc->tx_rings, M_DEVBUF, sc->num_queues * sizeof(struct tx_ring)); 1798 sc->tx_rings = NULL; 1799 fail: 1800 return (ENOMEM); 1801 } 1802 1803 /********************************************************************* 1804 * 1805 * Allocate memory for tx_buffer structures. The tx_buffer stores all 1806 * the information needed to transmit a packet on the wire. This is 1807 * called only once at attach, setup is done every reset. 1808 * 1809 **********************************************************************/ 1810 int 1811 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 1812 { 1813 struct ix_softc *sc = txr->sc; 1814 struct ifnet *ifp = &sc->arpcom.ac_if; 1815 struct ixgbe_tx_buf *txbuf; 1816 int error, i; 1817 1818 if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc, 1819 sizeof(struct ixgbe_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 1820 printf("%s: Unable to allocate tx_buffer memory\n", 1821 ifp->if_xname); 1822 error = ENOMEM; 1823 goto fail; 1824 } 1825 txr->txtag = txr->txdma.dma_tag; 1826 1827 /* Create the descriptor buffer dma maps */ 1828 for (i = 0; i < sc->num_tx_desc; i++) { 1829 txbuf = &txr->tx_buffers[i]; 1830 error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE, 1831 sc->num_segs, PAGE_SIZE, 0, 1832 BUS_DMA_NOWAIT, &txbuf->map); 1833 1834 if (error != 0) { 1835 printf("%s: Unable to create TX DMA map\n", 1836 ifp->if_xname); 1837 goto fail; 1838 } 1839 } 1840 1841 return 0; 1842 fail: 1843 return (error); 1844 } 1845 1846 /********************************************************************* 1847 * 1848 * Initialize a transmit ring. 1849 * 1850 **********************************************************************/ 1851 int 1852 ixgbe_setup_transmit_ring(struct tx_ring *txr) 1853 { 1854 struct ix_softc *sc = txr->sc; 1855 int error; 1856 1857 /* Now allocate transmit buffers for the ring */ 1858 if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0) 1859 return (error); 1860 1861 /* Clear the old ring contents */ 1862 bzero((void *)txr->tx_base, 1863 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 1864 1865 /* Reset indices */ 1866 txr->next_avail_desc = 0; 1867 txr->next_to_clean = 0; 1868 1869 /* Set number of descriptors available */ 1870 txr->tx_avail = sc->num_tx_desc; 1871 1872 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1873 0, txr->txdma.dma_map->dm_mapsize, 1874 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1875 1876 return (0); 1877 } 1878 1879 /********************************************************************* 1880 * 1881 * Initialize all transmit rings. 1882 * 1883 **********************************************************************/ 1884 int 1885 ixgbe_setup_transmit_structures(struct ix_softc *sc) 1886 { 1887 struct tx_ring *txr = sc->tx_rings; 1888 int i, error; 1889 1890 for (i = 0; i < sc->num_queues; i++, txr++) { 1891 if ((error = ixgbe_setup_transmit_ring(txr)) != 0) 1892 goto fail; 1893 } 1894 1895 return (0); 1896 fail: 1897 ixgbe_free_transmit_structures(sc); 1898 return (error); 1899 } 1900 1901 /********************************************************************* 1902 * 1903 * Enable transmit unit. 1904 * 1905 **********************************************************************/ 1906 void 1907 ixgbe_initialize_transmit_units(struct ix_softc *sc) 1908 { 1909 struct ifnet *ifp = &sc->arpcom.ac_if; 1910 struct tx_ring *txr; 1911 struct ixgbe_hw *hw = &sc->hw; 1912 int i; 1913 uint64_t tdba; 1914 uint32_t txctrl; 1915 1916 /* Setup the Base and Length of the Tx Descriptor Ring */ 1917 1918 for (i = 0; i < sc->num_queues; i++) { 1919 txr = &sc->tx_rings[i]; 1920 1921 /* Setup descriptor base address */ 1922 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 1923 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 1924 (tdba & 0x00000000ffffffffULL)); 1925 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 1926 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 1927 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1928 1929 /* Setup the HW Tx Head and Tail descriptor pointers */ 1930 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 1931 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 1932 1933 /* Setup Transmit Descriptor Cmd Settings */ 1934 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 1935 txr->queue_status = IXGBE_QUEUE_IDLE; 1936 txr->watchdog_timer = 0; 1937 1938 /* Disable Head Writeback */ 1939 switch (hw->mac.type) { 1940 case ixgbe_mac_82598EB: 1941 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 1942 break; 1943 case ixgbe_mac_82599EB: 1944 case ixgbe_mac_X540: 1945 default: 1946 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 1947 break; 1948 } 1949 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1950 switch (hw->mac.type) { 1951 case ixgbe_mac_82598EB: 1952 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 1953 break; 1954 case ixgbe_mac_82599EB: 1955 case ixgbe_mac_X540: 1956 default: 1957 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 1958 break; 1959 } 1960 } 1961 ifp->if_timer = 0; 1962 1963 if (hw->mac.type != ixgbe_mac_82598EB) { 1964 uint32_t dmatxctl, rttdcs; 1965 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1966 dmatxctl |= IXGBE_DMATXCTL_TE; 1967 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 1968 /* Disable arbiter to set MTQC */ 1969 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 1970 rttdcs |= IXGBE_RTTDCS_ARBDIS; 1971 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 1972 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 1973 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 1974 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 1975 } 1976 } 1977 1978 /********************************************************************* 1979 * 1980 * Free all transmit rings. 1981 * 1982 **********************************************************************/ 1983 void 1984 ixgbe_free_transmit_structures(struct ix_softc *sc) 1985 { 1986 struct tx_ring *txr = sc->tx_rings; 1987 int i; 1988 1989 for (i = 0; i < sc->num_queues; i++, txr++) 1990 ixgbe_free_transmit_buffers(txr); 1991 } 1992 1993 /********************************************************************* 1994 * 1995 * Free transmit ring related data structures. 1996 * 1997 **********************************************************************/ 1998 void 1999 ixgbe_free_transmit_buffers(struct tx_ring *txr) 2000 { 2001 struct ix_softc *sc = txr->sc; 2002 struct ixgbe_tx_buf *tx_buffer; 2003 int i; 2004 2005 INIT_DEBUGOUT("free_transmit_ring: begin"); 2006 2007 if (txr->tx_buffers == NULL) 2008 return; 2009 2010 tx_buffer = txr->tx_buffers; 2011 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 2012 if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) { 2013 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2014 0, tx_buffer->map->dm_mapsize, 2015 BUS_DMASYNC_POSTWRITE); 2016 bus_dmamap_unload(txr->txdma.dma_tag, 2017 tx_buffer->map); 2018 } 2019 if (tx_buffer->m_head != NULL) { 2020 m_freem(tx_buffer->m_head); 2021 tx_buffer->m_head = NULL; 2022 } 2023 if (tx_buffer->map != NULL) { 2024 bus_dmamap_destroy(txr->txdma.dma_tag, 2025 tx_buffer->map); 2026 tx_buffer->map = NULL; 2027 } 2028 } 2029 2030 if (txr->tx_buffers != NULL) 2031 free(txr->tx_buffers, M_DEVBUF, 2032 sc->num_tx_desc * sizeof(struct ixgbe_tx_buf)); 2033 txr->tx_buffers = NULL; 2034 txr->txtag = NULL; 2035 } 2036 2037 /********************************************************************* 2038 * 2039 * Advanced Context Descriptor setup for VLAN or CSUM 2040 * 2041 **********************************************************************/ 2042 2043 int 2044 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, 2045 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2046 { 2047 struct ix_softc *sc = txr->sc; 2048 struct ixgbe_adv_tx_context_desc *TXD; 2049 struct ixgbe_tx_buf *tx_buffer; 2050 #if NVLAN > 0 2051 struct ether_vlan_header *eh; 2052 #else 2053 struct ether_header *eh; 2054 #endif 2055 struct ip *ip; 2056 #ifdef notyet 2057 struct ip6_hdr *ip6; 2058 #endif 2059 struct mbuf *m; 2060 int ipoff; 2061 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2062 int ehdrlen, ip_hlen = 0; 2063 uint16_t etype; 2064 uint8_t ipproto = 0; 2065 int offload = TRUE; 2066 int ctxd = txr->next_avail_desc; 2067 #if NVLAN > 0 2068 uint16_t vtag = 0; 2069 #endif 2070 2071 #if notyet 2072 /* First check if TSO is to be used */ 2073 if (mp->m_pkthdr.csum_flags & CSUM_TSO) 2074 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status)); 2075 #endif 2076 2077 if ((mp->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT)) == 0) 2078 offload = FALSE; 2079 2080 /* Indicate the whole packet as payload when not doing TSO */ 2081 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2082 2083 /* Now ready a context descriptor */ 2084 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2085 tx_buffer = &txr->tx_buffers[ctxd]; 2086 2087 /* 2088 * In advanced descriptors the vlan tag must 2089 * be placed into the descriptor itself. Hence 2090 * we need to make one even if not doing offloads. 2091 */ 2092 #if NVLAN > 0 2093 if (mp->m_flags & M_VLANTAG) { 2094 vtag = mp->m_pkthdr.ether_vtag; 2095 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2096 } else 2097 #endif 2098 if (offload == FALSE) 2099 return (0); /* No need for CTX */ 2100 2101 /* 2102 * Determine where frame payload starts. 2103 * Jump over vlan headers if already present, 2104 * helpful for QinQ too. 2105 */ 2106 if (mp->m_len < sizeof(struct ether_header)) 2107 return (1); 2108 #if NVLAN > 0 2109 eh = mtod(mp, struct ether_vlan_header *); 2110 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2111 if (mp->m_len < sizeof(struct ether_vlan_header)) 2112 return (1); 2113 etype = ntohs(eh->evl_proto); 2114 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2115 } else { 2116 etype = ntohs(eh->evl_encap_proto); 2117 ehdrlen = ETHER_HDR_LEN; 2118 } 2119 #else 2120 eh = mtod(mp, struct ether_header *); 2121 etype = ntohs(eh->ether_type); 2122 ehdrlen = ETHER_HDR_LEN; 2123 #endif 2124 2125 /* Set the ether header length */ 2126 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2127 2128 switch (etype) { 2129 case ETHERTYPE_IP: 2130 if (mp->m_pkthdr.len < ehdrlen + sizeof(*ip)) 2131 return (1); 2132 m = m_getptr(mp, ehdrlen, &ipoff); 2133 KASSERT(m != NULL && m->m_len - ipoff >= sizeof(*ip)); 2134 ip = (struct ip *)(m->m_data + ipoff); 2135 ip_hlen = ip->ip_hl << 2; 2136 ipproto = ip->ip_p; 2137 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2138 break; 2139 #ifdef notyet 2140 case ETHERTYPE_IPV6: 2141 if (mp->m_pkthdr.len < ehdrlen + sizeof(*ip6)) 2142 return (1); 2143 m = m_getptr(mp, ehdrlen, &ipoff); 2144 KASSERT(m != NULL && m->m_len - ipoff >= sizeof(*ip6)); 2145 ip6 = (struct ip6 *)(m->m_data + ipoff); 2146 ip_hlen = sizeof(*ip6); 2147 /* XXX-BZ this will go badly in case of ext hdrs. */ 2148 ipproto = ip6->ip6_nxt; 2149 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2150 break; 2151 #endif 2152 default: 2153 offload = FALSE; 2154 break; 2155 } 2156 2157 vlan_macip_lens |= ip_hlen; 2158 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2159 2160 switch (ipproto) { 2161 case IPPROTO_TCP: 2162 if (mp->m_pkthdr.csum_flags & M_TCP_CSUM_OUT) 2163 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2164 break; 2165 case IPPROTO_UDP: 2166 if (mp->m_pkthdr.csum_flags & M_UDP_CSUM_OUT) 2167 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2168 break; 2169 default: 2170 offload = FALSE; 2171 break; 2172 } 2173 2174 if (offload) /* For the TX descriptor setup */ 2175 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2176 2177 /* Now copy bits into descriptor */ 2178 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2179 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2180 TXD->seqnum_seed = htole32(0); 2181 TXD->mss_l4len_idx = htole32(0); 2182 2183 tx_buffer->m_head = NULL; 2184 tx_buffer->eop_index = -1; 2185 2186 membar_producer(); 2187 2188 /* We've consumed the first desc, adjust counters */ 2189 if (++ctxd == sc->num_tx_desc) 2190 ctxd = 0; 2191 txr->next_avail_desc = ctxd; 2192 atomic_dec_int(&txr->tx_avail); 2193 2194 return (0); 2195 } 2196 2197 #ifdef notyet 2198 /********************************************************************** 2199 * 2200 * Setup work for hardware segmentation offload (TSO) on 2201 * adapters using advanced tx descriptors 2202 * 2203 **********************************************************************/ 2204 int 2205 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, 2206 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2207 { 2208 struct ix_softc *sc = txr->sc; 2209 struct ixgbe_adv_tx_context_desc *TXD; 2210 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2211 uint32_t mss_l4len_idx = 0, paylen; 2212 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 2213 uint16_t etype; 2214 #if NVLAN > 0 2215 uint16_t vtag = 0; 2216 struct ether_vlan_header *eh; 2217 #else 2218 struct ether_header *eh; 2219 #endif 2220 struct ip *ip; 2221 struct ip6_hdr *ip6; 2222 struct tcphdr *th; 2223 2224 /* 2225 * Determine where frame payload starts. 2226 * Jump over vlan headers if already present 2227 */ 2228 #if NVLAN > 0 2229 eh = mtod(mp, struct ether_vlan_header *); 2230 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2231 etype = ntohs(eh->evl_proto); 2232 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2233 } else { 2234 etype = ntohs(eh->evl_encap_proto); 2235 ehdrlen = ETHER_HDR_LEN; 2236 } 2237 #else 2238 eh = mtod(mp, struct ether_header *); 2239 etype = ntohs(eh->ether_type); 2240 ehdrlen = ETHER_HDR_LEN; 2241 #endif 2242 2243 switch (etype) { 2244 case ETHERTYPE_IPV6: 2245 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2246 /* XXX-BZ For now we do not pretend to support ext. hdrs. */ 2247 if (ip6->ip6_nxt != IPPROTO_TCP) 2248 return (ENXIO); 2249 ip_hlen = sizeof(struct ip6_hdr); 2250 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2251 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 2252 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 2253 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2254 break; 2255 case ETHERTYPE_IP: 2256 ip = (struct ip *)(mp->m_data + ehdrlen); 2257 if (ip->ip_p != IPPROTO_TCP) 2258 return (ENXIO); 2259 ip->ip_sum = 0; 2260 ip_hlen = ip->ip_hl << 2; 2261 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2262 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2263 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2264 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2265 /* Tell transmit desc to also do IPv4 checksum. */ 2266 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2267 break; 2268 default: 2269 panic("%s: CSUM_TSO but no supported IP version (0x%04x)", 2270 __func__, ntohs(etype)); 2271 break; 2272 } 2273 2274 ctxd = txr->next_avail_desc; 2275 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2276 2277 tcp_hlen = th->th_off << 2; 2278 2279 /* This is used in the transmit desc in encap */ 2280 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; 2281 2282 #if NVLAN > 0 2283 /* VLAN MACLEN IPLEN */ 2284 if (mp->m_flags & M_VLANTAG) { 2285 vtag = mp->m_pkthdr.ether_vtag; 2286 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2287 } 2288 #endif 2289 2290 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2291 vlan_macip_lens |= ip_hlen; 2292 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2293 2294 /* ADV DTYPE TUCMD */ 2295 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2296 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2297 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2298 2299 /* MSS L4LEN IDX */ 2300 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2301 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2302 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2303 2304 TXD->seqnum_seed = htole32(0); 2305 2306 membar_producer(); 2307 2308 if (++ctxd == sc->num_tx_desc) 2309 ctxd = 0; 2310 2311 atomic_dec_int(&txr->tx_avail); 2312 txr->next_avail_desc = ctxd; 2313 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2314 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2315 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 2316 return TRUE; 2317 } 2318 #endif 2319 2320 /********************************************************************** 2321 * 2322 * Examine each tx_buffer in the used queue. If the hardware is done 2323 * processing the packet then free associated resources. The 2324 * tx_buffer is put back on the free queue. 2325 * 2326 **********************************************************************/ 2327 int 2328 ixgbe_txeof(struct tx_ring *txr) 2329 { 2330 struct ix_softc *sc = txr->sc; 2331 struct ifnet *ifp = &sc->arpcom.ac_if; 2332 uint32_t first, last, done, processed; 2333 uint32_t num_avail; 2334 struct ixgbe_tx_buf *tx_buffer; 2335 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2336 2337 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2338 return FALSE; 2339 2340 if (txr->tx_avail == sc->num_tx_desc) { 2341 txr->queue_status = IXGBE_QUEUE_IDLE; 2342 return FALSE; 2343 } 2344 2345 membar_consumer(); 2346 2347 processed = 0; 2348 first = txr->next_to_clean; 2349 /* was the txt queue cleaned up in the meantime */ 2350 if (txr->tx_buffers == NULL) 2351 return FALSE; 2352 tx_buffer = &txr->tx_buffers[first]; 2353 /* For cleanup we just use legacy struct */ 2354 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2355 last = tx_buffer->eop_index; 2356 if (last == -1) 2357 return FALSE; 2358 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2359 2360 /* 2361 * Get the index of the first descriptor 2362 * BEYOND the EOP and call that 'done'. 2363 * I do this so the comparison in the 2364 * inner while loop below can be simple 2365 */ 2366 if (++last == sc->num_tx_desc) last = 0; 2367 done = last; 2368 2369 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2370 0, txr->txdma.dma_map->dm_mapsize, 2371 BUS_DMASYNC_POSTREAD); 2372 2373 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2374 /* We clean the range of the packet */ 2375 while (first != done) { 2376 tx_desc->upper.data = 0; 2377 tx_desc->lower.data = 0; 2378 tx_desc->buffer_addr = 0; 2379 ++processed; 2380 2381 if (tx_buffer->m_head) { 2382 bus_dmamap_sync(txr->txdma.dma_tag, 2383 tx_buffer->map, 2384 0, tx_buffer->map->dm_mapsize, 2385 BUS_DMASYNC_POSTWRITE); 2386 bus_dmamap_unload(txr->txdma.dma_tag, 2387 tx_buffer->map); 2388 m_freem(tx_buffer->m_head); 2389 tx_buffer->m_head = NULL; 2390 } 2391 tx_buffer->eop_index = -1; 2392 2393 if (++first == sc->num_tx_desc) 2394 first = 0; 2395 2396 tx_buffer = &txr->tx_buffers[first]; 2397 tx_desc = (struct ixgbe_legacy_tx_desc *) 2398 &txr->tx_base[first]; 2399 } 2400 ++txr->packets; 2401 ++ifp->if_opackets; 2402 /* See if there is more work now */ 2403 last = tx_buffer->eop_index; 2404 if (last != -1) { 2405 eop_desc = 2406 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2407 /* Get next done point */ 2408 if (++last == sc->num_tx_desc) last = 0; 2409 done = last; 2410 } else 2411 break; 2412 } 2413 2414 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2415 0, txr->txdma.dma_map->dm_mapsize, 2416 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2417 2418 txr->next_to_clean = first; 2419 2420 num_avail = atomic_add_int_nv(&txr->tx_avail, processed); 2421 2422 /* All clean, turn off the timer */ 2423 if (num_avail == sc->num_tx_desc) 2424 ifp->if_timer = 0; 2425 2426 if (ifq_is_oactive(&ifp->if_snd)) 2427 ifq_restart(&ifp->if_snd); 2428 2429 return TRUE; 2430 } 2431 2432 /********************************************************************* 2433 * 2434 * Get a buffer from system mbuf buffer pool. 2435 * 2436 **********************************************************************/ 2437 int 2438 ixgbe_get_buf(struct rx_ring *rxr, int i) 2439 { 2440 struct ix_softc *sc = rxr->sc; 2441 struct ixgbe_rx_buf *rxbuf; 2442 struct mbuf *mp; 2443 int error; 2444 union ixgbe_adv_rx_desc *rxdesc; 2445 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 2446 2447 rxbuf = &rxr->rx_buffers[i]; 2448 rxdesc = &rxr->rx_base[i]; 2449 if (rxbuf->buf) { 2450 printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n", 2451 sc->dev.dv_xname, i); 2452 return (ENOBUFS); 2453 } 2454 2455 /* needed in any case so prealocate since this one will fail for sure */ 2456 mp = MCLGETI(NULL, M_DONTWAIT, NULL, sc->rx_mbuf_sz); 2457 if (!mp) 2458 return (ENOBUFS); 2459 2460 mp->m_len = mp->m_pkthdr.len = sc->rx_mbuf_sz; 2461 #ifdef __STRICT_ALIGNMENT 2462 m_adj(mp, ETHER_ALIGN); 2463 #endif 2464 2465 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, 2466 mp, BUS_DMA_NOWAIT); 2467 if (error) { 2468 m_freem(mp); 2469 return (error); 2470 } 2471 2472 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2473 0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 2474 rxbuf->buf = mp; 2475 2476 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2477 dsize * i, dsize, BUS_DMASYNC_POSTWRITE); 2478 2479 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 2480 2481 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2482 dsize * i, dsize, BUS_DMASYNC_PREWRITE); 2483 2484 return (0); 2485 } 2486 2487 /********************************************************************* 2488 * 2489 * Allocate memory for rx_buffer structures. Since we use one 2490 * rx_buffer per received packet, the maximum number of rx_buffer's 2491 * that we'll need is equal to the number of receive descriptors 2492 * that we've allocated. 2493 * 2494 **********************************************************************/ 2495 int 2496 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2497 { 2498 struct ix_softc *sc = rxr->sc; 2499 struct ifnet *ifp = &sc->arpcom.ac_if; 2500 struct ixgbe_rx_buf *rxbuf; 2501 int i, bsize, error; 2502 2503 bsize = sizeof(struct ixgbe_rx_buf) * sc->num_rx_desc; 2504 if (!(rxr->rx_buffers = mallocarray(sc->num_rx_desc, 2505 sizeof(struct ixgbe_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2506 printf("%s: Unable to allocate rx_buffer memory\n", 2507 ifp->if_xname); 2508 error = ENOMEM; 2509 goto fail; 2510 } 2511 bsize = sizeof(struct ixgbe_rx_buf) * sc->num_rx_desc; 2512 2513 rxbuf = rxr->rx_buffers; 2514 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2515 error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1, 2516 16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map); 2517 if (error) { 2518 printf("%s: Unable to create Pack DMA map\n", 2519 ifp->if_xname); 2520 goto fail; 2521 } 2522 } 2523 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2524 rxr->rxdma.dma_map->dm_mapsize, 2525 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2526 2527 return (0); 2528 2529 fail: 2530 return (error); 2531 } 2532 2533 /********************************************************************* 2534 * 2535 * Initialize a receive ring and its buffers. 2536 * 2537 **********************************************************************/ 2538 int 2539 ixgbe_setup_receive_ring(struct rx_ring *rxr) 2540 { 2541 struct ix_softc *sc = rxr->sc; 2542 struct ifnet *ifp = &sc->arpcom.ac_if; 2543 int rsize, error; 2544 2545 rsize = roundup2(sc->num_rx_desc * 2546 sizeof(union ixgbe_adv_rx_desc), 4096); 2547 /* Clear the ring contents */ 2548 bzero((void *)rxr->rx_base, rsize); 2549 2550 if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0) 2551 return (error); 2552 2553 /* Setup our descriptor indices */ 2554 rxr->next_to_check = 0; 2555 rxr->last_desc_filled = sc->num_rx_desc - 1; 2556 2557 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2558 sc->num_rx_desc); 2559 2560 ixgbe_rxfill(rxr); 2561 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 2562 printf("%s: unable to fill any rx descriptors\n", 2563 sc->dev.dv_xname); 2564 return (ENOBUFS); 2565 } 2566 2567 return (0); 2568 } 2569 2570 int 2571 ixgbe_rxfill(struct rx_ring *rxr) 2572 { 2573 struct ix_softc *sc = rxr->sc; 2574 int post = 0; 2575 u_int slots; 2576 int i; 2577 2578 i = rxr->last_desc_filled; 2579 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); 2580 slots > 0; slots--) { 2581 if (++i == sc->num_rx_desc) 2582 i = 0; 2583 2584 if (ixgbe_get_buf(rxr, i) != 0) 2585 break; 2586 2587 rxr->last_desc_filled = i; 2588 post = 1; 2589 } 2590 2591 if_rxr_put(&rxr->rx_ring, slots); 2592 2593 return (post); 2594 } 2595 2596 void 2597 ixgbe_rxrefill(void *xsc) 2598 { 2599 struct ix_softc *sc = xsc; 2600 struct ix_queue *que = sc->queues; 2601 int s; 2602 2603 s = splnet(); 2604 if (ixgbe_rxfill(que->rxr)) { 2605 /* Advance the Rx Queue "Tail Pointer" */ 2606 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(que->rxr->me), 2607 que->rxr->last_desc_filled); 2608 } else 2609 timeout_add(&sc->rx_refill, 1); 2610 splx(s); 2611 } 2612 2613 /********************************************************************* 2614 * 2615 * Initialize all receive rings. 2616 * 2617 **********************************************************************/ 2618 int 2619 ixgbe_setup_receive_structures(struct ix_softc *sc) 2620 { 2621 struct rx_ring *rxr = sc->rx_rings; 2622 int i; 2623 2624 for (i = 0; i < sc->num_queues; i++, rxr++) 2625 if (ixgbe_setup_receive_ring(rxr)) 2626 goto fail; 2627 2628 return (0); 2629 fail: 2630 ixgbe_free_receive_structures(sc); 2631 return (ENOBUFS); 2632 } 2633 2634 /********************************************************************* 2635 * 2636 * Setup receive registers and features. 2637 * 2638 **********************************************************************/ 2639 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2640 2641 void 2642 ixgbe_initialize_receive_units(struct ix_softc *sc) 2643 { 2644 struct rx_ring *rxr = sc->rx_rings; 2645 uint32_t bufsz, rxctrl, fctrl, srrctl, rxcsum; 2646 uint32_t reta, mrqc = 0, hlreg; 2647 uint32_t random[10]; 2648 int i; 2649 2650 /* 2651 * Make sure receives are disabled while 2652 * setting up the descriptor ring 2653 */ 2654 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 2655 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXCTRL, 2656 rxctrl & ~IXGBE_RXCTRL_RXEN); 2657 2658 /* Enable broadcasts */ 2659 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 2660 fctrl |= IXGBE_FCTRL_BAM; 2661 fctrl |= IXGBE_FCTRL_DPF; 2662 fctrl |= IXGBE_FCTRL_PMCF; 2663 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 2664 2665 /* Always enable jumbo frame reception */ 2666 hlreg = IXGBE_READ_REG(&sc->hw, IXGBE_HLREG0); 2667 hlreg |= IXGBE_HLREG0_JUMBOEN; 2668 IXGBE_WRITE_REG(&sc->hw, IXGBE_HLREG0, hlreg); 2669 2670 #ifdef __STRICT_ALIGNMENT 2671 bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2672 #else 2673 bufsz = sc->rx_mbuf_sz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2674 #endif 2675 2676 for (i = 0; i < sc->num_queues; i++, rxr++) { 2677 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2678 2679 /* Setup the Base and Length of the Rx Descriptor Ring */ 2680 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDBAL(i), 2681 (rdba & 0x00000000ffffffffULL)); 2682 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDBAH(i), (rdba >> 32)); 2683 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDLEN(i), 2684 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2685 2686 /* Set up the SRRCTL register */ 2687 srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2688 IXGBE_WRITE_REG(&sc->hw, IXGBE_SRRCTL(i), srrctl); 2689 2690 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2691 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDH(i), 0); 2692 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), 0); 2693 } 2694 2695 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 2696 uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR | 2697 IXGBE_PSRTYPE_UDPHDR | 2698 IXGBE_PSRTYPE_IPV4HDR | 2699 IXGBE_PSRTYPE_IPV6HDR; 2700 IXGBE_WRITE_REG(&sc->hw, IXGBE_PSRTYPE(0), psrtype); 2701 } 2702 2703 rxcsum = IXGBE_READ_REG(&sc->hw, IXGBE_RXCSUM); 2704 rxcsum &= ~IXGBE_RXCSUM_PCSD; 2705 2706 /* Setup RSS */ 2707 if (sc->num_queues > 1) { 2708 int j; 2709 reta = 0; 2710 /* set up random bits */ 2711 arc4random_buf(&random, sizeof(random)); 2712 2713 /* Set up the redirection table */ 2714 for (i = 0, j = 0; i < 128; i++, j++) { 2715 if (j == sc->num_queues) 2716 j = 0; 2717 reta = (reta << 8) | (j * 0x11); 2718 if ((i & 3) == 3) 2719 IXGBE_WRITE_REG(&sc->hw, IXGBE_RETA(i >> 2), reta); 2720 } 2721 2722 /* Now fill our hash function seeds */ 2723 for (i = 0; i < 10; i++) 2724 IXGBE_WRITE_REG(&sc->hw, IXGBE_RSSRK(i), random[i]); 2725 2726 /* Perform hash on these packet types */ 2727 mrqc = IXGBE_MRQC_RSSEN 2728 | IXGBE_MRQC_RSS_FIELD_IPV4 2729 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 2730 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 2731 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 2732 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 2733 | IXGBE_MRQC_RSS_FIELD_IPV6 2734 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 2735 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 2736 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2737 IXGBE_WRITE_REG(&sc->hw, IXGBE_MRQC, mrqc); 2738 2739 /* RSS and RX IPP Checksum are mutually exclusive */ 2740 rxcsum |= IXGBE_RXCSUM_PCSD; 2741 } 2742 2743 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2744 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2745 2746 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXCSUM, rxcsum); 2747 } 2748 2749 /********************************************************************* 2750 * 2751 * Free all receive rings. 2752 * 2753 **********************************************************************/ 2754 void 2755 ixgbe_free_receive_structures(struct ix_softc *sc) 2756 { 2757 struct rx_ring *rxr; 2758 int i; 2759 2760 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 2761 if_rxr_init(&rxr->rx_ring, 0, 0); 2762 2763 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 2764 ixgbe_free_receive_buffers(rxr); 2765 } 2766 2767 /********************************************************************* 2768 * 2769 * Free receive ring data structures 2770 * 2771 **********************************************************************/ 2772 void 2773 ixgbe_free_receive_buffers(struct rx_ring *rxr) 2774 { 2775 struct ix_softc *sc; 2776 struct ixgbe_rx_buf *rxbuf; 2777 int i; 2778 2779 sc = rxr->sc; 2780 if (rxr->rx_buffers != NULL) { 2781 for (i = 0; i < sc->num_rx_desc; i++) { 2782 rxbuf = &rxr->rx_buffers[i]; 2783 if (rxbuf->buf != NULL) { 2784 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2785 0, rxbuf->map->dm_mapsize, 2786 BUS_DMASYNC_POSTREAD); 2787 bus_dmamap_unload(rxr->rxdma.dma_tag, 2788 rxbuf->map); 2789 m_freem(rxbuf->buf); 2790 rxbuf->buf = NULL; 2791 } 2792 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 2793 rxbuf->map = NULL; 2794 } 2795 free(rxr->rx_buffers, M_DEVBUF, 2796 sc->num_rx_desc * sizeof(struct ixgbe_rx_buf)); 2797 rxr->rx_buffers = NULL; 2798 } 2799 } 2800 2801 /********************************************************************* 2802 * 2803 * This routine executes in interrupt context. It replenishes 2804 * the mbufs in the descriptor and sends data which has been 2805 * dma'ed into host memory to upper layer. 2806 * 2807 *********************************************************************/ 2808 int 2809 ixgbe_rxeof(struct ix_queue *que) 2810 { 2811 struct ix_softc *sc = que->sc; 2812 struct rx_ring *rxr = que->rxr; 2813 struct ifnet *ifp = &sc->arpcom.ac_if; 2814 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 2815 struct mbuf *mp, *sendmp; 2816 uint8_t eop = 0; 2817 uint16_t len, vtag; 2818 uint32_t staterr = 0, ptype; 2819 struct ixgbe_rx_buf *rxbuf, *nxbuf; 2820 union ixgbe_adv_rx_desc *rxdesc; 2821 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 2822 int i, nextp; 2823 2824 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2825 return FALSE; 2826 2827 i = rxr->next_to_check; 2828 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 2829 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2830 dsize * i, dsize, BUS_DMASYNC_POSTREAD); 2831 2832 rxdesc = &rxr->rx_base[i]; 2833 staterr = letoh32(rxdesc->wb.upper.status_error); 2834 if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) { 2835 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2836 dsize * i, dsize, 2837 BUS_DMASYNC_PREREAD); 2838 break; 2839 } 2840 2841 /* Zero out the receive descriptors status */ 2842 rxdesc->wb.upper.status_error = 0; 2843 rxbuf = &rxr->rx_buffers[i]; 2844 2845 /* pull the mbuf off the ring */ 2846 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 2847 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2848 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 2849 2850 mp = rxbuf->buf; 2851 len = letoh16(rxdesc->wb.upper.length); 2852 ptype = letoh32(rxdesc->wb.lower.lo_dword.data) & 2853 IXGBE_RXDADV_PKTTYPE_MASK; 2854 vtag = letoh16(rxdesc->wb.upper.vlan); 2855 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 2856 2857 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) { 2858 sc->dropped_pkts++; 2859 2860 if (rxbuf->fmp) { 2861 m_freem(rxbuf->fmp); 2862 rxbuf->fmp = NULL; 2863 } 2864 2865 m_freem(mp); 2866 rxbuf->buf = NULL; 2867 goto next_desc; 2868 } 2869 2870 if (mp == NULL) { 2871 panic("%s: ixgbe_rxeof: NULL mbuf in slot %d " 2872 "(nrx %d, filled %d)", sc->dev.dv_xname, 2873 i, if_rxr_inuse(&rxr->rx_ring), 2874 rxr->last_desc_filled); 2875 } 2876 2877 /* Currently no HW RSC support of 82599 */ 2878 if (!eop) { 2879 /* 2880 * Figure out the next descriptor of this frame. 2881 */ 2882 nextp = i + 1; 2883 if (nextp == sc->num_rx_desc) 2884 nextp = 0; 2885 nxbuf = &rxr->rx_buffers[nextp]; 2886 /* prefetch(nxbuf); */ 2887 } 2888 2889 /* 2890 * Rather than using the fmp/lmp global pointers 2891 * we now keep the head of a packet chain in the 2892 * buffer struct and pass this along from one 2893 * descriptor to the next, until we get EOP. 2894 */ 2895 mp->m_len = len; 2896 /* 2897 * See if there is a stored head 2898 * that determines what we are 2899 */ 2900 sendmp = rxbuf->fmp; 2901 rxbuf->buf = rxbuf->fmp = NULL; 2902 2903 if (sendmp != NULL) /* secondary frag */ 2904 sendmp->m_pkthdr.len += mp->m_len; 2905 else { 2906 /* first desc of a non-ps chain */ 2907 sendmp = mp; 2908 sendmp->m_pkthdr.len = mp->m_len; 2909 #if NVLAN > 0 2910 if (staterr & IXGBE_RXD_STAT_VP) { 2911 sendmp->m_pkthdr.ether_vtag = vtag; 2912 sendmp->m_flags |= M_VLANTAG; 2913 } 2914 #endif 2915 } 2916 2917 /* Pass the head pointer on */ 2918 if (eop == 0) { 2919 nxbuf->fmp = sendmp; 2920 sendmp = NULL; 2921 mp->m_next = nxbuf->buf; 2922 } else { /* Sending this frame? */ 2923 rxr->rx_packets++; 2924 /* capture data for AIM */ 2925 rxr->bytes += sendmp->m_pkthdr.len; 2926 rxr->rx_bytes += sendmp->m_pkthdr.len; 2927 2928 ixgbe_rx_checksum(staterr, sendmp, ptype); 2929 2930 ml_enqueue(&ml, sendmp); 2931 } 2932 next_desc: 2933 if_rxr_put(&rxr->rx_ring, 1); 2934 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2935 dsize * i, dsize, 2936 BUS_DMASYNC_PREREAD); 2937 2938 /* Advance our pointers to the next descriptor. */ 2939 if (++i == sc->num_rx_desc) 2940 i = 0; 2941 } 2942 rxr->next_to_check = i; 2943 2944 if_input(ifp, &ml); 2945 2946 if (!(staterr & IXGBE_RXD_STAT_DD)) 2947 return FALSE; 2948 2949 return TRUE; 2950 } 2951 2952 /********************************************************************* 2953 * 2954 * Verify that the hardware indicated that the checksum is valid. 2955 * Inform the stack about the status of checksum so that stack 2956 * doesn't spend time verifying the checksum. 2957 * 2958 *********************************************************************/ 2959 void 2960 ixgbe_rx_checksum(uint32_t staterr, struct mbuf * mp, uint32_t ptype) 2961 { 2962 uint16_t status = (uint16_t) staterr; 2963 uint8_t errors = (uint8_t) (staterr >> 24); 2964 2965 if (status & IXGBE_RXD_STAT_IPCS) { 2966 if (!(errors & IXGBE_RXD_ERR_IPE)) { 2967 /* IP Checksum Good */ 2968 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 2969 } else 2970 mp->m_pkthdr.csum_flags = 0; 2971 } 2972 if (status & IXGBE_RXD_STAT_L4CS) { 2973 if (!(errors & IXGBE_RXD_ERR_TCPE)) 2974 mp->m_pkthdr.csum_flags |= 2975 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 2976 } 2977 } 2978 2979 void 2980 ixgbe_setup_vlan_hw_support(struct ix_softc *sc) 2981 { 2982 uint32_t ctrl; 2983 int i; 2984 2985 /* 2986 * A soft reset zero's out the VFTA, so 2987 * we need to repopulate it now. 2988 */ 2989 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 2990 if (sc->shadow_vfta[i] != 0) 2991 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i), 2992 sc->shadow_vfta[i]); 2993 } 2994 2995 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 2996 #if 0 2997 /* Enable the Filter Table if enabled */ 2998 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 2999 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3000 ctrl |= IXGBE_VLNCTRL_VFE; 3001 } 3002 #endif 3003 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3004 ctrl |= IXGBE_VLNCTRL_VME; 3005 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 3006 3007 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 3008 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 3009 for (i = 0; i < sc->num_queues; i++) { 3010 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 3011 ctrl |= IXGBE_RXDCTL_VME; 3012 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl); 3013 } 3014 } 3015 } 3016 3017 void 3018 ixgbe_enable_intr(struct ix_softc *sc) 3019 { 3020 struct ixgbe_hw *hw = &sc->hw; 3021 struct ix_queue *que = sc->queues; 3022 uint32_t mask, fwsm; 3023 int i; 3024 3025 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3026 /* Enable Fan Failure detection */ 3027 if (hw->device_id == IXGBE_DEV_ID_82598AT) 3028 mask |= IXGBE_EIMS_GPI_SDP1; 3029 3030 switch (sc->hw.mac.type) { 3031 case ixgbe_mac_82599EB: 3032 mask |= IXGBE_EIMS_ECC; 3033 mask |= IXGBE_EIMS_GPI_SDP0; 3034 mask |= IXGBE_EIMS_GPI_SDP1; 3035 mask |= IXGBE_EIMS_GPI_SDP2; 3036 break; 3037 case ixgbe_mac_X540: 3038 mask |= IXGBE_EIMS_ECC; 3039 /* Detect if Thermal Sensor is enabled */ 3040 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3041 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3042 mask |= IXGBE_EIMS_TS; 3043 break; 3044 default: 3045 break; 3046 } 3047 3048 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3049 3050 /* With RSS we use auto clear */ 3051 if (sc->msix > 1) { 3052 mask = IXGBE_EIMS_ENABLE_MASK; 3053 /* Don't autoclear Link */ 3054 mask &= ~IXGBE_EIMS_OTHER; 3055 mask &= ~IXGBE_EIMS_LSC; 3056 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, mask); 3057 } 3058 3059 /* 3060 * Now enable all queues, this is done separately to 3061 * allow for handling the extended (beyond 32) MSIX 3062 * vectors that can be used by 82599 3063 */ 3064 for (i = 0; i < sc->num_queues; i++, que++) 3065 ixgbe_enable_queue(sc, que->msix); 3066 3067 IXGBE_WRITE_FLUSH(hw); 3068 } 3069 3070 void 3071 ixgbe_disable_intr(struct ix_softc *sc) 3072 { 3073 if (sc->msix > 1) 3074 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3075 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3076 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3077 } else { 3078 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3079 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3080 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3081 } 3082 IXGBE_WRITE_FLUSH(&sc->hw); 3083 } 3084 3085 uint16_t 3086 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3087 { 3088 struct pci_attach_args *pa; 3089 uint32_t value; 3090 int high = 0; 3091 3092 if (reg & 0x2) { 3093 high = 1; 3094 reg &= ~0x2; 3095 } 3096 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3097 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3098 3099 if (high) 3100 value >>= 16; 3101 3102 return (value & 0xffff); 3103 } 3104 3105 void 3106 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3107 { 3108 struct pci_attach_args *pa; 3109 uint32_t rv; 3110 int high = 0; 3111 3112 /* Need to do read/mask/write... because 16 vs 32 bit!!! */ 3113 if (reg & 0x2) { 3114 high = 1; 3115 reg &= ~0x2; 3116 } 3117 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3118 rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3119 if (!high) 3120 rv = (rv & 0xffff0000) | value; 3121 else 3122 rv = (rv & 0xffff) | ((uint32_t)value << 16); 3123 pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv); 3124 } 3125 3126 /* 3127 * Setup the correct IVAR register for a particular MSIX interrupt 3128 * (yes this is all very magic and confusing :) 3129 * - entry is the register array entry 3130 * - vector is the MSIX vector for this queue 3131 * - type is RX/TX/MISC 3132 */ 3133 void 3134 ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type) 3135 { 3136 struct ixgbe_hw *hw = &sc->hw; 3137 uint32_t ivar, index; 3138 3139 vector |= IXGBE_IVAR_ALLOC_VAL; 3140 3141 switch (hw->mac.type) { 3142 3143 case ixgbe_mac_82598EB: 3144 if (type == -1) 3145 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3146 else 3147 entry += (type * 64); 3148 index = (entry >> 2) & 0x1F; 3149 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3150 ivar &= ~(0xFF << (8 * (entry & 0x3))); 3151 ivar |= (vector << (8 * (entry & 0x3))); 3152 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 3153 break; 3154 3155 case ixgbe_mac_82599EB: 3156 case ixgbe_mac_X540: 3157 if (type == -1) { /* MISC IVAR */ 3158 index = (entry & 1) * 8; 3159 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3160 ivar &= ~(0xFF << index); 3161 ivar |= (vector << index); 3162 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3163 } else { /* RX/TX IVARS */ 3164 index = (16 * (entry & 1)) + (8 * type); 3165 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3166 ivar &= ~(0xFF << index); 3167 ivar |= (vector << index); 3168 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3169 } 3170 3171 default: 3172 break; 3173 } 3174 } 3175 3176 void 3177 ixgbe_configure_ivars(struct ix_softc *sc) 3178 { 3179 #if notyet 3180 struct ix_queue *que = sc->queues; 3181 uint32_t newitr; 3182 int i; 3183 3184 if (ixgbe_max_interrupt_rate > 0) 3185 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 3186 else 3187 newitr = 0; 3188 3189 for (i = 0; i < sc->num_queues; i++, que++) { 3190 /* First the RX queue entry */ 3191 ixgbe_set_ivar(sc, i, que->msix, 0); 3192 /* ... and the TX */ 3193 ixgbe_set_ivar(sc, i, que->msix, 1); 3194 /* Set an Initial EITR value */ 3195 IXGBE_WRITE_REG(&sc->hw, 3196 IXGBE_EITR(que->msix), newitr); 3197 } 3198 3199 /* For the Link interrupt */ 3200 ixgbe_set_ivar(sc, 1, sc->linkvec, -1); 3201 #endif 3202 } 3203 3204 /* 3205 * SFP module interrupts handler 3206 */ 3207 void 3208 ixgbe_handle_mod(struct ix_softc *sc) 3209 { 3210 struct ixgbe_hw *hw = &sc->hw; 3211 uint32_t err; 3212 3213 err = hw->phy.ops.identify_sfp(hw); 3214 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3215 printf("%s: Unsupported SFP+ module type was detected!\n", 3216 sc->dev.dv_xname); 3217 return; 3218 } 3219 err = hw->mac.ops.setup_sfp(hw); 3220 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3221 printf("%s: Setup failure - unsupported SFP+ module type!\n", 3222 sc->dev.dv_xname); 3223 return; 3224 } 3225 /* Set the optics type so system reports correctly */ 3226 ixgbe_setup_optics(sc); 3227 3228 ixgbe_handle_msf(sc); 3229 } 3230 3231 3232 /* 3233 * MSF (multispeed fiber) interrupts handler 3234 */ 3235 void 3236 ixgbe_handle_msf(struct ix_softc *sc) 3237 { 3238 struct ixgbe_hw *hw = &sc->hw; 3239 uint32_t autoneg; 3240 bool negotiate; 3241 3242 autoneg = hw->phy.autoneg_advertised; 3243 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) { 3244 if (hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate)) 3245 return; 3246 } 3247 if (hw->mac.ops.setup_link) 3248 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3249 } 3250 3251 /********************************************************************** 3252 * 3253 * Update the board statistics counters. 3254 * 3255 **********************************************************************/ 3256 void 3257 ixgbe_update_stats_counters(struct ix_softc *sc) 3258 { 3259 struct ifnet *ifp = &sc->arpcom.ac_if; 3260 struct ixgbe_hw *hw = &sc->hw; 3261 uint64_t total_missed_rx = 0; 3262 #ifdef IX_DEBUG 3263 uint32_t missed_rx = 0, bprc, lxon, lxoff, total; 3264 int i; 3265 #endif 3266 3267 sc->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3268 sc->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3269 3270 #ifdef IX_DEBUG 3271 for (i = 0; i < 8; i++) { 3272 uint32_t mp; 3273 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3274 /* missed_rx tallies misses for the gprc workaround */ 3275 missed_rx += mp; 3276 /* global total per queue */ 3277 sc->stats.mpc[i] += mp; 3278 /* running comprehensive total for stats display */ 3279 total_missed_rx += sc->stats.mpc[i]; 3280 if (hw->mac.type == ixgbe_mac_82598EB) 3281 sc->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3282 } 3283 3284 /* Hardware workaround, gprc counts missed packets */ 3285 sc->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 3286 sc->stats.gprc -= missed_rx; 3287 3288 if (hw->mac.type != ixgbe_mac_82598EB) { 3289 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) + 3290 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 3291 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 3292 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 3293 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) + 3294 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 3295 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3296 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3297 } else { 3298 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3299 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3300 /* 82598 only has a counter in the high register */ 3301 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3302 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3303 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3304 } 3305 3306 /* 3307 * Workaround: mprc hardware is incorrectly counting 3308 * broadcasts, so for now we subtract those. 3309 */ 3310 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3311 sc->stats.bprc += bprc; 3312 sc->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3313 if (hw->mac.type == ixgbe_mac_82598EB) 3314 sc->stats.mprc -= bprc; 3315 3316 sc->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3317 sc->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3318 sc->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3319 sc->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3320 sc->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3321 sc->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3322 sc->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3323 3324 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3325 sc->stats.lxontxc += lxon; 3326 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3327 sc->stats.lxofftxc += lxoff; 3328 total = lxon + lxoff; 3329 3330 sc->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3331 sc->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3332 sc->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3333 sc->stats.gptc -= total; 3334 sc->stats.mptc -= total; 3335 sc->stats.ptc64 -= total; 3336 sc->stats.gotc -= total * ETHER_MIN_LEN; 3337 3338 sc->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3339 sc->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3340 sc->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3341 sc->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3342 sc->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3343 sc->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3344 sc->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3345 sc->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3346 sc->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3347 sc->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3348 #endif 3349 3350 /* Fill out the OS statistics structure */ 3351 ifp->if_collisions = 0; 3352 ifp->if_oerrors = sc->watchdog_events; 3353 ifp->if_ierrors = total_missed_rx + sc->stats.crcerrs + sc->stats.rlec; 3354 } 3355 3356 #ifdef IX_DEBUG 3357 /********************************************************************** 3358 * 3359 * This routine is called only when ixgbe_display_debug_stats is enabled. 3360 * This routine provides a way to take a look at important statistics 3361 * maintained by the driver and hardware. 3362 * 3363 **********************************************************************/ 3364 void 3365 ixgbe_print_hw_stats(struct ix_softc * sc) 3366 { 3367 struct ifnet *ifp = &sc->arpcom.ac_if;; 3368 3369 printf("%s: missed pkts %llu, rx len errs %llu, crc errs %llu, " 3370 "dropped pkts %lu, watchdog timeouts %ld, " 3371 "XON rx %llu, XON tx %llu, XOFF rx %llu, XOFF tx %llu, " 3372 "total pkts rx %llu, good pkts rx %llu, good pkts tx %llu, " 3373 "tso tx %lu\n", 3374 ifp->if_xname, 3375 (long long)sc->stats.mpc[0], 3376 (long long)sc->stats.roc + (long long)sc->stats.ruc, 3377 (long long)sc->stats.crcerrs, 3378 sc->dropped_pkts, 3379 sc->watchdog_events, 3380 (long long)sc->stats.lxonrxc, 3381 (long long)sc->stats.lxontxc, 3382 (long long)sc->stats.lxoffrxc, 3383 (long long)sc->stats.lxofftxc, 3384 (long long)sc->stats.tpr, 3385 (long long)sc->stats.gprc, 3386 (long long)sc->stats.gptc, 3387 sc->tso_tx); 3388 } 3389 #endif 3390