1 /* $OpenBSD: if_ix.c,v 1.96 2014/07/13 23:10:23 deraadt Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2013, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */ 36 37 #include <dev/pci/if_ix.h> 38 #include <dev/pci/ixgbe_type.h> 39 40 /********************************************************************* 41 * Driver version 42 *********************************************************************/ 43 /* char ixgbe_driver_version[] = "2.5.13"; */ 44 45 /********************************************************************* 46 * PCI Device ID Table 47 * 48 * Used by probe to select devices to load on 49 *********************************************************************/ 50 51 const struct pci_matchid ixgbe_devices[] = { 52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 }, 53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX }, 54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT }, 57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT2 }, 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP }, 63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM }, 64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL }, 65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 }, 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BP }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_T3_LOM }, 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF2 }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_FCOE }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T }, 77 }; 78 79 /********************************************************************* 80 * Function prototypes 81 *********************************************************************/ 82 int ixgbe_probe(struct device *, void *, void *); 83 void ixgbe_attach(struct device *, struct device *, void *); 84 int ixgbe_detach(struct device *, int); 85 void ixgbe_start(struct ifnet *); 86 int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 87 void ixgbe_watchdog(struct ifnet *); 88 void ixgbe_init(void *); 89 void ixgbe_stop(void *); 90 void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 91 int ixgbe_media_change(struct ifnet *); 92 void ixgbe_identify_hardware(struct ix_softc *); 93 int ixgbe_allocate_pci_resources(struct ix_softc *); 94 int ixgbe_allocate_legacy(struct ix_softc *); 95 int ixgbe_allocate_queues(struct ix_softc *); 96 void ixgbe_free_pci_resources(struct ix_softc *); 97 void ixgbe_local_timer(void *); 98 void ixgbe_setup_interface(struct ix_softc *); 99 void ixgbe_config_link(struct ix_softc *sc); 100 101 int ixgbe_allocate_transmit_buffers(struct tx_ring *); 102 int ixgbe_setup_transmit_structures(struct ix_softc *); 103 int ixgbe_setup_transmit_ring(struct tx_ring *); 104 void ixgbe_initialize_transmit_units(struct ix_softc *); 105 void ixgbe_free_transmit_structures(struct ix_softc *); 106 void ixgbe_free_transmit_buffers(struct tx_ring *); 107 108 int ixgbe_allocate_receive_buffers(struct rx_ring *); 109 int ixgbe_setup_receive_structures(struct ix_softc *); 110 int ixgbe_setup_receive_ring(struct rx_ring *); 111 void ixgbe_initialize_receive_units(struct ix_softc *); 112 void ixgbe_free_receive_structures(struct ix_softc *); 113 void ixgbe_free_receive_buffers(struct rx_ring *); 114 int ixgbe_rxfill(struct rx_ring *); 115 void ixgbe_rxrefill(void *); 116 117 void ixgbe_enable_intr(struct ix_softc *); 118 void ixgbe_disable_intr(struct ix_softc *); 119 void ixgbe_update_stats_counters(struct ix_softc *); 120 int ixgbe_txeof(struct tx_ring *); 121 int ixgbe_rxeof(struct ix_queue *); 122 void ixgbe_rx_checksum(uint32_t, struct mbuf *, uint32_t); 123 void ixgbe_iff(struct ix_softc *); 124 #ifdef IX_DEBUG 125 void ixgbe_print_hw_stats(struct ix_softc *); 126 #endif 127 void ixgbe_update_link_status(struct ix_softc *); 128 int ixgbe_get_buf(struct rx_ring *, int); 129 int ixgbe_encap(struct tx_ring *, struct mbuf *); 130 int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 131 struct ixgbe_dma_alloc *, int); 132 void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 133 int ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *, uint32_t *, 134 uint32_t *); 135 int ixgbe_tso_setup(struct tx_ring *, struct mbuf *, uint32_t *, 136 uint32_t *); 137 void ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 138 void ixgbe_configure_ivars(struct ix_softc *); 139 uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 140 141 void ixgbe_setup_vlan_hw_support(struct ix_softc *); 142 143 /* Support for pluggable optic modules */ 144 bool ixgbe_sfp_probe(struct ix_softc *); 145 void ixgbe_setup_optics(struct ix_softc *); 146 void ixgbe_handle_mod(struct ix_softc *); 147 void ixgbe_handle_msf(struct ix_softc *); 148 149 /* Legacy (single vector interrupt handler */ 150 int ixgbe_intr(void *); 151 void ixgbe_enable_queue(struct ix_softc *, uint32_t); 152 void ixgbe_disable_queue(struct ix_softc *, uint32_t); 153 void ixgbe_rearm_queue(struct ix_softc *, uint32_t); 154 155 /********************************************************************* 156 * OpenBSD Device Interface Entry Points 157 *********************************************************************/ 158 159 struct cfdriver ix_cd = { 160 NULL, "ix", DV_IFNET 161 }; 162 163 struct cfattach ix_ca = { 164 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach 165 }; 166 167 int ixgbe_smart_speed = ixgbe_smart_speed_on; 168 169 /********************************************************************* 170 * Device identification routine 171 * 172 * ixgbe_probe determines if the driver should be loaded on 173 * adapter based on PCI vendor/device id of the adapter. 174 * 175 * return 0 on success, positive on failure 176 *********************************************************************/ 177 178 int 179 ixgbe_probe(struct device *parent, void *match, void *aux) 180 { 181 INIT_DEBUGOUT("ixgbe_probe: begin"); 182 183 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 184 nitems(ixgbe_devices))); 185 } 186 187 /********************************************************************* 188 * Device initialization routine 189 * 190 * The attach entry point is called when the driver is being loaded. 191 * This routine identifies the type of hardware, allocates all resources 192 * and initializes the hardware. 193 * 194 * return 0 on success, positive on failure 195 *********************************************************************/ 196 197 void 198 ixgbe_attach(struct device *parent, struct device *self, void *aux) 199 { 200 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 201 struct ix_softc *sc = (struct ix_softc *)self; 202 int error = 0; 203 uint16_t csum; 204 uint32_t ctrl_ext; 205 struct ixgbe_hw *hw = &sc->hw; 206 207 INIT_DEBUGOUT("ixgbe_attach: begin"); 208 209 sc->osdep.os_sc = sc; 210 sc->osdep.os_pa = *pa; 211 212 /* Set up the timer callout */ 213 timeout_set(&sc->timer, ixgbe_local_timer, sc); 214 timeout_set(&sc->rx_refill, ixgbe_rxrefill, sc); 215 216 /* Determine hardware revision */ 217 ixgbe_identify_hardware(sc); 218 219 /* Indicate to RX setup to use Jumbo Clusters */ 220 sc->num_tx_desc = DEFAULT_TXD; 221 sc->num_rx_desc = DEFAULT_RXD; 222 223 /* Do base PCI setup - map BAR0 */ 224 if (ixgbe_allocate_pci_resources(sc)) 225 goto err_out; 226 227 /* Allocate our TX/RX Queues */ 228 if (ixgbe_allocate_queues(sc)) 229 goto err_out; 230 231 /* Allocate multicast array memory. */ 232 sc->mta = malloc(sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS * 233 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 234 if (sc->mta == NULL) { 235 printf(": Can not allocate multicast setup array\n"); 236 goto err_late; 237 } 238 239 /* Initialize the shared code */ 240 error = ixgbe_init_shared_code(hw); 241 if (error) { 242 printf(": Unable to initialize the shared code\n"); 243 goto err_late; 244 } 245 246 /* Make sure we have a good EEPROM before we read from it */ 247 if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) { 248 printf(": The EEPROM Checksum Is Not Valid\n"); 249 goto err_late; 250 } 251 252 error = ixgbe_init_hw(hw); 253 if (error == IXGBE_ERR_EEPROM_VERSION) { 254 printf(": This device is a pre-production adapter/" 255 "LOM. Please be aware there may be issues associated " 256 "with your hardware.\n If you are experiencing problems " 257 "please contact your Intel or hardware representative " 258 "who provided you with this hardware.\n"); 259 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 260 printf("Unsupported SFP+ Module\n"); 261 } 262 263 if (error) { 264 printf(": Hardware Initialization Failure\n"); 265 goto err_late; 266 } 267 268 /* Detect and set physical type */ 269 ixgbe_setup_optics(sc); 270 271 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 272 IXGBE_ETH_LENGTH_OF_ADDRESS); 273 274 error = ixgbe_allocate_legacy(sc); 275 if (error) 276 goto err_late; 277 278 /* Setup OS specific network interface */ 279 ixgbe_setup_interface(sc); 280 281 /* Initialize statistics */ 282 ixgbe_update_stats_counters(sc); 283 284 /* Get the PCI-E bus info and determine LAN ID */ 285 hw->mac.ops.get_bus_info(hw); 286 287 /* Set an initial default flow control value */ 288 sc->fc = ixgbe_fc_full; 289 290 /* let hardware know driver is loaded */ 291 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 292 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 293 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 294 295 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 296 297 INIT_DEBUGOUT("ixgbe_attach: end"); 298 return; 299 300 err_late: 301 ixgbe_free_transmit_structures(sc); 302 ixgbe_free_receive_structures(sc); 303 err_out: 304 ixgbe_free_pci_resources(sc); 305 free(sc->mta, M_DEVBUF, 0); 306 } 307 308 /********************************************************************* 309 * Device removal routine 310 * 311 * The detach entry point is called when the driver is being removed. 312 * This routine stops the adapter and deallocates all the resources 313 * that were allocated for driver operation. 314 * 315 * return 0 on success, positive on failure 316 *********************************************************************/ 317 318 int 319 ixgbe_detach(struct device *self, int flags) 320 { 321 struct ix_softc *sc = (struct ix_softc *)self; 322 struct ifnet *ifp = &sc->arpcom.ac_if; 323 uint32_t ctrl_ext; 324 325 INIT_DEBUGOUT("ixgbe_detach: begin"); 326 327 ixgbe_stop(sc); 328 329 /* let hardware know driver is unloading */ 330 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 331 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 332 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 333 334 ether_ifdetach(ifp); 335 if_detach(ifp); 336 337 timeout_del(&sc->timer); 338 timeout_del(&sc->rx_refill); 339 ixgbe_free_pci_resources(sc); 340 341 ixgbe_free_transmit_structures(sc); 342 ixgbe_free_receive_structures(sc); 343 free(sc->mta, M_DEVBUF, 0); 344 345 return (0); 346 } 347 348 /********************************************************************* 349 * Transmit entry point 350 * 351 * ixgbe_start is called by the stack to initiate a transmit. 352 * The driver will remain in this routine as long as there are 353 * packets to transmit and transmit resources are available. 354 * In case resources are not available stack is notified and 355 * the packet is requeued. 356 **********************************************************************/ 357 358 void 359 ixgbe_start(struct ifnet * ifp) 360 { 361 struct ix_softc *sc = ifp->if_softc; 362 struct tx_ring *txr = sc->tx_rings; 363 struct mbuf *m_head; 364 int post = 0; 365 366 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING) 367 return; 368 if (!sc->link_up) 369 return; 370 371 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 372 txr->txdma.dma_map->dm_mapsize, 373 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 374 375 for (;;) { 376 IFQ_POLL(&ifp->if_snd, m_head); 377 if (m_head == NULL) 378 break; 379 380 if (ixgbe_encap(txr, m_head)) { 381 ifp->if_flags |= IFF_OACTIVE; 382 break; 383 } 384 385 IFQ_DEQUEUE(&ifp->if_snd, m_head); 386 387 #if NBPFILTER > 0 388 if (ifp->if_bpf) 389 bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 390 #endif 391 392 /* Set timeout in case hardware has problems transmitting */ 393 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 394 ifp->if_timer = IXGBE_TX_TIMEOUT; 395 396 post = 1; 397 } 398 399 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 400 0, txr->txdma.dma_map->dm_mapsize, 401 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 402 403 /* 404 * Advance the Transmit Descriptor Tail (Tdt), this tells the 405 * hardware that this frame is available to transmit. 406 */ 407 if (post) 408 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 409 txr->next_avail_desc); 410 } 411 412 /********************************************************************* 413 * Ioctl entry point 414 * 415 * ixgbe_ioctl is called when the user wants to configure the 416 * interface. 417 * 418 * return 0 on success, positive on failure 419 **********************************************************************/ 420 421 int 422 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 423 { 424 struct ix_softc *sc = ifp->if_softc; 425 struct ifaddr *ifa = (struct ifaddr *) data; 426 struct ifreq *ifr = (struct ifreq *) data; 427 int s, error = 0; 428 429 s = splnet(); 430 431 switch (command) { 432 case SIOCSIFADDR: 433 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 434 ifp->if_flags |= IFF_UP; 435 if (!(ifp->if_flags & IFF_RUNNING)) 436 ixgbe_init(sc); 437 #ifdef INET 438 if (ifa->ifa_addr->sa_family == AF_INET) 439 arp_ifinit(&sc->arpcom, ifa); 440 #endif 441 break; 442 443 case SIOCSIFMTU: 444 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 445 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > ifp->if_hardmtu) 446 error = EINVAL; 447 else if (ifp->if_mtu != ifr->ifr_mtu) { 448 ifp->if_mtu = ifr->ifr_mtu; 449 sc->max_frame_size = 450 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 451 ixgbe_init(sc); 452 } 453 break; 454 455 case SIOCSIFFLAGS: 456 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 457 if (ifp->if_flags & IFF_UP) { 458 if (ifp->if_flags & IFF_RUNNING) 459 error = ENETRESET; 460 else 461 ixgbe_init(sc); 462 } else { 463 if (ifp->if_flags & IFF_RUNNING) 464 ixgbe_stop(sc); 465 } 466 break; 467 468 case SIOCSIFMEDIA: 469 case SIOCGIFMEDIA: 470 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 471 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 472 break; 473 474 default: 475 error = ether_ioctl(ifp, &sc->arpcom, command, data); 476 } 477 478 if (error == ENETRESET) { 479 if (ifp->if_flags & IFF_RUNNING) { 480 ixgbe_disable_intr(sc); 481 ixgbe_iff(sc); 482 ixgbe_enable_intr(sc); 483 } 484 error = 0; 485 } 486 487 splx(s); 488 return (error); 489 } 490 491 /********************************************************************* 492 * Watchdog entry point 493 * 494 **********************************************************************/ 495 496 void 497 ixgbe_watchdog(struct ifnet * ifp) 498 { 499 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 500 struct tx_ring *txr = sc->tx_rings; 501 struct ixgbe_hw *hw = &sc->hw; 502 int tx_hang = FALSE; 503 int i; 504 505 /* 506 * The timer is set to 5 every time ixgbe_start() queues a packet. 507 * Then ixgbe_txeof() keeps resetting to 5 as long as it cleans at 508 * least one descriptor. 509 * Finally, anytime all descriptors are clean the timer is 510 * set to 0. 511 */ 512 for (i = 0; i < sc->num_queues; i++, txr++) { 513 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 514 continue; 515 else { 516 tx_hang = TRUE; 517 break; 518 } 519 } 520 if (tx_hang == FALSE) 521 return; 522 523 /* 524 * If we are in this routine because of pause frames, then don't 525 * reset the hardware. 526 */ 527 if (!(IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXON)) { 528 for (i = 0; i < sc->num_queues; i++, txr++) 529 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 530 ifp->if_timer = IXGBE_TX_TIMEOUT; 531 return; 532 } 533 534 535 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 536 for (i = 0; i < sc->num_queues; i++, txr++) { 537 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 538 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 539 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 540 printf("%s: TX(%d) desc avail = %d, Next TX to Clean = %d\n", ifp->if_xname, 541 i, txr->tx_avail, txr->next_to_clean); 542 } 543 ifp->if_flags &= ~IFF_RUNNING; 544 sc->watchdog_events++; 545 546 ixgbe_init(sc); 547 } 548 549 /********************************************************************* 550 * Init entry point 551 * 552 * This routine is used in two ways. It is used by the stack as 553 * init entry point in network interface structure. It is also used 554 * by the driver as a hw/sw initialization routine to get to a 555 * consistent state. 556 * 557 * return 0 on success, positive on failure 558 **********************************************************************/ 559 #define IXGBE_MHADD_MFS_SHIFT 16 560 561 void 562 ixgbe_init(void *arg) 563 { 564 struct ix_softc *sc = (struct ix_softc *)arg; 565 struct ifnet *ifp = &sc->arpcom.ac_if; 566 struct rx_ring *rxr = sc->rx_rings; 567 uint32_t k, txdctl, rxdctl, rxctrl, mhadd, gpie, itr; 568 int i, s, err; 569 570 INIT_DEBUGOUT("ixgbe_init: begin"); 571 572 s = splnet(); 573 574 ixgbe_stop(sc); 575 576 /* reprogram the RAR[0] in case user changed it. */ 577 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 578 579 /* Get the latest mac address, User can use a LAA */ 580 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 581 IXGBE_ETH_LENGTH_OF_ADDRESS); 582 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1); 583 sc->hw.addr_ctrl.rar_used_count = 1; 584 585 /* Prepare transmit descriptors and buffers */ 586 if (ixgbe_setup_transmit_structures(sc)) { 587 printf("%s: Could not setup transmit structures\n", 588 ifp->if_xname); 589 ixgbe_stop(sc); 590 splx(s); 591 return; 592 } 593 594 ixgbe_init_hw(&sc->hw); 595 ixgbe_initialize_transmit_units(sc); 596 597 /* Determine the correct mbuf pool for doing jumbo frames */ 598 if (sc->max_frame_size <= 2048) 599 sc->rx_mbuf_sz = MCLBYTES; 600 else if (sc->max_frame_size <= 4096) 601 sc->rx_mbuf_sz = 4096; 602 else if (sc->max_frame_size <= 9216) 603 sc->rx_mbuf_sz = 9216; 604 else 605 sc->rx_mbuf_sz = 16 * 1024; 606 607 /* Prepare receive descriptors and buffers */ 608 if (ixgbe_setup_receive_structures(sc)) { 609 printf("%s: Could not setup receive structures\n", 610 ifp->if_xname); 611 ixgbe_stop(sc); 612 splx(s); 613 return; 614 } 615 616 /* Configure RX settings */ 617 ixgbe_initialize_receive_units(sc); 618 619 /* Program promiscuous mode and multicast filters. */ 620 ixgbe_iff(sc); 621 622 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 623 624 /* Enable Fan Failure Interrupt */ 625 gpie |= IXGBE_SDP1_GPIEN; 626 627 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 628 /* Add for Module detection */ 629 gpie |= IXGBE_SDP2_GPIEN; 630 631 /* 632 * Set LL interval to max to reduce the number of low latency 633 * interrupts hitting the card when the ring is getting full. 634 */ 635 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 636 } 637 638 if (sc->hw.mac.type == ixgbe_mac_X540) { 639 /* Thermal Failure Detection */ 640 gpie |= IXGBE_SDP0_GPIEN; 641 642 /* 643 * Set LL interval to max to reduce the number of low latency 644 * interrupts hitting the card when the ring is getting full. 645 */ 646 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 647 } 648 649 if (sc->msix > 1) { 650 /* Enable Enhanced MSIX mode */ 651 gpie |= IXGBE_GPIE_MSIX_MODE; 652 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 653 IXGBE_GPIE_OCD; 654 } 655 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 656 657 /* Set MTU size */ 658 if (ifp->if_mtu > ETHERMTU) { 659 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 660 mhadd &= ~IXGBE_MHADD_MFS_MASK; 661 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 662 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 663 } 664 665 /* Now enable all the queues */ 666 667 for (i = 0; i < sc->num_queues; i++) { 668 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 669 txdctl |= IXGBE_TXDCTL_ENABLE; 670 /* Set WTHRESH to 8, burst writeback */ 671 txdctl |= (8 << 16); 672 /* 673 * When the internal queue falls below PTHRESH (16), 674 * start prefetching as long as there are at least 675 * HTHRESH (1) buffers ready. 676 */ 677 txdctl |= (16 << 0) | (1 << 8); 678 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 679 } 680 681 for (i = 0; i < sc->num_queues; i++) { 682 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 683 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 684 /* 685 * PTHRESH = 21 686 * HTHRESH = 4 687 * WTHRESH = 8 688 */ 689 rxdctl &= ~0x3FFFFF; 690 rxdctl |= 0x080420; 691 } 692 rxdctl |= IXGBE_RXDCTL_ENABLE; 693 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 694 for (k = 0; k < 10; k++) { 695 if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) & 696 IXGBE_RXDCTL_ENABLE) 697 break; 698 else 699 msec_delay(1); 700 } 701 IXGBE_WRITE_FLUSH(&sc->hw); 702 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled); 703 } 704 705 /* Set up VLAN support and filter */ 706 ixgbe_setup_vlan_hw_support(sc); 707 708 /* Enable Receive engine */ 709 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 710 if (sc->hw.mac.type == ixgbe_mac_82598EB) 711 rxctrl |= IXGBE_RXCTRL_DMBYPS; 712 rxctrl |= IXGBE_RXCTRL_RXEN; 713 sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl); 714 715 timeout_add_sec(&sc->timer, 1); 716 717 /* Set up MSI/X routing */ 718 if (sc->msix > 1) { 719 ixgbe_configure_ivars(sc); 720 /* Set up auto-mask */ 721 if (sc->hw.mac.type == ixgbe_mac_82598EB) 722 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 723 else { 724 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 725 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 726 } 727 } else { /* Simple settings for Legacy/MSI */ 728 ixgbe_set_ivar(sc, 0, 0, 0); 729 ixgbe_set_ivar(sc, 0, 0, 1); 730 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 731 } 732 733 /* Check on any SFP devices that need to be kick-started */ 734 if (sc->hw.phy.type == ixgbe_phy_none) { 735 err = sc->hw.phy.ops.identify(&sc->hw); 736 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 737 printf("Unsupported SFP+ module type was detected.\n"); 738 splx(s); 739 return; 740 } 741 } 742 743 /* Setup interrupt moderation */ 744 itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8; 745 if (sc->hw.mac.type != ixgbe_mac_82598EB) 746 itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS; 747 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr); 748 749 /* Config/Enable Link */ 750 ixgbe_config_link(sc); 751 752 /* Hardware Packet Buffer & Flow Control setup */ 753 { 754 uint32_t rxpb, frame, size, tmp; 755 756 frame = sc->max_frame_size; 757 758 /* Calculate High Water */ 759 if (sc->hw.mac.type == ixgbe_mac_X540) 760 tmp = IXGBE_DV_X540(frame, frame); 761 else 762 tmp = IXGBE_DV(frame, frame); 763 size = IXGBE_BT2KB(tmp); 764 rxpb = IXGBE_READ_REG(&sc->hw, IXGBE_RXPBSIZE(0)) >> 10; 765 sc->hw.fc.high_water[0] = rxpb - size; 766 767 /* Now calculate Low Water */ 768 if (sc->hw.mac.type == ixgbe_mac_X540) 769 tmp = IXGBE_LOW_DV_X540(frame); 770 else 771 tmp = IXGBE_LOW_DV(frame); 772 sc->hw.fc.low_water[0] = IXGBE_BT2KB(tmp); 773 774 sc->hw.fc.requested_mode = sc->fc; 775 sc->hw.fc.pause_time = IXGBE_FC_PAUSE; 776 sc->hw.fc.send_xon = TRUE; 777 } 778 /* Initialize the FC settings */ 779 sc->hw.mac.ops.start_hw(&sc->hw); 780 781 /* And now turn on interrupts */ 782 ixgbe_enable_intr(sc); 783 784 /* Now inform the stack we're ready */ 785 ifp->if_flags |= IFF_RUNNING; 786 ifp->if_flags &= ~IFF_OACTIVE; 787 788 splx(s); 789 } 790 791 /* 792 * MSIX Interrupt Handlers 793 */ 794 void 795 ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector) 796 { 797 uint64_t queue = 1ULL << vector; 798 uint32_t mask; 799 800 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 801 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 802 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask); 803 } else { 804 mask = (queue & 0xFFFFFFFF); 805 if (mask) 806 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask); 807 mask = (queue >> 32); 808 if (mask) 809 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask); 810 } 811 } 812 813 void 814 ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector) 815 { 816 uint64_t queue = 1ULL << vector; 817 uint32_t mask; 818 819 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 820 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 821 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask); 822 } else { 823 mask = (queue & 0xFFFFFFFF); 824 if (mask) 825 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask); 826 mask = (queue >> 32); 827 if (mask) 828 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask); 829 } 830 } 831 832 /********************************************************************* 833 * 834 * Legacy Interrupt Service routine 835 * 836 **********************************************************************/ 837 838 int 839 ixgbe_intr(void *arg) 840 { 841 struct ix_softc *sc = (struct ix_softc *)arg; 842 struct ix_queue *que = sc->queues; 843 struct ifnet *ifp = &sc->arpcom.ac_if; 844 struct tx_ring *txr = sc->tx_rings; 845 struct ixgbe_hw *hw = &sc->hw; 846 uint32_t reg_eicr; 847 int i, refill = 0; 848 849 reg_eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR); 850 if (reg_eicr == 0) { 851 ixgbe_enable_intr(sc); 852 return (0); 853 } 854 855 /* Link status change */ 856 if (reg_eicr & IXGBE_EICR_LSC) { 857 timeout_del(&sc->timer); 858 ixgbe_update_link_status(sc); 859 timeout_add_sec(&sc->timer, 1); 860 } 861 862 if (hw->mac.type != ixgbe_mac_82598EB) { 863 if (reg_eicr & IXGBE_EICR_GPI_SDP2) { 864 /* Clear the interrupt */ 865 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 866 ixgbe_handle_mod(sc); 867 } else if ((hw->phy.media_type != ixgbe_media_type_copper) && 868 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 869 /* Clear the interrupt */ 870 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 871 ixgbe_handle_msf(sc); 872 } 873 } 874 875 /* Check for fan failure */ 876 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 877 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 878 printf("%s: CRITICAL: FAN FAILURE!! " 879 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 880 IXGBE_WRITE_REG(&sc->hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 881 } 882 883 /* Check for over temp condition */ 884 if ((hw->mac.type == ixgbe_mac_X540) && 885 (reg_eicr & IXGBE_EICR_TS)) { 886 printf("%s: CRITICAL: OVER TEMP!! " 887 "PHY IS SHUT DOWN!!\n", ifp->if_xname); 888 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS); 889 } 890 891 if (ifp->if_flags & IFF_RUNNING) { 892 ixgbe_rxeof(que); 893 ixgbe_txeof(txr); 894 refill = 1; 895 } 896 897 if (refill) { 898 if (ixgbe_rxfill(que->rxr)) { 899 /* Advance the Rx Queue "Tail Pointer" */ 900 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(que->rxr->me), 901 que->rxr->last_desc_filled); 902 } else 903 timeout_add(&sc->rx_refill, 1); 904 } 905 906 if (ifp->if_flags & IFF_RUNNING && !IFQ_IS_EMPTY(&ifp->if_snd)) 907 ixgbe_start(ifp); 908 909 for (i = 0; i < sc->num_queues; i++, que++) 910 ixgbe_enable_queue(sc, que->msix); 911 912 return (1); 913 } 914 915 /********************************************************************* 916 * 917 * Media Ioctl callback 918 * 919 * This routine is called whenever the user queries the status of 920 * the interface using ifconfig. 921 * 922 **********************************************************************/ 923 void 924 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 925 { 926 struct ix_softc *sc = ifp->if_softc; 927 928 ifmr->ifm_active = IFM_ETHER; 929 ifmr->ifm_status = IFM_AVALID; 930 931 INIT_DEBUGOUT("ixgbe_media_status: begin"); 932 ixgbe_update_link_status(sc); 933 934 if (LINK_STATE_IS_UP(ifp->if_link_state)) { 935 ifmr->ifm_status |= IFM_ACTIVE; 936 937 switch (sc->link_speed) { 938 case IXGBE_LINK_SPEED_100_FULL: 939 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 940 break; 941 case IXGBE_LINK_SPEED_1GB_FULL: 942 ifmr->ifm_active |= ((sc->optics == IFM_1000_SX) ? 943 IFM_1000_SX : IFM_1000_T) | IFM_FDX; 944 break; 945 case IXGBE_LINK_SPEED_10GB_FULL: 946 ifmr->ifm_active |= sc->optics | IFM_FDX; 947 break; 948 } 949 950 switch (sc->hw.fc.current_mode) { 951 case ixgbe_fc_tx_pause: 952 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 953 break; 954 case ixgbe_fc_rx_pause: 955 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 956 break; 957 case ixgbe_fc_full: 958 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE | 959 IFM_ETH_TXPAUSE; 960 break; 961 default: 962 ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE | 963 IFM_ETH_TXPAUSE); 964 break; 965 } 966 } 967 } 968 969 /********************************************************************* 970 * 971 * Media Ioctl callback 972 * 973 * This routine is called when the user changes speed/duplex using 974 * media/mediopt option with ifconfig. 975 * 976 **********************************************************************/ 977 int 978 ixgbe_media_change(struct ifnet *ifp) 979 { 980 struct ix_softc *sc = ifp->if_softc; 981 struct ifmedia *ifm = &sc->media; 982 983 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 984 return (EINVAL); 985 986 switch (IFM_SUBTYPE(ifm->ifm_media)) { 987 case IFM_AUTO: 988 sc->hw.phy.autoneg_advertised = 989 IXGBE_LINK_SPEED_100_FULL | 990 IXGBE_LINK_SPEED_1GB_FULL | 991 IXGBE_LINK_SPEED_10GB_FULL; 992 break; 993 default: 994 return (EINVAL); 995 } 996 997 return (0); 998 } 999 1000 /********************************************************************* 1001 * 1002 * This routine maps the mbufs to tx descriptors, allowing the 1003 * TX engine to transmit the packets. 1004 * - return 0 on success, positive on failure 1005 * 1006 **********************************************************************/ 1007 1008 int 1009 ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head) 1010 { 1011 struct ix_softc *sc = txr->sc; 1012 uint32_t olinfo_status = 0, cmd_type_len; 1013 int i, j, error; 1014 int first, last = 0; 1015 bus_dmamap_t map; 1016 struct ixgbe_tx_buf *txbuf; 1017 union ixgbe_adv_tx_desc *txd = NULL; 1018 1019 /* Basic descriptor defines */ 1020 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1021 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1022 1023 #if NVLAN > 0 1024 if (m_head->m_flags & M_VLANTAG) 1025 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1026 #endif 1027 1028 /* 1029 * Force a cleanup if number of TX descriptors 1030 * available is below the threshold. If it fails 1031 * to get above, then abort transmit. 1032 */ 1033 if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) { 1034 ixgbe_txeof(txr); 1035 /* Make sure things have improved */ 1036 if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) 1037 return (ENOBUFS); 1038 } 1039 1040 /* 1041 * Important to capture the first descriptor 1042 * used because it will contain the index of 1043 * the one we tell the hardware to report back 1044 */ 1045 first = txr->next_avail_desc; 1046 txbuf = &txr->tx_buffers[first]; 1047 map = txbuf->map; 1048 1049 /* 1050 * Map the packet for DMA. 1051 */ 1052 error = bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, m_head, 1053 BUS_DMA_NOWAIT); 1054 switch (error) { 1055 case 0: 1056 break; 1057 case EFBIG: 1058 if (m_defrag(m_head, M_NOWAIT) == 0 && 1059 (error = bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1060 m_head, BUS_DMA_NOWAIT)) == 0) 1061 break; 1062 /* FALLTHROUGH */ 1063 default: 1064 sc->no_tx_dma_setup++; 1065 return (error); 1066 } 1067 1068 /* Make certain there are enough descriptors */ 1069 if (map->dm_nsegs > txr->tx_avail - 2) { 1070 error = ENOBUFS; 1071 goto xmit_fail; 1072 } 1073 1074 /* 1075 * Set the appropriate offload context 1076 * this will becomes the first descriptor. 1077 */ 1078 error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 1079 if (error) 1080 goto xmit_fail; 1081 1082 i = txr->next_avail_desc; 1083 for (j = 0; j < map->dm_nsegs; j++) { 1084 txbuf = &txr->tx_buffers[i]; 1085 txd = &txr->tx_base[i]; 1086 1087 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 1088 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1089 cmd_type_len | map->dm_segs[j].ds_len); 1090 txd->read.olinfo_status = htole32(olinfo_status); 1091 last = i; /* descriptor that will get completion IRQ */ 1092 1093 if (++i == sc->num_tx_desc) 1094 i = 0; 1095 1096 txbuf->m_head = NULL; 1097 txbuf->eop_index = -1; 1098 } 1099 1100 txd->read.cmd_type_len |= 1101 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1102 txr->tx_avail -= map->dm_nsegs; 1103 txr->next_avail_desc = i; 1104 1105 txbuf->m_head = m_head; 1106 /* 1107 * Here we swap the map so the last descriptor, 1108 * which gets the completion interrupt has the 1109 * real map, and the first descriptor gets the 1110 * unused map from this descriptor. 1111 */ 1112 txr->tx_buffers[first].map = txbuf->map; 1113 txbuf->map = map; 1114 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1115 BUS_DMASYNC_PREWRITE); 1116 1117 /* Set the index of the descriptor that will be marked done */ 1118 txbuf = &txr->tx_buffers[first]; 1119 txbuf->eop_index = last; 1120 1121 ++txr->tx_packets; 1122 return (0); 1123 1124 xmit_fail: 1125 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 1126 return (error); 1127 1128 } 1129 1130 void 1131 ixgbe_iff(struct ix_softc *sc) 1132 { 1133 struct ifnet *ifp = &sc->arpcom.ac_if; 1134 struct arpcom *ac = &sc->arpcom; 1135 uint32_t fctrl; 1136 uint8_t *mta; 1137 uint8_t *update_ptr; 1138 struct ether_multi *enm; 1139 struct ether_multistep step; 1140 int mcnt = 0; 1141 1142 IOCTL_DEBUGOUT("ixgbe_iff: begin"); 1143 1144 mta = sc->mta; 1145 bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1146 MAX_NUM_MULTICAST_ADDRESSES); 1147 1148 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1149 fctrl &= ~(IXGBE_FCTRL_MPE | IXGBE_FCTRL_UPE); 1150 ifp->if_flags &= ~IFF_ALLMULTI; 1151 1152 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1153 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1154 ifp->if_flags |= IFF_ALLMULTI; 1155 fctrl |= IXGBE_FCTRL_MPE; 1156 if (ifp->if_flags & IFF_PROMISC) 1157 fctrl |= IXGBE_FCTRL_UPE; 1158 } else { 1159 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1160 while (enm != NULL) { 1161 bcopy(enm->enm_addrlo, 1162 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1163 IXGBE_ETH_LENGTH_OF_ADDRESS); 1164 mcnt++; 1165 1166 ETHER_NEXT_MULTI(step, enm); 1167 } 1168 1169 update_ptr = mta; 1170 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1171 ixgbe_mc_array_itr, TRUE); 1172 } 1173 1174 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1175 } 1176 1177 /* 1178 * This is an iterator function now needed by the multicast 1179 * shared code. It simply feeds the shared code routine the 1180 * addresses in the array of ixgbe_iff() one by one. 1181 */ 1182 uint8_t * 1183 ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1184 { 1185 uint8_t *addr = *update_ptr; 1186 uint8_t *newptr; 1187 *vmdq = 0; 1188 1189 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1190 *update_ptr = newptr; 1191 return addr; 1192 } 1193 1194 1195 /********************************************************************* 1196 * Timer routine 1197 * 1198 * This routine checks for link status,updates statistics, 1199 * and runs the watchdog check. 1200 * 1201 **********************************************************************/ 1202 1203 void 1204 ixgbe_local_timer(void *arg) 1205 { 1206 struct ix_softc *sc = arg; 1207 #ifdef IX_DEBUG 1208 struct ifnet *ifp = &sc->arpcom.ac_if; 1209 #endif 1210 int s; 1211 1212 s = splnet(); 1213 1214 /* Check for pluggable optics */ 1215 if (sc->sfp_probe) 1216 if (!ixgbe_sfp_probe(sc)) 1217 goto out; /* Nothing to do */ 1218 1219 ixgbe_update_link_status(sc); 1220 ixgbe_update_stats_counters(sc); 1221 1222 out: 1223 #ifdef IX_DEBUG 1224 if ((ifp->if_flags & (IFF_RUNNING|IFF_DEBUG)) == 1225 (IFF_RUNNING|IFF_DEBUG)) 1226 ixgbe_print_hw_stats(sc); 1227 #endif 1228 timeout_add_sec(&sc->timer, 1); 1229 1230 splx(s); 1231 } 1232 1233 void 1234 ixgbe_update_link_status(struct ix_softc *sc) 1235 { 1236 struct ifnet *ifp = &sc->arpcom.ac_if; 1237 int link_state = LINK_STATE_DOWN; 1238 1239 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 1240 1241 ifp->if_baudrate = 0; 1242 if (sc->link_up) { 1243 link_state = LINK_STATE_FULL_DUPLEX; 1244 1245 switch (sc->link_speed) { 1246 case IXGBE_LINK_SPEED_UNKNOWN: 1247 ifp->if_baudrate = 0; 1248 break; 1249 case IXGBE_LINK_SPEED_100_FULL: 1250 ifp->if_baudrate = IF_Mbps(100); 1251 break; 1252 case IXGBE_LINK_SPEED_1GB_FULL: 1253 ifp->if_baudrate = IF_Gbps(1); 1254 break; 1255 case IXGBE_LINK_SPEED_10GB_FULL: 1256 ifp->if_baudrate = IF_Gbps(10); 1257 break; 1258 } 1259 1260 /* Update any Flow Control changes */ 1261 sc->hw.mac.ops.fc_enable(&sc->hw); 1262 } 1263 if (ifp->if_link_state != link_state) { 1264 ifp->if_link_state = link_state; 1265 if_link_state_change(ifp); 1266 } 1267 } 1268 1269 1270 /********************************************************************* 1271 * 1272 * This routine disables all traffic on the adapter by issuing a 1273 * global reset on the MAC and deallocates TX/RX buffers. 1274 * 1275 **********************************************************************/ 1276 1277 void 1278 ixgbe_stop(void *arg) 1279 { 1280 struct ix_softc *sc = arg; 1281 struct ifnet *ifp = &sc->arpcom.ac_if; 1282 1283 /* Tell the stack that the interface is no longer active */ 1284 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1285 1286 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1287 ixgbe_disable_intr(sc); 1288 1289 sc->hw.mac.ops.reset_hw(&sc->hw); 1290 sc->hw.adapter_stopped = FALSE; 1291 sc->hw.mac.ops.stop_adapter(&sc->hw); 1292 if (sc->hw.mac.type == ixgbe_mac_82599EB) 1293 sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw); 1294 /* Turn off the laser */ 1295 if (sc->hw.phy.multispeed_fiber) 1296 sc->hw.mac.ops.disable_tx_laser(&sc->hw); 1297 timeout_del(&sc->timer); 1298 timeout_del(&sc->rx_refill); 1299 1300 /* reprogram the RAR[0] in case user changed it. */ 1301 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1302 1303 /* Should we really clear all structures on stop? */ 1304 ixgbe_free_transmit_structures(sc); 1305 ixgbe_free_receive_structures(sc); 1306 } 1307 1308 1309 /********************************************************************* 1310 * 1311 * Determine hardware revision. 1312 * 1313 **********************************************************************/ 1314 void 1315 ixgbe_identify_hardware(struct ix_softc *sc) 1316 { 1317 struct ixgbe_osdep *os = &sc->osdep; 1318 struct pci_attach_args *pa = &os->os_pa; 1319 uint32_t reg; 1320 1321 /* Save off the information about this board */ 1322 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1323 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1324 1325 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1326 sc->hw.revision_id = PCI_REVISION(reg); 1327 1328 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1329 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1330 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1331 1332 switch (sc->hw.device_id) { 1333 case PCI_PRODUCT_INTEL_82598: 1334 case PCI_PRODUCT_INTEL_82598AF_DUAL: 1335 case PCI_PRODUCT_INTEL_82598_DA_DUAL: 1336 case PCI_PRODUCT_INTEL_82598AF: 1337 case PCI_PRODUCT_INTEL_82598_SR_DUAL_EM: 1338 case PCI_PRODUCT_INTEL_82598EB_SFP: 1339 case PCI_PRODUCT_INTEL_82598EB_CX4_DUAL: 1340 case PCI_PRODUCT_INTEL_82598EB_CX4: 1341 case PCI_PRODUCT_INTEL_82598EB_XF_LR: 1342 case PCI_PRODUCT_INTEL_82598AT: 1343 case PCI_PRODUCT_INTEL_82598AT2: 1344 case PCI_PRODUCT_INTEL_82598AT_DUAL: 1345 case PCI_PRODUCT_INTEL_82598_BX: 1346 sc->hw.mac.type = ixgbe_mac_82598EB; 1347 break; 1348 case PCI_PRODUCT_INTEL_82599_SFP: 1349 case PCI_PRODUCT_INTEL_82599_SFP_EM: 1350 case PCI_PRODUCT_INTEL_82599_SFP_FCOE: 1351 case PCI_PRODUCT_INTEL_82599_SFP_SF2: 1352 case PCI_PRODUCT_INTEL_82599_KX4: 1353 case PCI_PRODUCT_INTEL_82599_KX4_MEZZ: 1354 case PCI_PRODUCT_INTEL_82599_CX4: 1355 case PCI_PRODUCT_INTEL_82599_T3_LOM: 1356 case PCI_PRODUCT_INTEL_82599_XAUI: 1357 case PCI_PRODUCT_INTEL_82599_COMBO_BP: 1358 case PCI_PRODUCT_INTEL_82599_BPLANE_FCOE: 1359 sc->hw.mac.type = ixgbe_mac_82599EB; 1360 break; 1361 case PCI_PRODUCT_INTEL_82599VF: 1362 sc->hw.mac.type = ixgbe_mac_82599_vf; 1363 break; 1364 case PCI_PRODUCT_INTEL_X540T: 1365 sc->hw.mac.type = ixgbe_mac_X540; 1366 break; 1367 default: 1368 break; 1369 } 1370 1371 /* Pick up the 82599 and VF settings */ 1372 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 1373 sc->hw.phy.smart_speed = ixgbe_smart_speed; 1374 sc->num_segs = IXGBE_82599_SCATTER; 1375 } else 1376 sc->num_segs = IXGBE_82598_SCATTER; 1377 } 1378 1379 /********************************************************************* 1380 * 1381 * Determine optic type 1382 * 1383 **********************************************************************/ 1384 void 1385 ixgbe_setup_optics(struct ix_softc *sc) 1386 { 1387 struct ixgbe_hw *hw = &sc->hw; 1388 int layer; 1389 1390 layer = hw->mac.ops.get_supported_physical_layer(hw); 1391 1392 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1393 sc->optics = IFM_10G_T; 1394 else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1395 sc->optics = IFM_1000_T; 1396 else if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1397 sc->optics = IFM_100_TX; 1398 else if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) 1399 sc->optics = IFM_10G_SFP_CU; 1400 else if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR | 1401 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) 1402 sc->optics = IFM_10G_LR; 1403 else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) 1404 sc->optics = IFM_10G_SR; 1405 else if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 | 1406 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) 1407 sc->optics = IFM_10G_CX4; 1408 else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 1409 sc->optics = IFM_1000_SX; 1410 } 1411 1412 /********************************************************************* 1413 * 1414 * Setup the Legacy or MSI Interrupt handler 1415 * 1416 **********************************************************************/ 1417 int 1418 ixgbe_allocate_legacy(struct ix_softc *sc) 1419 { 1420 struct ifnet *ifp = &sc->arpcom.ac_if; 1421 struct ixgbe_osdep *os = &sc->osdep; 1422 struct pci_attach_args *pa = &os->os_pa; 1423 const char *intrstr = NULL; 1424 pci_chipset_tag_t pc = pa->pa_pc; 1425 pci_intr_handle_t ih; 1426 1427 /* We allocate a single interrupt resource */ 1428 if (pci_intr_map_msi(pa, &ih) != 0 && 1429 pci_intr_map(pa, &ih) != 0) { 1430 printf(": couldn't map interrupt\n"); 1431 return (ENXIO); 1432 } 1433 1434 #if 0 1435 /* XXX */ 1436 /* Tasklets for Link, SFP and Multispeed Fiber */ 1437 TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc); 1438 TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc); 1439 TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc); 1440 #endif 1441 1442 intrstr = pci_intr_string(pc, ih); 1443 sc->tag = pci_intr_establish(pc, ih, IPL_NET, 1444 ixgbe_intr, sc, ifp->if_xname); 1445 if (sc->tag == NULL) { 1446 printf(": couldn't establish interrupt"); 1447 if (intrstr != NULL) 1448 printf(" at %s", intrstr); 1449 printf("\n"); 1450 return (ENXIO); 1451 } 1452 printf(": %s", intrstr); 1453 1454 /* For simplicity in the handlers */ 1455 sc->que_mask = IXGBE_EIMS_ENABLE_MASK; 1456 1457 return (0); 1458 } 1459 1460 int 1461 ixgbe_allocate_pci_resources(struct ix_softc *sc) 1462 { 1463 struct ixgbe_osdep *os = &sc->osdep; 1464 struct pci_attach_args *pa = &os->os_pa; 1465 int val; 1466 1467 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1468 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM && 1469 PCI_MAPREG_TYPE(val) != PCI_MAPREG_MEM_TYPE_64BIT) { 1470 printf(": mmba is not mem space\n"); 1471 return (ENXIO); 1472 } 1473 1474 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1475 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1476 printf(": cannot find mem space\n"); 1477 return (ENXIO); 1478 } 1479 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1480 1481 /* Legacy defaults */ 1482 sc->num_queues = 1; 1483 sc->hw.back = os; 1484 1485 #ifdef notyet 1486 /* Now setup MSI or MSI/X, return us the number of supported vectors. */ 1487 sc->msix = ixgbe_setup_msix(sc); 1488 #endif 1489 1490 return (0); 1491 } 1492 1493 void 1494 ixgbe_free_pci_resources(struct ix_softc * sc) 1495 { 1496 struct ixgbe_osdep *os = &sc->osdep; 1497 struct pci_attach_args *pa = &os->os_pa; 1498 struct ix_queue *que = sc->queues; 1499 int i; 1500 1501 /* Release all msix queue resources: */ 1502 for (i = 0; i < sc->num_queues; i++, que++) { 1503 if (que->tag) 1504 pci_intr_disestablish(pa->pa_pc, que->tag); 1505 que->tag = NULL; 1506 } 1507 1508 if (sc->tag) 1509 pci_intr_disestablish(pa->pa_pc, sc->tag); 1510 sc->tag = NULL; 1511 if (os->os_membase != 0) 1512 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1513 os->os_membase = 0; 1514 } 1515 1516 /********************************************************************* 1517 * 1518 * Setup networking device structure and register an interface. 1519 * 1520 **********************************************************************/ 1521 void 1522 ixgbe_setup_interface(struct ix_softc *sc) 1523 { 1524 struct ifnet *ifp = &sc->arpcom.ac_if; 1525 1526 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1527 ifp->if_softc = sc; 1528 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1529 ifp->if_ioctl = ixgbe_ioctl; 1530 ifp->if_start = ixgbe_start; 1531 ifp->if_timer = 0; 1532 ifp->if_watchdog = ixgbe_watchdog; 1533 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1534 ETHER_HDR_LEN - ETHER_CRC_LEN; 1535 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1536 IFQ_SET_READY(&ifp->if_snd); 1537 1538 ifp->if_capabilities = IFCAP_VLAN_MTU; 1539 1540 #if NVLAN > 0 1541 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1542 #endif 1543 1544 #ifdef IX_CSUM_OFFLOAD 1545 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 1546 #endif 1547 1548 /* 1549 * Specify the media types supported by this sc and register 1550 * callbacks to update media and link information 1551 */ 1552 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1553 ixgbe_media_status); 1554 if (sc->optics) 1555 ifmedia_add(&sc->media, IFM_ETHER | sc->optics | 1556 IFM_FDX, 0, NULL); 1557 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1558 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1559 1560 if_attach(ifp); 1561 ether_ifattach(ifp); 1562 1563 sc->max_frame_size = 1564 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1565 } 1566 1567 void 1568 ixgbe_config_link(struct ix_softc *sc) 1569 { 1570 uint32_t autoneg, err = 0; 1571 bool sfp, negotiate; 1572 1573 switch (sc->hw.phy.type) { 1574 case ixgbe_phy_sfp_avago: 1575 case ixgbe_phy_sfp_ftl: 1576 case ixgbe_phy_sfp_intel: 1577 case ixgbe_phy_sfp_unknown: 1578 case ixgbe_phy_sfp_passive_tyco: 1579 case ixgbe_phy_sfp_passive_unknown: 1580 sfp = TRUE; 1581 break; 1582 default: 1583 sfp = FALSE; 1584 break; 1585 } 1586 1587 if (sfp) { 1588 if (&sc->hw.phy.multispeed_fiber) { 1589 sc->hw.mac.ops.setup_sfp(&sc->hw); 1590 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 1591 ixgbe_handle_msf(sc); 1592 } else 1593 ixgbe_handle_mod(sc); 1594 } else { 1595 if (sc->hw.mac.ops.check_link) 1596 err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg, 1597 &sc->link_up, FALSE); 1598 if (err) 1599 return; 1600 autoneg = sc->hw.phy.autoneg_advertised; 1601 if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities)) 1602 err = sc->hw.mac.ops.get_link_capabilities(&sc->hw, 1603 &autoneg, &negotiate); 1604 if (err) 1605 return; 1606 if (sc->hw.mac.ops.setup_link) 1607 err = sc->hw.mac.ops.setup_link(&sc->hw, 1608 autoneg, sc->link_up); 1609 } 1610 } 1611 1612 /******************************************************************** 1613 * Manage DMA'able memory. 1614 *******************************************************************/ 1615 int 1616 ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 1617 struct ixgbe_dma_alloc *dma, int mapflags) 1618 { 1619 struct ifnet *ifp = &sc->arpcom.ac_if; 1620 struct ixgbe_osdep *os = &sc->osdep; 1621 int r; 1622 1623 dma->dma_tag = os->os_pa.pa_dmat; 1624 r = bus_dmamap_create(dma->dma_tag, size, 1, 1625 size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 1626 if (r != 0) { 1627 printf("%s: ixgbe_dma_malloc: bus_dma_tag_create failed; " 1628 "error %u\n", ifp->if_xname, r); 1629 goto fail_0; 1630 } 1631 1632 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1633 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 1634 if (r != 0) { 1635 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 1636 "error %u\n", ifp->if_xname, r); 1637 goto fail_1; 1638 } 1639 1640 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1641 &dma->dma_vaddr, BUS_DMA_NOWAIT); 1642 if (r != 0) { 1643 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 1644 "error %u\n", ifp->if_xname, r); 1645 goto fail_2; 1646 } 1647 1648 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1649 size, NULL, mapflags | BUS_DMA_NOWAIT); 1650 if (r != 0) { 1651 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 1652 "error %u\n", ifp->if_xname, r); 1653 goto fail_3; 1654 } 1655 1656 dma->dma_size = size; 1657 return (0); 1658 fail_3: 1659 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1660 fail_2: 1661 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1662 fail_1: 1663 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1664 fail_0: 1665 dma->dma_map = NULL; 1666 dma->dma_tag = NULL; 1667 return (r); 1668 } 1669 1670 void 1671 ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 1672 { 1673 if (dma->dma_tag == NULL) 1674 return; 1675 1676 if (dma->dma_map != NULL) { 1677 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1678 dma->dma_map->dm_mapsize, 1679 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1680 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1681 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1682 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1683 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1684 dma->dma_map = NULL; 1685 } 1686 } 1687 1688 1689 /********************************************************************* 1690 * 1691 * Allocate memory for the transmit and receive rings, and then 1692 * the descriptors associated with each, called only once at attach. 1693 * 1694 **********************************************************************/ 1695 int 1696 ixgbe_allocate_queues(struct ix_softc *sc) 1697 { 1698 struct ifnet *ifp = &sc->arpcom.ac_if; 1699 struct ix_queue *que; 1700 struct tx_ring *txr; 1701 struct rx_ring *rxr; 1702 int rsize, tsize; 1703 int txconf = 0, rxconf = 0, i; 1704 1705 /* First allocate the top level queue structs */ 1706 if (!(sc->queues = mallocarray(sc->num_queues, 1707 sizeof(struct ix_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) { 1708 printf("%s: Unable to allocate queue memory\n", ifp->if_xname); 1709 goto fail; 1710 } 1711 1712 /* Then allocate the TX ring struct memory */ 1713 if (!(sc->tx_rings = mallocarray(sc->num_queues, 1714 sizeof(struct tx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 1715 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 1716 goto fail; 1717 } 1718 1719 /* Next allocate the RX */ 1720 if (!(sc->rx_rings = mallocarray(sc->num_queues, 1721 sizeof(struct rx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 1722 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 1723 goto rx_fail; 1724 } 1725 1726 /* For the ring itself */ 1727 tsize = roundup2(sc->num_tx_desc * 1728 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 1729 1730 /* 1731 * Now set up the TX queues, txconf is needed to handle the 1732 * possibility that things fail midcourse and we need to 1733 * undo memory gracefully 1734 */ 1735 for (i = 0; i < sc->num_queues; i++, txconf++) { 1736 /* Set up some basics */ 1737 txr = &sc->tx_rings[i]; 1738 txr->sc = sc; 1739 txr->me = i; 1740 1741 if (ixgbe_dma_malloc(sc, tsize, 1742 &txr->txdma, BUS_DMA_NOWAIT)) { 1743 printf("%s: Unable to allocate TX Descriptor memory\n", 1744 ifp->if_xname); 1745 goto err_tx_desc; 1746 } 1747 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 1748 bzero((void *)txr->tx_base, tsize); 1749 } 1750 1751 /* 1752 * Next the RX queues... 1753 */ 1754 rsize = roundup2(sc->num_rx_desc * 1755 sizeof(union ixgbe_adv_rx_desc), 4096); 1756 for (i = 0; i < sc->num_queues; i++, rxconf++) { 1757 rxr = &sc->rx_rings[i]; 1758 /* Set up some basics */ 1759 rxr->sc = sc; 1760 rxr->me = i; 1761 1762 if (ixgbe_dma_malloc(sc, rsize, 1763 &rxr->rxdma, BUS_DMA_NOWAIT)) { 1764 printf("%s: Unable to allocate RxDescriptor memory\n", 1765 ifp->if_xname); 1766 goto err_rx_desc; 1767 } 1768 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 1769 bzero((void *)rxr->rx_base, rsize); 1770 } 1771 1772 /* 1773 * Finally set up the queue holding structs 1774 */ 1775 for (i = 0; i < sc->num_queues; i++) { 1776 que = &sc->queues[i]; 1777 que->sc = sc; 1778 que->txr = &sc->tx_rings[i]; 1779 que->rxr = &sc->rx_rings[i]; 1780 } 1781 1782 return (0); 1783 1784 err_rx_desc: 1785 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 1786 ixgbe_dma_free(sc, &rxr->rxdma); 1787 err_tx_desc: 1788 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 1789 ixgbe_dma_free(sc, &txr->txdma); 1790 free(sc->rx_rings, M_DEVBUF, 0); 1791 sc->rx_rings = NULL; 1792 rx_fail: 1793 free(sc->tx_rings, M_DEVBUF, 0); 1794 sc->tx_rings = NULL; 1795 fail: 1796 return (ENOMEM); 1797 } 1798 1799 /********************************************************************* 1800 * 1801 * Allocate memory for tx_buffer structures. The tx_buffer stores all 1802 * the information needed to transmit a packet on the wire. This is 1803 * called only once at attach, setup is done every reset. 1804 * 1805 **********************************************************************/ 1806 int 1807 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 1808 { 1809 struct ix_softc *sc = txr->sc; 1810 struct ifnet *ifp = &sc->arpcom.ac_if; 1811 struct ixgbe_tx_buf *txbuf; 1812 int error, i; 1813 1814 if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc, 1815 sizeof(struct ixgbe_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 1816 printf("%s: Unable to allocate tx_buffer memory\n", 1817 ifp->if_xname); 1818 error = ENOMEM; 1819 goto fail; 1820 } 1821 txr->txtag = txr->txdma.dma_tag; 1822 1823 /* Create the descriptor buffer dma maps */ 1824 for (i = 0; i < sc->num_tx_desc; i++) { 1825 txbuf = &txr->tx_buffers[i]; 1826 error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE, 1827 sc->num_segs, PAGE_SIZE, 0, 1828 BUS_DMA_NOWAIT, &txbuf->map); 1829 1830 if (error != 0) { 1831 printf("%s: Unable to create TX DMA map\n", 1832 ifp->if_xname); 1833 goto fail; 1834 } 1835 } 1836 1837 return 0; 1838 fail: 1839 return (error); 1840 } 1841 1842 /********************************************************************* 1843 * 1844 * Initialize a transmit ring. 1845 * 1846 **********************************************************************/ 1847 int 1848 ixgbe_setup_transmit_ring(struct tx_ring *txr) 1849 { 1850 struct ix_softc *sc = txr->sc; 1851 int error; 1852 1853 /* Now allocate transmit buffers for the ring */ 1854 if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0) 1855 return (error); 1856 1857 /* Clear the old ring contents */ 1858 bzero((void *)txr->tx_base, 1859 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 1860 1861 /* Reset indices */ 1862 txr->next_avail_desc = 0; 1863 txr->next_to_clean = 0; 1864 1865 /* Set number of descriptors available */ 1866 txr->tx_avail = sc->num_tx_desc; 1867 1868 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1869 0, txr->txdma.dma_map->dm_mapsize, 1870 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1871 1872 return (0); 1873 } 1874 1875 /********************************************************************* 1876 * 1877 * Initialize all transmit rings. 1878 * 1879 **********************************************************************/ 1880 int 1881 ixgbe_setup_transmit_structures(struct ix_softc *sc) 1882 { 1883 struct tx_ring *txr = sc->tx_rings; 1884 int i, error; 1885 1886 for (i = 0; i < sc->num_queues; i++, txr++) { 1887 if ((error = ixgbe_setup_transmit_ring(txr)) != 0) 1888 goto fail; 1889 } 1890 1891 return (0); 1892 fail: 1893 ixgbe_free_transmit_structures(sc); 1894 return (error); 1895 } 1896 1897 /********************************************************************* 1898 * 1899 * Enable transmit unit. 1900 * 1901 **********************************************************************/ 1902 void 1903 ixgbe_initialize_transmit_units(struct ix_softc *sc) 1904 { 1905 struct ifnet *ifp = &sc->arpcom.ac_if; 1906 struct tx_ring *txr; 1907 struct ixgbe_hw *hw = &sc->hw; 1908 int i; 1909 uint64_t tdba; 1910 uint32_t txctrl; 1911 1912 /* Setup the Base and Length of the Tx Descriptor Ring */ 1913 1914 for (i = 0; i < sc->num_queues; i++) { 1915 txr = &sc->tx_rings[i]; 1916 1917 /* Setup descriptor base address */ 1918 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 1919 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 1920 (tdba & 0x00000000ffffffffULL)); 1921 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 1922 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 1923 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1924 1925 /* Setup the HW Tx Head and Tail descriptor pointers */ 1926 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 1927 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 1928 1929 /* Setup Transmit Descriptor Cmd Settings */ 1930 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 1931 txr->queue_status = IXGBE_QUEUE_IDLE; 1932 txr->watchdog_timer = 0; 1933 1934 /* Disable Head Writeback */ 1935 switch (hw->mac.type) { 1936 case ixgbe_mac_82598EB: 1937 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 1938 break; 1939 case ixgbe_mac_82599EB: 1940 case ixgbe_mac_X540: 1941 default: 1942 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 1943 break; 1944 } 1945 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1946 switch (hw->mac.type) { 1947 case ixgbe_mac_82598EB: 1948 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 1949 break; 1950 case ixgbe_mac_82599EB: 1951 case ixgbe_mac_X540: 1952 default: 1953 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 1954 break; 1955 } 1956 } 1957 ifp->if_timer = 0; 1958 1959 if (hw->mac.type != ixgbe_mac_82598EB) { 1960 uint32_t dmatxctl, rttdcs; 1961 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1962 dmatxctl |= IXGBE_DMATXCTL_TE; 1963 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 1964 /* Disable arbiter to set MTQC */ 1965 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 1966 rttdcs |= IXGBE_RTTDCS_ARBDIS; 1967 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 1968 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 1969 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 1970 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 1971 } 1972 } 1973 1974 /********************************************************************* 1975 * 1976 * Free all transmit rings. 1977 * 1978 **********************************************************************/ 1979 void 1980 ixgbe_free_transmit_structures(struct ix_softc *sc) 1981 { 1982 struct tx_ring *txr = sc->tx_rings; 1983 int i; 1984 1985 for (i = 0; i < sc->num_queues; i++, txr++) 1986 ixgbe_free_transmit_buffers(txr); 1987 } 1988 1989 /********************************************************************* 1990 * 1991 * Free transmit ring related data structures. 1992 * 1993 **********************************************************************/ 1994 void 1995 ixgbe_free_transmit_buffers(struct tx_ring *txr) 1996 { 1997 struct ix_softc *sc = txr->sc; 1998 struct ixgbe_tx_buf *tx_buffer; 1999 int i; 2000 2001 INIT_DEBUGOUT("free_transmit_ring: begin"); 2002 2003 if (txr->tx_buffers == NULL) 2004 return; 2005 2006 tx_buffer = txr->tx_buffers; 2007 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 2008 if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) { 2009 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2010 0, tx_buffer->map->dm_mapsize, 2011 BUS_DMASYNC_POSTWRITE); 2012 bus_dmamap_unload(txr->txdma.dma_tag, 2013 tx_buffer->map); 2014 } 2015 if (tx_buffer->m_head != NULL) { 2016 m_freem(tx_buffer->m_head); 2017 tx_buffer->m_head = NULL; 2018 } 2019 if (tx_buffer->map != NULL) { 2020 bus_dmamap_destroy(txr->txdma.dma_tag, 2021 tx_buffer->map); 2022 tx_buffer->map = NULL; 2023 } 2024 } 2025 2026 if (txr->tx_buffers != NULL) 2027 free(txr->tx_buffers, M_DEVBUF, 0); 2028 txr->tx_buffers = NULL; 2029 txr->txtag = NULL; 2030 } 2031 2032 /********************************************************************* 2033 * 2034 * Advanced Context Descriptor setup for VLAN or CSUM 2035 * 2036 **********************************************************************/ 2037 2038 int 2039 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, 2040 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2041 { 2042 struct ix_softc *sc = txr->sc; 2043 struct ixgbe_adv_tx_context_desc *TXD; 2044 struct ixgbe_tx_buf *tx_buffer; 2045 #if NVLAN > 0 2046 struct ether_vlan_header *eh; 2047 #else 2048 struct ether_header *eh; 2049 #endif 2050 struct ip *ip; 2051 #ifdef notyet 2052 struct ip6_hdr *ip6; 2053 #endif 2054 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2055 int ehdrlen, ip_hlen = 0; 2056 uint16_t etype; 2057 uint8_t ipproto = 0; 2058 int offload = TRUE; 2059 int ctxd = txr->next_avail_desc; 2060 #if NVLAN > 0 2061 uint16_t vtag = 0; 2062 #endif 2063 2064 #if notyet 2065 /* First check if TSO is to be used */ 2066 if (mp->m_pkthdr.csum_flags & CSUM_TSO) 2067 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status)); 2068 #endif 2069 2070 if ((mp->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT)) == 0) 2071 offload = FALSE; 2072 2073 /* Indicate the whole packet as payload when not doing TSO */ 2074 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2075 2076 /* Now ready a context descriptor */ 2077 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2078 tx_buffer = &txr->tx_buffers[ctxd]; 2079 2080 /* 2081 * In advanced descriptors the vlan tag must 2082 * be placed into the descriptor itself. Hence 2083 * we need to make one even if not doing offloads. 2084 */ 2085 #if NVLAN > 0 2086 if (mp->m_flags & M_VLANTAG) { 2087 vtag = htole16(mp->m_pkthdr.ether_vtag); 2088 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2089 } else 2090 #endif 2091 if (offload == FALSE) 2092 return (0); /* No need for CTX */ 2093 2094 /* 2095 * Determine where frame payload starts. 2096 * Jump over vlan headers if already present, 2097 * helpful for QinQ too. 2098 */ 2099 #if NVLAN > 0 2100 eh = mtod(mp, struct ether_vlan_header *); 2101 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2102 etype = ntohs(eh->evl_proto); 2103 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2104 } else { 2105 etype = ntohs(eh->evl_encap_proto); 2106 ehdrlen = ETHER_HDR_LEN; 2107 } 2108 #else 2109 eh = mtod(mp, struct ether_header *); 2110 etype = ntohs(eh->ether_type); 2111 ehdrlen = ETHER_HDR_LEN; 2112 #endif 2113 2114 /* Set the ether header length */ 2115 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2116 2117 switch (etype) { 2118 case ETHERTYPE_IP: 2119 ip = (struct ip *)(mp->m_data + ehdrlen); 2120 ip_hlen = ip->ip_hl << 2; 2121 ipproto = ip->ip_p; 2122 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2123 break; 2124 #ifdef notyet 2125 case ETHERTYPE_IPV6: 2126 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2127 ip_hlen = sizeof(struct ip6_hdr); 2128 /* XXX-BZ this will go badly in case of ext hdrs. */ 2129 ipproto = ip6->ip6_nxt; 2130 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2131 break; 2132 #endif 2133 default: 2134 offload = FALSE; 2135 break; 2136 } 2137 2138 vlan_macip_lens |= ip_hlen; 2139 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2140 2141 switch (ipproto) { 2142 case IPPROTO_TCP: 2143 if (mp->m_pkthdr.csum_flags & M_TCP_CSUM_OUT) 2144 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2145 break; 2146 case IPPROTO_UDP: 2147 if (mp->m_pkthdr.csum_flags & M_UDP_CSUM_OUT) 2148 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2149 break; 2150 default: 2151 offload = FALSE; 2152 break; 2153 } 2154 2155 if (offload) /* For the TX descriptor setup */ 2156 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2157 2158 /* Now copy bits into descriptor */ 2159 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2160 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2161 TXD->seqnum_seed = htole32(0); 2162 TXD->mss_l4len_idx = htole32(0); 2163 2164 tx_buffer->m_head = NULL; 2165 tx_buffer->eop_index = -1; 2166 2167 /* We've consumed the first desc, adjust counters */ 2168 if (++ctxd == sc->num_tx_desc) 2169 ctxd = 0; 2170 txr->next_avail_desc = ctxd; 2171 --txr->tx_avail; 2172 2173 return (0); 2174 } 2175 2176 #ifdef notyet 2177 /********************************************************************** 2178 * 2179 * Setup work for hardware segmentation offload (TSO) on 2180 * adapters using advanced tx descriptors 2181 * 2182 **********************************************************************/ 2183 int 2184 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, 2185 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2186 { 2187 struct ix_softc *sc = txr->sc; 2188 struct ixgbe_adv_tx_context_desc *TXD; 2189 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2190 uint32_t mss_l4len_idx = 0, paylen; 2191 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 2192 uint16_t etype; 2193 #if NVLAN > 0 2194 uint16_t vtag = 0; 2195 struct ether_vlan_header *eh; 2196 #else 2197 struct ether_header *eh; 2198 #endif 2199 struct ip *ip; 2200 struct ip6_hdr *ip6; 2201 struct tcphdr *th; 2202 2203 /* 2204 * Determine where frame payload starts. 2205 * Jump over vlan headers if already present 2206 */ 2207 #if NVLAN > 0 2208 eh = mtod(mp, struct ether_vlan_header *); 2209 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2210 etype = ntohs(eh->evl_proto); 2211 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2212 } else { 2213 etype = ntohs(eh->evl_encap_proto); 2214 ehdrlen = ETHER_HDR_LEN; 2215 } 2216 #else 2217 eh = mtod(mp, struct ether_header *); 2218 etype = ntohs(eh->ether_type); 2219 ehdrlen = ETHER_HDR_LEN; 2220 #endif 2221 2222 switch (etype) { 2223 case ETHERTYPE_IPV6: 2224 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2225 /* XXX-BZ For now we do not pretend to support ext. hdrs. */ 2226 if (ip6->ip6_nxt != IPPROTO_TCP) 2227 return (ENXIO); 2228 ip_hlen = sizeof(struct ip6_hdr); 2229 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2230 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 2231 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 2232 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2233 break; 2234 case ETHERTYPE_IP: 2235 ip = (struct ip *)(mp->m_data + ehdrlen); 2236 if (ip->ip_p != IPPROTO_TCP) 2237 return (ENXIO); 2238 ip->ip_sum = 0; 2239 ip_hlen = ip->ip_hl << 2; 2240 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2241 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2242 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2243 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2244 /* Tell transmit desc to also do IPv4 checksum. */ 2245 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2246 break; 2247 default: 2248 panic("%s: CSUM_TSO but no supported IP version (0x%04x)", 2249 __func__, ntohs(etype)); 2250 break; 2251 } 2252 2253 ctxd = txr->next_avail_desc; 2254 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2255 2256 tcp_hlen = th->th_off << 2; 2257 2258 /* This is used in the transmit desc in encap */ 2259 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; 2260 2261 #if NVLAN > 0 2262 /* VLAN MACLEN IPLEN */ 2263 if (mp->m_flags & M_VLANTAG) { 2264 vtag = htole16(mp->m_pkthdr.ether_vtag); 2265 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2266 } 2267 #endif 2268 2269 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2270 vlan_macip_lens |= ip_hlen; 2271 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2272 2273 /* ADV DTYPE TUCMD */ 2274 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2275 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2276 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2277 2278 /* MSS L4LEN IDX */ 2279 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2280 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2281 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2282 2283 TXD->seqnum_seed = htole32(0); 2284 2285 if (++ctxd == sc->num_tx_desc) 2286 ctxd = 0; 2287 2288 txr->tx_avail--; 2289 txr->next_avail_desc = ctxd; 2290 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2291 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2292 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 2293 return TRUE; 2294 } 2295 #endif 2296 2297 /********************************************************************** 2298 * 2299 * Examine each tx_buffer in the used queue. If the hardware is done 2300 * processing the packet then free associated resources. The 2301 * tx_buffer is put back on the free queue. 2302 * 2303 **********************************************************************/ 2304 int 2305 ixgbe_txeof(struct tx_ring *txr) 2306 { 2307 struct ix_softc *sc = txr->sc; 2308 struct ifnet *ifp = &sc->arpcom.ac_if; 2309 uint32_t first, last, done, processed; 2310 struct ixgbe_tx_buf *tx_buffer; 2311 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2312 2313 if (txr->tx_avail == sc->num_tx_desc) { 2314 txr->queue_status = IXGBE_QUEUE_IDLE; 2315 return FALSE; 2316 } 2317 2318 processed = 0; 2319 first = txr->next_to_clean; 2320 tx_buffer = &txr->tx_buffers[first]; 2321 /* For cleanup we just use legacy struct */ 2322 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2323 last = tx_buffer->eop_index; 2324 if (last == -1) 2325 return FALSE; 2326 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2327 2328 /* 2329 * Get the index of the first descriptor 2330 * BEYOND the EOP and call that 'done'. 2331 * I do this so the comparison in the 2332 * inner while loop below can be simple 2333 */ 2334 if (++last == sc->num_tx_desc) last = 0; 2335 done = last; 2336 2337 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2338 0, txr->txdma.dma_map->dm_mapsize, 2339 BUS_DMASYNC_POSTREAD); 2340 2341 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2342 /* We clean the range of the packet */ 2343 while (first != done) { 2344 tx_desc->upper.data = 0; 2345 tx_desc->lower.data = 0; 2346 tx_desc->buffer_addr = 0; 2347 ++txr->tx_avail; 2348 ++processed; 2349 2350 if (tx_buffer->m_head) { 2351 bus_dmamap_sync(txr->txdma.dma_tag, 2352 tx_buffer->map, 2353 0, tx_buffer->map->dm_mapsize, 2354 BUS_DMASYNC_POSTWRITE); 2355 bus_dmamap_unload(txr->txdma.dma_tag, 2356 tx_buffer->map); 2357 m_freem(tx_buffer->m_head); 2358 tx_buffer->m_head = NULL; 2359 } 2360 tx_buffer->eop_index = -1; 2361 2362 if (++first == sc->num_tx_desc) 2363 first = 0; 2364 2365 tx_buffer = &txr->tx_buffers[first]; 2366 tx_desc = (struct ixgbe_legacy_tx_desc *) 2367 &txr->tx_base[first]; 2368 } 2369 ++txr->packets; 2370 ++ifp->if_opackets; 2371 /* See if there is more work now */ 2372 last = tx_buffer->eop_index; 2373 if (last != -1) { 2374 eop_desc = 2375 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2376 /* Get next done point */ 2377 if (++last == sc->num_tx_desc) last = 0; 2378 done = last; 2379 } else 2380 break; 2381 } 2382 2383 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2384 0, txr->txdma.dma_map->dm_mapsize, 2385 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2386 2387 txr->next_to_clean = first; 2388 2389 /* 2390 * If we have enough room, clear IFF_OACTIVE to tell the stack that 2391 * it is OK to send packets. If there are no pending descriptors, 2392 * clear the timeout. Otherwise, if some descriptors have been freed, 2393 * restart the timeout. 2394 */ 2395 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) { 2396 ifp->if_flags &= ~IFF_OACTIVE; 2397 2398 /* If all are clean turn off the timer */ 2399 if (txr->tx_avail == sc->num_tx_desc) { 2400 ifp->if_timer = 0; 2401 txr->watchdog_timer = 0; 2402 return FALSE; 2403 } 2404 /* Some were cleaned, so reset timer */ 2405 else if (processed) { 2406 ifp->if_timer = IXGBE_TX_TIMEOUT; 2407 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 2408 } 2409 } 2410 2411 return TRUE; 2412 } 2413 2414 /********************************************************************* 2415 * 2416 * Get a buffer from system mbuf buffer pool. 2417 * 2418 **********************************************************************/ 2419 int 2420 ixgbe_get_buf(struct rx_ring *rxr, int i) 2421 { 2422 struct ix_softc *sc = rxr->sc; 2423 struct ixgbe_rx_buf *rxbuf; 2424 struct mbuf *mp; 2425 int error; 2426 union ixgbe_adv_rx_desc *rxdesc; 2427 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 2428 2429 rxbuf = &rxr->rx_buffers[i]; 2430 rxdesc = &rxr->rx_base[i]; 2431 if (rxbuf->buf) { 2432 printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n", 2433 sc->dev.dv_xname, i); 2434 return (ENOBUFS); 2435 } 2436 2437 /* needed in any case so prealocate since this one will fail for sure */ 2438 mp = MCLGETI(NULL, M_DONTWAIT, NULL, sc->rx_mbuf_sz); 2439 if (!mp) 2440 return (ENOBUFS); 2441 2442 mp->m_len = mp->m_pkthdr.len = sc->rx_mbuf_sz; 2443 if (sc->max_frame_size <= (sc->rx_mbuf_sz - ETHER_ALIGN)) 2444 m_adj(mp, ETHER_ALIGN); 2445 2446 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, 2447 mp, BUS_DMA_NOWAIT); 2448 if (error) { 2449 m_freem(mp); 2450 return (error); 2451 } 2452 2453 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2454 0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 2455 rxbuf->buf = mp; 2456 2457 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2458 dsize * i, dsize, BUS_DMASYNC_POSTWRITE); 2459 2460 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 2461 2462 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2463 dsize * i, dsize, BUS_DMASYNC_PREWRITE); 2464 2465 return (0); 2466 } 2467 2468 /********************************************************************* 2469 * 2470 * Allocate memory for rx_buffer structures. Since we use one 2471 * rx_buffer per received packet, the maximum number of rx_buffer's 2472 * that we'll need is equal to the number of receive descriptors 2473 * that we've allocated. 2474 * 2475 **********************************************************************/ 2476 int 2477 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2478 { 2479 struct ix_softc *sc = rxr->sc; 2480 struct ifnet *ifp = &sc->arpcom.ac_if; 2481 struct ixgbe_rx_buf *rxbuf; 2482 int i, bsize, error; 2483 2484 bsize = sizeof(struct ixgbe_rx_buf) * sc->num_rx_desc; 2485 if (!(rxr->rx_buffers = (struct ixgbe_rx_buf *) malloc(bsize, 2486 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2487 printf("%s: Unable to allocate rx_buffer memory\n", 2488 ifp->if_xname); 2489 error = ENOMEM; 2490 goto fail; 2491 } 2492 2493 rxbuf = rxr->rx_buffers; 2494 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2495 error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1, 2496 16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map); 2497 if (error) { 2498 printf("%s: Unable to create Pack DMA map\n", 2499 ifp->if_xname); 2500 goto fail; 2501 } 2502 } 2503 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2504 rxr->rxdma.dma_map->dm_mapsize, 2505 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2506 2507 return (0); 2508 2509 fail: 2510 return (error); 2511 } 2512 2513 /********************************************************************* 2514 * 2515 * Initialize a receive ring and its buffers. 2516 * 2517 **********************************************************************/ 2518 int 2519 ixgbe_setup_receive_ring(struct rx_ring *rxr) 2520 { 2521 struct ix_softc *sc = rxr->sc; 2522 struct ifnet *ifp = &sc->arpcom.ac_if; 2523 int rsize, error; 2524 2525 rsize = roundup2(sc->num_rx_desc * 2526 sizeof(union ixgbe_adv_rx_desc), 4096); 2527 /* Clear the ring contents */ 2528 bzero((void *)rxr->rx_base, rsize); 2529 2530 if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0) 2531 return (error); 2532 2533 /* Setup our descriptor indices */ 2534 rxr->next_to_check = 0; 2535 rxr->last_desc_filled = sc->num_rx_desc - 1; 2536 2537 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2538 sc->num_rx_desc); 2539 2540 ixgbe_rxfill(rxr); 2541 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 2542 printf("%s: unable to fill any rx descriptors\n", 2543 sc->dev.dv_xname); 2544 return (ENOBUFS); 2545 } 2546 2547 return (0); 2548 } 2549 2550 int 2551 ixgbe_rxfill(struct rx_ring *rxr) 2552 { 2553 struct ix_softc *sc = rxr->sc; 2554 int post = 0; 2555 u_int slots; 2556 int i; 2557 2558 i = rxr->last_desc_filled; 2559 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); 2560 slots > 0; slots--) { 2561 if (++i == sc->num_rx_desc) 2562 i = 0; 2563 2564 if (ixgbe_get_buf(rxr, i) != 0) 2565 break; 2566 2567 post = 1; 2568 } 2569 if_rxr_put(&rxr->rx_ring, slots); 2570 2571 rxr->last_desc_filled = i; 2572 2573 return (post); 2574 } 2575 2576 void 2577 ixgbe_rxrefill(void *xsc) 2578 { 2579 struct ix_softc *sc = xsc; 2580 struct ix_queue *que = sc->queues; 2581 int s; 2582 2583 s = splnet(); 2584 if (ixgbe_rxfill(que->rxr)) { 2585 /* Advance the Rx Queue "Tail Pointer" */ 2586 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(que->rxr->me), 2587 que->rxr->last_desc_filled); 2588 } else 2589 timeout_add(&sc->rx_refill, 1); 2590 splx(s); 2591 } 2592 2593 /********************************************************************* 2594 * 2595 * Initialize all receive rings. 2596 * 2597 **********************************************************************/ 2598 int 2599 ixgbe_setup_receive_structures(struct ix_softc *sc) 2600 { 2601 struct rx_ring *rxr = sc->rx_rings; 2602 int i; 2603 2604 for (i = 0; i < sc->num_queues; i++, rxr++) 2605 if (ixgbe_setup_receive_ring(rxr)) 2606 goto fail; 2607 2608 return (0); 2609 fail: 2610 ixgbe_free_receive_structures(sc); 2611 return (ENOBUFS); 2612 } 2613 2614 /********************************************************************* 2615 * 2616 * Setup receive registers and features. 2617 * 2618 **********************************************************************/ 2619 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2620 2621 void 2622 ixgbe_initialize_receive_units(struct ix_softc *sc) 2623 { 2624 struct rx_ring *rxr = sc->rx_rings; 2625 struct ifnet *ifp = &sc->arpcom.ac_if; 2626 uint32_t bufsz, rxctrl, fctrl, srrctl, rxcsum; 2627 uint32_t reta, mrqc = 0, hlreg; 2628 uint32_t random[10]; 2629 int i; 2630 2631 /* 2632 * Make sure receives are disabled while 2633 * setting up the descriptor ring 2634 */ 2635 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 2636 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXCTRL, 2637 rxctrl & ~IXGBE_RXCTRL_RXEN); 2638 2639 /* Enable broadcasts */ 2640 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 2641 fctrl |= IXGBE_FCTRL_BAM; 2642 fctrl |= IXGBE_FCTRL_DPF; 2643 fctrl |= IXGBE_FCTRL_PMCF; 2644 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 2645 2646 /* Set for Jumbo Frames? */ 2647 hlreg = IXGBE_READ_REG(&sc->hw, IXGBE_HLREG0); 2648 if (ifp->if_mtu > ETHERMTU) 2649 hlreg |= IXGBE_HLREG0_JUMBOEN; 2650 else 2651 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 2652 IXGBE_WRITE_REG(&sc->hw, IXGBE_HLREG0, hlreg); 2653 2654 bufsz = sc->rx_mbuf_sz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2655 2656 for (i = 0; i < sc->num_queues; i++, rxr++) { 2657 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2658 2659 /* Setup the Base and Length of the Rx Descriptor Ring */ 2660 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDBAL(i), 2661 (rdba & 0x00000000ffffffffULL)); 2662 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDBAH(i), (rdba >> 32)); 2663 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDLEN(i), 2664 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2665 2666 /* Set up the SRRCTL register */ 2667 srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2668 IXGBE_WRITE_REG(&sc->hw, IXGBE_SRRCTL(i), srrctl); 2669 2670 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2671 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDH(i), 0); 2672 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), 0); 2673 } 2674 2675 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 2676 uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR | 2677 IXGBE_PSRTYPE_UDPHDR | 2678 IXGBE_PSRTYPE_IPV4HDR | 2679 IXGBE_PSRTYPE_IPV6HDR; 2680 IXGBE_WRITE_REG(&sc->hw, IXGBE_PSRTYPE(0), psrtype); 2681 } 2682 2683 rxcsum = IXGBE_READ_REG(&sc->hw, IXGBE_RXCSUM); 2684 rxcsum &= ~IXGBE_RXCSUM_PCSD; 2685 2686 /* Setup RSS */ 2687 if (sc->num_queues > 1) { 2688 int j; 2689 reta = 0; 2690 /* set up random bits */ 2691 arc4random_buf(&random, sizeof(random)); 2692 2693 /* Set up the redirection table */ 2694 for (i = 0, j = 0; i < 128; i++, j++) { 2695 if (j == sc->num_queues) 2696 j = 0; 2697 reta = (reta << 8) | (j * 0x11); 2698 if ((i & 3) == 3) 2699 IXGBE_WRITE_REG(&sc->hw, IXGBE_RETA(i >> 2), reta); 2700 } 2701 2702 /* Now fill our hash function seeds */ 2703 for (i = 0; i < 10; i++) 2704 IXGBE_WRITE_REG(&sc->hw, IXGBE_RSSRK(i), random[i]); 2705 2706 /* Perform hash on these packet types */ 2707 mrqc = IXGBE_MRQC_RSSEN 2708 | IXGBE_MRQC_RSS_FIELD_IPV4 2709 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 2710 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 2711 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 2712 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 2713 | IXGBE_MRQC_RSS_FIELD_IPV6 2714 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 2715 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 2716 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2717 IXGBE_WRITE_REG(&sc->hw, IXGBE_MRQC, mrqc); 2718 2719 /* RSS and RX IPP Checksum are mutually exclusive */ 2720 rxcsum |= IXGBE_RXCSUM_PCSD; 2721 } 2722 2723 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2724 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2725 2726 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXCSUM, rxcsum); 2727 } 2728 2729 /********************************************************************* 2730 * 2731 * Free all receive rings. 2732 * 2733 **********************************************************************/ 2734 void 2735 ixgbe_free_receive_structures(struct ix_softc *sc) 2736 { 2737 struct rx_ring *rxr = sc->rx_rings; 2738 int i; 2739 2740 for (i = 0; i < sc->num_queues; i++, rxr++) 2741 ixgbe_free_receive_buffers(rxr); 2742 } 2743 2744 /********************************************************************* 2745 * 2746 * Free receive ring data structures 2747 * 2748 **********************************************************************/ 2749 void 2750 ixgbe_free_receive_buffers(struct rx_ring *rxr) 2751 { 2752 struct ix_softc *sc; 2753 struct ixgbe_rx_buf *rxbuf; 2754 int i; 2755 2756 sc = rxr->sc; 2757 if (rxr->rx_buffers != NULL) { 2758 for (i = 0; i < sc->num_rx_desc; i++) { 2759 rxbuf = &rxr->rx_buffers[i]; 2760 if (rxbuf->buf != NULL) { 2761 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2762 0, rxbuf->map->dm_mapsize, 2763 BUS_DMASYNC_POSTREAD); 2764 bus_dmamap_unload(rxr->rxdma.dma_tag, 2765 rxbuf->map); 2766 m_freem(rxbuf->buf); 2767 rxbuf->buf = NULL; 2768 } 2769 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 2770 rxbuf->map = NULL; 2771 } 2772 free(rxr->rx_buffers, M_DEVBUF, 0); 2773 rxr->rx_buffers = NULL; 2774 } 2775 } 2776 2777 /********************************************************************* 2778 * 2779 * This routine executes in interrupt context. It replenishes 2780 * the mbufs in the descriptor and sends data which has been 2781 * dma'ed into host memory to upper layer. 2782 * 2783 *********************************************************************/ 2784 int 2785 ixgbe_rxeof(struct ix_queue *que) 2786 { 2787 struct ix_softc *sc = que->sc; 2788 struct rx_ring *rxr = que->rxr; 2789 struct ifnet *ifp = &sc->arpcom.ac_if; 2790 struct mbuf *mp, *sendmp; 2791 uint8_t eop = 0; 2792 uint16_t len, vtag; 2793 uint32_t staterr = 0, ptype; 2794 struct ixgbe_rx_buf *rxbuf, *nxbuf; 2795 union ixgbe_adv_rx_desc *rxdesc; 2796 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 2797 int i, nextp; 2798 2799 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2800 return FALSE; 2801 2802 i = rxr->next_to_check; 2803 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 2804 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2805 dsize * i, dsize, BUS_DMASYNC_POSTREAD); 2806 2807 rxdesc = &rxr->rx_base[i]; 2808 staterr = letoh32(rxdesc->wb.upper.status_error); 2809 if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) { 2810 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2811 dsize * i, dsize, 2812 BUS_DMASYNC_PREREAD); 2813 break; 2814 } 2815 2816 /* Zero out the receive descriptors status */ 2817 rxdesc->wb.upper.status_error = 0; 2818 rxbuf = &rxr->rx_buffers[i]; 2819 2820 /* pull the mbuf off the ring */ 2821 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 2822 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2823 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 2824 2825 mp = rxbuf->buf; 2826 len = letoh16(rxdesc->wb.upper.length); 2827 ptype = letoh32(rxdesc->wb.lower.lo_dword.data) & 2828 IXGBE_RXDADV_PKTTYPE_MASK; 2829 vtag = letoh16(rxdesc->wb.upper.vlan); 2830 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 2831 2832 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) { 2833 ifp->if_ierrors++; 2834 sc->dropped_pkts++; 2835 2836 if (rxbuf->fmp) { 2837 m_freem(rxbuf->fmp); 2838 rxbuf->fmp = NULL; 2839 } 2840 2841 m_freem(mp); 2842 rxbuf->buf = NULL; 2843 goto next_desc; 2844 } 2845 2846 if (mp == NULL) { 2847 panic("%s: ixgbe_rxeof: NULL mbuf in slot %d " 2848 "(nrx %d, filled %d)", sc->dev.dv_xname, 2849 i, if_rxr_inuse(&rxr->rx_ring), 2850 rxr->last_desc_filled); 2851 } 2852 2853 /* XXX ixgbe_realign() STRICT_ALIGN */ 2854 /* Currently no HW RSC support of 82599 */ 2855 if (!eop) { 2856 /* 2857 * Figure out the next descriptor of this frame. 2858 */ 2859 nextp = i + 1; 2860 if (nextp == sc->num_rx_desc) 2861 nextp = 0; 2862 nxbuf = &rxr->rx_buffers[nextp]; 2863 /* prefetch(nxbuf); */ 2864 } 2865 2866 /* 2867 * Rather than using the fmp/lmp global pointers 2868 * we now keep the head of a packet chain in the 2869 * buffer struct and pass this along from one 2870 * descriptor to the next, until we get EOP. 2871 */ 2872 mp->m_len = len; 2873 /* 2874 * See if there is a stored head 2875 * that determines what we are 2876 */ 2877 sendmp = rxbuf->fmp; 2878 rxbuf->buf = rxbuf->fmp = NULL; 2879 2880 if (sendmp != NULL) /* secondary frag */ 2881 sendmp->m_pkthdr.len += mp->m_len; 2882 else { 2883 /* first desc of a non-ps chain */ 2884 sendmp = mp; 2885 sendmp->m_pkthdr.len = mp->m_len; 2886 #if NVLAN > 0 2887 if (staterr & IXGBE_RXD_STAT_VP) { 2888 sendmp->m_pkthdr.ether_vtag = vtag; 2889 sendmp->m_flags |= M_VLANTAG; 2890 } 2891 #endif 2892 } 2893 2894 /* Pass the head pointer on */ 2895 if (eop == 0) { 2896 nxbuf->fmp = sendmp; 2897 sendmp = NULL; 2898 mp->m_next = nxbuf->buf; 2899 } else { /* Sending this frame? */ 2900 sendmp->m_pkthdr.rcvif = ifp; 2901 ifp->if_ipackets++; 2902 rxr->rx_packets++; 2903 /* capture data for AIM */ 2904 rxr->bytes += sendmp->m_pkthdr.len; 2905 rxr->rx_bytes += sendmp->m_pkthdr.len; 2906 2907 ixgbe_rx_checksum(staterr, sendmp, ptype); 2908 2909 #if NBPFILTER > 0 2910 if (ifp->if_bpf) 2911 bpf_mtap_ether(ifp->if_bpf, sendmp, 2912 BPF_DIRECTION_IN); 2913 #endif 2914 2915 ether_input_mbuf(ifp, sendmp); 2916 } 2917 next_desc: 2918 if_rxr_put(&rxr->rx_ring, 1); 2919 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2920 dsize * i, dsize, 2921 BUS_DMASYNC_PREREAD); 2922 2923 /* Advance our pointers to the next descriptor. */ 2924 if (++i == sc->num_rx_desc) 2925 i = 0; 2926 } 2927 rxr->next_to_check = i; 2928 2929 if (!(staterr & IXGBE_RXD_STAT_DD)) 2930 return FALSE; 2931 2932 return TRUE; 2933 } 2934 2935 /********************************************************************* 2936 * 2937 * Verify that the hardware indicated that the checksum is valid. 2938 * Inform the stack about the status of checksum so that stack 2939 * doesn't spend time verifying the checksum. 2940 * 2941 *********************************************************************/ 2942 void 2943 ixgbe_rx_checksum(uint32_t staterr, struct mbuf * mp, uint32_t ptype) 2944 { 2945 uint16_t status = (uint16_t) staterr; 2946 uint8_t errors = (uint8_t) (staterr >> 24); 2947 2948 if (status & IXGBE_RXD_STAT_IPCS) { 2949 if (!(errors & IXGBE_RXD_ERR_IPE)) { 2950 /* IP Checksum Good */ 2951 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 2952 } else 2953 mp->m_pkthdr.csum_flags = 0; 2954 } 2955 if (status & IXGBE_RXD_STAT_L4CS) { 2956 if (!(errors & IXGBE_RXD_ERR_TCPE)) 2957 mp->m_pkthdr.csum_flags |= 2958 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 2959 } 2960 } 2961 2962 void 2963 ixgbe_setup_vlan_hw_support(struct ix_softc *sc) 2964 { 2965 uint32_t ctrl; 2966 int i; 2967 2968 /* 2969 * A soft reset zero's out the VFTA, so 2970 * we need to repopulate it now. 2971 */ 2972 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 2973 if (sc->shadow_vfta[i] != 0) 2974 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i), 2975 sc->shadow_vfta[i]); 2976 } 2977 2978 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 2979 #if 0 2980 /* Enable the Filter Table if enabled */ 2981 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 2982 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 2983 ctrl |= IXGBE_VLNCTRL_VFE; 2984 } 2985 #endif 2986 if (sc->hw.mac.type == ixgbe_mac_82598EB) 2987 ctrl |= IXGBE_VLNCTRL_VME; 2988 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 2989 2990 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 2991 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 2992 for (i = 0; i < sc->num_queues; i++) { 2993 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 2994 ctrl |= IXGBE_RXDCTL_VME; 2995 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl); 2996 } 2997 } 2998 } 2999 3000 void 3001 ixgbe_enable_intr(struct ix_softc *sc) 3002 { 3003 struct ixgbe_hw *hw = &sc->hw; 3004 struct ix_queue *que = sc->queues; 3005 uint32_t mask, fwsm; 3006 int i; 3007 3008 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3009 /* Enable Fan Failure detection */ 3010 if (hw->device_id == IXGBE_DEV_ID_82598AT) 3011 mask |= IXGBE_EIMS_GPI_SDP1; 3012 3013 switch (sc->hw.mac.type) { 3014 case ixgbe_mac_82599EB: 3015 mask |= IXGBE_EIMS_ECC; 3016 mask |= IXGBE_EIMS_GPI_SDP0; 3017 mask |= IXGBE_EIMS_GPI_SDP1; 3018 mask |= IXGBE_EIMS_GPI_SDP2; 3019 break; 3020 case ixgbe_mac_X540: 3021 mask |= IXGBE_EIMS_ECC; 3022 /* Detect if Thermal Sensor is enabled */ 3023 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3024 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3025 mask |= IXGBE_EIMS_TS; 3026 break; 3027 default: 3028 break; 3029 } 3030 3031 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3032 3033 /* With RSS we use auto clear */ 3034 if (sc->msix > 1) { 3035 mask = IXGBE_EIMS_ENABLE_MASK; 3036 /* Don't autoclear Link */ 3037 mask &= ~IXGBE_EIMS_OTHER; 3038 mask &= ~IXGBE_EIMS_LSC; 3039 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, mask); 3040 } 3041 3042 /* 3043 * Now enable all queues, this is done separately to 3044 * allow for handling the extended (beyond 32) MSIX 3045 * vectors that can be used by 82599 3046 */ 3047 for (i = 0; i < sc->num_queues; i++, que++) 3048 ixgbe_enable_queue(sc, que->msix); 3049 3050 IXGBE_WRITE_FLUSH(hw); 3051 } 3052 3053 void 3054 ixgbe_disable_intr(struct ix_softc *sc) 3055 { 3056 if (sc->msix > 1) 3057 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3058 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3059 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3060 } else { 3061 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3062 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3063 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3064 } 3065 IXGBE_WRITE_FLUSH(&sc->hw); 3066 } 3067 3068 uint16_t 3069 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3070 { 3071 struct pci_attach_args *pa; 3072 uint32_t value; 3073 int high = 0; 3074 3075 if (reg & 0x2) { 3076 high = 1; 3077 reg &= ~0x2; 3078 } 3079 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3080 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3081 3082 if (high) 3083 value >>= 16; 3084 3085 return (value & 0xffff); 3086 } 3087 3088 void 3089 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3090 { 3091 struct pci_attach_args *pa; 3092 uint32_t rv; 3093 int high = 0; 3094 3095 /* Need to do read/mask/write... because 16 vs 32 bit!!! */ 3096 if (reg & 0x2) { 3097 high = 1; 3098 reg &= ~0x2; 3099 } 3100 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3101 rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3102 if (!high) 3103 rv = (rv & 0xffff0000) | value; 3104 else 3105 rv = (rv & 0xffff) | ((uint32_t)value << 16); 3106 pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv); 3107 } 3108 3109 /* 3110 * Setup the correct IVAR register for a particular MSIX interrupt 3111 * (yes this is all very magic and confusing :) 3112 * - entry is the register array entry 3113 * - vector is the MSIX vector for this queue 3114 * - type is RX/TX/MISC 3115 */ 3116 void 3117 ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type) 3118 { 3119 struct ixgbe_hw *hw = &sc->hw; 3120 uint32_t ivar, index; 3121 3122 vector |= IXGBE_IVAR_ALLOC_VAL; 3123 3124 switch (hw->mac.type) { 3125 3126 case ixgbe_mac_82598EB: 3127 if (type == -1) 3128 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3129 else 3130 entry += (type * 64); 3131 index = (entry >> 2) & 0x1F; 3132 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3133 ivar &= ~(0xFF << (8 * (entry & 0x3))); 3134 ivar |= (vector << (8 * (entry & 0x3))); 3135 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 3136 break; 3137 3138 case ixgbe_mac_82599EB: 3139 case ixgbe_mac_X540: 3140 if (type == -1) { /* MISC IVAR */ 3141 index = (entry & 1) * 8; 3142 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3143 ivar &= ~(0xFF << index); 3144 ivar |= (vector << index); 3145 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3146 } else { /* RX/TX IVARS */ 3147 index = (16 * (entry & 1)) + (8 * type); 3148 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3149 ivar &= ~(0xFF << index); 3150 ivar |= (vector << index); 3151 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3152 } 3153 3154 default: 3155 break; 3156 } 3157 } 3158 3159 void 3160 ixgbe_configure_ivars(struct ix_softc *sc) 3161 { 3162 #if notyet 3163 struct ix_queue *que = sc->queues; 3164 uint32_t newitr; 3165 int i; 3166 3167 if (ixgbe_max_interrupt_rate > 0) 3168 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 3169 else 3170 newitr = 0; 3171 3172 for (i = 0; i < sc->num_queues; i++, que++) { 3173 /* First the RX queue entry */ 3174 ixgbe_set_ivar(sc, i, que->msix, 0); 3175 /* ... and the TX */ 3176 ixgbe_set_ivar(sc, i, que->msix, 1); 3177 /* Set an Initial EITR value */ 3178 IXGBE_WRITE_REG(&sc->hw, 3179 IXGBE_EITR(que->msix), newitr); 3180 } 3181 3182 /* For the Link interrupt */ 3183 ixgbe_set_ivar(sc, 1, sc->linkvec, -1); 3184 #endif 3185 } 3186 3187 /* 3188 * ixgbe_sfp_probe - called in the local timer to 3189 * determine if a port had optics inserted. 3190 */ 3191 bool 3192 ixgbe_sfp_probe(struct ix_softc *sc) 3193 { 3194 bool result = FALSE; 3195 3196 if ((sc->hw.phy.type == ixgbe_phy_nl) && 3197 (sc->hw.phy.sfp_type == ixgbe_sfp_type_not_present)) { 3198 int32_t ret = sc->hw.phy.ops.identify_sfp(&sc->hw); 3199 if (ret) 3200 goto out; 3201 ret = sc->hw.phy.ops.reset(&sc->hw); 3202 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3203 printf("%s: Unsupported SFP+ module detected!", 3204 sc->dev.dv_xname); 3205 goto out; 3206 } 3207 /* We now have supported optics */ 3208 sc->sfp_probe = FALSE; 3209 /* Set the optics type so system reports correctly */ 3210 ixgbe_setup_optics(sc); 3211 result = TRUE; 3212 } 3213 out: 3214 return (result); 3215 } 3216 3217 /* 3218 * SFP module interrupts handler 3219 */ 3220 void 3221 ixgbe_handle_mod(struct ix_softc *sc) 3222 { 3223 struct ixgbe_hw *hw = &sc->hw; 3224 uint32_t err; 3225 3226 err = hw->phy.ops.identify_sfp(hw); 3227 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3228 printf("%s: Unsupported SFP+ module type was detected!\n", 3229 sc->dev.dv_xname); 3230 return; 3231 } 3232 err = hw->mac.ops.setup_sfp(hw); 3233 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3234 printf("%s: Setup failure - unsupported SFP+ module type!\n", 3235 sc->dev.dv_xname); 3236 return; 3237 } 3238 /* Set the optics type so system reports correctly */ 3239 ixgbe_setup_optics(sc); 3240 3241 ixgbe_handle_msf(sc); 3242 } 3243 3244 3245 /* 3246 * MSF (multispeed fiber) interrupts handler 3247 */ 3248 void 3249 ixgbe_handle_msf(struct ix_softc *sc) 3250 { 3251 struct ixgbe_hw *hw = &sc->hw; 3252 uint32_t autoneg; 3253 bool negotiate; 3254 3255 autoneg = hw->phy.autoneg_advertised; 3256 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 3257 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 3258 if (hw->mac.ops.setup_link) 3259 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3260 } 3261 3262 /********************************************************************** 3263 * 3264 * Update the board statistics counters. 3265 * 3266 **********************************************************************/ 3267 void 3268 ixgbe_update_stats_counters(struct ix_softc *sc) 3269 { 3270 struct ifnet *ifp = &sc->arpcom.ac_if; 3271 struct ixgbe_hw *hw = &sc->hw; 3272 uint32_t missed_rx = 0, bprc, lxon, lxoff, total; 3273 uint64_t total_missed_rx = 0; 3274 int i; 3275 3276 sc->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3277 3278 for (i = 0; i < 8; i++) { 3279 uint32_t mp; 3280 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3281 /* missed_rx tallies misses for the gprc workaround */ 3282 missed_rx += mp; 3283 /* global total per queue */ 3284 sc->stats.mpc[i] += mp; 3285 /* running comprehensive total for stats display */ 3286 total_missed_rx += sc->stats.mpc[i]; 3287 if (hw->mac.type == ixgbe_mac_82598EB) 3288 sc->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3289 } 3290 3291 /* Hardware workaround, gprc counts missed packets */ 3292 sc->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 3293 sc->stats.gprc -= missed_rx; 3294 3295 if (hw->mac.type != ixgbe_mac_82598EB) { 3296 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) + 3297 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 3298 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 3299 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 3300 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) + 3301 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 3302 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3303 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3304 } else { 3305 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3306 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3307 /* 82598 only has a counter in the high register */ 3308 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3309 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3310 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3311 } 3312 3313 /* 3314 * Workaround: mprc hardware is incorrectly counting 3315 * broadcasts, so for now we subtract those. 3316 */ 3317 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3318 sc->stats.bprc += bprc; 3319 sc->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3320 if (hw->mac.type == ixgbe_mac_82598EB) 3321 sc->stats.mprc -= bprc; 3322 3323 sc->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3324 sc->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3325 sc->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3326 sc->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3327 sc->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3328 sc->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3329 sc->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3330 sc->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3331 3332 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3333 sc->stats.lxontxc += lxon; 3334 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3335 sc->stats.lxofftxc += lxoff; 3336 total = lxon + lxoff; 3337 3338 sc->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3339 sc->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3340 sc->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3341 sc->stats.gptc -= total; 3342 sc->stats.mptc -= total; 3343 sc->stats.ptc64 -= total; 3344 sc->stats.gotc -= total * ETHER_MIN_LEN; 3345 3346 sc->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3347 sc->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3348 sc->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3349 sc->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3350 sc->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3351 sc->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3352 sc->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3353 sc->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3354 sc->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3355 sc->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3356 3357 #if 0 3358 /* Fill out the OS statistics structure */ 3359 ifp->if_ipackets = sc->stats.gprc; 3360 ifp->if_opackets = sc->stats.gptc; 3361 ifp->if_ibytes = sc->stats.gorc; 3362 ifp->if_obytes = sc->stats.gotc; 3363 ifp->if_imcasts = sc->stats.mprc; 3364 #endif 3365 ifp->if_collisions = 0; 3366 ifp->if_oerrors = sc->watchdog_events; 3367 ifp->if_ierrors = total_missed_rx + sc->stats.crcerrs + sc->stats.rlec; 3368 } 3369 3370 #ifdef IX_DEBUG 3371 /********************************************************************** 3372 * 3373 * This routine is called only when ixgbe_display_debug_stats is enabled. 3374 * This routine provides a way to take a look at important statistics 3375 * maintained by the driver and hardware. 3376 * 3377 **********************************************************************/ 3378 void 3379 ixgbe_print_hw_stats(struct ix_softc * sc) 3380 { 3381 struct ifnet *ifp = &sc->arpcom.ac_if;; 3382 3383 printf("%s: missed pkts %llu, rx len errs %llu, crc errs %llu, " 3384 "dropped pkts %lu, watchdog timeouts %ld, " 3385 "XON rx %llu, XON tx %llu, XOFF rx %llu, XOFF tx %llu, " 3386 "total pkts rx %llu, good pkts rx %llu, good pkts tx %llu, " 3387 "tso tx %lu\n", 3388 ifp->if_xname, 3389 (long long)sc->stats.mpc[0], 3390 (long long)sc->stats.roc + (long long)sc->stats.ruc, 3391 (long long)sc->stats.crcerrs, 3392 sc->dropped_pkts, 3393 sc->watchdog_events, 3394 (long long)sc->stats.lxonrxc, 3395 (long long)sc->stats.lxontxc, 3396 (long long)sc->stats.lxoffrxc, 3397 (long long)sc->stats.lxofftxc, 3398 (long long)sc->stats.tpr, 3399 (long long)sc->stats.gprc, 3400 (long long)sc->stats.gptc, 3401 sc->tso_tx); 3402 } 3403 #endif 3404