1 /* $OpenBSD: if_ix.c,v 1.166 2020/06/07 23:52:05 dlg Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2013, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */ 36 37 #include <dev/pci/if_ix.h> 38 #include <dev/pci/ixgbe_type.h> 39 40 /********************************************************************* 41 * Driver version 42 *********************************************************************/ 43 /* char ixgbe_driver_version[] = "2.5.13"; */ 44 45 /********************************************************************* 46 * PCI Device ID Table 47 * 48 * Used by probe to select devices to load on 49 *********************************************************************/ 50 51 const struct pci_matchid ixgbe_devices[] = { 52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 }, 53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX }, 54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT }, 57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT2 }, 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP }, 63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM }, 64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL }, 65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 }, 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BP }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_T3_LOM }, 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF_QP }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF2 }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_FCOE }, 77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599EN_SFP }, 78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_QSFP_SF_QP }, 79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T }, 80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T1 }, 81 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T }, 82 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T1 }, 83 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KX4 }, 84 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KR }, 85 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_SFP }, 86 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_10G_T }, 87 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_1G_T }, 88 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR }, 89 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR_L }, 90 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP_N }, 91 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP }, 92 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII }, 93 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII_L }, 94 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_10G_T }, 95 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T }, 96 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T_L } 97 }; 98 99 /********************************************************************* 100 * Function prototypes 101 *********************************************************************/ 102 int ixgbe_probe(struct device *, void *, void *); 103 void ixgbe_attach(struct device *, struct device *, void *); 104 int ixgbe_detach(struct device *, int); 105 void ixgbe_start(struct ifqueue *); 106 int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 107 int ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *); 108 int ixgbe_get_sffpage(struct ix_softc *, struct if_sffpage *); 109 void ixgbe_watchdog(struct ifnet *); 110 void ixgbe_init(void *); 111 void ixgbe_stop(void *); 112 void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 113 int ixgbe_media_change(struct ifnet *); 114 void ixgbe_identify_hardware(struct ix_softc *); 115 int ixgbe_allocate_pci_resources(struct ix_softc *); 116 int ixgbe_allocate_legacy(struct ix_softc *); 117 int ixgbe_allocate_msix(struct ix_softc *); 118 int ixgbe_setup_msix(struct ix_softc *); 119 int ixgbe_allocate_queues(struct ix_softc *); 120 void ixgbe_free_pci_resources(struct ix_softc *); 121 void ixgbe_local_timer(void *); 122 void ixgbe_setup_interface(struct ix_softc *); 123 void ixgbe_config_gpie(struct ix_softc *); 124 void ixgbe_config_delay_values(struct ix_softc *); 125 void ixgbe_add_media_types(struct ix_softc *); 126 void ixgbe_config_link(struct ix_softc *); 127 128 int ixgbe_allocate_transmit_buffers(struct tx_ring *); 129 int ixgbe_setup_transmit_structures(struct ix_softc *); 130 int ixgbe_setup_transmit_ring(struct tx_ring *); 131 void ixgbe_initialize_transmit_units(struct ix_softc *); 132 void ixgbe_free_transmit_structures(struct ix_softc *); 133 void ixgbe_free_transmit_buffers(struct tx_ring *); 134 135 int ixgbe_allocate_receive_buffers(struct rx_ring *); 136 int ixgbe_setup_receive_structures(struct ix_softc *); 137 int ixgbe_setup_receive_ring(struct rx_ring *); 138 void ixgbe_initialize_receive_units(struct ix_softc *); 139 void ixgbe_free_receive_structures(struct ix_softc *); 140 void ixgbe_free_receive_buffers(struct rx_ring *); 141 void ixgbe_initialize_rss_mapping(struct ix_softc *); 142 int ixgbe_rxfill(struct rx_ring *); 143 void ixgbe_rxrefill(void *); 144 145 int ixgbe_intr(struct ix_softc *sc); 146 void ixgbe_enable_intr(struct ix_softc *); 147 void ixgbe_disable_intr(struct ix_softc *); 148 void ixgbe_update_stats_counters(struct ix_softc *); 149 int ixgbe_txeof(struct tx_ring *); 150 int ixgbe_rxeof(struct rx_ring *); 151 void ixgbe_rx_checksum(uint32_t, struct mbuf *, uint32_t); 152 void ixgbe_iff(struct ix_softc *); 153 #ifdef IX_DEBUG 154 void ixgbe_print_hw_stats(struct ix_softc *); 155 void ixgbe_map_queue_statistics(struct ix_softc *); 156 #endif 157 void ixgbe_update_link_status(struct ix_softc *); 158 int ixgbe_get_buf(struct rx_ring *, int); 159 int ixgbe_encap(struct tx_ring *, struct mbuf *); 160 int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 161 struct ixgbe_dma_alloc *, int); 162 void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 163 int ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *, uint32_t *, 164 uint32_t *); 165 int ixgbe_tso_setup(struct tx_ring *, struct mbuf *, uint32_t *, 166 uint32_t *); 167 void ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 168 void ixgbe_configure_ivars(struct ix_softc *); 169 uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 170 171 void ixgbe_setup_vlan_hw_support(struct ix_softc *); 172 173 /* Support for pluggable optic modules */ 174 void ixgbe_handle_mod(struct ix_softc *); 175 void ixgbe_handle_msf(struct ix_softc *); 176 void ixgbe_handle_phy(struct ix_softc *); 177 178 /* Legacy (single vector interrupt handler */ 179 int ixgbe_legacy_intr(void *); 180 void ixgbe_enable_queue(struct ix_softc *, uint32_t); 181 void ixgbe_enable_queues(struct ix_softc *); 182 void ixgbe_disable_queue(struct ix_softc *, uint32_t); 183 void ixgbe_rearm_queue(struct ix_softc *, uint32_t); 184 185 /* MSI-X (multiple vectors interrupt handlers) */ 186 int ixgbe_link_intr(void *); 187 int ixgbe_queue_intr(void *); 188 189 /********************************************************************* 190 * OpenBSD Device Interface Entry Points 191 *********************************************************************/ 192 193 struct cfdriver ix_cd = { 194 NULL, "ix", DV_IFNET 195 }; 196 197 struct cfattach ix_ca = { 198 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach 199 }; 200 201 int ixgbe_smart_speed = ixgbe_smart_speed_on; 202 int ixgbe_enable_msix = 0; 203 204 /********************************************************************* 205 * Device identification routine 206 * 207 * ixgbe_probe determines if the driver should be loaded on 208 * adapter based on PCI vendor/device id of the adapter. 209 * 210 * return 0 on success, positive on failure 211 *********************************************************************/ 212 213 int 214 ixgbe_probe(struct device *parent, void *match, void *aux) 215 { 216 INIT_DEBUGOUT("ixgbe_probe: begin"); 217 218 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 219 nitems(ixgbe_devices))); 220 } 221 222 /********************************************************************* 223 * Device initialization routine 224 * 225 * The attach entry point is called when the driver is being loaded. 226 * This routine identifies the type of hardware, allocates all resources 227 * and initializes the hardware. 228 * 229 * return 0 on success, positive on failure 230 *********************************************************************/ 231 232 void 233 ixgbe_attach(struct device *parent, struct device *self, void *aux) 234 { 235 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 236 struct ix_softc *sc = (struct ix_softc *)self; 237 int error = 0; 238 uint16_t csum; 239 uint32_t ctrl_ext; 240 struct ixgbe_hw *hw = &sc->hw; 241 242 INIT_DEBUGOUT("ixgbe_attach: begin"); 243 244 sc->osdep.os_sc = sc; 245 sc->osdep.os_pa = *pa; 246 247 rw_init(&sc->sfflock, "ixsff"); 248 249 /* Set up the timer callout */ 250 timeout_set(&sc->timer, ixgbe_local_timer, sc); 251 252 /* Determine hardware revision */ 253 ixgbe_identify_hardware(sc); 254 255 /* Indicate to RX setup to use Jumbo Clusters */ 256 sc->num_tx_desc = DEFAULT_TXD; 257 sc->num_rx_desc = DEFAULT_RXD; 258 259 /* Do base PCI setup - map BAR0 */ 260 if (ixgbe_allocate_pci_resources(sc)) 261 goto err_out; 262 263 /* Allocate our TX/RX Queues */ 264 if (ixgbe_allocate_queues(sc)) 265 goto err_out; 266 267 /* Allocate multicast array memory. */ 268 sc->mta = mallocarray(IXGBE_ETH_LENGTH_OF_ADDRESS, 269 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 270 if (sc->mta == NULL) { 271 printf(": Can not allocate multicast setup array\n"); 272 goto err_late; 273 } 274 275 /* Initialize the shared code */ 276 error = ixgbe_init_shared_code(hw); 277 if (error) { 278 printf(": Unable to initialize the shared code\n"); 279 goto err_late; 280 } 281 282 /* Make sure we have a good EEPROM before we read from it */ 283 if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) { 284 printf(": The EEPROM Checksum Is Not Valid\n"); 285 goto err_late; 286 } 287 288 error = ixgbe_init_hw(hw); 289 if (error == IXGBE_ERR_EEPROM_VERSION) { 290 printf(": This device is a pre-production adapter/" 291 "LOM. Please be aware there may be issues associated " 292 "with your hardware.\nIf you are experiencing problems " 293 "please contact your Intel or hardware representative " 294 "who provided you with this hardware.\n"); 295 } else if (error && (error != IXGBE_ERR_SFP_NOT_PRESENT && 296 error != IXGBE_ERR_SFP_NOT_SUPPORTED)) { 297 printf(": Hardware Initialization Failure\n"); 298 goto err_late; 299 } 300 301 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 302 IXGBE_ETH_LENGTH_OF_ADDRESS); 303 304 if (sc->msix > 1) 305 error = ixgbe_allocate_msix(sc); 306 else 307 error = ixgbe_allocate_legacy(sc); 308 if (error) 309 goto err_late; 310 311 /* Enable the optics for 82599 SFP+ fiber */ 312 if (sc->hw.mac.ops.enable_tx_laser) 313 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 314 315 /* Enable power to the phy */ 316 if (hw->phy.ops.set_phy_power) 317 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 318 319 /* Setup OS specific network interface */ 320 ixgbe_setup_interface(sc); 321 322 /* Initialize statistics */ 323 ixgbe_update_stats_counters(sc); 324 325 /* Get the PCI-E bus info and determine LAN ID */ 326 hw->mac.ops.get_bus_info(hw); 327 328 /* Set an initial default flow control value */ 329 sc->fc = ixgbe_fc_full; 330 331 /* let hardware know driver is loaded */ 332 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 333 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 334 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 335 336 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 337 338 INIT_DEBUGOUT("ixgbe_attach: end"); 339 return; 340 341 err_late: 342 ixgbe_free_transmit_structures(sc); 343 ixgbe_free_receive_structures(sc); 344 err_out: 345 ixgbe_free_pci_resources(sc); 346 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 347 MAX_NUM_MULTICAST_ADDRESSES); 348 } 349 350 /********************************************************************* 351 * Device removal routine 352 * 353 * The detach entry point is called when the driver is being removed. 354 * This routine stops the adapter and deallocates all the resources 355 * that were allocated for driver operation. 356 * 357 * return 0 on success, positive on failure 358 *********************************************************************/ 359 360 int 361 ixgbe_detach(struct device *self, int flags) 362 { 363 struct ix_softc *sc = (struct ix_softc *)self; 364 struct ifnet *ifp = &sc->arpcom.ac_if; 365 uint32_t ctrl_ext; 366 367 INIT_DEBUGOUT("ixgbe_detach: begin"); 368 369 ixgbe_stop(sc); 370 371 /* let hardware know driver is unloading */ 372 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 373 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 374 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 375 376 ether_ifdetach(ifp); 377 if_detach(ifp); 378 379 timeout_del(&sc->timer); 380 ixgbe_free_pci_resources(sc); 381 382 ixgbe_free_transmit_structures(sc); 383 ixgbe_free_receive_structures(sc); 384 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 385 MAX_NUM_MULTICAST_ADDRESSES); 386 387 return (0); 388 } 389 390 /********************************************************************* 391 * Transmit entry point 392 * 393 * ixgbe_start is called by the stack to initiate a transmit. 394 * The driver will remain in this routine as long as there are 395 * packets to transmit and transmit resources are available. 396 * In case resources are not available stack is notified and 397 * the packet is requeued. 398 **********************************************************************/ 399 400 void 401 ixgbe_start(struct ifqueue *ifq) 402 { 403 struct ifnet *ifp = ifq->ifq_if; 404 struct ix_softc *sc = ifp->if_softc; 405 struct tx_ring *txr = ifq->ifq_softc; 406 struct mbuf *m_head; 407 unsigned int head, free, used; 408 int post = 0; 409 410 if (!sc->link_up) 411 return; 412 413 head = txr->next_avail_desc; 414 free = txr->next_to_clean; 415 if (free <= head) 416 free += sc->num_tx_desc; 417 free -= head; 418 419 membar_consumer(); 420 421 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 422 0, txr->txdma.dma_map->dm_mapsize, 423 BUS_DMASYNC_POSTWRITE); 424 425 for (;;) { 426 /* Check that we have the minimal number of TX descriptors. */ 427 if (free <= IXGBE_TX_OP_THRESHOLD) { 428 ifq_set_oactive(ifq); 429 break; 430 } 431 432 m_head = ifq_dequeue(ifq); 433 if (m_head == NULL) 434 break; 435 436 used = ixgbe_encap(txr, m_head); 437 if (used == 0) { 438 m_freem(m_head); 439 continue; 440 } 441 442 free -= used; 443 444 #if NBPFILTER > 0 445 if (ifp->if_bpf) 446 bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 447 #endif 448 449 /* Set timeout in case hardware has problems transmitting */ 450 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 451 ifp->if_timer = IXGBE_TX_TIMEOUT; 452 453 post = 1; 454 } 455 456 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 457 0, txr->txdma.dma_map->dm_mapsize, 458 BUS_DMASYNC_PREWRITE); 459 460 /* 461 * Advance the Transmit Descriptor Tail (Tdt), this tells the 462 * hardware that this frame is available to transmit. 463 */ 464 if (post) 465 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 466 txr->next_avail_desc); 467 } 468 469 /********************************************************************* 470 * Ioctl entry point 471 * 472 * ixgbe_ioctl is called when the user wants to configure the 473 * interface. 474 * 475 * return 0 on success, positive on failure 476 **********************************************************************/ 477 478 int 479 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 480 { 481 struct ix_softc *sc = ifp->if_softc; 482 struct ifreq *ifr = (struct ifreq *) data; 483 int s, error = 0; 484 485 s = splnet(); 486 487 switch (command) { 488 case SIOCSIFADDR: 489 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 490 ifp->if_flags |= IFF_UP; 491 if (!(ifp->if_flags & IFF_RUNNING)) 492 ixgbe_init(sc); 493 break; 494 495 case SIOCSIFFLAGS: 496 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 497 if (ifp->if_flags & IFF_UP) { 498 if (ifp->if_flags & IFF_RUNNING) 499 error = ENETRESET; 500 else 501 ixgbe_init(sc); 502 } else { 503 if (ifp->if_flags & IFF_RUNNING) 504 ixgbe_stop(sc); 505 } 506 break; 507 508 case SIOCSIFMEDIA: 509 case SIOCGIFMEDIA: 510 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 511 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 512 break; 513 514 case SIOCGIFRXR: 515 error = ixgbe_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 516 break; 517 518 case SIOCGIFSFFPAGE: 519 error = rw_enter(&sc->sfflock, RW_WRITE|RW_INTR); 520 if (error != 0) 521 break; 522 523 error = ixgbe_get_sffpage(sc, (struct if_sffpage *)data); 524 rw_exit(&sc->sfflock); 525 break; 526 527 default: 528 error = ether_ioctl(ifp, &sc->arpcom, command, data); 529 } 530 531 if (error == ENETRESET) { 532 if (ifp->if_flags & IFF_RUNNING) { 533 ixgbe_disable_intr(sc); 534 ixgbe_iff(sc); 535 ixgbe_enable_intr(sc); 536 ixgbe_enable_queues(sc); 537 } 538 error = 0; 539 } 540 541 splx(s); 542 return (error); 543 } 544 545 int 546 ixgbe_get_sffpage(struct ix_softc *sc, struct if_sffpage *sff) 547 { 548 struct ixgbe_hw *hw = &sc->hw; 549 uint32_t swfw_mask = hw->phy.phy_semaphore_mask; 550 uint8_t page; 551 size_t i; 552 int error = EIO; 553 554 if (hw->phy.type == ixgbe_phy_fw) 555 return (ENODEV); 556 557 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 558 return (EBUSY); /* XXX */ 559 560 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 561 if (hw->phy.ops.read_i2c_byte_unlocked(hw, 127, 562 IFSFF_ADDR_EEPROM, &page)) 563 goto error; 564 if (page != sff->sff_page && 565 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 566 IFSFF_ADDR_EEPROM, sff->sff_page)) 567 goto error; 568 } 569 570 for (i = 0; i < sizeof(sff->sff_data); i++) { 571 if (hw->phy.ops.read_i2c_byte_unlocked(hw, i, 572 sff->sff_addr, &sff->sff_data[i])) 573 goto error; 574 } 575 576 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 577 if (page != sff->sff_page && 578 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 579 IFSFF_ADDR_EEPROM, page)) 580 goto error; 581 } 582 583 error = 0; 584 error: 585 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 586 return (error); 587 } 588 589 int 590 ixgbe_rxrinfo(struct ix_softc *sc, struct if_rxrinfo *ifri) 591 { 592 struct if_rxring_info *ifr, ifr1; 593 struct rx_ring *rxr; 594 int error, i; 595 u_int n = 0; 596 597 if (sc->num_queues > 1) { 598 if ((ifr = mallocarray(sc->num_queues, sizeof(*ifr), M_DEVBUF, 599 M_WAITOK | M_ZERO)) == NULL) 600 return (ENOMEM); 601 } else 602 ifr = &ifr1; 603 604 for (i = 0; i < sc->num_queues; i++) { 605 rxr = &sc->rx_rings[i]; 606 ifr[n].ifr_size = sc->rx_mbuf_sz; 607 snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "/%d", i); 608 ifr[n].ifr_info = rxr->rx_ring; 609 n++; 610 } 611 612 error = if_rxr_info_ioctl(ifri, sc->num_queues, ifr); 613 614 if (sc->num_queues > 1) 615 free(ifr, M_DEVBUF, sc->num_queues * sizeof(*ifr)); 616 return (error); 617 } 618 619 /********************************************************************* 620 * Watchdog entry point 621 * 622 **********************************************************************/ 623 624 void 625 ixgbe_watchdog(struct ifnet * ifp) 626 { 627 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 628 struct tx_ring *txr = sc->tx_rings; 629 struct ixgbe_hw *hw = &sc->hw; 630 int tx_hang = FALSE; 631 int i; 632 633 /* 634 * The timer is set to 5 every time ixgbe_start() queues a packet. 635 * Anytime all descriptors are clean the timer is set to 0. 636 */ 637 for (i = 0; i < sc->num_queues; i++, txr++) { 638 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 639 continue; 640 else { 641 tx_hang = TRUE; 642 break; 643 } 644 } 645 if (tx_hang == FALSE) 646 return; 647 648 /* 649 * If we are in this routine because of pause frames, then don't 650 * reset the hardware. 651 */ 652 if (!(IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXON)) { 653 for (i = 0; i < sc->num_queues; i++, txr++) 654 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 655 ifp->if_timer = IXGBE_TX_TIMEOUT; 656 return; 657 } 658 659 660 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 661 for (i = 0; i < sc->num_queues; i++, txr++) { 662 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 663 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 664 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 665 printf("%s: TX(%d) Next TX to Clean = %d\n", ifp->if_xname, 666 i, txr->next_to_clean); 667 } 668 ifp->if_flags &= ~IFF_RUNNING; 669 sc->watchdog_events++; 670 671 ixgbe_init(sc); 672 } 673 674 /********************************************************************* 675 * Init entry point 676 * 677 * This routine is used in two ways. It is used by the stack as 678 * init entry point in network interface structure. It is also used 679 * by the driver as a hw/sw initialization routine to get to a 680 * consistent state. 681 * 682 * return 0 on success, positive on failure 683 **********************************************************************/ 684 #define IXGBE_MHADD_MFS_SHIFT 16 685 686 void 687 ixgbe_init(void *arg) 688 { 689 struct ix_softc *sc = (struct ix_softc *)arg; 690 struct ifnet *ifp = &sc->arpcom.ac_if; 691 struct rx_ring *rxr = sc->rx_rings; 692 uint32_t k, txdctl, rxdctl, rxctrl, mhadd, itr; 693 int i, s, err; 694 695 INIT_DEBUGOUT("ixgbe_init: begin"); 696 697 s = splnet(); 698 699 ixgbe_stop(sc); 700 701 /* reprogram the RAR[0] in case user changed it. */ 702 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 703 704 /* Get the latest mac address, User can use a LAA */ 705 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 706 IXGBE_ETH_LENGTH_OF_ADDRESS); 707 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1); 708 sc->hw.addr_ctrl.rar_used_count = 1; 709 710 /* Prepare transmit descriptors and buffers */ 711 if (ixgbe_setup_transmit_structures(sc)) { 712 printf("%s: Could not setup transmit structures\n", 713 ifp->if_xname); 714 ixgbe_stop(sc); 715 splx(s); 716 return; 717 } 718 719 ixgbe_init_hw(&sc->hw); 720 ixgbe_initialize_transmit_units(sc); 721 722 /* Use 2k clusters, even for jumbo frames */ 723 sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN; 724 725 /* Prepare receive descriptors and buffers */ 726 if (ixgbe_setup_receive_structures(sc)) { 727 printf("%s: Could not setup receive structures\n", 728 ifp->if_xname); 729 ixgbe_stop(sc); 730 splx(s); 731 return; 732 } 733 734 /* Configure RX settings */ 735 ixgbe_initialize_receive_units(sc); 736 737 /* Enable SDP & MSIX interrupts based on adapter */ 738 ixgbe_config_gpie(sc); 739 740 /* Program promiscuous mode and multicast filters. */ 741 ixgbe_iff(sc); 742 743 /* Set MRU size */ 744 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 745 mhadd &= ~IXGBE_MHADD_MFS_MASK; 746 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 747 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 748 749 /* Now enable all the queues */ 750 for (i = 0; i < sc->num_queues; i++) { 751 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 752 txdctl |= IXGBE_TXDCTL_ENABLE; 753 /* Set WTHRESH to 8, burst writeback */ 754 txdctl |= (8 << 16); 755 /* 756 * When the internal queue falls below PTHRESH (16), 757 * start prefetching as long as there are at least 758 * HTHRESH (1) buffers ready. 759 */ 760 txdctl |= (16 << 0) | (1 << 8); 761 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 762 } 763 764 for (i = 0; i < sc->num_queues; i++) { 765 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 766 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 767 /* 768 * PTHRESH = 21 769 * HTHRESH = 4 770 * WTHRESH = 8 771 */ 772 rxdctl &= ~0x3FFFFF; 773 rxdctl |= 0x080420; 774 } 775 rxdctl |= IXGBE_RXDCTL_ENABLE; 776 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 777 for (k = 0; k < 10; k++) { 778 if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) & 779 IXGBE_RXDCTL_ENABLE) 780 break; 781 else 782 msec_delay(1); 783 } 784 IXGBE_WRITE_FLUSH(&sc->hw); 785 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled); 786 } 787 788 /* Set up VLAN support and filter */ 789 ixgbe_setup_vlan_hw_support(sc); 790 791 /* Enable Receive engine */ 792 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 793 if (sc->hw.mac.type == ixgbe_mac_82598EB) 794 rxctrl |= IXGBE_RXCTRL_DMBYPS; 795 rxctrl |= IXGBE_RXCTRL_RXEN; 796 sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl); 797 798 timeout_add_sec(&sc->timer, 1); 799 800 /* Set up MSI/X routing */ 801 if (sc->msix > 1) { 802 ixgbe_configure_ivars(sc); 803 /* Set up auto-mask */ 804 if (sc->hw.mac.type == ixgbe_mac_82598EB) 805 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 806 else { 807 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 808 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 809 } 810 } else { /* Simple settings for Legacy/MSI */ 811 ixgbe_set_ivar(sc, 0, 0, 0); 812 ixgbe_set_ivar(sc, 0, 0, 1); 813 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 814 } 815 816 /* Check on any SFP devices that need to be kick-started */ 817 if (sc->hw.phy.type == ixgbe_phy_none) { 818 err = sc->hw.phy.ops.identify(&sc->hw); 819 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 820 printf("Unsupported SFP+ module type was detected.\n"); 821 splx(s); 822 return; 823 } 824 } 825 826 /* Setup interrupt moderation */ 827 itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8; 828 if (sc->hw.mac.type != ixgbe_mac_82598EB) 829 itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS; 830 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr); 831 832 if (sc->msix > 1) { 833 /* Set moderation on the Link interrupt */ 834 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(sc->linkvec), 835 IXGBE_LINK_ITR); 836 } 837 838 /* Enable power to the phy */ 839 if (sc->hw.phy.ops.set_phy_power) 840 sc->hw.phy.ops.set_phy_power(&sc->hw, TRUE); 841 842 /* Config/Enable Link */ 843 ixgbe_config_link(sc); 844 845 /* Hardware Packet Buffer & Flow Control setup */ 846 ixgbe_config_delay_values(sc); 847 848 /* Initialize the FC settings */ 849 sc->hw.mac.ops.start_hw(&sc->hw); 850 851 /* And now turn on interrupts */ 852 ixgbe_enable_intr(sc); 853 ixgbe_enable_queues(sc); 854 855 /* Now inform the stack we're ready */ 856 ifp->if_flags |= IFF_RUNNING; 857 for (i = 0; i < sc->num_queues; i++) 858 ifq_clr_oactive(ifp->if_ifqs[i]); 859 860 splx(s); 861 } 862 863 void 864 ixgbe_config_gpie(struct ix_softc *sc) 865 { 866 struct ixgbe_hw *hw = &sc->hw; 867 uint32_t gpie; 868 869 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 870 871 /* Fan Failure Interrupt */ 872 if (hw->device_id == IXGBE_DEV_ID_82598AT) 873 gpie |= IXGBE_SDP1_GPIEN; 874 875 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 876 /* Add for Module detection */ 877 gpie |= IXGBE_SDP2_GPIEN; 878 879 /* Media ready */ 880 if (hw->device_id != IXGBE_DEV_ID_82599_QSFP_SF_QP) 881 gpie |= IXGBE_SDP1_GPIEN; 882 883 /* 884 * Set LL interval to max to reduce the number of low latency 885 * interrupts hitting the card when the ring is getting full. 886 */ 887 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 888 } 889 890 if (sc->hw.mac.type == ixgbe_mac_X540 || 891 sc->hw.mac.type == ixgbe_mac_X550EM_x || 892 sc->hw.mac.type == ixgbe_mac_X550EM_a) { 893 /* 894 * Thermal Failure Detection (X540) 895 * Link Detection (X552 SFP+, X552/X557-AT) 896 */ 897 gpie |= IXGBE_SDP0_GPIEN_X540; 898 899 /* 900 * Set LL interval to max to reduce the number of low latency 901 * interrupts hitting the card when the ring is getting full. 902 */ 903 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 904 } 905 906 if (sc->msix > 1) { 907 /* Enable Enhanced MSIX mode */ 908 gpie |= IXGBE_GPIE_MSIX_MODE; 909 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 910 IXGBE_GPIE_OCD; 911 } 912 913 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 914 } 915 916 /* 917 * Requires sc->max_frame_size to be set. 918 */ 919 void 920 ixgbe_config_delay_values(struct ix_softc *sc) 921 { 922 struct ixgbe_hw *hw = &sc->hw; 923 uint32_t rxpb, frame, size, tmp; 924 925 frame = sc->max_frame_size; 926 927 /* Calculate High Water */ 928 switch (hw->mac.type) { 929 case ixgbe_mac_X540: 930 case ixgbe_mac_X550: 931 case ixgbe_mac_X550EM_x: 932 case ixgbe_mac_X550EM_a: 933 tmp = IXGBE_DV_X540(frame, frame); 934 break; 935 default: 936 tmp = IXGBE_DV(frame, frame); 937 break; 938 } 939 size = IXGBE_BT2KB(tmp); 940 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 941 hw->fc.high_water[0] = rxpb - size; 942 943 /* Now calculate Low Water */ 944 switch (hw->mac.type) { 945 case ixgbe_mac_X540: 946 case ixgbe_mac_X550: 947 case ixgbe_mac_X550EM_x: 948 case ixgbe_mac_X550EM_a: 949 tmp = IXGBE_LOW_DV_X540(frame); 950 break; 951 default: 952 tmp = IXGBE_LOW_DV(frame); 953 break; 954 } 955 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 956 957 hw->fc.requested_mode = sc->fc; 958 hw->fc.pause_time = IXGBE_FC_PAUSE; 959 hw->fc.send_xon = TRUE; 960 } 961 962 /* 963 * MSIX Interrupt Handlers 964 */ 965 void 966 ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector) 967 { 968 uint64_t queue = 1ULL << vector; 969 uint32_t mask; 970 971 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 972 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 973 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask); 974 } else { 975 mask = (queue & 0xFFFFFFFF); 976 if (mask) 977 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask); 978 mask = (queue >> 32); 979 if (mask) 980 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask); 981 } 982 } 983 984 void 985 ixgbe_enable_queues(struct ix_softc *sc) 986 { 987 struct ix_queue *que; 988 int i; 989 990 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 991 ixgbe_enable_queue(sc, que->msix); 992 } 993 994 void 995 ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector) 996 { 997 uint64_t queue = 1ULL << vector; 998 uint32_t mask; 999 1000 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1001 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1002 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask); 1003 } else { 1004 mask = (queue & 0xFFFFFFFF); 1005 if (mask) 1006 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask); 1007 mask = (queue >> 32); 1008 if (mask) 1009 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask); 1010 } 1011 } 1012 1013 /* 1014 * MSIX Interrupt Handlers 1015 */ 1016 int 1017 ixgbe_link_intr(void *vsc) 1018 { 1019 struct ix_softc *sc = (struct ix_softc *)vsc; 1020 1021 return ixgbe_intr(sc); 1022 } 1023 1024 int 1025 ixgbe_queue_intr(void *vque) 1026 { 1027 struct ix_queue *que = vque; 1028 struct ix_softc *sc = que->sc; 1029 struct ifnet *ifp = &sc->arpcom.ac_if; 1030 struct rx_ring *rxr = que->rxr; 1031 struct tx_ring *txr = que->txr; 1032 1033 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1034 ixgbe_rxeof(rxr); 1035 ixgbe_txeof(txr); 1036 ixgbe_rxrefill(rxr); 1037 } 1038 1039 ixgbe_enable_queue(sc, que->msix); 1040 1041 return (1); 1042 } 1043 1044 /********************************************************************* 1045 * 1046 * Legacy Interrupt Service routine 1047 * 1048 **********************************************************************/ 1049 1050 int 1051 ixgbe_legacy_intr(void *arg) 1052 { 1053 struct ix_softc *sc = (struct ix_softc *)arg; 1054 struct ifnet *ifp = &sc->arpcom.ac_if; 1055 struct rx_ring *rxr = sc->rx_rings; 1056 struct tx_ring *txr = sc->tx_rings; 1057 int rv; 1058 1059 rv = ixgbe_intr(sc); 1060 if (rv == 0) { 1061 ixgbe_enable_queues(sc); 1062 return (0); 1063 } 1064 1065 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1066 ixgbe_rxeof(rxr); 1067 ixgbe_txeof(txr); 1068 ixgbe_rxrefill(rxr); 1069 } 1070 1071 ixgbe_enable_queues(sc); 1072 return (rv); 1073 } 1074 1075 int 1076 ixgbe_intr(struct ix_softc *sc) 1077 { 1078 struct ifnet *ifp = &sc->arpcom.ac_if; 1079 struct ixgbe_hw *hw = &sc->hw; 1080 uint32_t reg_eicr, mod_mask, msf_mask; 1081 1082 reg_eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR); 1083 if (reg_eicr == 0) { 1084 ixgbe_enable_intr(sc); 1085 return (0); 1086 } 1087 1088 /* Link status change */ 1089 if (reg_eicr & IXGBE_EICR_LSC) { 1090 KERNEL_LOCK(); 1091 ixgbe_update_link_status(sc); 1092 KERNEL_UNLOCK(); 1093 } 1094 1095 if (hw->mac.type != ixgbe_mac_82598EB) { 1096 if (reg_eicr & IXGBE_EICR_ECC) { 1097 printf("%s: CRITICAL: ECC ERROR!! " 1098 "Please Reboot!!\n", sc->dev.dv_xname); 1099 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1100 } 1101 /* Check for over temp condition */ 1102 if (reg_eicr & IXGBE_EICR_TS) { 1103 printf("%s: CRITICAL: OVER TEMP!! " 1104 "PHY IS SHUT DOWN!!\n", ifp->if_xname); 1105 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS); 1106 } 1107 } 1108 1109 /* Pluggable optics-related interrupt */ 1110 if (ixgbe_is_sfp(hw)) { 1111 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP) { 1112 mod_mask = IXGBE_EICR_GPI_SDP0_X540; 1113 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1114 } else if (hw->mac.type == ixgbe_mac_X540 || 1115 hw->mac.type == ixgbe_mac_X550 || 1116 hw->mac.type == ixgbe_mac_X550EM_x) { 1117 mod_mask = IXGBE_EICR_GPI_SDP2_X540; 1118 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1119 } else { 1120 mod_mask = IXGBE_EICR_GPI_SDP2; 1121 msf_mask = IXGBE_EICR_GPI_SDP1; 1122 } 1123 if (reg_eicr & mod_mask) { 1124 /* Clear the interrupt */ 1125 IXGBE_WRITE_REG(hw, IXGBE_EICR, mod_mask); 1126 KERNEL_LOCK(); 1127 ixgbe_handle_mod(sc); 1128 KERNEL_UNLOCK(); 1129 } else if ((hw->phy.media_type != ixgbe_media_type_copper) && 1130 (reg_eicr & msf_mask)) { 1131 /* Clear the interrupt */ 1132 IXGBE_WRITE_REG(hw, IXGBE_EICR, msf_mask); 1133 KERNEL_LOCK(); 1134 ixgbe_handle_msf(sc); 1135 KERNEL_UNLOCK(); 1136 } 1137 } 1138 1139 /* Check for fan failure */ 1140 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1141 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1142 printf("%s: CRITICAL: FAN FAILURE!! " 1143 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 1144 IXGBE_WRITE_REG(&sc->hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1145 } 1146 1147 /* External PHY interrupt */ 1148 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 1149 (reg_eicr & IXGBE_EICR_GPI_SDP0_X540)) { 1150 /* Clear the interrupt */ 1151 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540); 1152 KERNEL_LOCK(); 1153 ixgbe_handle_phy(sc); 1154 KERNEL_UNLOCK(); 1155 } 1156 1157 return (1); 1158 } 1159 1160 /********************************************************************* 1161 * 1162 * Media Ioctl callback 1163 * 1164 * This routine is called whenever the user queries the status of 1165 * the interface using ifconfig. 1166 * 1167 **********************************************************************/ 1168 void 1169 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq *ifmr) 1170 { 1171 struct ix_softc *sc = ifp->if_softc; 1172 uint64_t layer; 1173 1174 ifmr->ifm_active = IFM_ETHER; 1175 ifmr->ifm_status = IFM_AVALID; 1176 1177 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1178 ixgbe_update_link_status(sc); 1179 1180 if (!LINK_STATE_IS_UP(ifp->if_link_state)) 1181 return; 1182 1183 ifmr->ifm_status |= IFM_ACTIVE; 1184 layer = sc->phy_layer; 1185 1186 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 1187 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 1188 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 1189 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1190 switch (sc->link_speed) { 1191 case IXGBE_LINK_SPEED_10GB_FULL: 1192 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1193 break; 1194 case IXGBE_LINK_SPEED_1GB_FULL: 1195 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1196 break; 1197 case IXGBE_LINK_SPEED_100_FULL: 1198 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1199 break; 1200 case IXGBE_LINK_SPEED_10_FULL: 1201 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1202 break; 1203 } 1204 } 1205 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1206 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1207 switch (sc->link_speed) { 1208 case IXGBE_LINK_SPEED_10GB_FULL: 1209 ifmr->ifm_active |= IFM_10G_SFP_CU | IFM_FDX; 1210 break; 1211 } 1212 } 1213 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1214 switch (sc->link_speed) { 1215 case IXGBE_LINK_SPEED_10GB_FULL: 1216 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 1217 break; 1218 case IXGBE_LINK_SPEED_1GB_FULL: 1219 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1220 break; 1221 } 1222 } 1223 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 1224 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1225 switch (sc->link_speed) { 1226 case IXGBE_LINK_SPEED_10GB_FULL: 1227 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1228 break; 1229 case IXGBE_LINK_SPEED_1GB_FULL: 1230 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1231 break; 1232 } 1233 } 1234 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1235 switch (sc->link_speed) { 1236 case IXGBE_LINK_SPEED_10GB_FULL: 1237 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1238 break; 1239 } 1240 } 1241 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1242 switch (sc->link_speed) { 1243 case IXGBE_LINK_SPEED_10GB_FULL: 1244 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 1245 break; 1246 case IXGBE_LINK_SPEED_2_5GB_FULL: 1247 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1248 break; 1249 case IXGBE_LINK_SPEED_1GB_FULL: 1250 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1251 break; 1252 } 1253 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 1254 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 1255 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1256 switch (sc->link_speed) { 1257 case IXGBE_LINK_SPEED_10GB_FULL: 1258 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 1259 break; 1260 case IXGBE_LINK_SPEED_2_5GB_FULL: 1261 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1262 break; 1263 case IXGBE_LINK_SPEED_1GB_FULL: 1264 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1265 break; 1266 } 1267 } 1268 1269 switch (sc->hw.fc.current_mode) { 1270 case ixgbe_fc_tx_pause: 1271 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 1272 break; 1273 case ixgbe_fc_rx_pause: 1274 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 1275 break; 1276 case ixgbe_fc_full: 1277 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE | 1278 IFM_ETH_TXPAUSE; 1279 break; 1280 default: 1281 ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE | 1282 IFM_ETH_TXPAUSE); 1283 break; 1284 } 1285 } 1286 1287 /********************************************************************* 1288 * 1289 * Media Ioctl callback 1290 * 1291 * This routine is called when the user changes speed/duplex using 1292 * media/mediopt option with ifconfig. 1293 * 1294 **********************************************************************/ 1295 int 1296 ixgbe_media_change(struct ifnet *ifp) 1297 { 1298 struct ix_softc *sc = ifp->if_softc; 1299 struct ixgbe_hw *hw = &sc->hw; 1300 struct ifmedia *ifm = &sc->media; 1301 ixgbe_link_speed speed = 0; 1302 1303 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1304 return (EINVAL); 1305 1306 if (hw->phy.media_type == ixgbe_media_type_backplane) 1307 return (ENODEV); 1308 1309 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1310 case IFM_AUTO: 1311 case IFM_10G_T: 1312 speed |= IXGBE_LINK_SPEED_100_FULL; 1313 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1314 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1315 break; 1316 case IFM_10G_SR: 1317 case IFM_10G_KR: 1318 case IFM_10G_LR: 1319 case IFM_10G_LRM: 1320 case IFM_10G_CX4: 1321 case IFM_10G_KX4: 1322 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1323 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1324 break; 1325 case IFM_10G_SFP_CU: 1326 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1327 break; 1328 case IFM_1000_T: 1329 speed |= IXGBE_LINK_SPEED_100_FULL; 1330 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1331 break; 1332 case IFM_1000_LX: 1333 case IFM_1000_SX: 1334 case IFM_1000_CX: 1335 case IFM_1000_KX: 1336 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1337 break; 1338 case IFM_100_TX: 1339 speed |= IXGBE_LINK_SPEED_100_FULL; 1340 break; 1341 case IFM_10_T: 1342 speed |= IXGBE_LINK_SPEED_10_FULL; 1343 break; 1344 default: 1345 return (EINVAL); 1346 } 1347 1348 hw->mac.autotry_restart = TRUE; 1349 hw->mac.ops.setup_link(hw, speed, TRUE); 1350 1351 return (0); 1352 } 1353 1354 /********************************************************************* 1355 * 1356 * This routine maps the mbufs to tx descriptors, allowing the 1357 * TX engine to transmit the packets. 1358 * - return 0 on success, positive on failure 1359 * 1360 **********************************************************************/ 1361 1362 int 1363 ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head) 1364 { 1365 struct ix_softc *sc = txr->sc; 1366 uint32_t olinfo_status = 0, cmd_type_len; 1367 int i, j, ntxc; 1368 int first, last = 0; 1369 bus_dmamap_t map; 1370 struct ixgbe_tx_buf *txbuf; 1371 union ixgbe_adv_tx_desc *txd = NULL; 1372 1373 /* Basic descriptor defines */ 1374 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1375 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1376 1377 #if NVLAN > 0 1378 if (m_head->m_flags & M_VLANTAG) 1379 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1380 #endif 1381 1382 /* 1383 * Important to capture the first descriptor 1384 * used because it will contain the index of 1385 * the one we tell the hardware to report back 1386 */ 1387 first = txr->next_avail_desc; 1388 txbuf = &txr->tx_buffers[first]; 1389 map = txbuf->map; 1390 1391 /* 1392 * Map the packet for DMA. 1393 */ 1394 switch (bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1395 m_head, BUS_DMA_NOWAIT)) { 1396 case 0: 1397 break; 1398 case EFBIG: 1399 if (m_defrag(m_head, M_NOWAIT) == 0 && 1400 bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1401 m_head, BUS_DMA_NOWAIT) == 0) 1402 break; 1403 /* FALLTHROUGH */ 1404 default: 1405 sc->no_tx_dma_setup++; 1406 return (0); 1407 } 1408 1409 /* 1410 * Set the appropriate offload context 1411 * this will becomes the first descriptor. 1412 */ 1413 ntxc = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 1414 if (ntxc == -1) 1415 goto xmit_fail; 1416 1417 i = txr->next_avail_desc + ntxc; 1418 if (i >= sc->num_tx_desc) 1419 i -= sc->num_tx_desc; 1420 1421 for (j = 0; j < map->dm_nsegs; j++) { 1422 txd = &txr->tx_base[i]; 1423 1424 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 1425 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1426 cmd_type_len | map->dm_segs[j].ds_len); 1427 txd->read.olinfo_status = htole32(olinfo_status); 1428 last = i; /* descriptor that will get completion IRQ */ 1429 1430 if (++i == sc->num_tx_desc) 1431 i = 0; 1432 } 1433 1434 txd->read.cmd_type_len |= 1435 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1436 1437 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1438 BUS_DMASYNC_PREWRITE); 1439 1440 /* Set the index of the descriptor that will be marked done */ 1441 txbuf->m_head = m_head; 1442 txbuf->eop_index = last; 1443 1444 membar_producer(); 1445 1446 txr->next_avail_desc = i; 1447 1448 ++txr->tx_packets; 1449 return (ntxc + j); 1450 1451 xmit_fail: 1452 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 1453 return (0); 1454 } 1455 1456 void 1457 ixgbe_iff(struct ix_softc *sc) 1458 { 1459 struct ifnet *ifp = &sc->arpcom.ac_if; 1460 struct arpcom *ac = &sc->arpcom; 1461 uint32_t fctrl; 1462 uint8_t *mta; 1463 uint8_t *update_ptr; 1464 struct ether_multi *enm; 1465 struct ether_multistep step; 1466 int mcnt = 0; 1467 1468 IOCTL_DEBUGOUT("ixgbe_iff: begin"); 1469 1470 mta = sc->mta; 1471 bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1472 MAX_NUM_MULTICAST_ADDRESSES); 1473 1474 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1475 fctrl &= ~(IXGBE_FCTRL_MPE | IXGBE_FCTRL_UPE); 1476 ifp->if_flags &= ~IFF_ALLMULTI; 1477 1478 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1479 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1480 ifp->if_flags |= IFF_ALLMULTI; 1481 fctrl |= IXGBE_FCTRL_MPE; 1482 if (ifp->if_flags & IFF_PROMISC) 1483 fctrl |= IXGBE_FCTRL_UPE; 1484 } else { 1485 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1486 while (enm != NULL) { 1487 bcopy(enm->enm_addrlo, 1488 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1489 IXGBE_ETH_LENGTH_OF_ADDRESS); 1490 mcnt++; 1491 1492 ETHER_NEXT_MULTI(step, enm); 1493 } 1494 1495 update_ptr = mta; 1496 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1497 ixgbe_mc_array_itr, TRUE); 1498 } 1499 1500 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1501 } 1502 1503 /* 1504 * This is an iterator function now needed by the multicast 1505 * shared code. It simply feeds the shared code routine the 1506 * addresses in the array of ixgbe_iff() one by one. 1507 */ 1508 uint8_t * 1509 ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1510 { 1511 uint8_t *addr = *update_ptr; 1512 uint8_t *newptr; 1513 *vmdq = 0; 1514 1515 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1516 *update_ptr = newptr; 1517 return addr; 1518 } 1519 1520 void 1521 ixgbe_local_timer(void *arg) 1522 { 1523 struct ix_softc *sc = arg; 1524 #ifdef IX_DEBUG 1525 struct ifnet *ifp = &sc->arpcom.ac_if; 1526 #endif 1527 int s; 1528 1529 s = splnet(); 1530 1531 ixgbe_update_stats_counters(sc); 1532 1533 #ifdef IX_DEBUG 1534 if ((ifp->if_flags & (IFF_RUNNING|IFF_DEBUG)) == 1535 (IFF_RUNNING|IFF_DEBUG)) 1536 ixgbe_print_hw_stats(sc); 1537 #endif 1538 1539 timeout_add_sec(&sc->timer, 1); 1540 1541 splx(s); 1542 } 1543 1544 void 1545 ixgbe_update_link_status(struct ix_softc *sc) 1546 { 1547 struct ifnet *ifp = &sc->arpcom.ac_if; 1548 int link_state = LINK_STATE_DOWN; 1549 1550 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 1551 1552 ifp->if_baudrate = 0; 1553 if (sc->link_up) { 1554 link_state = LINK_STATE_FULL_DUPLEX; 1555 1556 switch (sc->link_speed) { 1557 case IXGBE_LINK_SPEED_UNKNOWN: 1558 ifp->if_baudrate = 0; 1559 break; 1560 case IXGBE_LINK_SPEED_100_FULL: 1561 ifp->if_baudrate = IF_Mbps(100); 1562 break; 1563 case IXGBE_LINK_SPEED_1GB_FULL: 1564 ifp->if_baudrate = IF_Gbps(1); 1565 break; 1566 case IXGBE_LINK_SPEED_10GB_FULL: 1567 ifp->if_baudrate = IF_Gbps(10); 1568 break; 1569 } 1570 1571 /* Update any Flow Control changes */ 1572 sc->hw.mac.ops.fc_enable(&sc->hw); 1573 } 1574 if (ifp->if_link_state != link_state) { 1575 ifp->if_link_state = link_state; 1576 if_link_state_change(ifp); 1577 } 1578 } 1579 1580 1581 /********************************************************************* 1582 * 1583 * This routine disables all traffic on the adapter by issuing a 1584 * global reset on the MAC and deallocates TX/RX buffers. 1585 * 1586 **********************************************************************/ 1587 1588 void 1589 ixgbe_stop(void *arg) 1590 { 1591 struct ix_softc *sc = arg; 1592 struct ifnet *ifp = &sc->arpcom.ac_if; 1593 int i; 1594 1595 /* Tell the stack that the interface is no longer active */ 1596 ifp->if_flags &= ~IFF_RUNNING; 1597 1598 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1599 ixgbe_disable_intr(sc); 1600 1601 sc->hw.mac.ops.reset_hw(&sc->hw); 1602 sc->hw.adapter_stopped = FALSE; 1603 sc->hw.mac.ops.stop_adapter(&sc->hw); 1604 if (sc->hw.mac.type == ixgbe_mac_82599EB) 1605 sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw); 1606 /* Turn off the laser */ 1607 if (sc->hw.mac.ops.disable_tx_laser) 1608 sc->hw.mac.ops.disable_tx_laser(&sc->hw); 1609 timeout_del(&sc->timer); 1610 1611 /* reprogram the RAR[0] in case user changed it. */ 1612 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1613 1614 intr_barrier(sc->tag); 1615 for (i = 0; i < sc->num_queues; i++) { 1616 struct ifqueue *ifq = ifp->if_ifqs[i]; 1617 ifq_barrier(ifq); 1618 ifq_clr_oactive(ifq); 1619 1620 timeout_del(&sc->rx_rings[i].rx_refill); 1621 } 1622 1623 KASSERT((ifp->if_flags & IFF_RUNNING) == 0); 1624 1625 /* Should we really clear all structures on stop? */ 1626 ixgbe_free_transmit_structures(sc); 1627 ixgbe_free_receive_structures(sc); 1628 } 1629 1630 1631 /********************************************************************* 1632 * 1633 * Determine hardware revision. 1634 * 1635 **********************************************************************/ 1636 void 1637 ixgbe_identify_hardware(struct ix_softc *sc) 1638 { 1639 struct ixgbe_osdep *os = &sc->osdep; 1640 struct pci_attach_args *pa = &os->os_pa; 1641 uint32_t reg; 1642 1643 /* Save off the information about this board */ 1644 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1645 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1646 1647 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1648 sc->hw.revision_id = PCI_REVISION(reg); 1649 1650 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1651 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1652 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1653 1654 /* We need this here to set the num_segs below */ 1655 ixgbe_set_mac_type(&sc->hw); 1656 1657 /* Pick up the 82599 and VF settings */ 1658 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1659 sc->hw.phy.smart_speed = ixgbe_smart_speed; 1660 sc->num_segs = IXGBE_82599_SCATTER; 1661 } 1662 1663 /********************************************************************* 1664 * 1665 * Setup the Legacy or MSI Interrupt handler 1666 * 1667 **********************************************************************/ 1668 int 1669 ixgbe_allocate_legacy(struct ix_softc *sc) 1670 { 1671 struct ixgbe_osdep *os = &sc->osdep; 1672 struct pci_attach_args *pa = &os->os_pa; 1673 const char *intrstr = NULL; 1674 pci_chipset_tag_t pc = pa->pa_pc; 1675 pci_intr_handle_t ih; 1676 1677 /* We allocate a single interrupt resource */ 1678 if (pci_intr_map_msi(pa, &ih) != 0 && 1679 pci_intr_map(pa, &ih) != 0) { 1680 printf(": couldn't map interrupt\n"); 1681 return (ENXIO); 1682 } 1683 1684 #if 0 1685 /* XXX */ 1686 /* Tasklets for Link, SFP and Multispeed Fiber */ 1687 TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc); 1688 TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc); 1689 TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc); 1690 #endif 1691 1692 intrstr = pci_intr_string(pc, ih); 1693 sc->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE, 1694 ixgbe_legacy_intr, sc, sc->dev.dv_xname); 1695 if (sc->tag == NULL) { 1696 printf(": couldn't establish interrupt"); 1697 if (intrstr != NULL) 1698 printf(" at %s", intrstr); 1699 printf("\n"); 1700 return (ENXIO); 1701 } 1702 printf(": %s", intrstr); 1703 1704 /* For simplicity in the handlers */ 1705 sc->que_mask = IXGBE_EIMS_ENABLE_MASK; 1706 1707 return (0); 1708 } 1709 1710 /********************************************************************* 1711 * 1712 * Setup the MSI-X Interrupt handlers 1713 * 1714 **********************************************************************/ 1715 int 1716 ixgbe_allocate_msix(struct ix_softc *sc) 1717 { 1718 struct ixgbe_osdep *os = &sc->osdep; 1719 struct pci_attach_args *pa = &os->os_pa; 1720 int vec, error = 0; 1721 struct ix_queue *que = sc->queues; 1722 const char *intrstr = NULL; 1723 pci_chipset_tag_t pc = pa->pa_pc; 1724 pci_intr_handle_t ih; 1725 1726 vec = 0; 1727 if (pci_intr_map_msix(pa, vec, &ih)) { 1728 printf(": couldn't map interrupt\n"); 1729 return (ENXIO); 1730 } 1731 1732 que->msix = vec; 1733 snprintf(que->name, sizeof(que->name), "%s:%d", sc->dev.dv_xname, vec); 1734 1735 intrstr = pci_intr_string(pc, ih); 1736 que->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE, 1737 ixgbe_queue_intr, que, que->name); 1738 if (que->tag == NULL) { 1739 printf(": couldn't establish interrupt"); 1740 if (intrstr != NULL) 1741 printf(" at %s", intrstr); 1742 printf("\n"); 1743 return (ENXIO); 1744 } 1745 1746 /* Now the link status/control last MSI-X vector */ 1747 vec++; 1748 if (pci_intr_map_msix(pa, vec, &ih)) { 1749 printf(": couldn't map link vector\n"); 1750 error = ENXIO; 1751 goto fail; 1752 } 1753 1754 intrstr = pci_intr_string(pc, ih); 1755 sc->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE, 1756 ixgbe_link_intr, sc, sc->dev.dv_xname); 1757 if (sc->tag == NULL) { 1758 printf(": couldn't establish interrupt"); 1759 if (intrstr != NULL) 1760 printf(" at %s", intrstr); 1761 printf("\n"); 1762 error = ENXIO; 1763 goto fail; 1764 } 1765 1766 sc->linkvec = vec; 1767 printf(", %s, %d queue%s", intrstr, vec, (vec > 1) ? "s" : ""); 1768 1769 return (0); 1770 fail: 1771 pci_intr_disestablish(pc, que->tag); 1772 que->tag = NULL; 1773 return (error); 1774 } 1775 1776 int 1777 ixgbe_setup_msix(struct ix_softc *sc) 1778 { 1779 struct ixgbe_osdep *os = &sc->osdep; 1780 struct pci_attach_args *pa = &os->os_pa; 1781 pci_intr_handle_t dummy; 1782 1783 if (!ixgbe_enable_msix) 1784 return (0); 1785 1786 /* 1787 * Try a dummy map, maybe this bus doesn't like MSI, this function 1788 * has no side effects. 1789 */ 1790 if (pci_intr_map_msix(pa, 0, &dummy)) 1791 return (0); 1792 1793 return (2); /* queue vector + link vector */ 1794 } 1795 1796 int 1797 ixgbe_allocate_pci_resources(struct ix_softc *sc) 1798 { 1799 struct ixgbe_osdep *os = &sc->osdep; 1800 struct pci_attach_args *pa = &os->os_pa; 1801 int val; 1802 1803 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1804 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) { 1805 printf(": mmba is not mem space\n"); 1806 return (ENXIO); 1807 } 1808 1809 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1810 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1811 printf(": cannot find mem space\n"); 1812 return (ENXIO); 1813 } 1814 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1815 1816 /* Legacy defaults */ 1817 sc->num_queues = 1; 1818 sc->hw.back = os; 1819 1820 /* Now setup MSI or MSI/X, return us the number of supported vectors. */ 1821 sc->msix = ixgbe_setup_msix(sc); 1822 1823 return (0); 1824 } 1825 1826 void 1827 ixgbe_free_pci_resources(struct ix_softc * sc) 1828 { 1829 struct ixgbe_osdep *os = &sc->osdep; 1830 struct pci_attach_args *pa = &os->os_pa; 1831 struct ix_queue *que = sc->queues; 1832 int i; 1833 1834 /* Release all msix queue resources: */ 1835 for (i = 0; i < sc->num_queues; i++, que++) { 1836 if (que->tag) 1837 pci_intr_disestablish(pa->pa_pc, que->tag); 1838 que->tag = NULL; 1839 } 1840 1841 if (sc->tag) 1842 pci_intr_disestablish(pa->pa_pc, sc->tag); 1843 sc->tag = NULL; 1844 if (os->os_membase != 0) 1845 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1846 os->os_membase = 0; 1847 } 1848 1849 /********************************************************************* 1850 * 1851 * Setup networking device structure and register an interface. 1852 * 1853 **********************************************************************/ 1854 void 1855 ixgbe_setup_interface(struct ix_softc *sc) 1856 { 1857 struct ifnet *ifp = &sc->arpcom.ac_if; 1858 int i; 1859 1860 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1861 ifp->if_softc = sc; 1862 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1863 ifp->if_xflags = IFXF_MPSAFE; 1864 ifp->if_ioctl = ixgbe_ioctl; 1865 ifp->if_qstart = ixgbe_start; 1866 ifp->if_timer = 0; 1867 ifp->if_watchdog = ixgbe_watchdog; 1868 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1869 ETHER_HDR_LEN - ETHER_CRC_LEN; 1870 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1871 1872 ifp->if_capabilities = IFCAP_VLAN_MTU; 1873 1874 #if NVLAN > 0 1875 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1876 #endif 1877 1878 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 1879 1880 /* 1881 * Specify the media types supported by this sc and register 1882 * callbacks to update media and link information 1883 */ 1884 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1885 ixgbe_media_status); 1886 ixgbe_add_media_types(sc); 1887 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1888 1889 if_attach(ifp); 1890 ether_ifattach(ifp); 1891 1892 if_attach_queues(ifp, sc->num_queues); 1893 if_attach_iqueues(ifp, sc->num_queues); 1894 for (i = 0; i < sc->num_queues; i++) { 1895 struct ifqueue *ifq = ifp->if_ifqs[i]; 1896 struct ifiqueue *ifiq = ifp->if_iqs[i]; 1897 struct tx_ring *txr = &sc->tx_rings[i]; 1898 struct rx_ring *rxr = &sc->rx_rings[i]; 1899 1900 ifq->ifq_softc = txr; 1901 txr->ifq = ifq; 1902 1903 ifiq->ifiq_softc = rxr; 1904 rxr->ifiq = ifiq; 1905 } 1906 1907 sc->max_frame_size = IXGBE_MAX_FRAME_SIZE; 1908 } 1909 1910 void 1911 ixgbe_add_media_types(struct ix_softc *sc) 1912 { 1913 struct ixgbe_hw *hw = &sc->hw; 1914 uint64_t layer; 1915 1916 sc->phy_layer = hw->mac.ops.get_supported_physical_layer(hw); 1917 layer = sc->phy_layer; 1918 1919 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1920 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL); 1921 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1922 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 1923 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1924 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1925 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1926 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 1927 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SFP_CU, 0, NULL); 1928 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1929 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL); 1930 if (hw->phy.multispeed_fiber) 1931 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_LX, 0, 1932 NULL); 1933 } 1934 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1935 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 1936 if (hw->phy.multispeed_fiber) 1937 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, 1938 NULL); 1939 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 1940 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 1941 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 1942 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); 1943 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 1944 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL); 1945 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) 1946 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); 1947 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 1948 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL); 1949 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) 1950 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL); 1951 1952 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 1953 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, 1954 NULL); 1955 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 1956 } 1957 1958 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1959 } 1960 1961 void 1962 ixgbe_config_link(struct ix_softc *sc) 1963 { 1964 uint32_t autoneg, err = 0; 1965 bool negotiate; 1966 1967 if (ixgbe_is_sfp(&sc->hw)) { 1968 if (sc->hw.phy.multispeed_fiber) { 1969 sc->hw.mac.ops.setup_sfp(&sc->hw); 1970 if (sc->hw.mac.ops.enable_tx_laser) 1971 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 1972 ixgbe_handle_msf(sc); 1973 } else 1974 ixgbe_handle_mod(sc); 1975 } else { 1976 if (sc->hw.mac.ops.check_link) 1977 err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg, 1978 &sc->link_up, FALSE); 1979 if (err) 1980 return; 1981 autoneg = sc->hw.phy.autoneg_advertised; 1982 if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities)) 1983 err = sc->hw.mac.ops.get_link_capabilities(&sc->hw, 1984 &autoneg, &negotiate); 1985 if (err) 1986 return; 1987 if (sc->hw.mac.ops.setup_link) 1988 sc->hw.mac.ops.setup_link(&sc->hw, 1989 autoneg, sc->link_up); 1990 } 1991 } 1992 1993 /******************************************************************** 1994 * Manage DMA'able memory. 1995 *******************************************************************/ 1996 int 1997 ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 1998 struct ixgbe_dma_alloc *dma, int mapflags) 1999 { 2000 struct ifnet *ifp = &sc->arpcom.ac_if; 2001 struct ixgbe_osdep *os = &sc->osdep; 2002 int r; 2003 2004 dma->dma_tag = os->os_pa.pa_dmat; 2005 r = bus_dmamap_create(dma->dma_tag, size, 1, 2006 size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 2007 if (r != 0) { 2008 printf("%s: ixgbe_dma_malloc: bus_dmamap_create failed; " 2009 "error %u\n", ifp->if_xname, r); 2010 goto fail_0; 2011 } 2012 2013 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 2014 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 2015 if (r != 0) { 2016 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2017 "error %u\n", ifp->if_xname, r); 2018 goto fail_1; 2019 } 2020 2021 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 2022 &dma->dma_vaddr, BUS_DMA_NOWAIT); 2023 if (r != 0) { 2024 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 2025 "error %u\n", ifp->if_xname, r); 2026 goto fail_2; 2027 } 2028 2029 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2030 size, NULL, mapflags | BUS_DMA_NOWAIT); 2031 if (r != 0) { 2032 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 2033 "error %u\n", ifp->if_xname, r); 2034 goto fail_3; 2035 } 2036 2037 dma->dma_size = size; 2038 return (0); 2039 fail_3: 2040 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 2041 fail_2: 2042 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2043 fail_1: 2044 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2045 fail_0: 2046 dma->dma_map = NULL; 2047 dma->dma_tag = NULL; 2048 return (r); 2049 } 2050 2051 void 2052 ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 2053 { 2054 if (dma->dma_tag == NULL) 2055 return; 2056 2057 if (dma->dma_map != NULL) { 2058 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 2059 dma->dma_map->dm_mapsize, 2060 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2061 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2062 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 2063 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2064 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2065 dma->dma_map = NULL; 2066 } 2067 } 2068 2069 2070 /********************************************************************* 2071 * 2072 * Allocate memory for the transmit and receive rings, and then 2073 * the descriptors associated with each, called only once at attach. 2074 * 2075 **********************************************************************/ 2076 int 2077 ixgbe_allocate_queues(struct ix_softc *sc) 2078 { 2079 struct ifnet *ifp = &sc->arpcom.ac_if; 2080 struct ix_queue *que; 2081 struct tx_ring *txr; 2082 struct rx_ring *rxr; 2083 int rsize, tsize; 2084 int txconf = 0, rxconf = 0, i; 2085 2086 /* First allocate the top level queue structs */ 2087 if (!(sc->queues = mallocarray(sc->num_queues, 2088 sizeof(struct ix_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2089 printf("%s: Unable to allocate queue memory\n", ifp->if_xname); 2090 goto fail; 2091 } 2092 2093 /* Then allocate the TX ring struct memory */ 2094 if (!(sc->tx_rings = mallocarray(sc->num_queues, 2095 sizeof(struct tx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2096 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 2097 goto fail; 2098 } 2099 2100 /* Next allocate the RX */ 2101 if (!(sc->rx_rings = mallocarray(sc->num_queues, 2102 sizeof(struct rx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2103 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 2104 goto rx_fail; 2105 } 2106 2107 /* For the ring itself */ 2108 tsize = roundup2(sc->num_tx_desc * 2109 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2110 2111 /* 2112 * Now set up the TX queues, txconf is needed to handle the 2113 * possibility that things fail midcourse and we need to 2114 * undo memory gracefully 2115 */ 2116 for (i = 0; i < sc->num_queues; i++, txconf++) { 2117 /* Set up some basics */ 2118 txr = &sc->tx_rings[i]; 2119 txr->sc = sc; 2120 txr->me = i; 2121 2122 if (ixgbe_dma_malloc(sc, tsize, 2123 &txr->txdma, BUS_DMA_NOWAIT)) { 2124 printf("%s: Unable to allocate TX Descriptor memory\n", 2125 ifp->if_xname); 2126 goto err_tx_desc; 2127 } 2128 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2129 bzero((void *)txr->tx_base, tsize); 2130 } 2131 2132 /* 2133 * Next the RX queues... 2134 */ 2135 rsize = roundup2(sc->num_rx_desc * 2136 sizeof(union ixgbe_adv_rx_desc), 4096); 2137 for (i = 0; i < sc->num_queues; i++, rxconf++) { 2138 rxr = &sc->rx_rings[i]; 2139 /* Set up some basics */ 2140 rxr->sc = sc; 2141 rxr->me = i; 2142 timeout_set(&rxr->rx_refill, ixgbe_rxrefill, rxr); 2143 2144 if (ixgbe_dma_malloc(sc, rsize, 2145 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2146 printf("%s: Unable to allocate RxDescriptor memory\n", 2147 ifp->if_xname); 2148 goto err_rx_desc; 2149 } 2150 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2151 bzero((void *)rxr->rx_base, rsize); 2152 } 2153 2154 /* 2155 * Finally set up the queue holding structs 2156 */ 2157 for (i = 0; i < sc->num_queues; i++) { 2158 que = &sc->queues[i]; 2159 que->sc = sc; 2160 que->txr = &sc->tx_rings[i]; 2161 que->rxr = &sc->rx_rings[i]; 2162 } 2163 2164 return (0); 2165 2166 err_rx_desc: 2167 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 2168 ixgbe_dma_free(sc, &rxr->rxdma); 2169 err_tx_desc: 2170 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 2171 ixgbe_dma_free(sc, &txr->txdma); 2172 free(sc->rx_rings, M_DEVBUF, sc->num_queues * sizeof(struct rx_ring)); 2173 sc->rx_rings = NULL; 2174 rx_fail: 2175 free(sc->tx_rings, M_DEVBUF, sc->num_queues * sizeof(struct tx_ring)); 2176 sc->tx_rings = NULL; 2177 fail: 2178 return (ENOMEM); 2179 } 2180 2181 /********************************************************************* 2182 * 2183 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2184 * the information needed to transmit a packet on the wire. This is 2185 * called only once at attach, setup is done every reset. 2186 * 2187 **********************************************************************/ 2188 int 2189 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2190 { 2191 struct ix_softc *sc = txr->sc; 2192 struct ifnet *ifp = &sc->arpcom.ac_if; 2193 struct ixgbe_tx_buf *txbuf; 2194 int error, i; 2195 2196 if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc, 2197 sizeof(struct ixgbe_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2198 printf("%s: Unable to allocate tx_buffer memory\n", 2199 ifp->if_xname); 2200 error = ENOMEM; 2201 goto fail; 2202 } 2203 txr->txtag = txr->txdma.dma_tag; 2204 2205 /* Create the descriptor buffer dma maps */ 2206 for (i = 0; i < sc->num_tx_desc; i++) { 2207 txbuf = &txr->tx_buffers[i]; 2208 error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE, 2209 sc->num_segs, PAGE_SIZE, 0, 2210 BUS_DMA_NOWAIT, &txbuf->map); 2211 2212 if (error != 0) { 2213 printf("%s: Unable to create TX DMA map\n", 2214 ifp->if_xname); 2215 goto fail; 2216 } 2217 } 2218 2219 return 0; 2220 fail: 2221 return (error); 2222 } 2223 2224 /********************************************************************* 2225 * 2226 * Initialize a transmit ring. 2227 * 2228 **********************************************************************/ 2229 int 2230 ixgbe_setup_transmit_ring(struct tx_ring *txr) 2231 { 2232 struct ix_softc *sc = txr->sc; 2233 int error; 2234 2235 /* Now allocate transmit buffers for the ring */ 2236 if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0) 2237 return (error); 2238 2239 /* Clear the old ring contents */ 2240 bzero((void *)txr->tx_base, 2241 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 2242 2243 /* Reset indices */ 2244 txr->next_avail_desc = 0; 2245 txr->next_to_clean = 0; 2246 2247 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2248 0, txr->txdma.dma_map->dm_mapsize, 2249 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2250 2251 return (0); 2252 } 2253 2254 /********************************************************************* 2255 * 2256 * Initialize all transmit rings. 2257 * 2258 **********************************************************************/ 2259 int 2260 ixgbe_setup_transmit_structures(struct ix_softc *sc) 2261 { 2262 struct tx_ring *txr = sc->tx_rings; 2263 int i, error; 2264 2265 for (i = 0; i < sc->num_queues; i++, txr++) { 2266 if ((error = ixgbe_setup_transmit_ring(txr)) != 0) 2267 goto fail; 2268 } 2269 2270 return (0); 2271 fail: 2272 ixgbe_free_transmit_structures(sc); 2273 return (error); 2274 } 2275 2276 /********************************************************************* 2277 * 2278 * Enable transmit unit. 2279 * 2280 **********************************************************************/ 2281 void 2282 ixgbe_initialize_transmit_units(struct ix_softc *sc) 2283 { 2284 struct ifnet *ifp = &sc->arpcom.ac_if; 2285 struct tx_ring *txr; 2286 struct ixgbe_hw *hw = &sc->hw; 2287 int i; 2288 uint64_t tdba; 2289 uint32_t txctrl; 2290 2291 /* Setup the Base and Length of the Tx Descriptor Ring */ 2292 2293 for (i = 0; i < sc->num_queues; i++) { 2294 txr = &sc->tx_rings[i]; 2295 2296 /* Setup descriptor base address */ 2297 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 2298 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 2299 (tdba & 0x00000000ffffffffULL)); 2300 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 2301 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2302 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 2303 2304 /* Setup the HW Tx Head and Tail descriptor pointers */ 2305 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2306 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2307 2308 /* Setup Transmit Descriptor Cmd Settings */ 2309 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2310 txr->queue_status = IXGBE_QUEUE_IDLE; 2311 txr->watchdog_timer = 0; 2312 2313 /* Disable Head Writeback */ 2314 switch (hw->mac.type) { 2315 case ixgbe_mac_82598EB: 2316 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2317 break; 2318 case ixgbe_mac_82599EB: 2319 case ixgbe_mac_X540: 2320 default: 2321 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2322 break; 2323 } 2324 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2325 switch (hw->mac.type) { 2326 case ixgbe_mac_82598EB: 2327 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2328 break; 2329 case ixgbe_mac_82599EB: 2330 case ixgbe_mac_X540: 2331 default: 2332 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2333 break; 2334 } 2335 } 2336 ifp->if_timer = 0; 2337 2338 if (hw->mac.type != ixgbe_mac_82598EB) { 2339 uint32_t dmatxctl, rttdcs; 2340 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2341 dmatxctl |= IXGBE_DMATXCTL_TE; 2342 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2343 /* Disable arbiter to set MTQC */ 2344 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2345 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2346 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2347 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2348 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2349 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2350 } 2351 } 2352 2353 /********************************************************************* 2354 * 2355 * Free all transmit rings. 2356 * 2357 **********************************************************************/ 2358 void 2359 ixgbe_free_transmit_structures(struct ix_softc *sc) 2360 { 2361 struct tx_ring *txr = sc->tx_rings; 2362 int i; 2363 2364 for (i = 0; i < sc->num_queues; i++, txr++) 2365 ixgbe_free_transmit_buffers(txr); 2366 } 2367 2368 /********************************************************************* 2369 * 2370 * Free transmit ring related data structures. 2371 * 2372 **********************************************************************/ 2373 void 2374 ixgbe_free_transmit_buffers(struct tx_ring *txr) 2375 { 2376 struct ix_softc *sc = txr->sc; 2377 struct ixgbe_tx_buf *tx_buffer; 2378 int i; 2379 2380 INIT_DEBUGOUT("free_transmit_ring: begin"); 2381 2382 if (txr->tx_buffers == NULL) 2383 return; 2384 2385 tx_buffer = txr->tx_buffers; 2386 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 2387 if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) { 2388 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2389 0, tx_buffer->map->dm_mapsize, 2390 BUS_DMASYNC_POSTWRITE); 2391 bus_dmamap_unload(txr->txdma.dma_tag, 2392 tx_buffer->map); 2393 } 2394 if (tx_buffer->m_head != NULL) { 2395 m_freem(tx_buffer->m_head); 2396 tx_buffer->m_head = NULL; 2397 } 2398 if (tx_buffer->map != NULL) { 2399 bus_dmamap_destroy(txr->txdma.dma_tag, 2400 tx_buffer->map); 2401 tx_buffer->map = NULL; 2402 } 2403 } 2404 2405 if (txr->tx_buffers != NULL) 2406 free(txr->tx_buffers, M_DEVBUF, 2407 sc->num_tx_desc * sizeof(struct ixgbe_tx_buf)); 2408 txr->tx_buffers = NULL; 2409 txr->txtag = NULL; 2410 } 2411 2412 /********************************************************************* 2413 * 2414 * Advanced Context Descriptor setup for VLAN or CSUM 2415 * 2416 **********************************************************************/ 2417 2418 int 2419 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, 2420 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2421 { 2422 struct ixgbe_adv_tx_context_desc *TXD; 2423 struct ixgbe_tx_buf *tx_buffer; 2424 #if NVLAN > 0 2425 struct ether_vlan_header *eh; 2426 #else 2427 struct ether_header *eh; 2428 #endif 2429 struct ip *ip; 2430 #ifdef notyet 2431 struct ip6_hdr *ip6; 2432 #endif 2433 struct mbuf *m; 2434 int ipoff; 2435 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2436 int ehdrlen, ip_hlen = 0; 2437 uint16_t etype; 2438 uint8_t ipproto = 0; 2439 int offload = TRUE; 2440 int ctxd = txr->next_avail_desc; 2441 #if NVLAN > 0 2442 uint16_t vtag = 0; 2443 #endif 2444 2445 #if notyet 2446 /* First check if TSO is to be used */ 2447 if (mp->m_pkthdr.csum_flags & CSUM_TSO) 2448 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status)); 2449 #endif 2450 2451 if ((mp->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT)) == 0) 2452 offload = FALSE; 2453 2454 /* Indicate the whole packet as payload when not doing TSO */ 2455 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2456 2457 /* Now ready a context descriptor */ 2458 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2459 tx_buffer = &txr->tx_buffers[ctxd]; 2460 2461 /* 2462 * In advanced descriptors the vlan tag must 2463 * be placed into the descriptor itself. Hence 2464 * we need to make one even if not doing offloads. 2465 */ 2466 #if NVLAN > 0 2467 if (mp->m_flags & M_VLANTAG) { 2468 vtag = mp->m_pkthdr.ether_vtag; 2469 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2470 } else 2471 #endif 2472 if (offload == FALSE) 2473 return (0); /* No need for CTX */ 2474 2475 /* 2476 * Determine where frame payload starts. 2477 * Jump over vlan headers if already present, 2478 * helpful for QinQ too. 2479 */ 2480 if (mp->m_len < sizeof(struct ether_header)) 2481 return (-1); 2482 #if NVLAN > 0 2483 eh = mtod(mp, struct ether_vlan_header *); 2484 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2485 if (mp->m_len < sizeof(struct ether_vlan_header)) 2486 return (-1); 2487 etype = ntohs(eh->evl_proto); 2488 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2489 } else { 2490 etype = ntohs(eh->evl_encap_proto); 2491 ehdrlen = ETHER_HDR_LEN; 2492 } 2493 #else 2494 eh = mtod(mp, struct ether_header *); 2495 etype = ntohs(eh->ether_type); 2496 ehdrlen = ETHER_HDR_LEN; 2497 #endif 2498 2499 /* Set the ether header length */ 2500 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2501 2502 switch (etype) { 2503 case ETHERTYPE_IP: 2504 if (mp->m_pkthdr.len < ehdrlen + sizeof(*ip)) 2505 return (-1); 2506 m = m_getptr(mp, ehdrlen, &ipoff); 2507 KASSERT(m != NULL && m->m_len - ipoff >= sizeof(*ip)); 2508 ip = (struct ip *)(m->m_data + ipoff); 2509 ip_hlen = ip->ip_hl << 2; 2510 ipproto = ip->ip_p; 2511 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2512 break; 2513 #ifdef notyet 2514 case ETHERTYPE_IPV6: 2515 if (mp->m_pkthdr.len < ehdrlen + sizeof(*ip6)) 2516 return (-1); 2517 m = m_getptr(mp, ehdrlen, &ipoff); 2518 KASSERT(m != NULL && m->m_len - ipoff >= sizeof(*ip6)); 2519 ip6 = (struct ip6 *)(m->m_data + ipoff); 2520 ip_hlen = sizeof(*ip6); 2521 /* XXX-BZ this will go badly in case of ext hdrs. */ 2522 ipproto = ip6->ip6_nxt; 2523 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2524 break; 2525 #endif 2526 default: 2527 offload = FALSE; 2528 break; 2529 } 2530 2531 vlan_macip_lens |= ip_hlen; 2532 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2533 2534 switch (ipproto) { 2535 case IPPROTO_TCP: 2536 if (mp->m_pkthdr.csum_flags & M_TCP_CSUM_OUT) 2537 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2538 break; 2539 case IPPROTO_UDP: 2540 if (mp->m_pkthdr.csum_flags & M_UDP_CSUM_OUT) 2541 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2542 break; 2543 default: 2544 offload = FALSE; 2545 break; 2546 } 2547 2548 if (offload) /* For the TX descriptor setup */ 2549 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2550 2551 /* Now copy bits into descriptor */ 2552 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2553 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2554 TXD->seqnum_seed = htole32(0); 2555 TXD->mss_l4len_idx = htole32(0); 2556 2557 tx_buffer->m_head = NULL; 2558 tx_buffer->eop_index = -1; 2559 2560 return (1); 2561 } 2562 2563 /********************************************************************** 2564 * 2565 * Examine each tx_buffer in the used queue. If the hardware is done 2566 * processing the packet then free associated resources. The 2567 * tx_buffer is put back on the free queue. 2568 * 2569 **********************************************************************/ 2570 int 2571 ixgbe_txeof(struct tx_ring *txr) 2572 { 2573 struct ix_softc *sc = txr->sc; 2574 struct ifqueue *ifq = txr->ifq; 2575 struct ifnet *ifp = &sc->arpcom.ac_if; 2576 unsigned int head, tail, last; 2577 struct ixgbe_tx_buf *tx_buffer; 2578 struct ixgbe_legacy_tx_desc *tx_desc; 2579 2580 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2581 return FALSE; 2582 2583 head = txr->next_avail_desc; 2584 tail = txr->next_to_clean; 2585 2586 membar_consumer(); 2587 2588 if (head == tail) 2589 return (FALSE); 2590 2591 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2592 0, txr->txdma.dma_map->dm_mapsize, 2593 BUS_DMASYNC_POSTREAD); 2594 2595 for (;;) { 2596 tx_buffer = &txr->tx_buffers[tail]; 2597 last = tx_buffer->eop_index; 2598 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2599 2600 if (!ISSET(tx_desc->upper.fields.status, IXGBE_TXD_STAT_DD)) 2601 break; 2602 2603 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2604 0, tx_buffer->map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2605 bus_dmamap_unload(txr->txdma.dma_tag, tx_buffer->map); 2606 m_freem(tx_buffer->m_head); 2607 2608 tx_buffer->m_head = NULL; 2609 tx_buffer->eop_index = -1; 2610 2611 tail = last + 1; 2612 if (tail == sc->num_tx_desc) 2613 tail = 0; 2614 if (head == tail) { 2615 /* All clean, turn off the timer */ 2616 ifp->if_timer = 0; 2617 break; 2618 } 2619 } 2620 2621 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2622 0, txr->txdma.dma_map->dm_mapsize, 2623 BUS_DMASYNC_PREREAD); 2624 2625 membar_producer(); 2626 2627 txr->next_to_clean = tail; 2628 2629 if (ifq_is_oactive(ifq)) 2630 ifq_restart(ifq); 2631 2632 return TRUE; 2633 } 2634 2635 /********************************************************************* 2636 * 2637 * Get a buffer from system mbuf buffer pool. 2638 * 2639 **********************************************************************/ 2640 int 2641 ixgbe_get_buf(struct rx_ring *rxr, int i) 2642 { 2643 struct ix_softc *sc = rxr->sc; 2644 struct ixgbe_rx_buf *rxbuf; 2645 struct mbuf *mp; 2646 int error; 2647 union ixgbe_adv_rx_desc *rxdesc; 2648 2649 rxbuf = &rxr->rx_buffers[i]; 2650 rxdesc = &rxr->rx_base[i]; 2651 if (rxbuf->buf) { 2652 printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n", 2653 sc->dev.dv_xname, i); 2654 return (ENOBUFS); 2655 } 2656 2657 /* needed in any case so prealocate since this one will fail for sure */ 2658 mp = MCLGETI(NULL, M_DONTWAIT, NULL, sc->rx_mbuf_sz); 2659 if (!mp) 2660 return (ENOBUFS); 2661 2662 mp->m_data += (mp->m_ext.ext_size - sc->rx_mbuf_sz); 2663 mp->m_len = mp->m_pkthdr.len = sc->rx_mbuf_sz; 2664 2665 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, 2666 mp, BUS_DMA_NOWAIT); 2667 if (error) { 2668 m_freem(mp); 2669 return (error); 2670 } 2671 2672 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2673 0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 2674 rxbuf->buf = mp; 2675 2676 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 2677 2678 return (0); 2679 } 2680 2681 /********************************************************************* 2682 * 2683 * Allocate memory for rx_buffer structures. Since we use one 2684 * rx_buffer per received packet, the maximum number of rx_buffer's 2685 * that we'll need is equal to the number of receive descriptors 2686 * that we've allocated. 2687 * 2688 **********************************************************************/ 2689 int 2690 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2691 { 2692 struct ix_softc *sc = rxr->sc; 2693 struct ifnet *ifp = &sc->arpcom.ac_if; 2694 struct ixgbe_rx_buf *rxbuf; 2695 int i, error; 2696 2697 if (!(rxr->rx_buffers = mallocarray(sc->num_rx_desc, 2698 sizeof(struct ixgbe_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2699 printf("%s: Unable to allocate rx_buffer memory\n", 2700 ifp->if_xname); 2701 error = ENOMEM; 2702 goto fail; 2703 } 2704 2705 rxbuf = rxr->rx_buffers; 2706 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2707 error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1, 2708 16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map); 2709 if (error) { 2710 printf("%s: Unable to create Pack DMA map\n", 2711 ifp->if_xname); 2712 goto fail; 2713 } 2714 } 2715 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2716 rxr->rxdma.dma_map->dm_mapsize, 2717 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2718 2719 return (0); 2720 2721 fail: 2722 return (error); 2723 } 2724 2725 /********************************************************************* 2726 * 2727 * Initialize a receive ring and its buffers. 2728 * 2729 **********************************************************************/ 2730 int 2731 ixgbe_setup_receive_ring(struct rx_ring *rxr) 2732 { 2733 struct ix_softc *sc = rxr->sc; 2734 struct ifnet *ifp = &sc->arpcom.ac_if; 2735 int rsize, error; 2736 2737 rsize = roundup2(sc->num_rx_desc * 2738 sizeof(union ixgbe_adv_rx_desc), 4096); 2739 /* Clear the ring contents */ 2740 bzero((void *)rxr->rx_base, rsize); 2741 2742 if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0) 2743 return (error); 2744 2745 /* Setup our descriptor indices */ 2746 rxr->next_to_check = 0; 2747 rxr->last_desc_filled = sc->num_rx_desc - 1; 2748 2749 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2750 sc->num_rx_desc - 1); 2751 2752 ixgbe_rxfill(rxr); 2753 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 2754 printf("%s: unable to fill any rx descriptors\n", 2755 sc->dev.dv_xname); 2756 return (ENOBUFS); 2757 } 2758 2759 return (0); 2760 } 2761 2762 int 2763 ixgbe_rxfill(struct rx_ring *rxr) 2764 { 2765 struct ix_softc *sc = rxr->sc; 2766 int post = 0; 2767 u_int slots; 2768 int i; 2769 2770 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2771 0, rxr->rxdma.dma_map->dm_mapsize, 2772 BUS_DMASYNC_POSTWRITE); 2773 2774 i = rxr->last_desc_filled; 2775 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); 2776 slots > 0; slots--) { 2777 if (++i == sc->num_rx_desc) 2778 i = 0; 2779 2780 if (ixgbe_get_buf(rxr, i) != 0) 2781 break; 2782 2783 rxr->last_desc_filled = i; 2784 post = 1; 2785 } 2786 2787 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2788 0, rxr->rxdma.dma_map->dm_mapsize, 2789 BUS_DMASYNC_PREWRITE); 2790 2791 if_rxr_put(&rxr->rx_ring, slots); 2792 2793 return (post); 2794 } 2795 2796 void 2797 ixgbe_rxrefill(void *xrxr) 2798 { 2799 struct rx_ring *rxr = xrxr; 2800 struct ix_softc *sc = rxr->sc; 2801 2802 if (ixgbe_rxfill(rxr)) { 2803 /* Advance the Rx Queue "Tail Pointer" */ 2804 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me), 2805 rxr->last_desc_filled); 2806 } else if (if_rxr_inuse(&rxr->rx_ring) == 0) 2807 timeout_add(&rxr->rx_refill, 1); 2808 2809 } 2810 2811 /********************************************************************* 2812 * 2813 * Initialize all receive rings. 2814 * 2815 **********************************************************************/ 2816 int 2817 ixgbe_setup_receive_structures(struct ix_softc *sc) 2818 { 2819 struct rx_ring *rxr = sc->rx_rings; 2820 int i; 2821 2822 for (i = 0; i < sc->num_queues; i++, rxr++) 2823 if (ixgbe_setup_receive_ring(rxr)) 2824 goto fail; 2825 2826 return (0); 2827 fail: 2828 ixgbe_free_receive_structures(sc); 2829 return (ENOBUFS); 2830 } 2831 2832 /********************************************************************* 2833 * 2834 * Setup receive registers and features. 2835 * 2836 **********************************************************************/ 2837 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2838 2839 void 2840 ixgbe_initialize_receive_units(struct ix_softc *sc) 2841 { 2842 struct rx_ring *rxr = sc->rx_rings; 2843 struct ixgbe_hw *hw = &sc->hw; 2844 uint32_t bufsz, fctrl, srrctl, rxcsum; 2845 uint32_t hlreg; 2846 int i; 2847 2848 /* 2849 * Make sure receives are disabled while 2850 * setting up the descriptor ring 2851 */ 2852 ixgbe_disable_rx(hw); 2853 2854 /* Enable broadcasts */ 2855 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2856 fctrl |= IXGBE_FCTRL_BAM; 2857 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 2858 fctrl |= IXGBE_FCTRL_DPF; 2859 fctrl |= IXGBE_FCTRL_PMCF; 2860 } 2861 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 2862 2863 /* Always enable jumbo frame reception */ 2864 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2865 hlreg |= IXGBE_HLREG0_JUMBOEN; 2866 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2867 2868 bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2869 2870 for (i = 0; i < sc->num_queues; i++, rxr++) { 2871 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2872 2873 /* Setup the Base and Length of the Rx Descriptor Ring */ 2874 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 2875 (rdba & 0x00000000ffffffffULL)); 2876 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 2877 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 2878 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2879 2880 /* Set up the SRRCTL register */ 2881 srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2882 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 2883 2884 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2885 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 2886 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 2887 } 2888 2889 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 2890 uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR | 2891 IXGBE_PSRTYPE_UDPHDR | 2892 IXGBE_PSRTYPE_IPV4HDR | 2893 IXGBE_PSRTYPE_IPV6HDR; 2894 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 2895 } 2896 2897 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2898 rxcsum &= ~IXGBE_RXCSUM_PCSD; 2899 2900 ixgbe_initialize_rss_mapping(sc); 2901 2902 /* Setup RSS */ 2903 if (sc->num_queues > 1) { 2904 /* RSS and RX IPP Checksum are mutually exclusive */ 2905 rxcsum |= IXGBE_RXCSUM_PCSD; 2906 } 2907 2908 #ifdef IX_DEBUG 2909 /* Map QPRC/QPRDC/QPTC on a per queue basis */ 2910 ixgbe_map_queue_statistics(sc); 2911 #endif 2912 2913 /* This is useful for calculating UDP/IP fragment checksums */ 2914 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2915 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2916 2917 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2918 } 2919 2920 void 2921 ixgbe_initialize_rss_mapping(struct ix_softc *sc) 2922 { 2923 struct ixgbe_hw *hw = &sc->hw; 2924 uint32_t reta = 0, mrqc, rss_key[10]; 2925 int i, j, queue_id, table_size, index_mult; 2926 2927 /* set up random bits */ 2928 arc4random_buf(&rss_key, sizeof(rss_key)); 2929 2930 /* Set multiplier for RETA setup and table size based on MAC */ 2931 index_mult = 0x1; 2932 table_size = 128; 2933 switch (sc->hw.mac.type) { 2934 case ixgbe_mac_82598EB: 2935 index_mult = 0x11; 2936 break; 2937 case ixgbe_mac_X550: 2938 case ixgbe_mac_X550EM_x: 2939 case ixgbe_mac_X550EM_a: 2940 table_size = 512; 2941 break; 2942 default: 2943 break; 2944 } 2945 2946 /* Set up the redirection table */ 2947 for (i = 0, j = 0; i < table_size; i++, j++) { 2948 if (j == sc->num_queues) j = 0; 2949 queue_id = (j * index_mult); 2950 /* 2951 * The low 8 bits are for hash value (n+0); 2952 * The next 8 bits are for hash value (n+1), etc. 2953 */ 2954 reta = reta >> 8; 2955 reta = reta | ( ((uint32_t) queue_id) << 24); 2956 if ((i & 3) == 3) { 2957 if (i < 128) 2958 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2959 else 2960 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 2961 reta); 2962 reta = 0; 2963 } 2964 } 2965 2966 /* Now fill our hash function seeds */ 2967 for (i = 0; i < 10; i++) 2968 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 2969 2970 /* 2971 * Disable UDP - IP fragments aren't currently being handled 2972 * and so we end up with a mix of 2-tuple and 4-tuple 2973 * traffic. 2974 */ 2975 mrqc = IXGBE_MRQC_RSSEN 2976 | IXGBE_MRQC_RSS_FIELD_IPV4 2977 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 2978 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 2979 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 2980 | IXGBE_MRQC_RSS_FIELD_IPV6 2981 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 2982 ; 2983 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2984 } 2985 2986 /********************************************************************* 2987 * 2988 * Free all receive rings. 2989 * 2990 **********************************************************************/ 2991 void 2992 ixgbe_free_receive_structures(struct ix_softc *sc) 2993 { 2994 struct rx_ring *rxr; 2995 int i; 2996 2997 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 2998 if_rxr_init(&rxr->rx_ring, 0, 0); 2999 3000 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3001 ixgbe_free_receive_buffers(rxr); 3002 } 3003 3004 /********************************************************************* 3005 * 3006 * Free receive ring data structures 3007 * 3008 **********************************************************************/ 3009 void 3010 ixgbe_free_receive_buffers(struct rx_ring *rxr) 3011 { 3012 struct ix_softc *sc; 3013 struct ixgbe_rx_buf *rxbuf; 3014 int i; 3015 3016 sc = rxr->sc; 3017 if (rxr->rx_buffers != NULL) { 3018 for (i = 0; i < sc->num_rx_desc; i++) { 3019 rxbuf = &rxr->rx_buffers[i]; 3020 if (rxbuf->buf != NULL) { 3021 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 3022 0, rxbuf->map->dm_mapsize, 3023 BUS_DMASYNC_POSTREAD); 3024 bus_dmamap_unload(rxr->rxdma.dma_tag, 3025 rxbuf->map); 3026 m_freem(rxbuf->buf); 3027 rxbuf->buf = NULL; 3028 } 3029 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 3030 rxbuf->map = NULL; 3031 } 3032 free(rxr->rx_buffers, M_DEVBUF, 3033 sc->num_rx_desc * sizeof(struct ixgbe_rx_buf)); 3034 rxr->rx_buffers = NULL; 3035 } 3036 } 3037 3038 /********************************************************************* 3039 * 3040 * This routine executes in interrupt context. It replenishes 3041 * the mbufs in the descriptor and sends data which has been 3042 * dma'ed into host memory to upper layer. 3043 * 3044 *********************************************************************/ 3045 int 3046 ixgbe_rxeof(struct rx_ring *rxr) 3047 { 3048 struct ix_softc *sc = rxr->sc; 3049 struct ifnet *ifp = &sc->arpcom.ac_if; 3050 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 3051 struct mbuf *mp, *sendmp; 3052 uint8_t eop = 0; 3053 uint16_t len, vtag; 3054 uint32_t staterr = 0, ptype; 3055 struct ixgbe_rx_buf *rxbuf, *nxbuf; 3056 union ixgbe_adv_rx_desc *rxdesc; 3057 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 3058 int i, nextp; 3059 3060 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3061 return FALSE; 3062 3063 i = rxr->next_to_check; 3064 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 3065 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3066 dsize * i, dsize, BUS_DMASYNC_POSTREAD); 3067 3068 rxdesc = &rxr->rx_base[i]; 3069 staterr = letoh32(rxdesc->wb.upper.status_error); 3070 if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) { 3071 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3072 dsize * i, dsize, 3073 BUS_DMASYNC_PREREAD); 3074 break; 3075 } 3076 3077 /* Zero out the receive descriptors status */ 3078 rxdesc->wb.upper.status_error = 0; 3079 rxbuf = &rxr->rx_buffers[i]; 3080 3081 /* pull the mbuf off the ring */ 3082 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 3083 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3084 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 3085 3086 mp = rxbuf->buf; 3087 len = letoh16(rxdesc->wb.upper.length); 3088 ptype = letoh32(rxdesc->wb.lower.lo_dword.data) & 3089 IXGBE_RXDADV_PKTTYPE_MASK; 3090 vtag = letoh16(rxdesc->wb.upper.vlan); 3091 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3092 3093 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) { 3094 sc->dropped_pkts++; 3095 3096 if (rxbuf->fmp) { 3097 m_freem(rxbuf->fmp); 3098 rxbuf->fmp = NULL; 3099 } 3100 3101 m_freem(mp); 3102 rxbuf->buf = NULL; 3103 goto next_desc; 3104 } 3105 3106 if (mp == NULL) { 3107 panic("%s: ixgbe_rxeof: NULL mbuf in slot %d " 3108 "(nrx %d, filled %d)", sc->dev.dv_xname, 3109 i, if_rxr_inuse(&rxr->rx_ring), 3110 rxr->last_desc_filled); 3111 } 3112 3113 /* Currently no HW RSC support of 82599 */ 3114 if (!eop) { 3115 /* 3116 * Figure out the next descriptor of this frame. 3117 */ 3118 nextp = i + 1; 3119 if (nextp == sc->num_rx_desc) 3120 nextp = 0; 3121 nxbuf = &rxr->rx_buffers[nextp]; 3122 /* prefetch(nxbuf); */ 3123 } 3124 3125 /* 3126 * Rather than using the fmp/lmp global pointers 3127 * we now keep the head of a packet chain in the 3128 * buffer struct and pass this along from one 3129 * descriptor to the next, until we get EOP. 3130 */ 3131 mp->m_len = len; 3132 /* 3133 * See if there is a stored head 3134 * that determines what we are 3135 */ 3136 sendmp = rxbuf->fmp; 3137 rxbuf->buf = rxbuf->fmp = NULL; 3138 3139 if (sendmp != NULL) /* secondary frag */ 3140 sendmp->m_pkthdr.len += mp->m_len; 3141 else { 3142 /* first desc of a non-ps chain */ 3143 sendmp = mp; 3144 sendmp->m_pkthdr.len = mp->m_len; 3145 #if NVLAN > 0 3146 if (staterr & IXGBE_RXD_STAT_VP) { 3147 sendmp->m_pkthdr.ether_vtag = vtag; 3148 sendmp->m_flags |= M_VLANTAG; 3149 } 3150 #endif 3151 } 3152 3153 /* Pass the head pointer on */ 3154 if (eop == 0) { 3155 nxbuf->fmp = sendmp; 3156 sendmp = NULL; 3157 mp->m_next = nxbuf->buf; 3158 } else { /* Sending this frame? */ 3159 rxr->rx_packets++; 3160 /* capture data for AIM */ 3161 rxr->bytes += sendmp->m_pkthdr.len; 3162 rxr->rx_bytes += sendmp->m_pkthdr.len; 3163 3164 ixgbe_rx_checksum(staterr, sendmp, ptype); 3165 3166 ml_enqueue(&ml, sendmp); 3167 } 3168 next_desc: 3169 if_rxr_put(&rxr->rx_ring, 1); 3170 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3171 dsize * i, dsize, 3172 BUS_DMASYNC_PREREAD); 3173 3174 /* Advance our pointers to the next descriptor. */ 3175 if (++i == sc->num_rx_desc) 3176 i = 0; 3177 } 3178 rxr->next_to_check = i; 3179 3180 if (ifiq_input(rxr->ifiq, &ml)) 3181 if_rxr_livelocked(&rxr->rx_ring); 3182 3183 if (!(staterr & IXGBE_RXD_STAT_DD)) 3184 return FALSE; 3185 3186 return TRUE; 3187 } 3188 3189 /********************************************************************* 3190 * 3191 * Verify that the hardware indicated that the checksum is valid. 3192 * Inform the stack about the status of checksum so that stack 3193 * doesn't spend time verifying the checksum. 3194 * 3195 *********************************************************************/ 3196 void 3197 ixgbe_rx_checksum(uint32_t staterr, struct mbuf * mp, uint32_t ptype) 3198 { 3199 uint16_t status = (uint16_t) staterr; 3200 uint8_t errors = (uint8_t) (staterr >> 24); 3201 3202 if (status & IXGBE_RXD_STAT_IPCS) { 3203 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3204 /* IP Checksum Good */ 3205 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 3206 } else 3207 mp->m_pkthdr.csum_flags = 0; 3208 } 3209 if (status & IXGBE_RXD_STAT_L4CS) { 3210 if (!(errors & IXGBE_RXD_ERR_TCPE)) 3211 mp->m_pkthdr.csum_flags |= 3212 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 3213 } 3214 } 3215 3216 void 3217 ixgbe_setup_vlan_hw_support(struct ix_softc *sc) 3218 { 3219 uint32_t ctrl; 3220 int i; 3221 3222 /* 3223 * A soft reset zero's out the VFTA, so 3224 * we need to repopulate it now. 3225 */ 3226 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 3227 if (sc->shadow_vfta[i] != 0) 3228 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i), 3229 sc->shadow_vfta[i]); 3230 } 3231 3232 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 3233 #if 0 3234 /* Enable the Filter Table if enabled */ 3235 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3236 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3237 ctrl |= IXGBE_VLNCTRL_VFE; 3238 } 3239 #endif 3240 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3241 ctrl |= IXGBE_VLNCTRL_VME; 3242 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 3243 3244 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 3245 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 3246 for (i = 0; i < sc->num_queues; i++) { 3247 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 3248 ctrl |= IXGBE_RXDCTL_VME; 3249 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl); 3250 } 3251 } 3252 } 3253 3254 void 3255 ixgbe_enable_intr(struct ix_softc *sc) 3256 { 3257 struct ixgbe_hw *hw = &sc->hw; 3258 uint32_t mask, fwsm; 3259 3260 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3261 /* Enable Fan Failure detection */ 3262 if (hw->device_id == IXGBE_DEV_ID_82598AT) 3263 mask |= IXGBE_EIMS_GPI_SDP1; 3264 3265 switch (sc->hw.mac.type) { 3266 case ixgbe_mac_82599EB: 3267 mask |= IXGBE_EIMS_ECC; 3268 /* Temperature sensor on some adapters */ 3269 mask |= IXGBE_EIMS_GPI_SDP0; 3270 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 3271 mask |= IXGBE_EIMS_GPI_SDP1; 3272 mask |= IXGBE_EIMS_GPI_SDP2; 3273 break; 3274 case ixgbe_mac_X540: 3275 mask |= IXGBE_EIMS_ECC; 3276 /* Detect if Thermal Sensor is enabled */ 3277 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3278 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3279 mask |= IXGBE_EIMS_TS; 3280 break; 3281 case ixgbe_mac_X550: 3282 case ixgbe_mac_X550EM_x: 3283 case ixgbe_mac_X550EM_a: 3284 mask |= IXGBE_EIMS_ECC; 3285 /* MAC thermal sensor is automatically enabled */ 3286 mask |= IXGBE_EIMS_TS; 3287 /* Some devices use SDP0 for important information */ 3288 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 3289 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 3290 mask |= IXGBE_EIMS_GPI_SDP0_X540; 3291 default: 3292 break; 3293 } 3294 3295 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3296 3297 /* With MSI-X we use auto clear */ 3298 if (sc->msix > 1) { 3299 mask = IXGBE_EIMS_ENABLE_MASK; 3300 /* Don't autoclear Link */ 3301 mask &= ~IXGBE_EIMS_OTHER; 3302 mask &= ~IXGBE_EIMS_LSC; 3303 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 3304 } 3305 3306 IXGBE_WRITE_FLUSH(hw); 3307 } 3308 3309 void 3310 ixgbe_disable_intr(struct ix_softc *sc) 3311 { 3312 if (sc->msix > 1) 3313 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3314 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3315 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3316 } else { 3317 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3318 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3319 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3320 } 3321 IXGBE_WRITE_FLUSH(&sc->hw); 3322 } 3323 3324 uint16_t 3325 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3326 { 3327 struct pci_attach_args *pa; 3328 uint32_t value; 3329 int high = 0; 3330 3331 if (reg & 0x2) { 3332 high = 1; 3333 reg &= ~0x2; 3334 } 3335 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3336 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3337 3338 if (high) 3339 value >>= 16; 3340 3341 return (value & 0xffff); 3342 } 3343 3344 void 3345 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3346 { 3347 struct pci_attach_args *pa; 3348 uint32_t rv; 3349 int high = 0; 3350 3351 /* Need to do read/mask/write... because 16 vs 32 bit!!! */ 3352 if (reg & 0x2) { 3353 high = 1; 3354 reg &= ~0x2; 3355 } 3356 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3357 rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3358 if (!high) 3359 rv = (rv & 0xffff0000) | value; 3360 else 3361 rv = (rv & 0xffff) | ((uint32_t)value << 16); 3362 pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv); 3363 } 3364 3365 /* 3366 * Setup the correct IVAR register for a particular MSIX interrupt 3367 * (yes this is all very magic and confusing :) 3368 * - entry is the register array entry 3369 * - vector is the MSIX vector for this queue 3370 * - type is RX/TX/MISC 3371 */ 3372 void 3373 ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type) 3374 { 3375 struct ixgbe_hw *hw = &sc->hw; 3376 uint32_t ivar, index; 3377 3378 vector |= IXGBE_IVAR_ALLOC_VAL; 3379 3380 switch (hw->mac.type) { 3381 3382 case ixgbe_mac_82598EB: 3383 if (type == -1) 3384 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3385 else 3386 entry += (type * 64); 3387 index = (entry >> 2) & 0x1F; 3388 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3389 ivar &= ~(0xFF << (8 * (entry & 0x3))); 3390 ivar |= (vector << (8 * (entry & 0x3))); 3391 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 3392 break; 3393 3394 case ixgbe_mac_82599EB: 3395 case ixgbe_mac_X540: 3396 case ixgbe_mac_X550: 3397 case ixgbe_mac_X550EM_x: 3398 case ixgbe_mac_X550EM_a: 3399 if (type == -1) { /* MISC IVAR */ 3400 index = (entry & 1) * 8; 3401 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3402 ivar &= ~(0xFF << index); 3403 ivar |= (vector << index); 3404 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3405 } else { /* RX/TX IVARS */ 3406 index = (16 * (entry & 1)) + (8 * type); 3407 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3408 ivar &= ~(0xFF << index); 3409 ivar |= (vector << index); 3410 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3411 } 3412 3413 default: 3414 break; 3415 } 3416 } 3417 3418 void 3419 ixgbe_configure_ivars(struct ix_softc *sc) 3420 { 3421 struct ix_queue *que = sc->queues; 3422 uint32_t newitr; 3423 int i; 3424 3425 newitr = (4000000 / IXGBE_INTS_PER_SEC) & 0x0FF8; 3426 3427 for (i = 0; i < sc->num_queues; i++, que++) { 3428 /* First the RX queue entry */ 3429 ixgbe_set_ivar(sc, i, que->msix, 0); 3430 /* ... and the TX */ 3431 ixgbe_set_ivar(sc, i, que->msix, 1); 3432 /* Set an Initial EITR value */ 3433 IXGBE_WRITE_REG(&sc->hw, 3434 IXGBE_EITR(que->msix), newitr); 3435 } 3436 3437 /* For the Link interrupt */ 3438 ixgbe_set_ivar(sc, 1, sc->linkvec, -1); 3439 } 3440 3441 /* 3442 * SFP module interrupts handler 3443 */ 3444 void 3445 ixgbe_handle_mod(struct ix_softc *sc) 3446 { 3447 struct ixgbe_hw *hw = &sc->hw; 3448 uint32_t err; 3449 3450 err = hw->phy.ops.identify_sfp(hw); 3451 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3452 printf("%s: Unsupported SFP+ module type was detected!\n", 3453 sc->dev.dv_xname); 3454 return; 3455 } 3456 err = hw->mac.ops.setup_sfp(hw); 3457 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3458 printf("%s: Setup failure - unsupported SFP+ module type!\n", 3459 sc->dev.dv_xname); 3460 return; 3461 } 3462 3463 ixgbe_handle_msf(sc); 3464 } 3465 3466 3467 /* 3468 * MSF (multispeed fiber) interrupts handler 3469 */ 3470 void 3471 ixgbe_handle_msf(struct ix_softc *sc) 3472 { 3473 struct ixgbe_hw *hw = &sc->hw; 3474 uint32_t autoneg; 3475 bool negotiate; 3476 3477 autoneg = hw->phy.autoneg_advertised; 3478 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) { 3479 if (hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate)) 3480 return; 3481 } 3482 if (hw->mac.ops.setup_link) 3483 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3484 3485 ifmedia_delete_instance(&sc->media, IFM_INST_ANY); 3486 ixgbe_add_media_types(sc); 3487 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 3488 } 3489 3490 /* 3491 * External PHY interrupts handler 3492 */ 3493 void 3494 ixgbe_handle_phy(struct ix_softc *sc) 3495 { 3496 struct ixgbe_hw *hw = &sc->hw; 3497 int error; 3498 3499 error = hw->phy.ops.handle_lasi(hw); 3500 if (error == IXGBE_ERR_OVERTEMP) 3501 printf("%s: CRITICAL: EXTERNAL PHY OVER TEMP!! " 3502 " PHY will downshift to lower power state!\n", 3503 sc->dev.dv_xname); 3504 else if (error) 3505 printf("%s: Error handling LASI interrupt: %d\n", 3506 sc->dev.dv_xname, error); 3507 3508 } 3509 3510 /********************************************************************** 3511 * 3512 * Update the board statistics counters. 3513 * 3514 **********************************************************************/ 3515 void 3516 ixgbe_update_stats_counters(struct ix_softc *sc) 3517 { 3518 struct ifnet *ifp = &sc->arpcom.ac_if; 3519 struct ixgbe_hw *hw = &sc->hw; 3520 uint64_t crcerrs, rlec, total_missed_rx = 0; 3521 #ifdef IX_DEBUG 3522 uint32_t missed_rx = 0, bprc, lxon, lxoff, total; 3523 int i; 3524 #endif 3525 3526 crcerrs = IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3527 sc->stats.crcerrs += crcerrs; 3528 rlec = IXGBE_READ_REG(hw, IXGBE_RLEC); 3529 sc->stats.rlec += rlec; 3530 3531 #ifdef IX_DEBUG 3532 for (i = 0; i < 8; i++) { 3533 uint32_t mp; 3534 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3535 /* missed_rx tallies misses for the gprc workaround */ 3536 missed_rx += mp; 3537 /* global total per queue */ 3538 sc->stats.mpc[i] += mp; 3539 /* running comprehensive total for stats display */ 3540 total_missed_rx += sc->stats.mpc[i]; 3541 if (hw->mac.type == ixgbe_mac_82598EB) 3542 sc->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3543 } 3544 3545 /* Hardware workaround, gprc counts missed packets */ 3546 sc->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 3547 sc->stats.gprc -= missed_rx; 3548 3549 if (hw->mac.type != ixgbe_mac_82598EB) { 3550 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) + 3551 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 3552 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 3553 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 3554 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) + 3555 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 3556 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3557 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3558 } else { 3559 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3560 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3561 /* 82598 only has a counter in the high register */ 3562 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3563 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3564 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3565 } 3566 3567 /* 3568 * Workaround: mprc hardware is incorrectly counting 3569 * broadcasts, so for now we subtract those. 3570 */ 3571 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3572 sc->stats.bprc += bprc; 3573 sc->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3574 if (hw->mac.type == ixgbe_mac_82598EB) 3575 sc->stats.mprc -= bprc; 3576 3577 sc->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3578 sc->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3579 sc->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3580 sc->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3581 sc->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3582 sc->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3583 sc->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3584 3585 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3586 sc->stats.lxontxc += lxon; 3587 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3588 sc->stats.lxofftxc += lxoff; 3589 total = lxon + lxoff; 3590 3591 sc->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3592 sc->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3593 sc->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3594 sc->stats.gptc -= total; 3595 sc->stats.mptc -= total; 3596 sc->stats.ptc64 -= total; 3597 sc->stats.gotc -= total * ETHER_MIN_LEN; 3598 3599 sc->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3600 sc->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3601 sc->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3602 sc->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3603 sc->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3604 sc->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3605 sc->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3606 sc->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3607 sc->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3608 sc->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3609 for (i = 0; i < 16; i++) { 3610 uint32_t dropped; 3611 3612 dropped = IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3613 sc->stats.qprdc[i] += dropped; 3614 missed_rx += dropped; 3615 sc->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3616 sc->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3617 } 3618 #endif 3619 3620 /* Fill out the OS statistics structure */ 3621 ifp->if_collisions = 0; 3622 ifp->if_oerrors = sc->watchdog_events; 3623 ifp->if_ierrors = total_missed_rx + crcerrs + rlec; 3624 } 3625 3626 #ifdef IX_DEBUG 3627 /********************************************************************** 3628 * 3629 * This routine is called only when ixgbe_display_debug_stats is enabled. 3630 * This routine provides a way to take a look at important statistics 3631 * maintained by the driver and hardware. 3632 * 3633 **********************************************************************/ 3634 void 3635 ixgbe_print_hw_stats(struct ix_softc * sc) 3636 { 3637 struct ifnet *ifp = &sc->arpcom.ac_if; 3638 int i; 3639 3640 printf("%s: missed pkts %llu, rx len errs %llu, crc errs %llu, " 3641 "dropped pkts %lu, watchdog timeouts %ld, " 3642 "XON rx %llu, XON tx %llu, XOFF rx %llu, XOFF tx %llu, " 3643 "total pkts rx %llu, good pkts rx %llu, good pkts tx %llu, " 3644 "tso tx %lu\n", 3645 ifp->if_xname, 3646 (long long)sc->stats.mpc[0], 3647 (long long)sc->stats.roc + (long long)sc->stats.ruc, 3648 (long long)sc->stats.crcerrs, 3649 sc->dropped_pkts, 3650 sc->watchdog_events, 3651 (long long)sc->stats.lxonrxc, 3652 (long long)sc->stats.lxontxc, 3653 (long long)sc->stats.lxoffrxc, 3654 (long long)sc->stats.lxofftxc, 3655 (long long)sc->stats.tpr, 3656 (long long)sc->stats.gprc, 3657 (long long)sc->stats.gptc, 3658 sc->tso_tx); 3659 3660 printf("%s: per queue statistics\n", ifp->if_xname); 3661 for (i = 0; i < sc->num_queues; i++) { 3662 printf("\tqueue %d: rx pkts %llu, rx drops %llu, " 3663 "tx pkts %llu\n", 3664 i, sc->stats.qprc[i], sc->stats.qprdc[i], 3665 sc->stats.qptc[i]); 3666 } 3667 } 3668 3669 void 3670 ixgbe_map_queue_statistics(struct ix_softc *sc) 3671 { 3672 int i; 3673 uint32_t r; 3674 3675 for (i = 0; i < 32; i++) { 3676 /* 3677 * Queues 0-15 are mapped 1:1 3678 * Queue 0 -> Counter 0 3679 * Queue 1 -> Counter 1 3680 * Queue 2 -> Counter 2.... 3681 * Queues 16-127 are mapped to Counter 0 3682 */ 3683 if (i < 4) { 3684 r = (i * 4 + 0); 3685 r |= (i * 4 + 1) << 8; 3686 r |= (i * 4 + 2) << 16; 3687 r |= (i * 4 + 3) << 24; 3688 } else 3689 r = 0; 3690 3691 IXGBE_WRITE_REG(&sc->hw, IXGBE_RQSMR(i), r); 3692 IXGBE_WRITE_REG(&sc->hw, IXGBE_TQSM(i), r); 3693 } 3694 } 3695 #endif 3696