1 /* $OpenBSD: if_ix.c,v 1.197 2023/06/01 09:05:33 jan Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2013, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */ 36 37 #include <dev/pci/if_ix.h> 38 #include <dev/pci/ixgbe_type.h> 39 40 /********************************************************************* 41 * Driver version 42 *********************************************************************/ 43 /* char ixgbe_driver_version[] = "2.5.13"; */ 44 45 /********************************************************************* 46 * PCI Device ID Table 47 * 48 * Used by probe to select devices to load on 49 *********************************************************************/ 50 51 const struct pci_matchid ixgbe_devices[] = { 52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 }, 53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX }, 54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT }, 57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT2 }, 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP }, 63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM }, 64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL }, 65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 }, 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BP }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_T3_LOM }, 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF_QP }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF2 }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_FCOE }, 77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599EN_SFP }, 78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_QSFP_SF_QP }, 79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T }, 80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T1 }, 81 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T }, 82 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T1 }, 83 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KX4 }, 84 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KR }, 85 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_SFP }, 86 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_10G_T }, 87 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_1G_T }, 88 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR }, 89 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR_L }, 90 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP_N }, 91 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP }, 92 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII }, 93 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII_L }, 94 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_10G_T }, 95 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T }, 96 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T_L } 97 }; 98 99 /********************************************************************* 100 * Function prototypes 101 *********************************************************************/ 102 int ixgbe_probe(struct device *, void *, void *); 103 void ixgbe_attach(struct device *, struct device *, void *); 104 int ixgbe_detach(struct device *, int); 105 int ixgbe_activate(struct device *, int); 106 void ixgbe_start(struct ifqueue *); 107 int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 108 int ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *); 109 int ixgbe_get_sffpage(struct ix_softc *, struct if_sffpage *); 110 void ixgbe_watchdog(struct ifnet *); 111 void ixgbe_init(void *); 112 void ixgbe_stop(void *); 113 void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 114 int ixgbe_media_change(struct ifnet *); 115 void ixgbe_identify_hardware(struct ix_softc *); 116 int ixgbe_allocate_pci_resources(struct ix_softc *); 117 int ixgbe_allocate_legacy(struct ix_softc *); 118 int ixgbe_allocate_msix(struct ix_softc *); 119 void ixgbe_setup_msix(struct ix_softc *); 120 int ixgbe_allocate_queues(struct ix_softc *); 121 void ixgbe_free_pci_resources(struct ix_softc *); 122 void ixgbe_local_timer(void *); 123 void ixgbe_setup_interface(struct ix_softc *); 124 void ixgbe_config_gpie(struct ix_softc *); 125 void ixgbe_config_delay_values(struct ix_softc *); 126 void ixgbe_add_media_types(struct ix_softc *); 127 void ixgbe_config_link(struct ix_softc *); 128 129 int ixgbe_allocate_transmit_buffers(struct tx_ring *); 130 int ixgbe_setup_transmit_structures(struct ix_softc *); 131 int ixgbe_setup_transmit_ring(struct tx_ring *); 132 void ixgbe_initialize_transmit_units(struct ix_softc *); 133 void ixgbe_free_transmit_structures(struct ix_softc *); 134 void ixgbe_free_transmit_buffers(struct tx_ring *); 135 136 int ixgbe_allocate_receive_buffers(struct rx_ring *); 137 int ixgbe_setup_receive_structures(struct ix_softc *); 138 int ixgbe_setup_receive_ring(struct rx_ring *); 139 void ixgbe_initialize_receive_units(struct ix_softc *); 140 void ixgbe_free_receive_structures(struct ix_softc *); 141 void ixgbe_free_receive_buffers(struct rx_ring *); 142 void ixgbe_initialize_rss_mapping(struct ix_softc *); 143 int ixgbe_rxfill(struct rx_ring *); 144 void ixgbe_rxrefill(void *); 145 146 int ixgbe_intr(struct ix_softc *sc); 147 void ixgbe_enable_intr(struct ix_softc *); 148 void ixgbe_disable_intr(struct ix_softc *); 149 int ixgbe_txeof(struct tx_ring *); 150 int ixgbe_rxeof(struct rx_ring *); 151 void ixgbe_rx_checksum(uint32_t, struct mbuf *); 152 void ixgbe_iff(struct ix_softc *); 153 void ixgbe_map_queue_statistics(struct ix_softc *); 154 void ixgbe_update_link_status(struct ix_softc *); 155 int ixgbe_get_buf(struct rx_ring *, int); 156 int ixgbe_encap(struct tx_ring *, struct mbuf *); 157 int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 158 struct ixgbe_dma_alloc *, int); 159 void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 160 static int 161 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *, uint32_t *, 162 uint32_t *); 163 void ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 164 void ixgbe_configure_ivars(struct ix_softc *); 165 uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 166 167 void ixgbe_setup_vlan_hw_support(struct ix_softc *); 168 169 /* Support for pluggable optic modules */ 170 void ixgbe_handle_mod(struct ix_softc *); 171 void ixgbe_handle_msf(struct ix_softc *); 172 void ixgbe_handle_phy(struct ix_softc *); 173 174 /* Legacy (single vector interrupt handler */ 175 int ixgbe_legacy_intr(void *); 176 void ixgbe_enable_queue(struct ix_softc *, uint32_t); 177 void ixgbe_enable_queues(struct ix_softc *); 178 void ixgbe_disable_queue(struct ix_softc *, uint32_t); 179 void ixgbe_rearm_queue(struct ix_softc *, uint32_t); 180 181 /* MSI-X (multiple vectors interrupt handlers) */ 182 int ixgbe_link_intr(void *); 183 int ixgbe_queue_intr(void *); 184 185 #if NKSTAT > 0 186 static void ix_kstats(struct ix_softc *); 187 static void ix_rxq_kstats(struct ix_softc *, struct rx_ring *); 188 static void ix_txq_kstats(struct ix_softc *, struct tx_ring *); 189 static void ix_kstats_tick(void *); 190 #endif 191 192 /********************************************************************* 193 * OpenBSD Device Interface Entry Points 194 *********************************************************************/ 195 196 struct cfdriver ix_cd = { 197 NULL, "ix", DV_IFNET 198 }; 199 200 const struct cfattach ix_ca = { 201 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach, 202 ixgbe_activate 203 }; 204 205 int ixgbe_smart_speed = ixgbe_smart_speed_on; 206 int ixgbe_enable_msix = 1; 207 208 /********************************************************************* 209 * Device identification routine 210 * 211 * ixgbe_probe determines if the driver should be loaded on 212 * adapter based on PCI vendor/device id of the adapter. 213 * 214 * return 0 on success, positive on failure 215 *********************************************************************/ 216 217 int 218 ixgbe_probe(struct device *parent, void *match, void *aux) 219 { 220 INIT_DEBUGOUT("ixgbe_probe: begin"); 221 222 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 223 nitems(ixgbe_devices))); 224 } 225 226 /********************************************************************* 227 * Device initialization routine 228 * 229 * The attach entry point is called when the driver is being loaded. 230 * This routine identifies the type of hardware, allocates all resources 231 * and initializes the hardware. 232 * 233 * return 0 on success, positive on failure 234 *********************************************************************/ 235 236 void 237 ixgbe_attach(struct device *parent, struct device *self, void *aux) 238 { 239 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 240 struct ix_softc *sc = (struct ix_softc *)self; 241 int error = 0; 242 uint16_t csum; 243 uint32_t ctrl_ext; 244 struct ixgbe_hw *hw = &sc->hw; 245 246 INIT_DEBUGOUT("ixgbe_attach: begin"); 247 248 sc->osdep.os_sc = sc; 249 sc->osdep.os_pa = *pa; 250 251 rw_init(&sc->sfflock, "ixsff"); 252 253 #if NKSTAT > 0 254 ix_kstats(sc); 255 #endif 256 257 /* Determine hardware revision */ 258 ixgbe_identify_hardware(sc); 259 260 /* Indicate to RX setup to use Jumbo Clusters */ 261 sc->num_tx_desc = DEFAULT_TXD; 262 sc->num_rx_desc = DEFAULT_RXD; 263 264 /* Do base PCI setup - map BAR0 */ 265 if (ixgbe_allocate_pci_resources(sc)) 266 goto err_out; 267 268 /* Allocate our TX/RX Queues */ 269 if (ixgbe_allocate_queues(sc)) 270 goto err_out; 271 272 /* Allocate multicast array memory. */ 273 sc->mta = mallocarray(IXGBE_ETH_LENGTH_OF_ADDRESS, 274 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 275 if (sc->mta == NULL) { 276 printf(": Can not allocate multicast setup array\n"); 277 goto err_late; 278 } 279 280 /* Initialize the shared code */ 281 error = ixgbe_init_shared_code(hw); 282 if (error) { 283 printf(": Unable to initialize the shared code\n"); 284 goto err_late; 285 } 286 287 /* Make sure we have a good EEPROM before we read from it */ 288 if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) { 289 printf(": The EEPROM Checksum Is Not Valid\n"); 290 goto err_late; 291 } 292 293 error = ixgbe_init_hw(hw); 294 if (error == IXGBE_ERR_EEPROM_VERSION) { 295 printf(": This device is a pre-production adapter/" 296 "LOM. Please be aware there may be issues associated " 297 "with your hardware.\nIf you are experiencing problems " 298 "please contact your Intel or hardware representative " 299 "who provided you with this hardware.\n"); 300 } else if (error && (error != IXGBE_ERR_SFP_NOT_PRESENT && 301 error != IXGBE_ERR_SFP_NOT_SUPPORTED)) { 302 printf(": Hardware Initialization Failure\n"); 303 goto err_late; 304 } 305 306 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 307 IXGBE_ETH_LENGTH_OF_ADDRESS); 308 309 if (sc->sc_intrmap) 310 error = ixgbe_allocate_msix(sc); 311 else 312 error = ixgbe_allocate_legacy(sc); 313 if (error) 314 goto err_late; 315 316 /* Enable the optics for 82599 SFP+ fiber */ 317 if (sc->hw.mac.ops.enable_tx_laser) 318 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 319 320 /* Enable power to the phy */ 321 if (hw->phy.ops.set_phy_power) 322 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 323 324 /* Setup OS specific network interface */ 325 ixgbe_setup_interface(sc); 326 327 /* Get the PCI-E bus info and determine LAN ID */ 328 hw->mac.ops.get_bus_info(hw); 329 330 /* Set an initial default flow control value */ 331 sc->fc = ixgbe_fc_full; 332 333 /* let hardware know driver is loaded */ 334 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 335 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 336 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 337 338 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 339 340 INIT_DEBUGOUT("ixgbe_attach: end"); 341 return; 342 343 err_late: 344 ixgbe_free_transmit_structures(sc); 345 ixgbe_free_receive_structures(sc); 346 err_out: 347 ixgbe_free_pci_resources(sc); 348 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 349 MAX_NUM_MULTICAST_ADDRESSES); 350 } 351 352 /********************************************************************* 353 * Device removal routine 354 * 355 * The detach entry point is called when the driver is being removed. 356 * This routine stops the adapter and deallocates all the resources 357 * that were allocated for driver operation. 358 * 359 * return 0 on success, positive on failure 360 *********************************************************************/ 361 362 int 363 ixgbe_detach(struct device *self, int flags) 364 { 365 struct ix_softc *sc = (struct ix_softc *)self; 366 struct ifnet *ifp = &sc->arpcom.ac_if; 367 uint32_t ctrl_ext; 368 369 INIT_DEBUGOUT("ixgbe_detach: begin"); 370 371 ixgbe_stop(sc); 372 373 /* let hardware know driver is unloading */ 374 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 375 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 376 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 377 378 ether_ifdetach(ifp); 379 if_detach(ifp); 380 381 ixgbe_free_pci_resources(sc); 382 383 ixgbe_free_transmit_structures(sc); 384 ixgbe_free_receive_structures(sc); 385 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 386 MAX_NUM_MULTICAST_ADDRESSES); 387 388 /* XXX kstat */ 389 390 return (0); 391 } 392 393 int 394 ixgbe_activate(struct device *self, int act) 395 { 396 struct ix_softc *sc = (struct ix_softc *)self; 397 struct ifnet *ifp = &sc->arpcom.ac_if; 398 struct ixgbe_hw *hw = &sc->hw; 399 uint32_t ctrl_ext; 400 int rv = 0; 401 402 switch (act) { 403 case DVACT_QUIESCE: 404 if (ifp->if_flags & IFF_RUNNING) 405 ixgbe_stop(sc); 406 break; 407 case DVACT_RESUME: 408 ixgbe_init_hw(hw); 409 410 /* Enable the optics for 82599 SFP+ fiber */ 411 if (sc->hw.mac.ops.enable_tx_laser) 412 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 413 414 /* Enable power to the phy */ 415 if (hw->phy.ops.set_phy_power) 416 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 417 418 /* Get the PCI-E bus info and determine LAN ID */ 419 hw->mac.ops.get_bus_info(hw); 420 421 /* let hardware know driver is loaded */ 422 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 423 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 424 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 425 426 if (ifp->if_flags & IFF_UP) 427 ixgbe_init(sc); 428 break; 429 default: 430 break; 431 } 432 return (rv); 433 } 434 435 /********************************************************************* 436 * Transmit entry point 437 * 438 * ixgbe_start is called by the stack to initiate a transmit. 439 * The driver will remain in this routine as long as there are 440 * packets to transmit and transmit resources are available. 441 * In case resources are not available stack is notified and 442 * the packet is requeued. 443 **********************************************************************/ 444 445 void 446 ixgbe_start(struct ifqueue *ifq) 447 { 448 struct ifnet *ifp = ifq->ifq_if; 449 struct ix_softc *sc = ifp->if_softc; 450 struct tx_ring *txr = ifq->ifq_softc; 451 struct mbuf *m_head; 452 unsigned int head, free, used; 453 int post = 0; 454 455 if (!sc->link_up) 456 return; 457 458 head = txr->next_avail_desc; 459 free = txr->next_to_clean; 460 if (free <= head) 461 free += sc->num_tx_desc; 462 free -= head; 463 464 membar_consumer(); 465 466 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 467 0, txr->txdma.dma_map->dm_mapsize, 468 BUS_DMASYNC_POSTWRITE); 469 470 for (;;) { 471 /* Check that we have the minimal number of TX descriptors. */ 472 if (free <= IXGBE_TX_OP_THRESHOLD) { 473 ifq_set_oactive(ifq); 474 break; 475 } 476 477 m_head = ifq_dequeue(ifq); 478 if (m_head == NULL) 479 break; 480 481 used = ixgbe_encap(txr, m_head); 482 if (used == 0) { 483 m_freem(m_head); 484 continue; 485 } 486 487 free -= used; 488 489 #if NBPFILTER > 0 490 if (ifp->if_bpf) 491 bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 492 #endif 493 494 /* Set timeout in case hardware has problems transmitting */ 495 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 496 ifp->if_timer = IXGBE_TX_TIMEOUT; 497 498 post = 1; 499 } 500 501 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 502 0, txr->txdma.dma_map->dm_mapsize, 503 BUS_DMASYNC_PREWRITE); 504 505 /* 506 * Advance the Transmit Descriptor Tail (Tdt), this tells the 507 * hardware that this frame is available to transmit. 508 */ 509 if (post) 510 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 511 txr->next_avail_desc); 512 } 513 514 /********************************************************************* 515 * Ioctl entry point 516 * 517 * ixgbe_ioctl is called when the user wants to configure the 518 * interface. 519 * 520 * return 0 on success, positive on failure 521 **********************************************************************/ 522 523 int 524 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 525 { 526 struct ix_softc *sc = ifp->if_softc; 527 struct ifreq *ifr = (struct ifreq *) data; 528 int s, error = 0; 529 530 s = splnet(); 531 532 switch (command) { 533 case SIOCSIFADDR: 534 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 535 ifp->if_flags |= IFF_UP; 536 if (!(ifp->if_flags & IFF_RUNNING)) 537 ixgbe_init(sc); 538 break; 539 540 case SIOCSIFFLAGS: 541 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 542 if (ifp->if_flags & IFF_UP) { 543 if (ifp->if_flags & IFF_RUNNING) 544 error = ENETRESET; 545 else 546 ixgbe_init(sc); 547 } else { 548 if (ifp->if_flags & IFF_RUNNING) 549 ixgbe_stop(sc); 550 } 551 break; 552 553 case SIOCSIFMEDIA: 554 case SIOCGIFMEDIA: 555 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 556 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 557 break; 558 559 case SIOCGIFRXR: 560 error = ixgbe_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 561 break; 562 563 case SIOCGIFSFFPAGE: 564 error = rw_enter(&sc->sfflock, RW_WRITE|RW_INTR); 565 if (error != 0) 566 break; 567 568 error = ixgbe_get_sffpage(sc, (struct if_sffpage *)data); 569 rw_exit(&sc->sfflock); 570 break; 571 572 default: 573 error = ether_ioctl(ifp, &sc->arpcom, command, data); 574 } 575 576 if (error == ENETRESET) { 577 if (ifp->if_flags & IFF_RUNNING) { 578 ixgbe_disable_intr(sc); 579 ixgbe_iff(sc); 580 ixgbe_enable_intr(sc); 581 ixgbe_enable_queues(sc); 582 } 583 error = 0; 584 } 585 586 splx(s); 587 return (error); 588 } 589 590 int 591 ixgbe_get_sffpage(struct ix_softc *sc, struct if_sffpage *sff) 592 { 593 struct ixgbe_hw *hw = &sc->hw; 594 uint32_t swfw_mask = hw->phy.phy_semaphore_mask; 595 uint8_t page; 596 size_t i; 597 int error = EIO; 598 599 if (hw->phy.type == ixgbe_phy_fw) 600 return (ENODEV); 601 602 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 603 return (EBUSY); /* XXX */ 604 605 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 606 if (hw->phy.ops.read_i2c_byte_unlocked(hw, 127, 607 IFSFF_ADDR_EEPROM, &page)) 608 goto error; 609 if (page != sff->sff_page && 610 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 611 IFSFF_ADDR_EEPROM, sff->sff_page)) 612 goto error; 613 } 614 615 for (i = 0; i < sizeof(sff->sff_data); i++) { 616 if (hw->phy.ops.read_i2c_byte_unlocked(hw, i, 617 sff->sff_addr, &sff->sff_data[i])) 618 goto error; 619 } 620 621 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 622 if (page != sff->sff_page && 623 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 624 IFSFF_ADDR_EEPROM, page)) 625 goto error; 626 } 627 628 error = 0; 629 error: 630 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 631 return (error); 632 } 633 634 int 635 ixgbe_rxrinfo(struct ix_softc *sc, struct if_rxrinfo *ifri) 636 { 637 struct if_rxring_info *ifr, ifr1; 638 struct rx_ring *rxr; 639 int error, i; 640 u_int n = 0; 641 642 if (sc->num_queues > 1) { 643 ifr = mallocarray(sc->num_queues, sizeof(*ifr), M_DEVBUF, 644 M_WAITOK | M_ZERO); 645 } else 646 ifr = &ifr1; 647 648 for (i = 0; i < sc->num_queues; i++) { 649 rxr = &sc->rx_rings[i]; 650 ifr[n].ifr_size = MCLBYTES; 651 snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i); 652 ifr[n].ifr_info = rxr->rx_ring; 653 n++; 654 } 655 656 error = if_rxr_info_ioctl(ifri, sc->num_queues, ifr); 657 658 if (sc->num_queues > 1) 659 free(ifr, M_DEVBUF, sc->num_queues * sizeof(*ifr)); 660 return (error); 661 } 662 663 /********************************************************************* 664 * Watchdog entry point 665 * 666 **********************************************************************/ 667 668 void 669 ixgbe_watchdog(struct ifnet * ifp) 670 { 671 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 672 struct tx_ring *txr = sc->tx_rings; 673 struct ixgbe_hw *hw = &sc->hw; 674 int tx_hang = FALSE; 675 int i; 676 677 /* 678 * The timer is set to 5 every time ixgbe_start() queues a packet. 679 * Anytime all descriptors are clean the timer is set to 0. 680 */ 681 for (i = 0; i < sc->num_queues; i++, txr++) { 682 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 683 continue; 684 else { 685 tx_hang = TRUE; 686 break; 687 } 688 } 689 if (tx_hang == FALSE) 690 return; 691 692 /* 693 * If we are in this routine because of pause frames, then don't 694 * reset the hardware. 695 */ 696 if (!(IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXON)) { 697 for (i = 0; i < sc->num_queues; i++, txr++) 698 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 699 ifp->if_timer = IXGBE_TX_TIMEOUT; 700 return; 701 } 702 703 704 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 705 for (i = 0; i < sc->num_queues; i++, txr++) { 706 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 707 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 708 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 709 printf("%s: TX(%d) Next TX to Clean = %d\n", ifp->if_xname, 710 i, txr->next_to_clean); 711 } 712 ifp->if_flags &= ~IFF_RUNNING; 713 714 ixgbe_init(sc); 715 } 716 717 /********************************************************************* 718 * Init entry point 719 * 720 * This routine is used in two ways. It is used by the stack as 721 * init entry point in network interface structure. It is also used 722 * by the driver as a hw/sw initialization routine to get to a 723 * consistent state. 724 * 725 * return 0 on success, positive on failure 726 **********************************************************************/ 727 #define IXGBE_MHADD_MFS_SHIFT 16 728 729 void 730 ixgbe_init(void *arg) 731 { 732 struct ix_softc *sc = (struct ix_softc *)arg; 733 struct ifnet *ifp = &sc->arpcom.ac_if; 734 struct rx_ring *rxr = sc->rx_rings; 735 uint32_t k, txdctl, rxdctl, rxctrl, mhadd, itr; 736 int i, s, err; 737 738 INIT_DEBUGOUT("ixgbe_init: begin"); 739 740 s = splnet(); 741 742 ixgbe_stop(sc); 743 744 /* reprogram the RAR[0] in case user changed it. */ 745 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 746 747 /* Get the latest mac address, User can use a LAA */ 748 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 749 IXGBE_ETH_LENGTH_OF_ADDRESS); 750 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1); 751 sc->hw.addr_ctrl.rar_used_count = 1; 752 753 /* Prepare transmit descriptors and buffers */ 754 if (ixgbe_setup_transmit_structures(sc)) { 755 printf("%s: Could not setup transmit structures\n", 756 ifp->if_xname); 757 ixgbe_stop(sc); 758 splx(s); 759 return; 760 } 761 762 ixgbe_init_hw(&sc->hw); 763 ixgbe_initialize_transmit_units(sc); 764 765 /* Use 2k clusters, even for jumbo frames */ 766 sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN; 767 768 /* Prepare receive descriptors and buffers */ 769 if (ixgbe_setup_receive_structures(sc)) { 770 printf("%s: Could not setup receive structures\n", 771 ifp->if_xname); 772 ixgbe_stop(sc); 773 splx(s); 774 return; 775 } 776 777 /* Configure RX settings */ 778 ixgbe_initialize_receive_units(sc); 779 780 /* Enable SDP & MSIX interrupts based on adapter */ 781 ixgbe_config_gpie(sc); 782 783 /* Program promiscuous mode and multicast filters. */ 784 ixgbe_iff(sc); 785 786 /* Set MRU size */ 787 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 788 mhadd &= ~IXGBE_MHADD_MFS_MASK; 789 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 790 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 791 792 /* Now enable all the queues */ 793 for (i = 0; i < sc->num_queues; i++) { 794 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 795 txdctl |= IXGBE_TXDCTL_ENABLE; 796 /* Set WTHRESH to 8, burst writeback */ 797 txdctl |= (8 << 16); 798 /* 799 * When the internal queue falls below PTHRESH (16), 800 * start prefetching as long as there are at least 801 * HTHRESH (1) buffers ready. 802 */ 803 txdctl |= (16 << 0) | (1 << 8); 804 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 805 } 806 807 for (i = 0; i < sc->num_queues; i++) { 808 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 809 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 810 /* 811 * PTHRESH = 21 812 * HTHRESH = 4 813 * WTHRESH = 8 814 */ 815 rxdctl &= ~0x3FFFFF; 816 rxdctl |= 0x080420; 817 } 818 rxdctl |= IXGBE_RXDCTL_ENABLE; 819 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 820 for (k = 0; k < 10; k++) { 821 if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) & 822 IXGBE_RXDCTL_ENABLE) 823 break; 824 else 825 msec_delay(1); 826 } 827 IXGBE_WRITE_FLUSH(&sc->hw); 828 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled); 829 } 830 831 /* Set up VLAN support and filter */ 832 ixgbe_setup_vlan_hw_support(sc); 833 834 /* Enable Receive engine */ 835 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 836 if (sc->hw.mac.type == ixgbe_mac_82598EB) 837 rxctrl |= IXGBE_RXCTRL_DMBYPS; 838 rxctrl |= IXGBE_RXCTRL_RXEN; 839 sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl); 840 841 /* Set up MSI/X routing */ 842 if (sc->sc_intrmap) { 843 ixgbe_configure_ivars(sc); 844 /* Set up auto-mask */ 845 if (sc->hw.mac.type == ixgbe_mac_82598EB) 846 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 847 else { 848 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 849 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 850 } 851 } else { /* Simple settings for Legacy/MSI */ 852 ixgbe_set_ivar(sc, 0, 0, 0); 853 ixgbe_set_ivar(sc, 0, 0, 1); 854 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 855 } 856 857 /* Check on any SFP devices that need to be kick-started */ 858 if (sc->hw.phy.type == ixgbe_phy_none) { 859 err = sc->hw.phy.ops.identify(&sc->hw); 860 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 861 printf("Unsupported SFP+ module type was detected.\n"); 862 splx(s); 863 return; 864 } 865 } 866 867 /* Setup interrupt moderation */ 868 itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8; 869 if (sc->hw.mac.type != ixgbe_mac_82598EB) 870 itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS; 871 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr); 872 873 if (sc->sc_intrmap) { 874 /* Set moderation on the Link interrupt */ 875 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(sc->linkvec), 876 IXGBE_LINK_ITR); 877 } 878 879 /* Enable power to the phy */ 880 if (sc->hw.phy.ops.set_phy_power) 881 sc->hw.phy.ops.set_phy_power(&sc->hw, TRUE); 882 883 /* Config/Enable Link */ 884 ixgbe_config_link(sc); 885 886 /* Hardware Packet Buffer & Flow Control setup */ 887 ixgbe_config_delay_values(sc); 888 889 /* Initialize the FC settings */ 890 sc->hw.mac.ops.start_hw(&sc->hw); 891 892 /* And now turn on interrupts */ 893 ixgbe_enable_intr(sc); 894 ixgbe_enable_queues(sc); 895 896 /* Now inform the stack we're ready */ 897 ifp->if_flags |= IFF_RUNNING; 898 for (i = 0; i < sc->num_queues; i++) 899 ifq_clr_oactive(ifp->if_ifqs[i]); 900 901 #if NKSTAT > 0 902 ix_kstats_tick(sc); 903 #endif 904 905 splx(s); 906 } 907 908 void 909 ixgbe_config_gpie(struct ix_softc *sc) 910 { 911 struct ixgbe_hw *hw = &sc->hw; 912 uint32_t gpie; 913 914 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 915 916 /* Fan Failure Interrupt */ 917 if (hw->device_id == IXGBE_DEV_ID_82598AT) 918 gpie |= IXGBE_SDP1_GPIEN; 919 920 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 921 /* Add for Module detection */ 922 gpie |= IXGBE_SDP2_GPIEN; 923 924 /* Media ready */ 925 if (hw->device_id != IXGBE_DEV_ID_82599_QSFP_SF_QP) 926 gpie |= IXGBE_SDP1_GPIEN; 927 928 /* 929 * Set LL interval to max to reduce the number of low latency 930 * interrupts hitting the card when the ring is getting full. 931 */ 932 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 933 } 934 935 if (sc->hw.mac.type == ixgbe_mac_X540 || 936 sc->hw.mac.type == ixgbe_mac_X550EM_x || 937 sc->hw.mac.type == ixgbe_mac_X550EM_a) { 938 /* 939 * Thermal Failure Detection (X540) 940 * Link Detection (X552 SFP+, X552/X557-AT) 941 */ 942 gpie |= IXGBE_SDP0_GPIEN_X540; 943 944 /* 945 * Set LL interval to max to reduce the number of low latency 946 * interrupts hitting the card when the ring is getting full. 947 */ 948 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 949 } 950 951 if (sc->sc_intrmap) { 952 /* Enable Enhanced MSIX mode */ 953 gpie |= IXGBE_GPIE_MSIX_MODE; 954 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 955 IXGBE_GPIE_OCD; 956 } 957 958 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 959 } 960 961 /* 962 * Requires sc->max_frame_size to be set. 963 */ 964 void 965 ixgbe_config_delay_values(struct ix_softc *sc) 966 { 967 struct ixgbe_hw *hw = &sc->hw; 968 uint32_t rxpb, frame, size, tmp; 969 970 frame = sc->max_frame_size; 971 972 /* Calculate High Water */ 973 switch (hw->mac.type) { 974 case ixgbe_mac_X540: 975 case ixgbe_mac_X550: 976 case ixgbe_mac_X550EM_x: 977 case ixgbe_mac_X550EM_a: 978 tmp = IXGBE_DV_X540(frame, frame); 979 break; 980 default: 981 tmp = IXGBE_DV(frame, frame); 982 break; 983 } 984 size = IXGBE_BT2KB(tmp); 985 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 986 hw->fc.high_water[0] = rxpb - size; 987 988 /* Now calculate Low Water */ 989 switch (hw->mac.type) { 990 case ixgbe_mac_X540: 991 case ixgbe_mac_X550: 992 case ixgbe_mac_X550EM_x: 993 case ixgbe_mac_X550EM_a: 994 tmp = IXGBE_LOW_DV_X540(frame); 995 break; 996 default: 997 tmp = IXGBE_LOW_DV(frame); 998 break; 999 } 1000 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 1001 1002 hw->fc.requested_mode = sc->fc; 1003 hw->fc.pause_time = IXGBE_FC_PAUSE; 1004 hw->fc.send_xon = TRUE; 1005 } 1006 1007 /* 1008 * MSIX Interrupt Handlers 1009 */ 1010 void 1011 ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector) 1012 { 1013 uint64_t queue = 1ULL << vector; 1014 uint32_t mask; 1015 1016 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1017 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1018 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask); 1019 } else { 1020 mask = (queue & 0xFFFFFFFF); 1021 if (mask) 1022 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask); 1023 mask = (queue >> 32); 1024 if (mask) 1025 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask); 1026 } 1027 } 1028 1029 void 1030 ixgbe_enable_queues(struct ix_softc *sc) 1031 { 1032 struct ix_queue *que; 1033 int i; 1034 1035 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 1036 ixgbe_enable_queue(sc, que->msix); 1037 } 1038 1039 void 1040 ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector) 1041 { 1042 uint64_t queue = 1ULL << vector; 1043 uint32_t mask; 1044 1045 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1046 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1047 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask); 1048 } else { 1049 mask = (queue & 0xFFFFFFFF); 1050 if (mask) 1051 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask); 1052 mask = (queue >> 32); 1053 if (mask) 1054 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask); 1055 } 1056 } 1057 1058 /* 1059 * MSIX Interrupt Handlers 1060 */ 1061 int 1062 ixgbe_link_intr(void *vsc) 1063 { 1064 struct ix_softc *sc = (struct ix_softc *)vsc; 1065 1066 return ixgbe_intr(sc); 1067 } 1068 1069 int 1070 ixgbe_queue_intr(void *vque) 1071 { 1072 struct ix_queue *que = vque; 1073 struct ix_softc *sc = que->sc; 1074 struct ifnet *ifp = &sc->arpcom.ac_if; 1075 struct rx_ring *rxr = que->rxr; 1076 struct tx_ring *txr = que->txr; 1077 1078 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1079 ixgbe_rxeof(rxr); 1080 ixgbe_txeof(txr); 1081 ixgbe_rxrefill(rxr); 1082 } 1083 1084 ixgbe_enable_queue(sc, que->msix); 1085 1086 return (1); 1087 } 1088 1089 /********************************************************************* 1090 * 1091 * Legacy Interrupt Service routine 1092 * 1093 **********************************************************************/ 1094 1095 int 1096 ixgbe_legacy_intr(void *arg) 1097 { 1098 struct ix_softc *sc = (struct ix_softc *)arg; 1099 struct ifnet *ifp = &sc->arpcom.ac_if; 1100 struct rx_ring *rxr = sc->rx_rings; 1101 struct tx_ring *txr = sc->tx_rings; 1102 int rv; 1103 1104 rv = ixgbe_intr(sc); 1105 if (rv == 0) { 1106 return (0); 1107 } 1108 1109 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1110 ixgbe_rxeof(rxr); 1111 ixgbe_txeof(txr); 1112 ixgbe_rxrefill(rxr); 1113 } 1114 1115 ixgbe_enable_queues(sc); 1116 return (rv); 1117 } 1118 1119 int 1120 ixgbe_intr(struct ix_softc *sc) 1121 { 1122 struct ifnet *ifp = &sc->arpcom.ac_if; 1123 struct ixgbe_hw *hw = &sc->hw; 1124 uint32_t reg_eicr, mod_mask, msf_mask; 1125 1126 if (sc->sc_intrmap) { 1127 /* Pause other interrupts */ 1128 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER); 1129 /* First get the cause */ 1130 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1131 /* Be sure the queue bits are not cleared */ 1132 reg_eicr &= ~IXGBE_EICR_RTX_QUEUE; 1133 /* Clear interrupt with write */ 1134 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1135 } else { 1136 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1137 if (reg_eicr == 0) { 1138 ixgbe_enable_intr(sc); 1139 ixgbe_enable_queues(sc); 1140 return (0); 1141 } 1142 } 1143 1144 /* Link status change */ 1145 if (reg_eicr & IXGBE_EICR_LSC) { 1146 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); 1147 KERNEL_LOCK(); 1148 ixgbe_update_link_status(sc); 1149 KERNEL_UNLOCK(); 1150 } 1151 1152 if (hw->mac.type != ixgbe_mac_82598EB) { 1153 if (reg_eicr & IXGBE_EICR_ECC) { 1154 printf("%s: CRITICAL: ECC ERROR!! " 1155 "Please Reboot!!\n", sc->dev.dv_xname); 1156 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1157 } 1158 /* Check for over temp condition */ 1159 if (reg_eicr & IXGBE_EICR_TS) { 1160 printf("%s: CRITICAL: OVER TEMP!! " 1161 "PHY IS SHUT DOWN!!\n", ifp->if_xname); 1162 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS); 1163 } 1164 } 1165 1166 /* Pluggable optics-related interrupt */ 1167 if (ixgbe_is_sfp(hw)) { 1168 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP) { 1169 mod_mask = IXGBE_EICR_GPI_SDP0_X540; 1170 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1171 } else if (hw->mac.type == ixgbe_mac_X540 || 1172 hw->mac.type == ixgbe_mac_X550 || 1173 hw->mac.type == ixgbe_mac_X550EM_x) { 1174 mod_mask = IXGBE_EICR_GPI_SDP2_X540; 1175 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1176 } else { 1177 mod_mask = IXGBE_EICR_GPI_SDP2; 1178 msf_mask = IXGBE_EICR_GPI_SDP1; 1179 } 1180 if (reg_eicr & mod_mask) { 1181 /* Clear the interrupt */ 1182 IXGBE_WRITE_REG(hw, IXGBE_EICR, mod_mask); 1183 KERNEL_LOCK(); 1184 ixgbe_handle_mod(sc); 1185 KERNEL_UNLOCK(); 1186 } else if ((hw->phy.media_type != ixgbe_media_type_copper) && 1187 (reg_eicr & msf_mask)) { 1188 /* Clear the interrupt */ 1189 IXGBE_WRITE_REG(hw, IXGBE_EICR, msf_mask); 1190 KERNEL_LOCK(); 1191 ixgbe_handle_msf(sc); 1192 KERNEL_UNLOCK(); 1193 } 1194 } 1195 1196 /* Check for fan failure */ 1197 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1198 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1199 printf("%s: CRITICAL: FAN FAILURE!! " 1200 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 1201 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1202 } 1203 1204 /* External PHY interrupt */ 1205 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 1206 (reg_eicr & IXGBE_EICR_GPI_SDP0_X540)) { 1207 /* Clear the interrupt */ 1208 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540); 1209 KERNEL_LOCK(); 1210 ixgbe_handle_phy(sc); 1211 KERNEL_UNLOCK(); 1212 } 1213 1214 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 1215 1216 return (1); 1217 } 1218 1219 /********************************************************************* 1220 * 1221 * Media Ioctl callback 1222 * 1223 * This routine is called whenever the user queries the status of 1224 * the interface using ifconfig. 1225 * 1226 **********************************************************************/ 1227 void 1228 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq *ifmr) 1229 { 1230 struct ix_softc *sc = ifp->if_softc; 1231 uint64_t layer; 1232 1233 ifmr->ifm_active = IFM_ETHER; 1234 ifmr->ifm_status = IFM_AVALID; 1235 1236 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1237 ixgbe_update_link_status(sc); 1238 1239 if (!LINK_STATE_IS_UP(ifp->if_link_state)) 1240 return; 1241 1242 ifmr->ifm_status |= IFM_ACTIVE; 1243 layer = sc->phy_layer; 1244 1245 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 1246 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 1247 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 1248 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1249 switch (sc->link_speed) { 1250 case IXGBE_LINK_SPEED_10GB_FULL: 1251 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1252 break; 1253 case IXGBE_LINK_SPEED_1GB_FULL: 1254 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1255 break; 1256 case IXGBE_LINK_SPEED_100_FULL: 1257 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1258 break; 1259 case IXGBE_LINK_SPEED_10_FULL: 1260 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1261 break; 1262 } 1263 } 1264 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1265 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1266 switch (sc->link_speed) { 1267 case IXGBE_LINK_SPEED_10GB_FULL: 1268 ifmr->ifm_active |= IFM_10G_SFP_CU | IFM_FDX; 1269 break; 1270 } 1271 } 1272 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1273 switch (sc->link_speed) { 1274 case IXGBE_LINK_SPEED_10GB_FULL: 1275 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 1276 break; 1277 case IXGBE_LINK_SPEED_1GB_FULL: 1278 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1279 break; 1280 } 1281 } 1282 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 1283 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1284 switch (sc->link_speed) { 1285 case IXGBE_LINK_SPEED_10GB_FULL: 1286 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1287 break; 1288 case IXGBE_LINK_SPEED_1GB_FULL: 1289 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1290 break; 1291 } 1292 } 1293 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1294 switch (sc->link_speed) { 1295 case IXGBE_LINK_SPEED_10GB_FULL: 1296 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1297 break; 1298 } 1299 } 1300 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1301 switch (sc->link_speed) { 1302 case IXGBE_LINK_SPEED_10GB_FULL: 1303 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 1304 break; 1305 case IXGBE_LINK_SPEED_2_5GB_FULL: 1306 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1307 break; 1308 case IXGBE_LINK_SPEED_1GB_FULL: 1309 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1310 break; 1311 } 1312 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 1313 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 1314 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1315 switch (sc->link_speed) { 1316 case IXGBE_LINK_SPEED_10GB_FULL: 1317 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 1318 break; 1319 case IXGBE_LINK_SPEED_2_5GB_FULL: 1320 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1321 break; 1322 case IXGBE_LINK_SPEED_1GB_FULL: 1323 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1324 break; 1325 } 1326 } 1327 1328 switch (sc->hw.fc.current_mode) { 1329 case ixgbe_fc_tx_pause: 1330 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 1331 break; 1332 case ixgbe_fc_rx_pause: 1333 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 1334 break; 1335 case ixgbe_fc_full: 1336 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE | 1337 IFM_ETH_TXPAUSE; 1338 break; 1339 default: 1340 ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE | 1341 IFM_ETH_TXPAUSE); 1342 break; 1343 } 1344 } 1345 1346 /********************************************************************* 1347 * 1348 * Media Ioctl callback 1349 * 1350 * This routine is called when the user changes speed/duplex using 1351 * media/mediopt option with ifconfig. 1352 * 1353 **********************************************************************/ 1354 int 1355 ixgbe_media_change(struct ifnet *ifp) 1356 { 1357 struct ix_softc *sc = ifp->if_softc; 1358 struct ixgbe_hw *hw = &sc->hw; 1359 struct ifmedia *ifm = &sc->media; 1360 ixgbe_link_speed speed = 0; 1361 1362 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1363 return (EINVAL); 1364 1365 if (hw->phy.media_type == ixgbe_media_type_backplane) 1366 return (ENODEV); 1367 1368 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1369 case IFM_AUTO: 1370 case IFM_10G_T: 1371 speed |= IXGBE_LINK_SPEED_100_FULL; 1372 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1373 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1374 break; 1375 case IFM_10G_SR: 1376 case IFM_10G_KR: 1377 case IFM_10G_LR: 1378 case IFM_10G_LRM: 1379 case IFM_10G_CX4: 1380 case IFM_10G_KX4: 1381 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1382 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1383 break; 1384 case IFM_10G_SFP_CU: 1385 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1386 break; 1387 case IFM_1000_T: 1388 speed |= IXGBE_LINK_SPEED_100_FULL; 1389 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1390 break; 1391 case IFM_1000_LX: 1392 case IFM_1000_SX: 1393 case IFM_1000_CX: 1394 case IFM_1000_KX: 1395 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1396 break; 1397 case IFM_100_TX: 1398 speed |= IXGBE_LINK_SPEED_100_FULL; 1399 break; 1400 case IFM_10_T: 1401 speed |= IXGBE_LINK_SPEED_10_FULL; 1402 break; 1403 default: 1404 return (EINVAL); 1405 } 1406 1407 hw->mac.autotry_restart = TRUE; 1408 hw->mac.ops.setup_link(hw, speed, TRUE); 1409 1410 return (0); 1411 } 1412 1413 /********************************************************************* 1414 * 1415 * This routine maps the mbufs to tx descriptors, allowing the 1416 * TX engine to transmit the packets. 1417 * - return 0 on success, positive on failure 1418 * 1419 **********************************************************************/ 1420 1421 int 1422 ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head) 1423 { 1424 struct ix_softc *sc = txr->sc; 1425 uint32_t olinfo_status = 0, cmd_type_len; 1426 int i, j, ntxc; 1427 int first, last = 0; 1428 bus_dmamap_t map; 1429 struct ixgbe_tx_buf *txbuf; 1430 union ixgbe_adv_tx_desc *txd = NULL; 1431 1432 /* Basic descriptor defines */ 1433 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1434 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1435 1436 /* 1437 * Important to capture the first descriptor 1438 * used because it will contain the index of 1439 * the one we tell the hardware to report back 1440 */ 1441 first = txr->next_avail_desc; 1442 txbuf = &txr->tx_buffers[first]; 1443 map = txbuf->map; 1444 1445 /* 1446 * Set the appropriate offload context 1447 * this will becomes the first descriptor. 1448 */ 1449 ntxc = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 1450 if (ntxc == -1) 1451 goto xmit_fail; 1452 1453 /* 1454 * Map the packet for DMA. 1455 */ 1456 switch (bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1457 m_head, BUS_DMA_NOWAIT)) { 1458 case 0: 1459 break; 1460 case EFBIG: 1461 if (m_defrag(m_head, M_NOWAIT) == 0 && 1462 bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1463 m_head, BUS_DMA_NOWAIT) == 0) 1464 break; 1465 /* FALLTHROUGH */ 1466 default: 1467 return (0); 1468 } 1469 1470 i = txr->next_avail_desc + ntxc; 1471 if (i >= sc->num_tx_desc) 1472 i -= sc->num_tx_desc; 1473 1474 for (j = 0; j < map->dm_nsegs; j++) { 1475 txd = &txr->tx_base[i]; 1476 1477 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 1478 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1479 cmd_type_len | map->dm_segs[j].ds_len); 1480 txd->read.olinfo_status = htole32(olinfo_status); 1481 last = i; /* descriptor that will get completion IRQ */ 1482 1483 if (++i == sc->num_tx_desc) 1484 i = 0; 1485 } 1486 1487 txd->read.cmd_type_len |= 1488 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1489 1490 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1491 BUS_DMASYNC_PREWRITE); 1492 1493 /* Set the index of the descriptor that will be marked done */ 1494 txbuf->m_head = m_head; 1495 txbuf->eop_index = last; 1496 1497 membar_producer(); 1498 1499 txr->next_avail_desc = i; 1500 1501 return (ntxc + j); 1502 1503 xmit_fail: 1504 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 1505 return (0); 1506 } 1507 1508 void 1509 ixgbe_iff(struct ix_softc *sc) 1510 { 1511 struct ifnet *ifp = &sc->arpcom.ac_if; 1512 struct arpcom *ac = &sc->arpcom; 1513 uint32_t fctrl; 1514 uint8_t *mta; 1515 uint8_t *update_ptr; 1516 struct ether_multi *enm; 1517 struct ether_multistep step; 1518 int mcnt = 0; 1519 1520 IOCTL_DEBUGOUT("ixgbe_iff: begin"); 1521 1522 mta = sc->mta; 1523 bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1524 MAX_NUM_MULTICAST_ADDRESSES); 1525 1526 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1527 fctrl &= ~(IXGBE_FCTRL_MPE | IXGBE_FCTRL_UPE); 1528 ifp->if_flags &= ~IFF_ALLMULTI; 1529 1530 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1531 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1532 ifp->if_flags |= IFF_ALLMULTI; 1533 fctrl |= IXGBE_FCTRL_MPE; 1534 if (ifp->if_flags & IFF_PROMISC) 1535 fctrl |= IXGBE_FCTRL_UPE; 1536 } else { 1537 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1538 while (enm != NULL) { 1539 bcopy(enm->enm_addrlo, 1540 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1541 IXGBE_ETH_LENGTH_OF_ADDRESS); 1542 mcnt++; 1543 1544 ETHER_NEXT_MULTI(step, enm); 1545 } 1546 1547 update_ptr = mta; 1548 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1549 ixgbe_mc_array_itr, TRUE); 1550 } 1551 1552 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1553 } 1554 1555 /* 1556 * This is an iterator function now needed by the multicast 1557 * shared code. It simply feeds the shared code routine the 1558 * addresses in the array of ixgbe_iff() one by one. 1559 */ 1560 uint8_t * 1561 ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1562 { 1563 uint8_t *addr = *update_ptr; 1564 uint8_t *newptr; 1565 *vmdq = 0; 1566 1567 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1568 *update_ptr = newptr; 1569 return addr; 1570 } 1571 1572 void 1573 ixgbe_update_link_status(struct ix_softc *sc) 1574 { 1575 struct ifnet *ifp = &sc->arpcom.ac_if; 1576 int link_state = LINK_STATE_DOWN; 1577 1578 splassert(IPL_NET); 1579 KERNEL_ASSERT_LOCKED(); 1580 1581 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 1582 1583 ifp->if_baudrate = 0; 1584 if (sc->link_up) { 1585 link_state = LINK_STATE_FULL_DUPLEX; 1586 1587 switch (sc->link_speed) { 1588 case IXGBE_LINK_SPEED_UNKNOWN: 1589 ifp->if_baudrate = 0; 1590 break; 1591 case IXGBE_LINK_SPEED_100_FULL: 1592 ifp->if_baudrate = IF_Mbps(100); 1593 break; 1594 case IXGBE_LINK_SPEED_1GB_FULL: 1595 ifp->if_baudrate = IF_Gbps(1); 1596 break; 1597 case IXGBE_LINK_SPEED_10GB_FULL: 1598 ifp->if_baudrate = IF_Gbps(10); 1599 break; 1600 } 1601 1602 /* Update any Flow Control changes */ 1603 sc->hw.mac.ops.fc_enable(&sc->hw); 1604 } 1605 if (ifp->if_link_state != link_state) { 1606 ifp->if_link_state = link_state; 1607 if_link_state_change(ifp); 1608 } 1609 } 1610 1611 1612 /********************************************************************* 1613 * 1614 * This routine disables all traffic on the adapter by issuing a 1615 * global reset on the MAC and deallocates TX/RX buffers. 1616 * 1617 **********************************************************************/ 1618 1619 void 1620 ixgbe_stop(void *arg) 1621 { 1622 struct ix_softc *sc = arg; 1623 struct ifnet *ifp = &sc->arpcom.ac_if; 1624 int i; 1625 1626 /* Tell the stack that the interface is no longer active */ 1627 ifp->if_flags &= ~IFF_RUNNING; 1628 1629 #if NKSTAT > 0 1630 timeout_del(&sc->sc_kstat_tmo); 1631 #endif 1632 ifp->if_timer = 0; 1633 1634 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1635 ixgbe_disable_intr(sc); 1636 1637 sc->hw.mac.ops.reset_hw(&sc->hw); 1638 sc->hw.adapter_stopped = FALSE; 1639 sc->hw.mac.ops.stop_adapter(&sc->hw); 1640 if (sc->hw.mac.type == ixgbe_mac_82599EB) 1641 sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw); 1642 /* Turn off the laser */ 1643 if (sc->hw.mac.ops.disable_tx_laser) 1644 sc->hw.mac.ops.disable_tx_laser(&sc->hw); 1645 1646 /* reprogram the RAR[0] in case user changed it. */ 1647 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1648 1649 intr_barrier(sc->tag); 1650 for (i = 0; i < sc->num_queues; i++) { 1651 struct ifqueue *ifq = ifp->if_ifqs[i]; 1652 ifq_barrier(ifq); 1653 ifq_clr_oactive(ifq); 1654 1655 if (sc->queues[i].tag != NULL) 1656 intr_barrier(sc->queues[i].tag); 1657 timeout_del(&sc->rx_rings[i].rx_refill); 1658 } 1659 1660 KASSERT((ifp->if_flags & IFF_RUNNING) == 0); 1661 1662 /* Should we really clear all structures on stop? */ 1663 ixgbe_free_transmit_structures(sc); 1664 ixgbe_free_receive_structures(sc); 1665 1666 ixgbe_update_link_status(sc); 1667 } 1668 1669 1670 /********************************************************************* 1671 * 1672 * Determine hardware revision. 1673 * 1674 **********************************************************************/ 1675 void 1676 ixgbe_identify_hardware(struct ix_softc *sc) 1677 { 1678 struct ixgbe_osdep *os = &sc->osdep; 1679 struct pci_attach_args *pa = &os->os_pa; 1680 uint32_t reg; 1681 1682 /* Save off the information about this board */ 1683 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1684 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1685 1686 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1687 sc->hw.revision_id = PCI_REVISION(reg); 1688 1689 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1690 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1691 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1692 1693 /* We need this here to set the num_segs below */ 1694 ixgbe_set_mac_type(&sc->hw); 1695 1696 /* Pick up the 82599 and VF settings */ 1697 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1698 sc->hw.phy.smart_speed = ixgbe_smart_speed; 1699 sc->num_segs = IXGBE_82599_SCATTER; 1700 } 1701 1702 /********************************************************************* 1703 * 1704 * Setup the Legacy or MSI Interrupt handler 1705 * 1706 **********************************************************************/ 1707 int 1708 ixgbe_allocate_legacy(struct ix_softc *sc) 1709 { 1710 struct ixgbe_osdep *os = &sc->osdep; 1711 struct pci_attach_args *pa = &os->os_pa; 1712 const char *intrstr = NULL; 1713 pci_chipset_tag_t pc = pa->pa_pc; 1714 pci_intr_handle_t ih; 1715 1716 /* We allocate a single interrupt resource */ 1717 if (pci_intr_map_msi(pa, &ih) != 0 && 1718 pci_intr_map(pa, &ih) != 0) { 1719 printf(": couldn't map interrupt\n"); 1720 return (ENXIO); 1721 } 1722 1723 #if 0 1724 /* XXX */ 1725 /* Tasklets for Link, SFP and Multispeed Fiber */ 1726 TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc); 1727 TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc); 1728 TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc); 1729 #endif 1730 1731 intrstr = pci_intr_string(pc, ih); 1732 sc->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE, 1733 ixgbe_legacy_intr, sc, sc->dev.dv_xname); 1734 if (sc->tag == NULL) { 1735 printf(": couldn't establish interrupt"); 1736 if (intrstr != NULL) 1737 printf(" at %s", intrstr); 1738 printf("\n"); 1739 return (ENXIO); 1740 } 1741 printf(": %s", intrstr); 1742 1743 /* For simplicity in the handlers */ 1744 sc->que_mask = IXGBE_EIMS_ENABLE_MASK; 1745 1746 return (0); 1747 } 1748 1749 /********************************************************************* 1750 * 1751 * Setup the MSI-X Interrupt handlers 1752 * 1753 **********************************************************************/ 1754 int 1755 ixgbe_allocate_msix(struct ix_softc *sc) 1756 { 1757 struct ixgbe_osdep *os = &sc->osdep; 1758 struct pci_attach_args *pa = &os->os_pa; 1759 int i = 0, error = 0; 1760 struct ix_queue *que; 1761 pci_intr_handle_t ih; 1762 1763 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) { 1764 if (pci_intr_map_msix(pa, i, &ih)) { 1765 printf("ixgbe_allocate_msix: " 1766 "pci_intr_map_msix vec %d failed\n", i); 1767 error = ENOMEM; 1768 goto fail; 1769 } 1770 1771 que->tag = pci_intr_establish_cpu(pa->pa_pc, ih, 1772 IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i), 1773 ixgbe_queue_intr, que, que->name); 1774 if (que->tag == NULL) { 1775 printf("ixgbe_allocate_msix: " 1776 "pci_intr_establish vec %d failed\n", i); 1777 error = ENOMEM; 1778 goto fail; 1779 } 1780 1781 que->msix = i; 1782 } 1783 1784 /* Now the link status/control last MSI-X vector */ 1785 if (pci_intr_map_msix(pa, i, &ih)) { 1786 printf("ixgbe_allocate_msix: " 1787 "pci_intr_map_msix link vector failed\n"); 1788 error = ENOMEM; 1789 goto fail; 1790 } 1791 1792 sc->tag = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE, 1793 ixgbe_link_intr, sc, sc->dev.dv_xname); 1794 if (sc->tag == NULL) { 1795 printf("ixgbe_allocate_msix: " 1796 "pci_intr_establish link vector failed\n"); 1797 error = ENOMEM; 1798 goto fail; 1799 } 1800 sc->linkvec = i; 1801 printf(", %s, %d queue%s", pci_intr_string(pa->pa_pc, ih), 1802 i, (i > 1) ? "s" : ""); 1803 1804 return (0); 1805 fail: 1806 for (que = sc->queues; i > 0; i--, que++) { 1807 if (que->tag == NULL) 1808 continue; 1809 pci_intr_disestablish(pa->pa_pc, que->tag); 1810 que->tag = NULL; 1811 } 1812 1813 return (error); 1814 } 1815 1816 void 1817 ixgbe_setup_msix(struct ix_softc *sc) 1818 { 1819 struct ixgbe_osdep *os = &sc->osdep; 1820 struct pci_attach_args *pa = &os->os_pa; 1821 int nmsix; 1822 unsigned int maxq; 1823 1824 if (!ixgbe_enable_msix) 1825 return; 1826 1827 nmsix = pci_intr_msix_count(pa); 1828 if (nmsix <= 1) 1829 return; 1830 1831 /* give one vector to events */ 1832 nmsix--; 1833 1834 /* XXX the number of queues is limited to what we can keep stats on */ 1835 maxq = (sc->hw.mac.type == ixgbe_mac_82598EB) ? 8 : 16; 1836 1837 sc->sc_intrmap = intrmap_create(&sc->dev, nmsix, maxq, 0); 1838 sc->num_queues = intrmap_count(sc->sc_intrmap); 1839 } 1840 1841 int 1842 ixgbe_allocate_pci_resources(struct ix_softc *sc) 1843 { 1844 struct ixgbe_osdep *os = &sc->osdep; 1845 struct pci_attach_args *pa = &os->os_pa; 1846 int val; 1847 1848 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1849 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) { 1850 printf(": mmba is not mem space\n"); 1851 return (ENXIO); 1852 } 1853 1854 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1855 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1856 printf(": cannot find mem space\n"); 1857 return (ENXIO); 1858 } 1859 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1860 1861 /* Legacy defaults */ 1862 sc->num_queues = 1; 1863 sc->hw.back = os; 1864 1865 /* Now setup MSI or MSI/X, return us the number of supported vectors. */ 1866 ixgbe_setup_msix(sc); 1867 1868 return (0); 1869 } 1870 1871 void 1872 ixgbe_free_pci_resources(struct ix_softc * sc) 1873 { 1874 struct ixgbe_osdep *os = &sc->osdep; 1875 struct pci_attach_args *pa = &os->os_pa; 1876 struct ix_queue *que = sc->queues; 1877 int i; 1878 1879 /* Release all msix queue resources: */ 1880 for (i = 0; i < sc->num_queues; i++, que++) { 1881 if (que->tag) 1882 pci_intr_disestablish(pa->pa_pc, que->tag); 1883 que->tag = NULL; 1884 } 1885 1886 if (sc->tag) 1887 pci_intr_disestablish(pa->pa_pc, sc->tag); 1888 sc->tag = NULL; 1889 if (os->os_membase != 0) 1890 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1891 os->os_membase = 0; 1892 } 1893 1894 /********************************************************************* 1895 * 1896 * Setup networking device structure and register an interface. 1897 * 1898 **********************************************************************/ 1899 void 1900 ixgbe_setup_interface(struct ix_softc *sc) 1901 { 1902 struct ifnet *ifp = &sc->arpcom.ac_if; 1903 int i; 1904 1905 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1906 ifp->if_softc = sc; 1907 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1908 ifp->if_xflags = IFXF_MPSAFE; 1909 ifp->if_ioctl = ixgbe_ioctl; 1910 ifp->if_qstart = ixgbe_start; 1911 ifp->if_timer = 0; 1912 ifp->if_watchdog = ixgbe_watchdog; 1913 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1914 ETHER_HDR_LEN - ETHER_CRC_LEN; 1915 ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1); 1916 1917 ifp->if_capabilities = IFCAP_VLAN_MTU; 1918 1919 #if NVLAN > 0 1920 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1921 #endif 1922 1923 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 1924 ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6; 1925 ifp->if_capabilities |= IFCAP_CSUM_IPv4; 1926 1927 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1928 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1929 ifp->if_capabilities |= IFCAP_LRO; 1930 1931 /* 1932 * Specify the media types supported by this sc and register 1933 * callbacks to update media and link information 1934 */ 1935 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1936 ixgbe_media_status); 1937 ixgbe_add_media_types(sc); 1938 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1939 1940 if_attach(ifp); 1941 ether_ifattach(ifp); 1942 1943 if_attach_queues(ifp, sc->num_queues); 1944 if_attach_iqueues(ifp, sc->num_queues); 1945 for (i = 0; i < sc->num_queues; i++) { 1946 struct ifqueue *ifq = ifp->if_ifqs[i]; 1947 struct ifiqueue *ifiq = ifp->if_iqs[i]; 1948 struct tx_ring *txr = &sc->tx_rings[i]; 1949 struct rx_ring *rxr = &sc->rx_rings[i]; 1950 1951 ifq->ifq_softc = txr; 1952 txr->ifq = ifq; 1953 1954 ifiq->ifiq_softc = rxr; 1955 rxr->ifiq = ifiq; 1956 1957 #if NKSTAT > 0 1958 ix_txq_kstats(sc, txr); 1959 ix_rxq_kstats(sc, rxr); 1960 #endif 1961 } 1962 1963 sc->max_frame_size = IXGBE_MAX_FRAME_SIZE; 1964 } 1965 1966 void 1967 ixgbe_add_media_types(struct ix_softc *sc) 1968 { 1969 struct ixgbe_hw *hw = &sc->hw; 1970 uint64_t layer; 1971 1972 sc->phy_layer = hw->mac.ops.get_supported_physical_layer(hw); 1973 layer = sc->phy_layer; 1974 1975 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1976 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL); 1977 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1978 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 1979 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1980 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1981 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1982 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 1983 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SFP_CU, 0, NULL); 1984 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1985 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL); 1986 if (hw->phy.multispeed_fiber) 1987 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_LX, 0, 1988 NULL); 1989 } 1990 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1991 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 1992 if (hw->phy.multispeed_fiber) 1993 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, 1994 NULL); 1995 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 1996 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 1997 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 1998 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); 1999 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 2000 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL); 2001 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) 2002 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); 2003 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 2004 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL); 2005 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) 2006 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL); 2007 2008 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2009 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, 2010 NULL); 2011 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 2012 } 2013 2014 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2015 } 2016 2017 void 2018 ixgbe_config_link(struct ix_softc *sc) 2019 { 2020 uint32_t autoneg, err = 0; 2021 bool negotiate; 2022 2023 if (ixgbe_is_sfp(&sc->hw)) { 2024 if (sc->hw.phy.multispeed_fiber) { 2025 sc->hw.mac.ops.setup_sfp(&sc->hw); 2026 if (sc->hw.mac.ops.enable_tx_laser) 2027 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 2028 ixgbe_handle_msf(sc); 2029 } else 2030 ixgbe_handle_mod(sc); 2031 } else { 2032 if (sc->hw.mac.ops.check_link) 2033 err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg, 2034 &sc->link_up, FALSE); 2035 if (err) 2036 return; 2037 autoneg = sc->hw.phy.autoneg_advertised; 2038 if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities)) 2039 err = sc->hw.mac.ops.get_link_capabilities(&sc->hw, 2040 &autoneg, &negotiate); 2041 if (err) 2042 return; 2043 if (sc->hw.mac.ops.setup_link) 2044 sc->hw.mac.ops.setup_link(&sc->hw, 2045 autoneg, sc->link_up); 2046 } 2047 } 2048 2049 /******************************************************************** 2050 * Manage DMA'able memory. 2051 *******************************************************************/ 2052 int 2053 ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 2054 struct ixgbe_dma_alloc *dma, int mapflags) 2055 { 2056 struct ifnet *ifp = &sc->arpcom.ac_if; 2057 struct ixgbe_osdep *os = &sc->osdep; 2058 int r; 2059 2060 dma->dma_tag = os->os_pa.pa_dmat; 2061 r = bus_dmamap_create(dma->dma_tag, size, 1, 2062 size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 2063 if (r != 0) { 2064 printf("%s: ixgbe_dma_malloc: bus_dmamap_create failed; " 2065 "error %u\n", ifp->if_xname, r); 2066 goto fail_0; 2067 } 2068 2069 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 2070 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 2071 if (r != 0) { 2072 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2073 "error %u\n", ifp->if_xname, r); 2074 goto fail_1; 2075 } 2076 2077 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 2078 &dma->dma_vaddr, BUS_DMA_NOWAIT); 2079 if (r != 0) { 2080 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 2081 "error %u\n", ifp->if_xname, r); 2082 goto fail_2; 2083 } 2084 2085 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2086 size, NULL, mapflags | BUS_DMA_NOWAIT); 2087 if (r != 0) { 2088 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 2089 "error %u\n", ifp->if_xname, r); 2090 goto fail_3; 2091 } 2092 2093 dma->dma_size = size; 2094 return (0); 2095 fail_3: 2096 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 2097 fail_2: 2098 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2099 fail_1: 2100 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2101 fail_0: 2102 dma->dma_map = NULL; 2103 dma->dma_tag = NULL; 2104 return (r); 2105 } 2106 2107 void 2108 ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 2109 { 2110 if (dma->dma_tag == NULL) 2111 return; 2112 2113 if (dma->dma_map != NULL) { 2114 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 2115 dma->dma_map->dm_mapsize, 2116 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2117 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2118 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 2119 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2120 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2121 dma->dma_map = NULL; 2122 } 2123 } 2124 2125 2126 /********************************************************************* 2127 * 2128 * Allocate memory for the transmit and receive rings, and then 2129 * the descriptors associated with each, called only once at attach. 2130 * 2131 **********************************************************************/ 2132 int 2133 ixgbe_allocate_queues(struct ix_softc *sc) 2134 { 2135 struct ifnet *ifp = &sc->arpcom.ac_if; 2136 struct ix_queue *que; 2137 struct tx_ring *txr; 2138 struct rx_ring *rxr; 2139 int rsize, tsize; 2140 int txconf = 0, rxconf = 0, i; 2141 2142 /* First allocate the top level queue structs */ 2143 if (!(sc->queues = mallocarray(sc->num_queues, 2144 sizeof(struct ix_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2145 printf("%s: Unable to allocate queue memory\n", ifp->if_xname); 2146 goto fail; 2147 } 2148 2149 /* Then allocate the TX ring struct memory */ 2150 if (!(sc->tx_rings = mallocarray(sc->num_queues, 2151 sizeof(struct tx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2152 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 2153 goto fail; 2154 } 2155 2156 /* Next allocate the RX */ 2157 if (!(sc->rx_rings = mallocarray(sc->num_queues, 2158 sizeof(struct rx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2159 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 2160 goto rx_fail; 2161 } 2162 2163 /* For the ring itself */ 2164 tsize = roundup2(sc->num_tx_desc * 2165 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2166 2167 /* 2168 * Now set up the TX queues, txconf is needed to handle the 2169 * possibility that things fail midcourse and we need to 2170 * undo memory gracefully 2171 */ 2172 for (i = 0; i < sc->num_queues; i++, txconf++) { 2173 /* Set up some basics */ 2174 txr = &sc->tx_rings[i]; 2175 txr->sc = sc; 2176 txr->me = i; 2177 2178 if (ixgbe_dma_malloc(sc, tsize, 2179 &txr->txdma, BUS_DMA_NOWAIT)) { 2180 printf("%s: Unable to allocate TX Descriptor memory\n", 2181 ifp->if_xname); 2182 goto err_tx_desc; 2183 } 2184 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2185 bzero((void *)txr->tx_base, tsize); 2186 } 2187 2188 /* 2189 * Next the RX queues... 2190 */ 2191 rsize = roundup2(sc->num_rx_desc * 2192 sizeof(union ixgbe_adv_rx_desc), 4096); 2193 for (i = 0; i < sc->num_queues; i++, rxconf++) { 2194 rxr = &sc->rx_rings[i]; 2195 /* Set up some basics */ 2196 rxr->sc = sc; 2197 rxr->me = i; 2198 timeout_set(&rxr->rx_refill, ixgbe_rxrefill, rxr); 2199 2200 if (ixgbe_dma_malloc(sc, rsize, 2201 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2202 printf("%s: Unable to allocate RxDescriptor memory\n", 2203 ifp->if_xname); 2204 goto err_rx_desc; 2205 } 2206 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2207 bzero((void *)rxr->rx_base, rsize); 2208 } 2209 2210 /* 2211 * Finally set up the queue holding structs 2212 */ 2213 for (i = 0; i < sc->num_queues; i++) { 2214 que = &sc->queues[i]; 2215 que->sc = sc; 2216 que->txr = &sc->tx_rings[i]; 2217 que->rxr = &sc->rx_rings[i]; 2218 snprintf(que->name, sizeof(que->name), "%s:%d", 2219 sc->dev.dv_xname, i); 2220 } 2221 2222 return (0); 2223 2224 err_rx_desc: 2225 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 2226 ixgbe_dma_free(sc, &rxr->rxdma); 2227 err_tx_desc: 2228 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 2229 ixgbe_dma_free(sc, &txr->txdma); 2230 free(sc->rx_rings, M_DEVBUF, sc->num_queues * sizeof(struct rx_ring)); 2231 sc->rx_rings = NULL; 2232 rx_fail: 2233 free(sc->tx_rings, M_DEVBUF, sc->num_queues * sizeof(struct tx_ring)); 2234 sc->tx_rings = NULL; 2235 fail: 2236 return (ENOMEM); 2237 } 2238 2239 /********************************************************************* 2240 * 2241 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2242 * the information needed to transmit a packet on the wire. This is 2243 * called only once at attach, setup is done every reset. 2244 * 2245 **********************************************************************/ 2246 int 2247 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2248 { 2249 struct ix_softc *sc = txr->sc; 2250 struct ifnet *ifp = &sc->arpcom.ac_if; 2251 struct ixgbe_tx_buf *txbuf; 2252 int error, i; 2253 2254 if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc, 2255 sizeof(struct ixgbe_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2256 printf("%s: Unable to allocate tx_buffer memory\n", 2257 ifp->if_xname); 2258 error = ENOMEM; 2259 goto fail; 2260 } 2261 txr->txtag = txr->txdma.dma_tag; 2262 2263 /* Create the descriptor buffer dma maps */ 2264 for (i = 0; i < sc->num_tx_desc; i++) { 2265 txbuf = &txr->tx_buffers[i]; 2266 error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE, 2267 sc->num_segs, PAGE_SIZE, 0, 2268 BUS_DMA_NOWAIT, &txbuf->map); 2269 2270 if (error != 0) { 2271 printf("%s: Unable to create TX DMA map\n", 2272 ifp->if_xname); 2273 goto fail; 2274 } 2275 } 2276 2277 return 0; 2278 fail: 2279 return (error); 2280 } 2281 2282 /********************************************************************* 2283 * 2284 * Initialize a transmit ring. 2285 * 2286 **********************************************************************/ 2287 int 2288 ixgbe_setup_transmit_ring(struct tx_ring *txr) 2289 { 2290 struct ix_softc *sc = txr->sc; 2291 int error; 2292 2293 /* Now allocate transmit buffers for the ring */ 2294 if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0) 2295 return (error); 2296 2297 /* Clear the old ring contents */ 2298 bzero((void *)txr->tx_base, 2299 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 2300 2301 /* Reset indices */ 2302 txr->next_avail_desc = 0; 2303 txr->next_to_clean = 0; 2304 2305 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2306 0, txr->txdma.dma_map->dm_mapsize, 2307 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2308 2309 return (0); 2310 } 2311 2312 /********************************************************************* 2313 * 2314 * Initialize all transmit rings. 2315 * 2316 **********************************************************************/ 2317 int 2318 ixgbe_setup_transmit_structures(struct ix_softc *sc) 2319 { 2320 struct tx_ring *txr = sc->tx_rings; 2321 int i, error; 2322 2323 for (i = 0; i < sc->num_queues; i++, txr++) { 2324 if ((error = ixgbe_setup_transmit_ring(txr)) != 0) 2325 goto fail; 2326 } 2327 2328 return (0); 2329 fail: 2330 ixgbe_free_transmit_structures(sc); 2331 return (error); 2332 } 2333 2334 /********************************************************************* 2335 * 2336 * Enable transmit unit. 2337 * 2338 **********************************************************************/ 2339 void 2340 ixgbe_initialize_transmit_units(struct ix_softc *sc) 2341 { 2342 struct ifnet *ifp = &sc->arpcom.ac_if; 2343 struct tx_ring *txr; 2344 struct ixgbe_hw *hw = &sc->hw; 2345 int i; 2346 uint64_t tdba; 2347 uint32_t txctrl; 2348 uint32_t hlreg; 2349 2350 /* Setup the Base and Length of the Tx Descriptor Ring */ 2351 2352 for (i = 0; i < sc->num_queues; i++) { 2353 txr = &sc->tx_rings[i]; 2354 2355 /* Setup descriptor base address */ 2356 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 2357 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 2358 (tdba & 0x00000000ffffffffULL)); 2359 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 2360 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2361 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 2362 2363 /* Setup the HW Tx Head and Tail descriptor pointers */ 2364 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2365 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2366 2367 /* Setup Transmit Descriptor Cmd Settings */ 2368 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2369 txr->queue_status = IXGBE_QUEUE_IDLE; 2370 txr->watchdog_timer = 0; 2371 2372 /* Disable Head Writeback */ 2373 switch (hw->mac.type) { 2374 case ixgbe_mac_82598EB: 2375 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2376 break; 2377 case ixgbe_mac_82599EB: 2378 case ixgbe_mac_X540: 2379 default: 2380 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2381 break; 2382 } 2383 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2384 switch (hw->mac.type) { 2385 case ixgbe_mac_82598EB: 2386 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2387 break; 2388 case ixgbe_mac_82599EB: 2389 case ixgbe_mac_X540: 2390 default: 2391 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2392 break; 2393 } 2394 } 2395 ifp->if_timer = 0; 2396 2397 if (hw->mac.type != ixgbe_mac_82598EB) { 2398 uint32_t dmatxctl, rttdcs; 2399 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2400 dmatxctl |= IXGBE_DMATXCTL_TE; 2401 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2402 /* Disable arbiter to set MTQC */ 2403 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2404 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2405 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2406 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2407 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2408 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2409 } 2410 2411 /* Enable TCP/UDP padding when using TSO */ 2412 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2413 hlreg |= IXGBE_HLREG0_TXPADEN; 2414 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2415 } 2416 2417 /********************************************************************* 2418 * 2419 * Free all transmit rings. 2420 * 2421 **********************************************************************/ 2422 void 2423 ixgbe_free_transmit_structures(struct ix_softc *sc) 2424 { 2425 struct tx_ring *txr = sc->tx_rings; 2426 int i; 2427 2428 for (i = 0; i < sc->num_queues; i++, txr++) 2429 ixgbe_free_transmit_buffers(txr); 2430 } 2431 2432 /********************************************************************* 2433 * 2434 * Free transmit ring related data structures. 2435 * 2436 **********************************************************************/ 2437 void 2438 ixgbe_free_transmit_buffers(struct tx_ring *txr) 2439 { 2440 struct ix_softc *sc = txr->sc; 2441 struct ixgbe_tx_buf *tx_buffer; 2442 int i; 2443 2444 INIT_DEBUGOUT("free_transmit_ring: begin"); 2445 2446 if (txr->tx_buffers == NULL) 2447 return; 2448 2449 tx_buffer = txr->tx_buffers; 2450 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 2451 if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) { 2452 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2453 0, tx_buffer->map->dm_mapsize, 2454 BUS_DMASYNC_POSTWRITE); 2455 bus_dmamap_unload(txr->txdma.dma_tag, 2456 tx_buffer->map); 2457 } 2458 if (tx_buffer->m_head != NULL) { 2459 m_freem(tx_buffer->m_head); 2460 tx_buffer->m_head = NULL; 2461 } 2462 if (tx_buffer->map != NULL) { 2463 bus_dmamap_destroy(txr->txdma.dma_tag, 2464 tx_buffer->map); 2465 tx_buffer->map = NULL; 2466 } 2467 } 2468 2469 if (txr->tx_buffers != NULL) 2470 free(txr->tx_buffers, M_DEVBUF, 2471 sc->num_tx_desc * sizeof(struct ixgbe_tx_buf)); 2472 txr->tx_buffers = NULL; 2473 txr->txtag = NULL; 2474 } 2475 2476 /********************************************************************* 2477 * 2478 * Advanced Context Descriptor setup for VLAN or CSUM 2479 * 2480 **********************************************************************/ 2481 2482 static inline int 2483 ixgbe_tx_offload(struct mbuf *mp, uint32_t *vlan_macip_lens, 2484 uint32_t *type_tucmd_mlhl, uint32_t *olinfo_status, uint32_t *cmd_type_len, 2485 uint32_t *mss_l4len_idx) 2486 { 2487 struct ether_extracted ext; 2488 int offload = 0; 2489 uint32_t ethlen, iphlen; 2490 2491 ether_extract_headers(mp, &ext); 2492 ethlen = sizeof(*ext.eh); 2493 2494 *vlan_macip_lens |= (ethlen << IXGBE_ADVTXD_MACLEN_SHIFT); 2495 2496 if (ext.ip4) { 2497 iphlen = ext.ip4->ip_hl << 2; 2498 2499 if (ISSET(mp->m_pkthdr.csum_flags, M_IPV4_CSUM_OUT)) { 2500 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2501 offload = 1; 2502 } 2503 2504 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2505 #ifdef INET6 2506 } else if (ext.ip6) { 2507 iphlen = sizeof(*ext.ip6); 2508 2509 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2510 #endif 2511 } else { 2512 if (mp->m_pkthdr.csum_flags & M_TCP_TSO) 2513 tcpstat_inc(tcps_outbadtso); 2514 return offload; 2515 } 2516 2517 *vlan_macip_lens |= iphlen; 2518 2519 if (ext.tcp) { 2520 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2521 if (ISSET(mp->m_pkthdr.csum_flags, M_TCP_CSUM_OUT)) { 2522 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2523 offload = 1; 2524 } 2525 } else if (ext.udp) { 2526 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2527 if (ISSET(mp->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) { 2528 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2529 offload = 1; 2530 } 2531 } 2532 2533 if (mp->m_pkthdr.csum_flags & M_TCP_TSO) { 2534 if (ext.tcp) { 2535 uint32_t hdrlen, thlen, paylen, outlen; 2536 2537 thlen = ext.tcp->th_off << 2; 2538 2539 outlen = mp->m_pkthdr.ph_mss; 2540 *mss_l4len_idx |= outlen << IXGBE_ADVTXD_MSS_SHIFT; 2541 *mss_l4len_idx |= thlen << IXGBE_ADVTXD_L4LEN_SHIFT; 2542 2543 hdrlen = ethlen + iphlen + thlen; 2544 paylen = mp->m_pkthdr.len - hdrlen; 2545 CLR(*olinfo_status, IXGBE_ADVTXD_PAYLEN_MASK 2546 << IXGBE_ADVTXD_PAYLEN_SHIFT); 2547 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 2548 2549 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2550 offload = 1; 2551 2552 tcpstat_add(tcps_outpkttso, 2553 (paylen + outlen - 1) / outlen); 2554 } else 2555 tcpstat_inc(tcps_outbadtso); 2556 } 2557 2558 return offload; 2559 } 2560 2561 static int 2562 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, 2563 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2564 { 2565 struct ixgbe_adv_tx_context_desc *TXD; 2566 struct ixgbe_tx_buf *tx_buffer; 2567 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2568 uint32_t mss_l4len_idx = 0; 2569 int ctxd = txr->next_avail_desc; 2570 int offload = 0; 2571 2572 /* Indicate the whole packet as payload when not doing TSO */ 2573 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2574 2575 #if NVLAN > 0 2576 if (ISSET(mp->m_flags, M_VLANTAG)) { 2577 uint32_t vtag = mp->m_pkthdr.ether_vtag; 2578 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2579 *cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 2580 offload |= 1; 2581 } 2582 #endif 2583 2584 offload |= ixgbe_tx_offload(mp, &vlan_macip_lens, &type_tucmd_mlhl, 2585 olinfo_status, cmd_type_len, &mss_l4len_idx); 2586 2587 if (!offload) 2588 return (0); 2589 2590 TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; 2591 tx_buffer = &txr->tx_buffers[ctxd]; 2592 2593 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2594 2595 /* Now copy bits into descriptor */ 2596 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2597 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2598 TXD->seqnum_seed = htole32(0); 2599 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2600 2601 tx_buffer->m_head = NULL; 2602 tx_buffer->eop_index = -1; 2603 2604 return (1); 2605 } 2606 2607 /********************************************************************** 2608 * 2609 * Examine each tx_buffer in the used queue. If the hardware is done 2610 * processing the packet then free associated resources. The 2611 * tx_buffer is put back on the free queue. 2612 * 2613 **********************************************************************/ 2614 int 2615 ixgbe_txeof(struct tx_ring *txr) 2616 { 2617 struct ix_softc *sc = txr->sc; 2618 struct ifqueue *ifq = txr->ifq; 2619 struct ifnet *ifp = &sc->arpcom.ac_if; 2620 unsigned int head, tail, last; 2621 struct ixgbe_tx_buf *tx_buffer; 2622 struct ixgbe_legacy_tx_desc *tx_desc; 2623 2624 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2625 return FALSE; 2626 2627 head = txr->next_avail_desc; 2628 tail = txr->next_to_clean; 2629 2630 membar_consumer(); 2631 2632 if (head == tail) 2633 return (FALSE); 2634 2635 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2636 0, txr->txdma.dma_map->dm_mapsize, 2637 BUS_DMASYNC_POSTREAD); 2638 2639 for (;;) { 2640 tx_buffer = &txr->tx_buffers[tail]; 2641 last = tx_buffer->eop_index; 2642 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2643 2644 if (!ISSET(tx_desc->upper.fields.status, IXGBE_TXD_STAT_DD)) 2645 break; 2646 2647 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2648 0, tx_buffer->map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2649 bus_dmamap_unload(txr->txdma.dma_tag, tx_buffer->map); 2650 m_freem(tx_buffer->m_head); 2651 2652 tx_buffer->m_head = NULL; 2653 tx_buffer->eop_index = -1; 2654 2655 tail = last + 1; 2656 if (tail == sc->num_tx_desc) 2657 tail = 0; 2658 if (head == tail) { 2659 /* All clean, turn off the timer */ 2660 ifp->if_timer = 0; 2661 break; 2662 } 2663 } 2664 2665 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2666 0, txr->txdma.dma_map->dm_mapsize, 2667 BUS_DMASYNC_PREREAD); 2668 2669 membar_producer(); 2670 2671 txr->next_to_clean = tail; 2672 2673 if (ifq_is_oactive(ifq)) 2674 ifq_restart(ifq); 2675 2676 return TRUE; 2677 } 2678 2679 /********************************************************************* 2680 * 2681 * Get a buffer from system mbuf buffer pool. 2682 * 2683 **********************************************************************/ 2684 int 2685 ixgbe_get_buf(struct rx_ring *rxr, int i) 2686 { 2687 struct ix_softc *sc = rxr->sc; 2688 struct ixgbe_rx_buf *rxbuf; 2689 struct mbuf *mp; 2690 int error; 2691 union ixgbe_adv_rx_desc *rxdesc; 2692 2693 rxbuf = &rxr->rx_buffers[i]; 2694 rxdesc = &rxr->rx_base[i]; 2695 if (rxbuf->buf) { 2696 printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n", 2697 sc->dev.dv_xname, i); 2698 return (ENOBUFS); 2699 } 2700 2701 /* needed in any case so prealocate since this one will fail for sure */ 2702 mp = MCLGETL(NULL, M_DONTWAIT, sc->rx_mbuf_sz); 2703 if (!mp) 2704 return (ENOBUFS); 2705 2706 mp->m_data += (mp->m_ext.ext_size - sc->rx_mbuf_sz); 2707 mp->m_len = mp->m_pkthdr.len = sc->rx_mbuf_sz; 2708 2709 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, 2710 mp, BUS_DMA_NOWAIT); 2711 if (error) { 2712 m_freem(mp); 2713 return (error); 2714 } 2715 2716 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2717 0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 2718 rxbuf->buf = mp; 2719 2720 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 2721 2722 return (0); 2723 } 2724 2725 /********************************************************************* 2726 * 2727 * Allocate memory for rx_buffer structures. Since we use one 2728 * rx_buffer per received packet, the maximum number of rx_buffer's 2729 * that we'll need is equal to the number of receive descriptors 2730 * that we've allocated. 2731 * 2732 **********************************************************************/ 2733 int 2734 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2735 { 2736 struct ix_softc *sc = rxr->sc; 2737 struct ifnet *ifp = &sc->arpcom.ac_if; 2738 struct ixgbe_rx_buf *rxbuf; 2739 int i, error; 2740 2741 if (!(rxr->rx_buffers = mallocarray(sc->num_rx_desc, 2742 sizeof(struct ixgbe_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2743 printf("%s: Unable to allocate rx_buffer memory\n", 2744 ifp->if_xname); 2745 error = ENOMEM; 2746 goto fail; 2747 } 2748 2749 rxbuf = rxr->rx_buffers; 2750 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2751 error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1, 2752 16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map); 2753 if (error) { 2754 printf("%s: Unable to create Pack DMA map\n", 2755 ifp->if_xname); 2756 goto fail; 2757 } 2758 } 2759 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2760 rxr->rxdma.dma_map->dm_mapsize, 2761 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2762 2763 return (0); 2764 2765 fail: 2766 return (error); 2767 } 2768 2769 /********************************************************************* 2770 * 2771 * Initialize a receive ring and its buffers. 2772 * 2773 **********************************************************************/ 2774 int 2775 ixgbe_setup_receive_ring(struct rx_ring *rxr) 2776 { 2777 struct ix_softc *sc = rxr->sc; 2778 struct ifnet *ifp = &sc->arpcom.ac_if; 2779 int rsize, error; 2780 2781 rsize = roundup2(sc->num_rx_desc * 2782 sizeof(union ixgbe_adv_rx_desc), 4096); 2783 /* Clear the ring contents */ 2784 bzero((void *)rxr->rx_base, rsize); 2785 2786 if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0) 2787 return (error); 2788 2789 /* Setup our descriptor indices */ 2790 rxr->next_to_check = 0; 2791 rxr->last_desc_filled = sc->num_rx_desc - 1; 2792 2793 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2794 sc->num_rx_desc - 1); 2795 2796 ixgbe_rxfill(rxr); 2797 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 2798 printf("%s: unable to fill any rx descriptors\n", 2799 sc->dev.dv_xname); 2800 return (ENOBUFS); 2801 } 2802 2803 return (0); 2804 } 2805 2806 int 2807 ixgbe_rxfill(struct rx_ring *rxr) 2808 { 2809 struct ix_softc *sc = rxr->sc; 2810 int post = 0; 2811 u_int slots; 2812 int i; 2813 2814 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2815 0, rxr->rxdma.dma_map->dm_mapsize, 2816 BUS_DMASYNC_POSTWRITE); 2817 2818 i = rxr->last_desc_filled; 2819 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); 2820 slots > 0; slots--) { 2821 if (++i == sc->num_rx_desc) 2822 i = 0; 2823 2824 if (ixgbe_get_buf(rxr, i) != 0) 2825 break; 2826 2827 rxr->last_desc_filled = i; 2828 post = 1; 2829 } 2830 2831 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2832 0, rxr->rxdma.dma_map->dm_mapsize, 2833 BUS_DMASYNC_PREWRITE); 2834 2835 if_rxr_put(&rxr->rx_ring, slots); 2836 2837 return (post); 2838 } 2839 2840 void 2841 ixgbe_rxrefill(void *xrxr) 2842 { 2843 struct rx_ring *rxr = xrxr; 2844 struct ix_softc *sc = rxr->sc; 2845 2846 if (ixgbe_rxfill(rxr)) { 2847 /* Advance the Rx Queue "Tail Pointer" */ 2848 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me), 2849 rxr->last_desc_filled); 2850 } else if (if_rxr_inuse(&rxr->rx_ring) == 0) 2851 timeout_add(&rxr->rx_refill, 1); 2852 2853 } 2854 2855 /********************************************************************* 2856 * 2857 * Initialize all receive rings. 2858 * 2859 **********************************************************************/ 2860 int 2861 ixgbe_setup_receive_structures(struct ix_softc *sc) 2862 { 2863 struct rx_ring *rxr = sc->rx_rings; 2864 int i; 2865 2866 for (i = 0; i < sc->num_queues; i++, rxr++) 2867 if (ixgbe_setup_receive_ring(rxr)) 2868 goto fail; 2869 2870 return (0); 2871 fail: 2872 ixgbe_free_receive_structures(sc); 2873 return (ENOBUFS); 2874 } 2875 2876 /********************************************************************* 2877 * 2878 * Setup receive registers and features. 2879 * 2880 **********************************************************************/ 2881 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2882 2883 void 2884 ixgbe_initialize_receive_units(struct ix_softc *sc) 2885 { 2886 struct ifnet *ifp = &sc->arpcom.ac_if; 2887 struct rx_ring *rxr = sc->rx_rings; 2888 struct ixgbe_hw *hw = &sc->hw; 2889 uint32_t bufsz, fctrl, srrctl, rxcsum, rdrxctl; 2890 uint32_t hlreg; 2891 int i; 2892 2893 /* 2894 * Make sure receives are disabled while 2895 * setting up the descriptor ring 2896 */ 2897 ixgbe_disable_rx(hw); 2898 2899 /* Enable broadcasts */ 2900 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2901 fctrl |= IXGBE_FCTRL_BAM; 2902 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 2903 fctrl |= IXGBE_FCTRL_DPF; 2904 fctrl |= IXGBE_FCTRL_PMCF; 2905 } 2906 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 2907 2908 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2909 /* Always enable jumbo frame reception */ 2910 hlreg |= IXGBE_HLREG0_JUMBOEN; 2911 /* Always enable CRC stripping */ 2912 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 2913 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2914 2915 if (ISSET(ifp->if_xflags, IFXF_LRO)) { 2916 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2917 2918 /* This field has to be set to zero. */ 2919 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2920 2921 /* RSC Coalescing on ACK Change */ 2922 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 2923 rdrxctl |= IXGBE_RDRXCTL_FCOE_WRFIX; 2924 2925 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 2926 } 2927 2928 bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2929 2930 for (i = 0; i < sc->num_queues; i++, rxr++) { 2931 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2932 2933 /* Setup the Base and Length of the Rx Descriptor Ring */ 2934 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 2935 (rdba & 0x00000000ffffffffULL)); 2936 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 2937 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 2938 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2939 2940 /* Set up the SRRCTL register */ 2941 srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2942 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 2943 2944 if (ISSET(ifp->if_xflags, IFXF_LRO)) { 2945 rdrxctl = IXGBE_READ_REG(&sc->hw, IXGBE_RSCCTL(i)); 2946 2947 /* Enable Receive Side Coalescing */ 2948 rdrxctl |= IXGBE_RSCCTL_RSCEN; 2949 rdrxctl |= IXGBE_RSCCTL_MAXDESC_16; 2950 2951 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), rdrxctl); 2952 } 2953 2954 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2955 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 2956 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 2957 } 2958 2959 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 2960 uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR | 2961 IXGBE_PSRTYPE_UDPHDR | 2962 IXGBE_PSRTYPE_IPV4HDR | 2963 IXGBE_PSRTYPE_IPV6HDR; 2964 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 2965 } 2966 2967 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2968 rxcsum &= ~IXGBE_RXCSUM_PCSD; 2969 2970 ixgbe_initialize_rss_mapping(sc); 2971 2972 /* Setup RSS */ 2973 if (sc->num_queues > 1) { 2974 /* RSS and RX IPP Checksum are mutually exclusive */ 2975 rxcsum |= IXGBE_RXCSUM_PCSD; 2976 } 2977 2978 /* Map QPRC/QPRDC/QPTC on a per queue basis */ 2979 ixgbe_map_queue_statistics(sc); 2980 2981 /* This is useful for calculating UDP/IP fragment checksums */ 2982 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2983 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2984 2985 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2986 } 2987 2988 void 2989 ixgbe_initialize_rss_mapping(struct ix_softc *sc) 2990 { 2991 struct ixgbe_hw *hw = &sc->hw; 2992 uint32_t reta = 0, mrqc, rss_key[10]; 2993 int i, j, queue_id, table_size, index_mult; 2994 2995 /* set up random bits */ 2996 stoeplitz_to_key(&rss_key, sizeof(rss_key)); 2997 2998 /* Set multiplier for RETA setup and table size based on MAC */ 2999 index_mult = 0x1; 3000 table_size = 128; 3001 switch (sc->hw.mac.type) { 3002 case ixgbe_mac_82598EB: 3003 index_mult = 0x11; 3004 break; 3005 case ixgbe_mac_X550: 3006 case ixgbe_mac_X550EM_x: 3007 case ixgbe_mac_X550EM_a: 3008 table_size = 512; 3009 break; 3010 default: 3011 break; 3012 } 3013 3014 /* Set up the redirection table */ 3015 for (i = 0, j = 0; i < table_size; i++, j++) { 3016 if (j == sc->num_queues) j = 0; 3017 queue_id = (j * index_mult); 3018 /* 3019 * The low 8 bits are for hash value (n+0); 3020 * The next 8 bits are for hash value (n+1), etc. 3021 */ 3022 reta = reta >> 8; 3023 reta = reta | ( ((uint32_t) queue_id) << 24); 3024 if ((i & 3) == 3) { 3025 if (i < 128) 3026 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 3027 else 3028 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 3029 reta); 3030 reta = 0; 3031 } 3032 } 3033 3034 /* Now fill our hash function seeds */ 3035 for (i = 0; i < 10; i++) 3036 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 3037 3038 /* 3039 * Disable UDP - IP fragments aren't currently being handled 3040 * and so we end up with a mix of 2-tuple and 4-tuple 3041 * traffic. 3042 */ 3043 mrqc = IXGBE_MRQC_RSSEN 3044 | IXGBE_MRQC_RSS_FIELD_IPV4 3045 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 3046 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 3047 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 3048 | IXGBE_MRQC_RSS_FIELD_IPV6 3049 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 3050 ; 3051 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3052 } 3053 3054 /********************************************************************* 3055 * 3056 * Free all receive rings. 3057 * 3058 **********************************************************************/ 3059 void 3060 ixgbe_free_receive_structures(struct ix_softc *sc) 3061 { 3062 struct rx_ring *rxr; 3063 int i; 3064 3065 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3066 if_rxr_init(&rxr->rx_ring, 0, 0); 3067 3068 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3069 ixgbe_free_receive_buffers(rxr); 3070 } 3071 3072 /********************************************************************* 3073 * 3074 * Free receive ring data structures 3075 * 3076 **********************************************************************/ 3077 void 3078 ixgbe_free_receive_buffers(struct rx_ring *rxr) 3079 { 3080 struct ix_softc *sc; 3081 struct ixgbe_rx_buf *rxbuf; 3082 int i; 3083 3084 sc = rxr->sc; 3085 if (rxr->rx_buffers != NULL) { 3086 for (i = 0; i < sc->num_rx_desc; i++) { 3087 rxbuf = &rxr->rx_buffers[i]; 3088 if (rxbuf->buf != NULL) { 3089 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 3090 0, rxbuf->map->dm_mapsize, 3091 BUS_DMASYNC_POSTREAD); 3092 bus_dmamap_unload(rxr->rxdma.dma_tag, 3093 rxbuf->map); 3094 m_freem(rxbuf->buf); 3095 rxbuf->buf = NULL; 3096 } 3097 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 3098 rxbuf->map = NULL; 3099 } 3100 free(rxr->rx_buffers, M_DEVBUF, 3101 sc->num_rx_desc * sizeof(struct ixgbe_rx_buf)); 3102 rxr->rx_buffers = NULL; 3103 } 3104 } 3105 3106 /********************************************************************* 3107 * 3108 * This routine executes in interrupt context. It replenishes 3109 * the mbufs in the descriptor and sends data which has been 3110 * dma'ed into host memory to upper layer. 3111 * 3112 *********************************************************************/ 3113 int 3114 ixgbe_rxeof(struct rx_ring *rxr) 3115 { 3116 struct ix_softc *sc = rxr->sc; 3117 struct ifnet *ifp = &sc->arpcom.ac_if; 3118 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 3119 struct mbuf *mp, *sendmp; 3120 uint8_t eop = 0; 3121 uint16_t len, vtag; 3122 uint32_t staterr = 0; 3123 struct ixgbe_rx_buf *rxbuf, *nxbuf; 3124 union ixgbe_adv_rx_desc *rxdesc; 3125 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 3126 int i, nextp, rsccnt; 3127 3128 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3129 return FALSE; 3130 3131 i = rxr->next_to_check; 3132 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 3133 uint32_t hash; 3134 uint16_t hashtype; 3135 3136 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3137 dsize * i, dsize, BUS_DMASYNC_POSTREAD); 3138 3139 rxdesc = &rxr->rx_base[i]; 3140 staterr = letoh32(rxdesc->wb.upper.status_error); 3141 if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) { 3142 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3143 dsize * i, dsize, 3144 BUS_DMASYNC_PREREAD); 3145 break; 3146 } 3147 3148 /* Zero out the receive descriptors status */ 3149 rxdesc->wb.upper.status_error = 0; 3150 rxbuf = &rxr->rx_buffers[i]; 3151 3152 /* pull the mbuf off the ring */ 3153 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 3154 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3155 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 3156 3157 mp = rxbuf->buf; 3158 len = letoh16(rxdesc->wb.upper.length); 3159 vtag = letoh16(rxdesc->wb.upper.vlan); 3160 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3161 hash = lemtoh32(&rxdesc->wb.lower.hi_dword.rss); 3162 hashtype = 3163 lemtoh16(&rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 3164 IXGBE_RXDADV_RSSTYPE_MASK; 3165 rsccnt = lemtoh32(&rxdesc->wb.lower.lo_dword.data) & 3166 IXGBE_RXDADV_RSCCNT_MASK; 3167 rsccnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; 3168 3169 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) { 3170 if (rxbuf->fmp) { 3171 m_freem(rxbuf->fmp); 3172 rxbuf->fmp = NULL; 3173 } 3174 3175 m_freem(mp); 3176 rxbuf->buf = NULL; 3177 goto next_desc; 3178 } 3179 3180 if (mp == NULL) { 3181 panic("%s: ixgbe_rxeof: NULL mbuf in slot %d " 3182 "(nrx %d, filled %d)", sc->dev.dv_xname, 3183 i, if_rxr_inuse(&rxr->rx_ring), 3184 rxr->last_desc_filled); 3185 } 3186 3187 if (!eop) { 3188 /* 3189 * Figure out the next descriptor of this frame. 3190 */ 3191 if (rsccnt) { 3192 nextp = staterr & IXGBE_RXDADV_NEXTP_MASK; 3193 nextp >>= IXGBE_RXDADV_NEXTP_SHIFT; 3194 } else { 3195 nextp = i + 1; 3196 } 3197 if (nextp == sc->num_rx_desc) 3198 nextp = 0; 3199 nxbuf = &rxr->rx_buffers[nextp]; 3200 /* prefetch(nxbuf); */ 3201 } 3202 3203 /* 3204 * Rather than using the fmp/lmp global pointers 3205 * we now keep the head of a packet chain in the 3206 * buffer struct and pass this along from one 3207 * descriptor to the next, until we get EOP. 3208 */ 3209 mp->m_len = len; 3210 /* 3211 * See if there is a stored head 3212 * that determines what we are 3213 */ 3214 sendmp = rxbuf->fmp; 3215 rxbuf->buf = rxbuf->fmp = NULL; 3216 3217 if (sendmp != NULL) { /* secondary frag */ 3218 sendmp->m_pkthdr.len += mp->m_len; 3219 3220 /* 3221 * This function iterates over interleaved descriptors. 3222 * Thus, we reuse ph_mss as global segment counter per 3223 * TCP connection, instead of introducing a new variable 3224 * in m_pkthdr. 3225 */ 3226 if (rsccnt) 3227 sendmp->m_pkthdr.ph_mss += rsccnt - 1; 3228 } else { 3229 /* first desc of a non-ps chain */ 3230 sendmp = mp; 3231 sendmp->m_pkthdr.len = mp->m_len; 3232 if (rsccnt) 3233 sendmp->m_pkthdr.ph_mss = rsccnt - 1; 3234 #if NVLAN > 0 3235 if (sc->vlan_stripping && staterr & IXGBE_RXD_STAT_VP) { 3236 sendmp->m_pkthdr.ether_vtag = vtag; 3237 sendmp->m_flags |= M_VLANTAG; 3238 } 3239 #endif 3240 } 3241 3242 /* Pass the head pointer on */ 3243 if (eop == 0) { 3244 nxbuf->fmp = sendmp; 3245 sendmp = NULL; 3246 mp->m_next = nxbuf->buf; 3247 } else { /* Sending this frame? */ 3248 uint16_t pkts; 3249 3250 ixgbe_rx_checksum(staterr, sendmp); 3251 3252 if (hashtype != IXGBE_RXDADV_RSSTYPE_NONE) { 3253 sendmp->m_pkthdr.ph_flowid = hash; 3254 SET(sendmp->m_pkthdr.csum_flags, M_FLOWID); 3255 } 3256 3257 pkts = sendmp->m_pkthdr.ph_mss; 3258 sendmp->m_pkthdr.ph_mss = 0; 3259 3260 if (pkts > 1) { 3261 struct ether_extracted ext; 3262 uint32_t hdrlen, paylen; 3263 3264 /* Calculate header size. */ 3265 ether_extract_headers(sendmp, &ext); 3266 hdrlen = sizeof(*ext.eh); 3267 if (ext.ip4) 3268 hdrlen += ext.ip4->ip_hl << 2; 3269 if (ext.ip6) 3270 hdrlen += sizeof(*ext.ip6); 3271 if (ext.tcp) { 3272 hdrlen += ext.tcp->th_off << 2; 3273 tcpstat_inc(tcps_inhwlro); 3274 tcpstat_add(tcps_inpktlro, pkts); 3275 } else { 3276 tcpstat_inc(tcps_inbadlro); 3277 } 3278 3279 /* 3280 * If we gonna forward this packet, we have to 3281 * mark it as TSO, set a correct mss, 3282 * and recalculate the TCP checksum. 3283 */ 3284 paylen = sendmp->m_pkthdr.len - hdrlen; 3285 if (ext.tcp && paylen >= pkts) { 3286 SET(sendmp->m_pkthdr.csum_flags, 3287 M_TCP_TSO); 3288 sendmp->m_pkthdr.ph_mss = paylen / pkts; 3289 } 3290 if (ext.tcp && 3291 ISSET(sendmp->m_pkthdr.csum_flags, 3292 M_TCP_CSUM_IN_OK)) { 3293 SET(sendmp->m_pkthdr.csum_flags, 3294 M_TCP_CSUM_OUT); 3295 } 3296 } 3297 3298 ml_enqueue(&ml, sendmp); 3299 } 3300 next_desc: 3301 if_rxr_put(&rxr->rx_ring, 1); 3302 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3303 dsize * i, dsize, 3304 BUS_DMASYNC_PREREAD); 3305 3306 /* Advance our pointers to the next descriptor. */ 3307 if (++i == sc->num_rx_desc) 3308 i = 0; 3309 } 3310 rxr->next_to_check = i; 3311 3312 if (ifiq_input(rxr->ifiq, &ml)) 3313 if_rxr_livelocked(&rxr->rx_ring); 3314 3315 if (!(staterr & IXGBE_RXD_STAT_DD)) 3316 return FALSE; 3317 3318 return TRUE; 3319 } 3320 3321 /********************************************************************* 3322 * 3323 * Verify that the hardware indicated that the checksum is valid. 3324 * Inform the stack about the status of checksum so that stack 3325 * doesn't spend time verifying the checksum. 3326 * 3327 *********************************************************************/ 3328 void 3329 ixgbe_rx_checksum(uint32_t staterr, struct mbuf * mp) 3330 { 3331 uint16_t status = (uint16_t) staterr; 3332 uint8_t errors = (uint8_t) (staterr >> 24); 3333 3334 if (status & IXGBE_RXD_STAT_IPCS) { 3335 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3336 /* IP Checksum Good */ 3337 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 3338 } else 3339 mp->m_pkthdr.csum_flags = 0; 3340 } 3341 if (status & IXGBE_RXD_STAT_L4CS) { 3342 if (!(errors & IXGBE_RXD_ERR_TCPE)) 3343 mp->m_pkthdr.csum_flags |= 3344 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 3345 } 3346 } 3347 3348 void 3349 ixgbe_setup_vlan_hw_support(struct ix_softc *sc) 3350 { 3351 struct ifnet *ifp = &sc->arpcom.ac_if; 3352 uint32_t ctrl; 3353 int i; 3354 3355 /* 3356 * We have to disable VLAN striping when using TCP offloading, due to a 3357 * firmware bug. 3358 */ 3359 if (ISSET(ifp->if_xflags, IFXF_LRO)) { 3360 sc->vlan_stripping = 0; 3361 return; 3362 } 3363 3364 sc->vlan_stripping = 1; 3365 3366 /* 3367 * A soft reset zero's out the VFTA, so 3368 * we need to repopulate it now. 3369 */ 3370 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 3371 if (sc->shadow_vfta[i] != 0) 3372 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i), 3373 sc->shadow_vfta[i]); 3374 } 3375 3376 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 3377 #if 0 3378 /* Enable the Filter Table if enabled */ 3379 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3380 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3381 ctrl |= IXGBE_VLNCTRL_VFE; 3382 } 3383 #endif 3384 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3385 ctrl |= IXGBE_VLNCTRL_VME; 3386 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 3387 3388 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 3389 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 3390 for (i = 0; i < sc->num_queues; i++) { 3391 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 3392 ctrl |= IXGBE_RXDCTL_VME; 3393 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl); 3394 } 3395 } 3396 } 3397 3398 void 3399 ixgbe_enable_intr(struct ix_softc *sc) 3400 { 3401 struct ixgbe_hw *hw = &sc->hw; 3402 uint32_t mask, fwsm; 3403 3404 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3405 /* Enable Fan Failure detection */ 3406 if (hw->device_id == IXGBE_DEV_ID_82598AT) 3407 mask |= IXGBE_EIMS_GPI_SDP1; 3408 3409 switch (sc->hw.mac.type) { 3410 case ixgbe_mac_82599EB: 3411 mask |= IXGBE_EIMS_ECC; 3412 /* Temperature sensor on some adapters */ 3413 mask |= IXGBE_EIMS_GPI_SDP0; 3414 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 3415 mask |= IXGBE_EIMS_GPI_SDP1; 3416 mask |= IXGBE_EIMS_GPI_SDP2; 3417 break; 3418 case ixgbe_mac_X540: 3419 mask |= IXGBE_EIMS_ECC; 3420 /* Detect if Thermal Sensor is enabled */ 3421 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3422 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3423 mask |= IXGBE_EIMS_TS; 3424 break; 3425 case ixgbe_mac_X550: 3426 case ixgbe_mac_X550EM_x: 3427 case ixgbe_mac_X550EM_a: 3428 mask |= IXGBE_EIMS_ECC; 3429 /* MAC thermal sensor is automatically enabled */ 3430 mask |= IXGBE_EIMS_TS; 3431 /* Some devices use SDP0 for important information */ 3432 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 3433 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 3434 mask |= IXGBE_EIMS_GPI_SDP0_X540; 3435 default: 3436 break; 3437 } 3438 3439 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3440 3441 /* With MSI-X we use auto clear */ 3442 if (sc->sc_intrmap) { 3443 mask = IXGBE_EIMS_ENABLE_MASK; 3444 /* Don't autoclear Link */ 3445 mask &= ~IXGBE_EIMS_OTHER; 3446 mask &= ~IXGBE_EIMS_LSC; 3447 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 3448 } 3449 3450 IXGBE_WRITE_FLUSH(hw); 3451 } 3452 3453 void 3454 ixgbe_disable_intr(struct ix_softc *sc) 3455 { 3456 if (sc->sc_intrmap) 3457 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3458 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3459 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3460 } else { 3461 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3462 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3463 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3464 } 3465 IXGBE_WRITE_FLUSH(&sc->hw); 3466 } 3467 3468 uint16_t 3469 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3470 { 3471 struct pci_attach_args *pa; 3472 uint32_t value; 3473 int high = 0; 3474 3475 if (reg & 0x2) { 3476 high = 1; 3477 reg &= ~0x2; 3478 } 3479 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3480 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3481 3482 if (high) 3483 value >>= 16; 3484 3485 return (value & 0xffff); 3486 } 3487 3488 void 3489 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3490 { 3491 struct pci_attach_args *pa; 3492 uint32_t rv; 3493 int high = 0; 3494 3495 /* Need to do read/mask/write... because 16 vs 32 bit!!! */ 3496 if (reg & 0x2) { 3497 high = 1; 3498 reg &= ~0x2; 3499 } 3500 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3501 rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3502 if (!high) 3503 rv = (rv & 0xffff0000) | value; 3504 else 3505 rv = (rv & 0xffff) | ((uint32_t)value << 16); 3506 pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv); 3507 } 3508 3509 /* 3510 * Setup the correct IVAR register for a particular MSIX interrupt 3511 * (yes this is all very magic and confusing :) 3512 * - entry is the register array entry 3513 * - vector is the MSIX vector for this queue 3514 * - type is RX/TX/MISC 3515 */ 3516 void 3517 ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type) 3518 { 3519 struct ixgbe_hw *hw = &sc->hw; 3520 uint32_t ivar, index; 3521 3522 vector |= IXGBE_IVAR_ALLOC_VAL; 3523 3524 switch (hw->mac.type) { 3525 3526 case ixgbe_mac_82598EB: 3527 if (type == -1) 3528 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3529 else 3530 entry += (type * 64); 3531 index = (entry >> 2) & 0x1F; 3532 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3533 ivar &= ~((uint32_t)0xFF << (8 * (entry & 0x3))); 3534 ivar |= ((uint32_t)vector << (8 * (entry & 0x3))); 3535 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 3536 break; 3537 3538 case ixgbe_mac_82599EB: 3539 case ixgbe_mac_X540: 3540 case ixgbe_mac_X550: 3541 case ixgbe_mac_X550EM_x: 3542 case ixgbe_mac_X550EM_a: 3543 if (type == -1) { /* MISC IVAR */ 3544 index = (entry & 1) * 8; 3545 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3546 ivar &= ~((uint32_t)0xFF << index); 3547 ivar |= ((uint32_t)vector << index); 3548 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3549 } else { /* RX/TX IVARS */ 3550 index = (16 * (entry & 1)) + (8 * type); 3551 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3552 ivar &= ~((uint32_t)0xFF << index); 3553 ivar |= ((uint32_t)vector << index); 3554 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3555 } 3556 3557 default: 3558 break; 3559 } 3560 } 3561 3562 void 3563 ixgbe_configure_ivars(struct ix_softc *sc) 3564 { 3565 struct ix_queue *que = sc->queues; 3566 uint32_t newitr; 3567 int i; 3568 3569 newitr = (4000000 / IXGBE_INTS_PER_SEC) & 0x0FF8; 3570 3571 for (i = 0; i < sc->num_queues; i++, que++) { 3572 /* First the RX queue entry */ 3573 ixgbe_set_ivar(sc, i, que->msix, 0); 3574 /* ... and the TX */ 3575 ixgbe_set_ivar(sc, i, que->msix, 1); 3576 /* Set an Initial EITR value */ 3577 IXGBE_WRITE_REG(&sc->hw, 3578 IXGBE_EITR(que->msix), newitr); 3579 } 3580 3581 /* For the Link interrupt */ 3582 ixgbe_set_ivar(sc, 1, sc->linkvec, -1); 3583 } 3584 3585 /* 3586 * SFP module interrupts handler 3587 */ 3588 void 3589 ixgbe_handle_mod(struct ix_softc *sc) 3590 { 3591 struct ixgbe_hw *hw = &sc->hw; 3592 uint32_t err; 3593 3594 err = hw->phy.ops.identify_sfp(hw); 3595 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3596 printf("%s: Unsupported SFP+ module type was detected!\n", 3597 sc->dev.dv_xname); 3598 return; 3599 } 3600 err = hw->mac.ops.setup_sfp(hw); 3601 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3602 printf("%s: Setup failure - unsupported SFP+ module type!\n", 3603 sc->dev.dv_xname); 3604 return; 3605 } 3606 3607 ixgbe_handle_msf(sc); 3608 } 3609 3610 3611 /* 3612 * MSF (multispeed fiber) interrupts handler 3613 */ 3614 void 3615 ixgbe_handle_msf(struct ix_softc *sc) 3616 { 3617 struct ixgbe_hw *hw = &sc->hw; 3618 uint32_t autoneg; 3619 bool negotiate; 3620 3621 autoneg = hw->phy.autoneg_advertised; 3622 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) { 3623 if (hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate)) 3624 return; 3625 } 3626 if (hw->mac.ops.setup_link) 3627 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3628 3629 ifmedia_delete_instance(&sc->media, IFM_INST_ANY); 3630 ixgbe_add_media_types(sc); 3631 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 3632 } 3633 3634 /* 3635 * External PHY interrupts handler 3636 */ 3637 void 3638 ixgbe_handle_phy(struct ix_softc *sc) 3639 { 3640 struct ixgbe_hw *hw = &sc->hw; 3641 int error; 3642 3643 error = hw->phy.ops.handle_lasi(hw); 3644 if (error == IXGBE_ERR_OVERTEMP) 3645 printf("%s: CRITICAL: EXTERNAL PHY OVER TEMP!! " 3646 " PHY will downshift to lower power state!\n", 3647 sc->dev.dv_xname); 3648 else if (error) 3649 printf("%s: Error handling LASI interrupt: %d\n", 3650 sc->dev.dv_xname, error); 3651 3652 } 3653 3654 #if NKSTAT > 0 3655 enum ix_counter_idx { 3656 ix_counter_crcerrs, 3657 ix_counter_lxontxc, 3658 ix_counter_lxonrxc, 3659 ix_counter_lxofftxc, 3660 ix_counter_lxoffrxc, 3661 ix_counter_prc64, 3662 ix_counter_prc127, 3663 ix_counter_prc255, 3664 ix_counter_prc511, 3665 ix_counter_prc1023, 3666 ix_counter_prc1522, 3667 ix_counter_gptc, 3668 ix_counter_gorc, 3669 ix_counter_gotc, 3670 ix_counter_ruc, 3671 ix_counter_rfc, 3672 ix_counter_roc, 3673 ix_counter_rjc, 3674 ix_counter_tor, 3675 ix_counter_tpr, 3676 ix_counter_tpt, 3677 ix_counter_gprc, 3678 ix_counter_bprc, 3679 ix_counter_mprc, 3680 ix_counter_ptc64, 3681 ix_counter_ptc127, 3682 ix_counter_ptc255, 3683 ix_counter_ptc511, 3684 ix_counter_ptc1023, 3685 ix_counter_ptc1522, 3686 ix_counter_mptc, 3687 ix_counter_bptc, 3688 3689 ix_counter_num, 3690 }; 3691 3692 CTASSERT(KSTAT_KV_U_PACKETS <= 0xff); 3693 CTASSERT(KSTAT_KV_U_BYTES <= 0xff); 3694 3695 struct ix_counter { 3696 char name[KSTAT_KV_NAMELEN]; 3697 uint32_t reg; 3698 uint8_t width; 3699 uint8_t unit; 3700 }; 3701 3702 static const struct ix_counter ix_counters[ix_counter_num] = { 3703 [ix_counter_crcerrs] = { "crc errs", IXGBE_CRCERRS, 32, 3704 KSTAT_KV_U_PACKETS }, 3705 [ix_counter_lxontxc] = { "tx link xon", IXGBE_LXONTXC, 32, 3706 KSTAT_KV_U_PACKETS }, 3707 [ix_counter_lxonrxc] = { "rx link xon", 0, 32, 3708 KSTAT_KV_U_PACKETS }, 3709 [ix_counter_lxofftxc] = { "tx link xoff", IXGBE_LXOFFTXC, 32, 3710 KSTAT_KV_U_PACKETS }, 3711 [ix_counter_lxoffrxc] = { "rx link xoff", 0, 32, 3712 KSTAT_KV_U_PACKETS }, 3713 [ix_counter_prc64] = { "rx 64B", IXGBE_PRC64, 32, 3714 KSTAT_KV_U_PACKETS }, 3715 [ix_counter_prc127] = { "rx 65-127B", IXGBE_PRC127, 32, 3716 KSTAT_KV_U_PACKETS }, 3717 [ix_counter_prc255] = { "rx 128-255B", IXGBE_PRC255, 32, 3718 KSTAT_KV_U_PACKETS }, 3719 [ix_counter_prc511] = { "rx 256-511B", IXGBE_PRC511, 32, 3720 KSTAT_KV_U_PACKETS }, 3721 [ix_counter_prc1023] = { "rx 512-1023B", IXGBE_PRC1023, 32, 3722 KSTAT_KV_U_PACKETS }, 3723 [ix_counter_prc1522] = { "rx 1024-maxB", IXGBE_PRC1522, 32, 3724 KSTAT_KV_U_PACKETS }, 3725 [ix_counter_gptc] = { "tx good", IXGBE_GPTC, 32, 3726 KSTAT_KV_U_PACKETS }, 3727 [ix_counter_gorc] = { "rx good", IXGBE_GORCL, 36, 3728 KSTAT_KV_U_BYTES }, 3729 [ix_counter_gotc] = { "tx good", IXGBE_GOTCL, 36, 3730 KSTAT_KV_U_BYTES }, 3731 [ix_counter_ruc] = { "rx undersize", IXGBE_RUC, 32, 3732 KSTAT_KV_U_PACKETS }, 3733 [ix_counter_rfc] = { "rx fragment", IXGBE_RFC, 32, 3734 KSTAT_KV_U_PACKETS }, 3735 [ix_counter_roc] = { "rx oversize", IXGBE_ROC, 32, 3736 KSTAT_KV_U_PACKETS }, 3737 [ix_counter_rjc] = { "rx jabber", IXGBE_RJC, 32, 3738 KSTAT_KV_U_PACKETS }, 3739 [ix_counter_tor] = { "rx total", IXGBE_TORL, 36, 3740 KSTAT_KV_U_BYTES }, 3741 [ix_counter_tpr] = { "rx total", IXGBE_TPR, 32, 3742 KSTAT_KV_U_PACKETS }, 3743 [ix_counter_tpt] = { "tx total", IXGBE_TPT, 32, 3744 KSTAT_KV_U_PACKETS }, 3745 [ix_counter_gprc] = { "rx good", IXGBE_GPRC, 32, 3746 KSTAT_KV_U_PACKETS }, 3747 [ix_counter_bprc] = { "rx bcast", IXGBE_BPRC, 32, 3748 KSTAT_KV_U_PACKETS }, 3749 [ix_counter_mprc] = { "rx mcast", IXGBE_MPRC, 32, 3750 KSTAT_KV_U_PACKETS }, 3751 [ix_counter_ptc64] = { "tx 64B", IXGBE_PTC64, 32, 3752 KSTAT_KV_U_PACKETS }, 3753 [ix_counter_ptc127] = { "tx 65-127B", IXGBE_PTC127, 32, 3754 KSTAT_KV_U_PACKETS }, 3755 [ix_counter_ptc255] = { "tx 128-255B", IXGBE_PTC255, 32, 3756 KSTAT_KV_U_PACKETS }, 3757 [ix_counter_ptc511] = { "tx 256-511B", IXGBE_PTC511, 32, 3758 KSTAT_KV_U_PACKETS }, 3759 [ix_counter_ptc1023] = { "tx 512-1023B", IXGBE_PTC1023, 32, 3760 KSTAT_KV_U_PACKETS }, 3761 [ix_counter_ptc1522] = { "tx 1024-maxB", IXGBE_PTC1522, 32, 3762 KSTAT_KV_U_PACKETS }, 3763 [ix_counter_mptc] = { "tx mcast", IXGBE_MPTC, 32, 3764 KSTAT_KV_U_PACKETS }, 3765 [ix_counter_bptc] = { "tx bcast", IXGBE_BPTC, 32, 3766 KSTAT_KV_U_PACKETS }, 3767 }; 3768 3769 struct ix_rxq_kstats { 3770 struct kstat_kv qprc; 3771 struct kstat_kv qbrc; 3772 struct kstat_kv qprdc; 3773 }; 3774 3775 static const struct ix_rxq_kstats ix_rxq_kstats_tpl = { 3776 KSTAT_KV_UNIT_INITIALIZER("packets", 3777 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3778 KSTAT_KV_UNIT_INITIALIZER("bytes", 3779 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3780 KSTAT_KV_UNIT_INITIALIZER("qdrops", 3781 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3782 }; 3783 3784 struct ix_txq_kstats { 3785 struct kstat_kv qptc; 3786 struct kstat_kv qbtc; 3787 }; 3788 3789 static const struct ix_txq_kstats ix_txq_kstats_tpl = { 3790 KSTAT_KV_UNIT_INITIALIZER("packets", 3791 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3792 KSTAT_KV_UNIT_INITIALIZER("bytes", 3793 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3794 }; 3795 3796 static int ix_kstats_read(struct kstat *ks); 3797 static int ix_rxq_kstats_read(struct kstat *ks); 3798 static int ix_txq_kstats_read(struct kstat *ks); 3799 3800 static void 3801 ix_kstats(struct ix_softc *sc) 3802 { 3803 struct kstat *ks; 3804 struct kstat_kv *kvs; 3805 unsigned int i; 3806 3807 mtx_init(&sc->sc_kstat_mtx, IPL_SOFTCLOCK); 3808 timeout_set(&sc->sc_kstat_tmo, ix_kstats_tick, sc); 3809 3810 ks = kstat_create(sc->dev.dv_xname, 0, "ix-stats", 0, 3811 KSTAT_T_KV, 0); 3812 if (ks == NULL) 3813 return; 3814 3815 kvs = mallocarray(nitems(ix_counters), sizeof(*kvs), 3816 M_DEVBUF, M_WAITOK|M_ZERO); 3817 3818 for (i = 0; i < nitems(ix_counters); i++) { 3819 const struct ix_counter *ixc = &ix_counters[i]; 3820 3821 kstat_kv_unit_init(&kvs[i], ixc->name, 3822 KSTAT_KV_T_COUNTER64, ixc->unit); 3823 } 3824 3825 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3826 ks->ks_softc = sc; 3827 ks->ks_data = kvs; 3828 ks->ks_datalen = nitems(ix_counters) * sizeof(*kvs); 3829 ks->ks_read = ix_kstats_read; 3830 3831 sc->sc_kstat = ks; 3832 kstat_install(ks); 3833 } 3834 3835 static void 3836 ix_rxq_kstats(struct ix_softc *sc, struct rx_ring *rxr) 3837 { 3838 struct ix_rxq_kstats *stats; 3839 struct kstat *ks; 3840 3841 ks = kstat_create(sc->dev.dv_xname, 0, "ix-rxq", rxr->me, 3842 KSTAT_T_KV, 0); 3843 if (ks == NULL) 3844 return; 3845 3846 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3847 *stats = ix_rxq_kstats_tpl; 3848 3849 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3850 ks->ks_softc = rxr; 3851 ks->ks_data = stats; 3852 ks->ks_datalen = sizeof(*stats); 3853 ks->ks_read = ix_rxq_kstats_read; 3854 3855 rxr->kstat = ks; 3856 kstat_install(ks); 3857 } 3858 3859 static void 3860 ix_txq_kstats(struct ix_softc *sc, struct tx_ring *txr) 3861 { 3862 struct ix_txq_kstats *stats; 3863 struct kstat *ks; 3864 3865 ks = kstat_create(sc->dev.dv_xname, 0, "ix-txq", txr->me, 3866 KSTAT_T_KV, 0); 3867 if (ks == NULL) 3868 return; 3869 3870 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3871 *stats = ix_txq_kstats_tpl; 3872 3873 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3874 ks->ks_softc = txr; 3875 ks->ks_data = stats; 3876 ks->ks_datalen = sizeof(*stats); 3877 ks->ks_read = ix_txq_kstats_read; 3878 3879 txr->kstat = ks; 3880 kstat_install(ks); 3881 } 3882 3883 /********************************************************************** 3884 * 3885 * Update the board statistics counters. 3886 * 3887 **********************************************************************/ 3888 3889 static void 3890 ix_kstats_tick(void *arg) 3891 { 3892 struct ix_softc *sc = arg; 3893 int i; 3894 3895 timeout_add_sec(&sc->sc_kstat_tmo, 1); 3896 3897 mtx_enter(&sc->sc_kstat_mtx); 3898 ix_kstats_read(sc->sc_kstat); 3899 for (i = 0; i < sc->num_queues; i++) { 3900 ix_rxq_kstats_read(sc->rx_rings[i].kstat); 3901 ix_txq_kstats_read(sc->tx_rings[i].kstat); 3902 } 3903 mtx_leave(&sc->sc_kstat_mtx); 3904 } 3905 3906 static uint64_t 3907 ix_read36(struct ixgbe_hw *hw, bus_size_t loreg, bus_size_t hireg) 3908 { 3909 uint64_t lo, hi; 3910 3911 lo = IXGBE_READ_REG(hw, loreg); 3912 hi = IXGBE_READ_REG(hw, hireg); 3913 3914 return (((hi & 0xf) << 32) | lo); 3915 } 3916 3917 static int 3918 ix_kstats_read(struct kstat *ks) 3919 { 3920 struct ix_softc *sc = ks->ks_softc; 3921 struct kstat_kv *kvs = ks->ks_data; 3922 struct ixgbe_hw *hw = &sc->hw; 3923 unsigned int i; 3924 3925 for (i = 0; i < nitems(ix_counters); i++) { 3926 const struct ix_counter *ixc = &ix_counters[i]; 3927 uint32_t reg = ixc->reg; 3928 uint64_t v; 3929 3930 if (reg == 0) 3931 continue; 3932 3933 if (ixc->width > 32) { 3934 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3935 v = IXGBE_READ_REG(hw, reg + 4); 3936 else 3937 v = ix_read36(hw, reg, reg + 4); 3938 } else 3939 v = IXGBE_READ_REG(hw, reg); 3940 3941 kstat_kv_u64(&kvs[i]) += v; 3942 } 3943 3944 /* handle the exceptions */ 3945 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3946 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3947 IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3948 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3949 IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3950 } else { 3951 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3952 IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3953 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3954 IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3955 } 3956 3957 getnanouptime(&ks->ks_updated); 3958 3959 return (0); 3960 } 3961 3962 int 3963 ix_rxq_kstats_read(struct kstat *ks) 3964 { 3965 struct ix_rxq_kstats *stats = ks->ks_data; 3966 struct rx_ring *rxr = ks->ks_softc; 3967 struct ix_softc *sc = rxr->sc; 3968 struct ixgbe_hw *hw = &sc->hw; 3969 uint32_t i = rxr->me; 3970 3971 kstat_kv_u64(&stats->qprc) += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3972 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3973 kstat_kv_u64(&stats->qprdc) += 3974 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3975 kstat_kv_u64(&stats->qbrc) += 3976 IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 3977 } else { 3978 kstat_kv_u64(&stats->qprdc) += 3979 IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3980 kstat_kv_u64(&stats->qbrc) += 3981 ix_read36(hw, IXGBE_QBRC_L(i), IXGBE_QBRC_H(i)); 3982 } 3983 3984 getnanouptime(&ks->ks_updated); 3985 3986 return (0); 3987 } 3988 3989 int 3990 ix_txq_kstats_read(struct kstat *ks) 3991 { 3992 struct ix_txq_kstats *stats = ks->ks_data; 3993 struct tx_ring *txr = ks->ks_softc; 3994 struct ix_softc *sc = txr->sc; 3995 struct ixgbe_hw *hw = &sc->hw; 3996 uint32_t i = txr->me; 3997 3998 kstat_kv_u64(&stats->qptc) += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3999 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 4000 kstat_kv_u64(&stats->qbtc) += 4001 IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 4002 } else { 4003 kstat_kv_u64(&stats->qbtc) += 4004 ix_read36(hw, IXGBE_QBTC_L(i), IXGBE_QBTC_H(i)); 4005 } 4006 4007 getnanouptime(&ks->ks_updated); 4008 4009 return (0); 4010 } 4011 #endif /* NKVSTAT > 0 */ 4012 4013 void 4014 ixgbe_map_queue_statistics(struct ix_softc *sc) 4015 { 4016 int i; 4017 uint32_t r; 4018 4019 for (i = 0; i < 32; i++) { 4020 /* 4021 * Queues 0-15 are mapped 1:1 4022 * Queue 0 -> Counter 0 4023 * Queue 1 -> Counter 1 4024 * Queue 2 -> Counter 2.... 4025 * Queues 16-127 are mapped to Counter 0 4026 */ 4027 if (i < 4) { 4028 r = (i * 4 + 0); 4029 r |= (i * 4 + 1) << 8; 4030 r |= (i * 4 + 2) << 16; 4031 r |= (i * 4 + 3) << 24; 4032 } else 4033 r = 0; 4034 4035 IXGBE_WRITE_REG(&sc->hw, IXGBE_RQSMR(i), r); 4036 IXGBE_WRITE_REG(&sc->hw, IXGBE_TQSM(i), r); 4037 } 4038 } 4039