1 /* $OpenBSD: if_ix.c,v 1.185 2022/03/15 11:22:10 jan Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2013, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */ 36 37 #include <dev/pci/if_ix.h> 38 #include <dev/pci/ixgbe_type.h> 39 40 /********************************************************************* 41 * Driver version 42 *********************************************************************/ 43 /* char ixgbe_driver_version[] = "2.5.13"; */ 44 45 /********************************************************************* 46 * PCI Device ID Table 47 * 48 * Used by probe to select devices to load on 49 *********************************************************************/ 50 51 const struct pci_matchid ixgbe_devices[] = { 52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 }, 53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX }, 54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT }, 57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT2 }, 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP }, 63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM }, 64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL }, 65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 }, 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BP }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_T3_LOM }, 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF_QP }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF2 }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_FCOE }, 77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599EN_SFP }, 78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_QSFP_SF_QP }, 79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T }, 80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T1 }, 81 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T }, 82 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T1 }, 83 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KX4 }, 84 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KR }, 85 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_SFP }, 86 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_10G_T }, 87 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_1G_T }, 88 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR }, 89 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR_L }, 90 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP_N }, 91 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP }, 92 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII }, 93 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII_L }, 94 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_10G_T }, 95 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T }, 96 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T_L } 97 }; 98 99 /********************************************************************* 100 * Function prototypes 101 *********************************************************************/ 102 int ixgbe_probe(struct device *, void *, void *); 103 void ixgbe_attach(struct device *, struct device *, void *); 104 int ixgbe_detach(struct device *, int); 105 int ixgbe_activate(struct device *, int); 106 void ixgbe_start(struct ifqueue *); 107 int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 108 int ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *); 109 int ixgbe_get_sffpage(struct ix_softc *, struct if_sffpage *); 110 void ixgbe_watchdog(struct ifnet *); 111 void ixgbe_init(void *); 112 void ixgbe_stop(void *); 113 void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 114 int ixgbe_media_change(struct ifnet *); 115 void ixgbe_identify_hardware(struct ix_softc *); 116 int ixgbe_allocate_pci_resources(struct ix_softc *); 117 int ixgbe_allocate_legacy(struct ix_softc *); 118 int ixgbe_allocate_msix(struct ix_softc *); 119 void ixgbe_setup_msix(struct ix_softc *); 120 int ixgbe_allocate_queues(struct ix_softc *); 121 void ixgbe_free_pci_resources(struct ix_softc *); 122 void ixgbe_local_timer(void *); 123 void ixgbe_setup_interface(struct ix_softc *); 124 void ixgbe_config_gpie(struct ix_softc *); 125 void ixgbe_config_delay_values(struct ix_softc *); 126 void ixgbe_add_media_types(struct ix_softc *); 127 void ixgbe_config_link(struct ix_softc *); 128 129 int ixgbe_allocate_transmit_buffers(struct tx_ring *); 130 int ixgbe_setup_transmit_structures(struct ix_softc *); 131 int ixgbe_setup_transmit_ring(struct tx_ring *); 132 void ixgbe_initialize_transmit_units(struct ix_softc *); 133 void ixgbe_free_transmit_structures(struct ix_softc *); 134 void ixgbe_free_transmit_buffers(struct tx_ring *); 135 136 int ixgbe_allocate_receive_buffers(struct rx_ring *); 137 int ixgbe_setup_receive_structures(struct ix_softc *); 138 int ixgbe_setup_receive_ring(struct rx_ring *); 139 void ixgbe_initialize_receive_units(struct ix_softc *); 140 void ixgbe_free_receive_structures(struct ix_softc *); 141 void ixgbe_free_receive_buffers(struct rx_ring *); 142 void ixgbe_initialize_rss_mapping(struct ix_softc *); 143 int ixgbe_rxfill(struct rx_ring *); 144 void ixgbe_rxrefill(void *); 145 146 int ixgbe_intr(struct ix_softc *sc); 147 void ixgbe_enable_intr(struct ix_softc *); 148 void ixgbe_disable_intr(struct ix_softc *); 149 int ixgbe_txeof(struct tx_ring *); 150 int ixgbe_rxeof(struct rx_ring *); 151 void ixgbe_rx_checksum(uint32_t, struct mbuf *, uint32_t); 152 void ixgbe_iff(struct ix_softc *); 153 void ixgbe_map_queue_statistics(struct ix_softc *); 154 void ixgbe_update_link_status(struct ix_softc *); 155 int ixgbe_get_buf(struct rx_ring *, int); 156 int ixgbe_encap(struct tx_ring *, struct mbuf *); 157 int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 158 struct ixgbe_dma_alloc *, int); 159 void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 160 static int 161 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *, uint32_t *, 162 uint32_t *); 163 int ixgbe_tso_setup(struct tx_ring *, struct mbuf *, uint32_t *, 164 uint32_t *); 165 void ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 166 void ixgbe_configure_ivars(struct ix_softc *); 167 uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 168 169 void ixgbe_setup_vlan_hw_support(struct ix_softc *); 170 171 /* Support for pluggable optic modules */ 172 void ixgbe_handle_mod(struct ix_softc *); 173 void ixgbe_handle_msf(struct ix_softc *); 174 void ixgbe_handle_phy(struct ix_softc *); 175 176 /* Legacy (single vector interrupt handler */ 177 int ixgbe_legacy_intr(void *); 178 void ixgbe_enable_queue(struct ix_softc *, uint32_t); 179 void ixgbe_enable_queues(struct ix_softc *); 180 void ixgbe_disable_queue(struct ix_softc *, uint32_t); 181 void ixgbe_rearm_queue(struct ix_softc *, uint32_t); 182 183 /* MSI-X (multiple vectors interrupt handlers) */ 184 int ixgbe_link_intr(void *); 185 int ixgbe_queue_intr(void *); 186 187 #if NKSTAT > 0 188 static void ix_kstats(struct ix_softc *); 189 static void ix_rxq_kstats(struct ix_softc *, struct rx_ring *); 190 static void ix_txq_kstats(struct ix_softc *, struct tx_ring *); 191 static void ix_kstats_tick(void *); 192 #endif 193 194 /********************************************************************* 195 * OpenBSD Device Interface Entry Points 196 *********************************************************************/ 197 198 struct cfdriver ix_cd = { 199 NULL, "ix", DV_IFNET 200 }; 201 202 const struct cfattach ix_ca = { 203 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach, 204 ixgbe_activate 205 }; 206 207 int ixgbe_smart_speed = ixgbe_smart_speed_on; 208 int ixgbe_enable_msix = 1; 209 210 /********************************************************************* 211 * Device identification routine 212 * 213 * ixgbe_probe determines if the driver should be loaded on 214 * adapter based on PCI vendor/device id of the adapter. 215 * 216 * return 0 on success, positive on failure 217 *********************************************************************/ 218 219 int 220 ixgbe_probe(struct device *parent, void *match, void *aux) 221 { 222 INIT_DEBUGOUT("ixgbe_probe: begin"); 223 224 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 225 nitems(ixgbe_devices))); 226 } 227 228 /********************************************************************* 229 * Device initialization routine 230 * 231 * The attach entry point is called when the driver is being loaded. 232 * This routine identifies the type of hardware, allocates all resources 233 * and initializes the hardware. 234 * 235 * return 0 on success, positive on failure 236 *********************************************************************/ 237 238 void 239 ixgbe_attach(struct device *parent, struct device *self, void *aux) 240 { 241 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 242 struct ix_softc *sc = (struct ix_softc *)self; 243 int error = 0; 244 uint16_t csum; 245 uint32_t ctrl_ext; 246 struct ixgbe_hw *hw = &sc->hw; 247 248 INIT_DEBUGOUT("ixgbe_attach: begin"); 249 250 sc->osdep.os_sc = sc; 251 sc->osdep.os_pa = *pa; 252 253 rw_init(&sc->sfflock, "ixsff"); 254 255 #if NKSTAT > 0 256 ix_kstats(sc); 257 #endif 258 259 /* Determine hardware revision */ 260 ixgbe_identify_hardware(sc); 261 262 /* Indicate to RX setup to use Jumbo Clusters */ 263 sc->num_tx_desc = DEFAULT_TXD; 264 sc->num_rx_desc = DEFAULT_RXD; 265 266 /* Do base PCI setup - map BAR0 */ 267 if (ixgbe_allocate_pci_resources(sc)) 268 goto err_out; 269 270 /* Allocate our TX/RX Queues */ 271 if (ixgbe_allocate_queues(sc)) 272 goto err_out; 273 274 /* Allocate multicast array memory. */ 275 sc->mta = mallocarray(IXGBE_ETH_LENGTH_OF_ADDRESS, 276 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 277 if (sc->mta == NULL) { 278 printf(": Can not allocate multicast setup array\n"); 279 goto err_late; 280 } 281 282 /* Initialize the shared code */ 283 error = ixgbe_init_shared_code(hw); 284 if (error) { 285 printf(": Unable to initialize the shared code\n"); 286 goto err_late; 287 } 288 289 /* Make sure we have a good EEPROM before we read from it */ 290 if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) { 291 printf(": The EEPROM Checksum Is Not Valid\n"); 292 goto err_late; 293 } 294 295 error = ixgbe_init_hw(hw); 296 if (error == IXGBE_ERR_EEPROM_VERSION) { 297 printf(": This device is a pre-production adapter/" 298 "LOM. Please be aware there may be issues associated " 299 "with your hardware.\nIf you are experiencing problems " 300 "please contact your Intel or hardware representative " 301 "who provided you with this hardware.\n"); 302 } else if (error && (error != IXGBE_ERR_SFP_NOT_PRESENT && 303 error != IXGBE_ERR_SFP_NOT_SUPPORTED)) { 304 printf(": Hardware Initialization Failure\n"); 305 goto err_late; 306 } 307 308 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 309 IXGBE_ETH_LENGTH_OF_ADDRESS); 310 311 if (sc->sc_intrmap) 312 error = ixgbe_allocate_msix(sc); 313 else 314 error = ixgbe_allocate_legacy(sc); 315 if (error) 316 goto err_late; 317 318 /* Enable the optics for 82599 SFP+ fiber */ 319 if (sc->hw.mac.ops.enable_tx_laser) 320 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 321 322 /* Enable power to the phy */ 323 if (hw->phy.ops.set_phy_power) 324 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 325 326 /* Setup OS specific network interface */ 327 ixgbe_setup_interface(sc); 328 329 /* Get the PCI-E bus info and determine LAN ID */ 330 hw->mac.ops.get_bus_info(hw); 331 332 /* Set an initial default flow control value */ 333 sc->fc = ixgbe_fc_full; 334 335 /* let hardware know driver is loaded */ 336 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 337 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 338 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 339 340 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 341 342 INIT_DEBUGOUT("ixgbe_attach: end"); 343 return; 344 345 err_late: 346 ixgbe_free_transmit_structures(sc); 347 ixgbe_free_receive_structures(sc); 348 err_out: 349 ixgbe_free_pci_resources(sc); 350 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 351 MAX_NUM_MULTICAST_ADDRESSES); 352 } 353 354 /********************************************************************* 355 * Device removal routine 356 * 357 * The detach entry point is called when the driver is being removed. 358 * This routine stops the adapter and deallocates all the resources 359 * that were allocated for driver operation. 360 * 361 * return 0 on success, positive on failure 362 *********************************************************************/ 363 364 int 365 ixgbe_detach(struct device *self, int flags) 366 { 367 struct ix_softc *sc = (struct ix_softc *)self; 368 struct ifnet *ifp = &sc->arpcom.ac_if; 369 uint32_t ctrl_ext; 370 371 INIT_DEBUGOUT("ixgbe_detach: begin"); 372 373 ixgbe_stop(sc); 374 375 /* let hardware know driver is unloading */ 376 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 377 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 378 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 379 380 ether_ifdetach(ifp); 381 if_detach(ifp); 382 383 ixgbe_free_pci_resources(sc); 384 385 ixgbe_free_transmit_structures(sc); 386 ixgbe_free_receive_structures(sc); 387 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 388 MAX_NUM_MULTICAST_ADDRESSES); 389 390 /* XXX kstat */ 391 392 return (0); 393 } 394 395 int 396 ixgbe_activate(struct device *self, int act) 397 { 398 struct ix_softc *sc = (struct ix_softc *)self; 399 struct ifnet *ifp = &sc->arpcom.ac_if; 400 struct ixgbe_hw *hw = &sc->hw; 401 uint32_t ctrl_ext; 402 int rv = 0; 403 404 switch (act) { 405 case DVACT_QUIESCE: 406 if (ifp->if_flags & IFF_RUNNING) 407 ixgbe_stop(sc); 408 break; 409 case DVACT_RESUME: 410 ixgbe_init_hw(hw); 411 412 /* Enable the optics for 82599 SFP+ fiber */ 413 if (sc->hw.mac.ops.enable_tx_laser) 414 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 415 416 /* Enable power to the phy */ 417 if (hw->phy.ops.set_phy_power) 418 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 419 420 /* Get the PCI-E bus info and determine LAN ID */ 421 hw->mac.ops.get_bus_info(hw); 422 423 /* let hardware know driver is loaded */ 424 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 425 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 426 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 427 428 if (ifp->if_flags & IFF_UP) 429 ixgbe_init(sc); 430 break; 431 default: 432 break; 433 } 434 return (rv); 435 } 436 437 /********************************************************************* 438 * Transmit entry point 439 * 440 * ixgbe_start is called by the stack to initiate a transmit. 441 * The driver will remain in this routine as long as there are 442 * packets to transmit and transmit resources are available. 443 * In case resources are not available stack is notified and 444 * the packet is requeued. 445 **********************************************************************/ 446 447 void 448 ixgbe_start(struct ifqueue *ifq) 449 { 450 struct ifnet *ifp = ifq->ifq_if; 451 struct ix_softc *sc = ifp->if_softc; 452 struct tx_ring *txr = ifq->ifq_softc; 453 struct mbuf *m_head; 454 unsigned int head, free, used; 455 int post = 0; 456 457 if (!sc->link_up) 458 return; 459 460 head = txr->next_avail_desc; 461 free = txr->next_to_clean; 462 if (free <= head) 463 free += sc->num_tx_desc; 464 free -= head; 465 466 membar_consumer(); 467 468 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 469 0, txr->txdma.dma_map->dm_mapsize, 470 BUS_DMASYNC_POSTWRITE); 471 472 for (;;) { 473 /* Check that we have the minimal number of TX descriptors. */ 474 if (free <= IXGBE_TX_OP_THRESHOLD) { 475 ifq_set_oactive(ifq); 476 break; 477 } 478 479 m_head = ifq_dequeue(ifq); 480 if (m_head == NULL) 481 break; 482 483 used = ixgbe_encap(txr, m_head); 484 if (used == 0) { 485 m_freem(m_head); 486 continue; 487 } 488 489 free -= used; 490 491 #if NBPFILTER > 0 492 if (ifp->if_bpf) 493 bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 494 #endif 495 496 /* Set timeout in case hardware has problems transmitting */ 497 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 498 ifp->if_timer = IXGBE_TX_TIMEOUT; 499 500 post = 1; 501 } 502 503 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 504 0, txr->txdma.dma_map->dm_mapsize, 505 BUS_DMASYNC_PREWRITE); 506 507 /* 508 * Advance the Transmit Descriptor Tail (Tdt), this tells the 509 * hardware that this frame is available to transmit. 510 */ 511 if (post) 512 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 513 txr->next_avail_desc); 514 } 515 516 /********************************************************************* 517 * Ioctl entry point 518 * 519 * ixgbe_ioctl is called when the user wants to configure the 520 * interface. 521 * 522 * return 0 on success, positive on failure 523 **********************************************************************/ 524 525 int 526 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 527 { 528 struct ix_softc *sc = ifp->if_softc; 529 struct ifreq *ifr = (struct ifreq *) data; 530 int s, error = 0; 531 532 s = splnet(); 533 534 switch (command) { 535 case SIOCSIFADDR: 536 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 537 ifp->if_flags |= IFF_UP; 538 if (!(ifp->if_flags & IFF_RUNNING)) 539 ixgbe_init(sc); 540 break; 541 542 case SIOCSIFFLAGS: 543 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 544 if (ifp->if_flags & IFF_UP) { 545 if (ifp->if_flags & IFF_RUNNING) 546 error = ENETRESET; 547 else 548 ixgbe_init(sc); 549 } else { 550 if (ifp->if_flags & IFF_RUNNING) 551 ixgbe_stop(sc); 552 } 553 break; 554 555 case SIOCSIFMEDIA: 556 case SIOCGIFMEDIA: 557 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 558 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 559 break; 560 561 case SIOCGIFRXR: 562 error = ixgbe_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 563 break; 564 565 case SIOCGIFSFFPAGE: 566 error = rw_enter(&sc->sfflock, RW_WRITE|RW_INTR); 567 if (error != 0) 568 break; 569 570 error = ixgbe_get_sffpage(sc, (struct if_sffpage *)data); 571 rw_exit(&sc->sfflock); 572 break; 573 574 default: 575 error = ether_ioctl(ifp, &sc->arpcom, command, data); 576 } 577 578 if (error == ENETRESET) { 579 if (ifp->if_flags & IFF_RUNNING) { 580 ixgbe_disable_intr(sc); 581 ixgbe_iff(sc); 582 ixgbe_enable_intr(sc); 583 ixgbe_enable_queues(sc); 584 } 585 error = 0; 586 } 587 588 splx(s); 589 return (error); 590 } 591 592 int 593 ixgbe_get_sffpage(struct ix_softc *sc, struct if_sffpage *sff) 594 { 595 struct ixgbe_hw *hw = &sc->hw; 596 uint32_t swfw_mask = hw->phy.phy_semaphore_mask; 597 uint8_t page; 598 size_t i; 599 int error = EIO; 600 601 if (hw->phy.type == ixgbe_phy_fw) 602 return (ENODEV); 603 604 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 605 return (EBUSY); /* XXX */ 606 607 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 608 if (hw->phy.ops.read_i2c_byte_unlocked(hw, 127, 609 IFSFF_ADDR_EEPROM, &page)) 610 goto error; 611 if (page != sff->sff_page && 612 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 613 IFSFF_ADDR_EEPROM, sff->sff_page)) 614 goto error; 615 } 616 617 for (i = 0; i < sizeof(sff->sff_data); i++) { 618 if (hw->phy.ops.read_i2c_byte_unlocked(hw, i, 619 sff->sff_addr, &sff->sff_data[i])) 620 goto error; 621 } 622 623 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 624 if (page != sff->sff_page && 625 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 626 IFSFF_ADDR_EEPROM, page)) 627 goto error; 628 } 629 630 error = 0; 631 error: 632 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 633 return (error); 634 } 635 636 int 637 ixgbe_rxrinfo(struct ix_softc *sc, struct if_rxrinfo *ifri) 638 { 639 struct if_rxring_info *ifr, ifr1; 640 struct rx_ring *rxr; 641 int error, i; 642 u_int n = 0; 643 644 if (sc->num_queues > 1) { 645 if ((ifr = mallocarray(sc->num_queues, sizeof(*ifr), M_DEVBUF, 646 M_WAITOK | M_ZERO)) == NULL) 647 return (ENOMEM); 648 } else 649 ifr = &ifr1; 650 651 for (i = 0; i < sc->num_queues; i++) { 652 rxr = &sc->rx_rings[i]; 653 ifr[n].ifr_size = MCLBYTES; 654 snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i); 655 ifr[n].ifr_info = rxr->rx_ring; 656 n++; 657 } 658 659 error = if_rxr_info_ioctl(ifri, sc->num_queues, ifr); 660 661 if (sc->num_queues > 1) 662 free(ifr, M_DEVBUF, sc->num_queues * sizeof(*ifr)); 663 return (error); 664 } 665 666 /********************************************************************* 667 * Watchdog entry point 668 * 669 **********************************************************************/ 670 671 void 672 ixgbe_watchdog(struct ifnet * ifp) 673 { 674 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 675 struct tx_ring *txr = sc->tx_rings; 676 struct ixgbe_hw *hw = &sc->hw; 677 int tx_hang = FALSE; 678 int i; 679 680 /* 681 * The timer is set to 5 every time ixgbe_start() queues a packet. 682 * Anytime all descriptors are clean the timer is set to 0. 683 */ 684 for (i = 0; i < sc->num_queues; i++, txr++) { 685 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 686 continue; 687 else { 688 tx_hang = TRUE; 689 break; 690 } 691 } 692 if (tx_hang == FALSE) 693 return; 694 695 /* 696 * If we are in this routine because of pause frames, then don't 697 * reset the hardware. 698 */ 699 if (!(IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXON)) { 700 for (i = 0; i < sc->num_queues; i++, txr++) 701 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 702 ifp->if_timer = IXGBE_TX_TIMEOUT; 703 return; 704 } 705 706 707 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 708 for (i = 0; i < sc->num_queues; i++, txr++) { 709 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 710 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 711 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 712 printf("%s: TX(%d) Next TX to Clean = %d\n", ifp->if_xname, 713 i, txr->next_to_clean); 714 } 715 ifp->if_flags &= ~IFF_RUNNING; 716 717 ixgbe_init(sc); 718 } 719 720 /********************************************************************* 721 * Init entry point 722 * 723 * This routine is used in two ways. It is used by the stack as 724 * init entry point in network interface structure. It is also used 725 * by the driver as a hw/sw initialization routine to get to a 726 * consistent state. 727 * 728 * return 0 on success, positive on failure 729 **********************************************************************/ 730 #define IXGBE_MHADD_MFS_SHIFT 16 731 732 void 733 ixgbe_init(void *arg) 734 { 735 struct ix_softc *sc = (struct ix_softc *)arg; 736 struct ifnet *ifp = &sc->arpcom.ac_if; 737 struct rx_ring *rxr = sc->rx_rings; 738 uint32_t k, txdctl, rxdctl, rxctrl, mhadd, itr; 739 int i, s, err; 740 741 INIT_DEBUGOUT("ixgbe_init: begin"); 742 743 s = splnet(); 744 745 ixgbe_stop(sc); 746 747 /* reprogram the RAR[0] in case user changed it. */ 748 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 749 750 /* Get the latest mac address, User can use a LAA */ 751 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 752 IXGBE_ETH_LENGTH_OF_ADDRESS); 753 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1); 754 sc->hw.addr_ctrl.rar_used_count = 1; 755 756 /* Prepare transmit descriptors and buffers */ 757 if (ixgbe_setup_transmit_structures(sc)) { 758 printf("%s: Could not setup transmit structures\n", 759 ifp->if_xname); 760 ixgbe_stop(sc); 761 splx(s); 762 return; 763 } 764 765 ixgbe_init_hw(&sc->hw); 766 ixgbe_initialize_transmit_units(sc); 767 768 /* Use 2k clusters, even for jumbo frames */ 769 sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN; 770 771 /* Prepare receive descriptors and buffers */ 772 if (ixgbe_setup_receive_structures(sc)) { 773 printf("%s: Could not setup receive structures\n", 774 ifp->if_xname); 775 ixgbe_stop(sc); 776 splx(s); 777 return; 778 } 779 780 /* Configure RX settings */ 781 ixgbe_initialize_receive_units(sc); 782 783 /* Enable SDP & MSIX interrupts based on adapter */ 784 ixgbe_config_gpie(sc); 785 786 /* Program promiscuous mode and multicast filters. */ 787 ixgbe_iff(sc); 788 789 /* Set MRU size */ 790 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 791 mhadd &= ~IXGBE_MHADD_MFS_MASK; 792 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 793 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 794 795 /* Now enable all the queues */ 796 for (i = 0; i < sc->num_queues; i++) { 797 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 798 txdctl |= IXGBE_TXDCTL_ENABLE; 799 /* Set WTHRESH to 8, burst writeback */ 800 txdctl |= (8 << 16); 801 /* 802 * When the internal queue falls below PTHRESH (16), 803 * start prefetching as long as there are at least 804 * HTHRESH (1) buffers ready. 805 */ 806 txdctl |= (16 << 0) | (1 << 8); 807 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 808 } 809 810 for (i = 0; i < sc->num_queues; i++) { 811 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 812 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 813 /* 814 * PTHRESH = 21 815 * HTHRESH = 4 816 * WTHRESH = 8 817 */ 818 rxdctl &= ~0x3FFFFF; 819 rxdctl |= 0x080420; 820 } 821 rxdctl |= IXGBE_RXDCTL_ENABLE; 822 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 823 for (k = 0; k < 10; k++) { 824 if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) & 825 IXGBE_RXDCTL_ENABLE) 826 break; 827 else 828 msec_delay(1); 829 } 830 IXGBE_WRITE_FLUSH(&sc->hw); 831 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled); 832 } 833 834 /* Set up VLAN support and filter */ 835 ixgbe_setup_vlan_hw_support(sc); 836 837 /* Enable Receive engine */ 838 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 839 if (sc->hw.mac.type == ixgbe_mac_82598EB) 840 rxctrl |= IXGBE_RXCTRL_DMBYPS; 841 rxctrl |= IXGBE_RXCTRL_RXEN; 842 sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl); 843 844 /* Set up MSI/X routing */ 845 if (sc->sc_intrmap) { 846 ixgbe_configure_ivars(sc); 847 /* Set up auto-mask */ 848 if (sc->hw.mac.type == ixgbe_mac_82598EB) 849 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 850 else { 851 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 852 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 853 } 854 } else { /* Simple settings for Legacy/MSI */ 855 ixgbe_set_ivar(sc, 0, 0, 0); 856 ixgbe_set_ivar(sc, 0, 0, 1); 857 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 858 } 859 860 /* Check on any SFP devices that need to be kick-started */ 861 if (sc->hw.phy.type == ixgbe_phy_none) { 862 err = sc->hw.phy.ops.identify(&sc->hw); 863 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 864 printf("Unsupported SFP+ module type was detected.\n"); 865 splx(s); 866 return; 867 } 868 } 869 870 /* Setup interrupt moderation */ 871 itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8; 872 if (sc->hw.mac.type != ixgbe_mac_82598EB) 873 itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS; 874 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr); 875 876 if (sc->sc_intrmap) { 877 /* Set moderation on the Link interrupt */ 878 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(sc->linkvec), 879 IXGBE_LINK_ITR); 880 } 881 882 /* Enable power to the phy */ 883 if (sc->hw.phy.ops.set_phy_power) 884 sc->hw.phy.ops.set_phy_power(&sc->hw, TRUE); 885 886 /* Config/Enable Link */ 887 ixgbe_config_link(sc); 888 889 /* Hardware Packet Buffer & Flow Control setup */ 890 ixgbe_config_delay_values(sc); 891 892 /* Initialize the FC settings */ 893 sc->hw.mac.ops.start_hw(&sc->hw); 894 895 /* And now turn on interrupts */ 896 ixgbe_enable_intr(sc); 897 ixgbe_enable_queues(sc); 898 899 /* Now inform the stack we're ready */ 900 ifp->if_flags |= IFF_RUNNING; 901 for (i = 0; i < sc->num_queues; i++) 902 ifq_clr_oactive(ifp->if_ifqs[i]); 903 904 #if NKSTAT > 0 905 ix_kstats_tick(sc); 906 #endif 907 908 splx(s); 909 } 910 911 void 912 ixgbe_config_gpie(struct ix_softc *sc) 913 { 914 struct ixgbe_hw *hw = &sc->hw; 915 uint32_t gpie; 916 917 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 918 919 /* Fan Failure Interrupt */ 920 if (hw->device_id == IXGBE_DEV_ID_82598AT) 921 gpie |= IXGBE_SDP1_GPIEN; 922 923 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 924 /* Add for Module detection */ 925 gpie |= IXGBE_SDP2_GPIEN; 926 927 /* Media ready */ 928 if (hw->device_id != IXGBE_DEV_ID_82599_QSFP_SF_QP) 929 gpie |= IXGBE_SDP1_GPIEN; 930 931 /* 932 * Set LL interval to max to reduce the number of low latency 933 * interrupts hitting the card when the ring is getting full. 934 */ 935 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 936 } 937 938 if (sc->hw.mac.type == ixgbe_mac_X540 || 939 sc->hw.mac.type == ixgbe_mac_X550EM_x || 940 sc->hw.mac.type == ixgbe_mac_X550EM_a) { 941 /* 942 * Thermal Failure Detection (X540) 943 * Link Detection (X552 SFP+, X552/X557-AT) 944 */ 945 gpie |= IXGBE_SDP0_GPIEN_X540; 946 947 /* 948 * Set LL interval to max to reduce the number of low latency 949 * interrupts hitting the card when the ring is getting full. 950 */ 951 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 952 } 953 954 if (sc->sc_intrmap) { 955 /* Enable Enhanced MSIX mode */ 956 gpie |= IXGBE_GPIE_MSIX_MODE; 957 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 958 IXGBE_GPIE_OCD; 959 } 960 961 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 962 } 963 964 /* 965 * Requires sc->max_frame_size to be set. 966 */ 967 void 968 ixgbe_config_delay_values(struct ix_softc *sc) 969 { 970 struct ixgbe_hw *hw = &sc->hw; 971 uint32_t rxpb, frame, size, tmp; 972 973 frame = sc->max_frame_size; 974 975 /* Calculate High Water */ 976 switch (hw->mac.type) { 977 case ixgbe_mac_X540: 978 case ixgbe_mac_X550: 979 case ixgbe_mac_X550EM_x: 980 case ixgbe_mac_X550EM_a: 981 tmp = IXGBE_DV_X540(frame, frame); 982 break; 983 default: 984 tmp = IXGBE_DV(frame, frame); 985 break; 986 } 987 size = IXGBE_BT2KB(tmp); 988 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 989 hw->fc.high_water[0] = rxpb - size; 990 991 /* Now calculate Low Water */ 992 switch (hw->mac.type) { 993 case ixgbe_mac_X540: 994 case ixgbe_mac_X550: 995 case ixgbe_mac_X550EM_x: 996 case ixgbe_mac_X550EM_a: 997 tmp = IXGBE_LOW_DV_X540(frame); 998 break; 999 default: 1000 tmp = IXGBE_LOW_DV(frame); 1001 break; 1002 } 1003 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 1004 1005 hw->fc.requested_mode = sc->fc; 1006 hw->fc.pause_time = IXGBE_FC_PAUSE; 1007 hw->fc.send_xon = TRUE; 1008 } 1009 1010 /* 1011 * MSIX Interrupt Handlers 1012 */ 1013 void 1014 ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector) 1015 { 1016 uint64_t queue = 1ULL << vector; 1017 uint32_t mask; 1018 1019 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1020 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1021 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask); 1022 } else { 1023 mask = (queue & 0xFFFFFFFF); 1024 if (mask) 1025 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask); 1026 mask = (queue >> 32); 1027 if (mask) 1028 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask); 1029 } 1030 } 1031 1032 void 1033 ixgbe_enable_queues(struct ix_softc *sc) 1034 { 1035 struct ix_queue *que; 1036 int i; 1037 1038 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 1039 ixgbe_enable_queue(sc, que->msix); 1040 } 1041 1042 void 1043 ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector) 1044 { 1045 uint64_t queue = 1ULL << vector; 1046 uint32_t mask; 1047 1048 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1049 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1050 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask); 1051 } else { 1052 mask = (queue & 0xFFFFFFFF); 1053 if (mask) 1054 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask); 1055 mask = (queue >> 32); 1056 if (mask) 1057 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask); 1058 } 1059 } 1060 1061 /* 1062 * MSIX Interrupt Handlers 1063 */ 1064 int 1065 ixgbe_link_intr(void *vsc) 1066 { 1067 struct ix_softc *sc = (struct ix_softc *)vsc; 1068 1069 return ixgbe_intr(sc); 1070 } 1071 1072 int 1073 ixgbe_queue_intr(void *vque) 1074 { 1075 struct ix_queue *que = vque; 1076 struct ix_softc *sc = que->sc; 1077 struct ifnet *ifp = &sc->arpcom.ac_if; 1078 struct rx_ring *rxr = que->rxr; 1079 struct tx_ring *txr = que->txr; 1080 1081 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1082 ixgbe_rxeof(rxr); 1083 ixgbe_txeof(txr); 1084 ixgbe_rxrefill(rxr); 1085 } 1086 1087 ixgbe_enable_queue(sc, que->msix); 1088 1089 return (1); 1090 } 1091 1092 /********************************************************************* 1093 * 1094 * Legacy Interrupt Service routine 1095 * 1096 **********************************************************************/ 1097 1098 int 1099 ixgbe_legacy_intr(void *arg) 1100 { 1101 struct ix_softc *sc = (struct ix_softc *)arg; 1102 struct ifnet *ifp = &sc->arpcom.ac_if; 1103 struct rx_ring *rxr = sc->rx_rings; 1104 struct tx_ring *txr = sc->tx_rings; 1105 int rv; 1106 1107 rv = ixgbe_intr(sc); 1108 if (rv == 0) { 1109 return (0); 1110 } 1111 1112 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1113 ixgbe_rxeof(rxr); 1114 ixgbe_txeof(txr); 1115 ixgbe_rxrefill(rxr); 1116 } 1117 1118 ixgbe_enable_queues(sc); 1119 return (rv); 1120 } 1121 1122 int 1123 ixgbe_intr(struct ix_softc *sc) 1124 { 1125 struct ifnet *ifp = &sc->arpcom.ac_if; 1126 struct ixgbe_hw *hw = &sc->hw; 1127 uint32_t reg_eicr, mod_mask, msf_mask; 1128 1129 if (sc->sc_intrmap) { 1130 /* Pause other interrupts */ 1131 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER); 1132 /* First get the cause */ 1133 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1134 /* Be sure the queue bits are not cleared */ 1135 reg_eicr &= ~IXGBE_EICR_RTX_QUEUE; 1136 /* Clear interrupt with write */ 1137 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1138 } else { 1139 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1140 if (reg_eicr == 0) { 1141 ixgbe_enable_intr(sc); 1142 ixgbe_enable_queues(sc); 1143 return (0); 1144 } 1145 } 1146 1147 /* Link status change */ 1148 if (reg_eicr & IXGBE_EICR_LSC) { 1149 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); 1150 KERNEL_LOCK(); 1151 ixgbe_update_link_status(sc); 1152 KERNEL_UNLOCK(); 1153 } 1154 1155 if (hw->mac.type != ixgbe_mac_82598EB) { 1156 if (reg_eicr & IXGBE_EICR_ECC) { 1157 printf("%s: CRITICAL: ECC ERROR!! " 1158 "Please Reboot!!\n", sc->dev.dv_xname); 1159 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1160 } 1161 /* Check for over temp condition */ 1162 if (reg_eicr & IXGBE_EICR_TS) { 1163 printf("%s: CRITICAL: OVER TEMP!! " 1164 "PHY IS SHUT DOWN!!\n", ifp->if_xname); 1165 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS); 1166 } 1167 } 1168 1169 /* Pluggable optics-related interrupt */ 1170 if (ixgbe_is_sfp(hw)) { 1171 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP) { 1172 mod_mask = IXGBE_EICR_GPI_SDP0_X540; 1173 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1174 } else if (hw->mac.type == ixgbe_mac_X540 || 1175 hw->mac.type == ixgbe_mac_X550 || 1176 hw->mac.type == ixgbe_mac_X550EM_x) { 1177 mod_mask = IXGBE_EICR_GPI_SDP2_X540; 1178 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1179 } else { 1180 mod_mask = IXGBE_EICR_GPI_SDP2; 1181 msf_mask = IXGBE_EICR_GPI_SDP1; 1182 } 1183 if (reg_eicr & mod_mask) { 1184 /* Clear the interrupt */ 1185 IXGBE_WRITE_REG(hw, IXGBE_EICR, mod_mask); 1186 KERNEL_LOCK(); 1187 ixgbe_handle_mod(sc); 1188 KERNEL_UNLOCK(); 1189 } else if ((hw->phy.media_type != ixgbe_media_type_copper) && 1190 (reg_eicr & msf_mask)) { 1191 /* Clear the interrupt */ 1192 IXGBE_WRITE_REG(hw, IXGBE_EICR, msf_mask); 1193 KERNEL_LOCK(); 1194 ixgbe_handle_msf(sc); 1195 KERNEL_UNLOCK(); 1196 } 1197 } 1198 1199 /* Check for fan failure */ 1200 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1201 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1202 printf("%s: CRITICAL: FAN FAILURE!! " 1203 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 1204 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1205 } 1206 1207 /* External PHY interrupt */ 1208 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 1209 (reg_eicr & IXGBE_EICR_GPI_SDP0_X540)) { 1210 /* Clear the interrupt */ 1211 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540); 1212 KERNEL_LOCK(); 1213 ixgbe_handle_phy(sc); 1214 KERNEL_UNLOCK(); 1215 } 1216 1217 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 1218 1219 return (1); 1220 } 1221 1222 /********************************************************************* 1223 * 1224 * Media Ioctl callback 1225 * 1226 * This routine is called whenever the user queries the status of 1227 * the interface using ifconfig. 1228 * 1229 **********************************************************************/ 1230 void 1231 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq *ifmr) 1232 { 1233 struct ix_softc *sc = ifp->if_softc; 1234 uint64_t layer; 1235 1236 ifmr->ifm_active = IFM_ETHER; 1237 ifmr->ifm_status = IFM_AVALID; 1238 1239 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1240 ixgbe_update_link_status(sc); 1241 1242 if (!LINK_STATE_IS_UP(ifp->if_link_state)) 1243 return; 1244 1245 ifmr->ifm_status |= IFM_ACTIVE; 1246 layer = sc->phy_layer; 1247 1248 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 1249 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 1250 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 1251 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1252 switch (sc->link_speed) { 1253 case IXGBE_LINK_SPEED_10GB_FULL: 1254 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1255 break; 1256 case IXGBE_LINK_SPEED_1GB_FULL: 1257 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1258 break; 1259 case IXGBE_LINK_SPEED_100_FULL: 1260 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1261 break; 1262 case IXGBE_LINK_SPEED_10_FULL: 1263 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1264 break; 1265 } 1266 } 1267 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1268 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1269 switch (sc->link_speed) { 1270 case IXGBE_LINK_SPEED_10GB_FULL: 1271 ifmr->ifm_active |= IFM_10G_SFP_CU | IFM_FDX; 1272 break; 1273 } 1274 } 1275 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1276 switch (sc->link_speed) { 1277 case IXGBE_LINK_SPEED_10GB_FULL: 1278 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 1279 break; 1280 case IXGBE_LINK_SPEED_1GB_FULL: 1281 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1282 break; 1283 } 1284 } 1285 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 1286 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1287 switch (sc->link_speed) { 1288 case IXGBE_LINK_SPEED_10GB_FULL: 1289 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1290 break; 1291 case IXGBE_LINK_SPEED_1GB_FULL: 1292 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1293 break; 1294 } 1295 } 1296 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1297 switch (sc->link_speed) { 1298 case IXGBE_LINK_SPEED_10GB_FULL: 1299 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1300 break; 1301 } 1302 } 1303 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1304 switch (sc->link_speed) { 1305 case IXGBE_LINK_SPEED_10GB_FULL: 1306 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 1307 break; 1308 case IXGBE_LINK_SPEED_2_5GB_FULL: 1309 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1310 break; 1311 case IXGBE_LINK_SPEED_1GB_FULL: 1312 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1313 break; 1314 } 1315 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 1316 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 1317 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1318 switch (sc->link_speed) { 1319 case IXGBE_LINK_SPEED_10GB_FULL: 1320 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 1321 break; 1322 case IXGBE_LINK_SPEED_2_5GB_FULL: 1323 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1324 break; 1325 case IXGBE_LINK_SPEED_1GB_FULL: 1326 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1327 break; 1328 } 1329 } 1330 1331 switch (sc->hw.fc.current_mode) { 1332 case ixgbe_fc_tx_pause: 1333 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 1334 break; 1335 case ixgbe_fc_rx_pause: 1336 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 1337 break; 1338 case ixgbe_fc_full: 1339 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE | 1340 IFM_ETH_TXPAUSE; 1341 break; 1342 default: 1343 ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE | 1344 IFM_ETH_TXPAUSE); 1345 break; 1346 } 1347 } 1348 1349 /********************************************************************* 1350 * 1351 * Media Ioctl callback 1352 * 1353 * This routine is called when the user changes speed/duplex using 1354 * media/mediopt option with ifconfig. 1355 * 1356 **********************************************************************/ 1357 int 1358 ixgbe_media_change(struct ifnet *ifp) 1359 { 1360 struct ix_softc *sc = ifp->if_softc; 1361 struct ixgbe_hw *hw = &sc->hw; 1362 struct ifmedia *ifm = &sc->media; 1363 ixgbe_link_speed speed = 0; 1364 1365 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1366 return (EINVAL); 1367 1368 if (hw->phy.media_type == ixgbe_media_type_backplane) 1369 return (ENODEV); 1370 1371 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1372 case IFM_AUTO: 1373 case IFM_10G_T: 1374 speed |= IXGBE_LINK_SPEED_100_FULL; 1375 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1376 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1377 break; 1378 case IFM_10G_SR: 1379 case IFM_10G_KR: 1380 case IFM_10G_LR: 1381 case IFM_10G_LRM: 1382 case IFM_10G_CX4: 1383 case IFM_10G_KX4: 1384 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1385 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1386 break; 1387 case IFM_10G_SFP_CU: 1388 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1389 break; 1390 case IFM_1000_T: 1391 speed |= IXGBE_LINK_SPEED_100_FULL; 1392 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1393 break; 1394 case IFM_1000_LX: 1395 case IFM_1000_SX: 1396 case IFM_1000_CX: 1397 case IFM_1000_KX: 1398 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1399 break; 1400 case IFM_100_TX: 1401 speed |= IXGBE_LINK_SPEED_100_FULL; 1402 break; 1403 case IFM_10_T: 1404 speed |= IXGBE_LINK_SPEED_10_FULL; 1405 break; 1406 default: 1407 return (EINVAL); 1408 } 1409 1410 hw->mac.autotry_restart = TRUE; 1411 hw->mac.ops.setup_link(hw, speed, TRUE); 1412 1413 return (0); 1414 } 1415 1416 /********************************************************************* 1417 * 1418 * This routine maps the mbufs to tx descriptors, allowing the 1419 * TX engine to transmit the packets. 1420 * - return 0 on success, positive on failure 1421 * 1422 **********************************************************************/ 1423 1424 int 1425 ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head) 1426 { 1427 struct ix_softc *sc = txr->sc; 1428 uint32_t olinfo_status = 0, cmd_type_len; 1429 int i, j, ntxc; 1430 int first, last = 0; 1431 bus_dmamap_t map; 1432 struct ixgbe_tx_buf *txbuf; 1433 union ixgbe_adv_tx_desc *txd = NULL; 1434 1435 /* Basic descriptor defines */ 1436 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1437 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1438 1439 /* 1440 * Important to capture the first descriptor 1441 * used because it will contain the index of 1442 * the one we tell the hardware to report back 1443 */ 1444 first = txr->next_avail_desc; 1445 txbuf = &txr->tx_buffers[first]; 1446 map = txbuf->map; 1447 1448 /* 1449 * Set the appropriate offload context 1450 * this will becomes the first descriptor. 1451 */ 1452 ntxc = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 1453 if (ntxc == -1) 1454 goto xmit_fail; 1455 1456 /* 1457 * Map the packet for DMA. 1458 */ 1459 switch (bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1460 m_head, BUS_DMA_NOWAIT)) { 1461 case 0: 1462 break; 1463 case EFBIG: 1464 if (m_defrag(m_head, M_NOWAIT) == 0 && 1465 bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1466 m_head, BUS_DMA_NOWAIT) == 0) 1467 break; 1468 /* FALLTHROUGH */ 1469 default: 1470 return (0); 1471 } 1472 1473 i = txr->next_avail_desc + ntxc; 1474 if (i >= sc->num_tx_desc) 1475 i -= sc->num_tx_desc; 1476 1477 for (j = 0; j < map->dm_nsegs; j++) { 1478 txd = &txr->tx_base[i]; 1479 1480 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 1481 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1482 cmd_type_len | map->dm_segs[j].ds_len); 1483 txd->read.olinfo_status = htole32(olinfo_status); 1484 last = i; /* descriptor that will get completion IRQ */ 1485 1486 if (++i == sc->num_tx_desc) 1487 i = 0; 1488 } 1489 1490 txd->read.cmd_type_len |= 1491 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1492 1493 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1494 BUS_DMASYNC_PREWRITE); 1495 1496 /* Set the index of the descriptor that will be marked done */ 1497 txbuf->m_head = m_head; 1498 txbuf->eop_index = last; 1499 1500 membar_producer(); 1501 1502 txr->next_avail_desc = i; 1503 1504 return (ntxc + j); 1505 1506 xmit_fail: 1507 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 1508 return (0); 1509 } 1510 1511 void 1512 ixgbe_iff(struct ix_softc *sc) 1513 { 1514 struct ifnet *ifp = &sc->arpcom.ac_if; 1515 struct arpcom *ac = &sc->arpcom; 1516 uint32_t fctrl; 1517 uint8_t *mta; 1518 uint8_t *update_ptr; 1519 struct ether_multi *enm; 1520 struct ether_multistep step; 1521 int mcnt = 0; 1522 1523 IOCTL_DEBUGOUT("ixgbe_iff: begin"); 1524 1525 mta = sc->mta; 1526 bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1527 MAX_NUM_MULTICAST_ADDRESSES); 1528 1529 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1530 fctrl &= ~(IXGBE_FCTRL_MPE | IXGBE_FCTRL_UPE); 1531 ifp->if_flags &= ~IFF_ALLMULTI; 1532 1533 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1534 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1535 ifp->if_flags |= IFF_ALLMULTI; 1536 fctrl |= IXGBE_FCTRL_MPE; 1537 if (ifp->if_flags & IFF_PROMISC) 1538 fctrl |= IXGBE_FCTRL_UPE; 1539 } else { 1540 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1541 while (enm != NULL) { 1542 bcopy(enm->enm_addrlo, 1543 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1544 IXGBE_ETH_LENGTH_OF_ADDRESS); 1545 mcnt++; 1546 1547 ETHER_NEXT_MULTI(step, enm); 1548 } 1549 1550 update_ptr = mta; 1551 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1552 ixgbe_mc_array_itr, TRUE); 1553 } 1554 1555 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1556 } 1557 1558 /* 1559 * This is an iterator function now needed by the multicast 1560 * shared code. It simply feeds the shared code routine the 1561 * addresses in the array of ixgbe_iff() one by one. 1562 */ 1563 uint8_t * 1564 ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1565 { 1566 uint8_t *addr = *update_ptr; 1567 uint8_t *newptr; 1568 *vmdq = 0; 1569 1570 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1571 *update_ptr = newptr; 1572 return addr; 1573 } 1574 1575 void 1576 ixgbe_update_link_status(struct ix_softc *sc) 1577 { 1578 struct ifnet *ifp = &sc->arpcom.ac_if; 1579 int link_state = LINK_STATE_DOWN; 1580 1581 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 1582 1583 ifp->if_baudrate = 0; 1584 if (sc->link_up) { 1585 link_state = LINK_STATE_FULL_DUPLEX; 1586 1587 switch (sc->link_speed) { 1588 case IXGBE_LINK_SPEED_UNKNOWN: 1589 ifp->if_baudrate = 0; 1590 break; 1591 case IXGBE_LINK_SPEED_100_FULL: 1592 ifp->if_baudrate = IF_Mbps(100); 1593 break; 1594 case IXGBE_LINK_SPEED_1GB_FULL: 1595 ifp->if_baudrate = IF_Gbps(1); 1596 break; 1597 case IXGBE_LINK_SPEED_10GB_FULL: 1598 ifp->if_baudrate = IF_Gbps(10); 1599 break; 1600 } 1601 1602 /* Update any Flow Control changes */ 1603 sc->hw.mac.ops.fc_enable(&sc->hw); 1604 } 1605 if (ifp->if_link_state != link_state) { 1606 ifp->if_link_state = link_state; 1607 if_link_state_change(ifp); 1608 } 1609 } 1610 1611 1612 /********************************************************************* 1613 * 1614 * This routine disables all traffic on the adapter by issuing a 1615 * global reset on the MAC and deallocates TX/RX buffers. 1616 * 1617 **********************************************************************/ 1618 1619 void 1620 ixgbe_stop(void *arg) 1621 { 1622 struct ix_softc *sc = arg; 1623 struct ifnet *ifp = &sc->arpcom.ac_if; 1624 int i; 1625 1626 /* Tell the stack that the interface is no longer active */ 1627 ifp->if_flags &= ~IFF_RUNNING; 1628 1629 #if NKSTAT > 0 1630 timeout_del(&sc->sc_kstat_tmo); 1631 #endif 1632 ifp->if_timer = 0; 1633 1634 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1635 ixgbe_disable_intr(sc); 1636 1637 sc->hw.mac.ops.reset_hw(&sc->hw); 1638 sc->hw.adapter_stopped = FALSE; 1639 sc->hw.mac.ops.stop_adapter(&sc->hw); 1640 if (sc->hw.mac.type == ixgbe_mac_82599EB) 1641 sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw); 1642 /* Turn off the laser */ 1643 if (sc->hw.mac.ops.disable_tx_laser) 1644 sc->hw.mac.ops.disable_tx_laser(&sc->hw); 1645 1646 /* reprogram the RAR[0] in case user changed it. */ 1647 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1648 1649 intr_barrier(sc->tag); 1650 for (i = 0; i < sc->num_queues; i++) { 1651 struct ifqueue *ifq = ifp->if_ifqs[i]; 1652 ifq_barrier(ifq); 1653 ifq_clr_oactive(ifq); 1654 1655 if (sc->queues[i].tag != NULL) 1656 intr_barrier(sc->queues[i].tag); 1657 timeout_del(&sc->rx_rings[i].rx_refill); 1658 } 1659 1660 KASSERT((ifp->if_flags & IFF_RUNNING) == 0); 1661 1662 /* Should we really clear all structures on stop? */ 1663 ixgbe_free_transmit_structures(sc); 1664 ixgbe_free_receive_structures(sc); 1665 1666 ixgbe_update_link_status(sc); 1667 } 1668 1669 1670 /********************************************************************* 1671 * 1672 * Determine hardware revision. 1673 * 1674 **********************************************************************/ 1675 void 1676 ixgbe_identify_hardware(struct ix_softc *sc) 1677 { 1678 struct ixgbe_osdep *os = &sc->osdep; 1679 struct pci_attach_args *pa = &os->os_pa; 1680 uint32_t reg; 1681 1682 /* Save off the information about this board */ 1683 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1684 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1685 1686 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1687 sc->hw.revision_id = PCI_REVISION(reg); 1688 1689 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1690 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1691 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1692 1693 /* We need this here to set the num_segs below */ 1694 ixgbe_set_mac_type(&sc->hw); 1695 1696 /* Pick up the 82599 and VF settings */ 1697 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1698 sc->hw.phy.smart_speed = ixgbe_smart_speed; 1699 sc->num_segs = IXGBE_82599_SCATTER; 1700 } 1701 1702 /********************************************************************* 1703 * 1704 * Setup the Legacy or MSI Interrupt handler 1705 * 1706 **********************************************************************/ 1707 int 1708 ixgbe_allocate_legacy(struct ix_softc *sc) 1709 { 1710 struct ixgbe_osdep *os = &sc->osdep; 1711 struct pci_attach_args *pa = &os->os_pa; 1712 const char *intrstr = NULL; 1713 pci_chipset_tag_t pc = pa->pa_pc; 1714 pci_intr_handle_t ih; 1715 1716 /* We allocate a single interrupt resource */ 1717 if (pci_intr_map_msi(pa, &ih) != 0 && 1718 pci_intr_map(pa, &ih) != 0) { 1719 printf(": couldn't map interrupt\n"); 1720 return (ENXIO); 1721 } 1722 1723 #if 0 1724 /* XXX */ 1725 /* Tasklets for Link, SFP and Multispeed Fiber */ 1726 TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc); 1727 TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc); 1728 TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc); 1729 #endif 1730 1731 intrstr = pci_intr_string(pc, ih); 1732 sc->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE, 1733 ixgbe_legacy_intr, sc, sc->dev.dv_xname); 1734 if (sc->tag == NULL) { 1735 printf(": couldn't establish interrupt"); 1736 if (intrstr != NULL) 1737 printf(" at %s", intrstr); 1738 printf("\n"); 1739 return (ENXIO); 1740 } 1741 printf(": %s", intrstr); 1742 1743 /* For simplicity in the handlers */ 1744 sc->que_mask = IXGBE_EIMS_ENABLE_MASK; 1745 1746 return (0); 1747 } 1748 1749 /********************************************************************* 1750 * 1751 * Setup the MSI-X Interrupt handlers 1752 * 1753 **********************************************************************/ 1754 int 1755 ixgbe_allocate_msix(struct ix_softc *sc) 1756 { 1757 struct ixgbe_osdep *os = &sc->osdep; 1758 struct pci_attach_args *pa = &os->os_pa; 1759 int i = 0, error = 0; 1760 struct ix_queue *que; 1761 pci_intr_handle_t ih; 1762 1763 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) { 1764 if (pci_intr_map_msix(pa, i, &ih)) { 1765 printf("ixgbe_allocate_msix: " 1766 "pci_intr_map_msix vec %d failed\n", i); 1767 error = ENOMEM; 1768 goto fail; 1769 } 1770 1771 que->tag = pci_intr_establish_cpu(pa->pa_pc, ih, 1772 IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i), 1773 ixgbe_queue_intr, que, que->name); 1774 if (que->tag == NULL) { 1775 printf("ixgbe_allocate_msix: " 1776 "pci_intr_establish vec %d failed\n", i); 1777 error = ENOMEM; 1778 goto fail; 1779 } 1780 1781 que->msix = i; 1782 } 1783 1784 /* Now the link status/control last MSI-X vector */ 1785 if (pci_intr_map_msix(pa, i, &ih)) { 1786 printf("ixgbe_allocate_msix: " 1787 "pci_intr_map_msix link vector failed\n"); 1788 error = ENOMEM; 1789 goto fail; 1790 } 1791 1792 sc->tag = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE, 1793 ixgbe_link_intr, sc, sc->dev.dv_xname); 1794 if (sc->tag == NULL) { 1795 printf("ixgbe_allocate_msix: " 1796 "pci_intr_establish link vector failed\n"); 1797 error = ENOMEM; 1798 goto fail; 1799 } 1800 sc->linkvec = i; 1801 printf(", %s, %d queue%s", pci_intr_string(pa->pa_pc, ih), 1802 i, (i > 1) ? "s" : ""); 1803 1804 return (0); 1805 fail: 1806 for (que = sc->queues; i > 0; i--, que++) { 1807 if (que->tag == NULL) 1808 continue; 1809 pci_intr_disestablish(pa->pa_pc, que->tag); 1810 que->tag = NULL; 1811 } 1812 1813 return (error); 1814 } 1815 1816 void 1817 ixgbe_setup_msix(struct ix_softc *sc) 1818 { 1819 struct ixgbe_osdep *os = &sc->osdep; 1820 struct pci_attach_args *pa = &os->os_pa; 1821 int nmsix; 1822 unsigned int maxq; 1823 1824 if (!ixgbe_enable_msix) 1825 return; 1826 1827 nmsix = pci_intr_msix_count(pa); 1828 if (nmsix <= 1) 1829 return; 1830 1831 /* give one vector to events */ 1832 nmsix--; 1833 1834 /* XXX the number of queues is limited to what we can keep stats on */ 1835 maxq = (sc->hw.mac.type == ixgbe_mac_82598EB) ? 8 : 16; 1836 1837 sc->sc_intrmap = intrmap_create(&sc->dev, nmsix, maxq, 0); 1838 sc->num_queues = intrmap_count(sc->sc_intrmap); 1839 } 1840 1841 int 1842 ixgbe_allocate_pci_resources(struct ix_softc *sc) 1843 { 1844 struct ixgbe_osdep *os = &sc->osdep; 1845 struct pci_attach_args *pa = &os->os_pa; 1846 int val; 1847 1848 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1849 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) { 1850 printf(": mmba is not mem space\n"); 1851 return (ENXIO); 1852 } 1853 1854 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1855 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1856 printf(": cannot find mem space\n"); 1857 return (ENXIO); 1858 } 1859 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1860 1861 /* Legacy defaults */ 1862 sc->num_queues = 1; 1863 sc->hw.back = os; 1864 1865 /* Now setup MSI or MSI/X, return us the number of supported vectors. */ 1866 ixgbe_setup_msix(sc); 1867 1868 return (0); 1869 } 1870 1871 void 1872 ixgbe_free_pci_resources(struct ix_softc * sc) 1873 { 1874 struct ixgbe_osdep *os = &sc->osdep; 1875 struct pci_attach_args *pa = &os->os_pa; 1876 struct ix_queue *que = sc->queues; 1877 int i; 1878 1879 /* Release all msix queue resources: */ 1880 for (i = 0; i < sc->num_queues; i++, que++) { 1881 if (que->tag) 1882 pci_intr_disestablish(pa->pa_pc, que->tag); 1883 que->tag = NULL; 1884 } 1885 1886 if (sc->tag) 1887 pci_intr_disestablish(pa->pa_pc, sc->tag); 1888 sc->tag = NULL; 1889 if (os->os_membase != 0) 1890 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1891 os->os_membase = 0; 1892 } 1893 1894 /********************************************************************* 1895 * 1896 * Setup networking device structure and register an interface. 1897 * 1898 **********************************************************************/ 1899 void 1900 ixgbe_setup_interface(struct ix_softc *sc) 1901 { 1902 struct ifnet *ifp = &sc->arpcom.ac_if; 1903 int i; 1904 1905 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1906 ifp->if_softc = sc; 1907 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1908 ifp->if_xflags = IFXF_MPSAFE; 1909 ifp->if_ioctl = ixgbe_ioctl; 1910 ifp->if_qstart = ixgbe_start; 1911 ifp->if_timer = 0; 1912 ifp->if_watchdog = ixgbe_watchdog; 1913 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1914 ETHER_HDR_LEN - ETHER_CRC_LEN; 1915 ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1); 1916 1917 ifp->if_capabilities = IFCAP_VLAN_MTU; 1918 1919 #if NVLAN > 0 1920 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1921 #endif 1922 1923 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 1924 ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6; 1925 ifp->if_capabilities |= IFCAP_CSUM_IPv4; 1926 1927 /* 1928 * Specify the media types supported by this sc and register 1929 * callbacks to update media and link information 1930 */ 1931 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1932 ixgbe_media_status); 1933 ixgbe_add_media_types(sc); 1934 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1935 1936 if_attach(ifp); 1937 ether_ifattach(ifp); 1938 1939 if_attach_queues(ifp, sc->num_queues); 1940 if_attach_iqueues(ifp, sc->num_queues); 1941 for (i = 0; i < sc->num_queues; i++) { 1942 struct ifqueue *ifq = ifp->if_ifqs[i]; 1943 struct ifiqueue *ifiq = ifp->if_iqs[i]; 1944 struct tx_ring *txr = &sc->tx_rings[i]; 1945 struct rx_ring *rxr = &sc->rx_rings[i]; 1946 1947 ifq->ifq_softc = txr; 1948 txr->ifq = ifq; 1949 1950 ifiq->ifiq_softc = rxr; 1951 rxr->ifiq = ifiq; 1952 1953 #if NKSTAT > 0 1954 ix_txq_kstats(sc, txr); 1955 ix_rxq_kstats(sc, rxr); 1956 #endif 1957 } 1958 1959 sc->max_frame_size = IXGBE_MAX_FRAME_SIZE; 1960 } 1961 1962 void 1963 ixgbe_add_media_types(struct ix_softc *sc) 1964 { 1965 struct ixgbe_hw *hw = &sc->hw; 1966 uint64_t layer; 1967 1968 sc->phy_layer = hw->mac.ops.get_supported_physical_layer(hw); 1969 layer = sc->phy_layer; 1970 1971 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1972 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL); 1973 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1974 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 1975 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1976 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1977 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1978 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 1979 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SFP_CU, 0, NULL); 1980 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1981 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL); 1982 if (hw->phy.multispeed_fiber) 1983 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_LX, 0, 1984 NULL); 1985 } 1986 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1987 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 1988 if (hw->phy.multispeed_fiber) 1989 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, 1990 NULL); 1991 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 1992 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 1993 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 1994 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); 1995 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 1996 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL); 1997 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) 1998 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); 1999 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 2000 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL); 2001 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) 2002 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL); 2003 2004 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2005 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, 2006 NULL); 2007 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 2008 } 2009 2010 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2011 } 2012 2013 void 2014 ixgbe_config_link(struct ix_softc *sc) 2015 { 2016 uint32_t autoneg, err = 0; 2017 bool negotiate; 2018 2019 if (ixgbe_is_sfp(&sc->hw)) { 2020 if (sc->hw.phy.multispeed_fiber) { 2021 sc->hw.mac.ops.setup_sfp(&sc->hw); 2022 if (sc->hw.mac.ops.enable_tx_laser) 2023 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 2024 ixgbe_handle_msf(sc); 2025 } else 2026 ixgbe_handle_mod(sc); 2027 } else { 2028 if (sc->hw.mac.ops.check_link) 2029 err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg, 2030 &sc->link_up, FALSE); 2031 if (err) 2032 return; 2033 autoneg = sc->hw.phy.autoneg_advertised; 2034 if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities)) 2035 err = sc->hw.mac.ops.get_link_capabilities(&sc->hw, 2036 &autoneg, &negotiate); 2037 if (err) 2038 return; 2039 if (sc->hw.mac.ops.setup_link) 2040 sc->hw.mac.ops.setup_link(&sc->hw, 2041 autoneg, sc->link_up); 2042 } 2043 } 2044 2045 /******************************************************************** 2046 * Manage DMA'able memory. 2047 *******************************************************************/ 2048 int 2049 ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 2050 struct ixgbe_dma_alloc *dma, int mapflags) 2051 { 2052 struct ifnet *ifp = &sc->arpcom.ac_if; 2053 struct ixgbe_osdep *os = &sc->osdep; 2054 int r; 2055 2056 dma->dma_tag = os->os_pa.pa_dmat; 2057 r = bus_dmamap_create(dma->dma_tag, size, 1, 2058 size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 2059 if (r != 0) { 2060 printf("%s: ixgbe_dma_malloc: bus_dmamap_create failed; " 2061 "error %u\n", ifp->if_xname, r); 2062 goto fail_0; 2063 } 2064 2065 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 2066 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 2067 if (r != 0) { 2068 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2069 "error %u\n", ifp->if_xname, r); 2070 goto fail_1; 2071 } 2072 2073 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 2074 &dma->dma_vaddr, BUS_DMA_NOWAIT); 2075 if (r != 0) { 2076 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 2077 "error %u\n", ifp->if_xname, r); 2078 goto fail_2; 2079 } 2080 2081 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2082 size, NULL, mapflags | BUS_DMA_NOWAIT); 2083 if (r != 0) { 2084 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 2085 "error %u\n", ifp->if_xname, r); 2086 goto fail_3; 2087 } 2088 2089 dma->dma_size = size; 2090 return (0); 2091 fail_3: 2092 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 2093 fail_2: 2094 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2095 fail_1: 2096 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2097 fail_0: 2098 dma->dma_map = NULL; 2099 dma->dma_tag = NULL; 2100 return (r); 2101 } 2102 2103 void 2104 ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 2105 { 2106 if (dma->dma_tag == NULL) 2107 return; 2108 2109 if (dma->dma_map != NULL) { 2110 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 2111 dma->dma_map->dm_mapsize, 2112 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2113 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2114 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 2115 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2116 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2117 dma->dma_map = NULL; 2118 } 2119 } 2120 2121 2122 /********************************************************************* 2123 * 2124 * Allocate memory for the transmit and receive rings, and then 2125 * the descriptors associated with each, called only once at attach. 2126 * 2127 **********************************************************************/ 2128 int 2129 ixgbe_allocate_queues(struct ix_softc *sc) 2130 { 2131 struct ifnet *ifp = &sc->arpcom.ac_if; 2132 struct ix_queue *que; 2133 struct tx_ring *txr; 2134 struct rx_ring *rxr; 2135 int rsize, tsize; 2136 int txconf = 0, rxconf = 0, i; 2137 2138 /* First allocate the top level queue structs */ 2139 if (!(sc->queues = mallocarray(sc->num_queues, 2140 sizeof(struct ix_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2141 printf("%s: Unable to allocate queue memory\n", ifp->if_xname); 2142 goto fail; 2143 } 2144 2145 /* Then allocate the TX ring struct memory */ 2146 if (!(sc->tx_rings = mallocarray(sc->num_queues, 2147 sizeof(struct tx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2148 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 2149 goto fail; 2150 } 2151 2152 /* Next allocate the RX */ 2153 if (!(sc->rx_rings = mallocarray(sc->num_queues, 2154 sizeof(struct rx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2155 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 2156 goto rx_fail; 2157 } 2158 2159 /* For the ring itself */ 2160 tsize = roundup2(sc->num_tx_desc * 2161 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2162 2163 /* 2164 * Now set up the TX queues, txconf is needed to handle the 2165 * possibility that things fail midcourse and we need to 2166 * undo memory gracefully 2167 */ 2168 for (i = 0; i < sc->num_queues; i++, txconf++) { 2169 /* Set up some basics */ 2170 txr = &sc->tx_rings[i]; 2171 txr->sc = sc; 2172 txr->me = i; 2173 2174 if (ixgbe_dma_malloc(sc, tsize, 2175 &txr->txdma, BUS_DMA_NOWAIT)) { 2176 printf("%s: Unable to allocate TX Descriptor memory\n", 2177 ifp->if_xname); 2178 goto err_tx_desc; 2179 } 2180 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2181 bzero((void *)txr->tx_base, tsize); 2182 } 2183 2184 /* 2185 * Next the RX queues... 2186 */ 2187 rsize = roundup2(sc->num_rx_desc * 2188 sizeof(union ixgbe_adv_rx_desc), 4096); 2189 for (i = 0; i < sc->num_queues; i++, rxconf++) { 2190 rxr = &sc->rx_rings[i]; 2191 /* Set up some basics */ 2192 rxr->sc = sc; 2193 rxr->me = i; 2194 timeout_set(&rxr->rx_refill, ixgbe_rxrefill, rxr); 2195 2196 if (ixgbe_dma_malloc(sc, rsize, 2197 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2198 printf("%s: Unable to allocate RxDescriptor memory\n", 2199 ifp->if_xname); 2200 goto err_rx_desc; 2201 } 2202 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2203 bzero((void *)rxr->rx_base, rsize); 2204 } 2205 2206 /* 2207 * Finally set up the queue holding structs 2208 */ 2209 for (i = 0; i < sc->num_queues; i++) { 2210 que = &sc->queues[i]; 2211 que->sc = sc; 2212 que->txr = &sc->tx_rings[i]; 2213 que->rxr = &sc->rx_rings[i]; 2214 snprintf(que->name, sizeof(que->name), "%s:%d", 2215 sc->dev.dv_xname, i); 2216 } 2217 2218 return (0); 2219 2220 err_rx_desc: 2221 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 2222 ixgbe_dma_free(sc, &rxr->rxdma); 2223 err_tx_desc: 2224 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 2225 ixgbe_dma_free(sc, &txr->txdma); 2226 free(sc->rx_rings, M_DEVBUF, sc->num_queues * sizeof(struct rx_ring)); 2227 sc->rx_rings = NULL; 2228 rx_fail: 2229 free(sc->tx_rings, M_DEVBUF, sc->num_queues * sizeof(struct tx_ring)); 2230 sc->tx_rings = NULL; 2231 fail: 2232 return (ENOMEM); 2233 } 2234 2235 /********************************************************************* 2236 * 2237 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2238 * the information needed to transmit a packet on the wire. This is 2239 * called only once at attach, setup is done every reset. 2240 * 2241 **********************************************************************/ 2242 int 2243 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2244 { 2245 struct ix_softc *sc = txr->sc; 2246 struct ifnet *ifp = &sc->arpcom.ac_if; 2247 struct ixgbe_tx_buf *txbuf; 2248 int error, i; 2249 2250 if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc, 2251 sizeof(struct ixgbe_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2252 printf("%s: Unable to allocate tx_buffer memory\n", 2253 ifp->if_xname); 2254 error = ENOMEM; 2255 goto fail; 2256 } 2257 txr->txtag = txr->txdma.dma_tag; 2258 2259 /* Create the descriptor buffer dma maps */ 2260 for (i = 0; i < sc->num_tx_desc; i++) { 2261 txbuf = &txr->tx_buffers[i]; 2262 error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE, 2263 sc->num_segs, PAGE_SIZE, 0, 2264 BUS_DMA_NOWAIT, &txbuf->map); 2265 2266 if (error != 0) { 2267 printf("%s: Unable to create TX DMA map\n", 2268 ifp->if_xname); 2269 goto fail; 2270 } 2271 } 2272 2273 return 0; 2274 fail: 2275 return (error); 2276 } 2277 2278 /********************************************************************* 2279 * 2280 * Initialize a transmit ring. 2281 * 2282 **********************************************************************/ 2283 int 2284 ixgbe_setup_transmit_ring(struct tx_ring *txr) 2285 { 2286 struct ix_softc *sc = txr->sc; 2287 int error; 2288 2289 /* Now allocate transmit buffers for the ring */ 2290 if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0) 2291 return (error); 2292 2293 /* Clear the old ring contents */ 2294 bzero((void *)txr->tx_base, 2295 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 2296 2297 /* Reset indices */ 2298 txr->next_avail_desc = 0; 2299 txr->next_to_clean = 0; 2300 2301 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2302 0, txr->txdma.dma_map->dm_mapsize, 2303 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2304 2305 return (0); 2306 } 2307 2308 /********************************************************************* 2309 * 2310 * Initialize all transmit rings. 2311 * 2312 **********************************************************************/ 2313 int 2314 ixgbe_setup_transmit_structures(struct ix_softc *sc) 2315 { 2316 struct tx_ring *txr = sc->tx_rings; 2317 int i, error; 2318 2319 for (i = 0; i < sc->num_queues; i++, txr++) { 2320 if ((error = ixgbe_setup_transmit_ring(txr)) != 0) 2321 goto fail; 2322 } 2323 2324 return (0); 2325 fail: 2326 ixgbe_free_transmit_structures(sc); 2327 return (error); 2328 } 2329 2330 /********************************************************************* 2331 * 2332 * Enable transmit unit. 2333 * 2334 **********************************************************************/ 2335 void 2336 ixgbe_initialize_transmit_units(struct ix_softc *sc) 2337 { 2338 struct ifnet *ifp = &sc->arpcom.ac_if; 2339 struct tx_ring *txr; 2340 struct ixgbe_hw *hw = &sc->hw; 2341 int i; 2342 uint64_t tdba; 2343 uint32_t txctrl; 2344 2345 /* Setup the Base and Length of the Tx Descriptor Ring */ 2346 2347 for (i = 0; i < sc->num_queues; i++) { 2348 txr = &sc->tx_rings[i]; 2349 2350 /* Setup descriptor base address */ 2351 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 2352 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 2353 (tdba & 0x00000000ffffffffULL)); 2354 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 2355 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2356 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 2357 2358 /* Setup the HW Tx Head and Tail descriptor pointers */ 2359 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2360 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2361 2362 /* Setup Transmit Descriptor Cmd Settings */ 2363 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2364 txr->queue_status = IXGBE_QUEUE_IDLE; 2365 txr->watchdog_timer = 0; 2366 2367 /* Disable Head Writeback */ 2368 switch (hw->mac.type) { 2369 case ixgbe_mac_82598EB: 2370 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2371 break; 2372 case ixgbe_mac_82599EB: 2373 case ixgbe_mac_X540: 2374 default: 2375 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2376 break; 2377 } 2378 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2379 switch (hw->mac.type) { 2380 case ixgbe_mac_82598EB: 2381 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2382 break; 2383 case ixgbe_mac_82599EB: 2384 case ixgbe_mac_X540: 2385 default: 2386 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2387 break; 2388 } 2389 } 2390 ifp->if_timer = 0; 2391 2392 if (hw->mac.type != ixgbe_mac_82598EB) { 2393 uint32_t dmatxctl, rttdcs; 2394 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2395 dmatxctl |= IXGBE_DMATXCTL_TE; 2396 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2397 /* Disable arbiter to set MTQC */ 2398 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2399 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2400 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2401 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2402 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2403 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2404 } 2405 } 2406 2407 /********************************************************************* 2408 * 2409 * Free all transmit rings. 2410 * 2411 **********************************************************************/ 2412 void 2413 ixgbe_free_transmit_structures(struct ix_softc *sc) 2414 { 2415 struct tx_ring *txr = sc->tx_rings; 2416 int i; 2417 2418 for (i = 0; i < sc->num_queues; i++, txr++) 2419 ixgbe_free_transmit_buffers(txr); 2420 } 2421 2422 /********************************************************************* 2423 * 2424 * Free transmit ring related data structures. 2425 * 2426 **********************************************************************/ 2427 void 2428 ixgbe_free_transmit_buffers(struct tx_ring *txr) 2429 { 2430 struct ix_softc *sc = txr->sc; 2431 struct ixgbe_tx_buf *tx_buffer; 2432 int i; 2433 2434 INIT_DEBUGOUT("free_transmit_ring: begin"); 2435 2436 if (txr->tx_buffers == NULL) 2437 return; 2438 2439 tx_buffer = txr->tx_buffers; 2440 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 2441 if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) { 2442 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2443 0, tx_buffer->map->dm_mapsize, 2444 BUS_DMASYNC_POSTWRITE); 2445 bus_dmamap_unload(txr->txdma.dma_tag, 2446 tx_buffer->map); 2447 } 2448 if (tx_buffer->m_head != NULL) { 2449 m_freem(tx_buffer->m_head); 2450 tx_buffer->m_head = NULL; 2451 } 2452 if (tx_buffer->map != NULL) { 2453 bus_dmamap_destroy(txr->txdma.dma_tag, 2454 tx_buffer->map); 2455 tx_buffer->map = NULL; 2456 } 2457 } 2458 2459 if (txr->tx_buffers != NULL) 2460 free(txr->tx_buffers, M_DEVBUF, 2461 sc->num_tx_desc * sizeof(struct ixgbe_tx_buf)); 2462 txr->tx_buffers = NULL; 2463 txr->txtag = NULL; 2464 } 2465 2466 /********************************************************************* 2467 * 2468 * Advanced Context Descriptor setup for VLAN or CSUM 2469 * 2470 **********************************************************************/ 2471 2472 static inline int 2473 ixgbe_csum_offload(struct mbuf *mp, uint32_t *vlan_macip_lens, 2474 uint32_t *type_tucmd_mlhl, uint32_t *olinfo_status) 2475 { 2476 struct ether_header *eh = mtod(mp, struct ether_header *); 2477 struct mbuf *m; 2478 int hoff; 2479 int offload = 0; 2480 uint32_t iphlen; 2481 uint8_t ipproto; 2482 2483 *vlan_macip_lens |= (sizeof(*eh) << IXGBE_ADVTXD_MACLEN_SHIFT); 2484 2485 switch (ntohs(eh->ether_type)) { 2486 case ETHERTYPE_IP: { 2487 struct ip *ip; 2488 2489 m = m_getptr(mp, sizeof(*eh), &hoff); 2490 KASSERT(m != NULL && m->m_len - hoff >= sizeof(*ip)); 2491 ip = (struct ip *)(mtod(m, caddr_t) + hoff); 2492 2493 iphlen = ip->ip_hl << 2; 2494 ipproto = ip->ip_p; 2495 2496 if (ISSET(mp->m_pkthdr.csum_flags, M_IPV4_CSUM_OUT)) { 2497 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2498 offload = 1; 2499 } 2500 2501 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2502 break; 2503 } 2504 2505 #ifdef INET6 2506 case ETHERTYPE_IPV6: { 2507 struct ip6_hdr *ip6; 2508 2509 m = m_getptr(mp, sizeof(*eh), &hoff); 2510 KASSERT(m != NULL && m->m_len - hoff >= sizeof(*ip6)); 2511 ip6 = (struct ip6_hdr *)(mtod(m, caddr_t) + hoff); 2512 2513 iphlen = sizeof(*ip6); 2514 ipproto = ip6->ip6_nxt; 2515 2516 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2517 break; 2518 } 2519 #endif 2520 2521 default: 2522 return offload; 2523 } 2524 2525 *vlan_macip_lens |= iphlen; 2526 2527 switch (ipproto) { 2528 case IPPROTO_TCP: 2529 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2530 if (ISSET(mp->m_pkthdr.csum_flags, M_TCP_CSUM_OUT)) { 2531 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2532 offload = 1; 2533 } 2534 break; 2535 case IPPROTO_UDP: 2536 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2537 if (ISSET(mp->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) { 2538 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2539 offload = 1; 2540 } 2541 break; 2542 } 2543 2544 return offload; 2545 } 2546 2547 static int 2548 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, 2549 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2550 { 2551 struct ixgbe_adv_tx_context_desc *TXD; 2552 struct ixgbe_tx_buf *tx_buffer; 2553 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2554 int ctxd = txr->next_avail_desc; 2555 int offload = 0; 2556 2557 /* Indicate the whole packet as payload when not doing TSO */ 2558 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2559 2560 #if NVLAN > 0 2561 if (ISSET(mp->m_flags, M_VLANTAG)) { 2562 uint32_t vtag = mp->m_pkthdr.ether_vtag; 2563 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2564 *cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 2565 offload |= 1; 2566 } 2567 #endif 2568 2569 offload |= ixgbe_csum_offload(mp, &vlan_macip_lens, &type_tucmd_mlhl, 2570 olinfo_status); 2571 2572 if (!offload) 2573 return (0); 2574 2575 TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; 2576 tx_buffer = &txr->tx_buffers[ctxd]; 2577 2578 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2579 2580 /* Now copy bits into descriptor */ 2581 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2582 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2583 TXD->seqnum_seed = htole32(0); 2584 TXD->mss_l4len_idx = htole32(0); 2585 2586 tx_buffer->m_head = NULL; 2587 tx_buffer->eop_index = -1; 2588 2589 return (1); 2590 } 2591 2592 /********************************************************************** 2593 * 2594 * Examine each tx_buffer in the used queue. If the hardware is done 2595 * processing the packet then free associated resources. The 2596 * tx_buffer is put back on the free queue. 2597 * 2598 **********************************************************************/ 2599 int 2600 ixgbe_txeof(struct tx_ring *txr) 2601 { 2602 struct ix_softc *sc = txr->sc; 2603 struct ifqueue *ifq = txr->ifq; 2604 struct ifnet *ifp = &sc->arpcom.ac_if; 2605 unsigned int head, tail, last; 2606 struct ixgbe_tx_buf *tx_buffer; 2607 struct ixgbe_legacy_tx_desc *tx_desc; 2608 2609 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2610 return FALSE; 2611 2612 head = txr->next_avail_desc; 2613 tail = txr->next_to_clean; 2614 2615 membar_consumer(); 2616 2617 if (head == tail) 2618 return (FALSE); 2619 2620 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2621 0, txr->txdma.dma_map->dm_mapsize, 2622 BUS_DMASYNC_POSTREAD); 2623 2624 for (;;) { 2625 tx_buffer = &txr->tx_buffers[tail]; 2626 last = tx_buffer->eop_index; 2627 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2628 2629 if (!ISSET(tx_desc->upper.fields.status, IXGBE_TXD_STAT_DD)) 2630 break; 2631 2632 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2633 0, tx_buffer->map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2634 bus_dmamap_unload(txr->txdma.dma_tag, tx_buffer->map); 2635 m_freem(tx_buffer->m_head); 2636 2637 tx_buffer->m_head = NULL; 2638 tx_buffer->eop_index = -1; 2639 2640 tail = last + 1; 2641 if (tail == sc->num_tx_desc) 2642 tail = 0; 2643 if (head == tail) { 2644 /* All clean, turn off the timer */ 2645 ifp->if_timer = 0; 2646 break; 2647 } 2648 } 2649 2650 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2651 0, txr->txdma.dma_map->dm_mapsize, 2652 BUS_DMASYNC_PREREAD); 2653 2654 membar_producer(); 2655 2656 txr->next_to_clean = tail; 2657 2658 if (ifq_is_oactive(ifq)) 2659 ifq_restart(ifq); 2660 2661 return TRUE; 2662 } 2663 2664 /********************************************************************* 2665 * 2666 * Get a buffer from system mbuf buffer pool. 2667 * 2668 **********************************************************************/ 2669 int 2670 ixgbe_get_buf(struct rx_ring *rxr, int i) 2671 { 2672 struct ix_softc *sc = rxr->sc; 2673 struct ixgbe_rx_buf *rxbuf; 2674 struct mbuf *mp; 2675 int error; 2676 union ixgbe_adv_rx_desc *rxdesc; 2677 2678 rxbuf = &rxr->rx_buffers[i]; 2679 rxdesc = &rxr->rx_base[i]; 2680 if (rxbuf->buf) { 2681 printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n", 2682 sc->dev.dv_xname, i); 2683 return (ENOBUFS); 2684 } 2685 2686 /* needed in any case so prealocate since this one will fail for sure */ 2687 mp = MCLGETL(NULL, M_DONTWAIT, sc->rx_mbuf_sz); 2688 if (!mp) 2689 return (ENOBUFS); 2690 2691 mp->m_data += (mp->m_ext.ext_size - sc->rx_mbuf_sz); 2692 mp->m_len = mp->m_pkthdr.len = sc->rx_mbuf_sz; 2693 2694 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, 2695 mp, BUS_DMA_NOWAIT); 2696 if (error) { 2697 m_freem(mp); 2698 return (error); 2699 } 2700 2701 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2702 0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 2703 rxbuf->buf = mp; 2704 2705 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 2706 2707 return (0); 2708 } 2709 2710 /********************************************************************* 2711 * 2712 * Allocate memory for rx_buffer structures. Since we use one 2713 * rx_buffer per received packet, the maximum number of rx_buffer's 2714 * that we'll need is equal to the number of receive descriptors 2715 * that we've allocated. 2716 * 2717 **********************************************************************/ 2718 int 2719 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2720 { 2721 struct ix_softc *sc = rxr->sc; 2722 struct ifnet *ifp = &sc->arpcom.ac_if; 2723 struct ixgbe_rx_buf *rxbuf; 2724 int i, error; 2725 2726 if (!(rxr->rx_buffers = mallocarray(sc->num_rx_desc, 2727 sizeof(struct ixgbe_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2728 printf("%s: Unable to allocate rx_buffer memory\n", 2729 ifp->if_xname); 2730 error = ENOMEM; 2731 goto fail; 2732 } 2733 2734 rxbuf = rxr->rx_buffers; 2735 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2736 error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1, 2737 16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map); 2738 if (error) { 2739 printf("%s: Unable to create Pack DMA map\n", 2740 ifp->if_xname); 2741 goto fail; 2742 } 2743 } 2744 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2745 rxr->rxdma.dma_map->dm_mapsize, 2746 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2747 2748 return (0); 2749 2750 fail: 2751 return (error); 2752 } 2753 2754 /********************************************************************* 2755 * 2756 * Initialize a receive ring and its buffers. 2757 * 2758 **********************************************************************/ 2759 int 2760 ixgbe_setup_receive_ring(struct rx_ring *rxr) 2761 { 2762 struct ix_softc *sc = rxr->sc; 2763 struct ifnet *ifp = &sc->arpcom.ac_if; 2764 int rsize, error; 2765 2766 rsize = roundup2(sc->num_rx_desc * 2767 sizeof(union ixgbe_adv_rx_desc), 4096); 2768 /* Clear the ring contents */ 2769 bzero((void *)rxr->rx_base, rsize); 2770 2771 if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0) 2772 return (error); 2773 2774 /* Setup our descriptor indices */ 2775 rxr->next_to_check = 0; 2776 rxr->last_desc_filled = sc->num_rx_desc - 1; 2777 2778 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2779 sc->num_rx_desc - 1); 2780 2781 ixgbe_rxfill(rxr); 2782 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 2783 printf("%s: unable to fill any rx descriptors\n", 2784 sc->dev.dv_xname); 2785 return (ENOBUFS); 2786 } 2787 2788 return (0); 2789 } 2790 2791 int 2792 ixgbe_rxfill(struct rx_ring *rxr) 2793 { 2794 struct ix_softc *sc = rxr->sc; 2795 int post = 0; 2796 u_int slots; 2797 int i; 2798 2799 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2800 0, rxr->rxdma.dma_map->dm_mapsize, 2801 BUS_DMASYNC_POSTWRITE); 2802 2803 i = rxr->last_desc_filled; 2804 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); 2805 slots > 0; slots--) { 2806 if (++i == sc->num_rx_desc) 2807 i = 0; 2808 2809 if (ixgbe_get_buf(rxr, i) != 0) 2810 break; 2811 2812 rxr->last_desc_filled = i; 2813 post = 1; 2814 } 2815 2816 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2817 0, rxr->rxdma.dma_map->dm_mapsize, 2818 BUS_DMASYNC_PREWRITE); 2819 2820 if_rxr_put(&rxr->rx_ring, slots); 2821 2822 return (post); 2823 } 2824 2825 void 2826 ixgbe_rxrefill(void *xrxr) 2827 { 2828 struct rx_ring *rxr = xrxr; 2829 struct ix_softc *sc = rxr->sc; 2830 2831 if (ixgbe_rxfill(rxr)) { 2832 /* Advance the Rx Queue "Tail Pointer" */ 2833 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me), 2834 rxr->last_desc_filled); 2835 } else if (if_rxr_inuse(&rxr->rx_ring) == 0) 2836 timeout_add(&rxr->rx_refill, 1); 2837 2838 } 2839 2840 /********************************************************************* 2841 * 2842 * Initialize all receive rings. 2843 * 2844 **********************************************************************/ 2845 int 2846 ixgbe_setup_receive_structures(struct ix_softc *sc) 2847 { 2848 struct rx_ring *rxr = sc->rx_rings; 2849 int i; 2850 2851 for (i = 0; i < sc->num_queues; i++, rxr++) 2852 if (ixgbe_setup_receive_ring(rxr)) 2853 goto fail; 2854 2855 return (0); 2856 fail: 2857 ixgbe_free_receive_structures(sc); 2858 return (ENOBUFS); 2859 } 2860 2861 /********************************************************************* 2862 * 2863 * Setup receive registers and features. 2864 * 2865 **********************************************************************/ 2866 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2867 2868 void 2869 ixgbe_initialize_receive_units(struct ix_softc *sc) 2870 { 2871 struct rx_ring *rxr = sc->rx_rings; 2872 struct ixgbe_hw *hw = &sc->hw; 2873 uint32_t bufsz, fctrl, srrctl, rxcsum; 2874 uint32_t hlreg; 2875 int i; 2876 2877 /* 2878 * Make sure receives are disabled while 2879 * setting up the descriptor ring 2880 */ 2881 ixgbe_disable_rx(hw); 2882 2883 /* Enable broadcasts */ 2884 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2885 fctrl |= IXGBE_FCTRL_BAM; 2886 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 2887 fctrl |= IXGBE_FCTRL_DPF; 2888 fctrl |= IXGBE_FCTRL_PMCF; 2889 } 2890 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 2891 2892 /* Always enable jumbo frame reception */ 2893 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2894 hlreg |= IXGBE_HLREG0_JUMBOEN; 2895 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2896 2897 bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2898 2899 for (i = 0; i < sc->num_queues; i++, rxr++) { 2900 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2901 2902 /* Setup the Base and Length of the Rx Descriptor Ring */ 2903 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 2904 (rdba & 0x00000000ffffffffULL)); 2905 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 2906 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 2907 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2908 2909 /* Set up the SRRCTL register */ 2910 srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2911 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 2912 2913 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2914 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 2915 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 2916 } 2917 2918 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 2919 uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR | 2920 IXGBE_PSRTYPE_UDPHDR | 2921 IXGBE_PSRTYPE_IPV4HDR | 2922 IXGBE_PSRTYPE_IPV6HDR; 2923 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 2924 } 2925 2926 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2927 rxcsum &= ~IXGBE_RXCSUM_PCSD; 2928 2929 ixgbe_initialize_rss_mapping(sc); 2930 2931 /* Setup RSS */ 2932 if (sc->num_queues > 1) { 2933 /* RSS and RX IPP Checksum are mutually exclusive */ 2934 rxcsum |= IXGBE_RXCSUM_PCSD; 2935 } 2936 2937 /* Map QPRC/QPRDC/QPTC on a per queue basis */ 2938 ixgbe_map_queue_statistics(sc); 2939 2940 /* This is useful for calculating UDP/IP fragment checksums */ 2941 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2942 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2943 2944 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2945 } 2946 2947 void 2948 ixgbe_initialize_rss_mapping(struct ix_softc *sc) 2949 { 2950 struct ixgbe_hw *hw = &sc->hw; 2951 uint32_t reta = 0, mrqc, rss_key[10]; 2952 int i, j, queue_id, table_size, index_mult; 2953 2954 /* set up random bits */ 2955 stoeplitz_to_key(&rss_key, sizeof(rss_key)); 2956 2957 /* Set multiplier for RETA setup and table size based on MAC */ 2958 index_mult = 0x1; 2959 table_size = 128; 2960 switch (sc->hw.mac.type) { 2961 case ixgbe_mac_82598EB: 2962 index_mult = 0x11; 2963 break; 2964 case ixgbe_mac_X550: 2965 case ixgbe_mac_X550EM_x: 2966 case ixgbe_mac_X550EM_a: 2967 table_size = 512; 2968 break; 2969 default: 2970 break; 2971 } 2972 2973 /* Set up the redirection table */ 2974 for (i = 0, j = 0; i < table_size; i++, j++) { 2975 if (j == sc->num_queues) j = 0; 2976 queue_id = (j * index_mult); 2977 /* 2978 * The low 8 bits are for hash value (n+0); 2979 * The next 8 bits are for hash value (n+1), etc. 2980 */ 2981 reta = reta >> 8; 2982 reta = reta | ( ((uint32_t) queue_id) << 24); 2983 if ((i & 3) == 3) { 2984 if (i < 128) 2985 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2986 else 2987 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 2988 reta); 2989 reta = 0; 2990 } 2991 } 2992 2993 /* Now fill our hash function seeds */ 2994 for (i = 0; i < 10; i++) 2995 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 2996 2997 /* 2998 * Disable UDP - IP fragments aren't currently being handled 2999 * and so we end up with a mix of 2-tuple and 4-tuple 3000 * traffic. 3001 */ 3002 mrqc = IXGBE_MRQC_RSSEN 3003 | IXGBE_MRQC_RSS_FIELD_IPV4 3004 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 3005 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 3006 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 3007 | IXGBE_MRQC_RSS_FIELD_IPV6 3008 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 3009 ; 3010 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3011 } 3012 3013 /********************************************************************* 3014 * 3015 * Free all receive rings. 3016 * 3017 **********************************************************************/ 3018 void 3019 ixgbe_free_receive_structures(struct ix_softc *sc) 3020 { 3021 struct rx_ring *rxr; 3022 int i; 3023 3024 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3025 if_rxr_init(&rxr->rx_ring, 0, 0); 3026 3027 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3028 ixgbe_free_receive_buffers(rxr); 3029 } 3030 3031 /********************************************************************* 3032 * 3033 * Free receive ring data structures 3034 * 3035 **********************************************************************/ 3036 void 3037 ixgbe_free_receive_buffers(struct rx_ring *rxr) 3038 { 3039 struct ix_softc *sc; 3040 struct ixgbe_rx_buf *rxbuf; 3041 int i; 3042 3043 sc = rxr->sc; 3044 if (rxr->rx_buffers != NULL) { 3045 for (i = 0; i < sc->num_rx_desc; i++) { 3046 rxbuf = &rxr->rx_buffers[i]; 3047 if (rxbuf->buf != NULL) { 3048 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 3049 0, rxbuf->map->dm_mapsize, 3050 BUS_DMASYNC_POSTREAD); 3051 bus_dmamap_unload(rxr->rxdma.dma_tag, 3052 rxbuf->map); 3053 m_freem(rxbuf->buf); 3054 rxbuf->buf = NULL; 3055 } 3056 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 3057 rxbuf->map = NULL; 3058 } 3059 free(rxr->rx_buffers, M_DEVBUF, 3060 sc->num_rx_desc * sizeof(struct ixgbe_rx_buf)); 3061 rxr->rx_buffers = NULL; 3062 } 3063 } 3064 3065 /********************************************************************* 3066 * 3067 * This routine executes in interrupt context. It replenishes 3068 * the mbufs in the descriptor and sends data which has been 3069 * dma'ed into host memory to upper layer. 3070 * 3071 *********************************************************************/ 3072 int 3073 ixgbe_rxeof(struct rx_ring *rxr) 3074 { 3075 struct ix_softc *sc = rxr->sc; 3076 struct ifnet *ifp = &sc->arpcom.ac_if; 3077 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 3078 struct mbuf *mp, *sendmp; 3079 uint8_t eop = 0; 3080 uint16_t len, vtag; 3081 uint32_t staterr = 0, ptype; 3082 struct ixgbe_rx_buf *rxbuf, *nxbuf; 3083 union ixgbe_adv_rx_desc *rxdesc; 3084 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 3085 int i, nextp; 3086 3087 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3088 return FALSE; 3089 3090 i = rxr->next_to_check; 3091 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 3092 uint32_t hash; 3093 uint16_t hashtype; 3094 3095 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3096 dsize * i, dsize, BUS_DMASYNC_POSTREAD); 3097 3098 rxdesc = &rxr->rx_base[i]; 3099 staterr = letoh32(rxdesc->wb.upper.status_error); 3100 if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) { 3101 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3102 dsize * i, dsize, 3103 BUS_DMASYNC_PREREAD); 3104 break; 3105 } 3106 3107 /* Zero out the receive descriptors status */ 3108 rxdesc->wb.upper.status_error = 0; 3109 rxbuf = &rxr->rx_buffers[i]; 3110 3111 /* pull the mbuf off the ring */ 3112 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 3113 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3114 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 3115 3116 mp = rxbuf->buf; 3117 len = letoh16(rxdesc->wb.upper.length); 3118 ptype = letoh32(rxdesc->wb.lower.lo_dword.data) & 3119 IXGBE_RXDADV_PKTTYPE_MASK; 3120 vtag = letoh16(rxdesc->wb.upper.vlan); 3121 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3122 hash = lemtoh32(&rxdesc->wb.lower.hi_dword.rss); 3123 hashtype = 3124 lemtoh16(&rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 3125 IXGBE_RXDADV_RSSTYPE_MASK; 3126 3127 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) { 3128 if (rxbuf->fmp) { 3129 m_freem(rxbuf->fmp); 3130 rxbuf->fmp = NULL; 3131 } 3132 3133 m_freem(mp); 3134 rxbuf->buf = NULL; 3135 goto next_desc; 3136 } 3137 3138 if (mp == NULL) { 3139 panic("%s: ixgbe_rxeof: NULL mbuf in slot %d " 3140 "(nrx %d, filled %d)", sc->dev.dv_xname, 3141 i, if_rxr_inuse(&rxr->rx_ring), 3142 rxr->last_desc_filled); 3143 } 3144 3145 /* Currently no HW RSC support of 82599 */ 3146 if (!eop) { 3147 /* 3148 * Figure out the next descriptor of this frame. 3149 */ 3150 nextp = i + 1; 3151 if (nextp == sc->num_rx_desc) 3152 nextp = 0; 3153 nxbuf = &rxr->rx_buffers[nextp]; 3154 /* prefetch(nxbuf); */ 3155 } 3156 3157 /* 3158 * Rather than using the fmp/lmp global pointers 3159 * we now keep the head of a packet chain in the 3160 * buffer struct and pass this along from one 3161 * descriptor to the next, until we get EOP. 3162 */ 3163 mp->m_len = len; 3164 /* 3165 * See if there is a stored head 3166 * that determines what we are 3167 */ 3168 sendmp = rxbuf->fmp; 3169 rxbuf->buf = rxbuf->fmp = NULL; 3170 3171 if (sendmp != NULL) /* secondary frag */ 3172 sendmp->m_pkthdr.len += mp->m_len; 3173 else { 3174 /* first desc of a non-ps chain */ 3175 sendmp = mp; 3176 sendmp->m_pkthdr.len = mp->m_len; 3177 #if NVLAN > 0 3178 if (staterr & IXGBE_RXD_STAT_VP) { 3179 sendmp->m_pkthdr.ether_vtag = vtag; 3180 sendmp->m_flags |= M_VLANTAG; 3181 } 3182 #endif 3183 } 3184 3185 /* Pass the head pointer on */ 3186 if (eop == 0) { 3187 nxbuf->fmp = sendmp; 3188 sendmp = NULL; 3189 mp->m_next = nxbuf->buf; 3190 } else { /* Sending this frame? */ 3191 ixgbe_rx_checksum(staterr, sendmp, ptype); 3192 3193 if (hashtype != IXGBE_RXDADV_RSSTYPE_NONE) { 3194 sendmp->m_pkthdr.ph_flowid = hash; 3195 SET(sendmp->m_pkthdr.csum_flags, M_FLOWID); 3196 } 3197 3198 ml_enqueue(&ml, sendmp); 3199 } 3200 next_desc: 3201 if_rxr_put(&rxr->rx_ring, 1); 3202 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3203 dsize * i, dsize, 3204 BUS_DMASYNC_PREREAD); 3205 3206 /* Advance our pointers to the next descriptor. */ 3207 if (++i == sc->num_rx_desc) 3208 i = 0; 3209 } 3210 rxr->next_to_check = i; 3211 3212 if (ifiq_input(rxr->ifiq, &ml)) 3213 if_rxr_livelocked(&rxr->rx_ring); 3214 3215 if (!(staterr & IXGBE_RXD_STAT_DD)) 3216 return FALSE; 3217 3218 return TRUE; 3219 } 3220 3221 /********************************************************************* 3222 * 3223 * Verify that the hardware indicated that the checksum is valid. 3224 * Inform the stack about the status of checksum so that stack 3225 * doesn't spend time verifying the checksum. 3226 * 3227 *********************************************************************/ 3228 void 3229 ixgbe_rx_checksum(uint32_t staterr, struct mbuf * mp, uint32_t ptype) 3230 { 3231 uint16_t status = (uint16_t) staterr; 3232 uint8_t errors = (uint8_t) (staterr >> 24); 3233 3234 if (status & IXGBE_RXD_STAT_IPCS) { 3235 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3236 /* IP Checksum Good */ 3237 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 3238 } else 3239 mp->m_pkthdr.csum_flags = 0; 3240 } 3241 if (status & IXGBE_RXD_STAT_L4CS) { 3242 if (!(errors & IXGBE_RXD_ERR_TCPE)) 3243 mp->m_pkthdr.csum_flags |= 3244 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 3245 } 3246 } 3247 3248 void 3249 ixgbe_setup_vlan_hw_support(struct ix_softc *sc) 3250 { 3251 uint32_t ctrl; 3252 int i; 3253 3254 /* 3255 * A soft reset zero's out the VFTA, so 3256 * we need to repopulate it now. 3257 */ 3258 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 3259 if (sc->shadow_vfta[i] != 0) 3260 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i), 3261 sc->shadow_vfta[i]); 3262 } 3263 3264 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 3265 #if 0 3266 /* Enable the Filter Table if enabled */ 3267 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3268 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3269 ctrl |= IXGBE_VLNCTRL_VFE; 3270 } 3271 #endif 3272 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3273 ctrl |= IXGBE_VLNCTRL_VME; 3274 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 3275 3276 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 3277 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 3278 for (i = 0; i < sc->num_queues; i++) { 3279 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 3280 ctrl |= IXGBE_RXDCTL_VME; 3281 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl); 3282 } 3283 } 3284 } 3285 3286 void 3287 ixgbe_enable_intr(struct ix_softc *sc) 3288 { 3289 struct ixgbe_hw *hw = &sc->hw; 3290 uint32_t mask, fwsm; 3291 3292 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3293 /* Enable Fan Failure detection */ 3294 if (hw->device_id == IXGBE_DEV_ID_82598AT) 3295 mask |= IXGBE_EIMS_GPI_SDP1; 3296 3297 switch (sc->hw.mac.type) { 3298 case ixgbe_mac_82599EB: 3299 mask |= IXGBE_EIMS_ECC; 3300 /* Temperature sensor on some adapters */ 3301 mask |= IXGBE_EIMS_GPI_SDP0; 3302 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 3303 mask |= IXGBE_EIMS_GPI_SDP1; 3304 mask |= IXGBE_EIMS_GPI_SDP2; 3305 break; 3306 case ixgbe_mac_X540: 3307 mask |= IXGBE_EIMS_ECC; 3308 /* Detect if Thermal Sensor is enabled */ 3309 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3310 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3311 mask |= IXGBE_EIMS_TS; 3312 break; 3313 case ixgbe_mac_X550: 3314 case ixgbe_mac_X550EM_x: 3315 case ixgbe_mac_X550EM_a: 3316 mask |= IXGBE_EIMS_ECC; 3317 /* MAC thermal sensor is automatically enabled */ 3318 mask |= IXGBE_EIMS_TS; 3319 /* Some devices use SDP0 for important information */ 3320 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 3321 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 3322 mask |= IXGBE_EIMS_GPI_SDP0_X540; 3323 default: 3324 break; 3325 } 3326 3327 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3328 3329 /* With MSI-X we use auto clear */ 3330 if (sc->sc_intrmap) { 3331 mask = IXGBE_EIMS_ENABLE_MASK; 3332 /* Don't autoclear Link */ 3333 mask &= ~IXGBE_EIMS_OTHER; 3334 mask &= ~IXGBE_EIMS_LSC; 3335 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 3336 } 3337 3338 IXGBE_WRITE_FLUSH(hw); 3339 } 3340 3341 void 3342 ixgbe_disable_intr(struct ix_softc *sc) 3343 { 3344 if (sc->sc_intrmap) 3345 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3346 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3347 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3348 } else { 3349 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3350 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3351 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3352 } 3353 IXGBE_WRITE_FLUSH(&sc->hw); 3354 } 3355 3356 uint16_t 3357 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3358 { 3359 struct pci_attach_args *pa; 3360 uint32_t value; 3361 int high = 0; 3362 3363 if (reg & 0x2) { 3364 high = 1; 3365 reg &= ~0x2; 3366 } 3367 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3368 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3369 3370 if (high) 3371 value >>= 16; 3372 3373 return (value & 0xffff); 3374 } 3375 3376 void 3377 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3378 { 3379 struct pci_attach_args *pa; 3380 uint32_t rv; 3381 int high = 0; 3382 3383 /* Need to do read/mask/write... because 16 vs 32 bit!!! */ 3384 if (reg & 0x2) { 3385 high = 1; 3386 reg &= ~0x2; 3387 } 3388 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3389 rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3390 if (!high) 3391 rv = (rv & 0xffff0000) | value; 3392 else 3393 rv = (rv & 0xffff) | ((uint32_t)value << 16); 3394 pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv); 3395 } 3396 3397 /* 3398 * Setup the correct IVAR register for a particular MSIX interrupt 3399 * (yes this is all very magic and confusing :) 3400 * - entry is the register array entry 3401 * - vector is the MSIX vector for this queue 3402 * - type is RX/TX/MISC 3403 */ 3404 void 3405 ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type) 3406 { 3407 struct ixgbe_hw *hw = &sc->hw; 3408 uint32_t ivar, index; 3409 3410 vector |= IXGBE_IVAR_ALLOC_VAL; 3411 3412 switch (hw->mac.type) { 3413 3414 case ixgbe_mac_82598EB: 3415 if (type == -1) 3416 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3417 else 3418 entry += (type * 64); 3419 index = (entry >> 2) & 0x1F; 3420 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3421 ivar &= ~((uint32_t)0xFF << (8 * (entry & 0x3))); 3422 ivar |= ((uint32_t)vector << (8 * (entry & 0x3))); 3423 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 3424 break; 3425 3426 case ixgbe_mac_82599EB: 3427 case ixgbe_mac_X540: 3428 case ixgbe_mac_X550: 3429 case ixgbe_mac_X550EM_x: 3430 case ixgbe_mac_X550EM_a: 3431 if (type == -1) { /* MISC IVAR */ 3432 index = (entry & 1) * 8; 3433 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3434 ivar &= ~((uint32_t)0xFF << index); 3435 ivar |= ((uint32_t)vector << index); 3436 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3437 } else { /* RX/TX IVARS */ 3438 index = (16 * (entry & 1)) + (8 * type); 3439 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3440 ivar &= ~((uint32_t)0xFF << index); 3441 ivar |= ((uint32_t)vector << index); 3442 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3443 } 3444 3445 default: 3446 break; 3447 } 3448 } 3449 3450 void 3451 ixgbe_configure_ivars(struct ix_softc *sc) 3452 { 3453 struct ix_queue *que = sc->queues; 3454 uint32_t newitr; 3455 int i; 3456 3457 newitr = (4000000 / IXGBE_INTS_PER_SEC) & 0x0FF8; 3458 3459 for (i = 0; i < sc->num_queues; i++, que++) { 3460 /* First the RX queue entry */ 3461 ixgbe_set_ivar(sc, i, que->msix, 0); 3462 /* ... and the TX */ 3463 ixgbe_set_ivar(sc, i, que->msix, 1); 3464 /* Set an Initial EITR value */ 3465 IXGBE_WRITE_REG(&sc->hw, 3466 IXGBE_EITR(que->msix), newitr); 3467 } 3468 3469 /* For the Link interrupt */ 3470 ixgbe_set_ivar(sc, 1, sc->linkvec, -1); 3471 } 3472 3473 /* 3474 * SFP module interrupts handler 3475 */ 3476 void 3477 ixgbe_handle_mod(struct ix_softc *sc) 3478 { 3479 struct ixgbe_hw *hw = &sc->hw; 3480 uint32_t err; 3481 3482 err = hw->phy.ops.identify_sfp(hw); 3483 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3484 printf("%s: Unsupported SFP+ module type was detected!\n", 3485 sc->dev.dv_xname); 3486 return; 3487 } 3488 err = hw->mac.ops.setup_sfp(hw); 3489 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3490 printf("%s: Setup failure - unsupported SFP+ module type!\n", 3491 sc->dev.dv_xname); 3492 return; 3493 } 3494 3495 ixgbe_handle_msf(sc); 3496 } 3497 3498 3499 /* 3500 * MSF (multispeed fiber) interrupts handler 3501 */ 3502 void 3503 ixgbe_handle_msf(struct ix_softc *sc) 3504 { 3505 struct ixgbe_hw *hw = &sc->hw; 3506 uint32_t autoneg; 3507 bool negotiate; 3508 3509 autoneg = hw->phy.autoneg_advertised; 3510 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) { 3511 if (hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate)) 3512 return; 3513 } 3514 if (hw->mac.ops.setup_link) 3515 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3516 3517 ifmedia_delete_instance(&sc->media, IFM_INST_ANY); 3518 ixgbe_add_media_types(sc); 3519 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 3520 } 3521 3522 /* 3523 * External PHY interrupts handler 3524 */ 3525 void 3526 ixgbe_handle_phy(struct ix_softc *sc) 3527 { 3528 struct ixgbe_hw *hw = &sc->hw; 3529 int error; 3530 3531 error = hw->phy.ops.handle_lasi(hw); 3532 if (error == IXGBE_ERR_OVERTEMP) 3533 printf("%s: CRITICAL: EXTERNAL PHY OVER TEMP!! " 3534 " PHY will downshift to lower power state!\n", 3535 sc->dev.dv_xname); 3536 else if (error) 3537 printf("%s: Error handling LASI interrupt: %d\n", 3538 sc->dev.dv_xname, error); 3539 3540 } 3541 3542 #if NKSTAT > 0 3543 enum ix_counter_idx { 3544 ix_counter_crcerrs, 3545 ix_counter_lxontxc, 3546 ix_counter_lxonrxc, 3547 ix_counter_lxofftxc, 3548 ix_counter_lxoffrxc, 3549 ix_counter_prc64, 3550 ix_counter_prc127, 3551 ix_counter_prc255, 3552 ix_counter_prc511, 3553 ix_counter_prc1023, 3554 ix_counter_prc1522, 3555 ix_counter_gptc, 3556 ix_counter_gorc, 3557 ix_counter_gotc, 3558 ix_counter_ruc, 3559 ix_counter_rfc, 3560 ix_counter_roc, 3561 ix_counter_rjc, 3562 ix_counter_tor, 3563 ix_counter_tpr, 3564 ix_counter_tpt, 3565 ix_counter_gprc, 3566 ix_counter_bprc, 3567 ix_counter_mprc, 3568 ix_counter_ptc64, 3569 ix_counter_ptc127, 3570 ix_counter_ptc255, 3571 ix_counter_ptc511, 3572 ix_counter_ptc1023, 3573 ix_counter_ptc1522, 3574 ix_counter_mptc, 3575 ix_counter_bptc, 3576 3577 ix_counter_num, 3578 }; 3579 3580 CTASSERT(KSTAT_KV_U_PACKETS <= 0xff); 3581 CTASSERT(KSTAT_KV_U_BYTES <= 0xff); 3582 3583 struct ix_counter { 3584 char name[KSTAT_KV_NAMELEN]; 3585 uint32_t reg; 3586 uint8_t width; 3587 uint8_t unit; 3588 }; 3589 3590 static const struct ix_counter ix_counters[ix_counter_num] = { 3591 [ix_counter_crcerrs] = { "crc errs", IXGBE_CRCERRS, 32, 3592 KSTAT_KV_U_PACKETS }, 3593 [ix_counter_lxontxc] = { "tx link xon", IXGBE_LXONTXC, 32, 3594 KSTAT_KV_U_PACKETS }, 3595 [ix_counter_lxonrxc] = { "rx link xon", 0, 32, 3596 KSTAT_KV_U_PACKETS }, 3597 [ix_counter_lxofftxc] = { "tx link xoff", IXGBE_LXOFFTXC, 32, 3598 KSTAT_KV_U_PACKETS }, 3599 [ix_counter_lxoffrxc] = { "rx link xoff", 0, 32, 3600 KSTAT_KV_U_PACKETS }, 3601 [ix_counter_prc64] = { "rx 64B", IXGBE_PRC64, 32, 3602 KSTAT_KV_U_PACKETS }, 3603 [ix_counter_prc127] = { "rx 65-127B", IXGBE_PRC127, 32, 3604 KSTAT_KV_U_PACKETS }, 3605 [ix_counter_prc255] = { "rx 128-255B", IXGBE_PRC255, 32, 3606 KSTAT_KV_U_PACKETS }, 3607 [ix_counter_prc511] = { "rx 256-511B", IXGBE_PRC511, 32, 3608 KSTAT_KV_U_PACKETS }, 3609 [ix_counter_prc1023] = { "rx 512-1023B", IXGBE_PRC1023, 32, 3610 KSTAT_KV_U_PACKETS }, 3611 [ix_counter_prc1522] = { "rx 1024-maxB", IXGBE_PRC1522, 32, 3612 KSTAT_KV_U_PACKETS }, 3613 [ix_counter_gptc] = { "tx good", IXGBE_GPTC, 32, 3614 KSTAT_KV_U_PACKETS }, 3615 [ix_counter_gorc] = { "rx good", IXGBE_GORCL, 36, 3616 KSTAT_KV_U_BYTES }, 3617 [ix_counter_gotc] = { "tx good", IXGBE_GOTCL, 36, 3618 KSTAT_KV_U_BYTES }, 3619 [ix_counter_ruc] = { "rx undersize", IXGBE_RUC, 32, 3620 KSTAT_KV_U_PACKETS }, 3621 [ix_counter_rfc] = { "rx fragment", IXGBE_RFC, 32, 3622 KSTAT_KV_U_PACKETS }, 3623 [ix_counter_roc] = { "rx oversize", IXGBE_ROC, 32, 3624 KSTAT_KV_U_PACKETS }, 3625 [ix_counter_rjc] = { "rx jabber", IXGBE_RJC, 32, 3626 KSTAT_KV_U_PACKETS }, 3627 [ix_counter_tor] = { "rx total", IXGBE_TORL, 36, 3628 KSTAT_KV_U_BYTES }, 3629 [ix_counter_tpr] = { "rx total", IXGBE_TPR, 32, 3630 KSTAT_KV_U_PACKETS }, 3631 [ix_counter_tpt] = { "tx total", IXGBE_TPT, 32, 3632 KSTAT_KV_U_PACKETS }, 3633 [ix_counter_gprc] = { "rx good", IXGBE_GPRC, 32, 3634 KSTAT_KV_U_PACKETS }, 3635 [ix_counter_bprc] = { "rx bcast", IXGBE_BPRC, 32, 3636 KSTAT_KV_U_PACKETS }, 3637 [ix_counter_mprc] = { "rx mcast", IXGBE_MPRC, 32, 3638 KSTAT_KV_U_PACKETS }, 3639 [ix_counter_ptc64] = { "tx 64B", IXGBE_PTC64, 32, 3640 KSTAT_KV_U_PACKETS }, 3641 [ix_counter_ptc127] = { "tx 65-127B", IXGBE_PTC127, 32, 3642 KSTAT_KV_U_PACKETS }, 3643 [ix_counter_ptc255] = { "tx 128-255B", IXGBE_PTC255, 32, 3644 KSTAT_KV_U_PACKETS }, 3645 [ix_counter_ptc511] = { "tx 256-511B", IXGBE_PTC511, 32, 3646 KSTAT_KV_U_PACKETS }, 3647 [ix_counter_ptc1023] = { "tx 512-1023B", IXGBE_PTC1023, 32, 3648 KSTAT_KV_U_PACKETS }, 3649 [ix_counter_ptc1522] = { "tx 1024-maxB", IXGBE_PTC1522, 32, 3650 KSTAT_KV_U_PACKETS }, 3651 [ix_counter_mptc] = { "tx mcast", IXGBE_MPTC, 32, 3652 KSTAT_KV_U_PACKETS }, 3653 [ix_counter_bptc] = { "tx bcast", IXGBE_BPTC, 32, 3654 KSTAT_KV_U_PACKETS }, 3655 }; 3656 3657 struct ix_rxq_kstats { 3658 struct kstat_kv qprc; 3659 struct kstat_kv qbrc; 3660 struct kstat_kv qprdc; 3661 }; 3662 3663 static const struct ix_rxq_kstats ix_rxq_kstats_tpl = { 3664 KSTAT_KV_UNIT_INITIALIZER("packets", 3665 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3666 KSTAT_KV_UNIT_INITIALIZER("bytes", 3667 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3668 KSTAT_KV_UNIT_INITIALIZER("qdrops", 3669 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3670 }; 3671 3672 struct ix_txq_kstats { 3673 struct kstat_kv qptc; 3674 struct kstat_kv qbtc; 3675 }; 3676 3677 static const struct ix_txq_kstats ix_txq_kstats_tpl = { 3678 KSTAT_KV_UNIT_INITIALIZER("packets", 3679 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3680 KSTAT_KV_UNIT_INITIALIZER("bytes", 3681 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3682 }; 3683 3684 static int ix_kstats_read(struct kstat *ks); 3685 static int ix_rxq_kstats_read(struct kstat *ks); 3686 static int ix_txq_kstats_read(struct kstat *ks); 3687 3688 static void 3689 ix_kstats(struct ix_softc *sc) 3690 { 3691 struct kstat *ks; 3692 struct kstat_kv *kvs; 3693 unsigned int i; 3694 3695 mtx_init(&sc->sc_kstat_mtx, IPL_SOFTCLOCK); 3696 timeout_set(&sc->sc_kstat_tmo, ix_kstats_tick, sc); 3697 3698 ks = kstat_create(sc->dev.dv_xname, 0, "ix-stats", 0, 3699 KSTAT_T_KV, 0); 3700 if (ks == NULL) 3701 return; 3702 3703 kvs = mallocarray(nitems(ix_counters), sizeof(*kvs), 3704 M_DEVBUF, M_WAITOK|M_ZERO); 3705 3706 for (i = 0; i < nitems(ix_counters); i++) { 3707 const struct ix_counter *ixc = &ix_counters[i]; 3708 3709 kstat_kv_unit_init(&kvs[i], ixc->name, 3710 KSTAT_KV_T_COUNTER64, ixc->unit); 3711 } 3712 3713 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3714 ks->ks_softc = sc; 3715 ks->ks_data = kvs; 3716 ks->ks_datalen = nitems(ix_counters) * sizeof(*kvs); 3717 ks->ks_read = ix_kstats_read; 3718 3719 sc->sc_kstat = ks; 3720 kstat_install(ks); 3721 } 3722 3723 static void 3724 ix_rxq_kstats(struct ix_softc *sc, struct rx_ring *rxr) 3725 { 3726 struct ix_rxq_kstats *stats; 3727 struct kstat *ks; 3728 3729 ks = kstat_create(sc->dev.dv_xname, 0, "ix-rxq", rxr->me, 3730 KSTAT_T_KV, 0); 3731 if (ks == NULL) 3732 return; 3733 3734 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3735 *stats = ix_rxq_kstats_tpl; 3736 3737 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3738 ks->ks_softc = rxr; 3739 ks->ks_data = stats; 3740 ks->ks_datalen = sizeof(*stats); 3741 ks->ks_read = ix_rxq_kstats_read; 3742 3743 rxr->kstat = ks; 3744 kstat_install(ks); 3745 } 3746 3747 static void 3748 ix_txq_kstats(struct ix_softc *sc, struct tx_ring *txr) 3749 { 3750 struct ix_txq_kstats *stats; 3751 struct kstat *ks; 3752 3753 ks = kstat_create(sc->dev.dv_xname, 0, "ix-txq", txr->me, 3754 KSTAT_T_KV, 0); 3755 if (ks == NULL) 3756 return; 3757 3758 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3759 *stats = ix_txq_kstats_tpl; 3760 3761 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3762 ks->ks_softc = txr; 3763 ks->ks_data = stats; 3764 ks->ks_datalen = sizeof(*stats); 3765 ks->ks_read = ix_txq_kstats_read; 3766 3767 txr->kstat = ks; 3768 kstat_install(ks); 3769 } 3770 3771 /********************************************************************** 3772 * 3773 * Update the board statistics counters. 3774 * 3775 **********************************************************************/ 3776 3777 static void 3778 ix_kstats_tick(void *arg) 3779 { 3780 struct ix_softc *sc = arg; 3781 int i; 3782 3783 timeout_add_sec(&sc->sc_kstat_tmo, 1); 3784 3785 mtx_enter(&sc->sc_kstat_mtx); 3786 ix_kstats_read(sc->sc_kstat); 3787 for (i = 0; i < sc->num_queues; i++) { 3788 ix_rxq_kstats_read(sc->rx_rings[i].kstat); 3789 ix_txq_kstats_read(sc->tx_rings[i].kstat); 3790 } 3791 mtx_leave(&sc->sc_kstat_mtx); 3792 } 3793 3794 static uint64_t 3795 ix_read36(struct ixgbe_hw *hw, bus_size_t loreg, bus_size_t hireg) 3796 { 3797 uint64_t lo, hi; 3798 3799 lo = IXGBE_READ_REG(hw, loreg); 3800 hi = IXGBE_READ_REG(hw, hireg); 3801 3802 return (((hi & 0xf) << 32) | lo); 3803 } 3804 3805 static int 3806 ix_kstats_read(struct kstat *ks) 3807 { 3808 struct ix_softc *sc = ks->ks_softc; 3809 struct kstat_kv *kvs = ks->ks_data; 3810 struct ixgbe_hw *hw = &sc->hw; 3811 unsigned int i; 3812 3813 for (i = 0; i < nitems(ix_counters); i++) { 3814 const struct ix_counter *ixc = &ix_counters[i]; 3815 uint32_t reg = ixc->reg; 3816 uint64_t v; 3817 3818 if (reg == 0) 3819 continue; 3820 3821 if (ixc->width > 32) { 3822 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3823 v = IXGBE_READ_REG(hw, reg + 4); 3824 else 3825 v = ix_read36(hw, reg, reg + 4); 3826 } else 3827 v = IXGBE_READ_REG(hw, reg); 3828 3829 kstat_kv_u64(&kvs[i]) += v; 3830 } 3831 3832 /* handle the exceptions */ 3833 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3834 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3835 IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3836 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3837 IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3838 } else { 3839 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3840 IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3841 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3842 IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3843 } 3844 3845 getnanouptime(&ks->ks_updated); 3846 3847 return (0); 3848 } 3849 3850 int 3851 ix_rxq_kstats_read(struct kstat *ks) 3852 { 3853 struct ix_rxq_kstats *stats = ks->ks_data; 3854 struct rx_ring *rxr = ks->ks_softc; 3855 struct ix_softc *sc = rxr->sc; 3856 struct ixgbe_hw *hw = &sc->hw; 3857 uint32_t i = rxr->me; 3858 3859 kstat_kv_u64(&stats->qprc) += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3860 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3861 kstat_kv_u64(&stats->qprdc) += 3862 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3863 kstat_kv_u64(&stats->qbrc) += 3864 IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 3865 } else { 3866 kstat_kv_u64(&stats->qprdc) += 3867 IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3868 kstat_kv_u64(&stats->qbrc) += 3869 ix_read36(hw, IXGBE_QBRC_L(i), IXGBE_QBRC_H(i)); 3870 } 3871 3872 getnanouptime(&ks->ks_updated); 3873 3874 return (0); 3875 } 3876 3877 int 3878 ix_txq_kstats_read(struct kstat *ks) 3879 { 3880 struct ix_txq_kstats *stats = ks->ks_data; 3881 struct tx_ring *txr = ks->ks_softc; 3882 struct ix_softc *sc = txr->sc; 3883 struct ixgbe_hw *hw = &sc->hw; 3884 uint32_t i = txr->me; 3885 3886 kstat_kv_u64(&stats->qptc) += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3887 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3888 kstat_kv_u64(&stats->qbtc) += 3889 IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 3890 } else { 3891 kstat_kv_u64(&stats->qbtc) += 3892 ix_read36(hw, IXGBE_QBTC_L(i), IXGBE_QBTC_H(i)); 3893 } 3894 3895 getnanouptime(&ks->ks_updated); 3896 3897 return (0); 3898 } 3899 #endif /* NKVSTAT > 0 */ 3900 3901 void 3902 ixgbe_map_queue_statistics(struct ix_softc *sc) 3903 { 3904 int i; 3905 uint32_t r; 3906 3907 for (i = 0; i < 32; i++) { 3908 /* 3909 * Queues 0-15 are mapped 1:1 3910 * Queue 0 -> Counter 0 3911 * Queue 1 -> Counter 1 3912 * Queue 2 -> Counter 2.... 3913 * Queues 16-127 are mapped to Counter 0 3914 */ 3915 if (i < 4) { 3916 r = (i * 4 + 0); 3917 r |= (i * 4 + 1) << 8; 3918 r |= (i * 4 + 2) << 16; 3919 r |= (i * 4 + 3) << 24; 3920 } else 3921 r = 0; 3922 3923 IXGBE_WRITE_REG(&sc->hw, IXGBE_RQSMR(i), r); 3924 IXGBE_WRITE_REG(&sc->hw, IXGBE_TQSM(i), r); 3925 } 3926 } 3927