1 /* $OpenBSD: if_ix.c,v 1.186 2022/06/27 15:11:23 jan Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2013, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */ 36 37 #include <dev/pci/if_ix.h> 38 #include <dev/pci/ixgbe_type.h> 39 40 /********************************************************************* 41 * Driver version 42 *********************************************************************/ 43 /* char ixgbe_driver_version[] = "2.5.13"; */ 44 45 /********************************************************************* 46 * PCI Device ID Table 47 * 48 * Used by probe to select devices to load on 49 *********************************************************************/ 50 51 const struct pci_matchid ixgbe_devices[] = { 52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 }, 53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX }, 54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT }, 57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT2 }, 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP }, 63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM }, 64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL }, 65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 }, 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BP }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_T3_LOM }, 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF_QP }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF2 }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_FCOE }, 77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599EN_SFP }, 78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_QSFP_SF_QP }, 79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T }, 80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T1 }, 81 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T }, 82 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T1 }, 83 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KX4 }, 84 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KR }, 85 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_SFP }, 86 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_10G_T }, 87 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_1G_T }, 88 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR }, 89 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR_L }, 90 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP_N }, 91 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP }, 92 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII }, 93 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII_L }, 94 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_10G_T }, 95 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T }, 96 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T_L } 97 }; 98 99 /********************************************************************* 100 * Function prototypes 101 *********************************************************************/ 102 int ixgbe_probe(struct device *, void *, void *); 103 void ixgbe_attach(struct device *, struct device *, void *); 104 int ixgbe_detach(struct device *, int); 105 int ixgbe_activate(struct device *, int); 106 void ixgbe_start(struct ifqueue *); 107 int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 108 int ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *); 109 int ixgbe_get_sffpage(struct ix_softc *, struct if_sffpage *); 110 void ixgbe_watchdog(struct ifnet *); 111 void ixgbe_init(void *); 112 void ixgbe_stop(void *); 113 void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 114 int ixgbe_media_change(struct ifnet *); 115 void ixgbe_identify_hardware(struct ix_softc *); 116 int ixgbe_allocate_pci_resources(struct ix_softc *); 117 int ixgbe_allocate_legacy(struct ix_softc *); 118 int ixgbe_allocate_msix(struct ix_softc *); 119 void ixgbe_setup_msix(struct ix_softc *); 120 int ixgbe_allocate_queues(struct ix_softc *); 121 void ixgbe_free_pci_resources(struct ix_softc *); 122 void ixgbe_local_timer(void *); 123 void ixgbe_setup_interface(struct ix_softc *); 124 void ixgbe_config_gpie(struct ix_softc *); 125 void ixgbe_config_delay_values(struct ix_softc *); 126 void ixgbe_add_media_types(struct ix_softc *); 127 void ixgbe_config_link(struct ix_softc *); 128 129 int ixgbe_allocate_transmit_buffers(struct tx_ring *); 130 int ixgbe_setup_transmit_structures(struct ix_softc *); 131 int ixgbe_setup_transmit_ring(struct tx_ring *); 132 void ixgbe_initialize_transmit_units(struct ix_softc *); 133 void ixgbe_free_transmit_structures(struct ix_softc *); 134 void ixgbe_free_transmit_buffers(struct tx_ring *); 135 136 int ixgbe_allocate_receive_buffers(struct rx_ring *); 137 int ixgbe_setup_receive_structures(struct ix_softc *); 138 int ixgbe_setup_receive_ring(struct rx_ring *); 139 void ixgbe_initialize_receive_units(struct ix_softc *); 140 void ixgbe_free_receive_structures(struct ix_softc *); 141 void ixgbe_free_receive_buffers(struct rx_ring *); 142 void ixgbe_initialize_rss_mapping(struct ix_softc *); 143 int ixgbe_rxfill(struct rx_ring *); 144 void ixgbe_rxrefill(void *); 145 146 int ixgbe_intr(struct ix_softc *sc); 147 void ixgbe_enable_intr(struct ix_softc *); 148 void ixgbe_disable_intr(struct ix_softc *); 149 int ixgbe_txeof(struct tx_ring *); 150 int ixgbe_rxeof(struct rx_ring *); 151 void ixgbe_rx_checksum(uint32_t, struct mbuf *, uint32_t); 152 void ixgbe_iff(struct ix_softc *); 153 void ixgbe_map_queue_statistics(struct ix_softc *); 154 void ixgbe_update_link_status(struct ix_softc *); 155 int ixgbe_get_buf(struct rx_ring *, int); 156 int ixgbe_encap(struct tx_ring *, struct mbuf *); 157 int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 158 struct ixgbe_dma_alloc *, int); 159 void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 160 static int 161 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *, uint32_t *, 162 uint32_t *); 163 void ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 164 void ixgbe_configure_ivars(struct ix_softc *); 165 uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 166 167 void ixgbe_setup_vlan_hw_support(struct ix_softc *); 168 169 /* Support for pluggable optic modules */ 170 void ixgbe_handle_mod(struct ix_softc *); 171 void ixgbe_handle_msf(struct ix_softc *); 172 void ixgbe_handle_phy(struct ix_softc *); 173 174 /* Legacy (single vector interrupt handler */ 175 int ixgbe_legacy_intr(void *); 176 void ixgbe_enable_queue(struct ix_softc *, uint32_t); 177 void ixgbe_enable_queues(struct ix_softc *); 178 void ixgbe_disable_queue(struct ix_softc *, uint32_t); 179 void ixgbe_rearm_queue(struct ix_softc *, uint32_t); 180 181 /* MSI-X (multiple vectors interrupt handlers) */ 182 int ixgbe_link_intr(void *); 183 int ixgbe_queue_intr(void *); 184 185 #if NKSTAT > 0 186 static void ix_kstats(struct ix_softc *); 187 static void ix_rxq_kstats(struct ix_softc *, struct rx_ring *); 188 static void ix_txq_kstats(struct ix_softc *, struct tx_ring *); 189 static void ix_kstats_tick(void *); 190 #endif 191 192 /********************************************************************* 193 * OpenBSD Device Interface Entry Points 194 *********************************************************************/ 195 196 struct cfdriver ix_cd = { 197 NULL, "ix", DV_IFNET 198 }; 199 200 const struct cfattach ix_ca = { 201 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach, 202 ixgbe_activate 203 }; 204 205 int ixgbe_smart_speed = ixgbe_smart_speed_on; 206 int ixgbe_enable_msix = 1; 207 208 /********************************************************************* 209 * Device identification routine 210 * 211 * ixgbe_probe determines if the driver should be loaded on 212 * adapter based on PCI vendor/device id of the adapter. 213 * 214 * return 0 on success, positive on failure 215 *********************************************************************/ 216 217 int 218 ixgbe_probe(struct device *parent, void *match, void *aux) 219 { 220 INIT_DEBUGOUT("ixgbe_probe: begin"); 221 222 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 223 nitems(ixgbe_devices))); 224 } 225 226 /********************************************************************* 227 * Device initialization routine 228 * 229 * The attach entry point is called when the driver is being loaded. 230 * This routine identifies the type of hardware, allocates all resources 231 * and initializes the hardware. 232 * 233 * return 0 on success, positive on failure 234 *********************************************************************/ 235 236 void 237 ixgbe_attach(struct device *parent, struct device *self, void *aux) 238 { 239 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 240 struct ix_softc *sc = (struct ix_softc *)self; 241 int error = 0; 242 uint16_t csum; 243 uint32_t ctrl_ext; 244 struct ixgbe_hw *hw = &sc->hw; 245 246 INIT_DEBUGOUT("ixgbe_attach: begin"); 247 248 sc->osdep.os_sc = sc; 249 sc->osdep.os_pa = *pa; 250 251 rw_init(&sc->sfflock, "ixsff"); 252 253 #if NKSTAT > 0 254 ix_kstats(sc); 255 #endif 256 257 /* Determine hardware revision */ 258 ixgbe_identify_hardware(sc); 259 260 /* Indicate to RX setup to use Jumbo Clusters */ 261 sc->num_tx_desc = DEFAULT_TXD; 262 sc->num_rx_desc = DEFAULT_RXD; 263 264 /* Do base PCI setup - map BAR0 */ 265 if (ixgbe_allocate_pci_resources(sc)) 266 goto err_out; 267 268 /* Allocate our TX/RX Queues */ 269 if (ixgbe_allocate_queues(sc)) 270 goto err_out; 271 272 /* Allocate multicast array memory. */ 273 sc->mta = mallocarray(IXGBE_ETH_LENGTH_OF_ADDRESS, 274 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 275 if (sc->mta == NULL) { 276 printf(": Can not allocate multicast setup array\n"); 277 goto err_late; 278 } 279 280 /* Initialize the shared code */ 281 error = ixgbe_init_shared_code(hw); 282 if (error) { 283 printf(": Unable to initialize the shared code\n"); 284 goto err_late; 285 } 286 287 /* Make sure we have a good EEPROM before we read from it */ 288 if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) { 289 printf(": The EEPROM Checksum Is Not Valid\n"); 290 goto err_late; 291 } 292 293 error = ixgbe_init_hw(hw); 294 if (error == IXGBE_ERR_EEPROM_VERSION) { 295 printf(": This device is a pre-production adapter/" 296 "LOM. Please be aware there may be issues associated " 297 "with your hardware.\nIf you are experiencing problems " 298 "please contact your Intel or hardware representative " 299 "who provided you with this hardware.\n"); 300 } else if (error && (error != IXGBE_ERR_SFP_NOT_PRESENT && 301 error != IXGBE_ERR_SFP_NOT_SUPPORTED)) { 302 printf(": Hardware Initialization Failure\n"); 303 goto err_late; 304 } 305 306 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 307 IXGBE_ETH_LENGTH_OF_ADDRESS); 308 309 if (sc->sc_intrmap) 310 error = ixgbe_allocate_msix(sc); 311 else 312 error = ixgbe_allocate_legacy(sc); 313 if (error) 314 goto err_late; 315 316 /* Enable the optics for 82599 SFP+ fiber */ 317 if (sc->hw.mac.ops.enable_tx_laser) 318 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 319 320 /* Enable power to the phy */ 321 if (hw->phy.ops.set_phy_power) 322 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 323 324 /* Setup OS specific network interface */ 325 ixgbe_setup_interface(sc); 326 327 /* Get the PCI-E bus info and determine LAN ID */ 328 hw->mac.ops.get_bus_info(hw); 329 330 /* Set an initial default flow control value */ 331 sc->fc = ixgbe_fc_full; 332 333 /* let hardware know driver is loaded */ 334 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 335 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 336 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 337 338 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 339 340 INIT_DEBUGOUT("ixgbe_attach: end"); 341 return; 342 343 err_late: 344 ixgbe_free_transmit_structures(sc); 345 ixgbe_free_receive_structures(sc); 346 err_out: 347 ixgbe_free_pci_resources(sc); 348 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 349 MAX_NUM_MULTICAST_ADDRESSES); 350 } 351 352 /********************************************************************* 353 * Device removal routine 354 * 355 * The detach entry point is called when the driver is being removed. 356 * This routine stops the adapter and deallocates all the resources 357 * that were allocated for driver operation. 358 * 359 * return 0 on success, positive on failure 360 *********************************************************************/ 361 362 int 363 ixgbe_detach(struct device *self, int flags) 364 { 365 struct ix_softc *sc = (struct ix_softc *)self; 366 struct ifnet *ifp = &sc->arpcom.ac_if; 367 uint32_t ctrl_ext; 368 369 INIT_DEBUGOUT("ixgbe_detach: begin"); 370 371 ixgbe_stop(sc); 372 373 /* let hardware know driver is unloading */ 374 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 375 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 376 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 377 378 ether_ifdetach(ifp); 379 if_detach(ifp); 380 381 ixgbe_free_pci_resources(sc); 382 383 ixgbe_free_transmit_structures(sc); 384 ixgbe_free_receive_structures(sc); 385 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 386 MAX_NUM_MULTICAST_ADDRESSES); 387 388 /* XXX kstat */ 389 390 return (0); 391 } 392 393 int 394 ixgbe_activate(struct device *self, int act) 395 { 396 struct ix_softc *sc = (struct ix_softc *)self; 397 struct ifnet *ifp = &sc->arpcom.ac_if; 398 struct ixgbe_hw *hw = &sc->hw; 399 uint32_t ctrl_ext; 400 int rv = 0; 401 402 switch (act) { 403 case DVACT_QUIESCE: 404 if (ifp->if_flags & IFF_RUNNING) 405 ixgbe_stop(sc); 406 break; 407 case DVACT_RESUME: 408 ixgbe_init_hw(hw); 409 410 /* Enable the optics for 82599 SFP+ fiber */ 411 if (sc->hw.mac.ops.enable_tx_laser) 412 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 413 414 /* Enable power to the phy */ 415 if (hw->phy.ops.set_phy_power) 416 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 417 418 /* Get the PCI-E bus info and determine LAN ID */ 419 hw->mac.ops.get_bus_info(hw); 420 421 /* let hardware know driver is loaded */ 422 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 423 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 424 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 425 426 if (ifp->if_flags & IFF_UP) 427 ixgbe_init(sc); 428 break; 429 default: 430 break; 431 } 432 return (rv); 433 } 434 435 /********************************************************************* 436 * Transmit entry point 437 * 438 * ixgbe_start is called by the stack to initiate a transmit. 439 * The driver will remain in this routine as long as there are 440 * packets to transmit and transmit resources are available. 441 * In case resources are not available stack is notified and 442 * the packet is requeued. 443 **********************************************************************/ 444 445 void 446 ixgbe_start(struct ifqueue *ifq) 447 { 448 struct ifnet *ifp = ifq->ifq_if; 449 struct ix_softc *sc = ifp->if_softc; 450 struct tx_ring *txr = ifq->ifq_softc; 451 struct mbuf *m_head; 452 unsigned int head, free, used; 453 int post = 0; 454 455 if (!sc->link_up) 456 return; 457 458 head = txr->next_avail_desc; 459 free = txr->next_to_clean; 460 if (free <= head) 461 free += sc->num_tx_desc; 462 free -= head; 463 464 membar_consumer(); 465 466 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 467 0, txr->txdma.dma_map->dm_mapsize, 468 BUS_DMASYNC_POSTWRITE); 469 470 for (;;) { 471 /* Check that we have the minimal number of TX descriptors. */ 472 if (free <= IXGBE_TX_OP_THRESHOLD) { 473 ifq_set_oactive(ifq); 474 break; 475 } 476 477 m_head = ifq_dequeue(ifq); 478 if (m_head == NULL) 479 break; 480 481 used = ixgbe_encap(txr, m_head); 482 if (used == 0) { 483 m_freem(m_head); 484 continue; 485 } 486 487 free -= used; 488 489 #if NBPFILTER > 0 490 if (ifp->if_bpf) 491 bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 492 #endif 493 494 /* Set timeout in case hardware has problems transmitting */ 495 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 496 ifp->if_timer = IXGBE_TX_TIMEOUT; 497 498 post = 1; 499 } 500 501 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 502 0, txr->txdma.dma_map->dm_mapsize, 503 BUS_DMASYNC_PREWRITE); 504 505 /* 506 * Advance the Transmit Descriptor Tail (Tdt), this tells the 507 * hardware that this frame is available to transmit. 508 */ 509 if (post) 510 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 511 txr->next_avail_desc); 512 } 513 514 /********************************************************************* 515 * Ioctl entry point 516 * 517 * ixgbe_ioctl is called when the user wants to configure the 518 * interface. 519 * 520 * return 0 on success, positive on failure 521 **********************************************************************/ 522 523 int 524 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 525 { 526 struct ix_softc *sc = ifp->if_softc; 527 struct ifreq *ifr = (struct ifreq *) data; 528 int s, error = 0; 529 530 s = splnet(); 531 532 switch (command) { 533 case SIOCSIFADDR: 534 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 535 ifp->if_flags |= IFF_UP; 536 if (!(ifp->if_flags & IFF_RUNNING)) 537 ixgbe_init(sc); 538 break; 539 540 case SIOCSIFFLAGS: 541 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 542 if (ifp->if_flags & IFF_UP) { 543 if (ifp->if_flags & IFF_RUNNING) 544 error = ENETRESET; 545 else 546 ixgbe_init(sc); 547 } else { 548 if (ifp->if_flags & IFF_RUNNING) 549 ixgbe_stop(sc); 550 } 551 break; 552 553 case SIOCSIFMEDIA: 554 case SIOCGIFMEDIA: 555 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 556 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 557 break; 558 559 case SIOCGIFRXR: 560 error = ixgbe_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 561 break; 562 563 case SIOCGIFSFFPAGE: 564 error = rw_enter(&sc->sfflock, RW_WRITE|RW_INTR); 565 if (error != 0) 566 break; 567 568 error = ixgbe_get_sffpage(sc, (struct if_sffpage *)data); 569 rw_exit(&sc->sfflock); 570 break; 571 572 default: 573 error = ether_ioctl(ifp, &sc->arpcom, command, data); 574 } 575 576 if (error == ENETRESET) { 577 if (ifp->if_flags & IFF_RUNNING) { 578 ixgbe_disable_intr(sc); 579 ixgbe_iff(sc); 580 ixgbe_enable_intr(sc); 581 ixgbe_enable_queues(sc); 582 } 583 error = 0; 584 } 585 586 splx(s); 587 return (error); 588 } 589 590 int 591 ixgbe_get_sffpage(struct ix_softc *sc, struct if_sffpage *sff) 592 { 593 struct ixgbe_hw *hw = &sc->hw; 594 uint32_t swfw_mask = hw->phy.phy_semaphore_mask; 595 uint8_t page; 596 size_t i; 597 int error = EIO; 598 599 if (hw->phy.type == ixgbe_phy_fw) 600 return (ENODEV); 601 602 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 603 return (EBUSY); /* XXX */ 604 605 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 606 if (hw->phy.ops.read_i2c_byte_unlocked(hw, 127, 607 IFSFF_ADDR_EEPROM, &page)) 608 goto error; 609 if (page != sff->sff_page && 610 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 611 IFSFF_ADDR_EEPROM, sff->sff_page)) 612 goto error; 613 } 614 615 for (i = 0; i < sizeof(sff->sff_data); i++) { 616 if (hw->phy.ops.read_i2c_byte_unlocked(hw, i, 617 sff->sff_addr, &sff->sff_data[i])) 618 goto error; 619 } 620 621 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 622 if (page != sff->sff_page && 623 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 624 IFSFF_ADDR_EEPROM, page)) 625 goto error; 626 } 627 628 error = 0; 629 error: 630 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 631 return (error); 632 } 633 634 int 635 ixgbe_rxrinfo(struct ix_softc *sc, struct if_rxrinfo *ifri) 636 { 637 struct if_rxring_info *ifr, ifr1; 638 struct rx_ring *rxr; 639 int error, i; 640 u_int n = 0; 641 642 if (sc->num_queues > 1) { 643 if ((ifr = mallocarray(sc->num_queues, sizeof(*ifr), M_DEVBUF, 644 M_WAITOK | M_ZERO)) == NULL) 645 return (ENOMEM); 646 } else 647 ifr = &ifr1; 648 649 for (i = 0; i < sc->num_queues; i++) { 650 rxr = &sc->rx_rings[i]; 651 ifr[n].ifr_size = MCLBYTES; 652 snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i); 653 ifr[n].ifr_info = rxr->rx_ring; 654 n++; 655 } 656 657 error = if_rxr_info_ioctl(ifri, sc->num_queues, ifr); 658 659 if (sc->num_queues > 1) 660 free(ifr, M_DEVBUF, sc->num_queues * sizeof(*ifr)); 661 return (error); 662 } 663 664 /********************************************************************* 665 * Watchdog entry point 666 * 667 **********************************************************************/ 668 669 void 670 ixgbe_watchdog(struct ifnet * ifp) 671 { 672 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 673 struct tx_ring *txr = sc->tx_rings; 674 struct ixgbe_hw *hw = &sc->hw; 675 int tx_hang = FALSE; 676 int i; 677 678 /* 679 * The timer is set to 5 every time ixgbe_start() queues a packet. 680 * Anytime all descriptors are clean the timer is set to 0. 681 */ 682 for (i = 0; i < sc->num_queues; i++, txr++) { 683 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 684 continue; 685 else { 686 tx_hang = TRUE; 687 break; 688 } 689 } 690 if (tx_hang == FALSE) 691 return; 692 693 /* 694 * If we are in this routine because of pause frames, then don't 695 * reset the hardware. 696 */ 697 if (!(IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXON)) { 698 for (i = 0; i < sc->num_queues; i++, txr++) 699 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 700 ifp->if_timer = IXGBE_TX_TIMEOUT; 701 return; 702 } 703 704 705 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 706 for (i = 0; i < sc->num_queues; i++, txr++) { 707 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 708 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 709 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 710 printf("%s: TX(%d) Next TX to Clean = %d\n", ifp->if_xname, 711 i, txr->next_to_clean); 712 } 713 ifp->if_flags &= ~IFF_RUNNING; 714 715 ixgbe_init(sc); 716 } 717 718 /********************************************************************* 719 * Init entry point 720 * 721 * This routine is used in two ways. It is used by the stack as 722 * init entry point in network interface structure. It is also used 723 * by the driver as a hw/sw initialization routine to get to a 724 * consistent state. 725 * 726 * return 0 on success, positive on failure 727 **********************************************************************/ 728 #define IXGBE_MHADD_MFS_SHIFT 16 729 730 void 731 ixgbe_init(void *arg) 732 { 733 struct ix_softc *sc = (struct ix_softc *)arg; 734 struct ifnet *ifp = &sc->arpcom.ac_if; 735 struct rx_ring *rxr = sc->rx_rings; 736 uint32_t k, txdctl, rxdctl, rxctrl, mhadd, itr; 737 int i, s, err; 738 739 INIT_DEBUGOUT("ixgbe_init: begin"); 740 741 s = splnet(); 742 743 ixgbe_stop(sc); 744 745 /* reprogram the RAR[0] in case user changed it. */ 746 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 747 748 /* Get the latest mac address, User can use a LAA */ 749 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 750 IXGBE_ETH_LENGTH_OF_ADDRESS); 751 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1); 752 sc->hw.addr_ctrl.rar_used_count = 1; 753 754 /* Prepare transmit descriptors and buffers */ 755 if (ixgbe_setup_transmit_structures(sc)) { 756 printf("%s: Could not setup transmit structures\n", 757 ifp->if_xname); 758 ixgbe_stop(sc); 759 splx(s); 760 return; 761 } 762 763 ixgbe_init_hw(&sc->hw); 764 ixgbe_initialize_transmit_units(sc); 765 766 /* Use 2k clusters, even for jumbo frames */ 767 sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN; 768 769 /* Prepare receive descriptors and buffers */ 770 if (ixgbe_setup_receive_structures(sc)) { 771 printf("%s: Could not setup receive structures\n", 772 ifp->if_xname); 773 ixgbe_stop(sc); 774 splx(s); 775 return; 776 } 777 778 /* Configure RX settings */ 779 ixgbe_initialize_receive_units(sc); 780 781 /* Enable SDP & MSIX interrupts based on adapter */ 782 ixgbe_config_gpie(sc); 783 784 /* Program promiscuous mode and multicast filters. */ 785 ixgbe_iff(sc); 786 787 /* Set MRU size */ 788 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 789 mhadd &= ~IXGBE_MHADD_MFS_MASK; 790 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 791 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 792 793 /* Now enable all the queues */ 794 for (i = 0; i < sc->num_queues; i++) { 795 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 796 txdctl |= IXGBE_TXDCTL_ENABLE; 797 /* Set WTHRESH to 8, burst writeback */ 798 txdctl |= (8 << 16); 799 /* 800 * When the internal queue falls below PTHRESH (16), 801 * start prefetching as long as there are at least 802 * HTHRESH (1) buffers ready. 803 */ 804 txdctl |= (16 << 0) | (1 << 8); 805 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 806 } 807 808 for (i = 0; i < sc->num_queues; i++) { 809 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 810 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 811 /* 812 * PTHRESH = 21 813 * HTHRESH = 4 814 * WTHRESH = 8 815 */ 816 rxdctl &= ~0x3FFFFF; 817 rxdctl |= 0x080420; 818 } 819 rxdctl |= IXGBE_RXDCTL_ENABLE; 820 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 821 for (k = 0; k < 10; k++) { 822 if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) & 823 IXGBE_RXDCTL_ENABLE) 824 break; 825 else 826 msec_delay(1); 827 } 828 IXGBE_WRITE_FLUSH(&sc->hw); 829 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled); 830 } 831 832 /* Set up VLAN support and filter */ 833 ixgbe_setup_vlan_hw_support(sc); 834 835 /* Enable Receive engine */ 836 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 837 if (sc->hw.mac.type == ixgbe_mac_82598EB) 838 rxctrl |= IXGBE_RXCTRL_DMBYPS; 839 rxctrl |= IXGBE_RXCTRL_RXEN; 840 sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl); 841 842 /* Set up MSI/X routing */ 843 if (sc->sc_intrmap) { 844 ixgbe_configure_ivars(sc); 845 /* Set up auto-mask */ 846 if (sc->hw.mac.type == ixgbe_mac_82598EB) 847 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 848 else { 849 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 850 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 851 } 852 } else { /* Simple settings for Legacy/MSI */ 853 ixgbe_set_ivar(sc, 0, 0, 0); 854 ixgbe_set_ivar(sc, 0, 0, 1); 855 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 856 } 857 858 /* Check on any SFP devices that need to be kick-started */ 859 if (sc->hw.phy.type == ixgbe_phy_none) { 860 err = sc->hw.phy.ops.identify(&sc->hw); 861 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 862 printf("Unsupported SFP+ module type was detected.\n"); 863 splx(s); 864 return; 865 } 866 } 867 868 /* Setup interrupt moderation */ 869 itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8; 870 if (sc->hw.mac.type != ixgbe_mac_82598EB) 871 itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS; 872 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr); 873 874 if (sc->sc_intrmap) { 875 /* Set moderation on the Link interrupt */ 876 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(sc->linkvec), 877 IXGBE_LINK_ITR); 878 } 879 880 /* Enable power to the phy */ 881 if (sc->hw.phy.ops.set_phy_power) 882 sc->hw.phy.ops.set_phy_power(&sc->hw, TRUE); 883 884 /* Config/Enable Link */ 885 ixgbe_config_link(sc); 886 887 /* Hardware Packet Buffer & Flow Control setup */ 888 ixgbe_config_delay_values(sc); 889 890 /* Initialize the FC settings */ 891 sc->hw.mac.ops.start_hw(&sc->hw); 892 893 /* And now turn on interrupts */ 894 ixgbe_enable_intr(sc); 895 ixgbe_enable_queues(sc); 896 897 /* Now inform the stack we're ready */ 898 ifp->if_flags |= IFF_RUNNING; 899 for (i = 0; i < sc->num_queues; i++) 900 ifq_clr_oactive(ifp->if_ifqs[i]); 901 902 #if NKSTAT > 0 903 ix_kstats_tick(sc); 904 #endif 905 906 splx(s); 907 } 908 909 void 910 ixgbe_config_gpie(struct ix_softc *sc) 911 { 912 struct ixgbe_hw *hw = &sc->hw; 913 uint32_t gpie; 914 915 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 916 917 /* Fan Failure Interrupt */ 918 if (hw->device_id == IXGBE_DEV_ID_82598AT) 919 gpie |= IXGBE_SDP1_GPIEN; 920 921 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 922 /* Add for Module detection */ 923 gpie |= IXGBE_SDP2_GPIEN; 924 925 /* Media ready */ 926 if (hw->device_id != IXGBE_DEV_ID_82599_QSFP_SF_QP) 927 gpie |= IXGBE_SDP1_GPIEN; 928 929 /* 930 * Set LL interval to max to reduce the number of low latency 931 * interrupts hitting the card when the ring is getting full. 932 */ 933 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 934 } 935 936 if (sc->hw.mac.type == ixgbe_mac_X540 || 937 sc->hw.mac.type == ixgbe_mac_X550EM_x || 938 sc->hw.mac.type == ixgbe_mac_X550EM_a) { 939 /* 940 * Thermal Failure Detection (X540) 941 * Link Detection (X552 SFP+, X552/X557-AT) 942 */ 943 gpie |= IXGBE_SDP0_GPIEN_X540; 944 945 /* 946 * Set LL interval to max to reduce the number of low latency 947 * interrupts hitting the card when the ring is getting full. 948 */ 949 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 950 } 951 952 if (sc->sc_intrmap) { 953 /* Enable Enhanced MSIX mode */ 954 gpie |= IXGBE_GPIE_MSIX_MODE; 955 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 956 IXGBE_GPIE_OCD; 957 } 958 959 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 960 } 961 962 /* 963 * Requires sc->max_frame_size to be set. 964 */ 965 void 966 ixgbe_config_delay_values(struct ix_softc *sc) 967 { 968 struct ixgbe_hw *hw = &sc->hw; 969 uint32_t rxpb, frame, size, tmp; 970 971 frame = sc->max_frame_size; 972 973 /* Calculate High Water */ 974 switch (hw->mac.type) { 975 case ixgbe_mac_X540: 976 case ixgbe_mac_X550: 977 case ixgbe_mac_X550EM_x: 978 case ixgbe_mac_X550EM_a: 979 tmp = IXGBE_DV_X540(frame, frame); 980 break; 981 default: 982 tmp = IXGBE_DV(frame, frame); 983 break; 984 } 985 size = IXGBE_BT2KB(tmp); 986 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 987 hw->fc.high_water[0] = rxpb - size; 988 989 /* Now calculate Low Water */ 990 switch (hw->mac.type) { 991 case ixgbe_mac_X540: 992 case ixgbe_mac_X550: 993 case ixgbe_mac_X550EM_x: 994 case ixgbe_mac_X550EM_a: 995 tmp = IXGBE_LOW_DV_X540(frame); 996 break; 997 default: 998 tmp = IXGBE_LOW_DV(frame); 999 break; 1000 } 1001 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 1002 1003 hw->fc.requested_mode = sc->fc; 1004 hw->fc.pause_time = IXGBE_FC_PAUSE; 1005 hw->fc.send_xon = TRUE; 1006 } 1007 1008 /* 1009 * MSIX Interrupt Handlers 1010 */ 1011 void 1012 ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector) 1013 { 1014 uint64_t queue = 1ULL << vector; 1015 uint32_t mask; 1016 1017 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1018 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1019 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask); 1020 } else { 1021 mask = (queue & 0xFFFFFFFF); 1022 if (mask) 1023 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask); 1024 mask = (queue >> 32); 1025 if (mask) 1026 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask); 1027 } 1028 } 1029 1030 void 1031 ixgbe_enable_queues(struct ix_softc *sc) 1032 { 1033 struct ix_queue *que; 1034 int i; 1035 1036 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 1037 ixgbe_enable_queue(sc, que->msix); 1038 } 1039 1040 void 1041 ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector) 1042 { 1043 uint64_t queue = 1ULL << vector; 1044 uint32_t mask; 1045 1046 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1047 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1048 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask); 1049 } else { 1050 mask = (queue & 0xFFFFFFFF); 1051 if (mask) 1052 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask); 1053 mask = (queue >> 32); 1054 if (mask) 1055 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask); 1056 } 1057 } 1058 1059 /* 1060 * MSIX Interrupt Handlers 1061 */ 1062 int 1063 ixgbe_link_intr(void *vsc) 1064 { 1065 struct ix_softc *sc = (struct ix_softc *)vsc; 1066 1067 return ixgbe_intr(sc); 1068 } 1069 1070 int 1071 ixgbe_queue_intr(void *vque) 1072 { 1073 struct ix_queue *que = vque; 1074 struct ix_softc *sc = que->sc; 1075 struct ifnet *ifp = &sc->arpcom.ac_if; 1076 struct rx_ring *rxr = que->rxr; 1077 struct tx_ring *txr = que->txr; 1078 1079 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1080 ixgbe_rxeof(rxr); 1081 ixgbe_txeof(txr); 1082 ixgbe_rxrefill(rxr); 1083 } 1084 1085 ixgbe_enable_queue(sc, que->msix); 1086 1087 return (1); 1088 } 1089 1090 /********************************************************************* 1091 * 1092 * Legacy Interrupt Service routine 1093 * 1094 **********************************************************************/ 1095 1096 int 1097 ixgbe_legacy_intr(void *arg) 1098 { 1099 struct ix_softc *sc = (struct ix_softc *)arg; 1100 struct ifnet *ifp = &sc->arpcom.ac_if; 1101 struct rx_ring *rxr = sc->rx_rings; 1102 struct tx_ring *txr = sc->tx_rings; 1103 int rv; 1104 1105 rv = ixgbe_intr(sc); 1106 if (rv == 0) { 1107 return (0); 1108 } 1109 1110 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1111 ixgbe_rxeof(rxr); 1112 ixgbe_txeof(txr); 1113 ixgbe_rxrefill(rxr); 1114 } 1115 1116 ixgbe_enable_queues(sc); 1117 return (rv); 1118 } 1119 1120 int 1121 ixgbe_intr(struct ix_softc *sc) 1122 { 1123 struct ifnet *ifp = &sc->arpcom.ac_if; 1124 struct ixgbe_hw *hw = &sc->hw; 1125 uint32_t reg_eicr, mod_mask, msf_mask; 1126 1127 if (sc->sc_intrmap) { 1128 /* Pause other interrupts */ 1129 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER); 1130 /* First get the cause */ 1131 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1132 /* Be sure the queue bits are not cleared */ 1133 reg_eicr &= ~IXGBE_EICR_RTX_QUEUE; 1134 /* Clear interrupt with write */ 1135 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1136 } else { 1137 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1138 if (reg_eicr == 0) { 1139 ixgbe_enable_intr(sc); 1140 ixgbe_enable_queues(sc); 1141 return (0); 1142 } 1143 } 1144 1145 /* Link status change */ 1146 if (reg_eicr & IXGBE_EICR_LSC) { 1147 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); 1148 KERNEL_LOCK(); 1149 ixgbe_update_link_status(sc); 1150 KERNEL_UNLOCK(); 1151 } 1152 1153 if (hw->mac.type != ixgbe_mac_82598EB) { 1154 if (reg_eicr & IXGBE_EICR_ECC) { 1155 printf("%s: CRITICAL: ECC ERROR!! " 1156 "Please Reboot!!\n", sc->dev.dv_xname); 1157 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1158 } 1159 /* Check for over temp condition */ 1160 if (reg_eicr & IXGBE_EICR_TS) { 1161 printf("%s: CRITICAL: OVER TEMP!! " 1162 "PHY IS SHUT DOWN!!\n", ifp->if_xname); 1163 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS); 1164 } 1165 } 1166 1167 /* Pluggable optics-related interrupt */ 1168 if (ixgbe_is_sfp(hw)) { 1169 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP) { 1170 mod_mask = IXGBE_EICR_GPI_SDP0_X540; 1171 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1172 } else if (hw->mac.type == ixgbe_mac_X540 || 1173 hw->mac.type == ixgbe_mac_X550 || 1174 hw->mac.type == ixgbe_mac_X550EM_x) { 1175 mod_mask = IXGBE_EICR_GPI_SDP2_X540; 1176 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1177 } else { 1178 mod_mask = IXGBE_EICR_GPI_SDP2; 1179 msf_mask = IXGBE_EICR_GPI_SDP1; 1180 } 1181 if (reg_eicr & mod_mask) { 1182 /* Clear the interrupt */ 1183 IXGBE_WRITE_REG(hw, IXGBE_EICR, mod_mask); 1184 KERNEL_LOCK(); 1185 ixgbe_handle_mod(sc); 1186 KERNEL_UNLOCK(); 1187 } else if ((hw->phy.media_type != ixgbe_media_type_copper) && 1188 (reg_eicr & msf_mask)) { 1189 /* Clear the interrupt */ 1190 IXGBE_WRITE_REG(hw, IXGBE_EICR, msf_mask); 1191 KERNEL_LOCK(); 1192 ixgbe_handle_msf(sc); 1193 KERNEL_UNLOCK(); 1194 } 1195 } 1196 1197 /* Check for fan failure */ 1198 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1199 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1200 printf("%s: CRITICAL: FAN FAILURE!! " 1201 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 1202 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1203 } 1204 1205 /* External PHY interrupt */ 1206 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 1207 (reg_eicr & IXGBE_EICR_GPI_SDP0_X540)) { 1208 /* Clear the interrupt */ 1209 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540); 1210 KERNEL_LOCK(); 1211 ixgbe_handle_phy(sc); 1212 KERNEL_UNLOCK(); 1213 } 1214 1215 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 1216 1217 return (1); 1218 } 1219 1220 /********************************************************************* 1221 * 1222 * Media Ioctl callback 1223 * 1224 * This routine is called whenever the user queries the status of 1225 * the interface using ifconfig. 1226 * 1227 **********************************************************************/ 1228 void 1229 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq *ifmr) 1230 { 1231 struct ix_softc *sc = ifp->if_softc; 1232 uint64_t layer; 1233 1234 ifmr->ifm_active = IFM_ETHER; 1235 ifmr->ifm_status = IFM_AVALID; 1236 1237 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1238 ixgbe_update_link_status(sc); 1239 1240 if (!LINK_STATE_IS_UP(ifp->if_link_state)) 1241 return; 1242 1243 ifmr->ifm_status |= IFM_ACTIVE; 1244 layer = sc->phy_layer; 1245 1246 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 1247 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 1248 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 1249 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1250 switch (sc->link_speed) { 1251 case IXGBE_LINK_SPEED_10GB_FULL: 1252 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1253 break; 1254 case IXGBE_LINK_SPEED_1GB_FULL: 1255 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1256 break; 1257 case IXGBE_LINK_SPEED_100_FULL: 1258 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1259 break; 1260 case IXGBE_LINK_SPEED_10_FULL: 1261 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1262 break; 1263 } 1264 } 1265 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1266 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1267 switch (sc->link_speed) { 1268 case IXGBE_LINK_SPEED_10GB_FULL: 1269 ifmr->ifm_active |= IFM_10G_SFP_CU | IFM_FDX; 1270 break; 1271 } 1272 } 1273 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1274 switch (sc->link_speed) { 1275 case IXGBE_LINK_SPEED_10GB_FULL: 1276 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 1277 break; 1278 case IXGBE_LINK_SPEED_1GB_FULL: 1279 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1280 break; 1281 } 1282 } 1283 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 1284 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1285 switch (sc->link_speed) { 1286 case IXGBE_LINK_SPEED_10GB_FULL: 1287 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1288 break; 1289 case IXGBE_LINK_SPEED_1GB_FULL: 1290 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1291 break; 1292 } 1293 } 1294 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1295 switch (sc->link_speed) { 1296 case IXGBE_LINK_SPEED_10GB_FULL: 1297 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1298 break; 1299 } 1300 } 1301 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1302 switch (sc->link_speed) { 1303 case IXGBE_LINK_SPEED_10GB_FULL: 1304 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 1305 break; 1306 case IXGBE_LINK_SPEED_2_5GB_FULL: 1307 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1308 break; 1309 case IXGBE_LINK_SPEED_1GB_FULL: 1310 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1311 break; 1312 } 1313 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 1314 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 1315 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1316 switch (sc->link_speed) { 1317 case IXGBE_LINK_SPEED_10GB_FULL: 1318 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 1319 break; 1320 case IXGBE_LINK_SPEED_2_5GB_FULL: 1321 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1322 break; 1323 case IXGBE_LINK_SPEED_1GB_FULL: 1324 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1325 break; 1326 } 1327 } 1328 1329 switch (sc->hw.fc.current_mode) { 1330 case ixgbe_fc_tx_pause: 1331 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 1332 break; 1333 case ixgbe_fc_rx_pause: 1334 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 1335 break; 1336 case ixgbe_fc_full: 1337 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE | 1338 IFM_ETH_TXPAUSE; 1339 break; 1340 default: 1341 ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE | 1342 IFM_ETH_TXPAUSE); 1343 break; 1344 } 1345 } 1346 1347 /********************************************************************* 1348 * 1349 * Media Ioctl callback 1350 * 1351 * This routine is called when the user changes speed/duplex using 1352 * media/mediopt option with ifconfig. 1353 * 1354 **********************************************************************/ 1355 int 1356 ixgbe_media_change(struct ifnet *ifp) 1357 { 1358 struct ix_softc *sc = ifp->if_softc; 1359 struct ixgbe_hw *hw = &sc->hw; 1360 struct ifmedia *ifm = &sc->media; 1361 ixgbe_link_speed speed = 0; 1362 1363 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1364 return (EINVAL); 1365 1366 if (hw->phy.media_type == ixgbe_media_type_backplane) 1367 return (ENODEV); 1368 1369 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1370 case IFM_AUTO: 1371 case IFM_10G_T: 1372 speed |= IXGBE_LINK_SPEED_100_FULL; 1373 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1374 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1375 break; 1376 case IFM_10G_SR: 1377 case IFM_10G_KR: 1378 case IFM_10G_LR: 1379 case IFM_10G_LRM: 1380 case IFM_10G_CX4: 1381 case IFM_10G_KX4: 1382 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1383 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1384 break; 1385 case IFM_10G_SFP_CU: 1386 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1387 break; 1388 case IFM_1000_T: 1389 speed |= IXGBE_LINK_SPEED_100_FULL; 1390 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1391 break; 1392 case IFM_1000_LX: 1393 case IFM_1000_SX: 1394 case IFM_1000_CX: 1395 case IFM_1000_KX: 1396 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1397 break; 1398 case IFM_100_TX: 1399 speed |= IXGBE_LINK_SPEED_100_FULL; 1400 break; 1401 case IFM_10_T: 1402 speed |= IXGBE_LINK_SPEED_10_FULL; 1403 break; 1404 default: 1405 return (EINVAL); 1406 } 1407 1408 hw->mac.autotry_restart = TRUE; 1409 hw->mac.ops.setup_link(hw, speed, TRUE); 1410 1411 return (0); 1412 } 1413 1414 /********************************************************************* 1415 * 1416 * This routine maps the mbufs to tx descriptors, allowing the 1417 * TX engine to transmit the packets. 1418 * - return 0 on success, positive on failure 1419 * 1420 **********************************************************************/ 1421 1422 int 1423 ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head) 1424 { 1425 struct ix_softc *sc = txr->sc; 1426 uint32_t olinfo_status = 0, cmd_type_len; 1427 int i, j, ntxc; 1428 int first, last = 0; 1429 bus_dmamap_t map; 1430 struct ixgbe_tx_buf *txbuf; 1431 union ixgbe_adv_tx_desc *txd = NULL; 1432 1433 /* Basic descriptor defines */ 1434 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1435 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1436 1437 /* 1438 * Important to capture the first descriptor 1439 * used because it will contain the index of 1440 * the one we tell the hardware to report back 1441 */ 1442 first = txr->next_avail_desc; 1443 txbuf = &txr->tx_buffers[first]; 1444 map = txbuf->map; 1445 1446 /* 1447 * Set the appropriate offload context 1448 * this will becomes the first descriptor. 1449 */ 1450 ntxc = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 1451 if (ntxc == -1) 1452 goto xmit_fail; 1453 1454 /* 1455 * Map the packet for DMA. 1456 */ 1457 switch (bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1458 m_head, BUS_DMA_NOWAIT)) { 1459 case 0: 1460 break; 1461 case EFBIG: 1462 if (m_defrag(m_head, M_NOWAIT) == 0 && 1463 bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1464 m_head, BUS_DMA_NOWAIT) == 0) 1465 break; 1466 /* FALLTHROUGH */ 1467 default: 1468 return (0); 1469 } 1470 1471 i = txr->next_avail_desc + ntxc; 1472 if (i >= sc->num_tx_desc) 1473 i -= sc->num_tx_desc; 1474 1475 for (j = 0; j < map->dm_nsegs; j++) { 1476 txd = &txr->tx_base[i]; 1477 1478 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 1479 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1480 cmd_type_len | map->dm_segs[j].ds_len); 1481 txd->read.olinfo_status = htole32(olinfo_status); 1482 last = i; /* descriptor that will get completion IRQ */ 1483 1484 if (++i == sc->num_tx_desc) 1485 i = 0; 1486 } 1487 1488 txd->read.cmd_type_len |= 1489 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1490 1491 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1492 BUS_DMASYNC_PREWRITE); 1493 1494 /* Set the index of the descriptor that will be marked done */ 1495 txbuf->m_head = m_head; 1496 txbuf->eop_index = last; 1497 1498 membar_producer(); 1499 1500 txr->next_avail_desc = i; 1501 1502 return (ntxc + j); 1503 1504 xmit_fail: 1505 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 1506 return (0); 1507 } 1508 1509 void 1510 ixgbe_iff(struct ix_softc *sc) 1511 { 1512 struct ifnet *ifp = &sc->arpcom.ac_if; 1513 struct arpcom *ac = &sc->arpcom; 1514 uint32_t fctrl; 1515 uint8_t *mta; 1516 uint8_t *update_ptr; 1517 struct ether_multi *enm; 1518 struct ether_multistep step; 1519 int mcnt = 0; 1520 1521 IOCTL_DEBUGOUT("ixgbe_iff: begin"); 1522 1523 mta = sc->mta; 1524 bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1525 MAX_NUM_MULTICAST_ADDRESSES); 1526 1527 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1528 fctrl &= ~(IXGBE_FCTRL_MPE | IXGBE_FCTRL_UPE); 1529 ifp->if_flags &= ~IFF_ALLMULTI; 1530 1531 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1532 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1533 ifp->if_flags |= IFF_ALLMULTI; 1534 fctrl |= IXGBE_FCTRL_MPE; 1535 if (ifp->if_flags & IFF_PROMISC) 1536 fctrl |= IXGBE_FCTRL_UPE; 1537 } else { 1538 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1539 while (enm != NULL) { 1540 bcopy(enm->enm_addrlo, 1541 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1542 IXGBE_ETH_LENGTH_OF_ADDRESS); 1543 mcnt++; 1544 1545 ETHER_NEXT_MULTI(step, enm); 1546 } 1547 1548 update_ptr = mta; 1549 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1550 ixgbe_mc_array_itr, TRUE); 1551 } 1552 1553 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1554 } 1555 1556 /* 1557 * This is an iterator function now needed by the multicast 1558 * shared code. It simply feeds the shared code routine the 1559 * addresses in the array of ixgbe_iff() one by one. 1560 */ 1561 uint8_t * 1562 ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1563 { 1564 uint8_t *addr = *update_ptr; 1565 uint8_t *newptr; 1566 *vmdq = 0; 1567 1568 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1569 *update_ptr = newptr; 1570 return addr; 1571 } 1572 1573 void 1574 ixgbe_update_link_status(struct ix_softc *sc) 1575 { 1576 struct ifnet *ifp = &sc->arpcom.ac_if; 1577 int link_state = LINK_STATE_DOWN; 1578 1579 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 1580 1581 ifp->if_baudrate = 0; 1582 if (sc->link_up) { 1583 link_state = LINK_STATE_FULL_DUPLEX; 1584 1585 switch (sc->link_speed) { 1586 case IXGBE_LINK_SPEED_UNKNOWN: 1587 ifp->if_baudrate = 0; 1588 break; 1589 case IXGBE_LINK_SPEED_100_FULL: 1590 ifp->if_baudrate = IF_Mbps(100); 1591 break; 1592 case IXGBE_LINK_SPEED_1GB_FULL: 1593 ifp->if_baudrate = IF_Gbps(1); 1594 break; 1595 case IXGBE_LINK_SPEED_10GB_FULL: 1596 ifp->if_baudrate = IF_Gbps(10); 1597 break; 1598 } 1599 1600 /* Update any Flow Control changes */ 1601 sc->hw.mac.ops.fc_enable(&sc->hw); 1602 } 1603 if (ifp->if_link_state != link_state) { 1604 ifp->if_link_state = link_state; 1605 if_link_state_change(ifp); 1606 } 1607 } 1608 1609 1610 /********************************************************************* 1611 * 1612 * This routine disables all traffic on the adapter by issuing a 1613 * global reset on the MAC and deallocates TX/RX buffers. 1614 * 1615 **********************************************************************/ 1616 1617 void 1618 ixgbe_stop(void *arg) 1619 { 1620 struct ix_softc *sc = arg; 1621 struct ifnet *ifp = &sc->arpcom.ac_if; 1622 int i; 1623 1624 /* Tell the stack that the interface is no longer active */ 1625 ifp->if_flags &= ~IFF_RUNNING; 1626 1627 #if NKSTAT > 0 1628 timeout_del(&sc->sc_kstat_tmo); 1629 #endif 1630 ifp->if_timer = 0; 1631 1632 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1633 ixgbe_disable_intr(sc); 1634 1635 sc->hw.mac.ops.reset_hw(&sc->hw); 1636 sc->hw.adapter_stopped = FALSE; 1637 sc->hw.mac.ops.stop_adapter(&sc->hw); 1638 if (sc->hw.mac.type == ixgbe_mac_82599EB) 1639 sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw); 1640 /* Turn off the laser */ 1641 if (sc->hw.mac.ops.disable_tx_laser) 1642 sc->hw.mac.ops.disable_tx_laser(&sc->hw); 1643 1644 /* reprogram the RAR[0] in case user changed it. */ 1645 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1646 1647 intr_barrier(sc->tag); 1648 for (i = 0; i < sc->num_queues; i++) { 1649 struct ifqueue *ifq = ifp->if_ifqs[i]; 1650 ifq_barrier(ifq); 1651 ifq_clr_oactive(ifq); 1652 1653 if (sc->queues[i].tag != NULL) 1654 intr_barrier(sc->queues[i].tag); 1655 timeout_del(&sc->rx_rings[i].rx_refill); 1656 } 1657 1658 KASSERT((ifp->if_flags & IFF_RUNNING) == 0); 1659 1660 /* Should we really clear all structures on stop? */ 1661 ixgbe_free_transmit_structures(sc); 1662 ixgbe_free_receive_structures(sc); 1663 1664 ixgbe_update_link_status(sc); 1665 } 1666 1667 1668 /********************************************************************* 1669 * 1670 * Determine hardware revision. 1671 * 1672 **********************************************************************/ 1673 void 1674 ixgbe_identify_hardware(struct ix_softc *sc) 1675 { 1676 struct ixgbe_osdep *os = &sc->osdep; 1677 struct pci_attach_args *pa = &os->os_pa; 1678 uint32_t reg; 1679 1680 /* Save off the information about this board */ 1681 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1682 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1683 1684 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1685 sc->hw.revision_id = PCI_REVISION(reg); 1686 1687 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1688 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1689 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1690 1691 /* We need this here to set the num_segs below */ 1692 ixgbe_set_mac_type(&sc->hw); 1693 1694 /* Pick up the 82599 and VF settings */ 1695 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1696 sc->hw.phy.smart_speed = ixgbe_smart_speed; 1697 sc->num_segs = IXGBE_82599_SCATTER; 1698 } 1699 1700 /********************************************************************* 1701 * 1702 * Setup the Legacy or MSI Interrupt handler 1703 * 1704 **********************************************************************/ 1705 int 1706 ixgbe_allocate_legacy(struct ix_softc *sc) 1707 { 1708 struct ixgbe_osdep *os = &sc->osdep; 1709 struct pci_attach_args *pa = &os->os_pa; 1710 const char *intrstr = NULL; 1711 pci_chipset_tag_t pc = pa->pa_pc; 1712 pci_intr_handle_t ih; 1713 1714 /* We allocate a single interrupt resource */ 1715 if (pci_intr_map_msi(pa, &ih) != 0 && 1716 pci_intr_map(pa, &ih) != 0) { 1717 printf(": couldn't map interrupt\n"); 1718 return (ENXIO); 1719 } 1720 1721 #if 0 1722 /* XXX */ 1723 /* Tasklets for Link, SFP and Multispeed Fiber */ 1724 TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc); 1725 TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc); 1726 TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc); 1727 #endif 1728 1729 intrstr = pci_intr_string(pc, ih); 1730 sc->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE, 1731 ixgbe_legacy_intr, sc, sc->dev.dv_xname); 1732 if (sc->tag == NULL) { 1733 printf(": couldn't establish interrupt"); 1734 if (intrstr != NULL) 1735 printf(" at %s", intrstr); 1736 printf("\n"); 1737 return (ENXIO); 1738 } 1739 printf(": %s", intrstr); 1740 1741 /* For simplicity in the handlers */ 1742 sc->que_mask = IXGBE_EIMS_ENABLE_MASK; 1743 1744 return (0); 1745 } 1746 1747 /********************************************************************* 1748 * 1749 * Setup the MSI-X Interrupt handlers 1750 * 1751 **********************************************************************/ 1752 int 1753 ixgbe_allocate_msix(struct ix_softc *sc) 1754 { 1755 struct ixgbe_osdep *os = &sc->osdep; 1756 struct pci_attach_args *pa = &os->os_pa; 1757 int i = 0, error = 0; 1758 struct ix_queue *que; 1759 pci_intr_handle_t ih; 1760 1761 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) { 1762 if (pci_intr_map_msix(pa, i, &ih)) { 1763 printf("ixgbe_allocate_msix: " 1764 "pci_intr_map_msix vec %d failed\n", i); 1765 error = ENOMEM; 1766 goto fail; 1767 } 1768 1769 que->tag = pci_intr_establish_cpu(pa->pa_pc, ih, 1770 IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i), 1771 ixgbe_queue_intr, que, que->name); 1772 if (que->tag == NULL) { 1773 printf("ixgbe_allocate_msix: " 1774 "pci_intr_establish vec %d failed\n", i); 1775 error = ENOMEM; 1776 goto fail; 1777 } 1778 1779 que->msix = i; 1780 } 1781 1782 /* Now the link status/control last MSI-X vector */ 1783 if (pci_intr_map_msix(pa, i, &ih)) { 1784 printf("ixgbe_allocate_msix: " 1785 "pci_intr_map_msix link vector failed\n"); 1786 error = ENOMEM; 1787 goto fail; 1788 } 1789 1790 sc->tag = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE, 1791 ixgbe_link_intr, sc, sc->dev.dv_xname); 1792 if (sc->tag == NULL) { 1793 printf("ixgbe_allocate_msix: " 1794 "pci_intr_establish link vector failed\n"); 1795 error = ENOMEM; 1796 goto fail; 1797 } 1798 sc->linkvec = i; 1799 printf(", %s, %d queue%s", pci_intr_string(pa->pa_pc, ih), 1800 i, (i > 1) ? "s" : ""); 1801 1802 return (0); 1803 fail: 1804 for (que = sc->queues; i > 0; i--, que++) { 1805 if (que->tag == NULL) 1806 continue; 1807 pci_intr_disestablish(pa->pa_pc, que->tag); 1808 que->tag = NULL; 1809 } 1810 1811 return (error); 1812 } 1813 1814 void 1815 ixgbe_setup_msix(struct ix_softc *sc) 1816 { 1817 struct ixgbe_osdep *os = &sc->osdep; 1818 struct pci_attach_args *pa = &os->os_pa; 1819 int nmsix; 1820 unsigned int maxq; 1821 1822 if (!ixgbe_enable_msix) 1823 return; 1824 1825 nmsix = pci_intr_msix_count(pa); 1826 if (nmsix <= 1) 1827 return; 1828 1829 /* give one vector to events */ 1830 nmsix--; 1831 1832 /* XXX the number of queues is limited to what we can keep stats on */ 1833 maxq = (sc->hw.mac.type == ixgbe_mac_82598EB) ? 8 : 16; 1834 1835 sc->sc_intrmap = intrmap_create(&sc->dev, nmsix, maxq, 0); 1836 sc->num_queues = intrmap_count(sc->sc_intrmap); 1837 } 1838 1839 int 1840 ixgbe_allocate_pci_resources(struct ix_softc *sc) 1841 { 1842 struct ixgbe_osdep *os = &sc->osdep; 1843 struct pci_attach_args *pa = &os->os_pa; 1844 int val; 1845 1846 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1847 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) { 1848 printf(": mmba is not mem space\n"); 1849 return (ENXIO); 1850 } 1851 1852 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1853 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1854 printf(": cannot find mem space\n"); 1855 return (ENXIO); 1856 } 1857 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1858 1859 /* Legacy defaults */ 1860 sc->num_queues = 1; 1861 sc->hw.back = os; 1862 1863 /* Now setup MSI or MSI/X, return us the number of supported vectors. */ 1864 ixgbe_setup_msix(sc); 1865 1866 return (0); 1867 } 1868 1869 void 1870 ixgbe_free_pci_resources(struct ix_softc * sc) 1871 { 1872 struct ixgbe_osdep *os = &sc->osdep; 1873 struct pci_attach_args *pa = &os->os_pa; 1874 struct ix_queue *que = sc->queues; 1875 int i; 1876 1877 /* Release all msix queue resources: */ 1878 for (i = 0; i < sc->num_queues; i++, que++) { 1879 if (que->tag) 1880 pci_intr_disestablish(pa->pa_pc, que->tag); 1881 que->tag = NULL; 1882 } 1883 1884 if (sc->tag) 1885 pci_intr_disestablish(pa->pa_pc, sc->tag); 1886 sc->tag = NULL; 1887 if (os->os_membase != 0) 1888 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1889 os->os_membase = 0; 1890 } 1891 1892 /********************************************************************* 1893 * 1894 * Setup networking device structure and register an interface. 1895 * 1896 **********************************************************************/ 1897 void 1898 ixgbe_setup_interface(struct ix_softc *sc) 1899 { 1900 struct ifnet *ifp = &sc->arpcom.ac_if; 1901 int i; 1902 1903 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1904 ifp->if_softc = sc; 1905 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1906 ifp->if_xflags = IFXF_MPSAFE; 1907 ifp->if_ioctl = ixgbe_ioctl; 1908 ifp->if_qstart = ixgbe_start; 1909 ifp->if_timer = 0; 1910 ifp->if_watchdog = ixgbe_watchdog; 1911 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1912 ETHER_HDR_LEN - ETHER_CRC_LEN; 1913 ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1); 1914 1915 ifp->if_capabilities = IFCAP_VLAN_MTU; 1916 1917 #if NVLAN > 0 1918 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1919 #endif 1920 1921 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 1922 ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6; 1923 ifp->if_capabilities |= IFCAP_CSUM_IPv4; 1924 1925 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1926 ifp->if_capabilities |= IFCAP_TSO; 1927 1928 /* 1929 * Specify the media types supported by this sc and register 1930 * callbacks to update media and link information 1931 */ 1932 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1933 ixgbe_media_status); 1934 ixgbe_add_media_types(sc); 1935 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1936 1937 if_attach(ifp); 1938 ether_ifattach(ifp); 1939 1940 if_attach_queues(ifp, sc->num_queues); 1941 if_attach_iqueues(ifp, sc->num_queues); 1942 for (i = 0; i < sc->num_queues; i++) { 1943 struct ifqueue *ifq = ifp->if_ifqs[i]; 1944 struct ifiqueue *ifiq = ifp->if_iqs[i]; 1945 struct tx_ring *txr = &sc->tx_rings[i]; 1946 struct rx_ring *rxr = &sc->rx_rings[i]; 1947 1948 ifq->ifq_softc = txr; 1949 txr->ifq = ifq; 1950 1951 ifiq->ifiq_softc = rxr; 1952 rxr->ifiq = ifiq; 1953 1954 #if NKSTAT > 0 1955 ix_txq_kstats(sc, txr); 1956 ix_rxq_kstats(sc, rxr); 1957 #endif 1958 } 1959 1960 sc->max_frame_size = IXGBE_MAX_FRAME_SIZE; 1961 } 1962 1963 void 1964 ixgbe_add_media_types(struct ix_softc *sc) 1965 { 1966 struct ixgbe_hw *hw = &sc->hw; 1967 uint64_t layer; 1968 1969 sc->phy_layer = hw->mac.ops.get_supported_physical_layer(hw); 1970 layer = sc->phy_layer; 1971 1972 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1973 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL); 1974 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1975 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 1976 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1977 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1978 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1979 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 1980 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SFP_CU, 0, NULL); 1981 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1982 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL); 1983 if (hw->phy.multispeed_fiber) 1984 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_LX, 0, 1985 NULL); 1986 } 1987 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1988 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 1989 if (hw->phy.multispeed_fiber) 1990 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, 1991 NULL); 1992 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 1993 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 1994 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 1995 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); 1996 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 1997 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL); 1998 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) 1999 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); 2000 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 2001 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL); 2002 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) 2003 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL); 2004 2005 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2006 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, 2007 NULL); 2008 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 2009 } 2010 2011 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2012 } 2013 2014 void 2015 ixgbe_config_link(struct ix_softc *sc) 2016 { 2017 uint32_t autoneg, err = 0; 2018 bool negotiate; 2019 2020 if (ixgbe_is_sfp(&sc->hw)) { 2021 if (sc->hw.phy.multispeed_fiber) { 2022 sc->hw.mac.ops.setup_sfp(&sc->hw); 2023 if (sc->hw.mac.ops.enable_tx_laser) 2024 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 2025 ixgbe_handle_msf(sc); 2026 } else 2027 ixgbe_handle_mod(sc); 2028 } else { 2029 if (sc->hw.mac.ops.check_link) 2030 err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg, 2031 &sc->link_up, FALSE); 2032 if (err) 2033 return; 2034 autoneg = sc->hw.phy.autoneg_advertised; 2035 if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities)) 2036 err = sc->hw.mac.ops.get_link_capabilities(&sc->hw, 2037 &autoneg, &negotiate); 2038 if (err) 2039 return; 2040 if (sc->hw.mac.ops.setup_link) 2041 sc->hw.mac.ops.setup_link(&sc->hw, 2042 autoneg, sc->link_up); 2043 } 2044 } 2045 2046 /******************************************************************** 2047 * Manage DMA'able memory. 2048 *******************************************************************/ 2049 int 2050 ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 2051 struct ixgbe_dma_alloc *dma, int mapflags) 2052 { 2053 struct ifnet *ifp = &sc->arpcom.ac_if; 2054 struct ixgbe_osdep *os = &sc->osdep; 2055 int r; 2056 2057 dma->dma_tag = os->os_pa.pa_dmat; 2058 r = bus_dmamap_create(dma->dma_tag, size, 1, 2059 size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 2060 if (r != 0) { 2061 printf("%s: ixgbe_dma_malloc: bus_dmamap_create failed; " 2062 "error %u\n", ifp->if_xname, r); 2063 goto fail_0; 2064 } 2065 2066 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 2067 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 2068 if (r != 0) { 2069 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2070 "error %u\n", ifp->if_xname, r); 2071 goto fail_1; 2072 } 2073 2074 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 2075 &dma->dma_vaddr, BUS_DMA_NOWAIT); 2076 if (r != 0) { 2077 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 2078 "error %u\n", ifp->if_xname, r); 2079 goto fail_2; 2080 } 2081 2082 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2083 size, NULL, mapflags | BUS_DMA_NOWAIT); 2084 if (r != 0) { 2085 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 2086 "error %u\n", ifp->if_xname, r); 2087 goto fail_3; 2088 } 2089 2090 dma->dma_size = size; 2091 return (0); 2092 fail_3: 2093 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 2094 fail_2: 2095 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2096 fail_1: 2097 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2098 fail_0: 2099 dma->dma_map = NULL; 2100 dma->dma_tag = NULL; 2101 return (r); 2102 } 2103 2104 void 2105 ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 2106 { 2107 if (dma->dma_tag == NULL) 2108 return; 2109 2110 if (dma->dma_map != NULL) { 2111 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 2112 dma->dma_map->dm_mapsize, 2113 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2114 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2115 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 2116 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2117 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2118 dma->dma_map = NULL; 2119 } 2120 } 2121 2122 2123 /********************************************************************* 2124 * 2125 * Allocate memory for the transmit and receive rings, and then 2126 * the descriptors associated with each, called only once at attach. 2127 * 2128 **********************************************************************/ 2129 int 2130 ixgbe_allocate_queues(struct ix_softc *sc) 2131 { 2132 struct ifnet *ifp = &sc->arpcom.ac_if; 2133 struct ix_queue *que; 2134 struct tx_ring *txr; 2135 struct rx_ring *rxr; 2136 int rsize, tsize; 2137 int txconf = 0, rxconf = 0, i; 2138 2139 /* First allocate the top level queue structs */ 2140 if (!(sc->queues = mallocarray(sc->num_queues, 2141 sizeof(struct ix_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2142 printf("%s: Unable to allocate queue memory\n", ifp->if_xname); 2143 goto fail; 2144 } 2145 2146 /* Then allocate the TX ring struct memory */ 2147 if (!(sc->tx_rings = mallocarray(sc->num_queues, 2148 sizeof(struct tx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2149 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 2150 goto fail; 2151 } 2152 2153 /* Next allocate the RX */ 2154 if (!(sc->rx_rings = mallocarray(sc->num_queues, 2155 sizeof(struct rx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2156 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 2157 goto rx_fail; 2158 } 2159 2160 /* For the ring itself */ 2161 tsize = roundup2(sc->num_tx_desc * 2162 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2163 2164 /* 2165 * Now set up the TX queues, txconf is needed to handle the 2166 * possibility that things fail midcourse and we need to 2167 * undo memory gracefully 2168 */ 2169 for (i = 0; i < sc->num_queues; i++, txconf++) { 2170 /* Set up some basics */ 2171 txr = &sc->tx_rings[i]; 2172 txr->sc = sc; 2173 txr->me = i; 2174 2175 if (ixgbe_dma_malloc(sc, tsize, 2176 &txr->txdma, BUS_DMA_NOWAIT)) { 2177 printf("%s: Unable to allocate TX Descriptor memory\n", 2178 ifp->if_xname); 2179 goto err_tx_desc; 2180 } 2181 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2182 bzero((void *)txr->tx_base, tsize); 2183 } 2184 2185 /* 2186 * Next the RX queues... 2187 */ 2188 rsize = roundup2(sc->num_rx_desc * 2189 sizeof(union ixgbe_adv_rx_desc), 4096); 2190 for (i = 0; i < sc->num_queues; i++, rxconf++) { 2191 rxr = &sc->rx_rings[i]; 2192 /* Set up some basics */ 2193 rxr->sc = sc; 2194 rxr->me = i; 2195 timeout_set(&rxr->rx_refill, ixgbe_rxrefill, rxr); 2196 2197 if (ixgbe_dma_malloc(sc, rsize, 2198 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2199 printf("%s: Unable to allocate RxDescriptor memory\n", 2200 ifp->if_xname); 2201 goto err_rx_desc; 2202 } 2203 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2204 bzero((void *)rxr->rx_base, rsize); 2205 } 2206 2207 /* 2208 * Finally set up the queue holding structs 2209 */ 2210 for (i = 0; i < sc->num_queues; i++) { 2211 que = &sc->queues[i]; 2212 que->sc = sc; 2213 que->txr = &sc->tx_rings[i]; 2214 que->rxr = &sc->rx_rings[i]; 2215 snprintf(que->name, sizeof(que->name), "%s:%d", 2216 sc->dev.dv_xname, i); 2217 } 2218 2219 return (0); 2220 2221 err_rx_desc: 2222 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 2223 ixgbe_dma_free(sc, &rxr->rxdma); 2224 err_tx_desc: 2225 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 2226 ixgbe_dma_free(sc, &txr->txdma); 2227 free(sc->rx_rings, M_DEVBUF, sc->num_queues * sizeof(struct rx_ring)); 2228 sc->rx_rings = NULL; 2229 rx_fail: 2230 free(sc->tx_rings, M_DEVBUF, sc->num_queues * sizeof(struct tx_ring)); 2231 sc->tx_rings = NULL; 2232 fail: 2233 return (ENOMEM); 2234 } 2235 2236 /********************************************************************* 2237 * 2238 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2239 * the information needed to transmit a packet on the wire. This is 2240 * called only once at attach, setup is done every reset. 2241 * 2242 **********************************************************************/ 2243 int 2244 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2245 { 2246 struct ix_softc *sc = txr->sc; 2247 struct ifnet *ifp = &sc->arpcom.ac_if; 2248 struct ixgbe_tx_buf *txbuf; 2249 int error, i; 2250 2251 if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc, 2252 sizeof(struct ixgbe_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2253 printf("%s: Unable to allocate tx_buffer memory\n", 2254 ifp->if_xname); 2255 error = ENOMEM; 2256 goto fail; 2257 } 2258 txr->txtag = txr->txdma.dma_tag; 2259 2260 /* Create the descriptor buffer dma maps */ 2261 for (i = 0; i < sc->num_tx_desc; i++) { 2262 txbuf = &txr->tx_buffers[i]; 2263 error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE, 2264 sc->num_segs, PAGE_SIZE, 0, 2265 BUS_DMA_NOWAIT, &txbuf->map); 2266 2267 if (error != 0) { 2268 printf("%s: Unable to create TX DMA map\n", 2269 ifp->if_xname); 2270 goto fail; 2271 } 2272 } 2273 2274 return 0; 2275 fail: 2276 return (error); 2277 } 2278 2279 /********************************************************************* 2280 * 2281 * Initialize a transmit ring. 2282 * 2283 **********************************************************************/ 2284 int 2285 ixgbe_setup_transmit_ring(struct tx_ring *txr) 2286 { 2287 struct ix_softc *sc = txr->sc; 2288 int error; 2289 2290 /* Now allocate transmit buffers for the ring */ 2291 if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0) 2292 return (error); 2293 2294 /* Clear the old ring contents */ 2295 bzero((void *)txr->tx_base, 2296 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 2297 2298 /* Reset indices */ 2299 txr->next_avail_desc = 0; 2300 txr->next_to_clean = 0; 2301 2302 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2303 0, txr->txdma.dma_map->dm_mapsize, 2304 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2305 2306 return (0); 2307 } 2308 2309 /********************************************************************* 2310 * 2311 * Initialize all transmit rings. 2312 * 2313 **********************************************************************/ 2314 int 2315 ixgbe_setup_transmit_structures(struct ix_softc *sc) 2316 { 2317 struct tx_ring *txr = sc->tx_rings; 2318 int i, error; 2319 2320 for (i = 0; i < sc->num_queues; i++, txr++) { 2321 if ((error = ixgbe_setup_transmit_ring(txr)) != 0) 2322 goto fail; 2323 } 2324 2325 return (0); 2326 fail: 2327 ixgbe_free_transmit_structures(sc); 2328 return (error); 2329 } 2330 2331 /********************************************************************* 2332 * 2333 * Enable transmit unit. 2334 * 2335 **********************************************************************/ 2336 void 2337 ixgbe_initialize_transmit_units(struct ix_softc *sc) 2338 { 2339 struct ifnet *ifp = &sc->arpcom.ac_if; 2340 struct tx_ring *txr; 2341 struct ixgbe_hw *hw = &sc->hw; 2342 int i; 2343 uint64_t tdba; 2344 uint32_t txctrl; 2345 2346 /* Setup the Base and Length of the Tx Descriptor Ring */ 2347 2348 for (i = 0; i < sc->num_queues; i++) { 2349 txr = &sc->tx_rings[i]; 2350 2351 /* Setup descriptor base address */ 2352 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 2353 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 2354 (tdba & 0x00000000ffffffffULL)); 2355 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 2356 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2357 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 2358 2359 /* Setup the HW Tx Head and Tail descriptor pointers */ 2360 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2361 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2362 2363 /* Setup Transmit Descriptor Cmd Settings */ 2364 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2365 txr->queue_status = IXGBE_QUEUE_IDLE; 2366 txr->watchdog_timer = 0; 2367 2368 /* Disable Head Writeback */ 2369 switch (hw->mac.type) { 2370 case ixgbe_mac_82598EB: 2371 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2372 break; 2373 case ixgbe_mac_82599EB: 2374 case ixgbe_mac_X540: 2375 default: 2376 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2377 break; 2378 } 2379 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2380 switch (hw->mac.type) { 2381 case ixgbe_mac_82598EB: 2382 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2383 break; 2384 case ixgbe_mac_82599EB: 2385 case ixgbe_mac_X540: 2386 default: 2387 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2388 break; 2389 } 2390 } 2391 ifp->if_timer = 0; 2392 2393 if (hw->mac.type != ixgbe_mac_82598EB) { 2394 uint32_t dmatxctl, rttdcs; 2395 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2396 dmatxctl |= IXGBE_DMATXCTL_TE; 2397 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2398 /* Disable arbiter to set MTQC */ 2399 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2400 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2401 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2402 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2403 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2404 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2405 } 2406 } 2407 2408 /********************************************************************* 2409 * 2410 * Free all transmit rings. 2411 * 2412 **********************************************************************/ 2413 void 2414 ixgbe_free_transmit_structures(struct ix_softc *sc) 2415 { 2416 struct tx_ring *txr = sc->tx_rings; 2417 int i; 2418 2419 for (i = 0; i < sc->num_queues; i++, txr++) 2420 ixgbe_free_transmit_buffers(txr); 2421 } 2422 2423 /********************************************************************* 2424 * 2425 * Free transmit ring related data structures. 2426 * 2427 **********************************************************************/ 2428 void 2429 ixgbe_free_transmit_buffers(struct tx_ring *txr) 2430 { 2431 struct ix_softc *sc = txr->sc; 2432 struct ixgbe_tx_buf *tx_buffer; 2433 int i; 2434 2435 INIT_DEBUGOUT("free_transmit_ring: begin"); 2436 2437 if (txr->tx_buffers == NULL) 2438 return; 2439 2440 tx_buffer = txr->tx_buffers; 2441 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 2442 if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) { 2443 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2444 0, tx_buffer->map->dm_mapsize, 2445 BUS_DMASYNC_POSTWRITE); 2446 bus_dmamap_unload(txr->txdma.dma_tag, 2447 tx_buffer->map); 2448 } 2449 if (tx_buffer->m_head != NULL) { 2450 m_freem(tx_buffer->m_head); 2451 tx_buffer->m_head = NULL; 2452 } 2453 if (tx_buffer->map != NULL) { 2454 bus_dmamap_destroy(txr->txdma.dma_tag, 2455 tx_buffer->map); 2456 tx_buffer->map = NULL; 2457 } 2458 } 2459 2460 if (txr->tx_buffers != NULL) 2461 free(txr->tx_buffers, M_DEVBUF, 2462 sc->num_tx_desc * sizeof(struct ixgbe_tx_buf)); 2463 txr->tx_buffers = NULL; 2464 txr->txtag = NULL; 2465 } 2466 2467 /********************************************************************* 2468 * 2469 * Advanced Context Descriptor setup for VLAN or CSUM 2470 * 2471 **********************************************************************/ 2472 2473 static inline int 2474 ixgbe_csum_offload(struct mbuf *mp, uint32_t *vlan_macip_lens, 2475 uint32_t *type_tucmd_mlhl, uint32_t *olinfo_status) 2476 { 2477 struct ether_header *eh = mtod(mp, struct ether_header *); 2478 struct mbuf *m; 2479 int hoff; 2480 int offload = 0; 2481 uint32_t iphlen; 2482 uint8_t ipproto; 2483 2484 *vlan_macip_lens |= (sizeof(*eh) << IXGBE_ADVTXD_MACLEN_SHIFT); 2485 2486 switch (ntohs(eh->ether_type)) { 2487 case ETHERTYPE_IP: { 2488 struct ip *ip; 2489 2490 m = m_getptr(mp, sizeof(*eh), &hoff); 2491 KASSERT(m != NULL && m->m_len - hoff >= sizeof(*ip)); 2492 ip = (struct ip *)(mtod(m, caddr_t) + hoff); 2493 2494 iphlen = ip->ip_hl << 2; 2495 ipproto = ip->ip_p; 2496 2497 if (ISSET(mp->m_pkthdr.csum_flags, M_IPV4_CSUM_OUT)) { 2498 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2499 offload = 1; 2500 } 2501 2502 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2503 break; 2504 } 2505 2506 #ifdef INET6 2507 case ETHERTYPE_IPV6: { 2508 struct ip6_hdr *ip6; 2509 2510 m = m_getptr(mp, sizeof(*eh), &hoff); 2511 KASSERT(m != NULL && m->m_len - hoff >= sizeof(*ip6)); 2512 ip6 = (struct ip6_hdr *)(mtod(m, caddr_t) + hoff); 2513 2514 iphlen = sizeof(*ip6); 2515 ipproto = ip6->ip6_nxt; 2516 2517 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2518 break; 2519 } 2520 #endif 2521 2522 default: 2523 return offload; 2524 } 2525 2526 *vlan_macip_lens |= iphlen; 2527 2528 switch (ipproto) { 2529 case IPPROTO_TCP: 2530 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2531 if (ISSET(mp->m_pkthdr.csum_flags, M_TCP_CSUM_OUT)) { 2532 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2533 offload = 1; 2534 } 2535 break; 2536 case IPPROTO_UDP: 2537 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2538 if (ISSET(mp->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) { 2539 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2540 offload = 1; 2541 } 2542 break; 2543 } 2544 2545 return offload; 2546 } 2547 2548 static int 2549 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, 2550 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2551 { 2552 struct ixgbe_adv_tx_context_desc *TXD; 2553 struct ixgbe_tx_buf *tx_buffer; 2554 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2555 int ctxd = txr->next_avail_desc; 2556 int offload = 0; 2557 2558 /* Indicate the whole packet as payload when not doing TSO */ 2559 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2560 2561 #if NVLAN > 0 2562 if (ISSET(mp->m_flags, M_VLANTAG)) { 2563 uint32_t vtag = mp->m_pkthdr.ether_vtag; 2564 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2565 *cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 2566 offload |= 1; 2567 } 2568 #endif 2569 2570 offload |= ixgbe_csum_offload(mp, &vlan_macip_lens, &type_tucmd_mlhl, 2571 olinfo_status); 2572 2573 if (!offload) 2574 return (0); 2575 2576 TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; 2577 tx_buffer = &txr->tx_buffers[ctxd]; 2578 2579 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2580 2581 /* Now copy bits into descriptor */ 2582 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2583 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2584 TXD->seqnum_seed = htole32(0); 2585 TXD->mss_l4len_idx = htole32(0); 2586 2587 tx_buffer->m_head = NULL; 2588 tx_buffer->eop_index = -1; 2589 2590 return (1); 2591 } 2592 2593 /********************************************************************** 2594 * 2595 * Examine each tx_buffer in the used queue. If the hardware is done 2596 * processing the packet then free associated resources. The 2597 * tx_buffer is put back on the free queue. 2598 * 2599 **********************************************************************/ 2600 int 2601 ixgbe_txeof(struct tx_ring *txr) 2602 { 2603 struct ix_softc *sc = txr->sc; 2604 struct ifqueue *ifq = txr->ifq; 2605 struct ifnet *ifp = &sc->arpcom.ac_if; 2606 unsigned int head, tail, last; 2607 struct ixgbe_tx_buf *tx_buffer; 2608 struct ixgbe_legacy_tx_desc *tx_desc; 2609 2610 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2611 return FALSE; 2612 2613 head = txr->next_avail_desc; 2614 tail = txr->next_to_clean; 2615 2616 membar_consumer(); 2617 2618 if (head == tail) 2619 return (FALSE); 2620 2621 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2622 0, txr->txdma.dma_map->dm_mapsize, 2623 BUS_DMASYNC_POSTREAD); 2624 2625 for (;;) { 2626 tx_buffer = &txr->tx_buffers[tail]; 2627 last = tx_buffer->eop_index; 2628 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2629 2630 if (!ISSET(tx_desc->upper.fields.status, IXGBE_TXD_STAT_DD)) 2631 break; 2632 2633 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2634 0, tx_buffer->map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2635 bus_dmamap_unload(txr->txdma.dma_tag, tx_buffer->map); 2636 m_freem(tx_buffer->m_head); 2637 2638 tx_buffer->m_head = NULL; 2639 tx_buffer->eop_index = -1; 2640 2641 tail = last + 1; 2642 if (tail == sc->num_tx_desc) 2643 tail = 0; 2644 if (head == tail) { 2645 /* All clean, turn off the timer */ 2646 ifp->if_timer = 0; 2647 break; 2648 } 2649 } 2650 2651 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2652 0, txr->txdma.dma_map->dm_mapsize, 2653 BUS_DMASYNC_PREREAD); 2654 2655 membar_producer(); 2656 2657 txr->next_to_clean = tail; 2658 2659 if (ifq_is_oactive(ifq)) 2660 ifq_restart(ifq); 2661 2662 return TRUE; 2663 } 2664 2665 /********************************************************************* 2666 * 2667 * Get a buffer from system mbuf buffer pool. 2668 * 2669 **********************************************************************/ 2670 int 2671 ixgbe_get_buf(struct rx_ring *rxr, int i) 2672 { 2673 struct ix_softc *sc = rxr->sc; 2674 struct ixgbe_rx_buf *rxbuf; 2675 struct mbuf *mp; 2676 int error; 2677 union ixgbe_adv_rx_desc *rxdesc; 2678 2679 rxbuf = &rxr->rx_buffers[i]; 2680 rxdesc = &rxr->rx_base[i]; 2681 if (rxbuf->buf) { 2682 printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n", 2683 sc->dev.dv_xname, i); 2684 return (ENOBUFS); 2685 } 2686 2687 /* needed in any case so prealocate since this one will fail for sure */ 2688 mp = MCLGETL(NULL, M_DONTWAIT, sc->rx_mbuf_sz); 2689 if (!mp) 2690 return (ENOBUFS); 2691 2692 mp->m_data += (mp->m_ext.ext_size - sc->rx_mbuf_sz); 2693 mp->m_len = mp->m_pkthdr.len = sc->rx_mbuf_sz; 2694 2695 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, 2696 mp, BUS_DMA_NOWAIT); 2697 if (error) { 2698 m_freem(mp); 2699 return (error); 2700 } 2701 2702 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2703 0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 2704 rxbuf->buf = mp; 2705 2706 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 2707 2708 return (0); 2709 } 2710 2711 /********************************************************************* 2712 * 2713 * Allocate memory for rx_buffer structures. Since we use one 2714 * rx_buffer per received packet, the maximum number of rx_buffer's 2715 * that we'll need is equal to the number of receive descriptors 2716 * that we've allocated. 2717 * 2718 **********************************************************************/ 2719 int 2720 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2721 { 2722 struct ix_softc *sc = rxr->sc; 2723 struct ifnet *ifp = &sc->arpcom.ac_if; 2724 struct ixgbe_rx_buf *rxbuf; 2725 int i, error; 2726 2727 if (!(rxr->rx_buffers = mallocarray(sc->num_rx_desc, 2728 sizeof(struct ixgbe_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2729 printf("%s: Unable to allocate rx_buffer memory\n", 2730 ifp->if_xname); 2731 error = ENOMEM; 2732 goto fail; 2733 } 2734 2735 rxbuf = rxr->rx_buffers; 2736 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2737 error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1, 2738 16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map); 2739 if (error) { 2740 printf("%s: Unable to create Pack DMA map\n", 2741 ifp->if_xname); 2742 goto fail; 2743 } 2744 } 2745 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2746 rxr->rxdma.dma_map->dm_mapsize, 2747 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2748 2749 return (0); 2750 2751 fail: 2752 return (error); 2753 } 2754 2755 /********************************************************************* 2756 * 2757 * Initialize a receive ring and its buffers. 2758 * 2759 **********************************************************************/ 2760 int 2761 ixgbe_setup_receive_ring(struct rx_ring *rxr) 2762 { 2763 struct ix_softc *sc = rxr->sc; 2764 struct ifnet *ifp = &sc->arpcom.ac_if; 2765 int rsize, error; 2766 2767 rsize = roundup2(sc->num_rx_desc * 2768 sizeof(union ixgbe_adv_rx_desc), 4096); 2769 /* Clear the ring contents */ 2770 bzero((void *)rxr->rx_base, rsize); 2771 2772 if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0) 2773 return (error); 2774 2775 /* Setup our descriptor indices */ 2776 rxr->next_to_check = 0; 2777 rxr->last_desc_filled = sc->num_rx_desc - 1; 2778 2779 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2780 sc->num_rx_desc - 1); 2781 2782 ixgbe_rxfill(rxr); 2783 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 2784 printf("%s: unable to fill any rx descriptors\n", 2785 sc->dev.dv_xname); 2786 return (ENOBUFS); 2787 } 2788 2789 return (0); 2790 } 2791 2792 int 2793 ixgbe_rxfill(struct rx_ring *rxr) 2794 { 2795 struct ix_softc *sc = rxr->sc; 2796 int post = 0; 2797 u_int slots; 2798 int i; 2799 2800 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2801 0, rxr->rxdma.dma_map->dm_mapsize, 2802 BUS_DMASYNC_POSTWRITE); 2803 2804 i = rxr->last_desc_filled; 2805 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); 2806 slots > 0; slots--) { 2807 if (++i == sc->num_rx_desc) 2808 i = 0; 2809 2810 if (ixgbe_get_buf(rxr, i) != 0) 2811 break; 2812 2813 rxr->last_desc_filled = i; 2814 post = 1; 2815 } 2816 2817 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2818 0, rxr->rxdma.dma_map->dm_mapsize, 2819 BUS_DMASYNC_PREWRITE); 2820 2821 if_rxr_put(&rxr->rx_ring, slots); 2822 2823 return (post); 2824 } 2825 2826 void 2827 ixgbe_rxrefill(void *xrxr) 2828 { 2829 struct rx_ring *rxr = xrxr; 2830 struct ix_softc *sc = rxr->sc; 2831 2832 if (ixgbe_rxfill(rxr)) { 2833 /* Advance the Rx Queue "Tail Pointer" */ 2834 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me), 2835 rxr->last_desc_filled); 2836 } else if (if_rxr_inuse(&rxr->rx_ring) == 0) 2837 timeout_add(&rxr->rx_refill, 1); 2838 2839 } 2840 2841 /********************************************************************* 2842 * 2843 * Initialize all receive rings. 2844 * 2845 **********************************************************************/ 2846 int 2847 ixgbe_setup_receive_structures(struct ix_softc *sc) 2848 { 2849 struct rx_ring *rxr = sc->rx_rings; 2850 int i; 2851 2852 for (i = 0; i < sc->num_queues; i++, rxr++) 2853 if (ixgbe_setup_receive_ring(rxr)) 2854 goto fail; 2855 2856 return (0); 2857 fail: 2858 ixgbe_free_receive_structures(sc); 2859 return (ENOBUFS); 2860 } 2861 2862 /********************************************************************* 2863 * 2864 * Setup receive registers and features. 2865 * 2866 **********************************************************************/ 2867 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2868 2869 void 2870 ixgbe_initialize_receive_units(struct ix_softc *sc) 2871 { 2872 struct ifnet *ifp = &sc->arpcom.ac_if; 2873 struct rx_ring *rxr = sc->rx_rings; 2874 struct ixgbe_hw *hw = &sc->hw; 2875 uint32_t bufsz, fctrl, srrctl, rxcsum, rdrxctl; 2876 uint32_t hlreg; 2877 int i; 2878 2879 /* 2880 * Make sure receives are disabled while 2881 * setting up the descriptor ring 2882 */ 2883 ixgbe_disable_rx(hw); 2884 2885 /* Enable broadcasts */ 2886 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2887 fctrl |= IXGBE_FCTRL_BAM; 2888 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 2889 fctrl |= IXGBE_FCTRL_DPF; 2890 fctrl |= IXGBE_FCTRL_PMCF; 2891 } 2892 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 2893 2894 /* Always enable jumbo frame reception */ 2895 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2896 hlreg |= IXGBE_HLREG0_JUMBOEN; 2897 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2898 2899 if (ISSET(ifp->if_xflags, IFXF_TSO)) { 2900 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2901 2902 /* This field has to be set to zero. */ 2903 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2904 2905 /* Enable TSO Receive Offloading */ 2906 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 2907 rdrxctl |= IXGBE_RDRXCTL_FCOE_WRFIX; 2908 2909 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 2910 } 2911 2912 bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2913 2914 for (i = 0; i < sc->num_queues; i++, rxr++) { 2915 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2916 2917 /* Setup the Base and Length of the Rx Descriptor Ring */ 2918 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 2919 (rdba & 0x00000000ffffffffULL)); 2920 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 2921 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 2922 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2923 2924 /* Set up the SRRCTL register */ 2925 srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2926 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 2927 2928 if (ISSET(ifp->if_xflags, IFXF_TSO)) { 2929 rdrxctl = IXGBE_READ_REG(&sc->hw, IXGBE_RSCCTL(i)); 2930 2931 /* Enable TSO Receive Side Coalescing */ 2932 rdrxctl |= IXGBE_RSCCTL_RSCEN; 2933 rdrxctl |= IXGBE_RSCCTL_MAXDESC_16; 2934 2935 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), rdrxctl); 2936 } 2937 2938 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2939 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 2940 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 2941 } 2942 2943 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 2944 uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR | 2945 IXGBE_PSRTYPE_UDPHDR | 2946 IXGBE_PSRTYPE_IPV4HDR | 2947 IXGBE_PSRTYPE_IPV6HDR; 2948 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 2949 } 2950 2951 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2952 rxcsum &= ~IXGBE_RXCSUM_PCSD; 2953 2954 ixgbe_initialize_rss_mapping(sc); 2955 2956 /* Setup RSS */ 2957 if (sc->num_queues > 1) { 2958 /* RSS and RX IPP Checksum are mutually exclusive */ 2959 rxcsum |= IXGBE_RXCSUM_PCSD; 2960 } 2961 2962 /* Map QPRC/QPRDC/QPTC on a per queue basis */ 2963 ixgbe_map_queue_statistics(sc); 2964 2965 /* This is useful for calculating UDP/IP fragment checksums */ 2966 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2967 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2968 2969 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2970 } 2971 2972 void 2973 ixgbe_initialize_rss_mapping(struct ix_softc *sc) 2974 { 2975 struct ixgbe_hw *hw = &sc->hw; 2976 uint32_t reta = 0, mrqc, rss_key[10]; 2977 int i, j, queue_id, table_size, index_mult; 2978 2979 /* set up random bits */ 2980 stoeplitz_to_key(&rss_key, sizeof(rss_key)); 2981 2982 /* Set multiplier for RETA setup and table size based on MAC */ 2983 index_mult = 0x1; 2984 table_size = 128; 2985 switch (sc->hw.mac.type) { 2986 case ixgbe_mac_82598EB: 2987 index_mult = 0x11; 2988 break; 2989 case ixgbe_mac_X550: 2990 case ixgbe_mac_X550EM_x: 2991 case ixgbe_mac_X550EM_a: 2992 table_size = 512; 2993 break; 2994 default: 2995 break; 2996 } 2997 2998 /* Set up the redirection table */ 2999 for (i = 0, j = 0; i < table_size; i++, j++) { 3000 if (j == sc->num_queues) j = 0; 3001 queue_id = (j * index_mult); 3002 /* 3003 * The low 8 bits are for hash value (n+0); 3004 * The next 8 bits are for hash value (n+1), etc. 3005 */ 3006 reta = reta >> 8; 3007 reta = reta | ( ((uint32_t) queue_id) << 24); 3008 if ((i & 3) == 3) { 3009 if (i < 128) 3010 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 3011 else 3012 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 3013 reta); 3014 reta = 0; 3015 } 3016 } 3017 3018 /* Now fill our hash function seeds */ 3019 for (i = 0; i < 10; i++) 3020 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 3021 3022 /* 3023 * Disable UDP - IP fragments aren't currently being handled 3024 * and so we end up with a mix of 2-tuple and 4-tuple 3025 * traffic. 3026 */ 3027 mrqc = IXGBE_MRQC_RSSEN 3028 | IXGBE_MRQC_RSS_FIELD_IPV4 3029 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 3030 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 3031 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 3032 | IXGBE_MRQC_RSS_FIELD_IPV6 3033 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 3034 ; 3035 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3036 } 3037 3038 /********************************************************************* 3039 * 3040 * Free all receive rings. 3041 * 3042 **********************************************************************/ 3043 void 3044 ixgbe_free_receive_structures(struct ix_softc *sc) 3045 { 3046 struct rx_ring *rxr; 3047 int i; 3048 3049 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3050 if_rxr_init(&rxr->rx_ring, 0, 0); 3051 3052 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3053 ixgbe_free_receive_buffers(rxr); 3054 } 3055 3056 /********************************************************************* 3057 * 3058 * Free receive ring data structures 3059 * 3060 **********************************************************************/ 3061 void 3062 ixgbe_free_receive_buffers(struct rx_ring *rxr) 3063 { 3064 struct ix_softc *sc; 3065 struct ixgbe_rx_buf *rxbuf; 3066 int i; 3067 3068 sc = rxr->sc; 3069 if (rxr->rx_buffers != NULL) { 3070 for (i = 0; i < sc->num_rx_desc; i++) { 3071 rxbuf = &rxr->rx_buffers[i]; 3072 if (rxbuf->buf != NULL) { 3073 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 3074 0, rxbuf->map->dm_mapsize, 3075 BUS_DMASYNC_POSTREAD); 3076 bus_dmamap_unload(rxr->rxdma.dma_tag, 3077 rxbuf->map); 3078 m_freem(rxbuf->buf); 3079 rxbuf->buf = NULL; 3080 } 3081 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 3082 rxbuf->map = NULL; 3083 } 3084 free(rxr->rx_buffers, M_DEVBUF, 3085 sc->num_rx_desc * sizeof(struct ixgbe_rx_buf)); 3086 rxr->rx_buffers = NULL; 3087 } 3088 } 3089 3090 /********************************************************************* 3091 * 3092 * This routine executes in interrupt context. It replenishes 3093 * the mbufs in the descriptor and sends data which has been 3094 * dma'ed into host memory to upper layer. 3095 * 3096 *********************************************************************/ 3097 int 3098 ixgbe_rxeof(struct rx_ring *rxr) 3099 { 3100 struct ix_softc *sc = rxr->sc; 3101 struct ifnet *ifp = &sc->arpcom.ac_if; 3102 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 3103 struct mbuf *mp, *sendmp; 3104 uint8_t eop = 0; 3105 uint16_t len, vtag; 3106 uint32_t staterr = 0, ptype; 3107 struct ixgbe_rx_buf *rxbuf, *nxbuf; 3108 union ixgbe_adv_rx_desc *rxdesc; 3109 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 3110 int i, nextp; 3111 3112 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3113 return FALSE; 3114 3115 i = rxr->next_to_check; 3116 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 3117 uint32_t hash; 3118 uint16_t hashtype; 3119 3120 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3121 dsize * i, dsize, BUS_DMASYNC_POSTREAD); 3122 3123 rxdesc = &rxr->rx_base[i]; 3124 staterr = letoh32(rxdesc->wb.upper.status_error); 3125 if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) { 3126 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3127 dsize * i, dsize, 3128 BUS_DMASYNC_PREREAD); 3129 break; 3130 } 3131 3132 /* Zero out the receive descriptors status */ 3133 rxdesc->wb.upper.status_error = 0; 3134 rxbuf = &rxr->rx_buffers[i]; 3135 3136 /* pull the mbuf off the ring */ 3137 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 3138 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3139 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 3140 3141 mp = rxbuf->buf; 3142 len = letoh16(rxdesc->wb.upper.length); 3143 ptype = letoh32(rxdesc->wb.lower.lo_dword.data) & 3144 IXGBE_RXDADV_PKTTYPE_MASK; 3145 vtag = letoh16(rxdesc->wb.upper.vlan); 3146 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3147 hash = lemtoh32(&rxdesc->wb.lower.hi_dword.rss); 3148 hashtype = 3149 lemtoh16(&rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 3150 IXGBE_RXDADV_RSSTYPE_MASK; 3151 3152 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) { 3153 if (rxbuf->fmp) { 3154 m_freem(rxbuf->fmp); 3155 rxbuf->fmp = NULL; 3156 } 3157 3158 m_freem(mp); 3159 rxbuf->buf = NULL; 3160 goto next_desc; 3161 } 3162 3163 if (mp == NULL) { 3164 panic("%s: ixgbe_rxeof: NULL mbuf in slot %d " 3165 "(nrx %d, filled %d)", sc->dev.dv_xname, 3166 i, if_rxr_inuse(&rxr->rx_ring), 3167 rxr->last_desc_filled); 3168 } 3169 3170 /* Currently no HW RSC support of 82599 */ 3171 if (!eop) { 3172 /* 3173 * Figure out the next descriptor of this frame. 3174 */ 3175 nextp = i + 1; 3176 if (nextp == sc->num_rx_desc) 3177 nextp = 0; 3178 nxbuf = &rxr->rx_buffers[nextp]; 3179 /* prefetch(nxbuf); */ 3180 } 3181 3182 /* 3183 * Rather than using the fmp/lmp global pointers 3184 * we now keep the head of a packet chain in the 3185 * buffer struct and pass this along from one 3186 * descriptor to the next, until we get EOP. 3187 */ 3188 mp->m_len = len; 3189 /* 3190 * See if there is a stored head 3191 * that determines what we are 3192 */ 3193 sendmp = rxbuf->fmp; 3194 rxbuf->buf = rxbuf->fmp = NULL; 3195 3196 if (sendmp != NULL) /* secondary frag */ 3197 sendmp->m_pkthdr.len += mp->m_len; 3198 else { 3199 /* first desc of a non-ps chain */ 3200 sendmp = mp; 3201 sendmp->m_pkthdr.len = mp->m_len; 3202 #if NVLAN > 0 3203 if (sc->vlan_stripping && staterr & IXGBE_RXD_STAT_VP) { 3204 sendmp->m_pkthdr.ether_vtag = vtag; 3205 sendmp->m_flags |= M_VLANTAG; 3206 } 3207 #endif 3208 } 3209 3210 /* Pass the head pointer on */ 3211 if (eop == 0) { 3212 nxbuf->fmp = sendmp; 3213 sendmp = NULL; 3214 mp->m_next = nxbuf->buf; 3215 } else { /* Sending this frame? */ 3216 ixgbe_rx_checksum(staterr, sendmp, ptype); 3217 3218 if (hashtype != IXGBE_RXDADV_RSSTYPE_NONE) { 3219 sendmp->m_pkthdr.ph_flowid = hash; 3220 SET(sendmp->m_pkthdr.csum_flags, M_FLOWID); 3221 } 3222 3223 ml_enqueue(&ml, sendmp); 3224 } 3225 next_desc: 3226 if_rxr_put(&rxr->rx_ring, 1); 3227 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3228 dsize * i, dsize, 3229 BUS_DMASYNC_PREREAD); 3230 3231 /* Advance our pointers to the next descriptor. */ 3232 if (++i == sc->num_rx_desc) 3233 i = 0; 3234 } 3235 rxr->next_to_check = i; 3236 3237 if (ifiq_input(rxr->ifiq, &ml)) 3238 if_rxr_livelocked(&rxr->rx_ring); 3239 3240 if (!(staterr & IXGBE_RXD_STAT_DD)) 3241 return FALSE; 3242 3243 return TRUE; 3244 } 3245 3246 /********************************************************************* 3247 * 3248 * Verify that the hardware indicated that the checksum is valid. 3249 * Inform the stack about the status of checksum so that stack 3250 * doesn't spend time verifying the checksum. 3251 * 3252 *********************************************************************/ 3253 void 3254 ixgbe_rx_checksum(uint32_t staterr, struct mbuf * mp, uint32_t ptype) 3255 { 3256 uint16_t status = (uint16_t) staterr; 3257 uint8_t errors = (uint8_t) (staterr >> 24); 3258 3259 if (status & IXGBE_RXD_STAT_IPCS) { 3260 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3261 /* IP Checksum Good */ 3262 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 3263 } else 3264 mp->m_pkthdr.csum_flags = 0; 3265 } 3266 if (status & IXGBE_RXD_STAT_L4CS) { 3267 if (!(errors & IXGBE_RXD_ERR_TCPE)) 3268 mp->m_pkthdr.csum_flags |= 3269 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 3270 } 3271 } 3272 3273 void 3274 ixgbe_setup_vlan_hw_support(struct ix_softc *sc) 3275 { 3276 struct ifnet *ifp = &sc->arpcom.ac_if; 3277 uint32_t ctrl; 3278 int i; 3279 3280 /* 3281 * We have to disable VLAN striping when using TCP offloading, due to a 3282 * firmware bug. 3283 */ 3284 if (ISSET(ifp->if_xflags, IFXF_TSO)) { 3285 sc->vlan_stripping = 0; 3286 return; 3287 } 3288 3289 sc->vlan_stripping = 1; 3290 3291 /* 3292 * A soft reset zero's out the VFTA, so 3293 * we need to repopulate it now. 3294 */ 3295 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 3296 if (sc->shadow_vfta[i] != 0) 3297 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i), 3298 sc->shadow_vfta[i]); 3299 } 3300 3301 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 3302 #if 0 3303 /* Enable the Filter Table if enabled */ 3304 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3305 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3306 ctrl |= IXGBE_VLNCTRL_VFE; 3307 } 3308 #endif 3309 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3310 ctrl |= IXGBE_VLNCTRL_VME; 3311 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 3312 3313 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 3314 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 3315 for (i = 0; i < sc->num_queues; i++) { 3316 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 3317 ctrl |= IXGBE_RXDCTL_VME; 3318 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl); 3319 } 3320 } 3321 } 3322 3323 void 3324 ixgbe_enable_intr(struct ix_softc *sc) 3325 { 3326 struct ixgbe_hw *hw = &sc->hw; 3327 uint32_t mask, fwsm; 3328 3329 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3330 /* Enable Fan Failure detection */ 3331 if (hw->device_id == IXGBE_DEV_ID_82598AT) 3332 mask |= IXGBE_EIMS_GPI_SDP1; 3333 3334 switch (sc->hw.mac.type) { 3335 case ixgbe_mac_82599EB: 3336 mask |= IXGBE_EIMS_ECC; 3337 /* Temperature sensor on some adapters */ 3338 mask |= IXGBE_EIMS_GPI_SDP0; 3339 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 3340 mask |= IXGBE_EIMS_GPI_SDP1; 3341 mask |= IXGBE_EIMS_GPI_SDP2; 3342 break; 3343 case ixgbe_mac_X540: 3344 mask |= IXGBE_EIMS_ECC; 3345 /* Detect if Thermal Sensor is enabled */ 3346 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3347 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3348 mask |= IXGBE_EIMS_TS; 3349 break; 3350 case ixgbe_mac_X550: 3351 case ixgbe_mac_X550EM_x: 3352 case ixgbe_mac_X550EM_a: 3353 mask |= IXGBE_EIMS_ECC; 3354 /* MAC thermal sensor is automatically enabled */ 3355 mask |= IXGBE_EIMS_TS; 3356 /* Some devices use SDP0 for important information */ 3357 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 3358 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 3359 mask |= IXGBE_EIMS_GPI_SDP0_X540; 3360 default: 3361 break; 3362 } 3363 3364 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3365 3366 /* With MSI-X we use auto clear */ 3367 if (sc->sc_intrmap) { 3368 mask = IXGBE_EIMS_ENABLE_MASK; 3369 /* Don't autoclear Link */ 3370 mask &= ~IXGBE_EIMS_OTHER; 3371 mask &= ~IXGBE_EIMS_LSC; 3372 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 3373 } 3374 3375 IXGBE_WRITE_FLUSH(hw); 3376 } 3377 3378 void 3379 ixgbe_disable_intr(struct ix_softc *sc) 3380 { 3381 if (sc->sc_intrmap) 3382 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3383 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3384 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3385 } else { 3386 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3387 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3388 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3389 } 3390 IXGBE_WRITE_FLUSH(&sc->hw); 3391 } 3392 3393 uint16_t 3394 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3395 { 3396 struct pci_attach_args *pa; 3397 uint32_t value; 3398 int high = 0; 3399 3400 if (reg & 0x2) { 3401 high = 1; 3402 reg &= ~0x2; 3403 } 3404 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3405 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3406 3407 if (high) 3408 value >>= 16; 3409 3410 return (value & 0xffff); 3411 } 3412 3413 void 3414 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3415 { 3416 struct pci_attach_args *pa; 3417 uint32_t rv; 3418 int high = 0; 3419 3420 /* Need to do read/mask/write... because 16 vs 32 bit!!! */ 3421 if (reg & 0x2) { 3422 high = 1; 3423 reg &= ~0x2; 3424 } 3425 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3426 rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3427 if (!high) 3428 rv = (rv & 0xffff0000) | value; 3429 else 3430 rv = (rv & 0xffff) | ((uint32_t)value << 16); 3431 pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv); 3432 } 3433 3434 /* 3435 * Setup the correct IVAR register for a particular MSIX interrupt 3436 * (yes this is all very magic and confusing :) 3437 * - entry is the register array entry 3438 * - vector is the MSIX vector for this queue 3439 * - type is RX/TX/MISC 3440 */ 3441 void 3442 ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type) 3443 { 3444 struct ixgbe_hw *hw = &sc->hw; 3445 uint32_t ivar, index; 3446 3447 vector |= IXGBE_IVAR_ALLOC_VAL; 3448 3449 switch (hw->mac.type) { 3450 3451 case ixgbe_mac_82598EB: 3452 if (type == -1) 3453 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3454 else 3455 entry += (type * 64); 3456 index = (entry >> 2) & 0x1F; 3457 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3458 ivar &= ~((uint32_t)0xFF << (8 * (entry & 0x3))); 3459 ivar |= ((uint32_t)vector << (8 * (entry & 0x3))); 3460 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 3461 break; 3462 3463 case ixgbe_mac_82599EB: 3464 case ixgbe_mac_X540: 3465 case ixgbe_mac_X550: 3466 case ixgbe_mac_X550EM_x: 3467 case ixgbe_mac_X550EM_a: 3468 if (type == -1) { /* MISC IVAR */ 3469 index = (entry & 1) * 8; 3470 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3471 ivar &= ~((uint32_t)0xFF << index); 3472 ivar |= ((uint32_t)vector << index); 3473 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3474 } else { /* RX/TX IVARS */ 3475 index = (16 * (entry & 1)) + (8 * type); 3476 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3477 ivar &= ~((uint32_t)0xFF << index); 3478 ivar |= ((uint32_t)vector << index); 3479 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3480 } 3481 3482 default: 3483 break; 3484 } 3485 } 3486 3487 void 3488 ixgbe_configure_ivars(struct ix_softc *sc) 3489 { 3490 struct ix_queue *que = sc->queues; 3491 uint32_t newitr; 3492 int i; 3493 3494 newitr = (4000000 / IXGBE_INTS_PER_SEC) & 0x0FF8; 3495 3496 for (i = 0; i < sc->num_queues; i++, que++) { 3497 /* First the RX queue entry */ 3498 ixgbe_set_ivar(sc, i, que->msix, 0); 3499 /* ... and the TX */ 3500 ixgbe_set_ivar(sc, i, que->msix, 1); 3501 /* Set an Initial EITR value */ 3502 IXGBE_WRITE_REG(&sc->hw, 3503 IXGBE_EITR(que->msix), newitr); 3504 } 3505 3506 /* For the Link interrupt */ 3507 ixgbe_set_ivar(sc, 1, sc->linkvec, -1); 3508 } 3509 3510 /* 3511 * SFP module interrupts handler 3512 */ 3513 void 3514 ixgbe_handle_mod(struct ix_softc *sc) 3515 { 3516 struct ixgbe_hw *hw = &sc->hw; 3517 uint32_t err; 3518 3519 err = hw->phy.ops.identify_sfp(hw); 3520 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3521 printf("%s: Unsupported SFP+ module type was detected!\n", 3522 sc->dev.dv_xname); 3523 return; 3524 } 3525 err = hw->mac.ops.setup_sfp(hw); 3526 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3527 printf("%s: Setup failure - unsupported SFP+ module type!\n", 3528 sc->dev.dv_xname); 3529 return; 3530 } 3531 3532 ixgbe_handle_msf(sc); 3533 } 3534 3535 3536 /* 3537 * MSF (multispeed fiber) interrupts handler 3538 */ 3539 void 3540 ixgbe_handle_msf(struct ix_softc *sc) 3541 { 3542 struct ixgbe_hw *hw = &sc->hw; 3543 uint32_t autoneg; 3544 bool negotiate; 3545 3546 autoneg = hw->phy.autoneg_advertised; 3547 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) { 3548 if (hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate)) 3549 return; 3550 } 3551 if (hw->mac.ops.setup_link) 3552 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3553 3554 ifmedia_delete_instance(&sc->media, IFM_INST_ANY); 3555 ixgbe_add_media_types(sc); 3556 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 3557 } 3558 3559 /* 3560 * External PHY interrupts handler 3561 */ 3562 void 3563 ixgbe_handle_phy(struct ix_softc *sc) 3564 { 3565 struct ixgbe_hw *hw = &sc->hw; 3566 int error; 3567 3568 error = hw->phy.ops.handle_lasi(hw); 3569 if (error == IXGBE_ERR_OVERTEMP) 3570 printf("%s: CRITICAL: EXTERNAL PHY OVER TEMP!! " 3571 " PHY will downshift to lower power state!\n", 3572 sc->dev.dv_xname); 3573 else if (error) 3574 printf("%s: Error handling LASI interrupt: %d\n", 3575 sc->dev.dv_xname, error); 3576 3577 } 3578 3579 #if NKSTAT > 0 3580 enum ix_counter_idx { 3581 ix_counter_crcerrs, 3582 ix_counter_lxontxc, 3583 ix_counter_lxonrxc, 3584 ix_counter_lxofftxc, 3585 ix_counter_lxoffrxc, 3586 ix_counter_prc64, 3587 ix_counter_prc127, 3588 ix_counter_prc255, 3589 ix_counter_prc511, 3590 ix_counter_prc1023, 3591 ix_counter_prc1522, 3592 ix_counter_gptc, 3593 ix_counter_gorc, 3594 ix_counter_gotc, 3595 ix_counter_ruc, 3596 ix_counter_rfc, 3597 ix_counter_roc, 3598 ix_counter_rjc, 3599 ix_counter_tor, 3600 ix_counter_tpr, 3601 ix_counter_tpt, 3602 ix_counter_gprc, 3603 ix_counter_bprc, 3604 ix_counter_mprc, 3605 ix_counter_ptc64, 3606 ix_counter_ptc127, 3607 ix_counter_ptc255, 3608 ix_counter_ptc511, 3609 ix_counter_ptc1023, 3610 ix_counter_ptc1522, 3611 ix_counter_mptc, 3612 ix_counter_bptc, 3613 3614 ix_counter_num, 3615 }; 3616 3617 CTASSERT(KSTAT_KV_U_PACKETS <= 0xff); 3618 CTASSERT(KSTAT_KV_U_BYTES <= 0xff); 3619 3620 struct ix_counter { 3621 char name[KSTAT_KV_NAMELEN]; 3622 uint32_t reg; 3623 uint8_t width; 3624 uint8_t unit; 3625 }; 3626 3627 static const struct ix_counter ix_counters[ix_counter_num] = { 3628 [ix_counter_crcerrs] = { "crc errs", IXGBE_CRCERRS, 32, 3629 KSTAT_KV_U_PACKETS }, 3630 [ix_counter_lxontxc] = { "tx link xon", IXGBE_LXONTXC, 32, 3631 KSTAT_KV_U_PACKETS }, 3632 [ix_counter_lxonrxc] = { "rx link xon", 0, 32, 3633 KSTAT_KV_U_PACKETS }, 3634 [ix_counter_lxofftxc] = { "tx link xoff", IXGBE_LXOFFTXC, 32, 3635 KSTAT_KV_U_PACKETS }, 3636 [ix_counter_lxoffrxc] = { "rx link xoff", 0, 32, 3637 KSTAT_KV_U_PACKETS }, 3638 [ix_counter_prc64] = { "rx 64B", IXGBE_PRC64, 32, 3639 KSTAT_KV_U_PACKETS }, 3640 [ix_counter_prc127] = { "rx 65-127B", IXGBE_PRC127, 32, 3641 KSTAT_KV_U_PACKETS }, 3642 [ix_counter_prc255] = { "rx 128-255B", IXGBE_PRC255, 32, 3643 KSTAT_KV_U_PACKETS }, 3644 [ix_counter_prc511] = { "rx 256-511B", IXGBE_PRC511, 32, 3645 KSTAT_KV_U_PACKETS }, 3646 [ix_counter_prc1023] = { "rx 512-1023B", IXGBE_PRC1023, 32, 3647 KSTAT_KV_U_PACKETS }, 3648 [ix_counter_prc1522] = { "rx 1024-maxB", IXGBE_PRC1522, 32, 3649 KSTAT_KV_U_PACKETS }, 3650 [ix_counter_gptc] = { "tx good", IXGBE_GPTC, 32, 3651 KSTAT_KV_U_PACKETS }, 3652 [ix_counter_gorc] = { "rx good", IXGBE_GORCL, 36, 3653 KSTAT_KV_U_BYTES }, 3654 [ix_counter_gotc] = { "tx good", IXGBE_GOTCL, 36, 3655 KSTAT_KV_U_BYTES }, 3656 [ix_counter_ruc] = { "rx undersize", IXGBE_RUC, 32, 3657 KSTAT_KV_U_PACKETS }, 3658 [ix_counter_rfc] = { "rx fragment", IXGBE_RFC, 32, 3659 KSTAT_KV_U_PACKETS }, 3660 [ix_counter_roc] = { "rx oversize", IXGBE_ROC, 32, 3661 KSTAT_KV_U_PACKETS }, 3662 [ix_counter_rjc] = { "rx jabber", IXGBE_RJC, 32, 3663 KSTAT_KV_U_PACKETS }, 3664 [ix_counter_tor] = { "rx total", IXGBE_TORL, 36, 3665 KSTAT_KV_U_BYTES }, 3666 [ix_counter_tpr] = { "rx total", IXGBE_TPR, 32, 3667 KSTAT_KV_U_PACKETS }, 3668 [ix_counter_tpt] = { "tx total", IXGBE_TPT, 32, 3669 KSTAT_KV_U_PACKETS }, 3670 [ix_counter_gprc] = { "rx good", IXGBE_GPRC, 32, 3671 KSTAT_KV_U_PACKETS }, 3672 [ix_counter_bprc] = { "rx bcast", IXGBE_BPRC, 32, 3673 KSTAT_KV_U_PACKETS }, 3674 [ix_counter_mprc] = { "rx mcast", IXGBE_MPRC, 32, 3675 KSTAT_KV_U_PACKETS }, 3676 [ix_counter_ptc64] = { "tx 64B", IXGBE_PTC64, 32, 3677 KSTAT_KV_U_PACKETS }, 3678 [ix_counter_ptc127] = { "tx 65-127B", IXGBE_PTC127, 32, 3679 KSTAT_KV_U_PACKETS }, 3680 [ix_counter_ptc255] = { "tx 128-255B", IXGBE_PTC255, 32, 3681 KSTAT_KV_U_PACKETS }, 3682 [ix_counter_ptc511] = { "tx 256-511B", IXGBE_PTC511, 32, 3683 KSTAT_KV_U_PACKETS }, 3684 [ix_counter_ptc1023] = { "tx 512-1023B", IXGBE_PTC1023, 32, 3685 KSTAT_KV_U_PACKETS }, 3686 [ix_counter_ptc1522] = { "tx 1024-maxB", IXGBE_PTC1522, 32, 3687 KSTAT_KV_U_PACKETS }, 3688 [ix_counter_mptc] = { "tx mcast", IXGBE_MPTC, 32, 3689 KSTAT_KV_U_PACKETS }, 3690 [ix_counter_bptc] = { "tx bcast", IXGBE_BPTC, 32, 3691 KSTAT_KV_U_PACKETS }, 3692 }; 3693 3694 struct ix_rxq_kstats { 3695 struct kstat_kv qprc; 3696 struct kstat_kv qbrc; 3697 struct kstat_kv qprdc; 3698 }; 3699 3700 static const struct ix_rxq_kstats ix_rxq_kstats_tpl = { 3701 KSTAT_KV_UNIT_INITIALIZER("packets", 3702 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3703 KSTAT_KV_UNIT_INITIALIZER("bytes", 3704 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3705 KSTAT_KV_UNIT_INITIALIZER("qdrops", 3706 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3707 }; 3708 3709 struct ix_txq_kstats { 3710 struct kstat_kv qptc; 3711 struct kstat_kv qbtc; 3712 }; 3713 3714 static const struct ix_txq_kstats ix_txq_kstats_tpl = { 3715 KSTAT_KV_UNIT_INITIALIZER("packets", 3716 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3717 KSTAT_KV_UNIT_INITIALIZER("bytes", 3718 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3719 }; 3720 3721 static int ix_kstats_read(struct kstat *ks); 3722 static int ix_rxq_kstats_read(struct kstat *ks); 3723 static int ix_txq_kstats_read(struct kstat *ks); 3724 3725 static void 3726 ix_kstats(struct ix_softc *sc) 3727 { 3728 struct kstat *ks; 3729 struct kstat_kv *kvs; 3730 unsigned int i; 3731 3732 mtx_init(&sc->sc_kstat_mtx, IPL_SOFTCLOCK); 3733 timeout_set(&sc->sc_kstat_tmo, ix_kstats_tick, sc); 3734 3735 ks = kstat_create(sc->dev.dv_xname, 0, "ix-stats", 0, 3736 KSTAT_T_KV, 0); 3737 if (ks == NULL) 3738 return; 3739 3740 kvs = mallocarray(nitems(ix_counters), sizeof(*kvs), 3741 M_DEVBUF, M_WAITOK|M_ZERO); 3742 3743 for (i = 0; i < nitems(ix_counters); i++) { 3744 const struct ix_counter *ixc = &ix_counters[i]; 3745 3746 kstat_kv_unit_init(&kvs[i], ixc->name, 3747 KSTAT_KV_T_COUNTER64, ixc->unit); 3748 } 3749 3750 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3751 ks->ks_softc = sc; 3752 ks->ks_data = kvs; 3753 ks->ks_datalen = nitems(ix_counters) * sizeof(*kvs); 3754 ks->ks_read = ix_kstats_read; 3755 3756 sc->sc_kstat = ks; 3757 kstat_install(ks); 3758 } 3759 3760 static void 3761 ix_rxq_kstats(struct ix_softc *sc, struct rx_ring *rxr) 3762 { 3763 struct ix_rxq_kstats *stats; 3764 struct kstat *ks; 3765 3766 ks = kstat_create(sc->dev.dv_xname, 0, "ix-rxq", rxr->me, 3767 KSTAT_T_KV, 0); 3768 if (ks == NULL) 3769 return; 3770 3771 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3772 *stats = ix_rxq_kstats_tpl; 3773 3774 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3775 ks->ks_softc = rxr; 3776 ks->ks_data = stats; 3777 ks->ks_datalen = sizeof(*stats); 3778 ks->ks_read = ix_rxq_kstats_read; 3779 3780 rxr->kstat = ks; 3781 kstat_install(ks); 3782 } 3783 3784 static void 3785 ix_txq_kstats(struct ix_softc *sc, struct tx_ring *txr) 3786 { 3787 struct ix_txq_kstats *stats; 3788 struct kstat *ks; 3789 3790 ks = kstat_create(sc->dev.dv_xname, 0, "ix-txq", txr->me, 3791 KSTAT_T_KV, 0); 3792 if (ks == NULL) 3793 return; 3794 3795 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3796 *stats = ix_txq_kstats_tpl; 3797 3798 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3799 ks->ks_softc = txr; 3800 ks->ks_data = stats; 3801 ks->ks_datalen = sizeof(*stats); 3802 ks->ks_read = ix_txq_kstats_read; 3803 3804 txr->kstat = ks; 3805 kstat_install(ks); 3806 } 3807 3808 /********************************************************************** 3809 * 3810 * Update the board statistics counters. 3811 * 3812 **********************************************************************/ 3813 3814 static void 3815 ix_kstats_tick(void *arg) 3816 { 3817 struct ix_softc *sc = arg; 3818 int i; 3819 3820 timeout_add_sec(&sc->sc_kstat_tmo, 1); 3821 3822 mtx_enter(&sc->sc_kstat_mtx); 3823 ix_kstats_read(sc->sc_kstat); 3824 for (i = 0; i < sc->num_queues; i++) { 3825 ix_rxq_kstats_read(sc->rx_rings[i].kstat); 3826 ix_txq_kstats_read(sc->tx_rings[i].kstat); 3827 } 3828 mtx_leave(&sc->sc_kstat_mtx); 3829 } 3830 3831 static uint64_t 3832 ix_read36(struct ixgbe_hw *hw, bus_size_t loreg, bus_size_t hireg) 3833 { 3834 uint64_t lo, hi; 3835 3836 lo = IXGBE_READ_REG(hw, loreg); 3837 hi = IXGBE_READ_REG(hw, hireg); 3838 3839 return (((hi & 0xf) << 32) | lo); 3840 } 3841 3842 static int 3843 ix_kstats_read(struct kstat *ks) 3844 { 3845 struct ix_softc *sc = ks->ks_softc; 3846 struct kstat_kv *kvs = ks->ks_data; 3847 struct ixgbe_hw *hw = &sc->hw; 3848 unsigned int i; 3849 3850 for (i = 0; i < nitems(ix_counters); i++) { 3851 const struct ix_counter *ixc = &ix_counters[i]; 3852 uint32_t reg = ixc->reg; 3853 uint64_t v; 3854 3855 if (reg == 0) 3856 continue; 3857 3858 if (ixc->width > 32) { 3859 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3860 v = IXGBE_READ_REG(hw, reg + 4); 3861 else 3862 v = ix_read36(hw, reg, reg + 4); 3863 } else 3864 v = IXGBE_READ_REG(hw, reg); 3865 3866 kstat_kv_u64(&kvs[i]) += v; 3867 } 3868 3869 /* handle the exceptions */ 3870 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3871 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3872 IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3873 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3874 IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3875 } else { 3876 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3877 IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3878 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3879 IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3880 } 3881 3882 getnanouptime(&ks->ks_updated); 3883 3884 return (0); 3885 } 3886 3887 int 3888 ix_rxq_kstats_read(struct kstat *ks) 3889 { 3890 struct ix_rxq_kstats *stats = ks->ks_data; 3891 struct rx_ring *rxr = ks->ks_softc; 3892 struct ix_softc *sc = rxr->sc; 3893 struct ixgbe_hw *hw = &sc->hw; 3894 uint32_t i = rxr->me; 3895 3896 kstat_kv_u64(&stats->qprc) += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3897 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3898 kstat_kv_u64(&stats->qprdc) += 3899 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3900 kstat_kv_u64(&stats->qbrc) += 3901 IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 3902 } else { 3903 kstat_kv_u64(&stats->qprdc) += 3904 IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3905 kstat_kv_u64(&stats->qbrc) += 3906 ix_read36(hw, IXGBE_QBRC_L(i), IXGBE_QBRC_H(i)); 3907 } 3908 3909 getnanouptime(&ks->ks_updated); 3910 3911 return (0); 3912 } 3913 3914 int 3915 ix_txq_kstats_read(struct kstat *ks) 3916 { 3917 struct ix_txq_kstats *stats = ks->ks_data; 3918 struct tx_ring *txr = ks->ks_softc; 3919 struct ix_softc *sc = txr->sc; 3920 struct ixgbe_hw *hw = &sc->hw; 3921 uint32_t i = txr->me; 3922 3923 kstat_kv_u64(&stats->qptc) += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3924 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3925 kstat_kv_u64(&stats->qbtc) += 3926 IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 3927 } else { 3928 kstat_kv_u64(&stats->qbtc) += 3929 ix_read36(hw, IXGBE_QBTC_L(i), IXGBE_QBTC_H(i)); 3930 } 3931 3932 getnanouptime(&ks->ks_updated); 3933 3934 return (0); 3935 } 3936 #endif /* NKVSTAT > 0 */ 3937 3938 void 3939 ixgbe_map_queue_statistics(struct ix_softc *sc) 3940 { 3941 int i; 3942 uint32_t r; 3943 3944 for (i = 0; i < 32; i++) { 3945 /* 3946 * Queues 0-15 are mapped 1:1 3947 * Queue 0 -> Counter 0 3948 * Queue 1 -> Counter 1 3949 * Queue 2 -> Counter 2.... 3950 * Queues 16-127 are mapped to Counter 0 3951 */ 3952 if (i < 4) { 3953 r = (i * 4 + 0); 3954 r |= (i * 4 + 1) << 8; 3955 r |= (i * 4 + 2) << 16; 3956 r |= (i * 4 + 3) << 24; 3957 } else 3958 r = 0; 3959 3960 IXGBE_WRITE_REG(&sc->hw, IXGBE_RQSMR(i), r); 3961 IXGBE_WRITE_REG(&sc->hw, IXGBE_TQSM(i), r); 3962 } 3963 } 3964