1 /* $OpenBSD: if_ix.c,v 1.191 2023/01/26 07:32:39 deraadt Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2013, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */ 36 37 #include <dev/pci/if_ix.h> 38 #include <dev/pci/ixgbe_type.h> 39 40 /********************************************************************* 41 * Driver version 42 *********************************************************************/ 43 /* char ixgbe_driver_version[] = "2.5.13"; */ 44 45 /********************************************************************* 46 * PCI Device ID Table 47 * 48 * Used by probe to select devices to load on 49 *********************************************************************/ 50 51 const struct pci_matchid ixgbe_devices[] = { 52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 }, 53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX }, 54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT }, 57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT2 }, 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP }, 63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM }, 64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL }, 65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 }, 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BP }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_T3_LOM }, 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF_QP }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF2 }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_FCOE }, 77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599EN_SFP }, 78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_QSFP_SF_QP }, 79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T }, 80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T1 }, 81 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T }, 82 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T1 }, 83 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KX4 }, 84 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KR }, 85 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_SFP }, 86 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_10G_T }, 87 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_1G_T }, 88 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR }, 89 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR_L }, 90 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP_N }, 91 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP }, 92 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII }, 93 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII_L }, 94 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_10G_T }, 95 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T }, 96 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T_L } 97 }; 98 99 /********************************************************************* 100 * Function prototypes 101 *********************************************************************/ 102 int ixgbe_probe(struct device *, void *, void *); 103 void ixgbe_attach(struct device *, struct device *, void *); 104 int ixgbe_detach(struct device *, int); 105 int ixgbe_activate(struct device *, int); 106 void ixgbe_start(struct ifqueue *); 107 int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 108 int ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *); 109 int ixgbe_get_sffpage(struct ix_softc *, struct if_sffpage *); 110 void ixgbe_watchdog(struct ifnet *); 111 void ixgbe_init(void *); 112 void ixgbe_stop(void *); 113 void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 114 int ixgbe_media_change(struct ifnet *); 115 void ixgbe_identify_hardware(struct ix_softc *); 116 int ixgbe_allocate_pci_resources(struct ix_softc *); 117 int ixgbe_allocate_legacy(struct ix_softc *); 118 int ixgbe_allocate_msix(struct ix_softc *); 119 void ixgbe_setup_msix(struct ix_softc *); 120 int ixgbe_allocate_queues(struct ix_softc *); 121 void ixgbe_free_pci_resources(struct ix_softc *); 122 void ixgbe_local_timer(void *); 123 void ixgbe_setup_interface(struct ix_softc *); 124 void ixgbe_config_gpie(struct ix_softc *); 125 void ixgbe_config_delay_values(struct ix_softc *); 126 void ixgbe_add_media_types(struct ix_softc *); 127 void ixgbe_config_link(struct ix_softc *); 128 129 int ixgbe_allocate_transmit_buffers(struct tx_ring *); 130 int ixgbe_setup_transmit_structures(struct ix_softc *); 131 int ixgbe_setup_transmit_ring(struct tx_ring *); 132 void ixgbe_initialize_transmit_units(struct ix_softc *); 133 void ixgbe_free_transmit_structures(struct ix_softc *); 134 void ixgbe_free_transmit_buffers(struct tx_ring *); 135 136 int ixgbe_allocate_receive_buffers(struct rx_ring *); 137 int ixgbe_setup_receive_structures(struct ix_softc *); 138 int ixgbe_setup_receive_ring(struct rx_ring *); 139 void ixgbe_initialize_receive_units(struct ix_softc *); 140 void ixgbe_free_receive_structures(struct ix_softc *); 141 void ixgbe_free_receive_buffers(struct rx_ring *); 142 void ixgbe_initialize_rss_mapping(struct ix_softc *); 143 int ixgbe_rxfill(struct rx_ring *); 144 void ixgbe_rxrefill(void *); 145 146 int ixgbe_intr(struct ix_softc *sc); 147 void ixgbe_enable_intr(struct ix_softc *); 148 void ixgbe_disable_intr(struct ix_softc *); 149 int ixgbe_txeof(struct tx_ring *); 150 int ixgbe_rxeof(struct rx_ring *); 151 void ixgbe_rx_checksum(uint32_t, struct mbuf *); 152 void ixgbe_iff(struct ix_softc *); 153 void ixgbe_map_queue_statistics(struct ix_softc *); 154 void ixgbe_update_link_status(struct ix_softc *); 155 int ixgbe_get_buf(struct rx_ring *, int); 156 int ixgbe_encap(struct tx_ring *, struct mbuf *); 157 int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 158 struct ixgbe_dma_alloc *, int); 159 void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 160 static int 161 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *, uint32_t *, 162 uint32_t *); 163 void ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 164 void ixgbe_configure_ivars(struct ix_softc *); 165 uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 166 167 void ixgbe_setup_vlan_hw_support(struct ix_softc *); 168 169 /* Support for pluggable optic modules */ 170 void ixgbe_handle_mod(struct ix_softc *); 171 void ixgbe_handle_msf(struct ix_softc *); 172 void ixgbe_handle_phy(struct ix_softc *); 173 174 /* Legacy (single vector interrupt handler */ 175 int ixgbe_legacy_intr(void *); 176 void ixgbe_enable_queue(struct ix_softc *, uint32_t); 177 void ixgbe_enable_queues(struct ix_softc *); 178 void ixgbe_disable_queue(struct ix_softc *, uint32_t); 179 void ixgbe_rearm_queue(struct ix_softc *, uint32_t); 180 181 /* MSI-X (multiple vectors interrupt handlers) */ 182 int ixgbe_link_intr(void *); 183 int ixgbe_queue_intr(void *); 184 185 #if NKSTAT > 0 186 static void ix_kstats(struct ix_softc *); 187 static void ix_rxq_kstats(struct ix_softc *, struct rx_ring *); 188 static void ix_txq_kstats(struct ix_softc *, struct tx_ring *); 189 static void ix_kstats_tick(void *); 190 #endif 191 192 /********************************************************************* 193 * OpenBSD Device Interface Entry Points 194 *********************************************************************/ 195 196 struct cfdriver ix_cd = { 197 NULL, "ix", DV_IFNET 198 }; 199 200 const struct cfattach ix_ca = { 201 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach, 202 ixgbe_activate 203 }; 204 205 int ixgbe_smart_speed = ixgbe_smart_speed_on; 206 int ixgbe_enable_msix = 1; 207 208 /********************************************************************* 209 * Device identification routine 210 * 211 * ixgbe_probe determines if the driver should be loaded on 212 * adapter based on PCI vendor/device id of the adapter. 213 * 214 * return 0 on success, positive on failure 215 *********************************************************************/ 216 217 int 218 ixgbe_probe(struct device *parent, void *match, void *aux) 219 { 220 INIT_DEBUGOUT("ixgbe_probe: begin"); 221 222 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 223 nitems(ixgbe_devices))); 224 } 225 226 /********************************************************************* 227 * Device initialization routine 228 * 229 * The attach entry point is called when the driver is being loaded. 230 * This routine identifies the type of hardware, allocates all resources 231 * and initializes the hardware. 232 * 233 * return 0 on success, positive on failure 234 *********************************************************************/ 235 236 void 237 ixgbe_attach(struct device *parent, struct device *self, void *aux) 238 { 239 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 240 struct ix_softc *sc = (struct ix_softc *)self; 241 int error = 0; 242 uint16_t csum; 243 uint32_t ctrl_ext; 244 struct ixgbe_hw *hw = &sc->hw; 245 246 INIT_DEBUGOUT("ixgbe_attach: begin"); 247 248 sc->osdep.os_sc = sc; 249 sc->osdep.os_pa = *pa; 250 251 rw_init(&sc->sfflock, "ixsff"); 252 253 #if NKSTAT > 0 254 ix_kstats(sc); 255 #endif 256 257 /* Determine hardware revision */ 258 ixgbe_identify_hardware(sc); 259 260 /* Indicate to RX setup to use Jumbo Clusters */ 261 sc->num_tx_desc = DEFAULT_TXD; 262 sc->num_rx_desc = DEFAULT_RXD; 263 264 /* Do base PCI setup - map BAR0 */ 265 if (ixgbe_allocate_pci_resources(sc)) 266 goto err_out; 267 268 /* Allocate our TX/RX Queues */ 269 if (ixgbe_allocate_queues(sc)) 270 goto err_out; 271 272 /* Allocate multicast array memory. */ 273 sc->mta = mallocarray(IXGBE_ETH_LENGTH_OF_ADDRESS, 274 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 275 if (sc->mta == NULL) { 276 printf(": Can not allocate multicast setup array\n"); 277 goto err_late; 278 } 279 280 /* Initialize the shared code */ 281 error = ixgbe_init_shared_code(hw); 282 if (error) { 283 printf(": Unable to initialize the shared code\n"); 284 goto err_late; 285 } 286 287 /* Make sure we have a good EEPROM before we read from it */ 288 if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) { 289 printf(": The EEPROM Checksum Is Not Valid\n"); 290 goto err_late; 291 } 292 293 error = ixgbe_init_hw(hw); 294 if (error == IXGBE_ERR_EEPROM_VERSION) { 295 printf(": This device is a pre-production adapter/" 296 "LOM. Please be aware there may be issues associated " 297 "with your hardware.\nIf you are experiencing problems " 298 "please contact your Intel or hardware representative " 299 "who provided you with this hardware.\n"); 300 } else if (error && (error != IXGBE_ERR_SFP_NOT_PRESENT && 301 error != IXGBE_ERR_SFP_NOT_SUPPORTED)) { 302 printf(": Hardware Initialization Failure\n"); 303 goto err_late; 304 } 305 306 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 307 IXGBE_ETH_LENGTH_OF_ADDRESS); 308 309 if (sc->sc_intrmap) 310 error = ixgbe_allocate_msix(sc); 311 else 312 error = ixgbe_allocate_legacy(sc); 313 if (error) 314 goto err_late; 315 316 /* Enable the optics for 82599 SFP+ fiber */ 317 if (sc->hw.mac.ops.enable_tx_laser) 318 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 319 320 /* Enable power to the phy */ 321 if (hw->phy.ops.set_phy_power) 322 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 323 324 /* Setup OS specific network interface */ 325 ixgbe_setup_interface(sc); 326 327 /* Get the PCI-E bus info and determine LAN ID */ 328 hw->mac.ops.get_bus_info(hw); 329 330 /* Set an initial default flow control value */ 331 sc->fc = ixgbe_fc_full; 332 333 /* let hardware know driver is loaded */ 334 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 335 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 336 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 337 338 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 339 340 INIT_DEBUGOUT("ixgbe_attach: end"); 341 return; 342 343 err_late: 344 ixgbe_free_transmit_structures(sc); 345 ixgbe_free_receive_structures(sc); 346 err_out: 347 ixgbe_free_pci_resources(sc); 348 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 349 MAX_NUM_MULTICAST_ADDRESSES); 350 } 351 352 /********************************************************************* 353 * Device removal routine 354 * 355 * The detach entry point is called when the driver is being removed. 356 * This routine stops the adapter and deallocates all the resources 357 * that were allocated for driver operation. 358 * 359 * return 0 on success, positive on failure 360 *********************************************************************/ 361 362 int 363 ixgbe_detach(struct device *self, int flags) 364 { 365 struct ix_softc *sc = (struct ix_softc *)self; 366 struct ifnet *ifp = &sc->arpcom.ac_if; 367 uint32_t ctrl_ext; 368 369 INIT_DEBUGOUT("ixgbe_detach: begin"); 370 371 ixgbe_stop(sc); 372 373 /* let hardware know driver is unloading */ 374 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 375 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 376 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 377 378 ether_ifdetach(ifp); 379 if_detach(ifp); 380 381 ixgbe_free_pci_resources(sc); 382 383 ixgbe_free_transmit_structures(sc); 384 ixgbe_free_receive_structures(sc); 385 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 386 MAX_NUM_MULTICAST_ADDRESSES); 387 388 /* XXX kstat */ 389 390 return (0); 391 } 392 393 int 394 ixgbe_activate(struct device *self, int act) 395 { 396 struct ix_softc *sc = (struct ix_softc *)self; 397 struct ifnet *ifp = &sc->arpcom.ac_if; 398 struct ixgbe_hw *hw = &sc->hw; 399 uint32_t ctrl_ext; 400 int rv = 0; 401 402 switch (act) { 403 case DVACT_QUIESCE: 404 if (ifp->if_flags & IFF_RUNNING) 405 ixgbe_stop(sc); 406 break; 407 case DVACT_RESUME: 408 ixgbe_init_hw(hw); 409 410 /* Enable the optics for 82599 SFP+ fiber */ 411 if (sc->hw.mac.ops.enable_tx_laser) 412 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 413 414 /* Enable power to the phy */ 415 if (hw->phy.ops.set_phy_power) 416 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 417 418 /* Get the PCI-E bus info and determine LAN ID */ 419 hw->mac.ops.get_bus_info(hw); 420 421 /* let hardware know driver is loaded */ 422 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 423 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 424 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 425 426 if (ifp->if_flags & IFF_UP) 427 ixgbe_init(sc); 428 break; 429 default: 430 break; 431 } 432 return (rv); 433 } 434 435 /********************************************************************* 436 * Transmit entry point 437 * 438 * ixgbe_start is called by the stack to initiate a transmit. 439 * The driver will remain in this routine as long as there are 440 * packets to transmit and transmit resources are available. 441 * In case resources are not available stack is notified and 442 * the packet is requeued. 443 **********************************************************************/ 444 445 void 446 ixgbe_start(struct ifqueue *ifq) 447 { 448 struct ifnet *ifp = ifq->ifq_if; 449 struct ix_softc *sc = ifp->if_softc; 450 struct tx_ring *txr = ifq->ifq_softc; 451 struct mbuf *m_head; 452 unsigned int head, free, used; 453 int post = 0; 454 455 if (!sc->link_up) 456 return; 457 458 head = txr->next_avail_desc; 459 free = txr->next_to_clean; 460 if (free <= head) 461 free += sc->num_tx_desc; 462 free -= head; 463 464 membar_consumer(); 465 466 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 467 0, txr->txdma.dma_map->dm_mapsize, 468 BUS_DMASYNC_POSTWRITE); 469 470 for (;;) { 471 /* Check that we have the minimal number of TX descriptors. */ 472 if (free <= IXGBE_TX_OP_THRESHOLD) { 473 ifq_set_oactive(ifq); 474 break; 475 } 476 477 m_head = ifq_dequeue(ifq); 478 if (m_head == NULL) 479 break; 480 481 used = ixgbe_encap(txr, m_head); 482 if (used == 0) { 483 m_freem(m_head); 484 continue; 485 } 486 487 free -= used; 488 489 #if NBPFILTER > 0 490 if (ifp->if_bpf) 491 bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 492 #endif 493 494 /* Set timeout in case hardware has problems transmitting */ 495 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 496 ifp->if_timer = IXGBE_TX_TIMEOUT; 497 498 post = 1; 499 } 500 501 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 502 0, txr->txdma.dma_map->dm_mapsize, 503 BUS_DMASYNC_PREWRITE); 504 505 /* 506 * Advance the Transmit Descriptor Tail (Tdt), this tells the 507 * hardware that this frame is available to transmit. 508 */ 509 if (post) 510 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 511 txr->next_avail_desc); 512 } 513 514 /********************************************************************* 515 * Ioctl entry point 516 * 517 * ixgbe_ioctl is called when the user wants to configure the 518 * interface. 519 * 520 * return 0 on success, positive on failure 521 **********************************************************************/ 522 523 int 524 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 525 { 526 struct ix_softc *sc = ifp->if_softc; 527 struct ifreq *ifr = (struct ifreq *) data; 528 int s, error = 0; 529 530 s = splnet(); 531 532 switch (command) { 533 case SIOCSIFADDR: 534 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 535 ifp->if_flags |= IFF_UP; 536 if (!(ifp->if_flags & IFF_RUNNING)) 537 ixgbe_init(sc); 538 break; 539 540 case SIOCSIFFLAGS: 541 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 542 if (ifp->if_flags & IFF_UP) { 543 if (ifp->if_flags & IFF_RUNNING) 544 error = ENETRESET; 545 else 546 ixgbe_init(sc); 547 } else { 548 if (ifp->if_flags & IFF_RUNNING) 549 ixgbe_stop(sc); 550 } 551 break; 552 553 case SIOCSIFMEDIA: 554 case SIOCGIFMEDIA: 555 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 556 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 557 break; 558 559 case SIOCGIFRXR: 560 error = ixgbe_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 561 break; 562 563 case SIOCGIFSFFPAGE: 564 error = rw_enter(&sc->sfflock, RW_WRITE|RW_INTR); 565 if (error != 0) 566 break; 567 568 error = ixgbe_get_sffpage(sc, (struct if_sffpage *)data); 569 rw_exit(&sc->sfflock); 570 break; 571 572 default: 573 error = ether_ioctl(ifp, &sc->arpcom, command, data); 574 } 575 576 if (error == ENETRESET) { 577 if (ifp->if_flags & IFF_RUNNING) { 578 ixgbe_disable_intr(sc); 579 ixgbe_iff(sc); 580 ixgbe_enable_intr(sc); 581 ixgbe_enable_queues(sc); 582 } 583 error = 0; 584 } 585 586 splx(s); 587 return (error); 588 } 589 590 int 591 ixgbe_get_sffpage(struct ix_softc *sc, struct if_sffpage *sff) 592 { 593 struct ixgbe_hw *hw = &sc->hw; 594 uint32_t swfw_mask = hw->phy.phy_semaphore_mask; 595 uint8_t page; 596 size_t i; 597 int error = EIO; 598 599 if (hw->phy.type == ixgbe_phy_fw) 600 return (ENODEV); 601 602 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 603 return (EBUSY); /* XXX */ 604 605 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 606 if (hw->phy.ops.read_i2c_byte_unlocked(hw, 127, 607 IFSFF_ADDR_EEPROM, &page)) 608 goto error; 609 if (page != sff->sff_page && 610 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 611 IFSFF_ADDR_EEPROM, sff->sff_page)) 612 goto error; 613 } 614 615 for (i = 0; i < sizeof(sff->sff_data); i++) { 616 if (hw->phy.ops.read_i2c_byte_unlocked(hw, i, 617 sff->sff_addr, &sff->sff_data[i])) 618 goto error; 619 } 620 621 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 622 if (page != sff->sff_page && 623 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 624 IFSFF_ADDR_EEPROM, page)) 625 goto error; 626 } 627 628 error = 0; 629 error: 630 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 631 return (error); 632 } 633 634 int 635 ixgbe_rxrinfo(struct ix_softc *sc, struct if_rxrinfo *ifri) 636 { 637 struct if_rxring_info *ifr, ifr1; 638 struct rx_ring *rxr; 639 int error, i; 640 u_int n = 0; 641 642 if (sc->num_queues > 1) { 643 if ((ifr = mallocarray(sc->num_queues, sizeof(*ifr), M_DEVBUF, 644 M_WAITOK | M_ZERO)) == NULL) 645 return (ENOMEM); 646 } else 647 ifr = &ifr1; 648 649 for (i = 0; i < sc->num_queues; i++) { 650 rxr = &sc->rx_rings[i]; 651 ifr[n].ifr_size = MCLBYTES; 652 snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i); 653 ifr[n].ifr_info = rxr->rx_ring; 654 n++; 655 } 656 657 error = if_rxr_info_ioctl(ifri, sc->num_queues, ifr); 658 659 if (sc->num_queues > 1) 660 free(ifr, M_DEVBUF, sc->num_queues * sizeof(*ifr)); 661 return (error); 662 } 663 664 /********************************************************************* 665 * Watchdog entry point 666 * 667 **********************************************************************/ 668 669 void 670 ixgbe_watchdog(struct ifnet * ifp) 671 { 672 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 673 struct tx_ring *txr = sc->tx_rings; 674 struct ixgbe_hw *hw = &sc->hw; 675 int tx_hang = FALSE; 676 int i; 677 678 /* 679 * The timer is set to 5 every time ixgbe_start() queues a packet. 680 * Anytime all descriptors are clean the timer is set to 0. 681 */ 682 for (i = 0; i < sc->num_queues; i++, txr++) { 683 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 684 continue; 685 else { 686 tx_hang = TRUE; 687 break; 688 } 689 } 690 if (tx_hang == FALSE) 691 return; 692 693 /* 694 * If we are in this routine because of pause frames, then don't 695 * reset the hardware. 696 */ 697 if (!(IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXON)) { 698 for (i = 0; i < sc->num_queues; i++, txr++) 699 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 700 ifp->if_timer = IXGBE_TX_TIMEOUT; 701 return; 702 } 703 704 705 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 706 for (i = 0; i < sc->num_queues; i++, txr++) { 707 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 708 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 709 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 710 printf("%s: TX(%d) Next TX to Clean = %d\n", ifp->if_xname, 711 i, txr->next_to_clean); 712 } 713 ifp->if_flags &= ~IFF_RUNNING; 714 715 ixgbe_init(sc); 716 } 717 718 /********************************************************************* 719 * Init entry point 720 * 721 * This routine is used in two ways. It is used by the stack as 722 * init entry point in network interface structure. It is also used 723 * by the driver as a hw/sw initialization routine to get to a 724 * consistent state. 725 * 726 * return 0 on success, positive on failure 727 **********************************************************************/ 728 #define IXGBE_MHADD_MFS_SHIFT 16 729 730 void 731 ixgbe_init(void *arg) 732 { 733 struct ix_softc *sc = (struct ix_softc *)arg; 734 struct ifnet *ifp = &sc->arpcom.ac_if; 735 struct rx_ring *rxr = sc->rx_rings; 736 uint32_t k, txdctl, rxdctl, rxctrl, mhadd, itr; 737 int i, s, err; 738 739 INIT_DEBUGOUT("ixgbe_init: begin"); 740 741 s = splnet(); 742 743 ixgbe_stop(sc); 744 745 /* reprogram the RAR[0] in case user changed it. */ 746 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 747 748 /* Get the latest mac address, User can use a LAA */ 749 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 750 IXGBE_ETH_LENGTH_OF_ADDRESS); 751 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1); 752 sc->hw.addr_ctrl.rar_used_count = 1; 753 754 /* Prepare transmit descriptors and buffers */ 755 if (ixgbe_setup_transmit_structures(sc)) { 756 printf("%s: Could not setup transmit structures\n", 757 ifp->if_xname); 758 ixgbe_stop(sc); 759 splx(s); 760 return; 761 } 762 763 ixgbe_init_hw(&sc->hw); 764 ixgbe_initialize_transmit_units(sc); 765 766 /* Use 2k clusters, even for jumbo frames */ 767 sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN; 768 769 /* Prepare receive descriptors and buffers */ 770 if (ixgbe_setup_receive_structures(sc)) { 771 printf("%s: Could not setup receive structures\n", 772 ifp->if_xname); 773 ixgbe_stop(sc); 774 splx(s); 775 return; 776 } 777 778 /* Configure RX settings */ 779 ixgbe_initialize_receive_units(sc); 780 781 /* Enable SDP & MSIX interrupts based on adapter */ 782 ixgbe_config_gpie(sc); 783 784 /* Program promiscuous mode and multicast filters. */ 785 ixgbe_iff(sc); 786 787 /* Set MRU size */ 788 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 789 mhadd &= ~IXGBE_MHADD_MFS_MASK; 790 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 791 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 792 793 /* Now enable all the queues */ 794 for (i = 0; i < sc->num_queues; i++) { 795 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 796 txdctl |= IXGBE_TXDCTL_ENABLE; 797 /* Set WTHRESH to 8, burst writeback */ 798 txdctl |= (8 << 16); 799 /* 800 * When the internal queue falls below PTHRESH (16), 801 * start prefetching as long as there are at least 802 * HTHRESH (1) buffers ready. 803 */ 804 txdctl |= (16 << 0) | (1 << 8); 805 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 806 } 807 808 for (i = 0; i < sc->num_queues; i++) { 809 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 810 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 811 /* 812 * PTHRESH = 21 813 * HTHRESH = 4 814 * WTHRESH = 8 815 */ 816 rxdctl &= ~0x3FFFFF; 817 rxdctl |= 0x080420; 818 } 819 rxdctl |= IXGBE_RXDCTL_ENABLE; 820 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 821 for (k = 0; k < 10; k++) { 822 if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) & 823 IXGBE_RXDCTL_ENABLE) 824 break; 825 else 826 msec_delay(1); 827 } 828 IXGBE_WRITE_FLUSH(&sc->hw); 829 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled); 830 } 831 832 /* Set up VLAN support and filter */ 833 ixgbe_setup_vlan_hw_support(sc); 834 835 /* Enable Receive engine */ 836 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 837 if (sc->hw.mac.type == ixgbe_mac_82598EB) 838 rxctrl |= IXGBE_RXCTRL_DMBYPS; 839 rxctrl |= IXGBE_RXCTRL_RXEN; 840 sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl); 841 842 /* Set up MSI/X routing */ 843 if (sc->sc_intrmap) { 844 ixgbe_configure_ivars(sc); 845 /* Set up auto-mask */ 846 if (sc->hw.mac.type == ixgbe_mac_82598EB) 847 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 848 else { 849 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 850 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 851 } 852 } else { /* Simple settings for Legacy/MSI */ 853 ixgbe_set_ivar(sc, 0, 0, 0); 854 ixgbe_set_ivar(sc, 0, 0, 1); 855 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 856 } 857 858 /* Check on any SFP devices that need to be kick-started */ 859 if (sc->hw.phy.type == ixgbe_phy_none) { 860 err = sc->hw.phy.ops.identify(&sc->hw); 861 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 862 printf("Unsupported SFP+ module type was detected.\n"); 863 splx(s); 864 return; 865 } 866 } 867 868 /* Setup interrupt moderation */ 869 itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8; 870 if (sc->hw.mac.type != ixgbe_mac_82598EB) 871 itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS; 872 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr); 873 874 if (sc->sc_intrmap) { 875 /* Set moderation on the Link interrupt */ 876 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(sc->linkvec), 877 IXGBE_LINK_ITR); 878 } 879 880 /* Enable power to the phy */ 881 if (sc->hw.phy.ops.set_phy_power) 882 sc->hw.phy.ops.set_phy_power(&sc->hw, TRUE); 883 884 /* Config/Enable Link */ 885 ixgbe_config_link(sc); 886 887 /* Hardware Packet Buffer & Flow Control setup */ 888 ixgbe_config_delay_values(sc); 889 890 /* Initialize the FC settings */ 891 sc->hw.mac.ops.start_hw(&sc->hw); 892 893 /* And now turn on interrupts */ 894 ixgbe_enable_intr(sc); 895 ixgbe_enable_queues(sc); 896 897 /* Now inform the stack we're ready */ 898 ifp->if_flags |= IFF_RUNNING; 899 for (i = 0; i < sc->num_queues; i++) 900 ifq_clr_oactive(ifp->if_ifqs[i]); 901 902 #if NKSTAT > 0 903 ix_kstats_tick(sc); 904 #endif 905 906 splx(s); 907 } 908 909 void 910 ixgbe_config_gpie(struct ix_softc *sc) 911 { 912 struct ixgbe_hw *hw = &sc->hw; 913 uint32_t gpie; 914 915 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 916 917 /* Fan Failure Interrupt */ 918 if (hw->device_id == IXGBE_DEV_ID_82598AT) 919 gpie |= IXGBE_SDP1_GPIEN; 920 921 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 922 /* Add for Module detection */ 923 gpie |= IXGBE_SDP2_GPIEN; 924 925 /* Media ready */ 926 if (hw->device_id != IXGBE_DEV_ID_82599_QSFP_SF_QP) 927 gpie |= IXGBE_SDP1_GPIEN; 928 929 /* 930 * Set LL interval to max to reduce the number of low latency 931 * interrupts hitting the card when the ring is getting full. 932 */ 933 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 934 } 935 936 if (sc->hw.mac.type == ixgbe_mac_X540 || 937 sc->hw.mac.type == ixgbe_mac_X550EM_x || 938 sc->hw.mac.type == ixgbe_mac_X550EM_a) { 939 /* 940 * Thermal Failure Detection (X540) 941 * Link Detection (X552 SFP+, X552/X557-AT) 942 */ 943 gpie |= IXGBE_SDP0_GPIEN_X540; 944 945 /* 946 * Set LL interval to max to reduce the number of low latency 947 * interrupts hitting the card when the ring is getting full. 948 */ 949 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 950 } 951 952 if (sc->sc_intrmap) { 953 /* Enable Enhanced MSIX mode */ 954 gpie |= IXGBE_GPIE_MSIX_MODE; 955 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 956 IXGBE_GPIE_OCD; 957 } 958 959 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 960 } 961 962 /* 963 * Requires sc->max_frame_size to be set. 964 */ 965 void 966 ixgbe_config_delay_values(struct ix_softc *sc) 967 { 968 struct ixgbe_hw *hw = &sc->hw; 969 uint32_t rxpb, frame, size, tmp; 970 971 frame = sc->max_frame_size; 972 973 /* Calculate High Water */ 974 switch (hw->mac.type) { 975 case ixgbe_mac_X540: 976 case ixgbe_mac_X550: 977 case ixgbe_mac_X550EM_x: 978 case ixgbe_mac_X550EM_a: 979 tmp = IXGBE_DV_X540(frame, frame); 980 break; 981 default: 982 tmp = IXGBE_DV(frame, frame); 983 break; 984 } 985 size = IXGBE_BT2KB(tmp); 986 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 987 hw->fc.high_water[0] = rxpb - size; 988 989 /* Now calculate Low Water */ 990 switch (hw->mac.type) { 991 case ixgbe_mac_X540: 992 case ixgbe_mac_X550: 993 case ixgbe_mac_X550EM_x: 994 case ixgbe_mac_X550EM_a: 995 tmp = IXGBE_LOW_DV_X540(frame); 996 break; 997 default: 998 tmp = IXGBE_LOW_DV(frame); 999 break; 1000 } 1001 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 1002 1003 hw->fc.requested_mode = sc->fc; 1004 hw->fc.pause_time = IXGBE_FC_PAUSE; 1005 hw->fc.send_xon = TRUE; 1006 } 1007 1008 /* 1009 * MSIX Interrupt Handlers 1010 */ 1011 void 1012 ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector) 1013 { 1014 uint64_t queue = 1ULL << vector; 1015 uint32_t mask; 1016 1017 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1018 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1019 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask); 1020 } else { 1021 mask = (queue & 0xFFFFFFFF); 1022 if (mask) 1023 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask); 1024 mask = (queue >> 32); 1025 if (mask) 1026 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask); 1027 } 1028 } 1029 1030 void 1031 ixgbe_enable_queues(struct ix_softc *sc) 1032 { 1033 struct ix_queue *que; 1034 int i; 1035 1036 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 1037 ixgbe_enable_queue(sc, que->msix); 1038 } 1039 1040 void 1041 ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector) 1042 { 1043 uint64_t queue = 1ULL << vector; 1044 uint32_t mask; 1045 1046 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1047 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1048 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask); 1049 } else { 1050 mask = (queue & 0xFFFFFFFF); 1051 if (mask) 1052 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask); 1053 mask = (queue >> 32); 1054 if (mask) 1055 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask); 1056 } 1057 } 1058 1059 /* 1060 * MSIX Interrupt Handlers 1061 */ 1062 int 1063 ixgbe_link_intr(void *vsc) 1064 { 1065 struct ix_softc *sc = (struct ix_softc *)vsc; 1066 1067 return ixgbe_intr(sc); 1068 } 1069 1070 int 1071 ixgbe_queue_intr(void *vque) 1072 { 1073 struct ix_queue *que = vque; 1074 struct ix_softc *sc = que->sc; 1075 struct ifnet *ifp = &sc->arpcom.ac_if; 1076 struct rx_ring *rxr = que->rxr; 1077 struct tx_ring *txr = que->txr; 1078 1079 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1080 ixgbe_rxeof(rxr); 1081 ixgbe_txeof(txr); 1082 ixgbe_rxrefill(rxr); 1083 } 1084 1085 ixgbe_enable_queue(sc, que->msix); 1086 1087 return (1); 1088 } 1089 1090 /********************************************************************* 1091 * 1092 * Legacy Interrupt Service routine 1093 * 1094 **********************************************************************/ 1095 1096 int 1097 ixgbe_legacy_intr(void *arg) 1098 { 1099 struct ix_softc *sc = (struct ix_softc *)arg; 1100 struct ifnet *ifp = &sc->arpcom.ac_if; 1101 struct rx_ring *rxr = sc->rx_rings; 1102 struct tx_ring *txr = sc->tx_rings; 1103 int rv; 1104 1105 rv = ixgbe_intr(sc); 1106 if (rv == 0) { 1107 return (0); 1108 } 1109 1110 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1111 ixgbe_rxeof(rxr); 1112 ixgbe_txeof(txr); 1113 ixgbe_rxrefill(rxr); 1114 } 1115 1116 ixgbe_enable_queues(sc); 1117 return (rv); 1118 } 1119 1120 int 1121 ixgbe_intr(struct ix_softc *sc) 1122 { 1123 struct ifnet *ifp = &sc->arpcom.ac_if; 1124 struct ixgbe_hw *hw = &sc->hw; 1125 uint32_t reg_eicr, mod_mask, msf_mask; 1126 1127 if (sc->sc_intrmap) { 1128 /* Pause other interrupts */ 1129 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER); 1130 /* First get the cause */ 1131 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1132 /* Be sure the queue bits are not cleared */ 1133 reg_eicr &= ~IXGBE_EICR_RTX_QUEUE; 1134 /* Clear interrupt with write */ 1135 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1136 } else { 1137 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1138 if (reg_eicr == 0) { 1139 ixgbe_enable_intr(sc); 1140 ixgbe_enable_queues(sc); 1141 return (0); 1142 } 1143 } 1144 1145 /* Link status change */ 1146 if (reg_eicr & IXGBE_EICR_LSC) { 1147 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); 1148 KERNEL_LOCK(); 1149 ixgbe_update_link_status(sc); 1150 KERNEL_UNLOCK(); 1151 } 1152 1153 if (hw->mac.type != ixgbe_mac_82598EB) { 1154 if (reg_eicr & IXGBE_EICR_ECC) { 1155 printf("%s: CRITICAL: ECC ERROR!! " 1156 "Please Reboot!!\n", sc->dev.dv_xname); 1157 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1158 } 1159 /* Check for over temp condition */ 1160 if (reg_eicr & IXGBE_EICR_TS) { 1161 printf("%s: CRITICAL: OVER TEMP!! " 1162 "PHY IS SHUT DOWN!!\n", ifp->if_xname); 1163 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS); 1164 } 1165 } 1166 1167 /* Pluggable optics-related interrupt */ 1168 if (ixgbe_is_sfp(hw)) { 1169 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP) { 1170 mod_mask = IXGBE_EICR_GPI_SDP0_X540; 1171 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1172 } else if (hw->mac.type == ixgbe_mac_X540 || 1173 hw->mac.type == ixgbe_mac_X550 || 1174 hw->mac.type == ixgbe_mac_X550EM_x) { 1175 mod_mask = IXGBE_EICR_GPI_SDP2_X540; 1176 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1177 } else { 1178 mod_mask = IXGBE_EICR_GPI_SDP2; 1179 msf_mask = IXGBE_EICR_GPI_SDP1; 1180 } 1181 if (reg_eicr & mod_mask) { 1182 /* Clear the interrupt */ 1183 IXGBE_WRITE_REG(hw, IXGBE_EICR, mod_mask); 1184 KERNEL_LOCK(); 1185 ixgbe_handle_mod(sc); 1186 KERNEL_UNLOCK(); 1187 } else if ((hw->phy.media_type != ixgbe_media_type_copper) && 1188 (reg_eicr & msf_mask)) { 1189 /* Clear the interrupt */ 1190 IXGBE_WRITE_REG(hw, IXGBE_EICR, msf_mask); 1191 KERNEL_LOCK(); 1192 ixgbe_handle_msf(sc); 1193 KERNEL_UNLOCK(); 1194 } 1195 } 1196 1197 /* Check for fan failure */ 1198 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1199 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1200 printf("%s: CRITICAL: FAN FAILURE!! " 1201 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 1202 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1203 } 1204 1205 /* External PHY interrupt */ 1206 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 1207 (reg_eicr & IXGBE_EICR_GPI_SDP0_X540)) { 1208 /* Clear the interrupt */ 1209 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540); 1210 KERNEL_LOCK(); 1211 ixgbe_handle_phy(sc); 1212 KERNEL_UNLOCK(); 1213 } 1214 1215 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 1216 1217 return (1); 1218 } 1219 1220 /********************************************************************* 1221 * 1222 * Media Ioctl callback 1223 * 1224 * This routine is called whenever the user queries the status of 1225 * the interface using ifconfig. 1226 * 1227 **********************************************************************/ 1228 void 1229 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq *ifmr) 1230 { 1231 struct ix_softc *sc = ifp->if_softc; 1232 uint64_t layer; 1233 1234 ifmr->ifm_active = IFM_ETHER; 1235 ifmr->ifm_status = IFM_AVALID; 1236 1237 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1238 ixgbe_update_link_status(sc); 1239 1240 if (!LINK_STATE_IS_UP(ifp->if_link_state)) 1241 return; 1242 1243 ifmr->ifm_status |= IFM_ACTIVE; 1244 layer = sc->phy_layer; 1245 1246 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 1247 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 1248 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 1249 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1250 switch (sc->link_speed) { 1251 case IXGBE_LINK_SPEED_10GB_FULL: 1252 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1253 break; 1254 case IXGBE_LINK_SPEED_1GB_FULL: 1255 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1256 break; 1257 case IXGBE_LINK_SPEED_100_FULL: 1258 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1259 break; 1260 case IXGBE_LINK_SPEED_10_FULL: 1261 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1262 break; 1263 } 1264 } 1265 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1266 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1267 switch (sc->link_speed) { 1268 case IXGBE_LINK_SPEED_10GB_FULL: 1269 ifmr->ifm_active |= IFM_10G_SFP_CU | IFM_FDX; 1270 break; 1271 } 1272 } 1273 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1274 switch (sc->link_speed) { 1275 case IXGBE_LINK_SPEED_10GB_FULL: 1276 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 1277 break; 1278 case IXGBE_LINK_SPEED_1GB_FULL: 1279 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1280 break; 1281 } 1282 } 1283 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 1284 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1285 switch (sc->link_speed) { 1286 case IXGBE_LINK_SPEED_10GB_FULL: 1287 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1288 break; 1289 case IXGBE_LINK_SPEED_1GB_FULL: 1290 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1291 break; 1292 } 1293 } 1294 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1295 switch (sc->link_speed) { 1296 case IXGBE_LINK_SPEED_10GB_FULL: 1297 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1298 break; 1299 } 1300 } 1301 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1302 switch (sc->link_speed) { 1303 case IXGBE_LINK_SPEED_10GB_FULL: 1304 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 1305 break; 1306 case IXGBE_LINK_SPEED_2_5GB_FULL: 1307 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1308 break; 1309 case IXGBE_LINK_SPEED_1GB_FULL: 1310 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1311 break; 1312 } 1313 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 1314 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 1315 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1316 switch (sc->link_speed) { 1317 case IXGBE_LINK_SPEED_10GB_FULL: 1318 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 1319 break; 1320 case IXGBE_LINK_SPEED_2_5GB_FULL: 1321 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1322 break; 1323 case IXGBE_LINK_SPEED_1GB_FULL: 1324 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1325 break; 1326 } 1327 } 1328 1329 switch (sc->hw.fc.current_mode) { 1330 case ixgbe_fc_tx_pause: 1331 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 1332 break; 1333 case ixgbe_fc_rx_pause: 1334 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 1335 break; 1336 case ixgbe_fc_full: 1337 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE | 1338 IFM_ETH_TXPAUSE; 1339 break; 1340 default: 1341 ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE | 1342 IFM_ETH_TXPAUSE); 1343 break; 1344 } 1345 } 1346 1347 /********************************************************************* 1348 * 1349 * Media Ioctl callback 1350 * 1351 * This routine is called when the user changes speed/duplex using 1352 * media/mediopt option with ifconfig. 1353 * 1354 **********************************************************************/ 1355 int 1356 ixgbe_media_change(struct ifnet *ifp) 1357 { 1358 struct ix_softc *sc = ifp->if_softc; 1359 struct ixgbe_hw *hw = &sc->hw; 1360 struct ifmedia *ifm = &sc->media; 1361 ixgbe_link_speed speed = 0; 1362 1363 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1364 return (EINVAL); 1365 1366 if (hw->phy.media_type == ixgbe_media_type_backplane) 1367 return (ENODEV); 1368 1369 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1370 case IFM_AUTO: 1371 case IFM_10G_T: 1372 speed |= IXGBE_LINK_SPEED_100_FULL; 1373 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1374 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1375 break; 1376 case IFM_10G_SR: 1377 case IFM_10G_KR: 1378 case IFM_10G_LR: 1379 case IFM_10G_LRM: 1380 case IFM_10G_CX4: 1381 case IFM_10G_KX4: 1382 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1383 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1384 break; 1385 case IFM_10G_SFP_CU: 1386 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1387 break; 1388 case IFM_1000_T: 1389 speed |= IXGBE_LINK_SPEED_100_FULL; 1390 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1391 break; 1392 case IFM_1000_LX: 1393 case IFM_1000_SX: 1394 case IFM_1000_CX: 1395 case IFM_1000_KX: 1396 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1397 break; 1398 case IFM_100_TX: 1399 speed |= IXGBE_LINK_SPEED_100_FULL; 1400 break; 1401 case IFM_10_T: 1402 speed |= IXGBE_LINK_SPEED_10_FULL; 1403 break; 1404 default: 1405 return (EINVAL); 1406 } 1407 1408 hw->mac.autotry_restart = TRUE; 1409 hw->mac.ops.setup_link(hw, speed, TRUE); 1410 1411 return (0); 1412 } 1413 1414 /********************************************************************* 1415 * 1416 * This routine maps the mbufs to tx descriptors, allowing the 1417 * TX engine to transmit the packets. 1418 * - return 0 on success, positive on failure 1419 * 1420 **********************************************************************/ 1421 1422 int 1423 ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head) 1424 { 1425 struct ix_softc *sc = txr->sc; 1426 uint32_t olinfo_status = 0, cmd_type_len; 1427 int i, j, ntxc; 1428 int first, last = 0; 1429 bus_dmamap_t map; 1430 struct ixgbe_tx_buf *txbuf; 1431 union ixgbe_adv_tx_desc *txd = NULL; 1432 1433 /* Basic descriptor defines */ 1434 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1435 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1436 1437 /* 1438 * Important to capture the first descriptor 1439 * used because it will contain the index of 1440 * the one we tell the hardware to report back 1441 */ 1442 first = txr->next_avail_desc; 1443 txbuf = &txr->tx_buffers[first]; 1444 map = txbuf->map; 1445 1446 /* 1447 * Set the appropriate offload context 1448 * this will becomes the first descriptor. 1449 */ 1450 ntxc = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 1451 if (ntxc == -1) 1452 goto xmit_fail; 1453 1454 /* 1455 * Map the packet for DMA. 1456 */ 1457 switch (bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1458 m_head, BUS_DMA_NOWAIT)) { 1459 case 0: 1460 break; 1461 case EFBIG: 1462 if (m_defrag(m_head, M_NOWAIT) == 0 && 1463 bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1464 m_head, BUS_DMA_NOWAIT) == 0) 1465 break; 1466 /* FALLTHROUGH */ 1467 default: 1468 return (0); 1469 } 1470 1471 i = txr->next_avail_desc + ntxc; 1472 if (i >= sc->num_tx_desc) 1473 i -= sc->num_tx_desc; 1474 1475 for (j = 0; j < map->dm_nsegs; j++) { 1476 txd = &txr->tx_base[i]; 1477 1478 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 1479 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1480 cmd_type_len | map->dm_segs[j].ds_len); 1481 txd->read.olinfo_status = htole32(olinfo_status); 1482 last = i; /* descriptor that will get completion IRQ */ 1483 1484 if (++i == sc->num_tx_desc) 1485 i = 0; 1486 } 1487 1488 txd->read.cmd_type_len |= 1489 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1490 1491 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1492 BUS_DMASYNC_PREWRITE); 1493 1494 /* Set the index of the descriptor that will be marked done */ 1495 txbuf->m_head = m_head; 1496 txbuf->eop_index = last; 1497 1498 membar_producer(); 1499 1500 txr->next_avail_desc = i; 1501 1502 return (ntxc + j); 1503 1504 xmit_fail: 1505 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 1506 return (0); 1507 } 1508 1509 void 1510 ixgbe_iff(struct ix_softc *sc) 1511 { 1512 struct ifnet *ifp = &sc->arpcom.ac_if; 1513 struct arpcom *ac = &sc->arpcom; 1514 uint32_t fctrl; 1515 uint8_t *mta; 1516 uint8_t *update_ptr; 1517 struct ether_multi *enm; 1518 struct ether_multistep step; 1519 int mcnt = 0; 1520 1521 IOCTL_DEBUGOUT("ixgbe_iff: begin"); 1522 1523 mta = sc->mta; 1524 bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1525 MAX_NUM_MULTICAST_ADDRESSES); 1526 1527 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1528 fctrl &= ~(IXGBE_FCTRL_MPE | IXGBE_FCTRL_UPE); 1529 ifp->if_flags &= ~IFF_ALLMULTI; 1530 1531 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1532 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1533 ifp->if_flags |= IFF_ALLMULTI; 1534 fctrl |= IXGBE_FCTRL_MPE; 1535 if (ifp->if_flags & IFF_PROMISC) 1536 fctrl |= IXGBE_FCTRL_UPE; 1537 } else { 1538 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1539 while (enm != NULL) { 1540 bcopy(enm->enm_addrlo, 1541 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1542 IXGBE_ETH_LENGTH_OF_ADDRESS); 1543 mcnt++; 1544 1545 ETHER_NEXT_MULTI(step, enm); 1546 } 1547 1548 update_ptr = mta; 1549 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1550 ixgbe_mc_array_itr, TRUE); 1551 } 1552 1553 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1554 } 1555 1556 /* 1557 * This is an iterator function now needed by the multicast 1558 * shared code. It simply feeds the shared code routine the 1559 * addresses in the array of ixgbe_iff() one by one. 1560 */ 1561 uint8_t * 1562 ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1563 { 1564 uint8_t *addr = *update_ptr; 1565 uint8_t *newptr; 1566 *vmdq = 0; 1567 1568 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1569 *update_ptr = newptr; 1570 return addr; 1571 } 1572 1573 void 1574 ixgbe_update_link_status(struct ix_softc *sc) 1575 { 1576 struct ifnet *ifp = &sc->arpcom.ac_if; 1577 int link_state = LINK_STATE_DOWN; 1578 1579 splassert(IPL_NET); 1580 KERNEL_ASSERT_LOCKED(); 1581 1582 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 1583 1584 ifp->if_baudrate = 0; 1585 if (sc->link_up) { 1586 link_state = LINK_STATE_FULL_DUPLEX; 1587 1588 switch (sc->link_speed) { 1589 case IXGBE_LINK_SPEED_UNKNOWN: 1590 ifp->if_baudrate = 0; 1591 break; 1592 case IXGBE_LINK_SPEED_100_FULL: 1593 ifp->if_baudrate = IF_Mbps(100); 1594 break; 1595 case IXGBE_LINK_SPEED_1GB_FULL: 1596 ifp->if_baudrate = IF_Gbps(1); 1597 break; 1598 case IXGBE_LINK_SPEED_10GB_FULL: 1599 ifp->if_baudrate = IF_Gbps(10); 1600 break; 1601 } 1602 1603 /* Update any Flow Control changes */ 1604 sc->hw.mac.ops.fc_enable(&sc->hw); 1605 } 1606 if (ifp->if_link_state != link_state) { 1607 ifp->if_link_state = link_state; 1608 if_link_state_change(ifp); 1609 } 1610 } 1611 1612 1613 /********************************************************************* 1614 * 1615 * This routine disables all traffic on the adapter by issuing a 1616 * global reset on the MAC and deallocates TX/RX buffers. 1617 * 1618 **********************************************************************/ 1619 1620 void 1621 ixgbe_stop(void *arg) 1622 { 1623 struct ix_softc *sc = arg; 1624 struct ifnet *ifp = &sc->arpcom.ac_if; 1625 int i; 1626 1627 /* Tell the stack that the interface is no longer active */ 1628 ifp->if_flags &= ~IFF_RUNNING; 1629 1630 #if NKSTAT > 0 1631 timeout_del(&sc->sc_kstat_tmo); 1632 #endif 1633 ifp->if_timer = 0; 1634 1635 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1636 ixgbe_disable_intr(sc); 1637 1638 sc->hw.mac.ops.reset_hw(&sc->hw); 1639 sc->hw.adapter_stopped = FALSE; 1640 sc->hw.mac.ops.stop_adapter(&sc->hw); 1641 if (sc->hw.mac.type == ixgbe_mac_82599EB) 1642 sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw); 1643 /* Turn off the laser */ 1644 if (sc->hw.mac.ops.disable_tx_laser) 1645 sc->hw.mac.ops.disable_tx_laser(&sc->hw); 1646 1647 /* reprogram the RAR[0] in case user changed it. */ 1648 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1649 1650 intr_barrier(sc->tag); 1651 for (i = 0; i < sc->num_queues; i++) { 1652 struct ifqueue *ifq = ifp->if_ifqs[i]; 1653 ifq_barrier(ifq); 1654 ifq_clr_oactive(ifq); 1655 1656 if (sc->queues[i].tag != NULL) 1657 intr_barrier(sc->queues[i].tag); 1658 timeout_del(&sc->rx_rings[i].rx_refill); 1659 } 1660 1661 KASSERT((ifp->if_flags & IFF_RUNNING) == 0); 1662 1663 /* Should we really clear all structures on stop? */ 1664 ixgbe_free_transmit_structures(sc); 1665 ixgbe_free_receive_structures(sc); 1666 1667 ixgbe_update_link_status(sc); 1668 } 1669 1670 1671 /********************************************************************* 1672 * 1673 * Determine hardware revision. 1674 * 1675 **********************************************************************/ 1676 void 1677 ixgbe_identify_hardware(struct ix_softc *sc) 1678 { 1679 struct ixgbe_osdep *os = &sc->osdep; 1680 struct pci_attach_args *pa = &os->os_pa; 1681 uint32_t reg; 1682 1683 /* Save off the information about this board */ 1684 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1685 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1686 1687 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1688 sc->hw.revision_id = PCI_REVISION(reg); 1689 1690 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1691 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1692 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1693 1694 /* We need this here to set the num_segs below */ 1695 ixgbe_set_mac_type(&sc->hw); 1696 1697 /* Pick up the 82599 and VF settings */ 1698 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1699 sc->hw.phy.smart_speed = ixgbe_smart_speed; 1700 sc->num_segs = IXGBE_82599_SCATTER; 1701 } 1702 1703 /********************************************************************* 1704 * 1705 * Setup the Legacy or MSI Interrupt handler 1706 * 1707 **********************************************************************/ 1708 int 1709 ixgbe_allocate_legacy(struct ix_softc *sc) 1710 { 1711 struct ixgbe_osdep *os = &sc->osdep; 1712 struct pci_attach_args *pa = &os->os_pa; 1713 const char *intrstr = NULL; 1714 pci_chipset_tag_t pc = pa->pa_pc; 1715 pci_intr_handle_t ih; 1716 1717 /* We allocate a single interrupt resource */ 1718 if (pci_intr_map_msi(pa, &ih) != 0 && 1719 pci_intr_map(pa, &ih) != 0) { 1720 printf(": couldn't map interrupt\n"); 1721 return (ENXIO); 1722 } 1723 1724 #if 0 1725 /* XXX */ 1726 /* Tasklets for Link, SFP and Multispeed Fiber */ 1727 TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc); 1728 TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc); 1729 TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc); 1730 #endif 1731 1732 intrstr = pci_intr_string(pc, ih); 1733 sc->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE, 1734 ixgbe_legacy_intr, sc, sc->dev.dv_xname); 1735 if (sc->tag == NULL) { 1736 printf(": couldn't establish interrupt"); 1737 if (intrstr != NULL) 1738 printf(" at %s", intrstr); 1739 printf("\n"); 1740 return (ENXIO); 1741 } 1742 printf(": %s", intrstr); 1743 1744 /* For simplicity in the handlers */ 1745 sc->que_mask = IXGBE_EIMS_ENABLE_MASK; 1746 1747 return (0); 1748 } 1749 1750 /********************************************************************* 1751 * 1752 * Setup the MSI-X Interrupt handlers 1753 * 1754 **********************************************************************/ 1755 int 1756 ixgbe_allocate_msix(struct ix_softc *sc) 1757 { 1758 struct ixgbe_osdep *os = &sc->osdep; 1759 struct pci_attach_args *pa = &os->os_pa; 1760 int i = 0, error = 0; 1761 struct ix_queue *que; 1762 pci_intr_handle_t ih; 1763 1764 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) { 1765 if (pci_intr_map_msix(pa, i, &ih)) { 1766 printf("ixgbe_allocate_msix: " 1767 "pci_intr_map_msix vec %d failed\n", i); 1768 error = ENOMEM; 1769 goto fail; 1770 } 1771 1772 que->tag = pci_intr_establish_cpu(pa->pa_pc, ih, 1773 IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i), 1774 ixgbe_queue_intr, que, que->name); 1775 if (que->tag == NULL) { 1776 printf("ixgbe_allocate_msix: " 1777 "pci_intr_establish vec %d failed\n", i); 1778 error = ENOMEM; 1779 goto fail; 1780 } 1781 1782 que->msix = i; 1783 } 1784 1785 /* Now the link status/control last MSI-X vector */ 1786 if (pci_intr_map_msix(pa, i, &ih)) { 1787 printf("ixgbe_allocate_msix: " 1788 "pci_intr_map_msix link vector failed\n"); 1789 error = ENOMEM; 1790 goto fail; 1791 } 1792 1793 sc->tag = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE, 1794 ixgbe_link_intr, sc, sc->dev.dv_xname); 1795 if (sc->tag == NULL) { 1796 printf("ixgbe_allocate_msix: " 1797 "pci_intr_establish link vector failed\n"); 1798 error = ENOMEM; 1799 goto fail; 1800 } 1801 sc->linkvec = i; 1802 printf(", %s, %d queue%s", pci_intr_string(pa->pa_pc, ih), 1803 i, (i > 1) ? "s" : ""); 1804 1805 return (0); 1806 fail: 1807 for (que = sc->queues; i > 0; i--, que++) { 1808 if (que->tag == NULL) 1809 continue; 1810 pci_intr_disestablish(pa->pa_pc, que->tag); 1811 que->tag = NULL; 1812 } 1813 1814 return (error); 1815 } 1816 1817 void 1818 ixgbe_setup_msix(struct ix_softc *sc) 1819 { 1820 struct ixgbe_osdep *os = &sc->osdep; 1821 struct pci_attach_args *pa = &os->os_pa; 1822 int nmsix; 1823 unsigned int maxq; 1824 1825 if (!ixgbe_enable_msix) 1826 return; 1827 1828 nmsix = pci_intr_msix_count(pa); 1829 if (nmsix <= 1) 1830 return; 1831 1832 /* give one vector to events */ 1833 nmsix--; 1834 1835 /* XXX the number of queues is limited to what we can keep stats on */ 1836 maxq = (sc->hw.mac.type == ixgbe_mac_82598EB) ? 8 : 16; 1837 1838 sc->sc_intrmap = intrmap_create(&sc->dev, nmsix, maxq, 0); 1839 sc->num_queues = intrmap_count(sc->sc_intrmap); 1840 } 1841 1842 int 1843 ixgbe_allocate_pci_resources(struct ix_softc *sc) 1844 { 1845 struct ixgbe_osdep *os = &sc->osdep; 1846 struct pci_attach_args *pa = &os->os_pa; 1847 int val; 1848 1849 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1850 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) { 1851 printf(": mmba is not mem space\n"); 1852 return (ENXIO); 1853 } 1854 1855 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1856 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1857 printf(": cannot find mem space\n"); 1858 return (ENXIO); 1859 } 1860 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1861 1862 /* Legacy defaults */ 1863 sc->num_queues = 1; 1864 sc->hw.back = os; 1865 1866 /* Now setup MSI or MSI/X, return us the number of supported vectors. */ 1867 ixgbe_setup_msix(sc); 1868 1869 return (0); 1870 } 1871 1872 void 1873 ixgbe_free_pci_resources(struct ix_softc * sc) 1874 { 1875 struct ixgbe_osdep *os = &sc->osdep; 1876 struct pci_attach_args *pa = &os->os_pa; 1877 struct ix_queue *que = sc->queues; 1878 int i; 1879 1880 /* Release all msix queue resources: */ 1881 for (i = 0; i < sc->num_queues; i++, que++) { 1882 if (que->tag) 1883 pci_intr_disestablish(pa->pa_pc, que->tag); 1884 que->tag = NULL; 1885 } 1886 1887 if (sc->tag) 1888 pci_intr_disestablish(pa->pa_pc, sc->tag); 1889 sc->tag = NULL; 1890 if (os->os_membase != 0) 1891 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1892 os->os_membase = 0; 1893 } 1894 1895 /********************************************************************* 1896 * 1897 * Setup networking device structure and register an interface. 1898 * 1899 **********************************************************************/ 1900 void 1901 ixgbe_setup_interface(struct ix_softc *sc) 1902 { 1903 struct ifnet *ifp = &sc->arpcom.ac_if; 1904 int i; 1905 1906 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1907 ifp->if_softc = sc; 1908 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1909 ifp->if_xflags = IFXF_MPSAFE; 1910 ifp->if_ioctl = ixgbe_ioctl; 1911 ifp->if_qstart = ixgbe_start; 1912 ifp->if_timer = 0; 1913 ifp->if_watchdog = ixgbe_watchdog; 1914 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1915 ETHER_HDR_LEN - ETHER_CRC_LEN; 1916 ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1); 1917 1918 ifp->if_capabilities = IFCAP_VLAN_MTU; 1919 1920 #if NVLAN > 0 1921 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1922 #endif 1923 1924 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 1925 ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6; 1926 ifp->if_capabilities |= IFCAP_CSUM_IPv4; 1927 1928 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1929 ifp->if_capabilities |= IFCAP_TSO; 1930 1931 /* 1932 * Specify the media types supported by this sc and register 1933 * callbacks to update media and link information 1934 */ 1935 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1936 ixgbe_media_status); 1937 ixgbe_add_media_types(sc); 1938 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1939 1940 if_attach(ifp); 1941 ether_ifattach(ifp); 1942 1943 if_attach_queues(ifp, sc->num_queues); 1944 if_attach_iqueues(ifp, sc->num_queues); 1945 for (i = 0; i < sc->num_queues; i++) { 1946 struct ifqueue *ifq = ifp->if_ifqs[i]; 1947 struct ifiqueue *ifiq = ifp->if_iqs[i]; 1948 struct tx_ring *txr = &sc->tx_rings[i]; 1949 struct rx_ring *rxr = &sc->rx_rings[i]; 1950 1951 ifq->ifq_softc = txr; 1952 txr->ifq = ifq; 1953 1954 ifiq->ifiq_softc = rxr; 1955 rxr->ifiq = ifiq; 1956 1957 #if NKSTAT > 0 1958 ix_txq_kstats(sc, txr); 1959 ix_rxq_kstats(sc, rxr); 1960 #endif 1961 } 1962 1963 sc->max_frame_size = IXGBE_MAX_FRAME_SIZE; 1964 } 1965 1966 void 1967 ixgbe_add_media_types(struct ix_softc *sc) 1968 { 1969 struct ixgbe_hw *hw = &sc->hw; 1970 uint64_t layer; 1971 1972 sc->phy_layer = hw->mac.ops.get_supported_physical_layer(hw); 1973 layer = sc->phy_layer; 1974 1975 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1976 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL); 1977 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1978 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 1979 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1980 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1981 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1982 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 1983 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SFP_CU, 0, NULL); 1984 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1985 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL); 1986 if (hw->phy.multispeed_fiber) 1987 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_LX, 0, 1988 NULL); 1989 } 1990 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1991 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 1992 if (hw->phy.multispeed_fiber) 1993 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, 1994 NULL); 1995 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 1996 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 1997 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 1998 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); 1999 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 2000 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL); 2001 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) 2002 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); 2003 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 2004 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL); 2005 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) 2006 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL); 2007 2008 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2009 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, 2010 NULL); 2011 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 2012 } 2013 2014 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2015 } 2016 2017 void 2018 ixgbe_config_link(struct ix_softc *sc) 2019 { 2020 uint32_t autoneg, err = 0; 2021 bool negotiate; 2022 2023 if (ixgbe_is_sfp(&sc->hw)) { 2024 if (sc->hw.phy.multispeed_fiber) { 2025 sc->hw.mac.ops.setup_sfp(&sc->hw); 2026 if (sc->hw.mac.ops.enable_tx_laser) 2027 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 2028 ixgbe_handle_msf(sc); 2029 } else 2030 ixgbe_handle_mod(sc); 2031 } else { 2032 if (sc->hw.mac.ops.check_link) 2033 err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg, 2034 &sc->link_up, FALSE); 2035 if (err) 2036 return; 2037 autoneg = sc->hw.phy.autoneg_advertised; 2038 if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities)) 2039 err = sc->hw.mac.ops.get_link_capabilities(&sc->hw, 2040 &autoneg, &negotiate); 2041 if (err) 2042 return; 2043 if (sc->hw.mac.ops.setup_link) 2044 sc->hw.mac.ops.setup_link(&sc->hw, 2045 autoneg, sc->link_up); 2046 } 2047 } 2048 2049 /******************************************************************** 2050 * Manage DMA'able memory. 2051 *******************************************************************/ 2052 int 2053 ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 2054 struct ixgbe_dma_alloc *dma, int mapflags) 2055 { 2056 struct ifnet *ifp = &sc->arpcom.ac_if; 2057 struct ixgbe_osdep *os = &sc->osdep; 2058 int r; 2059 2060 dma->dma_tag = os->os_pa.pa_dmat; 2061 r = bus_dmamap_create(dma->dma_tag, size, 1, 2062 size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 2063 if (r != 0) { 2064 printf("%s: ixgbe_dma_malloc: bus_dmamap_create failed; " 2065 "error %u\n", ifp->if_xname, r); 2066 goto fail_0; 2067 } 2068 2069 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 2070 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 2071 if (r != 0) { 2072 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2073 "error %u\n", ifp->if_xname, r); 2074 goto fail_1; 2075 } 2076 2077 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 2078 &dma->dma_vaddr, BUS_DMA_NOWAIT); 2079 if (r != 0) { 2080 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 2081 "error %u\n", ifp->if_xname, r); 2082 goto fail_2; 2083 } 2084 2085 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2086 size, NULL, mapflags | BUS_DMA_NOWAIT); 2087 if (r != 0) { 2088 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 2089 "error %u\n", ifp->if_xname, r); 2090 goto fail_3; 2091 } 2092 2093 dma->dma_size = size; 2094 return (0); 2095 fail_3: 2096 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 2097 fail_2: 2098 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2099 fail_1: 2100 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2101 fail_0: 2102 dma->dma_map = NULL; 2103 dma->dma_tag = NULL; 2104 return (r); 2105 } 2106 2107 void 2108 ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 2109 { 2110 if (dma->dma_tag == NULL) 2111 return; 2112 2113 if (dma->dma_map != NULL) { 2114 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 2115 dma->dma_map->dm_mapsize, 2116 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2117 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2118 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 2119 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2120 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2121 dma->dma_map = NULL; 2122 } 2123 } 2124 2125 2126 /********************************************************************* 2127 * 2128 * Allocate memory for the transmit and receive rings, and then 2129 * the descriptors associated with each, called only once at attach. 2130 * 2131 **********************************************************************/ 2132 int 2133 ixgbe_allocate_queues(struct ix_softc *sc) 2134 { 2135 struct ifnet *ifp = &sc->arpcom.ac_if; 2136 struct ix_queue *que; 2137 struct tx_ring *txr; 2138 struct rx_ring *rxr; 2139 int rsize, tsize; 2140 int txconf = 0, rxconf = 0, i; 2141 2142 /* First allocate the top level queue structs */ 2143 if (!(sc->queues = mallocarray(sc->num_queues, 2144 sizeof(struct ix_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2145 printf("%s: Unable to allocate queue memory\n", ifp->if_xname); 2146 goto fail; 2147 } 2148 2149 /* Then allocate the TX ring struct memory */ 2150 if (!(sc->tx_rings = mallocarray(sc->num_queues, 2151 sizeof(struct tx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2152 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 2153 goto fail; 2154 } 2155 2156 /* Next allocate the RX */ 2157 if (!(sc->rx_rings = mallocarray(sc->num_queues, 2158 sizeof(struct rx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2159 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 2160 goto rx_fail; 2161 } 2162 2163 /* For the ring itself */ 2164 tsize = roundup2(sc->num_tx_desc * 2165 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2166 2167 /* 2168 * Now set up the TX queues, txconf is needed to handle the 2169 * possibility that things fail midcourse and we need to 2170 * undo memory gracefully 2171 */ 2172 for (i = 0; i < sc->num_queues; i++, txconf++) { 2173 /* Set up some basics */ 2174 txr = &sc->tx_rings[i]; 2175 txr->sc = sc; 2176 txr->me = i; 2177 2178 if (ixgbe_dma_malloc(sc, tsize, 2179 &txr->txdma, BUS_DMA_NOWAIT)) { 2180 printf("%s: Unable to allocate TX Descriptor memory\n", 2181 ifp->if_xname); 2182 goto err_tx_desc; 2183 } 2184 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2185 bzero((void *)txr->tx_base, tsize); 2186 } 2187 2188 /* 2189 * Next the RX queues... 2190 */ 2191 rsize = roundup2(sc->num_rx_desc * 2192 sizeof(union ixgbe_adv_rx_desc), 4096); 2193 for (i = 0; i < sc->num_queues; i++, rxconf++) { 2194 rxr = &sc->rx_rings[i]; 2195 /* Set up some basics */ 2196 rxr->sc = sc; 2197 rxr->me = i; 2198 timeout_set(&rxr->rx_refill, ixgbe_rxrefill, rxr); 2199 2200 if (ixgbe_dma_malloc(sc, rsize, 2201 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2202 printf("%s: Unable to allocate RxDescriptor memory\n", 2203 ifp->if_xname); 2204 goto err_rx_desc; 2205 } 2206 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2207 bzero((void *)rxr->rx_base, rsize); 2208 } 2209 2210 /* 2211 * Finally set up the queue holding structs 2212 */ 2213 for (i = 0; i < sc->num_queues; i++) { 2214 que = &sc->queues[i]; 2215 que->sc = sc; 2216 que->txr = &sc->tx_rings[i]; 2217 que->rxr = &sc->rx_rings[i]; 2218 snprintf(que->name, sizeof(que->name), "%s:%d", 2219 sc->dev.dv_xname, i); 2220 } 2221 2222 return (0); 2223 2224 err_rx_desc: 2225 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 2226 ixgbe_dma_free(sc, &rxr->rxdma); 2227 err_tx_desc: 2228 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 2229 ixgbe_dma_free(sc, &txr->txdma); 2230 free(sc->rx_rings, M_DEVBUF, sc->num_queues * sizeof(struct rx_ring)); 2231 sc->rx_rings = NULL; 2232 rx_fail: 2233 free(sc->tx_rings, M_DEVBUF, sc->num_queues * sizeof(struct tx_ring)); 2234 sc->tx_rings = NULL; 2235 fail: 2236 return (ENOMEM); 2237 } 2238 2239 /********************************************************************* 2240 * 2241 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2242 * the information needed to transmit a packet on the wire. This is 2243 * called only once at attach, setup is done every reset. 2244 * 2245 **********************************************************************/ 2246 int 2247 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2248 { 2249 struct ix_softc *sc = txr->sc; 2250 struct ifnet *ifp = &sc->arpcom.ac_if; 2251 struct ixgbe_tx_buf *txbuf; 2252 int error, i; 2253 2254 if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc, 2255 sizeof(struct ixgbe_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2256 printf("%s: Unable to allocate tx_buffer memory\n", 2257 ifp->if_xname); 2258 error = ENOMEM; 2259 goto fail; 2260 } 2261 txr->txtag = txr->txdma.dma_tag; 2262 2263 /* Create the descriptor buffer dma maps */ 2264 for (i = 0; i < sc->num_tx_desc; i++) { 2265 txbuf = &txr->tx_buffers[i]; 2266 error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE, 2267 sc->num_segs, PAGE_SIZE, 0, 2268 BUS_DMA_NOWAIT, &txbuf->map); 2269 2270 if (error != 0) { 2271 printf("%s: Unable to create TX DMA map\n", 2272 ifp->if_xname); 2273 goto fail; 2274 } 2275 } 2276 2277 return 0; 2278 fail: 2279 return (error); 2280 } 2281 2282 /********************************************************************* 2283 * 2284 * Initialize a transmit ring. 2285 * 2286 **********************************************************************/ 2287 int 2288 ixgbe_setup_transmit_ring(struct tx_ring *txr) 2289 { 2290 struct ix_softc *sc = txr->sc; 2291 int error; 2292 2293 /* Now allocate transmit buffers for the ring */ 2294 if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0) 2295 return (error); 2296 2297 /* Clear the old ring contents */ 2298 bzero((void *)txr->tx_base, 2299 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 2300 2301 /* Reset indices */ 2302 txr->next_avail_desc = 0; 2303 txr->next_to_clean = 0; 2304 2305 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2306 0, txr->txdma.dma_map->dm_mapsize, 2307 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2308 2309 return (0); 2310 } 2311 2312 /********************************************************************* 2313 * 2314 * Initialize all transmit rings. 2315 * 2316 **********************************************************************/ 2317 int 2318 ixgbe_setup_transmit_structures(struct ix_softc *sc) 2319 { 2320 struct tx_ring *txr = sc->tx_rings; 2321 int i, error; 2322 2323 for (i = 0; i < sc->num_queues; i++, txr++) { 2324 if ((error = ixgbe_setup_transmit_ring(txr)) != 0) 2325 goto fail; 2326 } 2327 2328 return (0); 2329 fail: 2330 ixgbe_free_transmit_structures(sc); 2331 return (error); 2332 } 2333 2334 /********************************************************************* 2335 * 2336 * Enable transmit unit. 2337 * 2338 **********************************************************************/ 2339 void 2340 ixgbe_initialize_transmit_units(struct ix_softc *sc) 2341 { 2342 struct ifnet *ifp = &sc->arpcom.ac_if; 2343 struct tx_ring *txr; 2344 struct ixgbe_hw *hw = &sc->hw; 2345 int i; 2346 uint64_t tdba; 2347 uint32_t txctrl; 2348 2349 /* Setup the Base and Length of the Tx Descriptor Ring */ 2350 2351 for (i = 0; i < sc->num_queues; i++) { 2352 txr = &sc->tx_rings[i]; 2353 2354 /* Setup descriptor base address */ 2355 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 2356 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 2357 (tdba & 0x00000000ffffffffULL)); 2358 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 2359 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2360 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 2361 2362 /* Setup the HW Tx Head and Tail descriptor pointers */ 2363 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2364 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2365 2366 /* Setup Transmit Descriptor Cmd Settings */ 2367 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2368 txr->queue_status = IXGBE_QUEUE_IDLE; 2369 txr->watchdog_timer = 0; 2370 2371 /* Disable Head Writeback */ 2372 switch (hw->mac.type) { 2373 case ixgbe_mac_82598EB: 2374 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2375 break; 2376 case ixgbe_mac_82599EB: 2377 case ixgbe_mac_X540: 2378 default: 2379 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2380 break; 2381 } 2382 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2383 switch (hw->mac.type) { 2384 case ixgbe_mac_82598EB: 2385 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2386 break; 2387 case ixgbe_mac_82599EB: 2388 case ixgbe_mac_X540: 2389 default: 2390 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2391 break; 2392 } 2393 } 2394 ifp->if_timer = 0; 2395 2396 if (hw->mac.type != ixgbe_mac_82598EB) { 2397 uint32_t dmatxctl, rttdcs; 2398 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2399 dmatxctl |= IXGBE_DMATXCTL_TE; 2400 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2401 /* Disable arbiter to set MTQC */ 2402 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2403 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2404 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2405 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2406 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2407 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2408 } 2409 } 2410 2411 /********************************************************************* 2412 * 2413 * Free all transmit rings. 2414 * 2415 **********************************************************************/ 2416 void 2417 ixgbe_free_transmit_structures(struct ix_softc *sc) 2418 { 2419 struct tx_ring *txr = sc->tx_rings; 2420 int i; 2421 2422 for (i = 0; i < sc->num_queues; i++, txr++) 2423 ixgbe_free_transmit_buffers(txr); 2424 } 2425 2426 /********************************************************************* 2427 * 2428 * Free transmit ring related data structures. 2429 * 2430 **********************************************************************/ 2431 void 2432 ixgbe_free_transmit_buffers(struct tx_ring *txr) 2433 { 2434 struct ix_softc *sc = txr->sc; 2435 struct ixgbe_tx_buf *tx_buffer; 2436 int i; 2437 2438 INIT_DEBUGOUT("free_transmit_ring: begin"); 2439 2440 if (txr->tx_buffers == NULL) 2441 return; 2442 2443 tx_buffer = txr->tx_buffers; 2444 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 2445 if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) { 2446 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2447 0, tx_buffer->map->dm_mapsize, 2448 BUS_DMASYNC_POSTWRITE); 2449 bus_dmamap_unload(txr->txdma.dma_tag, 2450 tx_buffer->map); 2451 } 2452 if (tx_buffer->m_head != NULL) { 2453 m_freem(tx_buffer->m_head); 2454 tx_buffer->m_head = NULL; 2455 } 2456 if (tx_buffer->map != NULL) { 2457 bus_dmamap_destroy(txr->txdma.dma_tag, 2458 tx_buffer->map); 2459 tx_buffer->map = NULL; 2460 } 2461 } 2462 2463 if (txr->tx_buffers != NULL) 2464 free(txr->tx_buffers, M_DEVBUF, 2465 sc->num_tx_desc * sizeof(struct ixgbe_tx_buf)); 2466 txr->tx_buffers = NULL; 2467 txr->txtag = NULL; 2468 } 2469 2470 /********************************************************************* 2471 * 2472 * Advanced Context Descriptor setup for VLAN or CSUM 2473 * 2474 **********************************************************************/ 2475 2476 static inline int 2477 ixgbe_csum_offload(struct mbuf *mp, uint32_t *vlan_macip_lens, 2478 uint32_t *type_tucmd_mlhl, uint32_t *olinfo_status) 2479 { 2480 struct ether_header *eh = mtod(mp, struct ether_header *); 2481 struct mbuf *m; 2482 int hoff; 2483 int offload = 0; 2484 uint32_t iphlen; 2485 uint8_t ipproto; 2486 2487 *vlan_macip_lens |= (sizeof(*eh) << IXGBE_ADVTXD_MACLEN_SHIFT); 2488 2489 switch (ntohs(eh->ether_type)) { 2490 case ETHERTYPE_IP: { 2491 struct ip *ip; 2492 2493 m = m_getptr(mp, sizeof(*eh), &hoff); 2494 KASSERT(m != NULL && m->m_len - hoff >= sizeof(*ip)); 2495 ip = (struct ip *)(mtod(m, caddr_t) + hoff); 2496 2497 iphlen = ip->ip_hl << 2; 2498 ipproto = ip->ip_p; 2499 2500 if (ISSET(mp->m_pkthdr.csum_flags, M_IPV4_CSUM_OUT)) { 2501 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2502 offload = 1; 2503 } 2504 2505 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2506 break; 2507 } 2508 2509 #ifdef INET6 2510 case ETHERTYPE_IPV6: { 2511 struct ip6_hdr *ip6; 2512 2513 m = m_getptr(mp, sizeof(*eh), &hoff); 2514 KASSERT(m != NULL && m->m_len - hoff >= sizeof(*ip6)); 2515 ip6 = (struct ip6_hdr *)(mtod(m, caddr_t) + hoff); 2516 2517 iphlen = sizeof(*ip6); 2518 ipproto = ip6->ip6_nxt; 2519 2520 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2521 break; 2522 } 2523 #endif 2524 2525 default: 2526 return offload; 2527 } 2528 2529 *vlan_macip_lens |= iphlen; 2530 2531 switch (ipproto) { 2532 case IPPROTO_TCP: 2533 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2534 if (ISSET(mp->m_pkthdr.csum_flags, M_TCP_CSUM_OUT)) { 2535 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2536 offload = 1; 2537 } 2538 break; 2539 case IPPROTO_UDP: 2540 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2541 if (ISSET(mp->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) { 2542 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2543 offload = 1; 2544 } 2545 break; 2546 } 2547 2548 return offload; 2549 } 2550 2551 static int 2552 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, 2553 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2554 { 2555 struct ixgbe_adv_tx_context_desc *TXD; 2556 struct ixgbe_tx_buf *tx_buffer; 2557 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2558 int ctxd = txr->next_avail_desc; 2559 int offload = 0; 2560 2561 /* Indicate the whole packet as payload when not doing TSO */ 2562 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2563 2564 #if NVLAN > 0 2565 if (ISSET(mp->m_flags, M_VLANTAG)) { 2566 uint32_t vtag = mp->m_pkthdr.ether_vtag; 2567 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2568 *cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 2569 offload |= 1; 2570 } 2571 #endif 2572 2573 offload |= ixgbe_csum_offload(mp, &vlan_macip_lens, &type_tucmd_mlhl, 2574 olinfo_status); 2575 2576 if (!offload) 2577 return (0); 2578 2579 TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; 2580 tx_buffer = &txr->tx_buffers[ctxd]; 2581 2582 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2583 2584 /* Now copy bits into descriptor */ 2585 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2586 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2587 TXD->seqnum_seed = htole32(0); 2588 TXD->mss_l4len_idx = htole32(0); 2589 2590 tx_buffer->m_head = NULL; 2591 tx_buffer->eop_index = -1; 2592 2593 return (1); 2594 } 2595 2596 /********************************************************************** 2597 * 2598 * Examine each tx_buffer in the used queue. If the hardware is done 2599 * processing the packet then free associated resources. The 2600 * tx_buffer is put back on the free queue. 2601 * 2602 **********************************************************************/ 2603 int 2604 ixgbe_txeof(struct tx_ring *txr) 2605 { 2606 struct ix_softc *sc = txr->sc; 2607 struct ifqueue *ifq = txr->ifq; 2608 struct ifnet *ifp = &sc->arpcom.ac_if; 2609 unsigned int head, tail, last; 2610 struct ixgbe_tx_buf *tx_buffer; 2611 struct ixgbe_legacy_tx_desc *tx_desc; 2612 2613 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2614 return FALSE; 2615 2616 head = txr->next_avail_desc; 2617 tail = txr->next_to_clean; 2618 2619 membar_consumer(); 2620 2621 if (head == tail) 2622 return (FALSE); 2623 2624 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2625 0, txr->txdma.dma_map->dm_mapsize, 2626 BUS_DMASYNC_POSTREAD); 2627 2628 for (;;) { 2629 tx_buffer = &txr->tx_buffers[tail]; 2630 last = tx_buffer->eop_index; 2631 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2632 2633 if (!ISSET(tx_desc->upper.fields.status, IXGBE_TXD_STAT_DD)) 2634 break; 2635 2636 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2637 0, tx_buffer->map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2638 bus_dmamap_unload(txr->txdma.dma_tag, tx_buffer->map); 2639 m_freem(tx_buffer->m_head); 2640 2641 tx_buffer->m_head = NULL; 2642 tx_buffer->eop_index = -1; 2643 2644 tail = last + 1; 2645 if (tail == sc->num_tx_desc) 2646 tail = 0; 2647 if (head == tail) { 2648 /* All clean, turn off the timer */ 2649 ifp->if_timer = 0; 2650 break; 2651 } 2652 } 2653 2654 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2655 0, txr->txdma.dma_map->dm_mapsize, 2656 BUS_DMASYNC_PREREAD); 2657 2658 membar_producer(); 2659 2660 txr->next_to_clean = tail; 2661 2662 if (ifq_is_oactive(ifq)) 2663 ifq_restart(ifq); 2664 2665 return TRUE; 2666 } 2667 2668 /********************************************************************* 2669 * 2670 * Get a buffer from system mbuf buffer pool. 2671 * 2672 **********************************************************************/ 2673 int 2674 ixgbe_get_buf(struct rx_ring *rxr, int i) 2675 { 2676 struct ix_softc *sc = rxr->sc; 2677 struct ixgbe_rx_buf *rxbuf; 2678 struct mbuf *mp; 2679 int error; 2680 union ixgbe_adv_rx_desc *rxdesc; 2681 2682 rxbuf = &rxr->rx_buffers[i]; 2683 rxdesc = &rxr->rx_base[i]; 2684 if (rxbuf->buf) { 2685 printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n", 2686 sc->dev.dv_xname, i); 2687 return (ENOBUFS); 2688 } 2689 2690 /* needed in any case so prealocate since this one will fail for sure */ 2691 mp = MCLGETL(NULL, M_DONTWAIT, sc->rx_mbuf_sz); 2692 if (!mp) 2693 return (ENOBUFS); 2694 2695 mp->m_data += (mp->m_ext.ext_size - sc->rx_mbuf_sz); 2696 mp->m_len = mp->m_pkthdr.len = sc->rx_mbuf_sz; 2697 2698 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, 2699 mp, BUS_DMA_NOWAIT); 2700 if (error) { 2701 m_freem(mp); 2702 return (error); 2703 } 2704 2705 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2706 0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 2707 rxbuf->buf = mp; 2708 2709 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 2710 2711 return (0); 2712 } 2713 2714 /********************************************************************* 2715 * 2716 * Allocate memory for rx_buffer structures. Since we use one 2717 * rx_buffer per received packet, the maximum number of rx_buffer's 2718 * that we'll need is equal to the number of receive descriptors 2719 * that we've allocated. 2720 * 2721 **********************************************************************/ 2722 int 2723 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2724 { 2725 struct ix_softc *sc = rxr->sc; 2726 struct ifnet *ifp = &sc->arpcom.ac_if; 2727 struct ixgbe_rx_buf *rxbuf; 2728 int i, error; 2729 2730 if (!(rxr->rx_buffers = mallocarray(sc->num_rx_desc, 2731 sizeof(struct ixgbe_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2732 printf("%s: Unable to allocate rx_buffer memory\n", 2733 ifp->if_xname); 2734 error = ENOMEM; 2735 goto fail; 2736 } 2737 2738 rxbuf = rxr->rx_buffers; 2739 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2740 error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1, 2741 16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map); 2742 if (error) { 2743 printf("%s: Unable to create Pack DMA map\n", 2744 ifp->if_xname); 2745 goto fail; 2746 } 2747 } 2748 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2749 rxr->rxdma.dma_map->dm_mapsize, 2750 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2751 2752 return (0); 2753 2754 fail: 2755 return (error); 2756 } 2757 2758 /********************************************************************* 2759 * 2760 * Initialize a receive ring and its buffers. 2761 * 2762 **********************************************************************/ 2763 int 2764 ixgbe_setup_receive_ring(struct rx_ring *rxr) 2765 { 2766 struct ix_softc *sc = rxr->sc; 2767 struct ifnet *ifp = &sc->arpcom.ac_if; 2768 int rsize, error; 2769 2770 rsize = roundup2(sc->num_rx_desc * 2771 sizeof(union ixgbe_adv_rx_desc), 4096); 2772 /* Clear the ring contents */ 2773 bzero((void *)rxr->rx_base, rsize); 2774 2775 if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0) 2776 return (error); 2777 2778 /* Setup our descriptor indices */ 2779 rxr->next_to_check = 0; 2780 rxr->last_desc_filled = sc->num_rx_desc - 1; 2781 2782 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2783 sc->num_rx_desc - 1); 2784 2785 ixgbe_rxfill(rxr); 2786 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 2787 printf("%s: unable to fill any rx descriptors\n", 2788 sc->dev.dv_xname); 2789 return (ENOBUFS); 2790 } 2791 2792 return (0); 2793 } 2794 2795 int 2796 ixgbe_rxfill(struct rx_ring *rxr) 2797 { 2798 struct ix_softc *sc = rxr->sc; 2799 int post = 0; 2800 u_int slots; 2801 int i; 2802 2803 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2804 0, rxr->rxdma.dma_map->dm_mapsize, 2805 BUS_DMASYNC_POSTWRITE); 2806 2807 i = rxr->last_desc_filled; 2808 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); 2809 slots > 0; slots--) { 2810 if (++i == sc->num_rx_desc) 2811 i = 0; 2812 2813 if (ixgbe_get_buf(rxr, i) != 0) 2814 break; 2815 2816 rxr->last_desc_filled = i; 2817 post = 1; 2818 } 2819 2820 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2821 0, rxr->rxdma.dma_map->dm_mapsize, 2822 BUS_DMASYNC_PREWRITE); 2823 2824 if_rxr_put(&rxr->rx_ring, slots); 2825 2826 return (post); 2827 } 2828 2829 void 2830 ixgbe_rxrefill(void *xrxr) 2831 { 2832 struct rx_ring *rxr = xrxr; 2833 struct ix_softc *sc = rxr->sc; 2834 2835 if (ixgbe_rxfill(rxr)) { 2836 /* Advance the Rx Queue "Tail Pointer" */ 2837 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me), 2838 rxr->last_desc_filled); 2839 } else if (if_rxr_inuse(&rxr->rx_ring) == 0) 2840 timeout_add(&rxr->rx_refill, 1); 2841 2842 } 2843 2844 /********************************************************************* 2845 * 2846 * Initialize all receive rings. 2847 * 2848 **********************************************************************/ 2849 int 2850 ixgbe_setup_receive_structures(struct ix_softc *sc) 2851 { 2852 struct rx_ring *rxr = sc->rx_rings; 2853 int i; 2854 2855 for (i = 0; i < sc->num_queues; i++, rxr++) 2856 if (ixgbe_setup_receive_ring(rxr)) 2857 goto fail; 2858 2859 return (0); 2860 fail: 2861 ixgbe_free_receive_structures(sc); 2862 return (ENOBUFS); 2863 } 2864 2865 /********************************************************************* 2866 * 2867 * Setup receive registers and features. 2868 * 2869 **********************************************************************/ 2870 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2871 2872 void 2873 ixgbe_initialize_receive_units(struct ix_softc *sc) 2874 { 2875 struct ifnet *ifp = &sc->arpcom.ac_if; 2876 struct rx_ring *rxr = sc->rx_rings; 2877 struct ixgbe_hw *hw = &sc->hw; 2878 uint32_t bufsz, fctrl, srrctl, rxcsum, rdrxctl; 2879 uint32_t hlreg; 2880 int i; 2881 2882 /* 2883 * Make sure receives are disabled while 2884 * setting up the descriptor ring 2885 */ 2886 ixgbe_disable_rx(hw); 2887 2888 /* Enable broadcasts */ 2889 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2890 fctrl |= IXGBE_FCTRL_BAM; 2891 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 2892 fctrl |= IXGBE_FCTRL_DPF; 2893 fctrl |= IXGBE_FCTRL_PMCF; 2894 } 2895 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 2896 2897 /* Always enable jumbo frame reception */ 2898 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2899 hlreg |= IXGBE_HLREG0_JUMBOEN; 2900 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2901 2902 if (ISSET(ifp->if_xflags, IFXF_TSO)) { 2903 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2904 2905 /* This field has to be set to zero. */ 2906 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2907 2908 /* Enable TSO Receive Offloading */ 2909 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 2910 rdrxctl |= IXGBE_RDRXCTL_FCOE_WRFIX; 2911 2912 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 2913 } 2914 2915 bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2916 2917 for (i = 0; i < sc->num_queues; i++, rxr++) { 2918 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2919 2920 /* Setup the Base and Length of the Rx Descriptor Ring */ 2921 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 2922 (rdba & 0x00000000ffffffffULL)); 2923 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 2924 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 2925 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2926 2927 /* Set up the SRRCTL register */ 2928 srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2929 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 2930 2931 if (ISSET(ifp->if_xflags, IFXF_TSO)) { 2932 rdrxctl = IXGBE_READ_REG(&sc->hw, IXGBE_RSCCTL(i)); 2933 2934 /* Enable TSO Receive Side Coalescing */ 2935 rdrxctl |= IXGBE_RSCCTL_RSCEN; 2936 rdrxctl |= IXGBE_RSCCTL_MAXDESC_16; 2937 2938 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), rdrxctl); 2939 } 2940 2941 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2942 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 2943 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 2944 } 2945 2946 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 2947 uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR | 2948 IXGBE_PSRTYPE_UDPHDR | 2949 IXGBE_PSRTYPE_IPV4HDR | 2950 IXGBE_PSRTYPE_IPV6HDR; 2951 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 2952 } 2953 2954 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2955 rxcsum &= ~IXGBE_RXCSUM_PCSD; 2956 2957 ixgbe_initialize_rss_mapping(sc); 2958 2959 /* Setup RSS */ 2960 if (sc->num_queues > 1) { 2961 /* RSS and RX IPP Checksum are mutually exclusive */ 2962 rxcsum |= IXGBE_RXCSUM_PCSD; 2963 } 2964 2965 /* Map QPRC/QPRDC/QPTC on a per queue basis */ 2966 ixgbe_map_queue_statistics(sc); 2967 2968 /* This is useful for calculating UDP/IP fragment checksums */ 2969 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2970 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2971 2972 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2973 } 2974 2975 void 2976 ixgbe_initialize_rss_mapping(struct ix_softc *sc) 2977 { 2978 struct ixgbe_hw *hw = &sc->hw; 2979 uint32_t reta = 0, mrqc, rss_key[10]; 2980 int i, j, queue_id, table_size, index_mult; 2981 2982 /* set up random bits */ 2983 stoeplitz_to_key(&rss_key, sizeof(rss_key)); 2984 2985 /* Set multiplier for RETA setup and table size based on MAC */ 2986 index_mult = 0x1; 2987 table_size = 128; 2988 switch (sc->hw.mac.type) { 2989 case ixgbe_mac_82598EB: 2990 index_mult = 0x11; 2991 break; 2992 case ixgbe_mac_X550: 2993 case ixgbe_mac_X550EM_x: 2994 case ixgbe_mac_X550EM_a: 2995 table_size = 512; 2996 break; 2997 default: 2998 break; 2999 } 3000 3001 /* Set up the redirection table */ 3002 for (i = 0, j = 0; i < table_size; i++, j++) { 3003 if (j == sc->num_queues) j = 0; 3004 queue_id = (j * index_mult); 3005 /* 3006 * The low 8 bits are for hash value (n+0); 3007 * The next 8 bits are for hash value (n+1), etc. 3008 */ 3009 reta = reta >> 8; 3010 reta = reta | ( ((uint32_t) queue_id) << 24); 3011 if ((i & 3) == 3) { 3012 if (i < 128) 3013 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 3014 else 3015 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 3016 reta); 3017 reta = 0; 3018 } 3019 } 3020 3021 /* Now fill our hash function seeds */ 3022 for (i = 0; i < 10; i++) 3023 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 3024 3025 /* 3026 * Disable UDP - IP fragments aren't currently being handled 3027 * and so we end up with a mix of 2-tuple and 4-tuple 3028 * traffic. 3029 */ 3030 mrqc = IXGBE_MRQC_RSSEN 3031 | IXGBE_MRQC_RSS_FIELD_IPV4 3032 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 3033 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 3034 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 3035 | IXGBE_MRQC_RSS_FIELD_IPV6 3036 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 3037 ; 3038 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3039 } 3040 3041 /********************************************************************* 3042 * 3043 * Free all receive rings. 3044 * 3045 **********************************************************************/ 3046 void 3047 ixgbe_free_receive_structures(struct ix_softc *sc) 3048 { 3049 struct rx_ring *rxr; 3050 int i; 3051 3052 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3053 if_rxr_init(&rxr->rx_ring, 0, 0); 3054 3055 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3056 ixgbe_free_receive_buffers(rxr); 3057 } 3058 3059 /********************************************************************* 3060 * 3061 * Free receive ring data structures 3062 * 3063 **********************************************************************/ 3064 void 3065 ixgbe_free_receive_buffers(struct rx_ring *rxr) 3066 { 3067 struct ix_softc *sc; 3068 struct ixgbe_rx_buf *rxbuf; 3069 int i; 3070 3071 sc = rxr->sc; 3072 if (rxr->rx_buffers != NULL) { 3073 for (i = 0; i < sc->num_rx_desc; i++) { 3074 rxbuf = &rxr->rx_buffers[i]; 3075 if (rxbuf->buf != NULL) { 3076 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 3077 0, rxbuf->map->dm_mapsize, 3078 BUS_DMASYNC_POSTREAD); 3079 bus_dmamap_unload(rxr->rxdma.dma_tag, 3080 rxbuf->map); 3081 m_freem(rxbuf->buf); 3082 rxbuf->buf = NULL; 3083 } 3084 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 3085 rxbuf->map = NULL; 3086 } 3087 free(rxr->rx_buffers, M_DEVBUF, 3088 sc->num_rx_desc * sizeof(struct ixgbe_rx_buf)); 3089 rxr->rx_buffers = NULL; 3090 } 3091 } 3092 3093 /********************************************************************* 3094 * 3095 * This routine executes in interrupt context. It replenishes 3096 * the mbufs in the descriptor and sends data which has been 3097 * dma'ed into host memory to upper layer. 3098 * 3099 *********************************************************************/ 3100 int 3101 ixgbe_rxeof(struct rx_ring *rxr) 3102 { 3103 struct ix_softc *sc = rxr->sc; 3104 struct ifnet *ifp = &sc->arpcom.ac_if; 3105 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 3106 struct mbuf *mp, *sendmp; 3107 uint8_t eop = 0; 3108 uint16_t len, vtag; 3109 uint32_t staterr = 0; 3110 struct ixgbe_rx_buf *rxbuf, *nxbuf; 3111 union ixgbe_adv_rx_desc *rxdesc; 3112 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 3113 int i, nextp, rsccnt; 3114 3115 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3116 return FALSE; 3117 3118 i = rxr->next_to_check; 3119 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 3120 uint32_t hash; 3121 uint16_t hashtype; 3122 3123 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3124 dsize * i, dsize, BUS_DMASYNC_POSTREAD); 3125 3126 rxdesc = &rxr->rx_base[i]; 3127 staterr = letoh32(rxdesc->wb.upper.status_error); 3128 if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) { 3129 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3130 dsize * i, dsize, 3131 BUS_DMASYNC_PREREAD); 3132 break; 3133 } 3134 3135 /* Zero out the receive descriptors status */ 3136 rxdesc->wb.upper.status_error = 0; 3137 rxbuf = &rxr->rx_buffers[i]; 3138 3139 /* pull the mbuf off the ring */ 3140 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 3141 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3142 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 3143 3144 mp = rxbuf->buf; 3145 len = letoh16(rxdesc->wb.upper.length); 3146 vtag = letoh16(rxdesc->wb.upper.vlan); 3147 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3148 hash = lemtoh32(&rxdesc->wb.lower.hi_dword.rss); 3149 hashtype = 3150 lemtoh16(&rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 3151 IXGBE_RXDADV_RSSTYPE_MASK; 3152 rsccnt = lemtoh32(&rxdesc->wb.lower.lo_dword.data) & 3153 IXGBE_RXDADV_RSCCNT_MASK; 3154 rsccnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; 3155 3156 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) { 3157 if (rxbuf->fmp) { 3158 m_freem(rxbuf->fmp); 3159 rxbuf->fmp = NULL; 3160 } 3161 3162 m_freem(mp); 3163 rxbuf->buf = NULL; 3164 goto next_desc; 3165 } 3166 3167 if (mp == NULL) { 3168 panic("%s: ixgbe_rxeof: NULL mbuf in slot %d " 3169 "(nrx %d, filled %d)", sc->dev.dv_xname, 3170 i, if_rxr_inuse(&rxr->rx_ring), 3171 rxr->last_desc_filled); 3172 } 3173 3174 if (!eop) { 3175 /* 3176 * Figure out the next descriptor of this frame. 3177 */ 3178 if (rsccnt) { 3179 nextp = staterr & IXGBE_RXDADV_NEXTP_MASK; 3180 nextp >>= IXGBE_RXDADV_NEXTP_SHIFT; 3181 } else { 3182 nextp = i + 1; 3183 } 3184 if (nextp == sc->num_rx_desc) 3185 nextp = 0; 3186 nxbuf = &rxr->rx_buffers[nextp]; 3187 /* prefetch(nxbuf); */ 3188 } 3189 3190 /* 3191 * Rather than using the fmp/lmp global pointers 3192 * we now keep the head of a packet chain in the 3193 * buffer struct and pass this along from one 3194 * descriptor to the next, until we get EOP. 3195 */ 3196 mp->m_len = len; 3197 /* 3198 * See if there is a stored head 3199 * that determines what we are 3200 */ 3201 sendmp = rxbuf->fmp; 3202 rxbuf->buf = rxbuf->fmp = NULL; 3203 3204 if (sendmp != NULL) /* secondary frag */ 3205 sendmp->m_pkthdr.len += mp->m_len; 3206 else { 3207 /* first desc of a non-ps chain */ 3208 sendmp = mp; 3209 sendmp->m_pkthdr.len = mp->m_len; 3210 #if NVLAN > 0 3211 if (sc->vlan_stripping && staterr & IXGBE_RXD_STAT_VP) { 3212 sendmp->m_pkthdr.ether_vtag = vtag; 3213 sendmp->m_flags |= M_VLANTAG; 3214 } 3215 #endif 3216 } 3217 3218 /* Pass the head pointer on */ 3219 if (eop == 0) { 3220 nxbuf->fmp = sendmp; 3221 sendmp = NULL; 3222 mp->m_next = nxbuf->buf; 3223 } else { /* Sending this frame? */ 3224 ixgbe_rx_checksum(staterr, sendmp); 3225 3226 if (hashtype != IXGBE_RXDADV_RSSTYPE_NONE) { 3227 sendmp->m_pkthdr.ph_flowid = hash; 3228 SET(sendmp->m_pkthdr.csum_flags, M_FLOWID); 3229 } 3230 3231 ml_enqueue(&ml, sendmp); 3232 } 3233 next_desc: 3234 if_rxr_put(&rxr->rx_ring, 1); 3235 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3236 dsize * i, dsize, 3237 BUS_DMASYNC_PREREAD); 3238 3239 /* Advance our pointers to the next descriptor. */ 3240 if (++i == sc->num_rx_desc) 3241 i = 0; 3242 } 3243 rxr->next_to_check = i; 3244 3245 if (ifiq_input(rxr->ifiq, &ml)) 3246 if_rxr_livelocked(&rxr->rx_ring); 3247 3248 if (!(staterr & IXGBE_RXD_STAT_DD)) 3249 return FALSE; 3250 3251 return TRUE; 3252 } 3253 3254 /********************************************************************* 3255 * 3256 * Verify that the hardware indicated that the checksum is valid. 3257 * Inform the stack about the status of checksum so that stack 3258 * doesn't spend time verifying the checksum. 3259 * 3260 *********************************************************************/ 3261 void 3262 ixgbe_rx_checksum(uint32_t staterr, struct mbuf * mp) 3263 { 3264 uint16_t status = (uint16_t) staterr; 3265 uint8_t errors = (uint8_t) (staterr >> 24); 3266 3267 if (status & IXGBE_RXD_STAT_IPCS) { 3268 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3269 /* IP Checksum Good */ 3270 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 3271 } else 3272 mp->m_pkthdr.csum_flags = 0; 3273 } 3274 if (status & IXGBE_RXD_STAT_L4CS) { 3275 if (!(errors & IXGBE_RXD_ERR_TCPE)) 3276 mp->m_pkthdr.csum_flags |= 3277 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 3278 } 3279 } 3280 3281 void 3282 ixgbe_setup_vlan_hw_support(struct ix_softc *sc) 3283 { 3284 struct ifnet *ifp = &sc->arpcom.ac_if; 3285 uint32_t ctrl; 3286 int i; 3287 3288 /* 3289 * We have to disable VLAN striping when using TCP offloading, due to a 3290 * firmware bug. 3291 */ 3292 if (ISSET(ifp->if_xflags, IFXF_TSO)) { 3293 sc->vlan_stripping = 0; 3294 return; 3295 } 3296 3297 sc->vlan_stripping = 1; 3298 3299 /* 3300 * A soft reset zero's out the VFTA, so 3301 * we need to repopulate it now. 3302 */ 3303 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 3304 if (sc->shadow_vfta[i] != 0) 3305 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i), 3306 sc->shadow_vfta[i]); 3307 } 3308 3309 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 3310 #if 0 3311 /* Enable the Filter Table if enabled */ 3312 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3313 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3314 ctrl |= IXGBE_VLNCTRL_VFE; 3315 } 3316 #endif 3317 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3318 ctrl |= IXGBE_VLNCTRL_VME; 3319 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 3320 3321 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 3322 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 3323 for (i = 0; i < sc->num_queues; i++) { 3324 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 3325 ctrl |= IXGBE_RXDCTL_VME; 3326 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl); 3327 } 3328 } 3329 } 3330 3331 void 3332 ixgbe_enable_intr(struct ix_softc *sc) 3333 { 3334 struct ixgbe_hw *hw = &sc->hw; 3335 uint32_t mask, fwsm; 3336 3337 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3338 /* Enable Fan Failure detection */ 3339 if (hw->device_id == IXGBE_DEV_ID_82598AT) 3340 mask |= IXGBE_EIMS_GPI_SDP1; 3341 3342 switch (sc->hw.mac.type) { 3343 case ixgbe_mac_82599EB: 3344 mask |= IXGBE_EIMS_ECC; 3345 /* Temperature sensor on some adapters */ 3346 mask |= IXGBE_EIMS_GPI_SDP0; 3347 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 3348 mask |= IXGBE_EIMS_GPI_SDP1; 3349 mask |= IXGBE_EIMS_GPI_SDP2; 3350 break; 3351 case ixgbe_mac_X540: 3352 mask |= IXGBE_EIMS_ECC; 3353 /* Detect if Thermal Sensor is enabled */ 3354 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3355 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3356 mask |= IXGBE_EIMS_TS; 3357 break; 3358 case ixgbe_mac_X550: 3359 case ixgbe_mac_X550EM_x: 3360 case ixgbe_mac_X550EM_a: 3361 mask |= IXGBE_EIMS_ECC; 3362 /* MAC thermal sensor is automatically enabled */ 3363 mask |= IXGBE_EIMS_TS; 3364 /* Some devices use SDP0 for important information */ 3365 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 3366 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 3367 mask |= IXGBE_EIMS_GPI_SDP0_X540; 3368 default: 3369 break; 3370 } 3371 3372 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3373 3374 /* With MSI-X we use auto clear */ 3375 if (sc->sc_intrmap) { 3376 mask = IXGBE_EIMS_ENABLE_MASK; 3377 /* Don't autoclear Link */ 3378 mask &= ~IXGBE_EIMS_OTHER; 3379 mask &= ~IXGBE_EIMS_LSC; 3380 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 3381 } 3382 3383 IXGBE_WRITE_FLUSH(hw); 3384 } 3385 3386 void 3387 ixgbe_disable_intr(struct ix_softc *sc) 3388 { 3389 if (sc->sc_intrmap) 3390 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3391 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3392 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3393 } else { 3394 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3395 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3396 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3397 } 3398 IXGBE_WRITE_FLUSH(&sc->hw); 3399 } 3400 3401 uint16_t 3402 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3403 { 3404 struct pci_attach_args *pa; 3405 uint32_t value; 3406 int high = 0; 3407 3408 if (reg & 0x2) { 3409 high = 1; 3410 reg &= ~0x2; 3411 } 3412 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3413 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3414 3415 if (high) 3416 value >>= 16; 3417 3418 return (value & 0xffff); 3419 } 3420 3421 void 3422 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3423 { 3424 struct pci_attach_args *pa; 3425 uint32_t rv; 3426 int high = 0; 3427 3428 /* Need to do read/mask/write... because 16 vs 32 bit!!! */ 3429 if (reg & 0x2) { 3430 high = 1; 3431 reg &= ~0x2; 3432 } 3433 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3434 rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3435 if (!high) 3436 rv = (rv & 0xffff0000) | value; 3437 else 3438 rv = (rv & 0xffff) | ((uint32_t)value << 16); 3439 pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv); 3440 } 3441 3442 /* 3443 * Setup the correct IVAR register for a particular MSIX interrupt 3444 * (yes this is all very magic and confusing :) 3445 * - entry is the register array entry 3446 * - vector is the MSIX vector for this queue 3447 * - type is RX/TX/MISC 3448 */ 3449 void 3450 ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type) 3451 { 3452 struct ixgbe_hw *hw = &sc->hw; 3453 uint32_t ivar, index; 3454 3455 vector |= IXGBE_IVAR_ALLOC_VAL; 3456 3457 switch (hw->mac.type) { 3458 3459 case ixgbe_mac_82598EB: 3460 if (type == -1) 3461 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3462 else 3463 entry += (type * 64); 3464 index = (entry >> 2) & 0x1F; 3465 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3466 ivar &= ~((uint32_t)0xFF << (8 * (entry & 0x3))); 3467 ivar |= ((uint32_t)vector << (8 * (entry & 0x3))); 3468 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 3469 break; 3470 3471 case ixgbe_mac_82599EB: 3472 case ixgbe_mac_X540: 3473 case ixgbe_mac_X550: 3474 case ixgbe_mac_X550EM_x: 3475 case ixgbe_mac_X550EM_a: 3476 if (type == -1) { /* MISC IVAR */ 3477 index = (entry & 1) * 8; 3478 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3479 ivar &= ~((uint32_t)0xFF << index); 3480 ivar |= ((uint32_t)vector << index); 3481 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3482 } else { /* RX/TX IVARS */ 3483 index = (16 * (entry & 1)) + (8 * type); 3484 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3485 ivar &= ~((uint32_t)0xFF << index); 3486 ivar |= ((uint32_t)vector << index); 3487 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3488 } 3489 3490 default: 3491 break; 3492 } 3493 } 3494 3495 void 3496 ixgbe_configure_ivars(struct ix_softc *sc) 3497 { 3498 struct ix_queue *que = sc->queues; 3499 uint32_t newitr; 3500 int i; 3501 3502 newitr = (4000000 / IXGBE_INTS_PER_SEC) & 0x0FF8; 3503 3504 for (i = 0; i < sc->num_queues; i++, que++) { 3505 /* First the RX queue entry */ 3506 ixgbe_set_ivar(sc, i, que->msix, 0); 3507 /* ... and the TX */ 3508 ixgbe_set_ivar(sc, i, que->msix, 1); 3509 /* Set an Initial EITR value */ 3510 IXGBE_WRITE_REG(&sc->hw, 3511 IXGBE_EITR(que->msix), newitr); 3512 } 3513 3514 /* For the Link interrupt */ 3515 ixgbe_set_ivar(sc, 1, sc->linkvec, -1); 3516 } 3517 3518 /* 3519 * SFP module interrupts handler 3520 */ 3521 void 3522 ixgbe_handle_mod(struct ix_softc *sc) 3523 { 3524 struct ixgbe_hw *hw = &sc->hw; 3525 uint32_t err; 3526 3527 err = hw->phy.ops.identify_sfp(hw); 3528 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3529 printf("%s: Unsupported SFP+ module type was detected!\n", 3530 sc->dev.dv_xname); 3531 return; 3532 } 3533 err = hw->mac.ops.setup_sfp(hw); 3534 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3535 printf("%s: Setup failure - unsupported SFP+ module type!\n", 3536 sc->dev.dv_xname); 3537 return; 3538 } 3539 3540 ixgbe_handle_msf(sc); 3541 } 3542 3543 3544 /* 3545 * MSF (multispeed fiber) interrupts handler 3546 */ 3547 void 3548 ixgbe_handle_msf(struct ix_softc *sc) 3549 { 3550 struct ixgbe_hw *hw = &sc->hw; 3551 uint32_t autoneg; 3552 bool negotiate; 3553 3554 autoneg = hw->phy.autoneg_advertised; 3555 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) { 3556 if (hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate)) 3557 return; 3558 } 3559 if (hw->mac.ops.setup_link) 3560 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3561 3562 ifmedia_delete_instance(&sc->media, IFM_INST_ANY); 3563 ixgbe_add_media_types(sc); 3564 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 3565 } 3566 3567 /* 3568 * External PHY interrupts handler 3569 */ 3570 void 3571 ixgbe_handle_phy(struct ix_softc *sc) 3572 { 3573 struct ixgbe_hw *hw = &sc->hw; 3574 int error; 3575 3576 error = hw->phy.ops.handle_lasi(hw); 3577 if (error == IXGBE_ERR_OVERTEMP) 3578 printf("%s: CRITICAL: EXTERNAL PHY OVER TEMP!! " 3579 " PHY will downshift to lower power state!\n", 3580 sc->dev.dv_xname); 3581 else if (error) 3582 printf("%s: Error handling LASI interrupt: %d\n", 3583 sc->dev.dv_xname, error); 3584 3585 } 3586 3587 #if NKSTAT > 0 3588 enum ix_counter_idx { 3589 ix_counter_crcerrs, 3590 ix_counter_lxontxc, 3591 ix_counter_lxonrxc, 3592 ix_counter_lxofftxc, 3593 ix_counter_lxoffrxc, 3594 ix_counter_prc64, 3595 ix_counter_prc127, 3596 ix_counter_prc255, 3597 ix_counter_prc511, 3598 ix_counter_prc1023, 3599 ix_counter_prc1522, 3600 ix_counter_gptc, 3601 ix_counter_gorc, 3602 ix_counter_gotc, 3603 ix_counter_ruc, 3604 ix_counter_rfc, 3605 ix_counter_roc, 3606 ix_counter_rjc, 3607 ix_counter_tor, 3608 ix_counter_tpr, 3609 ix_counter_tpt, 3610 ix_counter_gprc, 3611 ix_counter_bprc, 3612 ix_counter_mprc, 3613 ix_counter_ptc64, 3614 ix_counter_ptc127, 3615 ix_counter_ptc255, 3616 ix_counter_ptc511, 3617 ix_counter_ptc1023, 3618 ix_counter_ptc1522, 3619 ix_counter_mptc, 3620 ix_counter_bptc, 3621 3622 ix_counter_num, 3623 }; 3624 3625 CTASSERT(KSTAT_KV_U_PACKETS <= 0xff); 3626 CTASSERT(KSTAT_KV_U_BYTES <= 0xff); 3627 3628 struct ix_counter { 3629 char name[KSTAT_KV_NAMELEN]; 3630 uint32_t reg; 3631 uint8_t width; 3632 uint8_t unit; 3633 }; 3634 3635 static const struct ix_counter ix_counters[ix_counter_num] = { 3636 [ix_counter_crcerrs] = { "crc errs", IXGBE_CRCERRS, 32, 3637 KSTAT_KV_U_PACKETS }, 3638 [ix_counter_lxontxc] = { "tx link xon", IXGBE_LXONTXC, 32, 3639 KSTAT_KV_U_PACKETS }, 3640 [ix_counter_lxonrxc] = { "rx link xon", 0, 32, 3641 KSTAT_KV_U_PACKETS }, 3642 [ix_counter_lxofftxc] = { "tx link xoff", IXGBE_LXOFFTXC, 32, 3643 KSTAT_KV_U_PACKETS }, 3644 [ix_counter_lxoffrxc] = { "rx link xoff", 0, 32, 3645 KSTAT_KV_U_PACKETS }, 3646 [ix_counter_prc64] = { "rx 64B", IXGBE_PRC64, 32, 3647 KSTAT_KV_U_PACKETS }, 3648 [ix_counter_prc127] = { "rx 65-127B", IXGBE_PRC127, 32, 3649 KSTAT_KV_U_PACKETS }, 3650 [ix_counter_prc255] = { "rx 128-255B", IXGBE_PRC255, 32, 3651 KSTAT_KV_U_PACKETS }, 3652 [ix_counter_prc511] = { "rx 256-511B", IXGBE_PRC511, 32, 3653 KSTAT_KV_U_PACKETS }, 3654 [ix_counter_prc1023] = { "rx 512-1023B", IXGBE_PRC1023, 32, 3655 KSTAT_KV_U_PACKETS }, 3656 [ix_counter_prc1522] = { "rx 1024-maxB", IXGBE_PRC1522, 32, 3657 KSTAT_KV_U_PACKETS }, 3658 [ix_counter_gptc] = { "tx good", IXGBE_GPTC, 32, 3659 KSTAT_KV_U_PACKETS }, 3660 [ix_counter_gorc] = { "rx good", IXGBE_GORCL, 36, 3661 KSTAT_KV_U_BYTES }, 3662 [ix_counter_gotc] = { "tx good", IXGBE_GOTCL, 36, 3663 KSTAT_KV_U_BYTES }, 3664 [ix_counter_ruc] = { "rx undersize", IXGBE_RUC, 32, 3665 KSTAT_KV_U_PACKETS }, 3666 [ix_counter_rfc] = { "rx fragment", IXGBE_RFC, 32, 3667 KSTAT_KV_U_PACKETS }, 3668 [ix_counter_roc] = { "rx oversize", IXGBE_ROC, 32, 3669 KSTAT_KV_U_PACKETS }, 3670 [ix_counter_rjc] = { "rx jabber", IXGBE_RJC, 32, 3671 KSTAT_KV_U_PACKETS }, 3672 [ix_counter_tor] = { "rx total", IXGBE_TORL, 36, 3673 KSTAT_KV_U_BYTES }, 3674 [ix_counter_tpr] = { "rx total", IXGBE_TPR, 32, 3675 KSTAT_KV_U_PACKETS }, 3676 [ix_counter_tpt] = { "tx total", IXGBE_TPT, 32, 3677 KSTAT_KV_U_PACKETS }, 3678 [ix_counter_gprc] = { "rx good", IXGBE_GPRC, 32, 3679 KSTAT_KV_U_PACKETS }, 3680 [ix_counter_bprc] = { "rx bcast", IXGBE_BPRC, 32, 3681 KSTAT_KV_U_PACKETS }, 3682 [ix_counter_mprc] = { "rx mcast", IXGBE_MPRC, 32, 3683 KSTAT_KV_U_PACKETS }, 3684 [ix_counter_ptc64] = { "tx 64B", IXGBE_PTC64, 32, 3685 KSTAT_KV_U_PACKETS }, 3686 [ix_counter_ptc127] = { "tx 65-127B", IXGBE_PTC127, 32, 3687 KSTAT_KV_U_PACKETS }, 3688 [ix_counter_ptc255] = { "tx 128-255B", IXGBE_PTC255, 32, 3689 KSTAT_KV_U_PACKETS }, 3690 [ix_counter_ptc511] = { "tx 256-511B", IXGBE_PTC511, 32, 3691 KSTAT_KV_U_PACKETS }, 3692 [ix_counter_ptc1023] = { "tx 512-1023B", IXGBE_PTC1023, 32, 3693 KSTAT_KV_U_PACKETS }, 3694 [ix_counter_ptc1522] = { "tx 1024-maxB", IXGBE_PTC1522, 32, 3695 KSTAT_KV_U_PACKETS }, 3696 [ix_counter_mptc] = { "tx mcast", IXGBE_MPTC, 32, 3697 KSTAT_KV_U_PACKETS }, 3698 [ix_counter_bptc] = { "tx bcast", IXGBE_BPTC, 32, 3699 KSTAT_KV_U_PACKETS }, 3700 }; 3701 3702 struct ix_rxq_kstats { 3703 struct kstat_kv qprc; 3704 struct kstat_kv qbrc; 3705 struct kstat_kv qprdc; 3706 }; 3707 3708 static const struct ix_rxq_kstats ix_rxq_kstats_tpl = { 3709 KSTAT_KV_UNIT_INITIALIZER("packets", 3710 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3711 KSTAT_KV_UNIT_INITIALIZER("bytes", 3712 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3713 KSTAT_KV_UNIT_INITIALIZER("qdrops", 3714 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3715 }; 3716 3717 struct ix_txq_kstats { 3718 struct kstat_kv qptc; 3719 struct kstat_kv qbtc; 3720 }; 3721 3722 static const struct ix_txq_kstats ix_txq_kstats_tpl = { 3723 KSTAT_KV_UNIT_INITIALIZER("packets", 3724 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3725 KSTAT_KV_UNIT_INITIALIZER("bytes", 3726 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3727 }; 3728 3729 static int ix_kstats_read(struct kstat *ks); 3730 static int ix_rxq_kstats_read(struct kstat *ks); 3731 static int ix_txq_kstats_read(struct kstat *ks); 3732 3733 static void 3734 ix_kstats(struct ix_softc *sc) 3735 { 3736 struct kstat *ks; 3737 struct kstat_kv *kvs; 3738 unsigned int i; 3739 3740 mtx_init(&sc->sc_kstat_mtx, IPL_SOFTCLOCK); 3741 timeout_set(&sc->sc_kstat_tmo, ix_kstats_tick, sc); 3742 3743 ks = kstat_create(sc->dev.dv_xname, 0, "ix-stats", 0, 3744 KSTAT_T_KV, 0); 3745 if (ks == NULL) 3746 return; 3747 3748 kvs = mallocarray(nitems(ix_counters), sizeof(*kvs), 3749 M_DEVBUF, M_WAITOK|M_ZERO); 3750 3751 for (i = 0; i < nitems(ix_counters); i++) { 3752 const struct ix_counter *ixc = &ix_counters[i]; 3753 3754 kstat_kv_unit_init(&kvs[i], ixc->name, 3755 KSTAT_KV_T_COUNTER64, ixc->unit); 3756 } 3757 3758 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3759 ks->ks_softc = sc; 3760 ks->ks_data = kvs; 3761 ks->ks_datalen = nitems(ix_counters) * sizeof(*kvs); 3762 ks->ks_read = ix_kstats_read; 3763 3764 sc->sc_kstat = ks; 3765 kstat_install(ks); 3766 } 3767 3768 static void 3769 ix_rxq_kstats(struct ix_softc *sc, struct rx_ring *rxr) 3770 { 3771 struct ix_rxq_kstats *stats; 3772 struct kstat *ks; 3773 3774 ks = kstat_create(sc->dev.dv_xname, 0, "ix-rxq", rxr->me, 3775 KSTAT_T_KV, 0); 3776 if (ks == NULL) 3777 return; 3778 3779 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3780 *stats = ix_rxq_kstats_tpl; 3781 3782 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3783 ks->ks_softc = rxr; 3784 ks->ks_data = stats; 3785 ks->ks_datalen = sizeof(*stats); 3786 ks->ks_read = ix_rxq_kstats_read; 3787 3788 rxr->kstat = ks; 3789 kstat_install(ks); 3790 } 3791 3792 static void 3793 ix_txq_kstats(struct ix_softc *sc, struct tx_ring *txr) 3794 { 3795 struct ix_txq_kstats *stats; 3796 struct kstat *ks; 3797 3798 ks = kstat_create(sc->dev.dv_xname, 0, "ix-txq", txr->me, 3799 KSTAT_T_KV, 0); 3800 if (ks == NULL) 3801 return; 3802 3803 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3804 *stats = ix_txq_kstats_tpl; 3805 3806 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3807 ks->ks_softc = txr; 3808 ks->ks_data = stats; 3809 ks->ks_datalen = sizeof(*stats); 3810 ks->ks_read = ix_txq_kstats_read; 3811 3812 txr->kstat = ks; 3813 kstat_install(ks); 3814 } 3815 3816 /********************************************************************** 3817 * 3818 * Update the board statistics counters. 3819 * 3820 **********************************************************************/ 3821 3822 static void 3823 ix_kstats_tick(void *arg) 3824 { 3825 struct ix_softc *sc = arg; 3826 int i; 3827 3828 timeout_add_sec(&sc->sc_kstat_tmo, 1); 3829 3830 mtx_enter(&sc->sc_kstat_mtx); 3831 ix_kstats_read(sc->sc_kstat); 3832 for (i = 0; i < sc->num_queues; i++) { 3833 ix_rxq_kstats_read(sc->rx_rings[i].kstat); 3834 ix_txq_kstats_read(sc->tx_rings[i].kstat); 3835 } 3836 mtx_leave(&sc->sc_kstat_mtx); 3837 } 3838 3839 static uint64_t 3840 ix_read36(struct ixgbe_hw *hw, bus_size_t loreg, bus_size_t hireg) 3841 { 3842 uint64_t lo, hi; 3843 3844 lo = IXGBE_READ_REG(hw, loreg); 3845 hi = IXGBE_READ_REG(hw, hireg); 3846 3847 return (((hi & 0xf) << 32) | lo); 3848 } 3849 3850 static int 3851 ix_kstats_read(struct kstat *ks) 3852 { 3853 struct ix_softc *sc = ks->ks_softc; 3854 struct kstat_kv *kvs = ks->ks_data; 3855 struct ixgbe_hw *hw = &sc->hw; 3856 unsigned int i; 3857 3858 for (i = 0; i < nitems(ix_counters); i++) { 3859 const struct ix_counter *ixc = &ix_counters[i]; 3860 uint32_t reg = ixc->reg; 3861 uint64_t v; 3862 3863 if (reg == 0) 3864 continue; 3865 3866 if (ixc->width > 32) { 3867 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3868 v = IXGBE_READ_REG(hw, reg + 4); 3869 else 3870 v = ix_read36(hw, reg, reg + 4); 3871 } else 3872 v = IXGBE_READ_REG(hw, reg); 3873 3874 kstat_kv_u64(&kvs[i]) += v; 3875 } 3876 3877 /* handle the exceptions */ 3878 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3879 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3880 IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3881 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3882 IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3883 } else { 3884 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3885 IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3886 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3887 IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3888 } 3889 3890 getnanouptime(&ks->ks_updated); 3891 3892 return (0); 3893 } 3894 3895 int 3896 ix_rxq_kstats_read(struct kstat *ks) 3897 { 3898 struct ix_rxq_kstats *stats = ks->ks_data; 3899 struct rx_ring *rxr = ks->ks_softc; 3900 struct ix_softc *sc = rxr->sc; 3901 struct ixgbe_hw *hw = &sc->hw; 3902 uint32_t i = rxr->me; 3903 3904 kstat_kv_u64(&stats->qprc) += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3905 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3906 kstat_kv_u64(&stats->qprdc) += 3907 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3908 kstat_kv_u64(&stats->qbrc) += 3909 IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 3910 } else { 3911 kstat_kv_u64(&stats->qprdc) += 3912 IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3913 kstat_kv_u64(&stats->qbrc) += 3914 ix_read36(hw, IXGBE_QBRC_L(i), IXGBE_QBRC_H(i)); 3915 } 3916 3917 getnanouptime(&ks->ks_updated); 3918 3919 return (0); 3920 } 3921 3922 int 3923 ix_txq_kstats_read(struct kstat *ks) 3924 { 3925 struct ix_txq_kstats *stats = ks->ks_data; 3926 struct tx_ring *txr = ks->ks_softc; 3927 struct ix_softc *sc = txr->sc; 3928 struct ixgbe_hw *hw = &sc->hw; 3929 uint32_t i = txr->me; 3930 3931 kstat_kv_u64(&stats->qptc) += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3932 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3933 kstat_kv_u64(&stats->qbtc) += 3934 IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 3935 } else { 3936 kstat_kv_u64(&stats->qbtc) += 3937 ix_read36(hw, IXGBE_QBTC_L(i), IXGBE_QBTC_H(i)); 3938 } 3939 3940 getnanouptime(&ks->ks_updated); 3941 3942 return (0); 3943 } 3944 #endif /* NKVSTAT > 0 */ 3945 3946 void 3947 ixgbe_map_queue_statistics(struct ix_softc *sc) 3948 { 3949 int i; 3950 uint32_t r; 3951 3952 for (i = 0; i < 32; i++) { 3953 /* 3954 * Queues 0-15 are mapped 1:1 3955 * Queue 0 -> Counter 0 3956 * Queue 1 -> Counter 1 3957 * Queue 2 -> Counter 2.... 3958 * Queues 16-127 are mapped to Counter 0 3959 */ 3960 if (i < 4) { 3961 r = (i * 4 + 0); 3962 r |= (i * 4 + 1) << 8; 3963 r |= (i * 4 + 2) << 16; 3964 r |= (i * 4 + 3) << 24; 3965 } else 3966 r = 0; 3967 3968 IXGBE_WRITE_REG(&sc->hw, IXGBE_RQSMR(i), r); 3969 IXGBE_WRITE_REG(&sc->hw, IXGBE_TQSM(i), r); 3970 } 3971 } 3972