1 /* $OpenBSD: if_igc.c,v 1.24 2024/05/21 11:19:39 bluhm Exp $ */ 2 /*- 3 * SPDX-License-Identifier: BSD-2-Clause 4 * 5 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 6 * All rights reserved. 7 * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include "bpfilter.h" 32 #include "vlan.h" 33 #include "kstat.h" 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/sockio.h> 38 #include <sys/mbuf.h> 39 #include <sys/malloc.h> 40 #include <sys/kernel.h> 41 #include <sys/socket.h> 42 #include <sys/device.h> 43 #include <sys/endian.h> 44 #include <sys/intrmap.h> 45 #include <sys/kstat.h> 46 47 #include <net/if.h> 48 #include <net/if_media.h> 49 #include <net/route.h> 50 #include <net/toeplitz.h> 51 52 #include <netinet/in.h> 53 #include <netinet/if_ether.h> 54 #include <netinet/tcp.h> 55 #include <netinet/tcp_timer.h> 56 #include <netinet/tcp_var.h> 57 58 #if NBPFILTER > 0 59 #include <net/bpf.h> 60 #endif 61 62 #include <machine/bus.h> 63 #include <machine/intr.h> 64 65 #include <dev/pci/pcivar.h> 66 #include <dev/pci/pcireg.h> 67 #include <dev/pci/pcidevs.h> 68 #include <dev/pci/if_igc.h> 69 #include <dev/pci/igc_hw.h> 70 71 const struct pci_matchid igc_devices[] = { 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT }, 77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K }, 78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2 }, 79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM }, 80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP }, 81 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V }, 82 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM }, 83 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT }, 84 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM }, 85 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K }, 86 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V } 87 }; 88 89 /********************************************************************* 90 * Function Prototypes 91 *********************************************************************/ 92 int igc_match(struct device *, void *, void *); 93 void igc_attach(struct device *, struct device *, void *); 94 int igc_detach(struct device *, int); 95 96 void igc_identify_hardware(struct igc_softc *); 97 int igc_allocate_pci_resources(struct igc_softc *); 98 int igc_allocate_queues(struct igc_softc *); 99 void igc_free_pci_resources(struct igc_softc *); 100 void igc_reset(struct igc_softc *); 101 void igc_init_dmac(struct igc_softc *, uint32_t); 102 int igc_allocate_msix(struct igc_softc *); 103 void igc_setup_msix(struct igc_softc *); 104 int igc_dma_malloc(struct igc_softc *, bus_size_t, struct igc_dma_alloc *); 105 void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 106 void igc_setup_interface(struct igc_softc *); 107 108 void igc_init(void *); 109 void igc_start(struct ifqueue *); 110 int igc_txeof(struct igc_txring *); 111 void igc_stop(struct igc_softc *); 112 int igc_ioctl(struct ifnet *, u_long, caddr_t); 113 int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 114 int igc_rxfill(struct igc_rxring *); 115 void igc_rxrefill(void *); 116 int igc_rxeof(struct igc_rxring *); 117 void igc_rx_checksum(uint32_t, struct mbuf *, uint32_t); 118 void igc_watchdog(struct ifnet *); 119 void igc_media_status(struct ifnet *, struct ifmediareq *); 120 int igc_media_change(struct ifnet *); 121 void igc_iff(struct igc_softc *); 122 void igc_update_link_status(struct igc_softc *); 123 int igc_get_buf(struct igc_rxring *, int); 124 int igc_tx_ctx_setup(struct igc_txring *, struct mbuf *, int, uint32_t *, 125 uint32_t *); 126 127 void igc_configure_queues(struct igc_softc *); 128 void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 129 void igc_enable_queue(struct igc_softc *, uint32_t); 130 void igc_enable_intr(struct igc_softc *); 131 void igc_disable_intr(struct igc_softc *); 132 int igc_intr_link(void *); 133 int igc_intr_queue(void *); 134 135 int igc_allocate_transmit_buffers(struct igc_txring *); 136 int igc_setup_transmit_structures(struct igc_softc *); 137 int igc_setup_transmit_ring(struct igc_txring *); 138 void igc_initialize_transmit_unit(struct igc_softc *); 139 void igc_free_transmit_structures(struct igc_softc *); 140 void igc_free_transmit_buffers(struct igc_txring *); 141 int igc_allocate_receive_buffers(struct igc_rxring *); 142 int igc_setup_receive_structures(struct igc_softc *); 143 int igc_setup_receive_ring(struct igc_rxring *); 144 void igc_initialize_receive_unit(struct igc_softc *); 145 void igc_free_receive_structures(struct igc_softc *); 146 void igc_free_receive_buffers(struct igc_rxring *); 147 void igc_initialize_rss_mapping(struct igc_softc *); 148 149 void igc_get_hw_control(struct igc_softc *); 150 void igc_release_hw_control(struct igc_softc *); 151 int igc_is_valid_ether_addr(uint8_t *); 152 153 #if NKSTAT > 0 154 void igc_kstat_attach(struct igc_softc *); 155 #endif 156 157 /********************************************************************* 158 * OpenBSD Device Interface Entry Points 159 *********************************************************************/ 160 161 struct cfdriver igc_cd = { 162 NULL, "igc", DV_IFNET 163 }; 164 165 const struct cfattach igc_ca = { 166 sizeof(struct igc_softc), igc_match, igc_attach, igc_detach 167 }; 168 169 /********************************************************************* 170 * Device identification routine 171 * 172 * igc_match determines if the driver should be loaded on 173 * adapter based on PCI vendor/device id of the adapter. 174 * 175 * return 0 on success, positive on failure 176 *********************************************************************/ 177 int 178 igc_match(struct device *parent, void *match, void *aux) 179 { 180 return pci_matchbyid((struct pci_attach_args *)aux, igc_devices, 181 nitems(igc_devices)); 182 } 183 184 /********************************************************************* 185 * Device initialization routine 186 * 187 * The attach entry point is called when the driver is being loaded. 188 * This routine identifies the type of hardware, allocates all resources 189 * and initializes the hardware. 190 * 191 * return 0 on success, positive on failure 192 *********************************************************************/ 193 void 194 igc_attach(struct device *parent, struct device *self, void *aux) 195 { 196 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 197 struct igc_softc *sc = (struct igc_softc *)self; 198 struct igc_hw *hw = &sc->hw; 199 200 sc->osdep.os_sc = sc; 201 sc->osdep.os_pa = *pa; 202 203 /* Determine hardware and mac info */ 204 igc_identify_hardware(sc); 205 206 sc->num_tx_desc = IGC_DEFAULT_TXD; 207 sc->num_rx_desc = IGC_DEFAULT_RXD; 208 209 /* Setup PCI resources */ 210 if (igc_allocate_pci_resources(sc)) 211 goto err_pci; 212 213 /* Allocate TX/RX queues */ 214 if (igc_allocate_queues(sc)) 215 goto err_pci; 216 217 /* Do shared code initialization */ 218 if (igc_setup_init_funcs(hw, true)) { 219 printf(": Setup of shared code failed\n"); 220 goto err_pci; 221 } 222 223 hw->mac.autoneg = DO_AUTO_NEG; 224 hw->phy.autoneg_wait_to_complete = false; 225 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 226 227 /* Copper options. */ 228 if (hw->phy.media_type == igc_media_type_copper) 229 hw->phy.mdix = AUTO_ALL_MODES; 230 231 /* Set the max frame size. */ 232 sc->hw.mac.max_frame_size = 9234; 233 234 /* Allocate multicast array memory. */ 235 sc->mta = mallocarray(ETHER_ADDR_LEN, MAX_NUM_MULTICAST_ADDRESSES, 236 M_DEVBUF, M_NOWAIT); 237 if (sc->mta == NULL) { 238 printf(": Can not allocate multicast setup array\n"); 239 goto err_late; 240 } 241 242 /* Check SOL/IDER usage. */ 243 if (igc_check_reset_block(hw)) 244 printf(": PHY reset is blocked due to SOL/IDER session\n"); 245 246 /* Disable Energy Efficient Ethernet. */ 247 sc->hw.dev_spec._i225.eee_disable = true; 248 249 igc_reset_hw(hw); 250 251 /* Make sure we have a good EEPROM before we read from it. */ 252 if (igc_validate_nvm_checksum(hw) < 0) { 253 /* 254 * Some PCI-E parts fail the first check due to 255 * the link being in sleep state, call it again, 256 * if it fails a second time its a real issue. 257 */ 258 if (igc_validate_nvm_checksum(hw) < 0) { 259 printf(": The EEPROM checksum is not valid\n"); 260 goto err_late; 261 } 262 } 263 264 /* Copy the permanent MAC address out of the EEPROM. */ 265 if (igc_read_mac_addr(hw) < 0) { 266 printf(": EEPROM read error while reading MAC address\n"); 267 goto err_late; 268 } 269 270 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 271 printf(": Invalid MAC address\n"); 272 goto err_late; 273 } 274 275 memcpy(sc->sc_ac.ac_enaddr, sc->hw.mac.addr, ETHER_ADDR_LEN); 276 277 if (igc_allocate_msix(sc)) 278 goto err_late; 279 280 /* Setup OS specific network interface. */ 281 igc_setup_interface(sc); 282 283 igc_reset(sc); 284 hw->mac.get_link_status = true; 285 igc_update_link_status(sc); 286 287 /* The driver can now take control from firmware. */ 288 igc_get_hw_control(sc); 289 290 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 291 292 #if NKSTAT > 0 293 igc_kstat_attach(sc); 294 #endif 295 return; 296 297 err_late: 298 igc_release_hw_control(sc); 299 err_pci: 300 igc_free_pci_resources(sc); 301 free(sc->mta, M_DEVBUF, ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 302 } 303 304 /********************************************************************* 305 * Device removal routine 306 * 307 * The detach entry point is called when the driver is being removed. 308 * This routine stops the adapter and deallocates all the resources 309 * that were allocated for driver operation. 310 * 311 * return 0 on success, positive on failure 312 *********************************************************************/ 313 int 314 igc_detach(struct device *self, int flags) 315 { 316 struct igc_softc *sc = (struct igc_softc *)self; 317 struct ifnet *ifp = &sc->sc_ac.ac_if; 318 319 igc_stop(sc); 320 321 igc_phy_hw_reset(&sc->hw); 322 igc_release_hw_control(sc); 323 324 ether_ifdetach(ifp); 325 if_detach(ifp); 326 327 igc_free_pci_resources(sc); 328 329 igc_free_transmit_structures(sc); 330 igc_free_receive_structures(sc); 331 free(sc->mta, M_DEVBUF, ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 332 333 return 0; 334 } 335 336 void 337 igc_identify_hardware(struct igc_softc *sc) 338 { 339 struct igc_osdep *os = &sc->osdep; 340 struct pci_attach_args *pa = &os->os_pa; 341 342 /* Save off the information about this board. */ 343 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 344 345 /* Do shared code init and setup. */ 346 if (igc_set_mac_type(&sc->hw)) { 347 printf(": Setup init failure\n"); 348 return; 349 } 350 } 351 352 int 353 igc_allocate_pci_resources(struct igc_softc *sc) 354 { 355 struct igc_osdep *os = &sc->osdep; 356 struct pci_attach_args *pa = &os->os_pa; 357 pcireg_t memtype; 358 359 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 360 if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 361 &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 362 printf(": unable to map registers\n"); 363 return ENXIO; 364 } 365 sc->hw.hw_addr = (uint8_t *)os->os_membase; 366 sc->hw.back = os; 367 368 igc_setup_msix(sc); 369 370 return 0; 371 } 372 373 int 374 igc_allocate_queues(struct igc_softc *sc) 375 { 376 struct igc_queue *iq; 377 struct igc_txring *txr; 378 struct igc_rxring *rxr; 379 int i, rsize, rxconf, tsize, txconf; 380 381 /* Allocate the top level queue structs. */ 382 sc->queues = mallocarray(sc->sc_nqueues, sizeof(struct igc_queue), 383 M_DEVBUF, M_NOWAIT | M_ZERO); 384 if (sc->queues == NULL) { 385 printf("%s: unable to allocate queue\n", DEVNAME(sc)); 386 goto fail; 387 } 388 389 /* Allocate the TX ring. */ 390 sc->tx_rings = mallocarray(sc->sc_nqueues, sizeof(struct igc_txring), 391 M_DEVBUF, M_NOWAIT | M_ZERO); 392 if (sc->tx_rings == NULL) { 393 printf("%s: unable to allocate TX ring\n", DEVNAME(sc)); 394 goto fail; 395 } 396 397 /* Allocate the RX ring. */ 398 sc->rx_rings = mallocarray(sc->sc_nqueues, sizeof(struct igc_rxring), 399 M_DEVBUF, M_NOWAIT | M_ZERO); 400 if (sc->rx_rings == NULL) { 401 printf("%s: unable to allocate RX ring\n", DEVNAME(sc)); 402 goto rx_fail; 403 } 404 405 txconf = rxconf = 0; 406 407 /* Set up the TX queues. */ 408 tsize = roundup2(sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 409 IGC_DBA_ALIGN); 410 for (i = 0; i < sc->sc_nqueues; i++, txconf++) { 411 txr = &sc->tx_rings[i]; 412 txr->sc = sc; 413 txr->me = i; 414 415 if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 416 printf("%s: unable to allocate TX descriptor\n", 417 DEVNAME(sc)); 418 goto err_tx_desc; 419 } 420 txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 421 bzero((void *)txr->tx_base, tsize); 422 } 423 424 /* Set up the RX queues. */ 425 rsize = roundup2(sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 426 IGC_DBA_ALIGN); 427 for (i = 0; i < sc->sc_nqueues; i++, rxconf++) { 428 rxr = &sc->rx_rings[i]; 429 rxr->sc = sc; 430 rxr->me = i; 431 timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 432 433 if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 434 printf("%s: unable to allocate RX descriptor\n", 435 DEVNAME(sc)); 436 goto err_rx_desc; 437 } 438 rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 439 bzero((void *)rxr->rx_base, rsize); 440 } 441 442 /* Set up the queue holding structs. */ 443 for (i = 0; i < sc->sc_nqueues; i++) { 444 iq = &sc->queues[i]; 445 iq->sc = sc; 446 iq->txr = &sc->tx_rings[i]; 447 iq->rxr = &sc->rx_rings[i]; 448 snprintf(iq->name, sizeof(iq->name), "%s:%d", DEVNAME(sc), i); 449 } 450 451 return 0; 452 453 err_rx_desc: 454 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 455 igc_dma_free(sc, &rxr->rxdma); 456 err_tx_desc: 457 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 458 igc_dma_free(sc, &txr->txdma); 459 free(sc->rx_rings, M_DEVBUF, 460 sc->sc_nqueues * sizeof(struct igc_rxring)); 461 sc->rx_rings = NULL; 462 rx_fail: 463 free(sc->tx_rings, M_DEVBUF, 464 sc->sc_nqueues * sizeof(struct igc_txring)); 465 sc->tx_rings = NULL; 466 fail: 467 return ENOMEM; 468 } 469 470 void 471 igc_free_pci_resources(struct igc_softc *sc) 472 { 473 struct igc_osdep *os = &sc->osdep; 474 struct pci_attach_args *pa = &os->os_pa; 475 struct igc_queue *iq = sc->queues; 476 int i; 477 478 /* Release all msix queue resources. */ 479 for (i = 0; i < sc->sc_nqueues; i++, iq++) { 480 if (iq->tag) 481 pci_intr_disestablish(pa->pa_pc, iq->tag); 482 iq->tag = NULL; 483 } 484 485 if (sc->tag) 486 pci_intr_disestablish(pa->pa_pc, sc->tag); 487 sc->tag = NULL; 488 if (os->os_membase != 0) 489 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 490 os->os_membase = 0; 491 } 492 493 /********************************************************************* 494 * 495 * Initialize the hardware to a configuration as specified by the 496 * adapter structure. 497 * 498 **********************************************************************/ 499 void 500 igc_reset(struct igc_softc *sc) 501 { 502 struct igc_hw *hw = &sc->hw; 503 uint32_t pba; 504 uint16_t rx_buffer_size; 505 506 /* Let the firmware know the OS is in control */ 507 igc_get_hw_control(sc); 508 509 /* 510 * Packet Buffer Allocation (PBA) 511 * Writing PBA sets the receive portion of the buffer 512 * the remainder is used for the transmit buffer. 513 */ 514 pba = IGC_PBA_34K; 515 516 /* 517 * These parameters control the automatic generation (Tx) and 518 * response (Rx) to Ethernet PAUSE frames. 519 * - High water mark should allow for at least two frames to be 520 * received after sending an XOFF. 521 * - Low water mark works best when it is very near the high water mark. 522 * This allows the receiver to restart by sending XON when it has 523 * drained a bit. Here we use an arbitrary value of 1500 which will 524 * restart after one full frame is pulled from the buffer. There 525 * could be several smaller frames in the buffer and if so they will 526 * not trigger the XON until their total number reduces the buffer 527 * by 1500. 528 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 529 */ 530 rx_buffer_size = (pba & 0xffff) << 10; 531 hw->fc.high_water = rx_buffer_size - 532 roundup2(sc->hw.mac.max_frame_size, 1024); 533 /* 16-byte granularity */ 534 hw->fc.low_water = hw->fc.high_water - 16; 535 536 if (sc->fc) /* locally set flow control value? */ 537 hw->fc.requested_mode = sc->fc; 538 else 539 hw->fc.requested_mode = igc_fc_full; 540 541 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 542 543 hw->fc.send_xon = true; 544 545 /* Issue a global reset */ 546 igc_reset_hw(hw); 547 IGC_WRITE_REG(hw, IGC_WUC, 0); 548 549 /* and a re-init */ 550 if (igc_init_hw(hw) < 0) { 551 printf(": Hardware Initialization Failed\n"); 552 return; 553 } 554 555 /* Setup DMA Coalescing */ 556 igc_init_dmac(sc, pba); 557 558 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 559 igc_get_phy_info(hw); 560 igc_check_for_link(hw); 561 } 562 563 /********************************************************************* 564 * 565 * Initialize the DMA Coalescing feature 566 * 567 **********************************************************************/ 568 void 569 igc_init_dmac(struct igc_softc *sc, uint32_t pba) 570 { 571 struct igc_hw *hw = &sc->hw; 572 uint32_t dmac, reg = ~IGC_DMACR_DMAC_EN; 573 uint16_t hwm, max_frame_size; 574 int status; 575 576 max_frame_size = sc->hw.mac.max_frame_size; 577 578 if (sc->dmac == 0) { /* Disabling it */ 579 IGC_WRITE_REG(hw, IGC_DMACR, reg); 580 return; 581 } else 582 printf(": DMA Coalescing enabled\n"); 583 584 /* Set starting threshold */ 585 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 586 587 hwm = 64 * pba - max_frame_size / 16; 588 if (hwm < 64 * (pba - 6)) 589 hwm = 64 * (pba - 6); 590 reg = IGC_READ_REG(hw, IGC_FCRTC); 591 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 592 reg |= ((hwm << IGC_FCRTC_RTH_COAL_SHIFT) 593 & IGC_FCRTC_RTH_COAL_MASK); 594 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 595 596 dmac = pba - max_frame_size / 512; 597 if (dmac < pba - 10) 598 dmac = pba - 10; 599 reg = IGC_READ_REG(hw, IGC_DMACR); 600 reg &= ~IGC_DMACR_DMACTHR_MASK; 601 reg |= ((dmac << IGC_DMACR_DMACTHR_SHIFT) 602 & IGC_DMACR_DMACTHR_MASK); 603 604 /* transition to L0x or L1 if available..*/ 605 reg |= (IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK); 606 607 /* Check if status is 2.5Gb backplane connection 608 * before configuration of watchdog timer, which is 609 * in msec values in 12.8usec intervals 610 * watchdog timer= msec values in 32usec intervals 611 * for non 2.5Gb connection 612 */ 613 status = IGC_READ_REG(hw, IGC_STATUS); 614 if ((status & IGC_STATUS_2P5_SKU) && 615 (!(status & IGC_STATUS_2P5_SKU_OVER))) 616 reg |= ((sc->dmac * 5) >> 6); 617 else 618 reg |= (sc->dmac >> 5); 619 620 IGC_WRITE_REG(hw, IGC_DMACR, reg); 621 622 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 623 624 /* Set the interval before transition */ 625 reg = IGC_READ_REG(hw, IGC_DMCTLX); 626 reg |= IGC_DMCTLX_DCFLUSH_DIS; 627 628 /* 629 ** in 2.5Gb connection, TTLX unit is 0.4 usec 630 ** which is 0x4*2 = 0xA. But delay is still 4 usec 631 */ 632 status = IGC_READ_REG(hw, IGC_STATUS); 633 if ((status & IGC_STATUS_2P5_SKU) && 634 (!(status & IGC_STATUS_2P5_SKU_OVER))) 635 reg |= 0xA; 636 else 637 reg |= 0x4; 638 639 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 640 641 /* free space in tx packet buffer to wake from DMA coal */ 642 IGC_WRITE_REG(hw, IGC_DMCTXTH, (IGC_TXPBSIZE - 643 (2 * max_frame_size)) >> 6); 644 645 /* make low power state decision controlled by DMA coal */ 646 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 647 reg &= ~IGC_PCIEMISC_LX_DECISION; 648 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 649 } 650 651 int 652 igc_allocate_msix(struct igc_softc *sc) 653 { 654 struct igc_osdep *os = &sc->osdep; 655 struct pci_attach_args *pa = &os->os_pa; 656 struct igc_queue *iq; 657 pci_intr_handle_t ih; 658 int i, error = 0; 659 660 for (i = 0, iq = sc->queues; i < sc->sc_nqueues; i++, iq++) { 661 if (pci_intr_map_msix(pa, i, &ih)) { 662 printf("%s: unable to map msi-x vector %d\n", 663 DEVNAME(sc), i); 664 error = ENOMEM; 665 goto fail; 666 } 667 668 iq->tag = pci_intr_establish_cpu(pa->pa_pc, ih, 669 IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i), 670 igc_intr_queue, iq, iq->name); 671 if (iq->tag == NULL) { 672 printf("%s: unable to establish interrupt %d\n", 673 DEVNAME(sc), i); 674 error = ENOMEM; 675 goto fail; 676 } 677 678 iq->msix = i; 679 iq->eims = 1 << i; 680 } 681 682 /* Now the link status/control last MSI-X vector. */ 683 if (pci_intr_map_msix(pa, i, &ih)) { 684 printf("%s: unable to map link vector\n", DEVNAME(sc)); 685 error = ENOMEM; 686 goto fail; 687 } 688 689 sc->tag = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE, 690 igc_intr_link, sc, sc->sc_dev.dv_xname); 691 if (sc->tag == NULL) { 692 printf("%s: unable to establish link interrupt\n", DEVNAME(sc)); 693 error = ENOMEM; 694 goto fail; 695 } 696 697 sc->linkvec = i; 698 printf(", %s, %d queue%s", pci_intr_string(pa->pa_pc, ih), 699 i, (i > 1) ? "s" : ""); 700 701 return 0; 702 fail: 703 for (iq = sc->queues; i > 0; i--, iq++) { 704 if (iq->tag == NULL) 705 continue; 706 pci_intr_disestablish(pa->pa_pc, iq->tag); 707 iq->tag = NULL; 708 } 709 710 return error; 711 } 712 713 void 714 igc_setup_msix(struct igc_softc *sc) 715 { 716 struct igc_osdep *os = &sc->osdep; 717 struct pci_attach_args *pa = &os->os_pa; 718 int nmsix; 719 720 nmsix = pci_intr_msix_count(pa); 721 if (nmsix <= 1) 722 printf(": not enough msi-x vectors\n"); 723 724 /* Give one vector to events. */ 725 nmsix--; 726 727 sc->sc_intrmap = intrmap_create(&sc->sc_dev, nmsix, IGC_MAX_VECTORS, 728 INTRMAP_POWEROF2); 729 sc->sc_nqueues = intrmap_count(sc->sc_intrmap); 730 } 731 732 int 733 igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 734 { 735 struct igc_osdep *os = &sc->osdep; 736 737 dma->dma_tag = os->os_pa.pa_dmat; 738 739 if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, BUS_DMA_NOWAIT, 740 &dma->dma_map)) 741 return 1; 742 if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 743 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) 744 goto destroy; 745 if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 746 &dma->dma_vaddr, BUS_DMA_NOWAIT | BUS_DMA_COHERENT)) 747 goto free; 748 if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 749 NULL, BUS_DMA_NOWAIT)) 750 goto unmap; 751 752 dma->dma_size = size; 753 754 return 0; 755 unmap: 756 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 757 free: 758 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 759 destroy: 760 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 761 dma->dma_map = NULL; 762 dma->dma_tag = NULL; 763 return 1; 764 } 765 766 void 767 igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 768 { 769 if (dma->dma_tag == NULL) 770 return; 771 772 if (dma->dma_map != NULL) { 773 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 774 dma->dma_map->dm_mapsize, 775 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 776 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 777 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 778 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 779 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 780 dma->dma_map = NULL; 781 } 782 } 783 784 /********************************************************************* 785 * 786 * Setup networking device structure and register an interface. 787 * 788 **********************************************************************/ 789 void 790 igc_setup_interface(struct igc_softc *sc) 791 { 792 struct ifnet *ifp = &sc->sc_ac.ac_if; 793 int i; 794 795 ifp->if_softc = sc; 796 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); 797 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 798 ifp->if_xflags = IFXF_MPSAFE; 799 ifp->if_ioctl = igc_ioctl; 800 ifp->if_qstart = igc_start; 801 ifp->if_watchdog = igc_watchdog; 802 ifp->if_hardmtu = sc->hw.mac.max_frame_size - ETHER_HDR_LEN - 803 ETHER_CRC_LEN; 804 ifq_init_maxlen(&ifp->if_snd, sc->num_tx_desc - 1); 805 806 ifp->if_capabilities = IFCAP_VLAN_MTU; 807 808 #if NVLAN > 0 809 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 810 #endif 811 812 ifp->if_capabilities |= IFCAP_CSUM_IPv4; 813 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 814 ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6; 815 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 816 817 /* Initialize ifmedia structures. */ 818 ifmedia_init(&sc->media, IFM_IMASK, igc_media_change, igc_media_status); 819 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 820 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 821 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 822 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 823 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 824 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 825 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T, 0, NULL); 826 827 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 828 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 829 830 if_attach(ifp); 831 ether_ifattach(ifp); 832 833 if_attach_queues(ifp, sc->sc_nqueues); 834 if_attach_iqueues(ifp, sc->sc_nqueues); 835 for (i = 0; i < sc->sc_nqueues; i++) { 836 struct ifqueue *ifq = ifp->if_ifqs[i]; 837 struct ifiqueue *ifiq = ifp->if_iqs[i]; 838 struct igc_txring *txr = &sc->tx_rings[i]; 839 struct igc_rxring *rxr = &sc->rx_rings[i]; 840 841 ifq->ifq_softc = txr; 842 txr->ifq = ifq; 843 844 ifiq->ifiq_softc = rxr; 845 rxr->ifiq = ifiq; 846 } 847 } 848 849 void 850 igc_init(void *arg) 851 { 852 struct igc_softc *sc = (struct igc_softc *)arg; 853 struct ifnet *ifp = &sc->sc_ac.ac_if; 854 struct igc_rxring *rxr; 855 uint32_t ctrl = 0; 856 int i, s; 857 858 s = splnet(); 859 860 igc_stop(sc); 861 862 /* Get the latest mac address, user can use a LAA. */ 863 bcopy(sc->sc_ac.ac_enaddr, sc->hw.mac.addr, ETHER_ADDR_LEN); 864 865 /* Put the address into the receive address array. */ 866 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 867 868 /* Initialize the hardware. */ 869 igc_reset(sc); 870 igc_update_link_status(sc); 871 872 /* Setup VLAN support, basic and offload if available. */ 873 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 874 875 /* Prepare transmit descriptors and buffers. */ 876 if (igc_setup_transmit_structures(sc)) { 877 printf("%s: Could not setup transmit structures\n", 878 DEVNAME(sc)); 879 igc_stop(sc); 880 splx(s); 881 return; 882 } 883 igc_initialize_transmit_unit(sc); 884 885 sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN; 886 /* Prepare receive descriptors and buffers. */ 887 if (igc_setup_receive_structures(sc)) { 888 printf("%s: Could not setup receive structures\n", 889 DEVNAME(sc)); 890 igc_stop(sc); 891 splx(s); 892 return; 893 } 894 igc_initialize_receive_unit(sc); 895 896 if (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) { 897 ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 898 ctrl |= IGC_CTRL_VME; 899 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 900 } 901 902 /* Setup multicast table. */ 903 igc_iff(sc); 904 905 igc_clear_hw_cntrs_base_generic(&sc->hw); 906 907 igc_configure_queues(sc); 908 909 /* This clears any pending interrupts */ 910 IGC_READ_REG(&sc->hw, IGC_ICR); 911 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 912 913 /* The driver can now take control from firmware. */ 914 igc_get_hw_control(sc); 915 916 /* Set Energy Efficient Ethernet. */ 917 igc_set_eee_i225(&sc->hw, true, true, true); 918 919 for (i = 0; i < sc->sc_nqueues; i++) { 920 rxr = &sc->rx_rings[i]; 921 igc_rxfill(rxr); 922 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 923 printf("%s: Unable to fill any rx descriptors\n", 924 DEVNAME(sc)); 925 igc_stop(sc); 926 splx(s); 927 } 928 IGC_WRITE_REG(&sc->hw, IGC_RDT(i), 929 (rxr->last_desc_filled + 1) % sc->num_rx_desc); 930 } 931 932 igc_enable_intr(sc); 933 934 ifp->if_flags |= IFF_RUNNING; 935 for (i = 0; i < sc->sc_nqueues; i++) 936 ifq_clr_oactive(ifp->if_ifqs[i]); 937 938 splx(s); 939 } 940 941 static inline int 942 igc_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m) 943 { 944 int error; 945 946 error = bus_dmamap_load_mbuf(dmat, map, m, 947 BUS_DMA_STREAMING | BUS_DMA_NOWAIT); 948 if (error != EFBIG) 949 return (error); 950 951 error = m_defrag(m, M_DONTWAIT); 952 if (error != 0) 953 return (error); 954 955 return (bus_dmamap_load_mbuf(dmat, map, m, 956 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)); 957 } 958 959 void 960 igc_start(struct ifqueue *ifq) 961 { 962 struct ifnet *ifp = ifq->ifq_if; 963 struct igc_softc *sc = ifp->if_softc; 964 struct igc_txring *txr = ifq->ifq_softc; 965 union igc_adv_tx_desc *txdesc; 966 struct igc_tx_buf *txbuf; 967 bus_dmamap_t map; 968 struct mbuf *m; 969 unsigned int prod, free, last, i; 970 unsigned int mask; 971 uint32_t cmd_type_len; 972 uint32_t olinfo_status; 973 int post = 0; 974 #if NBPFILTER > 0 975 caddr_t if_bpf; 976 #endif 977 978 if (!sc->link_active) { 979 ifq_purge(ifq); 980 return; 981 } 982 983 prod = txr->next_avail_desc; 984 free = txr->next_to_clean; 985 if (free <= prod) 986 free += sc->num_tx_desc; 987 free -= prod; 988 989 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 990 txr->txdma.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 991 992 mask = sc->num_tx_desc - 1; 993 994 for (;;) { 995 if (free <= IGC_MAX_SCATTER + 1) { 996 ifq_set_oactive(ifq); 997 break; 998 } 999 1000 m = ifq_dequeue(ifq); 1001 if (m == NULL) 1002 break; 1003 1004 txbuf = &txr->tx_buffers[prod]; 1005 map = txbuf->map; 1006 1007 if (igc_load_mbuf(txr->txdma.dma_tag, map, m) != 0) { 1008 ifq->ifq_errors++; 1009 m_freem(m); 1010 continue; 1011 } 1012 1013 olinfo_status = m->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 1014 1015 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1016 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1017 1018 cmd_type_len = IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1019 IGC_ADVTXD_DCMD_DEXT; 1020 1021 if (igc_tx_ctx_setup(txr, m, prod, &cmd_type_len, 1022 &olinfo_status)) { 1023 /* Consume the first descriptor */ 1024 prod++; 1025 prod &= mask; 1026 free--; 1027 } 1028 1029 for (i = 0; i < map->dm_nsegs; i++) { 1030 txdesc = &txr->tx_base[prod]; 1031 1032 CLR(cmd_type_len, IGC_ADVTXD_DTALEN_MASK); 1033 cmd_type_len |= map->dm_segs[i].ds_len; 1034 if (i == map->dm_nsegs - 1) 1035 cmd_type_len |= IGC_ADVTXD_DCMD_EOP | 1036 IGC_ADVTXD_DCMD_RS; 1037 1038 htolem64(&txdesc->read.buffer_addr, 1039 map->dm_segs[i].ds_addr); 1040 htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1041 htolem32(&txdesc->read.olinfo_status, olinfo_status); 1042 1043 last = prod; 1044 1045 prod++; 1046 prod &= mask; 1047 } 1048 1049 txbuf->m_head = m; 1050 txbuf->eop_index = last; 1051 1052 #if NBPFILTER > 0 1053 if_bpf = ifp->if_bpf; 1054 if (if_bpf) 1055 bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_OUT); 1056 #endif 1057 1058 free -= i; 1059 post = 1; 1060 } 1061 1062 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 1063 txr->txdma.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1064 1065 if (post) { 1066 txr->next_avail_desc = prod; 1067 IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1068 } 1069 } 1070 1071 int 1072 igc_txeof(struct igc_txring *txr) 1073 { 1074 struct igc_softc *sc = txr->sc; 1075 struct ifqueue *ifq = txr->ifq; 1076 union igc_adv_tx_desc *txdesc; 1077 struct igc_tx_buf *txbuf; 1078 bus_dmamap_t map; 1079 unsigned int cons, prod, last; 1080 unsigned int mask; 1081 int done = 0; 1082 1083 prod = txr->next_avail_desc; 1084 cons = txr->next_to_clean; 1085 1086 if (cons == prod) 1087 return (0); 1088 1089 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 1090 txr->txdma.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1091 1092 mask = sc->num_tx_desc - 1; 1093 1094 do { 1095 txbuf = &txr->tx_buffers[cons]; 1096 last = txbuf->eop_index; 1097 txdesc = &txr->tx_base[last]; 1098 1099 if (!(txdesc->wb.status & htole32(IGC_TXD_STAT_DD))) 1100 break; 1101 1102 map = txbuf->map; 1103 1104 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1105 BUS_DMASYNC_POSTWRITE); 1106 bus_dmamap_unload(txr->txdma.dma_tag, map); 1107 m_freem(txbuf->m_head); 1108 1109 txbuf->m_head = NULL; 1110 txbuf->eop_index = -1; 1111 1112 cons = last + 1; 1113 cons &= mask; 1114 1115 done = 1; 1116 } while (cons != prod); 1117 1118 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 1119 txr->txdma.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 1120 1121 txr->next_to_clean = cons; 1122 1123 if (ifq_is_oactive(ifq)) 1124 ifq_restart(ifq); 1125 1126 return (done); 1127 } 1128 1129 /********************************************************************* 1130 * 1131 * This routine disables all traffic on the adapter by issuing a 1132 * global reset on the MAC. 1133 * 1134 **********************************************************************/ 1135 void 1136 igc_stop(struct igc_softc *sc) 1137 { 1138 struct ifnet *ifp = &sc->sc_ac.ac_if; 1139 int i; 1140 1141 /* Tell the stack that the interface is no longer active. */ 1142 ifp->if_flags &= ~IFF_RUNNING; 1143 1144 igc_disable_intr(sc); 1145 1146 igc_reset_hw(&sc->hw); 1147 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1148 1149 intr_barrier(sc->tag); 1150 for (i = 0; i < sc->sc_nqueues; i++) { 1151 struct ifqueue *ifq = ifp->if_ifqs[i]; 1152 ifq_barrier(ifq); 1153 ifq_clr_oactive(ifq); 1154 1155 if (sc->queues[i].tag != NULL) 1156 intr_barrier(sc->queues[i].tag); 1157 timeout_del(&sc->rx_rings[i].rx_refill); 1158 } 1159 1160 igc_free_transmit_structures(sc); 1161 igc_free_receive_structures(sc); 1162 1163 igc_update_link_status(sc); 1164 } 1165 1166 /********************************************************************* 1167 * Ioctl entry point 1168 * 1169 * igc_ioctl is called when the user wants to configure the 1170 * interface. 1171 * 1172 * return 0 on success, positive on failure 1173 **********************************************************************/ 1174 int 1175 igc_ioctl(struct ifnet * ifp, u_long cmd, caddr_t data) 1176 { 1177 struct igc_softc *sc = ifp->if_softc; 1178 struct ifreq *ifr = (struct ifreq *)data; 1179 int s, error = 0; 1180 1181 s = splnet(); 1182 1183 switch (cmd) { 1184 case SIOCSIFADDR: 1185 ifp->if_flags |= IFF_UP; 1186 if (!(ifp->if_flags & IFF_RUNNING)) 1187 igc_init(sc); 1188 break; 1189 case SIOCSIFFLAGS: 1190 if (ifp->if_flags & IFF_UP) { 1191 if (ifp->if_flags & IFF_RUNNING) 1192 error = ENETRESET; 1193 else 1194 igc_init(sc); 1195 } else { 1196 if (ifp->if_flags & IFF_RUNNING) 1197 igc_stop(sc); 1198 } 1199 break; 1200 case SIOCSIFMEDIA: 1201 case SIOCGIFMEDIA: 1202 error = ifmedia_ioctl(ifp, ifr, &sc->media, cmd); 1203 break; 1204 case SIOCGIFRXR: 1205 error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1206 break; 1207 default: 1208 error = ether_ioctl(ifp, &sc->sc_ac, cmd, data); 1209 } 1210 1211 if (error == ENETRESET) { 1212 if (ifp->if_flags & IFF_RUNNING) { 1213 igc_disable_intr(sc); 1214 igc_iff(sc); 1215 igc_enable_intr(sc); 1216 } 1217 error = 0; 1218 } 1219 1220 splx(s); 1221 return error; 1222 } 1223 1224 int 1225 igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 1226 { 1227 struct if_rxring_info *ifr; 1228 struct igc_rxring *rxr; 1229 int error, i, n = 0; 1230 1231 ifr = mallocarray(sc->sc_nqueues, sizeof(*ifr), M_DEVBUF, 1232 M_WAITOK | M_ZERO); 1233 1234 for (i = 0; i < sc->sc_nqueues; i++) { 1235 rxr = &sc->rx_rings[i]; 1236 ifr[n].ifr_size = MCLBYTES; 1237 snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i); 1238 ifr[n].ifr_info = rxr->rx_ring; 1239 n++; 1240 } 1241 1242 error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 1243 free(ifr, M_DEVBUF, sc->sc_nqueues * sizeof(*ifr)); 1244 1245 return error; 1246 } 1247 1248 int 1249 igc_rxfill(struct igc_rxring *rxr) 1250 { 1251 struct igc_softc *sc = rxr->sc; 1252 int i, post = 0; 1253 u_int slots; 1254 1255 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 1256 rxr->rxdma.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1257 1258 i = rxr->last_desc_filled; 1259 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); slots > 0; 1260 slots--) { 1261 if (++i == sc->num_rx_desc) 1262 i = 0; 1263 1264 if (igc_get_buf(rxr, i) != 0) 1265 break; 1266 1267 rxr->last_desc_filled = i; 1268 post = 1; 1269 } 1270 1271 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 1272 rxr->rxdma.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1273 1274 if_rxr_put(&rxr->rx_ring, slots); 1275 1276 return post; 1277 } 1278 1279 void 1280 igc_rxrefill(void *xrxr) 1281 { 1282 struct igc_rxring *rxr = xrxr; 1283 struct igc_softc *sc = rxr->sc; 1284 1285 if (igc_rxfill(rxr)) { 1286 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), 1287 (rxr->last_desc_filled + 1) % sc->num_rx_desc); 1288 } 1289 else if (if_rxr_inuse(&rxr->rx_ring) == 0) 1290 timeout_add(&rxr->rx_refill, 1); 1291 } 1292 1293 /********************************************************************* 1294 * 1295 * This routine executes in interrupt context. It replenishes 1296 * the mbufs in the descriptor and sends data which has been 1297 * dma'ed into host memory to upper layer. 1298 * 1299 *********************************************************************/ 1300 int 1301 igc_rxeof(struct igc_rxring *rxr) 1302 { 1303 struct igc_softc *sc = rxr->sc; 1304 struct ifnet *ifp = &sc->sc_ac.ac_if; 1305 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 1306 struct mbuf *mp, *m; 1307 struct igc_rx_buf *rxbuf, *nxbuf; 1308 union igc_adv_rx_desc *rxdesc; 1309 uint32_t ptype, staterr = 0; 1310 uint16_t len, vtag; 1311 uint8_t eop = 0; 1312 int i, nextp; 1313 1314 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 1315 return 0; 1316 1317 i = rxr->next_to_check; 1318 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 1319 uint32_t hash; 1320 uint16_t hashtype; 1321 1322 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 1323 i * sizeof(union igc_adv_rx_desc), 1324 sizeof(union igc_adv_rx_desc), BUS_DMASYNC_POSTREAD); 1325 1326 rxdesc = &rxr->rx_base[i]; 1327 staterr = letoh32(rxdesc->wb.upper.status_error); 1328 if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 1329 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 1330 i * sizeof(union igc_adv_rx_desc), 1331 sizeof(union igc_adv_rx_desc), BUS_DMASYNC_PREREAD); 1332 break; 1333 } 1334 1335 /* Zero out the receive descriptors status. */ 1336 rxdesc->wb.upper.status_error = 0; 1337 rxbuf = &rxr->rx_buffers[i]; 1338 1339 /* Pull the mbuf off the ring. */ 1340 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 1341 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1342 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 1343 1344 mp = rxbuf->buf; 1345 len = letoh16(rxdesc->wb.upper.length); 1346 vtag = letoh16(rxdesc->wb.upper.vlan); 1347 eop = ((staterr & IGC_RXD_STAT_EOP) == IGC_RXD_STAT_EOP); 1348 ptype = letoh32(rxdesc->wb.lower.lo_dword.data) & 1349 IGC_PKTTYPE_MASK; 1350 hash = letoh32(rxdesc->wb.lower.hi_dword.rss); 1351 hashtype = le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 1352 IGC_RXDADV_RSSTYPE_MASK; 1353 1354 if (staterr & IGC_RXDEXT_STATERR_RXE) { 1355 if (rxbuf->fmp) { 1356 m_freem(rxbuf->fmp); 1357 rxbuf->fmp = NULL; 1358 } 1359 1360 m_freem(mp); 1361 rxbuf->buf = NULL; 1362 goto next_desc; 1363 } 1364 1365 if (mp == NULL) { 1366 panic("%s: igc_rxeof: NULL mbuf in slot %d " 1367 "(nrx %d, filled %d)", DEVNAME(sc), i, 1368 if_rxr_inuse(&rxr->rx_ring), rxr->last_desc_filled); 1369 } 1370 1371 if (!eop) { 1372 /* 1373 * Figure out the next descriptor of this frame. 1374 */ 1375 nextp = i + 1; 1376 if (nextp == sc->num_rx_desc) 1377 nextp = 0; 1378 nxbuf = &rxr->rx_buffers[nextp]; 1379 /* prefetch(nxbuf); */ 1380 } 1381 1382 mp->m_len = len; 1383 1384 m = rxbuf->fmp; 1385 rxbuf->buf = rxbuf->fmp = NULL; 1386 1387 if (m != NULL) 1388 m->m_pkthdr.len += mp->m_len; 1389 else { 1390 m = mp; 1391 m->m_pkthdr.len = mp->m_len; 1392 #if NVLAN > 0 1393 if (staterr & IGC_RXD_STAT_VP) { 1394 m->m_pkthdr.ether_vtag = vtag; 1395 m->m_flags |= M_VLANTAG; 1396 } 1397 #endif 1398 } 1399 1400 /* Pass the head pointer on */ 1401 if (eop == 0) { 1402 nxbuf->fmp = m; 1403 m = NULL; 1404 mp->m_next = nxbuf->buf; 1405 } else { 1406 igc_rx_checksum(staterr, m, ptype); 1407 1408 if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 1409 m->m_pkthdr.ph_flowid = hash; 1410 SET(m->m_pkthdr.csum_flags, M_FLOWID); 1411 } 1412 1413 ml_enqueue(&ml, m); 1414 } 1415 next_desc: 1416 if_rxr_put(&rxr->rx_ring, 1); 1417 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 1418 i * sizeof(union igc_adv_rx_desc), 1419 sizeof(union igc_adv_rx_desc), BUS_DMASYNC_PREREAD); 1420 1421 /* Advance our pointers to the next descriptor. */ 1422 if (++i == sc->num_rx_desc) 1423 i = 0; 1424 } 1425 rxr->next_to_check = i; 1426 1427 if (ifiq_input(rxr->ifiq, &ml)) 1428 if_rxr_livelocked(&rxr->rx_ring); 1429 1430 if (!(staterr & IGC_RXD_STAT_DD)) 1431 return 0; 1432 1433 return 1; 1434 } 1435 1436 /********************************************************************* 1437 * 1438 * Verify that the hardware indicated that the checksum is valid. 1439 * Inform the stack about the status of checksum so that stack 1440 * doesn't spend time verifying the checksum. 1441 * 1442 *********************************************************************/ 1443 void 1444 igc_rx_checksum(uint32_t staterr, struct mbuf *m, uint32_t ptype) 1445 { 1446 uint16_t status = (uint16_t)staterr; 1447 uint8_t errors = (uint8_t)(staterr >> 24); 1448 1449 if (status & IGC_RXD_STAT_IPCS) { 1450 if (!(errors & IGC_RXD_ERR_IPE)) { 1451 /* IP Checksum Good */ 1452 m->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 1453 } else 1454 m->m_pkthdr.csum_flags = 0; 1455 } 1456 1457 if (status & (IGC_RXD_STAT_TCPCS | IGC_RXD_STAT_UDPCS)) { 1458 if (!(errors & IGC_RXD_ERR_TCPE)) 1459 m->m_pkthdr.csum_flags |= 1460 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 1461 } 1462 } 1463 1464 void 1465 igc_watchdog(struct ifnet * ifp) 1466 { 1467 } 1468 1469 /********************************************************************* 1470 * 1471 * Media Ioctl callback 1472 * 1473 * This routine is called whenever the user queries the status of 1474 * the interface using ifconfig. 1475 * 1476 **********************************************************************/ 1477 void 1478 igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1479 { 1480 struct igc_softc *sc = ifp->if_softc; 1481 1482 igc_update_link_status(sc); 1483 1484 ifmr->ifm_status = IFM_AVALID; 1485 ifmr->ifm_active = IFM_ETHER; 1486 1487 if (!sc->link_active) { 1488 ifmr->ifm_active |= IFM_NONE; 1489 return; 1490 } 1491 1492 ifmr->ifm_status |= IFM_ACTIVE; 1493 1494 switch (sc->link_speed) { 1495 case 10: 1496 ifmr->ifm_active |= IFM_10_T; 1497 break; 1498 case 100: 1499 ifmr->ifm_active |= IFM_100_TX; 1500 break; 1501 case 1000: 1502 ifmr->ifm_active |= IFM_1000_T; 1503 break; 1504 case 2500: 1505 ifmr->ifm_active |= IFM_2500_T; 1506 break; 1507 } 1508 1509 if (sc->link_duplex == FULL_DUPLEX) 1510 ifmr->ifm_active |= IFM_FDX; 1511 else 1512 ifmr->ifm_active |= IFM_HDX; 1513 1514 switch (sc->hw.fc.current_mode) { 1515 case igc_fc_tx_pause: 1516 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 1517 break; 1518 case igc_fc_rx_pause: 1519 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 1520 break; 1521 case igc_fc_full: 1522 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE | 1523 IFM_ETH_TXPAUSE; 1524 break; 1525 default: 1526 ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE | 1527 IFM_ETH_TXPAUSE); 1528 break; 1529 } 1530 } 1531 1532 /********************************************************************* 1533 * 1534 * Media Ioctl callback 1535 * 1536 * This routine is called when the user changes speed/duplex using 1537 * media/mediopt option with ifconfig. 1538 * 1539 **********************************************************************/ 1540 int 1541 igc_media_change(struct ifnet *ifp) 1542 { 1543 struct igc_softc *sc = ifp->if_softc; 1544 struct ifmedia *ifm = &sc->media; 1545 1546 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1547 return (EINVAL); 1548 1549 sc->hw.mac.autoneg = DO_AUTO_NEG; 1550 1551 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1552 case IFM_AUTO: 1553 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1554 break; 1555 case IFM_2500_T: 1556 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 1557 break; 1558 case IFM_1000_T: 1559 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1560 break; 1561 case IFM_100_TX: 1562 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1563 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 1564 else 1565 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 1566 break; 1567 case IFM_10_T: 1568 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1569 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 1570 else 1571 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 1572 break; 1573 default: 1574 return EINVAL; 1575 } 1576 1577 igc_init(sc); 1578 1579 return 0; 1580 } 1581 1582 void 1583 igc_iff(struct igc_softc *sc) 1584 { 1585 struct ifnet *ifp = &sc->sc_ac.ac_if; 1586 struct arpcom *ac = &sc->sc_ac; 1587 struct ether_multi *enm; 1588 struct ether_multistep step; 1589 uint32_t reg_rctl = 0; 1590 uint8_t *mta; 1591 int mcnt = 0; 1592 1593 mta = sc->mta; 1594 bzero(mta, sizeof(uint8_t) * ETHER_ADDR_LEN * 1595 MAX_NUM_MULTICAST_ADDRESSES); 1596 1597 reg_rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 1598 reg_rctl &= ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 1599 ifp->if_flags &= ~IFF_ALLMULTI; 1600 1601 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1602 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1603 ifp->if_flags |= IFF_ALLMULTI; 1604 reg_rctl |= IGC_RCTL_MPE; 1605 if (ifp->if_flags & IFF_PROMISC) 1606 reg_rctl |= IGC_RCTL_UPE; 1607 } else { 1608 ETHER_FIRST_MULTI(step, ac, enm); 1609 while (enm != NULL) { 1610 bcopy(enm->enm_addrlo, 1611 &mta[mcnt * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 1612 mcnt++; 1613 1614 ETHER_NEXT_MULTI(step, enm); 1615 } 1616 1617 igc_update_mc_addr_list(&sc->hw, mta, mcnt); 1618 } 1619 1620 IGC_WRITE_REG(&sc->hw, IGC_RCTL, reg_rctl); 1621 } 1622 1623 void 1624 igc_update_link_status(struct igc_softc *sc) 1625 { 1626 struct ifnet *ifp = &sc->sc_ac.ac_if; 1627 struct igc_hw *hw = &sc->hw; 1628 int link_state; 1629 1630 if (hw->mac.get_link_status == true) 1631 igc_check_for_link(hw); 1632 1633 if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 1634 if (sc->link_active == 0) { 1635 igc_get_speed_and_duplex(hw, &sc->link_speed, 1636 &sc->link_duplex); 1637 sc->link_active = 1; 1638 ifp->if_baudrate = IF_Mbps(sc->link_speed); 1639 } 1640 link_state = (sc->link_duplex == FULL_DUPLEX) ? 1641 LINK_STATE_FULL_DUPLEX : LINK_STATE_HALF_DUPLEX; 1642 } else { 1643 if (sc->link_active == 1) { 1644 ifp->if_baudrate = sc->link_speed = 0; 1645 sc->link_duplex = 0; 1646 sc->link_active = 0; 1647 } 1648 link_state = LINK_STATE_DOWN; 1649 } 1650 if (ifp->if_link_state != link_state) { 1651 ifp->if_link_state = link_state; 1652 if_link_state_change(ifp); 1653 } 1654 } 1655 1656 /********************************************************************* 1657 * 1658 * Get a buffer from system mbuf buffer pool. 1659 * 1660 **********************************************************************/ 1661 int 1662 igc_get_buf(struct igc_rxring *rxr, int i) 1663 { 1664 struct igc_softc *sc = rxr->sc; 1665 struct igc_rx_buf *rxbuf; 1666 struct mbuf *m; 1667 union igc_adv_rx_desc *rxdesc; 1668 int error; 1669 1670 rxbuf = &rxr->rx_buffers[i]; 1671 rxdesc = &rxr->rx_base[i]; 1672 if (rxbuf->buf) { 1673 printf("%s: slot %d already has an mbuf\n", DEVNAME(sc), i); 1674 return ENOBUFS; 1675 } 1676 1677 m = MCLGETL(NULL, M_DONTWAIT, sc->rx_mbuf_sz); 1678 if (!m) 1679 return ENOBUFS; 1680 1681 m->m_data += (m->m_ext.ext_size - sc->rx_mbuf_sz); 1682 m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 1683 1684 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, m, 1685 BUS_DMA_NOWAIT); 1686 if (error) { 1687 m_freem(m); 1688 return error; 1689 } 1690 1691 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 1692 rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 1693 rxbuf->buf = m; 1694 1695 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 1696 1697 return 0; 1698 } 1699 1700 void 1701 igc_configure_queues(struct igc_softc *sc) 1702 { 1703 struct igc_hw *hw = &sc->hw; 1704 struct igc_queue *iq = sc->queues; 1705 uint32_t ivar, newitr = 0; 1706 int i; 1707 1708 /* First turn on RSS capability */ 1709 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 1710 IGC_GPIE_PBA | IGC_GPIE_NSICR); 1711 1712 /* Set the starting interrupt rate */ 1713 newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 1714 1715 newitr |= IGC_EITR_CNT_IGNR; 1716 1717 /* Turn on MSI-X */ 1718 for (i = 0; i < sc->sc_nqueues; i++, iq++) { 1719 /* RX entries */ 1720 igc_set_queues(sc, i, iq->msix, 0); 1721 /* TX entries */ 1722 igc_set_queues(sc, i, iq->msix, 1); 1723 sc->msix_queuesmask |= iq->eims; 1724 IGC_WRITE_REG(hw, IGC_EITR(iq->msix), newitr); 1725 } 1726 1727 /* And for the link interrupt */ 1728 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 1729 sc->msix_linkmask = 1 << sc->linkvec; 1730 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 1731 } 1732 1733 void 1734 igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 1735 { 1736 struct igc_hw *hw = &sc->hw; 1737 uint32_t ivar, index; 1738 1739 index = entry >> 1; 1740 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 1741 if (type) { 1742 if (entry & 1) { 1743 ivar &= 0x00FFFFFF; 1744 ivar |= (vector | IGC_IVAR_VALID) << 24; 1745 } else { 1746 ivar &= 0xFFFF00FF; 1747 ivar |= (vector | IGC_IVAR_VALID) << 8; 1748 } 1749 } else { 1750 if (entry & 1) { 1751 ivar &= 0xFF00FFFF; 1752 ivar |= (vector | IGC_IVAR_VALID) << 16; 1753 } else { 1754 ivar &= 0xFFFFFF00; 1755 ivar |= vector | IGC_IVAR_VALID; 1756 } 1757 } 1758 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 1759 } 1760 1761 void 1762 igc_enable_queue(struct igc_softc *sc, uint32_t eims) 1763 { 1764 IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 1765 } 1766 1767 void 1768 igc_enable_intr(struct igc_softc *sc) 1769 { 1770 struct igc_hw *hw = &sc->hw; 1771 uint32_t mask; 1772 1773 mask = (sc->msix_queuesmask | sc->msix_linkmask); 1774 IGC_WRITE_REG(hw, IGC_EIAC, mask); 1775 IGC_WRITE_REG(hw, IGC_EIAM, mask); 1776 IGC_WRITE_REG(hw, IGC_EIMS, mask); 1777 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 1778 IGC_WRITE_FLUSH(hw); 1779 } 1780 1781 void 1782 igc_disable_intr(struct igc_softc *sc) 1783 { 1784 struct igc_hw *hw = &sc->hw; 1785 1786 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 1787 IGC_WRITE_REG(hw, IGC_EIAC, 0); 1788 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 1789 IGC_WRITE_FLUSH(hw); 1790 } 1791 1792 int 1793 igc_intr_link(void *arg) 1794 { 1795 struct igc_softc *sc = (struct igc_softc *)arg; 1796 uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 1797 1798 if (reg_icr & IGC_ICR_LSC) { 1799 KERNEL_LOCK(); 1800 sc->hw.mac.get_link_status = true; 1801 igc_update_link_status(sc); 1802 KERNEL_UNLOCK(); 1803 } 1804 1805 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 1806 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 1807 1808 return 1; 1809 } 1810 1811 int 1812 igc_intr_queue(void *arg) 1813 { 1814 struct igc_queue *iq = arg; 1815 struct igc_softc *sc = iq->sc; 1816 struct ifnet *ifp = &sc->sc_ac.ac_if; 1817 struct igc_rxring *rxr = iq->rxr; 1818 struct igc_txring *txr = iq->txr; 1819 1820 if (ifp->if_flags & IFF_RUNNING) { 1821 igc_txeof(txr); 1822 igc_rxeof(rxr); 1823 igc_rxrefill(rxr); 1824 } 1825 1826 igc_enable_queue(sc, iq->eims); 1827 1828 return 1; 1829 } 1830 1831 /********************************************************************* 1832 * 1833 * Allocate memory for tx_buffer structures. The tx_buffer stores all 1834 * the information needed to transmit a packet on the wire. 1835 * 1836 **********************************************************************/ 1837 int 1838 igc_allocate_transmit_buffers(struct igc_txring *txr) 1839 { 1840 struct igc_softc *sc = txr->sc; 1841 struct igc_tx_buf *txbuf; 1842 int error, i; 1843 1844 txr->tx_buffers = mallocarray(sc->num_tx_desc, 1845 sizeof(struct igc_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO); 1846 if (txr->tx_buffers == NULL) { 1847 printf("%s: Unable to allocate tx_buffer memory\n", 1848 DEVNAME(sc)); 1849 error = ENOMEM; 1850 goto fail; 1851 } 1852 txr->txtag = txr->txdma.dma_tag; 1853 1854 /* Create the descriptor buffer dma maps. */ 1855 for (i = 0; i < sc->num_tx_desc; i++) { 1856 txbuf = &txr->tx_buffers[i]; 1857 error = bus_dmamap_create(txr->txdma.dma_tag, IGC_TSO_SIZE, 1858 IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 1859 if (error != 0) { 1860 printf("%s: Unable to create TX DMA map\n", 1861 DEVNAME(sc)); 1862 goto fail; 1863 } 1864 } 1865 1866 return 0; 1867 fail: 1868 return error; 1869 } 1870 1871 1872 /********************************************************************* 1873 * 1874 * Allocate and initialize transmit structures. 1875 * 1876 **********************************************************************/ 1877 int 1878 igc_setup_transmit_structures(struct igc_softc *sc) 1879 { 1880 struct igc_txring *txr = sc->tx_rings; 1881 int i; 1882 1883 for (i = 0; i < sc->sc_nqueues; i++, txr++) { 1884 if (igc_setup_transmit_ring(txr)) 1885 goto fail; 1886 } 1887 1888 return 0; 1889 fail: 1890 igc_free_transmit_structures(sc); 1891 return ENOBUFS; 1892 } 1893 1894 /********************************************************************* 1895 * 1896 * Initialize a transmit ring. 1897 * 1898 **********************************************************************/ 1899 int 1900 igc_setup_transmit_ring(struct igc_txring *txr) 1901 { 1902 struct igc_softc *sc = txr->sc; 1903 1904 /* Now allocate transmit buffers for the ring. */ 1905 if (igc_allocate_transmit_buffers(txr)) 1906 return ENOMEM; 1907 1908 /* Clear the old ring contents */ 1909 bzero((void *)txr->tx_base, 1910 (sizeof(union igc_adv_tx_desc)) * sc->num_tx_desc); 1911 1912 /* Reset indices. */ 1913 txr->next_avail_desc = 0; 1914 txr->next_to_clean = 0; 1915 1916 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 1917 txr->txdma.dma_map->dm_mapsize, 1918 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1919 1920 return 0; 1921 } 1922 1923 /********************************************************************* 1924 * 1925 * Enable transmit unit. 1926 * 1927 **********************************************************************/ 1928 void 1929 igc_initialize_transmit_unit(struct igc_softc *sc) 1930 { 1931 struct ifnet *ifp = &sc->sc_ac.ac_if; 1932 struct igc_txring *txr; 1933 struct igc_hw *hw = &sc->hw; 1934 uint64_t bus_addr; 1935 uint32_t tctl, txdctl = 0; 1936 int i; 1937 1938 /* Setup the Base and Length of the TX descriptor ring. */ 1939 for (i = 0; i < sc->sc_nqueues; i++) { 1940 txr = &sc->tx_rings[i]; 1941 1942 bus_addr = txr->txdma.dma_map->dm_segs[0].ds_addr; 1943 1944 /* Base and len of TX ring */ 1945 IGC_WRITE_REG(hw, IGC_TDLEN(i), 1946 sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 1947 IGC_WRITE_REG(hw, IGC_TDBAH(i), (uint32_t)(bus_addr >> 32)); 1948 IGC_WRITE_REG(hw, IGC_TDBAL(i), (uint32_t)bus_addr); 1949 1950 /* Init the HEAD/TAIL indices */ 1951 IGC_WRITE_REG(hw, IGC_TDT(i), 0); 1952 IGC_WRITE_REG(hw, IGC_TDH(i), 0); 1953 1954 txr->watchdog_timer = 0; 1955 1956 txdctl = 0; /* Clear txdctl */ 1957 txdctl |= 0x1f; /* PTHRESH */ 1958 txdctl |= 1 << 8; /* HTHRESH */ 1959 txdctl |= 1 << 16; /* WTHRESH */ 1960 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 1961 txdctl |= IGC_TXDCTL_GRAN; 1962 txdctl |= 1 << 25; /* LWTHRESH */ 1963 1964 IGC_WRITE_REG(hw, IGC_TXDCTL(i), txdctl); 1965 } 1966 ifp->if_timer = 0; 1967 1968 /* Program the Transmit Control Register */ 1969 tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 1970 tctl &= ~IGC_TCTL_CT; 1971 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 1972 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 1973 1974 /* This write will effectively turn on the transmit unit. */ 1975 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 1976 } 1977 1978 /********************************************************************* 1979 * 1980 * Free all transmit rings. 1981 * 1982 **********************************************************************/ 1983 void 1984 igc_free_transmit_structures(struct igc_softc *sc) 1985 { 1986 struct igc_txring *txr = sc->tx_rings; 1987 int i; 1988 1989 for (i = 0; i < sc->sc_nqueues; i++, txr++) 1990 igc_free_transmit_buffers(txr); 1991 } 1992 1993 /********************************************************************* 1994 * 1995 * Free transmit ring related data structures. 1996 * 1997 **********************************************************************/ 1998 void 1999 igc_free_transmit_buffers(struct igc_txring *txr) 2000 { 2001 struct igc_softc *sc = txr->sc; 2002 struct igc_tx_buf *txbuf; 2003 int i; 2004 2005 if (txr->tx_buffers == NULL) 2006 return; 2007 2008 txbuf = txr->tx_buffers; 2009 for (i = 0; i < sc->num_tx_desc; i++, txbuf++) { 2010 if (txbuf->map != NULL && txbuf->map->dm_nsegs > 0) { 2011 bus_dmamap_sync(txr->txdma.dma_tag, txbuf->map, 2012 0, txbuf->map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2013 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 2014 } 2015 if (txbuf->m_head != NULL) { 2016 m_freem(txbuf->m_head); 2017 txbuf->m_head = NULL; 2018 } 2019 if (txbuf->map != NULL) { 2020 bus_dmamap_destroy(txr->txdma.dma_tag, txbuf->map); 2021 txbuf->map = NULL; 2022 } 2023 } 2024 2025 if (txr->tx_buffers != NULL) 2026 free(txr->tx_buffers, M_DEVBUF, 2027 sc->num_tx_desc * sizeof(struct igc_tx_buf)); 2028 txr->tx_buffers = NULL; 2029 txr->txtag = NULL; 2030 } 2031 2032 2033 /********************************************************************* 2034 * 2035 * Advanced Context Descriptor setup for VLAN, CSUM or TSO 2036 * 2037 **********************************************************************/ 2038 2039 int 2040 igc_tx_ctx_setup(struct igc_txring *txr, struct mbuf *mp, int prod, 2041 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2042 { 2043 struct ether_extracted ext; 2044 struct igc_adv_tx_context_desc *txdesc; 2045 uint32_t mss_l4len_idx = 0; 2046 uint32_t type_tucmd_mlhl = 0; 2047 uint32_t vlan_macip_lens = 0; 2048 int off = 0; 2049 2050 /* 2051 * In advanced descriptors the vlan tag must 2052 * be placed into the context descriptor. Hence 2053 * we need to make one even if not doing offloads. 2054 */ 2055 #if NVLAN > 0 2056 if (ISSET(mp->m_flags, M_VLANTAG)) { 2057 uint32_t vtag = mp->m_pkthdr.ether_vtag; 2058 vlan_macip_lens |= (vtag << IGC_ADVTXD_VLAN_SHIFT); 2059 *cmd_type_len |= IGC_ADVTXD_DCMD_VLE; 2060 off = 1; 2061 } 2062 #endif 2063 2064 ether_extract_headers(mp, &ext); 2065 2066 vlan_macip_lens |= (sizeof(*ext.eh) << IGC_ADVTXD_MACLEN_SHIFT); 2067 2068 if (ext.ip4) { 2069 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 2070 if (ISSET(mp->m_pkthdr.csum_flags, M_IPV4_CSUM_OUT)) { 2071 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 2072 off = 1; 2073 } 2074 #ifdef INET6 2075 } else if (ext.ip6) { 2076 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 2077 #endif 2078 } 2079 2080 vlan_macip_lens |= ext.iphlen; 2081 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 2082 2083 if (ext.tcp) { 2084 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 2085 if (ISSET(mp->m_pkthdr.csum_flags, M_TCP_CSUM_OUT)) { 2086 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 2087 off = 1; 2088 } 2089 } else if (ext.udp) { 2090 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 2091 if (ISSET(mp->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) { 2092 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 2093 off = 1; 2094 } 2095 } 2096 2097 if (ISSET(mp->m_pkthdr.csum_flags, M_TCP_TSO)) { 2098 if (ext.tcp && mp->m_pkthdr.ph_mss > 0) { 2099 uint32_t hdrlen, thlen, paylen, outlen; 2100 2101 thlen = ext.tcphlen; 2102 2103 outlen = mp->m_pkthdr.ph_mss; 2104 mss_l4len_idx |= outlen << IGC_ADVTXD_MSS_SHIFT; 2105 mss_l4len_idx |= thlen << IGC_ADVTXD_L4LEN_SHIFT; 2106 2107 hdrlen = sizeof(*ext.eh) + ext.iphlen + thlen; 2108 paylen = mp->m_pkthdr.len - hdrlen; 2109 CLR(*olinfo_status, IGC_ADVTXD_PAYLEN_MASK); 2110 *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 2111 2112 *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 2113 off = 1; 2114 2115 tcpstat_add(tcps_outpkttso, 2116 (paylen + outlen - 1) / outlen); 2117 } else 2118 tcpstat_inc(tcps_outbadtso); 2119 } 2120 2121 if (off == 0) 2122 return 0; 2123 2124 /* Now ready a context descriptor */ 2125 txdesc = (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 2126 2127 /* Now copy bits into descriptor */ 2128 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 2129 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 2130 htolem32(&txdesc->seqnum_seed, 0); 2131 htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 2132 2133 return 1; 2134 } 2135 2136 /********************************************************************* 2137 * 2138 * Allocate memory for rx_buffer structures. Since we use one 2139 * rx_buffer per received packet, the maximum number of rx_buffer's 2140 * that we'll need is equal to the number of receive descriptors 2141 * that we've allocated. 2142 * 2143 **********************************************************************/ 2144 int 2145 igc_allocate_receive_buffers(struct igc_rxring *rxr) 2146 { 2147 struct igc_softc *sc = rxr->sc; 2148 struct igc_rx_buf *rxbuf; 2149 int i, error; 2150 2151 rxr->rx_buffers = mallocarray(sc->num_rx_desc, 2152 sizeof(struct igc_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO); 2153 if (rxr->rx_buffers == NULL) { 2154 printf("%s: Unable to allocate rx_buffer memory\n", 2155 DEVNAME(sc)); 2156 error = ENOMEM; 2157 goto fail; 2158 } 2159 2160 rxbuf = rxr->rx_buffers; 2161 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2162 error = bus_dmamap_create(rxr->rxdma.dma_tag, 2163 MAX_JUMBO_FRAME_SIZE, 1, MAX_JUMBO_FRAME_SIZE, 0, 2164 BUS_DMA_NOWAIT, &rxbuf->map); 2165 if (error) { 2166 printf("%s: Unable to create RX DMA map\n", 2167 DEVNAME(sc)); 2168 goto fail; 2169 } 2170 } 2171 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2172 rxr->rxdma.dma_map->dm_mapsize, 2173 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2174 2175 return 0; 2176 fail: 2177 return error; 2178 } 2179 2180 /********************************************************************* 2181 * 2182 * Allocate and initialize receive structures. 2183 * 2184 **********************************************************************/ 2185 int 2186 igc_setup_receive_structures(struct igc_softc *sc) 2187 { 2188 struct igc_rxring *rxr = sc->rx_rings; 2189 int i; 2190 2191 for (i = 0; i < sc->sc_nqueues; i++, rxr++) { 2192 if (igc_setup_receive_ring(rxr)) 2193 goto fail; 2194 } 2195 2196 return 0; 2197 fail: 2198 igc_free_receive_structures(sc); 2199 return ENOBUFS; 2200 } 2201 2202 /********************************************************************* 2203 * 2204 * Initialize a receive ring and its buffers. 2205 * 2206 **********************************************************************/ 2207 int 2208 igc_setup_receive_ring(struct igc_rxring *rxr) 2209 { 2210 struct igc_softc *sc = rxr->sc; 2211 struct ifnet *ifp = &sc->sc_ac.ac_if; 2212 int rsize; 2213 2214 rsize = roundup2(sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 2215 IGC_DBA_ALIGN); 2216 2217 /* Clear the ring contents. */ 2218 bzero((void *)rxr->rx_base, rsize); 2219 2220 if (igc_allocate_receive_buffers(rxr)) 2221 return ENOMEM; 2222 2223 /* Setup our descriptor indices. */ 2224 rxr->next_to_check = 0; 2225 rxr->last_desc_filled = sc->num_rx_desc - 1; 2226 2227 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2228 sc->num_rx_desc - 1); 2229 2230 return 0; 2231 } 2232 2233 /********************************************************************* 2234 * 2235 * Enable receive unit. 2236 * 2237 **********************************************************************/ 2238 #define BSIZEPKT_ROUNDUP ((1 << IGC_SRRCTL_BSIZEPKT_SHIFT) - 1) 2239 2240 void 2241 igc_initialize_receive_unit(struct igc_softc *sc) 2242 { 2243 struct igc_rxring *rxr = sc->rx_rings; 2244 struct igc_hw *hw = &sc->hw; 2245 uint32_t rctl, rxcsum, srrctl = 0; 2246 int i; 2247 2248 /* 2249 * Make sure receives are disabled while setting 2250 * up the descriptor ring. 2251 */ 2252 rctl = IGC_READ_REG(hw, IGC_RCTL); 2253 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 2254 2255 /* Setup the Receive Control Register */ 2256 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 2257 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 2258 IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 2259 2260 /* Do not store bad packets */ 2261 rctl &= ~IGC_RCTL_SBP; 2262 2263 /* Enable Long Packet receive */ 2264 if (sc->hw.mac.max_frame_size != ETHER_MAX_LEN) 2265 rctl |= IGC_RCTL_LPE; 2266 2267 /* Strip the CRC */ 2268 rctl |= IGC_RCTL_SECRC; 2269 2270 /* 2271 * Set the interrupt throttling rate. Value is calculated 2272 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 2273 */ 2274 IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 2275 2276 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 2277 rxcsum &= ~IGC_RXCSUM_PCSD; 2278 2279 if (sc->sc_nqueues > 1) 2280 rxcsum |= IGC_RXCSUM_PCSD; 2281 2282 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 2283 2284 if (sc->sc_nqueues > 1) 2285 igc_initialize_rss_mapping(sc); 2286 2287 /* Set maximum packet buffer len */ 2288 srrctl |= (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 2289 IGC_SRRCTL_BSIZEPKT_SHIFT; 2290 /* srrctl above overrides this but set the register to a sane value */ 2291 rctl |= IGC_RCTL_SZ_2048; 2292 2293 /* 2294 * If TX flow control is disabled and there's > 1 queue defined, 2295 * enable DROP. 2296 * 2297 * This drops frames rather than hanging the RX MAC for all queues. 2298 */ 2299 if ((sc->sc_nqueues > 1) && (sc->fc == igc_fc_none || 2300 sc->fc == igc_fc_rx_pause)) { 2301 srrctl |= IGC_SRRCTL_DROP_EN; 2302 } 2303 2304 /* Setup the Base and Length of the RX descriptor rings. */ 2305 for (i = 0; i < sc->sc_nqueues; i++, rxr++) { 2306 IGC_WRITE_REG(hw, IGC_RXDCTL(i), 0); 2307 uint64_t bus_addr = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2308 uint32_t rxdctl; 2309 2310 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 2311 2312 IGC_WRITE_REG(hw, IGC_RDLEN(i), 2313 sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 2314 IGC_WRITE_REG(hw, IGC_RDBAH(i), (uint32_t)(bus_addr >> 32)); 2315 IGC_WRITE_REG(hw, IGC_RDBAL(i), (uint32_t)bus_addr); 2316 IGC_WRITE_REG(hw, IGC_SRRCTL(i), srrctl); 2317 2318 /* Setup the Head and Tail Descriptor Pointers */ 2319 IGC_WRITE_REG(hw, IGC_RDH(i), 0); 2320 IGC_WRITE_REG(hw, IGC_RDT(i), 0); 2321 2322 /* Enable this Queue */ 2323 rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(i)); 2324 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 2325 rxdctl &= 0xFFF00000; 2326 rxdctl |= IGC_RX_PTHRESH; 2327 rxdctl |= IGC_RX_HTHRESH << 8; 2328 rxdctl |= IGC_RX_WTHRESH << 16; 2329 IGC_WRITE_REG(hw, IGC_RXDCTL(i), rxdctl); 2330 } 2331 2332 /* Make sure VLAN Filters are off */ 2333 rctl &= ~IGC_RCTL_VFE; 2334 2335 /* Write out the settings */ 2336 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 2337 } 2338 2339 /********************************************************************* 2340 * 2341 * Free all receive rings. 2342 * 2343 **********************************************************************/ 2344 void 2345 igc_free_receive_structures(struct igc_softc *sc) 2346 { 2347 struct igc_rxring *rxr; 2348 int i; 2349 2350 for (i = 0, rxr = sc->rx_rings; i < sc->sc_nqueues; i++, rxr++) 2351 if_rxr_init(&rxr->rx_ring, 0, 0); 2352 2353 for (i = 0, rxr = sc->rx_rings; i < sc->sc_nqueues; i++, rxr++) 2354 igc_free_receive_buffers(rxr); 2355 } 2356 2357 /********************************************************************* 2358 * 2359 * Free receive ring data structures 2360 * 2361 **********************************************************************/ 2362 void 2363 igc_free_receive_buffers(struct igc_rxring *rxr) 2364 { 2365 struct igc_softc *sc = rxr->sc; 2366 struct igc_rx_buf *rxbuf; 2367 int i; 2368 2369 if (rxr->rx_buffers != NULL) { 2370 for (i = 0; i < sc->num_rx_desc; i++) { 2371 rxbuf = &rxr->rx_buffers[i]; 2372 if (rxbuf->buf != NULL) { 2373 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2374 0, rxbuf->map->dm_mapsize, 2375 BUS_DMASYNC_POSTREAD); 2376 bus_dmamap_unload(rxr->rxdma.dma_tag, 2377 rxbuf->map); 2378 m_freem(rxbuf->buf); 2379 rxbuf->buf = NULL; 2380 } 2381 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 2382 rxbuf->map = NULL; 2383 } 2384 free(rxr->rx_buffers, M_DEVBUF, 2385 sc->num_rx_desc * sizeof(struct igc_rx_buf)); 2386 rxr->rx_buffers = NULL; 2387 } 2388 } 2389 2390 /* 2391 * Initialise the RSS mapping for NICs that support multiple transmit/ 2392 * receive rings. 2393 */ 2394 void 2395 igc_initialize_rss_mapping(struct igc_softc *sc) 2396 { 2397 struct igc_hw *hw = &sc->hw; 2398 uint32_t rss_key[10], mrqc, reta, shift = 0; 2399 int i, queue_id; 2400 2401 /* 2402 * The redirection table controls which destination 2403 * queue each bucket redirects traffic to. 2404 * Each DWORD represents four queues, with the LSB 2405 * being the first queue in the DWORD. 2406 * 2407 * This just allocates buckets to queues using round-robin 2408 * allocation. 2409 * 2410 * NOTE: It Just Happens to line up with the default 2411 * RSS allocation method. 2412 */ 2413 2414 /* Warning FM follows */ 2415 reta = 0; 2416 for (i = 0; i < 128; i++) { 2417 queue_id = (i % sc->sc_nqueues); 2418 /* Adjust if required */ 2419 queue_id = queue_id << shift; 2420 2421 /* 2422 * The low 8 bits are for hash value (n+0); 2423 * The next 8 bits are for hash value (n+1), etc. 2424 */ 2425 reta = reta >> 8; 2426 reta = reta | ( ((uint32_t) queue_id) << 24); 2427 if ((i & 3) == 3) { 2428 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 2429 reta = 0; 2430 } 2431 } 2432 2433 /* 2434 * MRQC: Multiple Receive Queues Command 2435 * Set queuing to RSS control, number depends on the device. 2436 */ 2437 mrqc = IGC_MRQC_ENABLE_RSS_4Q; 2438 2439 /* Set up random bits */ 2440 stoeplitz_to_key(&rss_key, sizeof(rss_key)); 2441 2442 /* Now fill our hash function seeds */ 2443 for (i = 0; i < 10; i++) 2444 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 2445 2446 /* 2447 * Configure the RSS fields to hash upon. 2448 */ 2449 mrqc |= (IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP); 2450 mrqc |= (IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP); 2451 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 2452 2453 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 2454 } 2455 2456 /* 2457 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 2458 * For ASF and Pass Through versions of f/w this means 2459 * that the driver is loaded. For AMT version type f/w 2460 * this means that the network i/f is open. 2461 */ 2462 void 2463 igc_get_hw_control(struct igc_softc *sc) 2464 { 2465 uint32_t ctrl_ext; 2466 2467 ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 2468 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 2469 } 2470 2471 /* 2472 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 2473 * For ASF and Pass Through versions of f/w this means that 2474 * the driver is no longer loaded. For AMT versions of the 2475 * f/w this means that the network i/f is closed. 2476 */ 2477 void 2478 igc_release_hw_control(struct igc_softc *sc) 2479 { 2480 uint32_t ctrl_ext; 2481 2482 ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 2483 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 2484 } 2485 2486 int 2487 igc_is_valid_ether_addr(uint8_t *addr) 2488 { 2489 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 2490 2491 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 2492 return 0; 2493 } 2494 2495 return 1; 2496 } 2497 2498 #if NKSTAT > 0 2499 2500 /* 2501 * the below are read to clear, so they need to be accumulated for 2502 * userland to see counters. periodically fetch the counters from a 2503 * timeout to avoid a 32 roll-over between kstat reads. 2504 */ 2505 2506 enum igc_stat { 2507 igc_stat_crcerrs, 2508 igc_stat_algnerrc, 2509 igc_stat_rxerrc, 2510 igc_stat_mpc, 2511 igc_stat_scc, 2512 igc_stat_ecol, 2513 igc_stat_mcc, 2514 igc_stat_latecol, 2515 igc_stat_colc, 2516 igc_stat_rerc, 2517 igc_stat_dc, 2518 igc_stat_tncrs, 2519 igc_stat_htdpmc, 2520 igc_stat_rlec, 2521 igc_stat_xonrxc, 2522 igc_stat_xontxc, 2523 igc_stat_xoffrxc, 2524 igc_stat_xofftxc, 2525 igc_stat_fcruc, 2526 igc_stat_prc64, 2527 igc_stat_prc127, 2528 igc_stat_prc255, 2529 igc_stat_prc511, 2530 igc_stat_prc1023, 2531 igc_stat_prc1522, 2532 igc_stat_gprc, 2533 igc_stat_bprc, 2534 igc_stat_mprc, 2535 igc_stat_gptc, 2536 igc_stat_gorc, 2537 igc_stat_gotc, 2538 igc_stat_rnbc, 2539 igc_stat_ruc, 2540 igc_stat_rfc, 2541 igc_stat_roc, 2542 igc_stat_rjc, 2543 igc_stat_mgtprc, 2544 igc_stat_mgtpdc, 2545 igc_stat_mgtptc, 2546 igc_stat_tor, 2547 igc_stat_tot, 2548 igc_stat_tpr, 2549 igc_stat_tpt, 2550 igc_stat_ptc64, 2551 igc_stat_ptc127, 2552 igc_stat_ptc255, 2553 igc_stat_ptc511, 2554 igc_stat_ptc1023, 2555 igc_stat_ptc1522, 2556 igc_stat_mptc, 2557 igc_stat_bptc, 2558 igc_stat_tsctc, 2559 2560 igc_stat_iac, 2561 igc_stat_rpthc, 2562 igc_stat_tlpic, 2563 igc_stat_rlpic, 2564 igc_stat_hgptc, 2565 igc_stat_rxdmtc, 2566 igc_stat_hgorc, 2567 igc_stat_hgotc, 2568 igc_stat_lenerrs, 2569 2570 igc_stat_count 2571 }; 2572 2573 struct igc_counter { 2574 const char *name; 2575 enum kstat_kv_unit unit; 2576 uint32_t reg; 2577 }; 2578 2579 static const struct igc_counter igc_counters[igc_stat_count] = { 2580 [igc_stat_crcerrs] = 2581 { "crc errs", KSTAT_KV_U_NONE, IGC_CRCERRS }, 2582 [igc_stat_algnerrc] = 2583 { "alignment errs", KSTAT_KV_U_NONE, IGC_ALGNERRC }, 2584 [igc_stat_rxerrc] = 2585 { "rx errs", KSTAT_KV_U_NONE, IGC_RXERRC }, 2586 [igc_stat_mpc] = 2587 { "missed pkts", KSTAT_KV_U_NONE, IGC_MPC }, 2588 [igc_stat_scc] = 2589 { "single colls", KSTAT_KV_U_NONE, IGC_SCC }, 2590 [igc_stat_ecol] = 2591 { "excessive colls", KSTAT_KV_U_NONE, IGC_ECOL }, 2592 [igc_stat_mcc] = 2593 { "multiple colls", KSTAT_KV_U_NONE, IGC_MCC }, 2594 [igc_stat_latecol] = 2595 { "late colls", KSTAT_KV_U_NONE, IGC_LATECOL }, 2596 [igc_stat_colc] = 2597 { "collisions", KSTAT_KV_U_NONE, IGC_COLC }, 2598 [igc_stat_rerc] = 2599 { "recv errs", KSTAT_KV_U_NONE, IGC_RERC }, 2600 [igc_stat_dc] = 2601 { "defers", KSTAT_KV_U_NONE, IGC_DC }, 2602 [igc_stat_tncrs] = 2603 { "tx no crs", KSTAT_KV_U_NONE, IGC_TNCRS}, 2604 [igc_stat_htdpmc] = 2605 { "host tx discards", KSTAT_KV_U_NONE, IGC_HTDPMC }, 2606 [igc_stat_rlec] = 2607 { "recv len errs", KSTAT_KV_U_NONE, IGC_RLEC }, 2608 [igc_stat_xonrxc] = 2609 { "xon rx", KSTAT_KV_U_NONE, IGC_XONRXC }, 2610 [igc_stat_xontxc] = 2611 { "xon tx", KSTAT_KV_U_NONE, IGC_XONTXC }, 2612 [igc_stat_xoffrxc] = 2613 { "xoff rx", KSTAT_KV_U_NONE, IGC_XOFFRXC }, 2614 [igc_stat_xofftxc] = 2615 { "xoff tx", KSTAT_KV_U_NONE, IGC_XOFFTXC }, 2616 [igc_stat_fcruc] = 2617 { "fc rx unsupp", KSTAT_KV_U_NONE, IGC_FCRUC }, 2618 [igc_stat_prc64] = 2619 { "rx 64B", KSTAT_KV_U_PACKETS, IGC_PRC64 }, 2620 [igc_stat_prc127] = 2621 { "rx 65-127B", KSTAT_KV_U_PACKETS, IGC_PRC127 }, 2622 [igc_stat_prc255] = 2623 { "rx 128-255B", KSTAT_KV_U_PACKETS, IGC_PRC255 }, 2624 [igc_stat_prc511] = 2625 { "rx 256-511B", KSTAT_KV_U_PACKETS, IGC_PRC511 }, 2626 [igc_stat_prc1023] = 2627 { "rx 512-1023B", KSTAT_KV_U_PACKETS, IGC_PRC1023 }, 2628 [igc_stat_prc1522] = 2629 { "rx 1024-maxB", KSTAT_KV_U_PACKETS, IGC_PRC1522 }, 2630 [igc_stat_gprc] = 2631 { "rx good", KSTAT_KV_U_PACKETS, IGC_GPRC }, 2632 [igc_stat_bprc] = 2633 { "rx bcast", KSTAT_KV_U_PACKETS, IGC_BPRC }, 2634 [igc_stat_mprc] = 2635 { "rx mcast", KSTAT_KV_U_PACKETS, IGC_MPRC }, 2636 [igc_stat_gptc] = 2637 { "tx good", KSTAT_KV_U_PACKETS, IGC_GPTC }, 2638 [igc_stat_gorc] = 2639 { "rx good bytes", KSTAT_KV_U_BYTES, 0 }, 2640 [igc_stat_gotc] = 2641 { "tx good bytes", KSTAT_KV_U_BYTES, 0 }, 2642 [igc_stat_rnbc] = 2643 { "rx no bufs", KSTAT_KV_U_NONE, IGC_RNBC }, 2644 [igc_stat_ruc] = 2645 { "rx undersize", KSTAT_KV_U_NONE, IGC_RUC }, 2646 [igc_stat_rfc] = 2647 { "rx frags", KSTAT_KV_U_NONE, IGC_RFC }, 2648 [igc_stat_roc] = 2649 { "rx oversize", KSTAT_KV_U_NONE, IGC_ROC }, 2650 [igc_stat_rjc] = 2651 { "rx jabbers", KSTAT_KV_U_NONE, IGC_RJC }, 2652 [igc_stat_mgtprc] = 2653 { "rx mgmt", KSTAT_KV_U_PACKETS, IGC_MGTPRC }, 2654 [igc_stat_mgtpdc] = 2655 { "rx mgmt drops", KSTAT_KV_U_PACKETS, IGC_MGTPDC }, 2656 [igc_stat_mgtptc] = 2657 { "tx mgmt", KSTAT_KV_U_PACKETS, IGC_MGTPTC }, 2658 [igc_stat_tor] = 2659 { "rx total bytes", KSTAT_KV_U_BYTES, 0 }, 2660 [igc_stat_tot] = 2661 { "tx total bytes", KSTAT_KV_U_BYTES, 0 }, 2662 [igc_stat_tpr] = 2663 { "rx total", KSTAT_KV_U_PACKETS, IGC_TPR }, 2664 [igc_stat_tpt] = 2665 { "tx total", KSTAT_KV_U_PACKETS, IGC_TPT }, 2666 [igc_stat_ptc64] = 2667 { "tx 64B", KSTAT_KV_U_PACKETS, IGC_PTC64 }, 2668 [igc_stat_ptc127] = 2669 { "tx 65-127B", KSTAT_KV_U_PACKETS, IGC_PTC127 }, 2670 [igc_stat_ptc255] = 2671 { "tx 128-255B", KSTAT_KV_U_PACKETS, IGC_PTC255 }, 2672 [igc_stat_ptc511] = 2673 { "tx 256-511B", KSTAT_KV_U_PACKETS, IGC_PTC511 }, 2674 [igc_stat_ptc1023] = 2675 { "tx 512-1023B", KSTAT_KV_U_PACKETS, IGC_PTC1023 }, 2676 [igc_stat_ptc1522] = 2677 { "tx 1024-maxB", KSTAT_KV_U_PACKETS, IGC_PTC1522 }, 2678 [igc_stat_mptc] = 2679 { "tx mcast", KSTAT_KV_U_PACKETS, IGC_MPTC }, 2680 [igc_stat_bptc] = 2681 { "tx bcast", KSTAT_KV_U_PACKETS, IGC_BPTC }, 2682 [igc_stat_tsctc] = 2683 { "tx tso ctx", KSTAT_KV_U_NONE, IGC_TSCTC }, 2684 2685 [igc_stat_iac] = 2686 { "interrupts", KSTAT_KV_U_NONE, IGC_IAC }, 2687 [igc_stat_rpthc] = 2688 { "rx to host", KSTAT_KV_U_PACKETS, IGC_RPTHC }, 2689 [igc_stat_tlpic] = 2690 { "eee tx lpi", KSTAT_KV_U_NONE, IGC_TLPIC }, 2691 [igc_stat_rlpic] = 2692 { "eee rx lpi", KSTAT_KV_U_NONE, IGC_RLPIC }, 2693 [igc_stat_hgptc] = 2694 { "host rx", KSTAT_KV_U_PACKETS, IGC_HGPTC }, 2695 [igc_stat_rxdmtc] = 2696 { "rxd min thresh", KSTAT_KV_U_NONE, IGC_RXDMTC }, 2697 [igc_stat_hgorc] = 2698 { "host good rx", KSTAT_KV_U_BYTES, 0 }, 2699 [igc_stat_hgotc] = 2700 { "host good tx", KSTAT_KV_U_BYTES, 0 }, 2701 [igc_stat_lenerrs] = 2702 { "len errs", KSTAT_KV_U_NONE, IGC_LENERRS }, 2703 }; 2704 2705 static void 2706 igc_stat_read(struct igc_softc *sc) 2707 { 2708 struct igc_hw *hw = &sc->hw; 2709 struct kstat *ks = sc->ks; 2710 struct kstat_kv *kvs = ks->ks_data; 2711 uint32_t hi, lo; 2712 unsigned int i; 2713 2714 for (i = 0; i < nitems(igc_counters); i++) { 2715 const struct igc_counter *c = &igc_counters[i]; 2716 if (c->reg == 0) 2717 continue; 2718 2719 kstat_kv_u64(&kvs[i]) += IGC_READ_REG(hw, c->reg); 2720 } 2721 2722 lo = IGC_READ_REG(hw, IGC_GORCL); 2723 hi = IGC_READ_REG(hw, IGC_GORCH); 2724 kstat_kv_u64(&kvs[igc_stat_gorc]) += 2725 ((uint64_t)hi << 32) | ((uint64_t)lo << 0); 2726 2727 lo = IGC_READ_REG(hw, IGC_GOTCL); 2728 hi = IGC_READ_REG(hw, IGC_GOTCH); 2729 kstat_kv_u64(&kvs[igc_stat_gotc]) += 2730 ((uint64_t)hi << 32) | ((uint64_t)lo << 0); 2731 2732 lo = IGC_READ_REG(hw, IGC_TORL); 2733 hi = IGC_READ_REG(hw, IGC_TORH); 2734 kstat_kv_u64(&kvs[igc_stat_tor]) += 2735 ((uint64_t)hi << 32) | ((uint64_t)lo << 0); 2736 2737 lo = IGC_READ_REG(hw, IGC_TOTL); 2738 hi = IGC_READ_REG(hw, IGC_TOTH); 2739 kstat_kv_u64(&kvs[igc_stat_tot]) += 2740 ((uint64_t)hi << 32) | ((uint64_t)lo << 0); 2741 2742 lo = IGC_READ_REG(hw, IGC_HGORCL); 2743 hi = IGC_READ_REG(hw, IGC_HGORCH); 2744 kstat_kv_u64(&kvs[igc_stat_hgorc]) += 2745 ((uint64_t)hi << 32) | ((uint64_t)lo << 0); 2746 2747 lo = IGC_READ_REG(hw, IGC_HGOTCL); 2748 hi = IGC_READ_REG(hw, IGC_HGOTCH); 2749 kstat_kv_u64(&kvs[igc_stat_hgotc]) += 2750 ((uint64_t)hi << 32) | ((uint64_t)lo << 0); 2751 } 2752 2753 static void 2754 igc_kstat_tick(void *arg) 2755 { 2756 struct igc_softc *sc = arg; 2757 2758 if (mtx_enter_try(&sc->ks_mtx)) { 2759 igc_stat_read(sc); 2760 mtx_leave(&sc->ks_mtx); 2761 } 2762 2763 timeout_add_sec(&sc->ks_tmo, 4); 2764 } 2765 2766 static int 2767 igc_kstat_read(struct kstat *ks) 2768 { 2769 struct igc_softc *sc = ks->ks_softc; 2770 2771 igc_stat_read(sc); 2772 nanouptime(&ks->ks_updated); 2773 2774 return (0); 2775 } 2776 2777 void 2778 igc_kstat_attach(struct igc_softc *sc) 2779 { 2780 struct kstat *ks; 2781 struct kstat_kv *kvs; 2782 size_t len; 2783 unsigned int i; 2784 2785 mtx_init(&sc->ks_mtx, IPL_SOFTCLOCK); 2786 timeout_set(&sc->ks_tmo, igc_kstat_tick, sc); 2787 2788 kvs = mallocarray(sizeof(*kvs), nitems(igc_counters), M_DEVBUF, 2789 M_WAITOK|M_ZERO|M_CANFAIL); 2790 if (kvs == NULL) { 2791 printf("%s: unable to allocate igc kstats\n", DEVNAME(sc)); 2792 return; 2793 } 2794 len = sizeof(*kvs) * nitems(igc_counters); 2795 2796 ks = kstat_create(DEVNAME(sc), 0, "igc-stats", 0, KSTAT_T_KV, 0); 2797 if (ks == NULL) { 2798 printf("%s: unable to create igc kstats\n", DEVNAME(sc)); 2799 free(kvs, M_DEVBUF, len); 2800 return; 2801 } 2802 2803 for (i = 0; i < nitems(igc_counters); i++) { 2804 const struct igc_counter *c = &igc_counters[i]; 2805 kstat_kv_unit_init(&kvs[i], c->name, 2806 KSTAT_KV_T_COUNTER64, c->unit); 2807 } 2808 2809 ks->ks_softc = sc; 2810 ks->ks_data = kvs; 2811 ks->ks_datalen = len; 2812 ks->ks_read = igc_kstat_read; 2813 kstat_set_mutex(ks, &sc->ks_mtx); 2814 2815 kstat_install(ks); 2816 2817 sc->ks = ks; 2818 2819 igc_kstat_tick(sc); /* let's gooo */ 2820 } 2821 #endif /* NKSTAT > 0 */ 2822