1 /* $OpenBSD: if_igc.c,v 1.5 2021/11/22 14:00:52 jsg Exp $ */ 2 /*- 3 * SPDX-License-Identifier: BSD-2-Clause 4 * 5 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 6 * All rights reserved. 7 * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include "bpfilter.h" 32 #include "vlan.h" 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/sockio.h> 37 #include <sys/mbuf.h> 38 #include <sys/malloc.h> 39 #include <sys/kernel.h> 40 #include <sys/socket.h> 41 #include <sys/device.h> 42 #include <sys/endian.h> 43 #include <sys/intrmap.h> 44 45 #include <net/if.h> 46 #include <net/if_media.h> 47 #include <net/toeplitz.h> 48 49 #include <netinet/in.h> 50 #include <netinet/if_ether.h> 51 52 #if NBPFILTER > 0 53 #include <net/bpf.h> 54 #endif 55 56 #include <machine/bus.h> 57 #include <machine/intr.h> 58 59 #include <dev/pci/pcivar.h> 60 #include <dev/pci/pcireg.h> 61 #include <dev/pci/pcidevs.h> 62 #include <dev/pci/if_igc.h> 63 #include <dev/pci/igc_hw.h> 64 65 const struct pci_matchid igc_devices[] = { 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K }, 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2 }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM }, 77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT }, 78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM }, 79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K }, 80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V } 81 }; 82 83 /********************************************************************* 84 * Function Prototypes 85 *********************************************************************/ 86 int igc_match(struct device *, void *, void *); 87 void igc_attach(struct device *, struct device *, void *); 88 int igc_detach(struct device *, int); 89 90 void igc_identify_hardware(struct igc_softc *); 91 int igc_allocate_pci_resources(struct igc_softc *); 92 int igc_allocate_queues(struct igc_softc *); 93 void igc_free_pci_resources(struct igc_softc *); 94 void igc_reset(struct igc_softc *); 95 void igc_init_dmac(struct igc_softc *, uint32_t); 96 int igc_allocate_msix(struct igc_softc *); 97 void igc_setup_msix(struct igc_softc *); 98 int igc_dma_malloc(struct igc_softc *, bus_size_t, struct igc_dma_alloc *); 99 void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 100 void igc_setup_interface(struct igc_softc *); 101 102 void igc_init(void *); 103 void igc_start(struct ifqueue *); 104 int igc_txeof(struct tx_ring *); 105 void igc_stop(struct igc_softc *); 106 int igc_ioctl(struct ifnet *, u_long, caddr_t); 107 int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 108 int igc_rxfill(struct rx_ring *); 109 void igc_rxrefill(void *); 110 int igc_rxeof(struct rx_ring *); 111 void igc_rx_checksum(uint32_t, struct mbuf *, uint32_t); 112 void igc_watchdog(struct ifnet *); 113 void igc_media_status(struct ifnet *, struct ifmediareq *); 114 int igc_media_change(struct ifnet *); 115 void igc_iff(struct igc_softc *); 116 void igc_update_link_status(struct igc_softc *); 117 int igc_get_buf(struct rx_ring *, int); 118 119 void igc_configure_queues(struct igc_softc *); 120 void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 121 void igc_enable_queue(struct igc_softc *, uint32_t); 122 void igc_enable_intr(struct igc_softc *); 123 void igc_disable_intr(struct igc_softc *); 124 int igc_intr_link(void *); 125 int igc_intr_queue(void *); 126 127 int igc_allocate_transmit_buffers(struct tx_ring *); 128 int igc_setup_transmit_structures(struct igc_softc *); 129 int igc_setup_transmit_ring(struct tx_ring *); 130 void igc_initialize_transmit_unit(struct igc_softc *); 131 void igc_free_transmit_structures(struct igc_softc *); 132 void igc_free_transmit_buffers(struct tx_ring *); 133 int igc_allocate_receive_buffers(struct rx_ring *); 134 int igc_setup_receive_structures(struct igc_softc *); 135 int igc_setup_receive_ring(struct rx_ring *); 136 void igc_initialize_receive_unit(struct igc_softc *); 137 void igc_free_receive_structures(struct igc_softc *); 138 void igc_free_receive_buffers(struct rx_ring *); 139 void igc_initialize_rss_mapping(struct igc_softc *); 140 141 void igc_get_hw_control(struct igc_softc *); 142 void igc_release_hw_control(struct igc_softc *); 143 int igc_is_valid_ether_addr(uint8_t *); 144 145 /********************************************************************* 146 * OpenBSD Device Interface Entry Points 147 *********************************************************************/ 148 149 struct cfdriver igc_cd = { 150 NULL, "igc", DV_IFNET 151 }; 152 153 struct cfattach igc_ca = { 154 sizeof(struct igc_softc), igc_match, igc_attach, igc_detach 155 }; 156 157 /********************************************************************* 158 * Device identification routine 159 * 160 * igc_match determines if the driver should be loaded on 161 * adapter based on PCI vendor/device id of the adapter. 162 * 163 * return 0 on success, positive on failure 164 *********************************************************************/ 165 int 166 igc_match(struct device *parent, void *match, void *aux) 167 { 168 return pci_matchbyid((struct pci_attach_args *)aux, igc_devices, 169 nitems(igc_devices)); 170 } 171 172 /********************************************************************* 173 * Device initialization routine 174 * 175 * The attach entry point is called when the driver is being loaded. 176 * This routine identifies the type of hardware, allocates all resources 177 * and initializes the hardware. 178 * 179 * return 0 on success, positive on failure 180 *********************************************************************/ 181 void 182 igc_attach(struct device *parent, struct device *self, void *aux) 183 { 184 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 185 struct igc_softc *sc = (struct igc_softc *)self; 186 struct igc_hw *hw = &sc->hw; 187 188 sc->osdep.os_sc = sc; 189 sc->osdep.os_pa = *pa; 190 191 /* Determine hardware and mac info */ 192 igc_identify_hardware(sc); 193 194 sc->num_tx_desc = IGC_DEFAULT_TXD; 195 sc->num_rx_desc = IGC_DEFAULT_RXD; 196 197 /* Setup PCI resources */ 198 if (igc_allocate_pci_resources(sc)) 199 goto err_pci; 200 201 /* Allocate TX/RX queues */ 202 if (igc_allocate_queues(sc)) 203 goto err_pci; 204 205 /* Do shared code initialization */ 206 if (igc_setup_init_funcs(hw, true)) { 207 printf(": Setup of shared code failed\n"); 208 goto err_pci; 209 } 210 211 hw->mac.autoneg = DO_AUTO_NEG; 212 hw->phy.autoneg_wait_to_complete = false; 213 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 214 215 /* Copper options. */ 216 if (hw->phy.media_type == igc_media_type_copper) 217 hw->phy.mdix = AUTO_ALL_MODES; 218 219 /* Set the max frame size. */ 220 sc->hw.mac.max_frame_size = 9234; 221 222 /* Allocate multicast array memory. */ 223 sc->mta = mallocarray(ETHER_ADDR_LEN, MAX_NUM_MULTICAST_ADDRESSES, 224 M_DEVBUF, M_NOWAIT); 225 if (sc->mta == NULL) { 226 printf(": Can not allocate multicast setup array\n"); 227 goto err_late; 228 } 229 230 /* Check SOL/IDER usage. */ 231 if (igc_check_reset_block(hw)) 232 printf(": PHY reset is blocked due to SOL/IDER session\n"); 233 234 /* Enable Energy Efficient Ethernet. */ 235 sc->hw.dev_spec._i225.eee_disable = true; 236 237 igc_reset_hw(hw); 238 239 /* Make sure we have a good EEPROM before we read from it. */ 240 if (igc_validate_nvm_checksum(hw) < 0) { 241 /* 242 * Some PCI-E parts fail the first check due to 243 * the link being in sleep state, call it again, 244 * if it fails a second time its a real issue. 245 */ 246 if (igc_validate_nvm_checksum(hw) < 0) { 247 printf(": The EEPROM checksum is not valid\n"); 248 goto err_late; 249 } 250 } 251 252 /* Copy the permanent MAC address out of the EEPROM. */ 253 if (igc_read_mac_addr(hw) < 0) { 254 printf(": EEPROM read error while reading MAC address\n"); 255 goto err_late; 256 } 257 258 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 259 printf(": Invalid MAC address\n"); 260 goto err_late; 261 } 262 263 memcpy(sc->sc_ac.ac_enaddr, sc->hw.mac.addr, ETHER_ADDR_LEN); 264 265 if (igc_allocate_msix(sc)) 266 goto err_late; 267 268 /* Setup OS specific network interface. */ 269 igc_setup_interface(sc); 270 271 igc_reset(sc); 272 hw->mac.get_link_status = true; 273 igc_update_link_status(sc); 274 275 /* The driver can now take control from firmware. */ 276 igc_get_hw_control(sc); 277 278 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 279 return; 280 281 err_late: 282 igc_release_hw_control(sc); 283 err_pci: 284 igc_free_pci_resources(sc); 285 free(sc->mta, M_DEVBUF, ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 286 } 287 288 /********************************************************************* 289 * Device removal routine 290 * 291 * The detach entry point is called when the driver is being removed. 292 * This routine stops the adapter and deallocates all the resources 293 * that were allocated for driver operation. 294 * 295 * return 0 on success, positive on failure 296 *********************************************************************/ 297 int 298 igc_detach(struct device *self, int flags) 299 { 300 struct igc_softc *sc = (struct igc_softc *)self; 301 struct ifnet *ifp = &sc->sc_ac.ac_if; 302 303 igc_stop(sc); 304 305 igc_phy_hw_reset(&sc->hw); 306 igc_release_hw_control(sc); 307 308 ether_ifdetach(ifp); 309 if_detach(ifp); 310 311 igc_free_pci_resources(sc); 312 313 igc_free_transmit_structures(sc); 314 igc_free_receive_structures(sc); 315 free(sc->mta, M_DEVBUF, ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 316 317 return 0; 318 } 319 320 void 321 igc_identify_hardware(struct igc_softc *sc) 322 { 323 struct igc_osdep *os = &sc->osdep; 324 struct pci_attach_args *pa = &os->os_pa; 325 326 /* Save off the information about this board. */ 327 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 328 329 /* Do shared code init and setup. */ 330 if (igc_set_mac_type(&sc->hw)) { 331 printf(": Setup init failure\n"); 332 return; 333 } 334 } 335 336 int 337 igc_allocate_pci_resources(struct igc_softc *sc) 338 { 339 struct igc_osdep *os = &sc->osdep; 340 struct pci_attach_args *pa = &os->os_pa; 341 pcireg_t memtype; 342 343 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 344 if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 345 &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 346 printf(": unable to map registers\n"); 347 return ENXIO; 348 } 349 sc->hw.hw_addr = (uint8_t *)os->os_membase; 350 sc->hw.back = os; 351 352 igc_setup_msix(sc); 353 354 return 0; 355 } 356 357 int 358 igc_allocate_queues(struct igc_softc *sc) 359 { 360 struct igc_queue *iq; 361 struct tx_ring *txr; 362 struct rx_ring *rxr; 363 int i, rsize, rxconf, tsize, txconf; 364 365 /* Allocate the top level queue structs. */ 366 sc->queues = mallocarray(sc->sc_nqueues, sizeof(struct igc_queue), 367 M_DEVBUF, M_NOWAIT | M_ZERO); 368 if (sc->queues == NULL) { 369 printf("%s: unable to allocate queue\n", DEVNAME(sc)); 370 goto fail; 371 } 372 373 /* Allocate the TX ring. */ 374 sc->tx_rings = mallocarray(sc->sc_nqueues, sizeof(struct tx_ring), 375 M_DEVBUF, M_NOWAIT | M_ZERO); 376 if (sc->tx_rings == NULL) { 377 printf("%s: unable to allocate TX ring\n", DEVNAME(sc)); 378 goto fail; 379 } 380 381 /* Allocate the RX ring. */ 382 sc->rx_rings = mallocarray(sc->sc_nqueues, sizeof(struct rx_ring), 383 M_DEVBUF, M_NOWAIT | M_ZERO); 384 if (sc->rx_rings == NULL) { 385 printf("%s: unable to allocate RX ring\n", DEVNAME(sc)); 386 goto rx_fail; 387 } 388 389 txconf = rxconf = 0; 390 391 /* Set up the TX queues. */ 392 tsize = roundup2(sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 393 IGC_DBA_ALIGN); 394 for (i = 0; i < sc->sc_nqueues; i++, txconf++) { 395 txr = &sc->tx_rings[i]; 396 txr->sc = sc; 397 txr->me = i; 398 399 if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 400 printf("%s: unable to allocate TX descriptor\n", 401 DEVNAME(sc)); 402 goto err_tx_desc; 403 } 404 txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 405 bzero((void *)txr->tx_base, tsize); 406 } 407 408 /* Set up the RX queues. */ 409 rsize = roundup2(sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 410 IGC_DBA_ALIGN); 411 for (i = 0; i < sc->sc_nqueues; i++, rxconf++) { 412 rxr = &sc->rx_rings[i]; 413 rxr->sc = sc; 414 rxr->me = i; 415 timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 416 417 if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 418 printf("%s: unable to allocate RX descriptor\n", 419 DEVNAME(sc)); 420 goto err_rx_desc; 421 } 422 rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 423 bzero((void *)rxr->rx_base, rsize); 424 } 425 426 /* Set up the queue holding structs. */ 427 for (i = 0; i < sc->sc_nqueues; i++) { 428 iq = &sc->queues[i]; 429 iq->sc = sc; 430 iq->txr = &sc->tx_rings[i]; 431 iq->rxr = &sc->rx_rings[i]; 432 snprintf(iq->name, sizeof(iq->name), "%s:%d", DEVNAME(sc), i); 433 } 434 435 return 0; 436 437 err_rx_desc: 438 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 439 igc_dma_free(sc, &rxr->rxdma); 440 err_tx_desc: 441 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 442 igc_dma_free(sc, &txr->txdma); 443 free(sc->rx_rings, M_DEVBUF, sc->sc_nqueues * sizeof(struct rx_ring)); 444 sc->rx_rings = NULL; 445 rx_fail: 446 free(sc->tx_rings, M_DEVBUF, sc->sc_nqueues * sizeof(struct tx_ring)); 447 sc->tx_rings = NULL; 448 fail: 449 return ENOMEM; 450 } 451 452 void 453 igc_free_pci_resources(struct igc_softc *sc) 454 { 455 struct igc_osdep *os = &sc->osdep; 456 struct pci_attach_args *pa = &os->os_pa; 457 struct igc_queue *iq = sc->queues; 458 int i; 459 460 /* Release all msix queue resources. */ 461 for (i = 0; i < sc->sc_nqueues; i++, iq++) { 462 if (iq->tag) 463 pci_intr_disestablish(pa->pa_pc, iq->tag); 464 iq->tag = NULL; 465 } 466 467 if (sc->tag) 468 pci_intr_disestablish(pa->pa_pc, sc->tag); 469 sc->tag = NULL; 470 if (os->os_membase != 0) 471 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 472 os->os_membase = 0; 473 } 474 475 /********************************************************************* 476 * 477 * Initialize the hardware to a configuration as specified by the 478 * adapter structure. 479 * 480 **********************************************************************/ 481 void 482 igc_reset(struct igc_softc *sc) 483 { 484 struct igc_hw *hw = &sc->hw; 485 uint32_t pba; 486 uint16_t rx_buffer_size; 487 488 /* Let the firmware know the OS is in control */ 489 igc_get_hw_control(sc); 490 491 /* 492 * Packet Buffer Allocation (PBA) 493 * Writing PBA sets the receive portion of the buffer 494 * the remainder is used for the transmit buffer. 495 */ 496 pba = IGC_PBA_34K; 497 498 /* 499 * These parameters control the automatic generation (Tx) and 500 * response (Rx) to Ethernet PAUSE frames. 501 * - High water mark should allow for at least two frames to be 502 * received after sending an XOFF. 503 * - Low water mark works best when it is very near the high water mark. 504 * This allows the receiver to restart by sending XON when it has 505 * drained a bit. Here we use an arbitrary value of 1500 which will 506 * restart after one full frame is pulled from the buffer. There 507 * could be several smaller frames in the buffer and if so they will 508 * not trigger the XON until their total number reduces the buffer 509 * by 1500. 510 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 511 */ 512 rx_buffer_size = (pba & 0xffff) << 10; 513 hw->fc.high_water = rx_buffer_size - 514 roundup2(sc->hw.mac.max_frame_size, 1024); 515 /* 16-byte granularity */ 516 hw->fc.low_water = hw->fc.high_water - 16; 517 518 if (sc->fc) /* locally set flow control value? */ 519 hw->fc.requested_mode = sc->fc; 520 else 521 hw->fc.requested_mode = igc_fc_full; 522 523 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 524 525 hw->fc.send_xon = true; 526 527 /* Issue a global reset */ 528 igc_reset_hw(hw); 529 IGC_WRITE_REG(hw, IGC_WUC, 0); 530 531 /* and a re-init */ 532 if (igc_init_hw(hw) < 0) { 533 printf(": Hardware Initialization Failed\n"); 534 return; 535 } 536 537 /* Setup DMA Coalescing */ 538 igc_init_dmac(sc, pba); 539 540 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 541 igc_get_phy_info(hw); 542 igc_check_for_link(hw); 543 } 544 545 /********************************************************************* 546 * 547 * Initialize the DMA Coalescing feature 548 * 549 **********************************************************************/ 550 void 551 igc_init_dmac(struct igc_softc *sc, uint32_t pba) 552 { 553 struct igc_hw *hw = &sc->hw; 554 uint32_t dmac, reg = ~IGC_DMACR_DMAC_EN; 555 uint16_t hwm, max_frame_size; 556 int status; 557 558 max_frame_size = sc->hw.mac.max_frame_size; 559 560 if (sc->dmac == 0) { /* Disabling it */ 561 IGC_WRITE_REG(hw, IGC_DMACR, reg); 562 return; 563 } else 564 printf(": DMA Coalescing enabled\n"); 565 566 /* Set starting threshold */ 567 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 568 569 hwm = 64 * pba - max_frame_size / 16; 570 if (hwm < 64 * (pba - 6)) 571 hwm = 64 * (pba - 6); 572 reg = IGC_READ_REG(hw, IGC_FCRTC); 573 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 574 reg |= ((hwm << IGC_FCRTC_RTH_COAL_SHIFT) 575 & IGC_FCRTC_RTH_COAL_MASK); 576 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 577 578 dmac = pba - max_frame_size / 512; 579 if (dmac < pba - 10) 580 dmac = pba - 10; 581 reg = IGC_READ_REG(hw, IGC_DMACR); 582 reg &= ~IGC_DMACR_DMACTHR_MASK; 583 reg |= ((dmac << IGC_DMACR_DMACTHR_SHIFT) 584 & IGC_DMACR_DMACTHR_MASK); 585 586 /* transition to L0x or L1 if available..*/ 587 reg |= (IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK); 588 589 /* Check if status is 2.5Gb backplane connection 590 * before configuration of watchdog timer, which is 591 * in msec values in 12.8usec intervals 592 * watchdog timer= msec values in 32usec intervals 593 * for non 2.5Gb connection 594 */ 595 status = IGC_READ_REG(hw, IGC_STATUS); 596 if ((status & IGC_STATUS_2P5_SKU) && 597 (!(status & IGC_STATUS_2P5_SKU_OVER))) 598 reg |= ((sc->dmac * 5) >> 6); 599 else 600 reg |= (sc->dmac >> 5); 601 602 IGC_WRITE_REG(hw, IGC_DMACR, reg); 603 604 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 605 606 /* Set the interval before transition */ 607 reg = IGC_READ_REG(hw, IGC_DMCTLX); 608 reg |= IGC_DMCTLX_DCFLUSH_DIS; 609 610 /* 611 ** in 2.5Gb connection, TTLX unit is 0.4 usec 612 ** which is 0x4*2 = 0xA. But delay is still 4 usec 613 */ 614 status = IGC_READ_REG(hw, IGC_STATUS); 615 if ((status & IGC_STATUS_2P5_SKU) && 616 (!(status & IGC_STATUS_2P5_SKU_OVER))) 617 reg |= 0xA; 618 else 619 reg |= 0x4; 620 621 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 622 623 /* free space in tx packet buffer to wake from DMA coal */ 624 IGC_WRITE_REG(hw, IGC_DMCTXTH, (IGC_TXPBSIZE - 625 (2 * max_frame_size)) >> 6); 626 627 /* make low power state decision controlled by DMA coal */ 628 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 629 reg &= ~IGC_PCIEMISC_LX_DECISION; 630 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 631 } 632 633 int 634 igc_allocate_msix(struct igc_softc *sc) 635 { 636 struct igc_osdep *os = &sc->osdep; 637 struct pci_attach_args *pa = &os->os_pa; 638 struct igc_queue *iq; 639 pci_intr_handle_t ih; 640 int i, error = 0; 641 642 for (i = 0, iq = sc->queues; i < sc->sc_nqueues; i++, iq++) { 643 if (pci_intr_map_msix(pa, i, &ih)) { 644 printf("%s: unable to map msi-x vector %d\n", 645 DEVNAME(sc), i); 646 error = ENOMEM; 647 goto fail; 648 } 649 650 iq->tag = pci_intr_establish_cpu(pa->pa_pc, ih, 651 IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i), 652 igc_intr_queue, iq, iq->name); 653 if (iq->tag == NULL) { 654 printf("%s: unable to establish interrupt %d\n", 655 DEVNAME(sc), i); 656 error = ENOMEM; 657 goto fail; 658 } 659 660 iq->msix = i; 661 iq->eims = 1 << i; 662 } 663 664 /* Now the link status/control last MSI-X vector. */ 665 if (pci_intr_map_msix(pa, i, &ih)) { 666 printf("%s: unable to map link vector\n", DEVNAME(sc)); 667 error = ENOMEM; 668 goto fail; 669 } 670 671 sc->tag = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE, 672 igc_intr_link, sc, sc->sc_dev.dv_xname); 673 if (sc->tag == NULL) { 674 printf("%s: unable to establish link interrupt\n", DEVNAME(sc)); 675 error = ENOMEM; 676 goto fail; 677 } 678 679 sc->linkvec = i; 680 printf(", %s, %d queue%s", pci_intr_string(pa->pa_pc, ih), 681 i, (i > 1) ? "s" : ""); 682 683 return 0; 684 fail: 685 for (iq = sc->queues; i > 0; i--, iq++) { 686 if (iq->tag == NULL) 687 continue; 688 pci_intr_disestablish(pa->pa_pc, iq->tag); 689 iq->tag = NULL; 690 } 691 692 return error; 693 } 694 695 void 696 igc_setup_msix(struct igc_softc *sc) 697 { 698 struct igc_osdep *os = &sc->osdep; 699 struct pci_attach_args *pa = &os->os_pa; 700 int nmsix; 701 702 nmsix = pci_intr_msix_count(pa); 703 if (nmsix <= 1) 704 printf(": not enough msi-x vectors\n"); 705 706 /* Give one vector to events. */ 707 nmsix--; 708 709 sc->sc_intrmap = intrmap_create(&sc->sc_dev, nmsix, IGC_MAX_VECTORS, 710 INTRMAP_POWEROF2); 711 sc->sc_nqueues = intrmap_count(sc->sc_intrmap); 712 } 713 714 int 715 igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 716 { 717 struct igc_osdep *os = &sc->osdep; 718 719 dma->dma_tag = os->os_pa.pa_dmat; 720 721 if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, BUS_DMA_NOWAIT, 722 &dma->dma_map)) 723 return 1; 724 if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 725 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) 726 goto destroy; 727 if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 728 &dma->dma_vaddr, BUS_DMA_NOWAIT)) 729 goto free; 730 if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 731 NULL, BUS_DMA_NOWAIT)) 732 goto unmap; 733 734 dma->dma_size = size; 735 736 return 0; 737 unmap: 738 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 739 free: 740 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 741 destroy: 742 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 743 dma->dma_map = NULL; 744 dma->dma_tag = NULL; 745 return 1; 746 } 747 748 void 749 igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 750 { 751 if (dma->dma_tag == NULL) 752 return; 753 754 if (dma->dma_map != NULL) { 755 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 756 dma->dma_map->dm_mapsize, 757 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 758 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 759 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 760 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 761 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 762 dma->dma_map = NULL; 763 } 764 } 765 766 /********************************************************************* 767 * 768 * Setup networking device structure and register an interface. 769 * 770 **********************************************************************/ 771 void 772 igc_setup_interface(struct igc_softc *sc) 773 { 774 struct ifnet *ifp = &sc->sc_ac.ac_if; 775 int i; 776 777 ifp->if_softc = sc; 778 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); 779 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 780 ifp->if_xflags = IFXF_MPSAFE; 781 ifp->if_ioctl = igc_ioctl; 782 ifp->if_qstart = igc_start; 783 ifp->if_watchdog = igc_watchdog; 784 ifp->if_hardmtu = sc->hw.mac.max_frame_size - ETHER_HDR_LEN - 785 ETHER_CRC_LEN; 786 ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1); 787 788 ifp->if_capabilities = IFCAP_VLAN_MTU; 789 790 #ifdef notyet 791 #if NVLAN > 0 792 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 793 #endif 794 795 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 796 #endif 797 798 /* Initialize ifmedia structures. */ 799 ifmedia_init(&sc->media, IFM_IMASK, igc_media_change, igc_media_status); 800 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 801 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 802 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 803 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 804 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 805 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 806 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T, 0, NULL); 807 808 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 809 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 810 811 if_attach(ifp); 812 ether_ifattach(ifp); 813 814 if_attach_queues(ifp, sc->sc_nqueues); 815 if_attach_iqueues(ifp, sc->sc_nqueues); 816 for (i = 0; i < sc->sc_nqueues; i++) { 817 struct ifqueue *ifq = ifp->if_ifqs[i]; 818 struct ifiqueue *ifiq = ifp->if_iqs[i]; 819 struct tx_ring *txr = &sc->tx_rings[i]; 820 struct rx_ring *rxr = &sc->rx_rings[i]; 821 822 ifq->ifq_softc = txr; 823 txr->ifq = ifq; 824 825 ifiq->ifiq_softc = rxr; 826 rxr->ifiq = ifiq; 827 } 828 } 829 830 void 831 igc_init(void *arg) 832 { 833 struct igc_softc *sc = (struct igc_softc *)arg; 834 struct ifnet *ifp = &sc->sc_ac.ac_if; 835 struct rx_ring *rxr; 836 uint32_t ctrl = 0; 837 int i, s; 838 839 s = splnet(); 840 841 igc_stop(sc); 842 843 /* Get the latest mac address, user can use a LAA. */ 844 bcopy(sc->sc_ac.ac_enaddr, sc->hw.mac.addr, ETHER_ADDR_LEN); 845 846 /* Put the address into the receive address array. */ 847 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 848 849 /* Initialize the hardware. */ 850 igc_reset(sc); 851 igc_update_link_status(sc); 852 853 /* Setup VLAN support, basic and offload if available. */ 854 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 855 856 /* Prepare transmit descriptors and buffers. */ 857 if (igc_setup_transmit_structures(sc)) { 858 printf("%s: Could not setup transmit structures\n", 859 DEVNAME(sc)); 860 igc_stop(sc); 861 splx(s); 862 return; 863 } 864 igc_initialize_transmit_unit(sc); 865 866 sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN; 867 /* Prepare receive descriptors and buffers. */ 868 if (igc_setup_receive_structures(sc)) { 869 printf("%s: Could not setup receive structures\n", 870 DEVNAME(sc)); 871 igc_stop(sc); 872 splx(s); 873 return; 874 } 875 igc_initialize_receive_unit(sc); 876 877 if (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) { 878 ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 879 ctrl |= IGC_CTRL_VME; 880 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 881 } 882 883 /* Setup multicast table. */ 884 igc_iff(sc); 885 886 igc_clear_hw_cntrs_base_generic(&sc->hw); 887 888 igc_configure_queues(sc); 889 890 /* This clears any pending interrupts */ 891 IGC_READ_REG(&sc->hw, IGC_ICR); 892 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 893 894 /* The driver can now take control from firmware. */ 895 igc_get_hw_control(sc); 896 897 /* Set Energy Efficient Ethernet. */ 898 igc_set_eee_i225(&sc->hw, true, true, true); 899 900 for (i = 0; i < sc->sc_nqueues; i++) { 901 rxr = &sc->rx_rings[i]; 902 igc_rxfill(rxr); 903 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 904 printf("%s: Unable to fill any rx descriptors\n", 905 DEVNAME(sc)); 906 igc_stop(sc); 907 splx(s); 908 } 909 IGC_WRITE_REG(&sc->hw, IGC_RDT(i), 910 (rxr->last_desc_filled + 1) % sc->num_rx_desc); 911 } 912 913 igc_enable_intr(sc); 914 915 ifp->if_flags |= IFF_RUNNING; 916 for (i = 0; i < sc->sc_nqueues; i++) 917 ifq_clr_oactive(ifp->if_ifqs[i]); 918 919 splx(s); 920 } 921 922 static inline int 923 igc_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m) 924 { 925 int error; 926 927 error = bus_dmamap_load_mbuf(dmat, map, m, 928 BUS_DMA_STREAMING | BUS_DMA_NOWAIT); 929 if (error != EFBIG) 930 return (error); 931 932 error = m_defrag(m, M_DONTWAIT); 933 if (error != 0) 934 return (error); 935 936 return (bus_dmamap_load_mbuf(dmat, map, m, 937 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)); 938 } 939 940 void 941 igc_start(struct ifqueue *ifq) 942 { 943 struct ifnet *ifp = ifq->ifq_if; 944 struct igc_softc *sc = ifp->if_softc; 945 struct tx_ring *txr = ifq->ifq_softc; 946 union igc_adv_tx_desc *txdesc; 947 struct igc_tx_buf *txbuf; 948 bus_dmamap_t map; 949 struct mbuf *m; 950 unsigned int prod, free, last, i; 951 unsigned int mask; 952 uint32_t cmd_type_len; 953 uint32_t olinfo_status; 954 int post = 0; 955 #if NBPFILTER > 0 956 caddr_t if_bpf; 957 #endif 958 959 if (!sc->link_active) { 960 ifq_purge(ifq); 961 return; 962 } 963 964 prod = txr->next_avail_desc; 965 free = txr->next_to_clean; 966 if (free <= prod) 967 free += sc->num_tx_desc; 968 free -= prod; 969 970 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 971 txr->txdma.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 972 973 mask = sc->num_tx_desc - 1; 974 975 for (;;) { 976 if (free <= IGC_MAX_SCATTER) { 977 ifq_set_oactive(ifq); 978 break; 979 } 980 981 m = ifq_dequeue(ifq); 982 if (m == NULL) 983 break; 984 985 txbuf = &txr->tx_buffers[prod]; 986 map = txbuf->map; 987 988 if (igc_load_mbuf(txr->txdma.dma_tag, map, m) != 0) { 989 ifq->ifq_errors++; 990 m_freem(m); 991 continue; 992 } 993 994 olinfo_status = m->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 995 996 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 997 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 998 999 for (i = 0; i < map->dm_nsegs; i++) { 1000 txdesc = &txr->tx_base[prod]; 1001 1002 cmd_type_len = IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1003 IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1004 if (i == map->dm_nsegs - 1) 1005 cmd_type_len |= IGC_ADVTXD_DCMD_EOP | 1006 IGC_ADVTXD_DCMD_RS; 1007 1008 htolem64(&txdesc->read.buffer_addr, map->dm_segs[i].ds_addr); 1009 htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1010 htolem32(&txdesc->read.olinfo_status, olinfo_status); 1011 1012 last = prod; 1013 1014 prod++; 1015 prod &= mask; 1016 } 1017 1018 txbuf->m_head = m; 1019 txbuf->eop_index = last; 1020 1021 #if NBPFILTER > 0 1022 if_bpf = ifp->if_bpf; 1023 if (if_bpf) 1024 bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_OUT); 1025 #endif 1026 1027 free -= i; 1028 post = 1; 1029 } 1030 1031 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 1032 txr->txdma.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1033 1034 if (post) { 1035 txr->next_avail_desc = prod; 1036 IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1037 } 1038 } 1039 1040 int 1041 igc_txeof(struct tx_ring *txr) 1042 { 1043 struct igc_softc *sc = txr->sc; 1044 struct ifqueue *ifq = txr->ifq; 1045 union igc_adv_tx_desc *txdesc; 1046 struct igc_tx_buf *txbuf; 1047 bus_dmamap_t map; 1048 unsigned int cons, prod, last; 1049 unsigned int mask; 1050 int done = 0; 1051 1052 prod = txr->next_avail_desc; 1053 cons = txr->next_to_clean; 1054 1055 if (cons == prod) 1056 return (0); 1057 1058 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 1059 txr->txdma.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1060 1061 mask = sc->num_tx_desc - 1; 1062 1063 do { 1064 txbuf = &txr->tx_buffers[cons]; 1065 last = txbuf->eop_index; 1066 txdesc = &txr->tx_base[last]; 1067 1068 if (!(txdesc->wb.status & htole32(IGC_TXD_STAT_DD))) 1069 break; 1070 1071 map = txbuf->map; 1072 1073 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1074 BUS_DMASYNC_POSTWRITE); 1075 bus_dmamap_unload(txr->txdma.dma_tag, map); 1076 m_freem(txbuf->m_head); 1077 1078 txbuf->m_head = NULL; 1079 txbuf->eop_index = -1; 1080 1081 cons = last + 1; 1082 cons &= mask; 1083 1084 done = 1; 1085 } while (cons != prod); 1086 1087 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 1088 txr->txdma.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 1089 1090 txr->next_to_clean = cons; 1091 1092 if (ifq_is_oactive(ifq)) 1093 ifq_restart(ifq); 1094 1095 return (done); 1096 } 1097 1098 /********************************************************************* 1099 * 1100 * This routine disables all traffic on the adapter by issuing a 1101 * global reset on the MAC. 1102 * 1103 **********************************************************************/ 1104 void 1105 igc_stop(struct igc_softc *sc) 1106 { 1107 struct ifnet *ifp = &sc->sc_ac.ac_if; 1108 int i; 1109 1110 /* Tell the stack that the interface is no longer active. */ 1111 ifp->if_flags &= ~IFF_RUNNING; 1112 1113 igc_disable_intr(sc); 1114 1115 igc_reset_hw(&sc->hw); 1116 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1117 1118 intr_barrier(sc->tag); 1119 for (i = 0; i < sc->sc_nqueues; i++) { 1120 struct ifqueue *ifq = ifp->if_ifqs[i]; 1121 ifq_barrier(ifq); 1122 ifq_clr_oactive(ifq); 1123 1124 if (sc->queues[i].tag != NULL) 1125 intr_barrier(sc->queues[i].tag); 1126 timeout_del(&sc->rx_rings[i].rx_refill); 1127 } 1128 1129 igc_free_transmit_structures(sc); 1130 igc_free_receive_structures(sc); 1131 1132 igc_update_link_status(sc); 1133 } 1134 1135 /********************************************************************* 1136 * Ioctl entry point 1137 * 1138 * igc_ioctl is called when the user wants to configure the 1139 * interface. 1140 * 1141 * return 0 on success, positive on failure 1142 **********************************************************************/ 1143 int 1144 igc_ioctl(struct ifnet * ifp, u_long cmd, caddr_t data) 1145 { 1146 struct igc_softc *sc = ifp->if_softc; 1147 struct ifreq *ifr = (struct ifreq *)data; 1148 int s, error = 0; 1149 1150 s = splnet(); 1151 1152 switch (cmd) { 1153 case SIOCSIFADDR: 1154 ifp->if_flags |= IFF_UP; 1155 if (!(ifp->if_flags & IFF_RUNNING)) 1156 igc_init(sc); 1157 break; 1158 case SIOCSIFFLAGS: 1159 if (ifp->if_flags & IFF_UP) { 1160 if (ifp->if_flags & IFF_RUNNING) 1161 error = ENETRESET; 1162 else 1163 igc_init(sc); 1164 } else { 1165 if (ifp->if_flags & IFF_RUNNING) 1166 igc_stop(sc); 1167 } 1168 break; 1169 case SIOCSIFMEDIA: 1170 case SIOCGIFMEDIA: 1171 error = ifmedia_ioctl(ifp, ifr, &sc->media, cmd); 1172 break; 1173 case SIOCGIFRXR: 1174 error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1175 break; 1176 default: 1177 error = ether_ioctl(ifp, &sc->sc_ac, cmd, data); 1178 } 1179 1180 if (error == ENETRESET) { 1181 if (ifp->if_flags & IFF_RUNNING) { 1182 igc_disable_intr(sc); 1183 igc_iff(sc); 1184 igc_enable_intr(sc); 1185 } 1186 error = 0; 1187 } 1188 1189 splx(s); 1190 return error; 1191 } 1192 1193 int 1194 igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 1195 { 1196 struct if_rxring_info *ifr, ifr1; 1197 struct rx_ring *rxr; 1198 int error, i, n = 0; 1199 1200 if (sc->sc_nqueues > 1) { 1201 if ((ifr = mallocarray(sc->sc_nqueues, sizeof(*ifr), M_DEVBUF, 1202 M_WAITOK | M_ZERO)) == NULL) 1203 return ENOMEM; 1204 } else 1205 ifr = &ifr1; 1206 1207 for (i = 0; i < sc->sc_nqueues; i++) { 1208 rxr = &sc->rx_rings[i]; 1209 ifr[n].ifr_size = MCLBYTES; 1210 snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i); 1211 ifr[n].ifr_info = rxr->rx_ring; 1212 n++; 1213 } 1214 1215 error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 1216 if (sc->sc_nqueues > 1) 1217 free(ifr, M_DEVBUF, sc->sc_nqueues * sizeof(*ifr)); 1218 1219 return error; 1220 } 1221 1222 int 1223 igc_rxfill(struct rx_ring *rxr) 1224 { 1225 struct igc_softc *sc = rxr->sc; 1226 int i, post = 0; 1227 u_int slots; 1228 1229 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 1230 rxr->rxdma.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1231 1232 i = rxr->last_desc_filled; 1233 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); slots > 0; 1234 slots--) { 1235 if (++i == sc->num_rx_desc) 1236 i = 0; 1237 1238 if (igc_get_buf(rxr, i) != 0) 1239 break; 1240 1241 rxr->last_desc_filled = i; 1242 post = 1; 1243 } 1244 1245 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 1246 rxr->rxdma.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1247 1248 if_rxr_put(&rxr->rx_ring, slots); 1249 1250 return post; 1251 } 1252 1253 void 1254 igc_rxrefill(void *xrxr) 1255 { 1256 struct rx_ring *rxr = xrxr; 1257 struct igc_softc *sc = rxr->sc; 1258 1259 if (igc_rxfill(rxr)) { 1260 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), 1261 (rxr->last_desc_filled + 1) % sc->num_rx_desc); 1262 } 1263 else if (if_rxr_inuse(&rxr->rx_ring) == 0) 1264 timeout_add(&rxr->rx_refill, 1); 1265 } 1266 1267 /********************************************************************* 1268 * 1269 * This routine executes in interrupt context. It replenishes 1270 * the mbufs in the descriptor and sends data which has been 1271 * dma'ed into host memory to upper layer. 1272 * 1273 *********************************************************************/ 1274 int 1275 igc_rxeof(struct rx_ring *rxr) 1276 { 1277 struct igc_softc *sc = rxr->sc; 1278 struct ifnet *ifp = &sc->sc_ac.ac_if; 1279 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 1280 struct mbuf *mp, *m; 1281 struct igc_rx_buf *rxbuf, *nxbuf; 1282 union igc_adv_rx_desc *rxdesc; 1283 uint32_t ptype, staterr = 0; 1284 uint16_t len, vtag; 1285 uint8_t eop = 0; 1286 int i, nextp; 1287 1288 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 1289 return 0; 1290 1291 i = rxr->next_to_check; 1292 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 1293 uint32_t hash; 1294 uint16_t hashtype; 1295 1296 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 1297 i * sizeof(union igc_adv_rx_desc), 1298 sizeof(union igc_adv_rx_desc), BUS_DMASYNC_POSTREAD); 1299 1300 rxdesc = &rxr->rx_base[i]; 1301 staterr = letoh32(rxdesc->wb.upper.status_error); 1302 if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 1303 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 1304 i * sizeof(union igc_adv_rx_desc), 1305 sizeof(union igc_adv_rx_desc), BUS_DMASYNC_PREREAD); 1306 break; 1307 } 1308 1309 /* Zero out the receive descriptors status. */ 1310 rxdesc->wb.upper.status_error = 0; 1311 rxbuf = &rxr->rx_buffers[i]; 1312 1313 /* Pull the mbuf off the ring. */ 1314 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 1315 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1316 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 1317 1318 mp = rxbuf->buf; 1319 len = letoh16(rxdesc->wb.upper.length); 1320 vtag = letoh16(rxdesc->wb.upper.vlan); 1321 eop = ((staterr & IGC_RXD_STAT_EOP) == IGC_RXD_STAT_EOP); 1322 ptype = letoh32(rxdesc->wb.lower.lo_dword.data) & 1323 IGC_PKTTYPE_MASK; 1324 hash = letoh32(rxdesc->wb.lower.hi_dword.rss); 1325 hashtype = le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 1326 IGC_RXDADV_RSSTYPE_MASK; 1327 1328 if (staterr & IGC_RXDEXT_STATERR_RXE) { 1329 if (rxbuf->fmp) { 1330 m_freem(rxbuf->fmp); 1331 rxbuf->fmp = NULL; 1332 } 1333 1334 m_freem(mp); 1335 rxbuf->buf = NULL; 1336 goto next_desc; 1337 } 1338 1339 if (mp == NULL) { 1340 panic("%s: igc_rxeof: NULL mbuf in slot %d " 1341 "(nrx %d, filled %d)", DEVNAME(sc), i, 1342 if_rxr_inuse(&rxr->rx_ring), rxr->last_desc_filled); 1343 } 1344 1345 if (!eop) { 1346 /* 1347 * Figure out the next descriptor of this frame. 1348 */ 1349 nextp = i + 1; 1350 if (nextp == sc->num_rx_desc) 1351 nextp = 0; 1352 nxbuf = &rxr->rx_buffers[nextp]; 1353 /* prefetch(nxbuf); */ 1354 } 1355 1356 mp->m_len = len; 1357 1358 m = rxbuf->fmp; 1359 rxbuf->buf = rxbuf->fmp = NULL; 1360 1361 if (m != NULL) 1362 m->m_pkthdr.len += mp->m_len; 1363 else { 1364 m = mp; 1365 m->m_pkthdr.len = mp->m_len; 1366 #if NVLAN > 0 1367 if (staterr & IGC_RXD_STAT_VP) { 1368 m->m_pkthdr.ether_vtag = vtag; 1369 m->m_flags |= M_VLANTAG; 1370 } 1371 #endif 1372 } 1373 1374 /* Pass the head pointer on */ 1375 if (eop == 0) { 1376 nxbuf->fmp = m; 1377 m = NULL; 1378 mp->m_next = nxbuf->buf; 1379 } else { 1380 igc_rx_checksum(staterr, m, ptype); 1381 1382 if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 1383 m->m_pkthdr.ph_flowid = hash; 1384 SET(m->m_pkthdr.csum_flags, M_FLOWID); 1385 } 1386 1387 ml_enqueue(&ml, m); 1388 } 1389 next_desc: 1390 if_rxr_put(&rxr->rx_ring, 1); 1391 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 1392 i * sizeof(union igc_adv_rx_desc), 1393 sizeof(union igc_adv_rx_desc), BUS_DMASYNC_PREREAD); 1394 1395 /* Advance our pointers to the next descriptor. */ 1396 if (++i == sc->num_rx_desc) 1397 i = 0; 1398 } 1399 rxr->next_to_check = i; 1400 1401 if (ifiq_input(rxr->ifiq, &ml)) 1402 if_rxr_livelocked(&rxr->rx_ring); 1403 1404 if (!(staterr & IGC_RXD_STAT_DD)) 1405 return 0; 1406 1407 return 1; 1408 } 1409 1410 /********************************************************************* 1411 * 1412 * Verify that the hardware indicated that the checksum is valid. 1413 * Inform the stack about the status of checksum so that stack 1414 * doesn't spend time verifying the checksum. 1415 * 1416 *********************************************************************/ 1417 void 1418 igc_rx_checksum(uint32_t staterr, struct mbuf *m, uint32_t ptype) 1419 { 1420 uint16_t status = (uint16_t)staterr; 1421 uint8_t errors = (uint8_t)(staterr >> 24); 1422 1423 if (status & IGC_RXD_STAT_IPCS) { 1424 if (!(errors & IGC_RXD_ERR_IPE)) { 1425 /* IP Checksum Good */ 1426 m->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 1427 } else 1428 m->m_pkthdr.csum_flags = 0; 1429 } 1430 1431 if (status & (IGC_RXD_STAT_TCPCS | IGC_RXD_STAT_UDPCS)) { 1432 if (!(errors & IGC_RXD_ERR_TCPE)) 1433 m->m_pkthdr.csum_flags |= 1434 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 1435 } 1436 } 1437 1438 void 1439 igc_watchdog(struct ifnet * ifp) 1440 { 1441 } 1442 1443 /********************************************************************* 1444 * 1445 * Media Ioctl callback 1446 * 1447 * This routine is called whenever the user queries the status of 1448 * the interface using ifconfig. 1449 * 1450 **********************************************************************/ 1451 void 1452 igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1453 { 1454 struct igc_softc *sc = ifp->if_softc; 1455 1456 igc_update_link_status(sc); 1457 1458 ifmr->ifm_status = IFM_AVALID; 1459 ifmr->ifm_active = IFM_ETHER; 1460 1461 if (!sc->link_active) { 1462 ifmr->ifm_active |= IFM_NONE; 1463 return; 1464 } 1465 1466 ifmr->ifm_status |= IFM_ACTIVE; 1467 1468 switch (sc->link_speed) { 1469 case 10: 1470 ifmr->ifm_active |= IFM_10_T; 1471 break; 1472 case 100: 1473 ifmr->ifm_active |= IFM_100_TX; 1474 break; 1475 case 1000: 1476 ifmr->ifm_active |= IFM_1000_T; 1477 break; 1478 case 2500: 1479 ifmr->ifm_active |= IFM_2500_T; 1480 break; 1481 } 1482 1483 if (sc->link_duplex == FULL_DUPLEX) 1484 ifmr->ifm_active |= IFM_FDX; 1485 else 1486 ifmr->ifm_active |= IFM_HDX; 1487 } 1488 1489 /********************************************************************* 1490 * 1491 * Media Ioctl callback 1492 * 1493 * This routine is called when the user changes speed/duplex using 1494 * media/mediopt option with ifconfig. 1495 * 1496 **********************************************************************/ 1497 int 1498 igc_media_change(struct ifnet *ifp) 1499 { 1500 struct igc_softc *sc = ifp->if_softc; 1501 struct ifmedia *ifm = &sc->media; 1502 1503 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1504 return (EINVAL); 1505 1506 sc->hw.mac.autoneg = DO_AUTO_NEG; 1507 1508 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1509 case IFM_AUTO: 1510 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1511 break; 1512 case IFM_2500_T: 1513 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 1514 break; 1515 case IFM_1000_T: 1516 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1517 break; 1518 case IFM_100_TX: 1519 if ((ifm->ifm_media & IFM_GMASK) == IFM_HDX) 1520 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 1521 else 1522 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 1523 break; 1524 case IFM_10_T: 1525 if ((ifm->ifm_media & IFM_GMASK) == IFM_HDX) 1526 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 1527 else 1528 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 1529 break; 1530 default: 1531 return EINVAL; 1532 } 1533 1534 igc_init(sc); 1535 1536 return 0; 1537 } 1538 1539 void 1540 igc_iff(struct igc_softc *sc) 1541 { 1542 struct ifnet *ifp = &sc->sc_ac.ac_if; 1543 struct arpcom *ac = &sc->sc_ac; 1544 struct ether_multi *enm; 1545 struct ether_multistep step; 1546 uint32_t reg_rctl = 0; 1547 uint8_t *mta; 1548 int mcnt = 0; 1549 1550 mta = sc->mta; 1551 bzero(mta, sizeof(uint8_t) * ETHER_ADDR_LEN * 1552 MAX_NUM_MULTICAST_ADDRESSES); 1553 1554 reg_rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 1555 reg_rctl &= ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 1556 ifp->if_flags &= ~IFF_ALLMULTI; 1557 1558 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1559 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1560 ifp->if_flags |= IFF_ALLMULTI; 1561 reg_rctl |= IGC_RCTL_MPE; 1562 if (ifp->if_flags & IFF_PROMISC) 1563 reg_rctl |= IGC_RCTL_UPE; 1564 } else { 1565 ETHER_FIRST_MULTI(step, ac, enm); 1566 while (enm != NULL) { 1567 bcopy(enm->enm_addrlo, 1568 &mta[mcnt * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 1569 mcnt++; 1570 1571 ETHER_NEXT_MULTI(step, enm); 1572 } 1573 1574 igc_update_mc_addr_list(&sc->hw, mta, mcnt); 1575 } 1576 1577 IGC_WRITE_REG(&sc->hw, IGC_RCTL, reg_rctl); 1578 } 1579 1580 void 1581 igc_update_link_status(struct igc_softc *sc) 1582 { 1583 struct ifnet *ifp = &sc->sc_ac.ac_if; 1584 struct igc_hw *hw = &sc->hw; 1585 int link_state; 1586 1587 if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 1588 if (sc->link_active == 0) { 1589 igc_get_speed_and_duplex(hw, &sc->link_speed, 1590 &sc->link_duplex); 1591 sc->link_active = 1; 1592 ifp->if_baudrate = IF_Mbps(sc->link_speed); 1593 } 1594 link_state = (sc->link_duplex == FULL_DUPLEX) ? 1595 LINK_STATE_FULL_DUPLEX : LINK_STATE_HALF_DUPLEX; 1596 } else { 1597 if (sc->link_active == 1) { 1598 ifp->if_baudrate = sc->link_speed = 0; 1599 sc->link_duplex = 0; 1600 sc->link_active = 0; 1601 } 1602 link_state = LINK_STATE_DOWN; 1603 } 1604 if (ifp->if_link_state != link_state) { 1605 ifp->if_link_state = link_state; 1606 if_link_state_change(ifp); 1607 } 1608 } 1609 1610 /********************************************************************* 1611 * 1612 * Get a buffer from system mbuf buffer pool. 1613 * 1614 **********************************************************************/ 1615 int 1616 igc_get_buf(struct rx_ring *rxr, int i) 1617 { 1618 struct igc_softc *sc = rxr->sc; 1619 struct igc_rx_buf *rxbuf; 1620 struct mbuf *m; 1621 union igc_adv_rx_desc *rxdesc; 1622 int error; 1623 1624 rxbuf = &rxr->rx_buffers[i]; 1625 rxdesc = &rxr->rx_base[i]; 1626 if (rxbuf->buf) { 1627 printf("%s: slot %d already has an mbuf\n", DEVNAME(sc), i); 1628 return ENOBUFS; 1629 } 1630 1631 m = MCLGETL(NULL, M_DONTWAIT, sc->rx_mbuf_sz); 1632 if (!m) 1633 return ENOBUFS; 1634 1635 m->m_data += (m->m_ext.ext_size - sc->rx_mbuf_sz); 1636 m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 1637 1638 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, m, 1639 BUS_DMA_NOWAIT); 1640 if (error) { 1641 m_freem(m); 1642 return error; 1643 } 1644 1645 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 1646 rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 1647 rxbuf->buf = m; 1648 1649 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 1650 1651 return 0; 1652 } 1653 1654 void 1655 igc_configure_queues(struct igc_softc *sc) 1656 { 1657 struct igc_hw *hw = &sc->hw; 1658 struct igc_queue *iq = sc->queues; 1659 uint32_t ivar, newitr = 0; 1660 int i; 1661 1662 /* First turn on RSS capability */ 1663 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 1664 IGC_GPIE_PBA | IGC_GPIE_NSICR); 1665 1666 /* Set the starting interrupt rate */ 1667 newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 1668 1669 newitr |= IGC_EITR_CNT_IGNR; 1670 1671 /* Turn on MSI-X */ 1672 for (i = 0; i < sc->sc_nqueues; i++, iq++) { 1673 /* RX entries */ 1674 igc_set_queues(sc, i, iq->msix, 0); 1675 /* TX entries */ 1676 igc_set_queues(sc, i, iq->msix, 1); 1677 sc->msix_queuesmask |= iq->eims; 1678 IGC_WRITE_REG(hw, IGC_EITR(iq->msix), newitr); 1679 } 1680 1681 /* And for the link interrupt */ 1682 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 1683 sc->msix_linkmask = 1 << sc->linkvec; 1684 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 1685 } 1686 1687 void 1688 igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 1689 { 1690 struct igc_hw *hw = &sc->hw; 1691 uint32_t ivar, index; 1692 1693 index = entry >> 1; 1694 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 1695 if (type) { 1696 if (entry & 1) { 1697 ivar &= 0x00FFFFFF; 1698 ivar |= (vector | IGC_IVAR_VALID) << 24; 1699 } else { 1700 ivar &= 0xFFFF00FF; 1701 ivar |= (vector | IGC_IVAR_VALID) << 8; 1702 } 1703 } else { 1704 if (entry & 1) { 1705 ivar &= 0xFF00FFFF; 1706 ivar |= (vector | IGC_IVAR_VALID) << 16; 1707 } else { 1708 ivar &= 0xFFFFFF00; 1709 ivar |= vector | IGC_IVAR_VALID; 1710 } 1711 } 1712 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 1713 } 1714 1715 void 1716 igc_enable_queue(struct igc_softc *sc, uint32_t eims) 1717 { 1718 IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 1719 } 1720 1721 void 1722 igc_enable_intr(struct igc_softc *sc) 1723 { 1724 struct igc_hw *hw = &sc->hw; 1725 uint32_t mask; 1726 1727 mask = (sc->msix_queuesmask | sc->msix_linkmask); 1728 IGC_WRITE_REG(hw, IGC_EIAC, mask); 1729 IGC_WRITE_REG(hw, IGC_EIAM, mask); 1730 IGC_WRITE_REG(hw, IGC_EIMS, mask); 1731 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 1732 IGC_WRITE_FLUSH(hw); 1733 } 1734 1735 void 1736 igc_disable_intr(struct igc_softc *sc) 1737 { 1738 struct igc_hw *hw = &sc->hw; 1739 1740 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 1741 IGC_WRITE_REG(hw, IGC_EIAC, 0); 1742 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 1743 IGC_WRITE_FLUSH(hw); 1744 } 1745 1746 int 1747 igc_intr_link(void *arg) 1748 { 1749 struct igc_softc *sc = (struct igc_softc *)arg; 1750 uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 1751 1752 if (reg_icr & IGC_ICR_LSC) { 1753 KERNEL_LOCK(); 1754 sc->hw.mac.get_link_status = true; 1755 igc_update_link_status(sc); 1756 KERNEL_UNLOCK(); 1757 } 1758 1759 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 1760 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 1761 1762 return 1; 1763 } 1764 1765 int 1766 igc_intr_queue(void *arg) 1767 { 1768 struct igc_queue *iq = arg; 1769 struct igc_softc *sc = iq->sc; 1770 struct ifnet *ifp = &sc->sc_ac.ac_if; 1771 struct rx_ring *rxr = iq->rxr; 1772 struct tx_ring *txr = iq->txr; 1773 1774 if (ifp->if_flags & IFF_RUNNING) { 1775 igc_txeof(txr); 1776 igc_rxeof(rxr); 1777 igc_rxrefill(rxr); 1778 } 1779 1780 igc_enable_queue(sc, iq->eims); 1781 1782 return 1; 1783 } 1784 1785 /********************************************************************* 1786 * 1787 * Allocate memory for tx_buffer structures. The tx_buffer stores all 1788 * the information needed to transmit a packet on the wire. 1789 * 1790 **********************************************************************/ 1791 int 1792 igc_allocate_transmit_buffers(struct tx_ring *txr) 1793 { 1794 struct igc_softc *sc = txr->sc; 1795 struct igc_tx_buf *txbuf; 1796 int error, i; 1797 1798 txr->tx_buffers = mallocarray(sc->num_tx_desc, 1799 sizeof(struct igc_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO); 1800 if (txr->tx_buffers == NULL) { 1801 printf("%s: Unable to allocate tx_buffer memory\n", 1802 DEVNAME(sc)); 1803 error = ENOMEM; 1804 goto fail; 1805 } 1806 txr->txtag = txr->txdma.dma_tag; 1807 1808 /* Create the descriptor buffer dma maps. */ 1809 for (i = 0; i < sc->num_tx_desc; i++) { 1810 txbuf = &txr->tx_buffers[i]; 1811 error = bus_dmamap_create(txr->txdma.dma_tag, IGC_TSO_SIZE, 1812 IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 1813 if (error != 0) { 1814 printf("%s: Unable to create TX DMA map\n", 1815 DEVNAME(sc)); 1816 goto fail; 1817 } 1818 } 1819 1820 return 0; 1821 fail: 1822 return error; 1823 } 1824 1825 1826 /********************************************************************* 1827 * 1828 * Allocate and initialize transmit structures. 1829 * 1830 **********************************************************************/ 1831 int 1832 igc_setup_transmit_structures(struct igc_softc *sc) 1833 { 1834 struct tx_ring *txr = sc->tx_rings; 1835 int i; 1836 1837 for (i = 0; i < sc->sc_nqueues; i++, txr++) { 1838 if (igc_setup_transmit_ring(txr)) 1839 goto fail; 1840 } 1841 1842 return 0; 1843 fail: 1844 igc_free_transmit_structures(sc); 1845 return ENOBUFS; 1846 } 1847 1848 /********************************************************************* 1849 * 1850 * Initialize a transmit ring. 1851 * 1852 **********************************************************************/ 1853 int 1854 igc_setup_transmit_ring(struct tx_ring *txr) 1855 { 1856 struct igc_softc *sc = txr->sc; 1857 1858 /* Now allocate transmit buffers for the ring. */ 1859 if (igc_allocate_transmit_buffers(txr)) 1860 return ENOMEM; 1861 1862 /* Clear the old ring contents */ 1863 bzero((void *)txr->tx_base, 1864 (sizeof(union igc_adv_tx_desc)) * sc->num_tx_desc); 1865 1866 /* Reset indices. */ 1867 txr->next_avail_desc = 0; 1868 txr->next_to_clean = 0; 1869 1870 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 1871 txr->txdma.dma_map->dm_mapsize, 1872 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1873 1874 return 0; 1875 } 1876 1877 /********************************************************************* 1878 * 1879 * Enable transmit unit. 1880 * 1881 **********************************************************************/ 1882 void 1883 igc_initialize_transmit_unit(struct igc_softc *sc) 1884 { 1885 struct ifnet *ifp = &sc->sc_ac.ac_if; 1886 struct tx_ring *txr; 1887 struct igc_hw *hw = &sc->hw; 1888 uint64_t bus_addr; 1889 uint32_t tctl, txdctl = 0; 1890 int i; 1891 1892 /* Setup the Base and Length of the TX descriptor ring. */ 1893 for (i = 0; i < sc->sc_nqueues; i++) { 1894 txr = &sc->tx_rings[i]; 1895 1896 bus_addr = txr->txdma.dma_map->dm_segs[0].ds_addr; 1897 1898 /* Base and len of TX ring */ 1899 IGC_WRITE_REG(hw, IGC_TDLEN(i), 1900 sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 1901 IGC_WRITE_REG(hw, IGC_TDBAH(i), (uint32_t)(bus_addr >> 32)); 1902 IGC_WRITE_REG(hw, IGC_TDBAL(i), (uint32_t)bus_addr); 1903 1904 /* Init the HEAD/TAIL indices */ 1905 IGC_WRITE_REG(hw, IGC_TDT(i), 0); 1906 IGC_WRITE_REG(hw, IGC_TDH(i), 0); 1907 1908 txr->watchdog_timer = 0; 1909 1910 txdctl = 0; /* Clear txdctl */ 1911 txdctl |= 0x1f; /* PTHRESH */ 1912 txdctl |= 1 << 8; /* HTHRESH */ 1913 txdctl |= 1 << 16; /* WTHRESH */ 1914 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 1915 txdctl |= IGC_TXDCTL_GRAN; 1916 txdctl |= 1 << 25; /* LWTHRESH */ 1917 1918 IGC_WRITE_REG(hw, IGC_TXDCTL(i), txdctl); 1919 } 1920 ifp->if_timer = 0; 1921 1922 /* Program the Transmit Control Register */ 1923 tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 1924 tctl &= ~IGC_TCTL_CT; 1925 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 1926 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 1927 1928 /* This write will effectively turn on the transmit unit. */ 1929 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 1930 } 1931 1932 /********************************************************************* 1933 * 1934 * Free all transmit rings. 1935 * 1936 **********************************************************************/ 1937 void 1938 igc_free_transmit_structures(struct igc_softc *sc) 1939 { 1940 struct tx_ring *txr = sc->tx_rings; 1941 int i; 1942 1943 for (i = 0; i < sc->sc_nqueues; i++, txr++) 1944 igc_free_transmit_buffers(txr); 1945 } 1946 1947 /********************************************************************* 1948 * 1949 * Free transmit ring related data structures. 1950 * 1951 **********************************************************************/ 1952 void 1953 igc_free_transmit_buffers(struct tx_ring *txr) 1954 { 1955 struct igc_softc *sc = txr->sc; 1956 struct igc_tx_buf *txbuf; 1957 int i; 1958 1959 if (txr->tx_buffers == NULL) 1960 return; 1961 1962 txbuf = txr->tx_buffers; 1963 for (i = 0; i < sc->num_tx_desc; i++, txbuf++) { 1964 if (txbuf->map != NULL && txbuf->map->dm_nsegs > 0) { 1965 bus_dmamap_sync(txr->txdma.dma_tag, txbuf->map, 1966 0, txbuf->map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1967 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 1968 } 1969 if (txbuf->m_head != NULL) { 1970 m_freem(txbuf->m_head); 1971 txbuf->m_head = NULL; 1972 } 1973 if (txbuf->map != NULL) { 1974 bus_dmamap_destroy(txr->txdma.dma_tag, txbuf->map); 1975 txbuf->map = NULL; 1976 } 1977 } 1978 1979 if (txr->tx_buffers != NULL) 1980 free(txr->tx_buffers, M_DEVBUF, 1981 sc->num_tx_desc * sizeof(struct igc_tx_buf)); 1982 txr->tx_buffers = NULL; 1983 txr->txtag = NULL; 1984 } 1985 1986 /********************************************************************* 1987 * 1988 * Allocate memory for rx_buffer structures. Since we use one 1989 * rx_buffer per received packet, the maximum number of rx_buffer's 1990 * that we'll need is equal to the number of receive descriptors 1991 * that we've allocated. 1992 * 1993 **********************************************************************/ 1994 int 1995 igc_allocate_receive_buffers(struct rx_ring *rxr) 1996 { 1997 struct igc_softc *sc = rxr->sc; 1998 struct igc_rx_buf *rxbuf; 1999 int i, error; 2000 2001 rxr->rx_buffers = mallocarray(sc->num_rx_desc, 2002 sizeof(struct igc_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO); 2003 if (rxr->rx_buffers == NULL) { 2004 printf("%s: Unable to allocate rx_buffer memory\n", 2005 DEVNAME(sc)); 2006 error = ENOMEM; 2007 goto fail; 2008 } 2009 2010 rxbuf = rxr->rx_buffers; 2011 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2012 error = bus_dmamap_create(rxr->rxdma.dma_tag, 2013 MAX_JUMBO_FRAME_SIZE, 1, MAX_JUMBO_FRAME_SIZE, 0, 2014 BUS_DMA_NOWAIT, &rxbuf->map); 2015 if (error) { 2016 printf("%s: Unable to create RX DMA map\n", 2017 DEVNAME(sc)); 2018 goto fail; 2019 } 2020 } 2021 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2022 rxr->rxdma.dma_map->dm_mapsize, 2023 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2024 2025 return 0; 2026 fail: 2027 return error; 2028 } 2029 2030 /********************************************************************* 2031 * 2032 * Allocate and initialize receive structures. 2033 * 2034 **********************************************************************/ 2035 int 2036 igc_setup_receive_structures(struct igc_softc *sc) 2037 { 2038 struct rx_ring *rxr = sc->rx_rings; 2039 int i; 2040 2041 for (i = 0; i < sc->sc_nqueues; i++, rxr++) { 2042 if (igc_setup_receive_ring(rxr)) 2043 goto fail; 2044 } 2045 2046 return 0; 2047 fail: 2048 igc_free_receive_structures(sc); 2049 return ENOBUFS; 2050 } 2051 2052 /********************************************************************* 2053 * 2054 * Initialize a receive ring and its buffers. 2055 * 2056 **********************************************************************/ 2057 int 2058 igc_setup_receive_ring(struct rx_ring *rxr) 2059 { 2060 struct igc_softc *sc = rxr->sc; 2061 struct ifnet *ifp = &sc->sc_ac.ac_if; 2062 int rsize; 2063 2064 rsize = roundup2(sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 2065 IGC_DBA_ALIGN); 2066 2067 /* Clear the ring contents. */ 2068 bzero((void *)rxr->rx_base, rsize); 2069 2070 if (igc_allocate_receive_buffers(rxr)) 2071 return ENOMEM; 2072 2073 /* Setup our descriptor indices. */ 2074 rxr->next_to_check = 0; 2075 rxr->last_desc_filled = sc->num_rx_desc - 1; 2076 2077 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2078 sc->num_rx_desc - 1); 2079 2080 return 0; 2081 } 2082 2083 /********************************************************************* 2084 * 2085 * Enable receive unit. 2086 * 2087 **********************************************************************/ 2088 void 2089 igc_initialize_receive_unit(struct igc_softc *sc) 2090 { 2091 struct rx_ring *rxr = sc->rx_rings; 2092 struct igc_hw *hw = &sc->hw; 2093 uint32_t rctl, rxcsum, srrctl = 0; 2094 int i; 2095 2096 /* 2097 * Make sure receives are disabled while setting 2098 * up the descriptor ring. 2099 */ 2100 rctl = IGC_READ_REG(hw, IGC_RCTL); 2101 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 2102 2103 /* Setup the Receive Control Register */ 2104 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 2105 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 2106 IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 2107 2108 /* Do not store bad packets */ 2109 rctl &= ~IGC_RCTL_SBP; 2110 2111 /* Enable Long Packet receive */ 2112 if (sc->hw.mac.max_frame_size != ETHER_MAX_LEN) 2113 rctl |= IGC_RCTL_LPE; 2114 2115 /* Strip the CRC */ 2116 rctl |= IGC_RCTL_SECRC; 2117 2118 /* 2119 * Set the interrupt throttling rate. Value is calculated 2120 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 2121 */ 2122 IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 2123 2124 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 2125 rxcsum &= ~IGC_RXCSUM_PCSD; 2126 2127 if (sc->sc_nqueues > 1) 2128 rxcsum |= IGC_RXCSUM_PCSD; 2129 2130 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 2131 2132 if (sc->sc_nqueues > 1) 2133 igc_initialize_rss_mapping(sc); 2134 2135 #if 0 2136 srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 2137 rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 2138 #endif 2139 2140 srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 2141 rctl |= IGC_RCTL_SZ_2048; 2142 2143 /* 2144 * If TX flow control is disabled and there's > 1 queue defined, 2145 * enable DROP. 2146 * 2147 * This drops frames rather than hanging the RX MAC for all queues. 2148 */ 2149 if ((sc->sc_nqueues > 1) && (sc->fc == igc_fc_none || 2150 sc->fc == igc_fc_rx_pause)) { 2151 srrctl |= IGC_SRRCTL_DROP_EN; 2152 } 2153 2154 /* Setup the Base and Length of the RX descriptor rings. */ 2155 for (i = 0; i < sc->sc_nqueues; i++, rxr++) { 2156 IGC_WRITE_REG(hw, IGC_RXDCTL(i), 0); 2157 uint64_t bus_addr = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2158 uint32_t rxdctl; 2159 2160 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 2161 2162 IGC_WRITE_REG(hw, IGC_RDLEN(i), 2163 sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 2164 IGC_WRITE_REG(hw, IGC_RDBAH(i), (uint32_t)(bus_addr >> 32)); 2165 IGC_WRITE_REG(hw, IGC_RDBAL(i), (uint32_t)bus_addr); 2166 IGC_WRITE_REG(hw, IGC_SRRCTL(i), srrctl); 2167 2168 /* Setup the Head and Tail Descriptor Pointers */ 2169 IGC_WRITE_REG(hw, IGC_RDH(i), 0); 2170 IGC_WRITE_REG(hw, IGC_RDT(i), 0); 2171 2172 /* Enable this Queue */ 2173 rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(i)); 2174 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 2175 rxdctl &= 0xFFF00000; 2176 rxdctl |= IGC_RX_PTHRESH; 2177 rxdctl |= IGC_RX_HTHRESH << 8; 2178 rxdctl |= IGC_RX_WTHRESH << 16; 2179 IGC_WRITE_REG(hw, IGC_RXDCTL(i), rxdctl); 2180 } 2181 2182 /* Make sure VLAN Filters are off */ 2183 rctl &= ~IGC_RCTL_VFE; 2184 2185 /* Write out the settings */ 2186 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 2187 } 2188 2189 /********************************************************************* 2190 * 2191 * Free all receive rings. 2192 * 2193 **********************************************************************/ 2194 void 2195 igc_free_receive_structures(struct igc_softc *sc) 2196 { 2197 struct rx_ring *rxr; 2198 int i; 2199 2200 for (i = 0, rxr = sc->rx_rings; i < sc->sc_nqueues; i++, rxr++) 2201 if_rxr_init(&rxr->rx_ring, 0, 0); 2202 2203 for (i = 0, rxr = sc->rx_rings; i < sc->sc_nqueues; i++, rxr++) 2204 igc_free_receive_buffers(rxr); 2205 } 2206 2207 /********************************************************************* 2208 * 2209 * Free receive ring data structures 2210 * 2211 **********************************************************************/ 2212 void 2213 igc_free_receive_buffers(struct rx_ring *rxr) 2214 { 2215 struct igc_softc *sc = rxr->sc; 2216 struct igc_rx_buf *rxbuf; 2217 int i; 2218 2219 if (rxr->rx_buffers != NULL) { 2220 for (i = 0; i < sc->num_rx_desc; i++) { 2221 rxbuf = &rxr->rx_buffers[i]; 2222 if (rxbuf->buf != NULL) { 2223 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2224 0, rxbuf->map->dm_mapsize, 2225 BUS_DMASYNC_POSTREAD); 2226 bus_dmamap_unload(rxr->rxdma.dma_tag, 2227 rxbuf->map); 2228 m_freem(rxbuf->buf); 2229 rxbuf->buf = NULL; 2230 } 2231 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 2232 rxbuf->map = NULL; 2233 } 2234 free(rxr->rx_buffers, M_DEVBUF, 2235 sc->num_rx_desc * sizeof(struct igc_rx_buf)); 2236 rxr->rx_buffers = NULL; 2237 } 2238 } 2239 2240 /* 2241 * Initialise the RSS mapping for NICs that support multiple transmit/ 2242 * receive rings. 2243 */ 2244 void 2245 igc_initialize_rss_mapping(struct igc_softc *sc) 2246 { 2247 struct igc_hw *hw = &sc->hw; 2248 uint32_t rss_key[10], mrqc, reta, shift = 0; 2249 int i, queue_id; 2250 2251 /* 2252 * The redirection table controls which destination 2253 * queue each bucket redirects traffic to. 2254 * Each DWORD represents four queues, with the LSB 2255 * being the first queue in the DWORD. 2256 * 2257 * This just allocates buckets to queues using round-robin 2258 * allocation. 2259 * 2260 * NOTE: It Just Happens to line up with the default 2261 * RSS allocation method. 2262 */ 2263 2264 /* Warning FM follows */ 2265 reta = 0; 2266 for (i = 0; i < 128; i++) { 2267 queue_id = (i % sc->sc_nqueues); 2268 /* Adjust if required */ 2269 queue_id = queue_id << shift; 2270 2271 /* 2272 * The low 8 bits are for hash value (n+0); 2273 * The next 8 bits are for hash value (n+1), etc. 2274 */ 2275 reta = reta >> 8; 2276 reta = reta | ( ((uint32_t) queue_id) << 24); 2277 if ((i & 3) == 3) { 2278 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 2279 reta = 0; 2280 } 2281 } 2282 2283 /* 2284 * MRQC: Multiple Receive Queues Command 2285 * Set queuing to RSS control, number depends on the device. 2286 */ 2287 mrqc = IGC_MRQC_ENABLE_RSS_4Q; 2288 2289 /* Set up random bits */ 2290 stoeplitz_to_key(&rss_key, sizeof(rss_key)); 2291 2292 /* Now fill our hash function seeds */ 2293 for (i = 0; i < 10; i++) 2294 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 2295 2296 /* 2297 * Configure the RSS fields to hash upon. 2298 */ 2299 mrqc |= (IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP); 2300 mrqc |= (IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP); 2301 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 2302 2303 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 2304 } 2305 2306 /* 2307 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 2308 * For ASF and Pass Through versions of f/w this means 2309 * that the driver is loaded. For AMT version type f/w 2310 * this means that the network i/f is open. 2311 */ 2312 void 2313 igc_get_hw_control(struct igc_softc *sc) 2314 { 2315 uint32_t ctrl_ext; 2316 2317 ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 2318 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 2319 } 2320 2321 /* 2322 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 2323 * For ASF and Pass Through versions of f/w this means that 2324 * the driver is no longer loaded. For AMT versions of the 2325 * f/w this means that the network i/f is closed. 2326 */ 2327 void 2328 igc_release_hw_control(struct igc_softc *sc) 2329 { 2330 uint32_t ctrl_ext; 2331 2332 ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 2333 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 2334 } 2335 2336 int 2337 igc_is_valid_ether_addr(uint8_t *addr) 2338 { 2339 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 2340 2341 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 2342 return 0; 2343 } 2344 2345 return 1; 2346 } 2347