1 /*- 2 * Copyright (c) 2008, Pyun YongHyeon <yongari@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD: src/sys/dev/jme/if_jme.c,v 1.2 2008/07/18 04:20:48 yongari Exp $ 28 */ 29 30 #include "opt_ifpoll.h" 31 #include "opt_jme.h" 32 33 #include <sys/param.h> 34 #include <sys/endian.h> 35 #include <sys/kernel.h> 36 #include <sys/bus.h> 37 #include <sys/interrupt.h> 38 #include <sys/malloc.h> 39 #include <sys/proc.h> 40 #include <sys/rman.h> 41 #include <sys/serialize.h> 42 #include <sys/serialize2.h> 43 #include <sys/socket.h> 44 #include <sys/sockio.h> 45 #include <sys/sysctl.h> 46 47 #include <net/ethernet.h> 48 #include <net/if.h> 49 #include <net/bpf.h> 50 #include <net/if_arp.h> 51 #include <net/if_dl.h> 52 #include <net/if_media.h> 53 #include <net/if_poll.h> 54 #include <net/ifq_var.h> 55 #include <net/toeplitz.h> 56 #include <net/toeplitz2.h> 57 #include <net/vlan/if_vlan_var.h> 58 #include <net/vlan/if_vlan_ether.h> 59 60 #include <netinet/ip.h> 61 #include <netinet/tcp.h> 62 63 #include <dev/netif/mii_layer/miivar.h> 64 #include <dev/netif/mii_layer/jmphyreg.h> 65 66 #include <bus/pci/pcireg.h> 67 #include <bus/pci/pcivar.h> 68 #include <bus/pci/pcidevs.h> 69 70 #include <dev/netif/jme/if_jmereg.h> 71 #include <dev/netif/jme/if_jmevar.h> 72 73 #include "miibus_if.h" 74 75 #define JME_TICK_CPUID 0 /* DO NOT CHANGE THIS */ 76 77 #define JME_TX_SERIALIZE 1 78 #define JME_RX_SERIALIZE 2 79 80 #define JME_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP) 81 82 #ifdef JME_RSS_DEBUG 83 #define JME_RSS_DPRINTF(sc, lvl, fmt, ...) \ 84 do { \ 85 if ((sc)->jme_rss_debug >= (lvl)) \ 86 if_printf(&(sc)->arpcom.ac_if, fmt, __VA_ARGS__); \ 87 } while (0) 88 #else /* !JME_RSS_DEBUG */ 89 #define JME_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0) 90 #endif /* JME_RSS_DEBUG */ 91 92 static int jme_probe(device_t); 93 static int jme_attach(device_t); 94 static int jme_detach(device_t); 95 static int jme_shutdown(device_t); 96 static int jme_suspend(device_t); 97 static int jme_resume(device_t); 98 99 static int jme_miibus_readreg(device_t, int, int); 100 static int jme_miibus_writereg(device_t, int, int, int); 101 static void jme_miibus_statchg(device_t); 102 103 static void jme_init(void *); 104 static int jme_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 105 static void jme_start(struct ifnet *); 106 static void jme_watchdog(struct ifnet *); 107 static void jme_mediastatus(struct ifnet *, struct ifmediareq *); 108 static int jme_mediachange(struct ifnet *); 109 #ifdef IFPOLL_ENABLE 110 static void jme_npoll(struct ifnet *, struct ifpoll_info *); 111 static void jme_npoll_status(struct ifnet *); 112 static void jme_npoll_rx(struct ifnet *, void *, int); 113 static void jme_npoll_tx(struct ifnet *, void *, int); 114 #endif 115 static void jme_serialize(struct ifnet *, enum ifnet_serialize); 116 static void jme_deserialize(struct ifnet *, enum ifnet_serialize); 117 static int jme_tryserialize(struct ifnet *, enum ifnet_serialize); 118 #ifdef INVARIANTS 119 static void jme_serialize_assert(struct ifnet *, enum ifnet_serialize, 120 boolean_t); 121 #endif 122 123 static void jme_intr(void *); 124 static void jme_msix_tx(void *); 125 static void jme_msix_rx(void *); 126 static void jme_msix_status(void *); 127 static void jme_txeof(struct jme_txdata *); 128 static void jme_rxeof(struct jme_rxdata *, int); 129 static void jme_rx_intr(struct jme_softc *, uint32_t); 130 static void jme_enable_intr(struct jme_softc *); 131 static void jme_disable_intr(struct jme_softc *); 132 static void jme_rx_restart(struct jme_softc *, uint32_t); 133 134 static int jme_msix_setup(device_t); 135 static void jme_msix_teardown(device_t, int); 136 static int jme_intr_setup(device_t); 137 static void jme_intr_teardown(device_t); 138 static void jme_msix_try_alloc(device_t); 139 static void jme_msix_free(device_t); 140 static int jme_intr_alloc(device_t); 141 static void jme_intr_free(device_t); 142 static int jme_dma_alloc(struct jme_softc *); 143 static void jme_dma_free(struct jme_softc *); 144 static int jme_init_rx_ring(struct jme_rxdata *); 145 static void jme_init_tx_ring(struct jme_txdata *); 146 static void jme_init_ssb(struct jme_softc *); 147 static int jme_newbuf(struct jme_rxdata *, struct jme_rxdesc *, int); 148 static int jme_encap(struct jme_txdata *, struct mbuf **); 149 static void jme_rxpkt(struct jme_rxdata *); 150 static int jme_rxring_dma_alloc(struct jme_rxdata *); 151 static int jme_rxbuf_dma_alloc(struct jme_rxdata *); 152 static int jme_rxbuf_dma_filter(void *, bus_addr_t); 153 154 static void jme_tick(void *); 155 static void jme_stop(struct jme_softc *); 156 static void jme_reset(struct jme_softc *); 157 static void jme_set_msinum(struct jme_softc *); 158 static void jme_set_vlan(struct jme_softc *); 159 static void jme_set_filter(struct jme_softc *); 160 static void jme_stop_tx(struct jme_softc *); 161 static void jme_stop_rx(struct jme_softc *); 162 static void jme_mac_config(struct jme_softc *); 163 static void jme_reg_macaddr(struct jme_softc *, uint8_t[]); 164 static int jme_eeprom_macaddr(struct jme_softc *, uint8_t[]); 165 static int jme_eeprom_read_byte(struct jme_softc *, uint8_t, uint8_t *); 166 #ifdef notyet 167 static void jme_setwol(struct jme_softc *); 168 static void jme_setlinkspeed(struct jme_softc *); 169 #endif 170 static void jme_set_tx_coal(struct jme_softc *); 171 static void jme_set_rx_coal(struct jme_softc *); 172 static void jme_enable_rss(struct jme_softc *); 173 static void jme_disable_rss(struct jme_softc *); 174 static void jme_serialize_skipmain(struct jme_softc *); 175 static void jme_deserialize_skipmain(struct jme_softc *); 176 177 static void jme_sysctl_node(struct jme_softc *); 178 static int jme_sysctl_tx_coal_to(SYSCTL_HANDLER_ARGS); 179 static int jme_sysctl_tx_coal_pkt(SYSCTL_HANDLER_ARGS); 180 static int jme_sysctl_rx_coal_to(SYSCTL_HANDLER_ARGS); 181 static int jme_sysctl_rx_coal_pkt(SYSCTL_HANDLER_ARGS); 182 #ifdef IFPOLL_ENABLE 183 static int jme_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS); 184 static int jme_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS); 185 #endif 186 187 /* 188 * Devices supported by this driver. 189 */ 190 static const struct jme_dev { 191 uint16_t jme_vendorid; 192 uint16_t jme_deviceid; 193 uint32_t jme_caps; 194 const char *jme_name; 195 } jme_devs[] = { 196 { PCI_VENDOR_JMICRON, PCI_PRODUCT_JMICRON_JMC250, 197 JME_CAP_JUMBO, 198 "JMicron Inc, JMC250 Gigabit Ethernet" }, 199 { PCI_VENDOR_JMICRON, PCI_PRODUCT_JMICRON_JMC260, 200 JME_CAP_FASTETH, 201 "JMicron Inc, JMC260 Fast Ethernet" }, 202 { 0, 0, 0, NULL } 203 }; 204 205 static device_method_t jme_methods[] = { 206 /* Device interface. */ 207 DEVMETHOD(device_probe, jme_probe), 208 DEVMETHOD(device_attach, jme_attach), 209 DEVMETHOD(device_detach, jme_detach), 210 DEVMETHOD(device_shutdown, jme_shutdown), 211 DEVMETHOD(device_suspend, jme_suspend), 212 DEVMETHOD(device_resume, jme_resume), 213 214 /* Bus interface. */ 215 DEVMETHOD(bus_print_child, bus_generic_print_child), 216 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 217 218 /* MII interface. */ 219 DEVMETHOD(miibus_readreg, jme_miibus_readreg), 220 DEVMETHOD(miibus_writereg, jme_miibus_writereg), 221 DEVMETHOD(miibus_statchg, jme_miibus_statchg), 222 223 { NULL, NULL } 224 }; 225 226 static driver_t jme_driver = { 227 "jme", 228 jme_methods, 229 sizeof(struct jme_softc) 230 }; 231 232 static devclass_t jme_devclass; 233 234 DECLARE_DUMMY_MODULE(if_jme); 235 MODULE_DEPEND(if_jme, miibus, 1, 1, 1); 236 DRIVER_MODULE(if_jme, pci, jme_driver, jme_devclass, NULL, NULL); 237 DRIVER_MODULE(miibus, jme, miibus_driver, miibus_devclass, NULL, NULL); 238 239 static const struct { 240 uint32_t jme_coal; 241 uint32_t jme_comp; 242 uint32_t jme_empty; 243 } jme_rx_status[JME_NRXRING_MAX] = { 244 { INTR_RXQ0_COAL | INTR_RXQ0_COAL_TO, INTR_RXQ0_COMP, 245 INTR_RXQ0_DESC_EMPTY }, 246 { INTR_RXQ1_COAL | INTR_RXQ1_COAL_TO, INTR_RXQ1_COMP, 247 INTR_RXQ1_DESC_EMPTY }, 248 { INTR_RXQ2_COAL | INTR_RXQ2_COAL_TO, INTR_RXQ2_COMP, 249 INTR_RXQ2_DESC_EMPTY }, 250 { INTR_RXQ3_COAL | INTR_RXQ3_COAL_TO, INTR_RXQ3_COMP, 251 INTR_RXQ3_DESC_EMPTY } 252 }; 253 254 static int jme_rx_desc_count = JME_RX_DESC_CNT_DEF; 255 static int jme_tx_desc_count = JME_TX_DESC_CNT_DEF; 256 static int jme_rx_ring_count = 0; 257 static int jme_msi_enable = 1; 258 static int jme_msix_enable = 1; 259 260 TUNABLE_INT("hw.jme.rx_desc_count", &jme_rx_desc_count); 261 TUNABLE_INT("hw.jme.tx_desc_count", &jme_tx_desc_count); 262 TUNABLE_INT("hw.jme.rx_ring_count", &jme_rx_ring_count); 263 TUNABLE_INT("hw.jme.msi.enable", &jme_msi_enable); 264 TUNABLE_INT("hw.jme.msix.enable", &jme_msix_enable); 265 266 static __inline void 267 jme_setup_rxdesc(struct jme_rxdesc *rxd) 268 { 269 struct jme_desc *desc; 270 271 desc = rxd->rx_desc; 272 desc->buflen = htole32(MCLBYTES); 273 desc->addr_lo = htole32(JME_ADDR_LO(rxd->rx_paddr)); 274 desc->addr_hi = htole32(JME_ADDR_HI(rxd->rx_paddr)); 275 desc->flags = htole32(JME_RD_OWN | JME_RD_INTR | JME_RD_64BIT); 276 } 277 278 /* 279 * Read a PHY register on the MII of the JMC250. 280 */ 281 static int 282 jme_miibus_readreg(device_t dev, int phy, int reg) 283 { 284 struct jme_softc *sc = device_get_softc(dev); 285 uint32_t val; 286 int i; 287 288 /* For FPGA version, PHY address 0 should be ignored. */ 289 if (sc->jme_caps & JME_CAP_FPGA) { 290 if (phy == 0) 291 return (0); 292 } else { 293 if (sc->jme_phyaddr != phy) 294 return (0); 295 } 296 297 CSR_WRITE_4(sc, JME_SMI, SMI_OP_READ | SMI_OP_EXECUTE | 298 SMI_PHY_ADDR(phy) | SMI_REG_ADDR(reg)); 299 300 for (i = JME_PHY_TIMEOUT; i > 0; i--) { 301 DELAY(1); 302 if (((val = CSR_READ_4(sc, JME_SMI)) & SMI_OP_EXECUTE) == 0) 303 break; 304 } 305 if (i == 0) { 306 device_printf(sc->jme_dev, "phy read timeout: " 307 "phy %d, reg %d\n", phy, reg); 308 return (0); 309 } 310 311 return ((val & SMI_DATA_MASK) >> SMI_DATA_SHIFT); 312 } 313 314 /* 315 * Write a PHY register on the MII of the JMC250. 316 */ 317 static int 318 jme_miibus_writereg(device_t dev, int phy, int reg, int val) 319 { 320 struct jme_softc *sc = device_get_softc(dev); 321 int i; 322 323 /* For FPGA version, PHY address 0 should be ignored. */ 324 if (sc->jme_caps & JME_CAP_FPGA) { 325 if (phy == 0) 326 return (0); 327 } else { 328 if (sc->jme_phyaddr != phy) 329 return (0); 330 } 331 332 CSR_WRITE_4(sc, JME_SMI, SMI_OP_WRITE | SMI_OP_EXECUTE | 333 ((val << SMI_DATA_SHIFT) & SMI_DATA_MASK) | 334 SMI_PHY_ADDR(phy) | SMI_REG_ADDR(reg)); 335 336 for (i = JME_PHY_TIMEOUT; i > 0; i--) { 337 DELAY(1); 338 if (((val = CSR_READ_4(sc, JME_SMI)) & SMI_OP_EXECUTE) == 0) 339 break; 340 } 341 if (i == 0) { 342 device_printf(sc->jme_dev, "phy write timeout: " 343 "phy %d, reg %d\n", phy, reg); 344 } 345 346 return (0); 347 } 348 349 /* 350 * Callback from MII layer when media changes. 351 */ 352 static void 353 jme_miibus_statchg(device_t dev) 354 { 355 struct jme_softc *sc = device_get_softc(dev); 356 struct ifnet *ifp = &sc->arpcom.ac_if; 357 struct jme_txdata *tdata = &sc->jme_cdata.jme_tx_data; 358 struct mii_data *mii; 359 struct jme_txdesc *txd; 360 bus_addr_t paddr; 361 int i, r; 362 363 if (sc->jme_in_tick) 364 jme_serialize_skipmain(sc); 365 ASSERT_IFNET_SERIALIZED_ALL(ifp); 366 367 if ((ifp->if_flags & IFF_RUNNING) == 0) 368 goto done; 369 370 mii = device_get_softc(sc->jme_miibus); 371 372 sc->jme_has_link = FALSE; 373 if ((mii->mii_media_status & IFM_AVALID) != 0) { 374 switch (IFM_SUBTYPE(mii->mii_media_active)) { 375 case IFM_10_T: 376 case IFM_100_TX: 377 sc->jme_has_link = TRUE; 378 break; 379 case IFM_1000_T: 380 if (sc->jme_caps & JME_CAP_FASTETH) 381 break; 382 sc->jme_has_link = TRUE; 383 break; 384 default: 385 break; 386 } 387 } 388 389 /* 390 * Disabling Rx/Tx MACs have a side-effect of resetting 391 * JME_TXNDA/JME_RXNDA register to the first address of 392 * Tx/Rx descriptor address. So driver should reset its 393 * internal procucer/consumer pointer and reclaim any 394 * allocated resources. Note, just saving the value of 395 * JME_TXNDA and JME_RXNDA registers before stopping MAC 396 * and restoring JME_TXNDA/JME_RXNDA register is not 397 * sufficient to make sure correct MAC state because 398 * stopping MAC operation can take a while and hardware 399 * might have updated JME_TXNDA/JME_RXNDA registers 400 * during the stop operation. 401 */ 402 403 /* Disable interrupts */ 404 CSR_WRITE_4(sc, JME_INTR_MASK_CLR, JME_INTRS); 405 406 /* Stop driver */ 407 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 408 ifp->if_timer = 0; 409 callout_stop(&sc->jme_tick_ch); 410 411 /* Stop receiver/transmitter. */ 412 jme_stop_rx(sc); 413 jme_stop_tx(sc); 414 415 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 416 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[r]; 417 418 jme_rxeof(rdata, -1); 419 if (rdata->jme_rxhead != NULL) 420 m_freem(rdata->jme_rxhead); 421 JME_RXCHAIN_RESET(rdata); 422 423 /* 424 * Reuse configured Rx descriptors and reset 425 * procuder/consumer index. 426 */ 427 rdata->jme_rx_cons = 0; 428 } 429 if (JME_ENABLE_HWRSS(sc)) 430 jme_enable_rss(sc); 431 else 432 jme_disable_rss(sc); 433 434 jme_txeof(tdata); 435 if (tdata->jme_tx_cnt != 0) { 436 /* Remove queued packets for transmit. */ 437 for (i = 0; i < tdata->jme_tx_desc_cnt; i++) { 438 txd = &tdata->jme_txdesc[i]; 439 if (txd->tx_m != NULL) { 440 bus_dmamap_unload( tdata->jme_tx_tag, 441 txd->tx_dmamap); 442 m_freem(txd->tx_m); 443 txd->tx_m = NULL; 444 txd->tx_ndesc = 0; 445 ifp->if_oerrors++; 446 } 447 } 448 } 449 jme_init_tx_ring(tdata); 450 451 /* Initialize shadow status block. */ 452 jme_init_ssb(sc); 453 454 /* Program MAC with resolved speed/duplex/flow-control. */ 455 if (sc->jme_has_link) { 456 jme_mac_config(sc); 457 458 CSR_WRITE_4(sc, JME_TXCSR, sc->jme_txcsr); 459 460 /* Set Tx ring address to the hardware. */ 461 paddr = tdata->jme_tx_ring_paddr; 462 CSR_WRITE_4(sc, JME_TXDBA_HI, JME_ADDR_HI(paddr)); 463 CSR_WRITE_4(sc, JME_TXDBA_LO, JME_ADDR_LO(paddr)); 464 465 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 466 CSR_WRITE_4(sc, JME_RXCSR, 467 sc->jme_rxcsr | RXCSR_RXQ_N_SEL(r)); 468 469 /* Set Rx ring address to the hardware. */ 470 paddr = sc->jme_cdata.jme_rx_data[r].jme_rx_ring_paddr; 471 CSR_WRITE_4(sc, JME_RXDBA_HI, JME_ADDR_HI(paddr)); 472 CSR_WRITE_4(sc, JME_RXDBA_LO, JME_ADDR_LO(paddr)); 473 } 474 475 /* Restart receiver/transmitter. */ 476 CSR_WRITE_4(sc, JME_RXCSR, sc->jme_rxcsr | RXCSR_RX_ENB | 477 RXCSR_RXQ_START); 478 CSR_WRITE_4(sc, JME_TXCSR, sc->jme_txcsr | TXCSR_TX_ENB); 479 } 480 481 ifp->if_flags |= IFF_RUNNING; 482 ifp->if_flags &= ~IFF_OACTIVE; 483 callout_reset_bycpu(&sc->jme_tick_ch, hz, jme_tick, sc, 484 JME_TICK_CPUID); 485 486 #ifdef IFPOLL_ENABLE 487 if (!(ifp->if_flags & IFF_NPOLLING)) 488 #endif 489 /* Reenable interrupts. */ 490 CSR_WRITE_4(sc, JME_INTR_MASK_SET, JME_INTRS); 491 492 done: 493 if (sc->jme_in_tick) 494 jme_deserialize_skipmain(sc); 495 } 496 497 /* 498 * Get the current interface media status. 499 */ 500 static void 501 jme_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr) 502 { 503 struct jme_softc *sc = ifp->if_softc; 504 struct mii_data *mii = device_get_softc(sc->jme_miibus); 505 506 ASSERT_IFNET_SERIALIZED_ALL(ifp); 507 508 mii_pollstat(mii); 509 ifmr->ifm_status = mii->mii_media_status; 510 ifmr->ifm_active = mii->mii_media_active; 511 } 512 513 /* 514 * Set hardware to newly-selected media. 515 */ 516 static int 517 jme_mediachange(struct ifnet *ifp) 518 { 519 struct jme_softc *sc = ifp->if_softc; 520 struct mii_data *mii = device_get_softc(sc->jme_miibus); 521 int error; 522 523 ASSERT_IFNET_SERIALIZED_ALL(ifp); 524 525 if (mii->mii_instance != 0) { 526 struct mii_softc *miisc; 527 528 LIST_FOREACH(miisc, &mii->mii_phys, mii_list) 529 mii_phy_reset(miisc); 530 } 531 error = mii_mediachg(mii); 532 533 return (error); 534 } 535 536 static int 537 jme_probe(device_t dev) 538 { 539 const struct jme_dev *sp; 540 uint16_t vid, did; 541 542 vid = pci_get_vendor(dev); 543 did = pci_get_device(dev); 544 for (sp = jme_devs; sp->jme_name != NULL; ++sp) { 545 if (vid == sp->jme_vendorid && did == sp->jme_deviceid) { 546 struct jme_softc *sc = device_get_softc(dev); 547 548 sc->jme_caps = sp->jme_caps; 549 device_set_desc(dev, sp->jme_name); 550 return (0); 551 } 552 } 553 return (ENXIO); 554 } 555 556 static int 557 jme_eeprom_read_byte(struct jme_softc *sc, uint8_t addr, uint8_t *val) 558 { 559 uint32_t reg; 560 int i; 561 562 *val = 0; 563 for (i = JME_TIMEOUT; i > 0; i--) { 564 reg = CSR_READ_4(sc, JME_SMBCSR); 565 if ((reg & SMBCSR_HW_BUSY_MASK) == SMBCSR_HW_IDLE) 566 break; 567 DELAY(1); 568 } 569 570 if (i == 0) { 571 device_printf(sc->jme_dev, "EEPROM idle timeout!\n"); 572 return (ETIMEDOUT); 573 } 574 575 reg = ((uint32_t)addr << SMBINTF_ADDR_SHIFT) & SMBINTF_ADDR_MASK; 576 CSR_WRITE_4(sc, JME_SMBINTF, reg | SMBINTF_RD | SMBINTF_CMD_TRIGGER); 577 for (i = JME_TIMEOUT; i > 0; i--) { 578 DELAY(1); 579 reg = CSR_READ_4(sc, JME_SMBINTF); 580 if ((reg & SMBINTF_CMD_TRIGGER) == 0) 581 break; 582 } 583 584 if (i == 0) { 585 device_printf(sc->jme_dev, "EEPROM read timeout!\n"); 586 return (ETIMEDOUT); 587 } 588 589 reg = CSR_READ_4(sc, JME_SMBINTF); 590 *val = (reg & SMBINTF_RD_DATA_MASK) >> SMBINTF_RD_DATA_SHIFT; 591 592 return (0); 593 } 594 595 static int 596 jme_eeprom_macaddr(struct jme_softc *sc, uint8_t eaddr[]) 597 { 598 uint8_t fup, reg, val; 599 uint32_t offset; 600 int match; 601 602 offset = 0; 603 if (jme_eeprom_read_byte(sc, offset++, &fup) != 0 || 604 fup != JME_EEPROM_SIG0) 605 return (ENOENT); 606 if (jme_eeprom_read_byte(sc, offset++, &fup) != 0 || 607 fup != JME_EEPROM_SIG1) 608 return (ENOENT); 609 match = 0; 610 do { 611 if (jme_eeprom_read_byte(sc, offset, &fup) != 0) 612 break; 613 if (JME_EEPROM_MKDESC(JME_EEPROM_FUNC0, JME_EEPROM_PAGE_BAR1) == 614 (fup & (JME_EEPROM_FUNC_MASK | JME_EEPROM_PAGE_MASK))) { 615 if (jme_eeprom_read_byte(sc, offset + 1, ®) != 0) 616 break; 617 if (reg >= JME_PAR0 && 618 reg < JME_PAR0 + ETHER_ADDR_LEN) { 619 if (jme_eeprom_read_byte(sc, offset + 2, 620 &val) != 0) 621 break; 622 eaddr[reg - JME_PAR0] = val; 623 match++; 624 } 625 } 626 /* Check for the end of EEPROM descriptor. */ 627 if ((fup & JME_EEPROM_DESC_END) == JME_EEPROM_DESC_END) 628 break; 629 /* Try next eeprom descriptor. */ 630 offset += JME_EEPROM_DESC_BYTES; 631 } while (match != ETHER_ADDR_LEN && offset < JME_EEPROM_END); 632 633 if (match == ETHER_ADDR_LEN) 634 return (0); 635 636 return (ENOENT); 637 } 638 639 static void 640 jme_reg_macaddr(struct jme_softc *sc, uint8_t eaddr[]) 641 { 642 uint32_t par0, par1; 643 644 /* Read station address. */ 645 par0 = CSR_READ_4(sc, JME_PAR0); 646 par1 = CSR_READ_4(sc, JME_PAR1); 647 par1 &= 0xFFFF; 648 if ((par0 == 0 && par1 == 0) || (par0 & 0x1)) { 649 device_printf(sc->jme_dev, 650 "generating fake ethernet address.\n"); 651 par0 = karc4random(); 652 /* Set OUI to JMicron. */ 653 eaddr[0] = 0x00; 654 eaddr[1] = 0x1B; 655 eaddr[2] = 0x8C; 656 eaddr[3] = (par0 >> 16) & 0xff; 657 eaddr[4] = (par0 >> 8) & 0xff; 658 eaddr[5] = par0 & 0xff; 659 } else { 660 eaddr[0] = (par0 >> 0) & 0xFF; 661 eaddr[1] = (par0 >> 8) & 0xFF; 662 eaddr[2] = (par0 >> 16) & 0xFF; 663 eaddr[3] = (par0 >> 24) & 0xFF; 664 eaddr[4] = (par1 >> 0) & 0xFF; 665 eaddr[5] = (par1 >> 8) & 0xFF; 666 } 667 } 668 669 static int 670 jme_attach(device_t dev) 671 { 672 struct jme_softc *sc = device_get_softc(dev); 673 struct ifnet *ifp = &sc->arpcom.ac_if; 674 uint32_t reg; 675 uint16_t did; 676 uint8_t pcie_ptr, rev; 677 int error = 0, i, j, rx_desc_cnt, coal_max; 678 uint8_t eaddr[ETHER_ADDR_LEN]; 679 #ifdef IFPOLL_ENABLE 680 int offset, offset_def; 681 #endif 682 683 /* 684 * Initialize serializers 685 */ 686 lwkt_serialize_init(&sc->jme_serialize); 687 lwkt_serialize_init(&sc->jme_cdata.jme_tx_data.jme_tx_serialize); 688 for (i = 0; i < JME_NRXRING_MAX; ++i) { 689 lwkt_serialize_init( 690 &sc->jme_cdata.jme_rx_data[i].jme_rx_serialize); 691 } 692 693 /* 694 * Get # of RX ring descriptors 695 */ 696 rx_desc_cnt = device_getenv_int(dev, "rx_desc_count", 697 jme_rx_desc_count); 698 rx_desc_cnt = roundup(rx_desc_cnt, JME_NDESC_ALIGN); 699 if (rx_desc_cnt > JME_NDESC_MAX) 700 rx_desc_cnt = JME_NDESC_MAX; 701 702 /* 703 * Get # of TX ring descriptors 704 */ 705 sc->jme_cdata.jme_tx_data.jme_tx_desc_cnt = 706 device_getenv_int(dev, "tx_desc_count", jme_tx_desc_count); 707 sc->jme_cdata.jme_tx_data.jme_tx_desc_cnt = 708 roundup(sc->jme_cdata.jme_tx_data.jme_tx_desc_cnt, JME_NDESC_ALIGN); 709 if (sc->jme_cdata.jme_tx_data.jme_tx_desc_cnt > JME_NDESC_MAX) 710 sc->jme_cdata.jme_tx_data.jme_tx_desc_cnt = JME_NDESC_MAX; 711 712 /* 713 * Get # of RX rings 714 */ 715 sc->jme_cdata.jme_rx_ring_cnt = device_getenv_int(dev, "rx_ring_count", 716 jme_rx_ring_count); 717 sc->jme_cdata.jme_rx_ring_cnt = 718 if_ring_count2(sc->jme_cdata.jme_rx_ring_cnt, JME_NRXRING_MAX); 719 720 /* 721 * Initialize serializer array 722 */ 723 i = 0; 724 sc->jme_serialize_arr[i++] = &sc->jme_serialize; 725 726 KKASSERT(i == JME_TX_SERIALIZE); 727 sc->jme_serialize_arr[i++] = 728 &sc->jme_cdata.jme_tx_data.jme_tx_serialize; 729 730 KKASSERT(i == JME_RX_SERIALIZE); 731 for (j = 0; j < sc->jme_cdata.jme_rx_ring_cnt; ++j) { 732 sc->jme_serialize_arr[i++] = 733 &sc->jme_cdata.jme_rx_data[j].jme_rx_serialize; 734 } 735 KKASSERT(i <= JME_NSERIALIZE); 736 sc->jme_serialize_cnt = i; 737 738 /* 739 * Setup TX ring specific data 740 */ 741 sc->jme_cdata.jme_tx_data.jme_sc = sc; 742 743 /* 744 * Setup RX rings specific data 745 */ 746 for (i = 0; i < sc->jme_cdata.jme_rx_ring_cnt; ++i) { 747 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[i]; 748 749 rdata->jme_sc = sc; 750 rdata->jme_rx_coal = jme_rx_status[i].jme_coal; 751 rdata->jme_rx_comp = jme_rx_status[i].jme_comp; 752 rdata->jme_rx_empty = jme_rx_status[i].jme_empty; 753 rdata->jme_rx_idx = i; 754 rdata->jme_rx_desc_cnt = rx_desc_cnt; 755 } 756 757 sc->jme_dev = dev; 758 sc->jme_lowaddr = BUS_SPACE_MAXADDR; 759 760 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 761 762 callout_init(&sc->jme_tick_ch); 763 764 #ifndef BURN_BRIDGES 765 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) { 766 uint32_t irq, mem; 767 768 irq = pci_read_config(dev, PCIR_INTLINE, 4); 769 mem = pci_read_config(dev, JME_PCIR_BAR, 4); 770 771 device_printf(dev, "chip is in D%d power mode " 772 "-- setting to D0\n", pci_get_powerstate(dev)); 773 774 pci_set_powerstate(dev, PCI_POWERSTATE_D0); 775 776 pci_write_config(dev, PCIR_INTLINE, irq, 4); 777 pci_write_config(dev, JME_PCIR_BAR, mem, 4); 778 } 779 #endif /* !BURN_BRIDGE */ 780 781 /* Enable bus mastering */ 782 pci_enable_busmaster(dev); 783 784 /* 785 * Allocate IO memory 786 * 787 * JMC250 supports both memory mapped and I/O register space 788 * access. Because I/O register access should use different 789 * BARs to access registers it's waste of time to use I/O 790 * register spce access. JMC250 uses 16K to map entire memory 791 * space. 792 */ 793 sc->jme_mem_rid = JME_PCIR_BAR; 794 sc->jme_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 795 &sc->jme_mem_rid, RF_ACTIVE); 796 if (sc->jme_mem_res == NULL) { 797 device_printf(dev, "can't allocate IO memory\n"); 798 return ENXIO; 799 } 800 sc->jme_mem_bt = rman_get_bustag(sc->jme_mem_res); 801 sc->jme_mem_bh = rman_get_bushandle(sc->jme_mem_res); 802 803 /* 804 * Allocate IRQ 805 */ 806 error = jme_intr_alloc(dev); 807 if (error) 808 goto fail; 809 810 /* 811 * Extract revisions 812 */ 813 reg = CSR_READ_4(sc, JME_CHIPMODE); 814 if (((reg & CHIPMODE_FPGA_REV_MASK) >> CHIPMODE_FPGA_REV_SHIFT) != 815 CHIPMODE_NOT_FPGA) { 816 sc->jme_caps |= JME_CAP_FPGA; 817 if (bootverbose) { 818 device_printf(dev, "FPGA revision: 0x%04x\n", 819 (reg & CHIPMODE_FPGA_REV_MASK) >> 820 CHIPMODE_FPGA_REV_SHIFT); 821 } 822 } 823 824 /* NOTE: FM revision is put in the upper 4 bits */ 825 rev = ((reg & CHIPMODE_REVFM_MASK) >> CHIPMODE_REVFM_SHIFT) << 4; 826 rev |= (reg & CHIPMODE_REVECO_MASK) >> CHIPMODE_REVECO_SHIFT; 827 if (bootverbose) 828 device_printf(dev, "Revision (FM/ECO): 0x%02x\n", rev); 829 830 did = pci_get_device(dev); 831 switch (did) { 832 case PCI_PRODUCT_JMICRON_JMC250: 833 if (rev == JME_REV1_A2) 834 sc->jme_workaround |= JME_WA_EXTFIFO | JME_WA_HDX; 835 break; 836 837 case PCI_PRODUCT_JMICRON_JMC260: 838 if (rev == JME_REV2) 839 sc->jme_lowaddr = BUS_SPACE_MAXADDR_32BIT; 840 break; 841 842 default: 843 panic("unknown device id 0x%04x", did); 844 } 845 if (rev >= JME_REV2) { 846 sc->jme_clksrc = GHC_TXOFL_CLKSRC | GHC_TXMAC_CLKSRC; 847 sc->jme_clksrc_1000 = GHC_TXOFL_CLKSRC_1000 | 848 GHC_TXMAC_CLKSRC_1000; 849 } 850 851 /* Reset the ethernet controller. */ 852 jme_reset(sc); 853 854 /* Map MSI/MSI-X vectors */ 855 jme_set_msinum(sc); 856 857 /* Get station address. */ 858 reg = CSR_READ_4(sc, JME_SMBCSR); 859 if (reg & SMBCSR_EEPROM_PRESENT) 860 error = jme_eeprom_macaddr(sc, eaddr); 861 if (error != 0 || (reg & SMBCSR_EEPROM_PRESENT) == 0) { 862 if (error != 0 && (bootverbose)) { 863 device_printf(dev, "ethernet hardware address " 864 "not found in EEPROM.\n"); 865 } 866 jme_reg_macaddr(sc, eaddr); 867 } 868 869 /* 870 * Save PHY address. 871 * Integrated JR0211 has fixed PHY address whereas FPGA version 872 * requires PHY probing to get correct PHY address. 873 */ 874 if ((sc->jme_caps & JME_CAP_FPGA) == 0) { 875 sc->jme_phyaddr = CSR_READ_4(sc, JME_GPREG0) & 876 GPREG0_PHY_ADDR_MASK; 877 if (bootverbose) { 878 device_printf(dev, "PHY is at address %d.\n", 879 sc->jme_phyaddr); 880 } 881 } else { 882 sc->jme_phyaddr = 0; 883 } 884 885 /* Set max allowable DMA size. */ 886 pcie_ptr = pci_get_pciecap_ptr(dev); 887 if (pcie_ptr != 0) { 888 uint16_t ctrl; 889 890 sc->jme_caps |= JME_CAP_PCIE; 891 ctrl = pci_read_config(dev, pcie_ptr + PCIER_DEVCTRL, 2); 892 if (bootverbose) { 893 device_printf(dev, "Read request size : %d bytes.\n", 894 128 << ((ctrl >> 12) & 0x07)); 895 device_printf(dev, "TLP payload size : %d bytes.\n", 896 128 << ((ctrl >> 5) & 0x07)); 897 } 898 switch (ctrl & PCIEM_DEVCTL_MAX_READRQ_MASK) { 899 case PCIEM_DEVCTL_MAX_READRQ_128: 900 sc->jme_tx_dma_size = TXCSR_DMA_SIZE_128; 901 break; 902 case PCIEM_DEVCTL_MAX_READRQ_256: 903 sc->jme_tx_dma_size = TXCSR_DMA_SIZE_256; 904 break; 905 default: 906 sc->jme_tx_dma_size = TXCSR_DMA_SIZE_512; 907 break; 908 } 909 sc->jme_rx_dma_size = RXCSR_DMA_SIZE_128; 910 } else { 911 sc->jme_tx_dma_size = TXCSR_DMA_SIZE_512; 912 sc->jme_rx_dma_size = RXCSR_DMA_SIZE_128; 913 } 914 915 #ifdef notyet 916 if (pci_find_extcap(dev, PCIY_PMG, &pmc) == 0) 917 sc->jme_caps |= JME_CAP_PMCAP; 918 #endif 919 920 #ifdef IFPOLL_ENABLE 921 /* 922 * NPOLLING RX CPU offset 923 */ 924 if (sc->jme_cdata.jme_rx_ring_cnt == ncpus2) { 925 offset = 0; 926 } else { 927 offset_def = (sc->jme_cdata.jme_rx_ring_cnt * 928 device_get_unit(dev)) % ncpus2; 929 offset = device_getenv_int(dev, "npoll.rxoff", offset_def); 930 if (offset >= ncpus2 || 931 offset % sc->jme_cdata.jme_rx_ring_cnt != 0) { 932 device_printf(dev, "invalid npoll.rxoff %d, use %d\n", 933 offset, offset_def); 934 offset = offset_def; 935 } 936 } 937 sc->jme_npoll_rxoff = offset; 938 939 /* 940 * NPOLLING TX CPU offset 941 */ 942 offset_def = sc->jme_npoll_rxoff; 943 offset = device_getenv_int(dev, "npoll.txoff", offset_def); 944 if (offset >= ncpus2) { 945 device_printf(dev, "invalid npoll.txoff %d, use %d\n", 946 offset, offset_def); 947 offset = offset_def; 948 } 949 sc->jme_npoll_txoff = offset; 950 #endif 951 952 /* 953 * Set default coalesce valves 954 */ 955 sc->jme_tx_coal_to = PCCTX_COAL_TO_DEFAULT; 956 sc->jme_tx_coal_pkt = PCCTX_COAL_PKT_DEFAULT; 957 sc->jme_rx_coal_to = PCCRX_COAL_TO_DEFAULT; 958 sc->jme_rx_coal_pkt = PCCRX_COAL_PKT_DEFAULT; 959 960 /* 961 * Adjust coalesce valves, in case that the number of TX/RX 962 * descs are set to small values by users. 963 * 964 * NOTE: coal_max will not be zero, since number of descs 965 * must aligned by JME_NDESC_ALIGN (16 currently) 966 */ 967 coal_max = sc->jme_cdata.jme_tx_data.jme_tx_desc_cnt / 2; 968 if (coal_max < sc->jme_tx_coal_pkt) 969 sc->jme_tx_coal_pkt = coal_max; 970 971 coal_max = sc->jme_cdata.jme_rx_data[0].jme_rx_desc_cnt / 2; 972 if (coal_max < sc->jme_rx_coal_pkt) 973 sc->jme_rx_coal_pkt = coal_max; 974 975 /* 976 * Create sysctl tree 977 */ 978 jme_sysctl_node(sc); 979 980 /* Allocate DMA stuffs */ 981 error = jme_dma_alloc(sc); 982 if (error) 983 goto fail; 984 985 ifp->if_softc = sc; 986 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 987 ifp->if_init = jme_init; 988 ifp->if_ioctl = jme_ioctl; 989 ifp->if_start = jme_start; 990 #ifdef IFPOLL_ENABLE 991 ifp->if_npoll = jme_npoll; 992 #endif 993 ifp->if_watchdog = jme_watchdog; 994 ifp->if_serialize = jme_serialize; 995 ifp->if_deserialize = jme_deserialize; 996 ifp->if_tryserialize = jme_tryserialize; 997 #ifdef INVARIANTS 998 ifp->if_serialize_assert = jme_serialize_assert; 999 #endif 1000 ifq_set_maxlen(&ifp->if_snd, 1001 sc->jme_cdata.jme_tx_data.jme_tx_desc_cnt - JME_TXD_RSVD); 1002 ifq_set_ready(&ifp->if_snd); 1003 1004 /* JMC250 supports Tx/Rx checksum offload and hardware vlan tagging. */ 1005 ifp->if_capabilities = IFCAP_HWCSUM | 1006 IFCAP_TSO | 1007 IFCAP_VLAN_MTU | 1008 IFCAP_VLAN_HWTAGGING; 1009 if (sc->jme_cdata.jme_rx_ring_cnt > JME_NRXRING_MIN) 1010 ifp->if_capabilities |= IFCAP_RSS; 1011 ifp->if_capenable = ifp->if_capabilities; 1012 1013 /* 1014 * Disable TXCSUM by default to improve bulk data 1015 * transmit performance (+20Mbps improvement). 1016 */ 1017 ifp->if_capenable &= ~IFCAP_TXCSUM; 1018 1019 if (ifp->if_capenable & IFCAP_TXCSUM) 1020 ifp->if_hwassist |= JME_CSUM_FEATURES; 1021 ifp->if_hwassist |= CSUM_TSO; 1022 1023 /* Set up MII bus. */ 1024 error = mii_phy_probe(dev, &sc->jme_miibus, 1025 jme_mediachange, jme_mediastatus); 1026 if (error) { 1027 device_printf(dev, "no PHY found!\n"); 1028 goto fail; 1029 } 1030 1031 /* 1032 * Save PHYADDR for FPGA mode PHY. 1033 */ 1034 if (sc->jme_caps & JME_CAP_FPGA) { 1035 struct mii_data *mii = device_get_softc(sc->jme_miibus); 1036 1037 if (mii->mii_instance != 0) { 1038 struct mii_softc *miisc; 1039 1040 LIST_FOREACH(miisc, &mii->mii_phys, mii_list) { 1041 if (miisc->mii_phy != 0) { 1042 sc->jme_phyaddr = miisc->mii_phy; 1043 break; 1044 } 1045 } 1046 if (sc->jme_phyaddr != 0) { 1047 device_printf(sc->jme_dev, 1048 "FPGA PHY is at %d\n", sc->jme_phyaddr); 1049 /* vendor magic. */ 1050 jme_miibus_writereg(dev, sc->jme_phyaddr, 1051 JMPHY_CONF, JMPHY_CONF_DEFFIFO); 1052 1053 /* XXX should we clear JME_WA_EXTFIFO */ 1054 } 1055 } 1056 } 1057 1058 ether_ifattach(ifp, eaddr, NULL); 1059 1060 /* Tell the upper layer(s) we support long frames. */ 1061 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1062 1063 error = jme_intr_setup(dev); 1064 if (error) { 1065 ether_ifdetach(ifp); 1066 goto fail; 1067 } 1068 1069 return 0; 1070 fail: 1071 jme_detach(dev); 1072 return (error); 1073 } 1074 1075 static int 1076 jme_detach(device_t dev) 1077 { 1078 struct jme_softc *sc = device_get_softc(dev); 1079 1080 if (device_is_attached(dev)) { 1081 struct ifnet *ifp = &sc->arpcom.ac_if; 1082 1083 ifnet_serialize_all(ifp); 1084 jme_stop(sc); 1085 jme_intr_teardown(dev); 1086 ifnet_deserialize_all(ifp); 1087 1088 ether_ifdetach(ifp); 1089 } 1090 1091 if (sc->jme_sysctl_tree != NULL) 1092 sysctl_ctx_free(&sc->jme_sysctl_ctx); 1093 1094 if (sc->jme_miibus != NULL) 1095 device_delete_child(dev, sc->jme_miibus); 1096 bus_generic_detach(dev); 1097 1098 jme_intr_free(dev); 1099 1100 if (sc->jme_mem_res != NULL) { 1101 bus_release_resource(dev, SYS_RES_MEMORY, sc->jme_mem_rid, 1102 sc->jme_mem_res); 1103 } 1104 1105 jme_dma_free(sc); 1106 1107 return (0); 1108 } 1109 1110 static void 1111 jme_sysctl_node(struct jme_softc *sc) 1112 { 1113 #ifdef JME_RSS_DEBUG 1114 int r; 1115 #endif 1116 1117 sysctl_ctx_init(&sc->jme_sysctl_ctx); 1118 sc->jme_sysctl_tree = SYSCTL_ADD_NODE(&sc->jme_sysctl_ctx, 1119 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, 1120 device_get_nameunit(sc->jme_dev), 1121 CTLFLAG_RD, 0, ""); 1122 if (sc->jme_sysctl_tree == NULL) { 1123 device_printf(sc->jme_dev, "can't add sysctl node\n"); 1124 return; 1125 } 1126 1127 SYSCTL_ADD_PROC(&sc->jme_sysctl_ctx, 1128 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1129 "tx_coal_to", CTLTYPE_INT | CTLFLAG_RW, 1130 sc, 0, jme_sysctl_tx_coal_to, "I", "jme tx coalescing timeout"); 1131 1132 SYSCTL_ADD_PROC(&sc->jme_sysctl_ctx, 1133 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1134 "tx_coal_pkt", CTLTYPE_INT | CTLFLAG_RW, 1135 sc, 0, jme_sysctl_tx_coal_pkt, "I", "jme tx coalescing packet"); 1136 1137 SYSCTL_ADD_PROC(&sc->jme_sysctl_ctx, 1138 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1139 "rx_coal_to", CTLTYPE_INT | CTLFLAG_RW, 1140 sc, 0, jme_sysctl_rx_coal_to, "I", "jme rx coalescing timeout"); 1141 1142 SYSCTL_ADD_PROC(&sc->jme_sysctl_ctx, 1143 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1144 "rx_coal_pkt", CTLTYPE_INT | CTLFLAG_RW, 1145 sc, 0, jme_sysctl_rx_coal_pkt, "I", "jme rx coalescing packet"); 1146 1147 SYSCTL_ADD_INT(&sc->jme_sysctl_ctx, 1148 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1149 "rx_desc_count", CTLFLAG_RD, 1150 &sc->jme_cdata.jme_rx_data[0].jme_rx_desc_cnt, 1151 0, "RX desc count"); 1152 SYSCTL_ADD_INT(&sc->jme_sysctl_ctx, 1153 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1154 "tx_desc_count", CTLFLAG_RD, 1155 &sc->jme_cdata.jme_tx_data.jme_tx_desc_cnt, 1156 0, "TX desc count"); 1157 SYSCTL_ADD_INT(&sc->jme_sysctl_ctx, 1158 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1159 "rx_ring_count", CTLFLAG_RD, 1160 &sc->jme_cdata.jme_rx_ring_cnt, 1161 0, "RX ring count"); 1162 1163 #ifdef JME_RSS_DEBUG 1164 SYSCTL_ADD_INT(&sc->jme_sysctl_ctx, 1165 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1166 "rss_debug", CTLFLAG_RW, &sc->jme_rss_debug, 1167 0, "RSS debug level"); 1168 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 1169 char rx_ring_desc[32]; 1170 1171 ksnprintf(rx_ring_desc, sizeof(rx_ring_desc), 1172 "rx_ring%d_pkt", r); 1173 SYSCTL_ADD_ULONG(&sc->jme_sysctl_ctx, 1174 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1175 rx_ring_desc, CTLFLAG_RW, 1176 &sc->jme_cdata.jme_rx_data[r].jme_rx_pkt, "RXed packets"); 1177 1178 ksnprintf(rx_ring_desc, sizeof(rx_ring_desc), 1179 "rx_ring%d_emp", r); 1180 SYSCTL_ADD_ULONG(&sc->jme_sysctl_ctx, 1181 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1182 rx_ring_desc, CTLFLAG_RW, 1183 &sc->jme_cdata.jme_rx_data[r].jme_rx_emp, 1184 "# of time RX ring empty"); 1185 } 1186 #endif 1187 1188 #ifdef IFPOLL_ENABLE 1189 SYSCTL_ADD_PROC(&sc->jme_sysctl_ctx, 1190 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1191 "npoll_rxoff", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 1192 jme_sysctl_npoll_rxoff, "I", "NPOLLING RX cpu offset"); 1193 SYSCTL_ADD_PROC(&sc->jme_sysctl_ctx, 1194 SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO, 1195 "npoll_txoff", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 1196 jme_sysctl_npoll_txoff, "I", "NPOLLING TX cpu offset"); 1197 #endif 1198 } 1199 1200 static int 1201 jme_dma_alloc(struct jme_softc *sc) 1202 { 1203 struct jme_txdata *tdata = &sc->jme_cdata.jme_tx_data; 1204 struct jme_txdesc *txd; 1205 bus_dmamem_t dmem; 1206 int error, i, asize; 1207 1208 asize = __VM_CACHELINE_ALIGN( 1209 tdata->jme_tx_desc_cnt * sizeof(struct jme_txdesc)); 1210 tdata->jme_txdesc = kmalloc_cachealign(asize, M_DEVBUF, 1211 M_WAITOK | M_ZERO); 1212 1213 for (i = 0; i < sc->jme_cdata.jme_rx_ring_cnt; ++i) { 1214 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[i]; 1215 1216 asize = __VM_CACHELINE_ALIGN( 1217 rdata->jme_rx_desc_cnt * sizeof(struct jme_rxdesc)); 1218 rdata->jme_rxdesc = kmalloc_cachealign(asize, M_DEVBUF, 1219 M_WAITOK | M_ZERO); 1220 } 1221 1222 /* Create parent ring tag. */ 1223 error = bus_dma_tag_create(NULL,/* parent */ 1224 1, JME_RING_BOUNDARY, /* algnmnt, boundary */ 1225 sc->jme_lowaddr, /* lowaddr */ 1226 BUS_SPACE_MAXADDR, /* highaddr */ 1227 NULL, NULL, /* filter, filterarg */ 1228 BUS_SPACE_MAXSIZE_32BIT, /* maxsize */ 1229 0, /* nsegments */ 1230 BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 1231 0, /* flags */ 1232 &sc->jme_cdata.jme_ring_tag); 1233 if (error) { 1234 device_printf(sc->jme_dev, 1235 "could not create parent ring DMA tag.\n"); 1236 return error; 1237 } 1238 1239 /* 1240 * Create DMA stuffs for TX ring 1241 */ 1242 asize = roundup2(JME_TX_RING_SIZE(tdata), JME_TX_RING_ALIGN); 1243 error = bus_dmamem_coherent(sc->jme_cdata.jme_ring_tag, 1244 JME_TX_RING_ALIGN, 0, 1245 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 1246 asize, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem); 1247 if (error) { 1248 device_printf(sc->jme_dev, "could not allocate Tx ring.\n"); 1249 return error; 1250 } 1251 tdata->jme_tx_ring_tag = dmem.dmem_tag; 1252 tdata->jme_tx_ring_map = dmem.dmem_map; 1253 tdata->jme_tx_ring = dmem.dmem_addr; 1254 tdata->jme_tx_ring_paddr = dmem.dmem_busaddr; 1255 1256 /* 1257 * Create DMA stuffs for RX rings 1258 */ 1259 for (i = 0; i < sc->jme_cdata.jme_rx_ring_cnt; ++i) { 1260 error = jme_rxring_dma_alloc(&sc->jme_cdata.jme_rx_data[i]); 1261 if (error) 1262 return error; 1263 } 1264 1265 /* Create parent buffer tag. */ 1266 error = bus_dma_tag_create(NULL,/* parent */ 1267 1, 0, /* algnmnt, boundary */ 1268 sc->jme_lowaddr, /* lowaddr */ 1269 BUS_SPACE_MAXADDR, /* highaddr */ 1270 NULL, NULL, /* filter, filterarg */ 1271 BUS_SPACE_MAXSIZE_32BIT, /* maxsize */ 1272 0, /* nsegments */ 1273 BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 1274 0, /* flags */ 1275 &sc->jme_cdata.jme_buffer_tag); 1276 if (error) { 1277 device_printf(sc->jme_dev, 1278 "could not create parent buffer DMA tag.\n"); 1279 return error; 1280 } 1281 1282 /* 1283 * Create DMA stuffs for shadow status block 1284 */ 1285 asize = roundup2(JME_SSB_SIZE, JME_SSB_ALIGN); 1286 error = bus_dmamem_coherent(sc->jme_cdata.jme_buffer_tag, 1287 JME_SSB_ALIGN, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 1288 asize, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem); 1289 if (error) { 1290 device_printf(sc->jme_dev, 1291 "could not create shadow status block.\n"); 1292 return error; 1293 } 1294 sc->jme_cdata.jme_ssb_tag = dmem.dmem_tag; 1295 sc->jme_cdata.jme_ssb_map = dmem.dmem_map; 1296 sc->jme_cdata.jme_ssb_block = dmem.dmem_addr; 1297 sc->jme_cdata.jme_ssb_block_paddr = dmem.dmem_busaddr; 1298 1299 /* 1300 * Create DMA stuffs for TX buffers 1301 */ 1302 1303 /* Create tag for Tx buffers. */ 1304 error = bus_dma_tag_create(sc->jme_cdata.jme_buffer_tag,/* parent */ 1305 1, 0, /* algnmnt, boundary */ 1306 BUS_SPACE_MAXADDR, /* lowaddr */ 1307 BUS_SPACE_MAXADDR, /* highaddr */ 1308 NULL, NULL, /* filter, filterarg */ 1309 JME_TSO_MAXSIZE, /* maxsize */ 1310 JME_MAXTXSEGS, /* nsegments */ 1311 JME_MAXSEGSIZE, /* maxsegsize */ 1312 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,/* flags */ 1313 &tdata->jme_tx_tag); 1314 if (error != 0) { 1315 device_printf(sc->jme_dev, "could not create Tx DMA tag.\n"); 1316 return error; 1317 } 1318 1319 /* Create DMA maps for Tx buffers. */ 1320 for (i = 0; i < tdata->jme_tx_desc_cnt; i++) { 1321 txd = &tdata->jme_txdesc[i]; 1322 error = bus_dmamap_create(tdata->jme_tx_tag, 1323 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 1324 &txd->tx_dmamap); 1325 if (error) { 1326 int j; 1327 1328 device_printf(sc->jme_dev, 1329 "could not create %dth Tx dmamap.\n", i); 1330 1331 for (j = 0; j < i; ++j) { 1332 txd = &tdata->jme_txdesc[j]; 1333 bus_dmamap_destroy(tdata->jme_tx_tag, 1334 txd->tx_dmamap); 1335 } 1336 bus_dma_tag_destroy(tdata->jme_tx_tag); 1337 tdata->jme_tx_tag = NULL; 1338 return error; 1339 } 1340 } 1341 1342 /* 1343 * Create DMA stuffs for RX buffers 1344 */ 1345 for (i = 0; i < sc->jme_cdata.jme_rx_ring_cnt; ++i) { 1346 error = jme_rxbuf_dma_alloc(&sc->jme_cdata.jme_rx_data[i]); 1347 if (error) 1348 return error; 1349 } 1350 return 0; 1351 } 1352 1353 static void 1354 jme_dma_free(struct jme_softc *sc) 1355 { 1356 struct jme_txdata *tdata = &sc->jme_cdata.jme_tx_data; 1357 struct jme_txdesc *txd; 1358 struct jme_rxdesc *rxd; 1359 struct jme_rxdata *rdata; 1360 int i, r; 1361 1362 /* Tx ring */ 1363 if (tdata->jme_tx_ring_tag != NULL) { 1364 bus_dmamap_unload(tdata->jme_tx_ring_tag, 1365 tdata->jme_tx_ring_map); 1366 bus_dmamem_free(tdata->jme_tx_ring_tag, 1367 tdata->jme_tx_ring, tdata->jme_tx_ring_map); 1368 bus_dma_tag_destroy(tdata->jme_tx_ring_tag); 1369 tdata->jme_tx_ring_tag = NULL; 1370 } 1371 1372 /* Rx ring */ 1373 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 1374 rdata = &sc->jme_cdata.jme_rx_data[r]; 1375 if (rdata->jme_rx_ring_tag != NULL) { 1376 bus_dmamap_unload(rdata->jme_rx_ring_tag, 1377 rdata->jme_rx_ring_map); 1378 bus_dmamem_free(rdata->jme_rx_ring_tag, 1379 rdata->jme_rx_ring, 1380 rdata->jme_rx_ring_map); 1381 bus_dma_tag_destroy(rdata->jme_rx_ring_tag); 1382 rdata->jme_rx_ring_tag = NULL; 1383 } 1384 } 1385 1386 /* Tx buffers */ 1387 if (tdata->jme_tx_tag != NULL) { 1388 for (i = 0; i < tdata->jme_tx_desc_cnt; i++) { 1389 txd = &tdata->jme_txdesc[i]; 1390 bus_dmamap_destroy(tdata->jme_tx_tag, txd->tx_dmamap); 1391 } 1392 bus_dma_tag_destroy(tdata->jme_tx_tag); 1393 tdata->jme_tx_tag = NULL; 1394 } 1395 1396 /* Rx buffers */ 1397 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 1398 rdata = &sc->jme_cdata.jme_rx_data[r]; 1399 if (rdata->jme_rx_tag != NULL) { 1400 for (i = 0; i < rdata->jme_rx_desc_cnt; i++) { 1401 rxd = &rdata->jme_rxdesc[i]; 1402 bus_dmamap_destroy(rdata->jme_rx_tag, 1403 rxd->rx_dmamap); 1404 } 1405 bus_dmamap_destroy(rdata->jme_rx_tag, 1406 rdata->jme_rx_sparemap); 1407 bus_dma_tag_destroy(rdata->jme_rx_tag); 1408 rdata->jme_rx_tag = NULL; 1409 } 1410 } 1411 1412 /* Shadow status block. */ 1413 if (sc->jme_cdata.jme_ssb_tag != NULL) { 1414 bus_dmamap_unload(sc->jme_cdata.jme_ssb_tag, 1415 sc->jme_cdata.jme_ssb_map); 1416 bus_dmamem_free(sc->jme_cdata.jme_ssb_tag, 1417 sc->jme_cdata.jme_ssb_block, 1418 sc->jme_cdata.jme_ssb_map); 1419 bus_dma_tag_destroy(sc->jme_cdata.jme_ssb_tag); 1420 sc->jme_cdata.jme_ssb_tag = NULL; 1421 } 1422 1423 if (sc->jme_cdata.jme_buffer_tag != NULL) { 1424 bus_dma_tag_destroy(sc->jme_cdata.jme_buffer_tag); 1425 sc->jme_cdata.jme_buffer_tag = NULL; 1426 } 1427 if (sc->jme_cdata.jme_ring_tag != NULL) { 1428 bus_dma_tag_destroy(sc->jme_cdata.jme_ring_tag); 1429 sc->jme_cdata.jme_ring_tag = NULL; 1430 } 1431 1432 if (tdata->jme_txdesc != NULL) { 1433 kfree(tdata->jme_txdesc, M_DEVBUF); 1434 tdata->jme_txdesc = NULL; 1435 } 1436 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 1437 rdata = &sc->jme_cdata.jme_rx_data[r]; 1438 if (rdata->jme_rxdesc != NULL) { 1439 kfree(rdata->jme_rxdesc, M_DEVBUF); 1440 rdata->jme_rxdesc = NULL; 1441 } 1442 } 1443 } 1444 1445 /* 1446 * Make sure the interface is stopped at reboot time. 1447 */ 1448 static int 1449 jme_shutdown(device_t dev) 1450 { 1451 return jme_suspend(dev); 1452 } 1453 1454 #ifdef notyet 1455 /* 1456 * Unlike other ethernet controllers, JMC250 requires 1457 * explicit resetting link speed to 10/100Mbps as gigabit 1458 * link will cunsume more power than 375mA. 1459 * Note, we reset the link speed to 10/100Mbps with 1460 * auto-negotiation but we don't know whether that operation 1461 * would succeed or not as we have no control after powering 1462 * off. If the renegotiation fail WOL may not work. Running 1463 * at 1Gbps draws more power than 375mA at 3.3V which is 1464 * specified in PCI specification and that would result in 1465 * complete shutdowning power to ethernet controller. 1466 * 1467 * TODO 1468 * Save current negotiated media speed/duplex/flow-control 1469 * to softc and restore the same link again after resuming. 1470 * PHY handling such as power down/resetting to 100Mbps 1471 * may be better handled in suspend method in phy driver. 1472 */ 1473 static void 1474 jme_setlinkspeed(struct jme_softc *sc) 1475 { 1476 struct mii_data *mii; 1477 int aneg, i; 1478 1479 JME_LOCK_ASSERT(sc); 1480 1481 mii = device_get_softc(sc->jme_miibus); 1482 mii_pollstat(mii); 1483 aneg = 0; 1484 if ((mii->mii_media_status & IFM_AVALID) != 0) { 1485 switch IFM_SUBTYPE(mii->mii_media_active) { 1486 case IFM_10_T: 1487 case IFM_100_TX: 1488 return; 1489 case IFM_1000_T: 1490 aneg++; 1491 default: 1492 break; 1493 } 1494 } 1495 jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr, MII_100T2CR, 0); 1496 jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr, MII_ANAR, 1497 ANAR_TX_FD | ANAR_TX | ANAR_10_FD | ANAR_10 | ANAR_CSMA); 1498 jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr, MII_BMCR, 1499 BMCR_AUTOEN | BMCR_STARTNEG); 1500 DELAY(1000); 1501 if (aneg != 0) { 1502 /* Poll link state until jme(4) get a 10/100 link. */ 1503 for (i = 0; i < MII_ANEGTICKS_GIGE; i++) { 1504 mii_pollstat(mii); 1505 if ((mii->mii_media_status & IFM_AVALID) != 0) { 1506 switch (IFM_SUBTYPE(mii->mii_media_active)) { 1507 case IFM_10_T: 1508 case IFM_100_TX: 1509 jme_mac_config(sc); 1510 return; 1511 default: 1512 break; 1513 } 1514 } 1515 JME_UNLOCK(sc); 1516 pause("jmelnk", hz); 1517 JME_LOCK(sc); 1518 } 1519 if (i == MII_ANEGTICKS_GIGE) 1520 device_printf(sc->jme_dev, "establishing link failed, " 1521 "WOL may not work!"); 1522 } 1523 /* 1524 * No link, force MAC to have 100Mbps, full-duplex link. 1525 * This is the last resort and may/may not work. 1526 */ 1527 mii->mii_media_status = IFM_AVALID | IFM_ACTIVE; 1528 mii->mii_media_active = IFM_ETHER | IFM_100_TX | IFM_FDX; 1529 jme_mac_config(sc); 1530 } 1531 1532 static void 1533 jme_setwol(struct jme_softc *sc) 1534 { 1535 struct ifnet *ifp = &sc->arpcom.ac_if; 1536 uint32_t gpr, pmcs; 1537 uint16_t pmstat; 1538 int pmc; 1539 1540 if (pci_find_extcap(sc->jme_dev, PCIY_PMG, &pmc) != 0) { 1541 /* No PME capability, PHY power down. */ 1542 jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr, 1543 MII_BMCR, BMCR_PDOWN); 1544 return; 1545 } 1546 1547 gpr = CSR_READ_4(sc, JME_GPREG0) & ~GPREG0_PME_ENB; 1548 pmcs = CSR_READ_4(sc, JME_PMCS); 1549 pmcs &= ~PMCS_WOL_ENB_MASK; 1550 if ((ifp->if_capenable & IFCAP_WOL_MAGIC) != 0) { 1551 pmcs |= PMCS_MAGIC_FRAME | PMCS_MAGIC_FRAME_ENB; 1552 /* Enable PME message. */ 1553 gpr |= GPREG0_PME_ENB; 1554 /* For gigabit controllers, reset link speed to 10/100. */ 1555 if ((sc->jme_caps & JME_CAP_FASTETH) == 0) 1556 jme_setlinkspeed(sc); 1557 } 1558 1559 CSR_WRITE_4(sc, JME_PMCS, pmcs); 1560 CSR_WRITE_4(sc, JME_GPREG0, gpr); 1561 1562 /* Request PME. */ 1563 pmstat = pci_read_config(sc->jme_dev, pmc + PCIR_POWER_STATUS, 2); 1564 pmstat &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); 1565 if ((ifp->if_capenable & IFCAP_WOL) != 0) 1566 pmstat |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 1567 pci_write_config(sc->jme_dev, pmc + PCIR_POWER_STATUS, pmstat, 2); 1568 if ((ifp->if_capenable & IFCAP_WOL) == 0) { 1569 /* No WOL, PHY power down. */ 1570 jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr, 1571 MII_BMCR, BMCR_PDOWN); 1572 } 1573 } 1574 #endif 1575 1576 static int 1577 jme_suspend(device_t dev) 1578 { 1579 struct jme_softc *sc = device_get_softc(dev); 1580 struct ifnet *ifp = &sc->arpcom.ac_if; 1581 1582 ifnet_serialize_all(ifp); 1583 jme_stop(sc); 1584 #ifdef notyet 1585 jme_setwol(sc); 1586 #endif 1587 ifnet_deserialize_all(ifp); 1588 1589 return (0); 1590 } 1591 1592 static int 1593 jme_resume(device_t dev) 1594 { 1595 struct jme_softc *sc = device_get_softc(dev); 1596 struct ifnet *ifp = &sc->arpcom.ac_if; 1597 #ifdef notyet 1598 int pmc; 1599 #endif 1600 1601 ifnet_serialize_all(ifp); 1602 1603 #ifdef notyet 1604 if (pci_find_extcap(sc->jme_dev, PCIY_PMG, &pmc) != 0) { 1605 uint16_t pmstat; 1606 1607 pmstat = pci_read_config(sc->jme_dev, 1608 pmc + PCIR_POWER_STATUS, 2); 1609 /* Disable PME clear PME status. */ 1610 pmstat &= ~PCIM_PSTAT_PMEENABLE; 1611 pci_write_config(sc->jme_dev, 1612 pmc + PCIR_POWER_STATUS, pmstat, 2); 1613 } 1614 #endif 1615 1616 if (ifp->if_flags & IFF_UP) 1617 jme_init(sc); 1618 1619 ifnet_deserialize_all(ifp); 1620 1621 return (0); 1622 } 1623 1624 static __inline int 1625 jme_tso_pullup(struct mbuf **mp) 1626 { 1627 int hoff, iphlen, thoff; 1628 struct mbuf *m; 1629 1630 m = *mp; 1631 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable")); 1632 1633 iphlen = m->m_pkthdr.csum_iphlen; 1634 thoff = m->m_pkthdr.csum_thlen; 1635 hoff = m->m_pkthdr.csum_lhlen; 1636 1637 KASSERT(iphlen > 0, ("invalid ip hlen")); 1638 KASSERT(thoff > 0, ("invalid tcp hlen")); 1639 KASSERT(hoff > 0, ("invalid ether hlen")); 1640 1641 if (__predict_false(m->m_len < hoff + iphlen + thoff)) { 1642 m = m_pullup(m, hoff + iphlen + thoff); 1643 if (m == NULL) { 1644 *mp = NULL; 1645 return ENOBUFS; 1646 } 1647 *mp = m; 1648 } 1649 return 0; 1650 } 1651 1652 static int 1653 jme_encap(struct jme_txdata *tdata, struct mbuf **m_head) 1654 { 1655 struct jme_txdesc *txd; 1656 struct jme_desc *desc; 1657 struct mbuf *m; 1658 bus_dma_segment_t txsegs[JME_MAXTXSEGS]; 1659 int maxsegs, nsegs; 1660 int error, i, prod, symbol_desc; 1661 uint32_t cflags, flag64, mss; 1662 1663 M_ASSERTPKTHDR((*m_head)); 1664 1665 if ((*m_head)->m_pkthdr.csum_flags & CSUM_TSO) { 1666 /* XXX Is this necessary? */ 1667 error = jme_tso_pullup(m_head); 1668 if (error) 1669 return error; 1670 } 1671 1672 prod = tdata->jme_tx_prod; 1673 txd = &tdata->jme_txdesc[prod]; 1674 1675 if (tdata->jme_sc->jme_lowaddr != BUS_SPACE_MAXADDR_32BIT) 1676 symbol_desc = 1; 1677 else 1678 symbol_desc = 0; 1679 1680 maxsegs = (tdata->jme_tx_desc_cnt - tdata->jme_tx_cnt) - 1681 (JME_TXD_RSVD + symbol_desc); 1682 if (maxsegs > JME_MAXTXSEGS) 1683 maxsegs = JME_MAXTXSEGS; 1684 KASSERT(maxsegs >= (JME_TXD_SPARE - symbol_desc), 1685 ("not enough segments %d", maxsegs)); 1686 1687 error = bus_dmamap_load_mbuf_defrag(tdata->jme_tx_tag, 1688 txd->tx_dmamap, m_head, 1689 txsegs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1690 if (error) 1691 goto fail; 1692 1693 bus_dmamap_sync(tdata->jme_tx_tag, txd->tx_dmamap, 1694 BUS_DMASYNC_PREWRITE); 1695 1696 m = *m_head; 1697 cflags = 0; 1698 mss = 0; 1699 1700 /* Configure checksum offload. */ 1701 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1702 mss = (uint32_t)m->m_pkthdr.tso_segsz << JME_TD_MSS_SHIFT; 1703 cflags |= JME_TD_TSO; 1704 } else if (m->m_pkthdr.csum_flags & JME_CSUM_FEATURES) { 1705 if (m->m_pkthdr.csum_flags & CSUM_IP) 1706 cflags |= JME_TD_IPCSUM; 1707 if (m->m_pkthdr.csum_flags & CSUM_TCP) 1708 cflags |= JME_TD_TCPCSUM; 1709 if (m->m_pkthdr.csum_flags & CSUM_UDP) 1710 cflags |= JME_TD_UDPCSUM; 1711 } 1712 1713 /* Configure VLAN. */ 1714 if (m->m_flags & M_VLANTAG) { 1715 cflags |= (m->m_pkthdr.ether_vlantag & JME_TD_VLAN_MASK); 1716 cflags |= JME_TD_VLAN_TAG; 1717 } 1718 1719 desc = &tdata->jme_tx_ring[prod]; 1720 desc->flags = htole32(cflags); 1721 desc->addr_hi = htole32(m->m_pkthdr.len); 1722 if (tdata->jme_sc->jme_lowaddr != BUS_SPACE_MAXADDR_32BIT) { 1723 /* 1724 * Use 64bits TX desc chain format. 1725 * 1726 * The first TX desc of the chain, which is setup here, 1727 * is just a symbol TX desc carrying no payload. 1728 */ 1729 flag64 = JME_TD_64BIT; 1730 desc->buflen = htole32(mss); 1731 desc->addr_lo = 0; 1732 1733 /* No effective TX desc is consumed */ 1734 i = 0; 1735 } else { 1736 /* 1737 * Use 32bits TX desc chain format. 1738 * 1739 * The first TX desc of the chain, which is setup here, 1740 * is an effective TX desc carrying the first segment of 1741 * the mbuf chain. 1742 */ 1743 flag64 = 0; 1744 desc->buflen = htole32(mss | txsegs[0].ds_len); 1745 desc->addr_lo = htole32(JME_ADDR_LO(txsegs[0].ds_addr)); 1746 1747 /* One effective TX desc is consumed */ 1748 i = 1; 1749 } 1750 tdata->jme_tx_cnt++; 1751 KKASSERT(tdata->jme_tx_cnt - i < tdata->jme_tx_desc_cnt - JME_TXD_RSVD); 1752 JME_DESC_INC(prod, tdata->jme_tx_desc_cnt); 1753 1754 txd->tx_ndesc = 1 - i; 1755 for (; i < nsegs; i++) { 1756 desc = &tdata->jme_tx_ring[prod]; 1757 desc->buflen = htole32(txsegs[i].ds_len); 1758 desc->addr_hi = htole32(JME_ADDR_HI(txsegs[i].ds_addr)); 1759 desc->addr_lo = htole32(JME_ADDR_LO(txsegs[i].ds_addr)); 1760 desc->flags = htole32(JME_TD_OWN | flag64); 1761 1762 tdata->jme_tx_cnt++; 1763 KKASSERT(tdata->jme_tx_cnt <= 1764 tdata->jme_tx_desc_cnt - JME_TXD_RSVD); 1765 JME_DESC_INC(prod, tdata->jme_tx_desc_cnt); 1766 } 1767 1768 /* Update producer index. */ 1769 tdata->jme_tx_prod = prod; 1770 /* 1771 * Finally request interrupt and give the first descriptor 1772 * owenership to hardware. 1773 */ 1774 desc = txd->tx_desc; 1775 desc->flags |= htole32(JME_TD_OWN | JME_TD_INTR); 1776 1777 txd->tx_m = m; 1778 txd->tx_ndesc += nsegs; 1779 1780 return 0; 1781 fail: 1782 m_freem(*m_head); 1783 *m_head = NULL; 1784 return error; 1785 } 1786 1787 static void 1788 jme_start(struct ifnet *ifp) 1789 { 1790 struct jme_softc *sc = ifp->if_softc; 1791 struct jme_txdata *tdata = &sc->jme_cdata.jme_tx_data; 1792 struct mbuf *m_head; 1793 int enq = 0; 1794 1795 ASSERT_SERIALIZED(&tdata->jme_tx_serialize); 1796 1797 if (!sc->jme_has_link) { 1798 ifq_purge(&ifp->if_snd); 1799 return; 1800 } 1801 1802 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) 1803 return; 1804 1805 if (tdata->jme_tx_cnt >= JME_TX_DESC_HIWAT(tdata)) 1806 jme_txeof(tdata); 1807 1808 while (!ifq_is_empty(&ifp->if_snd)) { 1809 /* 1810 * Check number of available TX descs, always 1811 * leave JME_TXD_RSVD free TX descs. 1812 */ 1813 if (tdata->jme_tx_cnt + JME_TXD_SPARE > 1814 tdata->jme_tx_desc_cnt - JME_TXD_RSVD) { 1815 ifp->if_flags |= IFF_OACTIVE; 1816 break; 1817 } 1818 1819 m_head = ifq_dequeue(&ifp->if_snd, NULL); 1820 if (m_head == NULL) 1821 break; 1822 1823 /* 1824 * Pack the data into the transmit ring. If we 1825 * don't have room, set the OACTIVE flag and wait 1826 * for the NIC to drain the ring. 1827 */ 1828 if (jme_encap(tdata, &m_head)) { 1829 KKASSERT(m_head == NULL); 1830 ifp->if_oerrors++; 1831 ifp->if_flags |= IFF_OACTIVE; 1832 break; 1833 } 1834 enq++; 1835 1836 /* 1837 * If there's a BPF listener, bounce a copy of this frame 1838 * to him. 1839 */ 1840 ETHER_BPF_MTAP(ifp, m_head); 1841 } 1842 1843 if (enq > 0) { 1844 /* 1845 * Reading TXCSR takes very long time under heavy load 1846 * so cache TXCSR value and writes the ORed value with 1847 * the kick command to the TXCSR. This saves one register 1848 * access cycle. 1849 */ 1850 CSR_WRITE_4(sc, JME_TXCSR, sc->jme_txcsr | TXCSR_TX_ENB | 1851 TXCSR_TXQ_N_START(TXCSR_TXQ0)); 1852 /* Set a timeout in case the chip goes out to lunch. */ 1853 ifp->if_timer = JME_TX_TIMEOUT; 1854 } 1855 } 1856 1857 static void 1858 jme_watchdog(struct ifnet *ifp) 1859 { 1860 struct jme_softc *sc = ifp->if_softc; 1861 struct jme_txdata *tdata = &sc->jme_cdata.jme_tx_data; 1862 1863 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1864 1865 if (!sc->jme_has_link) { 1866 if_printf(ifp, "watchdog timeout (missed link)\n"); 1867 ifp->if_oerrors++; 1868 jme_init(sc); 1869 return; 1870 } 1871 1872 jme_txeof(tdata); 1873 if (tdata->jme_tx_cnt == 0) { 1874 if_printf(ifp, "watchdog timeout (missed Tx interrupts) " 1875 "-- recovering\n"); 1876 if (!ifq_is_empty(&ifp->if_snd)) 1877 if_devstart(ifp); 1878 return; 1879 } 1880 1881 if_printf(ifp, "watchdog timeout\n"); 1882 ifp->if_oerrors++; 1883 jme_init(sc); 1884 if (!ifq_is_empty(&ifp->if_snd)) 1885 if_devstart(ifp); 1886 } 1887 1888 static int 1889 jme_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr) 1890 { 1891 struct jme_softc *sc = ifp->if_softc; 1892 struct mii_data *mii = device_get_softc(sc->jme_miibus); 1893 struct ifreq *ifr = (struct ifreq *)data; 1894 int error = 0, mask; 1895 1896 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1897 1898 switch (cmd) { 1899 case SIOCSIFMTU: 1900 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > JME_JUMBO_MTU || 1901 (!(sc->jme_caps & JME_CAP_JUMBO) && 1902 ifr->ifr_mtu > JME_MAX_MTU)) { 1903 error = EINVAL; 1904 break; 1905 } 1906 1907 if (ifp->if_mtu != ifr->ifr_mtu) { 1908 /* 1909 * No special configuration is required when interface 1910 * MTU is changed but availability of Tx checksum 1911 * offload should be chcked against new MTU size as 1912 * FIFO size is just 2K. 1913 */ 1914 if (ifr->ifr_mtu >= JME_TX_FIFO_SIZE) { 1915 ifp->if_capenable &= 1916 ~(IFCAP_TXCSUM | IFCAP_TSO); 1917 ifp->if_hwassist &= 1918 ~(JME_CSUM_FEATURES | CSUM_TSO); 1919 } 1920 ifp->if_mtu = ifr->ifr_mtu; 1921 if (ifp->if_flags & IFF_RUNNING) 1922 jme_init(sc); 1923 } 1924 break; 1925 1926 case SIOCSIFFLAGS: 1927 if (ifp->if_flags & IFF_UP) { 1928 if (ifp->if_flags & IFF_RUNNING) { 1929 if ((ifp->if_flags ^ sc->jme_if_flags) & 1930 (IFF_PROMISC | IFF_ALLMULTI)) 1931 jme_set_filter(sc); 1932 } else { 1933 jme_init(sc); 1934 } 1935 } else { 1936 if (ifp->if_flags & IFF_RUNNING) 1937 jme_stop(sc); 1938 } 1939 sc->jme_if_flags = ifp->if_flags; 1940 break; 1941 1942 case SIOCADDMULTI: 1943 case SIOCDELMULTI: 1944 if (ifp->if_flags & IFF_RUNNING) 1945 jme_set_filter(sc); 1946 break; 1947 1948 case SIOCSIFMEDIA: 1949 case SIOCGIFMEDIA: 1950 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd); 1951 break; 1952 1953 case SIOCSIFCAP: 1954 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1955 1956 if ((mask & IFCAP_TXCSUM) && ifp->if_mtu < JME_TX_FIFO_SIZE) { 1957 ifp->if_capenable ^= IFCAP_TXCSUM; 1958 if (ifp->if_capenable & IFCAP_TXCSUM) 1959 ifp->if_hwassist |= JME_CSUM_FEATURES; 1960 else 1961 ifp->if_hwassist &= ~JME_CSUM_FEATURES; 1962 } 1963 if (mask & IFCAP_RXCSUM) { 1964 uint32_t reg; 1965 1966 ifp->if_capenable ^= IFCAP_RXCSUM; 1967 reg = CSR_READ_4(sc, JME_RXMAC); 1968 reg &= ~RXMAC_CSUM_ENB; 1969 if (ifp->if_capenable & IFCAP_RXCSUM) 1970 reg |= RXMAC_CSUM_ENB; 1971 CSR_WRITE_4(sc, JME_RXMAC, reg); 1972 } 1973 1974 if (mask & IFCAP_VLAN_HWTAGGING) { 1975 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1976 jme_set_vlan(sc); 1977 } 1978 1979 if ((mask & IFCAP_TSO) && ifp->if_mtu < JME_TX_FIFO_SIZE) { 1980 ifp->if_capenable ^= IFCAP_TSO; 1981 if (ifp->if_capenable & IFCAP_TSO) 1982 ifp->if_hwassist |= CSUM_TSO; 1983 else 1984 ifp->if_hwassist &= ~CSUM_TSO; 1985 } 1986 1987 if (mask & IFCAP_RSS) 1988 ifp->if_capenable ^= IFCAP_RSS; 1989 break; 1990 1991 default: 1992 error = ether_ioctl(ifp, cmd, data); 1993 break; 1994 } 1995 return (error); 1996 } 1997 1998 static void 1999 jme_mac_config(struct jme_softc *sc) 2000 { 2001 struct mii_data *mii; 2002 uint32_t ghc, rxmac, txmac, txpause, gp1; 2003 int phyconf = JMPHY_CONF_DEFFIFO, hdx = 0; 2004 2005 mii = device_get_softc(sc->jme_miibus); 2006 2007 CSR_WRITE_4(sc, JME_GHC, GHC_RESET); 2008 DELAY(10); 2009 CSR_WRITE_4(sc, JME_GHC, 0); 2010 ghc = 0; 2011 rxmac = CSR_READ_4(sc, JME_RXMAC); 2012 rxmac &= ~RXMAC_FC_ENB; 2013 txmac = CSR_READ_4(sc, JME_TXMAC); 2014 txmac &= ~(TXMAC_CARRIER_EXT | TXMAC_FRAME_BURST); 2015 txpause = CSR_READ_4(sc, JME_TXPFC); 2016 txpause &= ~TXPFC_PAUSE_ENB; 2017 if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0) { 2018 ghc |= GHC_FULL_DUPLEX; 2019 rxmac &= ~RXMAC_COLL_DET_ENB; 2020 txmac &= ~(TXMAC_COLL_ENB | TXMAC_CARRIER_SENSE | 2021 TXMAC_BACKOFF | TXMAC_CARRIER_EXT | 2022 TXMAC_FRAME_BURST); 2023 #ifdef notyet 2024 if ((IFM_OPTIONS(mii->mii_media_active) & IFM_ETH_TXPAUSE) != 0) 2025 txpause |= TXPFC_PAUSE_ENB; 2026 if ((IFM_OPTIONS(mii->mii_media_active) & IFM_ETH_RXPAUSE) != 0) 2027 rxmac |= RXMAC_FC_ENB; 2028 #endif 2029 /* Disable retry transmit timer/retry limit. */ 2030 CSR_WRITE_4(sc, JME_TXTRHD, CSR_READ_4(sc, JME_TXTRHD) & 2031 ~(TXTRHD_RT_PERIOD_ENB | TXTRHD_RT_LIMIT_ENB)); 2032 } else { 2033 rxmac |= RXMAC_COLL_DET_ENB; 2034 txmac |= TXMAC_COLL_ENB | TXMAC_CARRIER_SENSE | TXMAC_BACKOFF; 2035 /* Enable retry transmit timer/retry limit. */ 2036 CSR_WRITE_4(sc, JME_TXTRHD, CSR_READ_4(sc, JME_TXTRHD) | 2037 TXTRHD_RT_PERIOD_ENB | TXTRHD_RT_LIMIT_ENB); 2038 } 2039 2040 /* 2041 * Reprogram Tx/Rx MACs with resolved speed/duplex. 2042 */ 2043 gp1 = CSR_READ_4(sc, JME_GPREG1); 2044 gp1 &= ~GPREG1_WA_HDX; 2045 2046 if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) == 0) 2047 hdx = 1; 2048 2049 switch (IFM_SUBTYPE(mii->mii_media_active)) { 2050 case IFM_10_T: 2051 ghc |= GHC_SPEED_10 | sc->jme_clksrc; 2052 if (hdx) 2053 gp1 |= GPREG1_WA_HDX; 2054 break; 2055 2056 case IFM_100_TX: 2057 ghc |= GHC_SPEED_100 | sc->jme_clksrc; 2058 if (hdx) 2059 gp1 |= GPREG1_WA_HDX; 2060 2061 /* 2062 * Use extended FIFO depth to workaround CRC errors 2063 * emitted by chips before JMC250B 2064 */ 2065 phyconf = JMPHY_CONF_EXTFIFO; 2066 break; 2067 2068 case IFM_1000_T: 2069 if (sc->jme_caps & JME_CAP_FASTETH) 2070 break; 2071 2072 ghc |= GHC_SPEED_1000 | sc->jme_clksrc_1000; 2073 if (hdx) 2074 txmac |= TXMAC_CARRIER_EXT | TXMAC_FRAME_BURST; 2075 break; 2076 2077 default: 2078 break; 2079 } 2080 CSR_WRITE_4(sc, JME_GHC, ghc); 2081 CSR_WRITE_4(sc, JME_RXMAC, rxmac); 2082 CSR_WRITE_4(sc, JME_TXMAC, txmac); 2083 CSR_WRITE_4(sc, JME_TXPFC, txpause); 2084 2085 if (sc->jme_workaround & JME_WA_EXTFIFO) { 2086 jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr, 2087 JMPHY_CONF, phyconf); 2088 } 2089 if (sc->jme_workaround & JME_WA_HDX) 2090 CSR_WRITE_4(sc, JME_GPREG1, gp1); 2091 } 2092 2093 static void 2094 jme_intr(void *xsc) 2095 { 2096 struct jme_softc *sc = xsc; 2097 struct ifnet *ifp = &sc->arpcom.ac_if; 2098 uint32_t status; 2099 int r; 2100 2101 ASSERT_SERIALIZED(&sc->jme_serialize); 2102 2103 status = CSR_READ_4(sc, JME_INTR_REQ_STATUS); 2104 if (status == 0 || status == 0xFFFFFFFF) 2105 return; 2106 2107 /* Disable interrupts. */ 2108 CSR_WRITE_4(sc, JME_INTR_MASK_CLR, JME_INTRS); 2109 2110 status = CSR_READ_4(sc, JME_INTR_STATUS); 2111 if ((status & JME_INTRS) == 0 || status == 0xFFFFFFFF) 2112 goto back; 2113 2114 /* Reset PCC counter/timer and Ack interrupts. */ 2115 status &= ~(INTR_TXQ_COMP | INTR_RXQ_COMP); 2116 2117 if (status & (INTR_TXQ_COAL | INTR_TXQ_COAL_TO)) 2118 status |= INTR_TXQ_COAL | INTR_TXQ_COAL_TO | INTR_TXQ_COMP; 2119 2120 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 2121 if (status & jme_rx_status[r].jme_coal) { 2122 status |= jme_rx_status[r].jme_coal | 2123 jme_rx_status[r].jme_comp; 2124 } 2125 } 2126 2127 CSR_WRITE_4(sc, JME_INTR_STATUS, status); 2128 2129 if (ifp->if_flags & IFF_RUNNING) { 2130 struct jme_txdata *tdata = &sc->jme_cdata.jme_tx_data; 2131 2132 if (status & (INTR_RXQ_COAL | INTR_RXQ_COAL_TO)) 2133 jme_rx_intr(sc, status); 2134 2135 if (status & INTR_RXQ_DESC_EMPTY) { 2136 /* 2137 * Notify hardware availability of new Rx buffers. 2138 * Reading RXCSR takes very long time under heavy 2139 * load so cache RXCSR value and writes the ORed 2140 * value with the kick command to the RXCSR. This 2141 * saves one register access cycle. 2142 */ 2143 CSR_WRITE_4(sc, JME_RXCSR, sc->jme_rxcsr | 2144 RXCSR_RX_ENB | RXCSR_RXQ_START); 2145 } 2146 2147 if (status & (INTR_TXQ_COAL | INTR_TXQ_COAL_TO)) { 2148 lwkt_serialize_enter(&tdata->jme_tx_serialize); 2149 jme_txeof(tdata); 2150 if (!ifq_is_empty(&ifp->if_snd)) 2151 if_devstart(ifp); 2152 lwkt_serialize_exit(&tdata->jme_tx_serialize); 2153 } 2154 } 2155 back: 2156 /* Reenable interrupts. */ 2157 CSR_WRITE_4(sc, JME_INTR_MASK_SET, JME_INTRS); 2158 } 2159 2160 static void 2161 jme_txeof(struct jme_txdata *tdata) 2162 { 2163 struct ifnet *ifp = &tdata->jme_sc->arpcom.ac_if; 2164 int cons; 2165 2166 cons = tdata->jme_tx_cons; 2167 if (cons == tdata->jme_tx_prod) 2168 return; 2169 2170 /* 2171 * Go through our Tx list and free mbufs for those 2172 * frames which have been transmitted. 2173 */ 2174 while (cons != tdata->jme_tx_prod) { 2175 struct jme_txdesc *txd, *next_txd; 2176 uint32_t status, next_status; 2177 int next_cons, nsegs; 2178 2179 txd = &tdata->jme_txdesc[cons]; 2180 KASSERT(txd->tx_m != NULL, 2181 ("%s: freeing NULL mbuf!", __func__)); 2182 2183 status = le32toh(txd->tx_desc->flags); 2184 if ((status & JME_TD_OWN) == JME_TD_OWN) 2185 break; 2186 2187 /* 2188 * NOTE: 2189 * This chip will always update the TX descriptor's 2190 * buflen field and this updating always happens 2191 * after clearing the OWN bit, so even if the OWN 2192 * bit is cleared by the chip, we still don't sure 2193 * about whether the buflen field has been updated 2194 * by the chip or not. To avoid this race, we wait 2195 * for the next TX descriptor's OWN bit to be cleared 2196 * by the chip before reusing this TX descriptor. 2197 */ 2198 next_cons = cons; 2199 JME_DESC_ADD(next_cons, txd->tx_ndesc, tdata->jme_tx_desc_cnt); 2200 next_txd = &tdata->jme_txdesc[next_cons]; 2201 if (next_txd->tx_m == NULL) 2202 break; 2203 next_status = le32toh(next_txd->tx_desc->flags); 2204 if ((next_status & JME_TD_OWN) == JME_TD_OWN) 2205 break; 2206 2207 if (status & (JME_TD_TMOUT | JME_TD_RETRY_EXP)) { 2208 ifp->if_oerrors++; 2209 } else { 2210 ifp->if_opackets++; 2211 if (status & JME_TD_COLLISION) { 2212 ifp->if_collisions += 2213 le32toh(txd->tx_desc->buflen) & 2214 JME_TD_BUF_LEN_MASK; 2215 } 2216 } 2217 2218 /* 2219 * Only the first descriptor of multi-descriptor 2220 * transmission is updated so driver have to skip entire 2221 * chained buffers for the transmiited frame. In other 2222 * words, JME_TD_OWN bit is valid only at the first 2223 * descriptor of a multi-descriptor transmission. 2224 */ 2225 for (nsegs = 0; nsegs < txd->tx_ndesc; nsegs++) { 2226 tdata->jme_tx_ring[cons].flags = 0; 2227 JME_DESC_INC(cons, tdata->jme_tx_desc_cnt); 2228 } 2229 2230 /* Reclaim transferred mbufs. */ 2231 bus_dmamap_unload(tdata->jme_tx_tag, txd->tx_dmamap); 2232 m_freem(txd->tx_m); 2233 txd->tx_m = NULL; 2234 tdata->jme_tx_cnt -= txd->tx_ndesc; 2235 KASSERT(tdata->jme_tx_cnt >= 0, 2236 ("%s: Active Tx desc counter was garbled", __func__)); 2237 txd->tx_ndesc = 0; 2238 } 2239 tdata->jme_tx_cons = cons; 2240 2241 /* 1 for symbol TX descriptor */ 2242 if (tdata->jme_tx_cnt <= JME_MAXTXSEGS + 1) 2243 ifp->if_timer = 0; 2244 2245 if (tdata->jme_tx_cnt + JME_TXD_SPARE <= 2246 tdata->jme_tx_desc_cnt - JME_TXD_RSVD) 2247 ifp->if_flags &= ~IFF_OACTIVE; 2248 } 2249 2250 static __inline void 2251 jme_discard_rxbufs(struct jme_rxdata *rdata, int cons, int count) 2252 { 2253 int i; 2254 2255 for (i = 0; i < count; ++i) { 2256 jme_setup_rxdesc(&rdata->jme_rxdesc[cons]); 2257 JME_DESC_INC(cons, rdata->jme_rx_desc_cnt); 2258 } 2259 } 2260 2261 static __inline struct pktinfo * 2262 jme_pktinfo(struct pktinfo *pi, uint32_t flags) 2263 { 2264 if (flags & JME_RD_IPV4) 2265 pi->pi_netisr = NETISR_IP; 2266 else if (flags & JME_RD_IPV6) 2267 pi->pi_netisr = NETISR_IPV6; 2268 else 2269 return NULL; 2270 2271 pi->pi_flags = 0; 2272 pi->pi_l3proto = IPPROTO_UNKNOWN; 2273 2274 if (flags & JME_RD_MORE_FRAG) 2275 pi->pi_flags |= PKTINFO_FLAG_FRAG; 2276 else if (flags & JME_RD_TCP) 2277 pi->pi_l3proto = IPPROTO_TCP; 2278 else if (flags & JME_RD_UDP) 2279 pi->pi_l3proto = IPPROTO_UDP; 2280 else 2281 pi = NULL; 2282 return pi; 2283 } 2284 2285 /* Receive a frame. */ 2286 static void 2287 jme_rxpkt(struct jme_rxdata *rdata) 2288 { 2289 struct ifnet *ifp = &rdata->jme_sc->arpcom.ac_if; 2290 struct jme_desc *desc; 2291 struct jme_rxdesc *rxd; 2292 struct mbuf *mp, *m; 2293 uint32_t flags, status, hash, hashinfo; 2294 int cons, count, nsegs; 2295 2296 cons = rdata->jme_rx_cons; 2297 desc = &rdata->jme_rx_ring[cons]; 2298 2299 flags = le32toh(desc->flags); 2300 status = le32toh(desc->buflen); 2301 hash = le32toh(desc->addr_hi); 2302 hashinfo = le32toh(desc->addr_lo); 2303 nsegs = JME_RX_NSEGS(status); 2304 2305 if (nsegs > 1) { 2306 /* Skip the first descriptor. */ 2307 JME_DESC_INC(cons, rdata->jme_rx_desc_cnt); 2308 2309 /* 2310 * Clear the OWN bit of the following RX descriptors; 2311 * hardware will not clear the OWN bit except the first 2312 * RX descriptor. 2313 * 2314 * Since the first RX descriptor is setup, i.e. OWN bit 2315 * on, before its followins RX descriptors, leaving the 2316 * OWN bit on the following RX descriptors will trick 2317 * the hardware into thinking that the following RX 2318 * descriptors are ready to be used too. 2319 */ 2320 for (count = 1; count < nsegs; count++, 2321 JME_DESC_INC(cons, rdata->jme_rx_desc_cnt)) 2322 rdata->jme_rx_ring[cons].flags = 0; 2323 2324 cons = rdata->jme_rx_cons; 2325 } 2326 2327 JME_RSS_DPRINTF(rdata->jme_sc, 15, "ring%d, flags 0x%08x, " 2328 "hash 0x%08x, hash info 0x%08x\n", 2329 rdata->jme_rx_idx, flags, hash, hashinfo); 2330 2331 if (status & JME_RX_ERR_STAT) { 2332 ifp->if_ierrors++; 2333 jme_discard_rxbufs(rdata, cons, nsegs); 2334 #ifdef JME_SHOW_ERRORS 2335 if_printf(ifp, "%s : receive error = 0x%b\n", 2336 __func__, JME_RX_ERR(status), JME_RX_ERR_BITS); 2337 #endif 2338 rdata->jme_rx_cons += nsegs; 2339 rdata->jme_rx_cons %= rdata->jme_rx_desc_cnt; 2340 return; 2341 } 2342 2343 rdata->jme_rxlen = JME_RX_BYTES(status) - JME_RX_PAD_BYTES; 2344 for (count = 0; count < nsegs; count++, 2345 JME_DESC_INC(cons, rdata->jme_rx_desc_cnt)) { 2346 rxd = &rdata->jme_rxdesc[cons]; 2347 mp = rxd->rx_m; 2348 2349 /* Add a new receive buffer to the ring. */ 2350 if (jme_newbuf(rdata, rxd, 0) != 0) { 2351 ifp->if_iqdrops++; 2352 /* Reuse buffer. */ 2353 jme_discard_rxbufs(rdata, cons, nsegs - count); 2354 if (rdata->jme_rxhead != NULL) { 2355 m_freem(rdata->jme_rxhead); 2356 JME_RXCHAIN_RESET(rdata); 2357 } 2358 break; 2359 } 2360 2361 /* 2362 * Assume we've received a full sized frame. 2363 * Actual size is fixed when we encounter the end of 2364 * multi-segmented frame. 2365 */ 2366 mp->m_len = MCLBYTES; 2367 2368 /* Chain received mbufs. */ 2369 if (rdata->jme_rxhead == NULL) { 2370 rdata->jme_rxhead = mp; 2371 rdata->jme_rxtail = mp; 2372 } else { 2373 /* 2374 * Receive processor can receive a maximum frame 2375 * size of 65535 bytes. 2376 */ 2377 rdata->jme_rxtail->m_next = mp; 2378 rdata->jme_rxtail = mp; 2379 } 2380 2381 if (count == nsegs - 1) { 2382 struct pktinfo pi0, *pi; 2383 2384 /* Last desc. for this frame. */ 2385 m = rdata->jme_rxhead; 2386 m->m_pkthdr.len = rdata->jme_rxlen; 2387 if (nsegs > 1) { 2388 /* Set first mbuf size. */ 2389 m->m_len = MCLBYTES - JME_RX_PAD_BYTES; 2390 /* Set last mbuf size. */ 2391 mp->m_len = rdata->jme_rxlen - 2392 ((MCLBYTES - JME_RX_PAD_BYTES) + 2393 (MCLBYTES * (nsegs - 2))); 2394 } else { 2395 m->m_len = rdata->jme_rxlen; 2396 } 2397 m->m_pkthdr.rcvif = ifp; 2398 2399 /* 2400 * Account for 10bytes auto padding which is used 2401 * to align IP header on 32bit boundary. Also note, 2402 * CRC bytes is automatically removed by the 2403 * hardware. 2404 */ 2405 m->m_data += JME_RX_PAD_BYTES; 2406 2407 /* Set checksum information. */ 2408 if ((ifp->if_capenable & IFCAP_RXCSUM) && 2409 (flags & JME_RD_IPV4)) { 2410 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2411 if (flags & JME_RD_IPCSUM) 2412 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2413 if ((flags & JME_RD_MORE_FRAG) == 0 && 2414 ((flags & (JME_RD_TCP | JME_RD_TCPCSUM)) == 2415 (JME_RD_TCP | JME_RD_TCPCSUM) || 2416 (flags & (JME_RD_UDP | JME_RD_UDPCSUM)) == 2417 (JME_RD_UDP | JME_RD_UDPCSUM))) { 2418 m->m_pkthdr.csum_flags |= 2419 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 2420 m->m_pkthdr.csum_data = 0xffff; 2421 } 2422 } 2423 2424 /* Check for VLAN tagged packets. */ 2425 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) && 2426 (flags & JME_RD_VLAN_TAG)) { 2427 m->m_pkthdr.ether_vlantag = 2428 flags & JME_RD_VLAN_MASK; 2429 m->m_flags |= M_VLANTAG; 2430 } 2431 2432 ifp->if_ipackets++; 2433 2434 if (ifp->if_capenable & IFCAP_RSS) 2435 pi = jme_pktinfo(&pi0, flags); 2436 else 2437 pi = NULL; 2438 2439 if (pi != NULL && 2440 (hashinfo & JME_RD_HASH_FN_MASK) == 2441 JME_RD_HASH_FN_TOEPLITZ) { 2442 m->m_flags |= (M_HASH | M_CKHASH); 2443 m->m_pkthdr.hash = toeplitz_hash(hash); 2444 } 2445 2446 #ifdef JME_RSS_DEBUG 2447 if (pi != NULL) { 2448 JME_RSS_DPRINTF(rdata->jme_sc, 10, 2449 "isr %d flags %08x, l3 %d %s\n", 2450 pi->pi_netisr, pi->pi_flags, 2451 pi->pi_l3proto, 2452 (m->m_flags & M_HASH) ? "hash" : ""); 2453 } 2454 #endif 2455 2456 /* Pass it on. */ 2457 ether_input_pkt(ifp, m, pi); 2458 2459 /* Reset mbuf chains. */ 2460 JME_RXCHAIN_RESET(rdata); 2461 #ifdef JME_RSS_DEBUG 2462 rdata->jme_rx_pkt++; 2463 #endif 2464 } 2465 } 2466 2467 rdata->jme_rx_cons += nsegs; 2468 rdata->jme_rx_cons %= rdata->jme_rx_desc_cnt; 2469 } 2470 2471 static void 2472 jme_rxeof(struct jme_rxdata *rdata, int count) 2473 { 2474 struct jme_desc *desc; 2475 int nsegs, pktlen; 2476 2477 for (;;) { 2478 #ifdef IFPOLL_ENABLE 2479 if (count >= 0 && count-- == 0) 2480 break; 2481 #endif 2482 desc = &rdata->jme_rx_ring[rdata->jme_rx_cons]; 2483 if ((le32toh(desc->flags) & JME_RD_OWN) == JME_RD_OWN) 2484 break; 2485 if ((le32toh(desc->buflen) & JME_RD_VALID) == 0) 2486 break; 2487 2488 /* 2489 * Check number of segments against received bytes. 2490 * Non-matching value would indicate that hardware 2491 * is still trying to update Rx descriptors. I'm not 2492 * sure whether this check is needed. 2493 */ 2494 nsegs = JME_RX_NSEGS(le32toh(desc->buflen)); 2495 pktlen = JME_RX_BYTES(le32toh(desc->buflen)); 2496 if (nsegs != howmany(pktlen, MCLBYTES)) { 2497 if_printf(&rdata->jme_sc->arpcom.ac_if, 2498 "RX fragment count(%d) and " 2499 "packet size(%d) mismach\n", nsegs, pktlen); 2500 break; 2501 } 2502 2503 /* 2504 * NOTE: 2505 * RSS hash and hash information may _not_ be set by the 2506 * hardware even if the OWN bit is cleared and VALID bit 2507 * is set. 2508 * 2509 * If the RSS information is not delivered by the hardware 2510 * yet, we MUST NOT accept this packet, let alone reusing 2511 * its RX descriptor. If this packet was accepted and its 2512 * RX descriptor was reused before hardware delivering the 2513 * RSS information, the RX buffer's address would be trashed 2514 * by the RSS information delivered by the hardware. 2515 */ 2516 if (JME_ENABLE_HWRSS(rdata->jme_sc)) { 2517 struct jme_rxdesc *rxd; 2518 uint32_t hashinfo; 2519 2520 hashinfo = le32toh(desc->addr_lo); 2521 rxd = &rdata->jme_rxdesc[rdata->jme_rx_cons]; 2522 2523 /* 2524 * This test should be enough to detect the pending 2525 * RSS information delivery, given: 2526 * - If RSS hash is not calculated, the hashinfo 2527 * will be 0. Howvever, the lower 32bits of RX 2528 * buffers' physical address will never be 0. 2529 * (see jme_rxbuf_dma_filter) 2530 * - If RSS hash is calculated, the lowest 4 bits 2531 * of hashinfo will be set, while the RX buffers 2532 * are at least 2K aligned. 2533 */ 2534 if (hashinfo == JME_ADDR_LO(rxd->rx_paddr)) { 2535 #ifdef JME_SHOW_RSSWB 2536 if_printf(&rdata->jme_sc->arpcom.ac_if, 2537 "RSS is not written back yet\n"); 2538 #endif 2539 break; 2540 } 2541 } 2542 2543 /* Received a frame. */ 2544 jme_rxpkt(rdata); 2545 } 2546 } 2547 2548 static void 2549 jme_tick(void *xsc) 2550 { 2551 struct jme_softc *sc = xsc; 2552 struct mii_data *mii = device_get_softc(sc->jme_miibus); 2553 2554 lwkt_serialize_enter(&sc->jme_serialize); 2555 2556 KKASSERT(mycpuid == JME_TICK_CPUID); 2557 2558 sc->jme_in_tick = TRUE; 2559 mii_tick(mii); 2560 sc->jme_in_tick = FALSE; 2561 2562 callout_reset(&sc->jme_tick_ch, hz, jme_tick, sc); 2563 2564 lwkt_serialize_exit(&sc->jme_serialize); 2565 } 2566 2567 static void 2568 jme_reset(struct jme_softc *sc) 2569 { 2570 uint32_t val; 2571 2572 /* Make sure that TX and RX are stopped */ 2573 jme_stop_tx(sc); 2574 jme_stop_rx(sc); 2575 2576 /* Start reset */ 2577 CSR_WRITE_4(sc, JME_GHC, GHC_RESET); 2578 DELAY(20); 2579 2580 /* 2581 * Hold reset bit before stop reset 2582 */ 2583 2584 /* Disable TXMAC and TXOFL clock sources */ 2585 CSR_WRITE_4(sc, JME_GHC, GHC_RESET); 2586 /* Disable RXMAC clock source */ 2587 val = CSR_READ_4(sc, JME_GPREG1); 2588 CSR_WRITE_4(sc, JME_GPREG1, val | GPREG1_DIS_RXMAC_CLKSRC); 2589 /* Flush */ 2590 CSR_READ_4(sc, JME_GHC); 2591 2592 /* Stop reset */ 2593 CSR_WRITE_4(sc, JME_GHC, 0); 2594 /* Flush */ 2595 CSR_READ_4(sc, JME_GHC); 2596 2597 /* 2598 * Clear reset bit after stop reset 2599 */ 2600 2601 /* Enable TXMAC and TXOFL clock sources */ 2602 CSR_WRITE_4(sc, JME_GHC, GHC_TXOFL_CLKSRC | GHC_TXMAC_CLKSRC); 2603 /* Enable RXMAC clock source */ 2604 val = CSR_READ_4(sc, JME_GPREG1); 2605 CSR_WRITE_4(sc, JME_GPREG1, val & ~GPREG1_DIS_RXMAC_CLKSRC); 2606 /* Flush */ 2607 CSR_READ_4(sc, JME_GHC); 2608 2609 /* Disable TXMAC and TXOFL clock sources */ 2610 CSR_WRITE_4(sc, JME_GHC, 0); 2611 /* Disable RXMAC clock source */ 2612 val = CSR_READ_4(sc, JME_GPREG1); 2613 CSR_WRITE_4(sc, JME_GPREG1, val | GPREG1_DIS_RXMAC_CLKSRC); 2614 /* Flush */ 2615 CSR_READ_4(sc, JME_GHC); 2616 2617 /* Enable TX and RX */ 2618 val = CSR_READ_4(sc, JME_TXCSR); 2619 CSR_WRITE_4(sc, JME_TXCSR, val | TXCSR_TX_ENB); 2620 val = CSR_READ_4(sc, JME_RXCSR); 2621 CSR_WRITE_4(sc, JME_RXCSR, val | RXCSR_RX_ENB); 2622 /* Flush */ 2623 CSR_READ_4(sc, JME_TXCSR); 2624 CSR_READ_4(sc, JME_RXCSR); 2625 2626 /* Enable TXMAC and TXOFL clock sources */ 2627 CSR_WRITE_4(sc, JME_GHC, GHC_TXOFL_CLKSRC | GHC_TXMAC_CLKSRC); 2628 /* Eisable RXMAC clock source */ 2629 val = CSR_READ_4(sc, JME_GPREG1); 2630 CSR_WRITE_4(sc, JME_GPREG1, val & ~GPREG1_DIS_RXMAC_CLKSRC); 2631 /* Flush */ 2632 CSR_READ_4(sc, JME_GHC); 2633 2634 /* Stop TX and RX */ 2635 jme_stop_tx(sc); 2636 jme_stop_rx(sc); 2637 } 2638 2639 static void 2640 jme_init(void *xsc) 2641 { 2642 struct jme_softc *sc = xsc; 2643 struct ifnet *ifp = &sc->arpcom.ac_if; 2644 struct mii_data *mii; 2645 uint8_t eaddr[ETHER_ADDR_LEN]; 2646 bus_addr_t paddr; 2647 uint32_t reg; 2648 int error, r; 2649 2650 ASSERT_IFNET_SERIALIZED_ALL(ifp); 2651 2652 /* 2653 * Cancel any pending I/O. 2654 */ 2655 jme_stop(sc); 2656 2657 /* 2658 * Reset the chip to a known state. 2659 */ 2660 jme_reset(sc); 2661 2662 /* 2663 * Setup MSI/MSI-X vectors to interrupts mapping 2664 */ 2665 jme_set_msinum(sc); 2666 2667 if (JME_ENABLE_HWRSS(sc)) 2668 jme_enable_rss(sc); 2669 else 2670 jme_disable_rss(sc); 2671 2672 /* Init RX descriptors */ 2673 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 2674 error = jme_init_rx_ring(&sc->jme_cdata.jme_rx_data[r]); 2675 if (error) { 2676 if_printf(ifp, "initialization failed: " 2677 "no memory for %dth RX ring.\n", r); 2678 jme_stop(sc); 2679 return; 2680 } 2681 } 2682 2683 /* Init TX descriptors */ 2684 jme_init_tx_ring(&sc->jme_cdata.jme_tx_data); 2685 2686 /* Initialize shadow status block. */ 2687 jme_init_ssb(sc); 2688 2689 /* Reprogram the station address. */ 2690 bcopy(IF_LLADDR(ifp), eaddr, ETHER_ADDR_LEN); 2691 CSR_WRITE_4(sc, JME_PAR0, 2692 eaddr[3] << 24 | eaddr[2] << 16 | eaddr[1] << 8 | eaddr[0]); 2693 CSR_WRITE_4(sc, JME_PAR1, eaddr[5] << 8 | eaddr[4]); 2694 2695 /* 2696 * Configure Tx queue. 2697 * Tx priority queue weight value : 0 2698 * Tx FIFO threshold for processing next packet : 16QW 2699 * Maximum Tx DMA length : 512 2700 * Allow Tx DMA burst. 2701 */ 2702 sc->jme_txcsr = TXCSR_TXQ_N_SEL(TXCSR_TXQ0); 2703 sc->jme_txcsr |= TXCSR_TXQ_WEIGHT(TXCSR_TXQ_WEIGHT_MIN); 2704 sc->jme_txcsr |= TXCSR_FIFO_THRESH_16QW; 2705 sc->jme_txcsr |= sc->jme_tx_dma_size; 2706 sc->jme_txcsr |= TXCSR_DMA_BURST; 2707 CSR_WRITE_4(sc, JME_TXCSR, sc->jme_txcsr); 2708 2709 /* Set Tx descriptor counter. */ 2710 CSR_WRITE_4(sc, JME_TXQDC, sc->jme_cdata.jme_tx_data.jme_tx_desc_cnt); 2711 2712 /* Set Tx ring address to the hardware. */ 2713 paddr = sc->jme_cdata.jme_tx_data.jme_tx_ring_paddr; 2714 CSR_WRITE_4(sc, JME_TXDBA_HI, JME_ADDR_HI(paddr)); 2715 CSR_WRITE_4(sc, JME_TXDBA_LO, JME_ADDR_LO(paddr)); 2716 2717 /* Configure TxMAC parameters. */ 2718 reg = TXMAC_IFG1_DEFAULT | TXMAC_IFG2_DEFAULT | TXMAC_IFG_ENB; 2719 reg |= TXMAC_THRESH_1_PKT; 2720 reg |= TXMAC_CRC_ENB | TXMAC_PAD_ENB; 2721 CSR_WRITE_4(sc, JME_TXMAC, reg); 2722 2723 /* 2724 * Configure Rx queue. 2725 * FIFO full threshold for transmitting Tx pause packet : 128T 2726 * FIFO threshold for processing next packet : 128QW 2727 * Rx queue 0 select 2728 * Max Rx DMA length : 128 2729 * Rx descriptor retry : 32 2730 * Rx descriptor retry time gap : 256ns 2731 * Don't receive runt/bad frame. 2732 */ 2733 sc->jme_rxcsr = RXCSR_FIFO_FTHRESH_128T; 2734 #if 0 2735 /* 2736 * Since Rx FIFO size is 4K bytes, receiving frames larger 2737 * than 4K bytes will suffer from Rx FIFO overruns. So 2738 * decrease FIFO threshold to reduce the FIFO overruns for 2739 * frames larger than 4000 bytes. 2740 * For best performance of standard MTU sized frames use 2741 * maximum allowable FIFO threshold, 128QW. 2742 */ 2743 if ((ifp->if_mtu + ETHER_HDR_LEN + EVL_ENCAPLEN + ETHER_CRC_LEN) > 2744 JME_RX_FIFO_SIZE) 2745 sc->jme_rxcsr |= RXCSR_FIFO_THRESH_16QW; 2746 else 2747 sc->jme_rxcsr |= RXCSR_FIFO_THRESH_128QW; 2748 #else 2749 /* Improve PCI Express compatibility */ 2750 sc->jme_rxcsr |= RXCSR_FIFO_THRESH_16QW; 2751 #endif 2752 sc->jme_rxcsr |= sc->jme_rx_dma_size; 2753 sc->jme_rxcsr |= RXCSR_DESC_RT_CNT(RXCSR_DESC_RT_CNT_DEFAULT); 2754 sc->jme_rxcsr |= RXCSR_DESC_RT_GAP_256 & RXCSR_DESC_RT_GAP_MASK; 2755 /* XXX TODO DROP_BAD */ 2756 2757 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 2758 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[r]; 2759 2760 CSR_WRITE_4(sc, JME_RXCSR, sc->jme_rxcsr | RXCSR_RXQ_N_SEL(r)); 2761 2762 /* Set Rx descriptor counter. */ 2763 CSR_WRITE_4(sc, JME_RXQDC, rdata->jme_rx_desc_cnt); 2764 2765 /* Set Rx ring address to the hardware. */ 2766 paddr = rdata->jme_rx_ring_paddr; 2767 CSR_WRITE_4(sc, JME_RXDBA_HI, JME_ADDR_HI(paddr)); 2768 CSR_WRITE_4(sc, JME_RXDBA_LO, JME_ADDR_LO(paddr)); 2769 } 2770 2771 /* Clear receive filter. */ 2772 CSR_WRITE_4(sc, JME_RXMAC, 0); 2773 2774 /* Set up the receive filter. */ 2775 jme_set_filter(sc); 2776 jme_set_vlan(sc); 2777 2778 /* 2779 * Disable all WOL bits as WOL can interfere normal Rx 2780 * operation. Also clear WOL detection status bits. 2781 */ 2782 reg = CSR_READ_4(sc, JME_PMCS); 2783 reg &= ~PMCS_WOL_ENB_MASK; 2784 CSR_WRITE_4(sc, JME_PMCS, reg); 2785 2786 /* 2787 * Pad 10bytes right before received frame. This will greatly 2788 * help Rx performance on strict-alignment architectures as 2789 * it does not need to copy the frame to align the payload. 2790 */ 2791 reg = CSR_READ_4(sc, JME_RXMAC); 2792 reg |= RXMAC_PAD_10BYTES; 2793 2794 if (ifp->if_capenable & IFCAP_RXCSUM) 2795 reg |= RXMAC_CSUM_ENB; 2796 CSR_WRITE_4(sc, JME_RXMAC, reg); 2797 2798 /* Configure general purpose reg0 */ 2799 reg = CSR_READ_4(sc, JME_GPREG0); 2800 reg &= ~GPREG0_PCC_UNIT_MASK; 2801 /* Set PCC timer resolution to micro-seconds unit. */ 2802 reg |= GPREG0_PCC_UNIT_US; 2803 /* 2804 * Disable all shadow register posting as we have to read 2805 * JME_INTR_STATUS register in jme_intr. Also it seems 2806 * that it's hard to synchronize interrupt status between 2807 * hardware and software with shadow posting due to 2808 * requirements of bus_dmamap_sync(9). 2809 */ 2810 reg |= GPREG0_SH_POST_DW7_DIS | GPREG0_SH_POST_DW6_DIS | 2811 GPREG0_SH_POST_DW5_DIS | GPREG0_SH_POST_DW4_DIS | 2812 GPREG0_SH_POST_DW3_DIS | GPREG0_SH_POST_DW2_DIS | 2813 GPREG0_SH_POST_DW1_DIS | GPREG0_SH_POST_DW0_DIS; 2814 /* Disable posting of DW0. */ 2815 reg &= ~GPREG0_POST_DW0_ENB; 2816 /* Clear PME message. */ 2817 reg &= ~GPREG0_PME_ENB; 2818 /* Set PHY address. */ 2819 reg &= ~GPREG0_PHY_ADDR_MASK; 2820 reg |= sc->jme_phyaddr; 2821 CSR_WRITE_4(sc, JME_GPREG0, reg); 2822 2823 /* Configure Tx queue 0 packet completion coalescing. */ 2824 jme_set_tx_coal(sc); 2825 2826 /* Configure Rx queues packet completion coalescing. */ 2827 jme_set_rx_coal(sc); 2828 2829 /* Configure shadow status block but don't enable posting. */ 2830 paddr = sc->jme_cdata.jme_ssb_block_paddr; 2831 CSR_WRITE_4(sc, JME_SHBASE_ADDR_HI, JME_ADDR_HI(paddr)); 2832 CSR_WRITE_4(sc, JME_SHBASE_ADDR_LO, JME_ADDR_LO(paddr)); 2833 2834 /* Disable Timer 1 and Timer 2. */ 2835 CSR_WRITE_4(sc, JME_TIMER1, 0); 2836 CSR_WRITE_4(sc, JME_TIMER2, 0); 2837 2838 /* Configure retry transmit period, retry limit value. */ 2839 CSR_WRITE_4(sc, JME_TXTRHD, 2840 ((TXTRHD_RT_PERIOD_DEFAULT << TXTRHD_RT_PERIOD_SHIFT) & 2841 TXTRHD_RT_PERIOD_MASK) | 2842 ((TXTRHD_RT_LIMIT_DEFAULT << TXTRHD_RT_LIMIT_SHIFT) & 2843 TXTRHD_RT_LIMIT_SHIFT)); 2844 2845 #ifdef IFPOLL_ENABLE 2846 if (!(ifp->if_flags & IFF_NPOLLING)) 2847 #endif 2848 /* Initialize the interrupt mask. */ 2849 jme_enable_intr(sc); 2850 CSR_WRITE_4(sc, JME_INTR_STATUS, 0xFFFFFFFF); 2851 2852 /* 2853 * Enabling Tx/Rx DMA engines and Rx queue processing is 2854 * done after detection of valid link in jme_miibus_statchg. 2855 */ 2856 sc->jme_has_link = FALSE; 2857 2858 /* Set the current media. */ 2859 mii = device_get_softc(sc->jme_miibus); 2860 mii_mediachg(mii); 2861 2862 callout_reset_bycpu(&sc->jme_tick_ch, hz, jme_tick, sc, 2863 JME_TICK_CPUID); 2864 2865 ifp->if_flags |= IFF_RUNNING; 2866 ifp->if_flags &= ~IFF_OACTIVE; 2867 } 2868 2869 static void 2870 jme_stop(struct jme_softc *sc) 2871 { 2872 struct ifnet *ifp = &sc->arpcom.ac_if; 2873 struct jme_txdata *tdata = &sc->jme_cdata.jme_tx_data; 2874 struct jme_txdesc *txd; 2875 struct jme_rxdesc *rxd; 2876 struct jme_rxdata *rdata; 2877 int i, r; 2878 2879 ASSERT_IFNET_SERIALIZED_ALL(ifp); 2880 2881 /* 2882 * Mark the interface down and cancel the watchdog timer. 2883 */ 2884 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 2885 ifp->if_timer = 0; 2886 2887 callout_stop(&sc->jme_tick_ch); 2888 sc->jme_has_link = FALSE; 2889 2890 /* 2891 * Disable interrupts. 2892 */ 2893 jme_disable_intr(sc); 2894 CSR_WRITE_4(sc, JME_INTR_STATUS, 0xFFFFFFFF); 2895 2896 /* Disable updating shadow status block. */ 2897 CSR_WRITE_4(sc, JME_SHBASE_ADDR_LO, 2898 CSR_READ_4(sc, JME_SHBASE_ADDR_LO) & ~SHBASE_POST_ENB); 2899 2900 /* Stop receiver, transmitter. */ 2901 jme_stop_rx(sc); 2902 jme_stop_tx(sc); 2903 2904 /* 2905 * Free partial finished RX segments 2906 */ 2907 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 2908 rdata = &sc->jme_cdata.jme_rx_data[r]; 2909 if (rdata->jme_rxhead != NULL) 2910 m_freem(rdata->jme_rxhead); 2911 JME_RXCHAIN_RESET(rdata); 2912 } 2913 2914 /* 2915 * Free RX and TX mbufs still in the queues. 2916 */ 2917 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 2918 rdata = &sc->jme_cdata.jme_rx_data[r]; 2919 for (i = 0; i < rdata->jme_rx_desc_cnt; i++) { 2920 rxd = &rdata->jme_rxdesc[i]; 2921 if (rxd->rx_m != NULL) { 2922 bus_dmamap_unload(rdata->jme_rx_tag, 2923 rxd->rx_dmamap); 2924 m_freem(rxd->rx_m); 2925 rxd->rx_m = NULL; 2926 } 2927 } 2928 } 2929 for (i = 0; i < tdata->jme_tx_desc_cnt; i++) { 2930 txd = &tdata->jme_txdesc[i]; 2931 if (txd->tx_m != NULL) { 2932 bus_dmamap_unload(tdata->jme_tx_tag, txd->tx_dmamap); 2933 m_freem(txd->tx_m); 2934 txd->tx_m = NULL; 2935 txd->tx_ndesc = 0; 2936 } 2937 } 2938 } 2939 2940 static void 2941 jme_stop_tx(struct jme_softc *sc) 2942 { 2943 uint32_t reg; 2944 int i; 2945 2946 reg = CSR_READ_4(sc, JME_TXCSR); 2947 if ((reg & TXCSR_TX_ENB) == 0) 2948 return; 2949 reg &= ~TXCSR_TX_ENB; 2950 CSR_WRITE_4(sc, JME_TXCSR, reg); 2951 for (i = JME_TIMEOUT; i > 0; i--) { 2952 DELAY(1); 2953 if ((CSR_READ_4(sc, JME_TXCSR) & TXCSR_TX_ENB) == 0) 2954 break; 2955 } 2956 if (i == 0) 2957 device_printf(sc->jme_dev, "stopping transmitter timeout!\n"); 2958 } 2959 2960 static void 2961 jme_stop_rx(struct jme_softc *sc) 2962 { 2963 uint32_t reg; 2964 int i; 2965 2966 reg = CSR_READ_4(sc, JME_RXCSR); 2967 if ((reg & RXCSR_RX_ENB) == 0) 2968 return; 2969 reg &= ~RXCSR_RX_ENB; 2970 CSR_WRITE_4(sc, JME_RXCSR, reg); 2971 for (i = JME_TIMEOUT; i > 0; i--) { 2972 DELAY(1); 2973 if ((CSR_READ_4(sc, JME_RXCSR) & RXCSR_RX_ENB) == 0) 2974 break; 2975 } 2976 if (i == 0) 2977 device_printf(sc->jme_dev, "stopping recevier timeout!\n"); 2978 } 2979 2980 static void 2981 jme_init_tx_ring(struct jme_txdata *tdata) 2982 { 2983 struct jme_txdesc *txd; 2984 int i; 2985 2986 tdata->jme_tx_prod = 0; 2987 tdata->jme_tx_cons = 0; 2988 tdata->jme_tx_cnt = 0; 2989 2990 bzero(tdata->jme_tx_ring, JME_TX_RING_SIZE(tdata)); 2991 for (i = 0; i < tdata->jme_tx_desc_cnt; i++) { 2992 txd = &tdata->jme_txdesc[i]; 2993 txd->tx_m = NULL; 2994 txd->tx_desc = &tdata->jme_tx_ring[i]; 2995 txd->tx_ndesc = 0; 2996 } 2997 } 2998 2999 static void 3000 jme_init_ssb(struct jme_softc *sc) 3001 { 3002 struct jme_chain_data *cd; 3003 3004 cd = &sc->jme_cdata; 3005 bzero(cd->jme_ssb_block, JME_SSB_SIZE); 3006 } 3007 3008 static int 3009 jme_init_rx_ring(struct jme_rxdata *rdata) 3010 { 3011 struct jme_rxdesc *rxd; 3012 int i; 3013 3014 KKASSERT(rdata->jme_rxhead == NULL && 3015 rdata->jme_rxtail == NULL && 3016 rdata->jme_rxlen == 0); 3017 rdata->jme_rx_cons = 0; 3018 3019 bzero(rdata->jme_rx_ring, JME_RX_RING_SIZE(rdata)); 3020 for (i = 0; i < rdata->jme_rx_desc_cnt; i++) { 3021 int error; 3022 3023 rxd = &rdata->jme_rxdesc[i]; 3024 rxd->rx_m = NULL; 3025 rxd->rx_desc = &rdata->jme_rx_ring[i]; 3026 error = jme_newbuf(rdata, rxd, 1); 3027 if (error) 3028 return error; 3029 } 3030 return 0; 3031 } 3032 3033 static int 3034 jme_newbuf(struct jme_rxdata *rdata, struct jme_rxdesc *rxd, int init) 3035 { 3036 struct mbuf *m; 3037 bus_dma_segment_t segs; 3038 bus_dmamap_t map; 3039 int error, nsegs; 3040 3041 m = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR); 3042 if (m == NULL) 3043 return ENOBUFS; 3044 /* 3045 * JMC250 has 64bit boundary alignment limitation so jme(4) 3046 * takes advantage of 10 bytes padding feature of hardware 3047 * in order not to copy entire frame to align IP header on 3048 * 32bit boundary. 3049 */ 3050 m->m_len = m->m_pkthdr.len = MCLBYTES; 3051 3052 error = bus_dmamap_load_mbuf_segment(rdata->jme_rx_tag, 3053 rdata->jme_rx_sparemap, m, &segs, 1, &nsegs, 3054 BUS_DMA_NOWAIT); 3055 if (error) { 3056 m_freem(m); 3057 if (init) { 3058 if_printf(&rdata->jme_sc->arpcom.ac_if, 3059 "can't load RX mbuf\n"); 3060 } 3061 return error; 3062 } 3063 3064 if (rxd->rx_m != NULL) { 3065 bus_dmamap_sync(rdata->jme_rx_tag, rxd->rx_dmamap, 3066 BUS_DMASYNC_POSTREAD); 3067 bus_dmamap_unload(rdata->jme_rx_tag, rxd->rx_dmamap); 3068 } 3069 map = rxd->rx_dmamap; 3070 rxd->rx_dmamap = rdata->jme_rx_sparemap; 3071 rdata->jme_rx_sparemap = map; 3072 rxd->rx_m = m; 3073 rxd->rx_paddr = segs.ds_addr; 3074 3075 jme_setup_rxdesc(rxd); 3076 return 0; 3077 } 3078 3079 static void 3080 jme_set_vlan(struct jme_softc *sc) 3081 { 3082 struct ifnet *ifp = &sc->arpcom.ac_if; 3083 uint32_t reg; 3084 3085 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3086 3087 reg = CSR_READ_4(sc, JME_RXMAC); 3088 reg &= ~RXMAC_VLAN_ENB; 3089 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 3090 reg |= RXMAC_VLAN_ENB; 3091 CSR_WRITE_4(sc, JME_RXMAC, reg); 3092 } 3093 3094 static void 3095 jme_set_filter(struct jme_softc *sc) 3096 { 3097 struct ifnet *ifp = &sc->arpcom.ac_if; 3098 struct ifmultiaddr *ifma; 3099 uint32_t crc; 3100 uint32_t mchash[2]; 3101 uint32_t rxcfg; 3102 3103 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3104 3105 rxcfg = CSR_READ_4(sc, JME_RXMAC); 3106 rxcfg &= ~(RXMAC_BROADCAST | RXMAC_PROMISC | RXMAC_MULTICAST | 3107 RXMAC_ALLMULTI); 3108 3109 /* 3110 * Always accept frames destined to our station address. 3111 * Always accept broadcast frames. 3112 */ 3113 rxcfg |= RXMAC_UNICAST | RXMAC_BROADCAST; 3114 3115 if (ifp->if_flags & (IFF_PROMISC | IFF_ALLMULTI)) { 3116 if (ifp->if_flags & IFF_PROMISC) 3117 rxcfg |= RXMAC_PROMISC; 3118 if (ifp->if_flags & IFF_ALLMULTI) 3119 rxcfg |= RXMAC_ALLMULTI; 3120 CSR_WRITE_4(sc, JME_MAR0, 0xFFFFFFFF); 3121 CSR_WRITE_4(sc, JME_MAR1, 0xFFFFFFFF); 3122 CSR_WRITE_4(sc, JME_RXMAC, rxcfg); 3123 return; 3124 } 3125 3126 /* 3127 * Set up the multicast address filter by passing all multicast 3128 * addresses through a CRC generator, and then using the low-order 3129 * 6 bits as an index into the 64 bit multicast hash table. The 3130 * high order bits select the register, while the rest of the bits 3131 * select the bit within the register. 3132 */ 3133 rxcfg |= RXMAC_MULTICAST; 3134 bzero(mchash, sizeof(mchash)); 3135 3136 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3137 if (ifma->ifma_addr->sa_family != AF_LINK) 3138 continue; 3139 crc = ether_crc32_be(LLADDR((struct sockaddr_dl *) 3140 ifma->ifma_addr), ETHER_ADDR_LEN); 3141 3142 /* Just want the 6 least significant bits. */ 3143 crc &= 0x3f; 3144 3145 /* Set the corresponding bit in the hash table. */ 3146 mchash[crc >> 5] |= 1 << (crc & 0x1f); 3147 } 3148 3149 CSR_WRITE_4(sc, JME_MAR0, mchash[0]); 3150 CSR_WRITE_4(sc, JME_MAR1, mchash[1]); 3151 CSR_WRITE_4(sc, JME_RXMAC, rxcfg); 3152 } 3153 3154 static int 3155 jme_sysctl_tx_coal_to(SYSCTL_HANDLER_ARGS) 3156 { 3157 struct jme_softc *sc = arg1; 3158 struct ifnet *ifp = &sc->arpcom.ac_if; 3159 int error, v; 3160 3161 ifnet_serialize_all(ifp); 3162 3163 v = sc->jme_tx_coal_to; 3164 error = sysctl_handle_int(oidp, &v, 0, req); 3165 if (error || req->newptr == NULL) 3166 goto back; 3167 3168 if (v < PCCTX_COAL_TO_MIN || v > PCCTX_COAL_TO_MAX) { 3169 error = EINVAL; 3170 goto back; 3171 } 3172 3173 if (v != sc->jme_tx_coal_to) { 3174 sc->jme_tx_coal_to = v; 3175 if (ifp->if_flags & IFF_RUNNING) 3176 jme_set_tx_coal(sc); 3177 } 3178 back: 3179 ifnet_deserialize_all(ifp); 3180 return error; 3181 } 3182 3183 static int 3184 jme_sysctl_tx_coal_pkt(SYSCTL_HANDLER_ARGS) 3185 { 3186 struct jme_softc *sc = arg1; 3187 struct ifnet *ifp = &sc->arpcom.ac_if; 3188 int error, v; 3189 3190 ifnet_serialize_all(ifp); 3191 3192 v = sc->jme_tx_coal_pkt; 3193 error = sysctl_handle_int(oidp, &v, 0, req); 3194 if (error || req->newptr == NULL) 3195 goto back; 3196 3197 if (v < PCCTX_COAL_PKT_MIN || v > PCCTX_COAL_PKT_MAX) { 3198 error = EINVAL; 3199 goto back; 3200 } 3201 3202 if (v != sc->jme_tx_coal_pkt) { 3203 sc->jme_tx_coal_pkt = v; 3204 if (ifp->if_flags & IFF_RUNNING) 3205 jme_set_tx_coal(sc); 3206 } 3207 back: 3208 ifnet_deserialize_all(ifp); 3209 return error; 3210 } 3211 3212 static int 3213 jme_sysctl_rx_coal_to(SYSCTL_HANDLER_ARGS) 3214 { 3215 struct jme_softc *sc = arg1; 3216 struct ifnet *ifp = &sc->arpcom.ac_if; 3217 int error, v; 3218 3219 ifnet_serialize_all(ifp); 3220 3221 v = sc->jme_rx_coal_to; 3222 error = sysctl_handle_int(oidp, &v, 0, req); 3223 if (error || req->newptr == NULL) 3224 goto back; 3225 3226 if (v < PCCRX_COAL_TO_MIN || v > PCCRX_COAL_TO_MAX) { 3227 error = EINVAL; 3228 goto back; 3229 } 3230 3231 if (v != sc->jme_rx_coal_to) { 3232 sc->jme_rx_coal_to = v; 3233 if (ifp->if_flags & IFF_RUNNING) 3234 jme_set_rx_coal(sc); 3235 } 3236 back: 3237 ifnet_deserialize_all(ifp); 3238 return error; 3239 } 3240 3241 static int 3242 jme_sysctl_rx_coal_pkt(SYSCTL_HANDLER_ARGS) 3243 { 3244 struct jme_softc *sc = arg1; 3245 struct ifnet *ifp = &sc->arpcom.ac_if; 3246 int error, v; 3247 3248 ifnet_serialize_all(ifp); 3249 3250 v = sc->jme_rx_coal_pkt; 3251 error = sysctl_handle_int(oidp, &v, 0, req); 3252 if (error || req->newptr == NULL) 3253 goto back; 3254 3255 if (v < PCCRX_COAL_PKT_MIN || v > PCCRX_COAL_PKT_MAX) { 3256 error = EINVAL; 3257 goto back; 3258 } 3259 3260 if (v != sc->jme_rx_coal_pkt) { 3261 sc->jme_rx_coal_pkt = v; 3262 if (ifp->if_flags & IFF_RUNNING) 3263 jme_set_rx_coal(sc); 3264 } 3265 back: 3266 ifnet_deserialize_all(ifp); 3267 return error; 3268 } 3269 3270 static void 3271 jme_set_tx_coal(struct jme_softc *sc) 3272 { 3273 uint32_t reg; 3274 3275 reg = (sc->jme_tx_coal_to << PCCTX_COAL_TO_SHIFT) & 3276 PCCTX_COAL_TO_MASK; 3277 reg |= (sc->jme_tx_coal_pkt << PCCTX_COAL_PKT_SHIFT) & 3278 PCCTX_COAL_PKT_MASK; 3279 reg |= PCCTX_COAL_TXQ0; 3280 CSR_WRITE_4(sc, JME_PCCTX, reg); 3281 } 3282 3283 static void 3284 jme_set_rx_coal(struct jme_softc *sc) 3285 { 3286 uint32_t reg; 3287 int r; 3288 3289 reg = (sc->jme_rx_coal_to << PCCRX_COAL_TO_SHIFT) & 3290 PCCRX_COAL_TO_MASK; 3291 reg |= (sc->jme_rx_coal_pkt << PCCRX_COAL_PKT_SHIFT) & 3292 PCCRX_COAL_PKT_MASK; 3293 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) 3294 CSR_WRITE_4(sc, JME_PCCRX(r), reg); 3295 } 3296 3297 #ifdef IFPOLL_ENABLE 3298 3299 static void 3300 jme_npoll_status(struct ifnet *ifp) 3301 { 3302 struct jme_softc *sc = ifp->if_softc; 3303 uint32_t status; 3304 3305 ASSERT_SERIALIZED(&sc->jme_serialize); 3306 3307 status = CSR_READ_4(sc, JME_INTR_STATUS); 3308 if (status & INTR_RXQ_DESC_EMPTY) { 3309 CSR_WRITE_4(sc, JME_INTR_STATUS, status & INTR_RXQ_DESC_EMPTY); 3310 jme_rx_restart(sc, status); 3311 } 3312 } 3313 3314 static void 3315 jme_npoll_rx(struct ifnet *ifp __unused, void *arg, int cycle) 3316 { 3317 struct jme_rxdata *rdata = arg; 3318 3319 ASSERT_SERIALIZED(&rdata->jme_rx_serialize); 3320 3321 jme_rxeof(rdata, cycle); 3322 } 3323 3324 static void 3325 jme_npoll_tx(struct ifnet *ifp, void *arg, int cycle __unused) 3326 { 3327 struct jme_txdata *tdata = arg; 3328 3329 ASSERT_SERIALIZED(&tdata->jme_tx_serialize); 3330 3331 jme_txeof(tdata); 3332 if (!ifq_is_empty(&ifp->if_snd)) 3333 if_devstart(ifp); 3334 } 3335 3336 static void 3337 jme_npoll(struct ifnet *ifp, struct ifpoll_info *info) 3338 { 3339 struct jme_softc *sc = ifp->if_softc; 3340 3341 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3342 3343 if (info) { 3344 int i, off; 3345 3346 info->ifpi_status.status_func = jme_npoll_status; 3347 info->ifpi_status.serializer = &sc->jme_serialize; 3348 3349 off = sc->jme_npoll_txoff; 3350 KKASSERT(off <= ncpus2); 3351 info->ifpi_tx[off].poll_func = jme_npoll_tx; 3352 info->ifpi_tx[off].arg = &sc->jme_cdata.jme_tx_data; 3353 info->ifpi_tx[off].serializer = 3354 &sc->jme_cdata.jme_tx_data.jme_tx_serialize; 3355 3356 off = sc->jme_npoll_rxoff; 3357 for (i = 0; i < sc->jme_cdata.jme_rx_ring_cnt; ++i) { 3358 struct jme_rxdata *rdata = 3359 &sc->jme_cdata.jme_rx_data[i]; 3360 int idx = i + off; 3361 3362 info->ifpi_rx[idx].poll_func = jme_npoll_rx; 3363 info->ifpi_rx[idx].arg = rdata; 3364 info->ifpi_rx[idx].serializer = 3365 &rdata->jme_rx_serialize; 3366 } 3367 3368 if (ifp->if_flags & IFF_RUNNING) 3369 jme_disable_intr(sc); 3370 ifp->if_npoll_cpuid = sc->jme_npoll_txoff; 3371 } else { 3372 if (ifp->if_flags & IFF_RUNNING) 3373 jme_enable_intr(sc); 3374 ifp->if_npoll_cpuid = -1; 3375 } 3376 } 3377 3378 static int 3379 jme_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS) 3380 { 3381 struct jme_softc *sc = (void *)arg1; 3382 struct ifnet *ifp = &sc->arpcom.ac_if; 3383 int error, off; 3384 3385 off = sc->jme_npoll_rxoff; 3386 error = sysctl_handle_int(oidp, &off, 0, req); 3387 if (error || req->newptr == NULL) 3388 return error; 3389 if (off < 0) 3390 return EINVAL; 3391 3392 ifnet_serialize_all(ifp); 3393 if (off >= ncpus2 || off % sc->jme_cdata.jme_rx_ring_cnt != 0) { 3394 error = EINVAL; 3395 } else { 3396 error = 0; 3397 sc->jme_npoll_rxoff = off; 3398 } 3399 ifnet_deserialize_all(ifp); 3400 3401 return error; 3402 } 3403 3404 static int 3405 jme_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS) 3406 { 3407 struct jme_softc *sc = (void *)arg1; 3408 struct ifnet *ifp = &sc->arpcom.ac_if; 3409 int error, off; 3410 3411 off = sc->jme_npoll_txoff; 3412 error = sysctl_handle_int(oidp, &off, 0, req); 3413 if (error || req->newptr == NULL) 3414 return error; 3415 if (off < 0) 3416 return EINVAL; 3417 3418 ifnet_serialize_all(ifp); 3419 if (off >= ncpus2) { 3420 error = EINVAL; 3421 } else { 3422 error = 0; 3423 sc->jme_npoll_txoff = off; 3424 } 3425 ifnet_deserialize_all(ifp); 3426 3427 return error; 3428 } 3429 3430 #endif /* IFPOLL_ENABLE */ 3431 3432 static int 3433 jme_rxring_dma_alloc(struct jme_rxdata *rdata) 3434 { 3435 bus_dmamem_t dmem; 3436 int error, asize; 3437 3438 asize = roundup2(JME_RX_RING_SIZE(rdata), JME_RX_RING_ALIGN); 3439 error = bus_dmamem_coherent(rdata->jme_sc->jme_cdata.jme_ring_tag, 3440 JME_RX_RING_ALIGN, 0, 3441 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 3442 asize, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem); 3443 if (error) { 3444 device_printf(rdata->jme_sc->jme_dev, 3445 "could not allocate %dth Rx ring.\n", rdata->jme_rx_idx); 3446 return error; 3447 } 3448 rdata->jme_rx_ring_tag = dmem.dmem_tag; 3449 rdata->jme_rx_ring_map = dmem.dmem_map; 3450 rdata->jme_rx_ring = dmem.dmem_addr; 3451 rdata->jme_rx_ring_paddr = dmem.dmem_busaddr; 3452 3453 return 0; 3454 } 3455 3456 static int 3457 jme_rxbuf_dma_filter(void *arg __unused, bus_addr_t paddr) 3458 { 3459 if ((paddr & 0xffffffff) == 0) { 3460 /* 3461 * Don't allow lower 32bits of the RX buffer's 3462 * physical address to be 0, else it will break 3463 * hardware pending RSS information delivery 3464 * detection on RX path. 3465 */ 3466 return 1; 3467 } 3468 return 0; 3469 } 3470 3471 static int 3472 jme_rxbuf_dma_alloc(struct jme_rxdata *rdata) 3473 { 3474 bus_addr_t lowaddr; 3475 int i, error; 3476 3477 lowaddr = BUS_SPACE_MAXADDR; 3478 if (JME_ENABLE_HWRSS(rdata->jme_sc)) { 3479 /* jme_rxbuf_dma_filter will be called */ 3480 lowaddr = BUS_SPACE_MAXADDR_32BIT; 3481 } 3482 3483 /* Create tag for Rx buffers. */ 3484 error = bus_dma_tag_create( 3485 rdata->jme_sc->jme_cdata.jme_buffer_tag,/* parent */ 3486 JME_RX_BUF_ALIGN, 0, /* algnmnt, boundary */ 3487 lowaddr, /* lowaddr */ 3488 BUS_SPACE_MAXADDR, /* highaddr */ 3489 jme_rxbuf_dma_filter, NULL, /* filter, filterarg */ 3490 MCLBYTES, /* maxsize */ 3491 1, /* nsegments */ 3492 MCLBYTES, /* maxsegsize */ 3493 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ALIGNED,/* flags */ 3494 &rdata->jme_rx_tag); 3495 if (error) { 3496 device_printf(rdata->jme_sc->jme_dev, 3497 "could not create %dth Rx DMA tag.\n", rdata->jme_rx_idx); 3498 return error; 3499 } 3500 3501 /* Create DMA maps for Rx buffers. */ 3502 error = bus_dmamap_create(rdata->jme_rx_tag, BUS_DMA_WAITOK, 3503 &rdata->jme_rx_sparemap); 3504 if (error) { 3505 device_printf(rdata->jme_sc->jme_dev, 3506 "could not create %dth spare Rx dmamap.\n", 3507 rdata->jme_rx_idx); 3508 bus_dma_tag_destroy(rdata->jme_rx_tag); 3509 rdata->jme_rx_tag = NULL; 3510 return error; 3511 } 3512 for (i = 0; i < rdata->jme_rx_desc_cnt; i++) { 3513 struct jme_rxdesc *rxd = &rdata->jme_rxdesc[i]; 3514 3515 error = bus_dmamap_create(rdata->jme_rx_tag, BUS_DMA_WAITOK, 3516 &rxd->rx_dmamap); 3517 if (error) { 3518 int j; 3519 3520 device_printf(rdata->jme_sc->jme_dev, 3521 "could not create %dth Rx dmamap " 3522 "for %dth RX ring.\n", i, rdata->jme_rx_idx); 3523 3524 for (j = 0; j < i; ++j) { 3525 rxd = &rdata->jme_rxdesc[j]; 3526 bus_dmamap_destroy(rdata->jme_rx_tag, 3527 rxd->rx_dmamap); 3528 } 3529 bus_dmamap_destroy(rdata->jme_rx_tag, 3530 rdata->jme_rx_sparemap); 3531 bus_dma_tag_destroy(rdata->jme_rx_tag); 3532 rdata->jme_rx_tag = NULL; 3533 return error; 3534 } 3535 } 3536 return 0; 3537 } 3538 3539 static void 3540 jme_rx_intr(struct jme_softc *sc, uint32_t status) 3541 { 3542 int r; 3543 3544 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 3545 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[r]; 3546 3547 if (status & rdata->jme_rx_coal) { 3548 lwkt_serialize_enter(&rdata->jme_rx_serialize); 3549 jme_rxeof(rdata, -1); 3550 lwkt_serialize_exit(&rdata->jme_rx_serialize); 3551 } 3552 } 3553 } 3554 3555 static void 3556 jme_enable_rss(struct jme_softc *sc) 3557 { 3558 uint32_t rssc, ind; 3559 uint8_t key[RSSKEY_NREGS * RSSKEY_REGSIZE]; 3560 int i; 3561 3562 KASSERT(sc->jme_cdata.jme_rx_ring_cnt == JME_NRXRING_2 || 3563 sc->jme_cdata.jme_rx_ring_cnt == JME_NRXRING_4, 3564 ("%s: invalid # of RX rings (%d)", 3565 sc->arpcom.ac_if.if_xname, sc->jme_cdata.jme_rx_ring_cnt)); 3566 3567 rssc = RSSC_HASH_64_ENTRY; 3568 rssc |= RSSC_HASH_IPV4 | RSSC_HASH_IPV4_TCP; 3569 rssc |= sc->jme_cdata.jme_rx_ring_cnt >> 1; 3570 JME_RSS_DPRINTF(sc, 1, "rssc 0x%08x\n", rssc); 3571 CSR_WRITE_4(sc, JME_RSSC, rssc); 3572 3573 toeplitz_get_key(key, sizeof(key)); 3574 for (i = 0; i < RSSKEY_NREGS; ++i) { 3575 uint32_t keyreg; 3576 3577 keyreg = RSSKEY_REGVAL(key, i); 3578 JME_RSS_DPRINTF(sc, 5, "keyreg%d 0x%08x\n", i, keyreg); 3579 3580 CSR_WRITE_4(sc, RSSKEY_REG(i), keyreg); 3581 } 3582 3583 /* 3584 * Create redirect table in following fashion: 3585 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 3586 */ 3587 ind = 0; 3588 for (i = 0; i < RSSTBL_REGSIZE; ++i) { 3589 int q; 3590 3591 q = i % sc->jme_cdata.jme_rx_ring_cnt; 3592 ind |= q << (i * 8); 3593 } 3594 JME_RSS_DPRINTF(sc, 1, "ind 0x%08x\n", ind); 3595 3596 for (i = 0; i < RSSTBL_NREGS; ++i) 3597 CSR_WRITE_4(sc, RSSTBL_REG(i), ind); 3598 } 3599 3600 static void 3601 jme_disable_rss(struct jme_softc *sc) 3602 { 3603 CSR_WRITE_4(sc, JME_RSSC, RSSC_DIS_RSS); 3604 } 3605 3606 static void 3607 jme_serialize(struct ifnet *ifp, enum ifnet_serialize slz) 3608 { 3609 struct jme_softc *sc = ifp->if_softc; 3610 3611 ifnet_serialize_array_enter(sc->jme_serialize_arr, 3612 sc->jme_serialize_cnt, JME_TX_SERIALIZE, JME_RX_SERIALIZE, slz); 3613 } 3614 3615 static void 3616 jme_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) 3617 { 3618 struct jme_softc *sc = ifp->if_softc; 3619 3620 ifnet_serialize_array_exit(sc->jme_serialize_arr, 3621 sc->jme_serialize_cnt, JME_TX_SERIALIZE, JME_RX_SERIALIZE, slz); 3622 } 3623 3624 static int 3625 jme_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) 3626 { 3627 struct jme_softc *sc = ifp->if_softc; 3628 3629 return ifnet_serialize_array_try(sc->jme_serialize_arr, 3630 sc->jme_serialize_cnt, JME_TX_SERIALIZE, JME_RX_SERIALIZE, slz); 3631 } 3632 3633 #ifdef INVARIANTS 3634 3635 static void 3636 jme_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, 3637 boolean_t serialized) 3638 { 3639 struct jme_softc *sc = ifp->if_softc; 3640 3641 ifnet_serialize_array_assert(sc->jme_serialize_arr, 3642 sc->jme_serialize_cnt, JME_TX_SERIALIZE, JME_RX_SERIALIZE, 3643 slz, serialized); 3644 } 3645 3646 #endif /* INVARIANTS */ 3647 3648 static void 3649 jme_msix_try_alloc(device_t dev) 3650 { 3651 struct jme_softc *sc = device_get_softc(dev); 3652 struct jme_msix_data *msix; 3653 int error, i, r, msix_enable, msix_count; 3654 int offset, offset_def; 3655 3656 msix_count = JME_MSIXCNT(sc->jme_cdata.jme_rx_ring_cnt); 3657 KKASSERT(msix_count <= JME_NMSIX); 3658 3659 msix_enable = device_getenv_int(dev, "msix.enable", jme_msix_enable); 3660 3661 /* 3662 * We leave the 1st MSI-X vector unused, so we 3663 * actually need msix_count + 1 MSI-X vectors. 3664 */ 3665 if (!msix_enable || pci_msix_count(dev) < (msix_count + 1)) 3666 return; 3667 3668 for (i = 0; i < msix_count; ++i) 3669 sc->jme_msix[i].jme_msix_rid = -1; 3670 3671 i = 0; 3672 3673 /* 3674 * Setup status MSI-X 3675 */ 3676 3677 msix = &sc->jme_msix[i++]; 3678 msix->jme_msix_cpuid = 0; 3679 msix->jme_msix_arg = sc; 3680 msix->jme_msix_func = jme_msix_status; 3681 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 3682 msix->jme_msix_intrs |= 3683 sc->jme_cdata.jme_rx_data[r].jme_rx_empty; 3684 } 3685 msix->jme_msix_serialize = &sc->jme_serialize; 3686 ksnprintf(msix->jme_msix_desc, sizeof(msix->jme_msix_desc), "%s sts", 3687 device_get_nameunit(dev)); 3688 3689 /* 3690 * Setup TX MSI-X 3691 */ 3692 3693 offset_def = device_get_unit(dev) % ncpus2; 3694 offset = device_getenv_int(dev, "msix.txoff", offset_def); 3695 if (offset >= ncpus2) { 3696 device_printf(dev, "invalid msix.txoff %d, use %d\n", 3697 offset, offset_def); 3698 offset = offset_def; 3699 } 3700 3701 msix = &sc->jme_msix[i++]; 3702 msix->jme_msix_cpuid = offset; 3703 sc->jme_tx_cpuid = msix->jme_msix_cpuid; 3704 msix->jme_msix_arg = &sc->jme_cdata.jme_tx_data; 3705 msix->jme_msix_func = jme_msix_tx; 3706 msix->jme_msix_intrs = INTR_TXQ_COAL | INTR_TXQ_COAL_TO; 3707 msix->jme_msix_serialize = &sc->jme_cdata.jme_tx_data.jme_tx_serialize; 3708 ksnprintf(msix->jme_msix_desc, sizeof(msix->jme_msix_desc), "%s tx", 3709 device_get_nameunit(dev)); 3710 3711 /* 3712 * Setup RX MSI-X 3713 */ 3714 3715 if (sc->jme_cdata.jme_rx_ring_cnt == ncpus2) { 3716 offset = 0; 3717 } else { 3718 offset_def = (sc->jme_cdata.jme_rx_ring_cnt * 3719 device_get_unit(dev)) % ncpus2; 3720 3721 offset = device_getenv_int(dev, "msix.rxoff", offset_def); 3722 if (offset >= ncpus2 || 3723 offset % sc->jme_cdata.jme_rx_ring_cnt != 0) { 3724 device_printf(dev, "invalid msix.rxoff %d, use %d\n", 3725 offset, offset_def); 3726 offset = offset_def; 3727 } 3728 } 3729 3730 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) { 3731 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[r]; 3732 3733 msix = &sc->jme_msix[i++]; 3734 msix->jme_msix_cpuid = r + offset; 3735 KKASSERT(msix->jme_msix_cpuid < ncpus2); 3736 msix->jme_msix_arg = rdata; 3737 msix->jme_msix_func = jme_msix_rx; 3738 msix->jme_msix_intrs = rdata->jme_rx_coal; 3739 msix->jme_msix_serialize = &rdata->jme_rx_serialize; 3740 ksnprintf(msix->jme_msix_desc, sizeof(msix->jme_msix_desc), 3741 "%s rx%d", device_get_nameunit(dev), r); 3742 } 3743 3744 KKASSERT(i == msix_count); 3745 3746 error = pci_setup_msix(dev); 3747 if (error) 3748 return; 3749 3750 /* Setup jme_msix_cnt early, so we could cleanup */ 3751 sc->jme_msix_cnt = msix_count; 3752 3753 for (i = 0; i < msix_count; ++i) { 3754 msix = &sc->jme_msix[i]; 3755 3756 msix->jme_msix_vector = i + 1; 3757 error = pci_alloc_msix_vector(dev, msix->jme_msix_vector, 3758 &msix->jme_msix_rid, msix->jme_msix_cpuid); 3759 if (error) 3760 goto back; 3761 3762 msix->jme_msix_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 3763 &msix->jme_msix_rid, RF_ACTIVE); 3764 if (msix->jme_msix_res == NULL) { 3765 error = ENOMEM; 3766 goto back; 3767 } 3768 } 3769 3770 for (i = 0; i < JME_INTR_CNT; ++i) { 3771 uint32_t intr_mask = (1 << i); 3772 int x; 3773 3774 if ((JME_INTRS & intr_mask) == 0) 3775 continue; 3776 3777 for (x = 0; x < msix_count; ++x) { 3778 msix = &sc->jme_msix[x]; 3779 if (msix->jme_msix_intrs & intr_mask) { 3780 int reg, shift; 3781 3782 reg = i / JME_MSINUM_FACTOR; 3783 KKASSERT(reg < JME_MSINUM_CNT); 3784 3785 shift = (i % JME_MSINUM_FACTOR) * 4; 3786 3787 sc->jme_msinum[reg] |= 3788 (msix->jme_msix_vector << shift); 3789 3790 break; 3791 } 3792 } 3793 } 3794 3795 if (bootverbose) { 3796 for (i = 0; i < JME_MSINUM_CNT; ++i) { 3797 device_printf(dev, "MSINUM%d: %#x\n", i, 3798 sc->jme_msinum[i]); 3799 } 3800 } 3801 3802 pci_enable_msix(dev); 3803 sc->jme_irq_type = PCI_INTR_TYPE_MSIX; 3804 3805 back: 3806 if (error) 3807 jme_msix_free(dev); 3808 } 3809 3810 static int 3811 jme_intr_alloc(device_t dev) 3812 { 3813 struct jme_softc *sc = device_get_softc(dev); 3814 u_int irq_flags; 3815 3816 jme_msix_try_alloc(dev); 3817 3818 if (sc->jme_irq_type != PCI_INTR_TYPE_MSIX) { 3819 sc->jme_irq_type = pci_alloc_1intr(dev, jme_msi_enable, 3820 &sc->jme_irq_rid, &irq_flags); 3821 3822 sc->jme_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 3823 &sc->jme_irq_rid, irq_flags); 3824 if (sc->jme_irq_res == NULL) { 3825 device_printf(dev, "can't allocate irq\n"); 3826 return ENXIO; 3827 } 3828 } 3829 return 0; 3830 } 3831 3832 static void 3833 jme_msix_free(device_t dev) 3834 { 3835 struct jme_softc *sc = device_get_softc(dev); 3836 int i; 3837 3838 KKASSERT(sc->jme_msix_cnt > 1); 3839 3840 for (i = 0; i < sc->jme_msix_cnt; ++i) { 3841 struct jme_msix_data *msix = &sc->jme_msix[i]; 3842 3843 if (msix->jme_msix_res != NULL) { 3844 bus_release_resource(dev, SYS_RES_IRQ, 3845 msix->jme_msix_rid, msix->jme_msix_res); 3846 msix->jme_msix_res = NULL; 3847 } 3848 if (msix->jme_msix_rid >= 0) { 3849 pci_release_msix_vector(dev, msix->jme_msix_rid); 3850 msix->jme_msix_rid = -1; 3851 } 3852 } 3853 pci_teardown_msix(dev); 3854 } 3855 3856 static void 3857 jme_intr_free(device_t dev) 3858 { 3859 struct jme_softc *sc = device_get_softc(dev); 3860 3861 if (sc->jme_irq_type != PCI_INTR_TYPE_MSIX) { 3862 if (sc->jme_irq_res != NULL) { 3863 bus_release_resource(dev, SYS_RES_IRQ, sc->jme_irq_rid, 3864 sc->jme_irq_res); 3865 } 3866 if (sc->jme_irq_type == PCI_INTR_TYPE_MSI) 3867 pci_release_msi(dev); 3868 } else { 3869 jme_msix_free(dev); 3870 } 3871 } 3872 3873 static void 3874 jme_msix_tx(void *xtdata) 3875 { 3876 struct jme_txdata *tdata = xtdata; 3877 struct jme_softc *sc = tdata->jme_sc; 3878 struct ifnet *ifp = &sc->arpcom.ac_if; 3879 3880 ASSERT_SERIALIZED(&tdata->jme_tx_serialize); 3881 3882 CSR_WRITE_4(sc, JME_INTR_MASK_CLR, INTR_TXQ_COAL | INTR_TXQ_COAL_TO); 3883 3884 CSR_WRITE_4(sc, JME_INTR_STATUS, 3885 INTR_TXQ_COAL | INTR_TXQ_COAL_TO | INTR_TXQ_COMP); 3886 3887 if (ifp->if_flags & IFF_RUNNING) { 3888 jme_txeof(tdata); 3889 if (!ifq_is_empty(&ifp->if_snd)) 3890 if_devstart(ifp); 3891 } 3892 3893 CSR_WRITE_4(sc, JME_INTR_MASK_SET, INTR_TXQ_COAL | INTR_TXQ_COAL_TO); 3894 } 3895 3896 static void 3897 jme_msix_rx(void *xrdata) 3898 { 3899 struct jme_rxdata *rdata = xrdata; 3900 struct jme_softc *sc = rdata->jme_sc; 3901 struct ifnet *ifp = &sc->arpcom.ac_if; 3902 3903 ASSERT_SERIALIZED(&rdata->jme_rx_serialize); 3904 3905 CSR_WRITE_4(sc, JME_INTR_MASK_CLR, rdata->jme_rx_coal); 3906 3907 CSR_WRITE_4(sc, JME_INTR_STATUS, 3908 rdata->jme_rx_coal | rdata->jme_rx_comp); 3909 3910 if (ifp->if_flags & IFF_RUNNING) 3911 jme_rxeof(rdata, -1); 3912 3913 CSR_WRITE_4(sc, JME_INTR_MASK_SET, rdata->jme_rx_coal); 3914 } 3915 3916 static void 3917 jme_msix_status(void *xsc) 3918 { 3919 struct jme_softc *sc = xsc; 3920 struct ifnet *ifp = &sc->arpcom.ac_if; 3921 uint32_t status; 3922 3923 ASSERT_SERIALIZED(&sc->jme_serialize); 3924 3925 CSR_WRITE_4(sc, JME_INTR_MASK_CLR, INTR_RXQ_DESC_EMPTY); 3926 3927 status = CSR_READ_4(sc, JME_INTR_STATUS); 3928 3929 if (status & INTR_RXQ_DESC_EMPTY) { 3930 CSR_WRITE_4(sc, JME_INTR_STATUS, status & INTR_RXQ_DESC_EMPTY); 3931 if (ifp->if_flags & IFF_RUNNING) 3932 jme_rx_restart(sc, status); 3933 } 3934 3935 CSR_WRITE_4(sc, JME_INTR_MASK_SET, INTR_RXQ_DESC_EMPTY); 3936 } 3937 3938 static void 3939 jme_rx_restart(struct jme_softc *sc, uint32_t status) 3940 { 3941 int i; 3942 3943 for (i = 0; i < sc->jme_cdata.jme_rx_ring_cnt; ++i) { 3944 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[i]; 3945 3946 if (status & rdata->jme_rx_empty) { 3947 lwkt_serialize_enter(&rdata->jme_rx_serialize); 3948 jme_rxeof(rdata, -1); 3949 #ifdef JME_RSS_DEBUG 3950 rdata->jme_rx_emp++; 3951 #endif 3952 lwkt_serialize_exit(&rdata->jme_rx_serialize); 3953 } 3954 } 3955 CSR_WRITE_4(sc, JME_RXCSR, sc->jme_rxcsr | RXCSR_RX_ENB | 3956 RXCSR_RXQ_START); 3957 } 3958 3959 static void 3960 jme_set_msinum(struct jme_softc *sc) 3961 { 3962 int i; 3963 3964 for (i = 0; i < JME_MSINUM_CNT; ++i) 3965 CSR_WRITE_4(sc, JME_MSINUM(i), sc->jme_msinum[i]); 3966 } 3967 3968 static int 3969 jme_intr_setup(device_t dev) 3970 { 3971 struct jme_softc *sc = device_get_softc(dev); 3972 struct ifnet *ifp = &sc->arpcom.ac_if; 3973 int error; 3974 3975 if (sc->jme_irq_type == PCI_INTR_TYPE_MSIX) 3976 return jme_msix_setup(dev); 3977 3978 error = bus_setup_intr(dev, sc->jme_irq_res, INTR_MPSAFE, 3979 jme_intr, sc, &sc->jme_irq_handle, &sc->jme_serialize); 3980 if (error) { 3981 device_printf(dev, "could not set up interrupt handler.\n"); 3982 return error; 3983 } 3984 3985 ifp->if_cpuid = rman_get_cpuid(sc->jme_irq_res); 3986 KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); 3987 return 0; 3988 } 3989 3990 static void 3991 jme_intr_teardown(device_t dev) 3992 { 3993 struct jme_softc *sc = device_get_softc(dev); 3994 3995 if (sc->jme_irq_type == PCI_INTR_TYPE_MSIX) 3996 jme_msix_teardown(dev, sc->jme_msix_cnt); 3997 else 3998 bus_teardown_intr(dev, sc->jme_irq_res, sc->jme_irq_handle); 3999 } 4000 4001 static int 4002 jme_msix_setup(device_t dev) 4003 { 4004 struct jme_softc *sc = device_get_softc(dev); 4005 struct ifnet *ifp = &sc->arpcom.ac_if; 4006 int x; 4007 4008 for (x = 0; x < sc->jme_msix_cnt; ++x) { 4009 struct jme_msix_data *msix = &sc->jme_msix[x]; 4010 int error; 4011 4012 error = bus_setup_intr_descr(dev, msix->jme_msix_res, 4013 INTR_MPSAFE, msix->jme_msix_func, msix->jme_msix_arg, 4014 &msix->jme_msix_handle, msix->jme_msix_serialize, 4015 msix->jme_msix_desc); 4016 if (error) { 4017 device_printf(dev, "could not set up %s " 4018 "interrupt handler.\n", msix->jme_msix_desc); 4019 jme_msix_teardown(dev, x); 4020 return error; 4021 } 4022 } 4023 ifp->if_cpuid = sc->jme_tx_cpuid; 4024 return 0; 4025 } 4026 4027 static void 4028 jme_msix_teardown(device_t dev, int msix_count) 4029 { 4030 struct jme_softc *sc = device_get_softc(dev); 4031 int x; 4032 4033 for (x = 0; x < msix_count; ++x) { 4034 struct jme_msix_data *msix = &sc->jme_msix[x]; 4035 4036 bus_teardown_intr(dev, msix->jme_msix_res, 4037 msix->jme_msix_handle); 4038 } 4039 } 4040 4041 static void 4042 jme_serialize_skipmain(struct jme_softc *sc) 4043 { 4044 lwkt_serialize_array_enter(sc->jme_serialize_arr, 4045 sc->jme_serialize_cnt, 1); 4046 } 4047 4048 static void 4049 jme_deserialize_skipmain(struct jme_softc *sc) 4050 { 4051 lwkt_serialize_array_exit(sc->jme_serialize_arr, 4052 sc->jme_serialize_cnt, 1); 4053 } 4054 4055 static void 4056 jme_enable_intr(struct jme_softc *sc) 4057 { 4058 int i; 4059 4060 for (i = 0; i < sc->jme_serialize_cnt; ++i) 4061 lwkt_serialize_handler_enable(sc->jme_serialize_arr[i]); 4062 4063 CSR_WRITE_4(sc, JME_INTR_MASK_SET, JME_INTRS); 4064 } 4065 4066 static void 4067 jme_disable_intr(struct jme_softc *sc) 4068 { 4069 int i; 4070 4071 CSR_WRITE_4(sc, JME_INTR_MASK_CLR, JME_INTRS); 4072 4073 for (i = 0; i < sc->jme_serialize_cnt; ++i) 4074 lwkt_serialize_handler_disable(sc->jme_serialize_arr[i]); 4075 } 4076