1 /*- 2 * Copyright (c) 2012 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Matt Thomas of 3am Software Foundry. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #define _ARM32_BUS_DMA_PRIVATE 31 #define GMAC_PRIVATE 32 33 #include "locators.h" 34 #include "opt_broadcom.h" 35 36 #include <sys/cdefs.h> 37 38 __KERNEL_RCSID(1, "$NetBSD: bcm53xx_eth.c,v 1.25 2013/10/28 22:51:16 matt Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/atomic.h> 42 #include <sys/bus.h> 43 #include <sys/device.h> 44 #include <sys/ioctl.h> 45 #include <sys/intr.h> 46 #include <sys/kmem.h> 47 #include <sys/mutex.h> 48 #include <sys/socket.h> 49 #include <sys/systm.h> 50 #include <sys/workqueue.h> 51 52 #include <net/if.h> 53 #include <net/if_ether.h> 54 #include <net/if_media.h> 55 56 #include <net/if_dl.h> 57 58 #include <net/bpf.h> 59 60 #include <dev/mii/miivar.h> 61 62 #include <arm/broadcom/bcm53xx_reg.h> 63 #include <arm/broadcom/bcm53xx_var.h> 64 65 //#define BCMETH_MPSAFE 66 67 #ifdef BCMETH_COUNTERS 68 #define BCMETH_EVCNT_ADD(a,b) ((void)((a).ev_count += (b))) 69 #else 70 #define BCMETH_EVCNT_ADD(a,b) do { } while (/*CONSTCOND*/0) 71 #endif 72 #define BCMETH_EVCNT_INCR(a) BCMETH_EVCNT_ADD((a), 1) 73 74 #define BCMETH_MAXTXMBUFS 128 75 #define BCMETH_NTXSEGS 30 76 #define BCMETH_MAXRXMBUFS 255 77 #define BCMETH_MINRXMBUFS 64 78 #define BCMETH_NRXSEGS 1 79 #define BCMETH_RINGSIZE PAGE_SIZE 80 81 #if 1 82 #define BCMETH_RCVMAGIC 0xfeedface 83 #endif 84 85 static int bcmeth_ccb_match(device_t, cfdata_t, void *); 86 static void bcmeth_ccb_attach(device_t, device_t, void *); 87 88 struct bcmeth_txqueue { 89 bus_dmamap_t txq_descmap; 90 struct gmac_txdb *txq_consumer; 91 struct gmac_txdb *txq_producer; 92 struct gmac_txdb *txq_first; 93 struct gmac_txdb *txq_last; 94 struct ifqueue txq_mbufs; 95 struct mbuf *txq_next; 96 size_t txq_free; 97 size_t txq_threshold; 98 size_t txq_lastintr; 99 bus_size_t txq_reg_xmtaddrlo; 100 bus_size_t txq_reg_xmtptr; 101 bus_size_t txq_reg_xmtctl; 102 bus_size_t txq_reg_xmtsts0; 103 bus_size_t txq_reg_xmtsts1; 104 bus_dma_segment_t txq_descmap_seg; 105 }; 106 107 struct bcmeth_rxqueue { 108 bus_dmamap_t rxq_descmap; 109 struct gmac_rxdb *rxq_consumer; 110 struct gmac_rxdb *rxq_producer; 111 struct gmac_rxdb *rxq_first; 112 struct gmac_rxdb *rxq_last; 113 struct mbuf *rxq_mhead; 114 struct mbuf **rxq_mtail; 115 struct mbuf *rxq_mconsumer; 116 size_t rxq_inuse; 117 size_t rxq_threshold; 118 bus_size_t rxq_reg_rcvaddrlo; 119 bus_size_t rxq_reg_rcvptr; 120 bus_size_t rxq_reg_rcvctl; 121 bus_size_t rxq_reg_rcvsts0; 122 bus_size_t rxq_reg_rcvsts1; 123 bus_dma_segment_t rxq_descmap_seg; 124 }; 125 126 struct bcmeth_mapcache { 127 u_int dmc_nmaps; 128 u_int dmc_maxseg; 129 u_int dmc_maxmaps; 130 u_int dmc_maxmapsize; 131 bus_dmamap_t dmc_maps[0]; 132 }; 133 134 struct bcmeth_softc { 135 device_t sc_dev; 136 bus_space_tag_t sc_bst; 137 bus_space_handle_t sc_bsh; 138 bus_dma_tag_t sc_dmat; 139 kmutex_t *sc_lock; 140 kmutex_t *sc_hwlock; 141 struct ethercom sc_ec; 142 #define sc_if sc_ec.ec_if 143 struct ifmedia sc_media; 144 void *sc_soft_ih; 145 void *sc_ih; 146 147 struct bcmeth_rxqueue sc_rxq; 148 struct bcmeth_txqueue sc_txq; 149 150 size_t sc_rcvoffset; 151 uint32_t sc_macaddr[2]; 152 uint32_t sc_maxfrm; 153 uint32_t sc_cmdcfg; 154 uint32_t sc_intmask; 155 uint32_t sc_rcvlazy; 156 volatile uint32_t sc_soft_flags; 157 #define SOFT_RXINTR 0x01 158 #define SOFT_TXINTR 0x02 159 160 #ifdef BCMETH_COUNTERS 161 struct evcnt sc_ev_intr; 162 struct evcnt sc_ev_soft_intr; 163 struct evcnt sc_ev_work; 164 struct evcnt sc_ev_tx_stall; 165 struct evcnt sc_ev_rx_badmagic_lo; 166 struct evcnt sc_ev_rx_badmagic_hi; 167 #endif 168 169 struct ifqueue sc_rx_bufcache; 170 struct bcmeth_mapcache *sc_rx_mapcache; 171 struct bcmeth_mapcache *sc_tx_mapcache; 172 173 struct workqueue *sc_workq; 174 struct work sc_work; 175 176 volatile uint32_t sc_work_flags; 177 #define WORK_RXINTR 0x01 178 #define WORK_RXUNDERFLOW 0x02 179 #define WORK_REINIT 0x04 180 181 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 182 }; 183 184 static void bcmeth_ifstart(struct ifnet *); 185 static void bcmeth_ifwatchdog(struct ifnet *); 186 static int bcmeth_ifinit(struct ifnet *); 187 static void bcmeth_ifstop(struct ifnet *, int); 188 static int bcmeth_ifioctl(struct ifnet *, u_long, void *); 189 190 static int bcmeth_mapcache_create(struct bcmeth_softc *, 191 struct bcmeth_mapcache **, size_t, size_t, size_t); 192 static void bcmeth_mapcache_destroy(struct bcmeth_softc *, 193 struct bcmeth_mapcache *); 194 static bus_dmamap_t bcmeth_mapcache_get(struct bcmeth_softc *, 195 struct bcmeth_mapcache *); 196 static void bcmeth_mapcache_put(struct bcmeth_softc *, 197 struct bcmeth_mapcache *, bus_dmamap_t); 198 199 static int bcmeth_txq_attach(struct bcmeth_softc *, 200 struct bcmeth_txqueue *, u_int); 201 static void bcmeth_txq_purge(struct bcmeth_softc *, 202 struct bcmeth_txqueue *); 203 static void bcmeth_txq_reset(struct bcmeth_softc *, 204 struct bcmeth_txqueue *); 205 static bool bcmeth_txq_consume(struct bcmeth_softc *, 206 struct bcmeth_txqueue *); 207 static bool bcmeth_txq_produce(struct bcmeth_softc *, 208 struct bcmeth_txqueue *, struct mbuf *m); 209 static bool bcmeth_txq_active_p(struct bcmeth_softc *, 210 struct bcmeth_txqueue *); 211 212 static int bcmeth_rxq_attach(struct bcmeth_softc *, 213 struct bcmeth_rxqueue *, u_int); 214 static bool bcmeth_rxq_produce(struct bcmeth_softc *, 215 struct bcmeth_rxqueue *); 216 static void bcmeth_rxq_purge(struct bcmeth_softc *, 217 struct bcmeth_rxqueue *, bool); 218 static void bcmeth_rxq_reset(struct bcmeth_softc *, 219 struct bcmeth_rxqueue *); 220 221 static int bcmeth_intr(void *); 222 #ifdef BCMETH_MPSAFETX 223 static void bcmeth_soft_txintr(struct bcmeth_softc *); 224 #endif 225 static void bcmeth_soft_intr(void *); 226 static void bcmeth_worker(struct work *, void *); 227 228 static int bcmeth_mediachange(struct ifnet *); 229 static void bcmeth_mediastatus(struct ifnet *, struct ifmediareq *); 230 231 static inline uint32_t 232 bcmeth_read_4(struct bcmeth_softc *sc, bus_size_t o) 233 { 234 return bus_space_read_4(sc->sc_bst, sc->sc_bsh, o); 235 } 236 237 static inline void 238 bcmeth_write_4(struct bcmeth_softc *sc, bus_size_t o, uint32_t v) 239 { 240 bus_space_write_4(sc->sc_bst, sc->sc_bsh, o, v); 241 } 242 243 CFATTACH_DECL_NEW(bcmeth_ccb, sizeof(struct bcmeth_softc), 244 bcmeth_ccb_match, bcmeth_ccb_attach, NULL, NULL); 245 246 static int 247 bcmeth_ccb_match(device_t parent, cfdata_t cf, void *aux) 248 { 249 struct bcmccb_attach_args * const ccbaa = aux; 250 const struct bcm_locators * const loc = &ccbaa->ccbaa_loc; 251 252 if (strcmp(cf->cf_name, loc->loc_name)) 253 return 0; 254 255 #ifdef DIAGNOSTIC 256 const int port = cf->cf_loc[BCMCCBCF_PORT]; 257 #endif 258 KASSERT(port == BCMCCBCF_PORT_DEFAULT || port == loc->loc_port); 259 260 return 1; 261 } 262 263 static void 264 bcmeth_ccb_attach(device_t parent, device_t self, void *aux) 265 { 266 struct bcmeth_softc * const sc = device_private(self); 267 struct ethercom * const ec = &sc->sc_ec; 268 struct ifnet * const ifp = &ec->ec_if; 269 struct bcmccb_attach_args * const ccbaa = aux; 270 const struct bcm_locators * const loc = &ccbaa->ccbaa_loc; 271 const char * const xname = device_xname(self); 272 prop_dictionary_t dict = device_properties(self); 273 int error; 274 275 sc->sc_bst = ccbaa->ccbaa_ccb_bst; 276 sc->sc_dmat = ccbaa->ccbaa_dmat; 277 bus_space_subregion(sc->sc_bst, ccbaa->ccbaa_ccb_bsh, 278 loc->loc_offset, loc->loc_size, &sc->sc_bsh); 279 280 /* 281 * We need to use the coherent dma tag for the GMAC. 282 */ 283 sc->sc_dmat = &bcm53xx_coherent_dma_tag; 284 #if _ARM32_NEED_BUS_DMA_BOUNCE 285 if (device_cfdata(self)->cf_flags & 2) { 286 sc->sc_dmat = &bcm53xx_bounce_dma_tag; 287 } 288 #endif 289 290 prop_data_t eaprop = prop_dictionary_get(dict, "mac-address"); 291 if (eaprop == NULL) { 292 uint32_t mac0 = bcmeth_read_4(sc, UNIMAC_MAC_0); 293 uint32_t mac1 = bcmeth_read_4(sc, UNIMAC_MAC_1); 294 if ((mac0 == 0 && mac1 == 0) || (mac1 & 1)) { 295 aprint_error(": mac-address property is missing\n"); 296 return; 297 } 298 sc->sc_enaddr[0] = (mac0 >> 0) & 0xff; 299 sc->sc_enaddr[1] = (mac0 >> 8) & 0xff; 300 sc->sc_enaddr[2] = (mac0 >> 16) & 0xff; 301 sc->sc_enaddr[3] = (mac0 >> 24) & 0xff; 302 sc->sc_enaddr[4] = (mac1 >> 0) & 0xff; 303 sc->sc_enaddr[5] = (mac1 >> 8) & 0xff; 304 } else { 305 KASSERT(prop_object_type(eaprop) == PROP_TYPE_DATA); 306 KASSERT(prop_data_size(eaprop) == ETHER_ADDR_LEN); 307 memcpy(sc->sc_enaddr, prop_data_data_nocopy(eaprop), 308 ETHER_ADDR_LEN); 309 } 310 sc->sc_dev = self; 311 sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 312 sc->sc_hwlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_VM); 313 314 bcmeth_write_4(sc, GMAC_INTMASK, 0); // disable interrupts 315 316 aprint_naive("\n"); 317 aprint_normal(": Gigabit Ethernet Controller\n"); 318 319 error = bcmeth_rxq_attach(sc, &sc->sc_rxq, 0); 320 if (error) { 321 aprint_error(": failed to init rxq: %d\n", error); 322 return; 323 } 324 325 error = bcmeth_txq_attach(sc, &sc->sc_txq, 0); 326 if (error) { 327 aprint_error(": failed to init txq: %d\n", error); 328 return; 329 } 330 331 error = bcmeth_mapcache_create(sc, &sc->sc_rx_mapcache, 332 BCMETH_MAXRXMBUFS, MCLBYTES, BCMETH_NRXSEGS); 333 if (error) { 334 aprint_error(": failed to allocate rx dmamaps: %d\n", error); 335 return; 336 } 337 338 error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache, 339 BCMETH_MAXTXMBUFS, MCLBYTES, BCMETH_NTXSEGS); 340 if (error) { 341 aprint_error(": failed to allocate tx dmamaps: %d\n", error); 342 return; 343 } 344 345 error = workqueue_create(&sc->sc_workq, xname, bcmeth_worker, sc, 346 (PRI_USER + MAXPRI_USER) / 2, IPL_NET, WQ_MPSAFE|WQ_PERCPU); 347 if (error) { 348 aprint_error(": failed to create workqueue: %d\n", error); 349 return; 350 } 351 352 sc->sc_soft_ih = softint_establish(SOFTINT_MPSAFE | SOFTINT_NET, 353 bcmeth_soft_intr, sc); 354 355 sc->sc_ih = intr_establish(loc->loc_intrs[0], IPL_VM, IST_LEVEL, 356 bcmeth_intr, sc); 357 358 if (sc->sc_ih == NULL) { 359 aprint_error_dev(self, "failed to establish interrupt %d\n", 360 loc->loc_intrs[0]); 361 } else { 362 aprint_normal_dev(self, "interrupting on irq %d\n", 363 loc->loc_intrs[0]); 364 } 365 366 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 367 ether_sprintf(sc->sc_enaddr)); 368 369 /* 370 * Since each port in plugged into the switch/flow-accelerator, 371 * we hard code at Gige Full-Duplex with Flow Control enabled. 372 */ 373 int ifmedia = IFM_ETHER|IFM_1000_T|IFM_FDX; 374 //ifmedia |= IFM_FLOW|IFM_ETH_TXPAUSE|IFM_ETH_RXPAUSE; 375 ifmedia_init(&sc->sc_media, IFM_IMASK, bcmeth_mediachange, 376 bcmeth_mediastatus); 377 ifmedia_add(&sc->sc_media, ifmedia, 0, NULL); 378 ifmedia_set(&sc->sc_media, ifmedia); 379 380 ec->ec_capabilities = ETHERCAP_VLAN_MTU | ETHERCAP_JUMBO_MTU; 381 382 strlcpy(ifp->if_xname, xname, IFNAMSIZ); 383 ifp->if_softc = sc; 384 ifp->if_baudrate = IF_Mbps(1000); 385 ifp->if_capabilities = 0; 386 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 387 #ifdef BCMETH_MPSAFE 388 ifp->if_flags2 = IFF2_MPSAFE; 389 #endif 390 ifp->if_ioctl = bcmeth_ifioctl; 391 ifp->if_start = bcmeth_ifstart; 392 ifp->if_watchdog = bcmeth_ifwatchdog; 393 ifp->if_init = bcmeth_ifinit; 394 ifp->if_stop = bcmeth_ifstop; 395 IFQ_SET_READY(&ifp->if_snd); 396 397 bcmeth_ifstop(ifp, true); 398 399 /* 400 * Attach the interface. 401 */ 402 if_attach(ifp); 403 ether_ifattach(ifp, sc->sc_enaddr); 404 405 #ifdef BCMETH_COUNTERS 406 evcnt_attach_dynamic(&sc->sc_ev_intr, EVCNT_TYPE_INTR, 407 NULL, xname, "intr"); 408 evcnt_attach_dynamic(&sc->sc_ev_soft_intr, EVCNT_TYPE_INTR, 409 NULL, xname, "soft intr"); 410 evcnt_attach_dynamic(&sc->sc_ev_work, EVCNT_TYPE_MISC, 411 NULL, xname, "work items"); 412 evcnt_attach_dynamic(&sc->sc_ev_tx_stall, EVCNT_TYPE_MISC, 413 NULL, xname, "tx stalls"); 414 evcnt_attach_dynamic(&sc->sc_ev_rx_badmagic_lo, EVCNT_TYPE_MISC, 415 NULL, xname, "rx badmagic lo"); 416 evcnt_attach_dynamic(&sc->sc_ev_rx_badmagic_hi, EVCNT_TYPE_MISC, 417 NULL, xname, "rx badmagic hi"); 418 #endif 419 } 420 421 static int 422 bcmeth_mediachange(struct ifnet *ifp) 423 { 424 //struct bcmeth_softc * const sc = ifp->if_softc; 425 return 0; 426 } 427 428 static void 429 bcmeth_mediastatus(struct ifnet *ifp, struct ifmediareq *ifm) 430 { 431 //struct bcmeth_softc * const sc = ifp->if_softc; 432 433 ifm->ifm_status = IFM_AVALID | IFM_ACTIVE; 434 ifm->ifm_active = IFM_ETHER | IFM_FDX | IFM_1000_T; 435 } 436 437 static uint64_t 438 bcmeth_macaddr_create(const uint8_t *enaddr) 439 { 440 return (enaddr[3] << 0) // UNIMAC_MAC_0 441 | (enaddr[2] << 8) // UNIMAC_MAC_0 442 | (enaddr[1] << 16) // UNIMAC_MAC_0 443 | ((uint64_t)enaddr[0] << 24) // UNIMAC_MAC_0 444 | ((uint64_t)enaddr[5] << 32) // UNIMAC_MAC_1 445 | ((uint64_t)enaddr[4] << 40); // UNIMAC_MAC_1 446 } 447 448 static int 449 bcmeth_ifinit(struct ifnet *ifp) 450 { 451 struct bcmeth_softc * const sc = ifp->if_softc; 452 int error = 0; 453 454 sc->sc_maxfrm = max(ifp->if_mtu + 32, MCLBYTES); 455 if (ifp->if_mtu > ETHERMTU_JUMBO) 456 return error; 457 458 KASSERT(ifp->if_flags & IFF_UP); 459 460 /* 461 * Stop the interface 462 */ 463 bcmeth_ifstop(ifp, 0); 464 465 /* 466 * Reserve enough space at the front so that we can insert a maxsized 467 * link header and a VLAN tag. Also make sure we have enough room for 468 * the rcvsts field as well. 469 */ 470 KASSERT(ALIGN(max_linkhdr) == max_linkhdr); 471 KASSERTMSG(max_linkhdr > sizeof(struct ether_header), "%u > %zu", 472 max_linkhdr, sizeof(struct ether_header)); 473 sc->sc_rcvoffset = max_linkhdr + 4 - sizeof(struct ether_header); 474 if (sc->sc_rcvoffset <= 4) 475 sc->sc_rcvoffset += 4; 476 KASSERT((sc->sc_rcvoffset & 3) == 2); 477 KASSERT(sc->sc_rcvoffset <= __SHIFTOUT(RCVCTL_RCVOFFSET, RCVCTL_RCVOFFSET)); 478 KASSERT(sc->sc_rcvoffset >= 6); 479 480 /* 481 * If our frame size has changed (or it's our first time through) 482 * destroy the existing transmit mapcache. 483 */ 484 if (sc->sc_tx_mapcache != NULL 485 && sc->sc_maxfrm != sc->sc_tx_mapcache->dmc_maxmapsize) { 486 bcmeth_mapcache_destroy(sc, sc->sc_tx_mapcache); 487 sc->sc_tx_mapcache = NULL; 488 } 489 490 if (sc->sc_tx_mapcache == NULL) { 491 error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache, 492 BCMETH_MAXTXMBUFS, sc->sc_maxfrm, BCMETH_NTXSEGS); 493 if (error) 494 return error; 495 } 496 497 sc->sc_cmdcfg = NO_LENGTH_CHECK | PAUSE_IGNORE 498 | __SHIFTIN(ETH_SPEED_1000, ETH_SPEED) 499 | RX_ENA | TX_ENA; 500 501 if (ifp->if_flags & IFF_PROMISC) { 502 sc->sc_cmdcfg |= PROMISC_EN; 503 } else { 504 sc->sc_cmdcfg &= ~PROMISC_EN; 505 } 506 507 const uint8_t * const lladdr = CLLADDR(ifp->if_sadl); 508 const uint64_t macstnaddr = bcmeth_macaddr_create(lladdr); 509 510 /* 511 * We make sure that a received Ethernet packet start on a non-word 512 * boundary so that the packet payload will be on a word boundary. 513 * So to check the destination address we keep around two words to 514 * quickly compare with. 515 */ 516 #if __ARMEL__ 517 sc->sc_macaddr[0] = lladdr[0] | (lladdr[1] << 8); 518 sc->sc_macaddr[1] = lladdr[2] | (lladdr[3] << 8) 519 | (lladdr[4] << 16) | (lladdr[5] << 24); 520 #else 521 sc->sc_macaddr[0] = lladdr[1] | (lladdr[0] << 8); 522 sc->sc_macaddr[1] = lladdr[5] | (lladdr[4] << 8) 523 | (lladdr[1] << 16) | (lladdr[2] << 24); 524 #endif 525 526 sc->sc_intmask = DESCPROTOERR|DATAERR|DESCERR; 527 528 /* 5. Load RCVADDR_LO with new pointer */ 529 bcmeth_rxq_reset(sc, &sc->sc_rxq); 530 531 bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, 532 __SHIFTIN(sc->sc_rcvoffset, RCVCTL_RCVOFFSET) 533 | RCVCTL_PARITY_DIS 534 | RCVCTL_OFLOW_CONTINUE 535 | __SHIFTIN(3, RCVCTL_BURSTLEN)); 536 537 /* 6. Load XMTADDR_LO with new pointer */ 538 bcmeth_txq_reset(sc, &sc->sc_txq); 539 540 bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, XMTCTL_DMA_ACT_INDEX 541 | XMTCTL_PARITY_DIS 542 | __SHIFTIN(3, XMTCTL_BURSTLEN)); 543 544 /* 7. Setup other UNIMAC registers */ 545 bcmeth_write_4(sc, UNIMAC_FRAME_LEN, sc->sc_maxfrm); 546 bcmeth_write_4(sc, UNIMAC_MAC_0, (uint32_t)(macstnaddr >> 0)); 547 bcmeth_write_4(sc, UNIMAC_MAC_1, (uint32_t)(macstnaddr >> 32)); 548 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, sc->sc_cmdcfg); 549 550 uint32_t devctl = bcmeth_read_4(sc, GMAC_DEVCONTROL); 551 devctl |= RGMII_LINK_STATUS_SEL | NWAY_AUTO_POLL_EN | TXARB_STRICT_MODE; 552 devctl &= ~FLOW_CTRL_MODE; 553 devctl &= ~MIB_RD_RESET_EN; 554 devctl &= ~RXQ_OVERFLOW_CTRL_SEL; 555 devctl &= ~CPU_FLOW_CTRL_ON; 556 bcmeth_write_4(sc, GMAC_DEVCONTROL, devctl); 557 558 /* Setup lazy receive (at most 1ms). */ 559 const struct cpu_softc * const cpu = curcpu()->ci_softc; 560 sc->sc_rcvlazy = __SHIFTIN(4, INTRCVLAZY_FRAMECOUNT) 561 | __SHIFTIN(cpu->cpu_clk.clk_apb / 1000, INTRCVLAZY_TIMEOUT); 562 bcmeth_write_4(sc, GMAC_INTRCVLAZY, sc->sc_rcvlazy); 563 564 /* 11. Enable transmit queues in TQUEUE, and ensure that the transmit scheduling mode is correctly set in TCTRL. */ 565 sc->sc_intmask |= XMTINT_0|XMTUF; 566 bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, 567 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl) | XMTCTL_ENABLE); 568 569 570 /* 12. Enable receive queues in RQUEUE, */ 571 sc->sc_intmask |= RCVINT|RCVDESCUF|RCVFIFOOF; 572 bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, 573 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl) | RCVCTL_ENABLE); 574 575 bcmeth_rxq_produce(sc, &sc->sc_rxq); /* fill with rx buffers */ 576 577 #if 0 578 aprint_normal_dev(sc->sc_dev, 579 "devctl=%#x ucmdcfg=%#x xmtctl=%#x rcvctl=%#x\n", 580 devctl, sc->sc_cmdcfg, 581 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl), 582 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl)); 583 #endif 584 585 sc->sc_soft_flags = 0; 586 587 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 588 589 ifp->if_flags |= IFF_RUNNING; 590 591 return error; 592 } 593 594 static void 595 bcmeth_ifstop(struct ifnet *ifp, int disable) 596 { 597 struct bcmeth_softc * const sc = ifp->if_softc; 598 struct bcmeth_txqueue * const txq = &sc->sc_txq; 599 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 600 601 KASSERT(!cpu_intr_p()); 602 603 sc->sc_soft_flags = 0; 604 sc->sc_work_flags = 0; 605 606 /* Disable Rx processing */ 607 bcmeth_write_4(sc, rxq->rxq_reg_rcvctl, 608 bcmeth_read_4(sc, rxq->rxq_reg_rcvctl) & ~RCVCTL_ENABLE); 609 610 /* Disable Tx processing */ 611 bcmeth_write_4(sc, txq->txq_reg_xmtctl, 612 bcmeth_read_4(sc, txq->txq_reg_xmtctl) & ~XMTCTL_ENABLE); 613 614 /* Disable all interrupts */ 615 bcmeth_write_4(sc, GMAC_INTMASK, 0); 616 617 for (;;) { 618 uint32_t tx0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0); 619 uint32_t rx0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 620 if (__SHIFTOUT(tx0, XMTSTATE) == XMTSTATE_DIS 621 && __SHIFTOUT(rx0, RCVSTATE) == RCVSTATE_DIS) 622 break; 623 delay(50); 624 } 625 /* 626 * Now reset the controller. 627 * 628 * 3. Set SW_RESET bit in UNIMAC_COMMAND_CONFIG register 629 * 4. Clear SW_RESET bit in UNIMAC_COMMAND_CONFIG register 630 */ 631 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, SW_RESET); 632 bcmeth_write_4(sc, GMAC_INTSTATUS, ~0); 633 sc->sc_intmask = 0; 634 ifp->if_flags &= ~IFF_RUNNING; 635 636 /* 637 * Let's consume any remaining transmitted packets. And if we are 638 * disabling the interface, purge ourselves of any untransmitted 639 * packets. But don't consume any received packets, just drop them. 640 * If we aren't disabling the interface, save the mbufs in the 641 * receive queue for reuse. 642 */ 643 bcmeth_rxq_purge(sc, &sc->sc_rxq, disable); 644 bcmeth_txq_consume(sc, &sc->sc_txq); 645 if (disable) { 646 bcmeth_txq_purge(sc, &sc->sc_txq); 647 IF_PURGE(&ifp->if_snd); 648 } 649 650 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, 0); 651 } 652 653 static void 654 bcmeth_ifwatchdog(struct ifnet *ifp) 655 { 656 } 657 658 static int 659 bcmeth_ifioctl(struct ifnet *ifp, u_long cmd, void *data) 660 { 661 struct bcmeth_softc *sc = ifp->if_softc; 662 struct ifreq * const ifr = data; 663 const int s = splnet(); 664 int error; 665 666 switch (cmd) { 667 case SIOCSIFMEDIA: 668 case SIOCGIFMEDIA: 669 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 670 break; 671 672 default: 673 error = ether_ioctl(ifp, cmd, data); 674 if (error != ENETRESET) 675 break; 676 677 if (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI) { 678 error = 0; 679 break; 680 } 681 error = bcmeth_ifinit(ifp); 682 break; 683 } 684 685 splx(s); 686 return error; 687 } 688 689 static void 690 bcmeth_rxq_desc_presync( 691 struct bcmeth_softc *sc, 692 struct bcmeth_rxqueue *rxq, 693 struct gmac_rxdb *rxdb, 694 size_t count) 695 { 696 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap, 697 (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb), 698 BUS_DMASYNC_PREWRITE); 699 } 700 701 static void 702 bcmeth_rxq_desc_postsync( 703 struct bcmeth_softc *sc, 704 struct bcmeth_rxqueue *rxq, 705 struct gmac_rxdb *rxdb, 706 size_t count) 707 { 708 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap, 709 (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb), 710 BUS_DMASYNC_POSTWRITE); 711 } 712 713 static void 714 bcmeth_txq_desc_presync( 715 struct bcmeth_softc *sc, 716 struct bcmeth_txqueue *txq, 717 struct gmac_txdb *txdb, 718 size_t count) 719 { 720 bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap, 721 (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb), 722 BUS_DMASYNC_PREWRITE); 723 } 724 725 static void 726 bcmeth_txq_desc_postsync( 727 struct bcmeth_softc *sc, 728 struct bcmeth_txqueue *txq, 729 struct gmac_txdb *txdb, 730 size_t count) 731 { 732 bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap, 733 (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb), 734 BUS_DMASYNC_POSTWRITE); 735 } 736 737 static bus_dmamap_t 738 bcmeth_mapcache_get( 739 struct bcmeth_softc *sc, 740 struct bcmeth_mapcache *dmc) 741 { 742 KASSERT(dmc->dmc_nmaps > 0); 743 KASSERT(dmc->dmc_maps[dmc->dmc_nmaps-1] != NULL); 744 return dmc->dmc_maps[--dmc->dmc_nmaps]; 745 } 746 747 static void 748 bcmeth_mapcache_put( 749 struct bcmeth_softc *sc, 750 struct bcmeth_mapcache *dmc, 751 bus_dmamap_t map) 752 { 753 KASSERT(map != NULL); 754 KASSERT(dmc->dmc_nmaps < dmc->dmc_maxmaps); 755 dmc->dmc_maps[dmc->dmc_nmaps++] = map; 756 } 757 758 static void 759 bcmeth_mapcache_destroy( 760 struct bcmeth_softc *sc, 761 struct bcmeth_mapcache *dmc) 762 { 763 const size_t dmc_size = 764 offsetof(struct bcmeth_mapcache, dmc_maps[dmc->dmc_maxmaps]); 765 766 for (u_int i = 0; i < dmc->dmc_maxmaps; i++) { 767 bus_dmamap_destroy(sc->sc_dmat, dmc->dmc_maps[i]); 768 } 769 kmem_intr_free(dmc, dmc_size); 770 } 771 772 static int 773 bcmeth_mapcache_create( 774 struct bcmeth_softc *sc, 775 struct bcmeth_mapcache **dmc_p, 776 size_t maxmaps, 777 size_t maxmapsize, 778 size_t maxseg) 779 { 780 const size_t dmc_size = 781 offsetof(struct bcmeth_mapcache, dmc_maps[maxmaps]); 782 struct bcmeth_mapcache * const dmc = 783 kmem_intr_zalloc(dmc_size, KM_NOSLEEP); 784 785 dmc->dmc_maxmaps = maxmaps; 786 dmc->dmc_nmaps = maxmaps; 787 dmc->dmc_maxmapsize = maxmapsize; 788 dmc->dmc_maxseg = maxseg; 789 790 for (u_int i = 0; i < maxmaps; i++) { 791 int error = bus_dmamap_create(sc->sc_dmat, dmc->dmc_maxmapsize, 792 dmc->dmc_maxseg, dmc->dmc_maxmapsize, 0, 793 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW, &dmc->dmc_maps[i]); 794 if (error) { 795 aprint_error_dev(sc->sc_dev, 796 "failed to creat dma map cache " 797 "entry %u of %zu: %d\n", 798 i, maxmaps, error); 799 while (i-- > 0) { 800 bus_dmamap_destroy(sc->sc_dmat, 801 dmc->dmc_maps[i]); 802 } 803 kmem_intr_free(dmc, dmc_size); 804 return error; 805 } 806 KASSERT(dmc->dmc_maps[i] != NULL); 807 } 808 809 *dmc_p = dmc; 810 811 return 0; 812 } 813 814 #if 0 815 static void 816 bcmeth_dmamem_free( 817 bus_dma_tag_t dmat, 818 size_t map_size, 819 bus_dma_segment_t *seg, 820 bus_dmamap_t map, 821 void *kvap) 822 { 823 bus_dmamap_destroy(dmat, map); 824 bus_dmamem_unmap(dmat, kvap, map_size); 825 bus_dmamem_free(dmat, seg, 1); 826 } 827 #endif 828 829 static int 830 bcmeth_dmamem_alloc( 831 bus_dma_tag_t dmat, 832 size_t map_size, 833 bus_dma_segment_t *seg, 834 bus_dmamap_t *map, 835 void **kvap) 836 { 837 int error; 838 int nseg; 839 840 *kvap = NULL; 841 *map = NULL; 842 843 error = bus_dmamem_alloc(dmat, map_size, 2*PAGE_SIZE, 0, 844 seg, 1, &nseg, 0); 845 if (error) 846 return error; 847 848 KASSERT(nseg == 1); 849 850 error = bus_dmamem_map(dmat, seg, nseg, map_size, (void **)kvap, 0); 851 if (error == 0) { 852 error = bus_dmamap_create(dmat, map_size, 1, map_size, 0, 0, 853 map); 854 if (error == 0) { 855 error = bus_dmamap_load(dmat, *map, *kvap, map_size, 856 NULL, 0); 857 if (error == 0) 858 return 0; 859 bus_dmamap_destroy(dmat, *map); 860 *map = NULL; 861 } 862 bus_dmamem_unmap(dmat, *kvap, map_size); 863 *kvap = NULL; 864 } 865 bus_dmamem_free(dmat, seg, nseg); 866 return 0; 867 } 868 869 static struct mbuf * 870 bcmeth_rx_buf_alloc( 871 struct bcmeth_softc *sc) 872 { 873 struct mbuf *m = m_gethdr(M_DONTWAIT, MT_DATA); 874 if (m == NULL) { 875 printf("%s:%d: %s\n", __func__, __LINE__, "m_gethdr"); 876 return NULL; 877 } 878 MCLGET(m, M_DONTWAIT); 879 if ((m->m_flags & M_EXT) == 0) { 880 printf("%s:%d: %s\n", __func__, __LINE__, "MCLGET"); 881 m_freem(m); 882 return NULL; 883 } 884 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 885 886 bus_dmamap_t map = bcmeth_mapcache_get(sc, sc->sc_rx_mapcache); 887 if (map == NULL) { 888 printf("%s:%d: %s\n", __func__, __LINE__, "map get"); 889 m_freem(m); 890 return NULL; 891 } 892 M_SETCTX(m, map); 893 m->m_len = m->m_pkthdr.len = MCLBYTES; 894 int error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 895 BUS_DMA_READ|BUS_DMA_NOWAIT); 896 if (error) { 897 aprint_error_dev(sc->sc_dev, "fail to load rx dmamap: %d\n", 898 error); 899 M_SETCTX(m, NULL); 900 m_freem(m); 901 bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map); 902 return NULL; 903 } 904 KASSERT(map->dm_mapsize == MCLBYTES); 905 #ifdef BCMETH_RCVMAGIC 906 *mtod(m, uint32_t *) = htole32(BCMETH_RCVMAGIC); 907 bus_dmamap_sync(sc->sc_dmat, map, 0, sizeof(uint32_t), 908 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 909 bus_dmamap_sync(sc->sc_dmat, map, sizeof(uint32_t), 910 map->dm_mapsize - sizeof(uint32_t), BUS_DMASYNC_PREREAD); 911 #else 912 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 913 BUS_DMASYNC_PREREAD); 914 #endif 915 916 return m; 917 } 918 919 static void 920 bcmeth_rx_map_unload( 921 struct bcmeth_softc *sc, 922 struct mbuf *m) 923 { 924 KASSERT(m); 925 for (; m != NULL; m = m->m_next) { 926 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 927 KASSERT(map); 928 KASSERT(map->dm_mapsize == MCLBYTES); 929 bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_len, 930 BUS_DMASYNC_POSTREAD); 931 bus_dmamap_unload(sc->sc_dmat, map); 932 bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map); 933 M_SETCTX(m, NULL); 934 } 935 } 936 937 static bool 938 bcmeth_rxq_produce( 939 struct bcmeth_softc *sc, 940 struct bcmeth_rxqueue *rxq) 941 { 942 struct gmac_rxdb *producer = rxq->rxq_producer; 943 bool produced = false; 944 945 while (rxq->rxq_inuse < rxq->rxq_threshold) { 946 struct mbuf *m; 947 IF_DEQUEUE(&sc->sc_rx_bufcache, m); 948 if (m == NULL) { 949 m = bcmeth_rx_buf_alloc(sc); 950 if (m == NULL) { 951 printf("%s: bcmeth_rx_buf_alloc failed\n", __func__); 952 break; 953 } 954 } 955 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 956 KASSERT(map); 957 958 producer->rxdb_buflen = htole32(MCLBYTES); 959 producer->rxdb_addrlo = htole32(map->dm_segs[0].ds_addr); 960 producer->rxdb_flags &= htole32(RXDB_FLAG_ET); 961 *rxq->rxq_mtail = m; 962 rxq->rxq_mtail = &m->m_next; 963 m->m_len = MCLBYTES; 964 m->m_next = NULL; 965 rxq->rxq_inuse++; 966 if (++producer == rxq->rxq_last) { 967 membar_producer(); 968 bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer, 969 rxq->rxq_last - rxq->rxq_producer); 970 producer = rxq->rxq_producer = rxq->rxq_first; 971 } 972 produced = true; 973 } 974 if (produced) { 975 membar_producer(); 976 if (producer != rxq->rxq_producer) { 977 bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer, 978 producer - rxq->rxq_producer); 979 rxq->rxq_producer = producer; 980 } 981 bcmeth_write_4(sc, rxq->rxq_reg_rcvptr, 982 rxq->rxq_descmap->dm_segs[0].ds_addr 983 + ((uintptr_t)producer & RCVPTR)); 984 } 985 return true; 986 } 987 988 static void 989 bcmeth_rx_input( 990 struct bcmeth_softc *sc, 991 struct mbuf *m, 992 uint32_t rxdb_flags) 993 { 994 struct ifnet * const ifp = &sc->sc_if; 995 996 bcmeth_rx_map_unload(sc, m); 997 998 m_adj(m, sc->sc_rcvoffset); 999 1000 /* 1001 * If we are in promiscuous mode and this isn't a multicast, check the 1002 * destination address to make sure it matches our own. If it doesn't, 1003 * mark the packet as being received promiscuously. 1004 */ 1005 if ((sc->sc_cmdcfg & PROMISC_EN) 1006 && (m->m_data[0] & 1) == 0 1007 && (*(uint16_t *)&m->m_data[0] != sc->sc_macaddr[0] 1008 || *(uint32_t *)&m->m_data[2] != sc->sc_macaddr[1])) { 1009 m->m_flags |= M_PROMISC; 1010 } 1011 m->m_pkthdr.rcvif = ifp; 1012 1013 ifp->if_ipackets++; 1014 ifp->if_ibytes += m->m_pkthdr.len; 1015 1016 /* 1017 * Let's give it to the network subsystm to deal with. 1018 */ 1019 #ifdef BCMETH_MPSAFE 1020 mutex_exit(sc->sc_lock); 1021 (*ifp->if_input)(ifp, m); 1022 mutex_enter(sc->sc_lock); 1023 #else 1024 int s = splnet(); 1025 bpf_mtap(ifp, m); 1026 (*ifp->if_input)(ifp, m); 1027 splx(s); 1028 #endif 1029 } 1030 1031 static bool 1032 bcmeth_rxq_consume( 1033 struct bcmeth_softc *sc, 1034 struct bcmeth_rxqueue *rxq, 1035 size_t atmost) 1036 { 1037 struct ifnet * const ifp = &sc->sc_if; 1038 struct gmac_rxdb *consumer = rxq->rxq_consumer; 1039 size_t rxconsumed = 0; 1040 bool didconsume = false; 1041 1042 while (atmost-- > 0) { 1043 if (consumer == rxq->rxq_producer) { 1044 KASSERT(rxq->rxq_inuse == 0); 1045 break; 1046 } 1047 1048 uint32_t rcvsts0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 1049 uint32_t currdscr = __SHIFTOUT(rcvsts0, RCV_CURRDSCR); 1050 if (consumer == rxq->rxq_first + currdscr) { 1051 break; 1052 } 1053 bcmeth_rxq_desc_postsync(sc, rxq, consumer, 1); 1054 1055 /* 1056 * We own this packet again. Copy the rxsts word from it. 1057 */ 1058 rxconsumed++; 1059 didconsume = true; 1060 uint32_t rxsts; 1061 KASSERT(rxq->rxq_mhead != NULL); 1062 bus_dmamap_t map = M_GETCTX(rxq->rxq_mhead, bus_dmamap_t); 1063 bus_dmamap_sync(sc->sc_dmat, map, 0, arm_dcache_align, 1064 BUS_DMASYNC_POSTREAD); 1065 memcpy(&rxsts, rxq->rxq_mhead->m_data, 4); 1066 rxsts = le32toh(rxsts); 1067 #if 0 1068 KASSERTMSG(rxsts != BCMETH_RCVMAGIC, "currdscr=%u consumer=%zd", 1069 currdscr, consumer - rxq->rxq_first); 1070 #endif 1071 1072 /* 1073 * Get the count of descriptors. Fetch the correct number 1074 * of mbufs. 1075 */ 1076 #ifdef BCMETH_RCVMAGIC 1077 size_t desc_count = rxsts != BCMETH_RCVMAGIC ? __SHIFTOUT(rxsts, RXSTS_DESC_COUNT) + 1 : 1; 1078 #else 1079 size_t desc_count = __SHIFTOUT(rxsts, RXSTS_DESC_COUNT) + 1; 1080 #endif 1081 struct mbuf *m = rxq->rxq_mhead; 1082 struct mbuf *m_last = m; 1083 for (size_t i = 1; i < desc_count; i++) { 1084 if (++consumer == rxq->rxq_last) { 1085 consumer = rxq->rxq_first; 1086 } 1087 KASSERTMSG(consumer != rxq->rxq_first + currdscr, 1088 "i=%zu rxsts=%#x desc_count=%zu currdscr=%u consumer=%zd", 1089 i, rxsts, desc_count, currdscr, 1090 consumer - rxq->rxq_first); 1091 m_last = m_last->m_next; 1092 } 1093 1094 /* 1095 * Now remove it/them from the list of enqueued mbufs. 1096 */ 1097 if ((rxq->rxq_mhead = m_last->m_next) == NULL) 1098 rxq->rxq_mtail = &rxq->rxq_mhead; 1099 m_last->m_next = NULL; 1100 1101 #ifdef BCMETH_RCVMAGIC 1102 if (rxsts == BCMETH_RCVMAGIC) { 1103 ifp->if_ierrors++; 1104 if ((m->m_ext.ext_paddr >> 28) == 8) { 1105 BCMETH_EVCNT_INCR(sc->sc_ev_rx_badmagic_lo); 1106 } else { 1107 BCMETH_EVCNT_INCR( sc->sc_ev_rx_badmagic_hi); 1108 } 1109 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1110 } else 1111 #endif /* BCMETH_RCVMAGIC */ 1112 if (rxsts & (RXSTS_CRC_ERROR|RXSTS_OVERSIZED|RXSTS_PKT_OVERFLOW)) { 1113 aprint_error_dev(sc->sc_dev, "[%zu]: count=%zu rxsts=%#x\n", 1114 consumer - rxq->rxq_first, desc_count, rxsts); 1115 /* 1116 * We encountered an error, take the mbufs and add them 1117 * to the rx bufcache so we can quickly reuse them. 1118 */ 1119 ifp->if_ierrors++; 1120 do { 1121 struct mbuf *m0 = m->m_next; 1122 m->m_next = NULL; 1123 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1124 m = m0; 1125 } while (m); 1126 } else { 1127 uint32_t framelen = __SHIFTOUT(rxsts, RXSTS_FRAMELEN); 1128 framelen += sc->sc_rcvoffset; 1129 m->m_pkthdr.len = framelen; 1130 if (desc_count == 1) { 1131 KASSERT(framelen <= MCLBYTES); 1132 m->m_len = framelen; 1133 } else { 1134 m_last->m_len = framelen & (MCLBYTES - 1); 1135 } 1136 1137 #ifdef BCMETH_MPSAFE 1138 /* 1139 * Wrap at the last entry! 1140 */ 1141 if (++consumer == rxq->rxq_last) { 1142 KASSERT(consumer[-1].rxdb_flags & htole32(RXDB_FLAG_ET)); 1143 rxq->rxq_consumer = rxq->rxq_first; 1144 } else { 1145 rxq->rxq_consumer = consumer; 1146 } 1147 rxq->rxq_inuse -= rxconsumed; 1148 #endif /* BCMETH_MPSAFE */ 1149 1150 /* 1151 * Receive the packet (which releases our lock) 1152 */ 1153 bcmeth_rx_input(sc, m, rxsts); 1154 1155 #ifdef BCMETH_MPSAFE 1156 /* 1157 * Since we had to give up our lock, we need to 1158 * refresh these. 1159 */ 1160 consumer = rxq->rxq_consumer; 1161 rxconsumed = 0; 1162 continue; 1163 #endif /* BCMETH_MPSAFE */ 1164 } 1165 1166 /* 1167 * Wrap at the last entry! 1168 */ 1169 if (++consumer == rxq->rxq_last) { 1170 KASSERT(consumer[-1].rxdb_flags & htole32(RXDB_FLAG_ET)); 1171 consumer = rxq->rxq_first; 1172 } 1173 } 1174 1175 /* 1176 * Update queue info. 1177 */ 1178 rxq->rxq_consumer = consumer; 1179 rxq->rxq_inuse -= rxconsumed; 1180 1181 /* 1182 * Did we consume anything? 1183 */ 1184 return didconsume; 1185 } 1186 1187 static void 1188 bcmeth_rxq_purge( 1189 struct bcmeth_softc *sc, 1190 struct bcmeth_rxqueue *rxq, 1191 bool discard) 1192 { 1193 struct mbuf *m; 1194 1195 if ((m = rxq->rxq_mhead) != NULL) { 1196 if (discard) { 1197 bcmeth_rx_map_unload(sc, m); 1198 m_freem(m); 1199 } else { 1200 while (m != NULL) { 1201 struct mbuf *m0 = m->m_next; 1202 m->m_next = NULL; 1203 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1204 m = m0; 1205 } 1206 } 1207 1208 } 1209 1210 rxq->rxq_mhead = NULL; 1211 rxq->rxq_mtail = &rxq->rxq_mhead; 1212 rxq->rxq_inuse = 0; 1213 } 1214 1215 static void 1216 bcmeth_rxq_reset( 1217 struct bcmeth_softc *sc, 1218 struct bcmeth_rxqueue *rxq) 1219 { 1220 /* 1221 * sync all the descriptors 1222 */ 1223 bcmeth_rxq_desc_postsync(sc, rxq, rxq->rxq_first, 1224 rxq->rxq_last - rxq->rxq_first); 1225 1226 /* 1227 * Make sure we own all descriptors in the ring. 1228 */ 1229 struct gmac_rxdb *rxdb; 1230 for (rxdb = rxq->rxq_first; rxdb < rxq->rxq_last - 1; rxdb++) { 1231 rxdb->rxdb_flags = htole32(RXDB_FLAG_IC); 1232 } 1233 1234 /* 1235 * Last descriptor has the wrap flag. 1236 */ 1237 rxdb->rxdb_flags = htole32(RXDB_FLAG_ET|RXDB_FLAG_IC); 1238 1239 /* 1240 * Reset the producer consumer indexes. 1241 */ 1242 rxq->rxq_consumer = rxq->rxq_first; 1243 rxq->rxq_producer = rxq->rxq_first; 1244 rxq->rxq_inuse = 0; 1245 if (rxq->rxq_threshold < BCMETH_MINRXMBUFS) 1246 rxq->rxq_threshold = BCMETH_MINRXMBUFS; 1247 1248 sc->sc_intmask |= RCVINT|RCVFIFOOF|RCVDESCUF; 1249 1250 /* 1251 * Restart the receiver at the first descriptor 1252 */ 1253 bcmeth_write_4(sc, rxq->rxq_reg_rcvaddrlo, 1254 rxq->rxq_descmap->dm_segs[0].ds_addr); 1255 } 1256 1257 static int 1258 bcmeth_rxq_attach( 1259 struct bcmeth_softc *sc, 1260 struct bcmeth_rxqueue *rxq, 1261 u_int qno) 1262 { 1263 size_t desc_count = BCMETH_RINGSIZE / sizeof(rxq->rxq_first[0]); 1264 int error; 1265 void *descs; 1266 1267 KASSERT(desc_count == 256 || desc_count == 512); 1268 1269 error = bcmeth_dmamem_alloc(sc->sc_dmat, BCMETH_RINGSIZE, 1270 &rxq->rxq_descmap_seg, &rxq->rxq_descmap, &descs); 1271 if (error) 1272 return error; 1273 1274 memset(descs, 0, BCMETH_RINGSIZE); 1275 rxq->rxq_first = descs; 1276 rxq->rxq_last = rxq->rxq_first + desc_count; 1277 rxq->rxq_consumer = descs; 1278 rxq->rxq_producer = descs; 1279 1280 bcmeth_rxq_purge(sc, rxq, true); 1281 bcmeth_rxq_reset(sc, rxq); 1282 1283 rxq->rxq_reg_rcvaddrlo = GMAC_RCVADDR_LOW; 1284 rxq->rxq_reg_rcvctl = GMAC_RCVCONTROL; 1285 rxq->rxq_reg_rcvptr = GMAC_RCVPTR; 1286 rxq->rxq_reg_rcvsts0 = GMAC_RCVSTATUS0; 1287 rxq->rxq_reg_rcvsts1 = GMAC_RCVSTATUS1; 1288 1289 return 0; 1290 } 1291 1292 static bool 1293 bcmeth_txq_active_p( 1294 struct bcmeth_softc * const sc, 1295 struct bcmeth_txqueue *txq) 1296 { 1297 return !IF_IS_EMPTY(&txq->txq_mbufs); 1298 } 1299 1300 static bool 1301 bcmeth_txq_fillable_p( 1302 struct bcmeth_softc * const sc, 1303 struct bcmeth_txqueue *txq) 1304 { 1305 return txq->txq_free >= txq->txq_threshold; 1306 } 1307 1308 static int 1309 bcmeth_txq_attach( 1310 struct bcmeth_softc *sc, 1311 struct bcmeth_txqueue *txq, 1312 u_int qno) 1313 { 1314 size_t desc_count = BCMETH_RINGSIZE / sizeof(txq->txq_first[0]); 1315 int error; 1316 void *descs; 1317 1318 KASSERT(desc_count == 256 || desc_count == 512); 1319 1320 error = bcmeth_dmamem_alloc(sc->sc_dmat, BCMETH_RINGSIZE, 1321 &txq->txq_descmap_seg, &txq->txq_descmap, &descs); 1322 if (error) 1323 return error; 1324 1325 memset(descs, 0, BCMETH_RINGSIZE); 1326 txq->txq_first = descs; 1327 txq->txq_last = txq->txq_first + desc_count; 1328 txq->txq_consumer = descs; 1329 txq->txq_producer = descs; 1330 1331 IFQ_SET_MAXLEN(&txq->txq_mbufs, BCMETH_MAXTXMBUFS); 1332 1333 txq->txq_reg_xmtaddrlo = GMAC_XMTADDR_LOW; 1334 txq->txq_reg_xmtctl = GMAC_XMTCONTROL; 1335 txq->txq_reg_xmtptr = GMAC_XMTPTR; 1336 txq->txq_reg_xmtsts0 = GMAC_XMTSTATUS0; 1337 txq->txq_reg_xmtsts1 = GMAC_XMTSTATUS1; 1338 1339 bcmeth_txq_reset(sc, txq); 1340 1341 return 0; 1342 } 1343 1344 static int 1345 bcmeth_txq_map_load( 1346 struct bcmeth_softc *sc, 1347 struct bcmeth_txqueue *txq, 1348 struct mbuf *m) 1349 { 1350 bus_dmamap_t map; 1351 int error; 1352 1353 map = M_GETCTX(m, bus_dmamap_t); 1354 if (map != NULL) 1355 return 0; 1356 1357 map = bcmeth_mapcache_get(sc, sc->sc_tx_mapcache); 1358 if (map == NULL) 1359 return ENOMEM; 1360 1361 error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 1362 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1363 if (error) 1364 return error; 1365 1366 bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_pkthdr.len, 1367 BUS_DMASYNC_PREWRITE); 1368 M_SETCTX(m, map); 1369 return 0; 1370 } 1371 1372 static void 1373 bcmeth_txq_map_unload( 1374 struct bcmeth_softc *sc, 1375 struct bcmeth_txqueue *txq, 1376 struct mbuf *m) 1377 { 1378 KASSERT(m); 1379 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 1380 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 1381 BUS_DMASYNC_POSTWRITE); 1382 bus_dmamap_unload(sc->sc_dmat, map); 1383 bcmeth_mapcache_put(sc, sc->sc_tx_mapcache, map); 1384 } 1385 1386 static bool 1387 bcmeth_txq_produce( 1388 struct bcmeth_softc *sc, 1389 struct bcmeth_txqueue *txq, 1390 struct mbuf *m) 1391 { 1392 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 1393 1394 if (map->dm_nsegs > txq->txq_free) 1395 return false; 1396 1397 /* 1398 * TCP Offload flag must be set in the first descriptor. 1399 */ 1400 struct gmac_txdb *producer = txq->txq_producer; 1401 uint32_t first_flags = TXDB_FLAG_SF; 1402 uint32_t last_flags = TXDB_FLAG_EF; 1403 1404 /* 1405 * If we've produced enough descriptors without consuming any 1406 * we need to ask for an interrupt to reclaim some. 1407 */ 1408 txq->txq_lastintr += map->dm_nsegs; 1409 if (txq->txq_lastintr >= txq->txq_threshold 1410 || txq->txq_mbufs.ifq_len + 1 == txq->txq_mbufs.ifq_maxlen) { 1411 txq->txq_lastintr = 0; 1412 last_flags |= TXDB_FLAG_IC; 1413 } 1414 1415 KASSERT(producer != txq->txq_last); 1416 1417 struct gmac_txdb *start = producer; 1418 size_t count = map->dm_nsegs; 1419 producer->txdb_flags |= htole32(first_flags); 1420 producer->txdb_addrlo = htole32(map->dm_segs[0].ds_addr); 1421 producer->txdb_buflen = htole32(map->dm_segs[0].ds_len); 1422 for (u_int i = 1; i < map->dm_nsegs; i++) { 1423 #if 0 1424 printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first, 1425 le32toh(producer->txdb_flags), 1426 le32toh(producer->txdb_buflen), 1427 le32toh(producer->txdb_addrlo), 1428 le32toh(producer->txdb_addrhi)); 1429 #endif 1430 if (__predict_false(++producer == txq->txq_last)) { 1431 bcmeth_txq_desc_presync(sc, txq, start, 1432 txq->txq_last - start); 1433 count -= txq->txq_last - start; 1434 producer = txq->txq_first; 1435 start = txq->txq_first; 1436 } 1437 producer->txdb_addrlo = htole32(map->dm_segs[i].ds_addr); 1438 producer->txdb_buflen = htole32(map->dm_segs[i].ds_len); 1439 } 1440 producer->txdb_flags |= htole32(last_flags); 1441 #if 0 1442 printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first, 1443 le32toh(producer->txdb_flags), le32toh(producer->txdb_buflen), 1444 le32toh(producer->txdb_addrlo), le32toh(producer->txdb_addrhi)); 1445 #endif 1446 if (count) 1447 bcmeth_txq_desc_presync(sc, txq, start, count); 1448 1449 /* 1450 * Reduce free count by the number of segments we consumed. 1451 */ 1452 txq->txq_free -= map->dm_nsegs; 1453 KASSERT(map->dm_nsegs == 1 || txq->txq_producer != producer); 1454 KASSERT(map->dm_nsegs == 1 || (txq->txq_producer->txdb_flags & htole32(TXDB_FLAG_EF)) == 0); 1455 KASSERT(producer->txdb_flags & htole32(TXDB_FLAG_EF)); 1456 1457 #if 0 1458 printf("%s: mbuf %p: produced a %u byte packet in %u segments (%zd..%zd)\n", 1459 __func__, m, m->m_pkthdr.len, map->dm_nsegs, 1460 txq->txq_producer - txq->txq_first, producer - txq->txq_first); 1461 #endif 1462 1463 if (producer + 1 == txq->txq_last) 1464 txq->txq_producer = txq->txq_first; 1465 else 1466 txq->txq_producer = producer + 1; 1467 IF_ENQUEUE(&txq->txq_mbufs, m); 1468 1469 /* 1470 * Let the transmitter know there's more to do 1471 */ 1472 bcmeth_write_4(sc, txq->txq_reg_xmtptr, 1473 txq->txq_descmap->dm_segs[0].ds_addr 1474 + ((uintptr_t)txq->txq_producer & XMT_LASTDSCR)); 1475 1476 return true; 1477 } 1478 1479 static struct mbuf * 1480 bcmeth_copy_packet(struct mbuf *m) 1481 { 1482 struct mbuf *mext = NULL; 1483 size_t misalignment = 0; 1484 size_t hlen = 0; 1485 1486 for (mext = m; mext != NULL; mext = mext->m_next) { 1487 if (mext->m_flags & M_EXT) { 1488 misalignment = mtod(mext, vaddr_t) & arm_dcache_align; 1489 break; 1490 } 1491 hlen += m->m_len; 1492 } 1493 1494 struct mbuf *n = m->m_next; 1495 if (m != mext && hlen + misalignment <= MHLEN && false) { 1496 KASSERT(m->m_pktdat <= m->m_data && m->m_data <= &m->m_pktdat[MHLEN - m->m_len]); 1497 size_t oldoff = m->m_data - m->m_pktdat; 1498 size_t off; 1499 if (mext == NULL) { 1500 off = (oldoff + hlen > MHLEN) ? 0 : oldoff; 1501 } else { 1502 off = MHLEN - (hlen + misalignment); 1503 } 1504 KASSERT(off + hlen + misalignment <= MHLEN); 1505 if (((oldoff ^ off) & arm_dcache_align) != 0 || off < oldoff) { 1506 memmove(&m->m_pktdat[off], m->m_data, m->m_len); 1507 m->m_data = &m->m_pktdat[off]; 1508 } 1509 m_copydata(n, 0, hlen - m->m_len, &m->m_data[m->m_len]); 1510 m->m_len = hlen; 1511 m->m_next = mext; 1512 while (n != mext) { 1513 n = m_free(n); 1514 } 1515 return m; 1516 } 1517 1518 struct mbuf *m0 = m_gethdr(M_DONTWAIT, m->m_type); 1519 if (m0 == NULL) { 1520 return NULL; 1521 } 1522 M_COPY_PKTHDR(m0, m); 1523 MCLAIM(m0, m->m_owner); 1524 if (m0->m_pkthdr.len > MHLEN) { 1525 MCLGET(m0, M_DONTWAIT); 1526 if ((m0->m_flags & M_EXT) == 0) { 1527 m_freem(m0); 1528 return NULL; 1529 } 1530 } 1531 m0->m_len = m->m_pkthdr.len; 1532 m_copydata(m, 0, m0->m_len, mtod(m0, void *)); 1533 m_freem(m); 1534 return m0; 1535 } 1536 1537 static bool 1538 bcmeth_txq_enqueue( 1539 struct bcmeth_softc *sc, 1540 struct bcmeth_txqueue *txq) 1541 { 1542 for (;;) { 1543 if (IF_QFULL(&txq->txq_mbufs)) 1544 return false; 1545 struct mbuf *m = txq->txq_next; 1546 if (m == NULL) { 1547 int s = splnet(); 1548 IF_DEQUEUE(&sc->sc_if.if_snd, m); 1549 splx(s); 1550 if (m == NULL) 1551 return true; 1552 M_SETCTX(m, NULL); 1553 } else { 1554 txq->txq_next = NULL; 1555 } 1556 /* 1557 * If LINK2 is set and this packet uses multiple mbufs, 1558 * consolidate it into a single mbuf. 1559 */ 1560 if (m->m_next != NULL && (sc->sc_if.if_flags & IFF_LINK2)) { 1561 struct mbuf *m0 = bcmeth_copy_packet(m); 1562 if (m0 == NULL) { 1563 txq->txq_next = m; 1564 return true; 1565 } 1566 m = m0; 1567 } 1568 int error = bcmeth_txq_map_load(sc, txq, m); 1569 if (error) { 1570 aprint_error_dev(sc->sc_dev, 1571 "discarded packet due to " 1572 "dmamap load failure: %d\n", error); 1573 m_freem(m); 1574 continue; 1575 } 1576 KASSERT(txq->txq_next == NULL); 1577 if (!bcmeth_txq_produce(sc, txq, m)) { 1578 txq->txq_next = m; 1579 return false; 1580 } 1581 KASSERT(txq->txq_next == NULL); 1582 } 1583 } 1584 1585 static bool 1586 bcmeth_txq_consume( 1587 struct bcmeth_softc *sc, 1588 struct bcmeth_txqueue *txq) 1589 { 1590 struct ifnet * const ifp = &sc->sc_if; 1591 struct gmac_txdb *consumer = txq->txq_consumer; 1592 size_t txfree = 0; 1593 1594 #if 0 1595 printf("%s: entry: free=%zu\n", __func__, txq->txq_free); 1596 #endif 1597 1598 for (;;) { 1599 if (consumer == txq->txq_producer) { 1600 txq->txq_consumer = consumer; 1601 txq->txq_free += txfree; 1602 txq->txq_lastintr -= min(txq->txq_lastintr, txfree); 1603 #if 0 1604 printf("%s: empty: freed %zu descriptors going from %zu to %zu\n", 1605 __func__, txfree, txq->txq_free - txfree, txq->txq_free); 1606 #endif 1607 KASSERT(txq->txq_lastintr == 0); 1608 KASSERT(txq->txq_free == txq->txq_last - txq->txq_first - 1); 1609 return true; 1610 } 1611 bcmeth_txq_desc_postsync(sc, txq, consumer, 1); 1612 uint32_t s0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0); 1613 if (consumer == txq->txq_first + __SHIFTOUT(s0, XMT_CURRDSCR)) { 1614 txq->txq_consumer = consumer; 1615 txq->txq_free += txfree; 1616 txq->txq_lastintr -= min(txq->txq_lastintr, txfree); 1617 #if 0 1618 printf("%s: freed %zu descriptors\n", 1619 __func__, txfree); 1620 #endif 1621 return bcmeth_txq_fillable_p(sc, txq); 1622 } 1623 1624 /* 1625 * If this is the last descriptor in the chain, get the 1626 * mbuf, free its dmamap, and free the mbuf chain itself. 1627 */ 1628 const uint32_t txdb_flags = le32toh(consumer->txdb_flags); 1629 if (txdb_flags & TXDB_FLAG_EF) { 1630 struct mbuf *m; 1631 1632 IF_DEQUEUE(&txq->txq_mbufs, m); 1633 KASSERT(m); 1634 bcmeth_txq_map_unload(sc, txq, m); 1635 #if 0 1636 printf("%s: mbuf %p: consumed a %u byte packet\n", 1637 __func__, m, m->m_pkthdr.len); 1638 #endif 1639 bpf_mtap(ifp, m); 1640 ifp->if_opackets++; 1641 ifp->if_obytes += m->m_pkthdr.len; 1642 if (m->m_flags & M_MCAST) 1643 ifp->if_omcasts++; 1644 m_freem(m); 1645 } 1646 1647 /* 1648 * We own this packet again. Clear all flags except wrap. 1649 */ 1650 txfree++; 1651 1652 /* 1653 * Wrap at the last entry! 1654 */ 1655 if (txdb_flags & TXDB_FLAG_ET) { 1656 consumer->txdb_flags = htole32(TXDB_FLAG_ET); 1657 KASSERT(consumer + 1 == txq->txq_last); 1658 consumer = txq->txq_first; 1659 } else { 1660 consumer->txdb_flags = 0; 1661 consumer++; 1662 KASSERT(consumer < txq->txq_last); 1663 } 1664 } 1665 } 1666 1667 static void 1668 bcmeth_txq_purge( 1669 struct bcmeth_softc *sc, 1670 struct bcmeth_txqueue *txq) 1671 { 1672 struct mbuf *m; 1673 KASSERT((bcmeth_read_4(sc, UNIMAC_COMMAND_CONFIG) & TX_ENA) == 0); 1674 1675 for (;;) { 1676 IF_DEQUEUE(&txq->txq_mbufs, m); 1677 if (m == NULL) 1678 break; 1679 bcmeth_txq_map_unload(sc, txq, m); 1680 m_freem(m); 1681 } 1682 if ((m = txq->txq_next) != NULL) { 1683 txq->txq_next = NULL; 1684 bcmeth_txq_map_unload(sc, txq, m); 1685 m_freem(m); 1686 } 1687 } 1688 1689 static void 1690 bcmeth_txq_reset( 1691 struct bcmeth_softc *sc, 1692 struct bcmeth_txqueue *txq) 1693 { 1694 /* 1695 * sync all the descriptors 1696 */ 1697 bcmeth_txq_desc_postsync(sc, txq, txq->txq_first, 1698 txq->txq_last - txq->txq_first); 1699 1700 /* 1701 * Make sure we own all descriptors in the ring. 1702 */ 1703 struct gmac_txdb *txdb; 1704 for (txdb = txq->txq_first; txdb < txq->txq_last - 1; txdb++) { 1705 txdb->txdb_flags = 0; 1706 } 1707 1708 /* 1709 * Last descriptor has the wrap flag. 1710 */ 1711 txdb->txdb_flags = htole32(TXDB_FLAG_ET); 1712 1713 /* 1714 * Reset the producer consumer indexes. 1715 */ 1716 txq->txq_consumer = txq->txq_first; 1717 txq->txq_producer = txq->txq_first; 1718 txq->txq_free = txq->txq_last - txq->txq_first - 1; 1719 txq->txq_threshold = txq->txq_free / 2; 1720 txq->txq_lastintr = 0; 1721 1722 /* 1723 * What do we want to get interrupted on? 1724 */ 1725 sc->sc_intmask |= XMTINT_0 | XMTUF; 1726 1727 /* 1728 * Restart the transmiter at the first descriptor 1729 */ 1730 bcmeth_write_4(sc, txq->txq_reg_xmtaddrlo, 1731 txq->txq_descmap->dm_segs->ds_addr); 1732 } 1733 1734 static void 1735 bcmeth_ifstart(struct ifnet *ifp) 1736 { 1737 struct bcmeth_softc * const sc = ifp->if_softc; 1738 1739 if (__predict_false((ifp->if_flags & IFF_RUNNING) == 0)) { 1740 return; 1741 } 1742 1743 #ifdef BCMETH_MPSAFETX 1744 if (cpu_intr_p()) { 1745 #endif 1746 atomic_or_uint(&sc->sc_soft_flags, SOFT_TXINTR); 1747 softint_schedule(sc->sc_soft_ih); 1748 #ifdef BCMETH_MPSAFETX 1749 } else { 1750 /* 1751 * Either we are in a softintr thread already or some other 1752 * thread so just borrow it to do the send and save ourselves 1753 * the overhead of a fast soft int. 1754 */ 1755 bcmeth_soft_txintr(sc); 1756 } 1757 #endif 1758 } 1759 1760 int 1761 bcmeth_intr(void *arg) 1762 { 1763 struct bcmeth_softc * const sc = arg; 1764 uint32_t soft_flags = 0; 1765 uint32_t work_flags = 0; 1766 int rv = 0; 1767 1768 mutex_enter(sc->sc_hwlock); 1769 1770 uint32_t intmask = sc->sc_intmask; 1771 BCMETH_EVCNT_INCR(sc->sc_ev_intr); 1772 1773 for (;;) { 1774 uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS); 1775 intstatus &= intmask; 1776 bcmeth_write_4(sc, GMAC_INTSTATUS, intstatus); /* write 1 to clear */ 1777 if (intstatus == 0) { 1778 break; 1779 } 1780 #if 0 1781 aprint_normal_dev(sc->sc_dev, "%s: intstatus=%#x intmask=%#x\n", 1782 __func__, intstatus, bcmeth_read_4(sc, GMAC_INTMASK)); 1783 #endif 1784 if (intstatus & RCVINT) { 1785 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 1786 intmask &= ~RCVINT; 1787 1788 uint32_t rcvsts0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 1789 uint32_t descs = __SHIFTOUT(rcvsts0, RCV_CURRDSCR); 1790 if (descs < rxq->rxq_consumer - rxq->rxq_first) { 1791 /* 1792 * We wrapped at the end so count how far 1793 * we are from the end. 1794 */ 1795 descs += rxq->rxq_last - rxq->rxq_consumer; 1796 } else { 1797 descs -= rxq->rxq_consumer - rxq->rxq_first; 1798 } 1799 /* 1800 * If we "timedout" we can't be hogging so use 1801 * softints. If we exceeded then we might hogging 1802 * so let the workqueue deal with them. 1803 */ 1804 const uint32_t framecount = __SHIFTOUT(sc->sc_rcvlazy, INTRCVLAZY_FRAMECOUNT); 1805 if (descs < framecount 1806 || (curcpu()->ci_curlwp->l_flag & LW_IDLE)) { 1807 soft_flags |= SOFT_RXINTR; 1808 } else { 1809 work_flags |= WORK_RXINTR; 1810 } 1811 } 1812 1813 if (intstatus & XMTINT_0) { 1814 intmask &= ~XMTINT_0; 1815 soft_flags |= SOFT_TXINTR; 1816 } 1817 1818 if (intstatus & RCVDESCUF) { 1819 intmask &= ~RCVDESCUF; 1820 work_flags |= WORK_RXUNDERFLOW; 1821 } 1822 1823 intstatus &= intmask; 1824 if (intstatus) { 1825 aprint_error_dev(sc->sc_dev, 1826 "intr: intstatus=%#x\n", intstatus); 1827 aprint_error_dev(sc->sc_dev, 1828 "rcvbase=%p/%#lx rcvptr=%#x rcvsts=%#x/%#x\n", 1829 sc->sc_rxq.rxq_first, 1830 sc->sc_rxq.rxq_descmap->dm_segs[0].ds_addr, 1831 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvptr), 1832 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvsts0), 1833 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvsts1)); 1834 aprint_error_dev(sc->sc_dev, 1835 "xmtbase=%p/%#lx xmtptr=%#x xmtsts=%#x/%#x\n", 1836 sc->sc_txq.txq_first, 1837 sc->sc_txq.txq_descmap->dm_segs[0].ds_addr, 1838 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtptr), 1839 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtsts0), 1840 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtsts1)); 1841 intmask &= ~intstatus; 1842 work_flags |= WORK_REINIT; 1843 break; 1844 } 1845 } 1846 1847 if (intmask != sc->sc_intmask) { 1848 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1849 } 1850 1851 if (work_flags) { 1852 if (sc->sc_work_flags == 0) { 1853 workqueue_enqueue(sc->sc_workq, &sc->sc_work, NULL); 1854 } 1855 atomic_or_32(&sc->sc_work_flags, work_flags); 1856 rv = 1; 1857 } 1858 1859 if (soft_flags) { 1860 if (sc->sc_soft_flags == 0) { 1861 softint_schedule(sc->sc_soft_ih); 1862 } 1863 atomic_or_32(&sc->sc_soft_flags, soft_flags); 1864 rv = 1; 1865 } 1866 1867 mutex_exit(sc->sc_hwlock); 1868 1869 return rv; 1870 } 1871 1872 #ifdef BCMETH_MPSAFETX 1873 void 1874 bcmeth_soft_txintr(struct bcmeth_softc *sc) 1875 { 1876 mutex_enter(sc->sc_lock); 1877 /* 1878 * Let's do what we came here for. Consume transmitted 1879 * packets off the the transmit ring. 1880 */ 1881 if (!bcmeth_txq_consume(sc, &sc->sc_txq) 1882 || !bcmeth_txq_enqueue(sc, &sc->sc_txq)) { 1883 BCMETH_EVCNT_INCR(sc->sc_ev_tx_stall); 1884 sc->sc_if.if_flags |= IFF_OACTIVE; 1885 } else { 1886 sc->sc_if.if_flags &= ~IFF_OACTIVE; 1887 } 1888 if (sc->sc_if.if_flags & IFF_RUNNING) { 1889 mutex_spin_enter(sc->sc_hwlock); 1890 sc->sc_intmask |= XMTINT_0; 1891 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1892 mutex_spin_exit(sc->sc_hwlock); 1893 } 1894 mutex_exit(sc->sc_lock); 1895 } 1896 #endif /* BCMETH_MPSAFETX */ 1897 1898 void 1899 bcmeth_soft_intr(void *arg) 1900 { 1901 struct bcmeth_softc * const sc = arg; 1902 struct ifnet * const ifp = &sc->sc_if; 1903 uint32_t intmask = 0; 1904 1905 mutex_enter(sc->sc_lock); 1906 1907 u_int soft_flags = atomic_swap_uint(&sc->sc_soft_flags, 0); 1908 1909 BCMETH_EVCNT_INCR(sc->sc_ev_soft_intr); 1910 1911 if ((soft_flags & SOFT_TXINTR) 1912 || bcmeth_txq_active_p(sc, &sc->sc_txq)) { 1913 /* 1914 * Let's do what we came here for. Consume transmitted 1915 * packets off the the transmit ring. 1916 */ 1917 if (!bcmeth_txq_consume(sc, &sc->sc_txq) 1918 || !bcmeth_txq_enqueue(sc, &sc->sc_txq)) { 1919 BCMETH_EVCNT_INCR(sc->sc_ev_tx_stall); 1920 ifp->if_flags |= IFF_OACTIVE; 1921 } else { 1922 ifp->if_flags &= ~IFF_OACTIVE; 1923 } 1924 intmask |= XMTINT_0; 1925 } 1926 1927 if (soft_flags & SOFT_RXINTR) { 1928 /* 1929 * Let's consume 1930 */ 1931 while (bcmeth_rxq_consume(sc, &sc->sc_rxq, 1932 sc->sc_rxq.rxq_threshold / 4)) { 1933 /* 1934 * We've consumed a quarter of the ring and still have 1935 * more to do. Refill the ring. 1936 */ 1937 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1938 } 1939 intmask |= RCVINT; 1940 } 1941 1942 if (ifp->if_flags & IFF_RUNNING) { 1943 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1944 mutex_spin_enter(sc->sc_hwlock); 1945 sc->sc_intmask |= intmask; 1946 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1947 mutex_spin_exit(sc->sc_hwlock); 1948 } 1949 1950 mutex_exit(sc->sc_lock); 1951 } 1952 1953 void 1954 bcmeth_worker(struct work *wk, void *arg) 1955 { 1956 struct bcmeth_softc * const sc = arg; 1957 struct ifnet * const ifp = &sc->sc_if; 1958 uint32_t intmask = 0; 1959 1960 mutex_enter(sc->sc_lock); 1961 1962 BCMETH_EVCNT_INCR(sc->sc_ev_work); 1963 1964 uint32_t work_flags = atomic_swap_32(&sc->sc_work_flags, 0); 1965 if (work_flags & WORK_REINIT) { 1966 int s = splnet(); 1967 sc->sc_soft_flags = 0; 1968 bcmeth_ifinit(ifp); 1969 splx(s); 1970 work_flags &= ~WORK_RXUNDERFLOW; 1971 } 1972 1973 if (work_flags & WORK_RXUNDERFLOW) { 1974 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 1975 size_t threshold = 5 * rxq->rxq_threshold / 4; 1976 if (threshold >= rxq->rxq_last - rxq->rxq_first) { 1977 threshold = rxq->rxq_last - rxq->rxq_first - 1; 1978 } else { 1979 intmask |= RCVDESCUF; 1980 } 1981 aprint_normal_dev(sc->sc_dev, 1982 "increasing receive buffers from %zu to %zu\n", 1983 rxq->rxq_threshold, threshold); 1984 rxq->rxq_threshold = threshold; 1985 } 1986 1987 if (work_flags & WORK_RXINTR) { 1988 /* 1989 * Let's consume 1990 */ 1991 while (bcmeth_rxq_consume(sc, &sc->sc_rxq, 1992 sc->sc_rxq.rxq_threshold / 4)) { 1993 /* 1994 * We've consumed a quarter of the ring and still have 1995 * more to do. Refill the ring. 1996 */ 1997 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1998 } 1999 intmask |= RCVINT; 2000 } 2001 2002 if (ifp->if_flags & IFF_RUNNING) { 2003 bcmeth_rxq_produce(sc, &sc->sc_rxq); 2004 #if 0 2005 uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS); 2006 if (intstatus & RCVINT) { 2007 bcmeth_write_4(sc, GMAC_INTSTATUS, RCVINT); 2008 work_flags |= WORK_RXINTR; 2009 continue; 2010 } 2011 #endif 2012 mutex_spin_enter(sc->sc_hwlock); 2013 sc->sc_intmask |= intmask; 2014 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 2015 mutex_spin_exit(sc->sc_hwlock); 2016 } 2017 2018 mutex_exit(sc->sc_lock); 2019 } 2020