xref: /netbsd-src/sys/arch/arm/imx/if_enet.c (revision 90313c06e62e910bf0d1bb24faa9d17dcefd0ab6)

/*	$NetBSD: if_enet.c,v 1.37 2024/02/07 04:20:26 msaitoh Exp $	*/

/*
 * Copyright (c) 2014 Ryo Shimizu
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * i.MX6,7 10/100/1000-Mbps ethernet MAC (ENET)
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: if_enet.c,v 1.37 2024/02/07 04:20:26 msaitoh Exp $");

#include "vlan.h"

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/mbuf.h>
#include <sys/device.h>
#include <sys/sockio.h>
#include <sys/kernel.h>
#include <sys/rndsource.h>

#include <lib/libkern/libkern.h>

#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/if_ether.h>
#include <net/bpf.h>
#include <net/if_vlanvar.h>

#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>

#include <dev/mii/mii.h>
#include <dev/mii/miivar.h>

#include <arm/imx/if_enetreg.h>
#include <arm/imx/if_enetvar.h>

#undef DEBUG_ENET
#undef ENET_EVENT_COUNTER

#define ENET_TICK	hz

#ifdef DEBUG_ENET
int enet_debug = 0;
# define DEVICE_DPRINTF(args...)	\
	do { if (enet_debug) device_printf(sc->sc_dev, args); } while (0)
#else
# define DEVICE_DPRINTF(args...)
#endif


#define RXDESC_MAXBUFSIZE	0x07f0
				/* ENET does not work greather than 0x0800... */

#undef ENET_SUPPORT_JUMBO	/* JUMBO FRAME SUPPORT is unstable */
#ifdef ENET_SUPPORT_JUMBO
# define ENET_MAX_PKT_LEN	4034	/* MAX FIFO LEN */
#else
# define ENET_MAX_PKT_LEN	1522
#endif
#define ENET_DEFAULT_PKT_LEN	1522	/* including VLAN tag */
#define MTU2FRAMESIZE(n)	\
	((n) + ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN)


#define ENET_MAX_PKT_NSEGS	64

#define ENET_TX_NEXTIDX(idx)	\
	(((idx) >= (ENET_TX_RING_CNT - 1)) ? 0 : ((idx) + 1))
#define ENET_RX_NEXTIDX(idx)	\
	(((idx) >= (ENET_RX_RING_CNT - 1)) ? 0 : ((idx) + 1))

#define TXDESC_WRITEOUT(idx)					\
	bus_dmamap_sync(sc->sc_dmat, sc->sc_txdesc_dmamap,	\
	    sizeof(struct enet_txdesc) * (idx),			\
	    sizeof(struct enet_txdesc),				\
	    BUS_DMASYNC_PREWRITE)

#define TXDESC_READIN(idx)					\
	bus_dmamap_sync(sc->sc_dmat, sc->sc_txdesc_dmamap,	\
	    sizeof(struct enet_txdesc) * (idx),			\
	    sizeof(struct enet_txdesc),				\
	    BUS_DMASYNC_PREREAD)

#define RXDESC_WRITEOUT(idx)					\
	bus_dmamap_sync(sc->sc_dmat, sc->sc_rxdesc_dmamap,	\
	    sizeof(struct enet_rxdesc) * (idx),			\
	    sizeof(struct enet_rxdesc),				\
	    BUS_DMASYNC_PREWRITE)

#define RXDESC_READIN(idx)					\
	bus_dmamap_sync(sc->sc_dmat, sc->sc_rxdesc_dmamap,	\
	    sizeof(struct enet_rxdesc) * (idx),			\
	    sizeof(struct enet_rxdesc),				\
	    BUS_DMASYNC_PREREAD)

#define ENET_REG_READ(sc, reg)					\
	bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, reg)

#define ENET_REG_WRITE(sc, reg, value)				\
	bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, reg, value)

#ifdef ENET_EVENT_COUNTER
static void enet_attach_evcnt(struct enet_softc *);
static void enet_update_evcnt(struct enet_softc *);
#endif

static void enet_tick(void *);
static int enet_tx_intr(void *);
static int enet_rx_intr(void *);
static void enet_rx_csum(struct enet_softc *, struct ifnet *, struct mbuf *,
			 int);

static void enet_start(struct ifnet *);
static int enet_ifflags_cb(struct ethercom *);
static int enet_ioctl(struct ifnet *, u_long, void *);
static int enet_init(struct ifnet *);
static void enet_stop(struct ifnet *, int);
static void enet_watchdog(struct ifnet *);
static void enet_mediastatus(struct ifnet *, struct ifmediareq *);

static int enet_miibus_readreg(device_t, int, int, uint16_t *);
static int enet_miibus_writereg(device_t, int, int, uint16_t);
static void enet_miibus_statchg(struct ifnet *);

static void enet_gethwaddr(struct enet_softc *, uint8_t *);
static void enet_sethwaddr(struct enet_softc *, uint8_t *);
static void enet_setmulti(struct enet_softc *);
static int enet_encap_mbufalign(struct mbuf **);
static int enet_encap_txring(struct enet_softc *, struct mbuf **);
static int enet_init_regs(struct enet_softc *, int);
static int enet_alloc_ring(struct enet_softc *);
static void enet_init_txring(struct enet_softc *);
static int enet_init_rxring(struct enet_softc *);
static void enet_reset_rxdesc(struct enet_softc *, int);
static int enet_alloc_rxbuf(struct enet_softc *, int);
static void enet_drain_txbuf(struct enet_softc *);
static void enet_drain_rxbuf(struct enet_softc *);
static int enet_alloc_dma(struct enet_softc *, size_t, void **,
			  bus_dmamap_t *);

int
enet_attach_common(device_t self)
{
	struct enet_softc *sc = device_private(self);
	struct ifnet *ifp;
	struct mii_data * const mii = &sc->sc_mii;

	/* allocate dma buffer */
	if (enet_alloc_ring(sc))
		return -1;

#define IS_ENADDR_ZERO(enaddr)				\
	((enaddr[0] | enaddr[1] | enaddr[2] |		\
	 enaddr[3] | enaddr[4] | enaddr[5]) == 0)

	if (IS_ENADDR_ZERO(sc->sc_enaddr)) {
		/* by any chance, mac-address is already set by bootloader? */
		enet_gethwaddr(sc, sc->sc_enaddr);
		if (IS_ENADDR_ZERO(sc->sc_enaddr)) {
			/* give up. set randomly */
			uint32_t eaddr = random();
			/* not multicast */
			sc->sc_enaddr[0] = (eaddr >> 24) & 0xfc;
			sc->sc_enaddr[1] = eaddr >> 16;
			sc->sc_enaddr[2] = eaddr >> 8;
			sc->sc_enaddr[3] = eaddr;
			eaddr = random();
			sc->sc_enaddr[4] = eaddr >> 8;
			sc->sc_enaddr[5] = eaddr;

			aprint_error_dev(self,
			    "cannot get mac address. set randomly\n");
		}
	}
	enet_sethwaddr(sc, sc->sc_enaddr);

	aprint_normal_dev(self, "Ethernet address %s\n",
	    ether_sprintf(sc->sc_enaddr));

	enet_init_regs(sc, 1);

	/* callout will be scheduled from enet_init() */
	callout_init(&sc->sc_tick_ch, 0);
	callout_setfunc(&sc->sc_tick_ch, enet_tick, sc);

	/* setup ifp */
	ifp = &sc->sc_ethercom.ec_if;
	strlcpy(ifp->if_xname, device_xname(sc->sc_dev), IFNAMSIZ);
	ifp->if_softc = sc;
	ifp->if_mtu = ETHERMTU;
	ifp->if_baudrate = IF_Gbps(1);
	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
	ifp->if_ioctl = enet_ioctl;
	ifp->if_start = enet_start;
	ifp->if_init = enet_init;
	ifp->if_stop = enet_stop;
	ifp->if_watchdog = enet_watchdog;

	sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
#ifdef ENET_SUPPORT_JUMBO
	sc->sc_ethercom.ec_capabilities |= ETHERCAP_JUMBO_MTU;
#endif

	ifp->if_capabilities = IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx |
	    IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_UDPv4_Tx |
	    IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
	    IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_UDPv6_Tx |
	    IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx;

	IFQ_SET_MAXLEN(&ifp->if_snd, uimax(ENET_TX_RING_CNT, IFQ_MAXLEN));
	IFQ_SET_READY(&ifp->if_snd);

	/* setup MII */
	sc->sc_ethercom.ec_mii = mii;
	mii->mii_ifp = ifp;
	mii->mii_readreg = enet_miibus_readreg;
	mii->mii_writereg = enet_miibus_writereg;
	mii->mii_statchg = enet_miibus_statchg;
	ifmedia_init(&mii->mii_media, 0, ether_mediachange, enet_mediastatus);

	/* try to attach PHY */
	mii_attach(self, mii, 0xffffffff, sc->sc_phyid, MII_OFFSET_ANY, 0);
	if (LIST_FIRST(&mii->mii_phys) == NULL) {
		ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_MANUAL, 0, NULL);
		ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_MANUAL);
	} else {
		ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO);
	}

	if_attach(ifp);
	ether_ifattach(ifp, sc->sc_enaddr);
	ether_set_ifflags_cb(&sc->sc_ethercom, enet_ifflags_cb);

	rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev),
	    RND_TYPE_NET, RND_FLAG_DEFAULT);

#ifdef ENET_EVENT_COUNTER
	enet_attach_evcnt(sc);
#endif

	sc->sc_stopping = false;

	return 0;
}

#ifdef ENET_EVENT_COUNTER
static void
enet_attach_evcnt(struct enet_softc *sc)
{
	const char *xname;

	xname = device_xname(sc->sc_dev);

#define ENET_EVCNT_ATTACH(name)	\
	evcnt_attach_dynamic(&sc->sc_ev_ ## name, EVCNT_TYPE_MISC,	\
	    NULL, xname, #name);

	ENET_EVCNT_ATTACH(t_drop);
	ENET_EVCNT_ATTACH(t_packets);
	ENET_EVCNT_ATTACH(t_bc_pkt);
	ENET_EVCNT_ATTACH(t_mc_pkt);
	ENET_EVCNT_ATTACH(t_crc_align);
	ENET_EVCNT_ATTACH(t_undersize);
	ENET_EVCNT_ATTACH(t_oversize);
	ENET_EVCNT_ATTACH(t_frag);
	ENET_EVCNT_ATTACH(t_jab);
	ENET_EVCNT_ATTACH(t_col);
	ENET_EVCNT_ATTACH(t_p64);
	ENET_EVCNT_ATTACH(t_p65to127n);
	ENET_EVCNT_ATTACH(t_p128to255n);
	ENET_EVCNT_ATTACH(t_p256to511);
	ENET_EVCNT_ATTACH(t_p512to1023);
	ENET_EVCNT_ATTACH(t_p1024to2047);
	ENET_EVCNT_ATTACH(t_p_gte2048);
	ENET_EVCNT_ATTACH(t_octets);
	ENET_EVCNT_ATTACH(r_packets);
	ENET_EVCNT_ATTACH(r_bc_pkt);
	ENET_EVCNT_ATTACH(r_mc_pkt);
	ENET_EVCNT_ATTACH(r_crc_align);
	ENET_EVCNT_ATTACH(r_undersize);
	ENET_EVCNT_ATTACH(r_oversize);
	ENET_EVCNT_ATTACH(r_frag);
	ENET_EVCNT_ATTACH(r_jab);
	ENET_EVCNT_ATTACH(r_p64);
	ENET_EVCNT_ATTACH(r_p65to127);
	ENET_EVCNT_ATTACH(r_p128to255);
	ENET_EVCNT_ATTACH(r_p256to511);
	ENET_EVCNT_ATTACH(r_p512to1023);
	ENET_EVCNT_ATTACH(r_p1024to2047);
	ENET_EVCNT_ATTACH(r_p_gte2048);
	ENET_EVCNT_ATTACH(r_octets);
}

static void
enet_update_evcnt(struct enet_softc *sc)
{
	sc->sc_ev_t_drop.ev_count += ENET_REG_READ(sc, ENET_RMON_T_DROP);
	sc->sc_ev_t_packets.ev_count += ENET_REG_READ(sc, ENET_RMON_T_PACKETS);
	sc->sc_ev_t_bc_pkt.ev_count += ENET_REG_READ(sc, ENET_RMON_T_BC_PKT);
	sc->sc_ev_t_mc_pkt.ev_count += ENET_REG_READ(sc, ENET_RMON_T_MC_PKT);
	sc->sc_ev_t_crc_align.ev_count += ENET_REG_READ(sc, ENET_RMON_T_CRC_ALIGN);
	sc->sc_ev_t_undersize.ev_count += ENET_REG_READ(sc, ENET_RMON_T_UNDERSIZE);
	sc->sc_ev_t_oversize.ev_count += ENET_REG_READ(sc, ENET_RMON_T_OVERSIZE);
	sc->sc_ev_t_frag.ev_count += ENET_REG_READ(sc, ENET_RMON_T_FRAG);
	sc->sc_ev_t_jab.ev_count += ENET_REG_READ(sc, ENET_RMON_T_JAB);
	sc->sc_ev_t_col.ev_count += ENET_REG_READ(sc, ENET_RMON_T_COL);
	sc->sc_ev_t_p64.ev_count += ENET_REG_READ(sc, ENET_RMON_T_P64);
	sc->sc_ev_t_p65to127n.ev_count += ENET_REG_READ(sc, ENET_RMON_T_P65TO127N);
	sc->sc_ev_t_p128to255n.ev_count += ENET_REG_READ(sc, ENET_RMON_T_P128TO255N);
	sc->sc_ev_t_p256to511.ev_count += ENET_REG_READ(sc, ENET_RMON_T_P256TO511);
	sc->sc_ev_t_p512to1023.ev_count += ENET_REG_READ(sc, ENET_RMON_T_P512TO1023);
	sc->sc_ev_t_p1024to2047.ev_count += ENET_REG_READ(sc, ENET_RMON_T_P1024TO2047);
	sc->sc_ev_t_p_gte2048.ev_count += ENET_REG_READ(sc, ENET_RMON_T_P_GTE2048);
	sc->sc_ev_t_octets.ev_count += ENET_REG_READ(sc, ENET_RMON_T_OCTETS);
	sc->sc_ev_r_packets.ev_count += ENET_REG_READ(sc, ENET_RMON_R_PACKETS);
	sc->sc_ev_r_bc_pkt.ev_count += ENET_REG_READ(sc, ENET_RMON_R_BC_PKT);
	sc->sc_ev_r_mc_pkt.ev_count += ENET_REG_READ(sc, ENET_RMON_R_MC_PKT);
	sc->sc_ev_r_crc_align.ev_count += ENET_REG_READ(sc, ENET_RMON_R_CRC_ALIGN);
	sc->sc_ev_r_undersize.ev_count += ENET_REG_READ(sc, ENET_RMON_R_UNDERSIZE);
	sc->sc_ev_r_oversize.ev_count += ENET_REG_READ(sc, ENET_RMON_R_OVERSIZE);
	sc->sc_ev_r_frag.ev_count += ENET_REG_READ(sc, ENET_RMON_R_FRAG);
	sc->sc_ev_r_jab.ev_count += ENET_REG_READ(sc, ENET_RMON_R_JAB);
	sc->sc_ev_r_p64.ev_count += ENET_REG_READ(sc, ENET_RMON_R_P64);
	sc->sc_ev_r_p65to127.ev_count += ENET_REG_READ(sc, ENET_RMON_R_P65TO127);
	sc->sc_ev_r_p128to255.ev_count += ENET_REG_READ(sc, ENET_RMON_R_P128TO255);
	sc->sc_ev_r_p256to511.ev_count += ENET_REG_READ(sc, ENET_RMON_R_P256TO511);
	sc->sc_ev_r_p512to1023.ev_count += ENET_REG_READ(sc, ENET_RMON_R_P512TO1023);
	sc->sc_ev_r_p1024to2047.ev_count += ENET_REG_READ(sc, ENET_RMON_R_P1024TO2047);
	sc->sc_ev_r_p_gte2048.ev_count += ENET_REG_READ(sc, ENET_RMON_R_P_GTE2048);
	sc->sc_ev_r_octets.ev_count += ENET_REG_READ(sc, ENET_RMON_R_OCTETS);
}
#endif /* ENET_EVENT_COUNTER */

static void
enet_tick(void *arg)
{
	struct enet_softc *sc;
	struct mii_data *mii;
	struct ifnet *ifp;
	int s;

	sc = arg;
	mii = &sc->sc_mii;
	ifp = &sc->sc_ethercom.ec_if;

	s = splnet();

	if (sc->sc_stopping)
		goto out;

#ifdef ENET_EVENT_COUNTER
	enet_update_evcnt(sc);
#endif

	/* update counters */
	if_statadd(ifp, if_ierrors,
	    (uint64_t)ENET_REG_READ(sc, ENET_RMON_R_UNDERSIZE) +
	    (uint64_t)ENET_REG_READ(sc, ENET_RMON_R_FRAG) +
	    (uint64_t)ENET_REG_READ(sc, ENET_RMON_R_JAB));

	/* clear counters */
	ENET_REG_WRITE(sc, ENET_MIBC, ENET_MIBC_MIB_CLEAR);
	ENET_REG_WRITE(sc, ENET_MIBC, 0);

	mii_tick(mii);
 out:

	if (!sc->sc_stopping)
		callout_schedule(&sc->sc_tick_ch, ENET_TICK);

	splx(s);
}

int
enet_intr(void *arg)
{
	struct enet_softc *sc;
	struct ifnet *ifp;
	uint32_t status;

	sc = arg;
	status = ENET_REG_READ(sc, ENET_EIR);

	if (sc->sc_imxtype == 7) {
		if (status & (ENET_EIR_TXF | ENET_EIR_TXF1 | ENET_EIR_TXF2))
			enet_tx_intr(arg);
		if (status & (ENET_EIR_RXF | ENET_EIR_RXF1 | ENET_EIR_RXF2))
			enet_rx_intr(arg);
	} else {
		if (status & ENET_EIR_TXF)
			enet_tx_intr(arg);
		if (status & ENET_EIR_RXF)
			enet_rx_intr(arg);
	}

	if (status & ENET_EIR_EBERR) {
		device_printf(sc->sc_dev, "Ethernet Bus Error\n");
		ifp = &sc->sc_ethercom.ec_if;
		enet_stop(ifp, 1);
		enet_init(ifp);
	} else {
		ENET_REG_WRITE(sc, ENET_EIR, status);
	}

	rnd_add_uint32(&sc->sc_rnd_source, status);

	return 1;
}

static int
enet_tx_intr(void *arg)
{
	struct enet_softc *sc;
	struct ifnet *ifp;
	struct enet_txsoft *txs;
	int idx;

	sc = (struct enet_softc *)arg;
	ifp = &sc->sc_ethercom.ec_if;

	for (idx = sc->sc_tx_considx; idx != sc->sc_tx_prodidx;
	    idx = ENET_TX_NEXTIDX(idx)) {

		txs = &sc->sc_txsoft[idx];

		TXDESC_READIN(idx);
		if (sc->sc_txdesc_ring[idx].tx_flags1_len & TXFLAGS1_R) {
			/* This TX Descriptor has not been transmitted yet */
			break;
		}

		/* txsoft is available on first segment (TXFLAGS1_T1) */
		if (sc->sc_txdesc_ring[idx].tx_flags1_len & TXFLAGS1_T1) {
			bus_dmamap_unload(sc->sc_dmat,
			    txs->txs_dmamap);
			m_freem(txs->txs_mbuf);
			if_statinc(ifp, if_opackets);
		}

		/* checking error */
		if (sc->sc_txdesc_ring[idx].tx_flags1_len & TXFLAGS1_L) {
			uint32_t flags2;

			flags2 = sc->sc_txdesc_ring[idx].tx_flags2;

			if (flags2 & (TXFLAGS2_TXE |
			    TXFLAGS2_UE | TXFLAGS2_EE | TXFLAGS2_FE |
			    TXFLAGS2_LCE | TXFLAGS2_OE | TXFLAGS2_TSE)) {
#ifdef DEBUG_ENET
				if (enet_debug) {
					char flagsbuf[128];

					snprintb(flagsbuf, sizeof(flagsbuf),
					    "\20" "\20TRANSMIT" "\16UNDERFLOW"
					    "\15COLLISION" "\14FRAME"
					    "\13LATECOLLISION" "\12OVERFLOW",
					    flags2);

					device_printf(sc->sc_dev,
					    "txdesc[%d]: transmit error: "
					    "flags2=%s\n", idx, flagsbuf);
				}
#endif /* DEBUG_ENET */
				if_statinc(ifp, if_oerrors);
			}
		}

		sc->sc_tx_free++;
	}
	sc->sc_tx_considx = idx;

	if (sc->sc_tx_free > 0)
		sc->sc_txbusy = false;

	/*
	 * No more pending TX descriptor,
	 * cancel the watchdog timer.
	 */
	if (sc->sc_tx_free == ENET_TX_RING_CNT)
		ifp->if_timer = 0;

	return 1;
}

static int
enet_rx_intr(void *arg)
{
	struct enet_softc *sc;
	struct ifnet *ifp;
	struct enet_rxsoft *rxs;
	int idx, len, amount;
	uint32_t flags1, flags2;
	struct mbuf *m, *m0, *mprev;

	sc = arg;
	ifp = &sc->sc_ethercom.ec_if;

	m0 = mprev = NULL;
	amount = 0;
	for (idx = sc->sc_rx_readidx; ; idx = ENET_RX_NEXTIDX(idx)) {

		rxs = &sc->sc_rxsoft[idx];

		RXDESC_READIN(idx);
		if (sc->sc_rxdesc_ring[idx].rx_flags1_len & RXFLAGS1_E) {
			/* This RX Descriptor has not been received yet */
			break;
		}

		/*
		 * build mbuf from RX Descriptor if needed
		 */
		m = rxs->rxs_mbuf;
		rxs->rxs_mbuf = NULL;

		flags1 = sc->sc_rxdesc_ring[idx].rx_flags1_len;
		len = RXFLAGS1_LEN(flags1);

#define RACC_SHIFT16	2
		if (m0 == NULL) {
			m0 = m;
			m_adj(m0, RACC_SHIFT16);
			len -= RACC_SHIFT16;
			m->m_len = len;
			amount = len;
		} else {
			if (flags1 & RXFLAGS1_L)
				len = len - amount - RACC_SHIFT16;

			m->m_len = len;
			amount += len;
			if (m->m_flags & M_PKTHDR)
				m_remove_pkthdr(m);
			mprev->m_next = m;
		}
		mprev = m;

		flags2 = sc->sc_rxdesc_ring[idx].rx_flags2;

		if (flags1 & RXFLAGS1_L) {
			/* last buffer */
			if ((amount < ETHER_HDR_LEN) ||
			    ((flags1 & (RXFLAGS1_LG | RXFLAGS1_NO |
			    RXFLAGS1_CR | RXFLAGS1_OV | RXFLAGS1_TR)) ||
			    (flags2 & (RXFLAGS2_ME | RXFLAGS2_PE |
			    RXFLAGS2_CE)))) {

#ifdef DEBUG_ENET
				if (enet_debug) {
					char flags1buf[128], flags2buf[128];
					snprintb(flags1buf, sizeof(flags1buf),
					    "\20" "\31MISS" "\26LENGTHVIOLATION"
					    "\25NONOCTET" "\23CRC" "\22OVERRUN"
					    "\21TRUNCATED", flags1);
					snprintb(flags2buf, sizeof(flags2buf),
					    "\20" "\40MAC" "\33PHY"
					    "\32COLLISION", flags2);

					DEVICE_DPRINTF(
					    "rxdesc[%d]: receive error: "
					    "flags1=%s,flags2=%s,len=%d\n",
					    idx, flags1buf, flags2buf, amount);
				}
#endif /* DEBUG_ENET */
				if_statinc(ifp, if_ierrors);
				m_freem(m0);

			} else {
				/* packet receive ok */
				m_set_rcvif(m0, ifp);
				m0->m_pkthdr.len = amount;

				bus_dmamap_sync(sc->sc_dmat, rxs->rxs_dmamap, 0,
				    rxs->rxs_dmamap->dm_mapsize,
				    BUS_DMASYNC_PREREAD);

				if (ifp->if_csum_flags_rx & (M_CSUM_IPv4 |
				    M_CSUM_TCPv4 | M_CSUM_UDPv4 |
				    M_CSUM_TCPv6 | M_CSUM_UDPv6))
					enet_rx_csum(sc, ifp, m0, idx);

				if_percpuq_enqueue(ifp->if_percpuq, m0);
			}

			m0 = NULL;
			mprev = NULL;
			amount = 0;

		} else {
			/* continued from previous buffer */
			bus_dmamap_sync(sc->sc_dmat, rxs->rxs_dmamap, 0,
			    rxs->rxs_dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
		}

		bus_dmamap_unload(sc->sc_dmat, rxs->rxs_dmamap);
		if (enet_alloc_rxbuf(sc, idx) != 0) {
			panic("enet_alloc_rxbuf NULL\n");
		}
	}
	sc->sc_rx_readidx = idx;

	/* re-enable RX DMA to make sure */
	ENET_REG_WRITE(sc, ENET_RDAR, ENET_RDAR_ACTIVE);

	return 1;
}

static void
enet_rx_csum(struct enet_softc *sc, struct ifnet *ifp, struct mbuf *m, int idx)
{
	uint32_t flags2;
	uint8_t proto;

	flags2 = sc->sc_rxdesc_ring[idx].rx_flags2;

	if (flags2 & RXFLAGS2_IPV6) {
		proto = sc->sc_rxdesc_ring[idx].rx_proto;

		/* RXFLAGS2_PCR is valid when IPv6 and TCP/UDP */
		if ((proto == IPPROTO_TCP) &&
		    (ifp->if_csum_flags_rx & M_CSUM_TCPv6))
			m->m_pkthdr.csum_flags |= M_CSUM_TCPv6;
		else if ((proto == IPPROTO_UDP) &&
		    (ifp->if_csum_flags_rx & M_CSUM_UDPv6))
			m->m_pkthdr.csum_flags |= M_CSUM_UDPv6;
		else
			return;

		/* IPv6 protocol checksum error */
		if (flags2 & RXFLAGS2_PCR)
			m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;

	} else {
		struct ether_header *eh;
		uint8_t *ip;

		eh = mtod(m, struct ether_header *);

		/* XXX: is an IPv4? */
		if (ntohs(eh->ether_type) != ETHERTYPE_IP)
			return;
		ip = (uint8_t *)(eh + 1);
		if ((ip[0] & 0xf0) == 0x40)
			return;

		proto = sc->sc_rxdesc_ring[idx].rx_proto;
		if (flags2 & RXFLAGS2_ICE) {
			if (ifp->if_csum_flags_rx & M_CSUM_IPv4) {
				m->m_pkthdr.csum_flags |=
				    M_CSUM_IPv4 | M_CSUM_IPv4_BAD;
			}
		} else {
			if (ifp->if_csum_flags_rx & M_CSUM_IPv4) {
				m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
			}

			/*
			 * PCR is valid when
			 * ICE == 0 and FRAG == 0
			 */
			if (flags2 & RXFLAGS2_FRAG)
				return;

			/*
			 * PCR is valid when proto is TCP or UDP
			 */
			if ((proto == IPPROTO_TCP) &&
			    (ifp->if_csum_flags_rx & M_CSUM_TCPv4))
				m->m_pkthdr.csum_flags |= M_CSUM_TCPv4;
			else if ((proto == IPPROTO_UDP) &&
			    (ifp->if_csum_flags_rx & M_CSUM_UDPv4))
				m->m_pkthdr.csum_flags |= M_CSUM_UDPv4;
			else
				return;

			/* IPv4 protocol cksum error */
			if (flags2 & RXFLAGS2_PCR)
				m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;
		}
	}
}

static void
enet_setmulti(struct enet_softc *sc)
{
	struct ethercom *ec = &sc->sc_ethercom;
	struct ifnet *ifp = &ec->ec_if;
	struct ether_multi *enm;
	struct ether_multistep step;
	uint32_t crc, hashidx;
	uint32_t gaddr[2];

	if (ifp->if_flags & IFF_PROMISC) {
		/* receive all unicast packet */
		ENET_REG_WRITE(sc, ENET_IAUR, 0xffffffff);
		ENET_REG_WRITE(sc, ENET_IALR, 0xffffffff);
		/* receive all multicast packet */
		gaddr[0] = gaddr[1] = 0xffffffff;
	} else {
		gaddr[0] = gaddr[1] = 0;

		ETHER_LOCK(ec);
		ETHER_FIRST_MULTI(step, ec, enm);
		while (enm != NULL) {
			if (memcmp(enm->enm_addrlo, enm->enm_addrhi,
			    ETHER_ADDR_LEN)) {
				/*
				 * if specified by range, give up setting hash,
				 * and fallback to allmulti.
				 */
				gaddr[0] = gaddr[1] = 0xffffffff;
				break;
			}

			crc = ether_crc32_le(enm->enm_addrlo, ETHER_ADDR_LEN);
			hashidx = __SHIFTOUT(crc, __BITS(30,26));
			gaddr[__SHIFTOUT(crc, __BIT(31))] |= __BIT(hashidx);

			ETHER_NEXT_MULTI(step, enm);
		}
		ETHER_UNLOCK(ec);

		/* dont't receive any unicast packet (except own address) */
		ENET_REG_WRITE(sc, ENET_IAUR, 0);
		ENET_REG_WRITE(sc, ENET_IALR, 0);
	}

	if (gaddr[0] == 0xffffffff && gaddr[1] == 0xffffffff)
		ifp->if_flags |= IFF_ALLMULTI;
	else
		ifp->if_flags &= ~IFF_ALLMULTI;

	/* receive multicast packets according to multicast filter */
	ENET_REG_WRITE(sc, ENET_GAUR, gaddr[1]);
	ENET_REG_WRITE(sc, ENET_GALR, gaddr[0]);

}

static void
enet_gethwaddr(struct enet_softc *sc, uint8_t *hwaddr)
{
	uint32_t paddr;

	paddr = ENET_REG_READ(sc, ENET_PALR);
	hwaddr[0] = paddr >> 24;
	hwaddr[1] = paddr >> 16;
	hwaddr[2] = paddr >> 8;
	hwaddr[3] = paddr;

	paddr = ENET_REG_READ(sc, ENET_PAUR);
	hwaddr[4] = paddr >> 24;
	hwaddr[5] = paddr >> 16;
}

static void
enet_sethwaddr(struct enet_softc *sc, uint8_t *hwaddr)
{
	uint32_t paddr;

	paddr = (hwaddr[0] << 24) | (hwaddr[1] << 16) | (hwaddr[2] << 8) |
	    hwaddr[3];
	ENET_REG_WRITE(sc, ENET_PALR, paddr);
	paddr = (hwaddr[4] << 24) | (hwaddr[5] << 16);
	ENET_REG_WRITE(sc, ENET_PAUR, paddr);
}

/*
 * ifnet interfaces
 */
static int
enet_init(struct ifnet *ifp)
{
	struct enet_softc *sc;
	int s, error;

	sc = ifp->if_softc;

	s = splnet();

	enet_init_regs(sc, 0);
	enet_init_txring(sc);
	error = enet_init_rxring(sc);
	if (error != 0) {
		enet_drain_rxbuf(sc);
		device_printf(sc->sc_dev, "Cannot allocate mbuf cluster\n");
		goto init_failure;
	}

	/* reload mac address */
	memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
	enet_sethwaddr(sc, sc->sc_enaddr);

	/* program multicast address */
	enet_setmulti(sc);

	/* update if_flags */
	ifp->if_flags |= IFF_RUNNING;
	sc->sc_txbusy = false;

	/* update local copy of if_flags */
	sc->sc_if_flags = ifp->if_flags;

	/* mii */
	mii_mediachg(&sc->sc_mii);

	/* enable RX DMA */
	ENET_REG_WRITE(sc, ENET_RDAR, ENET_RDAR_ACTIVE);

	sc->sc_stopping = false;
	callout_schedule(&sc->sc_tick_ch, ENET_TICK);

 init_failure:
	splx(s);

	return error;
}

static void
enet_start(struct ifnet *ifp)
{
	struct enet_softc *sc;
	struct mbuf *m;
	int npkt;

	if ((ifp->if_flags & IFF_RUNNING) == 0)
		return;

	sc = ifp->if_softc;
	for (npkt = 0; !sc->sc_txbusy; npkt++) {
		IFQ_POLL(&ifp->if_snd, m);
		if (m == NULL)
			break;

		if (sc->sc_tx_free <= 0) {
			/* no tx descriptor now... */
			sc->sc_txbusy = true;
			DEVICE_DPRINTF("TX descriptor is full\n");
			break;
		}

		IFQ_DEQUEUE(&ifp->if_snd, m);

		if (enet_encap_txring(sc, &m) != 0) {
			/* too many mbuf chains? */
			sc->sc_txbusy = true;
			DEVICE_DPRINTF(
			    "TX descriptor is full. dropping packet\n");
			m_freem(m);
			if_statinc(ifp, if_oerrors);
			break;
		}

		/* Pass the packet to any BPF listeners */
		bpf_mtap(ifp, m, BPF_D_OUT);
	}

	if (npkt) {
		/* enable TX DMA */
		ENET_REG_WRITE(sc, ENET_TDAR, ENET_TDAR_ACTIVE);

		ifp->if_timer = 5;
	}
}

static void
enet_stop(struct ifnet *ifp, int disable)
{
	struct enet_softc *sc;
	int s;
	uint32_t v;

	sc = ifp->if_softc;

	s = splnet();

	sc->sc_stopping = true;
	callout_stop(&sc->sc_tick_ch);

	/* clear ENET_ECR[ETHEREN] to abort receive and transmit */
	v = ENET_REG_READ(sc, ENET_ECR);
	ENET_REG_WRITE(sc, ENET_ECR, v & ~ENET_ECR_ETHEREN);

	/* Mark the interface as down and cancel the watchdog timer. */
	ifp->if_flags &= ~IFF_RUNNING;
	ifp->if_timer = 0;
	sc->sc_txbusy = false;

	if (disable) {
		enet_drain_txbuf(sc);
		enet_drain_rxbuf(sc);
	}

	splx(s);
}

static void
enet_watchdog(struct ifnet *ifp)
{
	struct enet_softc *sc;
	int s;

	sc = ifp->if_softc;
	s = splnet();

	device_printf(sc->sc_dev, "watchdog timeout\n");
	if_statinc(ifp, if_oerrors);

	/* salvage packets left in descriptors */
	enet_tx_intr(sc);
	enet_rx_intr(sc);

	/* reset */
	enet_stop(ifp, 1);
	enet_init(ifp);

	splx(s);
}

static void
enet_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
{
	struct enet_softc *sc = ifp->if_softc;

	ether_mediastatus(ifp, ifmr);
	ifmr->ifm_active = (ifmr->ifm_active & ~IFM_ETH_FMASK)
	    | sc->sc_flowflags;
}

static int
enet_ifflags_cb(struct ethercom *ec)
{
	struct ifnet *ifp = &ec->ec_if;
	struct enet_softc *sc = ifp->if_softc;
	u_short change = ifp->if_flags ^ sc->sc_if_flags;

	if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0)
		return ENETRESET;
	else if ((change & (IFF_PROMISC | IFF_ALLMULTI)) == 0)
		return 0;

	enet_setmulti(sc);

	sc->sc_if_flags = ifp->if_flags;
	return 0;
}

static int
enet_ioctl(struct ifnet *ifp, u_long command, void *data)
{
	struct enet_softc *sc;
	struct ifreq *ifr;
	int s, error;
	uint32_t v;

	sc = ifp->if_softc;
	ifr = data;

	error = 0;

	s = splnet();

	switch (command) {
	case SIOCSIFMTU:
		if (MTU2FRAMESIZE(ifr->ifr_mtu) > ENET_MAX_PKT_LEN) {
			error = EINVAL;
		} else {
			ifp->if_mtu = ifr->ifr_mtu;

			/* set maximum frame length */
			v = MTU2FRAMESIZE(ifr->ifr_mtu);
			ENET_REG_WRITE(sc, ENET_FTRL, v);
			v = ENET_REG_READ(sc, ENET_RCR);
			v &= ~ENET_RCR_MAX_FL(0x3fff);
			v |= ENET_RCR_MAX_FL(ifp->if_mtu + ETHER_HDR_LEN +
			    ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN);
			ENET_REG_WRITE(sc, ENET_RCR, v);
		}
		break;
	case SIOCSIFMEDIA:
		/* Flow control requires full-duplex mode. */
		if (IFM_SUBTYPE(ifr->ifr_media) == IFM_AUTO ||
		    (ifr->ifr_media & IFM_FDX) == 0)
			ifr->ifr_media &= ~IFM_ETH_FMASK;
		if (IFM_SUBTYPE(ifr->ifr_media) != IFM_AUTO) {
			if ((ifr->ifr_media & IFM_ETH_FMASK) == IFM_FLOW) {
				/* We can do both TXPAUSE and RXPAUSE. */
				ifr->ifr_media |=
				    IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE;
			}
			sc->sc_flowflags = ifr->ifr_media & IFM_ETH_FMASK;
		}
		error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, command);
		break;
	default:
		error = ether_ioctl(ifp, command, data);
		if (error != ENETRESET)
			break;

		/* post-process */
		error = 0;
		switch (command) {
		case SIOCSIFCAP:
			error = if_init(ifp);
			break;
		case SIOCADDMULTI:
		case SIOCDELMULTI:
			if (ifp->if_flags & IFF_RUNNING)
				enet_setmulti(sc);
			break;
		}
		break;
	}

	splx(s);

	return error;
}

/*
 * for MII
 */
static int
enet_miibus_readreg(device_t dev, int phy, int reg, uint16_t *val)
{
	struct enet_softc *sc;
	int timeout;
	uint32_t status;

	sc = device_private(dev);

	/* clear MII update */
	ENET_REG_WRITE(sc, ENET_EIR, ENET_EIR_MII);

	/* read command */
	ENET_REG_WRITE(sc, ENET_MMFR,
	    ENET_MMFR_ST | ENET_MMFR_OP_READ | ENET_MMFR_TA |
	    ENET_MMFR_PHY_REG(reg) | ENET_MMFR_PHY_ADDR(phy));

	/* check MII update */
	for (timeout = 5000; timeout > 0; --timeout) {
		status = ENET_REG_READ(sc, ENET_EIR);
		if (status & ENET_EIR_MII)
			break;
	}
	if (timeout <= 0) {
		DEVICE_DPRINTF("MII read timeout: reg=0x%02x\n",
		    reg);
		return ETIMEDOUT;
	} else
		*val = ENET_REG_READ(sc, ENET_MMFR) & ENET_MMFR_DATAMASK;

	return 0;
}

static int
enet_miibus_writereg(device_t dev, int phy, int reg, uint16_t val)
{
	struct enet_softc *sc;
	int timeout;

	sc = device_private(dev);

	/* clear MII update */
	ENET_REG_WRITE(sc, ENET_EIR, ENET_EIR_MII);

	/* write command */
	ENET_REG_WRITE(sc, ENET_MMFR,
	    ENET_MMFR_ST | ENET_MMFR_OP_WRITE | ENET_MMFR_TA |
	    ENET_MMFR_PHY_REG(reg) | ENET_MMFR_PHY_ADDR(phy) |
	    (ENET_MMFR_DATAMASK & val));

	/* check MII update */
	for (timeout = 5000; timeout > 0; --timeout) {
		if (ENET_REG_READ(sc, ENET_EIR) & ENET_EIR_MII)
			break;
	}
	if (timeout <= 0) {
		DEVICE_DPRINTF("MII write timeout: reg=0x%02x\n", reg);
		return ETIMEDOUT;
	}

	return 0;
}

static void
enet_miibus_statchg(struct ifnet *ifp)
{
	struct enet_softc *sc;
	struct mii_data *mii;
	struct ifmedia_entry *ife;
	uint32_t ecr, ecr0;
	uint32_t rcr, rcr0;
	uint32_t tcr, tcr0;

	sc = ifp->if_softc;
	mii = &sc->sc_mii;
	ife = mii->mii_media.ifm_cur;

	/* get current status */
	ecr0 = ecr = ENET_REG_READ(sc, ENET_ECR) & ~ENET_ECR_RESET;
	rcr0 = rcr = ENET_REG_READ(sc, ENET_RCR);
	tcr0 = tcr = ENET_REG_READ(sc, ENET_TCR);

	if (IFM_SUBTYPE(mii->mii_media.ifm_cur->ifm_media) == IFM_AUTO &&
	    (mii->mii_media_active & IFM_ETH_FMASK) != sc->sc_flowflags) {
		sc->sc_flowflags = mii->mii_media_active & IFM_ETH_FMASK;
		mii->mii_media_active &= ~IFM_ETH_FMASK;
	}

	if ((ife->ifm_media & IFM_FDX) != 0) {
		tcr |= ENET_TCR_FDEN;	/* full duplex */
		rcr &= ~ENET_RCR_DRT;	/* enable receive on transmit */
	} else {
		tcr &= ~ENET_TCR_FDEN;	/* half duplex */
		rcr |= ENET_RCR_DRT;	/* disable receive on transmit */
	}

	if ((tcr ^ tcr0) & ENET_TCR_FDEN) {
		/*
		 * need to reset because
		 * FDEN can change when ECR[ETHEREN] is 0
		 */
		enet_init_regs(sc, 0);
		return;
	}

	switch (IFM_SUBTYPE(ife->ifm_media)) {
	case IFM_AUTO:
	case IFM_1000_T:
		ecr |= ENET_ECR_SPEED;		/* 1000Mbps mode */
		rcr &= ~ENET_RCR_RMII_10T;
		break;
	case IFM_100_TX:
		ecr &= ~ENET_ECR_SPEED;		/* 100Mbps mode */
		rcr &= ~ENET_RCR_RMII_10T;	/* 100Mbps mode */
		break;
	case IFM_10_T:
		ecr &= ~ENET_ECR_SPEED;		/* 10Mbps mode */
		rcr |= ENET_RCR_RMII_10T;	/* 10Mbps mode */
		break;
	default:
		ecr = ecr0;
		rcr = rcr0;
		tcr = tcr0;
		break;
	}

	if (sc->sc_rgmii == 0)
		ecr &= ~ENET_ECR_SPEED;

	if (sc->sc_flowflags & IFM_FLOW)
		rcr |= ENET_RCR_FCE;
	else
		rcr &= ~ENET_RCR_FCE;

	/* update registers if need change */
	if (ecr != ecr0)
		ENET_REG_WRITE(sc, ENET_ECR, ecr);
	if (rcr != rcr0)
		ENET_REG_WRITE(sc, ENET_RCR, rcr);
	if (tcr != tcr0)
		ENET_REG_WRITE(sc, ENET_TCR, tcr);
}

/*
 * handling descriptors
 */
static void
enet_init_txring(struct enet_softc *sc)
{
	int i;

	/* build TX ring */
	for (i = 0; i < ENET_TX_RING_CNT; i++) {
		sc->sc_txdesc_ring[i].tx_flags1_len =
		    ((i == (ENET_TX_RING_CNT - 1)) ? TXFLAGS1_W : 0);
		sc->sc_txdesc_ring[i].tx_databuf = 0;
		sc->sc_txdesc_ring[i].tx_flags2 = TXFLAGS2_INT;
		sc->sc_txdesc_ring[i].tx__reserved1 = 0;
		sc->sc_txdesc_ring[i].tx_flags3 = 0;
		sc->sc_txdesc_ring[i].tx_1588timestamp = 0;
		sc->sc_txdesc_ring[i].tx__reserved2 = 0;
		sc->sc_txdesc_ring[i].tx__reserved3 = 0;

		TXDESC_WRITEOUT(i);
	}

	sc->sc_tx_free = ENET_TX_RING_CNT;
	sc->sc_tx_considx = 0;
	sc->sc_tx_prodidx = 0;
}

static int
enet_init_rxring(struct enet_softc *sc)
{
	int i, error;

	/* build RX ring */
	for (i = 0; i < ENET_RX_RING_CNT; i++) {
		error = enet_alloc_rxbuf(sc, i);
		if (error != 0)
			return error;
	}

	sc->sc_rx_readidx = 0;

	return 0;
}

static int
enet_alloc_rxbuf(struct enet_softc *sc, int idx)
{
	struct mbuf *m;
	int error;

	KASSERT((idx >= 0) && (idx < ENET_RX_RING_CNT));

	/* free mbuf if already allocated */
	if (sc->sc_rxsoft[idx].rxs_mbuf != NULL) {
		bus_dmamap_unload(sc->sc_dmat, sc->sc_rxsoft[idx].rxs_dmamap);
		m_freem(sc->sc_rxsoft[idx].rxs_mbuf);
		sc->sc_rxsoft[idx].rxs_mbuf = NULL;
	}

	/* allocate new mbuf cluster */
	MGETHDR(m, M_DONTWAIT, MT_DATA);
	if (m == NULL)
		return ENOBUFS;
	MCLGET(m, M_DONTWAIT);
	if (!(m->m_flags & M_EXT)) {
		m_freem(m);
		return ENOBUFS;
	}
	m->m_len = MCLBYTES;
	m->m_next = NULL;

	error = bus_dmamap_load(sc->sc_dmat, sc->sc_rxsoft[idx].rxs_dmamap,
	    m->m_ext.ext_buf, m->m_ext.ext_size, NULL,
	    BUS_DMA_READ | BUS_DMA_NOWAIT);
	if (error) {
		m_freem(m);
		return error;
	}

	bus_dmamap_sync(sc->sc_dmat, sc->sc_rxsoft[idx].rxs_dmamap, 0,
	    sc->sc_rxsoft[idx].rxs_dmamap->dm_mapsize,
	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

	sc->sc_rxsoft[idx].rxs_mbuf = m;
	enet_reset_rxdesc(sc, idx);
	return 0;
}

static void
enet_reset_rxdesc(struct enet_softc *sc, int idx)
{
	uint32_t paddr;

	paddr = sc->sc_rxsoft[idx].rxs_dmamap->dm_segs[0].ds_addr;

	sc->sc_rxdesc_ring[idx].rx_flags1_len =
	    RXFLAGS1_E |
	    ((idx == (ENET_RX_RING_CNT - 1)) ? RXFLAGS1_W : 0);
	sc->sc_rxdesc_ring[idx].rx_databuf = paddr;
	sc->sc_rxdesc_ring[idx].rx_flags2 =
	    RXFLAGS2_INT;
	sc->sc_rxdesc_ring[idx].rx_hl = 0;
	sc->sc_rxdesc_ring[idx].rx_proto = 0;
	sc->sc_rxdesc_ring[idx].rx_cksum = 0;
	sc->sc_rxdesc_ring[idx].rx_flags3 = 0;
	sc->sc_rxdesc_ring[idx].rx_1588timestamp = 0;
	sc->sc_rxdesc_ring[idx].rx__reserved2 = 0;
	sc->sc_rxdesc_ring[idx].rx__reserved3 = 0;

	RXDESC_WRITEOUT(idx);
}

static void
enet_drain_txbuf(struct enet_softc *sc)
{
	int idx;
	struct enet_txsoft *txs;
	struct ifnet *ifp;

	ifp = &sc->sc_ethercom.ec_if;

	for (idx = sc->sc_tx_considx; idx != sc->sc_tx_prodidx;
	    idx = ENET_TX_NEXTIDX(idx)) {

		/* txsoft[] is used only first segment */
		txs = &sc->sc_txsoft[idx];
		TXDESC_READIN(idx);
		if (sc->sc_txdesc_ring[idx].tx_flags1_len & TXFLAGS1_T1) {
			sc->sc_txdesc_ring[idx].tx_flags1_len = 0;
			bus_dmamap_unload(sc->sc_dmat,
			    txs->txs_dmamap);
			m_freem(txs->txs_mbuf);

			if_statinc(ifp, if_oerrors);
		}
		sc->sc_tx_free++;
	}
}

static void
enet_drain_rxbuf(struct enet_softc *sc)
{
	int i;

	for (i = 0; i < ENET_RX_RING_CNT; i++) {
		if (sc->sc_rxsoft[i].rxs_mbuf != NULL) {
			sc->sc_rxdesc_ring[i].rx_flags1_len = 0;
			bus_dmamap_unload(sc->sc_dmat,
			    sc->sc_rxsoft[i].rxs_dmamap);
			m_freem(sc->sc_rxsoft[i].rxs_mbuf);
			sc->sc_rxsoft[i].rxs_mbuf = NULL;
		}
	}
}

static int
enet_alloc_ring(struct enet_softc *sc)
{
	int i, error;

	/*
	 * build DMA maps for TX.
	 * TX descriptor must be able to contain mbuf chains,
	 * so, make up ENET_MAX_PKT_NSEGS dmamap.
	 */
	for (i = 0; i < ENET_TX_RING_CNT; i++) {
		error = bus_dmamap_create(sc->sc_dmat, ENET_MAX_PKT_LEN,
		    ENET_MAX_PKT_NSEGS, ENET_MAX_PKT_LEN, 0, BUS_DMA_NOWAIT,
		    &sc->sc_txsoft[i].txs_dmamap);

		if (error) {
			aprint_error_dev(sc->sc_dev,
			    "can't create DMA map for TX descs\n");
			goto fail_1;
		}
	}

	/*
	 * build DMA maps for RX.
	 * RX descripter contains An mbuf cluster,
	 * and make up a dmamap.
	 */
	for (i = 0; i < ENET_RX_RING_CNT; i++) {
		error = bus_dmamap_create(sc->sc_dmat, MCLBYTES,
		    1, MCLBYTES, 0, BUS_DMA_NOWAIT,
		    &sc->sc_rxsoft[i].rxs_dmamap);
		if (error) {
			aprint_error_dev(sc->sc_dev,
			    "can't create DMA map for RX descs\n");
			goto fail_2;
		}
	}

	if (enet_alloc_dma(sc, sizeof(struct enet_txdesc) * ENET_TX_RING_CNT,
	    (void **)&(sc->sc_txdesc_ring), &(sc->sc_txdesc_dmamap)) != 0)
		return -1;
	memset(sc->sc_txdesc_ring, 0,
	    sizeof(struct enet_txdesc) * ENET_TX_RING_CNT);

	if (enet_alloc_dma(sc, sizeof(struct enet_rxdesc) * ENET_RX_RING_CNT,
	    (void **)&(sc->sc_rxdesc_ring), &(sc->sc_rxdesc_dmamap)) != 0)
		return -1;
	memset(sc->sc_rxdesc_ring, 0,
	    sizeof(struct enet_rxdesc) * ENET_RX_RING_CNT);

	return 0;

 fail_2:
	for (i = 0; i < ENET_RX_RING_CNT; i++) {
		if (sc->sc_rxsoft[i].rxs_dmamap != NULL)
			bus_dmamap_destroy(sc->sc_dmat,
			    sc->sc_rxsoft[i].rxs_dmamap);
	}
 fail_1:
	for (i = 0; i < ENET_TX_RING_CNT; i++) {
		if (sc->sc_txsoft[i].txs_dmamap != NULL)
			bus_dmamap_destroy(sc->sc_dmat,
			    sc->sc_txsoft[i].txs_dmamap);
	}
	return error;
}

static int
enet_encap_mbufalign(struct mbuf **mp)
{
	struct mbuf *m, *m0, *mt, *p, *x;
	void *ap;
	uint32_t alignoff, chiplen;

	/*
	 * iMX6 SoC ethernet controller requires
	 * address of buffer must aligned 8, and
	 * length of buffer must be greater than 10 (first fragment only?)
	 */
#define ALIGNBYTE	8
#define MINBUFSIZE	10
#define ALIGN_PTR(p, align)	\
	(void *)(((uintptr_t)(p) + ((align) - 1)) & -(align))

	m0 = *mp;
	mt = p = NULL;
	for (m = m0; m != NULL; m = m->m_next) {
		alignoff = (uintptr_t)m->m_data & (ALIGNBYTE - 1);
		if (m->m_len < (ALIGNBYTE * 2)) {
			/*
			 * rearrange mbuf data aligned
			 *
			 *        align 8 *       *       *       *       *
			 *               +0123456789abcdef0123456789abcdef0
			 * FROM m->m_data[___________abcdefghijklmn_______]
			 *
			 *               +0123456789abcdef0123456789abcdef0
			 * TO   m->m_data[________abcdefghijklm___________] or
			 *      m->m_data[________________abcdefghijklmn__]
			 */
			if ((alignoff != 0) && (m->m_len != 0)) {
				chiplen = ALIGNBYTE - alignoff;
				if (M_LEADINGSPACE(m) >= alignoff) {
					ap = m->m_data - alignoff;
					memmove(ap, m->m_data, m->m_len);
					m->m_data = ap;
				} else if (M_TRAILINGSPACE(m) >= chiplen) {
					ap = m->m_data + chiplen;
					memmove(ap, m->m_data, m->m_len);
					m->m_data = ap;
				} else {
					/*
					 * no space to align data. (M_READONLY?)
					 * allocate new mbuf aligned,
					 * and copy to it.
					 */
					MGET(x, M_DONTWAIT, m->m_type);
					if (x == NULL) {
						m_freem(m);
						return ENOBUFS;
					}
					MCLAIM(x, m->m_owner);
					if (m->m_flags & M_PKTHDR)
						m_move_pkthdr(x, m);
					x->m_len = m->m_len;
					x->m_data = ALIGN_PTR(x->m_data,
					    ALIGNBYTE);
					memcpy(mtod(x, void *), mtod(m, void *),
					    m->m_len);
					p->m_next = x;
					x->m_next = m_free(m);
					m = x;
				}
			}

			/*
			 * fill 1st mbuf at least 10byte
			 *
			 *        align 8 *       *       *       *       *
			 *               +0123456789abcdef0123456789abcdef0
			 * FROM m->m_data[________abcde___________________]
			 *      m->m_data[__fg____________________________]
			 *      m->m_data[_________________hi_____________]
			 *      m->m_data[__________jk____________________]
			 *      m->m_data[____l___________________________]
			 *
			 *               +0123456789abcdef0123456789abcdef0
			 * TO   m->m_data[________abcdefghij______________]
			 *      m->m_data[________________________________]
			 *      m->m_data[________________________________]
			 *      m->m_data[___________k____________________]
			 *      m->m_data[____l___________________________]
			 */
			if (mt == NULL) {
				mt = m;
				while (mt->m_len == 0) {
					mt = mt->m_next;
					if (mt == NULL) {
						m_freem(m);
						return ENOBUFS;
					}
				}

				/* mt = 1st mbuf, x = 2nd mbuf */
				x = mt->m_next;
				while (mt->m_len < MINBUFSIZE) {
					if (x == NULL) {
						m_freem(m);
						return ENOBUFS;
					}

					alignoff = (uintptr_t)x->m_data &
					    (ALIGNBYTE - 1);
					chiplen = ALIGNBYTE - alignoff;
					if (chiplen > x->m_len) {
						chiplen = x->m_len;
					} else if ((mt->m_len + chiplen) <
					    MINBUFSIZE) {
						/*
						 * next mbuf should be greater
						 * than ALIGNBYTE?
						 */
						if (x->m_len >= (chiplen +
						    ALIGNBYTE * 2))
							chiplen += ALIGNBYTE;
						else
							chiplen = x->m_len;
					}

					if (chiplen &&
					    (M_TRAILINGSPACE(mt) < chiplen)) {
						/*
						 * move data to the beginning of
						 * m_dat[] (aligned) to en-
						 * large trailingspace
						 */
						ap = M_BUFADDR(mt);
						ap = ALIGN_PTR(ap, ALIGNBYTE);
						memcpy(ap, mt->m_data,
						    mt->m_len);
						mt->m_data = ap;
					}

					if (chiplen &&
					    (M_TRAILINGSPACE(mt) >= chiplen)) {
						memcpy(mt->m_data + mt->m_len,
						    x->m_data, chiplen);
						mt->m_len += chiplen;
						m_adj(x, chiplen);
					}

					x = x->m_next;
				}
			}

		} else {
			mt = m;

			/*
			 * allocate new mbuf x, and rearrange as below;
			 *
			 *        align 8 *       *       *       *       *
			 *               +0123456789abcdef0123456789abcdef0
			 * FROM m->m_data[____________abcdefghijklmnopq___]
			 *
			 *               +0123456789abcdef0123456789abcdef0
			 * TO   x->m_data[________abcdefghijkl____________]
			 *      m->m_data[________________________mnopq___]
			 *
			 */
			if (alignoff != 0) {
				/* at least ALIGNBYTE */
				chiplen = ALIGNBYTE - alignoff + ALIGNBYTE;

				MGET(x, M_DONTWAIT, m->m_type);
				if (x == NULL) {
					m_freem(m);
					return ENOBUFS;
				}
				MCLAIM(x, m->m_owner);
				if (m->m_flags & M_PKTHDR)
					m_move_pkthdr(x, m);
				x->m_data = ALIGN_PTR(x->m_data, ALIGNBYTE);
				memcpy(mtod(x, void *), mtod(m, void *),
				    chiplen);
				x->m_len = chiplen;
				x->m_next = m;
				m_adj(m, chiplen);

				if (p == NULL)
					m0 = x;
				else
					p->m_next = x;
			}
		}
		p = m;
	}
	*mp = m0;

	return 0;
}

static int
enet_encap_txring(struct enet_softc *sc, struct mbuf **mp)
{
	bus_dmamap_t map;
	struct mbuf *m;
	int csumflags, idx, i, error;
	uint32_t flags1, flags2;

	idx = sc->sc_tx_prodidx;
	map = sc->sc_txsoft[idx].txs_dmamap;

	/* align mbuf data for claim of ENET */
	error = enet_encap_mbufalign(mp);
	if (error != 0)
		return error;

	m = *mp;
	csumflags = m->m_pkthdr.csum_flags;

	error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
	    BUS_DMA_NOWAIT);
	if (error != 0) {
		device_printf(sc->sc_dev,
		    "Error mapping mbuf into TX chain: error=%d\n", error);
		m_freem(m);
		return error;
	}

	if (map->dm_nsegs > sc->sc_tx_free) {
		bus_dmamap_unload(sc->sc_dmat, map);
		device_printf(sc->sc_dev,
		    "too many mbuf chain %d\n", map->dm_nsegs);
		m_freem(m);
		return ENOBUFS;
	}

	/* fill protocol cksum zero beforehand */
	if (csumflags & (M_CSUM_UDPv4 | M_CSUM_TCPv4 |
	    M_CSUM_UDPv6 | M_CSUM_TCPv6)) {
		int ehlen;
		uint16_t etype;

		m_copydata(m, ETHER_ADDR_LEN * 2, sizeof(etype), &etype);
		switch (ntohs(etype)) {
		case ETHERTYPE_IP:
		case ETHERTYPE_IPV6:
			ehlen = ETHER_HDR_LEN;
			break;
		case ETHERTYPE_VLAN:
			ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
			break;
		default:
			ehlen = 0;
			break;
		}

		if (ehlen) {
			const int off =
			    M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data) +
			    M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
			if (m->m_pkthdr.len >= ehlen + off + sizeof(uint16_t)) {
				uint16_t zero = 0;
				m_copyback(m, ehlen + off, sizeof(zero), &zero);
			}
		}
	}

	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
	    BUS_DMASYNC_PREWRITE);

	for (i = 0; i < map->dm_nsegs; i++) {
		flags1 = TXFLAGS1_R;
		flags2 = 0;

		if (i == 0) {
			flags1 |= TXFLAGS1_T1;	/* mark as first segment */
			sc->sc_txsoft[idx].txs_mbuf = m;
		}

		/* checksum offloading */
		if (csumflags & (M_CSUM_UDPv4 | M_CSUM_TCPv4 |
		    M_CSUM_UDPv6 | M_CSUM_TCPv6))
			flags2 |= TXFLAGS2_PINS;
		if (csumflags & (M_CSUM_IPv4))
			flags2 |= TXFLAGS2_IINS;

		if (i == map->dm_nsegs - 1) {
			/* mark last segment */
			flags1 |= TXFLAGS1_L | TXFLAGS1_TC;
			flags2 |= TXFLAGS2_INT;
		}
		if (idx == ENET_TX_RING_CNT - 1) {
			/* mark end of ring */
			flags1 |= TXFLAGS1_W;
		}

		sc->sc_txdesc_ring[idx].tx_databuf = map->dm_segs[i].ds_addr;
		sc->sc_txdesc_ring[idx].tx_flags2 = flags2;
		sc->sc_txdesc_ring[idx].tx_flags3 = 0;
		TXDESC_WRITEOUT(idx);

		sc->sc_txdesc_ring[idx].tx_flags1_len =
		    flags1 | TXFLAGS1_LEN(map->dm_segs[i].ds_len);
		TXDESC_WRITEOUT(idx);

		idx = ENET_TX_NEXTIDX(idx);
		sc->sc_tx_free--;
	}

	sc->sc_tx_prodidx = idx;

	return 0;
}

/*
 * device initialize
 */
static int
enet_init_regs(struct enet_softc *sc, int init)
{
	struct mii_data *mii;
	struct ifmedia_entry *ife;
	paddr_t paddr;
	uint32_t val;
	int miimode, fulldup, ecr_speed, rcr_speed, flowctrl;

	if (init) {
		fulldup = 1;
		ecr_speed = ENET_ECR_SPEED;
		rcr_speed = 0;
		flowctrl = 0;
	} else {
		mii = &sc->sc_mii;
		ife = mii->mii_media.ifm_cur;

		if ((ife->ifm_media & IFM_FDX) != 0)
			fulldup = 1;
		else
			fulldup = 0;

		switch (IFM_SUBTYPE(ife->ifm_media)) {
		case IFM_10_T:
			ecr_speed = 0;
			rcr_speed = ENET_RCR_RMII_10T;
			break;
		case IFM_100_TX:
			ecr_speed = 0;
			rcr_speed = 0;
			break;
		default:
			ecr_speed = ENET_ECR_SPEED;
			rcr_speed = 0;
			break;
		}

		flowctrl = sc->sc_flowflags & IFM_FLOW;
	}

	if (sc->sc_rgmii == 0)
		ecr_speed = 0;

	/* reset */
	ENET_REG_WRITE(sc, ENET_ECR, ecr_speed | ENET_ECR_RESET);

	/* mask and clear all interrupt */
	ENET_REG_WRITE(sc, ENET_EIMR, 0);
	ENET_REG_WRITE(sc, ENET_EIR, 0xffffffff);

	/* full duplex */
	ENET_REG_WRITE(sc, ENET_TCR, fulldup ? ENET_TCR_FDEN : 0);

	/* clear and enable MIB register */
	ENET_REG_WRITE(sc, ENET_MIBC, ENET_MIBC_MIB_CLEAR);
	ENET_REG_WRITE(sc, ENET_MIBC, 0);

	/* MII speed setup. MDCclk(=2.5MHz) = (internal module clock)/((val+1)*2) */
	val = (sc->sc_clock + (5000000 - 1)) / 5000000 - 1;
	ENET_REG_WRITE(sc, ENET_MSCR, __SHIFTIN(val, ENET_MSCR_MII_SPEED));

	/* Opcode/Pause Duration */
	ENET_REG_WRITE(sc, ENET_OPD, 0x00010020);

	/* Receive FIFO */
	ENET_REG_WRITE(sc, ENET_RSFL, 16);	/* RxFIFO Section Full */
	ENET_REG_WRITE(sc, ENET_RSEM, 0x84);	/* RxFIFO Section Empty */
	ENET_REG_WRITE(sc, ENET_RAEM, 8);	/* RxFIFO Almost Empty */
	ENET_REG_WRITE(sc, ENET_RAFL, 8);	/* RxFIFO Almost Full */

	/* Transmit FIFO */
	ENET_REG_WRITE(sc, ENET_TFWR, ENET_TFWR_STRFWD |
	    ENET_TFWR_FIFO(128));		/* TxFIFO Watermark */
	ENET_REG_WRITE(sc, ENET_TSEM, 0);	/* TxFIFO Section Empty */
	ENET_REG_WRITE(sc, ENET_TAEM, 256);	/* TxFIFO Almost Empty */
	ENET_REG_WRITE(sc, ENET_TAFL, 8);	/* TxFIFO Almost Full */
	ENET_REG_WRITE(sc, ENET_TIPG, 12);	/* Tx Inter-Packet Gap */

	/* hardware checksum is default off (override in TX descripter) */
	ENET_REG_WRITE(sc, ENET_TACC, 0);

	/*
	 * align ethernet payload on 32bit, discard frames with MAC layer error,
	 * and don't discard checksum error
	 */
	ENET_REG_WRITE(sc, ENET_RACC, ENET_RACC_SHIFT16 | ENET_RACC_LINEDIS);

	/* maximum frame size */
	val = ENET_DEFAULT_PKT_LEN;
	ENET_REG_WRITE(sc, ENET_FTRL, val);	/* Frame Truncation Length */

	if (sc->sc_rgmii == 0)
		miimode = ENET_RCR_RMII_MODE | ENET_RCR_MII_MODE;
	else
		miimode = ENET_RCR_RGMII_EN;
	ENET_REG_WRITE(sc, ENET_RCR,
	    ENET_RCR_PADEN |			/* RX frame padding remove */
	    miimode |
	    (flowctrl ? ENET_RCR_FCE : 0) |	/* flow control enable */
	    rcr_speed |
	    (fulldup ? 0 : ENET_RCR_DRT) |
	    ENET_RCR_MAX_FL(val));

	/* Maximum Receive BufSize per one descriptor */
	ENET_REG_WRITE(sc, ENET_MRBR, RXDESC_MAXBUFSIZE);


	/* TX/RX Descriptor Physical Address */
	paddr = sc->sc_txdesc_dmamap->dm_segs[0].ds_addr;
	ENET_REG_WRITE(sc, ENET_TDSR, paddr);
	paddr = sc->sc_rxdesc_dmamap->dm_segs[0].ds_addr;
	ENET_REG_WRITE(sc, ENET_RDSR, paddr);
	/* sync cache */
	bus_dmamap_sync(sc->sc_dmat, sc->sc_txdesc_dmamap, 0,
	    sc->sc_txdesc_dmamap->dm_mapsize, BUS_DMASYNC_PREWRITE);
	bus_dmamap_sync(sc->sc_dmat, sc->sc_rxdesc_dmamap, 0,
	    sc->sc_rxdesc_dmamap->dm_mapsize, BUS_DMASYNC_PREWRITE);

	/* enable interrupts */
	val = ENET_EIR_TXF | ENET_EIR_RXF | ENET_EIR_EBERR;
	if (sc->sc_imxtype == 7)
		val |= ENET_EIR_TXF2 | ENET_EIR_RXF2 | ENET_EIR_TXF1 |
		    ENET_EIR_RXF1;
	ENET_REG_WRITE(sc, ENET_EIMR, val);

	/* enable ether */
	ENET_REG_WRITE(sc, ENET_ECR,
#if _BYTE_ORDER == _LITTLE_ENDIAN
	    ENET_ECR_DBSWP |
#endif
	    ecr_speed |
	    ENET_ECR_EN1588 |	/* use enhanced TX/RX descriptor */
	    ENET_ECR_ETHEREN);	/* Ethernet Enable */

	return 0;
}

static int
enet_alloc_dma(struct enet_softc *sc, size_t size, void **addrp,
    bus_dmamap_t *mapp)
{
	bus_dma_segment_t seglist[1];
	int nsegs, error;

	if ((error = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, seglist,
	    1, &nsegs, M_NOWAIT)) != 0) {
		device_printf(sc->sc_dev,
		    "unable to allocate DMA buffer, error=%d\n", error);
		goto fail_alloc;
	}

	if ((error = bus_dmamem_map(sc->sc_dmat, seglist, 1, size, addrp,
	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT)) != 0) {
		device_printf(sc->sc_dev,
		    "unable to map DMA buffer, error=%d\n",
		    error);
		goto fail_map;
	}

	if ((error = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
	    BUS_DMA_NOWAIT, mapp)) != 0) {
		device_printf(sc->sc_dev,
		    "unable to create DMA map, error=%d\n", error);
		goto fail_create;
	}

	if ((error = bus_dmamap_load(sc->sc_dmat, *mapp, *addrp, size, NULL,
	    BUS_DMA_NOWAIT)) != 0) {
		aprint_error_dev(sc->sc_dev,
		    "unable to load DMA map, error=%d\n", error);
		goto fail_load;
	}

	return 0;

 fail_load:
	bus_dmamap_destroy(sc->sc_dmat, *mapp);
 fail_create:
	bus_dmamem_unmap(sc->sc_dmat, *addrp, size);
 fail_map:
	bus_dmamem_free(sc->sc_dmat, seglist, 1);
 fail_alloc:
	return error;
}