xref: /openbsd-src/sys/arch/octeon/dev/if_cnmac.c (revision a8b581972213782bec450017fe45a05ea431addf)

/*	$OpenBSD: if_cnmac.c,v 1.86 2024/05/20 23:13:33 jsg Exp $	*/

/*
 * Copyright (c) 2007 Internet Initiative Japan, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
#include "bpfilter.h"

/*
 * XXXSEIL
 * If no free send buffer is available, free all the sent buffer and bail out.
 */
#define OCTEON_ETH_SEND_QUEUE_CHECK

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/pool.h>
#include <sys/proc.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <sys/errno.h>
#include <sys/device.h>
#include <sys/queue.h>
#include <sys/conf.h>
#include <sys/stdint.h> /* uintptr_t */
#include <sys/syslog.h>
#include <sys/endian.h>
#include <sys/atomic.h>

#include <net/if.h>
#include <net/if_media.h>
#include <netinet/in.h>
#include <netinet/if_ether.h>

#if NBPFILTER > 0
#include <net/bpf.h>
#endif

#include <machine/bus.h>
#include <machine/intr.h>
#include <machine/octeonvar.h>
#include <machine/octeon_model.h>

#include <dev/mii/mii.h>
#include <dev/mii/miivar.h>

#include <octeon/dev/cn30xxciureg.h>
#include <octeon/dev/cn30xxnpireg.h>
#include <octeon/dev/cn30xxgmxreg.h>
#include <octeon/dev/cn30xxipdreg.h>
#include <octeon/dev/cn30xxpipreg.h>
#include <octeon/dev/cn30xxpowreg.h>
#include <octeon/dev/cn30xxfaureg.h>
#include <octeon/dev/cn30xxfpareg.h>
#include <octeon/dev/cn30xxbootbusreg.h>
#include <octeon/dev/cn30xxfpavar.h>
#include <octeon/dev/cn30xxgmxvar.h>
#include <octeon/dev/cn30xxfauvar.h>
#include <octeon/dev/cn30xxpowvar.h>
#include <octeon/dev/cn30xxipdvar.h>
#include <octeon/dev/cn30xxpipvar.h>
#include <octeon/dev/cn30xxpkovar.h>
#include <octeon/dev/cn30xxsmivar.h>
#include <octeon/dev/iobusvar.h>
#include <octeon/dev/if_cnmacvar.h>

#ifdef OCTEON_ETH_DEBUG
#define	OCTEON_ETH_KASSERT(x)	KASSERT(x)
#define	OCTEON_ETH_KDASSERT(x)	KDASSERT(x)
#else
#define	OCTEON_ETH_KASSERT(x)
#define	OCTEON_ETH_KDASSERT(x)
#endif

/*
 * Set the PKO to think command buffers are an odd length.  This makes it so we
 * never have to divide a command across two buffers.
 */
#define OCTEON_POOL_NWORDS_CMD	\
	    (((uint32_t)OCTEON_POOL_SIZE_CMD / sizeof(uint64_t)) - 1)
#define FPA_COMMAND_BUFFER_POOL_NWORDS	OCTEON_POOL_NWORDS_CMD	/* XXX */

CTASSERT(MCLBYTES >= OCTEON_POOL_SIZE_PKT + CACHELINESIZE);

void	cnmac_buf_init(struct cnmac_softc *);

int	cnmac_match(struct device *, void *, void *);
void	cnmac_attach(struct device *, struct device *, void *);
void	cnmac_pip_init(struct cnmac_softc *);
void	cnmac_ipd_init(struct cnmac_softc *);
void	cnmac_pko_init(struct cnmac_softc *);

void	cnmac_board_mac_addr(uint8_t *);

int	cnmac_mii_readreg(struct device *, int, int);
void	cnmac_mii_writereg(struct device *, int, int, int);
void	cnmac_mii_statchg(struct device *);

int	cnmac_mediainit(struct cnmac_softc *);
void	cnmac_mediastatus(struct ifnet *, struct ifmediareq *);
int	cnmac_mediachange(struct ifnet *);

void	cnmac_send_queue_flush_prefetch(struct cnmac_softc *);
void	cnmac_send_queue_flush_fetch(struct cnmac_softc *);
void	cnmac_send_queue_flush(struct cnmac_softc *);
int	cnmac_send_queue_is_full(struct cnmac_softc *);
void	cnmac_send_queue_add(struct cnmac_softc *,
	    struct mbuf *, uint64_t *);
void	cnmac_send_queue_del(struct cnmac_softc *,
	    struct mbuf **, uint64_t **);
int	cnmac_buf_free_work(struct cnmac_softc *, uint64_t *);

int	cnmac_ioctl(struct ifnet *, u_long, caddr_t);
void	cnmac_watchdog(struct ifnet *);
int	cnmac_init(struct ifnet *);
int	cnmac_stop(struct ifnet *, int);
void	cnmac_start(struct ifqueue *);

int	cnmac_send_cmd(struct cnmac_softc *, uint64_t, uint64_t);
uint64_t cnmac_send_makecmd_w1(int, paddr_t);
uint64_t cnmac_send_makecmd_w0(uint64_t, uint64_t, size_t, int, int);
int	cnmac_send_makecmd_gbuf(struct cnmac_softc *,
	    struct mbuf *, uint64_t *, int *);
int	cnmac_send_makecmd(struct cnmac_softc *,
	    struct mbuf *, uint64_t *, uint64_t *, uint64_t *);
int	cnmac_send_buf(struct cnmac_softc *,
	    struct mbuf *, uint64_t *);
int	cnmac_send(struct cnmac_softc *, struct mbuf *);

int	cnmac_reset(struct cnmac_softc *);
int	cnmac_configure(struct cnmac_softc *);
int	cnmac_configure_common(struct cnmac_softc *);

void	cnmac_free_task(void *);
void	cnmac_tick_free(void *arg);
void	cnmac_tick_misc(void *);

int	cnmac_recv_mbuf(struct cnmac_softc *,
	    uint64_t *, struct mbuf **, int *);
int	cnmac_recv_check(struct cnmac_softc *, uint64_t);
int	cnmac_recv(struct cnmac_softc *, uint64_t *, struct mbuf_list *);
int	cnmac_intr(void *);

int	cnmac_mbuf_alloc(int);

#if NKSTAT > 0
void	cnmac_kstat_attach(struct cnmac_softc *);
int	cnmac_kstat_read(struct kstat *);
void	cnmac_kstat_tick(struct cnmac_softc *);
#endif

/* device parameters */
int	cnmac_param_pko_cmd_w0_n2 = 1;

const struct cfattach cnmac_ca = {
	sizeof(struct cnmac_softc), cnmac_match, cnmac_attach
};

struct cfdriver cnmac_cd = { NULL, "cnmac", DV_IFNET };

/* ---- buffer management */

const struct cnmac_pool_param {
	int			poolno;
	size_t			size;
	size_t			nelems;
} cnmac_pool_params[] = {
#define	_ENTRY(x)	{ OCTEON_POOL_NO_##x, OCTEON_POOL_SIZE_##x, OCTEON_POOL_NELEMS_##x }
	_ENTRY(WQE),
	_ENTRY(CMD),
	_ENTRY(SG)
#undef	_ENTRY
};
struct cn30xxfpa_buf	*cnmac_pools[8];
#define	cnmac_fb_wqe	cnmac_pools[OCTEON_POOL_NO_WQE]
#define	cnmac_fb_cmd	cnmac_pools[OCTEON_POOL_NO_CMD]
#define	cnmac_fb_sg	cnmac_pools[OCTEON_POOL_NO_SG]

uint64_t cnmac_mac_addr = 0;
uint32_t cnmac_mac_addr_offset = 0;

int	cnmac_mbufs_to_alloc;
int	cnmac_npowgroups = 0;

void
cnmac_buf_init(struct cnmac_softc *sc)
{
	static int once;
	int i;
	const struct cnmac_pool_param *pp;
	struct cn30xxfpa_buf *fb;

	if (once == 1)
		return;
	once = 1;

	for (i = 0; i < (int)nitems(cnmac_pool_params); i++) {
		pp = &cnmac_pool_params[i];
		cn30xxfpa_buf_init(pp->poolno, pp->size, pp->nelems, &fb);
		cnmac_pools[pp->poolno] = fb;
	}
}

/* ---- autoconf */

int
cnmac_match(struct device *parent, void *match, void *aux)
{
	struct cfdata *cf = (struct cfdata *)match;
	struct cn30xxgmx_attach_args *ga = aux;

	if (strcmp(cf->cf_driver->cd_name, ga->ga_name) != 0) {
		return 0;
	}
	return 1;
}

void
cnmac_attach(struct device *parent, struct device *self, void *aux)
{
	struct cnmac_softc *sc = (void *)self;
	struct cn30xxgmx_attach_args *ga = aux;
	struct ifnet *ifp = &sc->sc_arpcom.ac_if;

	if (cnmac_npowgroups >= OCTEON_POW_GROUP_MAX) {
		printf(": out of POW groups\n");
		return;
	}

	atomic_add_int(&cnmac_mbufs_to_alloc,
	    cnmac_mbuf_alloc(CNMAC_MBUFS_PER_PORT));

	sc->sc_regt = ga->ga_regt;
	sc->sc_dmat = ga->ga_dmat;
	sc->sc_port = ga->ga_portno;
	sc->sc_port_type = ga->ga_port_type;
	sc->sc_gmx = ga->ga_gmx;
	sc->sc_gmx_port = ga->ga_gmx_port;
	sc->sc_smi = ga->ga_smi;
	sc->sc_phy_addr = ga->ga_phy_addr;
	sc->sc_powgroup = cnmac_npowgroups++;

	sc->sc_init_flag = 0;

	/*
	 * XXX
	 * Setting PIP_IP_OFFSET[OFFSET] to 8 causes panic ... why???
	 */
	sc->sc_ip_offset = 0/* XXX */;

	cnmac_board_mac_addr(sc->sc_arpcom.ac_enaddr);
	printf(", address %s\n", ether_sprintf(sc->sc_arpcom.ac_enaddr));

	ml_init(&sc->sc_sendq);
	sc->sc_soft_req_thresh = 15/* XXX */;
	sc->sc_ext_callback_cnt = 0;

	task_set(&sc->sc_free_task, cnmac_free_task, sc);
	timeout_set(&sc->sc_tick_misc_ch, cnmac_tick_misc, sc);
	timeout_set(&sc->sc_tick_free_ch, cnmac_tick_free, sc);

	cn30xxfau_op_init(&sc->sc_fau_done,
	    OCTEON_CVMSEG_ETHER_OFFSET(sc->sc_dev.dv_unit, csm_ether_fau_done),
	    OCT_FAU_REG_ADDR_END - (8 * (sc->sc_dev.dv_unit + 1))/* XXX */);
	cn30xxfau_op_set_8(&sc->sc_fau_done, 0);

	cnmac_pip_init(sc);
	cnmac_ipd_init(sc);
	cnmac_pko_init(sc);

	cnmac_configure_common(sc);

	sc->sc_gmx_port->sc_ipd = sc->sc_ipd;
	sc->sc_gmx_port->sc_port_mii = &sc->sc_mii;
	sc->sc_gmx_port->sc_port_ac = &sc->sc_arpcom;

	/* XXX */
	sc->sc_pow = &cn30xxpow_softc;

	cnmac_mediainit(sc);

	strncpy(ifp->if_xname, sc->sc_dev.dv_xname, sizeof(ifp->if_xname));
	ifp->if_softc = sc;
	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
	ifp->if_xflags = IFXF_MPSAFE;
	ifp->if_ioctl = cnmac_ioctl;
	ifp->if_qstart = cnmac_start;
	ifp->if_watchdog = cnmac_watchdog;
	ifp->if_hardmtu = CNMAC_MAX_MTU;
	ifq_init_maxlen(&ifp->if_snd, max(GATHER_QUEUE_SIZE, IFQ_MAXLEN));

	ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_CSUM_TCPv4 |
	    IFCAP_CSUM_UDPv4 | IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6;

	cn30xxgmx_set_filter(sc->sc_gmx_port);

	if_attach(ifp);
	ether_ifattach(ifp);

	cnmac_buf_init(sc);

#if NKSTAT > 0
	cnmac_kstat_attach(sc);
#endif

	sc->sc_ih = octeon_intr_establish(POW_WORKQ_IRQ(sc->sc_powgroup),
	    IPL_NET | IPL_MPSAFE, cnmac_intr, sc, sc->sc_dev.dv_xname);
	if (sc->sc_ih == NULL)
		panic("%s: could not set up interrupt", sc->sc_dev.dv_xname);
}

/* ---- submodules */

void
cnmac_pip_init(struct cnmac_softc *sc)
{
	struct cn30xxpip_attach_args pip_aa;

	pip_aa.aa_port = sc->sc_port;
	pip_aa.aa_regt = sc->sc_regt;
	pip_aa.aa_tag_type = POW_TAG_TYPE_ORDERED/* XXX */;
	pip_aa.aa_receive_group = sc->sc_powgroup;
	pip_aa.aa_ip_offset = sc->sc_ip_offset;
	cn30xxpip_init(&pip_aa, &sc->sc_pip);
	cn30xxpip_port_config(sc->sc_pip);
}

void
cnmac_ipd_init(struct cnmac_softc *sc)
{
	struct cn30xxipd_attach_args ipd_aa;

	ipd_aa.aa_port = sc->sc_port;
	ipd_aa.aa_regt = sc->sc_regt;
	ipd_aa.aa_first_mbuff_skip = 0/* XXX */;
	ipd_aa.aa_not_first_mbuff_skip = 0/* XXX */;
	cn30xxipd_init(&ipd_aa, &sc->sc_ipd);
}

void
cnmac_pko_init(struct cnmac_softc *sc)
{
	struct cn30xxpko_attach_args pko_aa;

	pko_aa.aa_port = sc->sc_port;
	pko_aa.aa_regt = sc->sc_regt;
	pko_aa.aa_cmdptr = &sc->sc_cmdptr;
	pko_aa.aa_cmd_buf_pool = OCTEON_POOL_NO_CMD;
	pko_aa.aa_cmd_buf_size = OCTEON_POOL_NWORDS_CMD;
	cn30xxpko_init(&pko_aa, &sc->sc_pko);
}

/* ---- XXX */

void
cnmac_board_mac_addr(uint8_t *enaddr)
{
	int id;

	/* Initialize MAC addresses from the global address base. */
	if (cnmac_mac_addr == 0) {
		memcpy((uint8_t *)&cnmac_mac_addr + 2,
		    octeon_boot_info->mac_addr_base, 6);

		/*
		 * Should be allowed to fail hard if couldn't read the
		 * mac_addr_base address...
		 */
		if (cnmac_mac_addr == 0)
			return;

		/*
		 * Calculate the offset from the mac_addr_base that will be used
		 * for the next sc->sc_port.
		 */
		id = octeon_get_chipid();

		switch (octeon_model_family(id)) {
		case OCTEON_MODEL_FAMILY_CN56XX:
			cnmac_mac_addr_offset = 1;
			break;
		/*
		case OCTEON_MODEL_FAMILY_CN52XX:
		case OCTEON_MODEL_FAMILY_CN63XX:
			cnmac_mac_addr_offset = 2;
			break;
		*/
		default:
			cnmac_mac_addr_offset = 0;
			break;
		}

		enaddr += cnmac_mac_addr_offset;
	}

	/* No more MAC addresses to assign. */
	if (cnmac_mac_addr_offset >= octeon_boot_info->mac_addr_count)
		return;

	if (enaddr)
		memcpy(enaddr, (uint8_t *)&cnmac_mac_addr + 2, 6);

	cnmac_mac_addr++;
	cnmac_mac_addr_offset++;
}

/* ---- media */

int
cnmac_mii_readreg(struct device *self, int phy_no, int reg)
{
	struct cnmac_softc *sc = (struct cnmac_softc *)self;
	return cn30xxsmi_read(sc->sc_smi, phy_no, reg);
}

void
cnmac_mii_writereg(struct device *self, int phy_no, int reg, int value)
{
	struct cnmac_softc *sc = (struct cnmac_softc *)self;
	cn30xxsmi_write(sc->sc_smi, phy_no, reg, value);
}

void
cnmac_mii_statchg(struct device *self)
{
	struct cnmac_softc *sc = (struct cnmac_softc *)self;

	cn30xxpko_port_enable(sc->sc_pko, 0);
	cn30xxgmx_port_enable(sc->sc_gmx_port, 0);

	cnmac_reset(sc);

	cn30xxpko_port_enable(sc->sc_pko, 1);
	cn30xxgmx_port_enable(sc->sc_gmx_port, 1);
}

int
cnmac_mediainit(struct cnmac_softc *sc)
{
	struct ifnet *ifp = &sc->sc_arpcom.ac_if;
	struct mii_softc *child;

	sc->sc_mii.mii_ifp = ifp;
	sc->sc_mii.mii_readreg = cnmac_mii_readreg;
	sc->sc_mii.mii_writereg = cnmac_mii_writereg;
	sc->sc_mii.mii_statchg = cnmac_mii_statchg;
	ifmedia_init(&sc->sc_mii.mii_media, 0, cnmac_mediachange,
	    cnmac_mediastatus);

	mii_attach(&sc->sc_dev, &sc->sc_mii,
	    0xffffffff, sc->sc_phy_addr, MII_OFFSET_ANY, MIIF_DOPAUSE);

	child = LIST_FIRST(&sc->sc_mii.mii_phys);
	if (child == NULL) {
                /* No PHY attached. */
		ifmedia_add(&sc->sc_mii.mii_media, IFM_ETHER | IFM_MANUAL,
			    0, NULL);
		ifmedia_set(&sc->sc_mii.mii_media, IFM_ETHER | IFM_MANUAL);
	} else {
		ifmedia_set(&sc->sc_mii.mii_media, IFM_ETHER | IFM_AUTO);
	}

	return 0;
}

void
cnmac_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
{
	struct cnmac_softc *sc = ifp->if_softc;

	mii_pollstat(&sc->sc_mii);
	ifmr->ifm_status = sc->sc_mii.mii_media_status;
	ifmr->ifm_active = sc->sc_mii.mii_media_active;
	ifmr->ifm_active = (sc->sc_mii.mii_media_active & ~IFM_ETH_FMASK) |
	    sc->sc_gmx_port->sc_port_flowflags;
}

int
cnmac_mediachange(struct ifnet *ifp)
{
	struct cnmac_softc *sc = ifp->if_softc;

	if ((ifp->if_flags & IFF_UP) == 0)
		return 0;

	return mii_mediachg(&sc->sc_mii);
}

/* ---- send buffer garbage collection */

void
cnmac_send_queue_flush_prefetch(struct cnmac_softc *sc)
{
	OCTEON_ETH_KASSERT(sc->sc_prefetch == 0);
	cn30xxfau_op_inc_fetch_8(&sc->sc_fau_done, 0);
	sc->sc_prefetch = 1;
}

void
cnmac_send_queue_flush_fetch(struct cnmac_softc *sc)
{
#ifndef  OCTEON_ETH_DEBUG
	if (!sc->sc_prefetch)
		return;
#endif
	OCTEON_ETH_KASSERT(sc->sc_prefetch == 1);
	sc->sc_hard_done_cnt = cn30xxfau_op_inc_read_8(&sc->sc_fau_done);
	OCTEON_ETH_KASSERT(sc->sc_hard_done_cnt <= 0);
	sc->sc_prefetch = 0;
}

void
cnmac_send_queue_flush(struct cnmac_softc *sc)
{
	const int64_t sent_count = sc->sc_hard_done_cnt;
	int i;

	OCTEON_ETH_KASSERT(sent_count <= 0);

	for (i = 0; i < 0 - sent_count; i++) {
		struct mbuf *m;
		uint64_t *gbuf;

		cnmac_send_queue_del(sc, &m, &gbuf);

		cn30xxfpa_buf_put_paddr(cnmac_fb_sg, XKPHYS_TO_PHYS(gbuf));

		m_freem(m);
	}

	cn30xxfau_op_add_8(&sc->sc_fau_done, i);
}

int
cnmac_send_queue_is_full(struct cnmac_softc *sc)
{
#ifdef OCTEON_ETH_SEND_QUEUE_CHECK
	int64_t nofree_cnt;

	nofree_cnt = ml_len(&sc->sc_sendq) + sc->sc_hard_done_cnt;

	if (__predict_false(nofree_cnt == GATHER_QUEUE_SIZE - 1)) {
		cnmac_send_queue_flush(sc);
		return 1;
	}

#endif
	return 0;
}

void
cnmac_send_queue_add(struct cnmac_softc *sc, struct mbuf *m,
    uint64_t *gbuf)
{
	OCTEON_ETH_KASSERT(m->m_flags & M_PKTHDR);

	m->m_pkthdr.ph_cookie = gbuf;
	ml_enqueue(&sc->sc_sendq, m);

	if (m->m_ext.ext_free_fn != 0)
		sc->sc_ext_callback_cnt++;
}

void
cnmac_send_queue_del(struct cnmac_softc *sc, struct mbuf **rm,
    uint64_t **rgbuf)
{
	struct mbuf *m;
	m = ml_dequeue(&sc->sc_sendq);
	OCTEON_ETH_KASSERT(m != NULL);

	*rm = m;
	*rgbuf = m->m_pkthdr.ph_cookie;

	if (m->m_ext.ext_free_fn != 0) {
		sc->sc_ext_callback_cnt--;
		OCTEON_ETH_KASSERT(sc->sc_ext_callback_cnt >= 0);
	}
}

int
cnmac_buf_free_work(struct cnmac_softc *sc, uint64_t *work)
{
	paddr_t addr, pktbuf;
	uint64_t word3;
	unsigned int back, nbufs;

	nbufs = (work[2] & PIP_WQE_WORD2_IP_BUFS) >>
	    PIP_WQE_WORD2_IP_BUFS_SHIFT;
	word3 = work[3];
	while (nbufs-- > 0) {
		addr = word3 & PIP_WQE_WORD3_ADDR;
		back = (word3 & PIP_WQE_WORD3_BACK) >>
		    PIP_WQE_WORD3_BACK_SHIFT;
		pktbuf = (addr & ~(CACHELINESIZE - 1)) - back * CACHELINESIZE;

		cn30xxfpa_store(pktbuf, OCTEON_POOL_NO_PKT,
		    OCTEON_POOL_SIZE_PKT / CACHELINESIZE);

		if (nbufs > 0)
			memcpy(&word3, (void *)PHYS_TO_XKPHYS(addr -
			    sizeof(word3), CCA_CACHED), sizeof(word3));
	}

	cn30xxfpa_buf_put_paddr(cnmac_fb_wqe, XKPHYS_TO_PHYS(work));

	return 0;
}

/* ---- ifnet interfaces */

int
cnmac_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
	struct cnmac_softc *sc = ifp->if_softc;
	struct ifreq *ifr = (struct ifreq *)data;
	int s, error = 0;

	s = splnet();

	switch (cmd) {
	case SIOCSIFADDR:
		ifp->if_flags |= IFF_UP;
		if (!(ifp->if_flags & IFF_RUNNING))
			cnmac_init(ifp);
		break;

	case SIOCSIFFLAGS:
		if (ifp->if_flags & IFF_UP) {
			if (ifp->if_flags & IFF_RUNNING)
				error = ENETRESET;
			else
				cnmac_init(ifp);
		} else {
			if (ifp->if_flags & IFF_RUNNING)
				cnmac_stop(ifp, 0);
		}
		break;

	case SIOCSIFMEDIA:
		/* Flow control requires full-duplex mode. */
		if (IFM_SUBTYPE(ifr->ifr_media) == IFM_AUTO ||
		    (ifr->ifr_media & IFM_FDX) == 0) {
			ifr->ifr_media &= ~IFM_ETH_FMASK;
		}
		if (IFM_SUBTYPE(ifr->ifr_media) != IFM_AUTO) {
			if ((ifr->ifr_media & IFM_ETH_FMASK) == IFM_FLOW) {
				ifr->ifr_media |=
				    IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE;
			}
			sc->sc_gmx_port->sc_port_flowflags =
				ifr->ifr_media & IFM_ETH_FMASK;
		}
		/* FALLTHROUGH */
	case SIOCGIFMEDIA:
		error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, cmd);
		break;

	default:
		error = ether_ioctl(ifp, &sc->sc_arpcom, cmd, data);
	}

	if (error == ENETRESET) {
		if (ISSET(ifp->if_flags, IFF_RUNNING))
			cn30xxgmx_set_filter(sc->sc_gmx_port);
		error = 0;
	}

	splx(s);
	return (error);
}

/* ---- send (output) */

uint64_t
cnmac_send_makecmd_w0(uint64_t fau0, uint64_t fau1, size_t len, int segs,
    int ipoffp1)
{
	return cn30xxpko_cmd_word0(
		OCT_FAU_OP_SIZE_64,		/* sz1 */
		OCT_FAU_OP_SIZE_64,		/* sz0 */
		1, fau1, 1, fau0,		/* s1, reg1, s0, reg0 */
		0,				/* le */
		cnmac_param_pko_cmd_w0_n2,	/* n2 */
		1, 0,				/* q, r */
		(segs == 1) ? 0 : 1,		/* g */
		ipoffp1, 0, 1,			/* ipoffp1, ii, df */
		segs, (int)len);		/* segs, totalbytes */
}

uint64_t
cnmac_send_makecmd_w1(int size, paddr_t addr)
{
	return cn30xxpko_cmd_word1(
		0, 0,				/* i, back */
		OCTEON_POOL_NO_SG,		/* pool */
		size, addr);			/* size, addr */
}

#define KVTOPHYS(addr)	cnmac_kvtophys((vaddr_t)(addr))

static inline paddr_t
cnmac_kvtophys(vaddr_t kva)
{
	KASSERT(IS_XKPHYS(kva));
	return XKPHYS_TO_PHYS(kva);
}

int
cnmac_send_makecmd_gbuf(struct cnmac_softc *sc, struct mbuf *m0,
    uint64_t *gbuf, int *rsegs)
{
	struct mbuf *m;
	int segs = 0;

	for (m = m0; m != NULL; m = m->m_next) {
		if (__predict_false(m->m_len == 0))
			continue;

		if (segs >= OCTEON_POOL_SIZE_SG / sizeof(uint64_t))
			goto defrag;
		gbuf[segs] = cnmac_send_makecmd_w1(m->m_len,
		    KVTOPHYS(m->m_data));
		segs++;
	}

	*rsegs = segs;

	return 0;

defrag:
	if (m_defrag(m0, M_DONTWAIT) != 0)
		return 1;
	gbuf[0] = cnmac_send_makecmd_w1(m0->m_len, KVTOPHYS(m0->m_data));
	*rsegs = 1;
	return 0;
}

int
cnmac_send_makecmd(struct cnmac_softc *sc, struct mbuf *m,
    uint64_t *gbuf, uint64_t *rpko_cmd_w0, uint64_t *rpko_cmd_w1)
{
	uint64_t pko_cmd_w0, pko_cmd_w1;
	int ipoffp1;
	int segs;
	int result = 0;

	if (cnmac_send_makecmd_gbuf(sc, m, gbuf, &segs)) {
		log(LOG_WARNING, "%s: large number of transmission"
		    " data segments", sc->sc_dev.dv_xname);
		result = 1;
		goto done;
	}

	/* Get the IP packet offset for TCP/UDP checksum offloading. */
	ipoffp1 = (m->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT))
	    ? (ETHER_HDR_LEN + 1) : 0;

	/*
	 * segs == 1	-> link mode (single continuous buffer)
	 *		   WORD1[size] is number of bytes pointed by segment
	 *
	 * segs > 1	-> gather mode (scatter-gather buffer)
	 *		   WORD1[size] is number of segments
	 */
	pko_cmd_w0 = cnmac_send_makecmd_w0(sc->sc_fau_done.fd_regno,
	    0, m->m_pkthdr.len, segs, ipoffp1);
	pko_cmd_w1 = cnmac_send_makecmd_w1(
	    (segs == 1) ? m->m_pkthdr.len : segs,
	    (segs == 1) ?
		KVTOPHYS(m->m_data) :
		XKPHYS_TO_PHYS(gbuf));

	*rpko_cmd_w0 = pko_cmd_w0;
	*rpko_cmd_w1 = pko_cmd_w1;

done:
	return result;
}

int
cnmac_send_cmd(struct cnmac_softc *sc, uint64_t pko_cmd_w0,
    uint64_t pko_cmd_w1)
{
	uint64_t *cmdptr;
	int result = 0;

	cmdptr = (uint64_t *)PHYS_TO_XKPHYS(sc->sc_cmdptr.cmdptr, CCA_CACHED);
	cmdptr += sc->sc_cmdptr.cmdptr_idx;

	OCTEON_ETH_KASSERT(cmdptr != NULL);

	*cmdptr++ = pko_cmd_w0;
	*cmdptr++ = pko_cmd_w1;

	OCTEON_ETH_KASSERT(sc->sc_cmdptr.cmdptr_idx + 2 <= FPA_COMMAND_BUFFER_POOL_NWORDS - 1);

	if (sc->sc_cmdptr.cmdptr_idx + 2 == FPA_COMMAND_BUFFER_POOL_NWORDS - 1) {
		paddr_t buf;

		buf = cn30xxfpa_buf_get_paddr(cnmac_fb_cmd);
		if (buf == 0) {
			log(LOG_WARNING,
			    "%s: cannot allocate command buffer from free pool allocator\n",
			    sc->sc_dev.dv_xname);
			result = 1;
			goto done;
		}
		*cmdptr++ = buf;
		sc->sc_cmdptr.cmdptr = (uint64_t)buf;
		sc->sc_cmdptr.cmdptr_idx = 0;
	} else {
		sc->sc_cmdptr.cmdptr_idx += 2;
	}

	cn30xxpko_op_doorbell_write(sc->sc_port, sc->sc_port, 2);

done:
	return result;
}

int
cnmac_send_buf(struct cnmac_softc *sc, struct mbuf *m, uint64_t *gbuf)
{
	int result = 0, error;
	uint64_t pko_cmd_w0, pko_cmd_w1;

	error = cnmac_send_makecmd(sc, m, gbuf, &pko_cmd_w0, &pko_cmd_w1);
	if (error != 0) {
		/* already logging */
		result = error;
		goto done;
	}

	error = cnmac_send_cmd(sc, pko_cmd_w0, pko_cmd_w1);
	if (error != 0) {
		/* already logging */
		result = error;
	}

done:
	return result;
}

int
cnmac_send(struct cnmac_softc *sc, struct mbuf *m)
{
	paddr_t gaddr = 0;
	uint64_t *gbuf = NULL;
	int result = 0, error;

	gaddr = cn30xxfpa_buf_get_paddr(cnmac_fb_sg);
	if (gaddr == 0) {
		log(LOG_WARNING,
		    "%s: cannot allocate gather buffer from free pool allocator\n",
		    sc->sc_dev.dv_xname);
		result = 1;
		goto done;
	}

	gbuf = (uint64_t *)(uintptr_t)PHYS_TO_XKPHYS(gaddr, CCA_CACHED);

	error = cnmac_send_buf(sc, m, gbuf);
	if (error != 0) {
		/* already logging */
		cn30xxfpa_buf_put_paddr(cnmac_fb_sg, gaddr);
		result = error;
		goto done;
	}

	cnmac_send_queue_add(sc, m, gbuf);

done:
	return result;
}

void
cnmac_start(struct ifqueue *ifq)
{
	struct ifnet *ifp = ifq->ifq_if;
	struct cnmac_softc *sc = ifp->if_softc;
	struct mbuf *m;

	if (__predict_false(!cn30xxgmx_link_status(sc->sc_gmx_port))) {
		ifq_purge(ifq);
		return;
	}

	/*
	 * performance tuning
	 * presend iobdma request
	 */
	cnmac_send_queue_flush_prefetch(sc);

	for (;;) {
		cnmac_send_queue_flush_fetch(sc); /* XXX */

		/*
		 * XXXSEIL
		 * If no free send buffer is available, free all the sent buffer
		 * and bail out.
		 */
		if (cnmac_send_queue_is_full(sc)) {
			ifq_set_oactive(ifq);
			timeout_add(&sc->sc_tick_free_ch, 1);
			return;
		}

		m = ifq_dequeue(ifq);
		if (m == NULL)
			return;

#if NBPFILTER > 0
		if (ifp->if_bpf != NULL)
			bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_OUT);
#endif

		/* XXX */
		if (ml_len(&sc->sc_sendq) > sc->sc_soft_req_thresh)
			cnmac_send_queue_flush(sc);
		if (cnmac_send(sc, m)) {
			ifp->if_oerrors++;
			m_freem(m);
			log(LOG_WARNING,
		  	  "%s: failed to transmit packet\n",
		    	  sc->sc_dev.dv_xname);
		}
		/* XXX */

		/*
		 * send next iobdma request
		 */
		cnmac_send_queue_flush_prefetch(sc);
	}

	cnmac_send_queue_flush_fetch(sc);
}

void
cnmac_watchdog(struct ifnet *ifp)
{
	struct cnmac_softc *sc = ifp->if_softc;

	printf("%s: device timeout\n", sc->sc_dev.dv_xname);

	cnmac_stop(ifp, 0);

	cnmac_configure(sc);

	SET(ifp->if_flags, IFF_RUNNING);
	ifp->if_timer = 0;

	ifq_restart(&ifp->if_snd);
}

int
cnmac_init(struct ifnet *ifp)
{
	struct cnmac_softc *sc = ifp->if_softc;

	/* XXX don't disable commonly used parts!!! XXX */
	if (sc->sc_init_flag == 0) {
		/* Cancel any pending I/O. */
		cnmac_stop(ifp, 0);

		/* Initialize the device */
		cnmac_configure(sc);

		cn30xxpko_enable(sc->sc_pko);
		cn30xxipd_enable(sc->sc_ipd);

		sc->sc_init_flag = 1;
	} else {
		cn30xxgmx_port_enable(sc->sc_gmx_port, 1);
	}
	cnmac_mediachange(ifp);

	cn30xxpip_stats_init(sc->sc_pip);
	cn30xxgmx_stats_init(sc->sc_gmx_port);
	cn30xxgmx_set_filter(sc->sc_gmx_port);

	timeout_add_sec(&sc->sc_tick_misc_ch, 1);
	timeout_add_sec(&sc->sc_tick_free_ch, 1);

	SET(ifp->if_flags, IFF_RUNNING);
	ifq_clr_oactive(&ifp->if_snd);

	return 0;
}

int
cnmac_stop(struct ifnet *ifp, int disable)
{
	struct cnmac_softc *sc = ifp->if_softc;

	CLR(ifp->if_flags, IFF_RUNNING);

	timeout_del(&sc->sc_tick_misc_ch);
	timeout_del(&sc->sc_tick_free_ch);

	mii_down(&sc->sc_mii);

	cn30xxgmx_port_enable(sc->sc_gmx_port, 0);

	intr_barrier(sc->sc_ih);
	ifq_barrier(&ifp->if_snd);

	ifq_clr_oactive(&ifp->if_snd);
	ifp->if_timer = 0;

	return 0;
}

/* ---- misc */

#define PKO_INDEX_MASK	((1ULL << 12/* XXX */) - 1)

int
cnmac_reset(struct cnmac_softc *sc)
{
	cn30xxgmx_reset_speed(sc->sc_gmx_port);
	cn30xxgmx_reset_flowctl(sc->sc_gmx_port);
	cn30xxgmx_reset_timing(sc->sc_gmx_port);

	return 0;
}

int
cnmac_configure(struct cnmac_softc *sc)
{
	cn30xxgmx_port_enable(sc->sc_gmx_port, 0);

	cnmac_reset(sc);

	cn30xxpko_port_config(sc->sc_pko);
	cn30xxpko_port_enable(sc->sc_pko, 1);
	cn30xxpow_config(sc->sc_pow, sc->sc_powgroup);

	cn30xxgmx_port_enable(sc->sc_gmx_port, 1);

	return 0;
}

int
cnmac_configure_common(struct cnmac_softc *sc)
{
	static int once;

	uint64_t reg;

	if (once == 1)
		return 0;
	once = 1;

	cn30xxipd_config(sc->sc_ipd);
	cn30xxpko_config(sc->sc_pko);

	/* Set padding for packets that Octeon does not recognize as IP. */
	reg = octeon_xkphys_read_8(PIP_GBL_CFG);
	reg &= ~PIP_GBL_CFG_NIP_SHF_MASK;
	reg |= ETHER_ALIGN << PIP_GBL_CFG_NIP_SHF_SHIFT;
	octeon_xkphys_write_8(PIP_GBL_CFG, reg);

	return 0;
}

int
cnmac_mbuf_alloc(int n)
{
	struct mbuf *m;
	paddr_t pktbuf;

	while (n > 0) {
		m = MCLGETL(NULL, M_NOWAIT,
		    OCTEON_POOL_SIZE_PKT + CACHELINESIZE);
		if (m == NULL || !ISSET(m->m_flags, M_EXT)) {
			m_freem(m);
			break;
		}

		m->m_data = (void *)(((vaddr_t)m->m_data + CACHELINESIZE) &
		    ~(CACHELINESIZE - 1));
		((struct mbuf **)m->m_data)[-1] = m;

		pktbuf = KVTOPHYS(m->m_data);
		m->m_pkthdr.ph_cookie = (void *)pktbuf;
		cn30xxfpa_store(pktbuf, OCTEON_POOL_NO_PKT,
		    OCTEON_POOL_SIZE_PKT / CACHELINESIZE);

		n--;
	}
	return n;
}

int
cnmac_recv_mbuf(struct cnmac_softc *sc, uint64_t *work,
    struct mbuf **rm, int *nmbuf)
{
	struct mbuf *m, *m0, *mprev, **pm;
	paddr_t addr, pktbuf;
	uint64_t word1 = work[1];
	uint64_t word2 = work[2];
	uint64_t word3 = work[3];
	unsigned int back, i, nbufs;
	unsigned int left, total, size;

	cn30xxfpa_buf_put_paddr(cnmac_fb_wqe, XKPHYS_TO_PHYS(work));

	nbufs = (word2 & PIP_WQE_WORD2_IP_BUFS) >> PIP_WQE_WORD2_IP_BUFS_SHIFT;
	if (nbufs == 0)
		panic("%s: dynamic short packet", __func__);

	m0 = mprev = NULL;
	total = left = (word1 & PIP_WQE_WORD1_LEN) >> 48;
	for (i = 0; i < nbufs; i++) {
		addr = word3 & PIP_WQE_WORD3_ADDR;
		back = (word3 & PIP_WQE_WORD3_BACK) >> PIP_WQE_WORD3_BACK_SHIFT;
		pktbuf = (addr & ~(CACHELINESIZE - 1)) - back * CACHELINESIZE;
		pm = (struct mbuf **)PHYS_TO_XKPHYS(pktbuf, CCA_CACHED) - 1;
		m = *pm;
		*pm = NULL;
		if ((paddr_t)m->m_pkthdr.ph_cookie != pktbuf)
			panic("%s: packet pool is corrupted, mbuf cookie %p != "
			    "pktbuf %p", __func__, m->m_pkthdr.ph_cookie,
			    (void *)pktbuf);

		/*
		 * Because of a hardware bug in some Octeon models the size
		 * field of word3 can be wrong (erratum PKI-100).
		 * However, the hardware uses all space in a buffer before
		 * moving to the next one so it is possible to derive
		 * the size of this data segment from the size
		 * of packet data buffers.
		 */
		size = OCTEON_POOL_SIZE_PKT - (addr - pktbuf);
		if (size > left)
			size = left;

		m->m_pkthdr.ph_cookie = NULL;
		m->m_data += addr - pktbuf;
		m->m_len = size;
		left -= size;

		if (m0 == NULL)
			m0 = m;
		else {
			m->m_flags &= ~M_PKTHDR;
			mprev->m_next = m;
		}
		mprev = m;

		if (i + 1 < nbufs)
			memcpy(&word3, (void *)PHYS_TO_XKPHYS(addr -
			    sizeof(word3), CCA_CACHED), sizeof(word3));
	}

	m0->m_pkthdr.len = total;
	*rm = m0;
	*nmbuf = nbufs;

	return 0;
}

int
cnmac_recv_check(struct cnmac_softc *sc, uint64_t word2)
{
	static struct timeval rxerr_log_interval = { 0, 250000 };
	uint64_t opecode;

	if (__predict_true(!ISSET(word2, PIP_WQE_WORD2_NOIP_RE)))
		return 0;

	opecode = word2 & PIP_WQE_WORD2_NOIP_OPECODE;
	if ((sc->sc_arpcom.ac_if.if_flags & IFF_DEBUG) &&
	    ratecheck(&sc->sc_rxerr_log_last, &rxerr_log_interval))
		log(LOG_DEBUG, "%s: rx error (%lld)\n", sc->sc_dev.dv_xname,
		    opecode);

	/* XXX harmless error? */
	if (opecode == PIP_WQE_WORD2_RE_OPCODE_OVRRUN)
		return 0;

	return 1;
}

int
cnmac_recv(struct cnmac_softc *sc, uint64_t *work, struct mbuf_list *ml)
{
	struct ifnet *ifp = &sc->sc_arpcom.ac_if;
	struct mbuf *m;
	uint64_t word2;
	int nmbuf = 0;

	word2 = work[2];

	if (!(ifp->if_flags & IFF_RUNNING))
		goto drop;

	if (__predict_false(cnmac_recv_check(sc, word2) != 0)) {
		ifp->if_ierrors++;
		goto drop;
	}

	/* On success, this releases the work queue entry. */
	if (__predict_false(cnmac_recv_mbuf(sc, work, &m, &nmbuf) != 0)) {
		ifp->if_ierrors++;
		goto drop;
	}

	m->m_pkthdr.csum_flags = 0;
	if (__predict_true(!ISSET(word2, PIP_WQE_WORD2_IP_NI))) {
		/* Check IP checksum status. */
		if (!ISSET(word2, PIP_WQE_WORD2_IP_V6) &&
		    !ISSET(word2, PIP_WQE_WORD2_IP_IE))
			m->m_pkthdr.csum_flags |= M_IPV4_CSUM_IN_OK;

		/* Check TCP/UDP checksum status. */
		if (ISSET(word2, PIP_WQE_WORD2_IP_TU) &&
		    !ISSET(word2, PIP_WQE_WORD2_IP_FR) &&
		    !ISSET(word2, PIP_WQE_WORD2_IP_LE))
			m->m_pkthdr.csum_flags |=
			    M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK;
	}

	ml_enqueue(ml, m);

	return nmbuf;

drop:
	cnmac_buf_free_work(sc, work);
	return 0;
}

int
cnmac_intr(void *arg)
{
	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
	struct cnmac_softc *sc = arg;
	struct ifnet *ifp = &sc->sc_arpcom.ac_if;
	uint64_t *work;
	uint64_t wqmask = 1ull << sc->sc_powgroup;
	uint32_t coreid = octeon_get_coreid();
	uint32_t port;
	int nmbuf = 0;

	_POW_WR8(sc->sc_pow, POW_PP_GRP_MSK_OFFSET(coreid), wqmask);

	cn30xxpow_tag_sw_wait();
	cn30xxpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
	    POW_NO_WAIT);

	for (;;) {
		work = (uint64_t *)cn30xxpow_work_response_async(
		    OCTEON_CVMSEG_OFFSET(csm_pow_intr));
		if (work == NULL)
			break;

		cn30xxpow_tag_sw_wait();
		cn30xxpow_work_request_async(
		    OCTEON_CVMSEG_OFFSET(csm_pow_intr), POW_NO_WAIT);

		port = (work[1] & PIP_WQE_WORD1_IPRT) >> 42;
		if (port != sc->sc_port) {
			printf("%s: unexpected wqe port %u, should be %u\n",
			    sc->sc_dev.dv_xname, port, sc->sc_port);
			goto wqe_error;
		}

		nmbuf += cnmac_recv(sc, work, &ml);
	}

	_POW_WR8(sc->sc_pow, POW_WQ_INT_OFFSET, wqmask);

	if_input(ifp, &ml);

	nmbuf = cnmac_mbuf_alloc(nmbuf);
	if (nmbuf != 0)
		atomic_add_int(&cnmac_mbufs_to_alloc, nmbuf);

	return 1;

wqe_error:
	printf("word0: 0x%016llx\n", work[0]);
	printf("word1: 0x%016llx\n", work[1]);
	printf("word2: 0x%016llx\n", work[2]);
	printf("word3: 0x%016llx\n", work[3]);
	panic("wqe error");
}

/* ---- tick */

void
cnmac_free_task(void *arg)
{
	struct cnmac_softc *sc = arg;
	struct ifnet *ifp = &sc->sc_arpcom.ac_if;
	struct ifqueue *ifq = &ifp->if_snd;
	int resched = 1;
	int timeout;

	if (ml_len(&sc->sc_sendq) > 0) {
		cnmac_send_queue_flush_prefetch(sc);
		cnmac_send_queue_flush_fetch(sc);
		cnmac_send_queue_flush(sc);
	}

	if (ifq_is_oactive(ifq)) {
		ifq_clr_oactive(ifq);
		cnmac_start(ifq);

		if (ifq_is_oactive(ifq)) {
			/* The start routine did rescheduling already. */
			resched = 0;
		}
	}

	if (resched) {
		timeout = (sc->sc_ext_callback_cnt > 0) ? 1 : hz;
		timeout_add(&sc->sc_tick_free_ch, timeout);
	}
}

/*
 * cnmac_tick_free
 *
 * => garbage collect send gather buffer / mbuf
 * => called at softclock
 */
void
cnmac_tick_free(void *arg)
{
	struct cnmac_softc *sc = arg;
	struct ifnet *ifp = &sc->sc_arpcom.ac_if;
	int to_alloc;

	ifq_serialize(&ifp->if_snd, &sc->sc_free_task);

	if (cnmac_mbufs_to_alloc != 0) {
		to_alloc = atomic_swap_uint(&cnmac_mbufs_to_alloc, 0);
		to_alloc = cnmac_mbuf_alloc(to_alloc);
		if (to_alloc != 0)
			atomic_add_int(&cnmac_mbufs_to_alloc, to_alloc);
	}
}

/*
 * cnmac_tick_misc
 *
 * => collect statistics
 * => check link status
 * => called at softclock
 */
void
cnmac_tick_misc(void *arg)
{
	struct cnmac_softc *sc = arg;
	int s;

	s = splnet();
	mii_tick(&sc->sc_mii);
	splx(s);

#if NKSTAT > 0
	cnmac_kstat_tick(sc);
#endif

	timeout_add_sec(&sc->sc_tick_misc_ch, 1);
}

#if NKSTAT > 0
#define KVE(n, t) \
	KSTAT_KV_UNIT_INITIALIZER((n), KSTAT_KV_T_COUNTER64, (t))

static const struct kstat_kv cnmac_kstat_tpl[cnmac_stat_count] = {
	[cnmac_stat_rx_toto_gmx]= KVE("rx total gmx",	KSTAT_KV_U_BYTES),
	[cnmac_stat_rx_totp_gmx]= KVE("rx total gmx",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_toto_pip]= KVE("rx total pip",	KSTAT_KV_U_BYTES),
	[cnmac_stat_rx_totp_pip]= KVE("rx total pip",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_h64]	= KVE("rx 64B",		KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_h127]	= KVE("rx 65-127B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_h255]	= KVE("rx 128-255B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_h511]	= KVE("rx 256-511B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_h1023]	= KVE("rx 512-1023B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_h1518]	= KVE("rx 1024-1518B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_hmax]	= KVE("rx 1519-maxB",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_bcast]	= KVE("rx bcast",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_mcast]	= KVE("rx mcast",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_qdpo]	= KVE("rx qos drop",	KSTAT_KV_U_BYTES),
	[cnmac_stat_rx_qdpp]	= KVE("rx qos drop",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_fcs]	= KVE("rx fcs err",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_frag]	= KVE("rx fcs undersize",KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_undersz]	= KVE("rx undersize",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_jabber]	= KVE("rx jabber",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_oversz]	= KVE("rx oversize",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_raw]	= KVE("rx raw",		KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_bad]	= KVE("rx bad",		KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_drop]	= KVE("rx drop",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_ctl]	= KVE("rx control",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_rx_dmac]	= KVE("rx dmac",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_toto]	= KVE("tx total",	KSTAT_KV_U_BYTES),
	[cnmac_stat_tx_totp]	= KVE("tx total",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_hmin]	= KVE("tx min-63B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_h64]	= KVE("tx 64B",		KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_h127]	= KVE("tx 65-127B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_h255]	= KVE("tx 128-255B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_h511]	= KVE("tx 256-511B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_h1023]	= KVE("tx 512-1023B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_h1518]	= KVE("tx 1024-1518B",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_hmax]	= KVE("tx 1519-maxB",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_bcast]	= KVE("tx bcast",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_mcast]	= KVE("tx mcast",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_coll]	= KVE("tx coll",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_defer]	= KVE("tx defer",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_scol]	= KVE("tx scoll",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_mcol]	= KVE("tx mcoll",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_ctl]	= KVE("tx control",	KSTAT_KV_U_PACKETS),
	[cnmac_stat_tx_uflow]	= KVE("tx underflow",	KSTAT_KV_U_PACKETS),
};

void
cnmac_kstat_attach(struct cnmac_softc *sc)
{
	struct kstat *ks;
	struct kstat_kv *kvs;

	mtx_init(&sc->sc_kstat_mtx, IPL_SOFTCLOCK);

	ks = kstat_create(sc->sc_dev.dv_xname, 0, "cnmac-stats", 0,
	    KSTAT_T_KV, 0);
	if (ks == NULL)
		return;

	kvs = malloc(sizeof(cnmac_kstat_tpl), M_DEVBUF, M_WAITOK | M_ZERO);
	memcpy(kvs, cnmac_kstat_tpl, sizeof(cnmac_kstat_tpl));

	kstat_set_mutex(ks, &sc->sc_kstat_mtx);
	ks->ks_softc = sc;
	ks->ks_data = kvs;
	ks->ks_datalen = sizeof(cnmac_kstat_tpl);
	ks->ks_read = cnmac_kstat_read;

	sc->sc_kstat = ks;
	kstat_install(ks);
}

int
cnmac_kstat_read(struct kstat *ks)
{
	struct cnmac_softc *sc = ks->ks_softc;
	struct kstat_kv *kvs = ks->ks_data;

	cn30xxpip_kstat_read(sc->sc_pip, kvs);
	cn30xxgmx_kstat_read(sc->sc_gmx_port, kvs);

	getnanouptime(&ks->ks_updated);

	return 0;
}

void
cnmac_kstat_tick(struct cnmac_softc *sc)
{
	if (sc->sc_kstat == NULL)
		return;
	if (!mtx_enter_try(&sc->sc_kstat_mtx))
		return;
	cnmac_kstat_read(sc->sc_kstat);
	mtx_leave(&sc->sc_kstat_mtx);
}
#endif