xref: /dpdk/drivers/common/sfc_efx/base/efx_ev.c (revision aa6dc1017cd3fa7b722ca877558e871f82eb1f02)
15e111ed8SAndrew Rybchenko /* SPDX-License-Identifier: BSD-3-Clause
25e111ed8SAndrew Rybchenko  *
3672386c1SAndrew Rybchenko  * Copyright(c) 2019-2021 Xilinx, Inc.
45e111ed8SAndrew Rybchenko  * Copyright(c) 2007-2019 Solarflare Communications Inc.
55e111ed8SAndrew Rybchenko  */
65e111ed8SAndrew Rybchenko 
75e111ed8SAndrew Rybchenko #include "efx.h"
85e111ed8SAndrew Rybchenko #include "efx_impl.h"
95e111ed8SAndrew Rybchenko #if EFSYS_OPT_MON_MCDI
105e111ed8SAndrew Rybchenko #include "mcdi_mon.h"
115e111ed8SAndrew Rybchenko #endif
125e111ed8SAndrew Rybchenko 
135e111ed8SAndrew Rybchenko #define	EFX_EV_PRESENT(_qword)						\
145e111ed8SAndrew Rybchenko 	(EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&	\
155e111ed8SAndrew Rybchenko 	EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
165e111ed8SAndrew Rybchenko 
175e111ed8SAndrew Rybchenko 
185e111ed8SAndrew Rybchenko 
195e111ed8SAndrew Rybchenko #if EFSYS_OPT_SIENA
205e111ed8SAndrew Rybchenko 
215e111ed8SAndrew Rybchenko static	__checkReturn	efx_rc_t
225e111ed8SAndrew Rybchenko siena_ev_init(
235e111ed8SAndrew Rybchenko 	__in		efx_nic_t *enp);
245e111ed8SAndrew Rybchenko 
255e111ed8SAndrew Rybchenko static			void
265e111ed8SAndrew Rybchenko siena_ev_fini(
275e111ed8SAndrew Rybchenko 	__in		efx_nic_t *enp);
285e111ed8SAndrew Rybchenko 
295e111ed8SAndrew Rybchenko static	__checkReturn	efx_rc_t
305e111ed8SAndrew Rybchenko siena_ev_qcreate(
315e111ed8SAndrew Rybchenko 	__in		efx_nic_t *enp,
325e111ed8SAndrew Rybchenko 	__in		unsigned int index,
335e111ed8SAndrew Rybchenko 	__in		efsys_mem_t *esmp,
345e111ed8SAndrew Rybchenko 	__in		size_t ndescs,
355e111ed8SAndrew Rybchenko 	__in		uint32_t id,
365e111ed8SAndrew Rybchenko 	__in		uint32_t us,
375e111ed8SAndrew Rybchenko 	__in		uint32_t flags,
38*aa6dc101SAndrew Rybchenko 	__in		uint32_t irq,
395e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep);
405e111ed8SAndrew Rybchenko 
415e111ed8SAndrew Rybchenko static			void
425e111ed8SAndrew Rybchenko siena_ev_qdestroy(
435e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep);
445e111ed8SAndrew Rybchenko 
455e111ed8SAndrew Rybchenko static	__checkReturn	efx_rc_t
465e111ed8SAndrew Rybchenko siena_ev_qprime(
475e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
485e111ed8SAndrew Rybchenko 	__in		unsigned int count);
495e111ed8SAndrew Rybchenko 
505e111ed8SAndrew Rybchenko static			void
515e111ed8SAndrew Rybchenko siena_ev_qpost(
525e111ed8SAndrew Rybchenko 	__in	efx_evq_t *eep,
535e111ed8SAndrew Rybchenko 	__in	uint16_t data);
545e111ed8SAndrew Rybchenko 
555e111ed8SAndrew Rybchenko static	__checkReturn	efx_rc_t
565e111ed8SAndrew Rybchenko siena_ev_qmoderate(
575e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
585e111ed8SAndrew Rybchenko 	__in		unsigned int us);
595e111ed8SAndrew Rybchenko 
605e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
615e111ed8SAndrew Rybchenko static			void
625e111ed8SAndrew Rybchenko siena_ev_qstats_update(
635e111ed8SAndrew Rybchenko 	__in				efx_evq_t *eep,
645e111ed8SAndrew Rybchenko 	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat);
655e111ed8SAndrew Rybchenko 
665e111ed8SAndrew Rybchenko #endif
675e111ed8SAndrew Rybchenko 
685e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
695e111ed8SAndrew Rybchenko 
70ad1e3ed8SAndrew Rybchenko #if EFX_OPTS_EF10() || EFSYS_OPT_SIENA
71ad1e3ed8SAndrew Rybchenko 
72ad1e3ed8SAndrew Rybchenko static			void
73ad1e3ed8SAndrew Rybchenko siena_ef10_ev_qpoll(
74ad1e3ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
75ad1e3ed8SAndrew Rybchenko 	__inout		unsigned int *countp,
76ad1e3ed8SAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
77ad1e3ed8SAndrew Rybchenko 	__in_opt	void *arg);
78ad1e3ed8SAndrew Rybchenko 
79ad1e3ed8SAndrew Rybchenko #endif	/* EFX_OPTS_EF10() || EFSYS_OPT_SIENA */
80ad1e3ed8SAndrew Rybchenko 
815e111ed8SAndrew Rybchenko #if EFSYS_OPT_SIENA
825e111ed8SAndrew Rybchenko static const efx_ev_ops_t	__efx_ev_siena_ops = {
835e111ed8SAndrew Rybchenko 	siena_ev_init,				/* eevo_init */
845e111ed8SAndrew Rybchenko 	siena_ev_fini,				/* eevo_fini */
855e111ed8SAndrew Rybchenko 	siena_ev_qcreate,			/* eevo_qcreate */
865e111ed8SAndrew Rybchenko 	siena_ev_qdestroy,			/* eevo_qdestroy */
875e111ed8SAndrew Rybchenko 	siena_ev_qprime,			/* eevo_qprime */
885e111ed8SAndrew Rybchenko 	siena_ev_qpost,				/* eevo_qpost */
89ad1e3ed8SAndrew Rybchenko 	siena_ef10_ev_qpoll,			/* eevo_qpoll */
905e111ed8SAndrew Rybchenko 	siena_ev_qmoderate,			/* eevo_qmoderate */
915e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
925e111ed8SAndrew Rybchenko 	siena_ev_qstats_update,			/* eevo_qstats_update */
935e111ed8SAndrew Rybchenko #endif
945e111ed8SAndrew Rybchenko };
955e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
965e111ed8SAndrew Rybchenko 
975e111ed8SAndrew Rybchenko #if EFX_OPTS_EF10()
985e111ed8SAndrew Rybchenko static const efx_ev_ops_t	__efx_ev_ef10_ops = {
995e111ed8SAndrew Rybchenko 	ef10_ev_init,				/* eevo_init */
1005e111ed8SAndrew Rybchenko 	ef10_ev_fini,				/* eevo_fini */
1015e111ed8SAndrew Rybchenko 	ef10_ev_qcreate,			/* eevo_qcreate */
1025e111ed8SAndrew Rybchenko 	ef10_ev_qdestroy,			/* eevo_qdestroy */
1035e111ed8SAndrew Rybchenko 	ef10_ev_qprime,				/* eevo_qprime */
1045e111ed8SAndrew Rybchenko 	ef10_ev_qpost,				/* eevo_qpost */
105ad1e3ed8SAndrew Rybchenko 	siena_ef10_ev_qpoll,			/* eevo_qpoll */
1065e111ed8SAndrew Rybchenko 	ef10_ev_qmoderate,			/* eevo_qmoderate */
1075e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
1085e111ed8SAndrew Rybchenko 	ef10_ev_qstats_update,			/* eevo_qstats_update */
1095e111ed8SAndrew Rybchenko #endif
1105e111ed8SAndrew Rybchenko };
1115e111ed8SAndrew Rybchenko #endif /* EFX_OPTS_EF10() */
1125e111ed8SAndrew Rybchenko 
113b97bf1caSAndrew Rybchenko #if EFSYS_OPT_RIVERHEAD
114b97bf1caSAndrew Rybchenko static const efx_ev_ops_t	__efx_ev_rhead_ops = {
115b97bf1caSAndrew Rybchenko 	rhead_ev_init,				/* eevo_init */
116b97bf1caSAndrew Rybchenko 	rhead_ev_fini,				/* eevo_fini */
117b97bf1caSAndrew Rybchenko 	rhead_ev_qcreate,			/* eevo_qcreate */
118b97bf1caSAndrew Rybchenko 	rhead_ev_qdestroy,			/* eevo_qdestroy */
119b97bf1caSAndrew Rybchenko 	rhead_ev_qprime,			/* eevo_qprime */
120b97bf1caSAndrew Rybchenko 	rhead_ev_qpost,				/* eevo_qpost */
121b97bf1caSAndrew Rybchenko 	rhead_ev_qpoll,				/* eevo_qpoll */
122b97bf1caSAndrew Rybchenko 	rhead_ev_qmoderate,			/* eevo_qmoderate */
123b97bf1caSAndrew Rybchenko #if EFSYS_OPT_QSTATS
124b97bf1caSAndrew Rybchenko 	rhead_ev_qstats_update,			/* eevo_qstats_update */
125b97bf1caSAndrew Rybchenko #endif
126b97bf1caSAndrew Rybchenko };
127b97bf1caSAndrew Rybchenko #endif /* EFSYS_OPT_RIVERHEAD */
128b97bf1caSAndrew Rybchenko 
1295e111ed8SAndrew Rybchenko 
1305e111ed8SAndrew Rybchenko 	__checkReturn	efx_rc_t
efx_ev_init(__in efx_nic_t * enp)1315e111ed8SAndrew Rybchenko efx_ev_init(
1325e111ed8SAndrew Rybchenko 	__in		efx_nic_t *enp)
1335e111ed8SAndrew Rybchenko {
1345e111ed8SAndrew Rybchenko 	const efx_ev_ops_t *eevop;
1355e111ed8SAndrew Rybchenko 	efx_rc_t rc;
1365e111ed8SAndrew Rybchenko 
1375e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1385e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
1395e111ed8SAndrew Rybchenko 
1405e111ed8SAndrew Rybchenko 	if (enp->en_mod_flags & EFX_MOD_EV) {
1415e111ed8SAndrew Rybchenko 		rc = EINVAL;
1425e111ed8SAndrew Rybchenko 		goto fail1;
1435e111ed8SAndrew Rybchenko 	}
1445e111ed8SAndrew Rybchenko 
1455e111ed8SAndrew Rybchenko 	switch (enp->en_family) {
1465e111ed8SAndrew Rybchenko #if EFSYS_OPT_SIENA
1475e111ed8SAndrew Rybchenko 	case EFX_FAMILY_SIENA:
1485e111ed8SAndrew Rybchenko 		eevop = &__efx_ev_siena_ops;
1495e111ed8SAndrew Rybchenko 		break;
1505e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
1515e111ed8SAndrew Rybchenko 
1525e111ed8SAndrew Rybchenko #if EFSYS_OPT_HUNTINGTON
1535e111ed8SAndrew Rybchenko 	case EFX_FAMILY_HUNTINGTON:
1545e111ed8SAndrew Rybchenko 		eevop = &__efx_ev_ef10_ops;
1555e111ed8SAndrew Rybchenko 		break;
1565e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_HUNTINGTON */
1575e111ed8SAndrew Rybchenko 
1585e111ed8SAndrew Rybchenko #if EFSYS_OPT_MEDFORD
1595e111ed8SAndrew Rybchenko 	case EFX_FAMILY_MEDFORD:
1605e111ed8SAndrew Rybchenko 		eevop = &__efx_ev_ef10_ops;
1615e111ed8SAndrew Rybchenko 		break;
1625e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_MEDFORD */
1635e111ed8SAndrew Rybchenko 
1645e111ed8SAndrew Rybchenko #if EFSYS_OPT_MEDFORD2
1655e111ed8SAndrew Rybchenko 	case EFX_FAMILY_MEDFORD2:
1665e111ed8SAndrew Rybchenko 		eevop = &__efx_ev_ef10_ops;
1675e111ed8SAndrew Rybchenko 		break;
1685e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_MEDFORD2 */
1695e111ed8SAndrew Rybchenko 
170b97bf1caSAndrew Rybchenko #if EFSYS_OPT_RIVERHEAD
171b97bf1caSAndrew Rybchenko 	case EFX_FAMILY_RIVERHEAD:
172b97bf1caSAndrew Rybchenko 		eevop = &__efx_ev_rhead_ops;
173b97bf1caSAndrew Rybchenko 		break;
174b97bf1caSAndrew Rybchenko #endif /* EFSYS_OPT_RIVERHEAD */
175b97bf1caSAndrew Rybchenko 
1765e111ed8SAndrew Rybchenko 	default:
1775e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(0);
1785e111ed8SAndrew Rybchenko 		rc = ENOTSUP;
1795e111ed8SAndrew Rybchenko 		goto fail1;
1805e111ed8SAndrew Rybchenko 	}
1815e111ed8SAndrew Rybchenko 
1825e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
1835e111ed8SAndrew Rybchenko 
1845e111ed8SAndrew Rybchenko 	if ((rc = eevop->eevo_init(enp)) != 0)
1855e111ed8SAndrew Rybchenko 		goto fail2;
1865e111ed8SAndrew Rybchenko 
1875e111ed8SAndrew Rybchenko 	enp->en_eevop = eevop;
1885e111ed8SAndrew Rybchenko 	enp->en_mod_flags |= EFX_MOD_EV;
1895e111ed8SAndrew Rybchenko 	return (0);
1905e111ed8SAndrew Rybchenko 
1915e111ed8SAndrew Rybchenko fail2:
1925e111ed8SAndrew Rybchenko 	EFSYS_PROBE(fail2);
1935e111ed8SAndrew Rybchenko 
1945e111ed8SAndrew Rybchenko fail1:
1955e111ed8SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1965e111ed8SAndrew Rybchenko 
1975e111ed8SAndrew Rybchenko 	enp->en_eevop = NULL;
1985e111ed8SAndrew Rybchenko 	enp->en_mod_flags &= ~EFX_MOD_EV;
1995e111ed8SAndrew Rybchenko 	return (rc);
2005e111ed8SAndrew Rybchenko }
2015e111ed8SAndrew Rybchenko 
2025e111ed8SAndrew Rybchenko 	__checkReturn	size_t
efx_evq_size(__in const efx_nic_t * enp,__in unsigned int ndescs,__in uint32_t flags)2035e111ed8SAndrew Rybchenko efx_evq_size(
2045e111ed8SAndrew Rybchenko 	__in	const efx_nic_t *enp,
205f8a60f76SAndy Moreton 	__in	unsigned int ndescs,
206f8a60f76SAndy Moreton 	__in	uint32_t flags)
2075e111ed8SAndrew Rybchenko {
2085e111ed8SAndrew Rybchenko 	const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
209f8a60f76SAndy Moreton 	size_t desc_size;
2105e111ed8SAndrew Rybchenko 
211f8a60f76SAndy Moreton 	desc_size = encp->enc_ev_desc_size;
212f8a60f76SAndy Moreton 
213f8a60f76SAndy Moreton #if EFSYS_OPT_EV_EXTENDED_WIDTH
214f8a60f76SAndy Moreton 	if (flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH)
215f8a60f76SAndy Moreton 		desc_size = encp->enc_ev_ew_desc_size;
216f8a60f76SAndy Moreton #else
217f8a60f76SAndy Moreton 	EFSYS_ASSERT((flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH) == 0);
218f8a60f76SAndy Moreton #endif
219f8a60f76SAndy Moreton 
220f8a60f76SAndy Moreton 	return (ndescs * desc_size);
2215e111ed8SAndrew Rybchenko }
2225e111ed8SAndrew Rybchenko 
2235e111ed8SAndrew Rybchenko 	__checkReturn	unsigned int
efx_evq_nbufs(__in const efx_nic_t * enp,__in unsigned int ndescs,__in uint32_t flags)2245e111ed8SAndrew Rybchenko efx_evq_nbufs(
2255e111ed8SAndrew Rybchenko 	__in	const efx_nic_t *enp,
226f8a60f76SAndy Moreton 	__in	unsigned int ndescs,
227f8a60f76SAndy Moreton 	__in	uint32_t flags)
2285e111ed8SAndrew Rybchenko {
229f8a60f76SAndy Moreton 	size_t size;
230f8a60f76SAndy Moreton 
231f8a60f76SAndy Moreton 	size = efx_evq_size(enp, ndescs, flags);
232f8a60f76SAndy Moreton 
233f8a60f76SAndy Moreton 	return (EFX_DIV_ROUND_UP(size, EFX_BUF_SIZE));
2345e111ed8SAndrew Rybchenko }
2355e111ed8SAndrew Rybchenko 
2365e111ed8SAndrew Rybchenko 		void
efx_ev_fini(__in efx_nic_t * enp)2375e111ed8SAndrew Rybchenko efx_ev_fini(
2385e111ed8SAndrew Rybchenko 	__in	efx_nic_t *enp)
2395e111ed8SAndrew Rybchenko {
2405e111ed8SAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
2415e111ed8SAndrew Rybchenko 
2425e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
2435e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
2445e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
2455e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
2465e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
2475e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
2485e111ed8SAndrew Rybchenko 
2495e111ed8SAndrew Rybchenko 	eevop->eevo_fini(enp);
2505e111ed8SAndrew Rybchenko 
2515e111ed8SAndrew Rybchenko 	enp->en_eevop = NULL;
2525e111ed8SAndrew Rybchenko 	enp->en_mod_flags &= ~EFX_MOD_EV;
2535e111ed8SAndrew Rybchenko }
2545e111ed8SAndrew Rybchenko 
2555e111ed8SAndrew Rybchenko 
2565e111ed8SAndrew Rybchenko 	__checkReturn	efx_rc_t
efx_ev_qcreate_irq(__in efx_nic_t * enp,__in unsigned int index,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in uint32_t us,__in uint32_t flags,__in uint32_t irq,__deref_out efx_evq_t ** eepp)257*aa6dc101SAndrew Rybchenko efx_ev_qcreate_irq(
2585e111ed8SAndrew Rybchenko 	__in		efx_nic_t *enp,
2595e111ed8SAndrew Rybchenko 	__in		unsigned int index,
2605e111ed8SAndrew Rybchenko 	__in		efsys_mem_t *esmp,
2615e111ed8SAndrew Rybchenko 	__in		size_t ndescs,
2625e111ed8SAndrew Rybchenko 	__in		uint32_t id,
2635e111ed8SAndrew Rybchenko 	__in		uint32_t us,
2645e111ed8SAndrew Rybchenko 	__in		uint32_t flags,
265*aa6dc101SAndrew Rybchenko 	__in		uint32_t irq,
2665e111ed8SAndrew Rybchenko 	__deref_out	efx_evq_t **eepp)
2675e111ed8SAndrew Rybchenko {
2685e111ed8SAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
2695e111ed8SAndrew Rybchenko 	efx_evq_t *eep;
2705e111ed8SAndrew Rybchenko 	const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
2715e111ed8SAndrew Rybchenko 	efx_rc_t rc;
2725e111ed8SAndrew Rybchenko 
2735e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
2745e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
2755e111ed8SAndrew Rybchenko 
2765e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <,
2775e111ed8SAndrew Rybchenko 	    enp->en_nic_cfg.enc_evq_limit);
2785e111ed8SAndrew Rybchenko 
2792e5819a5SAndrew Rybchenko 	if (index >= encp->enc_evq_limit) {
2802e5819a5SAndrew Rybchenko 		rc = EINVAL;
2812e5819a5SAndrew Rybchenko 		goto fail1;
2822e5819a5SAndrew Rybchenko 	}
2832e5819a5SAndrew Rybchenko 
2842e5819a5SAndrew Rybchenko 	if (us > encp->enc_evq_timer_max_us) {
2852e5819a5SAndrew Rybchenko 		rc = EINVAL;
2862e5819a5SAndrew Rybchenko 		goto fail2;
2872e5819a5SAndrew Rybchenko 	}
2882e5819a5SAndrew Rybchenko 
2895e111ed8SAndrew Rybchenko 	switch (flags & EFX_EVQ_FLAGS_NOTIFY_MASK) {
2905e111ed8SAndrew Rybchenko 	case EFX_EVQ_FLAGS_NOTIFY_INTERRUPT:
2915e111ed8SAndrew Rybchenko 		break;
2925e111ed8SAndrew Rybchenko 	case EFX_EVQ_FLAGS_NOTIFY_DISABLED:
2935e111ed8SAndrew Rybchenko 		if (us != 0) {
2945e111ed8SAndrew Rybchenko 			rc = EINVAL;
2952e5819a5SAndrew Rybchenko 			goto fail3;
2965e111ed8SAndrew Rybchenko 		}
2975e111ed8SAndrew Rybchenko 		break;
2985e111ed8SAndrew Rybchenko 	default:
2995e111ed8SAndrew Rybchenko 		rc = EINVAL;
3002e5819a5SAndrew Rybchenko 		goto fail4;
3015e111ed8SAndrew Rybchenko 	}
3025e111ed8SAndrew Rybchenko 
303f8a60f76SAndy Moreton 	if ((flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH) &&
304f8a60f76SAndy Moreton 	    (encp->enc_ev_ew_desc_size == 0)) {
305f8a60f76SAndy Moreton 		/* Extended width event descriptors are not supported. */
306f8a60f76SAndy Moreton 		rc = EINVAL;
307f8a60f76SAndy Moreton 		goto fail5;
308f8a60f76SAndy Moreton 	}
309f8a60f76SAndy Moreton 
3105e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(ISP2(encp->enc_evq_max_nevs));
3115e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(ISP2(encp->enc_evq_min_nevs));
3125e111ed8SAndrew Rybchenko 
3135e111ed8SAndrew Rybchenko 	if (!ISP2(ndescs) ||
3145e111ed8SAndrew Rybchenko 	    ndescs < encp->enc_evq_min_nevs ||
3155e111ed8SAndrew Rybchenko 	    ndescs > encp->enc_evq_max_nevs) {
3165e111ed8SAndrew Rybchenko 		rc = EINVAL;
317f8a60f76SAndy Moreton 		goto fail6;
318f8a60f76SAndy Moreton 	}
319f8a60f76SAndy Moreton 
320f8a60f76SAndy Moreton 	if (EFSYS_MEM_SIZE(esmp) < (ndescs * encp->enc_ev_desc_size)) {
321f8a60f76SAndy Moreton 		/* Buffer too small for event queue descriptors. */
322f8a60f76SAndy Moreton 		rc = EINVAL;
323f8a60f76SAndy Moreton 		goto fail7;
3245e111ed8SAndrew Rybchenko 	}
3255e111ed8SAndrew Rybchenko 
3265e111ed8SAndrew Rybchenko 	/* Allocate an EVQ object */
3275e111ed8SAndrew Rybchenko 	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
3285e111ed8SAndrew Rybchenko 	if (eep == NULL) {
3295e111ed8SAndrew Rybchenko 		rc = ENOMEM;
330f8a60f76SAndy Moreton 		goto fail8;
3315e111ed8SAndrew Rybchenko 	}
3325e111ed8SAndrew Rybchenko 
3335e111ed8SAndrew Rybchenko 	eep->ee_magic = EFX_EVQ_MAGIC;
3345e111ed8SAndrew Rybchenko 	eep->ee_enp = enp;
3355e111ed8SAndrew Rybchenko 	eep->ee_index = index;
3365e111ed8SAndrew Rybchenko 	eep->ee_mask = ndescs - 1;
3375e111ed8SAndrew Rybchenko 	eep->ee_flags = flags;
3385e111ed8SAndrew Rybchenko 	eep->ee_esmp = esmp;
3395e111ed8SAndrew Rybchenko 
3405e111ed8SAndrew Rybchenko 	/*
3415e111ed8SAndrew Rybchenko 	 * Set outputs before the queue is created because interrupts may be
3425e111ed8SAndrew Rybchenko 	 * raised for events immediately after the queue is created, before the
3435e111ed8SAndrew Rybchenko 	 * function call below returns. See bug58606.
3445e111ed8SAndrew Rybchenko 	 *
3455e111ed8SAndrew Rybchenko 	 * The eepp pointer passed in by the client must therefore point to data
3465e111ed8SAndrew Rybchenko 	 * shared with the client's event processing context.
3475e111ed8SAndrew Rybchenko 	 */
3485e111ed8SAndrew Rybchenko 	enp->en_ev_qcount++;
3495e111ed8SAndrew Rybchenko 	*eepp = eep;
3505e111ed8SAndrew Rybchenko 
3515e111ed8SAndrew Rybchenko 	if ((rc = eevop->eevo_qcreate(enp, index, esmp, ndescs, id, us, flags,
352*aa6dc101SAndrew Rybchenko 	    irq, eep)) != 0)
353f8a60f76SAndy Moreton 		goto fail9;
3545e111ed8SAndrew Rybchenko 
3555e111ed8SAndrew Rybchenko 	return (0);
3565e111ed8SAndrew Rybchenko 
357f8a60f76SAndy Moreton fail9:
358f8a60f76SAndy Moreton 	EFSYS_PROBE(fail9);
3595e111ed8SAndrew Rybchenko 
3605e111ed8SAndrew Rybchenko 	*eepp = NULL;
3615e111ed8SAndrew Rybchenko 	enp->en_ev_qcount--;
3625e111ed8SAndrew Rybchenko 	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
363f8a60f76SAndy Moreton fail8:
364f8a60f76SAndy Moreton 	EFSYS_PROBE(fail8);
365f8a60f76SAndy Moreton fail7:
366f8a60f76SAndy Moreton 	EFSYS_PROBE(fail7);
3672e5819a5SAndrew Rybchenko fail6:
3682e5819a5SAndrew Rybchenko 	EFSYS_PROBE(fail6);
3692e5819a5SAndrew Rybchenko fail5:
3702e5819a5SAndrew Rybchenko 	EFSYS_PROBE(fail5);
3715e111ed8SAndrew Rybchenko fail4:
3725e111ed8SAndrew Rybchenko 	EFSYS_PROBE(fail4);
3735e111ed8SAndrew Rybchenko fail3:
3745e111ed8SAndrew Rybchenko 	EFSYS_PROBE(fail3);
3755e111ed8SAndrew Rybchenko fail2:
3765e111ed8SAndrew Rybchenko 	EFSYS_PROBE(fail2);
3775e111ed8SAndrew Rybchenko fail1:
3785e111ed8SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
3795e111ed8SAndrew Rybchenko 	return (rc);
3805e111ed8SAndrew Rybchenko }
3815e111ed8SAndrew Rybchenko 
382*aa6dc101SAndrew Rybchenko 	__checkReturn	efx_rc_t
efx_ev_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in uint32_t us,__in uint32_t flags,__deref_out efx_evq_t ** eepp)383*aa6dc101SAndrew Rybchenko efx_ev_qcreate(
384*aa6dc101SAndrew Rybchenko 	__in		efx_nic_t *enp,
385*aa6dc101SAndrew Rybchenko 	__in		unsigned int index,
386*aa6dc101SAndrew Rybchenko 	__in		efsys_mem_t *esmp,
387*aa6dc101SAndrew Rybchenko 	__in		size_t ndescs,
388*aa6dc101SAndrew Rybchenko 	__in		uint32_t id,
389*aa6dc101SAndrew Rybchenko 	__in		uint32_t us,
390*aa6dc101SAndrew Rybchenko 	__in		uint32_t flags,
391*aa6dc101SAndrew Rybchenko 	__deref_out	efx_evq_t **eepp)
392*aa6dc101SAndrew Rybchenko {
393*aa6dc101SAndrew Rybchenko 	uint32_t irq = index;
394*aa6dc101SAndrew Rybchenko 
395*aa6dc101SAndrew Rybchenko 	return (efx_ev_qcreate_irq(enp, index, esmp, ndescs, id, us, flags,
396*aa6dc101SAndrew Rybchenko 	    irq, eepp));
397*aa6dc101SAndrew Rybchenko }
398*aa6dc101SAndrew Rybchenko 
3995e111ed8SAndrew Rybchenko 		void
efx_ev_qdestroy(__in efx_evq_t * eep)4005e111ed8SAndrew Rybchenko efx_ev_qdestroy(
4015e111ed8SAndrew Rybchenko 	__in	efx_evq_t *eep)
4025e111ed8SAndrew Rybchenko {
4035e111ed8SAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
4045e111ed8SAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
4055e111ed8SAndrew Rybchenko 
4065e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
4075e111ed8SAndrew Rybchenko 
4085e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(enp->en_ev_qcount != 0);
4095e111ed8SAndrew Rybchenko 	--enp->en_ev_qcount;
4105e111ed8SAndrew Rybchenko 
4115e111ed8SAndrew Rybchenko 	eevop->eevo_qdestroy(eep);
4125e111ed8SAndrew Rybchenko 
4135e111ed8SAndrew Rybchenko 	/* Free the EVQ object */
4145e111ed8SAndrew Rybchenko 	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
4155e111ed8SAndrew Rybchenko }
4165e111ed8SAndrew Rybchenko 
4175e111ed8SAndrew Rybchenko 	__checkReturn	efx_rc_t
efx_ev_qprime(__in efx_evq_t * eep,__in unsigned int count)4185e111ed8SAndrew Rybchenko efx_ev_qprime(
4195e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
4205e111ed8SAndrew Rybchenko 	__in		unsigned int count)
4215e111ed8SAndrew Rybchenko {
4225e111ed8SAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
4235e111ed8SAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
4245e111ed8SAndrew Rybchenko 	efx_rc_t rc;
4255e111ed8SAndrew Rybchenko 
4265e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
4275e111ed8SAndrew Rybchenko 
4285e111ed8SAndrew Rybchenko 	if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
4295e111ed8SAndrew Rybchenko 		rc = EINVAL;
4305e111ed8SAndrew Rybchenko 		goto fail1;
4315e111ed8SAndrew Rybchenko 	}
4325e111ed8SAndrew Rybchenko 
4335e111ed8SAndrew Rybchenko 	if ((rc = eevop->eevo_qprime(eep, count)) != 0)
4345e111ed8SAndrew Rybchenko 		goto fail2;
4355e111ed8SAndrew Rybchenko 
4365e111ed8SAndrew Rybchenko 	return (0);
4375e111ed8SAndrew Rybchenko 
4385e111ed8SAndrew Rybchenko fail2:
4395e111ed8SAndrew Rybchenko 	EFSYS_PROBE(fail2);
4405e111ed8SAndrew Rybchenko fail1:
4415e111ed8SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
4425e111ed8SAndrew Rybchenko 	return (rc);
4435e111ed8SAndrew Rybchenko }
4445e111ed8SAndrew Rybchenko 
4455e111ed8SAndrew Rybchenko 	__checkReturn	boolean_t
efx_ev_qpending(__in efx_evq_t * eep,__in unsigned int count)4465e111ed8SAndrew Rybchenko efx_ev_qpending(
4475e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
4485e111ed8SAndrew Rybchenko 	__in		unsigned int count)
4495e111ed8SAndrew Rybchenko {
4505e111ed8SAndrew Rybchenko 	size_t offset;
4515e111ed8SAndrew Rybchenko 	efx_qword_t qword;
4525e111ed8SAndrew Rybchenko 
4535e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
4545e111ed8SAndrew Rybchenko 
4555e111ed8SAndrew Rybchenko 	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
4565e111ed8SAndrew Rybchenko 	EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
4575e111ed8SAndrew Rybchenko 
4585e111ed8SAndrew Rybchenko 	return (EFX_EV_PRESENT(qword));
4595e111ed8SAndrew Rybchenko }
4605e111ed8SAndrew Rybchenko 
4615e111ed8SAndrew Rybchenko #if EFSYS_OPT_EV_PREFETCH
4625e111ed8SAndrew Rybchenko 
4635e111ed8SAndrew Rybchenko 			void
efx_ev_qprefetch(__in efx_evq_t * eep,__in unsigned int count)4645e111ed8SAndrew Rybchenko efx_ev_qprefetch(
4655e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
4665e111ed8SAndrew Rybchenko 	__in		unsigned int count)
4675e111ed8SAndrew Rybchenko {
4685e111ed8SAndrew Rybchenko 	unsigned int offset;
4695e111ed8SAndrew Rybchenko 
4705e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
4715e111ed8SAndrew Rybchenko 
4725e111ed8SAndrew Rybchenko 	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
4735e111ed8SAndrew Rybchenko 	EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
4745e111ed8SAndrew Rybchenko }
4755e111ed8SAndrew Rybchenko 
4765e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_EV_PREFETCH */
4775e111ed8SAndrew Rybchenko 
47892fedcd3SIvan Malov /*
47992fedcd3SIvan Malov  * This method is needed to ensure that eec_initialized callback
48092fedcd3SIvan Malov  * is invoked after queue creation. The callback will be invoked
48192fedcd3SIvan Malov  * on Riverhead boards which have no support for INIT_DONE events
48292fedcd3SIvan Malov  * and will do nothing on other boards.
48392fedcd3SIvan Malov  *
48492fedcd3SIvan Malov  * The client drivers must call this method after calling efx_ev_create().
48592fedcd3SIvan Malov  * The call must be done with the same locks being held (if any) which are
48692fedcd3SIvan Malov  * normally acquired around efx_ev_qpoll() calls to ensure that
48792fedcd3SIvan Malov  * eec_initialized callback is invoked within the same locking context.
48892fedcd3SIvan Malov  */
48992fedcd3SIvan Malov 			void
efx_ev_qcreate_check_init_done(__in efx_evq_t * eep,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)49092fedcd3SIvan Malov efx_ev_qcreate_check_init_done(
49192fedcd3SIvan Malov 	__in		efx_evq_t *eep,
49292fedcd3SIvan Malov 	__in		const efx_ev_callbacks_t *eecp,
49392fedcd3SIvan Malov 	__in_opt	void *arg)
49492fedcd3SIvan Malov {
49592fedcd3SIvan Malov 	const efx_nic_cfg_t *encp;
49692fedcd3SIvan Malov 
49792fedcd3SIvan Malov 	EFSYS_ASSERT(eep != NULL);
49892fedcd3SIvan Malov 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
49992fedcd3SIvan Malov 	EFSYS_ASSERT(eecp != NULL);
50092fedcd3SIvan Malov 	EFSYS_ASSERT(eecp->eec_initialized != NULL);
50192fedcd3SIvan Malov 
50292fedcd3SIvan Malov 	encp = efx_nic_cfg_get(eep->ee_enp);
50392fedcd3SIvan Malov 
50492fedcd3SIvan Malov 	if (encp->enc_evq_init_done_ev_supported == B_FALSE)
50592fedcd3SIvan Malov 		(void) eecp->eec_initialized(arg);
50692fedcd3SIvan Malov }
50792fedcd3SIvan Malov 
5085e111ed8SAndrew Rybchenko 			void
efx_ev_qpoll(__in efx_evq_t * eep,__inout unsigned int * countp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)5095e111ed8SAndrew Rybchenko efx_ev_qpoll(
5105e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
5115e111ed8SAndrew Rybchenko 	__inout		unsigned int *countp,
5125e111ed8SAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
5135e111ed8SAndrew Rybchenko 	__in_opt	void *arg)
5145e111ed8SAndrew Rybchenko {
515ad1e3ed8SAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
516ad1e3ed8SAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
5175e111ed8SAndrew Rybchenko 
5185e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
5195e111ed8SAndrew Rybchenko 
520ad1e3ed8SAndrew Rybchenko 	EFSYS_ASSERT(eevop != NULL &&
521ad1e3ed8SAndrew Rybchenko 	    eevop->eevo_qpoll != NULL);
5225e111ed8SAndrew Rybchenko 
523ad1e3ed8SAndrew Rybchenko 	eevop->eevo_qpoll(eep, countp, eecp, arg);
5245e111ed8SAndrew Rybchenko }
5255e111ed8SAndrew Rybchenko 
5265e111ed8SAndrew Rybchenko 			void
efx_ev_qpost(__in efx_evq_t * eep,__in uint16_t data)5275e111ed8SAndrew Rybchenko efx_ev_qpost(
5285e111ed8SAndrew Rybchenko 	__in	efx_evq_t *eep,
5295e111ed8SAndrew Rybchenko 	__in	uint16_t data)
5305e111ed8SAndrew Rybchenko {
5315e111ed8SAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
5325e111ed8SAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
5335e111ed8SAndrew Rybchenko 
5345e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
5355e111ed8SAndrew Rybchenko 
5365e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(eevop != NULL &&
5375e111ed8SAndrew Rybchenko 	    eevop->eevo_qpost != NULL);
5385e111ed8SAndrew Rybchenko 
5395e111ed8SAndrew Rybchenko 	eevop->eevo_qpost(eep, data);
5405e111ed8SAndrew Rybchenko }
5415e111ed8SAndrew Rybchenko 
5425e111ed8SAndrew Rybchenko 	__checkReturn	efx_rc_t
efx_ev_usecs_to_ticks(__in efx_nic_t * enp,__in unsigned int us,__out unsigned int * ticksp)5435e111ed8SAndrew Rybchenko efx_ev_usecs_to_ticks(
5445e111ed8SAndrew Rybchenko 	__in		efx_nic_t *enp,
5455e111ed8SAndrew Rybchenko 	__in		unsigned int us,
5465e111ed8SAndrew Rybchenko 	__out		unsigned int *ticksp)
5475e111ed8SAndrew Rybchenko {
5485e111ed8SAndrew Rybchenko 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
5495e111ed8SAndrew Rybchenko 	unsigned int ticks;
5505e111ed8SAndrew Rybchenko 	efx_rc_t rc;
5515e111ed8SAndrew Rybchenko 
5525e111ed8SAndrew Rybchenko 	if (encp->enc_evq_timer_quantum_ns == 0) {
5535e111ed8SAndrew Rybchenko 		rc = ENOTSUP;
5545e111ed8SAndrew Rybchenko 		goto fail1;
5555e111ed8SAndrew Rybchenko 	}
5565e111ed8SAndrew Rybchenko 
5575e111ed8SAndrew Rybchenko 	/* Convert microseconds to a timer tick count */
5585e111ed8SAndrew Rybchenko 	if (us == 0)
5595e111ed8SAndrew Rybchenko 		ticks = 0;
5605e111ed8SAndrew Rybchenko 	else if (us * 1000 < encp->enc_evq_timer_quantum_ns)
5615e111ed8SAndrew Rybchenko 		ticks = 1;	/* Never round down to zero */
5625e111ed8SAndrew Rybchenko 	else
5635e111ed8SAndrew Rybchenko 		ticks = us * 1000 / encp->enc_evq_timer_quantum_ns;
5645e111ed8SAndrew Rybchenko 
5655e111ed8SAndrew Rybchenko 	*ticksp = ticks;
5665e111ed8SAndrew Rybchenko 	return (0);
5675e111ed8SAndrew Rybchenko 
5685e111ed8SAndrew Rybchenko fail1:
5695e111ed8SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
5705e111ed8SAndrew Rybchenko 	return (rc);
5715e111ed8SAndrew Rybchenko }
5725e111ed8SAndrew Rybchenko 
5735e111ed8SAndrew Rybchenko 	__checkReturn	efx_rc_t
efx_ev_qmoderate(__in efx_evq_t * eep,__in unsigned int us)5745e111ed8SAndrew Rybchenko efx_ev_qmoderate(
5755e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
5765e111ed8SAndrew Rybchenko 	__in		unsigned int us)
5775e111ed8SAndrew Rybchenko {
5785e111ed8SAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
5795e111ed8SAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
5805e111ed8SAndrew Rybchenko 	efx_rc_t rc;
5815e111ed8SAndrew Rybchenko 
5825e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
5835e111ed8SAndrew Rybchenko 
5845e111ed8SAndrew Rybchenko 	if ((eep->ee_flags & EFX_EVQ_FLAGS_NOTIFY_MASK) ==
5855e111ed8SAndrew Rybchenko 	    EFX_EVQ_FLAGS_NOTIFY_DISABLED) {
5865e111ed8SAndrew Rybchenko 		rc = EINVAL;
5875e111ed8SAndrew Rybchenko 		goto fail1;
5885e111ed8SAndrew Rybchenko 	}
5895e111ed8SAndrew Rybchenko 
5905e111ed8SAndrew Rybchenko 	if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
5915e111ed8SAndrew Rybchenko 		goto fail2;
5925e111ed8SAndrew Rybchenko 
5935e111ed8SAndrew Rybchenko 	return (0);
5945e111ed8SAndrew Rybchenko 
5955e111ed8SAndrew Rybchenko fail2:
5965e111ed8SAndrew Rybchenko 	EFSYS_PROBE(fail2);
5975e111ed8SAndrew Rybchenko fail1:
5985e111ed8SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
5995e111ed8SAndrew Rybchenko 	return (rc);
6005e111ed8SAndrew Rybchenko }
6015e111ed8SAndrew Rybchenko 
6025e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
6035e111ed8SAndrew Rybchenko 					void
efx_ev_qstats_update(__in efx_evq_t * eep,__inout_ecount (EV_NQSTATS)efsys_stat_t * stat)6045e111ed8SAndrew Rybchenko efx_ev_qstats_update(
6055e111ed8SAndrew Rybchenko 	__in				efx_evq_t *eep,
6065e111ed8SAndrew Rybchenko 	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat)
6075e111ed8SAndrew Rybchenko 
6085e111ed8SAndrew Rybchenko {	efx_nic_t *enp = eep->ee_enp;
6095e111ed8SAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
6105e111ed8SAndrew Rybchenko 
6115e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
6125e111ed8SAndrew Rybchenko 
6135e111ed8SAndrew Rybchenko 	eevop->eevo_qstats_update(eep, stat);
6145e111ed8SAndrew Rybchenko }
6155e111ed8SAndrew Rybchenko 
6165e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_QSTATS */
6175e111ed8SAndrew Rybchenko 
6185e111ed8SAndrew Rybchenko #if EFSYS_OPT_SIENA
6195e111ed8SAndrew Rybchenko 
6205e111ed8SAndrew Rybchenko static	__checkReturn	efx_rc_t
siena_ev_init(__in efx_nic_t * enp)6215e111ed8SAndrew Rybchenko siena_ev_init(
6225e111ed8SAndrew Rybchenko 	__in		efx_nic_t *enp)
6235e111ed8SAndrew Rybchenko {
6245e111ed8SAndrew Rybchenko 	efx_oword_t oword;
6255e111ed8SAndrew Rybchenko 
6265e111ed8SAndrew Rybchenko 	/*
6275e111ed8SAndrew Rybchenko 	 * Program the event queue for receive and transmit queue
6285e111ed8SAndrew Rybchenko 	 * flush events.
6295e111ed8SAndrew Rybchenko 	 */
6305e111ed8SAndrew Rybchenko 	EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
6315e111ed8SAndrew Rybchenko 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
6325e111ed8SAndrew Rybchenko 	EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
6335e111ed8SAndrew Rybchenko 
6345e111ed8SAndrew Rybchenko 	return (0);
6355e111ed8SAndrew Rybchenko 
6365e111ed8SAndrew Rybchenko }
6375e111ed8SAndrew Rybchenko 
6385e111ed8SAndrew Rybchenko static  __checkReturn   boolean_t
siena_ev_rx_not_ok(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in uint32_t label,__in uint32_t id,__inout uint16_t * flagsp)6395e111ed8SAndrew Rybchenko siena_ev_rx_not_ok(
6405e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
6415e111ed8SAndrew Rybchenko 	__in		efx_qword_t *eqp,
6425e111ed8SAndrew Rybchenko 	__in		uint32_t label,
6435e111ed8SAndrew Rybchenko 	__in		uint32_t id,
6445e111ed8SAndrew Rybchenko 	__inout		uint16_t *flagsp)
6455e111ed8SAndrew Rybchenko {
6465e111ed8SAndrew Rybchenko 	boolean_t ignore = B_FALSE;
6475e111ed8SAndrew Rybchenko 
6485e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
6495e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
6505e111ed8SAndrew Rybchenko 		EFSYS_PROBE(tobe_disc);
6515e111ed8SAndrew Rybchenko 		/*
6525e111ed8SAndrew Rybchenko 		 * Assume this is a unicast address mismatch, unless below
6535e111ed8SAndrew Rybchenko 		 * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
6545e111ed8SAndrew Rybchenko 		 * EV_RX_PAUSE_FRM_ERR is set.
6555e111ed8SAndrew Rybchenko 		 */
6565e111ed8SAndrew Rybchenko 		(*flagsp) |= EFX_ADDR_MISMATCH;
6575e111ed8SAndrew Rybchenko 	}
6585e111ed8SAndrew Rybchenko 
6595e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
6605e111ed8SAndrew Rybchenko 		EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
6615e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
6625e111ed8SAndrew Rybchenko 		(*flagsp) |= EFX_DISCARD;
6635e111ed8SAndrew Rybchenko 
6645e111ed8SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
6655e111ed8SAndrew Rybchenko 		/*
6665e111ed8SAndrew Rybchenko 		 * Lookout for payload queue ran dry errors and ignore them.
6675e111ed8SAndrew Rybchenko 		 *
6685e111ed8SAndrew Rybchenko 		 * Sadly for the header/data split cases, the descriptor
6695e111ed8SAndrew Rybchenko 		 * pointer in this event refers to the header queue and
6705e111ed8SAndrew Rybchenko 		 * therefore cannot be easily detected as duplicate.
6715e111ed8SAndrew Rybchenko 		 * So we drop these and rely on the receive processing seeing
6725e111ed8SAndrew Rybchenko 		 * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
6735e111ed8SAndrew Rybchenko 		 * the partially received packet.
6745e111ed8SAndrew Rybchenko 		 */
6755e111ed8SAndrew Rybchenko 		if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
6765e111ed8SAndrew Rybchenko 		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
6775e111ed8SAndrew Rybchenko 		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
6785e111ed8SAndrew Rybchenko 			ignore = B_TRUE;
6795e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_RX_SCATTER */
6805e111ed8SAndrew Rybchenko 	}
6815e111ed8SAndrew Rybchenko 
6825e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
6835e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
6845e111ed8SAndrew Rybchenko 		EFSYS_PROBE(crc_err);
6855e111ed8SAndrew Rybchenko 		(*flagsp) &= ~EFX_ADDR_MISMATCH;
6865e111ed8SAndrew Rybchenko 		(*flagsp) |= EFX_DISCARD;
6875e111ed8SAndrew Rybchenko 	}
6885e111ed8SAndrew Rybchenko 
6895e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
6905e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
6915e111ed8SAndrew Rybchenko 		EFSYS_PROBE(pause_frm_err);
6925e111ed8SAndrew Rybchenko 		(*flagsp) &= ~EFX_ADDR_MISMATCH;
6935e111ed8SAndrew Rybchenko 		(*flagsp) |= EFX_DISCARD;
6945e111ed8SAndrew Rybchenko 	}
6955e111ed8SAndrew Rybchenko 
6965e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
6975e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
6985e111ed8SAndrew Rybchenko 		EFSYS_PROBE(owner_id_err);
6995e111ed8SAndrew Rybchenko 		(*flagsp) |= EFX_DISCARD;
7005e111ed8SAndrew Rybchenko 	}
7015e111ed8SAndrew Rybchenko 
7025e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
7035e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
7045e111ed8SAndrew Rybchenko 		EFSYS_PROBE(ipv4_err);
7055e111ed8SAndrew Rybchenko 		(*flagsp) &= ~EFX_CKSUM_IPV4;
7065e111ed8SAndrew Rybchenko 	}
7075e111ed8SAndrew Rybchenko 
7085e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
7095e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
7105e111ed8SAndrew Rybchenko 		EFSYS_PROBE(udp_chk_err);
7115e111ed8SAndrew Rybchenko 		(*flagsp) &= ~EFX_CKSUM_TCPUDP;
7125e111ed8SAndrew Rybchenko 	}
7135e111ed8SAndrew Rybchenko 
7145e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
7155e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
7165e111ed8SAndrew Rybchenko 
7175e111ed8SAndrew Rybchenko 		/*
7185e111ed8SAndrew Rybchenko 		 * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
7195e111ed8SAndrew Rybchenko 		 * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
7205e111ed8SAndrew Rybchenko 		 * condition.
7215e111ed8SAndrew Rybchenko 		 */
7225e111ed8SAndrew Rybchenko 		(*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
7235e111ed8SAndrew Rybchenko 	}
7245e111ed8SAndrew Rybchenko 
7255e111ed8SAndrew Rybchenko 	return (ignore);
7265e111ed8SAndrew Rybchenko }
7275e111ed8SAndrew Rybchenko 
7285e111ed8SAndrew Rybchenko static	__checkReturn	boolean_t
siena_ev_rx(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)7295e111ed8SAndrew Rybchenko siena_ev_rx(
7305e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
7315e111ed8SAndrew Rybchenko 	__in		efx_qword_t *eqp,
7325e111ed8SAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
7335e111ed8SAndrew Rybchenko 	__in_opt	void *arg)
7345e111ed8SAndrew Rybchenko {
7355e111ed8SAndrew Rybchenko 	uint32_t id;
7365e111ed8SAndrew Rybchenko 	uint32_t size;
7375e111ed8SAndrew Rybchenko 	uint32_t label;
7385e111ed8SAndrew Rybchenko 	boolean_t ok;
7395e111ed8SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
7405e111ed8SAndrew Rybchenko 	boolean_t sop;
7415e111ed8SAndrew Rybchenko 	boolean_t jumbo_cont;
7425e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_RX_SCATTER */
7435e111ed8SAndrew Rybchenko 	uint32_t hdr_type;
7445e111ed8SAndrew Rybchenko 	boolean_t is_v6;
7455e111ed8SAndrew Rybchenko 	uint16_t flags;
7465e111ed8SAndrew Rybchenko 	boolean_t ignore;
7475e111ed8SAndrew Rybchenko 	boolean_t should_abort;
7485e111ed8SAndrew Rybchenko 
7495e111ed8SAndrew Rybchenko 	EFX_EV_QSTAT_INCR(eep, EV_RX);
7505e111ed8SAndrew Rybchenko 
7515e111ed8SAndrew Rybchenko 	/* Basic packet information */
7525e111ed8SAndrew Rybchenko 	id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
7535e111ed8SAndrew Rybchenko 	size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
7545e111ed8SAndrew Rybchenko 	label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
7555e111ed8SAndrew Rybchenko 	ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
7565e111ed8SAndrew Rybchenko 
7575e111ed8SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
7585e111ed8SAndrew Rybchenko 	sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
7595e111ed8SAndrew Rybchenko 	jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
7605e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_RX_SCATTER */
7615e111ed8SAndrew Rybchenko 
7625e111ed8SAndrew Rybchenko 	hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
7635e111ed8SAndrew Rybchenko 
7645e111ed8SAndrew Rybchenko 	is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
7655e111ed8SAndrew Rybchenko 
7665e111ed8SAndrew Rybchenko 	/*
7675e111ed8SAndrew Rybchenko 	 * If packet is marked as OK and packet type is TCP/IP or
7685e111ed8SAndrew Rybchenko 	 * UDP/IP or other IP, then we can rely on the hardware checksums.
7695e111ed8SAndrew Rybchenko 	 */
7705e111ed8SAndrew Rybchenko 	switch (hdr_type) {
7715e111ed8SAndrew Rybchenko 	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
7725e111ed8SAndrew Rybchenko 		flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
7735e111ed8SAndrew Rybchenko 		if (is_v6) {
7745e111ed8SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
7755e111ed8SAndrew Rybchenko 			flags |= EFX_PKT_IPV6;
7765e111ed8SAndrew Rybchenko 		} else {
7775e111ed8SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
7785e111ed8SAndrew Rybchenko 			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
7795e111ed8SAndrew Rybchenko 		}
7805e111ed8SAndrew Rybchenko 		break;
7815e111ed8SAndrew Rybchenko 
7825e111ed8SAndrew Rybchenko 	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
7835e111ed8SAndrew Rybchenko 		flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
7845e111ed8SAndrew Rybchenko 		if (is_v6) {
7855e111ed8SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
7865e111ed8SAndrew Rybchenko 			flags |= EFX_PKT_IPV6;
7875e111ed8SAndrew Rybchenko 		} else {
7885e111ed8SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
7895e111ed8SAndrew Rybchenko 			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
7905e111ed8SAndrew Rybchenko 		}
7915e111ed8SAndrew Rybchenko 		break;
7925e111ed8SAndrew Rybchenko 
7935e111ed8SAndrew Rybchenko 	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
7945e111ed8SAndrew Rybchenko 		if (is_v6) {
7955e111ed8SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
7965e111ed8SAndrew Rybchenko 			flags = EFX_PKT_IPV6;
7975e111ed8SAndrew Rybchenko 		} else {
7985e111ed8SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
7995e111ed8SAndrew Rybchenko 			flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
8005e111ed8SAndrew Rybchenko 		}
8015e111ed8SAndrew Rybchenko 		break;
8025e111ed8SAndrew Rybchenko 
8035e111ed8SAndrew Rybchenko 	case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
8045e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
8055e111ed8SAndrew Rybchenko 		flags = 0;
8065e111ed8SAndrew Rybchenko 		break;
8075e111ed8SAndrew Rybchenko 
8085e111ed8SAndrew Rybchenko 	default:
8095e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(B_FALSE);
8105e111ed8SAndrew Rybchenko 		flags = 0;
8115e111ed8SAndrew Rybchenko 		break;
8125e111ed8SAndrew Rybchenko 	}
8135e111ed8SAndrew Rybchenko 
8145e111ed8SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
8155e111ed8SAndrew Rybchenko 	/* Report scatter and header/lookahead split buffer flags */
8165e111ed8SAndrew Rybchenko 	if (sop)
8175e111ed8SAndrew Rybchenko 		flags |= EFX_PKT_START;
8185e111ed8SAndrew Rybchenko 	if (jumbo_cont)
8195e111ed8SAndrew Rybchenko 		flags |= EFX_PKT_CONT;
8205e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_RX_SCATTER */
8215e111ed8SAndrew Rybchenko 
8225e111ed8SAndrew Rybchenko 	/* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
8235e111ed8SAndrew Rybchenko 	if (!ok) {
8245e111ed8SAndrew Rybchenko 		ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags);
8255e111ed8SAndrew Rybchenko 		if (ignore) {
8265e111ed8SAndrew Rybchenko 			EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
8275e111ed8SAndrew Rybchenko 			    uint32_t, size, uint16_t, flags);
8285e111ed8SAndrew Rybchenko 
8295e111ed8SAndrew Rybchenko 			return (B_FALSE);
8305e111ed8SAndrew Rybchenko 		}
8315e111ed8SAndrew Rybchenko 	}
8325e111ed8SAndrew Rybchenko 
8335e111ed8SAndrew Rybchenko 	/* If we're not discarding the packet then it is ok */
8345e111ed8SAndrew Rybchenko 	if (~flags & EFX_DISCARD)
8355e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
8365e111ed8SAndrew Rybchenko 
8375e111ed8SAndrew Rybchenko 	/* Detect multicast packets that didn't match the filter */
8385e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
8395e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
8405e111ed8SAndrew Rybchenko 
8415e111ed8SAndrew Rybchenko 		if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
8425e111ed8SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
8435e111ed8SAndrew Rybchenko 		} else {
8445e111ed8SAndrew Rybchenko 			EFSYS_PROBE(mcast_mismatch);
8455e111ed8SAndrew Rybchenko 			flags |= EFX_ADDR_MISMATCH;
8465e111ed8SAndrew Rybchenko 		}
8475e111ed8SAndrew Rybchenko 	} else {
8485e111ed8SAndrew Rybchenko 		flags |= EFX_PKT_UNICAST;
8495e111ed8SAndrew Rybchenko 	}
8505e111ed8SAndrew Rybchenko 
8515e111ed8SAndrew Rybchenko 	/*
8525e111ed8SAndrew Rybchenko 	 * The packet parser in Siena can abort parsing packets under
8535e111ed8SAndrew Rybchenko 	 * certain error conditions, setting the PKT_NOT_PARSED bit
8545e111ed8SAndrew Rybchenko 	 * (which clears PKT_OK). If this is set, then don't trust
8555e111ed8SAndrew Rybchenko 	 * the PKT_TYPE field.
8565e111ed8SAndrew Rybchenko 	 */
8575e111ed8SAndrew Rybchenko 	if (!ok) {
8585e111ed8SAndrew Rybchenko 		uint32_t parse_err;
8595e111ed8SAndrew Rybchenko 
8605e111ed8SAndrew Rybchenko 		parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
8615e111ed8SAndrew Rybchenko 		if (parse_err != 0)
8625e111ed8SAndrew Rybchenko 			flags |= EFX_CHECK_VLAN;
8635e111ed8SAndrew Rybchenko 	}
8645e111ed8SAndrew Rybchenko 
8655e111ed8SAndrew Rybchenko 	if (~flags & EFX_CHECK_VLAN) {
8665e111ed8SAndrew Rybchenko 		uint32_t pkt_type;
8675e111ed8SAndrew Rybchenko 
8685e111ed8SAndrew Rybchenko 		pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
8695e111ed8SAndrew Rybchenko 		if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
8705e111ed8SAndrew Rybchenko 			flags |= EFX_PKT_VLAN_TAGGED;
8715e111ed8SAndrew Rybchenko 	}
8725e111ed8SAndrew Rybchenko 
8735e111ed8SAndrew Rybchenko 	EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
8745e111ed8SAndrew Rybchenko 	    uint32_t, size, uint16_t, flags);
8755e111ed8SAndrew Rybchenko 
8765e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_rx != NULL);
8775e111ed8SAndrew Rybchenko 	should_abort = eecp->eec_rx(arg, label, id, size, flags);
8785e111ed8SAndrew Rybchenko 
8795e111ed8SAndrew Rybchenko 	return (should_abort);
8805e111ed8SAndrew Rybchenko }
8815e111ed8SAndrew Rybchenko 
8825e111ed8SAndrew Rybchenko static	__checkReturn	boolean_t
siena_ev_tx(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)8835e111ed8SAndrew Rybchenko siena_ev_tx(
8845e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
8855e111ed8SAndrew Rybchenko 	__in		efx_qword_t *eqp,
8865e111ed8SAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
8875e111ed8SAndrew Rybchenko 	__in_opt	void *arg)
8885e111ed8SAndrew Rybchenko {
8895e111ed8SAndrew Rybchenko 	uint32_t id;
8905e111ed8SAndrew Rybchenko 	uint32_t label;
8915e111ed8SAndrew Rybchenko 	boolean_t should_abort;
8925e111ed8SAndrew Rybchenko 
8935e111ed8SAndrew Rybchenko 	EFX_EV_QSTAT_INCR(eep, EV_TX);
8945e111ed8SAndrew Rybchenko 
8955e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
8965e111ed8SAndrew Rybchenko 	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
8975e111ed8SAndrew Rybchenko 	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
8985e111ed8SAndrew Rybchenko 	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
8995e111ed8SAndrew Rybchenko 
9005e111ed8SAndrew Rybchenko 		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
9015e111ed8SAndrew Rybchenko 		label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
9025e111ed8SAndrew Rybchenko 
9035e111ed8SAndrew Rybchenko 		EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
9045e111ed8SAndrew Rybchenko 
9055e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(eecp->eec_tx != NULL);
9065e111ed8SAndrew Rybchenko 		should_abort = eecp->eec_tx(arg, label, id);
9075e111ed8SAndrew Rybchenko 
9085e111ed8SAndrew Rybchenko 		return (should_abort);
9095e111ed8SAndrew Rybchenko 	}
9105e111ed8SAndrew Rybchenko 
9115e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
9125e111ed8SAndrew Rybchenko 		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
9135e111ed8SAndrew Rybchenko 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
9145e111ed8SAndrew Rybchenko 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
9155e111ed8SAndrew Rybchenko 
9165e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
9175e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
9185e111ed8SAndrew Rybchenko 
9195e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
9205e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
9215e111ed8SAndrew Rybchenko 
9225e111ed8SAndrew Rybchenko 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
9235e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
9245e111ed8SAndrew Rybchenko 
9255e111ed8SAndrew Rybchenko 	EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
9265e111ed8SAndrew Rybchenko 	return (B_FALSE);
9275e111ed8SAndrew Rybchenko }
9285e111ed8SAndrew Rybchenko 
9295e111ed8SAndrew Rybchenko static	__checkReturn	boolean_t
siena_ev_global(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)9305e111ed8SAndrew Rybchenko siena_ev_global(
9315e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
9325e111ed8SAndrew Rybchenko 	__in		efx_qword_t *eqp,
9335e111ed8SAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
9345e111ed8SAndrew Rybchenko 	__in_opt	void *arg)
9355e111ed8SAndrew Rybchenko {
9365e111ed8SAndrew Rybchenko 	_NOTE(ARGUNUSED(eqp, eecp, arg))
9375e111ed8SAndrew Rybchenko 
9385e111ed8SAndrew Rybchenko 	EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
9395e111ed8SAndrew Rybchenko 
9405e111ed8SAndrew Rybchenko 	return (B_FALSE);
9415e111ed8SAndrew Rybchenko }
9425e111ed8SAndrew Rybchenko 
9435e111ed8SAndrew Rybchenko static	__checkReturn	boolean_t
siena_ev_driver(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)9445e111ed8SAndrew Rybchenko siena_ev_driver(
9455e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
9465e111ed8SAndrew Rybchenko 	__in		efx_qword_t *eqp,
9475e111ed8SAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
9485e111ed8SAndrew Rybchenko 	__in_opt	void *arg)
9495e111ed8SAndrew Rybchenko {
9505e111ed8SAndrew Rybchenko 	boolean_t should_abort;
9515e111ed8SAndrew Rybchenko 
9525e111ed8SAndrew Rybchenko 	EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
9535e111ed8SAndrew Rybchenko 	should_abort = B_FALSE;
9545e111ed8SAndrew Rybchenko 
9555e111ed8SAndrew Rybchenko 	switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
9565e111ed8SAndrew Rybchenko 	case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
9575e111ed8SAndrew Rybchenko 		uint32_t txq_index;
9585e111ed8SAndrew Rybchenko 
9595e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
9605e111ed8SAndrew Rybchenko 
9615e111ed8SAndrew Rybchenko 		txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
9625e111ed8SAndrew Rybchenko 
9635e111ed8SAndrew Rybchenko 		EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
9645e111ed8SAndrew Rybchenko 
9655e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
9665e111ed8SAndrew Rybchenko 		should_abort = eecp->eec_txq_flush_done(arg, txq_index);
9675e111ed8SAndrew Rybchenko 
9685e111ed8SAndrew Rybchenko 		break;
9695e111ed8SAndrew Rybchenko 	}
9705e111ed8SAndrew Rybchenko 	case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
9715e111ed8SAndrew Rybchenko 		uint32_t rxq_index;
9725e111ed8SAndrew Rybchenko 		uint32_t failed;
9735e111ed8SAndrew Rybchenko 
9745e111ed8SAndrew Rybchenko 		rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
9755e111ed8SAndrew Rybchenko 		failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
9765e111ed8SAndrew Rybchenko 
9775e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
9785e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
9795e111ed8SAndrew Rybchenko 
9805e111ed8SAndrew Rybchenko 		if (failed) {
9815e111ed8SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
9825e111ed8SAndrew Rybchenko 
9835e111ed8SAndrew Rybchenko 			EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
9845e111ed8SAndrew Rybchenko 
9855e111ed8SAndrew Rybchenko 			should_abort = eecp->eec_rxq_flush_failed(arg,
9865e111ed8SAndrew Rybchenko 								    rxq_index);
9875e111ed8SAndrew Rybchenko 		} else {
9885e111ed8SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
9895e111ed8SAndrew Rybchenko 
9905e111ed8SAndrew Rybchenko 			EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
9915e111ed8SAndrew Rybchenko 
9925e111ed8SAndrew Rybchenko 			should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
9935e111ed8SAndrew Rybchenko 		}
9945e111ed8SAndrew Rybchenko 
9955e111ed8SAndrew Rybchenko 		break;
9965e111ed8SAndrew Rybchenko 	}
9975e111ed8SAndrew Rybchenko 	case FSE_AZ_EVQ_INIT_DONE_EV:
9985e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(eecp->eec_initialized != NULL);
9995e111ed8SAndrew Rybchenko 		should_abort = eecp->eec_initialized(arg);
10005e111ed8SAndrew Rybchenko 
10015e111ed8SAndrew Rybchenko 		break;
10025e111ed8SAndrew Rybchenko 
10035e111ed8SAndrew Rybchenko 	case FSE_AZ_EVQ_NOT_EN_EV:
10045e111ed8SAndrew Rybchenko 		EFSYS_PROBE(evq_not_en);
10055e111ed8SAndrew Rybchenko 		break;
10065e111ed8SAndrew Rybchenko 
10075e111ed8SAndrew Rybchenko 	case FSE_AZ_SRM_UPD_DONE_EV: {
10085e111ed8SAndrew Rybchenko 		uint32_t code;
10095e111ed8SAndrew Rybchenko 
10105e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
10115e111ed8SAndrew Rybchenko 
10125e111ed8SAndrew Rybchenko 		code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
10135e111ed8SAndrew Rybchenko 
10145e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(eecp->eec_sram != NULL);
10155e111ed8SAndrew Rybchenko 		should_abort = eecp->eec_sram(arg, code);
10165e111ed8SAndrew Rybchenko 
10175e111ed8SAndrew Rybchenko 		break;
10185e111ed8SAndrew Rybchenko 	}
10195e111ed8SAndrew Rybchenko 	case FSE_AZ_WAKE_UP_EV: {
10205e111ed8SAndrew Rybchenko 		uint32_t id;
10215e111ed8SAndrew Rybchenko 
10225e111ed8SAndrew Rybchenko 		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
10235e111ed8SAndrew Rybchenko 
10245e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(eecp->eec_wake_up != NULL);
10255e111ed8SAndrew Rybchenko 		should_abort = eecp->eec_wake_up(arg, id);
10265e111ed8SAndrew Rybchenko 
10275e111ed8SAndrew Rybchenko 		break;
10285e111ed8SAndrew Rybchenko 	}
10295e111ed8SAndrew Rybchenko 	case FSE_AZ_TX_PKT_NON_TCP_UDP:
10305e111ed8SAndrew Rybchenko 		EFSYS_PROBE(tx_pkt_non_tcp_udp);
10315e111ed8SAndrew Rybchenko 		break;
10325e111ed8SAndrew Rybchenko 
10335e111ed8SAndrew Rybchenko 	case FSE_AZ_TIMER_EV: {
10345e111ed8SAndrew Rybchenko 		uint32_t id;
10355e111ed8SAndrew Rybchenko 
10365e111ed8SAndrew Rybchenko 		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
10375e111ed8SAndrew Rybchenko 
10385e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(eecp->eec_timer != NULL);
10395e111ed8SAndrew Rybchenko 		should_abort = eecp->eec_timer(arg, id);
10405e111ed8SAndrew Rybchenko 
10415e111ed8SAndrew Rybchenko 		break;
10425e111ed8SAndrew Rybchenko 	}
10435e111ed8SAndrew Rybchenko 	case FSE_AZ_RX_DSC_ERROR_EV:
10445e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
10455e111ed8SAndrew Rybchenko 
10465e111ed8SAndrew Rybchenko 		EFSYS_PROBE(rx_dsc_error);
10475e111ed8SAndrew Rybchenko 
10485e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(eecp->eec_exception != NULL);
10495e111ed8SAndrew Rybchenko 		should_abort = eecp->eec_exception(arg,
10505e111ed8SAndrew Rybchenko 			EFX_EXCEPTION_RX_DSC_ERROR, 0);
10515e111ed8SAndrew Rybchenko 
10525e111ed8SAndrew Rybchenko 		break;
10535e111ed8SAndrew Rybchenko 
10545e111ed8SAndrew Rybchenko 	case FSE_AZ_TX_DSC_ERROR_EV:
10555e111ed8SAndrew Rybchenko 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
10565e111ed8SAndrew Rybchenko 
10575e111ed8SAndrew Rybchenko 		EFSYS_PROBE(tx_dsc_error);
10585e111ed8SAndrew Rybchenko 
10595e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(eecp->eec_exception != NULL);
10605e111ed8SAndrew Rybchenko 		should_abort = eecp->eec_exception(arg,
10615e111ed8SAndrew Rybchenko 			EFX_EXCEPTION_TX_DSC_ERROR, 0);
10625e111ed8SAndrew Rybchenko 
10635e111ed8SAndrew Rybchenko 		break;
10645e111ed8SAndrew Rybchenko 
10655e111ed8SAndrew Rybchenko 	default:
10665e111ed8SAndrew Rybchenko 		break;
10675e111ed8SAndrew Rybchenko 	}
10685e111ed8SAndrew Rybchenko 
10695e111ed8SAndrew Rybchenko 	return (should_abort);
10705e111ed8SAndrew Rybchenko }
10715e111ed8SAndrew Rybchenko 
10725e111ed8SAndrew Rybchenko static	__checkReturn	boolean_t
siena_ev_drv_gen(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)10735e111ed8SAndrew Rybchenko siena_ev_drv_gen(
10745e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
10755e111ed8SAndrew Rybchenko 	__in		efx_qword_t *eqp,
10765e111ed8SAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
10775e111ed8SAndrew Rybchenko 	__in_opt	void *arg)
10785e111ed8SAndrew Rybchenko {
10795e111ed8SAndrew Rybchenko 	uint32_t data;
10805e111ed8SAndrew Rybchenko 	boolean_t should_abort;
10815e111ed8SAndrew Rybchenko 
10825e111ed8SAndrew Rybchenko 	EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
10835e111ed8SAndrew Rybchenko 
10845e111ed8SAndrew Rybchenko 	data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
10855e111ed8SAndrew Rybchenko 	if (data >= ((uint32_t)1 << 16)) {
10865e111ed8SAndrew Rybchenko 		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
10875e111ed8SAndrew Rybchenko 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
10885e111ed8SAndrew Rybchenko 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
10895e111ed8SAndrew Rybchenko 		return (B_TRUE);
10905e111ed8SAndrew Rybchenko 	}
10915e111ed8SAndrew Rybchenko 
10925e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_software != NULL);
10935e111ed8SAndrew Rybchenko 	should_abort = eecp->eec_software(arg, (uint16_t)data);
10945e111ed8SAndrew Rybchenko 
10955e111ed8SAndrew Rybchenko 	return (should_abort);
10965e111ed8SAndrew Rybchenko }
10975e111ed8SAndrew Rybchenko 
10985e111ed8SAndrew Rybchenko #if EFSYS_OPT_MCDI
10995e111ed8SAndrew Rybchenko 
11005e111ed8SAndrew Rybchenko static	__checkReturn	boolean_t
siena_ev_mcdi(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)11015e111ed8SAndrew Rybchenko siena_ev_mcdi(
11025e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
11035e111ed8SAndrew Rybchenko 	__in		efx_qword_t *eqp,
11045e111ed8SAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
11055e111ed8SAndrew Rybchenko 	__in_opt	void *arg)
11065e111ed8SAndrew Rybchenko {
11075e111ed8SAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
11085e111ed8SAndrew Rybchenko 	unsigned int code;
11095e111ed8SAndrew Rybchenko 	boolean_t should_abort = B_FALSE;
11105e111ed8SAndrew Rybchenko 
11115e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
11125e111ed8SAndrew Rybchenko 
11135e111ed8SAndrew Rybchenko 	if (enp->en_family != EFX_FAMILY_SIENA)
11145e111ed8SAndrew Rybchenko 		goto out;
11155e111ed8SAndrew Rybchenko 
11165e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_link_change != NULL);
11175e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_exception != NULL);
11185e111ed8SAndrew Rybchenko #if EFSYS_OPT_MON_STATS
11195e111ed8SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_monitor != NULL);
11205e111ed8SAndrew Rybchenko #endif
11215e111ed8SAndrew Rybchenko 
11225e111ed8SAndrew Rybchenko 	EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
11235e111ed8SAndrew Rybchenko 
11245e111ed8SAndrew Rybchenko 	code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
11255e111ed8SAndrew Rybchenko 	switch (code) {
11265e111ed8SAndrew Rybchenko 	case MCDI_EVENT_CODE_BADSSERT:
11275e111ed8SAndrew Rybchenko 		efx_mcdi_ev_death(enp, EINTR);
11285e111ed8SAndrew Rybchenko 		break;
11295e111ed8SAndrew Rybchenko 
11305e111ed8SAndrew Rybchenko 	case MCDI_EVENT_CODE_CMDDONE:
11315e111ed8SAndrew Rybchenko 		efx_mcdi_ev_cpl(enp,
11325e111ed8SAndrew Rybchenko 		    MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
11335e111ed8SAndrew Rybchenko 		    MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
11345e111ed8SAndrew Rybchenko 		    MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
11355e111ed8SAndrew Rybchenko 		break;
11365e111ed8SAndrew Rybchenko 
11375e111ed8SAndrew Rybchenko 	case MCDI_EVENT_CODE_LINKCHANGE: {
11385e111ed8SAndrew Rybchenko 		efx_link_mode_t link_mode;
11395e111ed8SAndrew Rybchenko 
11405e111ed8SAndrew Rybchenko 		siena_phy_link_ev(enp, eqp, &link_mode);
11415e111ed8SAndrew Rybchenko 		should_abort = eecp->eec_link_change(arg, link_mode);
11425e111ed8SAndrew Rybchenko 		break;
11435e111ed8SAndrew Rybchenko 	}
11445e111ed8SAndrew Rybchenko 	case MCDI_EVENT_CODE_SENSOREVT: {
11455e111ed8SAndrew Rybchenko #if EFSYS_OPT_MON_STATS
11465e111ed8SAndrew Rybchenko 		efx_mon_stat_t id;
11475e111ed8SAndrew Rybchenko 		efx_mon_stat_value_t value;
11485e111ed8SAndrew Rybchenko 		efx_rc_t rc;
11495e111ed8SAndrew Rybchenko 
11505e111ed8SAndrew Rybchenko 		if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0)
11515e111ed8SAndrew Rybchenko 			should_abort = eecp->eec_monitor(arg, id, value);
11525e111ed8SAndrew Rybchenko 		else if (rc == ENOTSUP) {
11535e111ed8SAndrew Rybchenko 			should_abort = eecp->eec_exception(arg,
11545e111ed8SAndrew Rybchenko 				EFX_EXCEPTION_UNKNOWN_SENSOREVT,
11555e111ed8SAndrew Rybchenko 				MCDI_EV_FIELD(eqp, DATA));
11565e111ed8SAndrew Rybchenko 		} else
11575e111ed8SAndrew Rybchenko 			EFSYS_ASSERT(rc == ENODEV);	/* Wrong port */
11585e111ed8SAndrew Rybchenko #else
11595e111ed8SAndrew Rybchenko 		should_abort = B_FALSE;
11605e111ed8SAndrew Rybchenko #endif
11615e111ed8SAndrew Rybchenko 		break;
11625e111ed8SAndrew Rybchenko 	}
11635e111ed8SAndrew Rybchenko 	case MCDI_EVENT_CODE_SCHEDERR:
11645e111ed8SAndrew Rybchenko 		/* Informational only */
11655e111ed8SAndrew Rybchenko 		break;
11665e111ed8SAndrew Rybchenko 
11675e111ed8SAndrew Rybchenko 	case MCDI_EVENT_CODE_REBOOT:
11685e111ed8SAndrew Rybchenko 		efx_mcdi_ev_death(enp, EIO);
11695e111ed8SAndrew Rybchenko 		break;
11705e111ed8SAndrew Rybchenko 
11715e111ed8SAndrew Rybchenko 	case MCDI_EVENT_CODE_MAC_STATS_DMA:
11725e111ed8SAndrew Rybchenko #if EFSYS_OPT_MAC_STATS
11735e111ed8SAndrew Rybchenko 		if (eecp->eec_mac_stats != NULL) {
11745e111ed8SAndrew Rybchenko 			eecp->eec_mac_stats(arg,
11755e111ed8SAndrew Rybchenko 			    MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
11765e111ed8SAndrew Rybchenko 		}
11775e111ed8SAndrew Rybchenko #endif
11785e111ed8SAndrew Rybchenko 		break;
11795e111ed8SAndrew Rybchenko 
11805e111ed8SAndrew Rybchenko 	case MCDI_EVENT_CODE_FWALERT: {
11815e111ed8SAndrew Rybchenko 		uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
11825e111ed8SAndrew Rybchenko 
11835e111ed8SAndrew Rybchenko 		if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
11845e111ed8SAndrew Rybchenko 			should_abort = eecp->eec_exception(arg,
11855e111ed8SAndrew Rybchenko 				EFX_EXCEPTION_FWALERT_SRAM,
11865e111ed8SAndrew Rybchenko 				MCDI_EV_FIELD(eqp, FWALERT_DATA));
11875e111ed8SAndrew Rybchenko 		else
11885e111ed8SAndrew Rybchenko 			should_abort = eecp->eec_exception(arg,
11895e111ed8SAndrew Rybchenko 				EFX_EXCEPTION_UNKNOWN_FWALERT,
11905e111ed8SAndrew Rybchenko 				MCDI_EV_FIELD(eqp, DATA));
11915e111ed8SAndrew Rybchenko 		break;
11925e111ed8SAndrew Rybchenko 	}
11935e111ed8SAndrew Rybchenko 
11945e111ed8SAndrew Rybchenko 	default:
11955e111ed8SAndrew Rybchenko 		EFSYS_PROBE1(mc_pcol_error, int, code);
11965e111ed8SAndrew Rybchenko 		break;
11975e111ed8SAndrew Rybchenko 	}
11985e111ed8SAndrew Rybchenko 
11995e111ed8SAndrew Rybchenko out:
12005e111ed8SAndrew Rybchenko 	return (should_abort);
12015e111ed8SAndrew Rybchenko }
12025e111ed8SAndrew Rybchenko 
12035e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_MCDI */
12045e111ed8SAndrew Rybchenko 
12055e111ed8SAndrew Rybchenko static	__checkReturn	efx_rc_t
siena_ev_qprime(__in efx_evq_t * eep,__in unsigned int count)12065e111ed8SAndrew Rybchenko siena_ev_qprime(
12075e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
12085e111ed8SAndrew Rybchenko 	__in		unsigned int count)
12095e111ed8SAndrew Rybchenko {
12105e111ed8SAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
12115e111ed8SAndrew Rybchenko 	uint32_t rptr;
12125e111ed8SAndrew Rybchenko 	efx_dword_t dword;
12135e111ed8SAndrew Rybchenko 
12145e111ed8SAndrew Rybchenko 	rptr = count & eep->ee_mask;
12155e111ed8SAndrew Rybchenko 
12165e111ed8SAndrew Rybchenko 	EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
12175e111ed8SAndrew Rybchenko 
12185e111ed8SAndrew Rybchenko 	EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
12195e111ed8SAndrew Rybchenko 			    &dword, B_FALSE);
12205e111ed8SAndrew Rybchenko 
12215e111ed8SAndrew Rybchenko 	return (0);
12225e111ed8SAndrew Rybchenko }
12235e111ed8SAndrew Rybchenko 
12245e111ed8SAndrew Rybchenko static		void
siena_ev_qpost(__in efx_evq_t * eep,__in uint16_t data)12255e111ed8SAndrew Rybchenko siena_ev_qpost(
12265e111ed8SAndrew Rybchenko 	__in	efx_evq_t *eep,
12275e111ed8SAndrew Rybchenko 	__in	uint16_t data)
12285e111ed8SAndrew Rybchenko {
12295e111ed8SAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
12305e111ed8SAndrew Rybchenko 	efx_qword_t ev;
12315e111ed8SAndrew Rybchenko 	efx_oword_t oword;
12325e111ed8SAndrew Rybchenko 
12335e111ed8SAndrew Rybchenko 	EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
12345e111ed8SAndrew Rybchenko 	    FSF_AZ_EV_DATA_DW0, (uint32_t)data);
12355e111ed8SAndrew Rybchenko 
12365e111ed8SAndrew Rybchenko 	EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
12375e111ed8SAndrew Rybchenko 	    EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
12385e111ed8SAndrew Rybchenko 	    EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
12395e111ed8SAndrew Rybchenko 
12405e111ed8SAndrew Rybchenko 	EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
12415e111ed8SAndrew Rybchenko }
12425e111ed8SAndrew Rybchenko 
12435e111ed8SAndrew Rybchenko static	__checkReturn	efx_rc_t
siena_ev_qmoderate(__in efx_evq_t * eep,__in unsigned int us)12445e111ed8SAndrew Rybchenko siena_ev_qmoderate(
12455e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
12465e111ed8SAndrew Rybchenko 	__in		unsigned int us)
12475e111ed8SAndrew Rybchenko {
12485e111ed8SAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
12495e111ed8SAndrew Rybchenko 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
12505e111ed8SAndrew Rybchenko 	unsigned int locked;
12515e111ed8SAndrew Rybchenko 	efx_dword_t dword;
12525e111ed8SAndrew Rybchenko 	efx_rc_t rc;
12535e111ed8SAndrew Rybchenko 
12545e111ed8SAndrew Rybchenko 	if (us > encp->enc_evq_timer_max_us) {
12555e111ed8SAndrew Rybchenko 		rc = EINVAL;
12565e111ed8SAndrew Rybchenko 		goto fail1;
12575e111ed8SAndrew Rybchenko 	}
12585e111ed8SAndrew Rybchenko 
12595e111ed8SAndrew Rybchenko 	/* If the value is zero then disable the timer */
12605e111ed8SAndrew Rybchenko 	if (us == 0) {
12615e111ed8SAndrew Rybchenko 		EFX_POPULATE_DWORD_2(dword,
12625e111ed8SAndrew Rybchenko 		    FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
12635e111ed8SAndrew Rybchenko 		    FRF_CZ_TC_TIMER_VAL, 0);
12645e111ed8SAndrew Rybchenko 	} else {
12655e111ed8SAndrew Rybchenko 		unsigned int ticks;
12665e111ed8SAndrew Rybchenko 
12675e111ed8SAndrew Rybchenko 		if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0)
12685e111ed8SAndrew Rybchenko 			goto fail2;
12695e111ed8SAndrew Rybchenko 
12705e111ed8SAndrew Rybchenko 		EFSYS_ASSERT(ticks > 0);
12715e111ed8SAndrew Rybchenko 		EFX_POPULATE_DWORD_2(dword,
12725e111ed8SAndrew Rybchenko 		    FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
12735e111ed8SAndrew Rybchenko 		    FRF_CZ_TC_TIMER_VAL, ticks - 1);
12745e111ed8SAndrew Rybchenko 	}
12755e111ed8SAndrew Rybchenko 
12765e111ed8SAndrew Rybchenko 	locked = (eep->ee_index == 0) ? 1 : 0;
12775e111ed8SAndrew Rybchenko 
12785e111ed8SAndrew Rybchenko 	EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
12795e111ed8SAndrew Rybchenko 	    eep->ee_index, &dword, locked);
12805e111ed8SAndrew Rybchenko 
12815e111ed8SAndrew Rybchenko 	return (0);
12825e111ed8SAndrew Rybchenko 
12835e111ed8SAndrew Rybchenko fail2:
12845e111ed8SAndrew Rybchenko 	EFSYS_PROBE(fail2);
12855e111ed8SAndrew Rybchenko fail1:
12865e111ed8SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
12875e111ed8SAndrew Rybchenko 
12885e111ed8SAndrew Rybchenko 	return (rc);
12895e111ed8SAndrew Rybchenko }
12905e111ed8SAndrew Rybchenko 
12915e111ed8SAndrew Rybchenko static	__checkReturn	efx_rc_t
siena_ev_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in uint32_t us,__in uint32_t flags,__in uint32_t irq,__in efx_evq_t * eep)12925e111ed8SAndrew Rybchenko siena_ev_qcreate(
12935e111ed8SAndrew Rybchenko 	__in		efx_nic_t *enp,
12945e111ed8SAndrew Rybchenko 	__in		unsigned int index,
12955e111ed8SAndrew Rybchenko 	__in		efsys_mem_t *esmp,
12965e111ed8SAndrew Rybchenko 	__in		size_t ndescs,
12975e111ed8SAndrew Rybchenko 	__in		uint32_t id,
12985e111ed8SAndrew Rybchenko 	__in		uint32_t us,
12995e111ed8SAndrew Rybchenko 	__in		uint32_t flags,
1300*aa6dc101SAndrew Rybchenko 	__in		uint32_t irq,
13015e111ed8SAndrew Rybchenko 	__in		efx_evq_t *eep)
13025e111ed8SAndrew Rybchenko {
13035e111ed8SAndrew Rybchenko 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
13045e111ed8SAndrew Rybchenko 	uint32_t size;
13055e111ed8SAndrew Rybchenko 	efx_oword_t oword;
13065e111ed8SAndrew Rybchenko 	efx_rc_t rc;
13075e111ed8SAndrew Rybchenko 	boolean_t notify_mode;
13085e111ed8SAndrew Rybchenko 
13095e111ed8SAndrew Rybchenko 	_NOTE(ARGUNUSED(esmp))
13105e111ed8SAndrew Rybchenko 
1311f8a60f76SAndy Moreton 	EFSYS_ASSERT((flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH) == 0);
1312f8a60f76SAndy Moreton 
1313*aa6dc101SAndrew Rybchenko 	if (irq != index) {
1314*aa6dc101SAndrew Rybchenko 		rc = EINVAL;
1315*aa6dc101SAndrew Rybchenko 		goto fail1;
1316*aa6dc101SAndrew Rybchenko 	}
1317*aa6dc101SAndrew Rybchenko 
13185e111ed8SAndrew Rybchenko #if EFSYS_OPT_RX_SCALE
13195e111ed8SAndrew Rybchenko 	if (enp->en_intr.ei_type == EFX_INTR_LINE &&
13205e111ed8SAndrew Rybchenko 	    index >= EFX_MAXRSS_LEGACY) {
13215e111ed8SAndrew Rybchenko 		rc = EINVAL;
1322*aa6dc101SAndrew Rybchenko 		goto fail2;
13235e111ed8SAndrew Rybchenko 	}
13245e111ed8SAndrew Rybchenko #endif
13255e111ed8SAndrew Rybchenko 	for (size = 0;
13265e111ed8SAndrew Rybchenko 	    (1U << size) <= encp->enc_evq_max_nevs / encp->enc_evq_min_nevs;
13275e111ed8SAndrew Rybchenko 	    size++)
13285e111ed8SAndrew Rybchenko 		if ((1U << size) == (uint32_t)ndescs / encp->enc_evq_min_nevs)
13295e111ed8SAndrew Rybchenko 			break;
13305e111ed8SAndrew Rybchenko 	if (id + (1 << size) >= encp->enc_buftbl_limit) {
13315e111ed8SAndrew Rybchenko 		rc = EINVAL;
1332*aa6dc101SAndrew Rybchenko 		goto fail3;
13335e111ed8SAndrew Rybchenko 	}
13345e111ed8SAndrew Rybchenko 
13355e111ed8SAndrew Rybchenko 	/* Set up the handler table */
13365e111ed8SAndrew Rybchenko 	eep->ee_rx	= siena_ev_rx;
13375e111ed8SAndrew Rybchenko 	eep->ee_tx	= siena_ev_tx;
13385e111ed8SAndrew Rybchenko 	eep->ee_driver	= siena_ev_driver;
13395e111ed8SAndrew Rybchenko 	eep->ee_global	= siena_ev_global;
13405e111ed8SAndrew Rybchenko 	eep->ee_drv_gen	= siena_ev_drv_gen;
13415e111ed8SAndrew Rybchenko #if EFSYS_OPT_MCDI
13425e111ed8SAndrew Rybchenko 	eep->ee_mcdi	= siena_ev_mcdi;
13435e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_MCDI */
13445e111ed8SAndrew Rybchenko 
13455e111ed8SAndrew Rybchenko 	notify_mode = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) !=
13465e111ed8SAndrew Rybchenko 	    EFX_EVQ_FLAGS_NOTIFY_INTERRUPT);
13475e111ed8SAndrew Rybchenko 
13485e111ed8SAndrew Rybchenko 	/* Set up the new event queue */
13495e111ed8SAndrew Rybchenko 	EFX_POPULATE_OWORD_3(oword, FRF_CZ_TIMER_Q_EN, 1,
13505e111ed8SAndrew Rybchenko 	    FRF_CZ_HOST_NOTIFY_MODE, notify_mode,
13515e111ed8SAndrew Rybchenko 	    FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
13525e111ed8SAndrew Rybchenko 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE);
13535e111ed8SAndrew Rybchenko 
13545e111ed8SAndrew Rybchenko 	EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
13555e111ed8SAndrew Rybchenko 	    FRF_AZ_EVQ_BUF_BASE_ID, id);
13565e111ed8SAndrew Rybchenko 
13575e111ed8SAndrew Rybchenko 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE);
13585e111ed8SAndrew Rybchenko 
13595e111ed8SAndrew Rybchenko 	/* Set initial interrupt moderation */
13605e111ed8SAndrew Rybchenko 	siena_ev_qmoderate(eep, us);
13615e111ed8SAndrew Rybchenko 
13625e111ed8SAndrew Rybchenko 	return (0);
13635e111ed8SAndrew Rybchenko 
1364*aa6dc101SAndrew Rybchenko fail3:
1365*aa6dc101SAndrew Rybchenko 	EFSYS_PROBE(fail3);
1366*aa6dc101SAndrew Rybchenko #if EFSYS_OPT_RX_SCALE
13675e111ed8SAndrew Rybchenko fail2:
13685e111ed8SAndrew Rybchenko 	EFSYS_PROBE(fail2);
13692e5819a5SAndrew Rybchenko #endif
1370*aa6dc101SAndrew Rybchenko fail1:
13715e111ed8SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
13725e111ed8SAndrew Rybchenko 
13735e111ed8SAndrew Rybchenko 	return (rc);
13745e111ed8SAndrew Rybchenko }
13755e111ed8SAndrew Rybchenko 
13765e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
13775e111ed8SAndrew Rybchenko 
13785e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
13795e111ed8SAndrew Rybchenko #if EFSYS_OPT_NAMES
13805e111ed8SAndrew Rybchenko /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock ac223f7134058b4f */
13815e111ed8SAndrew Rybchenko static const char * const __efx_ev_qstat_name[] = {
13825e111ed8SAndrew Rybchenko 	"all",
13835e111ed8SAndrew Rybchenko 	"rx",
13845e111ed8SAndrew Rybchenko 	"rx_ok",
13855e111ed8SAndrew Rybchenko 	"rx_frm_trunc",
13865e111ed8SAndrew Rybchenko 	"rx_tobe_disc",
13875e111ed8SAndrew Rybchenko 	"rx_pause_frm_err",
13885e111ed8SAndrew Rybchenko 	"rx_buf_owner_id_err",
13895e111ed8SAndrew Rybchenko 	"rx_ipv4_hdr_chksum_err",
13905e111ed8SAndrew Rybchenko 	"rx_tcp_udp_chksum_err",
13915e111ed8SAndrew Rybchenko 	"rx_eth_crc_err",
13925e111ed8SAndrew Rybchenko 	"rx_ip_frag_err",
13935e111ed8SAndrew Rybchenko 	"rx_mcast_pkt",
13945e111ed8SAndrew Rybchenko 	"rx_mcast_hash_match",
13955e111ed8SAndrew Rybchenko 	"rx_tcp_ipv4",
13965e111ed8SAndrew Rybchenko 	"rx_tcp_ipv6",
13975e111ed8SAndrew Rybchenko 	"rx_udp_ipv4",
13985e111ed8SAndrew Rybchenko 	"rx_udp_ipv6",
13995e111ed8SAndrew Rybchenko 	"rx_other_ipv4",
14005e111ed8SAndrew Rybchenko 	"rx_other_ipv6",
14015e111ed8SAndrew Rybchenko 	"rx_non_ip",
14025e111ed8SAndrew Rybchenko 	"rx_batch",
14035e111ed8SAndrew Rybchenko 	"tx",
14045e111ed8SAndrew Rybchenko 	"tx_wq_ff_full",
14055e111ed8SAndrew Rybchenko 	"tx_pkt_err",
14065e111ed8SAndrew Rybchenko 	"tx_pkt_too_big",
14075e111ed8SAndrew Rybchenko 	"tx_unexpected",
14085e111ed8SAndrew Rybchenko 	"global",
14095e111ed8SAndrew Rybchenko 	"global_mnt",
14105e111ed8SAndrew Rybchenko 	"driver",
14115e111ed8SAndrew Rybchenko 	"driver_srm_upd_done",
14125e111ed8SAndrew Rybchenko 	"driver_tx_descq_fls_done",
14135e111ed8SAndrew Rybchenko 	"driver_rx_descq_fls_done",
14145e111ed8SAndrew Rybchenko 	"driver_rx_descq_fls_failed",
14155e111ed8SAndrew Rybchenko 	"driver_rx_dsc_error",
14165e111ed8SAndrew Rybchenko 	"driver_tx_dsc_error",
14175e111ed8SAndrew Rybchenko 	"drv_gen",
14185e111ed8SAndrew Rybchenko 	"mcdi_response",
14195e111ed8SAndrew Rybchenko 	"rx_parse_incomplete",
14205e111ed8SAndrew Rybchenko };
14215e111ed8SAndrew Rybchenko /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
14225e111ed8SAndrew Rybchenko 
14235e111ed8SAndrew Rybchenko 		const char *
efx_ev_qstat_name(__in efx_nic_t * enp,__in unsigned int id)14245e111ed8SAndrew Rybchenko efx_ev_qstat_name(
14255e111ed8SAndrew Rybchenko 	__in	efx_nic_t *enp,
14265e111ed8SAndrew Rybchenko 	__in	unsigned int id)
14275e111ed8SAndrew Rybchenko {
14285e111ed8SAndrew Rybchenko 	_NOTE(ARGUNUSED(enp))
14295e111ed8SAndrew Rybchenko 
14305e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
14315e111ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(id, <, EV_NQSTATS);
14325e111ed8SAndrew Rybchenko 
14335e111ed8SAndrew Rybchenko 	return (__efx_ev_qstat_name[id]);
14345e111ed8SAndrew Rybchenko }
14355e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_NAMES */
14365e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_QSTATS */
14375e111ed8SAndrew Rybchenko 
14385e111ed8SAndrew Rybchenko #if EFSYS_OPT_SIENA
14395e111ed8SAndrew Rybchenko 
14405e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
14415e111ed8SAndrew Rybchenko static					void
siena_ev_qstats_update(__in efx_evq_t * eep,__inout_ecount (EV_NQSTATS)efsys_stat_t * stat)14425e111ed8SAndrew Rybchenko siena_ev_qstats_update(
14435e111ed8SAndrew Rybchenko 	__in				efx_evq_t *eep,
14445e111ed8SAndrew Rybchenko 	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat)
14455e111ed8SAndrew Rybchenko {
14465e111ed8SAndrew Rybchenko 	unsigned int id;
14475e111ed8SAndrew Rybchenko 
14485e111ed8SAndrew Rybchenko 	for (id = 0; id < EV_NQSTATS; id++) {
14495e111ed8SAndrew Rybchenko 		efsys_stat_t *essp = &stat[id];
14505e111ed8SAndrew Rybchenko 
14515e111ed8SAndrew Rybchenko 		EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
14525e111ed8SAndrew Rybchenko 		eep->ee_stat[id] = 0;
14535e111ed8SAndrew Rybchenko 	}
14545e111ed8SAndrew Rybchenko }
14555e111ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_QSTATS */
14565e111ed8SAndrew Rybchenko 
14575e111ed8SAndrew Rybchenko static		void
siena_ev_qdestroy(__in efx_evq_t * eep)14585e111ed8SAndrew Rybchenko siena_ev_qdestroy(
14595e111ed8SAndrew Rybchenko 	__in	efx_evq_t *eep)
14605e111ed8SAndrew Rybchenko {
14615e111ed8SAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
14625e111ed8SAndrew Rybchenko 	efx_oword_t oword;
14635e111ed8SAndrew Rybchenko 
14645e111ed8SAndrew Rybchenko 	/* Purge event queue */
14655e111ed8SAndrew Rybchenko 	EFX_ZERO_OWORD(oword);
14665e111ed8SAndrew Rybchenko 
14675e111ed8SAndrew Rybchenko 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
14685e111ed8SAndrew Rybchenko 	    eep->ee_index, &oword, B_TRUE);
14695e111ed8SAndrew Rybchenko 
14705e111ed8SAndrew Rybchenko 	EFX_ZERO_OWORD(oword);
14715e111ed8SAndrew Rybchenko 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE);
14725e111ed8SAndrew Rybchenko }
14735e111ed8SAndrew Rybchenko 
14745e111ed8SAndrew Rybchenko static		void
siena_ev_fini(__in efx_nic_t * enp)14755e111ed8SAndrew Rybchenko siena_ev_fini(
14765e111ed8SAndrew Rybchenko 	__in	efx_nic_t *enp)
14775e111ed8SAndrew Rybchenko {
14785e111ed8SAndrew Rybchenko 	_NOTE(ARGUNUSED(enp))
14795e111ed8SAndrew Rybchenko }
14805e111ed8SAndrew Rybchenko 
14815e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
1482ad1e3ed8SAndrew Rybchenko 
1483ad1e3ed8SAndrew Rybchenko #if EFX_OPTS_EF10() || EFSYS_OPT_SIENA
1484ad1e3ed8SAndrew Rybchenko 
1485ad1e3ed8SAndrew Rybchenko #define	EFX_EV_BATCH	8
1486ad1e3ed8SAndrew Rybchenko 
1487ad1e3ed8SAndrew Rybchenko static			void
siena_ef10_ev_qpoll(__in efx_evq_t * eep,__inout unsigned int * countp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)1488ad1e3ed8SAndrew Rybchenko siena_ef10_ev_qpoll(
1489ad1e3ed8SAndrew Rybchenko 	__in		efx_evq_t *eep,
1490ad1e3ed8SAndrew Rybchenko 	__inout		unsigned int *countp,
1491ad1e3ed8SAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
1492ad1e3ed8SAndrew Rybchenko 	__in_opt	void *arg)
1493ad1e3ed8SAndrew Rybchenko {
1494ad1e3ed8SAndrew Rybchenko 	efx_qword_t ev[EFX_EV_BATCH];
1495ad1e3ed8SAndrew Rybchenko 	unsigned int batch;
1496ad1e3ed8SAndrew Rybchenko 	unsigned int total;
1497ad1e3ed8SAndrew Rybchenko 	unsigned int count;
1498ad1e3ed8SAndrew Rybchenko 	unsigned int index;
1499ad1e3ed8SAndrew Rybchenko 	size_t offset;
1500ad1e3ed8SAndrew Rybchenko 
1501ad1e3ed8SAndrew Rybchenko 	/* Ensure events codes match for EF10 (Huntington/Medford) and Siena */
1502ad1e3ed8SAndrew Rybchenko 	EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
1503ad1e3ed8SAndrew Rybchenko 	EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
1504ad1e3ed8SAndrew Rybchenko 
1505ad1e3ed8SAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
1506ad1e3ed8SAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
1507ad1e3ed8SAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
1508ad1e3ed8SAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
1509ad1e3ed8SAndrew Rybchenko 	    FSE_AZ_EV_CODE_DRV_GEN_EV);
1510ad1e3ed8SAndrew Rybchenko #if EFSYS_OPT_MCDI
1511ad1e3ed8SAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV ==
1512ad1e3ed8SAndrew Rybchenko 	    FSE_AZ_EV_CODE_MCDI_EVRESPONSE);
1513ad1e3ed8SAndrew Rybchenko #endif
1514ad1e3ed8SAndrew Rybchenko 
1515ad1e3ed8SAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
1516ad1e3ed8SAndrew Rybchenko 	EFSYS_ASSERT(countp != NULL);
1517ad1e3ed8SAndrew Rybchenko 	EFSYS_ASSERT(eecp != NULL);
1518ad1e3ed8SAndrew Rybchenko 
1519ad1e3ed8SAndrew Rybchenko 	count = *countp;
1520ad1e3ed8SAndrew Rybchenko 	do {
1521ad1e3ed8SAndrew Rybchenko 		/* Read up until the end of the batch period */
1522ad1e3ed8SAndrew Rybchenko 		batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
1523ad1e3ed8SAndrew Rybchenko 		offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1524ad1e3ed8SAndrew Rybchenko 		for (total = 0; total < batch; ++total) {
1525ad1e3ed8SAndrew Rybchenko 			EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
1526ad1e3ed8SAndrew Rybchenko 
1527ad1e3ed8SAndrew Rybchenko 			if (!EFX_EV_PRESENT(ev[total]))
1528ad1e3ed8SAndrew Rybchenko 				break;
1529ad1e3ed8SAndrew Rybchenko 
1530ad1e3ed8SAndrew Rybchenko 			EFSYS_PROBE3(event, unsigned int, eep->ee_index,
1531ad1e3ed8SAndrew Rybchenko 			    uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
1532ad1e3ed8SAndrew Rybchenko 			    uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
1533ad1e3ed8SAndrew Rybchenko 
1534ad1e3ed8SAndrew Rybchenko 			offset += sizeof (efx_qword_t);
1535ad1e3ed8SAndrew Rybchenko 		}
1536ad1e3ed8SAndrew Rybchenko 
1537ad1e3ed8SAndrew Rybchenko #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
1538ad1e3ed8SAndrew Rybchenko 		/*
1539ad1e3ed8SAndrew Rybchenko 		 * Prefetch the next batch when we get within PREFETCH_PERIOD
1540ad1e3ed8SAndrew Rybchenko 		 * of a completed batch. If the batch is smaller, then prefetch
1541ad1e3ed8SAndrew Rybchenko 		 * immediately.
1542ad1e3ed8SAndrew Rybchenko 		 */
1543ad1e3ed8SAndrew Rybchenko 		if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
1544ad1e3ed8SAndrew Rybchenko 			EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1545ad1e3ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_EV_PREFETCH */
1546ad1e3ed8SAndrew Rybchenko 
1547ad1e3ed8SAndrew Rybchenko 		/* Process the batch of events */
1548ad1e3ed8SAndrew Rybchenko 		for (index = 0; index < total; ++index) {
1549ad1e3ed8SAndrew Rybchenko 			boolean_t should_abort;
1550ad1e3ed8SAndrew Rybchenko 			uint32_t code;
1551ad1e3ed8SAndrew Rybchenko 
1552ad1e3ed8SAndrew Rybchenko #if EFSYS_OPT_EV_PREFETCH
1553ad1e3ed8SAndrew Rybchenko 			/* Prefetch if we've now reached the batch period */
1554ad1e3ed8SAndrew Rybchenko 			if (total == batch &&
1555ad1e3ed8SAndrew Rybchenko 			    index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
1556ad1e3ed8SAndrew Rybchenko 				offset = (count + batch) & eep->ee_mask;
1557ad1e3ed8SAndrew Rybchenko 				offset *= sizeof (efx_qword_t);
1558ad1e3ed8SAndrew Rybchenko 
1559ad1e3ed8SAndrew Rybchenko 				EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1560ad1e3ed8SAndrew Rybchenko 			}
1561ad1e3ed8SAndrew Rybchenko #endif	/* EFSYS_OPT_EV_PREFETCH */
1562ad1e3ed8SAndrew Rybchenko 
1563ad1e3ed8SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_ALL);
1564ad1e3ed8SAndrew Rybchenko 
1565ad1e3ed8SAndrew Rybchenko 			code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
1566ad1e3ed8SAndrew Rybchenko 			switch (code) {
1567ad1e3ed8SAndrew Rybchenko 			case FSE_AZ_EV_CODE_RX_EV:
1568ad1e3ed8SAndrew Rybchenko 				should_abort = eep->ee_rx(eep,
1569ad1e3ed8SAndrew Rybchenko 				    &(ev[index]), eecp, arg);
1570ad1e3ed8SAndrew Rybchenko 				break;
1571ad1e3ed8SAndrew Rybchenko 			case FSE_AZ_EV_CODE_TX_EV:
1572ad1e3ed8SAndrew Rybchenko 				should_abort = eep->ee_tx(eep,
1573ad1e3ed8SAndrew Rybchenko 				    &(ev[index]), eecp, arg);
1574ad1e3ed8SAndrew Rybchenko 				break;
1575ad1e3ed8SAndrew Rybchenko 			case FSE_AZ_EV_CODE_DRIVER_EV:
1576ad1e3ed8SAndrew Rybchenko 				should_abort = eep->ee_driver(eep,
1577ad1e3ed8SAndrew Rybchenko 				    &(ev[index]), eecp, arg);
1578ad1e3ed8SAndrew Rybchenko 				break;
1579ad1e3ed8SAndrew Rybchenko 			case FSE_AZ_EV_CODE_DRV_GEN_EV:
1580ad1e3ed8SAndrew Rybchenko 				should_abort = eep->ee_drv_gen(eep,
1581ad1e3ed8SAndrew Rybchenko 				    &(ev[index]), eecp, arg);
1582ad1e3ed8SAndrew Rybchenko 				break;
1583ad1e3ed8SAndrew Rybchenko #if EFSYS_OPT_MCDI
1584ad1e3ed8SAndrew Rybchenko 			case FSE_AZ_EV_CODE_MCDI_EVRESPONSE:
1585ad1e3ed8SAndrew Rybchenko 				should_abort = eep->ee_mcdi(eep,
1586ad1e3ed8SAndrew Rybchenko 				    &(ev[index]), eecp, arg);
1587ad1e3ed8SAndrew Rybchenko 				break;
1588ad1e3ed8SAndrew Rybchenko #endif
1589ad1e3ed8SAndrew Rybchenko 			case FSE_AZ_EV_CODE_GLOBAL_EV:
1590ad1e3ed8SAndrew Rybchenko 				if (eep->ee_global) {
1591ad1e3ed8SAndrew Rybchenko 					should_abort = eep->ee_global(eep,
1592ad1e3ed8SAndrew Rybchenko 					    &(ev[index]), eecp, arg);
1593ad1e3ed8SAndrew Rybchenko 					break;
1594ad1e3ed8SAndrew Rybchenko 				}
1595ad1e3ed8SAndrew Rybchenko 				/* else fallthrough */
1596ad1e3ed8SAndrew Rybchenko 			default:
1597ad1e3ed8SAndrew Rybchenko 				EFSYS_PROBE3(bad_event,
1598ad1e3ed8SAndrew Rybchenko 				    unsigned int, eep->ee_index,
1599ad1e3ed8SAndrew Rybchenko 				    uint32_t,
1600ad1e3ed8SAndrew Rybchenko 				    EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
1601ad1e3ed8SAndrew Rybchenko 				    uint32_t,
1602ad1e3ed8SAndrew Rybchenko 				    EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
1603ad1e3ed8SAndrew Rybchenko 
1604ad1e3ed8SAndrew Rybchenko 				EFSYS_ASSERT(eecp->eec_exception != NULL);
1605ad1e3ed8SAndrew Rybchenko 				(void) eecp->eec_exception(arg,
1606ad1e3ed8SAndrew Rybchenko 					EFX_EXCEPTION_EV_ERROR, code);
1607ad1e3ed8SAndrew Rybchenko 				should_abort = B_TRUE;
1608ad1e3ed8SAndrew Rybchenko 			}
1609ad1e3ed8SAndrew Rybchenko 			if (should_abort) {
1610ad1e3ed8SAndrew Rybchenko 				/* Ignore subsequent events */
1611ad1e3ed8SAndrew Rybchenko 				total = index + 1;
1612ad1e3ed8SAndrew Rybchenko 
1613ad1e3ed8SAndrew Rybchenko 				/*
1614ad1e3ed8SAndrew Rybchenko 				 * Poison batch to ensure the outer
1615ad1e3ed8SAndrew Rybchenko 				 * loop is broken out of.
1616ad1e3ed8SAndrew Rybchenko 				 */
1617ad1e3ed8SAndrew Rybchenko 				EFSYS_ASSERT(batch <= EFX_EV_BATCH);
1618ad1e3ed8SAndrew Rybchenko 				batch += (EFX_EV_BATCH << 1);
1619ad1e3ed8SAndrew Rybchenko 				EFSYS_ASSERT(total != batch);
1620ad1e3ed8SAndrew Rybchenko 				break;
1621ad1e3ed8SAndrew Rybchenko 			}
1622ad1e3ed8SAndrew Rybchenko 		}
1623ad1e3ed8SAndrew Rybchenko 
1624ad1e3ed8SAndrew Rybchenko 		/*
1625ad1e3ed8SAndrew Rybchenko 		 * Now that the hardware has most likely moved onto dma'ing
1626ad1e3ed8SAndrew Rybchenko 		 * into the next cache line, clear the processed events. Take
1627ad1e3ed8SAndrew Rybchenko 		 * care to only clear out events that we've processed
1628ad1e3ed8SAndrew Rybchenko 		 */
1629ad1e3ed8SAndrew Rybchenko 		EFX_SET_QWORD(ev[0]);
1630ad1e3ed8SAndrew Rybchenko 		offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1631ad1e3ed8SAndrew Rybchenko 		for (index = 0; index < total; ++index) {
1632ad1e3ed8SAndrew Rybchenko 			EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
1633ad1e3ed8SAndrew Rybchenko 			offset += sizeof (efx_qword_t);
1634ad1e3ed8SAndrew Rybchenko 		}
1635ad1e3ed8SAndrew Rybchenko 
1636ad1e3ed8SAndrew Rybchenko 		count += total;
1637ad1e3ed8SAndrew Rybchenko 
1638ad1e3ed8SAndrew Rybchenko 	} while (total == batch);
1639ad1e3ed8SAndrew Rybchenko 
1640ad1e3ed8SAndrew Rybchenko 	*countp = count;
1641ad1e3ed8SAndrew Rybchenko }
1642ad1e3ed8SAndrew Rybchenko 
1643ad1e3ed8SAndrew Rybchenko #endif	/* EFX_OPTS_EF10() || EFSYS_OPT_SIENA */
1644