15e111ed8SAndrew Rybchenko /* SPDX-License-Identifier: BSD-3-Clause
25e111ed8SAndrew Rybchenko *
3672386c1SAndrew Rybchenko * Copyright(c) 2019-2021 Xilinx, Inc.
45e111ed8SAndrew Rybchenko * Copyright(c) 2007-2019 Solarflare Communications Inc.
55e111ed8SAndrew Rybchenko */
65e111ed8SAndrew Rybchenko
75e111ed8SAndrew Rybchenko #include "efx.h"
85e111ed8SAndrew Rybchenko #include "efx_impl.h"
95e111ed8SAndrew Rybchenko #if EFSYS_OPT_MON_MCDI
105e111ed8SAndrew Rybchenko #include "mcdi_mon.h"
115e111ed8SAndrew Rybchenko #endif
125e111ed8SAndrew Rybchenko
135e111ed8SAndrew Rybchenko #define EFX_EV_PRESENT(_qword) \
145e111ed8SAndrew Rybchenko (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff && \
155e111ed8SAndrew Rybchenko EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
165e111ed8SAndrew Rybchenko
175e111ed8SAndrew Rybchenko
185e111ed8SAndrew Rybchenko
195e111ed8SAndrew Rybchenko #if EFSYS_OPT_SIENA
205e111ed8SAndrew Rybchenko
215e111ed8SAndrew Rybchenko static __checkReturn efx_rc_t
225e111ed8SAndrew Rybchenko siena_ev_init(
235e111ed8SAndrew Rybchenko __in efx_nic_t *enp);
245e111ed8SAndrew Rybchenko
255e111ed8SAndrew Rybchenko static void
265e111ed8SAndrew Rybchenko siena_ev_fini(
275e111ed8SAndrew Rybchenko __in efx_nic_t *enp);
285e111ed8SAndrew Rybchenko
295e111ed8SAndrew Rybchenko static __checkReturn efx_rc_t
305e111ed8SAndrew Rybchenko siena_ev_qcreate(
315e111ed8SAndrew Rybchenko __in efx_nic_t *enp,
325e111ed8SAndrew Rybchenko __in unsigned int index,
335e111ed8SAndrew Rybchenko __in efsys_mem_t *esmp,
345e111ed8SAndrew Rybchenko __in size_t ndescs,
355e111ed8SAndrew Rybchenko __in uint32_t id,
365e111ed8SAndrew Rybchenko __in uint32_t us,
375e111ed8SAndrew Rybchenko __in uint32_t flags,
38*aa6dc101SAndrew Rybchenko __in uint32_t irq,
395e111ed8SAndrew Rybchenko __in efx_evq_t *eep);
405e111ed8SAndrew Rybchenko
415e111ed8SAndrew Rybchenko static void
425e111ed8SAndrew Rybchenko siena_ev_qdestroy(
435e111ed8SAndrew Rybchenko __in efx_evq_t *eep);
445e111ed8SAndrew Rybchenko
455e111ed8SAndrew Rybchenko static __checkReturn efx_rc_t
465e111ed8SAndrew Rybchenko siena_ev_qprime(
475e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
485e111ed8SAndrew Rybchenko __in unsigned int count);
495e111ed8SAndrew Rybchenko
505e111ed8SAndrew Rybchenko static void
515e111ed8SAndrew Rybchenko siena_ev_qpost(
525e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
535e111ed8SAndrew Rybchenko __in uint16_t data);
545e111ed8SAndrew Rybchenko
555e111ed8SAndrew Rybchenko static __checkReturn efx_rc_t
565e111ed8SAndrew Rybchenko siena_ev_qmoderate(
575e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
585e111ed8SAndrew Rybchenko __in unsigned int us);
595e111ed8SAndrew Rybchenko
605e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
615e111ed8SAndrew Rybchenko static void
625e111ed8SAndrew Rybchenko siena_ev_qstats_update(
635e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
645e111ed8SAndrew Rybchenko __inout_ecount(EV_NQSTATS) efsys_stat_t *stat);
655e111ed8SAndrew Rybchenko
665e111ed8SAndrew Rybchenko #endif
675e111ed8SAndrew Rybchenko
685e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
695e111ed8SAndrew Rybchenko
70ad1e3ed8SAndrew Rybchenko #if EFX_OPTS_EF10() || EFSYS_OPT_SIENA
71ad1e3ed8SAndrew Rybchenko
72ad1e3ed8SAndrew Rybchenko static void
73ad1e3ed8SAndrew Rybchenko siena_ef10_ev_qpoll(
74ad1e3ed8SAndrew Rybchenko __in efx_evq_t *eep,
75ad1e3ed8SAndrew Rybchenko __inout unsigned int *countp,
76ad1e3ed8SAndrew Rybchenko __in const efx_ev_callbacks_t *eecp,
77ad1e3ed8SAndrew Rybchenko __in_opt void *arg);
78ad1e3ed8SAndrew Rybchenko
79ad1e3ed8SAndrew Rybchenko #endif /* EFX_OPTS_EF10() || EFSYS_OPT_SIENA */
80ad1e3ed8SAndrew Rybchenko
815e111ed8SAndrew Rybchenko #if EFSYS_OPT_SIENA
825e111ed8SAndrew Rybchenko static const efx_ev_ops_t __efx_ev_siena_ops = {
835e111ed8SAndrew Rybchenko siena_ev_init, /* eevo_init */
845e111ed8SAndrew Rybchenko siena_ev_fini, /* eevo_fini */
855e111ed8SAndrew Rybchenko siena_ev_qcreate, /* eevo_qcreate */
865e111ed8SAndrew Rybchenko siena_ev_qdestroy, /* eevo_qdestroy */
875e111ed8SAndrew Rybchenko siena_ev_qprime, /* eevo_qprime */
885e111ed8SAndrew Rybchenko siena_ev_qpost, /* eevo_qpost */
89ad1e3ed8SAndrew Rybchenko siena_ef10_ev_qpoll, /* eevo_qpoll */
905e111ed8SAndrew Rybchenko siena_ev_qmoderate, /* eevo_qmoderate */
915e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
925e111ed8SAndrew Rybchenko siena_ev_qstats_update, /* eevo_qstats_update */
935e111ed8SAndrew Rybchenko #endif
945e111ed8SAndrew Rybchenko };
955e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
965e111ed8SAndrew Rybchenko
975e111ed8SAndrew Rybchenko #if EFX_OPTS_EF10()
985e111ed8SAndrew Rybchenko static const efx_ev_ops_t __efx_ev_ef10_ops = {
995e111ed8SAndrew Rybchenko ef10_ev_init, /* eevo_init */
1005e111ed8SAndrew Rybchenko ef10_ev_fini, /* eevo_fini */
1015e111ed8SAndrew Rybchenko ef10_ev_qcreate, /* eevo_qcreate */
1025e111ed8SAndrew Rybchenko ef10_ev_qdestroy, /* eevo_qdestroy */
1035e111ed8SAndrew Rybchenko ef10_ev_qprime, /* eevo_qprime */
1045e111ed8SAndrew Rybchenko ef10_ev_qpost, /* eevo_qpost */
105ad1e3ed8SAndrew Rybchenko siena_ef10_ev_qpoll, /* eevo_qpoll */
1065e111ed8SAndrew Rybchenko ef10_ev_qmoderate, /* eevo_qmoderate */
1075e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
1085e111ed8SAndrew Rybchenko ef10_ev_qstats_update, /* eevo_qstats_update */
1095e111ed8SAndrew Rybchenko #endif
1105e111ed8SAndrew Rybchenko };
1115e111ed8SAndrew Rybchenko #endif /* EFX_OPTS_EF10() */
1125e111ed8SAndrew Rybchenko
113b97bf1caSAndrew Rybchenko #if EFSYS_OPT_RIVERHEAD
114b97bf1caSAndrew Rybchenko static const efx_ev_ops_t __efx_ev_rhead_ops = {
115b97bf1caSAndrew Rybchenko rhead_ev_init, /* eevo_init */
116b97bf1caSAndrew Rybchenko rhead_ev_fini, /* eevo_fini */
117b97bf1caSAndrew Rybchenko rhead_ev_qcreate, /* eevo_qcreate */
118b97bf1caSAndrew Rybchenko rhead_ev_qdestroy, /* eevo_qdestroy */
119b97bf1caSAndrew Rybchenko rhead_ev_qprime, /* eevo_qprime */
120b97bf1caSAndrew Rybchenko rhead_ev_qpost, /* eevo_qpost */
121b97bf1caSAndrew Rybchenko rhead_ev_qpoll, /* eevo_qpoll */
122b97bf1caSAndrew Rybchenko rhead_ev_qmoderate, /* eevo_qmoderate */
123b97bf1caSAndrew Rybchenko #if EFSYS_OPT_QSTATS
124b97bf1caSAndrew Rybchenko rhead_ev_qstats_update, /* eevo_qstats_update */
125b97bf1caSAndrew Rybchenko #endif
126b97bf1caSAndrew Rybchenko };
127b97bf1caSAndrew Rybchenko #endif /* EFSYS_OPT_RIVERHEAD */
128b97bf1caSAndrew Rybchenko
1295e111ed8SAndrew Rybchenko
1305e111ed8SAndrew Rybchenko __checkReturn efx_rc_t
efx_ev_init(__in efx_nic_t * enp)1315e111ed8SAndrew Rybchenko efx_ev_init(
1325e111ed8SAndrew Rybchenko __in efx_nic_t *enp)
1335e111ed8SAndrew Rybchenko {
1345e111ed8SAndrew Rybchenko const efx_ev_ops_t *eevop;
1355e111ed8SAndrew Rybchenko efx_rc_t rc;
1365e111ed8SAndrew Rybchenko
1375e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1385e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
1395e111ed8SAndrew Rybchenko
1405e111ed8SAndrew Rybchenko if (enp->en_mod_flags & EFX_MOD_EV) {
1415e111ed8SAndrew Rybchenko rc = EINVAL;
1425e111ed8SAndrew Rybchenko goto fail1;
1435e111ed8SAndrew Rybchenko }
1445e111ed8SAndrew Rybchenko
1455e111ed8SAndrew Rybchenko switch (enp->en_family) {
1465e111ed8SAndrew Rybchenko #if EFSYS_OPT_SIENA
1475e111ed8SAndrew Rybchenko case EFX_FAMILY_SIENA:
1485e111ed8SAndrew Rybchenko eevop = &__efx_ev_siena_ops;
1495e111ed8SAndrew Rybchenko break;
1505e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
1515e111ed8SAndrew Rybchenko
1525e111ed8SAndrew Rybchenko #if EFSYS_OPT_HUNTINGTON
1535e111ed8SAndrew Rybchenko case EFX_FAMILY_HUNTINGTON:
1545e111ed8SAndrew Rybchenko eevop = &__efx_ev_ef10_ops;
1555e111ed8SAndrew Rybchenko break;
1565e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_HUNTINGTON */
1575e111ed8SAndrew Rybchenko
1585e111ed8SAndrew Rybchenko #if EFSYS_OPT_MEDFORD
1595e111ed8SAndrew Rybchenko case EFX_FAMILY_MEDFORD:
1605e111ed8SAndrew Rybchenko eevop = &__efx_ev_ef10_ops;
1615e111ed8SAndrew Rybchenko break;
1625e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_MEDFORD */
1635e111ed8SAndrew Rybchenko
1645e111ed8SAndrew Rybchenko #if EFSYS_OPT_MEDFORD2
1655e111ed8SAndrew Rybchenko case EFX_FAMILY_MEDFORD2:
1665e111ed8SAndrew Rybchenko eevop = &__efx_ev_ef10_ops;
1675e111ed8SAndrew Rybchenko break;
1685e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_MEDFORD2 */
1695e111ed8SAndrew Rybchenko
170b97bf1caSAndrew Rybchenko #if EFSYS_OPT_RIVERHEAD
171b97bf1caSAndrew Rybchenko case EFX_FAMILY_RIVERHEAD:
172b97bf1caSAndrew Rybchenko eevop = &__efx_ev_rhead_ops;
173b97bf1caSAndrew Rybchenko break;
174b97bf1caSAndrew Rybchenko #endif /* EFSYS_OPT_RIVERHEAD */
175b97bf1caSAndrew Rybchenko
1765e111ed8SAndrew Rybchenko default:
1775e111ed8SAndrew Rybchenko EFSYS_ASSERT(0);
1785e111ed8SAndrew Rybchenko rc = ENOTSUP;
1795e111ed8SAndrew Rybchenko goto fail1;
1805e111ed8SAndrew Rybchenko }
1815e111ed8SAndrew Rybchenko
1825e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
1835e111ed8SAndrew Rybchenko
1845e111ed8SAndrew Rybchenko if ((rc = eevop->eevo_init(enp)) != 0)
1855e111ed8SAndrew Rybchenko goto fail2;
1865e111ed8SAndrew Rybchenko
1875e111ed8SAndrew Rybchenko enp->en_eevop = eevop;
1885e111ed8SAndrew Rybchenko enp->en_mod_flags |= EFX_MOD_EV;
1895e111ed8SAndrew Rybchenko return (0);
1905e111ed8SAndrew Rybchenko
1915e111ed8SAndrew Rybchenko fail2:
1925e111ed8SAndrew Rybchenko EFSYS_PROBE(fail2);
1935e111ed8SAndrew Rybchenko
1945e111ed8SAndrew Rybchenko fail1:
1955e111ed8SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc);
1965e111ed8SAndrew Rybchenko
1975e111ed8SAndrew Rybchenko enp->en_eevop = NULL;
1985e111ed8SAndrew Rybchenko enp->en_mod_flags &= ~EFX_MOD_EV;
1995e111ed8SAndrew Rybchenko return (rc);
2005e111ed8SAndrew Rybchenko }
2015e111ed8SAndrew Rybchenko
2025e111ed8SAndrew Rybchenko __checkReturn size_t
efx_evq_size(__in const efx_nic_t * enp,__in unsigned int ndescs,__in uint32_t flags)2035e111ed8SAndrew Rybchenko efx_evq_size(
2045e111ed8SAndrew Rybchenko __in const efx_nic_t *enp,
205f8a60f76SAndy Moreton __in unsigned int ndescs,
206f8a60f76SAndy Moreton __in uint32_t flags)
2075e111ed8SAndrew Rybchenko {
2085e111ed8SAndrew Rybchenko const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
209f8a60f76SAndy Moreton size_t desc_size;
2105e111ed8SAndrew Rybchenko
211f8a60f76SAndy Moreton desc_size = encp->enc_ev_desc_size;
212f8a60f76SAndy Moreton
213f8a60f76SAndy Moreton #if EFSYS_OPT_EV_EXTENDED_WIDTH
214f8a60f76SAndy Moreton if (flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH)
215f8a60f76SAndy Moreton desc_size = encp->enc_ev_ew_desc_size;
216f8a60f76SAndy Moreton #else
217f8a60f76SAndy Moreton EFSYS_ASSERT((flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH) == 0);
218f8a60f76SAndy Moreton #endif
219f8a60f76SAndy Moreton
220f8a60f76SAndy Moreton return (ndescs * desc_size);
2215e111ed8SAndrew Rybchenko }
2225e111ed8SAndrew Rybchenko
2235e111ed8SAndrew Rybchenko __checkReturn unsigned int
efx_evq_nbufs(__in const efx_nic_t * enp,__in unsigned int ndescs,__in uint32_t flags)2245e111ed8SAndrew Rybchenko efx_evq_nbufs(
2255e111ed8SAndrew Rybchenko __in const efx_nic_t *enp,
226f8a60f76SAndy Moreton __in unsigned int ndescs,
227f8a60f76SAndy Moreton __in uint32_t flags)
2285e111ed8SAndrew Rybchenko {
229f8a60f76SAndy Moreton size_t size;
230f8a60f76SAndy Moreton
231f8a60f76SAndy Moreton size = efx_evq_size(enp, ndescs, flags);
232f8a60f76SAndy Moreton
233f8a60f76SAndy Moreton return (EFX_DIV_ROUND_UP(size, EFX_BUF_SIZE));
2345e111ed8SAndrew Rybchenko }
2355e111ed8SAndrew Rybchenko
2365e111ed8SAndrew Rybchenko void
efx_ev_fini(__in efx_nic_t * enp)2375e111ed8SAndrew Rybchenko efx_ev_fini(
2385e111ed8SAndrew Rybchenko __in efx_nic_t *enp)
2395e111ed8SAndrew Rybchenko {
2405e111ed8SAndrew Rybchenko const efx_ev_ops_t *eevop = enp->en_eevop;
2415e111ed8SAndrew Rybchenko
2425e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
2435e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
2445e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
2455e111ed8SAndrew Rybchenko EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
2465e111ed8SAndrew Rybchenko EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
2475e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
2485e111ed8SAndrew Rybchenko
2495e111ed8SAndrew Rybchenko eevop->eevo_fini(enp);
2505e111ed8SAndrew Rybchenko
2515e111ed8SAndrew Rybchenko enp->en_eevop = NULL;
2525e111ed8SAndrew Rybchenko enp->en_mod_flags &= ~EFX_MOD_EV;
2535e111ed8SAndrew Rybchenko }
2545e111ed8SAndrew Rybchenko
2555e111ed8SAndrew Rybchenko
2565e111ed8SAndrew Rybchenko __checkReturn efx_rc_t
efx_ev_qcreate_irq(__in efx_nic_t * enp,__in unsigned int index,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in uint32_t us,__in uint32_t flags,__in uint32_t irq,__deref_out efx_evq_t ** eepp)257*aa6dc101SAndrew Rybchenko efx_ev_qcreate_irq(
2585e111ed8SAndrew Rybchenko __in efx_nic_t *enp,
2595e111ed8SAndrew Rybchenko __in unsigned int index,
2605e111ed8SAndrew Rybchenko __in efsys_mem_t *esmp,
2615e111ed8SAndrew Rybchenko __in size_t ndescs,
2625e111ed8SAndrew Rybchenko __in uint32_t id,
2635e111ed8SAndrew Rybchenko __in uint32_t us,
2645e111ed8SAndrew Rybchenko __in uint32_t flags,
265*aa6dc101SAndrew Rybchenko __in uint32_t irq,
2665e111ed8SAndrew Rybchenko __deref_out efx_evq_t **eepp)
2675e111ed8SAndrew Rybchenko {
2685e111ed8SAndrew Rybchenko const efx_ev_ops_t *eevop = enp->en_eevop;
2695e111ed8SAndrew Rybchenko efx_evq_t *eep;
2705e111ed8SAndrew Rybchenko const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
2715e111ed8SAndrew Rybchenko efx_rc_t rc;
2725e111ed8SAndrew Rybchenko
2735e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
2745e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
2755e111ed8SAndrew Rybchenko
2765e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <,
2775e111ed8SAndrew Rybchenko enp->en_nic_cfg.enc_evq_limit);
2785e111ed8SAndrew Rybchenko
2792e5819a5SAndrew Rybchenko if (index >= encp->enc_evq_limit) {
2802e5819a5SAndrew Rybchenko rc = EINVAL;
2812e5819a5SAndrew Rybchenko goto fail1;
2822e5819a5SAndrew Rybchenko }
2832e5819a5SAndrew Rybchenko
2842e5819a5SAndrew Rybchenko if (us > encp->enc_evq_timer_max_us) {
2852e5819a5SAndrew Rybchenko rc = EINVAL;
2862e5819a5SAndrew Rybchenko goto fail2;
2872e5819a5SAndrew Rybchenko }
2882e5819a5SAndrew Rybchenko
2895e111ed8SAndrew Rybchenko switch (flags & EFX_EVQ_FLAGS_NOTIFY_MASK) {
2905e111ed8SAndrew Rybchenko case EFX_EVQ_FLAGS_NOTIFY_INTERRUPT:
2915e111ed8SAndrew Rybchenko break;
2925e111ed8SAndrew Rybchenko case EFX_EVQ_FLAGS_NOTIFY_DISABLED:
2935e111ed8SAndrew Rybchenko if (us != 0) {
2945e111ed8SAndrew Rybchenko rc = EINVAL;
2952e5819a5SAndrew Rybchenko goto fail3;
2965e111ed8SAndrew Rybchenko }
2975e111ed8SAndrew Rybchenko break;
2985e111ed8SAndrew Rybchenko default:
2995e111ed8SAndrew Rybchenko rc = EINVAL;
3002e5819a5SAndrew Rybchenko goto fail4;
3015e111ed8SAndrew Rybchenko }
3025e111ed8SAndrew Rybchenko
303f8a60f76SAndy Moreton if ((flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH) &&
304f8a60f76SAndy Moreton (encp->enc_ev_ew_desc_size == 0)) {
305f8a60f76SAndy Moreton /* Extended width event descriptors are not supported. */
306f8a60f76SAndy Moreton rc = EINVAL;
307f8a60f76SAndy Moreton goto fail5;
308f8a60f76SAndy Moreton }
309f8a60f76SAndy Moreton
3105e111ed8SAndrew Rybchenko EFSYS_ASSERT(ISP2(encp->enc_evq_max_nevs));
3115e111ed8SAndrew Rybchenko EFSYS_ASSERT(ISP2(encp->enc_evq_min_nevs));
3125e111ed8SAndrew Rybchenko
3135e111ed8SAndrew Rybchenko if (!ISP2(ndescs) ||
3145e111ed8SAndrew Rybchenko ndescs < encp->enc_evq_min_nevs ||
3155e111ed8SAndrew Rybchenko ndescs > encp->enc_evq_max_nevs) {
3165e111ed8SAndrew Rybchenko rc = EINVAL;
317f8a60f76SAndy Moreton goto fail6;
318f8a60f76SAndy Moreton }
319f8a60f76SAndy Moreton
320f8a60f76SAndy Moreton if (EFSYS_MEM_SIZE(esmp) < (ndescs * encp->enc_ev_desc_size)) {
321f8a60f76SAndy Moreton /* Buffer too small for event queue descriptors. */
322f8a60f76SAndy Moreton rc = EINVAL;
323f8a60f76SAndy Moreton goto fail7;
3245e111ed8SAndrew Rybchenko }
3255e111ed8SAndrew Rybchenko
3265e111ed8SAndrew Rybchenko /* Allocate an EVQ object */
3275e111ed8SAndrew Rybchenko EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
3285e111ed8SAndrew Rybchenko if (eep == NULL) {
3295e111ed8SAndrew Rybchenko rc = ENOMEM;
330f8a60f76SAndy Moreton goto fail8;
3315e111ed8SAndrew Rybchenko }
3325e111ed8SAndrew Rybchenko
3335e111ed8SAndrew Rybchenko eep->ee_magic = EFX_EVQ_MAGIC;
3345e111ed8SAndrew Rybchenko eep->ee_enp = enp;
3355e111ed8SAndrew Rybchenko eep->ee_index = index;
3365e111ed8SAndrew Rybchenko eep->ee_mask = ndescs - 1;
3375e111ed8SAndrew Rybchenko eep->ee_flags = flags;
3385e111ed8SAndrew Rybchenko eep->ee_esmp = esmp;
3395e111ed8SAndrew Rybchenko
3405e111ed8SAndrew Rybchenko /*
3415e111ed8SAndrew Rybchenko * Set outputs before the queue is created because interrupts may be
3425e111ed8SAndrew Rybchenko * raised for events immediately after the queue is created, before the
3435e111ed8SAndrew Rybchenko * function call below returns. See bug58606.
3445e111ed8SAndrew Rybchenko *
3455e111ed8SAndrew Rybchenko * The eepp pointer passed in by the client must therefore point to data
3465e111ed8SAndrew Rybchenko * shared with the client's event processing context.
3475e111ed8SAndrew Rybchenko */
3485e111ed8SAndrew Rybchenko enp->en_ev_qcount++;
3495e111ed8SAndrew Rybchenko *eepp = eep;
3505e111ed8SAndrew Rybchenko
3515e111ed8SAndrew Rybchenko if ((rc = eevop->eevo_qcreate(enp, index, esmp, ndescs, id, us, flags,
352*aa6dc101SAndrew Rybchenko irq, eep)) != 0)
353f8a60f76SAndy Moreton goto fail9;
3545e111ed8SAndrew Rybchenko
3555e111ed8SAndrew Rybchenko return (0);
3565e111ed8SAndrew Rybchenko
357f8a60f76SAndy Moreton fail9:
358f8a60f76SAndy Moreton EFSYS_PROBE(fail9);
3595e111ed8SAndrew Rybchenko
3605e111ed8SAndrew Rybchenko *eepp = NULL;
3615e111ed8SAndrew Rybchenko enp->en_ev_qcount--;
3625e111ed8SAndrew Rybchenko EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
363f8a60f76SAndy Moreton fail8:
364f8a60f76SAndy Moreton EFSYS_PROBE(fail8);
365f8a60f76SAndy Moreton fail7:
366f8a60f76SAndy Moreton EFSYS_PROBE(fail7);
3672e5819a5SAndrew Rybchenko fail6:
3682e5819a5SAndrew Rybchenko EFSYS_PROBE(fail6);
3692e5819a5SAndrew Rybchenko fail5:
3702e5819a5SAndrew Rybchenko EFSYS_PROBE(fail5);
3715e111ed8SAndrew Rybchenko fail4:
3725e111ed8SAndrew Rybchenko EFSYS_PROBE(fail4);
3735e111ed8SAndrew Rybchenko fail3:
3745e111ed8SAndrew Rybchenko EFSYS_PROBE(fail3);
3755e111ed8SAndrew Rybchenko fail2:
3765e111ed8SAndrew Rybchenko EFSYS_PROBE(fail2);
3775e111ed8SAndrew Rybchenko fail1:
3785e111ed8SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc);
3795e111ed8SAndrew Rybchenko return (rc);
3805e111ed8SAndrew Rybchenko }
3815e111ed8SAndrew Rybchenko
382*aa6dc101SAndrew Rybchenko __checkReturn efx_rc_t
efx_ev_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in uint32_t us,__in uint32_t flags,__deref_out efx_evq_t ** eepp)383*aa6dc101SAndrew Rybchenko efx_ev_qcreate(
384*aa6dc101SAndrew Rybchenko __in efx_nic_t *enp,
385*aa6dc101SAndrew Rybchenko __in unsigned int index,
386*aa6dc101SAndrew Rybchenko __in efsys_mem_t *esmp,
387*aa6dc101SAndrew Rybchenko __in size_t ndescs,
388*aa6dc101SAndrew Rybchenko __in uint32_t id,
389*aa6dc101SAndrew Rybchenko __in uint32_t us,
390*aa6dc101SAndrew Rybchenko __in uint32_t flags,
391*aa6dc101SAndrew Rybchenko __deref_out efx_evq_t **eepp)
392*aa6dc101SAndrew Rybchenko {
393*aa6dc101SAndrew Rybchenko uint32_t irq = index;
394*aa6dc101SAndrew Rybchenko
395*aa6dc101SAndrew Rybchenko return (efx_ev_qcreate_irq(enp, index, esmp, ndescs, id, us, flags,
396*aa6dc101SAndrew Rybchenko irq, eepp));
397*aa6dc101SAndrew Rybchenko }
398*aa6dc101SAndrew Rybchenko
3995e111ed8SAndrew Rybchenko void
efx_ev_qdestroy(__in efx_evq_t * eep)4005e111ed8SAndrew Rybchenko efx_ev_qdestroy(
4015e111ed8SAndrew Rybchenko __in efx_evq_t *eep)
4025e111ed8SAndrew Rybchenko {
4035e111ed8SAndrew Rybchenko efx_nic_t *enp = eep->ee_enp;
4045e111ed8SAndrew Rybchenko const efx_ev_ops_t *eevop = enp->en_eevop;
4055e111ed8SAndrew Rybchenko
4065e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
4075e111ed8SAndrew Rybchenko
4085e111ed8SAndrew Rybchenko EFSYS_ASSERT(enp->en_ev_qcount != 0);
4095e111ed8SAndrew Rybchenko --enp->en_ev_qcount;
4105e111ed8SAndrew Rybchenko
4115e111ed8SAndrew Rybchenko eevop->eevo_qdestroy(eep);
4125e111ed8SAndrew Rybchenko
4135e111ed8SAndrew Rybchenko /* Free the EVQ object */
4145e111ed8SAndrew Rybchenko EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
4155e111ed8SAndrew Rybchenko }
4165e111ed8SAndrew Rybchenko
4175e111ed8SAndrew Rybchenko __checkReturn efx_rc_t
efx_ev_qprime(__in efx_evq_t * eep,__in unsigned int count)4185e111ed8SAndrew Rybchenko efx_ev_qprime(
4195e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
4205e111ed8SAndrew Rybchenko __in unsigned int count)
4215e111ed8SAndrew Rybchenko {
4225e111ed8SAndrew Rybchenko efx_nic_t *enp = eep->ee_enp;
4235e111ed8SAndrew Rybchenko const efx_ev_ops_t *eevop = enp->en_eevop;
4245e111ed8SAndrew Rybchenko efx_rc_t rc;
4255e111ed8SAndrew Rybchenko
4265e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
4275e111ed8SAndrew Rybchenko
4285e111ed8SAndrew Rybchenko if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
4295e111ed8SAndrew Rybchenko rc = EINVAL;
4305e111ed8SAndrew Rybchenko goto fail1;
4315e111ed8SAndrew Rybchenko }
4325e111ed8SAndrew Rybchenko
4335e111ed8SAndrew Rybchenko if ((rc = eevop->eevo_qprime(eep, count)) != 0)
4345e111ed8SAndrew Rybchenko goto fail2;
4355e111ed8SAndrew Rybchenko
4365e111ed8SAndrew Rybchenko return (0);
4375e111ed8SAndrew Rybchenko
4385e111ed8SAndrew Rybchenko fail2:
4395e111ed8SAndrew Rybchenko EFSYS_PROBE(fail2);
4405e111ed8SAndrew Rybchenko fail1:
4415e111ed8SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc);
4425e111ed8SAndrew Rybchenko return (rc);
4435e111ed8SAndrew Rybchenko }
4445e111ed8SAndrew Rybchenko
4455e111ed8SAndrew Rybchenko __checkReturn boolean_t
efx_ev_qpending(__in efx_evq_t * eep,__in unsigned int count)4465e111ed8SAndrew Rybchenko efx_ev_qpending(
4475e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
4485e111ed8SAndrew Rybchenko __in unsigned int count)
4495e111ed8SAndrew Rybchenko {
4505e111ed8SAndrew Rybchenko size_t offset;
4515e111ed8SAndrew Rybchenko efx_qword_t qword;
4525e111ed8SAndrew Rybchenko
4535e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
4545e111ed8SAndrew Rybchenko
4555e111ed8SAndrew Rybchenko offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
4565e111ed8SAndrew Rybchenko EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
4575e111ed8SAndrew Rybchenko
4585e111ed8SAndrew Rybchenko return (EFX_EV_PRESENT(qword));
4595e111ed8SAndrew Rybchenko }
4605e111ed8SAndrew Rybchenko
4615e111ed8SAndrew Rybchenko #if EFSYS_OPT_EV_PREFETCH
4625e111ed8SAndrew Rybchenko
4635e111ed8SAndrew Rybchenko void
efx_ev_qprefetch(__in efx_evq_t * eep,__in unsigned int count)4645e111ed8SAndrew Rybchenko efx_ev_qprefetch(
4655e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
4665e111ed8SAndrew Rybchenko __in unsigned int count)
4675e111ed8SAndrew Rybchenko {
4685e111ed8SAndrew Rybchenko unsigned int offset;
4695e111ed8SAndrew Rybchenko
4705e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
4715e111ed8SAndrew Rybchenko
4725e111ed8SAndrew Rybchenko offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
4735e111ed8SAndrew Rybchenko EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
4745e111ed8SAndrew Rybchenko }
4755e111ed8SAndrew Rybchenko
4765e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_EV_PREFETCH */
4775e111ed8SAndrew Rybchenko
47892fedcd3SIvan Malov /*
47992fedcd3SIvan Malov * This method is needed to ensure that eec_initialized callback
48092fedcd3SIvan Malov * is invoked after queue creation. The callback will be invoked
48192fedcd3SIvan Malov * on Riverhead boards which have no support for INIT_DONE events
48292fedcd3SIvan Malov * and will do nothing on other boards.
48392fedcd3SIvan Malov *
48492fedcd3SIvan Malov * The client drivers must call this method after calling efx_ev_create().
48592fedcd3SIvan Malov * The call must be done with the same locks being held (if any) which are
48692fedcd3SIvan Malov * normally acquired around efx_ev_qpoll() calls to ensure that
48792fedcd3SIvan Malov * eec_initialized callback is invoked within the same locking context.
48892fedcd3SIvan Malov */
48992fedcd3SIvan Malov void
efx_ev_qcreate_check_init_done(__in efx_evq_t * eep,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)49092fedcd3SIvan Malov efx_ev_qcreate_check_init_done(
49192fedcd3SIvan Malov __in efx_evq_t *eep,
49292fedcd3SIvan Malov __in const efx_ev_callbacks_t *eecp,
49392fedcd3SIvan Malov __in_opt void *arg)
49492fedcd3SIvan Malov {
49592fedcd3SIvan Malov const efx_nic_cfg_t *encp;
49692fedcd3SIvan Malov
49792fedcd3SIvan Malov EFSYS_ASSERT(eep != NULL);
49892fedcd3SIvan Malov EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
49992fedcd3SIvan Malov EFSYS_ASSERT(eecp != NULL);
50092fedcd3SIvan Malov EFSYS_ASSERT(eecp->eec_initialized != NULL);
50192fedcd3SIvan Malov
50292fedcd3SIvan Malov encp = efx_nic_cfg_get(eep->ee_enp);
50392fedcd3SIvan Malov
50492fedcd3SIvan Malov if (encp->enc_evq_init_done_ev_supported == B_FALSE)
50592fedcd3SIvan Malov (void) eecp->eec_initialized(arg);
50692fedcd3SIvan Malov }
50792fedcd3SIvan Malov
5085e111ed8SAndrew Rybchenko void
efx_ev_qpoll(__in efx_evq_t * eep,__inout unsigned int * countp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)5095e111ed8SAndrew Rybchenko efx_ev_qpoll(
5105e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
5115e111ed8SAndrew Rybchenko __inout unsigned int *countp,
5125e111ed8SAndrew Rybchenko __in const efx_ev_callbacks_t *eecp,
5135e111ed8SAndrew Rybchenko __in_opt void *arg)
5145e111ed8SAndrew Rybchenko {
515ad1e3ed8SAndrew Rybchenko efx_nic_t *enp = eep->ee_enp;
516ad1e3ed8SAndrew Rybchenko const efx_ev_ops_t *eevop = enp->en_eevop;
5175e111ed8SAndrew Rybchenko
5185e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
5195e111ed8SAndrew Rybchenko
520ad1e3ed8SAndrew Rybchenko EFSYS_ASSERT(eevop != NULL &&
521ad1e3ed8SAndrew Rybchenko eevop->eevo_qpoll != NULL);
5225e111ed8SAndrew Rybchenko
523ad1e3ed8SAndrew Rybchenko eevop->eevo_qpoll(eep, countp, eecp, arg);
5245e111ed8SAndrew Rybchenko }
5255e111ed8SAndrew Rybchenko
5265e111ed8SAndrew Rybchenko void
efx_ev_qpost(__in efx_evq_t * eep,__in uint16_t data)5275e111ed8SAndrew Rybchenko efx_ev_qpost(
5285e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
5295e111ed8SAndrew Rybchenko __in uint16_t data)
5305e111ed8SAndrew Rybchenko {
5315e111ed8SAndrew Rybchenko efx_nic_t *enp = eep->ee_enp;
5325e111ed8SAndrew Rybchenko const efx_ev_ops_t *eevop = enp->en_eevop;
5335e111ed8SAndrew Rybchenko
5345e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
5355e111ed8SAndrew Rybchenko
5365e111ed8SAndrew Rybchenko EFSYS_ASSERT(eevop != NULL &&
5375e111ed8SAndrew Rybchenko eevop->eevo_qpost != NULL);
5385e111ed8SAndrew Rybchenko
5395e111ed8SAndrew Rybchenko eevop->eevo_qpost(eep, data);
5405e111ed8SAndrew Rybchenko }
5415e111ed8SAndrew Rybchenko
5425e111ed8SAndrew Rybchenko __checkReturn efx_rc_t
efx_ev_usecs_to_ticks(__in efx_nic_t * enp,__in unsigned int us,__out unsigned int * ticksp)5435e111ed8SAndrew Rybchenko efx_ev_usecs_to_ticks(
5445e111ed8SAndrew Rybchenko __in efx_nic_t *enp,
5455e111ed8SAndrew Rybchenko __in unsigned int us,
5465e111ed8SAndrew Rybchenko __out unsigned int *ticksp)
5475e111ed8SAndrew Rybchenko {
5485e111ed8SAndrew Rybchenko efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
5495e111ed8SAndrew Rybchenko unsigned int ticks;
5505e111ed8SAndrew Rybchenko efx_rc_t rc;
5515e111ed8SAndrew Rybchenko
5525e111ed8SAndrew Rybchenko if (encp->enc_evq_timer_quantum_ns == 0) {
5535e111ed8SAndrew Rybchenko rc = ENOTSUP;
5545e111ed8SAndrew Rybchenko goto fail1;
5555e111ed8SAndrew Rybchenko }
5565e111ed8SAndrew Rybchenko
5575e111ed8SAndrew Rybchenko /* Convert microseconds to a timer tick count */
5585e111ed8SAndrew Rybchenko if (us == 0)
5595e111ed8SAndrew Rybchenko ticks = 0;
5605e111ed8SAndrew Rybchenko else if (us * 1000 < encp->enc_evq_timer_quantum_ns)
5615e111ed8SAndrew Rybchenko ticks = 1; /* Never round down to zero */
5625e111ed8SAndrew Rybchenko else
5635e111ed8SAndrew Rybchenko ticks = us * 1000 / encp->enc_evq_timer_quantum_ns;
5645e111ed8SAndrew Rybchenko
5655e111ed8SAndrew Rybchenko *ticksp = ticks;
5665e111ed8SAndrew Rybchenko return (0);
5675e111ed8SAndrew Rybchenko
5685e111ed8SAndrew Rybchenko fail1:
5695e111ed8SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc);
5705e111ed8SAndrew Rybchenko return (rc);
5715e111ed8SAndrew Rybchenko }
5725e111ed8SAndrew Rybchenko
5735e111ed8SAndrew Rybchenko __checkReturn efx_rc_t
efx_ev_qmoderate(__in efx_evq_t * eep,__in unsigned int us)5745e111ed8SAndrew Rybchenko efx_ev_qmoderate(
5755e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
5765e111ed8SAndrew Rybchenko __in unsigned int us)
5775e111ed8SAndrew Rybchenko {
5785e111ed8SAndrew Rybchenko efx_nic_t *enp = eep->ee_enp;
5795e111ed8SAndrew Rybchenko const efx_ev_ops_t *eevop = enp->en_eevop;
5805e111ed8SAndrew Rybchenko efx_rc_t rc;
5815e111ed8SAndrew Rybchenko
5825e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
5835e111ed8SAndrew Rybchenko
5845e111ed8SAndrew Rybchenko if ((eep->ee_flags & EFX_EVQ_FLAGS_NOTIFY_MASK) ==
5855e111ed8SAndrew Rybchenko EFX_EVQ_FLAGS_NOTIFY_DISABLED) {
5865e111ed8SAndrew Rybchenko rc = EINVAL;
5875e111ed8SAndrew Rybchenko goto fail1;
5885e111ed8SAndrew Rybchenko }
5895e111ed8SAndrew Rybchenko
5905e111ed8SAndrew Rybchenko if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
5915e111ed8SAndrew Rybchenko goto fail2;
5925e111ed8SAndrew Rybchenko
5935e111ed8SAndrew Rybchenko return (0);
5945e111ed8SAndrew Rybchenko
5955e111ed8SAndrew Rybchenko fail2:
5965e111ed8SAndrew Rybchenko EFSYS_PROBE(fail2);
5975e111ed8SAndrew Rybchenko fail1:
5985e111ed8SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc);
5995e111ed8SAndrew Rybchenko return (rc);
6005e111ed8SAndrew Rybchenko }
6015e111ed8SAndrew Rybchenko
6025e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
6035e111ed8SAndrew Rybchenko void
efx_ev_qstats_update(__in efx_evq_t * eep,__inout_ecount (EV_NQSTATS)efsys_stat_t * stat)6045e111ed8SAndrew Rybchenko efx_ev_qstats_update(
6055e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
6065e111ed8SAndrew Rybchenko __inout_ecount(EV_NQSTATS) efsys_stat_t *stat)
6075e111ed8SAndrew Rybchenko
6085e111ed8SAndrew Rybchenko { efx_nic_t *enp = eep->ee_enp;
6095e111ed8SAndrew Rybchenko const efx_ev_ops_t *eevop = enp->en_eevop;
6105e111ed8SAndrew Rybchenko
6115e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
6125e111ed8SAndrew Rybchenko
6135e111ed8SAndrew Rybchenko eevop->eevo_qstats_update(eep, stat);
6145e111ed8SAndrew Rybchenko }
6155e111ed8SAndrew Rybchenko
6165e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_QSTATS */
6175e111ed8SAndrew Rybchenko
6185e111ed8SAndrew Rybchenko #if EFSYS_OPT_SIENA
6195e111ed8SAndrew Rybchenko
6205e111ed8SAndrew Rybchenko static __checkReturn efx_rc_t
siena_ev_init(__in efx_nic_t * enp)6215e111ed8SAndrew Rybchenko siena_ev_init(
6225e111ed8SAndrew Rybchenko __in efx_nic_t *enp)
6235e111ed8SAndrew Rybchenko {
6245e111ed8SAndrew Rybchenko efx_oword_t oword;
6255e111ed8SAndrew Rybchenko
6265e111ed8SAndrew Rybchenko /*
6275e111ed8SAndrew Rybchenko * Program the event queue for receive and transmit queue
6285e111ed8SAndrew Rybchenko * flush events.
6295e111ed8SAndrew Rybchenko */
6305e111ed8SAndrew Rybchenko EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
6315e111ed8SAndrew Rybchenko EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
6325e111ed8SAndrew Rybchenko EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
6335e111ed8SAndrew Rybchenko
6345e111ed8SAndrew Rybchenko return (0);
6355e111ed8SAndrew Rybchenko
6365e111ed8SAndrew Rybchenko }
6375e111ed8SAndrew Rybchenko
6385e111ed8SAndrew Rybchenko static __checkReturn boolean_t
siena_ev_rx_not_ok(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in uint32_t label,__in uint32_t id,__inout uint16_t * flagsp)6395e111ed8SAndrew Rybchenko siena_ev_rx_not_ok(
6405e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
6415e111ed8SAndrew Rybchenko __in efx_qword_t *eqp,
6425e111ed8SAndrew Rybchenko __in uint32_t label,
6435e111ed8SAndrew Rybchenko __in uint32_t id,
6445e111ed8SAndrew Rybchenko __inout uint16_t *flagsp)
6455e111ed8SAndrew Rybchenko {
6465e111ed8SAndrew Rybchenko boolean_t ignore = B_FALSE;
6475e111ed8SAndrew Rybchenko
6485e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
6495e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
6505e111ed8SAndrew Rybchenko EFSYS_PROBE(tobe_disc);
6515e111ed8SAndrew Rybchenko /*
6525e111ed8SAndrew Rybchenko * Assume this is a unicast address mismatch, unless below
6535e111ed8SAndrew Rybchenko * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
6545e111ed8SAndrew Rybchenko * EV_RX_PAUSE_FRM_ERR is set.
6555e111ed8SAndrew Rybchenko */
6565e111ed8SAndrew Rybchenko (*flagsp) |= EFX_ADDR_MISMATCH;
6575e111ed8SAndrew Rybchenko }
6585e111ed8SAndrew Rybchenko
6595e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
6605e111ed8SAndrew Rybchenko EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
6615e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
6625e111ed8SAndrew Rybchenko (*flagsp) |= EFX_DISCARD;
6635e111ed8SAndrew Rybchenko
6645e111ed8SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
6655e111ed8SAndrew Rybchenko /*
6665e111ed8SAndrew Rybchenko * Lookout for payload queue ran dry errors and ignore them.
6675e111ed8SAndrew Rybchenko *
6685e111ed8SAndrew Rybchenko * Sadly for the header/data split cases, the descriptor
6695e111ed8SAndrew Rybchenko * pointer in this event refers to the header queue and
6705e111ed8SAndrew Rybchenko * therefore cannot be easily detected as duplicate.
6715e111ed8SAndrew Rybchenko * So we drop these and rely on the receive processing seeing
6725e111ed8SAndrew Rybchenko * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
6735e111ed8SAndrew Rybchenko * the partially received packet.
6745e111ed8SAndrew Rybchenko */
6755e111ed8SAndrew Rybchenko if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
6765e111ed8SAndrew Rybchenko (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
6775e111ed8SAndrew Rybchenko (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
6785e111ed8SAndrew Rybchenko ignore = B_TRUE;
6795e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCATTER */
6805e111ed8SAndrew Rybchenko }
6815e111ed8SAndrew Rybchenko
6825e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
6835e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
6845e111ed8SAndrew Rybchenko EFSYS_PROBE(crc_err);
6855e111ed8SAndrew Rybchenko (*flagsp) &= ~EFX_ADDR_MISMATCH;
6865e111ed8SAndrew Rybchenko (*flagsp) |= EFX_DISCARD;
6875e111ed8SAndrew Rybchenko }
6885e111ed8SAndrew Rybchenko
6895e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
6905e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
6915e111ed8SAndrew Rybchenko EFSYS_PROBE(pause_frm_err);
6925e111ed8SAndrew Rybchenko (*flagsp) &= ~EFX_ADDR_MISMATCH;
6935e111ed8SAndrew Rybchenko (*flagsp) |= EFX_DISCARD;
6945e111ed8SAndrew Rybchenko }
6955e111ed8SAndrew Rybchenko
6965e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
6975e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
6985e111ed8SAndrew Rybchenko EFSYS_PROBE(owner_id_err);
6995e111ed8SAndrew Rybchenko (*flagsp) |= EFX_DISCARD;
7005e111ed8SAndrew Rybchenko }
7015e111ed8SAndrew Rybchenko
7025e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
7035e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
7045e111ed8SAndrew Rybchenko EFSYS_PROBE(ipv4_err);
7055e111ed8SAndrew Rybchenko (*flagsp) &= ~EFX_CKSUM_IPV4;
7065e111ed8SAndrew Rybchenko }
7075e111ed8SAndrew Rybchenko
7085e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
7095e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
7105e111ed8SAndrew Rybchenko EFSYS_PROBE(udp_chk_err);
7115e111ed8SAndrew Rybchenko (*flagsp) &= ~EFX_CKSUM_TCPUDP;
7125e111ed8SAndrew Rybchenko }
7135e111ed8SAndrew Rybchenko
7145e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
7155e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
7165e111ed8SAndrew Rybchenko
7175e111ed8SAndrew Rybchenko /*
7185e111ed8SAndrew Rybchenko * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
7195e111ed8SAndrew Rybchenko * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
7205e111ed8SAndrew Rybchenko * condition.
7215e111ed8SAndrew Rybchenko */
7225e111ed8SAndrew Rybchenko (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
7235e111ed8SAndrew Rybchenko }
7245e111ed8SAndrew Rybchenko
7255e111ed8SAndrew Rybchenko return (ignore);
7265e111ed8SAndrew Rybchenko }
7275e111ed8SAndrew Rybchenko
7285e111ed8SAndrew Rybchenko static __checkReturn boolean_t
siena_ev_rx(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)7295e111ed8SAndrew Rybchenko siena_ev_rx(
7305e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
7315e111ed8SAndrew Rybchenko __in efx_qword_t *eqp,
7325e111ed8SAndrew Rybchenko __in const efx_ev_callbacks_t *eecp,
7335e111ed8SAndrew Rybchenko __in_opt void *arg)
7345e111ed8SAndrew Rybchenko {
7355e111ed8SAndrew Rybchenko uint32_t id;
7365e111ed8SAndrew Rybchenko uint32_t size;
7375e111ed8SAndrew Rybchenko uint32_t label;
7385e111ed8SAndrew Rybchenko boolean_t ok;
7395e111ed8SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
7405e111ed8SAndrew Rybchenko boolean_t sop;
7415e111ed8SAndrew Rybchenko boolean_t jumbo_cont;
7425e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCATTER */
7435e111ed8SAndrew Rybchenko uint32_t hdr_type;
7445e111ed8SAndrew Rybchenko boolean_t is_v6;
7455e111ed8SAndrew Rybchenko uint16_t flags;
7465e111ed8SAndrew Rybchenko boolean_t ignore;
7475e111ed8SAndrew Rybchenko boolean_t should_abort;
7485e111ed8SAndrew Rybchenko
7495e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX);
7505e111ed8SAndrew Rybchenko
7515e111ed8SAndrew Rybchenko /* Basic packet information */
7525e111ed8SAndrew Rybchenko id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
7535e111ed8SAndrew Rybchenko size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
7545e111ed8SAndrew Rybchenko label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
7555e111ed8SAndrew Rybchenko ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
7565e111ed8SAndrew Rybchenko
7575e111ed8SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
7585e111ed8SAndrew Rybchenko sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
7595e111ed8SAndrew Rybchenko jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
7605e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCATTER */
7615e111ed8SAndrew Rybchenko
7625e111ed8SAndrew Rybchenko hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
7635e111ed8SAndrew Rybchenko
7645e111ed8SAndrew Rybchenko is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
7655e111ed8SAndrew Rybchenko
7665e111ed8SAndrew Rybchenko /*
7675e111ed8SAndrew Rybchenko * If packet is marked as OK and packet type is TCP/IP or
7685e111ed8SAndrew Rybchenko * UDP/IP or other IP, then we can rely on the hardware checksums.
7695e111ed8SAndrew Rybchenko */
7705e111ed8SAndrew Rybchenko switch (hdr_type) {
7715e111ed8SAndrew Rybchenko case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
7725e111ed8SAndrew Rybchenko flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
7735e111ed8SAndrew Rybchenko if (is_v6) {
7745e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
7755e111ed8SAndrew Rybchenko flags |= EFX_PKT_IPV6;
7765e111ed8SAndrew Rybchenko } else {
7775e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
7785e111ed8SAndrew Rybchenko flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
7795e111ed8SAndrew Rybchenko }
7805e111ed8SAndrew Rybchenko break;
7815e111ed8SAndrew Rybchenko
7825e111ed8SAndrew Rybchenko case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
7835e111ed8SAndrew Rybchenko flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
7845e111ed8SAndrew Rybchenko if (is_v6) {
7855e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
7865e111ed8SAndrew Rybchenko flags |= EFX_PKT_IPV6;
7875e111ed8SAndrew Rybchenko } else {
7885e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
7895e111ed8SAndrew Rybchenko flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
7905e111ed8SAndrew Rybchenko }
7915e111ed8SAndrew Rybchenko break;
7925e111ed8SAndrew Rybchenko
7935e111ed8SAndrew Rybchenko case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
7945e111ed8SAndrew Rybchenko if (is_v6) {
7955e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
7965e111ed8SAndrew Rybchenko flags = EFX_PKT_IPV6;
7975e111ed8SAndrew Rybchenko } else {
7985e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
7995e111ed8SAndrew Rybchenko flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
8005e111ed8SAndrew Rybchenko }
8015e111ed8SAndrew Rybchenko break;
8025e111ed8SAndrew Rybchenko
8035e111ed8SAndrew Rybchenko case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
8045e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
8055e111ed8SAndrew Rybchenko flags = 0;
8065e111ed8SAndrew Rybchenko break;
8075e111ed8SAndrew Rybchenko
8085e111ed8SAndrew Rybchenko default:
8095e111ed8SAndrew Rybchenko EFSYS_ASSERT(B_FALSE);
8105e111ed8SAndrew Rybchenko flags = 0;
8115e111ed8SAndrew Rybchenko break;
8125e111ed8SAndrew Rybchenko }
8135e111ed8SAndrew Rybchenko
8145e111ed8SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
8155e111ed8SAndrew Rybchenko /* Report scatter and header/lookahead split buffer flags */
8165e111ed8SAndrew Rybchenko if (sop)
8175e111ed8SAndrew Rybchenko flags |= EFX_PKT_START;
8185e111ed8SAndrew Rybchenko if (jumbo_cont)
8195e111ed8SAndrew Rybchenko flags |= EFX_PKT_CONT;
8205e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_RX_SCATTER */
8215e111ed8SAndrew Rybchenko
8225e111ed8SAndrew Rybchenko /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
8235e111ed8SAndrew Rybchenko if (!ok) {
8245e111ed8SAndrew Rybchenko ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags);
8255e111ed8SAndrew Rybchenko if (ignore) {
8265e111ed8SAndrew Rybchenko EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
8275e111ed8SAndrew Rybchenko uint32_t, size, uint16_t, flags);
8285e111ed8SAndrew Rybchenko
8295e111ed8SAndrew Rybchenko return (B_FALSE);
8305e111ed8SAndrew Rybchenko }
8315e111ed8SAndrew Rybchenko }
8325e111ed8SAndrew Rybchenko
8335e111ed8SAndrew Rybchenko /* If we're not discarding the packet then it is ok */
8345e111ed8SAndrew Rybchenko if (~flags & EFX_DISCARD)
8355e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
8365e111ed8SAndrew Rybchenko
8375e111ed8SAndrew Rybchenko /* Detect multicast packets that didn't match the filter */
8385e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
8395e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
8405e111ed8SAndrew Rybchenko
8415e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
8425e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
8435e111ed8SAndrew Rybchenko } else {
8445e111ed8SAndrew Rybchenko EFSYS_PROBE(mcast_mismatch);
8455e111ed8SAndrew Rybchenko flags |= EFX_ADDR_MISMATCH;
8465e111ed8SAndrew Rybchenko }
8475e111ed8SAndrew Rybchenko } else {
8485e111ed8SAndrew Rybchenko flags |= EFX_PKT_UNICAST;
8495e111ed8SAndrew Rybchenko }
8505e111ed8SAndrew Rybchenko
8515e111ed8SAndrew Rybchenko /*
8525e111ed8SAndrew Rybchenko * The packet parser in Siena can abort parsing packets under
8535e111ed8SAndrew Rybchenko * certain error conditions, setting the PKT_NOT_PARSED bit
8545e111ed8SAndrew Rybchenko * (which clears PKT_OK). If this is set, then don't trust
8555e111ed8SAndrew Rybchenko * the PKT_TYPE field.
8565e111ed8SAndrew Rybchenko */
8575e111ed8SAndrew Rybchenko if (!ok) {
8585e111ed8SAndrew Rybchenko uint32_t parse_err;
8595e111ed8SAndrew Rybchenko
8605e111ed8SAndrew Rybchenko parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
8615e111ed8SAndrew Rybchenko if (parse_err != 0)
8625e111ed8SAndrew Rybchenko flags |= EFX_CHECK_VLAN;
8635e111ed8SAndrew Rybchenko }
8645e111ed8SAndrew Rybchenko
8655e111ed8SAndrew Rybchenko if (~flags & EFX_CHECK_VLAN) {
8665e111ed8SAndrew Rybchenko uint32_t pkt_type;
8675e111ed8SAndrew Rybchenko
8685e111ed8SAndrew Rybchenko pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
8695e111ed8SAndrew Rybchenko if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
8705e111ed8SAndrew Rybchenko flags |= EFX_PKT_VLAN_TAGGED;
8715e111ed8SAndrew Rybchenko }
8725e111ed8SAndrew Rybchenko
8735e111ed8SAndrew Rybchenko EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
8745e111ed8SAndrew Rybchenko uint32_t, size, uint16_t, flags);
8755e111ed8SAndrew Rybchenko
8765e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_rx != NULL);
8775e111ed8SAndrew Rybchenko should_abort = eecp->eec_rx(arg, label, id, size, flags);
8785e111ed8SAndrew Rybchenko
8795e111ed8SAndrew Rybchenko return (should_abort);
8805e111ed8SAndrew Rybchenko }
8815e111ed8SAndrew Rybchenko
8825e111ed8SAndrew Rybchenko static __checkReturn boolean_t
siena_ev_tx(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)8835e111ed8SAndrew Rybchenko siena_ev_tx(
8845e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
8855e111ed8SAndrew Rybchenko __in efx_qword_t *eqp,
8865e111ed8SAndrew Rybchenko __in const efx_ev_callbacks_t *eecp,
8875e111ed8SAndrew Rybchenko __in_opt void *arg)
8885e111ed8SAndrew Rybchenko {
8895e111ed8SAndrew Rybchenko uint32_t id;
8905e111ed8SAndrew Rybchenko uint32_t label;
8915e111ed8SAndrew Rybchenko boolean_t should_abort;
8925e111ed8SAndrew Rybchenko
8935e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_TX);
8945e111ed8SAndrew Rybchenko
8955e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
8965e111ed8SAndrew Rybchenko EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
8975e111ed8SAndrew Rybchenko EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
8985e111ed8SAndrew Rybchenko EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
8995e111ed8SAndrew Rybchenko
9005e111ed8SAndrew Rybchenko id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
9015e111ed8SAndrew Rybchenko label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
9025e111ed8SAndrew Rybchenko
9035e111ed8SAndrew Rybchenko EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
9045e111ed8SAndrew Rybchenko
9055e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_tx != NULL);
9065e111ed8SAndrew Rybchenko should_abort = eecp->eec_tx(arg, label, id);
9075e111ed8SAndrew Rybchenko
9085e111ed8SAndrew Rybchenko return (should_abort);
9095e111ed8SAndrew Rybchenko }
9105e111ed8SAndrew Rybchenko
9115e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
9125e111ed8SAndrew Rybchenko EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
9135e111ed8SAndrew Rybchenko uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
9145e111ed8SAndrew Rybchenko uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
9155e111ed8SAndrew Rybchenko
9165e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
9175e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
9185e111ed8SAndrew Rybchenko
9195e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
9205e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
9215e111ed8SAndrew Rybchenko
9225e111ed8SAndrew Rybchenko if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
9235e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
9245e111ed8SAndrew Rybchenko
9255e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
9265e111ed8SAndrew Rybchenko return (B_FALSE);
9275e111ed8SAndrew Rybchenko }
9285e111ed8SAndrew Rybchenko
9295e111ed8SAndrew Rybchenko static __checkReturn boolean_t
siena_ev_global(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)9305e111ed8SAndrew Rybchenko siena_ev_global(
9315e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
9325e111ed8SAndrew Rybchenko __in efx_qword_t *eqp,
9335e111ed8SAndrew Rybchenko __in const efx_ev_callbacks_t *eecp,
9345e111ed8SAndrew Rybchenko __in_opt void *arg)
9355e111ed8SAndrew Rybchenko {
9365e111ed8SAndrew Rybchenko _NOTE(ARGUNUSED(eqp, eecp, arg))
9375e111ed8SAndrew Rybchenko
9385e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
9395e111ed8SAndrew Rybchenko
9405e111ed8SAndrew Rybchenko return (B_FALSE);
9415e111ed8SAndrew Rybchenko }
9425e111ed8SAndrew Rybchenko
9435e111ed8SAndrew Rybchenko static __checkReturn boolean_t
siena_ev_driver(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)9445e111ed8SAndrew Rybchenko siena_ev_driver(
9455e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
9465e111ed8SAndrew Rybchenko __in efx_qword_t *eqp,
9475e111ed8SAndrew Rybchenko __in const efx_ev_callbacks_t *eecp,
9485e111ed8SAndrew Rybchenko __in_opt void *arg)
9495e111ed8SAndrew Rybchenko {
9505e111ed8SAndrew Rybchenko boolean_t should_abort;
9515e111ed8SAndrew Rybchenko
9525e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
9535e111ed8SAndrew Rybchenko should_abort = B_FALSE;
9545e111ed8SAndrew Rybchenko
9555e111ed8SAndrew Rybchenko switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
9565e111ed8SAndrew Rybchenko case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
9575e111ed8SAndrew Rybchenko uint32_t txq_index;
9585e111ed8SAndrew Rybchenko
9595e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
9605e111ed8SAndrew Rybchenko
9615e111ed8SAndrew Rybchenko txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
9625e111ed8SAndrew Rybchenko
9635e111ed8SAndrew Rybchenko EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
9645e111ed8SAndrew Rybchenko
9655e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
9665e111ed8SAndrew Rybchenko should_abort = eecp->eec_txq_flush_done(arg, txq_index);
9675e111ed8SAndrew Rybchenko
9685e111ed8SAndrew Rybchenko break;
9695e111ed8SAndrew Rybchenko }
9705e111ed8SAndrew Rybchenko case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
9715e111ed8SAndrew Rybchenko uint32_t rxq_index;
9725e111ed8SAndrew Rybchenko uint32_t failed;
9735e111ed8SAndrew Rybchenko
9745e111ed8SAndrew Rybchenko rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
9755e111ed8SAndrew Rybchenko failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
9765e111ed8SAndrew Rybchenko
9775e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
9785e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
9795e111ed8SAndrew Rybchenko
9805e111ed8SAndrew Rybchenko if (failed) {
9815e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
9825e111ed8SAndrew Rybchenko
9835e111ed8SAndrew Rybchenko EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
9845e111ed8SAndrew Rybchenko
9855e111ed8SAndrew Rybchenko should_abort = eecp->eec_rxq_flush_failed(arg,
9865e111ed8SAndrew Rybchenko rxq_index);
9875e111ed8SAndrew Rybchenko } else {
9885e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
9895e111ed8SAndrew Rybchenko
9905e111ed8SAndrew Rybchenko EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
9915e111ed8SAndrew Rybchenko
9925e111ed8SAndrew Rybchenko should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
9935e111ed8SAndrew Rybchenko }
9945e111ed8SAndrew Rybchenko
9955e111ed8SAndrew Rybchenko break;
9965e111ed8SAndrew Rybchenko }
9975e111ed8SAndrew Rybchenko case FSE_AZ_EVQ_INIT_DONE_EV:
9985e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_initialized != NULL);
9995e111ed8SAndrew Rybchenko should_abort = eecp->eec_initialized(arg);
10005e111ed8SAndrew Rybchenko
10015e111ed8SAndrew Rybchenko break;
10025e111ed8SAndrew Rybchenko
10035e111ed8SAndrew Rybchenko case FSE_AZ_EVQ_NOT_EN_EV:
10045e111ed8SAndrew Rybchenko EFSYS_PROBE(evq_not_en);
10055e111ed8SAndrew Rybchenko break;
10065e111ed8SAndrew Rybchenko
10075e111ed8SAndrew Rybchenko case FSE_AZ_SRM_UPD_DONE_EV: {
10085e111ed8SAndrew Rybchenko uint32_t code;
10095e111ed8SAndrew Rybchenko
10105e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
10115e111ed8SAndrew Rybchenko
10125e111ed8SAndrew Rybchenko code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
10135e111ed8SAndrew Rybchenko
10145e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_sram != NULL);
10155e111ed8SAndrew Rybchenko should_abort = eecp->eec_sram(arg, code);
10165e111ed8SAndrew Rybchenko
10175e111ed8SAndrew Rybchenko break;
10185e111ed8SAndrew Rybchenko }
10195e111ed8SAndrew Rybchenko case FSE_AZ_WAKE_UP_EV: {
10205e111ed8SAndrew Rybchenko uint32_t id;
10215e111ed8SAndrew Rybchenko
10225e111ed8SAndrew Rybchenko id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
10235e111ed8SAndrew Rybchenko
10245e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_wake_up != NULL);
10255e111ed8SAndrew Rybchenko should_abort = eecp->eec_wake_up(arg, id);
10265e111ed8SAndrew Rybchenko
10275e111ed8SAndrew Rybchenko break;
10285e111ed8SAndrew Rybchenko }
10295e111ed8SAndrew Rybchenko case FSE_AZ_TX_PKT_NON_TCP_UDP:
10305e111ed8SAndrew Rybchenko EFSYS_PROBE(tx_pkt_non_tcp_udp);
10315e111ed8SAndrew Rybchenko break;
10325e111ed8SAndrew Rybchenko
10335e111ed8SAndrew Rybchenko case FSE_AZ_TIMER_EV: {
10345e111ed8SAndrew Rybchenko uint32_t id;
10355e111ed8SAndrew Rybchenko
10365e111ed8SAndrew Rybchenko id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
10375e111ed8SAndrew Rybchenko
10385e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_timer != NULL);
10395e111ed8SAndrew Rybchenko should_abort = eecp->eec_timer(arg, id);
10405e111ed8SAndrew Rybchenko
10415e111ed8SAndrew Rybchenko break;
10425e111ed8SAndrew Rybchenko }
10435e111ed8SAndrew Rybchenko case FSE_AZ_RX_DSC_ERROR_EV:
10445e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
10455e111ed8SAndrew Rybchenko
10465e111ed8SAndrew Rybchenko EFSYS_PROBE(rx_dsc_error);
10475e111ed8SAndrew Rybchenko
10485e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_exception != NULL);
10495e111ed8SAndrew Rybchenko should_abort = eecp->eec_exception(arg,
10505e111ed8SAndrew Rybchenko EFX_EXCEPTION_RX_DSC_ERROR, 0);
10515e111ed8SAndrew Rybchenko
10525e111ed8SAndrew Rybchenko break;
10535e111ed8SAndrew Rybchenko
10545e111ed8SAndrew Rybchenko case FSE_AZ_TX_DSC_ERROR_EV:
10555e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
10565e111ed8SAndrew Rybchenko
10575e111ed8SAndrew Rybchenko EFSYS_PROBE(tx_dsc_error);
10585e111ed8SAndrew Rybchenko
10595e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_exception != NULL);
10605e111ed8SAndrew Rybchenko should_abort = eecp->eec_exception(arg,
10615e111ed8SAndrew Rybchenko EFX_EXCEPTION_TX_DSC_ERROR, 0);
10625e111ed8SAndrew Rybchenko
10635e111ed8SAndrew Rybchenko break;
10645e111ed8SAndrew Rybchenko
10655e111ed8SAndrew Rybchenko default:
10665e111ed8SAndrew Rybchenko break;
10675e111ed8SAndrew Rybchenko }
10685e111ed8SAndrew Rybchenko
10695e111ed8SAndrew Rybchenko return (should_abort);
10705e111ed8SAndrew Rybchenko }
10715e111ed8SAndrew Rybchenko
10725e111ed8SAndrew Rybchenko static __checkReturn boolean_t
siena_ev_drv_gen(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)10735e111ed8SAndrew Rybchenko siena_ev_drv_gen(
10745e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
10755e111ed8SAndrew Rybchenko __in efx_qword_t *eqp,
10765e111ed8SAndrew Rybchenko __in const efx_ev_callbacks_t *eecp,
10775e111ed8SAndrew Rybchenko __in_opt void *arg)
10785e111ed8SAndrew Rybchenko {
10795e111ed8SAndrew Rybchenko uint32_t data;
10805e111ed8SAndrew Rybchenko boolean_t should_abort;
10815e111ed8SAndrew Rybchenko
10825e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
10835e111ed8SAndrew Rybchenko
10845e111ed8SAndrew Rybchenko data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
10855e111ed8SAndrew Rybchenko if (data >= ((uint32_t)1 << 16)) {
10865e111ed8SAndrew Rybchenko EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
10875e111ed8SAndrew Rybchenko uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
10885e111ed8SAndrew Rybchenko uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
10895e111ed8SAndrew Rybchenko return (B_TRUE);
10905e111ed8SAndrew Rybchenko }
10915e111ed8SAndrew Rybchenko
10925e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_software != NULL);
10935e111ed8SAndrew Rybchenko should_abort = eecp->eec_software(arg, (uint16_t)data);
10945e111ed8SAndrew Rybchenko
10955e111ed8SAndrew Rybchenko return (should_abort);
10965e111ed8SAndrew Rybchenko }
10975e111ed8SAndrew Rybchenko
10985e111ed8SAndrew Rybchenko #if EFSYS_OPT_MCDI
10995e111ed8SAndrew Rybchenko
11005e111ed8SAndrew Rybchenko static __checkReturn boolean_t
siena_ev_mcdi(__in efx_evq_t * eep,__in efx_qword_t * eqp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)11015e111ed8SAndrew Rybchenko siena_ev_mcdi(
11025e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
11035e111ed8SAndrew Rybchenko __in efx_qword_t *eqp,
11045e111ed8SAndrew Rybchenko __in const efx_ev_callbacks_t *eecp,
11055e111ed8SAndrew Rybchenko __in_opt void *arg)
11065e111ed8SAndrew Rybchenko {
11075e111ed8SAndrew Rybchenko efx_nic_t *enp = eep->ee_enp;
11085e111ed8SAndrew Rybchenko unsigned int code;
11095e111ed8SAndrew Rybchenko boolean_t should_abort = B_FALSE;
11105e111ed8SAndrew Rybchenko
11115e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
11125e111ed8SAndrew Rybchenko
11135e111ed8SAndrew Rybchenko if (enp->en_family != EFX_FAMILY_SIENA)
11145e111ed8SAndrew Rybchenko goto out;
11155e111ed8SAndrew Rybchenko
11165e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_link_change != NULL);
11175e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_exception != NULL);
11185e111ed8SAndrew Rybchenko #if EFSYS_OPT_MON_STATS
11195e111ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_monitor != NULL);
11205e111ed8SAndrew Rybchenko #endif
11215e111ed8SAndrew Rybchenko
11225e111ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
11235e111ed8SAndrew Rybchenko
11245e111ed8SAndrew Rybchenko code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
11255e111ed8SAndrew Rybchenko switch (code) {
11265e111ed8SAndrew Rybchenko case MCDI_EVENT_CODE_BADSSERT:
11275e111ed8SAndrew Rybchenko efx_mcdi_ev_death(enp, EINTR);
11285e111ed8SAndrew Rybchenko break;
11295e111ed8SAndrew Rybchenko
11305e111ed8SAndrew Rybchenko case MCDI_EVENT_CODE_CMDDONE:
11315e111ed8SAndrew Rybchenko efx_mcdi_ev_cpl(enp,
11325e111ed8SAndrew Rybchenko MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
11335e111ed8SAndrew Rybchenko MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
11345e111ed8SAndrew Rybchenko MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
11355e111ed8SAndrew Rybchenko break;
11365e111ed8SAndrew Rybchenko
11375e111ed8SAndrew Rybchenko case MCDI_EVENT_CODE_LINKCHANGE: {
11385e111ed8SAndrew Rybchenko efx_link_mode_t link_mode;
11395e111ed8SAndrew Rybchenko
11405e111ed8SAndrew Rybchenko siena_phy_link_ev(enp, eqp, &link_mode);
11415e111ed8SAndrew Rybchenko should_abort = eecp->eec_link_change(arg, link_mode);
11425e111ed8SAndrew Rybchenko break;
11435e111ed8SAndrew Rybchenko }
11445e111ed8SAndrew Rybchenko case MCDI_EVENT_CODE_SENSOREVT: {
11455e111ed8SAndrew Rybchenko #if EFSYS_OPT_MON_STATS
11465e111ed8SAndrew Rybchenko efx_mon_stat_t id;
11475e111ed8SAndrew Rybchenko efx_mon_stat_value_t value;
11485e111ed8SAndrew Rybchenko efx_rc_t rc;
11495e111ed8SAndrew Rybchenko
11505e111ed8SAndrew Rybchenko if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0)
11515e111ed8SAndrew Rybchenko should_abort = eecp->eec_monitor(arg, id, value);
11525e111ed8SAndrew Rybchenko else if (rc == ENOTSUP) {
11535e111ed8SAndrew Rybchenko should_abort = eecp->eec_exception(arg,
11545e111ed8SAndrew Rybchenko EFX_EXCEPTION_UNKNOWN_SENSOREVT,
11555e111ed8SAndrew Rybchenko MCDI_EV_FIELD(eqp, DATA));
11565e111ed8SAndrew Rybchenko } else
11575e111ed8SAndrew Rybchenko EFSYS_ASSERT(rc == ENODEV); /* Wrong port */
11585e111ed8SAndrew Rybchenko #else
11595e111ed8SAndrew Rybchenko should_abort = B_FALSE;
11605e111ed8SAndrew Rybchenko #endif
11615e111ed8SAndrew Rybchenko break;
11625e111ed8SAndrew Rybchenko }
11635e111ed8SAndrew Rybchenko case MCDI_EVENT_CODE_SCHEDERR:
11645e111ed8SAndrew Rybchenko /* Informational only */
11655e111ed8SAndrew Rybchenko break;
11665e111ed8SAndrew Rybchenko
11675e111ed8SAndrew Rybchenko case MCDI_EVENT_CODE_REBOOT:
11685e111ed8SAndrew Rybchenko efx_mcdi_ev_death(enp, EIO);
11695e111ed8SAndrew Rybchenko break;
11705e111ed8SAndrew Rybchenko
11715e111ed8SAndrew Rybchenko case MCDI_EVENT_CODE_MAC_STATS_DMA:
11725e111ed8SAndrew Rybchenko #if EFSYS_OPT_MAC_STATS
11735e111ed8SAndrew Rybchenko if (eecp->eec_mac_stats != NULL) {
11745e111ed8SAndrew Rybchenko eecp->eec_mac_stats(arg,
11755e111ed8SAndrew Rybchenko MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
11765e111ed8SAndrew Rybchenko }
11775e111ed8SAndrew Rybchenko #endif
11785e111ed8SAndrew Rybchenko break;
11795e111ed8SAndrew Rybchenko
11805e111ed8SAndrew Rybchenko case MCDI_EVENT_CODE_FWALERT: {
11815e111ed8SAndrew Rybchenko uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
11825e111ed8SAndrew Rybchenko
11835e111ed8SAndrew Rybchenko if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
11845e111ed8SAndrew Rybchenko should_abort = eecp->eec_exception(arg,
11855e111ed8SAndrew Rybchenko EFX_EXCEPTION_FWALERT_SRAM,
11865e111ed8SAndrew Rybchenko MCDI_EV_FIELD(eqp, FWALERT_DATA));
11875e111ed8SAndrew Rybchenko else
11885e111ed8SAndrew Rybchenko should_abort = eecp->eec_exception(arg,
11895e111ed8SAndrew Rybchenko EFX_EXCEPTION_UNKNOWN_FWALERT,
11905e111ed8SAndrew Rybchenko MCDI_EV_FIELD(eqp, DATA));
11915e111ed8SAndrew Rybchenko break;
11925e111ed8SAndrew Rybchenko }
11935e111ed8SAndrew Rybchenko
11945e111ed8SAndrew Rybchenko default:
11955e111ed8SAndrew Rybchenko EFSYS_PROBE1(mc_pcol_error, int, code);
11965e111ed8SAndrew Rybchenko break;
11975e111ed8SAndrew Rybchenko }
11985e111ed8SAndrew Rybchenko
11995e111ed8SAndrew Rybchenko out:
12005e111ed8SAndrew Rybchenko return (should_abort);
12015e111ed8SAndrew Rybchenko }
12025e111ed8SAndrew Rybchenko
12035e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_MCDI */
12045e111ed8SAndrew Rybchenko
12055e111ed8SAndrew Rybchenko static __checkReturn efx_rc_t
siena_ev_qprime(__in efx_evq_t * eep,__in unsigned int count)12065e111ed8SAndrew Rybchenko siena_ev_qprime(
12075e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
12085e111ed8SAndrew Rybchenko __in unsigned int count)
12095e111ed8SAndrew Rybchenko {
12105e111ed8SAndrew Rybchenko efx_nic_t *enp = eep->ee_enp;
12115e111ed8SAndrew Rybchenko uint32_t rptr;
12125e111ed8SAndrew Rybchenko efx_dword_t dword;
12135e111ed8SAndrew Rybchenko
12145e111ed8SAndrew Rybchenko rptr = count & eep->ee_mask;
12155e111ed8SAndrew Rybchenko
12165e111ed8SAndrew Rybchenko EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
12175e111ed8SAndrew Rybchenko
12185e111ed8SAndrew Rybchenko EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
12195e111ed8SAndrew Rybchenko &dword, B_FALSE);
12205e111ed8SAndrew Rybchenko
12215e111ed8SAndrew Rybchenko return (0);
12225e111ed8SAndrew Rybchenko }
12235e111ed8SAndrew Rybchenko
12245e111ed8SAndrew Rybchenko static void
siena_ev_qpost(__in efx_evq_t * eep,__in uint16_t data)12255e111ed8SAndrew Rybchenko siena_ev_qpost(
12265e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
12275e111ed8SAndrew Rybchenko __in uint16_t data)
12285e111ed8SAndrew Rybchenko {
12295e111ed8SAndrew Rybchenko efx_nic_t *enp = eep->ee_enp;
12305e111ed8SAndrew Rybchenko efx_qword_t ev;
12315e111ed8SAndrew Rybchenko efx_oword_t oword;
12325e111ed8SAndrew Rybchenko
12335e111ed8SAndrew Rybchenko EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
12345e111ed8SAndrew Rybchenko FSF_AZ_EV_DATA_DW0, (uint32_t)data);
12355e111ed8SAndrew Rybchenko
12365e111ed8SAndrew Rybchenko EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
12375e111ed8SAndrew Rybchenko EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
12385e111ed8SAndrew Rybchenko EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
12395e111ed8SAndrew Rybchenko
12405e111ed8SAndrew Rybchenko EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
12415e111ed8SAndrew Rybchenko }
12425e111ed8SAndrew Rybchenko
12435e111ed8SAndrew Rybchenko static __checkReturn efx_rc_t
siena_ev_qmoderate(__in efx_evq_t * eep,__in unsigned int us)12445e111ed8SAndrew Rybchenko siena_ev_qmoderate(
12455e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
12465e111ed8SAndrew Rybchenko __in unsigned int us)
12475e111ed8SAndrew Rybchenko {
12485e111ed8SAndrew Rybchenko efx_nic_t *enp = eep->ee_enp;
12495e111ed8SAndrew Rybchenko efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
12505e111ed8SAndrew Rybchenko unsigned int locked;
12515e111ed8SAndrew Rybchenko efx_dword_t dword;
12525e111ed8SAndrew Rybchenko efx_rc_t rc;
12535e111ed8SAndrew Rybchenko
12545e111ed8SAndrew Rybchenko if (us > encp->enc_evq_timer_max_us) {
12555e111ed8SAndrew Rybchenko rc = EINVAL;
12565e111ed8SAndrew Rybchenko goto fail1;
12575e111ed8SAndrew Rybchenko }
12585e111ed8SAndrew Rybchenko
12595e111ed8SAndrew Rybchenko /* If the value is zero then disable the timer */
12605e111ed8SAndrew Rybchenko if (us == 0) {
12615e111ed8SAndrew Rybchenko EFX_POPULATE_DWORD_2(dword,
12625e111ed8SAndrew Rybchenko FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
12635e111ed8SAndrew Rybchenko FRF_CZ_TC_TIMER_VAL, 0);
12645e111ed8SAndrew Rybchenko } else {
12655e111ed8SAndrew Rybchenko unsigned int ticks;
12665e111ed8SAndrew Rybchenko
12675e111ed8SAndrew Rybchenko if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0)
12685e111ed8SAndrew Rybchenko goto fail2;
12695e111ed8SAndrew Rybchenko
12705e111ed8SAndrew Rybchenko EFSYS_ASSERT(ticks > 0);
12715e111ed8SAndrew Rybchenko EFX_POPULATE_DWORD_2(dword,
12725e111ed8SAndrew Rybchenko FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
12735e111ed8SAndrew Rybchenko FRF_CZ_TC_TIMER_VAL, ticks - 1);
12745e111ed8SAndrew Rybchenko }
12755e111ed8SAndrew Rybchenko
12765e111ed8SAndrew Rybchenko locked = (eep->ee_index == 0) ? 1 : 0;
12775e111ed8SAndrew Rybchenko
12785e111ed8SAndrew Rybchenko EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
12795e111ed8SAndrew Rybchenko eep->ee_index, &dword, locked);
12805e111ed8SAndrew Rybchenko
12815e111ed8SAndrew Rybchenko return (0);
12825e111ed8SAndrew Rybchenko
12835e111ed8SAndrew Rybchenko fail2:
12845e111ed8SAndrew Rybchenko EFSYS_PROBE(fail2);
12855e111ed8SAndrew Rybchenko fail1:
12865e111ed8SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc);
12875e111ed8SAndrew Rybchenko
12885e111ed8SAndrew Rybchenko return (rc);
12895e111ed8SAndrew Rybchenko }
12905e111ed8SAndrew Rybchenko
12915e111ed8SAndrew Rybchenko static __checkReturn efx_rc_t
siena_ev_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in uint32_t us,__in uint32_t flags,__in uint32_t irq,__in efx_evq_t * eep)12925e111ed8SAndrew Rybchenko siena_ev_qcreate(
12935e111ed8SAndrew Rybchenko __in efx_nic_t *enp,
12945e111ed8SAndrew Rybchenko __in unsigned int index,
12955e111ed8SAndrew Rybchenko __in efsys_mem_t *esmp,
12965e111ed8SAndrew Rybchenko __in size_t ndescs,
12975e111ed8SAndrew Rybchenko __in uint32_t id,
12985e111ed8SAndrew Rybchenko __in uint32_t us,
12995e111ed8SAndrew Rybchenko __in uint32_t flags,
1300*aa6dc101SAndrew Rybchenko __in uint32_t irq,
13015e111ed8SAndrew Rybchenko __in efx_evq_t *eep)
13025e111ed8SAndrew Rybchenko {
13035e111ed8SAndrew Rybchenko efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
13045e111ed8SAndrew Rybchenko uint32_t size;
13055e111ed8SAndrew Rybchenko efx_oword_t oword;
13065e111ed8SAndrew Rybchenko efx_rc_t rc;
13075e111ed8SAndrew Rybchenko boolean_t notify_mode;
13085e111ed8SAndrew Rybchenko
13095e111ed8SAndrew Rybchenko _NOTE(ARGUNUSED(esmp))
13105e111ed8SAndrew Rybchenko
1311f8a60f76SAndy Moreton EFSYS_ASSERT((flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH) == 0);
1312f8a60f76SAndy Moreton
1313*aa6dc101SAndrew Rybchenko if (irq != index) {
1314*aa6dc101SAndrew Rybchenko rc = EINVAL;
1315*aa6dc101SAndrew Rybchenko goto fail1;
1316*aa6dc101SAndrew Rybchenko }
1317*aa6dc101SAndrew Rybchenko
13185e111ed8SAndrew Rybchenko #if EFSYS_OPT_RX_SCALE
13195e111ed8SAndrew Rybchenko if (enp->en_intr.ei_type == EFX_INTR_LINE &&
13205e111ed8SAndrew Rybchenko index >= EFX_MAXRSS_LEGACY) {
13215e111ed8SAndrew Rybchenko rc = EINVAL;
1322*aa6dc101SAndrew Rybchenko goto fail2;
13235e111ed8SAndrew Rybchenko }
13245e111ed8SAndrew Rybchenko #endif
13255e111ed8SAndrew Rybchenko for (size = 0;
13265e111ed8SAndrew Rybchenko (1U << size) <= encp->enc_evq_max_nevs / encp->enc_evq_min_nevs;
13275e111ed8SAndrew Rybchenko size++)
13285e111ed8SAndrew Rybchenko if ((1U << size) == (uint32_t)ndescs / encp->enc_evq_min_nevs)
13295e111ed8SAndrew Rybchenko break;
13305e111ed8SAndrew Rybchenko if (id + (1 << size) >= encp->enc_buftbl_limit) {
13315e111ed8SAndrew Rybchenko rc = EINVAL;
1332*aa6dc101SAndrew Rybchenko goto fail3;
13335e111ed8SAndrew Rybchenko }
13345e111ed8SAndrew Rybchenko
13355e111ed8SAndrew Rybchenko /* Set up the handler table */
13365e111ed8SAndrew Rybchenko eep->ee_rx = siena_ev_rx;
13375e111ed8SAndrew Rybchenko eep->ee_tx = siena_ev_tx;
13385e111ed8SAndrew Rybchenko eep->ee_driver = siena_ev_driver;
13395e111ed8SAndrew Rybchenko eep->ee_global = siena_ev_global;
13405e111ed8SAndrew Rybchenko eep->ee_drv_gen = siena_ev_drv_gen;
13415e111ed8SAndrew Rybchenko #if EFSYS_OPT_MCDI
13425e111ed8SAndrew Rybchenko eep->ee_mcdi = siena_ev_mcdi;
13435e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_MCDI */
13445e111ed8SAndrew Rybchenko
13455e111ed8SAndrew Rybchenko notify_mode = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) !=
13465e111ed8SAndrew Rybchenko EFX_EVQ_FLAGS_NOTIFY_INTERRUPT);
13475e111ed8SAndrew Rybchenko
13485e111ed8SAndrew Rybchenko /* Set up the new event queue */
13495e111ed8SAndrew Rybchenko EFX_POPULATE_OWORD_3(oword, FRF_CZ_TIMER_Q_EN, 1,
13505e111ed8SAndrew Rybchenko FRF_CZ_HOST_NOTIFY_MODE, notify_mode,
13515e111ed8SAndrew Rybchenko FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
13525e111ed8SAndrew Rybchenko EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE);
13535e111ed8SAndrew Rybchenko
13545e111ed8SAndrew Rybchenko EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
13555e111ed8SAndrew Rybchenko FRF_AZ_EVQ_BUF_BASE_ID, id);
13565e111ed8SAndrew Rybchenko
13575e111ed8SAndrew Rybchenko EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE);
13585e111ed8SAndrew Rybchenko
13595e111ed8SAndrew Rybchenko /* Set initial interrupt moderation */
13605e111ed8SAndrew Rybchenko siena_ev_qmoderate(eep, us);
13615e111ed8SAndrew Rybchenko
13625e111ed8SAndrew Rybchenko return (0);
13635e111ed8SAndrew Rybchenko
1364*aa6dc101SAndrew Rybchenko fail3:
1365*aa6dc101SAndrew Rybchenko EFSYS_PROBE(fail3);
1366*aa6dc101SAndrew Rybchenko #if EFSYS_OPT_RX_SCALE
13675e111ed8SAndrew Rybchenko fail2:
13685e111ed8SAndrew Rybchenko EFSYS_PROBE(fail2);
13692e5819a5SAndrew Rybchenko #endif
1370*aa6dc101SAndrew Rybchenko fail1:
13715e111ed8SAndrew Rybchenko EFSYS_PROBE1(fail1, efx_rc_t, rc);
13725e111ed8SAndrew Rybchenko
13735e111ed8SAndrew Rybchenko return (rc);
13745e111ed8SAndrew Rybchenko }
13755e111ed8SAndrew Rybchenko
13765e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
13775e111ed8SAndrew Rybchenko
13785e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
13795e111ed8SAndrew Rybchenko #if EFSYS_OPT_NAMES
13805e111ed8SAndrew Rybchenko /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock ac223f7134058b4f */
13815e111ed8SAndrew Rybchenko static const char * const __efx_ev_qstat_name[] = {
13825e111ed8SAndrew Rybchenko "all",
13835e111ed8SAndrew Rybchenko "rx",
13845e111ed8SAndrew Rybchenko "rx_ok",
13855e111ed8SAndrew Rybchenko "rx_frm_trunc",
13865e111ed8SAndrew Rybchenko "rx_tobe_disc",
13875e111ed8SAndrew Rybchenko "rx_pause_frm_err",
13885e111ed8SAndrew Rybchenko "rx_buf_owner_id_err",
13895e111ed8SAndrew Rybchenko "rx_ipv4_hdr_chksum_err",
13905e111ed8SAndrew Rybchenko "rx_tcp_udp_chksum_err",
13915e111ed8SAndrew Rybchenko "rx_eth_crc_err",
13925e111ed8SAndrew Rybchenko "rx_ip_frag_err",
13935e111ed8SAndrew Rybchenko "rx_mcast_pkt",
13945e111ed8SAndrew Rybchenko "rx_mcast_hash_match",
13955e111ed8SAndrew Rybchenko "rx_tcp_ipv4",
13965e111ed8SAndrew Rybchenko "rx_tcp_ipv6",
13975e111ed8SAndrew Rybchenko "rx_udp_ipv4",
13985e111ed8SAndrew Rybchenko "rx_udp_ipv6",
13995e111ed8SAndrew Rybchenko "rx_other_ipv4",
14005e111ed8SAndrew Rybchenko "rx_other_ipv6",
14015e111ed8SAndrew Rybchenko "rx_non_ip",
14025e111ed8SAndrew Rybchenko "rx_batch",
14035e111ed8SAndrew Rybchenko "tx",
14045e111ed8SAndrew Rybchenko "tx_wq_ff_full",
14055e111ed8SAndrew Rybchenko "tx_pkt_err",
14065e111ed8SAndrew Rybchenko "tx_pkt_too_big",
14075e111ed8SAndrew Rybchenko "tx_unexpected",
14085e111ed8SAndrew Rybchenko "global",
14095e111ed8SAndrew Rybchenko "global_mnt",
14105e111ed8SAndrew Rybchenko "driver",
14115e111ed8SAndrew Rybchenko "driver_srm_upd_done",
14125e111ed8SAndrew Rybchenko "driver_tx_descq_fls_done",
14135e111ed8SAndrew Rybchenko "driver_rx_descq_fls_done",
14145e111ed8SAndrew Rybchenko "driver_rx_descq_fls_failed",
14155e111ed8SAndrew Rybchenko "driver_rx_dsc_error",
14165e111ed8SAndrew Rybchenko "driver_tx_dsc_error",
14175e111ed8SAndrew Rybchenko "drv_gen",
14185e111ed8SAndrew Rybchenko "mcdi_response",
14195e111ed8SAndrew Rybchenko "rx_parse_incomplete",
14205e111ed8SAndrew Rybchenko };
14215e111ed8SAndrew Rybchenko /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
14225e111ed8SAndrew Rybchenko
14235e111ed8SAndrew Rybchenko const char *
efx_ev_qstat_name(__in efx_nic_t * enp,__in unsigned int id)14245e111ed8SAndrew Rybchenko efx_ev_qstat_name(
14255e111ed8SAndrew Rybchenko __in efx_nic_t *enp,
14265e111ed8SAndrew Rybchenko __in unsigned int id)
14275e111ed8SAndrew Rybchenko {
14285e111ed8SAndrew Rybchenko _NOTE(ARGUNUSED(enp))
14295e111ed8SAndrew Rybchenko
14305e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
14315e111ed8SAndrew Rybchenko EFSYS_ASSERT3U(id, <, EV_NQSTATS);
14325e111ed8SAndrew Rybchenko
14335e111ed8SAndrew Rybchenko return (__efx_ev_qstat_name[id]);
14345e111ed8SAndrew Rybchenko }
14355e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_NAMES */
14365e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_QSTATS */
14375e111ed8SAndrew Rybchenko
14385e111ed8SAndrew Rybchenko #if EFSYS_OPT_SIENA
14395e111ed8SAndrew Rybchenko
14405e111ed8SAndrew Rybchenko #if EFSYS_OPT_QSTATS
14415e111ed8SAndrew Rybchenko static void
siena_ev_qstats_update(__in efx_evq_t * eep,__inout_ecount (EV_NQSTATS)efsys_stat_t * stat)14425e111ed8SAndrew Rybchenko siena_ev_qstats_update(
14435e111ed8SAndrew Rybchenko __in efx_evq_t *eep,
14445e111ed8SAndrew Rybchenko __inout_ecount(EV_NQSTATS) efsys_stat_t *stat)
14455e111ed8SAndrew Rybchenko {
14465e111ed8SAndrew Rybchenko unsigned int id;
14475e111ed8SAndrew Rybchenko
14485e111ed8SAndrew Rybchenko for (id = 0; id < EV_NQSTATS; id++) {
14495e111ed8SAndrew Rybchenko efsys_stat_t *essp = &stat[id];
14505e111ed8SAndrew Rybchenko
14515e111ed8SAndrew Rybchenko EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
14525e111ed8SAndrew Rybchenko eep->ee_stat[id] = 0;
14535e111ed8SAndrew Rybchenko }
14545e111ed8SAndrew Rybchenko }
14555e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_QSTATS */
14565e111ed8SAndrew Rybchenko
14575e111ed8SAndrew Rybchenko static void
siena_ev_qdestroy(__in efx_evq_t * eep)14585e111ed8SAndrew Rybchenko siena_ev_qdestroy(
14595e111ed8SAndrew Rybchenko __in efx_evq_t *eep)
14605e111ed8SAndrew Rybchenko {
14615e111ed8SAndrew Rybchenko efx_nic_t *enp = eep->ee_enp;
14625e111ed8SAndrew Rybchenko efx_oword_t oword;
14635e111ed8SAndrew Rybchenko
14645e111ed8SAndrew Rybchenko /* Purge event queue */
14655e111ed8SAndrew Rybchenko EFX_ZERO_OWORD(oword);
14665e111ed8SAndrew Rybchenko
14675e111ed8SAndrew Rybchenko EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
14685e111ed8SAndrew Rybchenko eep->ee_index, &oword, B_TRUE);
14695e111ed8SAndrew Rybchenko
14705e111ed8SAndrew Rybchenko EFX_ZERO_OWORD(oword);
14715e111ed8SAndrew Rybchenko EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE);
14725e111ed8SAndrew Rybchenko }
14735e111ed8SAndrew Rybchenko
14745e111ed8SAndrew Rybchenko static void
siena_ev_fini(__in efx_nic_t * enp)14755e111ed8SAndrew Rybchenko siena_ev_fini(
14765e111ed8SAndrew Rybchenko __in efx_nic_t *enp)
14775e111ed8SAndrew Rybchenko {
14785e111ed8SAndrew Rybchenko _NOTE(ARGUNUSED(enp))
14795e111ed8SAndrew Rybchenko }
14805e111ed8SAndrew Rybchenko
14815e111ed8SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
1482ad1e3ed8SAndrew Rybchenko
1483ad1e3ed8SAndrew Rybchenko #if EFX_OPTS_EF10() || EFSYS_OPT_SIENA
1484ad1e3ed8SAndrew Rybchenko
1485ad1e3ed8SAndrew Rybchenko #define EFX_EV_BATCH 8
1486ad1e3ed8SAndrew Rybchenko
1487ad1e3ed8SAndrew Rybchenko static void
siena_ef10_ev_qpoll(__in efx_evq_t * eep,__inout unsigned int * countp,__in const efx_ev_callbacks_t * eecp,__in_opt void * arg)1488ad1e3ed8SAndrew Rybchenko siena_ef10_ev_qpoll(
1489ad1e3ed8SAndrew Rybchenko __in efx_evq_t *eep,
1490ad1e3ed8SAndrew Rybchenko __inout unsigned int *countp,
1491ad1e3ed8SAndrew Rybchenko __in const efx_ev_callbacks_t *eecp,
1492ad1e3ed8SAndrew Rybchenko __in_opt void *arg)
1493ad1e3ed8SAndrew Rybchenko {
1494ad1e3ed8SAndrew Rybchenko efx_qword_t ev[EFX_EV_BATCH];
1495ad1e3ed8SAndrew Rybchenko unsigned int batch;
1496ad1e3ed8SAndrew Rybchenko unsigned int total;
1497ad1e3ed8SAndrew Rybchenko unsigned int count;
1498ad1e3ed8SAndrew Rybchenko unsigned int index;
1499ad1e3ed8SAndrew Rybchenko size_t offset;
1500ad1e3ed8SAndrew Rybchenko
1501ad1e3ed8SAndrew Rybchenko /* Ensure events codes match for EF10 (Huntington/Medford) and Siena */
1502ad1e3ed8SAndrew Rybchenko EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
1503ad1e3ed8SAndrew Rybchenko EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
1504ad1e3ed8SAndrew Rybchenko
1505ad1e3ed8SAndrew Rybchenko EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
1506ad1e3ed8SAndrew Rybchenko EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
1507ad1e3ed8SAndrew Rybchenko EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
1508ad1e3ed8SAndrew Rybchenko EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
1509ad1e3ed8SAndrew Rybchenko FSE_AZ_EV_CODE_DRV_GEN_EV);
1510ad1e3ed8SAndrew Rybchenko #if EFSYS_OPT_MCDI
1511ad1e3ed8SAndrew Rybchenko EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV ==
1512ad1e3ed8SAndrew Rybchenko FSE_AZ_EV_CODE_MCDI_EVRESPONSE);
1513ad1e3ed8SAndrew Rybchenko #endif
1514ad1e3ed8SAndrew Rybchenko
1515ad1e3ed8SAndrew Rybchenko EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
1516ad1e3ed8SAndrew Rybchenko EFSYS_ASSERT(countp != NULL);
1517ad1e3ed8SAndrew Rybchenko EFSYS_ASSERT(eecp != NULL);
1518ad1e3ed8SAndrew Rybchenko
1519ad1e3ed8SAndrew Rybchenko count = *countp;
1520ad1e3ed8SAndrew Rybchenko do {
1521ad1e3ed8SAndrew Rybchenko /* Read up until the end of the batch period */
1522ad1e3ed8SAndrew Rybchenko batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
1523ad1e3ed8SAndrew Rybchenko offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1524ad1e3ed8SAndrew Rybchenko for (total = 0; total < batch; ++total) {
1525ad1e3ed8SAndrew Rybchenko EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
1526ad1e3ed8SAndrew Rybchenko
1527ad1e3ed8SAndrew Rybchenko if (!EFX_EV_PRESENT(ev[total]))
1528ad1e3ed8SAndrew Rybchenko break;
1529ad1e3ed8SAndrew Rybchenko
1530ad1e3ed8SAndrew Rybchenko EFSYS_PROBE3(event, unsigned int, eep->ee_index,
1531ad1e3ed8SAndrew Rybchenko uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
1532ad1e3ed8SAndrew Rybchenko uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
1533ad1e3ed8SAndrew Rybchenko
1534ad1e3ed8SAndrew Rybchenko offset += sizeof (efx_qword_t);
1535ad1e3ed8SAndrew Rybchenko }
1536ad1e3ed8SAndrew Rybchenko
1537ad1e3ed8SAndrew Rybchenko #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
1538ad1e3ed8SAndrew Rybchenko /*
1539ad1e3ed8SAndrew Rybchenko * Prefetch the next batch when we get within PREFETCH_PERIOD
1540ad1e3ed8SAndrew Rybchenko * of a completed batch. If the batch is smaller, then prefetch
1541ad1e3ed8SAndrew Rybchenko * immediately.
1542ad1e3ed8SAndrew Rybchenko */
1543ad1e3ed8SAndrew Rybchenko if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
1544ad1e3ed8SAndrew Rybchenko EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1545ad1e3ed8SAndrew Rybchenko #endif /* EFSYS_OPT_EV_PREFETCH */
1546ad1e3ed8SAndrew Rybchenko
1547ad1e3ed8SAndrew Rybchenko /* Process the batch of events */
1548ad1e3ed8SAndrew Rybchenko for (index = 0; index < total; ++index) {
1549ad1e3ed8SAndrew Rybchenko boolean_t should_abort;
1550ad1e3ed8SAndrew Rybchenko uint32_t code;
1551ad1e3ed8SAndrew Rybchenko
1552ad1e3ed8SAndrew Rybchenko #if EFSYS_OPT_EV_PREFETCH
1553ad1e3ed8SAndrew Rybchenko /* Prefetch if we've now reached the batch period */
1554ad1e3ed8SAndrew Rybchenko if (total == batch &&
1555ad1e3ed8SAndrew Rybchenko index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
1556ad1e3ed8SAndrew Rybchenko offset = (count + batch) & eep->ee_mask;
1557ad1e3ed8SAndrew Rybchenko offset *= sizeof (efx_qword_t);
1558ad1e3ed8SAndrew Rybchenko
1559ad1e3ed8SAndrew Rybchenko EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1560ad1e3ed8SAndrew Rybchenko }
1561ad1e3ed8SAndrew Rybchenko #endif /* EFSYS_OPT_EV_PREFETCH */
1562ad1e3ed8SAndrew Rybchenko
1563ad1e3ed8SAndrew Rybchenko EFX_EV_QSTAT_INCR(eep, EV_ALL);
1564ad1e3ed8SAndrew Rybchenko
1565ad1e3ed8SAndrew Rybchenko code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
1566ad1e3ed8SAndrew Rybchenko switch (code) {
1567ad1e3ed8SAndrew Rybchenko case FSE_AZ_EV_CODE_RX_EV:
1568ad1e3ed8SAndrew Rybchenko should_abort = eep->ee_rx(eep,
1569ad1e3ed8SAndrew Rybchenko &(ev[index]), eecp, arg);
1570ad1e3ed8SAndrew Rybchenko break;
1571ad1e3ed8SAndrew Rybchenko case FSE_AZ_EV_CODE_TX_EV:
1572ad1e3ed8SAndrew Rybchenko should_abort = eep->ee_tx(eep,
1573ad1e3ed8SAndrew Rybchenko &(ev[index]), eecp, arg);
1574ad1e3ed8SAndrew Rybchenko break;
1575ad1e3ed8SAndrew Rybchenko case FSE_AZ_EV_CODE_DRIVER_EV:
1576ad1e3ed8SAndrew Rybchenko should_abort = eep->ee_driver(eep,
1577ad1e3ed8SAndrew Rybchenko &(ev[index]), eecp, arg);
1578ad1e3ed8SAndrew Rybchenko break;
1579ad1e3ed8SAndrew Rybchenko case FSE_AZ_EV_CODE_DRV_GEN_EV:
1580ad1e3ed8SAndrew Rybchenko should_abort = eep->ee_drv_gen(eep,
1581ad1e3ed8SAndrew Rybchenko &(ev[index]), eecp, arg);
1582ad1e3ed8SAndrew Rybchenko break;
1583ad1e3ed8SAndrew Rybchenko #if EFSYS_OPT_MCDI
1584ad1e3ed8SAndrew Rybchenko case FSE_AZ_EV_CODE_MCDI_EVRESPONSE:
1585ad1e3ed8SAndrew Rybchenko should_abort = eep->ee_mcdi(eep,
1586ad1e3ed8SAndrew Rybchenko &(ev[index]), eecp, arg);
1587ad1e3ed8SAndrew Rybchenko break;
1588ad1e3ed8SAndrew Rybchenko #endif
1589ad1e3ed8SAndrew Rybchenko case FSE_AZ_EV_CODE_GLOBAL_EV:
1590ad1e3ed8SAndrew Rybchenko if (eep->ee_global) {
1591ad1e3ed8SAndrew Rybchenko should_abort = eep->ee_global(eep,
1592ad1e3ed8SAndrew Rybchenko &(ev[index]), eecp, arg);
1593ad1e3ed8SAndrew Rybchenko break;
1594ad1e3ed8SAndrew Rybchenko }
1595ad1e3ed8SAndrew Rybchenko /* else fallthrough */
1596ad1e3ed8SAndrew Rybchenko default:
1597ad1e3ed8SAndrew Rybchenko EFSYS_PROBE3(bad_event,
1598ad1e3ed8SAndrew Rybchenko unsigned int, eep->ee_index,
1599ad1e3ed8SAndrew Rybchenko uint32_t,
1600ad1e3ed8SAndrew Rybchenko EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
1601ad1e3ed8SAndrew Rybchenko uint32_t,
1602ad1e3ed8SAndrew Rybchenko EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
1603ad1e3ed8SAndrew Rybchenko
1604ad1e3ed8SAndrew Rybchenko EFSYS_ASSERT(eecp->eec_exception != NULL);
1605ad1e3ed8SAndrew Rybchenko (void) eecp->eec_exception(arg,
1606ad1e3ed8SAndrew Rybchenko EFX_EXCEPTION_EV_ERROR, code);
1607ad1e3ed8SAndrew Rybchenko should_abort = B_TRUE;
1608ad1e3ed8SAndrew Rybchenko }
1609ad1e3ed8SAndrew Rybchenko if (should_abort) {
1610ad1e3ed8SAndrew Rybchenko /* Ignore subsequent events */
1611ad1e3ed8SAndrew Rybchenko total = index + 1;
1612ad1e3ed8SAndrew Rybchenko
1613ad1e3ed8SAndrew Rybchenko /*
1614ad1e3ed8SAndrew Rybchenko * Poison batch to ensure the outer
1615ad1e3ed8SAndrew Rybchenko * loop is broken out of.
1616ad1e3ed8SAndrew Rybchenko */
1617ad1e3ed8SAndrew Rybchenko EFSYS_ASSERT(batch <= EFX_EV_BATCH);
1618ad1e3ed8SAndrew Rybchenko batch += (EFX_EV_BATCH << 1);
1619ad1e3ed8SAndrew Rybchenko EFSYS_ASSERT(total != batch);
1620ad1e3ed8SAndrew Rybchenko break;
1621ad1e3ed8SAndrew Rybchenko }
1622ad1e3ed8SAndrew Rybchenko }
1623ad1e3ed8SAndrew Rybchenko
1624ad1e3ed8SAndrew Rybchenko /*
1625ad1e3ed8SAndrew Rybchenko * Now that the hardware has most likely moved onto dma'ing
1626ad1e3ed8SAndrew Rybchenko * into the next cache line, clear the processed events. Take
1627ad1e3ed8SAndrew Rybchenko * care to only clear out events that we've processed
1628ad1e3ed8SAndrew Rybchenko */
1629ad1e3ed8SAndrew Rybchenko EFX_SET_QWORD(ev[0]);
1630ad1e3ed8SAndrew Rybchenko offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1631ad1e3ed8SAndrew Rybchenko for (index = 0; index < total; ++index) {
1632ad1e3ed8SAndrew Rybchenko EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
1633ad1e3ed8SAndrew Rybchenko offset += sizeof (efx_qword_t);
1634ad1e3ed8SAndrew Rybchenko }
1635ad1e3ed8SAndrew Rybchenko
1636ad1e3ed8SAndrew Rybchenko count += total;
1637ad1e3ed8SAndrew Rybchenko
1638ad1e3ed8SAndrew Rybchenko } while (total == batch);
1639ad1e3ed8SAndrew Rybchenko
1640ad1e3ed8SAndrew Rybchenko *countp = count;
1641ad1e3ed8SAndrew Rybchenko }
1642ad1e3ed8SAndrew Rybchenko
1643ad1e3ed8SAndrew Rybchenko #endif /* EFX_OPTS_EF10() || EFSYS_OPT_SIENA */
1644