1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright(c) 2019-2021 Xilinx, Inc.
4 * Copyright(c) 2007-2019 Solarflare Communications Inc.
5 */
6
7 #include "efx.h"
8 #include "efx_impl.h"
9
10 #if EFSYS_OPT_QSTATS
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
12 do { \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
15 } while (B_FALSE)
16 #else
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
18 #endif
19
20 #if EFSYS_OPT_SIENA
21
22 static __checkReturn efx_rc_t
23 siena_tx_init(
24 __in efx_nic_t *enp);
25
26 static void
27 siena_tx_fini(
28 __in efx_nic_t *enp);
29
30 static __checkReturn efx_rc_t
31 siena_tx_qcreate(
32 __in efx_nic_t *enp,
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
36 __in size_t ndescs,
37 __in uint32_t id,
38 __in uint16_t flags,
39 __in efx_evq_t *eep,
40 __in efx_txq_t *etp,
41 __out unsigned int *addedp);
42
43 static void
44 siena_tx_qdestroy(
45 __in efx_txq_t *etp);
46
47 static __checkReturn efx_rc_t
48 siena_tx_qpost(
49 __in efx_txq_t *etp,
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
54
55 static void
56 siena_tx_qpush(
57 __in efx_txq_t *etp,
58 __in unsigned int added,
59 __in unsigned int pushed);
60
61 static __checkReturn efx_rc_t
62 siena_tx_qpace(
63 __in efx_txq_t *etp,
64 __in unsigned int ns);
65
66 static __checkReturn efx_rc_t
67 siena_tx_qflush(
68 __in efx_txq_t *etp);
69
70 static void
71 siena_tx_qenable(
72 __in efx_txq_t *etp);
73
74 __checkReturn efx_rc_t
75 siena_tx_qdesc_post(
76 __in efx_txq_t *etp,
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
81
82 void
83 siena_tx_qdesc_dma_create(
84 __in efx_txq_t *etp,
85 __in efsys_dma_addr_t addr,
86 __in size_t size,
87 __in boolean_t eop,
88 __out efx_desc_t *edp);
89
90 #if EFSYS_OPT_QSTATS
91 static void
92 siena_tx_qstats_update(
93 __in efx_txq_t *etp,
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
95 #endif
96
97 #endif /* EFSYS_OPT_SIENA */
98
99
100 #if EFSYS_OPT_SIENA
101 static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
120 NULL, /* etxo_qdesc_checksum_create */
121 #if EFSYS_OPT_QSTATS
122 siena_tx_qstats_update, /* etxo_qstats_update */
123 #endif
124 };
125 #endif /* EFSYS_OPT_SIENA */
126
127 #if EFSYS_OPT_HUNTINGTON
128 static const efx_tx_ops_t __efx_tx_hunt_ops = {
129 ef10_tx_init, /* etxo_init */
130 ef10_tx_fini, /* etxo_fini */
131 ef10_tx_qcreate, /* etxo_qcreate */
132 ef10_tx_qdestroy, /* etxo_qdestroy */
133 ef10_tx_qpost, /* etxo_qpost */
134 ef10_tx_qpush, /* etxo_qpush */
135 ef10_tx_qpace, /* etxo_qpace */
136 ef10_tx_qflush, /* etxo_qflush */
137 ef10_tx_qenable, /* etxo_qenable */
138 ef10_tx_qpio_enable, /* etxo_qpio_enable */
139 ef10_tx_qpio_disable, /* etxo_qpio_disable */
140 ef10_tx_qpio_write, /* etxo_qpio_write */
141 ef10_tx_qpio_post, /* etxo_qpio_post */
142 ef10_tx_qdesc_post, /* etxo_qdesc_post */
143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
148 #if EFSYS_OPT_QSTATS
149 ef10_tx_qstats_update, /* etxo_qstats_update */
150 #endif
151 };
152 #endif /* EFSYS_OPT_HUNTINGTON */
153
154 #if EFSYS_OPT_MEDFORD
155 static const efx_tx_ops_t __efx_tx_medford_ops = {
156 ef10_tx_init, /* etxo_init */
157 ef10_tx_fini, /* etxo_fini */
158 ef10_tx_qcreate, /* etxo_qcreate */
159 ef10_tx_qdestroy, /* etxo_qdestroy */
160 ef10_tx_qpost, /* etxo_qpost */
161 ef10_tx_qpush, /* etxo_qpush */
162 ef10_tx_qpace, /* etxo_qpace */
163 ef10_tx_qflush, /* etxo_qflush */
164 ef10_tx_qenable, /* etxo_qenable */
165 ef10_tx_qpio_enable, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable, /* etxo_qpio_disable */
167 ef10_tx_qpio_write, /* etxo_qpio_write */
168 ef10_tx_qpio_post, /* etxo_qpio_post */
169 ef10_tx_qdesc_post, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
171 NULL, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
175 #if EFSYS_OPT_QSTATS
176 ef10_tx_qstats_update, /* etxo_qstats_update */
177 #endif
178 };
179 #endif /* EFSYS_OPT_MEDFORD */
180
181 #if EFSYS_OPT_MEDFORD2
182 static const efx_tx_ops_t __efx_tx_medford2_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
202 #if EFSYS_OPT_QSTATS
203 ef10_tx_qstats_update, /* etxo_qstats_update */
204 #endif
205 };
206 #endif /* EFSYS_OPT_MEDFORD2 */
207
208 #if EFSYS_OPT_RIVERHEAD
209 static const efx_tx_ops_t __efx_tx_rhead_ops = {
210 rhead_tx_init, /* etxo_init */
211 rhead_tx_fini, /* etxo_fini */
212 rhead_tx_qcreate, /* etxo_qcreate */
213 rhead_tx_qdestroy, /* etxo_qdestroy */
214 rhead_tx_qpost, /* etxo_qpost */
215 rhead_tx_qpush, /* etxo_qpush */
216 rhead_tx_qpace, /* etxo_qpace */
217 rhead_tx_qflush, /* etxo_qflush */
218 rhead_tx_qenable, /* etxo_qenable */
219 NULL, /* etxo_qpio_enable */
220 NULL, /* etxo_qpio_disable */
221 NULL, /* etxo_qpio_write */
222 NULL, /* etxo_qpio_post */
223 rhead_tx_qdesc_post, /* etxo_qdesc_post */
224 NULL, /* etxo_qdesc_dma_create */
225 NULL, /* etxo_qdesc_tso_create */
226 NULL, /* etxo_qdesc_tso2_create */
227 NULL, /* etxo_qdesc_vlantci_create */
228 NULL, /* etxo_qdesc_checksum_create */
229 #if EFSYS_OPT_QSTATS
230 rhead_tx_qstats_update, /* etxo_qstats_update */
231 #endif
232 };
233 #endif /* EFSYS_OPT_RIVERHEAD */
234
235
236 __checkReturn efx_rc_t
efx_tx_init(__in efx_nic_t * enp)237 efx_tx_init(
238 __in efx_nic_t *enp)
239 {
240 const efx_tx_ops_t *etxop;
241 efx_rc_t rc;
242
243 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
244 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
245
246 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
247 rc = EINVAL;
248 goto fail1;
249 }
250
251 if (enp->en_mod_flags & EFX_MOD_TX) {
252 rc = EINVAL;
253 goto fail2;
254 }
255
256 switch (enp->en_family) {
257 #if EFSYS_OPT_SIENA
258 case EFX_FAMILY_SIENA:
259 etxop = &__efx_tx_siena_ops;
260 break;
261 #endif /* EFSYS_OPT_SIENA */
262
263 #if EFSYS_OPT_HUNTINGTON
264 case EFX_FAMILY_HUNTINGTON:
265 etxop = &__efx_tx_hunt_ops;
266 break;
267 #endif /* EFSYS_OPT_HUNTINGTON */
268
269 #if EFSYS_OPT_MEDFORD
270 case EFX_FAMILY_MEDFORD:
271 etxop = &__efx_tx_medford_ops;
272 break;
273 #endif /* EFSYS_OPT_MEDFORD */
274
275 #if EFSYS_OPT_MEDFORD2
276 case EFX_FAMILY_MEDFORD2:
277 etxop = &__efx_tx_medford2_ops;
278 break;
279 #endif /* EFSYS_OPT_MEDFORD2 */
280
281 #if EFSYS_OPT_RIVERHEAD
282 case EFX_FAMILY_RIVERHEAD:
283 etxop = &__efx_tx_rhead_ops;
284 break;
285 #endif /* EFSYS_OPT_RIVERHEAD */
286
287 default:
288 EFSYS_ASSERT(0);
289 rc = ENOTSUP;
290 goto fail3;
291 }
292
293 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
294
295 if ((rc = etxop->etxo_init(enp)) != 0)
296 goto fail4;
297
298 enp->en_etxop = etxop;
299 enp->en_mod_flags |= EFX_MOD_TX;
300 return (0);
301
302 fail4:
303 EFSYS_PROBE(fail4);
304 fail3:
305 EFSYS_PROBE(fail3);
306 fail2:
307 EFSYS_PROBE(fail2);
308 fail1:
309 EFSYS_PROBE1(fail1, efx_rc_t, rc);
310
311 enp->en_etxop = NULL;
312 enp->en_mod_flags &= ~EFX_MOD_TX;
313 return (rc);
314 }
315
316 void
efx_tx_fini(__in efx_nic_t * enp)317 efx_tx_fini(
318 __in efx_nic_t *enp)
319 {
320 const efx_tx_ops_t *etxop = enp->en_etxop;
321
322 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
323 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
324 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
325 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
326
327 etxop->etxo_fini(enp);
328
329 enp->en_etxop = NULL;
330 enp->en_mod_flags &= ~EFX_MOD_TX;
331 }
332
333 __checkReturn size_t
efx_txq_size(__in const efx_nic_t * enp,__in unsigned int ndescs)334 efx_txq_size(
335 __in const efx_nic_t *enp,
336 __in unsigned int ndescs)
337 {
338 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
339
340 return (ndescs * encp->enc_tx_desc_size);
341 }
342
343 __checkReturn unsigned int
efx_txq_nbufs(__in const efx_nic_t * enp,__in unsigned int ndescs)344 efx_txq_nbufs(
345 __in const efx_nic_t *enp,
346 __in unsigned int ndescs)
347 {
348 return (EFX_DIV_ROUND_UP(efx_txq_size(enp, ndescs), EFX_BUF_SIZE));
349 }
350
351 __checkReturn efx_rc_t
efx_tx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in uint16_t flags,__in efx_evq_t * eep,__deref_out efx_txq_t ** etpp,__out unsigned int * addedp)352 efx_tx_qcreate(
353 __in efx_nic_t *enp,
354 __in unsigned int index,
355 __in unsigned int label,
356 __in efsys_mem_t *esmp,
357 __in size_t ndescs,
358 __in uint32_t id,
359 __in uint16_t flags,
360 __in efx_evq_t *eep,
361 __deref_out efx_txq_t **etpp,
362 __out unsigned int *addedp)
363 {
364 const efx_tx_ops_t *etxop = enp->en_etxop;
365 efx_txq_t *etp;
366 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
367 efx_rc_t rc;
368
369 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
370 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
371
372 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
373 enp->en_nic_cfg.enc_txq_limit);
374
375 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
376 EFSYS_ASSERT(ISP2(encp->enc_txq_min_ndescs));
377
378 if (!ISP2(ndescs) ||
379 ndescs < encp->enc_txq_min_ndescs ||
380 ndescs > encp->enc_txq_max_ndescs) {
381 rc = EINVAL;
382 goto fail1;
383 }
384
385 /* Allocate an TXQ object */
386 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
387
388 if (etp == NULL) {
389 rc = ENOMEM;
390 goto fail2;
391 }
392
393 etp->et_magic = EFX_TXQ_MAGIC;
394 etp->et_enp = enp;
395 etp->et_index = index;
396 etp->et_mask = ndescs - 1;
397 etp->et_esmp = esmp;
398
399 /* Initial descriptor index may be modified by etxo_qcreate */
400 *addedp = 0;
401
402 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
403 ndescs, id, flags, eep, etp, addedp)) != 0)
404 goto fail3;
405
406 enp->en_tx_qcount++;
407 *etpp = etp;
408
409 return (0);
410
411 fail3:
412 EFSYS_PROBE(fail3);
413 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
414 fail2:
415 EFSYS_PROBE(fail2);
416 fail1:
417 EFSYS_PROBE1(fail1, efx_rc_t, rc);
418 return (rc);
419 }
420
421 void
efx_tx_qdestroy(__in efx_txq_t * etp)422 efx_tx_qdestroy(
423 __in efx_txq_t *etp)
424 {
425 efx_nic_t *enp = etp->et_enp;
426 const efx_tx_ops_t *etxop = enp->en_etxop;
427
428 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
429
430 EFSYS_ASSERT(enp->en_tx_qcount != 0);
431 --enp->en_tx_qcount;
432
433 etxop->etxo_qdestroy(etp);
434
435 /* Free the TXQ object */
436 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
437 }
438
439 __checkReturn efx_rc_t
efx_tx_qpost(__in efx_txq_t * etp,__in_ecount (ndescs)efx_buffer_t * eb,__in unsigned int ndescs,__in unsigned int completed,__inout unsigned int * addedp)440 efx_tx_qpost(
441 __in efx_txq_t *etp,
442 __in_ecount(ndescs) efx_buffer_t *eb,
443 __in unsigned int ndescs,
444 __in unsigned int completed,
445 __inout unsigned int *addedp)
446 {
447 efx_nic_t *enp = etp->et_enp;
448 const efx_tx_ops_t *etxop = enp->en_etxop;
449 efx_rc_t rc;
450
451 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
452
453 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
454 goto fail1;
455
456 return (0);
457
458 fail1:
459 EFSYS_PROBE1(fail1, efx_rc_t, rc);
460 return (rc);
461 }
462
463 void
efx_tx_qpush(__in efx_txq_t * etp,__in unsigned int added,__in unsigned int pushed)464 efx_tx_qpush(
465 __in efx_txq_t *etp,
466 __in unsigned int added,
467 __in unsigned int pushed)
468 {
469 efx_nic_t *enp = etp->et_enp;
470 const efx_tx_ops_t *etxop = enp->en_etxop;
471
472 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
473
474 etxop->etxo_qpush(etp, added, pushed);
475 }
476
477 __checkReturn efx_rc_t
efx_tx_qpace(__in efx_txq_t * etp,__in unsigned int ns)478 efx_tx_qpace(
479 __in efx_txq_t *etp,
480 __in unsigned int ns)
481 {
482 efx_nic_t *enp = etp->et_enp;
483 const efx_tx_ops_t *etxop = enp->en_etxop;
484 efx_rc_t rc;
485
486 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
487
488 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
489 goto fail1;
490
491 return (0);
492
493 fail1:
494 EFSYS_PROBE1(fail1, efx_rc_t, rc);
495 return (rc);
496 }
497
498 __checkReturn efx_rc_t
efx_tx_qflush(__in efx_txq_t * etp)499 efx_tx_qflush(
500 __in efx_txq_t *etp)
501 {
502 efx_nic_t *enp = etp->et_enp;
503 const efx_tx_ops_t *etxop = enp->en_etxop;
504 efx_rc_t rc;
505
506 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
507
508 if ((rc = etxop->etxo_qflush(etp)) != 0)
509 goto fail1;
510
511 return (0);
512
513 fail1:
514 EFSYS_PROBE1(fail1, efx_rc_t, rc);
515 return (rc);
516 }
517
518 void
efx_tx_qenable(__in efx_txq_t * etp)519 efx_tx_qenable(
520 __in efx_txq_t *etp)
521 {
522 efx_nic_t *enp = etp->et_enp;
523 const efx_tx_ops_t *etxop = enp->en_etxop;
524
525 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
526
527 etxop->etxo_qenable(etp);
528 }
529
530 __checkReturn efx_rc_t
efx_tx_qpio_enable(__in efx_txq_t * etp)531 efx_tx_qpio_enable(
532 __in efx_txq_t *etp)
533 {
534 efx_nic_t *enp = etp->et_enp;
535 const efx_tx_ops_t *etxop = enp->en_etxop;
536 efx_rc_t rc;
537
538 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
539
540 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
541 rc = ENOTSUP;
542 goto fail1;
543 }
544 if (etxop->etxo_qpio_enable == NULL) {
545 rc = ENOTSUP;
546 goto fail2;
547 }
548 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
549 goto fail3;
550
551 return (0);
552
553 fail3:
554 EFSYS_PROBE(fail3);
555 fail2:
556 EFSYS_PROBE(fail2);
557 fail1:
558 EFSYS_PROBE1(fail1, efx_rc_t, rc);
559 return (rc);
560 }
561
562 void
efx_tx_qpio_disable(__in efx_txq_t * etp)563 efx_tx_qpio_disable(
564 __in efx_txq_t *etp)
565 {
566 efx_nic_t *enp = etp->et_enp;
567 const efx_tx_ops_t *etxop = enp->en_etxop;
568
569 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
570
571 if (etxop->etxo_qpio_disable != NULL)
572 etxop->etxo_qpio_disable(etp);
573 }
574
575 __checkReturn efx_rc_t
efx_tx_qpio_write(__in efx_txq_t * etp,__in_ecount (buf_length)uint8_t * buffer,__in size_t buf_length,__in size_t pio_buf_offset)576 efx_tx_qpio_write(
577 __in efx_txq_t *etp,
578 __in_ecount(buf_length) uint8_t *buffer,
579 __in size_t buf_length,
580 __in size_t pio_buf_offset)
581 {
582 efx_nic_t *enp = etp->et_enp;
583 const efx_tx_ops_t *etxop = enp->en_etxop;
584 efx_rc_t rc;
585
586 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
587
588 if (etxop->etxo_qpio_write != NULL) {
589 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
590 pio_buf_offset)) != 0)
591 goto fail1;
592 return (0);
593 }
594
595 return (ENOTSUP);
596
597 fail1:
598 EFSYS_PROBE1(fail1, efx_rc_t, rc);
599 return (rc);
600 }
601
602 __checkReturn efx_rc_t
efx_tx_qpio_post(__in efx_txq_t * etp,__in size_t pkt_length,__in unsigned int completed,__inout unsigned int * addedp)603 efx_tx_qpio_post(
604 __in efx_txq_t *etp,
605 __in size_t pkt_length,
606 __in unsigned int completed,
607 __inout unsigned int *addedp)
608 {
609 efx_nic_t *enp = etp->et_enp;
610 const efx_tx_ops_t *etxop = enp->en_etxop;
611 efx_rc_t rc;
612
613 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
614
615 if (etxop->etxo_qpio_post != NULL) {
616 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
617 addedp)) != 0)
618 goto fail1;
619 return (0);
620 }
621
622 return (ENOTSUP);
623
624 fail1:
625 EFSYS_PROBE1(fail1, efx_rc_t, rc);
626 return (rc);
627 }
628
629 __checkReturn efx_rc_t
efx_tx_qdesc_post(__in efx_txq_t * etp,__in_ecount (ndescs)efx_desc_t * ed,__in unsigned int ndescs,__in unsigned int completed,__inout unsigned int * addedp)630 efx_tx_qdesc_post(
631 __in efx_txq_t *etp,
632 __in_ecount(ndescs) efx_desc_t *ed,
633 __in unsigned int ndescs,
634 __in unsigned int completed,
635 __inout unsigned int *addedp)
636 {
637 efx_nic_t *enp = etp->et_enp;
638 const efx_tx_ops_t *etxop = enp->en_etxop;
639
640 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
641
642 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp));
643 }
644
645 void
efx_tx_qdesc_dma_create(__in efx_txq_t * etp,__in efsys_dma_addr_t addr,__in size_t size,__in boolean_t eop,__out efx_desc_t * edp)646 efx_tx_qdesc_dma_create(
647 __in efx_txq_t *etp,
648 __in efsys_dma_addr_t addr,
649 __in size_t size,
650 __in boolean_t eop,
651 __out efx_desc_t *edp)
652 {
653 efx_nic_t *enp = etp->et_enp;
654 const efx_tx_ops_t *etxop = enp->en_etxop;
655
656 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
657 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
658
659 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
660 }
661
662 void
efx_tx_qdesc_tso_create(__in efx_txq_t * etp,__in uint16_t ipv4_id,__in uint32_t tcp_seq,__in uint8_t tcp_flags,__out efx_desc_t * edp)663 efx_tx_qdesc_tso_create(
664 __in efx_txq_t *etp,
665 __in uint16_t ipv4_id,
666 __in uint32_t tcp_seq,
667 __in uint8_t tcp_flags,
668 __out efx_desc_t *edp)
669 {
670 efx_nic_t *enp = etp->et_enp;
671 const efx_tx_ops_t *etxop = enp->en_etxop;
672
673 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
674 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
675
676 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
677 }
678
679 void
efx_tx_qdesc_tso2_create(__in efx_txq_t * etp,__in uint16_t ipv4_id,__in uint16_t outer_ipv4_id,__in uint32_t tcp_seq,__in uint16_t mss,__out_ecount (count)efx_desc_t * edp,__in int count)680 efx_tx_qdesc_tso2_create(
681 __in efx_txq_t *etp,
682 __in uint16_t ipv4_id,
683 __in uint16_t outer_ipv4_id,
684 __in uint32_t tcp_seq,
685 __in uint16_t mss,
686 __out_ecount(count) efx_desc_t *edp,
687 __in int count)
688 {
689 efx_nic_t *enp = etp->et_enp;
690 const efx_tx_ops_t *etxop = enp->en_etxop;
691
692 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
693 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
694
695 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id,
696 tcp_seq, mss, edp, count);
697 }
698
699 void
efx_tx_qdesc_vlantci_create(__in efx_txq_t * etp,__in uint16_t tci,__out efx_desc_t * edp)700 efx_tx_qdesc_vlantci_create(
701 __in efx_txq_t *etp,
702 __in uint16_t tci,
703 __out efx_desc_t *edp)
704 {
705 efx_nic_t *enp = etp->et_enp;
706 const efx_tx_ops_t *etxop = enp->en_etxop;
707
708 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
709 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
710
711 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
712 }
713
714 void
efx_tx_qdesc_checksum_create(__in efx_txq_t * etp,__in uint16_t flags,__out efx_desc_t * edp)715 efx_tx_qdesc_checksum_create(
716 __in efx_txq_t *etp,
717 __in uint16_t flags,
718 __out efx_desc_t *edp)
719 {
720 efx_nic_t *enp = etp->et_enp;
721 const efx_tx_ops_t *etxop = enp->en_etxop;
722
723 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
724 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
725
726 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
727 }
728
729
730 #if EFSYS_OPT_QSTATS
731 void
efx_tx_qstats_update(__in efx_txq_t * etp,__inout_ecount (TX_NQSTATS)efsys_stat_t * stat)732 efx_tx_qstats_update(
733 __in efx_txq_t *etp,
734 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
735 {
736 efx_nic_t *enp = etp->et_enp;
737 const efx_tx_ops_t *etxop = enp->en_etxop;
738
739 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
740
741 etxop->etxo_qstats_update(etp, stat);
742 }
743 #endif
744
745
746 #if EFSYS_OPT_SIENA
747
748 static __checkReturn efx_rc_t
siena_tx_init(__in efx_nic_t * enp)749 siena_tx_init(
750 __in efx_nic_t *enp)
751 {
752 efx_oword_t oword;
753
754 /*
755 * Disable the timer-based TX DMA backoff and allow TX DMA to be
756 * controlled by the RX FIFO fill level (although always allow a
757 * minimal trickle).
758 */
759 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
760 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
761 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
762 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
763 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
764 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
765 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
766 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
767
768 /*
769 * Filter all packets less than 14 bytes to avoid parsing
770 * errors.
771 */
772 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
773 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
774
775 /*
776 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
777 * descriptors (which is bad).
778 */
779 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
780 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
781 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
782
783 return (0);
784 }
785
786 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
787 do { \
788 unsigned int id; \
789 size_t offset; \
790 efx_qword_t qword; \
791 \
792 id = (_added)++ & (_etp)->et_mask; \
793 offset = id * sizeof (efx_qword_t); \
794 \
795 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
796 unsigned int, id, efsys_dma_addr_t, (_addr), \
797 size_t, (_size), boolean_t, (_eop)); \
798 \
799 EFX_POPULATE_QWORD_4(qword, \
800 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
801 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
802 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
803 (uint32_t)((_addr) & 0xffffffff), \
804 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
805 (uint32_t)((_addr) >> 32)); \
806 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
807 \
808 _NOTE(CONSTANTCONDITION) \
809 } while (B_FALSE)
810
811 static __checkReturn efx_rc_t
siena_tx_qpost(__in efx_txq_t * etp,__in_ecount (ndescs)efx_buffer_t * eb,__in unsigned int ndescs,__in unsigned int completed,__inout unsigned int * addedp)812 siena_tx_qpost(
813 __in efx_txq_t *etp,
814 __in_ecount(ndescs) efx_buffer_t *eb,
815 __in unsigned int ndescs,
816 __in unsigned int completed,
817 __inout unsigned int *addedp)
818 {
819 unsigned int added = *addedp;
820 unsigned int i;
821
822 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
823 return (ENOSPC);
824
825 for (i = 0; i < ndescs; i++) {
826 efx_buffer_t *ebp = &eb[i];
827 efsys_dma_addr_t start = ebp->eb_addr;
828 size_t size = ebp->eb_size;
829 efsys_dma_addr_t end = start + size;
830
831 /*
832 * Fragments must not span 4k boundaries.
833 * Here it is a stricter requirement than the maximum length.
834 */
835 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1,
836 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
837
838 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
839 }
840
841 EFX_TX_QSTAT_INCR(etp, TX_POST);
842
843 *addedp = added;
844 return (0);
845 }
846
847 static void
siena_tx_qpush(__in efx_txq_t * etp,__in unsigned int added,__in unsigned int pushed)848 siena_tx_qpush(
849 __in efx_txq_t *etp,
850 __in unsigned int added,
851 __in unsigned int pushed)
852 {
853 efx_nic_t *enp = etp->et_enp;
854 uint32_t wptr;
855 efx_dword_t dword;
856 efx_oword_t oword;
857
858 /* Push the populated descriptors out */
859 wptr = added & etp->et_mask;
860
861 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
862
863 /* Only write the third DWORD */
864 EFX_POPULATE_DWORD_1(dword,
865 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
866
867 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
868 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
869 SIENA_TXQ_DESC_SIZE, wptr, pushed & etp->et_mask);
870 EFSYS_PIO_WRITE_BARRIER();
871 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
872 etp->et_index, &dword, B_FALSE);
873 }
874
875 #define EFX_MAX_PACE_VALUE 20
876
877 static __checkReturn efx_rc_t
siena_tx_qpace(__in efx_txq_t * etp,__in unsigned int ns)878 siena_tx_qpace(
879 __in efx_txq_t *etp,
880 __in unsigned int ns)
881 {
882 efx_nic_t *enp = etp->et_enp;
883 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
884 efx_oword_t oword;
885 unsigned int pace_val;
886 unsigned int timer_period;
887 efx_rc_t rc;
888
889 if (ns == 0) {
890 pace_val = 0;
891 } else {
892 /*
893 * The pace_val to write into the table is s.t
894 * ns <= timer_period * (2 ^ pace_val)
895 */
896 timer_period = 104 / encp->enc_clk_mult;
897 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
898 if ((timer_period << pace_val) >= ns)
899 break;
900 }
901 }
902 if (pace_val > EFX_MAX_PACE_VALUE) {
903 rc = EINVAL;
904 goto fail1;
905 }
906
907 /* Update the pacing table */
908 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
909 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
910 &oword, B_TRUE);
911
912 return (0);
913
914 fail1:
915 EFSYS_PROBE1(fail1, efx_rc_t, rc);
916
917 return (rc);
918 }
919
920 static __checkReturn efx_rc_t
siena_tx_qflush(__in efx_txq_t * etp)921 siena_tx_qflush(
922 __in efx_txq_t *etp)
923 {
924 efx_nic_t *enp = etp->et_enp;
925 efx_oword_t oword;
926 uint32_t label;
927
928 efx_tx_qpace(etp, 0);
929
930 label = etp->et_index;
931
932 /* Flush the queue */
933 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
934 FRF_AZ_TX_FLUSH_DESCQ, label);
935 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
936
937 return (0);
938 }
939
940 static void
siena_tx_qenable(__in efx_txq_t * etp)941 siena_tx_qenable(
942 __in efx_txq_t *etp)
943 {
944 efx_nic_t *enp = etp->et_enp;
945 efx_oword_t oword;
946
947 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
948 etp->et_index, &oword, B_TRUE);
949
950 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
951 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
952 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
953 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
954 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
955
956 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
957 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
958 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
959
960 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
961 etp->et_index, &oword, B_TRUE);
962 }
963
964 static __checkReturn efx_rc_t
siena_tx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in uint16_t flags,__in efx_evq_t * eep,__in efx_txq_t * etp,__out unsigned int * addedp)965 siena_tx_qcreate(
966 __in efx_nic_t *enp,
967 __in unsigned int index,
968 __in unsigned int label,
969 __in efsys_mem_t *esmp,
970 __in size_t ndescs,
971 __in uint32_t id,
972 __in uint16_t flags,
973 __in efx_evq_t *eep,
974 __in efx_txq_t *etp,
975 __out unsigned int *addedp)
976 {
977 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
978 efx_oword_t oword;
979 uint32_t size;
980 uint16_t inner_csum;
981 efx_rc_t rc;
982
983 _NOTE(ARGUNUSED(esmp))
984
985 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
986 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
987 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
988
989 if (index >= encp->enc_txq_limit) {
990 rc = EINVAL;
991 goto fail1;
992 }
993 for (size = 0;
994 (1U << size) <= encp->enc_txq_max_ndescs / encp->enc_txq_min_ndescs;
995 size++)
996 if ((1U << size) == (uint32_t)ndescs / encp->enc_txq_min_ndescs)
997 break;
998 if (id + (1 << size) >= encp->enc_buftbl_limit) {
999 rc = EINVAL;
1000 goto fail2;
1001 }
1002
1003 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
1004 if ((flags & inner_csum) != 0) {
1005 rc = EINVAL;
1006 goto fail3;
1007 }
1008
1009 /* Set up the new descriptor queue */
1010 *addedp = 0;
1011
1012 EFX_POPULATE_OWORD_6(oword,
1013 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
1014 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
1015 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
1016 FRF_AZ_TX_DESCQ_LABEL, label,
1017 FRF_AZ_TX_DESCQ_SIZE, size,
1018 FRF_AZ_TX_DESCQ_TYPE, 0);
1019
1020 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
1021 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
1022 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
1023 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
1024 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
1025
1026 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1027 etp->et_index, &oword, B_TRUE);
1028
1029 return (0);
1030
1031 fail3:
1032 EFSYS_PROBE(fail3);
1033 fail2:
1034 EFSYS_PROBE(fail2);
1035 fail1:
1036 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1037
1038 return (rc);
1039 }
1040
1041 __checkReturn efx_rc_t
siena_tx_qdesc_post(__in efx_txq_t * etp,__in_ecount (ndescs)efx_desc_t * ed,__in unsigned int ndescs,__in unsigned int completed,__inout unsigned int * addedp)1042 siena_tx_qdesc_post(
1043 __in efx_txq_t *etp,
1044 __in_ecount(ndescs) efx_desc_t *ed,
1045 __in unsigned int ndescs,
1046 __in unsigned int completed,
1047 __inout unsigned int *addedp)
1048 {
1049 unsigned int added = *addedp;
1050 unsigned int i;
1051 efx_rc_t rc;
1052
1053 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1054 rc = ENOSPC;
1055 goto fail1;
1056 }
1057
1058 for (i = 0; i < ndescs; i++) {
1059 efx_desc_t *edp = &ed[i];
1060 unsigned int id;
1061 size_t offset;
1062
1063 id = added++ & etp->et_mask;
1064 offset = id * sizeof (efx_desc_t);
1065
1066 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1067 }
1068
1069 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1070 unsigned int, added, unsigned int, ndescs);
1071
1072 EFX_TX_QSTAT_INCR(etp, TX_POST);
1073
1074 *addedp = added;
1075 return (0);
1076
1077 fail1:
1078 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1079 return (rc);
1080 }
1081
1082 void
siena_tx_qdesc_dma_create(__in efx_txq_t * etp,__in efsys_dma_addr_t addr,__in size_t size,__in boolean_t eop,__out efx_desc_t * edp)1083 siena_tx_qdesc_dma_create(
1084 __in efx_txq_t *etp,
1085 __in efsys_dma_addr_t addr,
1086 __in size_t size,
1087 __in boolean_t eop,
1088 __out efx_desc_t *edp)
1089 {
1090 /*
1091 * Fragments must not span 4k boundaries.
1092 * Here it is a stricter requirement than the maximum length.
1093 */
1094 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1,
1095 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1096
1097 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1098 efsys_dma_addr_t, addr,
1099 size_t, size, boolean_t, eop);
1100
1101 EFX_POPULATE_QWORD_4(edp->ed_eq,
1102 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1103 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1104 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1105 (uint32_t)(addr & 0xffffffff),
1106 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1107 (uint32_t)(addr >> 32));
1108 }
1109
1110 #endif /* EFSYS_OPT_SIENA */
1111
1112 #if EFSYS_OPT_QSTATS
1113 #if EFSYS_OPT_NAMES
1114 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1115 static const char * const __efx_tx_qstat_name[] = {
1116 "post",
1117 "post_pio",
1118 };
1119 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1120
1121 const char *
efx_tx_qstat_name(__in efx_nic_t * enp,__in unsigned int id)1122 efx_tx_qstat_name(
1123 __in efx_nic_t *enp,
1124 __in unsigned int id)
1125 {
1126 _NOTE(ARGUNUSED(enp))
1127 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1128 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1129
1130 return (__efx_tx_qstat_name[id]);
1131 }
1132 #endif /* EFSYS_OPT_NAMES */
1133 #endif /* EFSYS_OPT_QSTATS */
1134
1135 #if EFSYS_OPT_SIENA
1136
1137 #if EFSYS_OPT_QSTATS
1138 static void
siena_tx_qstats_update(__in efx_txq_t * etp,__inout_ecount (TX_NQSTATS)efsys_stat_t * stat)1139 siena_tx_qstats_update(
1140 __in efx_txq_t *etp,
1141 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1142 {
1143 unsigned int id;
1144
1145 for (id = 0; id < TX_NQSTATS; id++) {
1146 efsys_stat_t *essp = &stat[id];
1147
1148 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1149 etp->et_stat[id] = 0;
1150 }
1151 }
1152 #endif /* EFSYS_OPT_QSTATS */
1153
1154 static void
siena_tx_qdestroy(__in efx_txq_t * etp)1155 siena_tx_qdestroy(
1156 __in efx_txq_t *etp)
1157 {
1158 efx_nic_t *enp = etp->et_enp;
1159 efx_oword_t oword;
1160
1161 /* Purge descriptor queue */
1162 EFX_ZERO_OWORD(oword);
1163
1164 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1165 etp->et_index, &oword, B_TRUE);
1166 }
1167
1168 static void
siena_tx_fini(__in efx_nic_t * enp)1169 siena_tx_fini(
1170 __in efx_nic_t *enp)
1171 {
1172 _NOTE(ARGUNUSED(enp))
1173 }
1174
1175 #endif /* EFSYS_OPT_SIENA */
1176