1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright(c) 2019-2021 Xilinx, Inc.
4 * Copyright(c) 2007-2019 Solarflare Communications Inc.
5 */
6
7 #include "efx.h"
8 #include "efx_impl.h"
9
10
11 #if EFSYS_OPT_SIENA
12
13 static __checkReturn efx_rc_t
14 siena_rx_init(
15 __in efx_nic_t *enp);
16
17 static void
18 siena_rx_fini(
19 __in efx_nic_t *enp);
20
21 #if EFSYS_OPT_RX_SCATTER
22 static __checkReturn efx_rc_t
23 siena_rx_scatter_enable(
24 __in efx_nic_t *enp,
25 __in unsigned int buf_size);
26 #endif /* EFSYS_OPT_RX_SCATTER */
27
28 #if EFSYS_OPT_RX_SCALE
29 static __checkReturn efx_rc_t
30 siena_rx_scale_mode_set(
31 __in efx_nic_t *enp,
32 __in uint32_t rss_context,
33 __in efx_rx_hash_alg_t alg,
34 __in efx_rx_hash_type_t type,
35 __in boolean_t insert);
36
37 static __checkReturn efx_rc_t
38 siena_rx_scale_key_set(
39 __in efx_nic_t *enp,
40 __in uint32_t rss_context,
41 __in_ecount(n) uint8_t *key,
42 __in size_t n);
43
44 static __checkReturn efx_rc_t
45 siena_rx_scale_tbl_set(
46 __in efx_nic_t *enp,
47 __in uint32_t rss_context,
48 __in_ecount(nentries) unsigned int *table,
49 __in size_t nentries);
50
51 static __checkReturn uint32_t
52 siena_rx_prefix_hash(
53 __in efx_nic_t *enp,
54 __in efx_rx_hash_alg_t func,
55 __in uint8_t *buffer);
56
57 #endif /* EFSYS_OPT_RX_SCALE */
58
59 static __checkReturn efx_rc_t
60 siena_rx_prefix_pktlen(
61 __in efx_nic_t *enp,
62 __in uint8_t *buffer,
63 __out uint16_t *lengthp);
64
65 static void
66 siena_rx_qpost(
67 __in efx_rxq_t *erp,
68 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
69 __in size_t size,
70 __in unsigned int ndescs,
71 __in unsigned int completed,
72 __in unsigned int added);
73
74 static void
75 siena_rx_qpush(
76 __in efx_rxq_t *erp,
77 __in unsigned int added,
78 __inout unsigned int *pushedp);
79
80 #if EFSYS_OPT_RX_PACKED_STREAM
81 static void
82 siena_rx_qpush_ps_credits(
83 __in efx_rxq_t *erp);
84
85 static __checkReturn uint8_t *
86 siena_rx_qps_packet_info(
87 __in efx_rxq_t *erp,
88 __in uint8_t *buffer,
89 __in uint32_t buffer_length,
90 __in uint32_t current_offset,
91 __out uint16_t *lengthp,
92 __out uint32_t *next_offsetp,
93 __out uint32_t *timestamp);
94 #endif
95
96 static __checkReturn efx_rc_t
97 siena_rx_qflush(
98 __in efx_rxq_t *erp);
99
100 static void
101 siena_rx_qenable(
102 __in efx_rxq_t *erp);
103
104 static __checkReturn efx_rc_t
105 siena_rx_qcreate(
106 __in efx_nic_t *enp,
107 __in unsigned int index,
108 __in unsigned int label,
109 __in efx_rxq_type_t type,
110 __in_opt const efx_rxq_type_data_t *type_data,
111 __in efsys_mem_t *esmp,
112 __in size_t ndescs,
113 __in uint32_t id,
114 __in unsigned int flags,
115 __in efx_evq_t *eep,
116 __in efx_rxq_t *erp);
117
118 static void
119 siena_rx_qdestroy(
120 __in efx_rxq_t *erp);
121
122 #endif /* EFSYS_OPT_SIENA */
123
124
125 #if EFSYS_OPT_SIENA
126 static const efx_rx_ops_t __efx_rx_siena_ops = {
127 siena_rx_init, /* erxo_init */
128 siena_rx_fini, /* erxo_fini */
129 #if EFSYS_OPT_RX_SCATTER
130 siena_rx_scatter_enable, /* erxo_scatter_enable */
131 #endif
132 #if EFSYS_OPT_RX_SCALE
133 NULL, /* erxo_scale_context_alloc */
134 NULL, /* erxo_scale_context_free */
135 siena_rx_scale_mode_set, /* erxo_scale_mode_set */
136 siena_rx_scale_key_set, /* erxo_scale_key_set */
137 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */
138 siena_rx_prefix_hash, /* erxo_prefix_hash */
139 #endif
140 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */
141 siena_rx_qpost, /* erxo_qpost */
142 siena_rx_qpush, /* erxo_qpush */
143 #if EFSYS_OPT_RX_PACKED_STREAM
144 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */
145 siena_rx_qps_packet_info, /* erxo_qps_packet_info */
146 #endif
147 siena_rx_qflush, /* erxo_qflush */
148 siena_rx_qenable, /* erxo_qenable */
149 siena_rx_qcreate, /* erxo_qcreate */
150 siena_rx_qdestroy, /* erxo_qdestroy */
151 };
152 #endif /* EFSYS_OPT_SIENA */
153
154 #if EFX_OPTS_EF10()
155 static const efx_rx_ops_t __efx_rx_ef10_ops = {
156 ef10_rx_init, /* erxo_init */
157 ef10_rx_fini, /* erxo_fini */
158 #if EFSYS_OPT_RX_SCATTER
159 ef10_rx_scatter_enable, /* erxo_scatter_enable */
160 #endif
161 #if EFSYS_OPT_RX_SCALE
162 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */
163 ef10_rx_scale_context_free, /* erxo_scale_context_free */
164 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */
165 ef10_rx_scale_key_set, /* erxo_scale_key_set */
166 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */
167 ef10_rx_prefix_hash, /* erxo_prefix_hash */
168 #endif
169 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */
170 ef10_rx_qpost, /* erxo_qpost */
171 ef10_rx_qpush, /* erxo_qpush */
172 #if EFSYS_OPT_RX_PACKED_STREAM
173 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */
174 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */
175 #endif
176 ef10_rx_qflush, /* erxo_qflush */
177 ef10_rx_qenable, /* erxo_qenable */
178 ef10_rx_qcreate, /* erxo_qcreate */
179 ef10_rx_qdestroy, /* erxo_qdestroy */
180 };
181 #endif /* EFX_OPTS_EF10() */
182
183 #if EFSYS_OPT_RIVERHEAD
184 static const efx_rx_ops_t __efx_rx_rhead_ops = {
185 rhead_rx_init, /* erxo_init */
186 rhead_rx_fini, /* erxo_fini */
187 #if EFSYS_OPT_RX_SCATTER
188 rhead_rx_scatter_enable, /* erxo_scatter_enable */
189 #endif
190 #if EFSYS_OPT_RX_SCALE
191 rhead_rx_scale_context_alloc, /* erxo_scale_context_alloc */
192 rhead_rx_scale_context_free, /* erxo_scale_context_free */
193 rhead_rx_scale_mode_set, /* erxo_scale_mode_set */
194 rhead_rx_scale_key_set, /* erxo_scale_key_set */
195 rhead_rx_scale_tbl_set, /* erxo_scale_tbl_set */
196 rhead_rx_prefix_hash, /* erxo_prefix_hash */
197 #endif
198 rhead_rx_prefix_pktlen, /* erxo_prefix_pktlen */
199 rhead_rx_qpost, /* erxo_qpost */
200 rhead_rx_qpush, /* erxo_qpush */
201 #if EFSYS_OPT_RX_PACKED_STREAM
202 NULL, /* erxo_qpush_ps_credits */
203 NULL, /* erxo_qps_packet_info */
204 #endif
205 rhead_rx_qflush, /* erxo_qflush */
206 rhead_rx_qenable, /* erxo_qenable */
207 rhead_rx_qcreate, /* erxo_qcreate */
208 rhead_rx_qdestroy, /* erxo_qdestroy */
209 };
210 #endif /* EFSYS_OPT_RIVERHEAD */
211
212
213 __checkReturn efx_rc_t
efx_rx_init(__inout efx_nic_t * enp)214 efx_rx_init(
215 __inout efx_nic_t *enp)
216 {
217 const efx_rx_ops_t *erxop;
218 efx_rc_t rc;
219
220 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
221 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
222
223 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
224 rc = EINVAL;
225 goto fail1;
226 }
227
228 if (enp->en_mod_flags & EFX_MOD_RX) {
229 rc = EINVAL;
230 goto fail2;
231 }
232
233 switch (enp->en_family) {
234 #if EFSYS_OPT_SIENA
235 case EFX_FAMILY_SIENA:
236 erxop = &__efx_rx_siena_ops;
237 break;
238 #endif /* EFSYS_OPT_SIENA */
239
240 #if EFSYS_OPT_HUNTINGTON
241 case EFX_FAMILY_HUNTINGTON:
242 erxop = &__efx_rx_ef10_ops;
243 break;
244 #endif /* EFSYS_OPT_HUNTINGTON */
245
246 #if EFSYS_OPT_MEDFORD
247 case EFX_FAMILY_MEDFORD:
248 erxop = &__efx_rx_ef10_ops;
249 break;
250 #endif /* EFSYS_OPT_MEDFORD */
251
252 #if EFSYS_OPT_MEDFORD2
253 case EFX_FAMILY_MEDFORD2:
254 erxop = &__efx_rx_ef10_ops;
255 break;
256 #endif /* EFSYS_OPT_MEDFORD2 */
257
258 #if EFSYS_OPT_RIVERHEAD
259 case EFX_FAMILY_RIVERHEAD:
260 erxop = &__efx_rx_rhead_ops;
261 break;
262 #endif /* EFSYS_OPT_RIVERHEAD */
263
264 default:
265 EFSYS_ASSERT(0);
266 rc = ENOTSUP;
267 goto fail3;
268 }
269
270 if ((rc = erxop->erxo_init(enp)) != 0)
271 goto fail4;
272
273 enp->en_erxop = erxop;
274 enp->en_mod_flags |= EFX_MOD_RX;
275 return (0);
276
277 fail4:
278 EFSYS_PROBE(fail4);
279 fail3:
280 EFSYS_PROBE(fail3);
281 fail2:
282 EFSYS_PROBE(fail2);
283 fail1:
284 EFSYS_PROBE1(fail1, efx_rc_t, rc);
285
286 enp->en_erxop = NULL;
287 enp->en_mod_flags &= ~EFX_MOD_RX;
288 return (rc);
289 }
290
291 void
efx_rx_fini(__in efx_nic_t * enp)292 efx_rx_fini(
293 __in efx_nic_t *enp)
294 {
295 const efx_rx_ops_t *erxop = enp->en_erxop;
296
297 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
298 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
299 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
300 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
301
302 erxop->erxo_fini(enp);
303
304 enp->en_erxop = NULL;
305 enp->en_mod_flags &= ~EFX_MOD_RX;
306 }
307
308 #if EFSYS_OPT_RX_SCATTER
309 __checkReturn efx_rc_t
efx_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)310 efx_rx_scatter_enable(
311 __in efx_nic_t *enp,
312 __in unsigned int buf_size)
313 {
314 const efx_rx_ops_t *erxop = enp->en_erxop;
315 efx_rc_t rc;
316
317 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
319
320 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
321 goto fail1;
322
323 return (0);
324
325 fail1:
326 EFSYS_PROBE1(fail1, efx_rc_t, rc);
327 return (rc);
328 }
329 #endif /* EFSYS_OPT_RX_SCATTER */
330
331 #if EFSYS_OPT_RX_SCALE
332 __checkReturn efx_rc_t
efx_rx_scale_hash_flags_get(__in efx_nic_t * enp,__in efx_rx_hash_alg_t hash_alg,__out_ecount_part (max_nflags,* nflagsp)unsigned int * flagsp,__in unsigned int max_nflags,__out unsigned int * nflagsp)333 efx_rx_scale_hash_flags_get(
334 __in efx_nic_t *enp,
335 __in efx_rx_hash_alg_t hash_alg,
336 __out_ecount_part(max_nflags, *nflagsp) unsigned int *flagsp,
337 __in unsigned int max_nflags,
338 __out unsigned int *nflagsp)
339 {
340 efx_nic_cfg_t *encp = &enp->en_nic_cfg;
341 unsigned int nflags = 0;
342 efx_rc_t rc;
343
344 if (flagsp == NULL || nflagsp == NULL) {
345 rc = EINVAL;
346 goto fail1;
347 }
348
349 if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) {
350 nflags = 0;
351 goto done;
352 }
353
354 /* Helper to add flags word to flags array without buffer overflow */
355 #define INSERT_FLAGS(_flags) \
356 do { \
357 if (nflags >= max_nflags) { \
358 rc = E2BIG; \
359 goto fail2; \
360 } \
361 *(flagsp + nflags) = (_flags); \
362 nflags++; \
363 \
364 _NOTE(CONSTANTCONDITION) \
365 } while (B_FALSE)
366
367 if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) {
368 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE));
369 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE));
370 }
371
372 if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) &&
373 (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) {
374 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST));
375 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC));
376
377 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST));
378 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC));
379
380 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE));
381 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST));
382 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC));
383
384 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE));
385 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST));
386 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC));
387 }
388
389 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE));
390 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE));
391
392 INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE));
393 INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE));
394
395 if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) {
396 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST));
397 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC));
398
399 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST));
400 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC));
401
402 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE));
403 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST));
404 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC));
405
406 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE));
407 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST));
408 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC));
409
410 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST));
411 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC));
412
413 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST));
414 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC));
415 }
416
417 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE));
418 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE));
419
420 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE));
421 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE));
422
423 INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE));
424 INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE));
425
426 #undef INSERT_FLAGS
427
428 done:
429 *nflagsp = nflags;
430 return (0);
431
432 fail2:
433 EFSYS_PROBE(fail2);
434 fail1:
435 EFSYS_PROBE1(fail1, efx_rc_t, rc);
436
437 return (rc);
438 }
439
440 __checkReturn efx_rc_t
efx_rx_hash_default_support_get(__in efx_nic_t * enp,__out efx_rx_hash_support_t * supportp)441 efx_rx_hash_default_support_get(
442 __in efx_nic_t *enp,
443 __out efx_rx_hash_support_t *supportp)
444 {
445 efx_rc_t rc;
446
447 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
448 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
449
450 if (supportp == NULL) {
451 rc = EINVAL;
452 goto fail1;
453 }
454
455 /*
456 * Report the hashing support the client gets by default if it
457 * does not allocate an RSS context itself.
458 */
459 *supportp = enp->en_hash_support;
460
461 return (0);
462
463 fail1:
464 EFSYS_PROBE1(fail1, efx_rc_t, rc);
465
466 return (rc);
467 }
468
469 __checkReturn efx_rc_t
efx_rx_scale_default_support_get(__in efx_nic_t * enp,__out efx_rx_scale_context_type_t * typep)470 efx_rx_scale_default_support_get(
471 __in efx_nic_t *enp,
472 __out efx_rx_scale_context_type_t *typep)
473 {
474 efx_rc_t rc;
475
476 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
477 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
478
479 if (typep == NULL) {
480 rc = EINVAL;
481 goto fail1;
482 }
483
484 /*
485 * Report the RSS support the client gets by default if it
486 * does not allocate an RSS context itself.
487 */
488 *typep = enp->en_rss_context_type;
489
490 return (0);
491
492 fail1:
493 EFSYS_PROBE1(fail1, efx_rc_t, rc);
494
495 return (rc);
496 }
497 #endif /* EFSYS_OPT_RX_SCALE */
498
499 #if EFSYS_OPT_RX_SCALE
500 __checkReturn efx_rc_t
efx_rx_scale_context_alloc(__in efx_nic_t * enp,__in efx_rx_scale_context_type_t type,__in uint32_t num_queues,__out uint32_t * rss_contextp)501 efx_rx_scale_context_alloc(
502 __in efx_nic_t *enp,
503 __in efx_rx_scale_context_type_t type,
504 __in uint32_t num_queues,
505 __out uint32_t *rss_contextp)
506 {
507 uint32_t table_nentries = EFX_RSS_TBL_SIZE;
508 const efx_rx_ops_t *erxop = enp->en_erxop;
509 efx_rc_t rc;
510
511 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
512 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
513
514 if (erxop->erxo_scale_context_alloc == NULL) {
515 rc = ENOTSUP;
516 goto fail1;
517 }
518
519 if (type == EFX_RX_SCALE_EVEN_SPREAD)
520 table_nentries = 0;
521
522 if ((rc = erxop->erxo_scale_context_alloc(enp, type, num_queues,
523 table_nentries, rss_contextp)) != 0) {
524 goto fail2;
525 }
526
527 return (0);
528
529 fail2:
530 EFSYS_PROBE(fail2);
531 fail1:
532 EFSYS_PROBE1(fail1, efx_rc_t, rc);
533 return (rc);
534 }
535 #endif /* EFSYS_OPT_RX_SCALE */
536
537 #if EFSYS_OPT_RX_SCALE
538 __checkReturn efx_rc_t
efx_rx_scale_context_alloc_v2(__in efx_nic_t * enp,__in efx_rx_scale_context_type_t type,__in uint32_t num_queues,__in uint32_t table_nentries,__out uint32_t * rss_contextp)539 efx_rx_scale_context_alloc_v2(
540 __in efx_nic_t *enp,
541 __in efx_rx_scale_context_type_t type,
542 __in uint32_t num_queues,
543 __in uint32_t table_nentries,
544 __out uint32_t *rss_contextp)
545 {
546 const efx_rx_ops_t *erxop = enp->en_erxop;
547 efx_rc_t rc;
548
549 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
550 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
551
552 if (erxop->erxo_scale_context_alloc == NULL) {
553 rc = ENOTSUP;
554 goto fail1;
555 }
556
557 if ((rc = erxop->erxo_scale_context_alloc(enp, type,
558 num_queues, table_nentries, rss_contextp)) != 0) {
559 goto fail2;
560 }
561
562 return (0);
563
564 fail2:
565 EFSYS_PROBE(fail2);
566 fail1:
567 EFSYS_PROBE1(fail1, efx_rc_t, rc);
568 return (rc);
569 }
570 #endif /* EFSYS_OPT_RX_SCALE */
571
572 #if EFSYS_OPT_RX_SCALE
573 __checkReturn efx_rc_t
efx_rx_scale_context_free(__in efx_nic_t * enp,__in uint32_t rss_context)574 efx_rx_scale_context_free(
575 __in efx_nic_t *enp,
576 __in uint32_t rss_context)
577 {
578 const efx_rx_ops_t *erxop = enp->en_erxop;
579 efx_rc_t rc;
580
581 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
582 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
583
584 if (erxop->erxo_scale_context_free == NULL) {
585 rc = ENOTSUP;
586 goto fail1;
587 }
588 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0)
589 goto fail2;
590
591 return (0);
592
593 fail2:
594 EFSYS_PROBE(fail2);
595 fail1:
596 EFSYS_PROBE1(fail1, efx_rc_t, rc);
597 return (rc);
598 }
599 #endif /* EFSYS_OPT_RX_SCALE */
600
601 #if EFSYS_OPT_RX_SCALE
602 __checkReturn efx_rc_t
efx_rx_scale_mode_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)603 efx_rx_scale_mode_set(
604 __in efx_nic_t *enp,
605 __in uint32_t rss_context,
606 __in efx_rx_hash_alg_t alg,
607 __in efx_rx_hash_type_t type,
608 __in boolean_t insert)
609 {
610 efx_nic_cfg_t *encp = &enp->en_nic_cfg;
611 const efx_rx_ops_t *erxop = enp->en_erxop;
612 efx_rx_hash_type_t type_check;
613 unsigned int i;
614 efx_rc_t rc;
615
616 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
617 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
618
619 /*
620 * Legacy flags and modern bits cannot be
621 * used at the same time in the hash type.
622 */
623 if ((type & EFX_RX_HASH_LEGACY_MASK) &&
624 (type & ~EFX_RX_HASH_LEGACY_MASK)) {
625 rc = EINVAL;
626 goto fail1;
627 }
628
629 /*
630 * If RSS hash type is represented by additional bits
631 * in the value, the latter need to be verified since
632 * not all bit combinations are valid RSS modes. Also,
633 * depending on the firmware, some valid combinations
634 * may be unsupported. Discern additional bits in the
635 * type value and try to recognise valid combinations.
636 * If some bits remain unrecognised, report the error.
637 */
638 type_check = type & ~EFX_RX_HASH_LEGACY_MASK;
639 if (type_check != 0) {
640 unsigned int type_flags[EFX_RX_HASH_NFLAGS];
641 unsigned int type_nflags;
642
643 rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags,
644 EFX_ARRAY_SIZE(type_flags), &type_nflags);
645 if (rc != 0)
646 goto fail2;
647
648 for (i = 0; i < type_nflags; ++i) {
649 if ((type_check & type_flags[i]) == type_flags[i])
650 type_check &= ~(type_flags[i]);
651 }
652
653 if (type_check != 0) {
654 rc = EINVAL;
655 goto fail3;
656 }
657 }
658
659 /*
660 * Translate EFX_RX_HASH() flags to their legacy counterparts
661 * provided that the FW claims no support for additional modes.
662 */
663 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) {
664 efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) |
665 EFX_RX_HASH(IPV4_TCP, 2TUPLE);
666 efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) |
667 EFX_RX_HASH(IPV6_TCP, 2TUPLE);
668 efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE);
669 efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE);
670
671 if ((type & t_ipv4) == t_ipv4)
672 type |= EFX_RX_HASH_IPV4;
673 if ((type & t_ipv6) == t_ipv6)
674 type |= EFX_RX_HASH_IPV6;
675
676 if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) {
677 if ((type & t_ipv4_tcp) == t_ipv4_tcp)
678 type |= EFX_RX_HASH_TCPIPV4;
679 if ((type & t_ipv6_tcp) == t_ipv6_tcp)
680 type |= EFX_RX_HASH_TCPIPV6;
681 }
682
683 type &= EFX_RX_HASH_LEGACY_MASK;
684 }
685
686 if (erxop->erxo_scale_mode_set != NULL) {
687 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg,
688 type, insert)) != 0)
689 goto fail4;
690 }
691
692 return (0);
693
694 fail4:
695 EFSYS_PROBE(fail4);
696 fail3:
697 EFSYS_PROBE(fail3);
698 fail2:
699 EFSYS_PROBE(fail2);
700 fail1:
701 EFSYS_PROBE1(fail1, efx_rc_t, rc);
702 return (rc);
703 }
704 #endif /* EFSYS_OPT_RX_SCALE */
705
706 #if EFSYS_OPT_RX_SCALE
707 __checkReturn efx_rc_t
efx_rx_scale_key_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)708 efx_rx_scale_key_set(
709 __in efx_nic_t *enp,
710 __in uint32_t rss_context,
711 __in_ecount(n) uint8_t *key,
712 __in size_t n)
713 {
714 const efx_rx_ops_t *erxop = enp->en_erxop;
715 efx_rc_t rc;
716
717 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
718 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
719
720 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0)
721 goto fail1;
722
723 return (0);
724
725 fail1:
726 EFSYS_PROBE1(fail1, efx_rc_t, rc);
727
728 return (rc);
729 }
730 #endif /* EFSYS_OPT_RX_SCALE */
731
732 #if EFSYS_OPT_RX_SCALE
733 __checkReturn efx_rc_t
efx_rx_scale_tbl_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (nentries)unsigned int * table,__in size_t nentries)734 efx_rx_scale_tbl_set(
735 __in efx_nic_t *enp,
736 __in uint32_t rss_context,
737 __in_ecount(nentries) unsigned int *table,
738 __in size_t nentries)
739 {
740 const efx_rx_ops_t *erxop = enp->en_erxop;
741 efx_rc_t rc;
742
743 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
744 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
745
746 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table,
747 nentries)) != 0)
748 goto fail1;
749
750 return (0);
751
752 fail1:
753 EFSYS_PROBE1(fail1, efx_rc_t, rc);
754
755 return (rc);
756 }
757 #endif /* EFSYS_OPT_RX_SCALE */
758
759 void
efx_rx_qpost(__in efx_rxq_t * erp,__in_ecount (ndescs)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int ndescs,__in unsigned int completed,__in unsigned int added)760 efx_rx_qpost(
761 __in efx_rxq_t *erp,
762 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
763 __in size_t size,
764 __in unsigned int ndescs,
765 __in unsigned int completed,
766 __in unsigned int added)
767 {
768 efx_nic_t *enp = erp->er_enp;
769 const efx_rx_ops_t *erxop = enp->en_erxop;
770
771 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
772 EFSYS_ASSERT(erp->er_buf_size == 0 || size == erp->er_buf_size);
773
774 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added);
775 }
776
777 #if EFSYS_OPT_RX_PACKED_STREAM
778
779 void
efx_rx_qpush_ps_credits(__in efx_rxq_t * erp)780 efx_rx_qpush_ps_credits(
781 __in efx_rxq_t *erp)
782 {
783 efx_nic_t *enp = erp->er_enp;
784 const efx_rx_ops_t *erxop = enp->en_erxop;
785
786 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
787
788 erxop->erxo_qpush_ps_credits(erp);
789 }
790
791 __checkReturn uint8_t *
efx_rx_qps_packet_info(__in efx_rxq_t * erp,__in uint8_t * buffer,__in uint32_t buffer_length,__in uint32_t current_offset,__out uint16_t * lengthp,__out uint32_t * next_offsetp,__out uint32_t * timestamp)792 efx_rx_qps_packet_info(
793 __in efx_rxq_t *erp,
794 __in uint8_t *buffer,
795 __in uint32_t buffer_length,
796 __in uint32_t current_offset,
797 __out uint16_t *lengthp,
798 __out uint32_t *next_offsetp,
799 __out uint32_t *timestamp)
800 {
801 efx_nic_t *enp = erp->er_enp;
802 const efx_rx_ops_t *erxop = enp->en_erxop;
803
804 return (erxop->erxo_qps_packet_info(erp, buffer,
805 buffer_length, current_offset, lengthp,
806 next_offsetp, timestamp));
807 }
808
809 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
810
811 void
efx_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)812 efx_rx_qpush(
813 __in efx_rxq_t *erp,
814 __in unsigned int added,
815 __inout unsigned int *pushedp)
816 {
817 efx_nic_t *enp = erp->er_enp;
818 const efx_rx_ops_t *erxop = enp->en_erxop;
819
820 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
821
822 erxop->erxo_qpush(erp, added, pushedp);
823 }
824
825 __checkReturn efx_rc_t
efx_rx_qflush(__in efx_rxq_t * erp)826 efx_rx_qflush(
827 __in efx_rxq_t *erp)
828 {
829 efx_nic_t *enp = erp->er_enp;
830 const efx_rx_ops_t *erxop = enp->en_erxop;
831 efx_rc_t rc;
832
833 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
834
835 if ((rc = erxop->erxo_qflush(erp)) != 0)
836 goto fail1;
837
838 return (0);
839
840 fail1:
841 EFSYS_PROBE1(fail1, efx_rc_t, rc);
842
843 return (rc);
844 }
845
846 __checkReturn size_t
efx_rxq_size(__in const efx_nic_t * enp,__in unsigned int ndescs)847 efx_rxq_size(
848 __in const efx_nic_t *enp,
849 __in unsigned int ndescs)
850 {
851 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
852
853 return (ndescs * encp->enc_rx_desc_size);
854 }
855
856 __checkReturn unsigned int
efx_rxq_nbufs(__in const efx_nic_t * enp,__in unsigned int ndescs)857 efx_rxq_nbufs(
858 __in const efx_nic_t *enp,
859 __in unsigned int ndescs)
860 {
861 return (EFX_DIV_ROUND_UP(efx_rxq_size(enp, ndescs), EFX_BUF_SIZE));
862 }
863
864 void
efx_rx_qenable(__in efx_rxq_t * erp)865 efx_rx_qenable(
866 __in efx_rxq_t *erp)
867 {
868 efx_nic_t *enp = erp->er_enp;
869 const efx_rx_ops_t *erxop = enp->en_erxop;
870
871 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
872
873 erxop->erxo_qenable(erp);
874 }
875
876 static __checkReturn efx_rc_t
efx_rx_qcreate_internal(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in_opt const efx_rxq_type_data_t * type_data,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)877 efx_rx_qcreate_internal(
878 __in efx_nic_t *enp,
879 __in unsigned int index,
880 __in unsigned int label,
881 __in efx_rxq_type_t type,
882 __in_opt const efx_rxq_type_data_t *type_data,
883 __in efsys_mem_t *esmp,
884 __in size_t ndescs,
885 __in uint32_t id,
886 __in unsigned int flags,
887 __in efx_evq_t *eep,
888 __deref_out efx_rxq_t **erpp)
889 {
890 const efx_rx_ops_t *erxop = enp->en_erxop;
891 efx_rxq_t *erp;
892 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
893 efx_rc_t rc;
894
895 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
896 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
897
898 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
899
900 EFSYS_ASSERT(ISP2(encp->enc_rxq_max_ndescs));
901 EFSYS_ASSERT(ISP2(encp->enc_rxq_min_ndescs));
902
903 if (index >= encp->enc_rxq_limit) {
904 rc = EINVAL;
905 goto fail1;
906 }
907
908 if (!ISP2(ndescs) ||
909 ndescs < encp->enc_rxq_min_ndescs ||
910 ndescs > encp->enc_rxq_max_ndescs) {
911 rc = EINVAL;
912 goto fail2;
913 }
914
915 /* Allocate an RXQ object */
916 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
917
918 if (erp == NULL) {
919 rc = ENOMEM;
920 goto fail3;
921 }
922
923 erp->er_magic = EFX_RXQ_MAGIC;
924 erp->er_enp = enp;
925 erp->er_index = index;
926 erp->er_mask = ndescs - 1;
927 erp->er_esmp = esmp;
928
929 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp,
930 ndescs, id, flags, eep, erp)) != 0)
931 goto fail4;
932
933 /* Sanity check queue creation result */
934 if (flags & EFX_RXQ_FLAG_RSS_HASH) {
935 const efx_rx_prefix_layout_t *erplp = &erp->er_prefix_layout;
936 const efx_rx_prefix_field_info_t *rss_hash_field;
937
938 rss_hash_field =
939 &erplp->erpl_fields[EFX_RX_PREFIX_FIELD_RSS_HASH];
940 if (rss_hash_field->erpfi_width_bits == 0) {
941 rc = ENOTSUP;
942 goto fail5;
943 }
944 }
945
946 if (flags & EFX_RXQ_FLAG_VLAN_STRIPPED_TCI) {
947 const efx_rx_prefix_layout_t *erplp = &erp->er_prefix_layout;
948 const efx_rx_prefix_field_info_t *vlan_tci_field;
949
950 vlan_tci_field =
951 &erplp->erpl_fields[EFX_RX_PREFIX_FIELD_VLAN_STRIP_TCI];
952 if (vlan_tci_field->erpfi_width_bits == 0) {
953 rc = ENOTSUP;
954 goto fail6;
955 }
956 }
957
958 enp->en_rx_qcount++;
959 *erpp = erp;
960
961 return (0);
962
963 fail6:
964 EFSYS_PROBE(fail6);
965 fail5:
966 EFSYS_PROBE(fail5);
967
968 erxop->erxo_qdestroy(erp);
969 fail4:
970 EFSYS_PROBE(fail4);
971
972 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
973 fail3:
974 EFSYS_PROBE(fail3);
975 fail2:
976 EFSYS_PROBE(fail2);
977 fail1:
978 EFSYS_PROBE1(fail1, efx_rc_t, rc);
979
980 return (rc);
981 }
982
983 __checkReturn efx_rc_t
efx_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in size_t buf_size,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)984 efx_rx_qcreate(
985 __in efx_nic_t *enp,
986 __in unsigned int index,
987 __in unsigned int label,
988 __in efx_rxq_type_t type,
989 __in size_t buf_size,
990 __in efsys_mem_t *esmp,
991 __in size_t ndescs,
992 __in uint32_t id,
993 __in unsigned int flags,
994 __in efx_evq_t *eep,
995 __deref_out efx_rxq_t **erpp)
996 {
997 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
998 efx_rxq_type_data_t type_data;
999 efx_rc_t rc;
1000
1001 if (buf_size > encp->enc_rx_dma_desc_size_max) {
1002 rc = EINVAL;
1003 goto fail1;
1004 }
1005
1006 memset(&type_data, 0, sizeof (type_data));
1007
1008 type_data.ertd_default.ed_buf_size = buf_size;
1009
1010 return efx_rx_qcreate_internal(enp, index, label, type, &type_data,
1011 esmp, ndescs, id, flags, eep, erpp);
1012
1013 fail1:
1014 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1015
1016 return (rc);
1017 }
1018
1019 #if EFSYS_OPT_RX_PACKED_STREAM
1020
1021 __checkReturn efx_rc_t
efx_rx_qcreate_packed_stream(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in uint32_t ps_buf_size,__in efsys_mem_t * esmp,__in size_t ndescs,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)1022 efx_rx_qcreate_packed_stream(
1023 __in efx_nic_t *enp,
1024 __in unsigned int index,
1025 __in unsigned int label,
1026 __in uint32_t ps_buf_size,
1027 __in efsys_mem_t *esmp,
1028 __in size_t ndescs,
1029 __in efx_evq_t *eep,
1030 __deref_out efx_rxq_t **erpp)
1031 {
1032 efx_rxq_type_data_t type_data;
1033
1034 memset(&type_data, 0, sizeof (type_data));
1035
1036 type_data.ertd_packed_stream.eps_buf_size = ps_buf_size;
1037
1038 return efx_rx_qcreate_internal(enp, index, label,
1039 EFX_RXQ_TYPE_PACKED_STREAM, &type_data, esmp, ndescs,
1040 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp);
1041 }
1042
1043 #endif
1044
1045 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1046
1047 __checkReturn efx_rc_t
efx_rx_qcreate_es_super_buffer(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in uint32_t n_bufs_per_desc,__in uint32_t max_dma_len,__in uint32_t buf_stride,__in uint32_t hol_block_timeout,__in efsys_mem_t * esmp,__in size_t ndescs,__in unsigned int flags,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)1048 efx_rx_qcreate_es_super_buffer(
1049 __in efx_nic_t *enp,
1050 __in unsigned int index,
1051 __in unsigned int label,
1052 __in uint32_t n_bufs_per_desc,
1053 __in uint32_t max_dma_len,
1054 __in uint32_t buf_stride,
1055 __in uint32_t hol_block_timeout,
1056 __in efsys_mem_t *esmp,
1057 __in size_t ndescs,
1058 __in unsigned int flags,
1059 __in efx_evq_t *eep,
1060 __deref_out efx_rxq_t **erpp)
1061 {
1062 efx_rc_t rc;
1063 efx_rxq_type_data_t type_data;
1064
1065 if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) {
1066 rc = EINVAL;
1067 goto fail1;
1068 }
1069
1070 memset(&type_data, 0, sizeof (type_data));
1071
1072 type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc;
1073 type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len;
1074 type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride;
1075 type_data.ertd_es_super_buffer.eessb_hol_block_timeout =
1076 hol_block_timeout;
1077
1078 rc = efx_rx_qcreate_internal(enp, index, label,
1079 EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs,
1080 0 /* id unused on EF10 */, flags, eep, erpp);
1081 if (rc != 0)
1082 goto fail2;
1083
1084 return (0);
1085
1086 fail2:
1087 EFSYS_PROBE(fail2);
1088 fail1:
1089 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1090
1091 return (rc);
1092 }
1093
1094 #endif
1095
1096
1097 void
efx_rx_qdestroy(__in efx_rxq_t * erp)1098 efx_rx_qdestroy(
1099 __in efx_rxq_t *erp)
1100 {
1101 efx_nic_t *enp = erp->er_enp;
1102 const efx_rx_ops_t *erxop = enp->en_erxop;
1103
1104 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1105
1106 EFSYS_ASSERT(enp->en_rx_qcount != 0);
1107 --enp->en_rx_qcount;
1108
1109 erxop->erxo_qdestroy(erp);
1110
1111 /* Free the RXQ object */
1112 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1113 }
1114
1115 __checkReturn efx_rc_t
efx_pseudo_hdr_pkt_length_get(__in efx_rxq_t * erp,__in uint8_t * buffer,__out uint16_t * lengthp)1116 efx_pseudo_hdr_pkt_length_get(
1117 __in efx_rxq_t *erp,
1118 __in uint8_t *buffer,
1119 __out uint16_t *lengthp)
1120 {
1121 efx_nic_t *enp = erp->er_enp;
1122 const efx_rx_ops_t *erxop = enp->en_erxop;
1123
1124 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1125
1126 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
1127 }
1128
1129 #if EFSYS_OPT_RX_SCALE
1130 __checkReturn uint32_t
efx_pseudo_hdr_hash_get(__in efx_rxq_t * erp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)1131 efx_pseudo_hdr_hash_get(
1132 __in efx_rxq_t *erp,
1133 __in efx_rx_hash_alg_t func,
1134 __in uint8_t *buffer)
1135 {
1136 efx_nic_t *enp = erp->er_enp;
1137 const efx_rx_ops_t *erxop = enp->en_erxop;
1138
1139 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1140
1141 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
1142 return (erxop->erxo_prefix_hash(enp, func, buffer));
1143 }
1144 #endif /* EFSYS_OPT_RX_SCALE */
1145
1146 __checkReturn efx_rc_t
efx_rx_prefix_get_layout(__in const efx_rxq_t * erp,__out efx_rx_prefix_layout_t * erplp)1147 efx_rx_prefix_get_layout(
1148 __in const efx_rxq_t *erp,
1149 __out efx_rx_prefix_layout_t *erplp)
1150 {
1151 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1152
1153 *erplp = erp->er_prefix_layout;
1154
1155 return (0);
1156 }
1157
1158 #if EFSYS_OPT_SIENA
1159
1160 static __checkReturn efx_rc_t
siena_rx_init(__in efx_nic_t * enp)1161 siena_rx_init(
1162 __in efx_nic_t *enp)
1163 {
1164 efx_oword_t oword;
1165 unsigned int index;
1166
1167 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
1168
1169 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
1170 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
1171 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
1172 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
1173 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
1174 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
1175 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
1176
1177 /* Zero the RSS table */
1178 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
1179 index++) {
1180 EFX_ZERO_OWORD(oword);
1181 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1182 index, &oword, B_TRUE);
1183 }
1184
1185 #if EFSYS_OPT_RX_SCALE
1186 /* The RSS key and indirection table are writable. */
1187 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
1188
1189 /* Hardware can insert RX hash with/without RSS */
1190 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
1191 #endif /* EFSYS_OPT_RX_SCALE */
1192
1193 return (0);
1194 }
1195
1196 #if EFSYS_OPT_RX_SCATTER
1197 static __checkReturn efx_rc_t
siena_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)1198 siena_rx_scatter_enable(
1199 __in efx_nic_t *enp,
1200 __in unsigned int buf_size)
1201 {
1202 unsigned int nbuf32;
1203 efx_oword_t oword;
1204 efx_rc_t rc;
1205
1206 nbuf32 = buf_size / 32;
1207 if ((nbuf32 == 0) ||
1208 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
1209 ((buf_size % 32) != 0)) {
1210 rc = EINVAL;
1211 goto fail1;
1212 }
1213
1214 if (enp->en_rx_qcount > 0) {
1215 rc = EBUSY;
1216 goto fail2;
1217 }
1218
1219 /* Set scatter buffer size */
1220 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
1221 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
1222 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
1223
1224 /* Enable scatter for packets not matching a filter */
1225 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
1226 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
1227 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
1228
1229 return (0);
1230
1231 fail2:
1232 EFSYS_PROBE(fail2);
1233 fail1:
1234 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1235
1236 return (rc);
1237 }
1238 #endif /* EFSYS_OPT_RX_SCATTER */
1239
1240
1241 #define EFX_RX_LFSR_HASH(_enp, _insert) \
1242 do { \
1243 efx_oword_t oword; \
1244 \
1245 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1246 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \
1247 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \
1248 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \
1249 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
1250 (_insert) ? 1 : 0); \
1251 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1252 \
1253 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \
1254 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
1255 &oword); \
1256 EFX_SET_OWORD_FIELD(oword, \
1257 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \
1258 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
1259 &oword); \
1260 } \
1261 \
1262 _NOTE(CONSTANTCONDITION) \
1263 } while (B_FALSE)
1264
1265 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \
1266 do { \
1267 efx_oword_t oword; \
1268 \
1269 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1270 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \
1271 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \
1272 (_ip) ? 1 : 0); \
1273 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \
1274 (_tcp) ? 0 : 1); \
1275 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
1276 (_insert) ? 1 : 0); \
1277 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1278 \
1279 _NOTE(CONSTANTCONDITION) \
1280 } while (B_FALSE)
1281
1282 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \
1283 do { \
1284 efx_oword_t oword; \
1285 \
1286 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
1287 EFX_SET_OWORD_FIELD(oword, \
1288 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \
1289 EFX_SET_OWORD_FIELD(oword, \
1290 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
1291 EFX_SET_OWORD_FIELD(oword, \
1292 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \
1293 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
1294 \
1295 (_rc) = 0; \
1296 \
1297 _NOTE(CONSTANTCONDITION) \
1298 } while (B_FALSE)
1299
1300
1301 #if EFSYS_OPT_RX_SCALE
1302
1303 static __checkReturn efx_rc_t
siena_rx_scale_mode_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)1304 siena_rx_scale_mode_set(
1305 __in efx_nic_t *enp,
1306 __in uint32_t rss_context,
1307 __in efx_rx_hash_alg_t alg,
1308 __in efx_rx_hash_type_t type,
1309 __in boolean_t insert)
1310 {
1311 efx_rc_t rc;
1312
1313 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1314 rc = EINVAL;
1315 goto fail1;
1316 }
1317
1318 switch (alg) {
1319 case EFX_RX_HASHALG_LFSR:
1320 EFX_RX_LFSR_HASH(enp, insert);
1321 break;
1322
1323 case EFX_RX_HASHALG_TOEPLITZ:
1324 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
1325 (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE,
1326 (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE);
1327
1328 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
1329 (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE,
1330 (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE,
1331 rc);
1332 if (rc != 0)
1333 goto fail2;
1334
1335 break;
1336
1337 default:
1338 rc = EINVAL;
1339 goto fail3;
1340 }
1341
1342 return (0);
1343
1344 fail3:
1345 EFSYS_PROBE(fail3);
1346 fail2:
1347 EFSYS_PROBE(fail2);
1348 fail1:
1349 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1350
1351 EFX_RX_LFSR_HASH(enp, B_FALSE);
1352
1353 return (rc);
1354 }
1355 #endif
1356
1357 #if EFSYS_OPT_RX_SCALE
1358 static __checkReturn efx_rc_t
siena_rx_scale_key_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)1359 siena_rx_scale_key_set(
1360 __in efx_nic_t *enp,
1361 __in uint32_t rss_context,
1362 __in_ecount(n) uint8_t *key,
1363 __in size_t n)
1364 {
1365 efx_oword_t oword;
1366 unsigned int byte;
1367 unsigned int offset;
1368 efx_rc_t rc;
1369
1370 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1371 rc = EINVAL;
1372 goto fail1;
1373 }
1374
1375 byte = 0;
1376
1377 /* Write Toeplitz IPv4 hash key */
1378 EFX_ZERO_OWORD(oword);
1379 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
1380 offset > 0 && byte < n;
1381 --offset)
1382 oword.eo_u8[offset - 1] = key[byte++];
1383
1384 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
1385
1386 byte = 0;
1387
1388 /* Verify Toeplitz IPv4 hash key */
1389 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
1390 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
1391 offset > 0 && byte < n;
1392 --offset) {
1393 if (oword.eo_u8[offset - 1] != key[byte++]) {
1394 rc = EFAULT;
1395 goto fail2;
1396 }
1397 }
1398
1399 if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
1400 goto done;
1401
1402 byte = 0;
1403
1404 /* Write Toeplitz IPv6 hash key 3 */
1405 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1406 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1407 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1408 offset > 0 && byte < n;
1409 --offset)
1410 oword.eo_u8[offset - 1] = key[byte++];
1411
1412 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1413
1414 /* Write Toeplitz IPv6 hash key 2 */
1415 EFX_ZERO_OWORD(oword);
1416 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1417 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1418 offset > 0 && byte < n;
1419 --offset)
1420 oword.eo_u8[offset - 1] = key[byte++];
1421
1422 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1423
1424 /* Write Toeplitz IPv6 hash key 1 */
1425 EFX_ZERO_OWORD(oword);
1426 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1427 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1428 offset > 0 && byte < n;
1429 --offset)
1430 oword.eo_u8[offset - 1] = key[byte++];
1431
1432 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1433
1434 byte = 0;
1435
1436 /* Verify Toeplitz IPv6 hash key 3 */
1437 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1438 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1439 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1440 offset > 0 && byte < n;
1441 --offset) {
1442 if (oword.eo_u8[offset - 1] != key[byte++]) {
1443 rc = EFAULT;
1444 goto fail3;
1445 }
1446 }
1447
1448 /* Verify Toeplitz IPv6 hash key 2 */
1449 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1450 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1451 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1452 offset > 0 && byte < n;
1453 --offset) {
1454 if (oword.eo_u8[offset - 1] != key[byte++]) {
1455 rc = EFAULT;
1456 goto fail4;
1457 }
1458 }
1459
1460 /* Verify Toeplitz IPv6 hash key 1 */
1461 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1462 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1463 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1464 offset > 0 && byte < n;
1465 --offset) {
1466 if (oword.eo_u8[offset - 1] != key[byte++]) {
1467 rc = EFAULT;
1468 goto fail5;
1469 }
1470 }
1471
1472 done:
1473 return (0);
1474
1475 fail5:
1476 EFSYS_PROBE(fail5);
1477 fail4:
1478 EFSYS_PROBE(fail4);
1479 fail3:
1480 EFSYS_PROBE(fail3);
1481 fail2:
1482 EFSYS_PROBE(fail2);
1483 fail1:
1484 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1485
1486 return (rc);
1487 }
1488 #endif
1489
1490 #if EFSYS_OPT_RX_SCALE
1491 static __checkReturn efx_rc_t
siena_rx_scale_tbl_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (nentries)unsigned int * table,__in size_t nentries)1492 siena_rx_scale_tbl_set(
1493 __in efx_nic_t *enp,
1494 __in uint32_t rss_context,
1495 __in_ecount(nentries) unsigned int *table,
1496 __in size_t nentries)
1497 {
1498 efx_oword_t oword;
1499 int index;
1500 efx_rc_t rc;
1501
1502 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
1503 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
1504
1505 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1506 rc = EINVAL;
1507 goto fail1;
1508 }
1509
1510 if (nentries > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
1511 rc = EINVAL;
1512 goto fail2;
1513 }
1514
1515 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
1516 uint32_t byte;
1517
1518 /* Calculate the entry to place in the table */
1519 byte = (nentries > 0) ? (uint32_t)table[index % nentries] : 0;
1520
1521 EFSYS_PROBE2(table, int, index, uint32_t, byte);
1522
1523 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
1524
1525 /* Write the table */
1526 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1527 index, &oword, B_TRUE);
1528 }
1529
1530 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1531 uint32_t byte;
1532
1533 /* Determine if we're starting a new batch */
1534 byte = (nentries > 0) ? (uint32_t)table[index % nentries] : 0;
1535
1536 /* Read the table */
1537 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1538 index, &oword, B_TRUE);
1539
1540 /* Verify the entry */
1541 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1542 rc = EFAULT;
1543 goto fail3;
1544 }
1545 }
1546
1547 return (0);
1548
1549 fail3:
1550 EFSYS_PROBE(fail3);
1551 fail2:
1552 EFSYS_PROBE(fail2);
1553 fail1:
1554 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1555
1556 return (rc);
1557 }
1558 #endif
1559
1560 /*
1561 * Falcon/Siena pseudo-header
1562 * --------------------------
1563 *
1564 * Receive packets are prefixed by an optional 16 byte pseudo-header.
1565 * The pseudo-header is a byte array of one of the forms:
1566 *
1567 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1568 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
1569 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
1570 *
1571 * where:
1572 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian)
1573 * LL.LL LFSR hash (16-bit big-endian)
1574 */
1575
1576 /*
1577 * Provide Rx prefix layout with Toeplitz hash only since LSFR is
1578 * used by no supported drivers.
1579 *
1580 * Siena does not support Rx prefix choice via MC_CMD_GET_RX_PREFIX_ID
1581 * and query its layout using MC_CMD_QUERY_RX_PREFIX_ID.
1582 */
1583 static const efx_rx_prefix_layout_t siena_toeplitz_rx_prefix_layout = {
1584 .erpl_id = 0,
1585 .erpl_length = 16,
1586 .erpl_fields = {
1587 [EFX_RX_PREFIX_FIELD_RSS_HASH] = { 12 * 8, 32, B_TRUE },
1588 }
1589 };
1590
1591 #if EFSYS_OPT_RX_SCALE
1592 static __checkReturn uint32_t
siena_rx_prefix_hash(__in efx_nic_t * enp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)1593 siena_rx_prefix_hash(
1594 __in efx_nic_t *enp,
1595 __in efx_rx_hash_alg_t func,
1596 __in uint8_t *buffer)
1597 {
1598 _NOTE(ARGUNUSED(enp))
1599
1600 switch (func) {
1601 case EFX_RX_HASHALG_TOEPLITZ:
1602 return ((buffer[12] << 24) |
1603 (buffer[13] << 16) |
1604 (buffer[14] << 8) |
1605 buffer[15]);
1606
1607 case EFX_RX_HASHALG_LFSR:
1608 return ((buffer[14] << 8) | buffer[15]);
1609
1610 default:
1611 EFSYS_ASSERT(0);
1612 return (0);
1613 }
1614 }
1615 #endif /* EFSYS_OPT_RX_SCALE */
1616
1617 static __checkReturn efx_rc_t
siena_rx_prefix_pktlen(__in efx_nic_t * enp,__in uint8_t * buffer,__out uint16_t * lengthp)1618 siena_rx_prefix_pktlen(
1619 __in efx_nic_t *enp,
1620 __in uint8_t *buffer,
1621 __out uint16_t *lengthp)
1622 {
1623 _NOTE(ARGUNUSED(enp, buffer, lengthp))
1624
1625 /* Not supported by Falcon/Siena hardware */
1626 EFSYS_ASSERT(0);
1627 return (ENOTSUP);
1628 }
1629
1630
1631 static void
siena_rx_qpost(__in efx_rxq_t * erp,__in_ecount (ndescs)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int ndescs,__in unsigned int completed,__in unsigned int added)1632 siena_rx_qpost(
1633 __in efx_rxq_t *erp,
1634 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
1635 __in size_t size,
1636 __in unsigned int ndescs,
1637 __in unsigned int completed,
1638 __in unsigned int added)
1639 {
1640 efx_qword_t qword;
1641 unsigned int i;
1642 unsigned int offset;
1643 unsigned int id;
1644
1645 /* The client driver must not overfill the queue */
1646 EFSYS_ASSERT3U(added - completed + ndescs, <=,
1647 EFX_RXQ_LIMIT(erp->er_mask + 1));
1648
1649 id = added & (erp->er_mask);
1650 for (i = 0; i < ndescs; i++) {
1651 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1652 unsigned int, id, efsys_dma_addr_t, addrp[i],
1653 size_t, size);
1654
1655 EFX_POPULATE_QWORD_3(qword,
1656 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1657 FSF_AZ_RX_KER_BUF_ADDR_DW0,
1658 (uint32_t)(addrp[i] & 0xffffffff),
1659 FSF_AZ_RX_KER_BUF_ADDR_DW1,
1660 (uint32_t)(addrp[i] >> 32));
1661
1662 offset = id * sizeof (efx_qword_t);
1663 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1664
1665 id = (id + 1) & (erp->er_mask);
1666 }
1667 }
1668
1669 static void
siena_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)1670 siena_rx_qpush(
1671 __in efx_rxq_t *erp,
1672 __in unsigned int added,
1673 __inout unsigned int *pushedp)
1674 {
1675 efx_nic_t *enp = erp->er_enp;
1676 unsigned int pushed = *pushedp;
1677 uint32_t wptr;
1678 efx_oword_t oword;
1679 efx_dword_t dword;
1680
1681 /* All descriptors are pushed */
1682 *pushedp = added;
1683
1684 /* Push the populated descriptors out */
1685 wptr = added & erp->er_mask;
1686
1687 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1688
1689 /* Only write the third DWORD */
1690 EFX_POPULATE_DWORD_1(dword,
1691 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1692
1693 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1694 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1695 SIENA_RXQ_DESC_SIZE, wptr, pushed & erp->er_mask);
1696 EFSYS_PIO_WRITE_BARRIER();
1697 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1698 erp->er_index, &dword, B_FALSE);
1699 }
1700
1701 #if EFSYS_OPT_RX_PACKED_STREAM
1702 static void
siena_rx_qpush_ps_credits(__in efx_rxq_t * erp)1703 siena_rx_qpush_ps_credits(
1704 __in efx_rxq_t *erp)
1705 {
1706 /* Not supported by Siena hardware */
1707 EFSYS_ASSERT(0);
1708 }
1709
1710 static uint8_t *
siena_rx_qps_packet_info(__in efx_rxq_t * erp,__in uint8_t * buffer,__in uint32_t buffer_length,__in uint32_t current_offset,__out uint16_t * lengthp,__out uint32_t * next_offsetp,__out uint32_t * timestamp)1711 siena_rx_qps_packet_info(
1712 __in efx_rxq_t *erp,
1713 __in uint8_t *buffer,
1714 __in uint32_t buffer_length,
1715 __in uint32_t current_offset,
1716 __out uint16_t *lengthp,
1717 __out uint32_t *next_offsetp,
1718 __out uint32_t *timestamp)
1719 {
1720 /* Not supported by Siena hardware */
1721 EFSYS_ASSERT(0);
1722
1723 return (NULL);
1724 }
1725 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1726
1727 static __checkReturn efx_rc_t
siena_rx_qflush(__in efx_rxq_t * erp)1728 siena_rx_qflush(
1729 __in efx_rxq_t *erp)
1730 {
1731 efx_nic_t *enp = erp->er_enp;
1732 efx_oword_t oword;
1733 uint32_t label;
1734
1735 label = erp->er_index;
1736
1737 /* Flush the queue */
1738 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1739 FRF_AZ_RX_FLUSH_DESCQ, label);
1740 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1741
1742 return (0);
1743 }
1744
1745 static void
siena_rx_qenable(__in efx_rxq_t * erp)1746 siena_rx_qenable(
1747 __in efx_rxq_t *erp)
1748 {
1749 efx_nic_t *enp = erp->er_enp;
1750 efx_oword_t oword;
1751
1752 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1753
1754 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1755 erp->er_index, &oword, B_TRUE);
1756
1757 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1758 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1759 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1760
1761 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1762 erp->er_index, &oword, B_TRUE);
1763 }
1764
1765 static __checkReturn efx_rc_t
siena_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in_opt const efx_rxq_type_data_t * type_data,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__in efx_rxq_t * erp)1766 siena_rx_qcreate(
1767 __in efx_nic_t *enp,
1768 __in unsigned int index,
1769 __in unsigned int label,
1770 __in efx_rxq_type_t type,
1771 __in_opt const efx_rxq_type_data_t *type_data,
1772 __in efsys_mem_t *esmp,
1773 __in size_t ndescs,
1774 __in uint32_t id,
1775 __in unsigned int flags,
1776 __in efx_evq_t *eep,
1777 __in efx_rxq_t *erp)
1778 {
1779 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1780 efx_oword_t oword;
1781 uint32_t size;
1782 boolean_t jumbo = B_FALSE;
1783 efx_rc_t rc;
1784
1785 _NOTE(ARGUNUSED(esmp))
1786
1787 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1788 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1789 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1790
1791 for (size = 0;
1792 (1U << size) <= encp->enc_rxq_max_ndescs / encp->enc_rxq_min_ndescs;
1793 size++)
1794 if ((1U << size) == (uint32_t)ndescs / encp->enc_rxq_min_ndescs)
1795 break;
1796 if (id + (1 << size) >= encp->enc_buftbl_limit) {
1797 rc = EINVAL;
1798 goto fail1;
1799 }
1800
1801 switch (type) {
1802 case EFX_RXQ_TYPE_DEFAULT:
1803 erp->er_buf_size = type_data->ertd_default.ed_buf_size;
1804 /*
1805 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated
1806 * it is always delivered from HW in the pseudo-header.
1807 */
1808 break;
1809
1810 default:
1811 rc = EINVAL;
1812 goto fail2;
1813 }
1814
1815 #if EFSYS_OPT_RX_SCATTER
1816 #define SUPPORTED_RXQ_FLAGS EFX_RXQ_FLAG_SCATTER
1817 #else
1818 #define SUPPORTED_RXQ_FLAGS EFX_RXQ_FLAG_NONE
1819 #endif
1820 /* Reject flags for unsupported queue features */
1821 if ((flags & ~SUPPORTED_RXQ_FLAGS) != 0) {
1822 rc = EINVAL;
1823 goto fail3;
1824 }
1825 #undef SUPPORTED_RXQ_FLAGS
1826
1827 if (flags & EFX_RXQ_FLAG_SCATTER)
1828 jumbo = B_TRUE;
1829
1830 /* Set up the new descriptor queue */
1831 EFX_POPULATE_OWORD_7(oword,
1832 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1833 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1834 FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1835 FRF_AZ_RX_DESCQ_LABEL, label,
1836 FRF_AZ_RX_DESCQ_SIZE, size,
1837 FRF_AZ_RX_DESCQ_TYPE, 0,
1838 FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1839
1840 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1841 erp->er_index, &oword, B_TRUE);
1842
1843 erp->er_prefix_layout = siena_toeplitz_rx_prefix_layout;
1844
1845 return (0);
1846
1847 fail3:
1848 EFSYS_PROBE(fail3);
1849 fail2:
1850 EFSYS_PROBE(fail2);
1851 fail1:
1852 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1853
1854 return (rc);
1855 }
1856
1857 static void
siena_rx_qdestroy(__in efx_rxq_t * erp)1858 siena_rx_qdestroy(
1859 __in efx_rxq_t *erp)
1860 {
1861 efx_nic_t *enp = erp->er_enp;
1862 efx_oword_t oword;
1863
1864 /* Purge descriptor queue */
1865 EFX_ZERO_OWORD(oword);
1866
1867 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1868 erp->er_index, &oword, B_TRUE);
1869 }
1870
1871 static void
siena_rx_fini(__in efx_nic_t * enp)1872 siena_rx_fini(
1873 __in efx_nic_t *enp)
1874 {
1875 _NOTE(ARGUNUSED(enp))
1876 }
1877
1878 #endif /* EFSYS_OPT_SIENA */
1879
1880 static __checkReturn boolean_t
efx_rx_prefix_layout_fields_match(__in const efx_rx_prefix_field_info_t * erpfip1,__in const efx_rx_prefix_field_info_t * erpfip2)1881 efx_rx_prefix_layout_fields_match(
1882 __in const efx_rx_prefix_field_info_t *erpfip1,
1883 __in const efx_rx_prefix_field_info_t *erpfip2)
1884 {
1885 if (erpfip1->erpfi_offset_bits != erpfip2->erpfi_offset_bits)
1886 return (B_FALSE);
1887
1888 if (erpfip1->erpfi_width_bits != erpfip2->erpfi_width_bits)
1889 return (B_FALSE);
1890
1891 if (erpfip1->erpfi_big_endian != erpfip2->erpfi_big_endian)
1892 return (B_FALSE);
1893
1894 return (B_TRUE);
1895 }
1896
1897 __checkReturn uint32_t
efx_rx_prefix_layout_check(__in const efx_rx_prefix_layout_t * available,__in const efx_rx_prefix_layout_t * wanted)1898 efx_rx_prefix_layout_check(
1899 __in const efx_rx_prefix_layout_t *available,
1900 __in const efx_rx_prefix_layout_t *wanted)
1901 {
1902 uint32_t result = 0;
1903 unsigned int i;
1904
1905 EFX_STATIC_ASSERT(EFX_RX_PREFIX_NFIELDS < sizeof (result) * 8);
1906 for (i = 0; i < EFX_RX_PREFIX_NFIELDS; ++i) {
1907 /* Skip the field if driver does not want to use it */
1908 if (wanted->erpl_fields[i].erpfi_width_bits == 0)
1909 continue;
1910
1911 if (efx_rx_prefix_layout_fields_match(
1912 &available->erpl_fields[i],
1913 &wanted->erpl_fields[i]) == B_FALSE)
1914 result |= (1U << i);
1915 }
1916
1917 return (result);
1918 }
1919