1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright(c) 2019-2021 Xilinx, Inc.
4 * Copyright(c) 2012-2019 Solarflare Communications Inc.
5 */
6
7 #include "efx.h"
8 #include "efx_impl.h"
9
10
11 #if EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10()
12
13 #if EFSYS_OPT_RX_SCALE
14 static __checkReturn efx_rc_t
efx_mcdi_rss_context_alloc(__in efx_nic_t * enp,__in efx_rx_scale_context_type_t type,__in uint32_t num_queues,__in uint32_t table_nentries,__out uint32_t * rss_contextp)15 efx_mcdi_rss_context_alloc(
16 __in efx_nic_t *enp,
17 __in efx_rx_scale_context_type_t type,
18 __in uint32_t num_queues,
19 __in uint32_t table_nentries,
20 __out uint32_t *rss_contextp)
21 {
22 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
23 efx_mcdi_req_t req;
24 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_V2_IN_LEN,
25 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
26 uint32_t table_nentries_min;
27 uint32_t table_nentries_max;
28 uint32_t num_queues_max;
29 uint32_t rss_context;
30 uint32_t context_type;
31 efx_rc_t rc;
32
33 switch (type) {
34 case EFX_RX_SCALE_EXCLUSIVE:
35 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
36 num_queues_max = encp->enc_rx_scale_indirection_max_nqueues;
37 table_nentries_min = encp->enc_rx_scale_tbl_min_nentries;
38 table_nentries_max = encp->enc_rx_scale_tbl_max_nentries;
39 break;
40 case EFX_RX_SCALE_SHARED:
41 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
42 num_queues_max = encp->enc_rx_scale_indirection_max_nqueues;
43 table_nentries_min = encp->enc_rx_scale_tbl_min_nentries;
44 table_nentries_max = encp->enc_rx_scale_tbl_max_nentries;
45 break;
46 case EFX_RX_SCALE_EVEN_SPREAD:
47 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EVEN_SPREADING;
48 num_queues_max = encp->enc_rx_scale_even_spread_max_nqueues;
49 table_nentries_min = 0;
50 table_nentries_max = 0;
51 break;
52 default:
53 rc = EINVAL;
54 goto fail1;
55 }
56
57 if (num_queues == 0 || num_queues > num_queues_max) {
58 rc = EINVAL;
59 goto fail2;
60 }
61
62 if (table_nentries < table_nentries_min ||
63 table_nentries > table_nentries_max ||
64 (table_nentries != 0 && !ISP2(table_nentries))) {
65 rc = EINVAL;
66 goto fail3;
67 }
68
69 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
70 req.emr_in_buf = payload;
71 req.emr_in_length =
72 (encp->enc_rx_scale_tbl_entry_count_is_selectable != B_FALSE) ?
73 MC_CMD_RSS_CONTEXT_ALLOC_V2_IN_LEN :
74 MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
75 req.emr_out_buf = payload;
76 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
77
78 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
79 enp->en_vport_id);
80 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
81
82 /*
83 * For exclusive contexts, NUM_QUEUES is only used to validate
84 * indirection table offsets.
85 * For shared contexts, the provided context will spread traffic over
86 * NUM_QUEUES many queues.
87 * For the even spread contexts, the provided context will spread
88 * traffic over NUM_QUEUES many queues, but that will not involve
89 * the use of precious indirection table resources in the adapter.
90 */
91 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
92
93 if (encp->enc_rx_scale_tbl_entry_count_is_selectable != B_FALSE) {
94 MCDI_IN_SET_DWORD(req,
95 RSS_CONTEXT_ALLOC_V2_IN_INDIRECTION_TABLE_SIZE,
96 table_nentries);
97 }
98
99 efx_mcdi_execute(enp, &req);
100
101 if (req.emr_rc != 0) {
102 rc = req.emr_rc;
103 goto fail4;
104 }
105
106 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
107 rc = EMSGSIZE;
108 goto fail5;
109 }
110
111 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
112 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
113 rc = ENOENT;
114 goto fail6;
115 }
116
117 *rss_contextp = rss_context;
118
119 return (0);
120
121 fail6:
122 EFSYS_PROBE(fail6);
123 fail5:
124 EFSYS_PROBE(fail5);
125 fail4:
126 EFSYS_PROBE(fail4);
127 fail3:
128 EFSYS_PROBE(fail3);
129 fail2:
130 EFSYS_PROBE(fail2);
131 fail1:
132 EFSYS_PROBE1(fail1, efx_rc_t, rc);
133
134 return (rc);
135 }
136 #endif /* EFSYS_OPT_RX_SCALE */
137
138 #if EFSYS_OPT_RX_SCALE
139 static efx_rc_t
efx_mcdi_rss_context_free(__in efx_nic_t * enp,__in uint32_t rss_context)140 efx_mcdi_rss_context_free(
141 __in efx_nic_t *enp,
142 __in uint32_t rss_context)
143 {
144 efx_mcdi_req_t req;
145 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
146 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
147 efx_rc_t rc;
148
149 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
150 rc = EINVAL;
151 goto fail1;
152 }
153
154 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
155 req.emr_in_buf = payload;
156 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
157 req.emr_out_buf = payload;
158 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
159
160 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
161
162 efx_mcdi_execute_quiet(enp, &req);
163
164 if (req.emr_rc != 0) {
165 rc = req.emr_rc;
166 goto fail2;
167 }
168
169 return (0);
170
171 fail2:
172 EFSYS_PROBE(fail2);
173 fail1:
174 EFSYS_PROBE1(fail1, efx_rc_t, rc);
175
176 return (rc);
177 }
178 #endif /* EFSYS_OPT_RX_SCALE */
179
180 #if EFSYS_OPT_RX_SCALE
181 static efx_rc_t
efx_mcdi_rss_context_set_flags(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_type_t type)182 efx_mcdi_rss_context_set_flags(
183 __in efx_nic_t *enp,
184 __in uint32_t rss_context,
185 __in efx_rx_hash_type_t type)
186 {
187 efx_nic_cfg_t *encp = &enp->en_nic_cfg;
188 efx_mcdi_req_t req;
189 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
190 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
191 efx_rc_t rc;
192
193 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN ==
194 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN);
195 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH ==
196 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH);
197 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN ==
198 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN);
199 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH ==
200 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH);
201 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN ==
202 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN);
203 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH ==
204 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH);
205 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN ==
206 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN);
207 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH ==
208 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH);
209
210 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
211 rc = EINVAL;
212 goto fail1;
213 }
214
215 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
216 req.emr_in_buf = payload;
217 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
218 req.emr_out_buf = payload;
219 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
220
221 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
222 rss_context);
223
224 /*
225 * If the firmware lacks support for additional modes, RSS_MODE
226 * fields must contain zeros, otherwise the operation will fail.
227 */
228 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE)
229 type &= EFX_RX_HASH_LEGACY_MASK;
230
231 MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
232 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
233 (type & EFX_RX_HASH_IPV4) ? 1 : 0,
234 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
235 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
236 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
237 (type & EFX_RX_HASH_IPV6) ? 1 : 0,
238 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
239 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0,
240 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE,
241 (type >> EFX_RX_CLASS_IPV4_TCP_LBN) &
242 EFX_MASK32(EFX_RX_CLASS_IPV4_TCP),
243 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE,
244 (type >> EFX_RX_CLASS_IPV4_UDP_LBN) &
245 EFX_MASK32(EFX_RX_CLASS_IPV4_UDP),
246 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE,
247 (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4),
248 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE,
249 (type >> EFX_RX_CLASS_IPV6_TCP_LBN) &
250 EFX_MASK32(EFX_RX_CLASS_IPV6_TCP),
251 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE,
252 (type >> EFX_RX_CLASS_IPV6_UDP_LBN) &
253 EFX_MASK32(EFX_RX_CLASS_IPV6_UDP),
254 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE,
255 (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6));
256
257 efx_mcdi_execute(enp, &req);
258
259 if (req.emr_rc != 0) {
260 rc = req.emr_rc;
261 goto fail2;
262 }
263
264 return (0);
265
266 fail2:
267 EFSYS_PROBE(fail2);
268 fail1:
269 EFSYS_PROBE1(fail1, efx_rc_t, rc);
270
271 return (rc);
272 }
273 #endif /* EFSYS_OPT_RX_SCALE */
274
275 #if EFSYS_OPT_RX_SCALE
276 static efx_rc_t
efx_mcdi_rss_context_set_key(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)277 efx_mcdi_rss_context_set_key(
278 __in efx_nic_t *enp,
279 __in uint32_t rss_context,
280 __in_ecount(n) uint8_t *key,
281 __in size_t n)
282 {
283 efx_mcdi_req_t req;
284 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
285 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
286 efx_rc_t rc;
287
288 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
289 rc = EINVAL;
290 goto fail1;
291 }
292
293 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
294 req.emr_in_buf = payload;
295 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
296 req.emr_out_buf = payload;
297 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
298
299 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
300 rss_context);
301
302 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
303 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
304 rc = EINVAL;
305 goto fail2;
306 }
307
308 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
309 key, n);
310
311 efx_mcdi_execute(enp, &req);
312
313 if (req.emr_rc != 0) {
314 rc = req.emr_rc;
315 goto fail3;
316 }
317
318 return (0);
319
320 fail3:
321 EFSYS_PROBE(fail3);
322 fail2:
323 EFSYS_PROBE(fail2);
324 fail1:
325 EFSYS_PROBE1(fail1, efx_rc_t, rc);
326
327 return (rc);
328 }
329 #endif /* EFSYS_OPT_RX_SCALE */
330
331 #if EFSYS_OPT_RX_SCALE
332 static efx_rc_t
efx_mcdi_rss_context_set_table(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (nentries)unsigned int * table,__in size_t nentries)333 efx_mcdi_rss_context_set_table(
334 __in efx_nic_t *enp,
335 __in uint32_t rss_context,
336 __in_ecount(nentries) unsigned int *table,
337 __in size_t nentries)
338 {
339 efx_mcdi_req_t req;
340 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
341 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
342 uint8_t *req_table;
343 int i, rc;
344
345 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
346 rc = EINVAL;
347 goto fail1;
348 }
349
350 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
351 req.emr_in_buf = payload;
352 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
353 req.emr_out_buf = payload;
354 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
355
356 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
357 rss_context);
358
359 req_table =
360 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
361
362 for (i = 0;
363 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
364 i++) {
365 req_table[i] = (nentries > 0) ?
366 (uint8_t)table[i % nentries] : 0;
367 }
368
369 efx_mcdi_execute(enp, &req);
370
371 if (req.emr_rc != 0) {
372 rc = req.emr_rc;
373 goto fail2;
374 }
375
376 return (0);
377
378 fail2:
379 EFSYS_PROBE(fail2);
380 fail1:
381 EFSYS_PROBE1(fail1, efx_rc_t, rc);
382
383 return (rc);
384 }
385 #endif /* EFSYS_OPT_RX_SCALE */
386
387 #if EFSYS_OPT_RX_SCALE
388 static __checkReturn efx_rc_t
efx_mcdi_rss_context_write_table(__in efx_nic_t * enp,__in uint32_t context,__in unsigned int start_idx,__in_ecount (nentries)unsigned int * table,__in unsigned int nentries)389 efx_mcdi_rss_context_write_table(
390 __in efx_nic_t *enp,
391 __in uint32_t context,
392 __in unsigned int start_idx,
393 __in_ecount(nentries) unsigned int *table,
394 __in unsigned int nentries)
395 {
396 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
397 efx_mcdi_req_t req;
398 EFX_MCDI_DECLARE_BUF(payload,
399 MC_CMD_RSS_CONTEXT_WRITE_TABLE_IN_LENMAX_MCDI2,
400 MC_CMD_RSS_CONTEXT_WRITE_TABLE_OUT_LEN);
401 unsigned int i;
402 int rc;
403
404 if (nentries >
405 MC_CMD_RSS_CONTEXT_WRITE_TABLE_IN_ENTRIES_MAXNUM_MCDI2) {
406 rc = EINVAL;
407 goto fail1;
408 }
409
410 if (start_idx + nentries >
411 encp->enc_rx_scale_tbl_max_nentries) {
412 rc = EINVAL;
413 goto fail2;
414 }
415
416 req.emr_cmd = MC_CMD_RSS_CONTEXT_WRITE_TABLE;
417 req.emr_in_buf = payload;
418 req.emr_in_length = MC_CMD_RSS_CONTEXT_WRITE_TABLE_IN_LEN(nentries);
419 req.emr_out_buf = payload;
420 req.emr_out_length = MC_CMD_RSS_CONTEXT_WRITE_TABLE_OUT_LEN;
421
422 MCDI_IN_SET_DWORD(req,
423 RSS_CONTEXT_WRITE_TABLE_IN_RSS_CONTEXT_ID, context);
424
425 for (i = 0; i < nentries; ++i) {
426 if (table[i] >= encp->enc_rx_scale_indirection_max_nqueues) {
427 rc = EINVAL;
428 goto fail3;
429 }
430
431 MCDI_IN_POPULATE_INDEXED_DWORD_2(req,
432 RSS_CONTEXT_WRITE_TABLE_IN_ENTRIES, i,
433 RSS_CONTEXT_WRITE_TABLE_ENTRY_INDEX, start_idx + i,
434 RSS_CONTEXT_WRITE_TABLE_ENTRY_VALUE, table[i]);
435 }
436
437 efx_mcdi_execute(enp, &req);
438 if (req.emr_rc != 0) {
439 rc = req.emr_rc;
440 goto fail4;
441 }
442
443 return (0);
444
445 fail4:
446 EFSYS_PROBE(fail4);
447 fail3:
448 EFSYS_PROBE(fail3);
449 fail2:
450 EFSYS_PROBE(fail2);
451 fail1:
452 EFSYS_PROBE1(fail1, efx_rc_t, rc);
453 return (rc);
454 }
455 #endif /* EFSYS_OPT_RX_SCALE */
456
457
458 __checkReturn efx_rc_t
ef10_rx_init(__in efx_nic_t * enp)459 ef10_rx_init(
460 __in efx_nic_t *enp)
461 {
462 #if EFSYS_OPT_RX_SCALE
463
464 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
465 EFX_RSS_TBL_SIZE, &enp->en_rss_context) == 0) {
466 /*
467 * Allocated an exclusive RSS context, which allows both the
468 * indirection table and key to be modified.
469 */
470 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
471 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
472 } else {
473 /*
474 * Failed to allocate an exclusive RSS context. Continue
475 * operation without support for RSS. The pseudo-header in
476 * received packets will not contain a Toeplitz hash value.
477 */
478 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
479 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
480 }
481
482 #endif /* EFSYS_OPT_RX_SCALE */
483
484 return (0);
485 }
486
487 #if EFX_OPTS_EF10()
488
489 #if EFSYS_OPT_RX_SCATTER
490 __checkReturn efx_rc_t
ef10_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)491 ef10_rx_scatter_enable(
492 __in efx_nic_t *enp,
493 __in unsigned int buf_size)
494 {
495 _NOTE(ARGUNUSED(enp, buf_size))
496 return (0);
497 }
498 #endif /* EFSYS_OPT_RX_SCATTER */
499
500 #endif /* EFX_OPTS_EF10() */
501
502 #if EFSYS_OPT_RX_SCALE
503 __checkReturn efx_rc_t
ef10_rx_scale_context_alloc(__in efx_nic_t * enp,__in efx_rx_scale_context_type_t type,__in uint32_t num_queues,__in uint32_t table_nentries,__out uint32_t * rss_contextp)504 ef10_rx_scale_context_alloc(
505 __in efx_nic_t *enp,
506 __in efx_rx_scale_context_type_t type,
507 __in uint32_t num_queues,
508 __in uint32_t table_nentries,
509 __out uint32_t *rss_contextp)
510 {
511 efx_rc_t rc;
512
513 rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, table_nentries,
514 rss_contextp);
515 if (rc != 0)
516 goto fail1;
517
518 return (0);
519
520 fail1:
521 EFSYS_PROBE1(fail1, efx_rc_t, rc);
522 return (rc);
523 }
524 #endif /* EFSYS_OPT_RX_SCALE */
525
526 #if EFSYS_OPT_RX_SCALE
527 __checkReturn efx_rc_t
ef10_rx_scale_context_free(__in efx_nic_t * enp,__in uint32_t rss_context)528 ef10_rx_scale_context_free(
529 __in efx_nic_t *enp,
530 __in uint32_t rss_context)
531 {
532 efx_rc_t rc;
533
534 rc = efx_mcdi_rss_context_free(enp, rss_context);
535 if (rc != 0)
536 goto fail1;
537
538 return (0);
539
540 fail1:
541 EFSYS_PROBE1(fail1, efx_rc_t, rc);
542 return (rc);
543 }
544 #endif /* EFSYS_OPT_RX_SCALE */
545
546 #if EFSYS_OPT_RX_SCALE
547 __checkReturn efx_rc_t
ef10_rx_scale_mode_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)548 ef10_rx_scale_mode_set(
549 __in efx_nic_t *enp,
550 __in uint32_t rss_context,
551 __in efx_rx_hash_alg_t alg,
552 __in efx_rx_hash_type_t type,
553 __in boolean_t insert)
554 {
555 efx_nic_cfg_t *encp = &enp->en_nic_cfg;
556 efx_rc_t rc;
557
558 EFSYS_ASSERT3U(insert, ==, B_TRUE);
559
560 if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 ||
561 insert == B_FALSE) {
562 rc = EINVAL;
563 goto fail1;
564 }
565
566 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
567 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
568 rc = ENOTSUP;
569 goto fail2;
570 }
571 rss_context = enp->en_rss_context;
572 }
573
574 if ((rc = efx_mcdi_rss_context_set_flags(enp,
575 rss_context, type)) != 0)
576 goto fail3;
577
578 return (0);
579
580 fail3:
581 EFSYS_PROBE(fail3);
582 fail2:
583 EFSYS_PROBE(fail2);
584 fail1:
585 EFSYS_PROBE1(fail1, efx_rc_t, rc);
586
587 return (rc);
588 }
589 #endif /* EFSYS_OPT_RX_SCALE */
590
591 #if EFSYS_OPT_RX_SCALE
592 __checkReturn efx_rc_t
ef10_rx_scale_key_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)593 ef10_rx_scale_key_set(
594 __in efx_nic_t *enp,
595 __in uint32_t rss_context,
596 __in_ecount(n) uint8_t *key,
597 __in size_t n)
598 {
599 efx_rc_t rc;
600
601 EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
602 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
603
604 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
605 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
606 rc = ENOTSUP;
607 goto fail1;
608 }
609 rss_context = enp->en_rss_context;
610 }
611
612 if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
613 goto fail2;
614
615 return (0);
616
617 fail2:
618 EFSYS_PROBE(fail2);
619 fail1:
620 EFSYS_PROBE1(fail1, efx_rc_t, rc);
621
622 return (rc);
623 }
624 #endif /* EFSYS_OPT_RX_SCALE */
625
626 #if EFSYS_OPT_RX_SCALE
627 __checkReturn efx_rc_t
ef10_rx_scale_tbl_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (nentries)unsigned int * table,__in size_t nentries)628 ef10_rx_scale_tbl_set(
629 __in efx_nic_t *enp,
630 __in uint32_t rss_context,
631 __in_ecount(nentries) unsigned int *table,
632 __in size_t nentries)
633 {
634 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
635 efx_rc_t rc;
636
637
638 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
639 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
640 rc = ENOTSUP;
641 goto fail1;
642 }
643 rss_context = enp->en_rss_context;
644 }
645
646 if (encp->enc_rx_scale_tbl_entry_count_is_selectable != B_FALSE) {
647 uint32_t index, remain, batch;
648
649 batch = MC_CMD_RSS_CONTEXT_WRITE_TABLE_IN_ENTRIES_MAXNUM_MCDI2;
650 index = 0;
651
652 for (remain = nentries; remain > 0; remain -= batch) {
653 if (batch > remain)
654 batch = remain;
655
656 rc = efx_mcdi_rss_context_write_table(enp, rss_context,
657 index, &table[index], batch);
658 if (rc != 0)
659 goto fail2;
660
661 index += batch;
662 }
663 } else {
664 rc = efx_mcdi_rss_context_set_table(enp, rss_context, table,
665 nentries);
666 if (rc != 0)
667 goto fail3;
668 }
669
670 return (0);
671
672 fail3:
673 EFSYS_PROBE(fail3);
674 fail2:
675 EFSYS_PROBE(fail2);
676 fail1:
677 EFSYS_PROBE1(fail1, efx_rc_t, rc);
678
679 return (rc);
680 }
681 #endif /* EFSYS_OPT_RX_SCALE */
682
683 #if EFX_OPTS_EF10()
684
685 /*
686 * EF10 RX pseudo-header (aka Rx prefix)
687 * -------------------------------------
688 *
689 * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
690 *
691 * +00: Toeplitz hash value.
692 * (32bit little-endian)
693 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
694 * (16bit big-endian)
695 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
696 * (16bit big-endian)
697 * +08: Packet Length. Zero if the RX datapath was in cut-through mode.
698 * (16bit little-endian)
699 * +10: MAC timestamp. Zero if timestamping is not enabled.
700 * (32bit little-endian)
701 *
702 * See "The RX Pseudo-header" in SF-109306-TC.
703 *
704 * EF10 does not support Rx prefix choice using MC_CMD_GET_RX_PREFIX_ID
705 * and query its layout using MC_CMD_QUERY_RX_PREFIX_ID.
706 */
707 static const efx_rx_prefix_layout_t ef10_default_rx_prefix_layout = {
708 .erpl_id = 0,
709 .erpl_length = 14,
710 .erpl_fields = {
711 [EFX_RX_PREFIX_FIELD_RSS_HASH] =
712 { 0, 32, B_FALSE },
713 [EFX_RX_PREFIX_FIELD_VLAN_STRIP_TCI] =
714 { 32, 16, B_TRUE },
715 [EFX_RX_PREFIX_FIELD_INNER_VLAN_STRIP_TCI] =
716 { 48, 16, B_TRUE },
717 [EFX_RX_PREFIX_FIELD_LENGTH] =
718 { 64, 16, B_FALSE },
719 [EFX_RX_PREFIX_FIELD_PARTIAL_TSTAMP] =
720 { 80, 32, B_FALSE },
721 }
722 };
723
724 #if EFSYS_OPT_RX_PACKED_STREAM
725
726 /*
727 * EF10 packed stream Rx prefix layout.
728 *
729 * See SF-112241-TC Full speed capture for Huntington and Medford section 4.5.
730 */
731 static const efx_rx_prefix_layout_t ef10_packed_stream_rx_prefix_layout = {
732 .erpl_id = 0,
733 .erpl_length = 8,
734 .erpl_fields = {
735 #define EF10_PS_RX_PREFIX_FIELD(_efx, _ef10) \
736 EFX_RX_PREFIX_FIELD(_efx, ES_DZ_PS_RX_PREFIX_ ## _ef10, B_FALSE)
737
738 EF10_PS_RX_PREFIX_FIELD(PARTIAL_TSTAMP, TSTAMP),
739 EF10_PS_RX_PREFIX_FIELD(LENGTH, CAP_LEN),
740 EF10_PS_RX_PREFIX_FIELD(ORIG_LENGTH, ORIG_LEN),
741
742 #undef EF10_PS_RX_PREFIX_FIELD
743 }
744 };
745
746 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
747
748 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
749
750 /*
751 * EF10 equal stride super-buffer Rx prefix layout.
752 *
753 * See SF-119419-TC DPDK Firmware Driver Interface section 3.4.
754 */
755 static const efx_rx_prefix_layout_t ef10_essb_rx_prefix_layout = {
756 .erpl_id = 0,
757 .erpl_length = ES_EZ_ESSB_RX_PREFIX_LEN,
758 .erpl_fields = {
759 #define EF10_ESSB_RX_PREFIX_FIELD(_efx, _ef10) \
760 EFX_RX_PREFIX_FIELD(_efx, ES_EZ_ESSB_RX_PREFIX_ ## _ef10, B_FALSE)
761
762 EF10_ESSB_RX_PREFIX_FIELD(LENGTH, DATA_LEN),
763 EF10_ESSB_RX_PREFIX_FIELD(USER_MARK, MARK),
764 EF10_ESSB_RX_PREFIX_FIELD(RSS_HASH_VALID, HASH_VALID),
765 EF10_ESSB_RX_PREFIX_FIELD(USER_MARK_VALID, MARK_VALID),
766 EF10_ESSB_RX_PREFIX_FIELD(USER_FLAG, MATCH_FLAG),
767 EF10_ESSB_RX_PREFIX_FIELD(RSS_HASH, HASH),
768
769 #undef EF10_ESSB_RX_PREFIX_FIELD
770 }
771 };
772
773 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
774
775 __checkReturn efx_rc_t
ef10_rx_prefix_pktlen(__in efx_nic_t * enp,__in uint8_t * buffer,__out uint16_t * lengthp)776 ef10_rx_prefix_pktlen(
777 __in efx_nic_t *enp,
778 __in uint8_t *buffer,
779 __out uint16_t *lengthp)
780 {
781 _NOTE(ARGUNUSED(enp))
782
783 /*
784 * The RX pseudo-header contains the packet length, excluding the
785 * pseudo-header. If the hardware receive datapath was operating in
786 * cut-through mode then the length in the RX pseudo-header will be
787 * zero, and the packet length must be obtained from the DMA length
788 * reported in the RX event.
789 */
790 *lengthp = buffer[8] | (buffer[9] << 8);
791 return (0);
792 }
793
794 #if EFSYS_OPT_RX_SCALE
795 __checkReturn uint32_t
ef10_rx_prefix_hash(__in efx_nic_t * enp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)796 ef10_rx_prefix_hash(
797 __in efx_nic_t *enp,
798 __in efx_rx_hash_alg_t func,
799 __in uint8_t *buffer)
800 {
801 _NOTE(ARGUNUSED(enp))
802
803 switch (func) {
804 case EFX_RX_HASHALG_PACKED_STREAM:
805 case EFX_RX_HASHALG_TOEPLITZ:
806 return (buffer[0] |
807 (buffer[1] << 8) |
808 (buffer[2] << 16) |
809 (buffer[3] << 24));
810
811 default:
812 EFSYS_ASSERT(0);
813 return (0);
814 }
815 }
816 #endif /* EFSYS_OPT_RX_SCALE */
817
818 #if EFSYS_OPT_RX_PACKED_STREAM
819 /*
820 * Fake length for RXQ descriptors in packed stream mode
821 * to make hardware happy
822 */
823 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
824 #endif
825
826 void
ef10_rx_qpost(__in efx_rxq_t * erp,__in_ecount (ndescs)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int ndescs,__in unsigned int completed,__in unsigned int added)827 ef10_rx_qpost(
828 __in efx_rxq_t *erp,
829 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
830 __in size_t size,
831 __in unsigned int ndescs,
832 __in unsigned int completed,
833 __in unsigned int added)
834 {
835 efx_qword_t qword;
836 unsigned int i;
837 unsigned int offset;
838 unsigned int id;
839
840 _NOTE(ARGUNUSED(completed))
841
842 #if EFSYS_OPT_RX_PACKED_STREAM
843 /*
844 * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
845 * and equal to 0 after applying mask. Hardware does not like it.
846 */
847 if (erp->er_ev_qstate->eers_rx_packed_stream)
848 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
849 #endif
850
851 /* The client driver must not overfill the queue */
852 EFSYS_ASSERT3U(added - completed + ndescs, <=,
853 EFX_RXQ_LIMIT(erp->er_mask + 1));
854
855 id = added & (erp->er_mask);
856 for (i = 0; i < ndescs; i++) {
857 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
858 unsigned int, id, efsys_dma_addr_t, addrp[i],
859 size_t, size);
860
861 EFX_POPULATE_QWORD_3(qword,
862 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
863 ESF_DZ_RX_KER_BUF_ADDR_DW0,
864 (uint32_t)(addrp[i] & 0xffffffff),
865 ESF_DZ_RX_KER_BUF_ADDR_DW1,
866 (uint32_t)(addrp[i] >> 32));
867
868 offset = id * sizeof (efx_qword_t);
869 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
870
871 id = (id + 1) & (erp->er_mask);
872 }
873 }
874
875 void
ef10_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)876 ef10_rx_qpush(
877 __in efx_rxq_t *erp,
878 __in unsigned int added,
879 __inout unsigned int *pushedp)
880 {
881 efx_nic_t *enp = erp->er_enp;
882 unsigned int pushed = *pushedp;
883 uint32_t wptr;
884 efx_dword_t dword;
885
886 /* Hardware has alignment restriction for WPTR */
887 wptr = EFX_P2ALIGN(unsigned int, added, EF10_RX_WPTR_ALIGN);
888 if (pushed == wptr)
889 return;
890
891 *pushedp = wptr;
892
893 /* Push the populated descriptors out */
894 wptr &= erp->er_mask;
895
896 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
897
898 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
899 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
900 EF10_RXQ_DESC_SIZE, wptr, pushed & erp->er_mask);
901 EFSYS_PIO_WRITE_BARRIER();
902 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
903 erp->er_index, &dword, B_FALSE);
904 }
905
906 #if EFSYS_OPT_RX_PACKED_STREAM
907
908 void
ef10_rx_qpush_ps_credits(__in efx_rxq_t * erp)909 ef10_rx_qpush_ps_credits(
910 __in efx_rxq_t *erp)
911 {
912 efx_nic_t *enp = erp->er_enp;
913 efx_dword_t dword;
914 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
915 uint32_t credits;
916
917 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
918
919 if (rxq_state->eers_rx_packed_stream_credits == 0)
920 return;
921
922 /*
923 * It is a bug if we think that FW has utilized more
924 * credits than it is allowed to have (maximum). However,
925 * make sure that we do not credit more than maximum anyway.
926 */
927 credits = MIN(rxq_state->eers_rx_packed_stream_credits,
928 EFX_RX_PACKED_STREAM_MAX_CREDITS);
929 EFX_POPULATE_DWORD_3(dword,
930 ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
931 ERF_DZ_RX_DESC_MAGIC_CMD,
932 ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
933 ERF_DZ_RX_DESC_MAGIC_DATA, credits);
934 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
935 erp->er_index, &dword, B_FALSE);
936
937 rxq_state->eers_rx_packed_stream_credits = 0;
938 }
939
940 /*
941 * In accordance with SF-112241-TC the received data has the following layout:
942 * - 8 byte pseudo-header which consist of:
943 * - 4 byte little-endian timestamp
944 * - 2 byte little-endian captured length in bytes
945 * - 2 byte little-endian original packet length in bytes
946 * - captured packet bytes
947 * - optional padding to align to 64 bytes boundary
948 * - 64 bytes scratch space for the host software
949 */
950 __checkReturn uint8_t *
ef10_rx_qps_packet_info(__in efx_rxq_t * erp,__in uint8_t * buffer,__in uint32_t buffer_length,__in uint32_t current_offset,__out uint16_t * lengthp,__out uint32_t * next_offsetp,__out uint32_t * timestamp)951 ef10_rx_qps_packet_info(
952 __in efx_rxq_t *erp,
953 __in uint8_t *buffer,
954 __in uint32_t buffer_length,
955 __in uint32_t current_offset,
956 __out uint16_t *lengthp,
957 __out uint32_t *next_offsetp,
958 __out uint32_t *timestamp)
959 {
960 uint16_t buf_len;
961 uint8_t *pkt_start;
962 efx_qword_t *qwordp;
963 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
964
965 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
966
967 buffer += current_offset;
968 pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
969
970 qwordp = (efx_qword_t *)buffer;
971 *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
972 *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
973 buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
974
975 buf_len = EFX_P2ROUNDUP(uint16_t,
976 buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
977 EFX_RX_PACKED_STREAM_ALIGNMENT);
978 *next_offsetp =
979 current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
980
981 EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
982 EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
983
984 if ((*next_offsetp ^ current_offset) &
985 EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
986 rxq_state->eers_rx_packed_stream_credits++;
987
988 return (pkt_start);
989 }
990
991
992 #endif
993
994 __checkReturn efx_rc_t
ef10_rx_qflush(__in efx_rxq_t * erp)995 ef10_rx_qflush(
996 __in efx_rxq_t *erp)
997 {
998 efx_nic_t *enp = erp->er_enp;
999 efx_rc_t rc;
1000
1001 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
1002 goto fail1;
1003
1004 return (0);
1005
1006 fail1:
1007 /*
1008 * EALREADY is not an error, but indicates that the MC has rebooted and
1009 * that the RXQ has already been destroyed. Callers need to know that
1010 * the RXQ flush has completed to avoid waiting until timeout for a
1011 * flush done event that will not be delivered.
1012 */
1013 if (rc != EALREADY)
1014 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1015
1016 return (rc);
1017 }
1018
1019 void
ef10_rx_qenable(__in efx_rxq_t * erp)1020 ef10_rx_qenable(
1021 __in efx_rxq_t *erp)
1022 {
1023 /* FIXME */
1024 _NOTE(ARGUNUSED(erp))
1025 /* FIXME */
1026 }
1027
1028 __checkReturn efx_rc_t
ef10_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in_opt const efx_rxq_type_data_t * type_data,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__in efx_rxq_t * erp)1029 ef10_rx_qcreate(
1030 __in efx_nic_t *enp,
1031 __in unsigned int index,
1032 __in unsigned int label,
1033 __in efx_rxq_type_t type,
1034 __in_opt const efx_rxq_type_data_t *type_data,
1035 __in efsys_mem_t *esmp,
1036 __in size_t ndescs,
1037 __in uint32_t id,
1038 __in unsigned int flags,
1039 __in efx_evq_t *eep,
1040 __in efx_rxq_t *erp)
1041 {
1042 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1043 efx_mcdi_init_rxq_params_t params;
1044 const efx_rx_prefix_layout_t *erpl;
1045 efx_rc_t rc;
1046
1047 _NOTE(ARGUNUSED(id, erp))
1048
1049 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
1050 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1051
1052 memset(¶ms, 0, sizeof (params));
1053 params.buf_size = erp->er_buf_size;
1054
1055 switch (type) {
1056 case EFX_RXQ_TYPE_DEFAULT:
1057 erpl = &ef10_default_rx_prefix_layout;
1058 if (type_data == NULL) {
1059 rc = EINVAL;
1060 goto fail1;
1061 }
1062 erp->er_buf_size = type_data->ertd_default.ed_buf_size;
1063 if (flags & EFX_RXQ_FLAG_USER_MARK) {
1064 rc = ENOTSUP;
1065 goto fail2;
1066 }
1067 if (flags & EFX_RXQ_FLAG_USER_FLAG) {
1068 rc = ENOTSUP;
1069 goto fail3;
1070 }
1071 /*
1072 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated
1073 * it is always delivered from HW in the pseudo-header.
1074 */
1075 break;
1076 #if EFSYS_OPT_RX_PACKED_STREAM
1077 case EFX_RXQ_TYPE_PACKED_STREAM:
1078 erpl = &ef10_packed_stream_rx_prefix_layout;
1079 if (type_data == NULL) {
1080 rc = EINVAL;
1081 goto fail4;
1082 }
1083 switch (type_data->ertd_packed_stream.eps_buf_size) {
1084 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
1085 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
1086 break;
1087 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
1088 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
1089 break;
1090 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
1091 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
1092 break;
1093 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
1094 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
1095 break;
1096 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
1097 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
1098 break;
1099 default:
1100 rc = ENOTSUP;
1101 goto fail5;
1102 }
1103 erp->er_buf_size = type_data->ertd_packed_stream.eps_buf_size;
1104 /* Packed stream pseudo header does not have RSS hash value */
1105 if (flags & EFX_RXQ_FLAG_RSS_HASH) {
1106 rc = ENOTSUP;
1107 goto fail6;
1108 }
1109 if (flags & EFX_RXQ_FLAG_USER_MARK) {
1110 rc = ENOTSUP;
1111 goto fail7;
1112 }
1113 if (flags & EFX_RXQ_FLAG_USER_FLAG) {
1114 rc = ENOTSUP;
1115 goto fail8;
1116 }
1117 break;
1118 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1119 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1120 case EFX_RXQ_TYPE_ES_SUPER_BUFFER:
1121 erpl = &ef10_essb_rx_prefix_layout;
1122 if (type_data == NULL) {
1123 rc = EINVAL;
1124 goto fail9;
1125 }
1126 params.es_bufs_per_desc =
1127 type_data->ertd_es_super_buffer.eessb_bufs_per_desc;
1128 params.es_max_dma_len =
1129 type_data->ertd_es_super_buffer.eessb_max_dma_len;
1130 params.es_buf_stride =
1131 type_data->ertd_es_super_buffer.eessb_buf_stride;
1132 params.hol_block_timeout =
1133 type_data->ertd_es_super_buffer.eessb_hol_block_timeout;
1134 /*
1135 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated
1136 * it is always delivered from HW in the pseudo-header.
1137 */
1138 break;
1139 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1140 default:
1141 rc = ENOTSUP;
1142 goto fail10;
1143 }
1144
1145 #if EFSYS_OPT_RX_PACKED_STREAM
1146 if (params.ps_buf_size != 0) {
1147 /* Check if datapath firmware supports packed stream mode */
1148 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
1149 rc = ENOTSUP;
1150 goto fail11;
1151 }
1152 /* Check if packed stream allows configurable buffer sizes */
1153 if ((params.ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
1154 (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
1155 rc = ENOTSUP;
1156 goto fail12;
1157 }
1158 }
1159 #else /* EFSYS_OPT_RX_PACKED_STREAM */
1160 EFSYS_ASSERT(params.ps_buf_size == 0);
1161 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1162
1163 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1164 if (params.es_bufs_per_desc > 0) {
1165 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) {
1166 rc = ENOTSUP;
1167 goto fail13;
1168 }
1169 if (!EFX_IS_P2ALIGNED(uint32_t, params.es_max_dma_len,
1170 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1171 rc = EINVAL;
1172 goto fail14;
1173 }
1174 if (!EFX_IS_P2ALIGNED(uint32_t, params.es_buf_stride,
1175 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1176 rc = EINVAL;
1177 goto fail15;
1178 }
1179 }
1180 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1181 EFSYS_ASSERT(params.es_bufs_per_desc == 0);
1182 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1183
1184 if (flags & EFX_RXQ_FLAG_INGRESS_MPORT) {
1185 rc = ENOTSUP;
1186 goto fail16;
1187 }
1188
1189 /* Scatter can only be disabled if the firmware supports doing so */
1190 if (flags & EFX_RXQ_FLAG_SCATTER)
1191 params.disable_scatter = B_FALSE;
1192 else
1193 params.disable_scatter = encp->enc_rx_disable_scatter_supported;
1194
1195 if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
1196 params.want_inner_classes = B_TRUE;
1197 else
1198 params.want_inner_classes = B_FALSE;
1199
1200 if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep, label, index,
1201 esmp, ¶ms)) != 0)
1202 goto fail17;
1203
1204 erp->er_eep = eep;
1205 erp->er_label = label;
1206
1207 ef10_ev_rxlabel_init(eep, erp, label, type);
1208
1209 erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
1210
1211 erp->er_prefix_layout = *erpl;
1212
1213 return (0);
1214
1215 fail17:
1216 EFSYS_PROBE(fail15);
1217 fail16:
1218 EFSYS_PROBE(fail14);
1219 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1220 fail15:
1221 EFSYS_PROBE(fail15);
1222 fail14:
1223 EFSYS_PROBE(fail14);
1224 fail13:
1225 EFSYS_PROBE(fail13);
1226 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1227 #if EFSYS_OPT_RX_PACKED_STREAM
1228 fail12:
1229 EFSYS_PROBE(fail12);
1230 fail11:
1231 EFSYS_PROBE(fail11);
1232 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1233 fail10:
1234 EFSYS_PROBE(fail10);
1235 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1236 fail9:
1237 EFSYS_PROBE(fail9);
1238 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1239 #if EFSYS_OPT_RX_PACKED_STREAM
1240 fail8:
1241 EFSYS_PROBE(fail8);
1242 fail7:
1243 EFSYS_PROBE(fail7);
1244 fail6:
1245 EFSYS_PROBE(fail6);
1246 fail5:
1247 EFSYS_PROBE(fail5);
1248 fail4:
1249 EFSYS_PROBE(fail4);
1250 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1251 fail3:
1252 EFSYS_PROBE(fail3);
1253 fail2:
1254 EFSYS_PROBE(fail2);
1255 fail1:
1256 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1257
1258 return (rc);
1259 }
1260
1261 void
ef10_rx_qdestroy(__in efx_rxq_t * erp)1262 ef10_rx_qdestroy(
1263 __in efx_rxq_t *erp)
1264 {
1265 efx_evq_t *eep = erp->er_eep;
1266 unsigned int label = erp->er_label;
1267
1268 ef10_ev_rxlabel_fini(eep, label);
1269 }
1270
1271 #endif /* EFX_OPTS_EF10() */
1272
1273 void
ef10_rx_fini(__in efx_nic_t * enp)1274 ef10_rx_fini(
1275 __in efx_nic_t *enp)
1276 {
1277 #if EFSYS_OPT_RX_SCALE
1278 if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1279 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1280 enp->en_rss_context = 0;
1281 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1282 #else
1283 _NOTE(ARGUNUSED(enp))
1284 #endif /* EFSYS_OPT_RX_SCALE */
1285 }
1286
1287 #endif /* EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10() */
1288