1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright(c) 2019-2021 Xilinx, Inc.
4 * Copyright(c) 2017-2019 Solarflare Communications Inc.
5 *
6 * This software was jointly developed between OKTET Labs (under contract
7 * for Solarflare) and Solarflare Communications, Inc.
8 */
9
10 /* EF10 equal stride packed stream receive native datapath implementation */
11
12 #include <stdbool.h>
13
14 #include <rte_byteorder.h>
15 #include <rte_mbuf.h>
16 #include <rte_io.h>
17
18 #include "efx_types.h"
19 #include "efx_regs_ef10.h"
20 #include "efx.h"
21
22 #include "sfc_debug.h"
23 #include "sfc_tweak.h"
24 #include "sfc_dp_rx.h"
25 #include "sfc_kvargs.h"
26 #include "sfc_ef10.h"
27
28 /* Tunnels are not supported */
29 #define SFC_EF10_RX_EV_ENCAP_SUPPORT 0
30 #include "sfc_ef10_rx_ev.h"
31
32 #define sfc_ef10_essb_rx_err(dpq, ...) \
33 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10_ESSB, ERR, dpq, __VA_ARGS__)
34
35 #define sfc_ef10_essb_rx_info(dpq, ...) \
36 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10_ESSB, INFO, dpq, __VA_ARGS__)
37
38 /*
39 * Fake length for RXQ descriptors in equal stride super-buffer mode
40 * to make hardware happy.
41 */
42 #define SFC_EF10_ESSB_RX_FAKE_BUF_SIZE 32
43
44 /**
45 * Minimum number of Rx buffers the datapath allows to use.
46 *
47 * Each HW Rx descriptor has many Rx buffers. The number of buffers
48 * in one HW Rx descriptor is equal to size of contiguous block
49 * provided by Rx buffers memory pool. The contiguous block size
50 * depends on RTE_DRIVER_MEMPOOL_BUCKET_SIZE_KB and rte_mbuf
51 * data size specified on the memory pool creation. Typical rte_mbuf
52 * data size is about 2k which makes a bit less than 32 buffers in
53 * contiguous block with default bucket size equal to 64k.
54 * Since HW Rx descriptors are pushed by 8 (see SFC_EF10_RX_WPTR_ALIGN),
55 * it makes about 256 as required minimum. Double it in advertised
56 * minimum to allow for at least 2 refill blocks.
57 */
58 #define SFC_EF10_ESSB_RX_DESCS_MIN 512
59
60 /**
61 * Number of Rx buffers should be aligned to.
62 *
63 * There are no extra requirements on alignment since actual number of
64 * pushed Rx buffers will be multiple by contiguous block size which
65 * is unknown beforehand.
66 */
67 #define SFC_EF10_ESSB_RX_DESCS_ALIGN 1
68
69 /**
70 * Maximum number of descriptors/buffers in the Rx ring.
71 * It should guarantee that corresponding event queue never overfill.
72 */
73 #define SFC_EF10_ESSB_RXQ_LIMIT(_nevs) \
74 ((_nevs) - 1 /* head must not step on tail */ - \
75 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \
76 1 /* Rx error */ - 1 /* flush */)
77
78 struct sfc_ef10_essb_rx_sw_desc {
79 struct rte_mbuf *first_mbuf;
80 };
81
82 struct sfc_ef10_essb_rxq {
83 /* Used on data path */
84 unsigned int flags;
85 #define SFC_EF10_ESSB_RXQ_STARTED 0x1
86 #define SFC_EF10_ESSB_RXQ_NOT_RUNNING 0x2
87 #define SFC_EF10_ESSB_RXQ_EXCEPTION 0x4
88 unsigned int rxq_ptr_mask;
89 unsigned int block_size;
90 unsigned int buf_stride;
91 unsigned int bufs_ptr;
92 unsigned int completed;
93 unsigned int pending_id;
94 unsigned int bufs_pending;
95 unsigned int left_in_completed;
96 unsigned int left_in_pending;
97 unsigned int evq_read_ptr;
98 unsigned int evq_ptr_mask;
99 efx_qword_t *evq_hw_ring;
100 struct sfc_ef10_essb_rx_sw_desc *sw_ring;
101 uint16_t port_id;
102
103 /* Used on refill */
104 unsigned int added;
105 unsigned int max_fill_level;
106 unsigned int refill_threshold;
107 struct rte_mempool *refill_mb_pool;
108 efx_qword_t *rxq_hw_ring;
109 volatile void *doorbell;
110
111 /* Datapath receive queue anchor */
112 struct sfc_dp_rxq dp;
113 };
114
115 static inline struct sfc_ef10_essb_rxq *
sfc_ef10_essb_rxq_by_dp_rxq(struct sfc_dp_rxq * dp_rxq)116 sfc_ef10_essb_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq)
117 {
118 return container_of(dp_rxq, struct sfc_ef10_essb_rxq, dp);
119 }
120
121 static struct rte_mbuf *
sfc_ef10_essb_next_mbuf(const struct sfc_ef10_essb_rxq * rxq,struct rte_mbuf * mbuf)122 sfc_ef10_essb_next_mbuf(const struct sfc_ef10_essb_rxq *rxq,
123 struct rte_mbuf *mbuf)
124 {
125 struct rte_mbuf *m;
126
127 m = (struct rte_mbuf *)((uintptr_t)mbuf + rxq->buf_stride);
128 __rte_mbuf_raw_sanity_check(m);
129 return m;
130 }
131
132 static struct rte_mbuf *
sfc_ef10_essb_mbuf_by_index(const struct sfc_ef10_essb_rxq * rxq,struct rte_mbuf * mbuf,unsigned int idx)133 sfc_ef10_essb_mbuf_by_index(const struct sfc_ef10_essb_rxq *rxq,
134 struct rte_mbuf *mbuf, unsigned int idx)
135 {
136 struct rte_mbuf *m;
137
138 m = (struct rte_mbuf *)((uintptr_t)mbuf + idx * rxq->buf_stride);
139 __rte_mbuf_raw_sanity_check(m);
140 return m;
141 }
142
143 static struct rte_mbuf *
sfc_ef10_essb_maybe_next_completed(struct sfc_ef10_essb_rxq * rxq)144 sfc_ef10_essb_maybe_next_completed(struct sfc_ef10_essb_rxq *rxq)
145 {
146 const struct sfc_ef10_essb_rx_sw_desc *rxd;
147
148 if (rxq->left_in_completed != 0) {
149 rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
150 return sfc_ef10_essb_mbuf_by_index(rxq, rxd->first_mbuf,
151 rxq->block_size - rxq->left_in_completed);
152 } else {
153 rxq->completed++;
154 rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
155 rxq->left_in_completed = rxq->block_size;
156 return rxd->first_mbuf;
157 }
158 }
159
160 static void
sfc_ef10_essb_rx_qrefill(struct sfc_ef10_essb_rxq * rxq)161 sfc_ef10_essb_rx_qrefill(struct sfc_ef10_essb_rxq *rxq)
162 {
163 const unsigned int rxq_ptr_mask = rxq->rxq_ptr_mask;
164 unsigned int free_space;
165 unsigned int bulks;
166 void *mbuf_blocks[SFC_EF10_RX_WPTR_ALIGN];
167 unsigned int added = rxq->added;
168
169 free_space = rxq->max_fill_level - (added - rxq->completed);
170
171 if (free_space < rxq->refill_threshold)
172 return;
173
174 bulks = free_space / RTE_DIM(mbuf_blocks);
175 /* refill_threshold guarantees that bulks is positive */
176 SFC_ASSERT(bulks > 0);
177
178 do {
179 unsigned int id;
180 unsigned int i;
181
182 if (unlikely(rte_mempool_get_contig_blocks(rxq->refill_mb_pool,
183 mbuf_blocks, RTE_DIM(mbuf_blocks)) < 0)) {
184 struct rte_eth_dev_data *dev_data =
185 rte_eth_devices[rxq->port_id].data;
186
187 /*
188 * It is hardly a safe way to increment counter
189 * from different contexts, but all PMDs do it.
190 */
191 dev_data->rx_mbuf_alloc_failed += RTE_DIM(mbuf_blocks);
192 /* Return if we have posted nothing yet */
193 if (added == rxq->added)
194 return;
195 /* Push posted */
196 break;
197 }
198
199 for (i = 0, id = added & rxq_ptr_mask;
200 i < RTE_DIM(mbuf_blocks);
201 ++i, ++id) {
202 struct rte_mbuf *m = mbuf_blocks[i];
203 struct sfc_ef10_essb_rx_sw_desc *rxd;
204
205 SFC_ASSERT((id & ~rxq_ptr_mask) == 0);
206 rxd = &rxq->sw_ring[id];
207 rxd->first_mbuf = m;
208
209 /* RX_KER_BYTE_CNT is ignored by firmware */
210 EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id],
211 ESF_DZ_RX_KER_BYTE_CNT,
212 SFC_EF10_ESSB_RX_FAKE_BUF_SIZE,
213 ESF_DZ_RX_KER_BUF_ADDR,
214 rte_mbuf_data_iova_default(m));
215 }
216
217 added += RTE_DIM(mbuf_blocks);
218
219 } while (--bulks > 0);
220
221 SFC_ASSERT(rxq->added != added);
222 rxq->added = added;
223 sfc_ef10_rx_qpush(rxq->doorbell, added, rxq_ptr_mask,
224 &rxq->dp.dpq.dbells);
225 }
226
227 static bool
sfc_ef10_essb_rx_event_get(struct sfc_ef10_essb_rxq * rxq,efx_qword_t * rx_ev)228 sfc_ef10_essb_rx_event_get(struct sfc_ef10_essb_rxq *rxq, efx_qword_t *rx_ev)
229 {
230 *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->evq_ptr_mask];
231
232 if (!sfc_ef10_ev_present(*rx_ev))
233 return false;
234
235 if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) !=
236 FSE_AZ_EV_CODE_RX_EV)) {
237 /*
238 * Do not move read_ptr to keep the event for exception
239 * handling
240 */
241 rxq->flags |= SFC_EF10_ESSB_RXQ_EXCEPTION;
242 sfc_ef10_essb_rx_err(&rxq->dp.dpq,
243 "RxQ exception at EvQ read ptr %#x",
244 rxq->evq_read_ptr);
245 return false;
246 }
247
248 rxq->evq_read_ptr++;
249 return true;
250 }
251
252 static void
sfc_ef10_essb_rx_process_ev(struct sfc_ef10_essb_rxq * rxq,efx_qword_t rx_ev)253 sfc_ef10_essb_rx_process_ev(struct sfc_ef10_essb_rxq *rxq, efx_qword_t rx_ev)
254 {
255 unsigned int ready;
256
257 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) -
258 rxq->bufs_ptr) &
259 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
260
261 rxq->bufs_ptr += ready;
262 rxq->bufs_pending += ready;
263
264 SFC_ASSERT(ready > 0);
265 do {
266 const struct sfc_ef10_essb_rx_sw_desc *rxd;
267 struct rte_mbuf *m;
268 unsigned int todo_bufs;
269 struct rte_mbuf *m0;
270
271 rxd = &rxq->sw_ring[rxq->pending_id];
272 m = sfc_ef10_essb_mbuf_by_index(rxq, rxd->first_mbuf,
273 rxq->block_size - rxq->left_in_pending);
274
275 if (ready < rxq->left_in_pending) {
276 todo_bufs = ready;
277 ready = 0;
278 rxq->left_in_pending -= todo_bufs;
279 } else {
280 todo_bufs = rxq->left_in_pending;
281 ready -= todo_bufs;
282 rxq->left_in_pending = rxq->block_size;
283 if (rxq->pending_id != rxq->rxq_ptr_mask)
284 rxq->pending_id++;
285 else
286 rxq->pending_id = 0;
287 }
288
289 SFC_ASSERT(todo_bufs > 0);
290 --todo_bufs;
291
292 sfc_ef10_rx_ev_to_offloads(rx_ev, m, ~0ull);
293
294 /* Prefetch pseudo-header */
295 rte_prefetch0((uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM);
296
297 m0 = m;
298 while (todo_bufs-- > 0) {
299 m = sfc_ef10_essb_next_mbuf(rxq, m);
300 m->ol_flags = m0->ol_flags;
301 m->packet_type = m0->packet_type;
302 /* Prefetch pseudo-header */
303 rte_prefetch0((uint8_t *)m->buf_addr +
304 RTE_PKTMBUF_HEADROOM);
305 }
306 } while (ready > 0);
307 }
308
309 /*
310 * Below function relies on the following length and layout of the
311 * Rx prefix.
312 */
313 static const efx_rx_prefix_layout_t sfc_ef10_essb_rx_prefix_layout = {
314 .erpl_length = ES_EZ_ESSB_RX_PREFIX_LEN,
315 .erpl_fields = {
316 #define SFC_EF10_ESSB_RX_PREFIX_FIELD(_efx, _ef10) \
317 EFX_RX_PREFIX_FIELD(_efx, ES_EZ_ESSB_RX_PREFIX_ ## _ef10, B_FALSE)
318
319 SFC_EF10_ESSB_RX_PREFIX_FIELD(LENGTH, DATA_LEN),
320 SFC_EF10_ESSB_RX_PREFIX_FIELD(USER_MARK, MARK),
321 SFC_EF10_ESSB_RX_PREFIX_FIELD(RSS_HASH_VALID, HASH_VALID),
322 SFC_EF10_ESSB_RX_PREFIX_FIELD(USER_MARK_VALID, MARK_VALID),
323 SFC_EF10_ESSB_RX_PREFIX_FIELD(USER_FLAG, MATCH_FLAG),
324 SFC_EF10_ESSB_RX_PREFIX_FIELD(RSS_HASH, HASH),
325
326 #undef SFC_EF10_ESSB_RX_PREFIX_FIELD
327 }
328 };
329
330 static unsigned int
sfc_ef10_essb_rx_get_pending(struct sfc_ef10_essb_rxq * rxq,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)331 sfc_ef10_essb_rx_get_pending(struct sfc_ef10_essb_rxq *rxq,
332 struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
333 {
334 unsigned int n_rx_pkts = 0;
335 unsigned int todo_bufs;
336 struct rte_mbuf *m;
337
338 while ((todo_bufs = RTE_MIN(nb_pkts - n_rx_pkts,
339 rxq->bufs_pending)) > 0) {
340 m = sfc_ef10_essb_maybe_next_completed(rxq);
341
342 todo_bufs = RTE_MIN(todo_bufs, rxq->left_in_completed);
343
344 rxq->bufs_pending -= todo_bufs;
345 rxq->left_in_completed -= todo_bufs;
346
347 SFC_ASSERT(todo_bufs > 0);
348 todo_bufs--;
349
350 do {
351 const efx_qword_t *qwordp;
352 uint16_t pkt_len;
353
354 /* Buffers to be discarded have 0 in packet type */
355 if (unlikely(m->packet_type == 0)) {
356 rte_mbuf_raw_free(m);
357 goto next_buf;
358 }
359
360 rx_pkts[n_rx_pkts++] = m;
361
362 /* Parse pseudo-header */
363 qwordp = (const efx_qword_t *)
364 ((uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM);
365 pkt_len =
366 EFX_QWORD_FIELD(*qwordp,
367 ES_EZ_ESSB_RX_PREFIX_DATA_LEN);
368
369 m->data_off = RTE_PKTMBUF_HEADROOM +
370 ES_EZ_ESSB_RX_PREFIX_LEN;
371 m->port = rxq->port_id;
372
373 rte_pktmbuf_pkt_len(m) = pkt_len;
374 rte_pktmbuf_data_len(m) = pkt_len;
375
376 m->ol_flags |=
377 (RTE_MBUF_F_RX_RSS_HASH *
378 !!EFX_TEST_QWORD_BIT(*qwordp,
379 ES_EZ_ESSB_RX_PREFIX_HASH_VALID_LBN)) |
380 (RTE_MBUF_F_RX_FDIR_ID *
381 !!EFX_TEST_QWORD_BIT(*qwordp,
382 ES_EZ_ESSB_RX_PREFIX_MARK_VALID_LBN)) |
383 (RTE_MBUF_F_RX_FDIR *
384 !!EFX_TEST_QWORD_BIT(*qwordp,
385 ES_EZ_ESSB_RX_PREFIX_MATCH_FLAG_LBN));
386
387 /* EFX_QWORD_FIELD converts little-endian to CPU */
388 m->hash.rss =
389 EFX_QWORD_FIELD(*qwordp,
390 ES_EZ_ESSB_RX_PREFIX_HASH);
391 m->hash.fdir.hi =
392 EFX_QWORD_FIELD(*qwordp,
393 ES_EZ_ESSB_RX_PREFIX_MARK);
394
395 next_buf:
396 m = sfc_ef10_essb_next_mbuf(rxq, m);
397 } while (todo_bufs-- > 0);
398 }
399
400 return n_rx_pkts;
401 }
402
403
404 static uint16_t
sfc_ef10_essb_recv_pkts(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)405 sfc_ef10_essb_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
406 uint16_t nb_pkts)
407 {
408 struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(rx_queue);
409 const unsigned int evq_old_read_ptr = rxq->evq_read_ptr;
410 uint16_t n_rx_pkts;
411 efx_qword_t rx_ev;
412
413 if (unlikely(rxq->flags & (SFC_EF10_ESSB_RXQ_NOT_RUNNING |
414 SFC_EF10_ESSB_RXQ_EXCEPTION)))
415 return 0;
416
417 n_rx_pkts = sfc_ef10_essb_rx_get_pending(rxq, rx_pkts, nb_pkts);
418
419 while (n_rx_pkts != nb_pkts &&
420 sfc_ef10_essb_rx_event_get(rxq, &rx_ev)) {
421 /*
422 * DROP_EVENT is an internal to the NIC, software should
423 * never see it and, therefore, may ignore it.
424 */
425
426 sfc_ef10_essb_rx_process_ev(rxq, rx_ev);
427 n_rx_pkts += sfc_ef10_essb_rx_get_pending(rxq,
428 rx_pkts + n_rx_pkts,
429 nb_pkts - n_rx_pkts);
430 }
431
432 sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->evq_ptr_mask,
433 evq_old_read_ptr, rxq->evq_read_ptr);
434
435 /* It is not a problem if we refill in the case of exception */
436 sfc_ef10_essb_rx_qrefill(rxq);
437
438 return n_rx_pkts;
439 }
440
441 static sfc_dp_rx_qdesc_npending_t sfc_ef10_essb_rx_qdesc_npending;
442 static unsigned int
sfc_ef10_essb_rx_qdesc_npending(struct sfc_dp_rxq * dp_rxq)443 sfc_ef10_essb_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq)
444 {
445 struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
446 const unsigned int evq_old_read_ptr = rxq->evq_read_ptr;
447 efx_qword_t rx_ev;
448
449 if (unlikely(rxq->flags & (SFC_EF10_ESSB_RXQ_NOT_RUNNING |
450 SFC_EF10_ESSB_RXQ_EXCEPTION)))
451 return rxq->bufs_pending;
452
453 while (sfc_ef10_essb_rx_event_get(rxq, &rx_ev)) {
454 /*
455 * DROP_EVENT is an internal to the NIC, software should
456 * never see it and, therefore, may ignore it.
457 */
458 sfc_ef10_essb_rx_process_ev(rxq, rx_ev);
459 }
460
461 sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->evq_ptr_mask,
462 evq_old_read_ptr, rxq->evq_read_ptr);
463
464 return rxq->bufs_pending;
465 }
466
467 static sfc_dp_rx_qdesc_status_t sfc_ef10_essb_rx_qdesc_status;
468 static int
sfc_ef10_essb_rx_qdesc_status(struct sfc_dp_rxq * dp_rxq,uint16_t offset)469 sfc_ef10_essb_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset)
470 {
471 struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
472 unsigned int pending = sfc_ef10_essb_rx_qdesc_npending(dp_rxq);
473
474 if (offset < pending)
475 return RTE_ETH_RX_DESC_DONE;
476
477 if (offset < (rxq->added - rxq->completed) * rxq->block_size +
478 rxq->left_in_completed - rxq->block_size)
479 return RTE_ETH_RX_DESC_AVAIL;
480
481 return RTE_ETH_RX_DESC_UNAVAIL;
482 }
483
484 static sfc_dp_rx_get_dev_info_t sfc_ef10_essb_rx_get_dev_info;
485 static void
sfc_ef10_essb_rx_get_dev_info(struct rte_eth_dev_info * dev_info)486 sfc_ef10_essb_rx_get_dev_info(struct rte_eth_dev_info *dev_info)
487 {
488 /*
489 * Number of descriptors just defines maximum number of pushed
490 * descriptors (fill level).
491 */
492 dev_info->rx_desc_lim.nb_min = SFC_EF10_ESSB_RX_DESCS_MIN;
493 dev_info->rx_desc_lim.nb_align = SFC_EF10_ESSB_RX_DESCS_ALIGN;
494 }
495
496 static sfc_dp_rx_pool_ops_supported_t sfc_ef10_essb_rx_pool_ops_supported;
497 static int
sfc_ef10_essb_rx_pool_ops_supported(const char * pool)498 sfc_ef10_essb_rx_pool_ops_supported(const char *pool)
499 {
500 SFC_ASSERT(pool != NULL);
501
502 if (strcmp(pool, "bucket") == 0)
503 return 0;
504
505 return -ENOTSUP;
506 }
507
508 static sfc_dp_rx_qsize_up_rings_t sfc_ef10_essb_rx_qsize_up_rings;
509 static int
sfc_ef10_essb_rx_qsize_up_rings(uint16_t nb_rx_desc,struct sfc_dp_rx_hw_limits * limits,struct rte_mempool * mb_pool,unsigned int * rxq_entries,unsigned int * evq_entries,unsigned int * rxq_max_fill_level)510 sfc_ef10_essb_rx_qsize_up_rings(uint16_t nb_rx_desc,
511 struct sfc_dp_rx_hw_limits *limits,
512 struct rte_mempool *mb_pool,
513 unsigned int *rxq_entries,
514 unsigned int *evq_entries,
515 unsigned int *rxq_max_fill_level)
516 {
517 int rc;
518 struct rte_mempool_info mp_info;
519 unsigned int nb_hw_rx_desc;
520 unsigned int max_events;
521
522 rc = rte_mempool_ops_get_info(mb_pool, &mp_info);
523 if (rc != 0)
524 return -rc;
525 if (mp_info.contig_block_size == 0)
526 return EINVAL;
527
528 /*
529 * Calculate required number of hardware Rx descriptors each
530 * carrying contig block size Rx buffers.
531 * It cannot be less than Rx write pointer alignment plus 1
532 * in order to avoid cases when the ring is guaranteed to be
533 * empty.
534 */
535 nb_hw_rx_desc = RTE_MAX(SFC_DIV_ROUND_UP(nb_rx_desc,
536 mp_info.contig_block_size),
537 SFC_EF10_RX_WPTR_ALIGN + 1);
538 if (nb_hw_rx_desc <= limits->rxq_min_entries) {
539 *rxq_entries = limits->rxq_min_entries;
540 } else {
541 *rxq_entries = rte_align32pow2(nb_hw_rx_desc);
542 if (*rxq_entries > limits->rxq_max_entries)
543 return EINVAL;
544 }
545
546 max_events = RTE_ALIGN_FLOOR(nb_hw_rx_desc, SFC_EF10_RX_WPTR_ALIGN) *
547 mp_info.contig_block_size +
548 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ +
549 1 /* Rx error */ + 1 /* flush */ + 1 /* head-tail space */;
550
551 *evq_entries = rte_align32pow2(max_events);
552 *evq_entries = RTE_MAX(*evq_entries, limits->evq_min_entries);
553 *evq_entries = RTE_MIN(*evq_entries, limits->evq_max_entries);
554
555 /*
556 * May be even maximum event queue size is insufficient to handle
557 * so many Rx descriptors. If so, we should limit Rx queue fill level.
558 */
559 *rxq_max_fill_level = RTE_MIN(nb_rx_desc,
560 SFC_EF10_ESSB_RXQ_LIMIT(*evq_entries));
561 return 0;
562 }
563
564 static sfc_dp_rx_qcreate_t sfc_ef10_essb_rx_qcreate;
565 static int
sfc_ef10_essb_rx_qcreate(uint16_t port_id,uint16_t queue_id,const struct rte_pci_addr * pci_addr,int socket_id,const struct sfc_dp_rx_qcreate_info * info,struct sfc_dp_rxq ** dp_rxqp)566 sfc_ef10_essb_rx_qcreate(uint16_t port_id, uint16_t queue_id,
567 const struct rte_pci_addr *pci_addr, int socket_id,
568 const struct sfc_dp_rx_qcreate_info *info,
569 struct sfc_dp_rxq **dp_rxqp)
570 {
571 struct rte_mempool * const mp = info->refill_mb_pool;
572 struct rte_mempool_info mp_info;
573 struct sfc_ef10_essb_rxq *rxq;
574 int rc;
575
576 rc = ENOTSUP;
577 if (info->nic_dma_info->nb_regions > 0)
578 goto fail_nic_dma;
579
580 rc = rte_mempool_ops_get_info(mp, &mp_info);
581 if (rc != 0) {
582 /* Positive errno is used in the driver */
583 rc = -rc;
584 goto fail_get_contig_block_size;
585 }
586
587 /* Check if the mempool provides block dequeue */
588 rc = EINVAL;
589 if (mp_info.contig_block_size == 0)
590 goto fail_no_block_dequeue;
591
592 rc = ENOMEM;
593 rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq),
594 RTE_CACHE_LINE_SIZE, socket_id);
595 if (rxq == NULL)
596 goto fail_rxq_alloc;
597
598 sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr);
599
600 rc = ENOMEM;
601 rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring",
602 info->rxq_entries,
603 sizeof(*rxq->sw_ring),
604 RTE_CACHE_LINE_SIZE, socket_id);
605 if (rxq->sw_ring == NULL)
606 goto fail_desc_alloc;
607
608 rxq->block_size = mp_info.contig_block_size;
609 rxq->buf_stride = mp->header_size + mp->elt_size + mp->trailer_size;
610 rxq->rxq_ptr_mask = info->rxq_entries - 1;
611 rxq->evq_ptr_mask = info->evq_entries - 1;
612 rxq->evq_hw_ring = info->evq_hw_ring;
613 rxq->port_id = port_id;
614
615 rxq->max_fill_level = info->max_fill_level / mp_info.contig_block_size;
616 rxq->refill_threshold =
617 RTE_MAX(info->refill_threshold / mp_info.contig_block_size,
618 SFC_EF10_RX_WPTR_ALIGN);
619 rxq->refill_mb_pool = mp;
620 rxq->rxq_hw_ring = info->rxq_hw_ring;
621
622 rxq->doorbell = (volatile uint8_t *)info->mem_bar +
623 ER_DZ_RX_DESC_UPD_REG_OFST +
624 (info->hw_index << info->vi_window_shift);
625
626 sfc_ef10_essb_rx_info(&rxq->dp.dpq, "RxQ doorbell is %p",
627 rxq->doorbell);
628 sfc_ef10_essb_rx_info(&rxq->dp.dpq,
629 "block size is %u, buf stride is %u",
630 rxq->block_size, rxq->buf_stride);
631 sfc_ef10_essb_rx_info(&rxq->dp.dpq,
632 "max fill level is %u descs (%u bufs), "
633 "refill threshold %u descs (%u bufs)",
634 rxq->max_fill_level,
635 rxq->max_fill_level * rxq->block_size,
636 rxq->refill_threshold,
637 rxq->refill_threshold * rxq->block_size);
638
639 *dp_rxqp = &rxq->dp;
640 return 0;
641
642 fail_desc_alloc:
643 rte_free(rxq);
644
645 fail_rxq_alloc:
646 fail_no_block_dequeue:
647 fail_get_contig_block_size:
648 fail_nic_dma:
649 return rc;
650 }
651
652 static sfc_dp_rx_qdestroy_t sfc_ef10_essb_rx_qdestroy;
653 static void
sfc_ef10_essb_rx_qdestroy(struct sfc_dp_rxq * dp_rxq)654 sfc_ef10_essb_rx_qdestroy(struct sfc_dp_rxq *dp_rxq)
655 {
656 struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
657
658 rte_free(rxq->sw_ring);
659 rte_free(rxq);
660 }
661
662 static sfc_dp_rx_qstart_t sfc_ef10_essb_rx_qstart;
663 static int
sfc_ef10_essb_rx_qstart(struct sfc_dp_rxq * dp_rxq,unsigned int evq_read_ptr,const efx_rx_prefix_layout_t * pinfo)664 sfc_ef10_essb_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr,
665 const efx_rx_prefix_layout_t *pinfo)
666 {
667 struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
668
669 if (pinfo->erpl_length != sfc_ef10_essb_rx_prefix_layout.erpl_length)
670 return ENOTSUP;
671
672 if (efx_rx_prefix_layout_check(pinfo,
673 &sfc_ef10_essb_rx_prefix_layout) != 0)
674 return ENOTSUP;
675
676 rxq->evq_read_ptr = evq_read_ptr;
677
678 /* Initialize before refill */
679 rxq->completed = rxq->pending_id = rxq->added = 0;
680 rxq->left_in_completed = rxq->left_in_pending = rxq->block_size;
681 rxq->bufs_ptr = UINT_MAX;
682 rxq->bufs_pending = 0;
683
684 sfc_ef10_essb_rx_qrefill(rxq);
685
686 rxq->flags |= SFC_EF10_ESSB_RXQ_STARTED;
687 rxq->flags &=
688 ~(SFC_EF10_ESSB_RXQ_NOT_RUNNING | SFC_EF10_ESSB_RXQ_EXCEPTION);
689
690 return 0;
691 }
692
693 static sfc_dp_rx_qstop_t sfc_ef10_essb_rx_qstop;
694 static void
sfc_ef10_essb_rx_qstop(struct sfc_dp_rxq * dp_rxq,unsigned int * evq_read_ptr)695 sfc_ef10_essb_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr)
696 {
697 struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
698
699 rxq->flags |= SFC_EF10_ESSB_RXQ_NOT_RUNNING;
700
701 *evq_read_ptr = rxq->evq_read_ptr;
702 }
703
704 static sfc_dp_rx_qrx_ev_t sfc_ef10_essb_rx_qrx_ev;
705 static bool
sfc_ef10_essb_rx_qrx_ev(struct sfc_dp_rxq * dp_rxq,__rte_unused unsigned int id)706 sfc_ef10_essb_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id)
707 {
708 __rte_unused struct sfc_ef10_essb_rxq *rxq;
709
710 rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
711 SFC_ASSERT(rxq->flags & SFC_EF10_ESSB_RXQ_NOT_RUNNING);
712
713 /*
714 * It is safe to ignore Rx event since we free all mbufs on
715 * queue purge anyway.
716 */
717
718 return false;
719 }
720
721 static sfc_dp_rx_qpurge_t sfc_ef10_essb_rx_qpurge;
722 static void
sfc_ef10_essb_rx_qpurge(struct sfc_dp_rxq * dp_rxq)723 sfc_ef10_essb_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
724 {
725 struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
726 unsigned int i;
727 const struct sfc_ef10_essb_rx_sw_desc *rxd;
728 struct rte_mbuf *m;
729
730 for (i = rxq->completed; i != rxq->added; ++i) {
731 rxd = &rxq->sw_ring[i & rxq->rxq_ptr_mask];
732 m = sfc_ef10_essb_mbuf_by_index(rxq, rxd->first_mbuf,
733 rxq->block_size - rxq->left_in_completed);
734 while (rxq->left_in_completed > 0) {
735 rte_mbuf_raw_free(m);
736 m = sfc_ef10_essb_next_mbuf(rxq, m);
737 rxq->left_in_completed--;
738 }
739 rxq->left_in_completed = rxq->block_size;
740 }
741
742 rxq->flags &= ~SFC_EF10_ESSB_RXQ_STARTED;
743 }
744
745 struct sfc_dp_rx sfc_ef10_essb_rx = {
746 .dp = {
747 .name = SFC_KVARG_DATAPATH_EF10_ESSB,
748 .type = SFC_DP_RX,
749 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10 |
750 SFC_DP_HW_FW_CAP_RX_ES_SUPER_BUFFER,
751 },
752 .features = SFC_DP_RX_FEAT_FLOW_FLAG |
753 SFC_DP_RX_FEAT_FLOW_MARK,
754 .dev_offload_capa = RTE_ETH_RX_OFFLOAD_CHECKSUM |
755 RTE_ETH_RX_OFFLOAD_RSS_HASH,
756 .queue_offload_capa = 0,
757 .get_dev_info = sfc_ef10_essb_rx_get_dev_info,
758 .pool_ops_supported = sfc_ef10_essb_rx_pool_ops_supported,
759 .qsize_up_rings = sfc_ef10_essb_rx_qsize_up_rings,
760 .qcreate = sfc_ef10_essb_rx_qcreate,
761 .qdestroy = sfc_ef10_essb_rx_qdestroy,
762 .qstart = sfc_ef10_essb_rx_qstart,
763 .qstop = sfc_ef10_essb_rx_qstop,
764 .qrx_ev = sfc_ef10_essb_rx_qrx_ev,
765 .qpurge = sfc_ef10_essb_rx_qpurge,
766 .supported_ptypes_get = sfc_ef10_supported_ptypes_get,
767 .qdesc_npending = sfc_ef10_essb_rx_qdesc_npending,
768 .qdesc_status = sfc_ef10_essb_rx_qdesc_status,
769 .pkt_burst = sfc_ef10_essb_recv_pkts,
770 };
771