1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright(c) 2019-2021 Xilinx, Inc.
4 * Copyright(c) 2016-2019 Solarflare Communications Inc.
5 *
6 * This software was jointly developed between OKTET Labs (under contract
7 * for Solarflare) and Solarflare Communications, Inc.
8 */
9
10 #include <stdbool.h>
11
12 #include <rte_mbuf.h>
13 #include <rte_io.h>
14 #include <rte_ip.h>
15 #include <rte_tcp.h>
16
17 #include "efx.h"
18 #include "efx_types.h"
19 #include "efx_regs.h"
20 #include "efx_regs_ef10.h"
21
22 #include "sfc_debug.h"
23 #include "sfc_dp_tx.h"
24 #include "sfc_tweak.h"
25 #include "sfc_kvargs.h"
26 #include "sfc_ef10.h"
27 #include "sfc_tso.h"
28
29 #define sfc_ef10_tx_err(dpq, ...) \
30 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__)
31
32 #define sfc_ef10_tx_info(dpq, ...) \
33 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, INFO, dpq, __VA_ARGS__)
34
35 /** Maximum length of the DMA descriptor data */
36 #define SFC_EF10_TX_DMA_DESC_LEN_MAX \
37 ((1u << ESF_DZ_TX_KER_BYTE_CNT_WIDTH) - 1)
38
39 /**
40 * Maximum number of descriptors/buffers in the Tx ring.
41 * It should guarantee that corresponding event queue never overfill.
42 * EF10 native datapath uses event queue of the same size as Tx queue.
43 * Maximum number of events on datapath can be estimated as number of
44 * Tx queue entries (one event per Tx buffer in the worst case) plus
45 * Tx error and flush events.
46 */
47 #define SFC_EF10_TXQ_LIMIT(_ndesc) \
48 ((_ndesc) - 1 /* head must not step on tail */ - \
49 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \
50 1 /* Rx error */ - 1 /* flush */)
51
52 struct sfc_ef10_tx_sw_desc {
53 struct rte_mbuf *mbuf;
54 };
55
56 struct sfc_ef10_txq {
57 unsigned int flags;
58 #define SFC_EF10_TXQ_STARTED 0x1
59 #define SFC_EF10_TXQ_NOT_RUNNING 0x2
60 #define SFC_EF10_TXQ_EXCEPTION 0x4
61
62 unsigned int ptr_mask;
63 unsigned int added;
64 unsigned int completed;
65 unsigned int max_fill_level;
66 unsigned int free_thresh;
67 unsigned int evq_read_ptr;
68 struct sfc_ef10_tx_sw_desc *sw_ring;
69 efx_qword_t *txq_hw_ring;
70 volatile void *doorbell;
71 efx_qword_t *evq_hw_ring;
72 uint8_t *tsoh;
73 rte_iova_t tsoh_iova;
74 uint16_t tso_tcp_header_offset_limit;
75
76 /* Datapath transmit queue anchor */
77 struct sfc_dp_txq dp;
78 };
79
80 static inline struct sfc_ef10_txq *
sfc_ef10_txq_by_dp_txq(struct sfc_dp_txq * dp_txq)81 sfc_ef10_txq_by_dp_txq(struct sfc_dp_txq *dp_txq)
82 {
83 return container_of(dp_txq, struct sfc_ef10_txq, dp);
84 }
85
86 static bool
sfc_ef10_tx_get_event(struct sfc_ef10_txq * txq,efx_qword_t * tx_ev)87 sfc_ef10_tx_get_event(struct sfc_ef10_txq *txq, efx_qword_t *tx_ev)
88 {
89 volatile efx_qword_t *evq_hw_ring = txq->evq_hw_ring;
90
91 /*
92 * Exception flag is set when reap is done.
93 * It is never done twice per packet burst get and absence of
94 * the flag is checked on burst get entry.
95 */
96 SFC_ASSERT((txq->flags & SFC_EF10_TXQ_EXCEPTION) == 0);
97
98 *tx_ev = evq_hw_ring[txq->evq_read_ptr & txq->ptr_mask];
99
100 if (!sfc_ef10_ev_present(*tx_ev))
101 return false;
102
103 if (unlikely(EFX_QWORD_FIELD(*tx_ev, FSF_AZ_EV_CODE) !=
104 FSE_AZ_EV_CODE_TX_EV)) {
105 /*
106 * Do not move read_ptr to keep the event for exception
107 * handling by the control path.
108 */
109 txq->flags |= SFC_EF10_TXQ_EXCEPTION;
110 sfc_ef10_tx_err(&txq->dp.dpq,
111 "TxQ exception at EvQ read ptr %#x",
112 txq->evq_read_ptr);
113 return false;
114 }
115
116 txq->evq_read_ptr++;
117 return true;
118 }
119
120 static unsigned int
sfc_ef10_tx_process_events(struct sfc_ef10_txq * txq)121 sfc_ef10_tx_process_events(struct sfc_ef10_txq *txq)
122 {
123 const unsigned int curr_done = txq->completed - 1;
124 unsigned int anew_done = curr_done;
125 efx_qword_t tx_ev;
126
127 while (sfc_ef10_tx_get_event(txq, &tx_ev)) {
128 /*
129 * DROP_EVENT is an internal to the NIC, software should
130 * never see it and, therefore, may ignore it.
131 */
132
133 /* Update the latest done descriptor */
134 anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX);
135 }
136 return (anew_done - curr_done) & txq->ptr_mask;
137 }
138
139 static void
sfc_ef10_tx_reap(struct sfc_ef10_txq * txq)140 sfc_ef10_tx_reap(struct sfc_ef10_txq *txq)
141 {
142 const unsigned int old_read_ptr = txq->evq_read_ptr;
143 const unsigned int ptr_mask = txq->ptr_mask;
144 unsigned int completed = txq->completed;
145 unsigned int pending = completed;
146
147 pending += sfc_ef10_tx_process_events(txq);
148
149 if (pending != completed) {
150 struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE];
151 unsigned int nb = 0;
152
153 do {
154 struct sfc_ef10_tx_sw_desc *txd;
155 struct rte_mbuf *m;
156
157 txd = &txq->sw_ring[completed & ptr_mask];
158 if (txd->mbuf == NULL)
159 continue;
160
161 m = rte_pktmbuf_prefree_seg(txd->mbuf);
162 txd->mbuf = NULL;
163 if (m == NULL)
164 continue;
165
166 if ((nb == RTE_DIM(bulk)) ||
167 ((nb != 0) && (m->pool != bulk[0]->pool))) {
168 rte_mempool_put_bulk(bulk[0]->pool,
169 (void *)bulk, nb);
170 nb = 0;
171 }
172
173 bulk[nb++] = m;
174 } while (++completed != pending);
175
176 if (nb != 0)
177 rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb);
178
179 txq->completed = completed;
180 }
181
182 sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr,
183 txq->evq_read_ptr);
184 }
185
186 static void
sfc_ef10_tx_qdesc_dma_create(rte_iova_t addr,uint16_t size,bool eop,efx_qword_t * edp)187 sfc_ef10_tx_qdesc_dma_create(rte_iova_t addr, uint16_t size, bool eop,
188 efx_qword_t *edp)
189 {
190 EFX_POPULATE_QWORD_4(*edp,
191 ESF_DZ_TX_KER_TYPE, 0,
192 ESF_DZ_TX_KER_CONT, !eop,
193 ESF_DZ_TX_KER_BYTE_CNT, size,
194 ESF_DZ_TX_KER_BUF_ADDR, addr);
195 }
196
197 static void
sfc_ef10_tx_qdesc_tso2_create(struct sfc_ef10_txq * const txq,unsigned int added,uint16_t ipv4_id,uint16_t outer_ipv4_id,uint32_t tcp_seq,uint16_t tcp_mss)198 sfc_ef10_tx_qdesc_tso2_create(struct sfc_ef10_txq * const txq,
199 unsigned int added, uint16_t ipv4_id,
200 uint16_t outer_ipv4_id, uint32_t tcp_seq,
201 uint16_t tcp_mss)
202 {
203 EFX_POPULATE_QWORD_5(txq->txq_hw_ring[added & txq->ptr_mask],
204 ESF_DZ_TX_DESC_IS_OPT, 1,
205 ESF_DZ_TX_OPTION_TYPE,
206 ESE_DZ_TX_OPTION_DESC_TSO,
207 ESF_DZ_TX_TSO_OPTION_TYPE,
208 ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A,
209 ESF_DZ_TX_TSO_IP_ID, ipv4_id,
210 ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq);
211 EFX_POPULATE_QWORD_5(txq->txq_hw_ring[(added + 1) & txq->ptr_mask],
212 ESF_DZ_TX_DESC_IS_OPT, 1,
213 ESF_DZ_TX_OPTION_TYPE,
214 ESE_DZ_TX_OPTION_DESC_TSO,
215 ESF_DZ_TX_TSO_OPTION_TYPE,
216 ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B,
217 ESF_DZ_TX_TSO_TCP_MSS, tcp_mss,
218 ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id);
219 }
220
221 static inline void
sfc_ef10_tx_qpush(struct sfc_ef10_txq * txq,unsigned int added,unsigned int pushed)222 sfc_ef10_tx_qpush(struct sfc_ef10_txq *txq, unsigned int added,
223 unsigned int pushed)
224 {
225 efx_qword_t desc;
226 efx_oword_t oword;
227
228 /*
229 * This improves performance by pushing a TX descriptor at the same
230 * time as the doorbell. The descriptor must be added to the TXQ,
231 * so that can be used if the hardware decides not to use the pushed
232 * descriptor.
233 */
234 desc.eq_u64[0] = txq->txq_hw_ring[pushed & txq->ptr_mask].eq_u64[0];
235 EFX_POPULATE_OWORD_3(oword,
236 ERF_DZ_TX_DESC_WPTR, added & txq->ptr_mask,
237 ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1),
238 ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0));
239
240 /* DMA sync to device is not required */
241
242 /*
243 * rte_io_wmb() which guarantees that the STORE operations
244 * (i.e. Tx and event descriptor updates) that precede
245 * the rte_io_wmb() call are visible to NIC before the STORE
246 * operations that follow it (i.e. doorbell write).
247 */
248 rte_io_wmb();
249
250 *(volatile efsys_uint128_t *)txq->doorbell = oword.eo_u128[0];
251 txq->dp.dpq.dbells++;
252 }
253
254 static unsigned int
sfc_ef10_tx_pkt_descs_max(const struct rte_mbuf * m)255 sfc_ef10_tx_pkt_descs_max(const struct rte_mbuf *m)
256 {
257 unsigned int extra_descs_per_seg;
258 unsigned int extra_descs_per_pkt;
259
260 /*
261 * VLAN offload is not supported yet, so no extra descriptors
262 * are required for VLAN option descriptor.
263 */
264
265 /** Maximum length of the mbuf segment data */
266 #define SFC_MBUF_SEG_LEN_MAX UINT16_MAX
267 RTE_BUILD_BUG_ON(sizeof(m->data_len) != 2);
268
269 /*
270 * Each segment is already counted once below. So, calculate
271 * how many extra DMA descriptors may be required per segment in
272 * the worst case because of maximum DMA descriptor length limit.
273 * If maximum segment length is less or equal to maximum DMA
274 * descriptor length, no extra DMA descriptors are required.
275 */
276 extra_descs_per_seg =
277 (SFC_MBUF_SEG_LEN_MAX - 1) / SFC_EF10_TX_DMA_DESC_LEN_MAX;
278
279 /** Maximum length of the packet */
280 #define SFC_MBUF_PKT_LEN_MAX UINT32_MAX
281 RTE_BUILD_BUG_ON(sizeof(m->pkt_len) != 4);
282
283 /*
284 * One more limitation on maximum number of extra DMA descriptors
285 * comes from slicing entire packet because of DMA descriptor length
286 * limit taking into account that there is at least one segment
287 * which is already counted below (so division of the maximum
288 * packet length minus one with round down).
289 * TSO is not supported yet, so packet length is limited by
290 * maximum PDU size.
291 */
292 extra_descs_per_pkt =
293 (RTE_MIN((unsigned int)EFX_MAC_PDU_MAX,
294 SFC_MBUF_PKT_LEN_MAX) - 1) /
295 SFC_EF10_TX_DMA_DESC_LEN_MAX;
296
297 return m->nb_segs + RTE_MIN(m->nb_segs * extra_descs_per_seg,
298 extra_descs_per_pkt);
299 }
300
301 static bool
sfc_ef10_try_reap(struct sfc_ef10_txq * const txq,unsigned int added,unsigned int needed_desc,unsigned int * dma_desc_space,bool * reap_done)302 sfc_ef10_try_reap(struct sfc_ef10_txq * const txq, unsigned int added,
303 unsigned int needed_desc, unsigned int *dma_desc_space,
304 bool *reap_done)
305 {
306 if (*reap_done)
307 return false;
308
309 if (added != txq->added) {
310 sfc_ef10_tx_qpush(txq, added, txq->added);
311 txq->added = added;
312 }
313
314 sfc_ef10_tx_reap(txq);
315 *reap_done = true;
316
317 /*
318 * Recalculate DMA descriptor space since Tx reap may change
319 * the number of completed descriptors
320 */
321 *dma_desc_space = txq->max_fill_level -
322 (added - txq->completed);
323
324 return (needed_desc <= *dma_desc_space);
325 }
326
327 static uint16_t
sfc_ef10_prepare_pkts(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)328 sfc_ef10_prepare_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
329 uint16_t nb_pkts)
330 {
331 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue);
332 uint16_t i;
333
334 for (i = 0; i < nb_pkts; i++) {
335 struct rte_mbuf *m = tx_pkts[i];
336 int ret;
337
338 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG
339 /*
340 * In non-TSO case, check that a packet segments do not exceed
341 * the size limit. Perform the check in debug mode since MTU
342 * more than 9k is not supported, but the limit here is 16k-1.
343 */
344 if (!(m->ol_flags & RTE_MBUF_F_TX_TCP_SEG)) {
345 struct rte_mbuf *m_seg;
346
347 for (m_seg = m; m_seg != NULL; m_seg = m_seg->next) {
348 if (m_seg->data_len >
349 SFC_EF10_TX_DMA_DESC_LEN_MAX) {
350 rte_errno = EINVAL;
351 break;
352 }
353 }
354 }
355 #endif
356 ret = sfc_dp_tx_prepare_pkt(m, 0, SFC_TSOH_STD_LEN,
357 txq->tso_tcp_header_offset_limit,
358 txq->max_fill_level,
359 SFC_EF10_TSO_OPT_DESCS_NUM, 0);
360 if (unlikely(ret != 0)) {
361 rte_errno = ret;
362 break;
363 }
364 }
365
366 return i;
367 }
368
369 static int
sfc_ef10_xmit_tso_pkt(struct sfc_ef10_txq * const txq,struct rte_mbuf * m_seg,unsigned int * added,unsigned int * dma_desc_space,bool * reap_done)370 sfc_ef10_xmit_tso_pkt(struct sfc_ef10_txq * const txq, struct rte_mbuf *m_seg,
371 unsigned int *added, unsigned int *dma_desc_space,
372 bool *reap_done)
373 {
374 size_t iph_off = ((m_seg->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ?
375 m_seg->outer_l2_len + m_seg->outer_l3_len : 0) +
376 m_seg->l2_len;
377 size_t tcph_off = iph_off + m_seg->l3_len;
378 size_t header_len = tcph_off + m_seg->l4_len;
379 /* Offset of the payload in the last segment that contains the header */
380 size_t in_off = 0;
381 const struct rte_tcp_hdr *th;
382 uint16_t packet_id = 0;
383 uint16_t outer_packet_id = 0;
384 uint32_t sent_seq;
385 uint8_t *hdr_addr;
386 rte_iova_t hdr_iova;
387 struct rte_mbuf *first_m_seg = m_seg;
388 unsigned int pkt_start = *added;
389 unsigned int needed_desc;
390 struct rte_mbuf *m_seg_to_free_up_to = first_m_seg;
391 bool eop;
392
393 /*
394 * Preliminary estimation of required DMA descriptors, including extra
395 * descriptor for TSO header that is needed when the header is
396 * separated from payload in one segment. It does not include
397 * extra descriptors that may appear when a big segment is split across
398 * several descriptors.
399 */
400 needed_desc = m_seg->nb_segs +
401 (unsigned int)SFC_EF10_TSO_OPT_DESCS_NUM +
402 (unsigned int)SFC_EF10_TSO_HDR_DESCS_NUM;
403
404 if (needed_desc > *dma_desc_space &&
405 !sfc_ef10_try_reap(txq, pkt_start, needed_desc,
406 dma_desc_space, reap_done)) {
407 /*
408 * If a future Tx reap may increase available DMA descriptor
409 * space, do not try to send the packet.
410 */
411 if (txq->completed != pkt_start)
412 return ENOSPC;
413 /*
414 * Do not allow to send packet if the maximum DMA
415 * descriptor space is not sufficient to hold TSO
416 * descriptors, header descriptor and at least 1
417 * segment descriptor.
418 */
419 if (*dma_desc_space < SFC_EF10_TSO_OPT_DESCS_NUM +
420 SFC_EF10_TSO_HDR_DESCS_NUM + 1)
421 return EMSGSIZE;
422 }
423
424 /* Check if the header is not fragmented */
425 if (rte_pktmbuf_data_len(m_seg) >= header_len) {
426 hdr_addr = rte_pktmbuf_mtod(m_seg, uint8_t *);
427 hdr_iova = rte_mbuf_data_iova(m_seg);
428 if (rte_pktmbuf_data_len(m_seg) == header_len) {
429 /* Cannot send a packet that consists only of header */
430 if (unlikely(m_seg->next == NULL))
431 return EMSGSIZE;
432 /*
433 * Associate header mbuf with header descriptor
434 * which is located after TSO descriptors.
435 */
436 txq->sw_ring[(pkt_start + SFC_EF10_TSO_OPT_DESCS_NUM) &
437 txq->ptr_mask].mbuf = m_seg;
438 m_seg = m_seg->next;
439 in_off = 0;
440
441 /*
442 * If there is no payload offset (payload starts at the
443 * beginning of a segment) then an extra descriptor for
444 * separated header is not needed.
445 */
446 needed_desc--;
447 } else {
448 in_off = header_len;
449 }
450 } else {
451 unsigned int copied_segs;
452 unsigned int hdr_addr_off = (*added & txq->ptr_mask) *
453 SFC_TSOH_STD_LEN;
454
455 /*
456 * Discard a packet if header linearization is needed but
457 * the header is too big.
458 * Duplicate Tx prepare check here to avoid spoil of
459 * memory if Tx prepare is skipped.
460 */
461 if (unlikely(header_len > SFC_TSOH_STD_LEN))
462 return EMSGSIZE;
463
464 hdr_addr = txq->tsoh + hdr_addr_off;
465 hdr_iova = txq->tsoh_iova + hdr_addr_off;
466 copied_segs = sfc_tso_prepare_header(hdr_addr, header_len,
467 &m_seg, &in_off);
468
469 /* Cannot send a packet that consists only of header */
470 if (unlikely(m_seg == NULL))
471 return EMSGSIZE;
472
473 m_seg_to_free_up_to = m_seg;
474 /*
475 * Reduce the number of needed descriptors by the number of
476 * segments that entirely consist of header data.
477 */
478 needed_desc -= copied_segs;
479
480 /* Extra descriptor for separated header is not needed */
481 if (in_off == 0)
482 needed_desc--;
483 }
484
485 /*
486 * 8000-series EF10 hardware requires that innermost IP length
487 * be greater than or equal to the value which each segment is
488 * supposed to have; otherwise, TCP checksum will be incorrect.
489 *
490 * The same concern applies to outer UDP datagram length field.
491 */
492 switch (m_seg->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
493 case RTE_MBUF_F_TX_TUNNEL_VXLAN:
494 /* FALLTHROUGH */
495 case RTE_MBUF_F_TX_TUNNEL_GENEVE:
496 sfc_tso_outer_udp_fix_len(first_m_seg, hdr_addr);
497 break;
498 default:
499 break;
500 }
501
502 sfc_tso_innermost_ip_fix_len(first_m_seg, hdr_addr, iph_off);
503
504 /*
505 * Tx prepare has debug-only checks that offload flags are correctly
506 * filled in TSO mbuf. Use zero IPID if there is no IPv4 flag.
507 * If the packet is still IPv4, HW will simply start from zero IPID.
508 */
509 if (first_m_seg->ol_flags & RTE_MBUF_F_TX_IPV4)
510 packet_id = sfc_tso_ip4_get_ipid(hdr_addr, iph_off);
511
512 if (first_m_seg->ol_flags & RTE_MBUF_F_TX_OUTER_IPV4)
513 outer_packet_id = sfc_tso_ip4_get_ipid(hdr_addr,
514 first_m_seg->outer_l2_len);
515
516 th = (const struct rte_tcp_hdr *)(hdr_addr + tcph_off);
517 rte_memcpy(&sent_seq, &th->sent_seq, sizeof(uint32_t));
518 sent_seq = rte_be_to_cpu_32(sent_seq);
519
520 sfc_ef10_tx_qdesc_tso2_create(txq, *added, packet_id, outer_packet_id,
521 sent_seq, first_m_seg->tso_segsz);
522 (*added) += SFC_EF10_TSO_OPT_DESCS_NUM;
523
524 sfc_ef10_tx_qdesc_dma_create(hdr_iova, header_len, false,
525 &txq->txq_hw_ring[(*added) & txq->ptr_mask]);
526 (*added)++;
527
528 do {
529 rte_iova_t next_frag = rte_mbuf_data_iova(m_seg);
530 unsigned int seg_len = rte_pktmbuf_data_len(m_seg);
531 unsigned int id;
532
533 next_frag += in_off;
534 seg_len -= in_off;
535 in_off = 0;
536
537 do {
538 rte_iova_t frag_addr = next_frag;
539 size_t frag_len;
540
541 frag_len = RTE_MIN(seg_len,
542 SFC_EF10_TX_DMA_DESC_LEN_MAX);
543
544 next_frag += frag_len;
545 seg_len -= frag_len;
546
547 eop = (seg_len == 0 && m_seg->next == NULL);
548
549 id = (*added) & txq->ptr_mask;
550 (*added)++;
551
552 /*
553 * Initially we assume that one DMA descriptor is needed
554 * for every segment. When the segment is split across
555 * several DMA descriptors, increase the estimation.
556 */
557 needed_desc += (seg_len != 0);
558
559 /*
560 * When no more descriptors can be added, but not all
561 * segments are processed.
562 */
563 if (*added - pkt_start == *dma_desc_space &&
564 !eop &&
565 !sfc_ef10_try_reap(txq, pkt_start, needed_desc,
566 dma_desc_space, reap_done)) {
567 struct rte_mbuf *m;
568 struct rte_mbuf *m_next;
569
570 if (txq->completed != pkt_start) {
571 unsigned int i;
572
573 /*
574 * Reset mbuf associations with added
575 * descriptors.
576 */
577 for (i = pkt_start; i != *added; i++) {
578 id = i & txq->ptr_mask;
579 txq->sw_ring[id].mbuf = NULL;
580 }
581 return ENOSPC;
582 }
583
584 /* Free the segments that cannot be sent */
585 for (m = m_seg->next; m != NULL; m = m_next) {
586 m_next = m->next;
587 rte_pktmbuf_free_seg(m);
588 }
589 eop = true;
590 /* Ignore the rest of the segment */
591 seg_len = 0;
592 }
593
594 sfc_ef10_tx_qdesc_dma_create(frag_addr, frag_len,
595 eop, &txq->txq_hw_ring[id]);
596
597 } while (seg_len != 0);
598
599 txq->sw_ring[id].mbuf = m_seg;
600
601 m_seg = m_seg->next;
602 } while (!eop);
603
604 /*
605 * Free segments which content was entirely copied to the TSO header
606 * memory space of Tx queue
607 */
608 for (m_seg = first_m_seg; m_seg != m_seg_to_free_up_to;) {
609 struct rte_mbuf *seg_to_free = m_seg;
610
611 m_seg = m_seg->next;
612 rte_pktmbuf_free_seg(seg_to_free);
613 }
614
615 return 0;
616 }
617
618 static uint16_t
sfc_ef10_xmit_pkts(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)619 sfc_ef10_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
620 {
621 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue);
622 unsigned int added;
623 unsigned int dma_desc_space;
624 bool reap_done;
625 struct rte_mbuf **pktp;
626 struct rte_mbuf **pktp_end;
627
628 if (unlikely(txq->flags &
629 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION)))
630 return 0;
631
632 added = txq->added;
633 dma_desc_space = txq->max_fill_level - (added - txq->completed);
634
635 reap_done = (dma_desc_space < txq->free_thresh);
636 if (reap_done) {
637 sfc_ef10_tx_reap(txq);
638 dma_desc_space = txq->max_fill_level - (added - txq->completed);
639 }
640
641 for (pktp = &tx_pkts[0], pktp_end = &tx_pkts[nb_pkts];
642 pktp != pktp_end;
643 ++pktp) {
644 struct rte_mbuf *m_seg = *pktp;
645 unsigned int pkt_start = added;
646 uint32_t pkt_len;
647
648 if (likely(pktp + 1 != pktp_end))
649 rte_mbuf_prefetch_part1(pktp[1]);
650
651 if (m_seg->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
652 int rc;
653
654 rc = sfc_ef10_xmit_tso_pkt(txq, m_seg, &added,
655 &dma_desc_space, &reap_done);
656 if (rc != 0) {
657 added = pkt_start;
658
659 /* Packet can be sent in following xmit calls */
660 if (likely(rc == ENOSPC))
661 break;
662
663 /*
664 * Packet cannot be sent, tell RTE that
665 * it is sent, but actually drop it and
666 * continue with another packet
667 */
668 rte_pktmbuf_free(*pktp);
669 continue;
670 }
671
672 goto dma_desc_space_update;
673 }
674
675 if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space) {
676 if (reap_done)
677 break;
678
679 /* Push already prepared descriptors before polling */
680 if (added != txq->added) {
681 sfc_ef10_tx_qpush(txq, added, txq->added);
682 txq->added = added;
683 }
684
685 sfc_ef10_tx_reap(txq);
686 reap_done = true;
687 dma_desc_space = txq->max_fill_level -
688 (added - txq->completed);
689 if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space)
690 break;
691 }
692
693 pkt_len = m_seg->pkt_len;
694 do {
695 rte_iova_t seg_addr = rte_mbuf_data_iova(m_seg);
696 unsigned int seg_len = rte_pktmbuf_data_len(m_seg);
697 unsigned int id = added & txq->ptr_mask;
698
699 SFC_ASSERT(seg_len <= SFC_EF10_TX_DMA_DESC_LEN_MAX);
700
701 pkt_len -= seg_len;
702
703 sfc_ef10_tx_qdesc_dma_create(seg_addr,
704 seg_len, (pkt_len == 0),
705 &txq->txq_hw_ring[id]);
706
707 /*
708 * rte_pktmbuf_free() is commonly used in DPDK for
709 * recycling packets - the function checks every
710 * segment's reference counter and returns the
711 * buffer to its pool whenever possible;
712 * nevertheless, freeing mbuf segments one by one
713 * may entail some performance decline;
714 * from this point, sfc_efx_tx_reap() does the same job
715 * on its own and frees buffers in bulks (all mbufs
716 * within a bulk belong to the same pool);
717 * from this perspective, individual segment pointers
718 * must be associated with the corresponding SW
719 * descriptors independently so that only one loop
720 * is sufficient on reap to inspect all the buffers
721 */
722 txq->sw_ring[id].mbuf = m_seg;
723
724 ++added;
725
726 } while ((m_seg = m_seg->next) != 0);
727
728 dma_desc_space_update:
729 dma_desc_space -= (added - pkt_start);
730 }
731
732 if (likely(added != txq->added)) {
733 sfc_ef10_tx_qpush(txq, added, txq->added);
734 txq->added = added;
735 }
736
737 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
738 if (!reap_done)
739 sfc_ef10_tx_reap(txq);
740 #endif
741
742 return pktp - &tx_pkts[0];
743 }
744
745 static void
sfc_ef10_simple_tx_reap(struct sfc_ef10_txq * txq)746 sfc_ef10_simple_tx_reap(struct sfc_ef10_txq *txq)
747 {
748 const unsigned int old_read_ptr = txq->evq_read_ptr;
749 const unsigned int ptr_mask = txq->ptr_mask;
750 unsigned int completed = txq->completed;
751 unsigned int pending = completed;
752
753 pending += sfc_ef10_tx_process_events(txq);
754
755 if (pending != completed) {
756 struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE];
757 unsigned int nb = 0;
758
759 do {
760 struct sfc_ef10_tx_sw_desc *txd;
761
762 txd = &txq->sw_ring[completed & ptr_mask];
763
764 if (nb == RTE_DIM(bulk)) {
765 rte_mempool_put_bulk(bulk[0]->pool,
766 (void *)bulk, nb);
767 nb = 0;
768 }
769
770 bulk[nb++] = txd->mbuf;
771 } while (++completed != pending);
772
773 rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb);
774
775 txq->completed = completed;
776 }
777
778 sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr,
779 txq->evq_read_ptr);
780 }
781
782 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG
783 static uint16_t
sfc_ef10_simple_prepare_pkts(__rte_unused void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)784 sfc_ef10_simple_prepare_pkts(__rte_unused void *tx_queue,
785 struct rte_mbuf **tx_pkts,
786 uint16_t nb_pkts)
787 {
788 uint16_t i;
789
790 for (i = 0; i < nb_pkts; i++) {
791 struct rte_mbuf *m = tx_pkts[i];
792 int ret;
793
794 ret = rte_validate_tx_offload(m);
795 if (unlikely(ret != 0)) {
796 /*
797 * Negative error code is returned by
798 * rte_validate_tx_offload(), but positive are used
799 * inside net/sfc PMD.
800 */
801 SFC_ASSERT(ret < 0);
802 rte_errno = -ret;
803 break;
804 }
805
806 /* ef10_simple does not support TSO and VLAN insertion */
807 if (unlikely(m->ol_flags &
808 (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_VLAN))) {
809 rte_errno = ENOTSUP;
810 break;
811 }
812
813 /* ef10_simple does not support scattered packets */
814 if (unlikely(m->nb_segs != 1)) {
815 rte_errno = ENOTSUP;
816 break;
817 }
818
819 /*
820 * ef10_simple requires fast-free which ignores reference
821 * counters
822 */
823 if (unlikely(rte_mbuf_refcnt_read(m) != 1)) {
824 rte_errno = ENOTSUP;
825 break;
826 }
827
828 /* ef10_simple requires single pool for all packets */
829 if (unlikely(m->pool != tx_pkts[0]->pool)) {
830 rte_errno = ENOTSUP;
831 break;
832 }
833 }
834
835 return i;
836 }
837 #endif
838
839 static uint16_t
sfc_ef10_simple_xmit_pkts(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)840 sfc_ef10_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
841 uint16_t nb_pkts)
842 {
843 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue);
844 unsigned int ptr_mask;
845 unsigned int added;
846 unsigned int dma_desc_space;
847 bool reap_done;
848 struct rte_mbuf **pktp;
849 struct rte_mbuf **pktp_end;
850
851 if (unlikely(txq->flags &
852 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION)))
853 return 0;
854
855 ptr_mask = txq->ptr_mask;
856 added = txq->added;
857 dma_desc_space = txq->max_fill_level - (added - txq->completed);
858
859 reap_done = (dma_desc_space < RTE_MAX(txq->free_thresh, nb_pkts));
860 if (reap_done) {
861 sfc_ef10_simple_tx_reap(txq);
862 dma_desc_space = txq->max_fill_level - (added - txq->completed);
863 }
864
865 pktp_end = &tx_pkts[MIN(nb_pkts, dma_desc_space)];
866 for (pktp = &tx_pkts[0]; pktp != pktp_end; ++pktp) {
867 struct rte_mbuf *pkt = *pktp;
868 unsigned int id = added & ptr_mask;
869
870 SFC_ASSERT(rte_pktmbuf_data_len(pkt) <=
871 SFC_EF10_TX_DMA_DESC_LEN_MAX);
872
873 sfc_ef10_tx_qdesc_dma_create(rte_mbuf_data_iova(pkt),
874 rte_pktmbuf_data_len(pkt),
875 true, &txq->txq_hw_ring[id]);
876
877 txq->sw_ring[id].mbuf = pkt;
878
879 ++added;
880 }
881
882 if (likely(added != txq->added)) {
883 sfc_ef10_tx_qpush(txq, added, txq->added);
884 txq->added = added;
885 }
886
887 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
888 if (!reap_done)
889 sfc_ef10_simple_tx_reap(txq);
890 #endif
891
892 return pktp - &tx_pkts[0];
893 }
894
895 static sfc_dp_tx_get_dev_info_t sfc_ef10_get_dev_info;
896 static void
sfc_ef10_get_dev_info(struct rte_eth_dev_info * dev_info)897 sfc_ef10_get_dev_info(struct rte_eth_dev_info *dev_info)
898 {
899 /*
900 * Number of descriptors just defines maximum number of pushed
901 * descriptors (fill level).
902 */
903 dev_info->tx_desc_lim.nb_min = 1;
904 dev_info->tx_desc_lim.nb_align = 1;
905 }
906
907 static sfc_dp_tx_qsize_up_rings_t sfc_ef10_tx_qsize_up_rings;
908 static int
sfc_ef10_tx_qsize_up_rings(uint16_t nb_tx_desc,struct sfc_dp_tx_hw_limits * limits,unsigned int * txq_entries,unsigned int * evq_entries,unsigned int * txq_max_fill_level)909 sfc_ef10_tx_qsize_up_rings(uint16_t nb_tx_desc,
910 struct sfc_dp_tx_hw_limits *limits,
911 unsigned int *txq_entries,
912 unsigned int *evq_entries,
913 unsigned int *txq_max_fill_level)
914 {
915 /*
916 * rte_ethdev API guarantees that the number meets min, max and
917 * alignment requirements.
918 */
919 if (nb_tx_desc <= limits->txq_min_entries)
920 *txq_entries = limits->txq_min_entries;
921 else
922 *txq_entries = rte_align32pow2(nb_tx_desc);
923
924 *evq_entries = *txq_entries;
925
926 *txq_max_fill_level = RTE_MIN(nb_tx_desc,
927 SFC_EF10_TXQ_LIMIT(*evq_entries));
928 return 0;
929 }
930
931 static sfc_dp_tx_qcreate_t sfc_ef10_tx_qcreate;
932 static int
sfc_ef10_tx_qcreate(uint16_t port_id,uint16_t queue_id,const struct rte_pci_addr * pci_addr,int socket_id,const struct sfc_dp_tx_qcreate_info * info,struct sfc_dp_txq ** dp_txqp)933 sfc_ef10_tx_qcreate(uint16_t port_id, uint16_t queue_id,
934 const struct rte_pci_addr *pci_addr, int socket_id,
935 const struct sfc_dp_tx_qcreate_info *info,
936 struct sfc_dp_txq **dp_txqp)
937 {
938 struct sfc_ef10_txq *txq;
939 int rc;
940
941 rc = EINVAL;
942 if (info->txq_entries != info->evq_entries)
943 goto fail_bad_args;
944
945 rc = ENOTSUP;
946 if (info->nic_dma_info->nb_regions > 0)
947 goto fail_nic_dma;
948
949 rc = ENOMEM;
950 txq = rte_zmalloc_socket("sfc-ef10-txq", sizeof(*txq),
951 RTE_CACHE_LINE_SIZE, socket_id);
952 if (txq == NULL)
953 goto fail_txq_alloc;
954
955 sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr);
956
957 rc = ENOMEM;
958 txq->sw_ring = rte_calloc_socket("sfc-ef10-txq-sw_ring",
959 info->txq_entries,
960 sizeof(*txq->sw_ring),
961 RTE_CACHE_LINE_SIZE, socket_id);
962 if (txq->sw_ring == NULL)
963 goto fail_sw_ring_alloc;
964
965 if (info->offloads & (RTE_ETH_TX_OFFLOAD_TCP_TSO |
966 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
967 RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO)) {
968 txq->tsoh = rte_calloc_socket("sfc-ef10-txq-tsoh",
969 info->txq_entries,
970 SFC_TSOH_STD_LEN,
971 RTE_CACHE_LINE_SIZE,
972 socket_id);
973 if (txq->tsoh == NULL)
974 goto fail_tsoh_alloc;
975
976 txq->tsoh_iova = rte_malloc_virt2iova(txq->tsoh);
977 }
978
979 txq->flags = SFC_EF10_TXQ_NOT_RUNNING;
980 txq->ptr_mask = info->txq_entries - 1;
981 txq->max_fill_level = info->max_fill_level;
982 txq->free_thresh = info->free_thresh;
983 txq->txq_hw_ring = info->txq_hw_ring;
984 txq->doorbell = (volatile uint8_t *)info->mem_bar +
985 ER_DZ_TX_DESC_UPD_REG_OFST +
986 (info->hw_index << info->vi_window_shift);
987 txq->evq_hw_ring = info->evq_hw_ring;
988 txq->tso_tcp_header_offset_limit = info->tso_tcp_header_offset_limit;
989
990 sfc_ef10_tx_info(&txq->dp.dpq, "TxQ doorbell is %p", txq->doorbell);
991
992 *dp_txqp = &txq->dp;
993 return 0;
994
995 fail_tsoh_alloc:
996 rte_free(txq->sw_ring);
997
998 fail_sw_ring_alloc:
999 rte_free(txq);
1000
1001 fail_txq_alloc:
1002 fail_nic_dma:
1003 fail_bad_args:
1004 return rc;
1005 }
1006
1007 static sfc_dp_tx_qdestroy_t sfc_ef10_tx_qdestroy;
1008 static void
sfc_ef10_tx_qdestroy(struct sfc_dp_txq * dp_txq)1009 sfc_ef10_tx_qdestroy(struct sfc_dp_txq *dp_txq)
1010 {
1011 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
1012
1013 rte_free(txq->tsoh);
1014 rte_free(txq->sw_ring);
1015 rte_free(txq);
1016 }
1017
1018 static sfc_dp_tx_qstart_t sfc_ef10_tx_qstart;
1019 static int
sfc_ef10_tx_qstart(struct sfc_dp_txq * dp_txq,unsigned int evq_read_ptr,unsigned int txq_desc_index)1020 sfc_ef10_tx_qstart(struct sfc_dp_txq *dp_txq, unsigned int evq_read_ptr,
1021 unsigned int txq_desc_index)
1022 {
1023 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
1024
1025 txq->evq_read_ptr = evq_read_ptr;
1026 txq->added = txq->completed = txq_desc_index;
1027
1028 txq->flags |= SFC_EF10_TXQ_STARTED;
1029 txq->flags &= ~(SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION);
1030
1031 return 0;
1032 }
1033
1034 static sfc_dp_tx_qstop_t sfc_ef10_tx_qstop;
1035 static void
sfc_ef10_tx_qstop(struct sfc_dp_txq * dp_txq,unsigned int * evq_read_ptr)1036 sfc_ef10_tx_qstop(struct sfc_dp_txq *dp_txq, unsigned int *evq_read_ptr)
1037 {
1038 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
1039
1040 txq->flags |= SFC_EF10_TXQ_NOT_RUNNING;
1041
1042 *evq_read_ptr = txq->evq_read_ptr;
1043 }
1044
1045 static sfc_dp_tx_qtx_ev_t sfc_ef10_tx_qtx_ev;
1046 static bool
sfc_ef10_tx_qtx_ev(struct sfc_dp_txq * dp_txq,__rte_unused unsigned int id)1047 sfc_ef10_tx_qtx_ev(struct sfc_dp_txq *dp_txq, __rte_unused unsigned int id)
1048 {
1049 __rte_unused struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
1050
1051 SFC_ASSERT(txq->flags & SFC_EF10_TXQ_NOT_RUNNING);
1052
1053 /*
1054 * It is safe to ignore Tx event since we reap all mbufs on
1055 * queue purge anyway.
1056 */
1057
1058 return false;
1059 }
1060
1061 static sfc_dp_tx_qreap_t sfc_ef10_tx_qreap;
1062 static void
sfc_ef10_tx_qreap(struct sfc_dp_txq * dp_txq)1063 sfc_ef10_tx_qreap(struct sfc_dp_txq *dp_txq)
1064 {
1065 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
1066 unsigned int completed;
1067
1068 for (completed = txq->completed; completed != txq->added; ++completed) {
1069 struct sfc_ef10_tx_sw_desc *txd;
1070
1071 txd = &txq->sw_ring[completed & txq->ptr_mask];
1072 if (txd->mbuf != NULL) {
1073 rte_pktmbuf_free_seg(txd->mbuf);
1074 txd->mbuf = NULL;
1075 }
1076 }
1077
1078 txq->flags &= ~SFC_EF10_TXQ_STARTED;
1079 }
1080
1081 static unsigned int
sfc_ef10_tx_qdesc_npending(struct sfc_ef10_txq * txq)1082 sfc_ef10_tx_qdesc_npending(struct sfc_ef10_txq *txq)
1083 {
1084 const unsigned int curr_done = txq->completed - 1;
1085 unsigned int anew_done = curr_done;
1086 efx_qword_t tx_ev;
1087 const unsigned int evq_old_read_ptr = txq->evq_read_ptr;
1088
1089 if (unlikely(txq->flags &
1090 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION)))
1091 return 0;
1092
1093 while (sfc_ef10_tx_get_event(txq, &tx_ev))
1094 anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX);
1095
1096 /*
1097 * The function does not process events, so return event queue read
1098 * pointer to the original position to allow the events that were
1099 * read to be processed later
1100 */
1101 txq->evq_read_ptr = evq_old_read_ptr;
1102
1103 return (anew_done - curr_done) & txq->ptr_mask;
1104 }
1105
1106 static sfc_dp_tx_qdesc_status_t sfc_ef10_tx_qdesc_status;
1107 static int
sfc_ef10_tx_qdesc_status(struct sfc_dp_txq * dp_txq,uint16_t offset)1108 sfc_ef10_tx_qdesc_status(struct sfc_dp_txq *dp_txq,
1109 uint16_t offset)
1110 {
1111 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
1112 unsigned int npending = sfc_ef10_tx_qdesc_npending(txq);
1113
1114 if (unlikely(offset > txq->ptr_mask))
1115 return -EINVAL;
1116
1117 if (unlikely(offset >= txq->max_fill_level))
1118 return RTE_ETH_TX_DESC_UNAVAIL;
1119
1120 if (unlikely(offset < npending))
1121 return RTE_ETH_TX_DESC_FULL;
1122
1123 return RTE_ETH_TX_DESC_DONE;
1124 }
1125
1126 struct sfc_dp_tx sfc_ef10_tx = {
1127 .dp = {
1128 .name = SFC_KVARG_DATAPATH_EF10,
1129 .type = SFC_DP_TX,
1130 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10,
1131 },
1132 .features = SFC_DP_TX_FEAT_MULTI_PROCESS,
1133 .dev_offload_capa = RTE_ETH_TX_OFFLOAD_MULTI_SEGS,
1134 .queue_offload_capa = RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
1135 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
1136 RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
1137 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1138 RTE_ETH_TX_OFFLOAD_TCP_TSO |
1139 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1140 RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO,
1141 .get_dev_info = sfc_ef10_get_dev_info,
1142 .qsize_up_rings = sfc_ef10_tx_qsize_up_rings,
1143 .qcreate = sfc_ef10_tx_qcreate,
1144 .qdestroy = sfc_ef10_tx_qdestroy,
1145 .qstart = sfc_ef10_tx_qstart,
1146 .qtx_ev = sfc_ef10_tx_qtx_ev,
1147 .qstop = sfc_ef10_tx_qstop,
1148 .qreap = sfc_ef10_tx_qreap,
1149 .qdesc_status = sfc_ef10_tx_qdesc_status,
1150 .pkt_prepare = sfc_ef10_prepare_pkts,
1151 .pkt_burst = sfc_ef10_xmit_pkts,
1152 };
1153
1154 struct sfc_dp_tx sfc_ef10_simple_tx = {
1155 .dp = {
1156 .name = SFC_KVARG_DATAPATH_EF10_SIMPLE,
1157 .type = SFC_DP_TX,
1158 },
1159 .features = SFC_DP_TX_FEAT_MULTI_PROCESS,
1160 .dev_offload_capa = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
1161 .queue_offload_capa = RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
1162 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
1163 RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
1164 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM,
1165 .get_dev_info = sfc_ef10_get_dev_info,
1166 .qsize_up_rings = sfc_ef10_tx_qsize_up_rings,
1167 .qcreate = sfc_ef10_tx_qcreate,
1168 .qdestroy = sfc_ef10_tx_qdestroy,
1169 .qstart = sfc_ef10_tx_qstart,
1170 .qtx_ev = sfc_ef10_tx_qtx_ev,
1171 .qstop = sfc_ef10_tx_qstop,
1172 .qreap = sfc_ef10_tx_qreap,
1173 .qdesc_status = sfc_ef10_tx_qdesc_status,
1174 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG
1175 .pkt_prepare = sfc_ef10_simple_prepare_pkts,
1176 #endif
1177 .pkt_burst = sfc_ef10_simple_xmit_pkts,
1178 };
1179