1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018-2022 Advanced Micro Devices, Inc.
3 */
4
5 #include <stdio.h>
6 #include <errno.h>
7 #include <stdint.h>
8 #include <assert.h>
9
10 #include <rte_common.h>
11 #include <rte_byteorder.h>
12 #include <rte_atomic.h>
13 #include <rte_mempool.h>
14 #include <rte_mbuf.h>
15 #include <rte_ether.h>
16 #include <rte_prefetch.h>
17
18 #include "ionic.h"
19 #include "ionic_ethdev.h"
20 #include "ionic_lif.h"
21 #include "ionic_rxtx.h"
22
23 static __rte_always_inline void
ionic_tx_flush_sg(struct ionic_tx_qcq * txq)24 ionic_tx_flush_sg(struct ionic_tx_qcq *txq)
25 {
26 struct ionic_cq *cq = &txq->qcq.cq;
27 struct ionic_queue *q = &txq->qcq.q;
28 struct ionic_tx_stats *stats = &txq->stats;
29 struct rte_mbuf *txm;
30 struct ionic_txq_comp *cq_desc_base = cq->base;
31 volatile struct ionic_txq_comp *cq_desc;
32 void **info;
33 uint32_t i;
34
35 cq_desc = &cq_desc_base[cq->tail_idx];
36
37 while (color_match(cq_desc->color, cq->done_color)) {
38 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
39 if (cq->tail_idx == 0)
40 cq->done_color = !cq->done_color;
41
42 /* Prefetch 4 x 16B comp at cq->tail_idx + 4 */
43 if ((cq->tail_idx & 0x3) == 0)
44 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
45
46 while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) {
47 /* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */
48 rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 2)));
49
50 /* Prefetch next mbuf */
51 void **next_info =
52 IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 1));
53 if (next_info[0])
54 rte_mbuf_prefetch_part2(next_info[0]);
55 if (next_info[1])
56 rte_mbuf_prefetch_part2(next_info[1]);
57
58 info = IONIC_INFO_PTR(q, q->tail_idx);
59 for (i = 0; i < q->num_segs; i++) {
60 txm = info[i];
61 if (!txm)
62 break;
63
64 if (txq->flags & IONIC_QCQ_F_FAST_FREE)
65 rte_mempool_put(txm->pool, txm);
66 else
67 rte_pktmbuf_free_seg(txm);
68
69 info[i] = NULL;
70 }
71
72 q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
73 }
74
75 cq_desc = &cq_desc_base[cq->tail_idx];
76 stats->comps++;
77 }
78 }
79
80 static __rte_always_inline int
ionic_tx_sg(struct ionic_tx_qcq * txq,struct rte_mbuf * txm)81 ionic_tx_sg(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
82 {
83 struct ionic_queue *q = &txq->qcq.q;
84 struct ionic_txq_desc *desc, *desc_base = q->base;
85 struct ionic_txq_sg_desc_v1 *sg_desc, *sg_desc_base = q->sg_base;
86 struct ionic_txq_sg_elem *elem;
87 struct ionic_tx_stats *stats = &txq->stats;
88 struct rte_mbuf *txm_seg;
89 rte_iova_t data_iova;
90 void **info;
91 uint64_t ol_flags = txm->ol_flags;
92 uint64_t addr, cmd;
93 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
94 uint8_t flags = 0;
95
96 desc = &desc_base[q->head_idx];
97 sg_desc = &sg_desc_base[q->head_idx];
98 info = IONIC_INFO_PTR(q, q->head_idx);
99
100 if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) &&
101 (txq->flags & IONIC_QCQ_F_CSUM_L3)) {
102 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
103 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
104 }
105
106 if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) &&
107 (txq->flags & IONIC_QCQ_F_CSUM_TCP)) ||
108 ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) &&
109 (txq->flags & IONIC_QCQ_F_CSUM_UDP))) {
110 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
111 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
112 }
113
114 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE)
115 stats->no_csum++;
116
117 if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) ||
118 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) &&
119 ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) ||
120 (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) {
121 flags |= IONIC_TXQ_DESC_FLAG_ENCAP;
122 }
123
124 if (ol_flags & RTE_MBUF_F_TX_VLAN) {
125 flags |= IONIC_TXQ_DESC_FLAG_VLAN;
126 desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci);
127 }
128
129 addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm));
130
131 cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr);
132 desc->cmd = rte_cpu_to_le_64(cmd);
133 desc->len = rte_cpu_to_le_16(txm->data_len);
134
135 info[0] = txm;
136
137 if (txm->nb_segs > 1) {
138 txm_seg = txm->next;
139
140 elem = sg_desc->elems;
141
142 while (txm_seg != NULL) {
143 /* Stash the mbuf ptr in the array */
144 info++;
145 *info = txm_seg;
146
147 /* Configure the SGE */
148 data_iova = rte_mbuf_data_iova(txm_seg);
149 elem->len = rte_cpu_to_le_16(txm_seg->data_len);
150 elem->addr = rte_cpu_to_le_64(data_iova);
151 elem++;
152
153 txm_seg = txm_seg->next;
154 }
155 }
156
157 q->head_idx = Q_NEXT_TO_POST(q, 1);
158
159 return 0;
160 }
161
162 uint16_t
ionic_xmit_pkts_sg(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)163 ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
164 uint16_t nb_pkts)
165 {
166 struct ionic_tx_qcq *txq = tx_queue;
167 struct ionic_queue *q = &txq->qcq.q;
168 struct ionic_txq_desc *desc_base = q->base;
169 struct ionic_tx_stats *stats = &txq->stats;
170 struct rte_mbuf *mbuf;
171 uint32_t bytes_tx = 0;
172 uint16_t nb_avail, nb_tx = 0;
173 uint64_t then, now, hz, delta;
174 int err;
175
176 rte_prefetch0(&desc_base[q->head_idx]);
177 rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx));
178
179 if (nb_pkts) {
180 rte_mbuf_prefetch_part1(tx_pkts[0]);
181 rte_mbuf_prefetch_part2(tx_pkts[0]);
182 }
183
184 if (ionic_q_space_avail(q) < txq->free_thresh) {
185 /* Cleaning old buffers */
186 ionic_tx_flush_sg(txq);
187 }
188
189 nb_avail = ionic_q_space_avail(q);
190 if (nb_avail < nb_pkts) {
191 stats->stop += nb_pkts - nb_avail;
192 nb_pkts = nb_avail;
193 }
194
195 while (nb_tx < nb_pkts) {
196 uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
197 rte_prefetch0(&desc_base[next_idx]);
198 rte_prefetch0(IONIC_INFO_PTR(q, next_idx));
199
200 if (nb_tx + 1 < nb_pkts) {
201 rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]);
202 rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]);
203 }
204
205 mbuf = tx_pkts[nb_tx];
206
207 if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)
208 err = ionic_tx_tso(txq, mbuf);
209 else
210 err = ionic_tx_sg(txq, mbuf);
211 if (err) {
212 stats->drop += nb_pkts - nb_tx;
213 break;
214 }
215
216 bytes_tx += mbuf->pkt_len;
217 nb_tx++;
218 }
219
220 if (nb_tx > 0) {
221 ionic_txq_flush(q);
222
223 txq->last_wdog_cycles = rte_get_timer_cycles();
224
225 stats->packets += nb_tx;
226 stats->bytes += bytes_tx;
227 } else {
228 /*
229 * Ring the doorbell again if no work could be posted and work
230 * is still pending after the deadline.
231 */
232 if (q->head_idx != q->tail_idx) {
233 then = txq->last_wdog_cycles;
234 now = rte_get_timer_cycles();
235 hz = rte_get_timer_hz();
236 delta = (now - then) * 1000;
237
238 if (delta >= hz * IONIC_Q_WDOG_MS) {
239 ionic_q_flush(q);
240 txq->last_wdog_cycles = now;
241 }
242 }
243 }
244
245 return nb_tx;
246 }
247
248 /*
249 * Cleans one descriptor. Connects the filled mbufs into a chain.
250 * Does not advance the tail index.
251 */
252 static __rte_always_inline void
ionic_rx_clean_one_sg(struct ionic_rx_qcq * rxq,volatile struct ionic_rxq_comp * cq_desc,struct ionic_rx_service * rx_svc)253 ionic_rx_clean_one_sg(struct ionic_rx_qcq *rxq,
254 volatile struct ionic_rxq_comp *cq_desc,
255 struct ionic_rx_service *rx_svc)
256 {
257 struct ionic_queue *q = &rxq->qcq.q;
258 struct rte_mbuf *rxm;
259 struct rte_mbuf *rxm_seg, *prev_rxm;
260 struct ionic_rx_stats *stats = &rxq->stats;
261 uint64_t pkt_flags = 0;
262 uint32_t pkt_type;
263 uint32_t left, i;
264 uint16_t cq_desc_len;
265 uint8_t ptype, cflags;
266 void **info;
267
268 cq_desc_len = rte_le_to_cpu_16(cq_desc->len);
269
270 info = IONIC_INFO_PTR(q, q->tail_idx);
271
272 rxm = info[0];
273
274 if (cq_desc->status) {
275 stats->bad_cq_status++;
276 return;
277 }
278
279 if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) {
280 stats->bad_len++;
281 return;
282 }
283
284 info[0] = NULL;
285
286 /* Set the mbuf metadata based on the cq entry */
287 rxm->rearm_data[0] = rxq->rearm_data;
288 rxm->pkt_len = cq_desc_len;
289 rxm->data_len = RTE_MIN(rxq->hdr_seg_size, cq_desc_len);
290 left = cq_desc_len - rxm->data_len;
291 rxm->nb_segs = cq_desc->num_sg_elems + 1;
292
293 prev_rxm = rxm;
294
295 for (i = 1; i < rxm->nb_segs && left; i++) {
296 rxm_seg = info[i];
297 info[i] = NULL;
298
299 /* Set the chained mbuf metadata */
300 rxm_seg->rearm_data[0] = rxq->rearm_seg_data;
301 rxm_seg->data_len = RTE_MIN(rxq->seg_size, left);
302 left -= rxm_seg->data_len;
303
304 /* Link the mbuf */
305 prev_rxm->next = rxm_seg;
306 prev_rxm = rxm_seg;
307 }
308
309 /* Terminate the mbuf chain */
310 prev_rxm->next = NULL;
311
312 /* RSS */
313 pkt_flags |= RTE_MBUF_F_RX_RSS_HASH;
314 rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash);
315
316 /* Vlan Strip */
317 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
318 pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
319 rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci);
320 }
321
322 /* Checksum */
323 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
324 cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK;
325 pkt_flags |= ionic_csum_flags[cflags];
326 }
327
328 rxm->ol_flags = pkt_flags;
329
330 /* Packet Type */
331 ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK;
332 pkt_type = ionic_ptype_table[ptype];
333 if (pkt_type == RTE_PTYPE_UNKNOWN) {
334 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
335 struct rte_ether_hdr *);
336 uint16_t ether_type = eth_h->ether_type;
337 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
338 pkt_type = RTE_PTYPE_L2_ETHER_ARP;
339 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP))
340 pkt_type = RTE_PTYPE_L2_ETHER_LLDP;
341 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588))
342 pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC;
343 stats->mtods++;
344 } else if (pkt_flags & RTE_MBUF_F_RX_VLAN) {
345 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN;
346 } else {
347 pkt_type |= RTE_PTYPE_L2_ETHER;
348 }
349
350 rxm->packet_type = pkt_type;
351
352 rx_svc->rx_pkts[rx_svc->nb_rx] = rxm;
353 rx_svc->nb_rx++;
354
355 stats->packets++;
356 stats->bytes += rxm->pkt_len;
357 }
358
359 /*
360 * Fills one descriptor with mbufs. Does not advance the head index.
361 */
362 static __rte_always_inline int
ionic_rx_fill_one_sg(struct ionic_rx_qcq * rxq)363 ionic_rx_fill_one_sg(struct ionic_rx_qcq *rxq)
364 {
365 struct ionic_queue *q = &rxq->qcq.q;
366 struct rte_mbuf *rxm;
367 struct rte_mbuf *rxm_seg;
368 struct ionic_rxq_desc *desc, *desc_base = q->base;
369 struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
370 rte_iova_t data_iova;
371 uint32_t i;
372 void **info;
373 int ret;
374
375 info = IONIC_INFO_PTR(q, q->head_idx);
376 desc = &desc_base[q->head_idx];
377 sg_desc = &sg_desc_base[q->head_idx];
378
379 /* mbuf is unused => whole chain is unused */
380 if (info[0])
381 return 0;
382
383 if (rxq->mb_idx == 0) {
384 ret = rte_mempool_get_bulk(rxq->mb_pool,
385 (void **)rxq->mbs,
386 IONIC_MBUF_BULK_ALLOC);
387 if (ret) {
388 assert(0);
389 return -ENOMEM;
390 }
391
392 rxq->mb_idx = IONIC_MBUF_BULK_ALLOC;
393 }
394
395 rxm = rxq->mbs[--rxq->mb_idx];
396 info[0] = rxm;
397
398 data_iova = rte_mbuf_data_iova_default(rxm);
399 desc->addr = rte_cpu_to_le_64(data_iova);
400
401 for (i = 1; i < q->num_segs; i++) {
402 /* mbuf is unused => rest of the chain is unused */
403 if (info[i])
404 return 0;
405
406 if (rxq->mb_idx == 0) {
407 ret = rte_mempool_get_bulk(rxq->mb_pool,
408 (void **)rxq->mbs,
409 IONIC_MBUF_BULK_ALLOC);
410 if (ret) {
411 assert(0);
412 return -ENOMEM;
413 }
414
415 rxq->mb_idx = IONIC_MBUF_BULK_ALLOC;
416 }
417
418 rxm_seg = rxq->mbs[--rxq->mb_idx];
419 info[i] = rxm_seg;
420
421 /* The data_off does not get set to 0 until later */
422 data_iova = rxm_seg->buf_iova;
423 sg_desc->elems[i - 1].addr = rte_cpu_to_le_64(data_iova);
424 }
425
426 return 0;
427 }
428
429 /*
430 * Walk the CQ to find completed receive descriptors.
431 * Any completed descriptor found is refilled.
432 */
433 static __rte_always_inline void
ionic_rxq_service_sg(struct ionic_rx_qcq * rxq,uint32_t work_to_do,struct ionic_rx_service * rx_svc)434 ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
435 struct ionic_rx_service *rx_svc)
436 {
437 struct ionic_cq *cq = &rxq->qcq.cq;
438 struct ionic_queue *q = &rxq->qcq.q;
439 struct ionic_rxq_desc *q_desc_base = q->base;
440 struct ionic_rxq_comp *cq_desc_base = cq->base;
441 volatile struct ionic_rxq_comp *cq_desc;
442 uint32_t work_done = 0;
443 uint64_t then, now, hz, delta;
444
445 cq_desc = &cq_desc_base[cq->tail_idx];
446
447 while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
448 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
449 if (cq->tail_idx == 0)
450 cq->done_color = !cq->done_color;
451
452 /* Prefetch 8 x 8B bufinfo */
453 rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 8)));
454 /* Prefetch 4 x 16B comp */
455 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
456 /* Prefetch 4 x 16B descriptors */
457 rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
458
459 /* Clean one descriptor */
460 ionic_rx_clean_one_sg(rxq, cq_desc, rx_svc);
461 q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
462
463 /* Fill one descriptor */
464 (void)ionic_rx_fill_one_sg(rxq);
465
466 q->head_idx = Q_NEXT_TO_POST(q, 1);
467
468 if (++work_done == work_to_do)
469 break;
470
471 cq_desc = &cq_desc_base[cq->tail_idx];
472 }
473
474 /* Update the queue indices and ring the doorbell */
475 if (work_done) {
476 ionic_rxq_flush(q);
477
478 rxq->last_wdog_cycles = rte_get_timer_cycles();
479 rxq->wdog_ms = IONIC_Q_WDOG_MS;
480 } else {
481 /*
482 * Ring the doorbell again if no recvs were posted and the
483 * recv queue is not empty after the deadline.
484 *
485 * Exponentially back off the deadline to avoid excessive
486 * doorbells when the recv queue is idle.
487 */
488 if (q->head_idx != q->tail_idx) {
489 then = rxq->last_wdog_cycles;
490 now = rte_get_timer_cycles();
491 hz = rte_get_timer_hz();
492 delta = (now - then) * 1000;
493
494 if (delta >= hz * rxq->wdog_ms) {
495 ionic_q_flush(q);
496 rxq->last_wdog_cycles = now;
497
498 delta = 2 * rxq->wdog_ms;
499 if (delta > IONIC_Q_WDOG_MAX_MS)
500 delta = IONIC_Q_WDOG_MAX_MS;
501
502 rxq->wdog_ms = delta;
503 }
504 }
505 }
506 }
507
508 uint16_t
ionic_recv_pkts_sg(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)509 ionic_recv_pkts_sg(void *rx_queue, struct rte_mbuf **rx_pkts,
510 uint16_t nb_pkts)
511 {
512 struct ionic_rx_qcq *rxq = rx_queue;
513 struct ionic_rx_service rx_svc;
514
515 rx_svc.rx_pkts = rx_pkts;
516 rx_svc.nb_rx = 0;
517
518 ionic_rxq_service_sg(rxq, nb_pkts, &rx_svc);
519
520 return rx_svc.nb_rx;
521 }
522
523 /*
524 * Fills all descriptors with mbufs.
525 */
526 int __rte_cold
ionic_rx_fill_sg(struct ionic_rx_qcq * rxq)527 ionic_rx_fill_sg(struct ionic_rx_qcq *rxq)
528 {
529 struct ionic_queue *q = &rxq->qcq.q;
530 uint32_t i;
531 int err = 0;
532
533 for (i = 0; i < q->num_descs - 1u; i++) {
534 err = ionic_rx_fill_one_sg(rxq);
535 if (err)
536 break;
537
538 q->head_idx = Q_NEXT_TO_POST(q, 1);
539 }
540
541 ionic_rxq_flush(q);
542
543 return err;
544 }
545