xref: /dpdk/drivers/net/ionic/ionic_rxtx_simple.c (revision 8e7a6b8ac8696684133fd54ea403c840e37758c2)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2018-2022 Advanced Micro Devices, Inc.
3  */
4 
5 #include <stdio.h>
6 #include <errno.h>
7 #include <stdint.h>
8 #include <assert.h>
9 
10 #include <rte_common.h>
11 #include <rte_byteorder.h>
12 #include <rte_atomic.h>
13 #include <rte_mempool.h>
14 #include <rte_mbuf.h>
15 #include <rte_ether.h>
16 #include <rte_prefetch.h>
17 
18 #include "ionic.h"
19 #include "ionic_ethdev.h"
20 #include "ionic_lif.h"
21 #include "ionic_rxtx.h"
22 
23 static __rte_always_inline void
ionic_tx_flush(struct ionic_tx_qcq * txq)24 ionic_tx_flush(struct ionic_tx_qcq *txq)
25 {
26 	struct ionic_cq *cq = &txq->qcq.cq;
27 	struct ionic_queue *q = &txq->qcq.q;
28 	struct ionic_tx_stats *stats = &txq->stats;
29 	struct rte_mbuf *txm;
30 	struct ionic_txq_comp *cq_desc_base = cq->base;
31 	volatile struct ionic_txq_comp *cq_desc;
32 	void **info;
33 
34 	cq_desc = &cq_desc_base[cq->tail_idx];
35 
36 	while (color_match(cq_desc->color, cq->done_color)) {
37 		cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
38 		if (cq->tail_idx == 0)
39 			cq->done_color = !cq->done_color;
40 
41 		/* Prefetch 4 x 16B comp at cq->tail_idx + 4 */
42 		if ((cq->tail_idx & 0x3) == 0)
43 			rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
44 
45 		while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) {
46 			/* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */
47 			rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 2)]);
48 
49 			/* Prefetch next mbuf */
50 			void **next_info =
51 				&q->info[Q_NEXT_TO_SRVC(q, 1)];
52 			if (next_info[0])
53 				rte_mbuf_prefetch_part2(next_info[0]);
54 
55 			info = &q->info[q->tail_idx];
56 			{
57 				txm = info[0];
58 
59 				if (txq->flags & IONIC_QCQ_F_FAST_FREE)
60 					rte_mempool_put(txm->pool, txm);
61 				else
62 					rte_pktmbuf_free_seg(txm);
63 
64 				info[0] = NULL;
65 			}
66 
67 			q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
68 		}
69 
70 		cq_desc = &cq_desc_base[cq->tail_idx];
71 		stats->comps++;
72 	}
73 }
74 
75 static __rte_always_inline int
ionic_tx(struct ionic_tx_qcq * txq,struct rte_mbuf * txm)76 ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
77 {
78 	struct ionic_queue *q = &txq->qcq.q;
79 	struct ionic_txq_desc *desc, *desc_base = q->base;
80 	struct ionic_tx_stats *stats = &txq->stats;
81 	void **info;
82 	uint64_t ol_flags = txm->ol_flags;
83 	uint64_t addr, cmd;
84 	uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
85 	uint8_t flags = 0;
86 
87 	if (txm->nb_segs > 1)
88 		return -EINVAL;
89 
90 	desc = &desc_base[q->head_idx];
91 	info = &q->info[q->head_idx];
92 
93 	if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) &&
94 	    (txq->flags & IONIC_QCQ_F_CSUM_L3)) {
95 		opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
96 		flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
97 	}
98 
99 	if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) &&
100 	     (txq->flags & IONIC_QCQ_F_CSUM_TCP)) ||
101 	    ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) &&
102 	     (txq->flags & IONIC_QCQ_F_CSUM_UDP))) {
103 		opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
104 		flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
105 	}
106 
107 	if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE)
108 		stats->no_csum++;
109 
110 	if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) ||
111 	     (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) &&
112 	    ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) ||
113 	     (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) {
114 		flags |= IONIC_TXQ_DESC_FLAG_ENCAP;
115 	}
116 
117 	if (ol_flags & RTE_MBUF_F_TX_VLAN) {
118 		flags |= IONIC_TXQ_DESC_FLAG_VLAN;
119 		desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci);
120 	}
121 
122 	addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm));
123 
124 	cmd = encode_txq_desc_cmd(opcode, flags, 0, addr);
125 	desc->cmd = rte_cpu_to_le_64(cmd);
126 	desc->len = rte_cpu_to_le_16(txm->data_len);
127 
128 	info[0] = txm;
129 
130 	q->head_idx = Q_NEXT_TO_POST(q, 1);
131 
132 	return 0;
133 }
134 
135 uint16_t
ionic_xmit_pkts(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)136 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
137 		uint16_t nb_pkts)
138 {
139 	struct ionic_tx_qcq *txq = tx_queue;
140 	struct ionic_queue *q = &txq->qcq.q;
141 	struct ionic_txq_desc *desc_base = q->base;
142 	struct ionic_tx_stats *stats = &txq->stats;
143 	struct rte_mbuf *mbuf;
144 	uint32_t bytes_tx = 0;
145 	uint16_t nb_avail, nb_tx = 0;
146 	uint64_t then, now, hz, delta;
147 	int err;
148 
149 	rte_prefetch0(&desc_base[q->head_idx]);
150 	rte_prefetch0(&q->info[q->head_idx]);
151 
152 	if (nb_pkts) {
153 		rte_mbuf_prefetch_part1(tx_pkts[0]);
154 		rte_mbuf_prefetch_part2(tx_pkts[0]);
155 	}
156 
157 	if (ionic_q_space_avail(q) < txq->free_thresh) {
158 		/* Cleaning old buffers */
159 		ionic_tx_flush(txq);
160 	}
161 
162 	nb_avail = ionic_q_space_avail(q);
163 	if (nb_avail < nb_pkts) {
164 		stats->stop += nb_pkts - nb_avail;
165 		nb_pkts = nb_avail;
166 	}
167 
168 	while (nb_tx < nb_pkts) {
169 		uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
170 		rte_prefetch0(&desc_base[next_idx]);
171 		rte_prefetch0(&q->info[next_idx]);
172 
173 		if (nb_tx + 1 < nb_pkts) {
174 			rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]);
175 			rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]);
176 		}
177 
178 		mbuf = tx_pkts[nb_tx];
179 
180 		if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)
181 			err = ionic_tx_tso(txq, mbuf);
182 		else
183 			err = ionic_tx(txq, mbuf);
184 		if (err) {
185 			stats->drop += nb_pkts - nb_tx;
186 			break;
187 		}
188 
189 		bytes_tx += mbuf->pkt_len;
190 		nb_tx++;
191 	}
192 
193 	if (nb_tx > 0) {
194 		ionic_txq_flush(q);
195 
196 		txq->last_wdog_cycles = rte_get_timer_cycles();
197 
198 		stats->packets += nb_tx;
199 		stats->bytes += bytes_tx;
200 	} else {
201 		/*
202 		 * Ring the doorbell again if no work could be posted and work
203 		 * is still pending after the deadline.
204 		 */
205 		if (q->head_idx != q->tail_idx) {
206 			then = txq->last_wdog_cycles;
207 			now = rte_get_timer_cycles();
208 			hz = rte_get_timer_hz();
209 			delta = (now - then) * 1000;
210 
211 			if (delta >= hz * IONIC_Q_WDOG_MS) {
212 				ionic_q_flush(q);
213 				txq->last_wdog_cycles = now;
214 			}
215 		}
216 	}
217 
218 	return nb_tx;
219 }
220 
221 /*
222  * Cleans one descriptor. Connects the filled mbufs into a chain.
223  * Does not advance the tail index.
224  */
225 static __rte_always_inline void
ionic_rx_clean_one(struct ionic_rx_qcq * rxq,volatile struct ionic_rxq_comp * cq_desc,struct ionic_rx_service * rx_svc)226 ionic_rx_clean_one(struct ionic_rx_qcq *rxq,
227 		volatile struct ionic_rxq_comp *cq_desc,
228 		struct ionic_rx_service *rx_svc)
229 {
230 	struct ionic_queue *q = &rxq->qcq.q;
231 	struct rte_mbuf *rxm;
232 	struct ionic_rx_stats *stats = &rxq->stats;
233 	uint64_t pkt_flags = 0;
234 	uint32_t pkt_type;
235 	uint16_t cq_desc_len;
236 	uint8_t ptype, cflags;
237 	void **info;
238 
239 	cq_desc_len = rte_le_to_cpu_16(cq_desc->len);
240 
241 	info = &q->info[q->tail_idx];
242 
243 	rxm = info[0];
244 
245 	if (cq_desc->status) {
246 		stats->bad_cq_status++;
247 		return;
248 	}
249 
250 	if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) {
251 		stats->bad_len++;
252 		return;
253 	}
254 
255 	info[0] = NULL;
256 
257 	/* Set the mbuf metadata based on the cq entry */
258 	rxm->rearm_data[0] = rxq->rearm_data;
259 	rxm->pkt_len = cq_desc_len;
260 	rxm->data_len = cq_desc_len;
261 
262 	/* RSS */
263 	pkt_flags |= RTE_MBUF_F_RX_RSS_HASH;
264 	rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash);
265 
266 	/* Vlan Strip */
267 	if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
268 		pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
269 		rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci);
270 	}
271 
272 	/* Checksum */
273 	if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
274 		cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK;
275 		pkt_flags |= ionic_csum_flags[cflags];
276 	}
277 
278 	rxm->ol_flags = pkt_flags;
279 
280 	/* Packet Type */
281 	ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK;
282 	pkt_type = ionic_ptype_table[ptype];
283 	if (pkt_type == RTE_PTYPE_UNKNOWN) {
284 		struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
285 				struct rte_ether_hdr *);
286 		uint16_t ether_type = eth_h->ether_type;
287 		if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
288 			pkt_type = RTE_PTYPE_L2_ETHER_ARP;
289 		else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP))
290 			pkt_type = RTE_PTYPE_L2_ETHER_LLDP;
291 		else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588))
292 			pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC;
293 		stats->mtods++;
294 	} else if (pkt_flags & RTE_MBUF_F_RX_VLAN) {
295 		pkt_type |= RTE_PTYPE_L2_ETHER_VLAN;
296 	} else {
297 		pkt_type |= RTE_PTYPE_L2_ETHER;
298 	}
299 
300 	rxm->packet_type = pkt_type;
301 
302 	rx_svc->rx_pkts[rx_svc->nb_rx] = rxm;
303 	rx_svc->nb_rx++;
304 
305 	stats->packets++;
306 	stats->bytes += rxm->pkt_len;
307 }
308 
309 /*
310  * Fills one descriptor with mbufs. Does not advance the head index.
311  */
312 static __rte_always_inline int
ionic_rx_fill_one(struct ionic_rx_qcq * rxq)313 ionic_rx_fill_one(struct ionic_rx_qcq *rxq)
314 {
315 	struct ionic_queue *q = &rxq->qcq.q;
316 	struct rte_mbuf *rxm;
317 	struct ionic_rxq_desc *desc, *desc_base = q->base;
318 	rte_iova_t data_iova;
319 	void **info;
320 	int ret;
321 
322 	info = &q->info[q->head_idx];
323 	desc = &desc_base[q->head_idx];
324 
325 	/* mbuf is unused */
326 	if (info[0])
327 		return 0;
328 
329 	if (rxq->mb_idx == 0) {
330 		ret = rte_mempool_get_bulk(rxq->mb_pool,
331 					(void **)rxq->mbs,
332 					IONIC_MBUF_BULK_ALLOC);
333 		if (ret) {
334 			assert(0);
335 			return -ENOMEM;
336 		}
337 
338 		rxq->mb_idx = IONIC_MBUF_BULK_ALLOC;
339 	}
340 
341 	rxm = rxq->mbs[--rxq->mb_idx];
342 	info[0] = rxm;
343 
344 	data_iova = rte_mbuf_data_iova_default(rxm);
345 	desc->addr = rte_cpu_to_le_64(data_iova);
346 
347 	return 0;
348 }
349 
350 /*
351  * Walk the CQ to find completed receive descriptors.
352  * Any completed descriptor found is refilled.
353  */
354 static __rte_always_inline void
ionic_rxq_service(struct ionic_rx_qcq * rxq,uint32_t work_to_do,struct ionic_rx_service * rx_svc)355 ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
356 		struct ionic_rx_service *rx_svc)
357 {
358 	struct ionic_cq *cq = &rxq->qcq.cq;
359 	struct ionic_queue *q = &rxq->qcq.q;
360 	struct ionic_rxq_desc *q_desc_base = q->base;
361 	struct ionic_rxq_comp *cq_desc_base = cq->base;
362 	volatile struct ionic_rxq_comp *cq_desc;
363 	uint32_t work_done = 0;
364 	uint64_t then, now, hz, delta;
365 
366 	cq_desc = &cq_desc_base[cq->tail_idx];
367 
368 	while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
369 		cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
370 		if (cq->tail_idx == 0)
371 			cq->done_color = !cq->done_color;
372 
373 		/* Prefetch 8 x 8B bufinfo */
374 		rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 8)]);
375 		/* Prefetch 4 x 16B comp */
376 		rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
377 		/* Prefetch 4 x 16B descriptors */
378 		rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
379 
380 		/* Clean one descriptor */
381 		ionic_rx_clean_one(rxq, cq_desc, rx_svc);
382 		q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
383 
384 		/* Fill one descriptor */
385 		(void)ionic_rx_fill_one(rxq);
386 
387 		q->head_idx = Q_NEXT_TO_POST(q, 1);
388 
389 		if (++work_done == work_to_do)
390 			break;
391 
392 		cq_desc = &cq_desc_base[cq->tail_idx];
393 	}
394 
395 	/* Update the queue indices and ring the doorbell */
396 	if (work_done) {
397 		ionic_rxq_flush(q);
398 
399 		rxq->last_wdog_cycles = rte_get_timer_cycles();
400 		rxq->wdog_ms = IONIC_Q_WDOG_MS;
401 	} else {
402 		/*
403 		 * Ring the doorbell again if no recvs were posted and the
404 		 * recv queue is not empty after the deadline.
405 		 *
406 		 * Exponentially back off the deadline to avoid excessive
407 		 * doorbells when the recv queue is idle.
408 		 */
409 		if (q->head_idx != q->tail_idx) {
410 			then = rxq->last_wdog_cycles;
411 			now = rte_get_timer_cycles();
412 			hz = rte_get_timer_hz();
413 			delta = (now - then) * 1000;
414 
415 			if (delta >= hz * rxq->wdog_ms) {
416 				ionic_q_flush(q);
417 				rxq->last_wdog_cycles = now;
418 
419 				delta = 2 * rxq->wdog_ms;
420 				if (delta > IONIC_Q_WDOG_MAX_MS)
421 					delta = IONIC_Q_WDOG_MAX_MS;
422 
423 				rxq->wdog_ms = delta;
424 			}
425 		}
426 	}
427 }
428 
429 uint16_t
ionic_recv_pkts(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)430 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
431 		uint16_t nb_pkts)
432 {
433 	struct ionic_rx_qcq *rxq = rx_queue;
434 	struct ionic_rx_service rx_svc;
435 
436 	rx_svc.rx_pkts = rx_pkts;
437 	rx_svc.nb_rx = 0;
438 
439 	ionic_rxq_service(rxq, nb_pkts, &rx_svc);
440 
441 	return rx_svc.nb_rx;
442 }
443 
444 /*
445  * Fills all descriptors with mbufs.
446  */
447 int __rte_cold
ionic_rx_fill(struct ionic_rx_qcq * rxq)448 ionic_rx_fill(struct ionic_rx_qcq *rxq)
449 {
450 	struct ionic_queue *q = &rxq->qcq.q;
451 	uint32_t i;
452 	int err = 0;
453 
454 	for (i = 0; i < q->num_descs - 1u; i++) {
455 		err = ionic_rx_fill_one(rxq);
456 		if (err)
457 			break;
458 
459 		q->head_idx = Q_NEXT_TO_POST(q, 1);
460 	}
461 
462 	ionic_rxq_flush(q);
463 
464 	return err;
465 }
466