xref: /dpdk/drivers/net/ionic/ionic_rxtx_simple.c (revision 8e7a6b8ac8696684133fd54ea403c840e37758c2)
1e86a6fccSAndrew Boyer /* SPDX-License-Identifier: BSD-3-Clause
2e86a6fccSAndrew Boyer  * Copyright 2018-2022 Advanced Micro Devices, Inc.
3e86a6fccSAndrew Boyer  */
4e86a6fccSAndrew Boyer 
5e86a6fccSAndrew Boyer #include <stdio.h>
6e86a6fccSAndrew Boyer #include <errno.h>
7e86a6fccSAndrew Boyer #include <stdint.h>
8e86a6fccSAndrew Boyer #include <assert.h>
9e86a6fccSAndrew Boyer 
10e86a6fccSAndrew Boyer #include <rte_common.h>
11e86a6fccSAndrew Boyer #include <rte_byteorder.h>
12e86a6fccSAndrew Boyer #include <rte_atomic.h>
13e86a6fccSAndrew Boyer #include <rte_mempool.h>
14e86a6fccSAndrew Boyer #include <rte_mbuf.h>
15e86a6fccSAndrew Boyer #include <rte_ether.h>
16e86a6fccSAndrew Boyer #include <rte_prefetch.h>
17e86a6fccSAndrew Boyer 
18e86a6fccSAndrew Boyer #include "ionic.h"
19*484027bfSAndrew Boyer #include "ionic_ethdev.h"
20e86a6fccSAndrew Boyer #include "ionic_lif.h"
21e86a6fccSAndrew Boyer #include "ionic_rxtx.h"
22e86a6fccSAndrew Boyer 
23e86a6fccSAndrew Boyer static __rte_always_inline void
ionic_tx_flush(struct ionic_tx_qcq * txq)24e86a6fccSAndrew Boyer ionic_tx_flush(struct ionic_tx_qcq *txq)
25e86a6fccSAndrew Boyer {
26e86a6fccSAndrew Boyer 	struct ionic_cq *cq = &txq->qcq.cq;
27e86a6fccSAndrew Boyer 	struct ionic_queue *q = &txq->qcq.q;
28ea81e9f2SAndrew Boyer 	struct ionic_tx_stats *stats = &txq->stats;
29e86a6fccSAndrew Boyer 	struct rte_mbuf *txm;
30463ad260SNeel Patel 	struct ionic_txq_comp *cq_desc_base = cq->base;
31463ad260SNeel Patel 	volatile struct ionic_txq_comp *cq_desc;
32e86a6fccSAndrew Boyer 	void **info;
33e86a6fccSAndrew Boyer 
34e86a6fccSAndrew Boyer 	cq_desc = &cq_desc_base[cq->tail_idx];
35e86a6fccSAndrew Boyer 
36e86a6fccSAndrew Boyer 	while (color_match(cq_desc->color, cq->done_color)) {
37e86a6fccSAndrew Boyer 		cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
38e86a6fccSAndrew Boyer 		if (cq->tail_idx == 0)
39e86a6fccSAndrew Boyer 			cq->done_color = !cq->done_color;
40e86a6fccSAndrew Boyer 
41e86a6fccSAndrew Boyer 		/* Prefetch 4 x 16B comp at cq->tail_idx + 4 */
42e86a6fccSAndrew Boyer 		if ((cq->tail_idx & 0x3) == 0)
43e86a6fccSAndrew Boyer 			rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
44e86a6fccSAndrew Boyer 
45e86a6fccSAndrew Boyer 		while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) {
46e86a6fccSAndrew Boyer 			/* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */
47e86a6fccSAndrew Boyer 			rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 2)]);
48e86a6fccSAndrew Boyer 
49e86a6fccSAndrew Boyer 			/* Prefetch next mbuf */
50e86a6fccSAndrew Boyer 			void **next_info =
51e86a6fccSAndrew Boyer 				&q->info[Q_NEXT_TO_SRVC(q, 1)];
52e86a6fccSAndrew Boyer 			if (next_info[0])
53e86a6fccSAndrew Boyer 				rte_mbuf_prefetch_part2(next_info[0]);
54e86a6fccSAndrew Boyer 
55e86a6fccSAndrew Boyer 			info = &q->info[q->tail_idx];
56e86a6fccSAndrew Boyer 			{
57e86a6fccSAndrew Boyer 				txm = info[0];
58e86a6fccSAndrew Boyer 
59e86a6fccSAndrew Boyer 				if (txq->flags & IONIC_QCQ_F_FAST_FREE)
60e86a6fccSAndrew Boyer 					rte_mempool_put(txm->pool, txm);
61e86a6fccSAndrew Boyer 				else
62e86a6fccSAndrew Boyer 					rte_pktmbuf_free_seg(txm);
63e86a6fccSAndrew Boyer 
64e86a6fccSAndrew Boyer 				info[0] = NULL;
65e86a6fccSAndrew Boyer 			}
66e86a6fccSAndrew Boyer 
67e86a6fccSAndrew Boyer 			q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
68e86a6fccSAndrew Boyer 		}
69e86a6fccSAndrew Boyer 
70e86a6fccSAndrew Boyer 		cq_desc = &cq_desc_base[cq->tail_idx];
71ea81e9f2SAndrew Boyer 		stats->comps++;
72e86a6fccSAndrew Boyer 	}
73e86a6fccSAndrew Boyer }
74e86a6fccSAndrew Boyer 
75e86a6fccSAndrew Boyer static __rte_always_inline int
ionic_tx(struct ionic_tx_qcq * txq,struct rte_mbuf * txm)76e86a6fccSAndrew Boyer ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
77e86a6fccSAndrew Boyer {
78e86a6fccSAndrew Boyer 	struct ionic_queue *q = &txq->qcq.q;
79e86a6fccSAndrew Boyer 	struct ionic_txq_desc *desc, *desc_base = q->base;
80e86a6fccSAndrew Boyer 	struct ionic_tx_stats *stats = &txq->stats;
81e86a6fccSAndrew Boyer 	void **info;
82e86a6fccSAndrew Boyer 	uint64_t ol_flags = txm->ol_flags;
83e86a6fccSAndrew Boyer 	uint64_t addr, cmd;
84e86a6fccSAndrew Boyer 	uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
85e86a6fccSAndrew Boyer 	uint8_t flags = 0;
86e86a6fccSAndrew Boyer 
87e86a6fccSAndrew Boyer 	if (txm->nb_segs > 1)
88e86a6fccSAndrew Boyer 		return -EINVAL;
89e86a6fccSAndrew Boyer 
90e86a6fccSAndrew Boyer 	desc = &desc_base[q->head_idx];
91e86a6fccSAndrew Boyer 	info = &q->info[q->head_idx];
92e86a6fccSAndrew Boyer 
93e86a6fccSAndrew Boyer 	if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) &&
94e86a6fccSAndrew Boyer 	    (txq->flags & IONIC_QCQ_F_CSUM_L3)) {
95e86a6fccSAndrew Boyer 		opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
96e86a6fccSAndrew Boyer 		flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
97e86a6fccSAndrew Boyer 	}
98e86a6fccSAndrew Boyer 
99e86a6fccSAndrew Boyer 	if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) &&
100e86a6fccSAndrew Boyer 	     (txq->flags & IONIC_QCQ_F_CSUM_TCP)) ||
101e86a6fccSAndrew Boyer 	    ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) &&
102e86a6fccSAndrew Boyer 	     (txq->flags & IONIC_QCQ_F_CSUM_UDP))) {
103e86a6fccSAndrew Boyer 		opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
104e86a6fccSAndrew Boyer 		flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
105e86a6fccSAndrew Boyer 	}
106e86a6fccSAndrew Boyer 
107e86a6fccSAndrew Boyer 	if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE)
108e86a6fccSAndrew Boyer 		stats->no_csum++;
109e86a6fccSAndrew Boyer 
110e86a6fccSAndrew Boyer 	if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) ||
111e86a6fccSAndrew Boyer 	     (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) &&
112e86a6fccSAndrew Boyer 	    ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) ||
113e86a6fccSAndrew Boyer 	     (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) {
114e86a6fccSAndrew Boyer 		flags |= IONIC_TXQ_DESC_FLAG_ENCAP;
115e86a6fccSAndrew Boyer 	}
116e86a6fccSAndrew Boyer 
117e86a6fccSAndrew Boyer 	if (ol_flags & RTE_MBUF_F_TX_VLAN) {
118e86a6fccSAndrew Boyer 		flags |= IONIC_TXQ_DESC_FLAG_VLAN;
119e86a6fccSAndrew Boyer 		desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci);
120e86a6fccSAndrew Boyer 	}
121e86a6fccSAndrew Boyer 
122e86a6fccSAndrew Boyer 	addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm));
123e86a6fccSAndrew Boyer 
124e86a6fccSAndrew Boyer 	cmd = encode_txq_desc_cmd(opcode, flags, 0, addr);
125e86a6fccSAndrew Boyer 	desc->cmd = rte_cpu_to_le_64(cmd);
126e86a6fccSAndrew Boyer 	desc->len = rte_cpu_to_le_16(txm->data_len);
127e86a6fccSAndrew Boyer 
128e86a6fccSAndrew Boyer 	info[0] = txm;
129e86a6fccSAndrew Boyer 
130e86a6fccSAndrew Boyer 	q->head_idx = Q_NEXT_TO_POST(q, 1);
131e86a6fccSAndrew Boyer 
132e86a6fccSAndrew Boyer 	return 0;
133e86a6fccSAndrew Boyer }
134e86a6fccSAndrew Boyer 
135e86a6fccSAndrew Boyer uint16_t
ionic_xmit_pkts(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)136e86a6fccSAndrew Boyer ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
137e86a6fccSAndrew Boyer 		uint16_t nb_pkts)
138e86a6fccSAndrew Boyer {
139e86a6fccSAndrew Boyer 	struct ionic_tx_qcq *txq = tx_queue;
140e86a6fccSAndrew Boyer 	struct ionic_queue *q = &txq->qcq.q;
14190fa040aSNeel Patel 	struct ionic_txq_desc *desc_base = q->base;
142e86a6fccSAndrew Boyer 	struct ionic_tx_stats *stats = &txq->stats;
143e86a6fccSAndrew Boyer 	struct rte_mbuf *mbuf;
144e86a6fccSAndrew Boyer 	uint32_t bytes_tx = 0;
145e86a6fccSAndrew Boyer 	uint16_t nb_avail, nb_tx = 0;
146a5b1ffd8SAndrew Boyer 	uint64_t then, now, hz, delta;
147e86a6fccSAndrew Boyer 	int err;
148e86a6fccSAndrew Boyer 
149e86a6fccSAndrew Boyer 	rte_prefetch0(&desc_base[q->head_idx]);
150e86a6fccSAndrew Boyer 	rte_prefetch0(&q->info[q->head_idx]);
151e86a6fccSAndrew Boyer 
1523c02593cSAndrew Boyer 	if (nb_pkts) {
153e86a6fccSAndrew Boyer 		rte_mbuf_prefetch_part1(tx_pkts[0]);
154e86a6fccSAndrew Boyer 		rte_mbuf_prefetch_part2(tx_pkts[0]);
155e86a6fccSAndrew Boyer 	}
156e86a6fccSAndrew Boyer 
157e86a6fccSAndrew Boyer 	if (ionic_q_space_avail(q) < txq->free_thresh) {
158e86a6fccSAndrew Boyer 		/* Cleaning old buffers */
159e86a6fccSAndrew Boyer 		ionic_tx_flush(txq);
160e86a6fccSAndrew Boyer 	}
161e86a6fccSAndrew Boyer 
162e86a6fccSAndrew Boyer 	nb_avail = ionic_q_space_avail(q);
163e86a6fccSAndrew Boyer 	if (nb_avail < nb_pkts) {
164e86a6fccSAndrew Boyer 		stats->stop += nb_pkts - nb_avail;
165e86a6fccSAndrew Boyer 		nb_pkts = nb_avail;
166e86a6fccSAndrew Boyer 	}
167e86a6fccSAndrew Boyer 
168e86a6fccSAndrew Boyer 	while (nb_tx < nb_pkts) {
169e86a6fccSAndrew Boyer 		uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
170e86a6fccSAndrew Boyer 		rte_prefetch0(&desc_base[next_idx]);
171e86a6fccSAndrew Boyer 		rte_prefetch0(&q->info[next_idx]);
172e86a6fccSAndrew Boyer 
173e86a6fccSAndrew Boyer 		if (nb_tx + 1 < nb_pkts) {
174e86a6fccSAndrew Boyer 			rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]);
175e86a6fccSAndrew Boyer 			rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]);
176e86a6fccSAndrew Boyer 		}
177e86a6fccSAndrew Boyer 
178e86a6fccSAndrew Boyer 		mbuf = tx_pkts[nb_tx];
179e86a6fccSAndrew Boyer 
180e86a6fccSAndrew Boyer 		if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)
181e86a6fccSAndrew Boyer 			err = ionic_tx_tso(txq, mbuf);
182e86a6fccSAndrew Boyer 		else
183e86a6fccSAndrew Boyer 			err = ionic_tx(txq, mbuf);
184e86a6fccSAndrew Boyer 		if (err) {
185e86a6fccSAndrew Boyer 			stats->drop += nb_pkts - nb_tx;
186e86a6fccSAndrew Boyer 			break;
187e86a6fccSAndrew Boyer 		}
188e86a6fccSAndrew Boyer 
189e86a6fccSAndrew Boyer 		bytes_tx += mbuf->pkt_len;
190e86a6fccSAndrew Boyer 		nb_tx++;
191e86a6fccSAndrew Boyer 	}
192e86a6fccSAndrew Boyer 
193e86a6fccSAndrew Boyer 	if (nb_tx > 0) {
19490fa040aSNeel Patel 		ionic_txq_flush(q);
195e86a6fccSAndrew Boyer 
196a5b1ffd8SAndrew Boyer 		txq->last_wdog_cycles = rte_get_timer_cycles();
197a5b1ffd8SAndrew Boyer 
198e86a6fccSAndrew Boyer 		stats->packets += nb_tx;
199e86a6fccSAndrew Boyer 		stats->bytes += bytes_tx;
200a5b1ffd8SAndrew Boyer 	} else {
201a5b1ffd8SAndrew Boyer 		/*
202a5b1ffd8SAndrew Boyer 		 * Ring the doorbell again if no work could be posted and work
203a5b1ffd8SAndrew Boyer 		 * is still pending after the deadline.
204a5b1ffd8SAndrew Boyer 		 */
205a5b1ffd8SAndrew Boyer 		if (q->head_idx != q->tail_idx) {
206a5b1ffd8SAndrew Boyer 			then = txq->last_wdog_cycles;
207a5b1ffd8SAndrew Boyer 			now = rte_get_timer_cycles();
208a5b1ffd8SAndrew Boyer 			hz = rte_get_timer_hz();
209a5b1ffd8SAndrew Boyer 			delta = (now - then) * 1000;
210a5b1ffd8SAndrew Boyer 
211a5b1ffd8SAndrew Boyer 			if (delta >= hz * IONIC_Q_WDOG_MS) {
212a5b1ffd8SAndrew Boyer 				ionic_q_flush(q);
213a5b1ffd8SAndrew Boyer 				txq->last_wdog_cycles = now;
214a5b1ffd8SAndrew Boyer 			}
215a5b1ffd8SAndrew Boyer 		}
216e86a6fccSAndrew Boyer 	}
217e86a6fccSAndrew Boyer 
218e86a6fccSAndrew Boyer 	return nb_tx;
219e86a6fccSAndrew Boyer }
220e86a6fccSAndrew Boyer 
221e86a6fccSAndrew Boyer /*
222e86a6fccSAndrew Boyer  * Cleans one descriptor. Connects the filled mbufs into a chain.
223e86a6fccSAndrew Boyer  * Does not advance the tail index.
224e86a6fccSAndrew Boyer  */
225e86a6fccSAndrew Boyer static __rte_always_inline void
ionic_rx_clean_one(struct ionic_rx_qcq * rxq,volatile struct ionic_rxq_comp * cq_desc,struct ionic_rx_service * rx_svc)226e86a6fccSAndrew Boyer ionic_rx_clean_one(struct ionic_rx_qcq *rxq,
227463ad260SNeel Patel 		volatile struct ionic_rxq_comp *cq_desc,
228e86a6fccSAndrew Boyer 		struct ionic_rx_service *rx_svc)
229e86a6fccSAndrew Boyer {
230e86a6fccSAndrew Boyer 	struct ionic_queue *q = &rxq->qcq.q;
231e86a6fccSAndrew Boyer 	struct rte_mbuf *rxm;
232e86a6fccSAndrew Boyer 	struct ionic_rx_stats *stats = &rxq->stats;
233e86a6fccSAndrew Boyer 	uint64_t pkt_flags = 0;
234e86a6fccSAndrew Boyer 	uint32_t pkt_type;
235e86a6fccSAndrew Boyer 	uint16_t cq_desc_len;
236e86a6fccSAndrew Boyer 	uint8_t ptype, cflags;
237e86a6fccSAndrew Boyer 	void **info;
238e86a6fccSAndrew Boyer 
239e86a6fccSAndrew Boyer 	cq_desc_len = rte_le_to_cpu_16(cq_desc->len);
240e86a6fccSAndrew Boyer 
241e86a6fccSAndrew Boyer 	info = &q->info[q->tail_idx];
242e86a6fccSAndrew Boyer 
243e86a6fccSAndrew Boyer 	rxm = info[0];
244e86a6fccSAndrew Boyer 
245e86a6fccSAndrew Boyer 	if (cq_desc->status) {
246e86a6fccSAndrew Boyer 		stats->bad_cq_status++;
247e86a6fccSAndrew Boyer 		return;
248e86a6fccSAndrew Boyer 	}
249e86a6fccSAndrew Boyer 
250e86a6fccSAndrew Boyer 	if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) {
251e86a6fccSAndrew Boyer 		stats->bad_len++;
252e86a6fccSAndrew Boyer 		return;
253e86a6fccSAndrew Boyer 	}
254e86a6fccSAndrew Boyer 
255e86a6fccSAndrew Boyer 	info[0] = NULL;
256e86a6fccSAndrew Boyer 
257e86a6fccSAndrew Boyer 	/* Set the mbuf metadata based on the cq entry */
258e86a6fccSAndrew Boyer 	rxm->rearm_data[0] = rxq->rearm_data;
259e86a6fccSAndrew Boyer 	rxm->pkt_len = cq_desc_len;
260e86a6fccSAndrew Boyer 	rxm->data_len = cq_desc_len;
261e86a6fccSAndrew Boyer 
262e86a6fccSAndrew Boyer 	/* RSS */
263e86a6fccSAndrew Boyer 	pkt_flags |= RTE_MBUF_F_RX_RSS_HASH;
264e86a6fccSAndrew Boyer 	rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash);
265e86a6fccSAndrew Boyer 
266e86a6fccSAndrew Boyer 	/* Vlan Strip */
267e86a6fccSAndrew Boyer 	if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
268e86a6fccSAndrew Boyer 		pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
269e86a6fccSAndrew Boyer 		rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci);
270e86a6fccSAndrew Boyer 	}
271e86a6fccSAndrew Boyer 
272e86a6fccSAndrew Boyer 	/* Checksum */
273e86a6fccSAndrew Boyer 	if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
274e86a6fccSAndrew Boyer 		cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK;
275e86a6fccSAndrew Boyer 		pkt_flags |= ionic_csum_flags[cflags];
276e86a6fccSAndrew Boyer 	}
277e86a6fccSAndrew Boyer 
278e86a6fccSAndrew Boyer 	rxm->ol_flags = pkt_flags;
279e86a6fccSAndrew Boyer 
280e86a6fccSAndrew Boyer 	/* Packet Type */
281e86a6fccSAndrew Boyer 	ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK;
282e86a6fccSAndrew Boyer 	pkt_type = ionic_ptype_table[ptype];
283e86a6fccSAndrew Boyer 	if (pkt_type == RTE_PTYPE_UNKNOWN) {
284e86a6fccSAndrew Boyer 		struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
285e86a6fccSAndrew Boyer 				struct rte_ether_hdr *);
286e86a6fccSAndrew Boyer 		uint16_t ether_type = eth_h->ether_type;
287e86a6fccSAndrew Boyer 		if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
288e86a6fccSAndrew Boyer 			pkt_type = RTE_PTYPE_L2_ETHER_ARP;
289e86a6fccSAndrew Boyer 		else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP))
290e86a6fccSAndrew Boyer 			pkt_type = RTE_PTYPE_L2_ETHER_LLDP;
291e86a6fccSAndrew Boyer 		else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588))
292e86a6fccSAndrew Boyer 			pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC;
293e86a6fccSAndrew Boyer 		stats->mtods++;
294e86a6fccSAndrew Boyer 	} else if (pkt_flags & RTE_MBUF_F_RX_VLAN) {
295e86a6fccSAndrew Boyer 		pkt_type |= RTE_PTYPE_L2_ETHER_VLAN;
296e86a6fccSAndrew Boyer 	} else {
297e86a6fccSAndrew Boyer 		pkt_type |= RTE_PTYPE_L2_ETHER;
298e86a6fccSAndrew Boyer 	}
299e86a6fccSAndrew Boyer 
300e86a6fccSAndrew Boyer 	rxm->packet_type = pkt_type;
301e86a6fccSAndrew Boyer 
302e86a6fccSAndrew Boyer 	rx_svc->rx_pkts[rx_svc->nb_rx] = rxm;
303e86a6fccSAndrew Boyer 	rx_svc->nb_rx++;
304e86a6fccSAndrew Boyer 
305e86a6fccSAndrew Boyer 	stats->packets++;
306e86a6fccSAndrew Boyer 	stats->bytes += rxm->pkt_len;
307e86a6fccSAndrew Boyer }
308e86a6fccSAndrew Boyer 
309e86a6fccSAndrew Boyer /*
310e86a6fccSAndrew Boyer  * Fills one descriptor with mbufs. Does not advance the head index.
311e86a6fccSAndrew Boyer  */
312e86a6fccSAndrew Boyer static __rte_always_inline int
ionic_rx_fill_one(struct ionic_rx_qcq * rxq)313e86a6fccSAndrew Boyer ionic_rx_fill_one(struct ionic_rx_qcq *rxq)
314e86a6fccSAndrew Boyer {
315e86a6fccSAndrew Boyer 	struct ionic_queue *q = &rxq->qcq.q;
316e86a6fccSAndrew Boyer 	struct rte_mbuf *rxm;
317e86a6fccSAndrew Boyer 	struct ionic_rxq_desc *desc, *desc_base = q->base;
318e86a6fccSAndrew Boyer 	rte_iova_t data_iova;
319e86a6fccSAndrew Boyer 	void **info;
320e86a6fccSAndrew Boyer 	int ret;
321e86a6fccSAndrew Boyer 
322e86a6fccSAndrew Boyer 	info = &q->info[q->head_idx];
323e86a6fccSAndrew Boyer 	desc = &desc_base[q->head_idx];
324e86a6fccSAndrew Boyer 
325e86a6fccSAndrew Boyer 	/* mbuf is unused */
326e86a6fccSAndrew Boyer 	if (info[0])
327e86a6fccSAndrew Boyer 		return 0;
328e86a6fccSAndrew Boyer 
329e86a6fccSAndrew Boyer 	if (rxq->mb_idx == 0) {
330e86a6fccSAndrew Boyer 		ret = rte_mempool_get_bulk(rxq->mb_pool,
331e86a6fccSAndrew Boyer 					(void **)rxq->mbs,
332e86a6fccSAndrew Boyer 					IONIC_MBUF_BULK_ALLOC);
333e86a6fccSAndrew Boyer 		if (ret) {
334e86a6fccSAndrew Boyer 			assert(0);
335e86a6fccSAndrew Boyer 			return -ENOMEM;
336e86a6fccSAndrew Boyer 		}
337e86a6fccSAndrew Boyer 
338e86a6fccSAndrew Boyer 		rxq->mb_idx = IONIC_MBUF_BULK_ALLOC;
339e86a6fccSAndrew Boyer 	}
340e86a6fccSAndrew Boyer 
341e86a6fccSAndrew Boyer 	rxm = rxq->mbs[--rxq->mb_idx];
342e86a6fccSAndrew Boyer 	info[0] = rxm;
343e86a6fccSAndrew Boyer 
344e86a6fccSAndrew Boyer 	data_iova = rte_mbuf_data_iova_default(rxm);
345e86a6fccSAndrew Boyer 	desc->addr = rte_cpu_to_le_64(data_iova);
346e86a6fccSAndrew Boyer 
347e86a6fccSAndrew Boyer 	return 0;
348e86a6fccSAndrew Boyer }
349e86a6fccSAndrew Boyer 
350e86a6fccSAndrew Boyer /*
351e86a6fccSAndrew Boyer  * Walk the CQ to find completed receive descriptors.
352e86a6fccSAndrew Boyer  * Any completed descriptor found is refilled.
353e86a6fccSAndrew Boyer  */
354e86a6fccSAndrew Boyer static __rte_always_inline void
ionic_rxq_service(struct ionic_rx_qcq * rxq,uint32_t work_to_do,struct ionic_rx_service * rx_svc)355e86a6fccSAndrew Boyer ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
356e86a6fccSAndrew Boyer 		struct ionic_rx_service *rx_svc)
357e86a6fccSAndrew Boyer {
358e86a6fccSAndrew Boyer 	struct ionic_cq *cq = &rxq->qcq.cq;
359e86a6fccSAndrew Boyer 	struct ionic_queue *q = &rxq->qcq.q;
360e86a6fccSAndrew Boyer 	struct ionic_rxq_desc *q_desc_base = q->base;
361463ad260SNeel Patel 	struct ionic_rxq_comp *cq_desc_base = cq->base;
362463ad260SNeel Patel 	volatile struct ionic_rxq_comp *cq_desc;
363e86a6fccSAndrew Boyer 	uint32_t work_done = 0;
364a5b1ffd8SAndrew Boyer 	uint64_t then, now, hz, delta;
365e86a6fccSAndrew Boyer 
366e86a6fccSAndrew Boyer 	cq_desc = &cq_desc_base[cq->tail_idx];
367e86a6fccSAndrew Boyer 
368e86a6fccSAndrew Boyer 	while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
369e86a6fccSAndrew Boyer 		cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
370e86a6fccSAndrew Boyer 		if (cq->tail_idx == 0)
371e86a6fccSAndrew Boyer 			cq->done_color = !cq->done_color;
372e86a6fccSAndrew Boyer 
373e86a6fccSAndrew Boyer 		/* Prefetch 8 x 8B bufinfo */
374e86a6fccSAndrew Boyer 		rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 8)]);
375e86a6fccSAndrew Boyer 		/* Prefetch 4 x 16B comp */
376e86a6fccSAndrew Boyer 		rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
377e86a6fccSAndrew Boyer 		/* Prefetch 4 x 16B descriptors */
378e86a6fccSAndrew Boyer 		rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
379e86a6fccSAndrew Boyer 
380e86a6fccSAndrew Boyer 		/* Clean one descriptor */
381e86a6fccSAndrew Boyer 		ionic_rx_clean_one(rxq, cq_desc, rx_svc);
382e86a6fccSAndrew Boyer 		q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
383e86a6fccSAndrew Boyer 
384e86a6fccSAndrew Boyer 		/* Fill one descriptor */
385e86a6fccSAndrew Boyer 		(void)ionic_rx_fill_one(rxq);
386e86a6fccSAndrew Boyer 
387e86a6fccSAndrew Boyer 		q->head_idx = Q_NEXT_TO_POST(q, 1);
388e86a6fccSAndrew Boyer 
389e86a6fccSAndrew Boyer 		if (++work_done == work_to_do)
390e86a6fccSAndrew Boyer 			break;
391e86a6fccSAndrew Boyer 
392e86a6fccSAndrew Boyer 		cq_desc = &cq_desc_base[cq->tail_idx];
393e86a6fccSAndrew Boyer 	}
394e86a6fccSAndrew Boyer 
395e86a6fccSAndrew Boyer 	/* Update the queue indices and ring the doorbell */
396a5b1ffd8SAndrew Boyer 	if (work_done) {
39790fa040aSNeel Patel 		ionic_rxq_flush(q);
39890fa040aSNeel Patel 
399a5b1ffd8SAndrew Boyer 		rxq->last_wdog_cycles = rte_get_timer_cycles();
400a5b1ffd8SAndrew Boyer 		rxq->wdog_ms = IONIC_Q_WDOG_MS;
401a5b1ffd8SAndrew Boyer 	} else {
402a5b1ffd8SAndrew Boyer 		/*
403a5b1ffd8SAndrew Boyer 		 * Ring the doorbell again if no recvs were posted and the
404a5b1ffd8SAndrew Boyer 		 * recv queue is not empty after the deadline.
405a5b1ffd8SAndrew Boyer 		 *
406a5b1ffd8SAndrew Boyer 		 * Exponentially back off the deadline to avoid excessive
407a5b1ffd8SAndrew Boyer 		 * doorbells when the recv queue is idle.
408a5b1ffd8SAndrew Boyer 		 */
409a5b1ffd8SAndrew Boyer 		if (q->head_idx != q->tail_idx) {
410a5b1ffd8SAndrew Boyer 			then = rxq->last_wdog_cycles;
411a5b1ffd8SAndrew Boyer 			now = rte_get_timer_cycles();
412a5b1ffd8SAndrew Boyer 			hz = rte_get_timer_hz();
413a5b1ffd8SAndrew Boyer 			delta = (now - then) * 1000;
414a5b1ffd8SAndrew Boyer 
415a5b1ffd8SAndrew Boyer 			if (delta >= hz * rxq->wdog_ms) {
416a5b1ffd8SAndrew Boyer 				ionic_q_flush(q);
417a5b1ffd8SAndrew Boyer 				rxq->last_wdog_cycles = now;
418a5b1ffd8SAndrew Boyer 
419a5b1ffd8SAndrew Boyer 				delta = 2 * rxq->wdog_ms;
420a5b1ffd8SAndrew Boyer 				if (delta > IONIC_Q_WDOG_MAX_MS)
421a5b1ffd8SAndrew Boyer 					delta = IONIC_Q_WDOG_MAX_MS;
422a5b1ffd8SAndrew Boyer 
423a5b1ffd8SAndrew Boyer 				rxq->wdog_ms = delta;
424a5b1ffd8SAndrew Boyer 			}
425a5b1ffd8SAndrew Boyer 		}
426a5b1ffd8SAndrew Boyer 	}
427e86a6fccSAndrew Boyer }
428e86a6fccSAndrew Boyer 
429e86a6fccSAndrew Boyer uint16_t
ionic_recv_pkts(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)430e86a6fccSAndrew Boyer ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
431e86a6fccSAndrew Boyer 		uint16_t nb_pkts)
432e86a6fccSAndrew Boyer {
433e86a6fccSAndrew Boyer 	struct ionic_rx_qcq *rxq = rx_queue;
434e86a6fccSAndrew Boyer 	struct ionic_rx_service rx_svc;
435e86a6fccSAndrew Boyer 
436e86a6fccSAndrew Boyer 	rx_svc.rx_pkts = rx_pkts;
437e86a6fccSAndrew Boyer 	rx_svc.nb_rx = 0;
438e86a6fccSAndrew Boyer 
439e86a6fccSAndrew Boyer 	ionic_rxq_service(rxq, nb_pkts, &rx_svc);
440e86a6fccSAndrew Boyer 
441e86a6fccSAndrew Boyer 	return rx_svc.nb_rx;
442e86a6fccSAndrew Boyer }
443e86a6fccSAndrew Boyer 
444e86a6fccSAndrew Boyer /*
445e86a6fccSAndrew Boyer  * Fills all descriptors with mbufs.
446e86a6fccSAndrew Boyer  */
447e86a6fccSAndrew Boyer int __rte_cold
ionic_rx_fill(struct ionic_rx_qcq * rxq)448e86a6fccSAndrew Boyer ionic_rx_fill(struct ionic_rx_qcq *rxq)
449e86a6fccSAndrew Boyer {
450e86a6fccSAndrew Boyer 	struct ionic_queue *q = &rxq->qcq.q;
451e86a6fccSAndrew Boyer 	uint32_t i;
452e86a6fccSAndrew Boyer 	int err = 0;
453e86a6fccSAndrew Boyer 
454e86a6fccSAndrew Boyer 	for (i = 0; i < q->num_descs - 1u; i++) {
455e86a6fccSAndrew Boyer 		err = ionic_rx_fill_one(rxq);
456e86a6fccSAndrew Boyer 		if (err)
457e86a6fccSAndrew Boyer 			break;
458e86a6fccSAndrew Boyer 
459e86a6fccSAndrew Boyer 		q->head_idx = Q_NEXT_TO_POST(q, 1);
460e86a6fccSAndrew Boyer 	}
461e86a6fccSAndrew Boyer 
46290fa040aSNeel Patel 	ionic_rxq_flush(q);
463e86a6fccSAndrew Boyer 
464e86a6fccSAndrew Boyer 	return err;
465e86a6fccSAndrew Boyer }
466