xref: /dpdk/drivers/net/ionic/ionic_rxtx.c (revision d46b9fa83f136beb0e6feedd0a7b3a228b0d8cd3)
176668754SAndrew Boyer /* SPDX-License-Identifier: BSD-3-Clause
2a5205992SAndrew Boyer  * Copyright 2018-2022 Advanced Micro Devices, Inc.
3a27d9013SAlfredo Cardigliano  */
4a27d9013SAlfredo Cardigliano 
5a27d9013SAlfredo Cardigliano #include <stdio.h>
6a27d9013SAlfredo Cardigliano #include <string.h>
7a27d9013SAlfredo Cardigliano #include <errno.h>
8a27d9013SAlfredo Cardigliano #include <stdint.h>
9a27d9013SAlfredo Cardigliano 
10a27d9013SAlfredo Cardigliano #include <rte_common.h>
11e86a6fccSAndrew Boyer #include <rte_byteorder.h>
12e86a6fccSAndrew Boyer #include <rte_errno.h>
13a27d9013SAlfredo Cardigliano #include <rte_log.h>
14a27d9013SAlfredo Cardigliano #include <rte_mbuf.h>
15a27d9013SAlfredo Cardigliano #include <rte_ether.h>
16a27d9013SAlfredo Cardigliano #include <rte_ip.h>
17e86a6fccSAndrew Boyer #include <rte_tcp.h>
18e86a6fccSAndrew Boyer #include <rte_ethdev.h>
19e86a6fccSAndrew Boyer #include <ethdev_driver.h>
20a27d9013SAlfredo Cardigliano 
21e86a6fccSAndrew Boyer #include "ionic.h"
22e86a6fccSAndrew Boyer #include "ionic_dev.h"
23a27d9013SAlfredo Cardigliano #include "ionic_lif.h"
24e86a6fccSAndrew Boyer #include "ionic_ethdev.h"
25a27d9013SAlfredo Cardigliano #include "ionic_rxtx.h"
26e86a6fccSAndrew Boyer #include "ionic_logs.h"
27a27d9013SAlfredo Cardigliano 
28e7222f94SAndrew Boyer static void
ionic_empty_array(void ** array,uint32_t free_idx,uint32_t zero_idx)29*d46b9fa8SAndrew Boyer ionic_empty_array(void **array, uint32_t free_idx, uint32_t zero_idx)
30e7222f94SAndrew Boyer {
31e7222f94SAndrew Boyer 	uint32_t i;
32e7222f94SAndrew Boyer 
33*d46b9fa8SAndrew Boyer 	for (i = 0; i < free_idx; i++)
34e7222f94SAndrew Boyer 		if (array[i])
35e7222f94SAndrew Boyer 			rte_pktmbuf_free_seg(array[i]);
36e7222f94SAndrew Boyer 
37*d46b9fa8SAndrew Boyer 	memset(array, 0, sizeof(void *) * zero_idx);
38e7222f94SAndrew Boyer }
39e7222f94SAndrew Boyer 
40e7222f94SAndrew Boyer static void __rte_cold
ionic_tx_empty(struct ionic_tx_qcq * txq)41e7222f94SAndrew Boyer ionic_tx_empty(struct ionic_tx_qcq *txq)
42e7222f94SAndrew Boyer {
43e7222f94SAndrew Boyer 	struct ionic_queue *q = &txq->qcq.q;
44*d46b9fa8SAndrew Boyer 	uint32_t info_len = q->num_descs * q->num_segs;
45e7222f94SAndrew Boyer 
46*d46b9fa8SAndrew Boyer 	ionic_empty_array(q->info, info_len, info_len);
47e7222f94SAndrew Boyer }
48e7222f94SAndrew Boyer 
49e7222f94SAndrew Boyer static void __rte_cold
ionic_rx_empty(struct ionic_rx_qcq * rxq)50e7222f94SAndrew Boyer ionic_rx_empty(struct ionic_rx_qcq *rxq)
51e7222f94SAndrew Boyer {
52e7222f94SAndrew Boyer 	struct ionic_queue *q = &rxq->qcq.q;
53*d46b9fa8SAndrew Boyer 	uint32_t info_len = q->num_descs * q->num_segs;
54e7222f94SAndrew Boyer 
557b20fc2fSAndrew Boyer 	/*
567b20fc2fSAndrew Boyer 	 * Walk the full info array so that the clean up includes any
577b20fc2fSAndrew Boyer 	 * fragments that were left dangling for later reuse
587b20fc2fSAndrew Boyer 	 */
59*d46b9fa8SAndrew Boyer 	ionic_empty_array(q->info, info_len, info_len);
60218afd82SAndrew Boyer 
61*d46b9fa8SAndrew Boyer 	ionic_empty_array((void **)rxq->mbs, rxq->mb_idx,
62*d46b9fa8SAndrew Boyer 			IONIC_MBUF_BULK_ALLOC);
63218afd82SAndrew Boyer 	rxq->mb_idx = 0;
64e7222f94SAndrew Boyer }
65e7222f94SAndrew Boyer 
66a27d9013SAlfredo Cardigliano /*********************************************************************
67a27d9013SAlfredo Cardigliano  *
68a27d9013SAlfredo Cardigliano  *  TX functions
69a27d9013SAlfredo Cardigliano  *
70a27d9013SAlfredo Cardigliano  **********************************************************************/
71a27d9013SAlfredo Cardigliano 
72a27d9013SAlfredo Cardigliano void
ionic_txq_info_get(struct rte_eth_dev * dev,uint16_t queue_id,struct rte_eth_txq_info * qinfo)73a27d9013SAlfredo Cardigliano ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
74a27d9013SAlfredo Cardigliano 		struct rte_eth_txq_info *qinfo)
75a27d9013SAlfredo Cardigliano {
76be39f75cSAndrew Boyer 	struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id];
77be39f75cSAndrew Boyer 	struct ionic_queue *q = &txq->qcq.q;
78a27d9013SAlfredo Cardigliano 
79a27d9013SAlfredo Cardigliano 	qinfo->nb_desc = q->num_descs;
8068591087SAndrew Boyer 	qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
819ac234eeSAndrew Boyer 	if (txq->flags & IONIC_QCQ_F_FAST_FREE)
829ac234eeSAndrew Boyer 		qinfo->conf.offloads |= RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
8302eabf57SAndrew Boyer 	qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED;
84a27d9013SAlfredo Cardigliano }
85a27d9013SAlfredo Cardigliano 
86ce6427ddSThomas Monjalon void __rte_cold
ionic_dev_tx_queue_release(struct rte_eth_dev * dev,uint16_t qid)877483341aSXueming Li ionic_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
88a27d9013SAlfredo Cardigliano {
897483341aSXueming Li 	struct ionic_tx_qcq *txq = dev->data->tx_queues[qid];
90a27d9013SAlfredo Cardigliano 
91a27d9013SAlfredo Cardigliano 	IONIC_PRINT_CALL();
92a27d9013SAlfredo Cardigliano 
93be39f75cSAndrew Boyer 	ionic_qcq_free(&txq->qcq);
94a27d9013SAlfredo Cardigliano }
95a27d9013SAlfredo Cardigliano 
96ce6427ddSThomas Monjalon int __rte_cold
ionic_dev_tx_queue_stop(struct rte_eth_dev * dev,uint16_t tx_queue_id)977bb08900SAndrew Boyer ionic_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
98a27d9013SAlfredo Cardigliano {
997bb08900SAndrew Boyer 	ionic_dev_tx_queue_stop_firsthalf(dev, tx_queue_id);
1007bb08900SAndrew Boyer 	ionic_dev_tx_queue_stop_secondhalf(dev, tx_queue_id);
1017bb08900SAndrew Boyer 
1027bb08900SAndrew Boyer 	return 0;
1037bb08900SAndrew Boyer }
1047bb08900SAndrew Boyer 
1057bb08900SAndrew Boyer void __rte_cold
ionic_dev_tx_queue_stop_firsthalf(struct rte_eth_dev * dev,uint16_t tx_queue_id)1067bb08900SAndrew Boyer ionic_dev_tx_queue_stop_firsthalf(struct rte_eth_dev *dev,
1077bb08900SAndrew Boyer 				uint16_t tx_queue_id)
1087bb08900SAndrew Boyer {
1097bb08900SAndrew Boyer 	struct ionic_tx_qcq *txq = dev->data->tx_queues[tx_queue_id];
110a27d9013SAlfredo Cardigliano 
1114ae96cb8SAndrew Boyer 	IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id);
112a27d9013SAlfredo Cardigliano 
1137bb08900SAndrew Boyer 	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
114a27d9013SAlfredo Cardigliano 
1157bb08900SAndrew Boyer 	ionic_lif_txq_deinit_nowait(txq);
1167bb08900SAndrew Boyer }
1179fdf11c4SAndrew Boyer 
1187bb08900SAndrew Boyer void __rte_cold
ionic_dev_tx_queue_stop_secondhalf(struct rte_eth_dev * dev,uint16_t tx_queue_id)1197bb08900SAndrew Boyer ionic_dev_tx_queue_stop_secondhalf(struct rte_eth_dev *dev,
1207bb08900SAndrew Boyer 				uint16_t tx_queue_id)
1217bb08900SAndrew Boyer {
1227bb08900SAndrew Boyer 	struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(dev);
1237bb08900SAndrew Boyer 	struct ionic_tx_qcq *txq = dev->data->tx_queues[tx_queue_id];
124a27d9013SAlfredo Cardigliano 
1257bb08900SAndrew Boyer 	ionic_adminq_wait(lif, &txq->admin_ctx);
126a27d9013SAlfredo Cardigliano 
127e7222f94SAndrew Boyer 	/* Free all buffers from descriptor ring */
128e7222f94SAndrew Boyer 	ionic_tx_empty(txq);
129e7222f94SAndrew Boyer 
1307bb08900SAndrew Boyer 	ionic_lif_txq_stats(txq);
131a27d9013SAlfredo Cardigliano }
132a27d9013SAlfredo Cardigliano 
133ce6427ddSThomas Monjalon int __rte_cold
ionic_dev_tx_queue_setup(struct rte_eth_dev * eth_dev,uint16_t tx_queue_id,uint16_t nb_desc,uint32_t socket_id,const struct rte_eth_txconf * tx_conf)134a27d9013SAlfredo Cardigliano ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id,
1354ae96cb8SAndrew Boyer 		uint16_t nb_desc, uint32_t socket_id,
136a27d9013SAlfredo Cardigliano 		const struct rte_eth_txconf *tx_conf)
137a27d9013SAlfredo Cardigliano {
138a27d9013SAlfredo Cardigliano 	struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
139be39f75cSAndrew Boyer 	struct ionic_tx_qcq *txq;
140a27d9013SAlfredo Cardigliano 	uint64_t offloads;
141a27d9013SAlfredo Cardigliano 	int err;
142a27d9013SAlfredo Cardigliano 
143a27d9013SAlfredo Cardigliano 	if (tx_queue_id >= lif->ntxqcqs) {
144a27d9013SAlfredo Cardigliano 		IONIC_PRINT(DEBUG, "Queue index %u not available "
145a27d9013SAlfredo Cardigliano 			"(max %u queues)",
146a27d9013SAlfredo Cardigliano 			tx_queue_id, lif->ntxqcqs);
147a27d9013SAlfredo Cardigliano 		return -EINVAL;
148a27d9013SAlfredo Cardigliano 	}
149a27d9013SAlfredo Cardigliano 
150a27d9013SAlfredo Cardigliano 	offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads;
1514ae96cb8SAndrew Boyer 	IONIC_PRINT(DEBUG,
1524ae96cb8SAndrew Boyer 		"Configuring skt %u TX queue %u with %u buffers, offloads %jx",
1534ae96cb8SAndrew Boyer 		socket_id, tx_queue_id, nb_desc, offloads);
154a27d9013SAlfredo Cardigliano 
155a27d9013SAlfredo Cardigliano 	/* Validate number of receive descriptors */
156a27d9013SAlfredo Cardigliano 	if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC)
157a27d9013SAlfredo Cardigliano 		return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
158a27d9013SAlfredo Cardigliano 
15907512941SAndrew Boyer 	if (tx_conf->tx_free_thresh > nb_desc) {
16007512941SAndrew Boyer 		IONIC_PRINT(ERR,
16107512941SAndrew Boyer 			"tx_free_thresh must be less than nb_desc (%u)",
16207512941SAndrew Boyer 			nb_desc);
16307512941SAndrew Boyer 		return -EINVAL;
16407512941SAndrew Boyer 	}
16507512941SAndrew Boyer 
166a27d9013SAlfredo Cardigliano 	/* Free memory prior to re-allocation if needed... */
167a27d9013SAlfredo Cardigliano 	if (eth_dev->data->tx_queues[tx_queue_id] != NULL) {
1687483341aSXueming Li 		ionic_dev_tx_queue_release(eth_dev, tx_queue_id);
169a27d9013SAlfredo Cardigliano 		eth_dev->data->tx_queues[tx_queue_id] = NULL;
170a27d9013SAlfredo Cardigliano 	}
171a27d9013SAlfredo Cardigliano 
1729fdf11c4SAndrew Boyer 	eth_dev->data->tx_queue_state[tx_queue_id] =
1739fdf11c4SAndrew Boyer 		RTE_ETH_QUEUE_STATE_STOPPED;
1749fdf11c4SAndrew Boyer 
1758ec5ad7fSAndrew Boyer 	err = ionic_tx_qcq_alloc(lif, socket_id, tx_queue_id, nb_desc, &txq);
176a27d9013SAlfredo Cardigliano 	if (err) {
177a27d9013SAlfredo Cardigliano 		IONIC_PRINT(DEBUG, "Queue allocation failure");
178a27d9013SAlfredo Cardigliano 		return -EINVAL;
179a27d9013SAlfredo Cardigliano 	}
180a27d9013SAlfredo Cardigliano 
181a27d9013SAlfredo Cardigliano 	/* Do not start queue with rte_eth_dev_start() */
18202eabf57SAndrew Boyer 	if (tx_conf->tx_deferred_start)
18302eabf57SAndrew Boyer 		txq->flags |= IONIC_QCQ_F_DEFERRED;
184a27d9013SAlfredo Cardigliano 
18568591087SAndrew Boyer 	/* Convert the offload flags into queue flags */
186295968d1SFerruh Yigit 	if (offloads & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM)
18768591087SAndrew Boyer 		txq->flags |= IONIC_QCQ_F_CSUM_L3;
188295968d1SFerruh Yigit 	if (offloads & RTE_ETH_TX_OFFLOAD_TCP_CKSUM)
18968591087SAndrew Boyer 		txq->flags |= IONIC_QCQ_F_CSUM_TCP;
190295968d1SFerruh Yigit 	if (offloads & RTE_ETH_TX_OFFLOAD_UDP_CKSUM)
19168591087SAndrew Boyer 		txq->flags |= IONIC_QCQ_F_CSUM_UDP;
1929ac234eeSAndrew Boyer 	if (offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)
1939ac234eeSAndrew Boyer 		txq->flags |= IONIC_QCQ_F_FAST_FREE;
194a27d9013SAlfredo Cardigliano 
19507512941SAndrew Boyer 	txq->free_thresh =
19607512941SAndrew Boyer 		tx_conf->tx_free_thresh ? tx_conf->tx_free_thresh :
19707512941SAndrew Boyer 		nb_desc - IONIC_DEF_TXRX_BURST;
19807512941SAndrew Boyer 
199a27d9013SAlfredo Cardigliano 	eth_dev->data->tx_queues[tx_queue_id] = txq;
200a27d9013SAlfredo Cardigliano 
201a27d9013SAlfredo Cardigliano 	return 0;
202a27d9013SAlfredo Cardigliano }
203a27d9013SAlfredo Cardigliano 
204a27d9013SAlfredo Cardigliano /*
205a27d9013SAlfredo Cardigliano  * Start Transmit Units for specified queue.
206a27d9013SAlfredo Cardigliano  */
207ce6427ddSThomas Monjalon int __rte_cold
ionic_dev_tx_queue_start(struct rte_eth_dev * dev,uint16_t tx_queue_id)2080033e92fSAndrew Boyer ionic_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
209a27d9013SAlfredo Cardigliano {
210a27d9013SAlfredo Cardigliano 	int err;
211a27d9013SAlfredo Cardigliano 
2120033e92fSAndrew Boyer 	err = ionic_dev_tx_queue_start_firsthalf(dev, tx_queue_id);
2130033e92fSAndrew Boyer 	if (err)
2140033e92fSAndrew Boyer 		return err;
2150033e92fSAndrew Boyer 
2160033e92fSAndrew Boyer 	return ionic_dev_tx_queue_start_secondhalf(dev, tx_queue_id);
2170033e92fSAndrew Boyer }
2180033e92fSAndrew Boyer 
2190033e92fSAndrew Boyer int __rte_cold
ionic_dev_tx_queue_start_firsthalf(struct rte_eth_dev * dev,uint16_t tx_queue_id)2200033e92fSAndrew Boyer ionic_dev_tx_queue_start_firsthalf(struct rte_eth_dev *dev,
2210033e92fSAndrew Boyer 				uint16_t tx_queue_id)
2220033e92fSAndrew Boyer {
2230033e92fSAndrew Boyer 	uint8_t *tx_queue_state = dev->data->tx_queue_state;
2240033e92fSAndrew Boyer 	struct ionic_tx_qcq *txq = dev->data->tx_queues[tx_queue_id];
2250033e92fSAndrew Boyer 
2269fdf11c4SAndrew Boyer 	if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
2279fdf11c4SAndrew Boyer 		IONIC_PRINT(DEBUG, "TX queue %u already started",
2289fdf11c4SAndrew Boyer 			tx_queue_id);
2299fdf11c4SAndrew Boyer 		return 0;
2309fdf11c4SAndrew Boyer 	}
2319fdf11c4SAndrew Boyer 
2324ae96cb8SAndrew Boyer 	IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs",
233be39f75cSAndrew Boyer 		tx_queue_id, txq->qcq.q.num_descs);
2344ae96cb8SAndrew Boyer 
2350033e92fSAndrew Boyer 	return ionic_lif_txq_init_nowait(txq);
2360033e92fSAndrew Boyer }
2370033e92fSAndrew Boyer 
2380033e92fSAndrew Boyer int __rte_cold
ionic_dev_tx_queue_start_secondhalf(struct rte_eth_dev * dev,uint16_t tx_queue_id)2390033e92fSAndrew Boyer ionic_dev_tx_queue_start_secondhalf(struct rte_eth_dev *dev,
2400033e92fSAndrew Boyer 				uint16_t tx_queue_id)
2410033e92fSAndrew Boyer {
2420033e92fSAndrew Boyer 	uint8_t *tx_queue_state = dev->data->tx_queue_state;
2430033e92fSAndrew Boyer 	struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(dev);
2440033e92fSAndrew Boyer 	struct ionic_tx_qcq *txq = dev->data->tx_queues[tx_queue_id];
2450033e92fSAndrew Boyer 	int err;
2460033e92fSAndrew Boyer 
2470033e92fSAndrew Boyer 	if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED)
2480033e92fSAndrew Boyer 		return 0;
2490033e92fSAndrew Boyer 
2500033e92fSAndrew Boyer 	err = ionic_adminq_wait(lif, &txq->admin_ctx);
251a27d9013SAlfredo Cardigliano 	if (err)
252a27d9013SAlfredo Cardigliano 		return err;
253a27d9013SAlfredo Cardigliano 
2540033e92fSAndrew Boyer 	ionic_lif_txq_init_done(txq);
2550033e92fSAndrew Boyer 
2569fdf11c4SAndrew Boyer 	tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
257a27d9013SAlfredo Cardigliano 
258a27d9013SAlfredo Cardigliano 	return 0;
259a27d9013SAlfredo Cardigliano }
260a27d9013SAlfredo Cardigliano 
261a27d9013SAlfredo Cardigliano static void
ionic_tx_tcp_pseudo_csum(struct rte_mbuf * txm)26264b08152SAlfredo Cardigliano ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm)
26364b08152SAlfredo Cardigliano {
26464b08152SAlfredo Cardigliano 	struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
26564b08152SAlfredo Cardigliano 	char *l3_hdr = ((char *)eth_hdr) + txm->l2_len;
26664b08152SAlfredo Cardigliano 	struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
26764b08152SAlfredo Cardigliano 		(l3_hdr + txm->l3_len);
26864b08152SAlfredo Cardigliano 
269daa02b5cSOlivier Matz 	if (txm->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
27064b08152SAlfredo Cardigliano 		struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
27164b08152SAlfredo Cardigliano 		ipv4_hdr->hdr_checksum = 0;
27264b08152SAlfredo Cardigliano 		tcp_hdr->cksum = 0;
27364b08152SAlfredo Cardigliano 		tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
27464b08152SAlfredo Cardigliano 	} else {
27564b08152SAlfredo Cardigliano 		struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
27664b08152SAlfredo Cardigliano 		tcp_hdr->cksum = 0;
27764b08152SAlfredo Cardigliano 		tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
27864b08152SAlfredo Cardigliano 	}
27964b08152SAlfredo Cardigliano }
28064b08152SAlfredo Cardigliano 
28164b08152SAlfredo Cardigliano static void
ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf * txm)28264b08152SAlfredo Cardigliano ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm)
28364b08152SAlfredo Cardigliano {
28464b08152SAlfredo Cardigliano 	struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
28564b08152SAlfredo Cardigliano 	char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len +
28664b08152SAlfredo Cardigliano 		txm->outer_l3_len + txm->l2_len;
28764b08152SAlfredo Cardigliano 	struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
28864b08152SAlfredo Cardigliano 		(l3_hdr + txm->l3_len);
28964b08152SAlfredo Cardigliano 
290daa02b5cSOlivier Matz 	if (txm->ol_flags & RTE_MBUF_F_TX_IPV4) {
29164b08152SAlfredo Cardigliano 		struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
29264b08152SAlfredo Cardigliano 		ipv4_hdr->hdr_checksum = 0;
29364b08152SAlfredo Cardigliano 		tcp_hdr->cksum = 0;
29464b08152SAlfredo Cardigliano 		tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
29564b08152SAlfredo Cardigliano 	} else {
29664b08152SAlfredo Cardigliano 		struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
29764b08152SAlfredo Cardigliano 		tcp_hdr->cksum = 0;
29864b08152SAlfredo Cardigliano 		tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
29964b08152SAlfredo Cardigliano 	}
30064b08152SAlfredo Cardigliano }
30164b08152SAlfredo Cardigliano 
30264b08152SAlfredo Cardigliano static void
ionic_tx_tso_post(struct ionic_queue * q,struct ionic_txq_desc * desc,struct rte_mbuf * txm,rte_iova_t addr,uint8_t nsge,uint16_t len,uint32_t hdrlen,uint32_t mss,bool encap,uint16_t vlan_tci,bool has_vlan,bool start,bool done)303a27d9013SAlfredo Cardigliano ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
304a27d9013SAlfredo Cardigliano 		struct rte_mbuf *txm,
305a27d9013SAlfredo Cardigliano 		rte_iova_t addr, uint8_t nsge, uint16_t len,
306a27d9013SAlfredo Cardigliano 		uint32_t hdrlen, uint32_t mss,
30764b08152SAlfredo Cardigliano 		bool encap,
308a27d9013SAlfredo Cardigliano 		uint16_t vlan_tci, bool has_vlan,
309a27d9013SAlfredo Cardigliano 		bool start, bool done)
310a27d9013SAlfredo Cardigliano {
311b4beb84aSAndrew Boyer 	struct rte_mbuf *txm_seg;
312dd10c5b4SAndrew Boyer 	void **info;
3134a735599SAndrew Boyer 	uint64_t cmd;
314a27d9013SAlfredo Cardigliano 	uint8_t flags = 0;
315b4beb84aSAndrew Boyer 	int i;
316b4beb84aSAndrew Boyer 
317a27d9013SAlfredo Cardigliano 	flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
31864b08152SAlfredo Cardigliano 	flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
319a27d9013SAlfredo Cardigliano 	flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
320a27d9013SAlfredo Cardigliano 	flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
321a27d9013SAlfredo Cardigliano 
3224a735599SAndrew Boyer 	cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO,
323a27d9013SAlfredo Cardigliano 		flags, nsge, addr);
3244a735599SAndrew Boyer 	desc->cmd = rte_cpu_to_le_64(cmd);
3254a735599SAndrew Boyer 	desc->len = rte_cpu_to_le_16(len);
3264a735599SAndrew Boyer 	desc->vlan_tci = rte_cpu_to_le_16(vlan_tci);
3274a735599SAndrew Boyer 	desc->hdr_len = rte_cpu_to_le_16(hdrlen);
3284a735599SAndrew Boyer 	desc->mss = rte_cpu_to_le_16(mss);
329a27d9013SAlfredo Cardigliano 
330dd10c5b4SAndrew Boyer 	if (done) {
331dd10c5b4SAndrew Boyer 		info = IONIC_INFO_PTR(q, q->head_idx);
332b4beb84aSAndrew Boyer 
333b4beb84aSAndrew Boyer 		/* Walk the mbuf chain to stash pointers in the array */
334b4beb84aSAndrew Boyer 		txm_seg = txm;
335b4beb84aSAndrew Boyer 		for (i = 0; i < txm->nb_segs; i++) {
336b4beb84aSAndrew Boyer 			info[i] = txm_seg;
337b4beb84aSAndrew Boyer 			txm_seg = txm_seg->next;
338b4beb84aSAndrew Boyer 		}
339dd10c5b4SAndrew Boyer 	}
340dd10c5b4SAndrew Boyer 
341dd10c5b4SAndrew Boyer 	q->head_idx = Q_NEXT_TO_POST(q, 1);
342a27d9013SAlfredo Cardigliano }
343a27d9013SAlfredo Cardigliano 
344a27d9013SAlfredo Cardigliano static struct ionic_txq_desc *
ionic_tx_tso_next(struct ionic_tx_qcq * txq,struct ionic_txq_sg_elem ** elem)345be39f75cSAndrew Boyer ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem)
346a27d9013SAlfredo Cardigliano {
347be39f75cSAndrew Boyer 	struct ionic_queue *q = &txq->qcq.q;
348a27d9013SAlfredo Cardigliano 	struct ionic_txq_desc *desc_base = q->base;
34956117636SAndrew Boyer 	struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;
350a27d9013SAlfredo Cardigliano 	struct ionic_txq_desc *desc = &desc_base[q->head_idx];
35156117636SAndrew Boyer 	struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx];
352a27d9013SAlfredo Cardigliano 
353a27d9013SAlfredo Cardigliano 	*elem = sg_desc->elems;
354a27d9013SAlfredo Cardigliano 	return desc;
355a27d9013SAlfredo Cardigliano }
356a27d9013SAlfredo Cardigliano 
357e86a6fccSAndrew Boyer int
ionic_tx_tso(struct ionic_tx_qcq * txq,struct rte_mbuf * txm)35877c60793SAndrew Boyer ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
359a27d9013SAlfredo Cardigliano {
360be39f75cSAndrew Boyer 	struct ionic_queue *q = &txq->qcq.q;
361be39f75cSAndrew Boyer 	struct ionic_tx_stats *stats = &txq->stats;
362a27d9013SAlfredo Cardigliano 	struct ionic_txq_desc *desc;
363a27d9013SAlfredo Cardigliano 	struct ionic_txq_sg_elem *elem;
364a27d9013SAlfredo Cardigliano 	struct rte_mbuf *txm_seg;
3657c3a867bSAndrew Boyer 	rte_iova_t data_iova;
3667c3a867bSAndrew Boyer 	uint64_t desc_addr = 0, next_addr;
367a27d9013SAlfredo Cardigliano 	uint16_t desc_len = 0;
368e86a6fccSAndrew Boyer 	uint8_t desc_nsge = 0;
369a27d9013SAlfredo Cardigliano 	uint32_t hdrlen;
370a27d9013SAlfredo Cardigliano 	uint32_t mss = txm->tso_segsz;
371a27d9013SAlfredo Cardigliano 	uint32_t frag_left = 0;
372a27d9013SAlfredo Cardigliano 	uint32_t left;
373a27d9013SAlfredo Cardigliano 	uint32_t seglen;
374a27d9013SAlfredo Cardigliano 	uint32_t len;
375a27d9013SAlfredo Cardigliano 	uint32_t offset = 0;
376a27d9013SAlfredo Cardigliano 	bool start, done;
37764b08152SAlfredo Cardigliano 	bool encap;
378daa02b5cSOlivier Matz 	bool has_vlan = !!(txm->ol_flags & RTE_MBUF_F_TX_VLAN);
379e86a6fccSAndrew Boyer 	bool use_sgl = !!(txq->flags & IONIC_QCQ_F_SG);
380a27d9013SAlfredo Cardigliano 	uint16_t vlan_tci = txm->vlan_tci;
38164b08152SAlfredo Cardigliano 	uint64_t ol_flags = txm->ol_flags;
382a27d9013SAlfredo Cardigliano 
383daa02b5cSOlivier Matz 	encap = ((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) ||
384daa02b5cSOlivier Matz 		 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) &&
385daa02b5cSOlivier Matz 		((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) ||
386daa02b5cSOlivier Matz 		 (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6));
38764b08152SAlfredo Cardigliano 
38864b08152SAlfredo Cardigliano 	/* Preload inner-most TCP csum field with IP pseudo hdr
38964b08152SAlfredo Cardigliano 	 * calculated with IP length set to zero.  HW will later
39064b08152SAlfredo Cardigliano 	 * add in length to each TCP segment resulting from the TSO.
39164b08152SAlfredo Cardigliano 	 */
39264b08152SAlfredo Cardigliano 
39364b08152SAlfredo Cardigliano 	if (encap) {
39464b08152SAlfredo Cardigliano 		ionic_tx_tcp_inner_pseudo_csum(txm);
39564b08152SAlfredo Cardigliano 		hdrlen = txm->outer_l2_len + txm->outer_l3_len +
39664b08152SAlfredo Cardigliano 			txm->l2_len + txm->l3_len + txm->l4_len;
39764b08152SAlfredo Cardigliano 	} else {
39864b08152SAlfredo Cardigliano 		ionic_tx_tcp_pseudo_csum(txm);
39964b08152SAlfredo Cardigliano 		hdrlen = txm->l2_len + txm->l3_len + txm->l4_len;
40064b08152SAlfredo Cardigliano 	}
401a27d9013SAlfredo Cardigliano 
402be39f75cSAndrew Boyer 	desc = ionic_tx_tso_next(txq, &elem);
403e86a6fccSAndrew Boyer 	txm_seg = txm;
404a27d9013SAlfredo Cardigliano 	start = true;
405e86a6fccSAndrew Boyer 	seglen = hdrlen + mss;
406a27d9013SAlfredo Cardigliano 
407e86a6fccSAndrew Boyer 	/* Walk the chain of mbufs */
408a27d9013SAlfredo Cardigliano 	while (txm_seg != NULL) {
409a27d9013SAlfredo Cardigliano 		offset = 0;
4107c3a867bSAndrew Boyer 		data_iova = rte_mbuf_data_iova(txm_seg);
411a27d9013SAlfredo Cardigliano 		left = txm_seg->data_len;
412a27d9013SAlfredo Cardigliano 
413e86a6fccSAndrew Boyer 		/* Split the mbuf data up into multiple descriptors */
414a27d9013SAlfredo Cardigliano 		while (left > 0) {
4157c3a867bSAndrew Boyer 			next_addr = rte_cpu_to_le_64(data_iova + offset);
416e86a6fccSAndrew Boyer 			if (frag_left > 0 && use_sgl) {
417e86a6fccSAndrew Boyer 				/* Fill previous descriptor's SGE */
418a27d9013SAlfredo Cardigliano 				len = RTE_MIN(frag_left, left);
419a27d9013SAlfredo Cardigliano 				frag_left -= len;
4207c3a867bSAndrew Boyer 				elem->addr = next_addr;
4214a735599SAndrew Boyer 				elem->len = rte_cpu_to_le_16(len);
422a27d9013SAlfredo Cardigliano 				elem++;
423a27d9013SAlfredo Cardigliano 				desc_nsge++;
424a27d9013SAlfredo Cardigliano 			} else {
425e86a6fccSAndrew Boyer 				/* Fill new descriptor's data field */
426e86a6fccSAndrew Boyer 				len = RTE_MIN(seglen, left);
427e86a6fccSAndrew Boyer 				frag_left = seglen - len;
4287c3a867bSAndrew Boyer 				desc_addr = next_addr;
429a27d9013SAlfredo Cardigliano 				desc_len = len;
430a27d9013SAlfredo Cardigliano 				desc_nsge = 0;
431a27d9013SAlfredo Cardigliano 			}
432a27d9013SAlfredo Cardigliano 			left -= len;
433a27d9013SAlfredo Cardigliano 			offset += len;
434e86a6fccSAndrew Boyer 
435e86a6fccSAndrew Boyer 			/* Pack the next mbuf's data into the descriptor */
436e86a6fccSAndrew Boyer 			if (txm_seg->next != NULL && frag_left > 0 && use_sgl)
437e86a6fccSAndrew Boyer 				break;
4387c3a867bSAndrew Boyer 
439a27d9013SAlfredo Cardigliano 			done = (txm_seg->next == NULL && left == 0);
440a27d9013SAlfredo Cardigliano 			ionic_tx_tso_post(q, desc, txm_seg,
441a27d9013SAlfredo Cardigliano 				desc_addr, desc_nsge, desc_len,
442a27d9013SAlfredo Cardigliano 				hdrlen, mss,
44364b08152SAlfredo Cardigliano 				encap,
444a27d9013SAlfredo Cardigliano 				vlan_tci, has_vlan,
44577c60793SAndrew Boyer 				start, done);
446be39f75cSAndrew Boyer 			desc = ionic_tx_tso_next(txq, &elem);
447a27d9013SAlfredo Cardigliano 			start = false;
448e86a6fccSAndrew Boyer 			seglen = mss;
449a27d9013SAlfredo Cardigliano 		}
450a27d9013SAlfredo Cardigliano 
451a27d9013SAlfredo Cardigliano 		txm_seg = txm_seg->next;
452a27d9013SAlfredo Cardigliano 	}
453a27d9013SAlfredo Cardigliano 
454a27d9013SAlfredo Cardigliano 	stats->tso++;
455a27d9013SAlfredo Cardigliano 
456a27d9013SAlfredo Cardigliano 	return 0;
457a27d9013SAlfredo Cardigliano }
458a27d9013SAlfredo Cardigliano 
459a27d9013SAlfredo Cardigliano /*********************************************************************
460a27d9013SAlfredo Cardigliano  *
461a27d9013SAlfredo Cardigliano  *  TX prep functions
462a27d9013SAlfredo Cardigliano  *
463a27d9013SAlfredo Cardigliano  **********************************************************************/
464a27d9013SAlfredo Cardigliano 
465daa02b5cSOlivier Matz #define IONIC_TX_OFFLOAD_MASK (RTE_MBUF_F_TX_IPV4 |		\
466daa02b5cSOlivier Matz 	RTE_MBUF_F_TX_IPV6 |		\
467daa02b5cSOlivier Matz 	RTE_MBUF_F_TX_VLAN |		\
468daa02b5cSOlivier Matz 	RTE_MBUF_F_TX_IP_CKSUM |	\
469daa02b5cSOlivier Matz 	RTE_MBUF_F_TX_TCP_SEG |	\
470daa02b5cSOlivier Matz 	RTE_MBUF_F_TX_L4_MASK)
471a27d9013SAlfredo Cardigliano 
472a27d9013SAlfredo Cardigliano #define IONIC_TX_OFFLOAD_NOTSUP_MASK \
473daa02b5cSOlivier Matz 	(RTE_MBUF_F_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK)
474a27d9013SAlfredo Cardigliano 
475a27d9013SAlfredo Cardigliano uint16_t
ionic_prep_pkts(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)476e19eea1eSAndrew Boyer ionic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
477a27d9013SAlfredo Cardigliano {
478e19eea1eSAndrew Boyer 	struct ionic_tx_qcq *txq = tx_queue;
479a27d9013SAlfredo Cardigliano 	struct rte_mbuf *txm;
480a27d9013SAlfredo Cardigliano 	uint64_t offloads;
481a27d9013SAlfredo Cardigliano 	int i = 0;
482a27d9013SAlfredo Cardigliano 
483a27d9013SAlfredo Cardigliano 	for (i = 0; i < nb_pkts; i++) {
484a27d9013SAlfredo Cardigliano 		txm = tx_pkts[i];
485a27d9013SAlfredo Cardigliano 
486e19eea1eSAndrew Boyer 		if (txm->nb_segs > txq->num_segs_fw) {
487a27d9013SAlfredo Cardigliano 			rte_errno = -EINVAL;
488a27d9013SAlfredo Cardigliano 			break;
489a27d9013SAlfredo Cardigliano 		}
490a27d9013SAlfredo Cardigliano 
491a27d9013SAlfredo Cardigliano 		offloads = txm->ol_flags;
492a27d9013SAlfredo Cardigliano 
493a27d9013SAlfredo Cardigliano 		if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) {
494a27d9013SAlfredo Cardigliano 			rte_errno = -ENOTSUP;
495a27d9013SAlfredo Cardigliano 			break;
496a27d9013SAlfredo Cardigliano 		}
497a27d9013SAlfredo Cardigliano 	}
498a27d9013SAlfredo Cardigliano 
499a27d9013SAlfredo Cardigliano 	return i;
500a27d9013SAlfredo Cardigliano }
501a27d9013SAlfredo Cardigliano 
502a27d9013SAlfredo Cardigliano /*********************************************************************
503a27d9013SAlfredo Cardigliano  *
504a27d9013SAlfredo Cardigliano  *  RX functions
505a27d9013SAlfredo Cardigliano  *
506a27d9013SAlfredo Cardigliano  **********************************************************************/
507a27d9013SAlfredo Cardigliano 
508a27d9013SAlfredo Cardigliano void
ionic_rxq_info_get(struct rte_eth_dev * dev,uint16_t queue_id,struct rte_eth_rxq_info * qinfo)509a27d9013SAlfredo Cardigliano ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
510a27d9013SAlfredo Cardigliano 		struct rte_eth_rxq_info *qinfo)
511a27d9013SAlfredo Cardigliano {
512be39f75cSAndrew Boyer 	struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id];
513be39f75cSAndrew Boyer 	struct ionic_queue *q = &rxq->qcq.q;
514a27d9013SAlfredo Cardigliano 
515a27d9013SAlfredo Cardigliano 	qinfo->mp = rxq->mb_pool;
516a27d9013SAlfredo Cardigliano 	qinfo->scattered_rx = dev->data->scattered_rx;
517a27d9013SAlfredo Cardigliano 	qinfo->nb_desc = q->num_descs;
51802eabf57SAndrew Boyer 	qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED;
51968591087SAndrew Boyer 	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
520a27d9013SAlfredo Cardigliano }
521a27d9013SAlfredo Cardigliano 
522ce6427ddSThomas Monjalon void __rte_cold
ionic_dev_rx_queue_release(struct rte_eth_dev * dev,uint16_t qid)5237483341aSXueming Li ionic_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
524a27d9013SAlfredo Cardigliano {
5257483341aSXueming Li 	struct ionic_rx_qcq *rxq = dev->data->rx_queues[qid];
526be39f75cSAndrew Boyer 
527be39f75cSAndrew Boyer 	if (!rxq)
528be39f75cSAndrew Boyer 		return;
529a27d9013SAlfredo Cardigliano 
530a27d9013SAlfredo Cardigliano 	IONIC_PRINT_CALL();
531a27d9013SAlfredo Cardigliano 
532be39f75cSAndrew Boyer 	ionic_qcq_free(&rxq->qcq);
533a27d9013SAlfredo Cardigliano }
534a27d9013SAlfredo Cardigliano 
535ce6427ddSThomas Monjalon int __rte_cold
ionic_dev_rx_queue_setup(struct rte_eth_dev * eth_dev,uint16_t rx_queue_id,uint16_t nb_desc,uint32_t socket_id,const struct rte_eth_rxconf * rx_conf,struct rte_mempool * mp)536a27d9013SAlfredo Cardigliano ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
537a27d9013SAlfredo Cardigliano 		uint16_t rx_queue_id,
538a27d9013SAlfredo Cardigliano 		uint16_t nb_desc,
5394ae96cb8SAndrew Boyer 		uint32_t socket_id,
540a27d9013SAlfredo Cardigliano 		const struct rte_eth_rxconf *rx_conf,
541a27d9013SAlfredo Cardigliano 		struct rte_mempool *mp)
542a27d9013SAlfredo Cardigliano {
543a27d9013SAlfredo Cardigliano 	struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
544be39f75cSAndrew Boyer 	struct ionic_rx_qcq *rxq;
545a27d9013SAlfredo Cardigliano 	uint64_t offloads;
546a27d9013SAlfredo Cardigliano 	int err;
547a27d9013SAlfredo Cardigliano 
548a27d9013SAlfredo Cardigliano 	if (rx_queue_id >= lif->nrxqcqs) {
549a27d9013SAlfredo Cardigliano 		IONIC_PRINT(ERR,
550a27d9013SAlfredo Cardigliano 			"Queue index %u not available (max %u queues)",
551a27d9013SAlfredo Cardigliano 			rx_queue_id, lif->nrxqcqs);
552a27d9013SAlfredo Cardigliano 		return -EINVAL;
553a27d9013SAlfredo Cardigliano 	}
554a27d9013SAlfredo Cardigliano 
555a27d9013SAlfredo Cardigliano 	offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads;
5564ae96cb8SAndrew Boyer 	IONIC_PRINT(DEBUG,
5574ae96cb8SAndrew Boyer 		"Configuring skt %u RX queue %u with %u buffers, offloads %jx",
5584ae96cb8SAndrew Boyer 		socket_id, rx_queue_id, nb_desc, offloads);
559a27d9013SAlfredo Cardigliano 
56018a44465SAndrew Boyer 	if (!rx_conf->rx_drop_en)
56118a44465SAndrew Boyer 		IONIC_PRINT(WARNING, "No-drop mode is not supported");
56218a44465SAndrew Boyer 
563a27d9013SAlfredo Cardigliano 	/* Validate number of receive descriptors */
564a27d9013SAlfredo Cardigliano 	if (!rte_is_power_of_2(nb_desc) ||
565a27d9013SAlfredo Cardigliano 			nb_desc < IONIC_MIN_RING_DESC ||
566a27d9013SAlfredo Cardigliano 			nb_desc > IONIC_MAX_RING_DESC) {
567a27d9013SAlfredo Cardigliano 		IONIC_PRINT(ERR,
5684ae96cb8SAndrew Boyer 			"Bad descriptor count (%u) for queue %u (min: %u)",
569a27d9013SAlfredo Cardigliano 			nb_desc, rx_queue_id, IONIC_MIN_RING_DESC);
570a27d9013SAlfredo Cardigliano 		return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
571a27d9013SAlfredo Cardigliano 	}
572a27d9013SAlfredo Cardigliano 
573a27d9013SAlfredo Cardigliano 	/* Free memory prior to re-allocation if needed... */
574a27d9013SAlfredo Cardigliano 	if (eth_dev->data->rx_queues[rx_queue_id] != NULL) {
5757483341aSXueming Li 		ionic_dev_rx_queue_release(eth_dev, rx_queue_id);
576a27d9013SAlfredo Cardigliano 		eth_dev->data->rx_queues[rx_queue_id] = NULL;
577a27d9013SAlfredo Cardigliano 	}
578a27d9013SAlfredo Cardigliano 
5799fdf11c4SAndrew Boyer 	eth_dev->data->rx_queue_state[rx_queue_id] =
5809fdf11c4SAndrew Boyer 		RTE_ETH_QUEUE_STATE_STOPPED;
5819fdf11c4SAndrew Boyer 
582d5850081SAndrew Boyer 	err = ionic_rx_qcq_alloc(lif, socket_id, rx_queue_id, nb_desc, mp,
583be39f75cSAndrew Boyer 			&rxq);
584a27d9013SAlfredo Cardigliano 	if (err) {
5854ae96cb8SAndrew Boyer 		IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id);
586a27d9013SAlfredo Cardigliano 		return -EINVAL;
587a27d9013SAlfredo Cardigliano 	}
588a27d9013SAlfredo Cardigliano 
589a27d9013SAlfredo Cardigliano 	rxq->mb_pool = mp;
590a5b1ffd8SAndrew Boyer 	rxq->wdog_ms = IONIC_Q_WDOG_MS;
591a27d9013SAlfredo Cardigliano 
592a27d9013SAlfredo Cardigliano 	/*
593a27d9013SAlfredo Cardigliano 	 * Note: the interface does not currently support
594295968d1SFerruh Yigit 	 * RTE_ETH_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN
595a27d9013SAlfredo Cardigliano 	 * when the adapter will be able to keep the CRC and subtract
596a27d9013SAlfredo Cardigliano 	 * it to the length for all received packets:
597a27d9013SAlfredo Cardigliano 	 * if (eth_dev->data->dev_conf.rxmode.offloads &
598295968d1SFerruh Yigit 	 *     RTE_ETH_RX_OFFLOAD_KEEP_CRC)
599a27d9013SAlfredo Cardigliano 	 *   rxq->crc_len = ETHER_CRC_LEN;
600a27d9013SAlfredo Cardigliano 	 */
601a27d9013SAlfredo Cardigliano 
602a27d9013SAlfredo Cardigliano 	/* Do not start queue with rte_eth_dev_start() */
60302eabf57SAndrew Boyer 	if (rx_conf->rx_deferred_start)
60402eabf57SAndrew Boyer 		rxq->flags |= IONIC_QCQ_F_DEFERRED;
605a27d9013SAlfredo Cardigliano 
606a27d9013SAlfredo Cardigliano 	eth_dev->data->rx_queues[rx_queue_id] = rxq;
607a27d9013SAlfredo Cardigliano 
608a27d9013SAlfredo Cardigliano 	return 0;
609a27d9013SAlfredo Cardigliano }
610a27d9013SAlfredo Cardigliano 
611bbdf955dSAndrew Boyer #define IONIC_CSUM_FLAG_MASK (IONIC_RXQ_COMP_CSUM_F_VLAN - 1)
61227595cd8STyler Retzlaff const alignas(RTE_CACHE_LINE_SIZE) uint64_t ionic_csum_flags[IONIC_CSUM_FLAG_MASK] = {
613bbdf955dSAndrew Boyer 	/* IP_BAD set */
614bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_IP_BAD] = RTE_MBUF_F_RX_IP_CKSUM_BAD,
615bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_IP_BAD | IONIC_RXQ_COMP_CSUM_F_TCP_OK] =
616bbdf955dSAndrew Boyer 			RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD,
617bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_IP_BAD | IONIC_RXQ_COMP_CSUM_F_TCP_BAD] =
618bbdf955dSAndrew Boyer 			RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD,
619bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_IP_BAD | IONIC_RXQ_COMP_CSUM_F_UDP_OK] =
620bbdf955dSAndrew Boyer 			RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD,
621bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_IP_BAD | IONIC_RXQ_COMP_CSUM_F_UDP_BAD] =
622bbdf955dSAndrew Boyer 			RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD,
623bbdf955dSAndrew Boyer 	/* IP_OK set */
624bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_IP_OK] = RTE_MBUF_F_RX_IP_CKSUM_GOOD,
625bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_IP_OK | IONIC_RXQ_COMP_CSUM_F_TCP_OK] =
626bbdf955dSAndrew Boyer 			RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD,
627bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_IP_OK | IONIC_RXQ_COMP_CSUM_F_TCP_BAD] =
628bbdf955dSAndrew Boyer 			RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD,
629bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_IP_OK | IONIC_RXQ_COMP_CSUM_F_UDP_OK] =
630bbdf955dSAndrew Boyer 			RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD,
631bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_IP_OK | IONIC_RXQ_COMP_CSUM_F_UDP_BAD] =
632bbdf955dSAndrew Boyer 			RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD,
633bbdf955dSAndrew Boyer 	/* No IP flag set */
634bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_TCP_OK] = RTE_MBUF_F_RX_L4_CKSUM_GOOD,
635bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_TCP_BAD] = RTE_MBUF_F_RX_L4_CKSUM_BAD,
636bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_UDP_OK] = RTE_MBUF_F_RX_L4_CKSUM_GOOD,
637bbdf955dSAndrew Boyer 	[IONIC_RXQ_COMP_CSUM_F_UDP_BAD] = RTE_MBUF_F_RX_L4_CKSUM_BAD,
638bbdf955dSAndrew Boyer };
639bbdf955dSAndrew Boyer 
64073b1c67eSAndrew Boyer /* RTE_PTYPE_UNKNOWN is 0x0 */
64127595cd8STyler Retzlaff const alignas(RTE_CACHE_LINE_SIZE) uint32_t ionic_ptype_table[IONIC_RXQ_COMP_PKT_TYPE_MASK] = {
64273b1c67eSAndrew Boyer 	[IONIC_PKT_TYPE_NON_IP]   = RTE_PTYPE_UNKNOWN,
64373b1c67eSAndrew Boyer 	[IONIC_PKT_TYPE_IPV4]     = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
64473b1c67eSAndrew Boyer 	[IONIC_PKT_TYPE_IPV4_TCP] =
64573b1c67eSAndrew Boyer 		RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP,
64673b1c67eSAndrew Boyer 	[IONIC_PKT_TYPE_IPV4_UDP] =
64773b1c67eSAndrew Boyer 		RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP,
64873b1c67eSAndrew Boyer 	[IONIC_PKT_TYPE_IPV6]     = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
64973b1c67eSAndrew Boyer 	[IONIC_PKT_TYPE_IPV6_TCP] =
65073b1c67eSAndrew Boyer 		RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP,
65173b1c67eSAndrew Boyer 	[IONIC_PKT_TYPE_IPV6_UDP] =
65273b1c67eSAndrew Boyer 		RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP,
65373b1c67eSAndrew Boyer };
65473b1c67eSAndrew Boyer 
655b5b56afdSAndrew Boyer const uint32_t *
ionic_dev_supported_ptypes_get(struct rte_eth_dev * dev __rte_unused,size_t * no_of_elements)656ba6a168aSSivaramakrishnan Venkat ionic_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused,
657ba6a168aSSivaramakrishnan Venkat 			       size_t *no_of_elements)
658b5b56afdSAndrew Boyer {
659b5b56afdSAndrew Boyer 	/* See ionic_ptype_table[] */
660b5b56afdSAndrew Boyer 	static const uint32_t ptypes[] = {
661b5b56afdSAndrew Boyer 		RTE_PTYPE_L2_ETHER,
662b5b56afdSAndrew Boyer 		RTE_PTYPE_L2_ETHER_TIMESYNC,
663b5b56afdSAndrew Boyer 		RTE_PTYPE_L2_ETHER_LLDP,
664b5b56afdSAndrew Boyer 		RTE_PTYPE_L2_ETHER_ARP,
665b5b56afdSAndrew Boyer 		RTE_PTYPE_L3_IPV4,
666b5b56afdSAndrew Boyer 		RTE_PTYPE_L3_IPV6,
667b5b56afdSAndrew Boyer 		RTE_PTYPE_L4_TCP,
668b5b56afdSAndrew Boyer 		RTE_PTYPE_L4_UDP,
669b5b56afdSAndrew Boyer 	};
670b5b56afdSAndrew Boyer 
671ba6a168aSSivaramakrishnan Venkat 	*no_of_elements = RTE_DIM(ptypes);
672b5b56afdSAndrew Boyer 	return ptypes;
673b5b56afdSAndrew Boyer }
674b5b56afdSAndrew Boyer 
6757b20fc2fSAndrew Boyer /*
6767b2eb674SAndrew Boyer  * Perform one-time initialization of descriptor fields
6777b2eb674SAndrew Boyer  * which will not change for the life of the queue.
6787b2eb674SAndrew Boyer  */
6797b2eb674SAndrew Boyer static void __rte_cold
ionic_rx_init_descriptors(struct ionic_rx_qcq * rxq)6807b2eb674SAndrew Boyer ionic_rx_init_descriptors(struct ionic_rx_qcq *rxq)
6817b2eb674SAndrew Boyer {
6827b2eb674SAndrew Boyer 	struct ionic_queue *q = &rxq->qcq.q;
6837b2eb674SAndrew Boyer 	struct ionic_rxq_desc *desc, *desc_base = q->base;
6847b2eb674SAndrew Boyer 	struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
6857b2eb674SAndrew Boyer 	uint32_t i, j;
6867b2eb674SAndrew Boyer 	uint8_t opcode;
6877b2eb674SAndrew Boyer 
6887b2eb674SAndrew Boyer 	opcode = (q->num_segs > 1) ?
6897b2eb674SAndrew Boyer 		IONIC_RXQ_DESC_OPCODE_SG : IONIC_RXQ_DESC_OPCODE_SIMPLE;
6907b2eb674SAndrew Boyer 
6917b2eb674SAndrew Boyer 	/*
6927b2eb674SAndrew Boyer 	 * NB: Only the first segment needs to leave headroom (hdr_seg_size).
6937b2eb674SAndrew Boyer 	 *     Later segments (seg_size) do not.
6947b2eb674SAndrew Boyer 	 */
6957b2eb674SAndrew Boyer 	for (i = 0; i < q->num_descs; i++) {
6967b2eb674SAndrew Boyer 		desc = &desc_base[i];
6977b2eb674SAndrew Boyer 		desc->len = rte_cpu_to_le_16(rxq->hdr_seg_size);
6987b2eb674SAndrew Boyer 		desc->opcode = opcode;
6997b2eb674SAndrew Boyer 
7007b2eb674SAndrew Boyer 		sg_desc = &sg_desc_base[i];
7017b2eb674SAndrew Boyer 		for (j = 0; j < q->num_segs - 1u; j++)
7027b2eb674SAndrew Boyer 			sg_desc->elems[j].len =
7037b2eb674SAndrew Boyer 				rte_cpu_to_le_16(rxq->seg_size);
7047b2eb674SAndrew Boyer 	}
7057b2eb674SAndrew Boyer }
7067b2eb674SAndrew Boyer 
7077b2eb674SAndrew Boyer /*
708a27d9013SAlfredo Cardigliano  * Start Receive Units for specified queue.
709a27d9013SAlfredo Cardigliano  */
710ce6427ddSThomas Monjalon int __rte_cold
ionic_dev_rx_queue_start(struct rte_eth_dev * dev,uint16_t rx_queue_id)7110033e92fSAndrew Boyer ionic_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
712a27d9013SAlfredo Cardigliano {
713a27d9013SAlfredo Cardigliano 	int err;
714a27d9013SAlfredo Cardigliano 
7150033e92fSAndrew Boyer 	err = ionic_dev_rx_queue_start_firsthalf(dev, rx_queue_id);
7160033e92fSAndrew Boyer 	if (err)
7170033e92fSAndrew Boyer 		return err;
7180033e92fSAndrew Boyer 
7190033e92fSAndrew Boyer 	return ionic_dev_rx_queue_start_secondhalf(dev, rx_queue_id);
7200033e92fSAndrew Boyer }
7210033e92fSAndrew Boyer 
7220033e92fSAndrew Boyer int __rte_cold
ionic_dev_rx_queue_start_firsthalf(struct rte_eth_dev * dev,uint16_t rx_queue_id)7230033e92fSAndrew Boyer ionic_dev_rx_queue_start_firsthalf(struct rte_eth_dev *dev,
7240033e92fSAndrew Boyer 				uint16_t rx_queue_id)
7250033e92fSAndrew Boyer {
7260033e92fSAndrew Boyer 	uint8_t *rx_queue_state = dev->data->rx_queue_state;
7270033e92fSAndrew Boyer 	struct ionic_rx_qcq *rxq = dev->data->rx_queues[rx_queue_id];
7280033e92fSAndrew Boyer 	struct ionic_queue *q = &rxq->qcq.q;
7290033e92fSAndrew Boyer 
7309fdf11c4SAndrew Boyer 	if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
7319fdf11c4SAndrew Boyer 		IONIC_PRINT(DEBUG, "RX queue %u already started",
7329fdf11c4SAndrew Boyer 			rx_queue_id);
7339fdf11c4SAndrew Boyer 		return 0;
7349fdf11c4SAndrew Boyer 	}
7359fdf11c4SAndrew Boyer 
736b671e69aSAndrew Boyer 	rxq->frame_size = rxq->qcq.lif->frame_size - RTE_ETHER_CRC_LEN;
737b671e69aSAndrew Boyer 
738d5850081SAndrew Boyer 	/* Recalculate segment count based on MTU */
739d5850081SAndrew Boyer 	q->num_segs = 1 +
740d5850081SAndrew Boyer 		(rxq->frame_size + RTE_PKTMBUF_HEADROOM - 1) / rxq->seg_size;
741d5850081SAndrew Boyer 
742d5850081SAndrew Boyer 	IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs, size %u segs %u",
743d5850081SAndrew Boyer 		rx_queue_id, q->num_descs, rxq->frame_size, q->num_segs);
7444ae96cb8SAndrew Boyer 
7457b2eb674SAndrew Boyer 	ionic_rx_init_descriptors(rxq);
7467b2eb674SAndrew Boyer 
7470033e92fSAndrew Boyer 	return ionic_lif_rxq_init_nowait(rxq);
7480033e92fSAndrew Boyer }
7490033e92fSAndrew Boyer 
7500033e92fSAndrew Boyer int __rte_cold
ionic_dev_rx_queue_start_secondhalf(struct rte_eth_dev * dev,uint16_t rx_queue_id)7510033e92fSAndrew Boyer ionic_dev_rx_queue_start_secondhalf(struct rte_eth_dev *dev,
7520033e92fSAndrew Boyer 				uint16_t rx_queue_id)
7530033e92fSAndrew Boyer {
7540033e92fSAndrew Boyer 	uint8_t *rx_queue_state = dev->data->rx_queue_state;
7550033e92fSAndrew Boyer 	struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(dev);
7560033e92fSAndrew Boyer 	struct ionic_rx_qcq *rxq = dev->data->rx_queues[rx_queue_id];
7570033e92fSAndrew Boyer 	int err;
7580033e92fSAndrew Boyer 
7590033e92fSAndrew Boyer 	if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED)
7600033e92fSAndrew Boyer 		return 0;
7610033e92fSAndrew Boyer 
7620033e92fSAndrew Boyer 	err = ionic_adminq_wait(lif, &rxq->admin_ctx);
763a27d9013SAlfredo Cardigliano 	if (err)
764a27d9013SAlfredo Cardigliano 		return err;
765a27d9013SAlfredo Cardigliano 
7660033e92fSAndrew Boyer 	ionic_lif_rxq_init_done(rxq);
7670033e92fSAndrew Boyer 
768e86a6fccSAndrew Boyer 	/* Allocate buffers for descriptor ring */
769e86a6fccSAndrew Boyer 	if (rxq->flags & IONIC_QCQ_F_SG)
770e86a6fccSAndrew Boyer 		err = ionic_rx_fill_sg(rxq);
771e86a6fccSAndrew Boyer 	else
772e86a6fccSAndrew Boyer 		err = ionic_rx_fill(rxq);
773e86a6fccSAndrew Boyer 	if (err != 0) {
774e86a6fccSAndrew Boyer 		IONIC_PRINT(ERR, "Could not fill queue %d", rx_queue_id);
775a27d9013SAlfredo Cardigliano 		return -1;
776a27d9013SAlfredo Cardigliano 	}
777a27d9013SAlfredo Cardigliano 
7789fdf11c4SAndrew Boyer 	rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
779a27d9013SAlfredo Cardigliano 
780a27d9013SAlfredo Cardigliano 	return 0;
781a27d9013SAlfredo Cardigliano }
782a27d9013SAlfredo Cardigliano 
7837b20fc2fSAndrew Boyer /*
784a27d9013SAlfredo Cardigliano  * Stop Receive Units for specified queue.
785a27d9013SAlfredo Cardigliano  */
786ce6427ddSThomas Monjalon int __rte_cold
ionic_dev_rx_queue_stop(struct rte_eth_dev * dev,uint16_t rx_queue_id)7877bb08900SAndrew Boyer ionic_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
788a27d9013SAlfredo Cardigliano {
7897bb08900SAndrew Boyer 	ionic_dev_rx_queue_stop_firsthalf(dev, rx_queue_id);
7907bb08900SAndrew Boyer 	ionic_dev_rx_queue_stop_secondhalf(dev, rx_queue_id);
7917bb08900SAndrew Boyer 
7927bb08900SAndrew Boyer 	return 0;
7937bb08900SAndrew Boyer }
7947bb08900SAndrew Boyer 
7957bb08900SAndrew Boyer void __rte_cold
ionic_dev_rx_queue_stop_firsthalf(struct rte_eth_dev * dev,uint16_t rx_queue_id)7967bb08900SAndrew Boyer ionic_dev_rx_queue_stop_firsthalf(struct rte_eth_dev *dev,
7977bb08900SAndrew Boyer 				uint16_t rx_queue_id)
7987bb08900SAndrew Boyer {
7997bb08900SAndrew Boyer 	struct ionic_rx_qcq *rxq = dev->data->rx_queues[rx_queue_id];
800a27d9013SAlfredo Cardigliano 
8014ae96cb8SAndrew Boyer 	IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id);
802a27d9013SAlfredo Cardigliano 
8037bb08900SAndrew Boyer 	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
804a27d9013SAlfredo Cardigliano 
8057bb08900SAndrew Boyer 	ionic_lif_rxq_deinit_nowait(rxq);
8067bb08900SAndrew Boyer }
8079fdf11c4SAndrew Boyer 
8087bb08900SAndrew Boyer void __rte_cold
ionic_dev_rx_queue_stop_secondhalf(struct rte_eth_dev * dev,uint16_t rx_queue_id)8097bb08900SAndrew Boyer ionic_dev_rx_queue_stop_secondhalf(struct rte_eth_dev *dev,
8107bb08900SAndrew Boyer 				uint16_t rx_queue_id)
8117bb08900SAndrew Boyer {
8127bb08900SAndrew Boyer 	struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(dev);
8137bb08900SAndrew Boyer 	struct ionic_rx_qcq *rxq = dev->data->rx_queues[rx_queue_id];
8147bb08900SAndrew Boyer 
8157bb08900SAndrew Boyer 	ionic_adminq_wait(lif, &rxq->admin_ctx);
816a27d9013SAlfredo Cardigliano 
817e7222f94SAndrew Boyer 	/* Free all buffers from descriptor ring */
818e7222f94SAndrew Boyer 	ionic_rx_empty(rxq);
819e7222f94SAndrew Boyer 
8207bb08900SAndrew Boyer 	ionic_lif_rxq_stats(rxq);
821a27d9013SAlfredo Cardigliano }
822a27d9013SAlfredo Cardigliano 
8230983a74aSAndrew Boyer int
ionic_dev_rx_descriptor_status(void * rx_queue,uint16_t offset)8240983a74aSAndrew Boyer ionic_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
8250983a74aSAndrew Boyer {
8260983a74aSAndrew Boyer 	struct ionic_rx_qcq *rxq = rx_queue;
8270983a74aSAndrew Boyer 	struct ionic_qcq *qcq = &rxq->qcq;
828463ad260SNeel Patel 	volatile struct ionic_rxq_comp *cq_desc;
8290983a74aSAndrew Boyer 	uint16_t mask, head, tail, pos;
8300983a74aSAndrew Boyer 	bool done_color;
8310983a74aSAndrew Boyer 
8320983a74aSAndrew Boyer 	mask = qcq->q.size_mask;
8330983a74aSAndrew Boyer 
8340983a74aSAndrew Boyer 	/* offset must be within the size of the ring */
8350983a74aSAndrew Boyer 	if (offset > mask)
8360983a74aSAndrew Boyer 		return -EINVAL;
8370983a74aSAndrew Boyer 
8380983a74aSAndrew Boyer 	head = qcq->q.head_idx;
8390983a74aSAndrew Boyer 	tail = qcq->q.tail_idx;
8400983a74aSAndrew Boyer 
8410983a74aSAndrew Boyer 	/* offset is beyond what is posted */
8420983a74aSAndrew Boyer 	if (offset >= ((head - tail) & mask))
8430983a74aSAndrew Boyer 		return RTE_ETH_RX_DESC_UNAVAIL;
8440983a74aSAndrew Boyer 
8450983a74aSAndrew Boyer 	/* interested in this absolute position in the rxq */
8460983a74aSAndrew Boyer 	pos = (tail + offset) & mask;
8470983a74aSAndrew Boyer 
8480983a74aSAndrew Boyer 	/* rx cq position == rx q position */
8490983a74aSAndrew Boyer 	cq_desc = qcq->cq.base;
8500983a74aSAndrew Boyer 	cq_desc = &cq_desc[pos];
8510983a74aSAndrew Boyer 
8520983a74aSAndrew Boyer 	/* expected done color at this position */
8530983a74aSAndrew Boyer 	done_color = qcq->cq.done_color != (pos < tail);
8540983a74aSAndrew Boyer 
8550983a74aSAndrew Boyer 	/* has the hw indicated the done color at this position? */
8560983a74aSAndrew Boyer 	if (color_match(cq_desc->pkt_type_color, done_color))
8570983a74aSAndrew Boyer 		return RTE_ETH_RX_DESC_DONE;
8580983a74aSAndrew Boyer 
8590983a74aSAndrew Boyer 	return RTE_ETH_RX_DESC_AVAIL;
8600983a74aSAndrew Boyer }
86160625147SAndrew Boyer 
86260625147SAndrew Boyer int
ionic_dev_tx_descriptor_status(void * tx_queue,uint16_t offset)86360625147SAndrew Boyer ionic_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
86460625147SAndrew Boyer {
86560625147SAndrew Boyer 	struct ionic_tx_qcq *txq = tx_queue;
86660625147SAndrew Boyer 	struct ionic_qcq *qcq = &txq->qcq;
867463ad260SNeel Patel 	volatile struct ionic_txq_comp *cq_desc;
86860625147SAndrew Boyer 	uint16_t mask, head, tail, pos, cq_pos;
86960625147SAndrew Boyer 	bool done_color;
87060625147SAndrew Boyer 
87160625147SAndrew Boyer 	mask = qcq->q.size_mask;
87260625147SAndrew Boyer 
87360625147SAndrew Boyer 	/* offset must be within the size of the ring */
87460625147SAndrew Boyer 	if (offset > mask)
87560625147SAndrew Boyer 		return -EINVAL;
87660625147SAndrew Boyer 
87760625147SAndrew Boyer 	head = qcq->q.head_idx;
87860625147SAndrew Boyer 	tail = qcq->q.tail_idx;
87960625147SAndrew Boyer 
88060625147SAndrew Boyer 	/* offset is beyond what is posted */
88160625147SAndrew Boyer 	if (offset >= ((head - tail) & mask))
88260625147SAndrew Boyer 		return RTE_ETH_TX_DESC_DONE;
88360625147SAndrew Boyer 
88460625147SAndrew Boyer 	/* interested in this absolute position in the txq */
88560625147SAndrew Boyer 	pos = (tail + offset) & mask;
88660625147SAndrew Boyer 
88760625147SAndrew Boyer 	/* tx cq position != tx q position, need to walk cq */
88860625147SAndrew Boyer 	cq_pos = qcq->cq.tail_idx;
88960625147SAndrew Boyer 	cq_desc = qcq->cq.base;
89060625147SAndrew Boyer 	cq_desc = &cq_desc[cq_pos];
89160625147SAndrew Boyer 
89260625147SAndrew Boyer 	/* how far behind is pos from head? */
89360625147SAndrew Boyer 	offset = (head - pos) & mask;
89460625147SAndrew Boyer 
89560625147SAndrew Boyer 	/* walk cq descriptors that match the expected done color */
89660625147SAndrew Boyer 	done_color = qcq->cq.done_color;
89760625147SAndrew Boyer 	while (color_match(cq_desc->color, done_color)) {
89860625147SAndrew Boyer 		/* is comp index no further behind than pos? */
89960625147SAndrew Boyer 		tail = rte_cpu_to_le_16(cq_desc->comp_index);
90060625147SAndrew Boyer 		if (((head - tail) & mask) <= offset)
90160625147SAndrew Boyer 			return RTE_ETH_TX_DESC_DONE;
90260625147SAndrew Boyer 
90360625147SAndrew Boyer 		cq_pos = (cq_pos + 1) & mask;
90460625147SAndrew Boyer 		cq_desc = qcq->cq.base;
90560625147SAndrew Boyer 		cq_desc = &cq_desc[cq_pos];
90660625147SAndrew Boyer 
90760625147SAndrew Boyer 		done_color = done_color != (cq_pos == 0);
90860625147SAndrew Boyer 	}
90960625147SAndrew Boyer 
91060625147SAndrew Boyer 	return RTE_ETH_TX_DESC_FULL;
91160625147SAndrew Boyer }
912