xref: /dpdk/drivers/net/dpaa/dpaa_rxtx.c (revision e7524271c3984ae3a77d42c7ea1df78ddcc89a5f)
1d81734caSHemant Agrawal /* SPDX-License-Identifier: BSD-3-Clause
237f9b54bSShreyansh Jain  *
337f9b54bSShreyansh Jain  *   Copyright 2016 Freescale Semiconductor, Inc. All rights reserved.
4f191d5abSHemant Agrawal  *   Copyright 2017,2019-2021 NXP
537f9b54bSShreyansh Jain  *
637f9b54bSShreyansh Jain  */
737f9b54bSShreyansh Jain 
837f9b54bSShreyansh Jain /* System headers */
937f9b54bSShreyansh Jain #include <inttypes.h>
1037f9b54bSShreyansh Jain #include <unistd.h>
1137f9b54bSShreyansh Jain #include <stdio.h>
1237f9b54bSShreyansh Jain #include <limits.h>
1337f9b54bSShreyansh Jain #include <sched.h>
1437f9b54bSShreyansh Jain #include <pthread.h>
1537f9b54bSShreyansh Jain 
1637f9b54bSShreyansh Jain #include <rte_byteorder.h>
1737f9b54bSShreyansh Jain #include <rte_common.h>
1837f9b54bSShreyansh Jain #include <rte_interrupts.h>
1937f9b54bSShreyansh Jain #include <rte_log.h>
2037f9b54bSShreyansh Jain #include <rte_debug.h>
2137f9b54bSShreyansh Jain #include <rte_pci.h>
2237f9b54bSShreyansh Jain #include <rte_atomic.h>
2337f9b54bSShreyansh Jain #include <rte_branch_prediction.h>
2437f9b54bSShreyansh Jain #include <rte_memory.h>
2537f9b54bSShreyansh Jain #include <rte_tailq.h>
2637f9b54bSShreyansh Jain #include <rte_eal.h>
2737f9b54bSShreyansh Jain #include <rte_alarm.h>
2837f9b54bSShreyansh Jain #include <rte_ether.h>
29df96fd0dSBruce Richardson #include <ethdev_driver.h>
3037f9b54bSShreyansh Jain #include <rte_malloc.h>
3137f9b54bSShreyansh Jain #include <rte_ring.h>
3237f9b54bSShreyansh Jain #include <rte_ip.h>
3337f9b54bSShreyansh Jain #include <rte_tcp.h>
3437f9b54bSShreyansh Jain #include <rte_udp.h>
35d565c887SAshish Jain #include <rte_net.h>
365e745593SSunil Kumar Kori #include <rte_eventdev.h>
3737f9b54bSShreyansh Jain 
3837f9b54bSShreyansh Jain #include "dpaa_ethdev.h"
3937f9b54bSShreyansh Jain #include "dpaa_rxtx.h"
40a2f1da7dSDavid Marchand #include <bus_dpaa_driver.h>
4137f9b54bSShreyansh Jain #include <dpaa_mempool.h>
4237f9b54bSShreyansh Jain 
435e745593SSunil Kumar Kori #include <qman.h>
4437f9b54bSShreyansh Jain #include <fsl_usd.h>
4537f9b54bSShreyansh Jain #include <fsl_qman.h>
4637f9b54bSShreyansh Jain #include <fsl_bman.h>
478c83f28cSHemant Agrawal #include <dpaa_of.h>
4837f9b54bSShreyansh Jain #include <netcfg.h>
4937f9b54bSShreyansh Jain 
5037f9b54bSShreyansh Jain #define DPAA_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) \
5137f9b54bSShreyansh Jain 	do { \
5237f9b54bSShreyansh Jain 		(_fd)->cmd = 0; \
5337f9b54bSShreyansh Jain 		(_fd)->opaque_addr = 0; \
5437f9b54bSShreyansh Jain 		(_fd)->opaque = QM_FD_CONTIG << DPAA_FD_FORMAT_SHIFT; \
5537f9b54bSShreyansh Jain 		(_fd)->opaque |= ((_mbuf)->data_off) << DPAA_FD_OFFSET_SHIFT; \
5637f9b54bSShreyansh Jain 		(_fd)->opaque |= (_mbuf)->pkt_len; \
57455da545SSantosh Shukla 		(_fd)->addr = (_mbuf)->buf_iova; \
5837f9b54bSShreyansh Jain 		(_fd)->bpid = _bpid; \
5937f9b54bSShreyansh Jain 	} while (0)
6037f9b54bSShreyansh Jain 
6177393f56SSachin Saxena #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER
6277393f56SSachin Saxena #define DISPLAY_PRINT printf
6377393f56SSachin Saxena static void dpaa_display_frame_info(const struct qm_fd *fd,
6477393f56SSachin Saxena 			uint32_t fqid, bool rx)
6505ba55bcSShreyansh Jain {
6605ba55bcSShreyansh Jain 	int ii;
6705ba55bcSShreyansh Jain 	char *ptr;
6877393f56SSachin Saxena 	struct annotations_t *annot = rte_dpaa_mem_ptov(fd->addr);
6977393f56SSachin Saxena 	uint8_t format;
7005ba55bcSShreyansh Jain 
7177393f56SSachin Saxena 	if (!fd->status) {
7277393f56SSachin Saxena 		/* Do not display correct packets.*/
7377393f56SSachin Saxena 		return;
7405ba55bcSShreyansh Jain 	}
7577393f56SSachin Saxena 
7677393f56SSachin Saxena 	format = (fd->opaque & DPAA_FD_FORMAT_MASK) >>
7777393f56SSachin Saxena 				DPAA_FD_FORMAT_SHIFT;
7877393f56SSachin Saxena 
7977393f56SSachin Saxena 	DISPLAY_PRINT("fqid %d bpid %d addr 0x%lx, format %d\r\n",
8077393f56SSachin Saxena 		      fqid, fd->bpid, (unsigned long)fd->addr, fd->format);
8177393f56SSachin Saxena 	DISPLAY_PRINT("off %d, len %d stat 0x%x\r\n",
8277393f56SSachin Saxena 		      fd->offset, fd->length20, fd->status);
8377393f56SSachin Saxena 	if (rx) {
8477393f56SSachin Saxena 		ptr = (char *)&annot->parse;
8577393f56SSachin Saxena 		DISPLAY_PRINT("RX parser result:\r\n");
8677393f56SSachin Saxena 		for (ii = 0; ii < (int)sizeof(struct dpaa_eth_parse_results_t);
8777393f56SSachin Saxena 			ii++) {
8877393f56SSachin Saxena 			DISPLAY_PRINT("%02x ", ptr[ii]);
8977393f56SSachin Saxena 			if (((ii + 1) % 16) == 0)
9077393f56SSachin Saxena 				DISPLAY_PRINT("\n");
9177393f56SSachin Saxena 		}
9277393f56SSachin Saxena 		DISPLAY_PRINT("\n");
9377393f56SSachin Saxena 	}
9477393f56SSachin Saxena 
9577393f56SSachin Saxena 	if (unlikely(format == qm_fd_sg)) {
9677393f56SSachin Saxena 		/*TBD:S/G display: to be implemented*/
9777393f56SSachin Saxena 		return;
9877393f56SSachin Saxena 	}
9977393f56SSachin Saxena 
10077393f56SSachin Saxena 	DISPLAY_PRINT("Frame payload:\r\n");
10177393f56SSachin Saxena 	ptr = (char *)annot;
10277393f56SSachin Saxena 	ptr += fd->offset;
10377393f56SSachin Saxena 	for (ii = 0; ii < fd->length20; ii++) {
10477393f56SSachin Saxena 		DISPLAY_PRINT("%02x ", ptr[ii]);
10577393f56SSachin Saxena 		if (((ii + 1) % 16) == 0)
10605ba55bcSShreyansh Jain 			printf("\n");
10705ba55bcSShreyansh Jain 	}
10877393f56SSachin Saxena 	DISPLAY_PRINT("\n");
10977393f56SSachin Saxena }
11005ba55bcSShreyansh Jain #else
11177393f56SSachin Saxena #define dpaa_display_frame_info(a, b, c)
11205ba55bcSShreyansh Jain #endif
11305ba55bcSShreyansh Jain 
114a7bdc3bdSShreyansh Jain static inline void dpaa_slow_parsing(struct rte_mbuf *m __rte_unused,
115a7bdc3bdSShreyansh Jain 				     uint64_t prs __rte_unused)
116a7bdc3bdSShreyansh Jain {
117a7bdc3bdSShreyansh Jain 	DPAA_DP_LOG(DEBUG, "Slow parsing");
118a7bdc3bdSShreyansh Jain 	/*TBD:XXX: to be implemented*/
119a7bdc3bdSShreyansh Jain }
120a7bdc3bdSShreyansh Jain 
1210e5607e4SHemant Agrawal static inline void dpaa_eth_packet_info(struct rte_mbuf *m, void *fd_virt_addr)
122a7bdc3bdSShreyansh Jain {
123a7bdc3bdSShreyansh Jain 	struct annotations_t *annot = GET_ANNOTATIONS(fd_virt_addr);
1240e5607e4SHemant Agrawal 	uint64_t prs = *((uintptr_t *)(&annot->parse)) & DPAA_PARSE_MASK;
125a7bdc3bdSShreyansh Jain 
126a7bdc3bdSShreyansh Jain 	DPAA_DP_LOG(DEBUG, " Parsing mbuf: %p with annotations: %p", m, annot);
127a7bdc3bdSShreyansh Jain 
128daa02b5cSOlivier Matz 	m->ol_flags = RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_IP_CKSUM_GOOD |
129daa02b5cSOlivier Matz 		RTE_MBUF_F_RX_L4_CKSUM_GOOD;
13095d226f0SNipun Gupta 
131a7bdc3bdSShreyansh Jain 	switch (prs) {
132a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4:
133a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
134a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV4;
135a7bdc3bdSShreyansh Jain 		break;
136a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6:
137a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
138a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV6;
139a7bdc3bdSShreyansh Jain 		break;
1409ac71da4SNipun Gupta 	case DPAA_PKT_TYPE_ETHER:
1419ac71da4SNipun Gupta 		m->packet_type = RTE_PTYPE_L2_ETHER;
1429ac71da4SNipun Gupta 		break;
143a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4_FRAG:
144a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4_FRAG_UDP:
145a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4_FRAG_TCP:
146a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4_FRAG_SCTP:
147a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
148a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG;
149a7bdc3bdSShreyansh Jain 		break;
150a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6_FRAG:
151a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6_FRAG_UDP:
152a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6_FRAG_TCP:
153a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6_FRAG_SCTP:
154a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
155a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG;
156a7bdc3bdSShreyansh Jain 		break;
157a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4_EXT:
158a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
159a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV4_EXT;
160a7bdc3bdSShreyansh Jain 		break;
161a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6_EXT:
162a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
163a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV6_EXT;
164a7bdc3bdSShreyansh Jain 		break;
165a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4_TCP:
166a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
167a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP;
168a7bdc3bdSShreyansh Jain 		break;
169a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6_TCP:
170a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
171a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP;
172a7bdc3bdSShreyansh Jain 		break;
173a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4_UDP:
174a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
175a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP;
176a7bdc3bdSShreyansh Jain 		break;
177a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6_UDP:
178a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
179a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP;
180a7bdc3bdSShreyansh Jain 		break;
181*e7524271SGagandeep Singh 	case DPAA_PKT_TYPE_IPSEC_IPV4:
182*e7524271SGagandeep Singh 		if (*((uintptr_t *)&annot->parse) & DPAA_PARSE_ESP_MASK)
183*e7524271SGagandeep Singh 			m->packet_type = RTE_PTYPE_L2_ETHER |
184*e7524271SGagandeep Singh 				RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_ESP;
185*e7524271SGagandeep Singh 		break;
186*e7524271SGagandeep Singh 	case DPAA_PKT_TYPE_IPSEC_IPV6:
187*e7524271SGagandeep Singh 		if (*((uintptr_t *)&annot->parse) & DPAA_PARSE_ESP_MASK)
188*e7524271SGagandeep Singh 			m->packet_type = RTE_PTYPE_L2_ETHER |
189*e7524271SGagandeep Singh 				RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_ESP;
190*e7524271SGagandeep Singh 		break;
191a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4_EXT_UDP:
192a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
193a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP;
194a7bdc3bdSShreyansh Jain 		break;
195a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6_EXT_UDP:
196a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
197a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP;
198a7bdc3bdSShreyansh Jain 		break;
199a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4_EXT_TCP:
200a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
201a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP;
202a7bdc3bdSShreyansh Jain 		break;
203a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6_EXT_TCP:
204a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
205a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP;
206a7bdc3bdSShreyansh Jain 		break;
207a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV4_SCTP:
208a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
209a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP;
210a7bdc3bdSShreyansh Jain 		break;
211a7bdc3bdSShreyansh Jain 	case DPAA_PKT_TYPE_IPV6_SCTP:
212a7bdc3bdSShreyansh Jain 		m->packet_type = RTE_PTYPE_L2_ETHER |
213a7bdc3bdSShreyansh Jain 			RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP;
214a7bdc3bdSShreyansh Jain 		break;
21595d226f0SNipun Gupta 	case DPAA_PKT_TYPE_IPV4_CSUM_ERR:
21695d226f0SNipun Gupta 	case DPAA_PKT_TYPE_IPV6_CSUM_ERR:
217daa02b5cSOlivier Matz 		m->ol_flags = RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_IP_CKSUM_BAD;
21895d226f0SNipun Gupta 		break;
21995d226f0SNipun Gupta 	case DPAA_PKT_TYPE_IPV4_TCP_CSUM_ERR:
22095d226f0SNipun Gupta 	case DPAA_PKT_TYPE_IPV6_TCP_CSUM_ERR:
22195d226f0SNipun Gupta 	case DPAA_PKT_TYPE_IPV4_UDP_CSUM_ERR:
22295d226f0SNipun Gupta 	case DPAA_PKT_TYPE_IPV6_UDP_CSUM_ERR:
223daa02b5cSOlivier Matz 		m->ol_flags = RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_L4_CKSUM_BAD;
22495d226f0SNipun Gupta 		break;
2259ac71da4SNipun Gupta 	case DPAA_PKT_TYPE_NONE:
2269ac71da4SNipun Gupta 		m->packet_type = 0;
2279ac71da4SNipun Gupta 		break;
228a7bdc3bdSShreyansh Jain 	/* More switch cases can be added */
229a7bdc3bdSShreyansh Jain 	default:
230a7bdc3bdSShreyansh Jain 		dpaa_slow_parsing(m, prs);
231a7bdc3bdSShreyansh Jain 	}
232a7bdc3bdSShreyansh Jain 
233a7bdc3bdSShreyansh Jain 	m->tx_offload = annot->parse.ip_off[0];
234a7bdc3bdSShreyansh Jain 	m->tx_offload |= (annot->parse.l4_off - annot->parse.ip_off[0])
235a7bdc3bdSShreyansh Jain 					<< DPAA_PKT_L3_LEN_SHIFT;
236a7bdc3bdSShreyansh Jain 
237a7bdc3bdSShreyansh Jain 	/* Set the hash values */
2389ac71da4SNipun Gupta 	m->hash.rss = (uint32_t)(annot->hash);
239a7bdc3bdSShreyansh Jain 
240a7bdc3bdSShreyansh Jain 	/* Check if Vlan is present */
241a7bdc3bdSShreyansh Jain 	if (prs & DPAA_PARSE_VLAN_MASK)
242daa02b5cSOlivier Matz 		m->ol_flags |= RTE_MBUF_F_RX_VLAN;
243a7bdc3bdSShreyansh Jain 	/* Packet received without stripping the vlan */
244a7bdc3bdSShreyansh Jain }
245a7bdc3bdSShreyansh Jain 
2465a8cf1beSShreyansh Jain static inline void dpaa_checksum(struct rte_mbuf *mbuf)
2475a8cf1beSShreyansh Jain {
2486d13ea8eSOlivier Matz 	struct rte_ether_hdr *eth_hdr =
2496d13ea8eSOlivier Matz 		rte_pktmbuf_mtod(mbuf, struct rte_ether_hdr *);
2505a8cf1beSShreyansh Jain 	char *l3_hdr = (char *)eth_hdr + mbuf->l2_len;
251a7c528e5SOlivier Matz 	struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
252a7c528e5SOlivier Matz 	struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
2535a8cf1beSShreyansh Jain 
2545a8cf1beSShreyansh Jain 	DPAA_DP_LOG(DEBUG, "Calculating checksum for mbuf: %p", mbuf);
2555a8cf1beSShreyansh Jain 
2565a8cf1beSShreyansh Jain 	if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) ||
2575a8cf1beSShreyansh Jain 	    ((mbuf->packet_type & RTE_PTYPE_L3_MASK) ==
2585a8cf1beSShreyansh Jain 	    RTE_PTYPE_L3_IPV4_EXT)) {
259a7c528e5SOlivier Matz 		ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
2605a8cf1beSShreyansh Jain 		ipv4_hdr->hdr_checksum = 0;
2615a8cf1beSShreyansh Jain 		ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr);
2625a8cf1beSShreyansh Jain 	} else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) ==
2635a8cf1beSShreyansh Jain 		   RTE_PTYPE_L3_IPV6) ||
2645a8cf1beSShreyansh Jain 		   ((mbuf->packet_type & RTE_PTYPE_L3_MASK) ==
2655a8cf1beSShreyansh Jain 		   RTE_PTYPE_L3_IPV6_EXT))
266a7c528e5SOlivier Matz 		ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
2675a8cf1beSShreyansh Jain 
2685a8cf1beSShreyansh Jain 	if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) {
269f41b5156SOlivier Matz 		struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)(l3_hdr +
2705a8cf1beSShreyansh Jain 					  mbuf->l3_len);
2715a8cf1beSShreyansh Jain 		tcp_hdr->cksum = 0;
2720c9da755SDavid Marchand 		if (eth_hdr->ether_type == htons(RTE_ETHER_TYPE_IPV4))
2735a8cf1beSShreyansh Jain 			tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr,
2745a8cf1beSShreyansh Jain 							       tcp_hdr);
2750c9da755SDavid Marchand 		else /* assume ethertype == RTE_ETHER_TYPE_IPV6 */
2765a8cf1beSShreyansh Jain 			tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr,
2775a8cf1beSShreyansh Jain 							       tcp_hdr);
2785a8cf1beSShreyansh Jain 	} else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) ==
2795a8cf1beSShreyansh Jain 		   RTE_PTYPE_L4_UDP) {
280e73e3547SOlivier Matz 		struct rte_udp_hdr *udp_hdr = (struct rte_udp_hdr *)(l3_hdr +
2815a8cf1beSShreyansh Jain 							     mbuf->l3_len);
2825a8cf1beSShreyansh Jain 		udp_hdr->dgram_cksum = 0;
2830c9da755SDavid Marchand 		if (eth_hdr->ether_type == htons(RTE_ETHER_TYPE_IPV4))
2845a8cf1beSShreyansh Jain 			udp_hdr->dgram_cksum = rte_ipv4_udptcp_cksum(ipv4_hdr,
2855a8cf1beSShreyansh Jain 								     udp_hdr);
2860c9da755SDavid Marchand 		else /* assume ethertype == RTE_ETHER_TYPE_IPV6 */
2875a8cf1beSShreyansh Jain 			udp_hdr->dgram_cksum = rte_ipv6_udptcp_cksum(ipv6_hdr,
2885a8cf1beSShreyansh Jain 								     udp_hdr);
2895a8cf1beSShreyansh Jain 	}
2905a8cf1beSShreyansh Jain }
2915a8cf1beSShreyansh Jain 
2925a8cf1beSShreyansh Jain static inline void dpaa_checksum_offload(struct rte_mbuf *mbuf,
2935a8cf1beSShreyansh Jain 					 struct qm_fd *fd, char *prs_buf)
2945a8cf1beSShreyansh Jain {
2955a8cf1beSShreyansh Jain 	struct dpaa_eth_parse_results_t *prs;
2965a8cf1beSShreyansh Jain 
2975a8cf1beSShreyansh Jain 	DPAA_DP_LOG(DEBUG, " Offloading checksum for mbuf: %p", mbuf);
2985a8cf1beSShreyansh Jain 
2995a8cf1beSShreyansh Jain 	prs = GET_TX_PRS(prs_buf);
3005a8cf1beSShreyansh Jain 	prs->l3r = 0;
3015a8cf1beSShreyansh Jain 	prs->l4r = 0;
3025a8cf1beSShreyansh Jain 	if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) ||
3035a8cf1beSShreyansh Jain 	   ((mbuf->packet_type & RTE_PTYPE_L3_MASK) ==
3045a8cf1beSShreyansh Jain 	   RTE_PTYPE_L3_IPV4_EXT))
3055a8cf1beSShreyansh Jain 		prs->l3r = DPAA_L3_PARSE_RESULT_IPV4;
3065a8cf1beSShreyansh Jain 	else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) ==
3075a8cf1beSShreyansh Jain 		   RTE_PTYPE_L3_IPV6) ||
3085a8cf1beSShreyansh Jain 		 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) ==
3095a8cf1beSShreyansh Jain 		RTE_PTYPE_L3_IPV6_EXT))
3105a8cf1beSShreyansh Jain 		prs->l3r = DPAA_L3_PARSE_RESULT_IPV6;
3115a8cf1beSShreyansh Jain 
3125a8cf1beSShreyansh Jain 	if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP)
3135a8cf1beSShreyansh Jain 		prs->l4r = DPAA_L4_PARSE_RESULT_TCP;
3145a8cf1beSShreyansh Jain 	else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP)
3155a8cf1beSShreyansh Jain 		prs->l4r = DPAA_L4_PARSE_RESULT_UDP;
3165a8cf1beSShreyansh Jain 
3175a8cf1beSShreyansh Jain 	prs->ip_off[0] = mbuf->l2_len;
3185a8cf1beSShreyansh Jain 	prs->l4_off = mbuf->l3_len + mbuf->l2_len;
3195a8cf1beSShreyansh Jain 	/* Enable L3 (and L4, if TCP or UDP) HW checksum*/
3205a8cf1beSShreyansh Jain 	fd->cmd = DPAA_FD_CMD_RPD | DPAA_FD_CMD_DTC;
3215a8cf1beSShreyansh Jain }
3225a8cf1beSShreyansh Jain 
3235e0789e9SNipun Gupta static inline void
3245e0789e9SNipun Gupta dpaa_unsegmented_checksum(struct rte_mbuf *mbuf, struct qm_fd *fd_arr)
3255e0789e9SNipun Gupta {
3265e0789e9SNipun Gupta 	if (!mbuf->packet_type) {
3275e0789e9SNipun Gupta 		struct rte_net_hdr_lens hdr_lens;
3285e0789e9SNipun Gupta 
3295e0789e9SNipun Gupta 		mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens,
3305e0789e9SNipun Gupta 				RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK
3315e0789e9SNipun Gupta 				| RTE_PTYPE_L4_MASK);
3325e0789e9SNipun Gupta 		mbuf->l2_len = hdr_lens.l2_len;
3335e0789e9SNipun Gupta 		mbuf->l3_len = hdr_lens.l3_len;
3345e0789e9SNipun Gupta 	}
3355e0789e9SNipun Gupta 	if (mbuf->data_off < (DEFAULT_TX_ICEOF +
3365e0789e9SNipun Gupta 	    sizeof(struct dpaa_eth_parse_results_t))) {
3375e0789e9SNipun Gupta 		DPAA_DP_LOG(DEBUG, "Checksum offload Err: "
3385e0789e9SNipun Gupta 			"Not enough Headroom "
3395e0789e9SNipun Gupta 			"space for correct Checksum offload."
3405e0789e9SNipun Gupta 			"So Calculating checksum in Software.");
3415e0789e9SNipun Gupta 		dpaa_checksum(mbuf);
3425e0789e9SNipun Gupta 	} else {
3435e0789e9SNipun Gupta 		dpaa_checksum_offload(mbuf, fd_arr, mbuf->buf_addr);
3445e0789e9SNipun Gupta 	}
3455e0789e9SNipun Gupta }
3465e0789e9SNipun Gupta 
347f191d5abSHemant Agrawal static struct rte_mbuf *
3489ac71da4SNipun Gupta dpaa_eth_sg_to_mbuf(const struct qm_fd *fd, uint32_t ifid)
3498cffdcbeSShreyansh Jain {
3508cffdcbeSShreyansh Jain 	struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid);
3518cffdcbeSShreyansh Jain 	struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp;
3528cffdcbeSShreyansh Jain 	struct qm_sg_entry *sgt, *sg_temp;
3538cffdcbeSShreyansh Jain 	void *vaddr, *sg_vaddr;
3548cffdcbeSShreyansh Jain 	int i = 0;
355287f4256SNipun Gupta 	uint16_t fd_offset = fd->offset;
3568cffdcbeSShreyansh Jain 
35741c9ee8dSHemant Agrawal 	vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd));
3588cffdcbeSShreyansh Jain 	if (!vaddr) {
3598cffdcbeSShreyansh Jain 		DPAA_PMD_ERR("unable to convert physical address");
3608cffdcbeSShreyansh Jain 		return NULL;
3618cffdcbeSShreyansh Jain 	}
3628cffdcbeSShreyansh Jain 	sgt = vaddr + fd_offset;
3638cffdcbeSShreyansh Jain 	sg_temp = &sgt[i++];
3648cffdcbeSShreyansh Jain 	hw_sg_to_cpu(sg_temp);
3658cffdcbeSShreyansh Jain 	temp = (struct rte_mbuf *)((char *)vaddr - bp_info->meta_data_size);
36641c9ee8dSHemant Agrawal 	sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_sg_entry_get64(sg_temp));
3678cffdcbeSShreyansh Jain 
3688cffdcbeSShreyansh Jain 	first_seg = (struct rte_mbuf *)((char *)sg_vaddr -
3698cffdcbeSShreyansh Jain 						bp_info->meta_data_size);
3708cffdcbeSShreyansh Jain 	first_seg->data_off = sg_temp->offset;
3718cffdcbeSShreyansh Jain 	first_seg->data_len = sg_temp->length;
3728cffdcbeSShreyansh Jain 	first_seg->pkt_len = sg_temp->length;
3738cffdcbeSShreyansh Jain 	rte_mbuf_refcnt_set(first_seg, 1);
3748cffdcbeSShreyansh Jain 
3758cffdcbeSShreyansh Jain 	first_seg->port = ifid;
3768cffdcbeSShreyansh Jain 	first_seg->nb_segs = 1;
3778cffdcbeSShreyansh Jain 	first_seg->ol_flags = 0;
3788cffdcbeSShreyansh Jain 	prev_seg = first_seg;
3798cffdcbeSShreyansh Jain 	while (i < DPAA_SGT_MAX_ENTRIES) {
3808cffdcbeSShreyansh Jain 		sg_temp = &sgt[i++];
3818cffdcbeSShreyansh Jain 		hw_sg_to_cpu(sg_temp);
38241c9ee8dSHemant Agrawal 		sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info,
38341c9ee8dSHemant Agrawal 					     qm_sg_entry_get64(sg_temp));
3848cffdcbeSShreyansh Jain 		cur_seg = (struct rte_mbuf *)((char *)sg_vaddr -
3858cffdcbeSShreyansh Jain 						      bp_info->meta_data_size);
3868cffdcbeSShreyansh Jain 		cur_seg->data_off = sg_temp->offset;
3878cffdcbeSShreyansh Jain 		cur_seg->data_len = sg_temp->length;
3888cffdcbeSShreyansh Jain 		first_seg->pkt_len += sg_temp->length;
3898cffdcbeSShreyansh Jain 		first_seg->nb_segs += 1;
3908cffdcbeSShreyansh Jain 		rte_mbuf_refcnt_set(cur_seg, 1);
3918cffdcbeSShreyansh Jain 		prev_seg->next = cur_seg;
3928cffdcbeSShreyansh Jain 		if (sg_temp->final) {
3938cffdcbeSShreyansh Jain 			cur_seg->next = NULL;
3948cffdcbeSShreyansh Jain 			break;
3958cffdcbeSShreyansh Jain 		}
3968cffdcbeSShreyansh Jain 		prev_seg = cur_seg;
3978cffdcbeSShreyansh Jain 	}
39855576ac2SHemant Agrawal 	DPAA_DP_LOG(DEBUG, "Received an SG frame len =%d, num_sg =%d",
39955576ac2SHemant Agrawal 			first_seg->pkt_len, first_seg->nb_segs);
4008cffdcbeSShreyansh Jain 
4010e5607e4SHemant Agrawal 	dpaa_eth_packet_info(first_seg, vaddr);
4028cffdcbeSShreyansh Jain 	rte_pktmbuf_free_seg(temp);
4038cffdcbeSShreyansh Jain 
4048cffdcbeSShreyansh Jain 	return first_seg;
4058cffdcbeSShreyansh Jain }
4068cffdcbeSShreyansh Jain 
4079ac71da4SNipun Gupta static inline struct rte_mbuf *
4089ac71da4SNipun Gupta dpaa_eth_fd_to_mbuf(const struct qm_fd *fd, uint32_t ifid)
40937f9b54bSShreyansh Jain {
41037f9b54bSShreyansh Jain 	struct rte_mbuf *mbuf;
4119ac71da4SNipun Gupta 	struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid);
41241c9ee8dSHemant Agrawal 	void *ptr;
4138cffdcbeSShreyansh Jain 	uint8_t format =
4148cffdcbeSShreyansh Jain 		(fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT;
4159ac71da4SNipun Gupta 	uint16_t offset;
4169ac71da4SNipun Gupta 	uint32_t length;
41737f9b54bSShreyansh Jain 
4188cffdcbeSShreyansh Jain 	if (unlikely(format == qm_fd_sg))
4198cffdcbeSShreyansh Jain 		return dpaa_eth_sg_to_mbuf(fd, ifid);
4208cffdcbeSShreyansh Jain 
4219ac71da4SNipun Gupta 	offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT;
4229ac71da4SNipun Gupta 	length = fd->opaque & DPAA_FD_LENGTH_MASK;
4239ac71da4SNipun Gupta 
42455576ac2SHemant Agrawal 	DPAA_DP_LOG(DEBUG, " FD--->MBUF off %d len = %d", offset, length);
42555576ac2SHemant Agrawal 
42637f9b54bSShreyansh Jain 	/* Ignoring case when format != qm_fd_contig */
4271ee09e39SHemant Agrawal 	ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd));
42837f9b54bSShreyansh Jain 
42937f9b54bSShreyansh Jain 	mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size);
4301ee09e39SHemant Agrawal 	/* Prefetch the Parse results and packet data to L1 */
4311ee09e39SHemant Agrawal 	rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF));
43237f9b54bSShreyansh Jain 
43337f9b54bSShreyansh Jain 	mbuf->data_off = offset;
43437f9b54bSShreyansh Jain 	mbuf->data_len = length;
43537f9b54bSShreyansh Jain 	mbuf->pkt_len = length;
43637f9b54bSShreyansh Jain 
43737f9b54bSShreyansh Jain 	mbuf->port = ifid;
43837f9b54bSShreyansh Jain 	mbuf->nb_segs = 1;
43937f9b54bSShreyansh Jain 	mbuf->ol_flags = 0;
44037f9b54bSShreyansh Jain 	mbuf->next = NULL;
44137f9b54bSShreyansh Jain 	rte_mbuf_refcnt_set(mbuf, 1);
4420e5607e4SHemant Agrawal 	dpaa_eth_packet_info(mbuf, mbuf->buf_addr);
44337f9b54bSShreyansh Jain 
44437f9b54bSShreyansh Jain 	return mbuf;
44537f9b54bSShreyansh Jain }
44637f9b54bSShreyansh Jain 
4479124e65dSGagandeep Singh uint16_t
4489124e65dSGagandeep Singh dpaa_free_mbuf(const struct qm_fd *fd)
4499124e65dSGagandeep Singh {
4509124e65dSGagandeep Singh 	struct rte_mbuf *mbuf;
4519124e65dSGagandeep Singh 	struct dpaa_bp_info *bp_info;
4529124e65dSGagandeep Singh 	uint8_t format;
4539124e65dSGagandeep Singh 	void *ptr;
4549124e65dSGagandeep Singh 
4559124e65dSGagandeep Singh 	bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid);
4569124e65dSGagandeep Singh 	format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT;
4579124e65dSGagandeep Singh 	if (unlikely(format == qm_fd_sg)) {
4589124e65dSGagandeep Singh 		struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp;
4599124e65dSGagandeep Singh 		struct qm_sg_entry *sgt, *sg_temp;
4609124e65dSGagandeep Singh 		void *vaddr, *sg_vaddr;
4619124e65dSGagandeep Singh 		int i = 0;
4629124e65dSGagandeep Singh 		uint16_t fd_offset = fd->offset;
4639124e65dSGagandeep Singh 
4649124e65dSGagandeep Singh 		vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd));
4659124e65dSGagandeep Singh 		if (!vaddr) {
4669124e65dSGagandeep Singh 			DPAA_PMD_ERR("unable to convert physical address");
4679124e65dSGagandeep Singh 			return -1;
4689124e65dSGagandeep Singh 		}
4699124e65dSGagandeep Singh 		sgt = vaddr + fd_offset;
4709124e65dSGagandeep Singh 		sg_temp = &sgt[i++];
4719124e65dSGagandeep Singh 		hw_sg_to_cpu(sg_temp);
4729124e65dSGagandeep Singh 		temp = (struct rte_mbuf *)
4739124e65dSGagandeep Singh 			((char *)vaddr - bp_info->meta_data_size);
4749124e65dSGagandeep Singh 		sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info,
4759124e65dSGagandeep Singh 						qm_sg_entry_get64(sg_temp));
4769124e65dSGagandeep Singh 
4779124e65dSGagandeep Singh 		first_seg = (struct rte_mbuf *)((char *)sg_vaddr -
4789124e65dSGagandeep Singh 						bp_info->meta_data_size);
4799124e65dSGagandeep Singh 		first_seg->nb_segs = 1;
4809124e65dSGagandeep Singh 		prev_seg = first_seg;
4819124e65dSGagandeep Singh 		while (i < DPAA_SGT_MAX_ENTRIES) {
4829124e65dSGagandeep Singh 			sg_temp = &sgt[i++];
4839124e65dSGagandeep Singh 			hw_sg_to_cpu(sg_temp);
4849124e65dSGagandeep Singh 			sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info,
4859124e65dSGagandeep Singh 						qm_sg_entry_get64(sg_temp));
4869124e65dSGagandeep Singh 			cur_seg = (struct rte_mbuf *)((char *)sg_vaddr -
4879124e65dSGagandeep Singh 						      bp_info->meta_data_size);
4889124e65dSGagandeep Singh 			first_seg->nb_segs += 1;
4899124e65dSGagandeep Singh 			prev_seg->next = cur_seg;
4909124e65dSGagandeep Singh 			if (sg_temp->final) {
4919124e65dSGagandeep Singh 				cur_seg->next = NULL;
4929124e65dSGagandeep Singh 				break;
4939124e65dSGagandeep Singh 			}
4949124e65dSGagandeep Singh 			prev_seg = cur_seg;
4959124e65dSGagandeep Singh 		}
4969124e65dSGagandeep Singh 
4979124e65dSGagandeep Singh 		rte_pktmbuf_free_seg(temp);
4989124e65dSGagandeep Singh 		rte_pktmbuf_free_seg(first_seg);
4999124e65dSGagandeep Singh 		return 0;
5009124e65dSGagandeep Singh 	}
5019124e65dSGagandeep Singh 
5029124e65dSGagandeep Singh 	ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd));
5039124e65dSGagandeep Singh 	mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size);
5049124e65dSGagandeep Singh 
5059124e65dSGagandeep Singh 	rte_pktmbuf_free(mbuf);
5069124e65dSGagandeep Singh 
5079124e65dSGagandeep Singh 	return 0;
5089124e65dSGagandeep Singh }
5099124e65dSGagandeep Singh 
51019b4aba2SHemant Agrawal /* Specific for LS1043 */
511b9083ea5SNipun Gupta void
51219b4aba2SHemant Agrawal dpaa_rx_cb_no_prefetch(struct qman_fq **fq, struct qm_dqrr_entry **dqrr,
513b9083ea5SNipun Gupta 	   void **bufs, int num_bufs)
5140c504f69SHemant Agrawal {
515b9083ea5SNipun Gupta 	struct rte_mbuf *mbuf;
516b9083ea5SNipun Gupta 	struct dpaa_bp_info *bp_info;
517b9083ea5SNipun Gupta 	const struct qm_fd *fd;
518b9083ea5SNipun Gupta 	void *ptr;
519b9083ea5SNipun Gupta 	struct dpaa_if *dpaa_intf;
520b9083ea5SNipun Gupta 	uint16_t offset, i;
521b9083ea5SNipun Gupta 	uint32_t length;
522b9083ea5SNipun Gupta 	uint8_t format;
5230c504f69SHemant Agrawal 
524b9083ea5SNipun Gupta 	bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[0]->fd.bpid);
525b9083ea5SNipun Gupta 	ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[0]->fd));
526b9083ea5SNipun Gupta 	rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF));
52719b4aba2SHemant Agrawal 	bufs[0] = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size);
528b9083ea5SNipun Gupta 
529b9083ea5SNipun Gupta 	for (i = 0; i < num_bufs; i++) {
53019b4aba2SHemant Agrawal 		if (i < num_bufs - 1) {
531b9083ea5SNipun Gupta 			bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[i + 1]->fd.bpid);
532b9083ea5SNipun Gupta 			ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[i + 1]->fd));
533b9083ea5SNipun Gupta 			rte_prefetch0((void *)((uint8_t *)ptr +
534b9083ea5SNipun Gupta 					DEFAULT_RX_ICEOF));
535b9083ea5SNipun Gupta 			bufs[i + 1] = (struct rte_mbuf *)((char *)ptr -
536b9083ea5SNipun Gupta 					bp_info->meta_data_size);
537b9083ea5SNipun Gupta 		}
538b9083ea5SNipun Gupta 
539b9083ea5SNipun Gupta 		fd = &dqrr[i]->fd;
5409abdad12SHemant Agrawal 		dpaa_intf = fq[0]->dpaa_intf;
541b9083ea5SNipun Gupta 		format = (fd->opaque & DPAA_FD_FORMAT_MASK) >>
542b9083ea5SNipun Gupta 				DPAA_FD_FORMAT_SHIFT;
543b9083ea5SNipun Gupta 		if (unlikely(format == qm_fd_sg)) {
544b9083ea5SNipun Gupta 			bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid);
545b9083ea5SNipun Gupta 			continue;
546b9083ea5SNipun Gupta 		}
547b9083ea5SNipun Gupta 
548b9083ea5SNipun Gupta 		offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >>
549b9083ea5SNipun Gupta 				DPAA_FD_OFFSET_SHIFT;
550b9083ea5SNipun Gupta 		length = fd->opaque & DPAA_FD_LENGTH_MASK;
551b9083ea5SNipun Gupta 
552b9083ea5SNipun Gupta 		mbuf = bufs[i];
553b9083ea5SNipun Gupta 		mbuf->data_off = offset;
554b9083ea5SNipun Gupta 		mbuf->data_len = length;
555b9083ea5SNipun Gupta 		mbuf->pkt_len = length;
556b9083ea5SNipun Gupta 		mbuf->port = dpaa_intf->ifid;
557b9083ea5SNipun Gupta 
558b9083ea5SNipun Gupta 		mbuf->nb_segs = 1;
559b9083ea5SNipun Gupta 		mbuf->ol_flags = 0;
560b9083ea5SNipun Gupta 		mbuf->next = NULL;
561b9083ea5SNipun Gupta 		rte_mbuf_refcnt_set(mbuf, 1);
5620e5607e4SHemant Agrawal 		dpaa_eth_packet_info(mbuf, mbuf->buf_addr);
56377393f56SSachin Saxena 		dpaa_display_frame_info(fd, fq[0]->fqid, true);
564b9083ea5SNipun Gupta 	}
565b9083ea5SNipun Gupta }
566b9083ea5SNipun Gupta 
56719b4aba2SHemant Agrawal void
56819b4aba2SHemant Agrawal dpaa_rx_cb(struct qman_fq **fq, struct qm_dqrr_entry **dqrr,
56919b4aba2SHemant Agrawal 	   void **bufs, int num_bufs)
57019b4aba2SHemant Agrawal {
57119b4aba2SHemant Agrawal 	struct rte_mbuf *mbuf;
57219b4aba2SHemant Agrawal 	const struct qm_fd *fd;
57319b4aba2SHemant Agrawal 	struct dpaa_if *dpaa_intf;
57419b4aba2SHemant Agrawal 	uint16_t offset, i;
57519b4aba2SHemant Agrawal 	uint32_t length;
57619b4aba2SHemant Agrawal 	uint8_t format;
57719b4aba2SHemant Agrawal 
57819b4aba2SHemant Agrawal 	for (i = 0; i < num_bufs; i++) {
57919b4aba2SHemant Agrawal 		fd = &dqrr[i]->fd;
58019b4aba2SHemant Agrawal 		dpaa_intf = fq[0]->dpaa_intf;
58119b4aba2SHemant Agrawal 		format = (fd->opaque & DPAA_FD_FORMAT_MASK) >>
58219b4aba2SHemant Agrawal 				DPAA_FD_FORMAT_SHIFT;
58319b4aba2SHemant Agrawal 		if (unlikely(format == qm_fd_sg)) {
58419b4aba2SHemant Agrawal 			bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid);
58519b4aba2SHemant Agrawal 			continue;
58619b4aba2SHemant Agrawal 		}
58719b4aba2SHemant Agrawal 
58819b4aba2SHemant Agrawal 		offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >>
58919b4aba2SHemant Agrawal 				DPAA_FD_OFFSET_SHIFT;
59019b4aba2SHemant Agrawal 		length = fd->opaque & DPAA_FD_LENGTH_MASK;
59119b4aba2SHemant Agrawal 
59219b4aba2SHemant Agrawal 		mbuf = bufs[i];
59319b4aba2SHemant Agrawal 		mbuf->data_off = offset;
59419b4aba2SHemant Agrawal 		mbuf->data_len = length;
59519b4aba2SHemant Agrawal 		mbuf->pkt_len = length;
59619b4aba2SHemant Agrawal 		mbuf->port = dpaa_intf->ifid;
59719b4aba2SHemant Agrawal 
59819b4aba2SHemant Agrawal 		mbuf->nb_segs = 1;
59919b4aba2SHemant Agrawal 		mbuf->ol_flags = 0;
60019b4aba2SHemant Agrawal 		mbuf->next = NULL;
60119b4aba2SHemant Agrawal 		rte_mbuf_refcnt_set(mbuf, 1);
60219b4aba2SHemant Agrawal 		dpaa_eth_packet_info(mbuf, mbuf->buf_addr);
60377393f56SSachin Saxena 		dpaa_display_frame_info(fd, fq[0]->fqid, true);
60419b4aba2SHemant Agrawal 	}
60519b4aba2SHemant Agrawal }
60619b4aba2SHemant Agrawal 
607b9083ea5SNipun Gupta void dpaa_rx_cb_prepare(struct qm_dqrr_entry *dq, void **bufs)
608b9083ea5SNipun Gupta {
609b9083ea5SNipun Gupta 	struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(dq->fd.bpid);
610b9083ea5SNipun Gupta 	void *ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dq->fd));
611b9083ea5SNipun Gupta 
612b9083ea5SNipun Gupta 	/* In case of LS1046, annotation stashing is disabled due to L2 cache
6137be78d02SJosh Soref 	 * being bottleneck in case of multicore scenario for this platform.
6147be78d02SJosh Soref 	 * So we prefetch the annotation beforehand, so that it is available
615b9083ea5SNipun Gupta 	 * in cache when accessed.
616b9083ea5SNipun Gupta 	 */
617b9083ea5SNipun Gupta 	rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF));
618b9083ea5SNipun Gupta 
619b9083ea5SNipun Gupta 	*bufs = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size);
6200c504f69SHemant Agrawal }
6210c504f69SHemant Agrawal 
6220c504f69SHemant Agrawal static uint16_t
6230c504f69SHemant Agrawal dpaa_eth_queue_portal_rx(struct qman_fq *fq,
6240c504f69SHemant Agrawal 			 struct rte_mbuf **bufs,
6250c504f69SHemant Agrawal 			 uint16_t nb_bufs)
6260c504f69SHemant Agrawal {
6270c504f69SHemant Agrawal 	int ret;
6280c504f69SHemant Agrawal 
629b9c94167SNipun Gupta 	if (unlikely(!fq->qp_initialized)) {
6300c504f69SHemant Agrawal 		ret = rte_dpaa_portal_fq_init((void *)0, fq);
6310c504f69SHemant Agrawal 		if (ret) {
6320c504f69SHemant Agrawal 			DPAA_PMD_ERR("Failure in affining portal %d", ret);
6330c504f69SHemant Agrawal 			return 0;
6340c504f69SHemant Agrawal 		}
635b9c94167SNipun Gupta 		fq->qp_initialized = 1;
6360c504f69SHemant Agrawal 	}
6370c504f69SHemant Agrawal 
6380c504f69SHemant Agrawal 	return qman_portal_poll_rx(nb_bufs, (void **)bufs, fq->qp);
6390c504f69SHemant Agrawal }
6400c504f69SHemant Agrawal 
6415e745593SSunil Kumar Kori enum qman_cb_dqrr_result
6425e745593SSunil Kumar Kori dpaa_rx_cb_parallel(void *event,
6435e745593SSunil Kumar Kori 		    struct qman_portal *qm __always_unused,
6445e745593SSunil Kumar Kori 		    struct qman_fq *fq,
6455e745593SSunil Kumar Kori 		    const struct qm_dqrr_entry *dqrr,
6465e745593SSunil Kumar Kori 		    void **bufs)
6475e745593SSunil Kumar Kori {
6485e745593SSunil Kumar Kori 	u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid;
6495e745593SSunil Kumar Kori 	struct rte_mbuf *mbuf;
6505e745593SSunil Kumar Kori 	struct rte_event *ev = (struct rte_event *)event;
6515e745593SSunil Kumar Kori 
6525e745593SSunil Kumar Kori 	mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid);
6535e745593SSunil Kumar Kori 	ev->event_ptr = (void *)mbuf;
6545e745593SSunil Kumar Kori 	ev->flow_id = fq->ev.flow_id;
6555e745593SSunil Kumar Kori 	ev->sub_event_type = fq->ev.sub_event_type;
6565e745593SSunil Kumar Kori 	ev->event_type = RTE_EVENT_TYPE_ETHDEV;
6575e745593SSunil Kumar Kori 	ev->op = RTE_EVENT_OP_NEW;
6585e745593SSunil Kumar Kori 	ev->sched_type = fq->ev.sched_type;
6595e745593SSunil Kumar Kori 	ev->queue_id = fq->ev.queue_id;
6605e745593SSunil Kumar Kori 	ev->priority = fq->ev.priority;
6615e745593SSunil Kumar Kori 	ev->impl_opaque = (uint8_t)DPAA_INVALID_MBUF_SEQN;
662c9a1c2e5SDavid Marchand 	*dpaa_seqn(mbuf) = DPAA_INVALID_MBUF_SEQN;
6635e745593SSunil Kumar Kori 	*bufs = mbuf;
6645e745593SSunil Kumar Kori 
6655e745593SSunil Kumar Kori 	return qman_cb_dqrr_consume;
6665e745593SSunil Kumar Kori }
6675e745593SSunil Kumar Kori 
6685e745593SSunil Kumar Kori enum qman_cb_dqrr_result
6695e745593SSunil Kumar Kori dpaa_rx_cb_atomic(void *event,
6705e745593SSunil Kumar Kori 		  struct qman_portal *qm __always_unused,
6715e745593SSunil Kumar Kori 		  struct qman_fq *fq,
6725e745593SSunil Kumar Kori 		  const struct qm_dqrr_entry *dqrr,
6735e745593SSunil Kumar Kori 		  void **bufs)
6745e745593SSunil Kumar Kori {
6755e745593SSunil Kumar Kori 	u8 index;
6765e745593SSunil Kumar Kori 	u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid;
6775e745593SSunil Kumar Kori 	struct rte_mbuf *mbuf;
6785e745593SSunil Kumar Kori 	struct rte_event *ev = (struct rte_event *)event;
6795e745593SSunil Kumar Kori 
6805e745593SSunil Kumar Kori 	mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid);
6815e745593SSunil Kumar Kori 	ev->event_ptr = (void *)mbuf;
6825e745593SSunil Kumar Kori 	ev->flow_id = fq->ev.flow_id;
6835e745593SSunil Kumar Kori 	ev->sub_event_type = fq->ev.sub_event_type;
6845e745593SSunil Kumar Kori 	ev->event_type = RTE_EVENT_TYPE_ETHDEV;
6855e745593SSunil Kumar Kori 	ev->op = RTE_EVENT_OP_NEW;
6865e745593SSunil Kumar Kori 	ev->sched_type = fq->ev.sched_type;
6875e745593SSunil Kumar Kori 	ev->queue_id = fq->ev.queue_id;
6885e745593SSunil Kumar Kori 	ev->priority = fq->ev.priority;
6895e745593SSunil Kumar Kori 
6905e745593SSunil Kumar Kori 	/* Save active dqrr entries */
6915e745593SSunil Kumar Kori 	index = DQRR_PTR2IDX(dqrr);
6925e745593SSunil Kumar Kori 	DPAA_PER_LCORE_DQRR_SIZE++;
6935e745593SSunil Kumar Kori 	DPAA_PER_LCORE_DQRR_HELD |= 1 << index;
6945e745593SSunil Kumar Kori 	DPAA_PER_LCORE_DQRR_MBUF(index) = mbuf;
6955e745593SSunil Kumar Kori 	ev->impl_opaque = index + 1;
696c9a1c2e5SDavid Marchand 	*dpaa_seqn(mbuf) = (uint32_t)index + 1;
6975e745593SSunil Kumar Kori 	*bufs = mbuf;
6985e745593SSunil Kumar Kori 
6995e745593SSunil Kumar Kori 	return qman_cb_dqrr_defer;
7005e745593SSunil Kumar Kori }
7015e745593SSunil Kumar Kori 
70277393f56SSachin Saxena #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER
70377393f56SSachin Saxena static inline void dpaa_eth_err_queue(struct dpaa_if *dpaa_intf)
70477393f56SSachin Saxena {
70577393f56SSachin Saxena 	struct rte_mbuf *mbuf;
70677393f56SSachin Saxena 	struct qman_fq *debug_fq;
70777393f56SSachin Saxena 	int ret, i;
70877393f56SSachin Saxena 	struct qm_dqrr_entry *dq;
70977393f56SSachin Saxena 	struct qm_fd *fd;
71077393f56SSachin Saxena 
71177393f56SSachin Saxena 	if (unlikely(!RTE_PER_LCORE(dpaa_io))) {
71277393f56SSachin Saxena 		ret = rte_dpaa_portal_init((void *)0);
71377393f56SSachin Saxena 		if (ret) {
71477393f56SSachin Saxena 			DPAA_PMD_ERR("Failure in affining portal");
71577393f56SSachin Saxena 			return;
71677393f56SSachin Saxena 		}
71777393f56SSachin Saxena 	}
71877393f56SSachin Saxena 	for (i = 0; i <= DPAA_DEBUG_FQ_TX_ERROR; i++) {
71977393f56SSachin Saxena 		debug_fq = &dpaa_intf->debug_queues[i];
72077393f56SSachin Saxena 		ret = qman_set_vdq(debug_fq, 4, QM_VDQCR_EXACT);
72177393f56SSachin Saxena 		if (ret)
72277393f56SSachin Saxena 			return;
72377393f56SSachin Saxena 
72477393f56SSachin Saxena 		do {
72577393f56SSachin Saxena 			dq = qman_dequeue(debug_fq);
72677393f56SSachin Saxena 			if (!dq)
72777393f56SSachin Saxena 				continue;
72877393f56SSachin Saxena 			fd = &dq->fd;
72977393f56SSachin Saxena 			if (i == DPAA_DEBUG_FQ_RX_ERROR)
73077393f56SSachin Saxena 				DPAA_PMD_ERR("RX ERROR status: 0x%08x",
73177393f56SSachin Saxena 					fd->status);
73277393f56SSachin Saxena 			else
73377393f56SSachin Saxena 				DPAA_PMD_ERR("TX ERROR status: 0x%08x",
73477393f56SSachin Saxena 					fd->status);
73577393f56SSachin Saxena 			dpaa_display_frame_info(fd, debug_fq->fqid,
73677393f56SSachin Saxena 				i == DPAA_DEBUG_FQ_RX_ERROR);
73777393f56SSachin Saxena 
73877393f56SSachin Saxena 			mbuf = dpaa_eth_fd_to_mbuf(fd, dpaa_intf->ifid);
73977393f56SSachin Saxena 			rte_pktmbuf_free(mbuf);
74077393f56SSachin Saxena 			qman_dqrr_consume(debug_fq, dq);
74177393f56SSachin Saxena 		} while (debug_fq->flags & QMAN_FQ_STATE_VDQCR);
74277393f56SSachin Saxena 	}
74377393f56SSachin Saxena }
74477393f56SSachin Saxena #endif
74577393f56SSachin Saxena 
74637f9b54bSShreyansh Jain uint16_t dpaa_eth_queue_rx(void *q,
74737f9b54bSShreyansh Jain 			   struct rte_mbuf **bufs,
74837f9b54bSShreyansh Jain 			   uint16_t nb_bufs)
74937f9b54bSShreyansh Jain {
75037f9b54bSShreyansh Jain 	struct qman_fq *fq = q;
75137f9b54bSShreyansh Jain 	struct qm_dqrr_entry *dq;
75237f9b54bSShreyansh Jain 	uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid;
753f40d5a53SNipun Gupta 	int num_rx_bufs, ret;
754f40d5a53SNipun Gupta 	uint32_t vdqcr_flags = 0;
75537f9b54bSShreyansh Jain 
756e1797f4bSAkhil Goyal 	if (unlikely(rte_dpaa_bpid_info == NULL &&
757e1797f4bSAkhil Goyal 				rte_eal_process_type() == RTE_PROC_SECONDARY))
758e1797f4bSAkhil Goyal 		rte_dpaa_bpid_info = fq->bp_array;
759e1797f4bSAkhil Goyal 
76077393f56SSachin Saxena #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER
76177393f56SSachin Saxena 	if (fq->fqid == ((struct dpaa_if *)fq->dpaa_intf)->rx_queues[0].fqid)
76277393f56SSachin Saxena 		dpaa_eth_err_queue((struct dpaa_if *)fq->dpaa_intf);
76377393f56SSachin Saxena #endif
76477393f56SSachin Saxena 
7650c504f69SHemant Agrawal 	if (likely(fq->is_static))
7660c504f69SHemant Agrawal 		return dpaa_eth_queue_portal_rx(fq, bufs, nb_bufs);
7670c504f69SHemant Agrawal 
768e5872221SRohit Raj 	if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
76937f9b54bSShreyansh Jain 		ret = rte_dpaa_portal_init((void *)0);
77037f9b54bSShreyansh Jain 		if (ret) {
77137f9b54bSShreyansh Jain 			DPAA_PMD_ERR("Failure in affining portal");
77237f9b54bSShreyansh Jain 			return 0;
77337f9b54bSShreyansh Jain 		}
7745d944582SNipun Gupta 	}
77537f9b54bSShreyansh Jain 
776f40d5a53SNipun Gupta 	/* Until request for four buffers, we provide exact number of buffers.
777f40d5a53SNipun Gupta 	 * Otherwise we do not set the QM_VDQCR_EXACT flag.
778f40d5a53SNipun Gupta 	 * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
779f40d5a53SNipun Gupta 	 * requested, so we request two less in this case.
780f40d5a53SNipun Gupta 	 */
781f40d5a53SNipun Gupta 	if (nb_bufs < 4) {
782f40d5a53SNipun Gupta 		vdqcr_flags = QM_VDQCR_EXACT;
783f40d5a53SNipun Gupta 		num_rx_bufs = nb_bufs;
784f40d5a53SNipun Gupta 	} else {
785f40d5a53SNipun Gupta 		num_rx_bufs = nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
786f40d5a53SNipun Gupta 			(DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_bufs - 2);
787f40d5a53SNipun Gupta 	}
788f40d5a53SNipun Gupta 	ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
78937f9b54bSShreyansh Jain 	if (ret)
79037f9b54bSShreyansh Jain 		return 0;
79137f9b54bSShreyansh Jain 
79237f9b54bSShreyansh Jain 	do {
79337f9b54bSShreyansh Jain 		dq = qman_dequeue(fq);
79437f9b54bSShreyansh Jain 		if (!dq)
79537f9b54bSShreyansh Jain 			continue;
79637f9b54bSShreyansh Jain 		bufs[num_rx++] = dpaa_eth_fd_to_mbuf(&dq->fd, ifid);
79777393f56SSachin Saxena 		dpaa_display_frame_info(&dq->fd, fq->fqid, true);
79837f9b54bSShreyansh Jain 		qman_dqrr_consume(fq, dq);
79937f9b54bSShreyansh Jain 	} while (fq->flags & QMAN_FQ_STATE_VDQCR);
80037f9b54bSShreyansh Jain 
80137f9b54bSShreyansh Jain 	return num_rx;
80237f9b54bSShreyansh Jain }
80337f9b54bSShreyansh Jain 
804f191d5abSHemant Agrawal static int
8058cffdcbeSShreyansh Jain dpaa_eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
8068cffdcbeSShreyansh Jain 		struct qm_fd *fd,
807f191d5abSHemant Agrawal 		struct dpaa_bp_info *bp_info)
8088cffdcbeSShreyansh Jain {
8098cffdcbeSShreyansh Jain 	struct rte_mbuf *cur_seg = mbuf, *prev_seg = NULL;
8108cffdcbeSShreyansh Jain 	struct rte_mbuf *temp, *mi;
8118cffdcbeSShreyansh Jain 	struct qm_sg_entry *sg_temp, *sgt;
8128cffdcbeSShreyansh Jain 	int i = 0;
8138cffdcbeSShreyansh Jain 
8148cffdcbeSShreyansh Jain 	DPAA_DP_LOG(DEBUG, "Creating SG FD to transmit");
8158cffdcbeSShreyansh Jain 
8168cffdcbeSShreyansh Jain 	temp = rte_pktmbuf_alloc(bp_info->mp);
8178cffdcbeSShreyansh Jain 	if (!temp) {
8188cffdcbeSShreyansh Jain 		DPAA_PMD_ERR("Failure in allocation of mbuf");
8198cffdcbeSShreyansh Jain 		return -1;
8208cffdcbeSShreyansh Jain 	}
8218cffdcbeSShreyansh Jain 	if (temp->buf_len < ((mbuf->nb_segs * sizeof(struct qm_sg_entry))
8228cffdcbeSShreyansh Jain 				+ temp->data_off)) {
8238cffdcbeSShreyansh Jain 		DPAA_PMD_ERR("Insufficient space in mbuf for SG entries");
8248cffdcbeSShreyansh Jain 		return -1;
8258cffdcbeSShreyansh Jain 	}
8268cffdcbeSShreyansh Jain 
8278cffdcbeSShreyansh Jain 	fd->cmd = 0;
8288cffdcbeSShreyansh Jain 	fd->opaque_addr = 0;
8298cffdcbeSShreyansh Jain 
8308cffdcbeSShreyansh Jain 	if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) {
831d565c887SAshish Jain 		if (!mbuf->packet_type) {
832d565c887SAshish Jain 			struct rte_net_hdr_lens hdr_lens;
833d565c887SAshish Jain 
834d565c887SAshish Jain 			mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens,
835d565c887SAshish Jain 					RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK
836d565c887SAshish Jain 					| RTE_PTYPE_L4_MASK);
837d565c887SAshish Jain 			mbuf->l2_len = hdr_lens.l2_len;
838d565c887SAshish Jain 			mbuf->l3_len = hdr_lens.l3_len;
839d565c887SAshish Jain 		}
8408cffdcbeSShreyansh Jain 		if (temp->data_off < DEFAULT_TX_ICEOF
8418cffdcbeSShreyansh Jain 			+ sizeof(struct dpaa_eth_parse_results_t))
8428cffdcbeSShreyansh Jain 			temp->data_off = DEFAULT_TX_ICEOF
8438cffdcbeSShreyansh Jain 				+ sizeof(struct dpaa_eth_parse_results_t);
8448cffdcbeSShreyansh Jain 		dcbz_64(temp->buf_addr);
8458cffdcbeSShreyansh Jain 		dpaa_checksum_offload(mbuf, fd, temp->buf_addr);
8468cffdcbeSShreyansh Jain 	}
8478cffdcbeSShreyansh Jain 
8488cffdcbeSShreyansh Jain 	sgt = temp->buf_addr + temp->data_off;
8498cffdcbeSShreyansh Jain 	fd->format = QM_FD_SG;
850455da545SSantosh Shukla 	fd->addr = temp->buf_iova;
8518cffdcbeSShreyansh Jain 	fd->offset = temp->data_off;
852f191d5abSHemant Agrawal 	fd->bpid = bp_info ? bp_info->bpid : 0xff;
8538cffdcbeSShreyansh Jain 	fd->length20 = mbuf->pkt_len;
8548cffdcbeSShreyansh Jain 
8558cffdcbeSShreyansh Jain 	while (i < DPAA_SGT_MAX_ENTRIES) {
8568cffdcbeSShreyansh Jain 		sg_temp = &sgt[i++];
8578cffdcbeSShreyansh Jain 		sg_temp->opaque = 0;
8588cffdcbeSShreyansh Jain 		sg_temp->val = 0;
859455da545SSantosh Shukla 		sg_temp->addr = cur_seg->buf_iova;
8608cffdcbeSShreyansh Jain 		sg_temp->offset = cur_seg->data_off;
8618cffdcbeSShreyansh Jain 		sg_temp->length = cur_seg->data_len;
8628cffdcbeSShreyansh Jain 		if (RTE_MBUF_DIRECT(cur_seg)) {
8638cffdcbeSShreyansh Jain 			if (rte_mbuf_refcnt_read(cur_seg) > 1) {
8648cffdcbeSShreyansh Jain 				/*If refcnt > 1, invalid bpid is set to ensure
8658cffdcbeSShreyansh Jain 				 * buffer is not freed by HW.
8668cffdcbeSShreyansh Jain 				 */
8678cffdcbeSShreyansh Jain 				sg_temp->bpid = 0xff;
8688cffdcbeSShreyansh Jain 				rte_mbuf_refcnt_update(cur_seg, -1);
8698cffdcbeSShreyansh Jain 			} else {
8708cffdcbeSShreyansh Jain 				sg_temp->bpid =
8718cffdcbeSShreyansh Jain 					DPAA_MEMPOOL_TO_BPID(cur_seg->pool);
8728cffdcbeSShreyansh Jain 			}
8738cffdcbeSShreyansh Jain 			cur_seg = cur_seg->next;
874f191d5abSHemant Agrawal 		} else if (RTE_MBUF_HAS_EXTBUF(cur_seg)) {
875f191d5abSHemant Agrawal 			sg_temp->bpid = 0xff;
876f191d5abSHemant Agrawal 			cur_seg = cur_seg->next;
8778cffdcbeSShreyansh Jain 		} else {
8788cffdcbeSShreyansh Jain 			/* Get owner MBUF from indirect buffer */
8798cffdcbeSShreyansh Jain 			mi = rte_mbuf_from_indirect(cur_seg);
8808cffdcbeSShreyansh Jain 			if (rte_mbuf_refcnt_read(mi) > 1) {
8818cffdcbeSShreyansh Jain 				/*If refcnt > 1, invalid bpid is set to ensure
8828cffdcbeSShreyansh Jain 				 * owner buffer is not freed by HW.
8838cffdcbeSShreyansh Jain 				 */
8848cffdcbeSShreyansh Jain 				sg_temp->bpid = 0xff;
8858cffdcbeSShreyansh Jain 			} else {
8868cffdcbeSShreyansh Jain 				sg_temp->bpid = DPAA_MEMPOOL_TO_BPID(mi->pool);
8878cffdcbeSShreyansh Jain 				rte_mbuf_refcnt_update(mi, 1);
8888cffdcbeSShreyansh Jain 			}
8898cffdcbeSShreyansh Jain 			prev_seg = cur_seg;
8908cffdcbeSShreyansh Jain 			cur_seg = cur_seg->next;
8918cffdcbeSShreyansh Jain 			prev_seg->next = NULL;
8928cffdcbeSShreyansh Jain 			rte_pktmbuf_free(prev_seg);
8938cffdcbeSShreyansh Jain 		}
8948cffdcbeSShreyansh Jain 		if (cur_seg == NULL) {
8958cffdcbeSShreyansh Jain 			sg_temp->final = 1;
8968cffdcbeSShreyansh Jain 			cpu_to_hw_sg(sg_temp);
8978cffdcbeSShreyansh Jain 			break;
8988cffdcbeSShreyansh Jain 		}
8998cffdcbeSShreyansh Jain 		cpu_to_hw_sg(sg_temp);
9008cffdcbeSShreyansh Jain 	}
9018cffdcbeSShreyansh Jain 	return 0;
9028cffdcbeSShreyansh Jain }
9038cffdcbeSShreyansh Jain 
90437f9b54bSShreyansh Jain /* Handle mbufs which are not segmented (non SG) */
90537f9b54bSShreyansh Jain static inline void
90637f9b54bSShreyansh Jain tx_on_dpaa_pool_unsegmented(struct rte_mbuf *mbuf,
90737f9b54bSShreyansh Jain 			    struct dpaa_bp_info *bp_info,
90837f9b54bSShreyansh Jain 			    struct qm_fd *fd_arr)
90937f9b54bSShreyansh Jain {
91037f9b54bSShreyansh Jain 	struct rte_mbuf *mi = NULL;
91137f9b54bSShreyansh Jain 
91237f9b54bSShreyansh Jain 	if (RTE_MBUF_DIRECT(mbuf)) {
91337f9b54bSShreyansh Jain 		if (rte_mbuf_refcnt_read(mbuf) > 1) {
91437f9b54bSShreyansh Jain 			/* In case of direct mbuf and mbuf being cloned,
91537f9b54bSShreyansh Jain 			 * BMAN should _not_ release buffer.
91637f9b54bSShreyansh Jain 			 */
91737f9b54bSShreyansh Jain 			DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff);
91837f9b54bSShreyansh Jain 			/* Buffer should be releasd by EAL */
91937f9b54bSShreyansh Jain 			rte_mbuf_refcnt_update(mbuf, -1);
92037f9b54bSShreyansh Jain 		} else {
92137f9b54bSShreyansh Jain 			/* In case of direct mbuf and no cloning, mbuf can be
92237f9b54bSShreyansh Jain 			 * released by BMAN.
92337f9b54bSShreyansh Jain 			 */
92437f9b54bSShreyansh Jain 			DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid);
92537f9b54bSShreyansh Jain 		}
926f191d5abSHemant Agrawal 	} else if (RTE_MBUF_HAS_EXTBUF(mbuf)) {
927f191d5abSHemant Agrawal 		DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr,
928f191d5abSHemant Agrawal 				bp_info ? bp_info->bpid : 0xff);
92937f9b54bSShreyansh Jain 	} else {
93037f9b54bSShreyansh Jain 		/* This is data-containing core mbuf: 'mi' */
93137f9b54bSShreyansh Jain 		mi = rte_mbuf_from_indirect(mbuf);
93237f9b54bSShreyansh Jain 		if (rte_mbuf_refcnt_read(mi) > 1) {
93337f9b54bSShreyansh Jain 			/* In case of indirect mbuf, and mbuf being cloned,
93437f9b54bSShreyansh Jain 			 * BMAN should _not_ release it and let EAL release
93537f9b54bSShreyansh Jain 			 * it through pktmbuf_free below.
93637f9b54bSShreyansh Jain 			 */
93737f9b54bSShreyansh Jain 			DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff);
93837f9b54bSShreyansh Jain 		} else {
93937f9b54bSShreyansh Jain 			/* In case of indirect mbuf, and no cloning, core mbuf
94037f9b54bSShreyansh Jain 			 * should be released by BMAN.
94137f9b54bSShreyansh Jain 			 * Increate refcnt of core mbuf so that when
94237f9b54bSShreyansh Jain 			 * pktmbuf_free is called and mbuf is released, EAL
94337f9b54bSShreyansh Jain 			 * doesn't try to release core mbuf which would have
94437f9b54bSShreyansh Jain 			 * been released by BMAN.
94537f9b54bSShreyansh Jain 			 */
94637f9b54bSShreyansh Jain 			rte_mbuf_refcnt_update(mi, 1);
947f191d5abSHemant Agrawal 			DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr,
948f191d5abSHemant Agrawal 						bp_info ? bp_info->bpid : 0xff);
94937f9b54bSShreyansh Jain 		}
95037f9b54bSShreyansh Jain 		rte_pktmbuf_free(mbuf);
95137f9b54bSShreyansh Jain 	}
9525a8cf1beSShreyansh Jain 
9535e0789e9SNipun Gupta 	if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK)
9545e0789e9SNipun Gupta 		dpaa_unsegmented_checksum(mbuf, fd_arr);
95537f9b54bSShreyansh Jain }
95637f9b54bSShreyansh Jain 
95737f9b54bSShreyansh Jain /* Handle all mbufs on dpaa BMAN managed pool */
95837f9b54bSShreyansh Jain static inline uint16_t
95937f9b54bSShreyansh Jain tx_on_dpaa_pool(struct rte_mbuf *mbuf,
96037f9b54bSShreyansh Jain 		struct dpaa_bp_info *bp_info,
96137f9b54bSShreyansh Jain 		struct qm_fd *fd_arr)
96237f9b54bSShreyansh Jain {
96337f9b54bSShreyansh Jain 	DPAA_DP_LOG(DEBUG, "BMAN offloaded buffer, mbuf: %p", mbuf);
96437f9b54bSShreyansh Jain 
96537f9b54bSShreyansh Jain 	if (mbuf->nb_segs == 1) {
96637f9b54bSShreyansh Jain 		/* Case for non-segmented buffers */
96737f9b54bSShreyansh Jain 		tx_on_dpaa_pool_unsegmented(mbuf, bp_info, fd_arr);
9688cffdcbeSShreyansh Jain 	} else if (mbuf->nb_segs > 1 &&
9698cffdcbeSShreyansh Jain 		   mbuf->nb_segs <= DPAA_SGT_MAX_ENTRIES) {
970f191d5abSHemant Agrawal 		if (dpaa_eth_mbuf_to_sg_fd(mbuf, fd_arr, bp_info)) {
9718cffdcbeSShreyansh Jain 			DPAA_PMD_DEBUG("Unable to create Scatter Gather FD");
9728cffdcbeSShreyansh Jain 			return 1;
9738cffdcbeSShreyansh Jain 		}
97437f9b54bSShreyansh Jain 	} else {
97537f9b54bSShreyansh Jain 		DPAA_PMD_DEBUG("Number of Segments not supported");
97637f9b54bSShreyansh Jain 		return 1;
97737f9b54bSShreyansh Jain 	}
97837f9b54bSShreyansh Jain 
97937f9b54bSShreyansh Jain 	return 0;
98037f9b54bSShreyansh Jain }
98137f9b54bSShreyansh Jain 
98237f9b54bSShreyansh Jain /* Handle all mbufs on an external pool (non-dpaa) */
983f8c7a17aSNipun Gupta static inline struct rte_mbuf *
984f8c7a17aSNipun Gupta reallocate_mbuf(struct qman_fq *txq, struct rte_mbuf *mbuf)
98537f9b54bSShreyansh Jain {
98637f9b54bSShreyansh Jain 	struct dpaa_if *dpaa_intf = txq->dpaa_intf;
987f8c7a17aSNipun Gupta 	struct dpaa_bp_info *bp_info = dpaa_intf->bp_info;
988f8c7a17aSNipun Gupta 	struct rte_mbuf *new_mbufs[DPAA_SGT_MAX_ENTRIES + 1] = {0};
989f8c7a17aSNipun Gupta 	struct rte_mbuf *temp_mbuf;
990f8c7a17aSNipun Gupta 	int num_new_segs, mbuf_greater, ret, extra_seg = 0, i = 0;
991f8c7a17aSNipun Gupta 	uint64_t mbufs_size, bytes_to_copy, offset1 = 0, offset2 = 0;
992f8c7a17aSNipun Gupta 	char *data;
99337f9b54bSShreyansh Jain 
994f8c7a17aSNipun Gupta 	DPAA_DP_LOG(DEBUG, "Reallocating transmit buffer");
995f8c7a17aSNipun Gupta 
996f8c7a17aSNipun Gupta 	mbufs_size = bp_info->size -
997f8c7a17aSNipun Gupta 		bp_info->meta_data_size - RTE_PKTMBUF_HEADROOM;
998f8c7a17aSNipun Gupta 	extra_seg = !!(mbuf->pkt_len % mbufs_size);
999f8c7a17aSNipun Gupta 	num_new_segs = (mbuf->pkt_len / mbufs_size) + extra_seg;
1000f8c7a17aSNipun Gupta 
1001f8c7a17aSNipun Gupta 	ret = rte_pktmbuf_alloc_bulk(bp_info->mp, new_mbufs, num_new_segs);
1002f8c7a17aSNipun Gupta 	if (ret != 0) {
1003f8c7a17aSNipun Gupta 		DPAA_DP_LOG(DEBUG, "Allocation for new buffers failed");
1004f8c7a17aSNipun Gupta 		return NULL;
100537f9b54bSShreyansh Jain 	}
100637f9b54bSShreyansh Jain 
1007f8c7a17aSNipun Gupta 	temp_mbuf = mbuf;
100837f9b54bSShreyansh Jain 
1009f8c7a17aSNipun Gupta 	while (temp_mbuf) {
1010f8c7a17aSNipun Gupta 		/* If mbuf data is less than new mbuf remaining memory */
1011f8c7a17aSNipun Gupta 		if ((temp_mbuf->data_len - offset1) < (mbufs_size - offset2)) {
1012f8c7a17aSNipun Gupta 			bytes_to_copy = temp_mbuf->data_len - offset1;
1013f8c7a17aSNipun Gupta 			mbuf_greater = -1;
1014f8c7a17aSNipun Gupta 		/* If mbuf data is greater than new mbuf remaining memory */
1015f8c7a17aSNipun Gupta 		} else if ((temp_mbuf->data_len - offset1) >
1016f8c7a17aSNipun Gupta 			   (mbufs_size - offset2)) {
1017f8c7a17aSNipun Gupta 			bytes_to_copy = mbufs_size - offset2;
1018f8c7a17aSNipun Gupta 			mbuf_greater = 1;
1019f8c7a17aSNipun Gupta 		/* if mbuf data is equal to new mbuf remaining memory */
1020f8c7a17aSNipun Gupta 		} else {
1021f8c7a17aSNipun Gupta 			bytes_to_copy = temp_mbuf->data_len - offset1;
1022f8c7a17aSNipun Gupta 			mbuf_greater = 0;
1023f8c7a17aSNipun Gupta 		}
1024f8c7a17aSNipun Gupta 
1025f8c7a17aSNipun Gupta 		/* Copy the data */
1026f8c7a17aSNipun Gupta 		data = rte_pktmbuf_append(new_mbufs[0], bytes_to_copy);
1027f8c7a17aSNipun Gupta 
1028f8c7a17aSNipun Gupta 		rte_memcpy((uint8_t *)data, rte_pktmbuf_mtod_offset(mbuf,
1029f8c7a17aSNipun Gupta 			   void *, offset1), bytes_to_copy);
1030f8c7a17aSNipun Gupta 
1031f8c7a17aSNipun Gupta 		/* Set new offsets and the temp buffers */
1032f8c7a17aSNipun Gupta 		if (mbuf_greater == -1) {
1033f8c7a17aSNipun Gupta 			offset1 = 0;
1034f8c7a17aSNipun Gupta 			offset2 += bytes_to_copy;
1035f8c7a17aSNipun Gupta 			temp_mbuf = temp_mbuf->next;
1036f8c7a17aSNipun Gupta 		} else if (mbuf_greater == 1) {
1037f8c7a17aSNipun Gupta 			offset2 = 0;
1038f8c7a17aSNipun Gupta 			offset1 += bytes_to_copy;
1039f8c7a17aSNipun Gupta 			new_mbufs[i]->next = new_mbufs[i + 1];
1040f8c7a17aSNipun Gupta 			new_mbufs[0]->nb_segs++;
1041f8c7a17aSNipun Gupta 			i++;
1042f8c7a17aSNipun Gupta 		} else {
1043f8c7a17aSNipun Gupta 			offset1 = 0;
1044f8c7a17aSNipun Gupta 			offset2 = 0;
1045f8c7a17aSNipun Gupta 			temp_mbuf = temp_mbuf->next;
1046f8c7a17aSNipun Gupta 			new_mbufs[i]->next = new_mbufs[i + 1];
1047f8c7a17aSNipun Gupta 			if (new_mbufs[i + 1])
1048f8c7a17aSNipun Gupta 				new_mbufs[0]->nb_segs++;
1049f8c7a17aSNipun Gupta 			i++;
1050f8c7a17aSNipun Gupta 		}
1051f8c7a17aSNipun Gupta 	}
1052f8c7a17aSNipun Gupta 
1053f8c7a17aSNipun Gupta 	/* Copy other required fields */
1054f8c7a17aSNipun Gupta 	new_mbufs[0]->ol_flags = mbuf->ol_flags;
1055f8c7a17aSNipun Gupta 	new_mbufs[0]->packet_type = mbuf->packet_type;
1056f8c7a17aSNipun Gupta 	new_mbufs[0]->tx_offload = mbuf->tx_offload;
1057f8c7a17aSNipun Gupta 
1058f8c7a17aSNipun Gupta 	rte_pktmbuf_free(mbuf);
1059f8c7a17aSNipun Gupta 
1060f8c7a17aSNipun Gupta 	return new_mbufs[0];
106137f9b54bSShreyansh Jain }
106237f9b54bSShreyansh Jain 
106337f9b54bSShreyansh Jain uint16_t
106437f9b54bSShreyansh Jain dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs)
106537f9b54bSShreyansh Jain {
106637f9b54bSShreyansh Jain 	struct rte_mbuf *mbuf, *mi = NULL;
106737f9b54bSShreyansh Jain 	struct rte_mempool *mp;
106837f9b54bSShreyansh Jain 	struct dpaa_bp_info *bp_info;
1069b0a87fe2SNipun Gupta 	struct qm_fd fd_arr[DPAA_TX_BURST_SIZE];
10705e0789e9SNipun Gupta 	uint32_t frames_to_send, loop, sent = 0;
107137f9b54bSShreyansh Jain 	uint16_t state;
1072f8c7a17aSNipun Gupta 	int ret, realloc_mbuf = 0;
10735e745593SSunil Kumar Kori 	uint32_t seqn, index, flags[DPAA_TX_BURST_SIZE] = {0};
1074f191d5abSHemant Agrawal 	struct rte_mbuf **orig_bufs = bufs;
107537f9b54bSShreyansh Jain 
1076e5872221SRohit Raj 	if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
107737f9b54bSShreyansh Jain 		ret = rte_dpaa_portal_init((void *)0);
107837f9b54bSShreyansh Jain 		if (ret) {
107937f9b54bSShreyansh Jain 			DPAA_PMD_ERR("Failure in affining portal");
108037f9b54bSShreyansh Jain 			return 0;
108137f9b54bSShreyansh Jain 		}
10825d944582SNipun Gupta 	}
108337f9b54bSShreyansh Jain 
108437f9b54bSShreyansh Jain 	DPAA_DP_LOG(DEBUG, "Transmitting %d buffers on queue: %p", nb_bufs, q);
108537f9b54bSShreyansh Jain 
108637f9b54bSShreyansh Jain 	while (nb_bufs) {
1087b0a87fe2SNipun Gupta 		frames_to_send = (nb_bufs > DPAA_TX_BURST_SIZE) ?
1088b0a87fe2SNipun Gupta 				DPAA_TX_BURST_SIZE : nb_bufs;
10895e0789e9SNipun Gupta 		for (loop = 0; loop < frames_to_send; loop++) {
10905e0789e9SNipun Gupta 			mbuf = *(bufs++);
1091f8c7a17aSNipun Gupta 			/* In case the data offset is not multiple of 16,
1092f8c7a17aSNipun Gupta 			 * FMAN can stall because of an errata. So reallocate
1093f8c7a17aSNipun Gupta 			 * the buffer in such case.
1094f8c7a17aSNipun Gupta 			 */
1095f8c7a17aSNipun Gupta 			if (dpaa_svr_family == SVR_LS1043A_FAMILY &&
109659267d7bSNipun Gupta 					(mbuf->data_off & 0x7F) != 0x0)
1097f8c7a17aSNipun Gupta 				realloc_mbuf = 1;
1098c9a1c2e5SDavid Marchand 			seqn = *dpaa_seqn(mbuf);
10999afce5aaSSunil Kumar Kori 			if (seqn != DPAA_INVALID_MBUF_SEQN) {
11009afce5aaSSunil Kumar Kori 				index = seqn - 1;
11019afce5aaSSunil Kumar Kori 				if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) {
11029afce5aaSSunil Kumar Kori 					flags[loop] =
11039afce5aaSSunil Kumar Kori 					   ((index & QM_EQCR_DCA_IDXMASK) << 8);
11049afce5aaSSunil Kumar Kori 					flags[loop] |= QMAN_ENQUEUE_FLAG_DCA;
11059afce5aaSSunil Kumar Kori 					DPAA_PER_LCORE_DQRR_SIZE--;
11069afce5aaSSunil Kumar Kori 					DPAA_PER_LCORE_DQRR_HELD &=
11079afce5aaSSunil Kumar Kori 								~(1 << index);
11089afce5aaSSunil Kumar Kori 				}
11099afce5aaSSunil Kumar Kori 			}
11109afce5aaSSunil Kumar Kori 
11115e0789e9SNipun Gupta 			if (likely(RTE_MBUF_DIRECT(mbuf))) {
111237f9b54bSShreyansh Jain 				mp = mbuf->pool;
11135e0789e9SNipun Gupta 				bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp);
11145e0789e9SNipun Gupta 				if (likely(mp->ops_index ==
11155e0789e9SNipun Gupta 						bp_info->dpaa_ops_index &&
11165e0789e9SNipun Gupta 					mbuf->nb_segs == 1 &&
1117f8c7a17aSNipun Gupta 					realloc_mbuf == 0 &&
11185e0789e9SNipun Gupta 					rte_mbuf_refcnt_read(mbuf) == 1)) {
11195e0789e9SNipun Gupta 					DPAA_MBUF_TO_CONTIG_FD(mbuf,
11205e0789e9SNipun Gupta 						&fd_arr[loop], bp_info->bpid);
11215e0789e9SNipun Gupta 					if (mbuf->ol_flags &
11225e0789e9SNipun Gupta 						DPAA_TX_CKSUM_OFFLOAD_MASK)
11235e0789e9SNipun Gupta 						dpaa_unsegmented_checksum(mbuf,
11245e0789e9SNipun Gupta 							&fd_arr[loop]);
11255e0789e9SNipun Gupta 					continue;
11265e0789e9SNipun Gupta 				}
112737f9b54bSShreyansh Jain 			} else {
112837f9b54bSShreyansh Jain 				mi = rte_mbuf_from_indirect(mbuf);
112937f9b54bSShreyansh Jain 				mp = mi->pool;
113037f9b54bSShreyansh Jain 			}
113137f9b54bSShreyansh Jain 
1132f191d5abSHemant Agrawal 			if (unlikely(RTE_MBUF_HAS_EXTBUF(mbuf))) {
1133f191d5abSHemant Agrawal 				bp_info = NULL;
1134f191d5abSHemant Agrawal 				goto indirect_buf;
1135f191d5abSHemant Agrawal 			}
1136f191d5abSHemant Agrawal 
113737f9b54bSShreyansh Jain 			bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp);
1138f8c7a17aSNipun Gupta 			if (unlikely(mp->ops_index != bp_info->dpaa_ops_index ||
1139f8c7a17aSNipun Gupta 				     realloc_mbuf == 1)) {
1140f8c7a17aSNipun Gupta 				struct rte_mbuf *temp_mbuf;
1141f8c7a17aSNipun Gupta 
1142f8c7a17aSNipun Gupta 				temp_mbuf = reallocate_mbuf(q, mbuf);
1143f8c7a17aSNipun Gupta 				if (!temp_mbuf) {
1144f8c7a17aSNipun Gupta 					/* Set frames_to_send & nb_bufs so
1145f8c7a17aSNipun Gupta 					 * that packets are transmitted till
1146f8c7a17aSNipun Gupta 					 * previous frame.
1147f8c7a17aSNipun Gupta 					 */
1148f8c7a17aSNipun Gupta 					frames_to_send = loop;
1149f8c7a17aSNipun Gupta 					nb_bufs = loop;
1150f8c7a17aSNipun Gupta 					goto send_pkts;
1151f8c7a17aSNipun Gupta 				}
1152f8c7a17aSNipun Gupta 				mbuf = temp_mbuf;
1153f8c7a17aSNipun Gupta 				realloc_mbuf = 0;
1154f8c7a17aSNipun Gupta 			}
1155f191d5abSHemant Agrawal indirect_buf:
115637f9b54bSShreyansh Jain 			state = tx_on_dpaa_pool(mbuf, bp_info,
115737f9b54bSShreyansh Jain 						&fd_arr[loop]);
115837f9b54bSShreyansh Jain 			if (unlikely(state)) {
115937f9b54bSShreyansh Jain 				/* Set frames_to_send & nb_bufs so
116037f9b54bSShreyansh Jain 				 * that packets are transmitted till
116137f9b54bSShreyansh Jain 				 * previous frame.
116237f9b54bSShreyansh Jain 				 */
116337f9b54bSShreyansh Jain 				frames_to_send = loop;
116437f9b54bSShreyansh Jain 				nb_bufs = loop;
116537f9b54bSShreyansh Jain 				goto send_pkts;
116637f9b54bSShreyansh Jain 			}
116737f9b54bSShreyansh Jain 		}
116837f9b54bSShreyansh Jain 
116937f9b54bSShreyansh Jain send_pkts:
117037f9b54bSShreyansh Jain 		loop = 0;
117137f9b54bSShreyansh Jain 		while (loop < frames_to_send) {
117237f9b54bSShreyansh Jain 			loop += qman_enqueue_multi(q, &fd_arr[loop],
11735e745593SSunil Kumar Kori 						   &flags[loop],
117437f9b54bSShreyansh Jain 						   frames_to_send - loop);
117537f9b54bSShreyansh Jain 		}
117637f9b54bSShreyansh Jain 		nb_bufs -= frames_to_send;
11775e0789e9SNipun Gupta 		sent += frames_to_send;
117837f9b54bSShreyansh Jain 	}
117937f9b54bSShreyansh Jain 
11805e0789e9SNipun Gupta 	DPAA_DP_LOG(DEBUG, "Transmitted %d buffers on queue: %p", sent, q);
118137f9b54bSShreyansh Jain 
1182f191d5abSHemant Agrawal 
1183f191d5abSHemant Agrawal 	loop = 0;
1184f191d5abSHemant Agrawal 	while (loop < sent) {
1185f191d5abSHemant Agrawal 		if (unlikely(RTE_MBUF_HAS_EXTBUF(*orig_bufs)))
1186f191d5abSHemant Agrawal 			rte_pktmbuf_free(*orig_bufs);
1187f191d5abSHemant Agrawal 		orig_bufs++;
1188f191d5abSHemant Agrawal 		loop++;
1189f191d5abSHemant Agrawal 	}
1190f191d5abSHemant Agrawal 
11915e0789e9SNipun Gupta 	return sent;
119237f9b54bSShreyansh Jain }
119337f9b54bSShreyansh Jain 
11949124e65dSGagandeep Singh uint16_t
11959124e65dSGagandeep Singh dpaa_eth_queue_tx_slow(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs)
11969124e65dSGagandeep Singh {
11979124e65dSGagandeep Singh 	qman_ern_poll_free();
11989124e65dSGagandeep Singh 
11999124e65dSGagandeep Singh 	return dpaa_eth_queue_tx(q, bufs, nb_bufs);
12009124e65dSGagandeep Singh }
12019124e65dSGagandeep Singh 
120237f9b54bSShreyansh Jain uint16_t dpaa_eth_tx_drop_all(void *q  __rte_unused,
120337f9b54bSShreyansh Jain 			      struct rte_mbuf **bufs __rte_unused,
120437f9b54bSShreyansh Jain 		uint16_t nb_bufs __rte_unused)
120537f9b54bSShreyansh Jain {
120637f9b54bSShreyansh Jain 	DPAA_DP_LOG(DEBUG, "Drop all packets");
120737f9b54bSShreyansh Jain 
120837f9b54bSShreyansh Jain 	/* Drop all incoming packets. No need to free packets here
120937f9b54bSShreyansh Jain 	 * because the rte_eth f/w frees up the packets through tx_buffer
121037f9b54bSShreyansh Jain 	 * callback in case this functions returns count less than nb_bufs
121137f9b54bSShreyansh Jain 	 */
121237f9b54bSShreyansh Jain 	return 0;
121337f9b54bSShreyansh Jain }
1214