xref: /dpdk/app/test-eventdev/test_pipeline_queue.c (revision a734e7388de7dcda400f1ae7c1e445b581752f40)
1d60b4185SPavan Nikhilesh /*
2d60b4185SPavan Nikhilesh  * SPDX-License-Identifier: BSD-3-Clause
3d60b4185SPavan Nikhilesh  * Copyright 2017 Cavium, Inc.
4d60b4185SPavan Nikhilesh  */
5d60b4185SPavan Nikhilesh 
6d60b4185SPavan Nikhilesh #include "test_pipeline_common.h"
7d60b4185SPavan Nikhilesh 
843d162bcSThomas Monjalon /* See http://doc.dpdk.org/guides/tools/testeventdev.html for test details */
9d60b4185SPavan Nikhilesh 
10d60b4185SPavan Nikhilesh static __rte_always_inline int
11d60b4185SPavan Nikhilesh pipeline_queue_nb_event_queues(struct evt_options *opt)
12d60b4185SPavan Nikhilesh {
13d9a42a69SThomas Monjalon 	uint16_t eth_count = rte_eth_dev_count_avail();
14d60b4185SPavan Nikhilesh 
15d60b4185SPavan Nikhilesh 	return (eth_count * opt->nb_stages) + eth_count;
16d60b4185SPavan Nikhilesh }
17d60b4185SPavan Nikhilesh 
182eaa37b8SPavan Nikhilesh typedef int (*pipeline_queue_worker_t)(void *arg);
192eaa37b8SPavan Nikhilesh 
20032a965aSPavan Nikhilesh static __rte_noinline int
21314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_tx(void *arg)
22314bcf58SPavan Nikhilesh {
23f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_INIT;
24314bcf58SPavan Nikhilesh 
25314bcf58SPavan Nikhilesh 	while (t->done == false) {
26314bcf58SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
27314bcf58SPavan Nikhilesh 
28314bcf58SPavan Nikhilesh 		if (!event) {
29314bcf58SPavan Nikhilesh 			rte_pause();
30314bcf58SPavan Nikhilesh 			continue;
31314bcf58SPavan Nikhilesh 		}
32314bcf58SPavan Nikhilesh 
33314bcf58SPavan Nikhilesh 		if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
34032a965aSPavan Nikhilesh 			pipeline_event_tx(dev, port, &ev);
35314bcf58SPavan Nikhilesh 			w->processed_pkts++;
36314bcf58SPavan Nikhilesh 		} else {
37314bcf58SPavan Nikhilesh 			ev.queue_id++;
38314bcf58SPavan Nikhilesh 			pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
39314bcf58SPavan Nikhilesh 			pipeline_event_enqueue(dev, port, &ev);
40314bcf58SPavan Nikhilesh 		}
41314bcf58SPavan Nikhilesh 	}
42314bcf58SPavan Nikhilesh 
43314bcf58SPavan Nikhilesh 	return 0;
44314bcf58SPavan Nikhilesh }
45314bcf58SPavan Nikhilesh 
46032a965aSPavan Nikhilesh static __rte_noinline int
47314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_fwd(void *arg)
48314bcf58SPavan Nikhilesh {
49f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_INIT;
50032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
51314bcf58SPavan Nikhilesh 
52314bcf58SPavan Nikhilesh 	while (t->done == false) {
53314bcf58SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
54314bcf58SPavan Nikhilesh 
55314bcf58SPavan Nikhilesh 		if (!event) {
56314bcf58SPavan Nikhilesh 			rte_pause();
57314bcf58SPavan Nikhilesh 			continue;
58314bcf58SPavan Nikhilesh 		}
59314bcf58SPavan Nikhilesh 
60032a965aSPavan Nikhilesh 		ev.queue_id = tx_queue[ev.mbuf->port];
61032a965aSPavan Nikhilesh 		rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0);
62314bcf58SPavan Nikhilesh 		pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
63314bcf58SPavan Nikhilesh 		pipeline_event_enqueue(dev, port, &ev);
64314bcf58SPavan Nikhilesh 		w->processed_pkts++;
65314bcf58SPavan Nikhilesh 	}
66314bcf58SPavan Nikhilesh 
67314bcf58SPavan Nikhilesh 	return 0;
68314bcf58SPavan Nikhilesh }
69314bcf58SPavan Nikhilesh 
70032a965aSPavan Nikhilesh static __rte_noinline int
71314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_tx(void *arg)
72314bcf58SPavan Nikhilesh {
73f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT;
74314bcf58SPavan Nikhilesh 
75314bcf58SPavan Nikhilesh 	while (t->done == false) {
76314bcf58SPavan Nikhilesh 		uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
77314bcf58SPavan Nikhilesh 				BURST_SIZE, 0);
78314bcf58SPavan Nikhilesh 
79314bcf58SPavan Nikhilesh 		if (!nb_rx) {
80314bcf58SPavan Nikhilesh 			rte_pause();
81314bcf58SPavan Nikhilesh 			continue;
82314bcf58SPavan Nikhilesh 		}
83314bcf58SPavan Nikhilesh 
84314bcf58SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
85314bcf58SPavan Nikhilesh 			rte_prefetch0(ev[i + 1].mbuf);
86314bcf58SPavan Nikhilesh 			if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
87032a965aSPavan Nikhilesh 				pipeline_event_tx(dev, port, &ev[i]);
88314bcf58SPavan Nikhilesh 				w->processed_pkts++;
89314bcf58SPavan Nikhilesh 			} else {
90314bcf58SPavan Nikhilesh 				ev[i].queue_id++;
91314bcf58SPavan Nikhilesh 				pipeline_fwd_event(&ev[i],
92314bcf58SPavan Nikhilesh 						RTE_SCHED_TYPE_ATOMIC);
9321b1ca48SFeifei Wang 				pipeline_event_enqueue_burst(dev, port, ev,
9421b1ca48SFeifei Wang 						nb_rx);
95314bcf58SPavan Nikhilesh 			}
96314bcf58SPavan Nikhilesh 		}
97314bcf58SPavan Nikhilesh 	}
98314bcf58SPavan Nikhilesh 
99314bcf58SPavan Nikhilesh 	return 0;
100314bcf58SPavan Nikhilesh }
101314bcf58SPavan Nikhilesh 
102032a965aSPavan Nikhilesh static __rte_noinline int
103314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_fwd(void *arg)
104314bcf58SPavan Nikhilesh {
105f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT;
106032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
107314bcf58SPavan Nikhilesh 
108314bcf58SPavan Nikhilesh 	while (t->done == false) {
109314bcf58SPavan Nikhilesh 		uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
110314bcf58SPavan Nikhilesh 				BURST_SIZE, 0);
111314bcf58SPavan Nikhilesh 
112314bcf58SPavan Nikhilesh 		if (!nb_rx) {
113314bcf58SPavan Nikhilesh 			rte_pause();
114314bcf58SPavan Nikhilesh 			continue;
115314bcf58SPavan Nikhilesh 		}
116314bcf58SPavan Nikhilesh 
117314bcf58SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
118314bcf58SPavan Nikhilesh 			rte_prefetch0(ev[i + 1].mbuf);
119032a965aSPavan Nikhilesh 			ev[i].queue_id = tx_queue[ev[i].mbuf->port];
120032a965aSPavan Nikhilesh 			rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0);
121314bcf58SPavan Nikhilesh 			pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC);
122314bcf58SPavan Nikhilesh 		}
123314bcf58SPavan Nikhilesh 
124314bcf58SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
125032a965aSPavan Nikhilesh 		w->processed_pkts += nb_rx;
126314bcf58SPavan Nikhilesh 	}
127314bcf58SPavan Nikhilesh 
128314bcf58SPavan Nikhilesh 	return 0;
129314bcf58SPavan Nikhilesh }
130314bcf58SPavan Nikhilesh 
1312eaa37b8SPavan Nikhilesh static __rte_noinline int
1322eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_tx_vector(void *arg)
1332eaa37b8SPavan Nikhilesh {
1342eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_INIT;
1352eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
1362eaa37b8SPavan Nikhilesh 
1372eaa37b8SPavan Nikhilesh 	while (!t->done) {
1382eaa37b8SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
1392eaa37b8SPavan Nikhilesh 
1402eaa37b8SPavan Nikhilesh 		if (!event) {
1412eaa37b8SPavan Nikhilesh 			rte_pause();
1422eaa37b8SPavan Nikhilesh 			continue;
1432eaa37b8SPavan Nikhilesh 		}
1442eaa37b8SPavan Nikhilesh 
1452eaa37b8SPavan Nikhilesh 		if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
1462eaa37b8SPavan Nikhilesh 			vector_sz = ev.vec->nb_elem;
1472eaa37b8SPavan Nikhilesh 			pipeline_event_tx_vector(dev, port, &ev);
1482eaa37b8SPavan Nikhilesh 			w->processed_pkts += vector_sz;
1492eaa37b8SPavan Nikhilesh 		} else {
1502eaa37b8SPavan Nikhilesh 			ev.queue_id++;
1512eaa37b8SPavan Nikhilesh 			pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC);
1522eaa37b8SPavan Nikhilesh 			pipeline_event_enqueue(dev, port, &ev);
1532eaa37b8SPavan Nikhilesh 		}
1542eaa37b8SPavan Nikhilesh 	}
1552eaa37b8SPavan Nikhilesh 
1562eaa37b8SPavan Nikhilesh 	return 0;
1572eaa37b8SPavan Nikhilesh }
1582eaa37b8SPavan Nikhilesh 
1592eaa37b8SPavan Nikhilesh static __rte_noinline int
1602eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_fwd_vector(void *arg)
1612eaa37b8SPavan Nikhilesh {
1622eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_INIT;
1632eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
1642eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
1652eaa37b8SPavan Nikhilesh 
1662eaa37b8SPavan Nikhilesh 	while (!t->done) {
1672eaa37b8SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
1682eaa37b8SPavan Nikhilesh 
1692eaa37b8SPavan Nikhilesh 		if (!event) {
1702eaa37b8SPavan Nikhilesh 			rte_pause();
1712eaa37b8SPavan Nikhilesh 			continue;
1722eaa37b8SPavan Nikhilesh 		}
1732eaa37b8SPavan Nikhilesh 
1742eaa37b8SPavan Nikhilesh 		ev.queue_id = tx_queue[ev.vec->port];
1752eaa37b8SPavan Nikhilesh 		ev.vec->queue = 0;
1762eaa37b8SPavan Nikhilesh 		vector_sz = ev.vec->nb_elem;
1772eaa37b8SPavan Nikhilesh 		pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC);
1782eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue(dev, port, &ev);
1792eaa37b8SPavan Nikhilesh 		w->processed_pkts += vector_sz;
1802eaa37b8SPavan Nikhilesh 	}
1812eaa37b8SPavan Nikhilesh 
1822eaa37b8SPavan Nikhilesh 	return 0;
1832eaa37b8SPavan Nikhilesh }
1842eaa37b8SPavan Nikhilesh 
1852eaa37b8SPavan Nikhilesh static __rte_noinline int
1862eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_tx_vector(void *arg)
1872eaa37b8SPavan Nikhilesh {
1882eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT;
1892eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
1902eaa37b8SPavan Nikhilesh 
1912eaa37b8SPavan Nikhilesh 	while (!t->done) {
1922eaa37b8SPavan Nikhilesh 		uint16_t nb_rx =
1932eaa37b8SPavan Nikhilesh 			rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);
1942eaa37b8SPavan Nikhilesh 
1952eaa37b8SPavan Nikhilesh 		if (!nb_rx) {
1962eaa37b8SPavan Nikhilesh 			rte_pause();
1972eaa37b8SPavan Nikhilesh 			continue;
1982eaa37b8SPavan Nikhilesh 		}
1992eaa37b8SPavan Nikhilesh 
2002eaa37b8SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
2012eaa37b8SPavan Nikhilesh 			if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
2022eaa37b8SPavan Nikhilesh 				vector_sz = ev[i].vec->nb_elem;
2032eaa37b8SPavan Nikhilesh 				pipeline_event_tx_vector(dev, port, &ev[i]);
2042eaa37b8SPavan Nikhilesh 				ev[i].op = RTE_EVENT_OP_RELEASE;
2052eaa37b8SPavan Nikhilesh 				w->processed_pkts += vector_sz;
2062eaa37b8SPavan Nikhilesh 			} else {
2072eaa37b8SPavan Nikhilesh 				ev[i].queue_id++;
2082eaa37b8SPavan Nikhilesh 				pipeline_fwd_event_vector(
2092eaa37b8SPavan Nikhilesh 					&ev[i], RTE_SCHED_TYPE_ATOMIC);
2102eaa37b8SPavan Nikhilesh 			}
2112eaa37b8SPavan Nikhilesh 		}
2122eaa37b8SPavan Nikhilesh 
2132eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
2142eaa37b8SPavan Nikhilesh 	}
2152eaa37b8SPavan Nikhilesh 
2162eaa37b8SPavan Nikhilesh 	return 0;
2172eaa37b8SPavan Nikhilesh }
2182eaa37b8SPavan Nikhilesh 
2192eaa37b8SPavan Nikhilesh static __rte_noinline int
2202eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_fwd_vector(void *arg)
2212eaa37b8SPavan Nikhilesh {
2222eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT;
2232eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
2242eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
2252eaa37b8SPavan Nikhilesh 
2262eaa37b8SPavan Nikhilesh 	while (!t->done) {
2272eaa37b8SPavan Nikhilesh 		uint16_t nb_rx =
2282eaa37b8SPavan Nikhilesh 			rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);
2292eaa37b8SPavan Nikhilesh 
2302eaa37b8SPavan Nikhilesh 		if (!nb_rx) {
2312eaa37b8SPavan Nikhilesh 			rte_pause();
2322eaa37b8SPavan Nikhilesh 			continue;
2332eaa37b8SPavan Nikhilesh 		}
2342eaa37b8SPavan Nikhilesh 
2352eaa37b8SPavan Nikhilesh 		vector_sz = 0;
2362eaa37b8SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
2372eaa37b8SPavan Nikhilesh 			ev[i].queue_id = tx_queue[ev[i].vec->port];
2382eaa37b8SPavan Nikhilesh 			ev[i].vec->queue = 0;
2392eaa37b8SPavan Nikhilesh 			vector_sz += ev[i].vec->nb_elem;
2402eaa37b8SPavan Nikhilesh 			pipeline_fwd_event_vector(&ev[i],
2412eaa37b8SPavan Nikhilesh 						  RTE_SCHED_TYPE_ATOMIC);
2422eaa37b8SPavan Nikhilesh 		}
2432eaa37b8SPavan Nikhilesh 
2442eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
2452eaa37b8SPavan Nikhilesh 		w->processed_pkts += vector_sz;
2462eaa37b8SPavan Nikhilesh 	}
2472eaa37b8SPavan Nikhilesh 
2482eaa37b8SPavan Nikhilesh 	return 0;
2492eaa37b8SPavan Nikhilesh }
250314bcf58SPavan Nikhilesh 
251032a965aSPavan Nikhilesh static __rte_noinline int
252314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_tx(void *arg)
253314bcf58SPavan Nikhilesh {
254f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_INIT;
255032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
256314bcf58SPavan Nikhilesh 
257314bcf58SPavan Nikhilesh 	while (t->done == false) {
258314bcf58SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
259314bcf58SPavan Nikhilesh 
260314bcf58SPavan Nikhilesh 		if (!event) {
261314bcf58SPavan Nikhilesh 			rte_pause();
262314bcf58SPavan Nikhilesh 			continue;
263314bcf58SPavan Nikhilesh 		}
264314bcf58SPavan Nikhilesh 
265314bcf58SPavan Nikhilesh 		cq_id = ev.queue_id % nb_stages;
266314bcf58SPavan Nikhilesh 
267032a965aSPavan Nikhilesh 		if (ev.queue_id == tx_queue[ev.mbuf->port]) {
268032a965aSPavan Nikhilesh 			pipeline_event_tx(dev, port, &ev);
269314bcf58SPavan Nikhilesh 			w->processed_pkts++;
270314bcf58SPavan Nikhilesh 			continue;
271314bcf58SPavan Nikhilesh 		}
272314bcf58SPavan Nikhilesh 
273032a965aSPavan Nikhilesh 		ev.queue_id++;
274032a965aSPavan Nikhilesh 		pipeline_fwd_event(&ev, cq_id != last_queue ?
275032a965aSPavan Nikhilesh 				sched_type_list[cq_id] :
276032a965aSPavan Nikhilesh 				RTE_SCHED_TYPE_ATOMIC);
277314bcf58SPavan Nikhilesh 		pipeline_event_enqueue(dev, port, &ev);
278314bcf58SPavan Nikhilesh 	}
279032a965aSPavan Nikhilesh 
280314bcf58SPavan Nikhilesh 	return 0;
281314bcf58SPavan Nikhilesh }
282314bcf58SPavan Nikhilesh 
283032a965aSPavan Nikhilesh static __rte_noinline int
284314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_fwd(void *arg)
285314bcf58SPavan Nikhilesh {
286f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_INIT;
287032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
288314bcf58SPavan Nikhilesh 
289314bcf58SPavan Nikhilesh 	while (t->done == false) {
290314bcf58SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
291314bcf58SPavan Nikhilesh 
292314bcf58SPavan Nikhilesh 		if (!event) {
293314bcf58SPavan Nikhilesh 			rte_pause();
294314bcf58SPavan Nikhilesh 			continue;
295314bcf58SPavan Nikhilesh 		}
296314bcf58SPavan Nikhilesh 
297314bcf58SPavan Nikhilesh 		cq_id = ev.queue_id % nb_stages;
298314bcf58SPavan Nikhilesh 
299314bcf58SPavan Nikhilesh 		if (cq_id == last_queue) {
300032a965aSPavan Nikhilesh 			ev.queue_id = tx_queue[ev.mbuf->port];
301032a965aSPavan Nikhilesh 			rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0);
302314bcf58SPavan Nikhilesh 			pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
303e0c05737SFeifei Wang 			pipeline_event_enqueue(dev, port, &ev);
304314bcf58SPavan Nikhilesh 			w->processed_pkts++;
305314bcf58SPavan Nikhilesh 		} else {
306314bcf58SPavan Nikhilesh 			ev.queue_id++;
307314bcf58SPavan Nikhilesh 			pipeline_fwd_event(&ev, sched_type_list[cq_id]);
308314bcf58SPavan Nikhilesh 			pipeline_event_enqueue(dev, port, &ev);
309314bcf58SPavan Nikhilesh 		}
310e0c05737SFeifei Wang 	}
311032a965aSPavan Nikhilesh 
312314bcf58SPavan Nikhilesh 	return 0;
313314bcf58SPavan Nikhilesh }
314314bcf58SPavan Nikhilesh 
315032a965aSPavan Nikhilesh static __rte_noinline int
316314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_tx(void *arg)
317314bcf58SPavan Nikhilesh {
318f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
319032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
320314bcf58SPavan Nikhilesh 
321314bcf58SPavan Nikhilesh 	while (t->done == false) {
322314bcf58SPavan Nikhilesh 		uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
323314bcf58SPavan Nikhilesh 				BURST_SIZE, 0);
324314bcf58SPavan Nikhilesh 
325314bcf58SPavan Nikhilesh 		if (!nb_rx) {
326314bcf58SPavan Nikhilesh 			rte_pause();
327314bcf58SPavan Nikhilesh 			continue;
328314bcf58SPavan Nikhilesh 		}
329314bcf58SPavan Nikhilesh 
330314bcf58SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
331314bcf58SPavan Nikhilesh 			rte_prefetch0(ev[i + 1].mbuf);
332314bcf58SPavan Nikhilesh 			cq_id = ev[i].queue_id % nb_stages;
333314bcf58SPavan Nikhilesh 
334032a965aSPavan Nikhilesh 			if (ev[i].queue_id == tx_queue[ev[i].mbuf->port]) {
335032a965aSPavan Nikhilesh 				pipeline_event_tx(dev, port, &ev[i]);
336314bcf58SPavan Nikhilesh 				w->processed_pkts++;
337314bcf58SPavan Nikhilesh 				continue;
338314bcf58SPavan Nikhilesh 			}
339314bcf58SPavan Nikhilesh 
340314bcf58SPavan Nikhilesh 			ev[i].queue_id++;
341032a965aSPavan Nikhilesh 			pipeline_fwd_event(&ev[i], cq_id != last_queue ?
342032a965aSPavan Nikhilesh 					sched_type_list[cq_id] :
343032a965aSPavan Nikhilesh 					RTE_SCHED_TYPE_ATOMIC);
344314bcf58SPavan Nikhilesh 			pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
345314bcf58SPavan Nikhilesh 		}
34621b1ca48SFeifei Wang 	}
347032a965aSPavan Nikhilesh 
348314bcf58SPavan Nikhilesh 	return 0;
349314bcf58SPavan Nikhilesh }
350314bcf58SPavan Nikhilesh 
351032a965aSPavan Nikhilesh static __rte_noinline int
352314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_fwd(void *arg)
353314bcf58SPavan Nikhilesh {
354f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
355032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
356314bcf58SPavan Nikhilesh 
357314bcf58SPavan Nikhilesh 	while (t->done == false) {
358e0c05737SFeifei Wang 		uint16_t processed_pkts = 0;
359314bcf58SPavan Nikhilesh 		uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
360314bcf58SPavan Nikhilesh 				BURST_SIZE, 0);
361314bcf58SPavan Nikhilesh 
362314bcf58SPavan Nikhilesh 		if (!nb_rx) {
363314bcf58SPavan Nikhilesh 			rte_pause();
364314bcf58SPavan Nikhilesh 			continue;
365314bcf58SPavan Nikhilesh 		}
366314bcf58SPavan Nikhilesh 
367314bcf58SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
368314bcf58SPavan Nikhilesh 			rte_prefetch0(ev[i + 1].mbuf);
369314bcf58SPavan Nikhilesh 			cq_id = ev[i].queue_id % nb_stages;
370314bcf58SPavan Nikhilesh 
371314bcf58SPavan Nikhilesh 			if (cq_id == last_queue) {
372032a965aSPavan Nikhilesh 				ev[i].queue_id = tx_queue[ev[i].mbuf->port];
373032a965aSPavan Nikhilesh 				rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0);
374314bcf58SPavan Nikhilesh 				pipeline_fwd_event(&ev[i],
375314bcf58SPavan Nikhilesh 						RTE_SCHED_TYPE_ATOMIC);
376e0c05737SFeifei Wang 				processed_pkts++;
377314bcf58SPavan Nikhilesh 			} else {
378314bcf58SPavan Nikhilesh 				ev[i].queue_id++;
379314bcf58SPavan Nikhilesh 				pipeline_fwd_event(&ev[i],
380314bcf58SPavan Nikhilesh 						sched_type_list[cq_id]);
381314bcf58SPavan Nikhilesh 			}
382314bcf58SPavan Nikhilesh 		}
383314bcf58SPavan Nikhilesh 
384314bcf58SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
385e0c05737SFeifei Wang 		w->processed_pkts += processed_pkts;
386314bcf58SPavan Nikhilesh 	}
387032a965aSPavan Nikhilesh 
388314bcf58SPavan Nikhilesh 	return 0;
389314bcf58SPavan Nikhilesh }
390314bcf58SPavan Nikhilesh 
3912eaa37b8SPavan Nikhilesh static __rte_noinline int
3922eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_tx_vector(void *arg)
3932eaa37b8SPavan Nikhilesh {
3942eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_INIT;
3952eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
3962eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
3972eaa37b8SPavan Nikhilesh 
3982eaa37b8SPavan Nikhilesh 	while (!t->done) {
3992eaa37b8SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
4002eaa37b8SPavan Nikhilesh 
4012eaa37b8SPavan Nikhilesh 		if (!event) {
4022eaa37b8SPavan Nikhilesh 			rte_pause();
4032eaa37b8SPavan Nikhilesh 			continue;
4042eaa37b8SPavan Nikhilesh 		}
4052eaa37b8SPavan Nikhilesh 
4062eaa37b8SPavan Nikhilesh 		cq_id = ev.queue_id % nb_stages;
4072eaa37b8SPavan Nikhilesh 
4082eaa37b8SPavan Nikhilesh 		if (ev.queue_id == tx_queue[ev.vec->port]) {
4092eaa37b8SPavan Nikhilesh 			vector_sz = ev.vec->nb_elem;
4102eaa37b8SPavan Nikhilesh 			pipeline_event_tx_vector(dev, port, &ev);
4112eaa37b8SPavan Nikhilesh 			w->processed_pkts += vector_sz;
4122eaa37b8SPavan Nikhilesh 			continue;
4132eaa37b8SPavan Nikhilesh 		}
4142eaa37b8SPavan Nikhilesh 
4152eaa37b8SPavan Nikhilesh 		ev.queue_id++;
4162eaa37b8SPavan Nikhilesh 		pipeline_fwd_event_vector(&ev, cq_id != last_queue
4172eaa37b8SPavan Nikhilesh 						       ? sched_type_list[cq_id]
4182eaa37b8SPavan Nikhilesh 						       : RTE_SCHED_TYPE_ATOMIC);
4192eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue(dev, port, &ev);
4202eaa37b8SPavan Nikhilesh 	}
4212eaa37b8SPavan Nikhilesh 
4222eaa37b8SPavan Nikhilesh 	return 0;
4232eaa37b8SPavan Nikhilesh }
4242eaa37b8SPavan Nikhilesh 
4252eaa37b8SPavan Nikhilesh static __rte_noinline int
4262eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_fwd_vector(void *arg)
4272eaa37b8SPavan Nikhilesh {
4282eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_INIT;
4292eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
4302eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
4312eaa37b8SPavan Nikhilesh 
4322eaa37b8SPavan Nikhilesh 	while (!t->done) {
4332eaa37b8SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
4342eaa37b8SPavan Nikhilesh 
4352eaa37b8SPavan Nikhilesh 		if (!event) {
4362eaa37b8SPavan Nikhilesh 			rte_pause();
4372eaa37b8SPavan Nikhilesh 			continue;
4382eaa37b8SPavan Nikhilesh 		}
4392eaa37b8SPavan Nikhilesh 
4402eaa37b8SPavan Nikhilesh 		cq_id = ev.queue_id % nb_stages;
4412eaa37b8SPavan Nikhilesh 
4422eaa37b8SPavan Nikhilesh 		if (cq_id == last_queue) {
4432eaa37b8SPavan Nikhilesh 			vector_sz = ev.vec->nb_elem;
4442eaa37b8SPavan Nikhilesh 			ev.queue_id = tx_queue[ev.vec->port];
4452eaa37b8SPavan Nikhilesh 			pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC);
4462eaa37b8SPavan Nikhilesh 			w->processed_pkts += vector_sz;
4472eaa37b8SPavan Nikhilesh 		} else {
4482eaa37b8SPavan Nikhilesh 			ev.queue_id++;
4492eaa37b8SPavan Nikhilesh 			pipeline_fwd_event_vector(&ev, sched_type_list[cq_id]);
4502eaa37b8SPavan Nikhilesh 		}
4512eaa37b8SPavan Nikhilesh 
4522eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue(dev, port, &ev);
4532eaa37b8SPavan Nikhilesh 	}
4542eaa37b8SPavan Nikhilesh 
4552eaa37b8SPavan Nikhilesh 	return 0;
4562eaa37b8SPavan Nikhilesh }
4572eaa37b8SPavan Nikhilesh 
4582eaa37b8SPavan Nikhilesh static __rte_noinline int
4592eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_tx_vector(void *arg)
4602eaa37b8SPavan Nikhilesh {
4612eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
4622eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
4632eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
4642eaa37b8SPavan Nikhilesh 
4652eaa37b8SPavan Nikhilesh 	while (!t->done) {
4662eaa37b8SPavan Nikhilesh 		uint16_t nb_rx =
4672eaa37b8SPavan Nikhilesh 			rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);
4682eaa37b8SPavan Nikhilesh 
4692eaa37b8SPavan Nikhilesh 		if (!nb_rx) {
4702eaa37b8SPavan Nikhilesh 			rte_pause();
4712eaa37b8SPavan Nikhilesh 			continue;
4722eaa37b8SPavan Nikhilesh 		}
4732eaa37b8SPavan Nikhilesh 
4742eaa37b8SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
4752eaa37b8SPavan Nikhilesh 			cq_id = ev[i].queue_id % nb_stages;
4762eaa37b8SPavan Nikhilesh 
4772eaa37b8SPavan Nikhilesh 			if (ev[i].queue_id == tx_queue[ev[i].vec->port]) {
4782eaa37b8SPavan Nikhilesh 				vector_sz = ev[i].vec->nb_elem;
4792eaa37b8SPavan Nikhilesh 				pipeline_event_tx_vector(dev, port, &ev[i]);
4802eaa37b8SPavan Nikhilesh 				ev[i].op = RTE_EVENT_OP_RELEASE;
4812eaa37b8SPavan Nikhilesh 				w->processed_pkts += vector_sz;
4822eaa37b8SPavan Nikhilesh 				continue;
4832eaa37b8SPavan Nikhilesh 			}
4842eaa37b8SPavan Nikhilesh 
4852eaa37b8SPavan Nikhilesh 			ev[i].queue_id++;
4862eaa37b8SPavan Nikhilesh 			pipeline_fwd_event_vector(
4872eaa37b8SPavan Nikhilesh 				&ev[i], cq_id != last_queue
4882eaa37b8SPavan Nikhilesh 						? sched_type_list[cq_id]
4892eaa37b8SPavan Nikhilesh 						: RTE_SCHED_TYPE_ATOMIC);
4902eaa37b8SPavan Nikhilesh 		}
4912eaa37b8SPavan Nikhilesh 
4922eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
4932eaa37b8SPavan Nikhilesh 	}
4942eaa37b8SPavan Nikhilesh 
4952eaa37b8SPavan Nikhilesh 	return 0;
4962eaa37b8SPavan Nikhilesh }
4972eaa37b8SPavan Nikhilesh 
4982eaa37b8SPavan Nikhilesh static __rte_noinline int
4992eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_fwd_vector(void *arg)
5002eaa37b8SPavan Nikhilesh {
5012eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
5022eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
5032eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
5042eaa37b8SPavan Nikhilesh 
5052eaa37b8SPavan Nikhilesh 	while (!t->done) {
5062eaa37b8SPavan Nikhilesh 		uint16_t nb_rx =
5072eaa37b8SPavan Nikhilesh 			rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);
5082eaa37b8SPavan Nikhilesh 
5092eaa37b8SPavan Nikhilesh 		if (!nb_rx) {
5102eaa37b8SPavan Nikhilesh 			rte_pause();
5112eaa37b8SPavan Nikhilesh 			continue;
5122eaa37b8SPavan Nikhilesh 		}
5132eaa37b8SPavan Nikhilesh 
5142eaa37b8SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
5152eaa37b8SPavan Nikhilesh 			cq_id = ev[i].queue_id % nb_stages;
5162eaa37b8SPavan Nikhilesh 
5172eaa37b8SPavan Nikhilesh 			if (cq_id == last_queue) {
5182eaa37b8SPavan Nikhilesh 				ev[i].queue_id = tx_queue[ev[i].vec->port];
5192eaa37b8SPavan Nikhilesh 				vector_sz = ev[i].vec->nb_elem;
5202eaa37b8SPavan Nikhilesh 				pipeline_fwd_event_vector(
5212eaa37b8SPavan Nikhilesh 					&ev[i], RTE_SCHED_TYPE_ATOMIC);
5222eaa37b8SPavan Nikhilesh 				w->processed_pkts += vector_sz;
5232eaa37b8SPavan Nikhilesh 			} else {
5242eaa37b8SPavan Nikhilesh 				ev[i].queue_id++;
5252eaa37b8SPavan Nikhilesh 				pipeline_fwd_event_vector(
5262eaa37b8SPavan Nikhilesh 					&ev[i], sched_type_list[cq_id]);
5272eaa37b8SPavan Nikhilesh 			}
5282eaa37b8SPavan Nikhilesh 		}
5292eaa37b8SPavan Nikhilesh 
5302eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
5312eaa37b8SPavan Nikhilesh 	}
5322eaa37b8SPavan Nikhilesh 
5332eaa37b8SPavan Nikhilesh 	return 0;
5342eaa37b8SPavan Nikhilesh }
5352eaa37b8SPavan Nikhilesh 
536314bcf58SPavan Nikhilesh static int
537d60b4185SPavan Nikhilesh worker_wrapper(void *arg)
538d60b4185SPavan Nikhilesh {
539314bcf58SPavan Nikhilesh 	struct worker_data *w  = arg;
540314bcf58SPavan Nikhilesh 	struct evt_options *opt = w->t->opt;
541314bcf58SPavan Nikhilesh 	const bool burst = evt_has_burst_mode(w->dev_id);
542032a965aSPavan Nikhilesh 	const bool internal_port = w->t->internal_port;
543314bcf58SPavan Nikhilesh 	const uint8_t nb_stages = opt->nb_stages;
5442eaa37b8SPavan Nikhilesh 	/*vector/burst/internal_port*/
5452eaa37b8SPavan Nikhilesh 	const pipeline_queue_worker_t
5462eaa37b8SPavan Nikhilesh 	pipeline_queue_worker_single_stage[2][2][2] = {
5472eaa37b8SPavan Nikhilesh 		[0][0][0] = pipeline_queue_worker_single_stage_fwd,
5482eaa37b8SPavan Nikhilesh 		[0][0][1] = pipeline_queue_worker_single_stage_tx,
5492eaa37b8SPavan Nikhilesh 		[0][1][0] = pipeline_queue_worker_single_stage_burst_fwd,
5502eaa37b8SPavan Nikhilesh 		[0][1][1] = pipeline_queue_worker_single_stage_burst_tx,
5512eaa37b8SPavan Nikhilesh 		[1][0][0] = pipeline_queue_worker_single_stage_fwd_vector,
5522eaa37b8SPavan Nikhilesh 		[1][0][1] = pipeline_queue_worker_single_stage_tx_vector,
5532eaa37b8SPavan Nikhilesh 		[1][1][0] = pipeline_queue_worker_single_stage_burst_fwd_vector,
5542eaa37b8SPavan Nikhilesh 		[1][1][1] = pipeline_queue_worker_single_stage_burst_tx_vector,
5552eaa37b8SPavan Nikhilesh 	};
5562eaa37b8SPavan Nikhilesh 	const pipeline_queue_worker_t
5572eaa37b8SPavan Nikhilesh 	pipeline_queue_worker_multi_stage[2][2][2] = {
5582eaa37b8SPavan Nikhilesh 		[0][0][0] = pipeline_queue_worker_multi_stage_fwd,
5592eaa37b8SPavan Nikhilesh 		[0][0][1] = pipeline_queue_worker_multi_stage_tx,
5602eaa37b8SPavan Nikhilesh 		[0][1][0] = pipeline_queue_worker_multi_stage_burst_fwd,
5612eaa37b8SPavan Nikhilesh 		[0][1][1] = pipeline_queue_worker_multi_stage_burst_tx,
5622eaa37b8SPavan Nikhilesh 		[1][0][0] = pipeline_queue_worker_multi_stage_fwd_vector,
5632eaa37b8SPavan Nikhilesh 		[1][0][1] = pipeline_queue_worker_multi_stage_tx_vector,
5642eaa37b8SPavan Nikhilesh 		[1][1][0] = pipeline_queue_worker_multi_stage_burst_fwd_vector,
5652eaa37b8SPavan Nikhilesh 		[1][1][1] = pipeline_queue_worker_multi_stage_burst_tx_vector,
5662eaa37b8SPavan Nikhilesh 	};
567314bcf58SPavan Nikhilesh 
5682eaa37b8SPavan Nikhilesh 	if (nb_stages == 1)
5692eaa37b8SPavan Nikhilesh 		return (pipeline_queue_worker_single_stage[opt->ena_vector]
5702eaa37b8SPavan Nikhilesh 							  [burst]
5712eaa37b8SPavan Nikhilesh 							  [internal_port])(arg);
5722eaa37b8SPavan Nikhilesh 	else
5732eaa37b8SPavan Nikhilesh 		return (pipeline_queue_worker_multi_stage[opt->ena_vector]
5742eaa37b8SPavan Nikhilesh 							 [burst]
5752eaa37b8SPavan Nikhilesh 							 [internal_port])(arg);
576314bcf58SPavan Nikhilesh 
577d60b4185SPavan Nikhilesh 	rte_panic("invalid worker\n");
578d60b4185SPavan Nikhilesh }
579d60b4185SPavan Nikhilesh 
580d60b4185SPavan Nikhilesh static int
581d60b4185SPavan Nikhilesh pipeline_queue_launch_lcores(struct evt_test *test, struct evt_options *opt)
582d60b4185SPavan Nikhilesh {
583d60b4185SPavan Nikhilesh 	return pipeline_launch_lcores(test, opt, worker_wrapper);
584d60b4185SPavan Nikhilesh }
585d60b4185SPavan Nikhilesh 
586d60b4185SPavan Nikhilesh static int
587d60b4185SPavan Nikhilesh pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
588d60b4185SPavan Nikhilesh {
589d60b4185SPavan Nikhilesh 	int ret;
590d60b4185SPavan Nikhilesh 	int nb_ports;
591d60b4185SPavan Nikhilesh 	int nb_queues;
592d60b4185SPavan Nikhilesh 	int nb_stages = opt->nb_stages;
593d60b4185SPavan Nikhilesh 	uint8_t queue;
594032a965aSPavan Nikhilesh 	uint8_t tx_evport_id = 0;
595032a965aSPavan Nikhilesh 	uint8_t tx_evqueue_id[RTE_MAX_ETHPORTS];
596d60b4185SPavan Nikhilesh 	uint8_t queue_arr[RTE_EVENT_MAX_QUEUES_PER_DEV];
597d60b4185SPavan Nikhilesh 	uint8_t nb_worker_queues = 0;
598032a965aSPavan Nikhilesh 	uint16_t prod = 0;
599032a965aSPavan Nikhilesh 	struct rte_event_dev_info info;
600032a965aSPavan Nikhilesh 	struct test_pipeline *t = evt_test_priv(test);
601d60b4185SPavan Nikhilesh 
602d60b4185SPavan Nikhilesh 	nb_ports = evt_nr_active_lcores(opt->wlcores);
603d9a42a69SThomas Monjalon 	nb_queues = rte_eth_dev_count_avail() * (nb_stages);
604d60b4185SPavan Nikhilesh 
605032a965aSPavan Nikhilesh 	/* One queue for Tx adapter per port */
606d9a42a69SThomas Monjalon 	nb_queues += rte_eth_dev_count_avail();
607d60b4185SPavan Nikhilesh 
608032a965aSPavan Nikhilesh 	memset(tx_evqueue_id, 0, sizeof(uint8_t) * RTE_MAX_ETHPORTS);
609032a965aSPavan Nikhilesh 	memset(queue_arr, 0, sizeof(uint8_t) * RTE_EVENT_MAX_QUEUES_PER_DEV);
610d60b4185SPavan Nikhilesh 
611032a965aSPavan Nikhilesh 	rte_event_dev_info_get(opt->dev_id, &info);
612f0959283SPavan Nikhilesh 	ret = evt_configure_eventdev(opt, nb_queues, nb_ports);
613d60b4185SPavan Nikhilesh 	if (ret) {
614d60b4185SPavan Nikhilesh 		evt_err("failed to configure eventdev %d", opt->dev_id);
615d60b4185SPavan Nikhilesh 		return ret;
616d60b4185SPavan Nikhilesh 	}
617d60b4185SPavan Nikhilesh 
618d60b4185SPavan Nikhilesh 	struct rte_event_queue_conf q_conf = {
619d60b4185SPavan Nikhilesh 			.priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
620d60b4185SPavan Nikhilesh 			.nb_atomic_flows = opt->nb_flows,
621d60b4185SPavan Nikhilesh 			.nb_atomic_order_sequences = opt->nb_flows,
622d60b4185SPavan Nikhilesh 	};
623d60b4185SPavan Nikhilesh 	/* queue configurations */
624d60b4185SPavan Nikhilesh 	for (queue = 0; queue < nb_queues; queue++) {
625d60b4185SPavan Nikhilesh 		uint8_t slot;
626d60b4185SPavan Nikhilesh 
627032a965aSPavan Nikhilesh 		q_conf.event_queue_cfg = 0;
628d60b4185SPavan Nikhilesh 		slot = queue % (nb_stages + 1);
629032a965aSPavan Nikhilesh 		if (slot == nb_stages) {
630d60b4185SPavan Nikhilesh 			q_conf.schedule_type = RTE_SCHED_TYPE_ATOMIC;
631032a965aSPavan Nikhilesh 			if (!t->internal_port) {
632d60b4185SPavan Nikhilesh 				q_conf.event_queue_cfg =
633d60b4185SPavan Nikhilesh 					RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
634032a965aSPavan Nikhilesh 			}
635032a965aSPavan Nikhilesh 			tx_evqueue_id[prod++] = queue;
636d60b4185SPavan Nikhilesh 		} else {
637032a965aSPavan Nikhilesh 			q_conf.schedule_type = opt->sched_type_list[slot];
638d60b4185SPavan Nikhilesh 			queue_arr[nb_worker_queues] = queue;
639d60b4185SPavan Nikhilesh 			nb_worker_queues++;
640d60b4185SPavan Nikhilesh 		}
641d60b4185SPavan Nikhilesh 
642d60b4185SPavan Nikhilesh 		ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf);
643d60b4185SPavan Nikhilesh 		if (ret) {
644d60b4185SPavan Nikhilesh 			evt_err("failed to setup queue=%d", queue);
645d60b4185SPavan Nikhilesh 			return ret;
646d60b4185SPavan Nikhilesh 		}
647d60b4185SPavan Nikhilesh 	}
648d60b4185SPavan Nikhilesh 
649535c630cSPavan Nikhilesh 	if (opt->wkr_deq_dep > info.max_event_port_dequeue_depth)
650535c630cSPavan Nikhilesh 		opt->wkr_deq_dep = info.max_event_port_dequeue_depth;
651535c630cSPavan Nikhilesh 
652d60b4185SPavan Nikhilesh 	/* port configuration */
653d60b4185SPavan Nikhilesh 	const struct rte_event_port_conf p_conf = {
654d60b4185SPavan Nikhilesh 			.dequeue_depth = opt->wkr_deq_dep,
655d60b4185SPavan Nikhilesh 			.enqueue_depth = info.max_event_port_dequeue_depth,
656d60b4185SPavan Nikhilesh 			.new_event_threshold = info.max_num_events,
657d60b4185SPavan Nikhilesh 	};
658d60b4185SPavan Nikhilesh 
659032a965aSPavan Nikhilesh 	if (!t->internal_port) {
660d60b4185SPavan Nikhilesh 		ret = pipeline_event_port_setup(test, opt, queue_arr,
661d60b4185SPavan Nikhilesh 				nb_worker_queues, p_conf);
662d60b4185SPavan Nikhilesh 		if (ret)
663d60b4185SPavan Nikhilesh 			return ret;
664d60b4185SPavan Nikhilesh 	} else
665d60b4185SPavan Nikhilesh 		ret = pipeline_event_port_setup(test, opt, NULL, nb_queues,
666d60b4185SPavan Nikhilesh 				p_conf);
667d60b4185SPavan Nikhilesh 
668d60b4185SPavan Nikhilesh 	if (ret)
669d60b4185SPavan Nikhilesh 		return ret;
670d60b4185SPavan Nikhilesh 	/*
671d60b4185SPavan Nikhilesh 	 * The pipelines are setup in the following manner:
672d60b4185SPavan Nikhilesh 	 *
673d60b4185SPavan Nikhilesh 	 * eth_dev_count = 2, nb_stages = 2.
674d60b4185SPavan Nikhilesh 	 *
675d60b4185SPavan Nikhilesh 	 *	queues = 6
676d60b4185SPavan Nikhilesh 	 *	stride = 3
677d60b4185SPavan Nikhilesh 	 *
678d60b4185SPavan Nikhilesh 	 *	event queue pipelines:
679d60b4185SPavan Nikhilesh 	 *	eth0 -> q0 -> q1 -> (q2->tx)
680d60b4185SPavan Nikhilesh 	 *	eth1 -> q3 -> q4 -> (q5->tx)
681d60b4185SPavan Nikhilesh 	 *
682032a965aSPavan Nikhilesh 	 *	q2, q5 configured as ATOMIC | SINGLE_LINK
683d60b4185SPavan Nikhilesh 	 *
684d60b4185SPavan Nikhilesh 	 */
685032a965aSPavan Nikhilesh 	ret = pipeline_event_rx_adapter_setup(opt, nb_stages + 1, p_conf);
686032a965aSPavan Nikhilesh 	if (ret)
687032a965aSPavan Nikhilesh 		return ret;
688032a965aSPavan Nikhilesh 
689032a965aSPavan Nikhilesh 	ret = pipeline_event_tx_adapter_setup(opt, p_conf);
690d60b4185SPavan Nikhilesh 	if (ret)
691d60b4185SPavan Nikhilesh 		return ret;
692d60b4185SPavan Nikhilesh 
693d60b4185SPavan Nikhilesh 	if (!evt_has_distributed_sched(opt->dev_id)) {
694d60b4185SPavan Nikhilesh 		uint32_t service_id;
695d60b4185SPavan Nikhilesh 		rte_event_dev_service_id_get(opt->dev_id, &service_id);
696d60b4185SPavan Nikhilesh 		ret = evt_service_setup(service_id);
697d60b4185SPavan Nikhilesh 		if (ret) {
698d60b4185SPavan Nikhilesh 			evt_err("No service lcore found to run event dev.");
699d60b4185SPavan Nikhilesh 			return ret;
700d60b4185SPavan Nikhilesh 		}
701d60b4185SPavan Nikhilesh 	}
702d60b4185SPavan Nikhilesh 
703032a965aSPavan Nikhilesh 	/* Connect the tx_evqueue_id to the Tx adapter port */
704032a965aSPavan Nikhilesh 	if (!t->internal_port) {
705032a965aSPavan Nikhilesh 		RTE_ETH_FOREACH_DEV(prod) {
706032a965aSPavan Nikhilesh 			ret = rte_event_eth_tx_adapter_event_port_get(prod,
707032a965aSPavan Nikhilesh 					&tx_evport_id);
708032a965aSPavan Nikhilesh 			if (ret) {
709032a965aSPavan Nikhilesh 				evt_err("Unable to get Tx adptr[%d] evprt[%d]",
710032a965aSPavan Nikhilesh 						prod, tx_evport_id);
711032a965aSPavan Nikhilesh 				return ret;
712032a965aSPavan Nikhilesh 			}
713032a965aSPavan Nikhilesh 
714032a965aSPavan Nikhilesh 			if (rte_event_port_link(opt->dev_id, tx_evport_id,
715032a965aSPavan Nikhilesh 						&tx_evqueue_id[prod],
716032a965aSPavan Nikhilesh 						NULL, 1) != 1) {
717032a965aSPavan Nikhilesh 				evt_err("Unable to link Tx adptr[%d] evprt[%d]",
718032a965aSPavan Nikhilesh 						prod, tx_evport_id);
719032a965aSPavan Nikhilesh 				return ret;
720032a965aSPavan Nikhilesh 			}
721032a965aSPavan Nikhilesh 		}
722032a965aSPavan Nikhilesh 	}
723032a965aSPavan Nikhilesh 
72466b82db2SPavan Nikhilesh 	ret = rte_event_dev_start(opt->dev_id);
72566b82db2SPavan Nikhilesh 	if (ret) {
72666b82db2SPavan Nikhilesh 		evt_err("failed to start eventdev %d", opt->dev_id);
72766b82db2SPavan Nikhilesh 		return ret;
72866b82db2SPavan Nikhilesh 	}
72966b82db2SPavan Nikhilesh 
73066b82db2SPavan Nikhilesh 
731032a965aSPavan Nikhilesh 	RTE_ETH_FOREACH_DEV(prod) {
732032a965aSPavan Nikhilesh 		ret = rte_eth_dev_start(prod);
733032a965aSPavan Nikhilesh 		if (ret) {
734032a965aSPavan Nikhilesh 			evt_err("Ethernet dev [%d] failed to start."
735032a965aSPavan Nikhilesh 					" Using synthetic producer", prod);
736032a965aSPavan Nikhilesh 			return ret;
737032a965aSPavan Nikhilesh 		}
738032a965aSPavan Nikhilesh 
739032a965aSPavan Nikhilesh 	}
740032a965aSPavan Nikhilesh 
741032a965aSPavan Nikhilesh 	RTE_ETH_FOREACH_DEV(prod) {
742032a965aSPavan Nikhilesh 		ret = rte_event_eth_rx_adapter_start(prod);
743032a965aSPavan Nikhilesh 		if (ret) {
744032a965aSPavan Nikhilesh 			evt_err("Rx adapter[%d] start failed", prod);
745032a965aSPavan Nikhilesh 			return ret;
746032a965aSPavan Nikhilesh 		}
747032a965aSPavan Nikhilesh 
748032a965aSPavan Nikhilesh 		ret = rte_event_eth_tx_adapter_start(prod);
749032a965aSPavan Nikhilesh 		if (ret) {
750032a965aSPavan Nikhilesh 			evt_err("Tx adapter[%d] start failed", prod);
751032a965aSPavan Nikhilesh 			return ret;
752032a965aSPavan Nikhilesh 		}
753032a965aSPavan Nikhilesh 	}
754032a965aSPavan Nikhilesh 
755032a965aSPavan Nikhilesh 	memcpy(t->tx_evqueue_id, tx_evqueue_id, sizeof(uint8_t) *
756032a965aSPavan Nikhilesh 			RTE_MAX_ETHPORTS);
757032a965aSPavan Nikhilesh 
758d60b4185SPavan Nikhilesh 	return 0;
759d60b4185SPavan Nikhilesh }
760d60b4185SPavan Nikhilesh 
761d60b4185SPavan Nikhilesh static void
762d60b4185SPavan Nikhilesh pipeline_queue_opt_dump(struct evt_options *opt)
763d60b4185SPavan Nikhilesh {
764d60b4185SPavan Nikhilesh 	pipeline_opt_dump(opt, pipeline_queue_nb_event_queues(opt));
765d60b4185SPavan Nikhilesh }
766d60b4185SPavan Nikhilesh 
767d60b4185SPavan Nikhilesh static int
768d60b4185SPavan Nikhilesh pipeline_queue_opt_check(struct evt_options *opt)
769d60b4185SPavan Nikhilesh {
770d60b4185SPavan Nikhilesh 	return pipeline_opt_check(opt, pipeline_queue_nb_event_queues(opt));
771d60b4185SPavan Nikhilesh }
772d60b4185SPavan Nikhilesh 
773d60b4185SPavan Nikhilesh static bool
774d60b4185SPavan Nikhilesh pipeline_queue_capability_check(struct evt_options *opt)
775d60b4185SPavan Nikhilesh {
776d60b4185SPavan Nikhilesh 	struct rte_event_dev_info dev_info;
777d60b4185SPavan Nikhilesh 
778d60b4185SPavan Nikhilesh 	rte_event_dev_info_get(opt->dev_id, &dev_info);
779d60b4185SPavan Nikhilesh 	if (dev_info.max_event_queues < pipeline_queue_nb_event_queues(opt) ||
780d60b4185SPavan Nikhilesh 			dev_info.max_event_ports <
781d60b4185SPavan Nikhilesh 			evt_nr_active_lcores(opt->wlcores)) {
782d60b4185SPavan Nikhilesh 		evt_err("not enough eventdev queues=%d/%d or ports=%d/%d",
783d60b4185SPavan Nikhilesh 			pipeline_queue_nb_event_queues(opt),
784d60b4185SPavan Nikhilesh 			dev_info.max_event_queues,
785d60b4185SPavan Nikhilesh 			evt_nr_active_lcores(opt->wlcores),
786d60b4185SPavan Nikhilesh 			dev_info.max_event_ports);
787d60b4185SPavan Nikhilesh 	}
788d60b4185SPavan Nikhilesh 
789d60b4185SPavan Nikhilesh 	return true;
790d60b4185SPavan Nikhilesh }
791d60b4185SPavan Nikhilesh 
792d60b4185SPavan Nikhilesh static const struct evt_test_ops pipeline_queue =  {
793d60b4185SPavan Nikhilesh 	.cap_check          = pipeline_queue_capability_check,
794d60b4185SPavan Nikhilesh 	.opt_check          = pipeline_queue_opt_check,
795d60b4185SPavan Nikhilesh 	.opt_dump           = pipeline_queue_opt_dump,
796d60b4185SPavan Nikhilesh 	.test_setup         = pipeline_test_setup,
797d60b4185SPavan Nikhilesh 	.mempool_setup      = pipeline_mempool_setup,
798d60b4185SPavan Nikhilesh 	.ethdev_setup	    = pipeline_ethdev_setup,
799d60b4185SPavan Nikhilesh 	.eventdev_setup     = pipeline_queue_eventdev_setup,
800d60b4185SPavan Nikhilesh 	.launch_lcores      = pipeline_queue_launch_lcores,
801*a734e738SPavan Nikhilesh 	.ethdev_rx_stop     = pipeline_ethdev_rx_stop,
802d60b4185SPavan Nikhilesh 	.eventdev_destroy   = pipeline_eventdev_destroy,
803d60b4185SPavan Nikhilesh 	.mempool_destroy    = pipeline_mempool_destroy,
804d60b4185SPavan Nikhilesh 	.ethdev_destroy	    = pipeline_ethdev_destroy,
805d60b4185SPavan Nikhilesh 	.test_result        = pipeline_test_result,
806d60b4185SPavan Nikhilesh 	.test_destroy       = pipeline_test_destroy,
807d60b4185SPavan Nikhilesh };
808d60b4185SPavan Nikhilesh 
809d60b4185SPavan Nikhilesh EVT_TEST_REGISTER(pipeline_queue);
810