xref: /dpdk/app/test-eventdev/test_pipeline_queue.c (revision 2eaa37b866353442bb479a64236ded22e4c79841)
1d60b4185SPavan Nikhilesh /*
2d60b4185SPavan Nikhilesh  * SPDX-License-Identifier: BSD-3-Clause
3d60b4185SPavan Nikhilesh  * Copyright 2017 Cavium, Inc.
4d60b4185SPavan Nikhilesh  */
5d60b4185SPavan Nikhilesh 
6d60b4185SPavan Nikhilesh #include "test_pipeline_common.h"
7d60b4185SPavan Nikhilesh 
843d162bcSThomas Monjalon /* See http://doc.dpdk.org/guides/tools/testeventdev.html for test details */
9d60b4185SPavan Nikhilesh 
10d60b4185SPavan Nikhilesh static __rte_always_inline int
11d60b4185SPavan Nikhilesh pipeline_queue_nb_event_queues(struct evt_options *opt)
12d60b4185SPavan Nikhilesh {
13d9a42a69SThomas Monjalon 	uint16_t eth_count = rte_eth_dev_count_avail();
14d60b4185SPavan Nikhilesh 
15d60b4185SPavan Nikhilesh 	return (eth_count * opt->nb_stages) + eth_count;
16d60b4185SPavan Nikhilesh }
17d60b4185SPavan Nikhilesh 
18*2eaa37b8SPavan Nikhilesh typedef int (*pipeline_queue_worker_t)(void *arg);
19*2eaa37b8SPavan Nikhilesh 
20032a965aSPavan Nikhilesh static __rte_noinline int
21314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_tx(void *arg)
22314bcf58SPavan Nikhilesh {
23f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_INIT;
24314bcf58SPavan Nikhilesh 
25314bcf58SPavan Nikhilesh 	while (t->done == false) {
26314bcf58SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
27314bcf58SPavan Nikhilesh 
28314bcf58SPavan Nikhilesh 		if (!event) {
29314bcf58SPavan Nikhilesh 			rte_pause();
30314bcf58SPavan Nikhilesh 			continue;
31314bcf58SPavan Nikhilesh 		}
32314bcf58SPavan Nikhilesh 
33314bcf58SPavan Nikhilesh 		if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
34032a965aSPavan Nikhilesh 			pipeline_event_tx(dev, port, &ev);
35314bcf58SPavan Nikhilesh 			w->processed_pkts++;
36314bcf58SPavan Nikhilesh 		} else {
37314bcf58SPavan Nikhilesh 			ev.queue_id++;
38314bcf58SPavan Nikhilesh 			pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
39314bcf58SPavan Nikhilesh 			pipeline_event_enqueue(dev, port, &ev);
40314bcf58SPavan Nikhilesh 		}
41314bcf58SPavan Nikhilesh 	}
42314bcf58SPavan Nikhilesh 
43314bcf58SPavan Nikhilesh 	return 0;
44314bcf58SPavan Nikhilesh }
45314bcf58SPavan Nikhilesh 
46032a965aSPavan Nikhilesh static __rte_noinline int
47314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_fwd(void *arg)
48314bcf58SPavan Nikhilesh {
49f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_INIT;
50032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
51314bcf58SPavan Nikhilesh 
52314bcf58SPavan Nikhilesh 	while (t->done == false) {
53314bcf58SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
54314bcf58SPavan Nikhilesh 
55314bcf58SPavan Nikhilesh 		if (!event) {
56314bcf58SPavan Nikhilesh 			rte_pause();
57314bcf58SPavan Nikhilesh 			continue;
58314bcf58SPavan Nikhilesh 		}
59314bcf58SPavan Nikhilesh 
60032a965aSPavan Nikhilesh 		ev.queue_id = tx_queue[ev.mbuf->port];
61032a965aSPavan Nikhilesh 		rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0);
62314bcf58SPavan Nikhilesh 		pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
63314bcf58SPavan Nikhilesh 		pipeline_event_enqueue(dev, port, &ev);
64314bcf58SPavan Nikhilesh 		w->processed_pkts++;
65314bcf58SPavan Nikhilesh 	}
66314bcf58SPavan Nikhilesh 
67314bcf58SPavan Nikhilesh 	return 0;
68314bcf58SPavan Nikhilesh }
69314bcf58SPavan Nikhilesh 
70032a965aSPavan Nikhilesh static __rte_noinline int
71314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_tx(void *arg)
72314bcf58SPavan Nikhilesh {
73f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT;
74314bcf58SPavan Nikhilesh 
75314bcf58SPavan Nikhilesh 	while (t->done == false) {
76314bcf58SPavan Nikhilesh 		uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
77314bcf58SPavan Nikhilesh 				BURST_SIZE, 0);
78314bcf58SPavan Nikhilesh 
79314bcf58SPavan Nikhilesh 		if (!nb_rx) {
80314bcf58SPavan Nikhilesh 			rte_pause();
81314bcf58SPavan Nikhilesh 			continue;
82314bcf58SPavan Nikhilesh 		}
83314bcf58SPavan Nikhilesh 
84314bcf58SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
85314bcf58SPavan Nikhilesh 			rte_prefetch0(ev[i + 1].mbuf);
86314bcf58SPavan Nikhilesh 			if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
87032a965aSPavan Nikhilesh 				pipeline_event_tx(dev, port, &ev[i]);
88314bcf58SPavan Nikhilesh 				w->processed_pkts++;
89314bcf58SPavan Nikhilesh 			} else {
90314bcf58SPavan Nikhilesh 				ev[i].queue_id++;
91314bcf58SPavan Nikhilesh 				pipeline_fwd_event(&ev[i],
92314bcf58SPavan Nikhilesh 						RTE_SCHED_TYPE_ATOMIC);
9321b1ca48SFeifei Wang 				pipeline_event_enqueue_burst(dev, port, ev,
9421b1ca48SFeifei Wang 						nb_rx);
95314bcf58SPavan Nikhilesh 			}
96314bcf58SPavan Nikhilesh 		}
97314bcf58SPavan Nikhilesh 	}
98314bcf58SPavan Nikhilesh 
99314bcf58SPavan Nikhilesh 	return 0;
100314bcf58SPavan Nikhilesh }
101314bcf58SPavan Nikhilesh 
102032a965aSPavan Nikhilesh static __rte_noinline int
103314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_fwd(void *arg)
104314bcf58SPavan Nikhilesh {
105f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT;
106032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
107314bcf58SPavan Nikhilesh 
108314bcf58SPavan Nikhilesh 	while (t->done == false) {
109314bcf58SPavan Nikhilesh 		uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
110314bcf58SPavan Nikhilesh 				BURST_SIZE, 0);
111314bcf58SPavan Nikhilesh 
112314bcf58SPavan Nikhilesh 		if (!nb_rx) {
113314bcf58SPavan Nikhilesh 			rte_pause();
114314bcf58SPavan Nikhilesh 			continue;
115314bcf58SPavan Nikhilesh 		}
116314bcf58SPavan Nikhilesh 
117314bcf58SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
118314bcf58SPavan Nikhilesh 			rte_prefetch0(ev[i + 1].mbuf);
119032a965aSPavan Nikhilesh 			ev[i].queue_id = tx_queue[ev[i].mbuf->port];
120032a965aSPavan Nikhilesh 			rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0);
121314bcf58SPavan Nikhilesh 			pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC);
122314bcf58SPavan Nikhilesh 		}
123314bcf58SPavan Nikhilesh 
124314bcf58SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
125032a965aSPavan Nikhilesh 		w->processed_pkts += nb_rx;
126314bcf58SPavan Nikhilesh 	}
127314bcf58SPavan Nikhilesh 
128314bcf58SPavan Nikhilesh 	return 0;
129314bcf58SPavan Nikhilesh }
130314bcf58SPavan Nikhilesh 
131*2eaa37b8SPavan Nikhilesh static __rte_noinline int
132*2eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_tx_vector(void *arg)
133*2eaa37b8SPavan Nikhilesh {
134*2eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_INIT;
135*2eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
136*2eaa37b8SPavan Nikhilesh 
137*2eaa37b8SPavan Nikhilesh 	while (!t->done) {
138*2eaa37b8SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
139*2eaa37b8SPavan Nikhilesh 
140*2eaa37b8SPavan Nikhilesh 		if (!event) {
141*2eaa37b8SPavan Nikhilesh 			rte_pause();
142*2eaa37b8SPavan Nikhilesh 			continue;
143*2eaa37b8SPavan Nikhilesh 		}
144*2eaa37b8SPavan Nikhilesh 
145*2eaa37b8SPavan Nikhilesh 		if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
146*2eaa37b8SPavan Nikhilesh 			vector_sz = ev.vec->nb_elem;
147*2eaa37b8SPavan Nikhilesh 			pipeline_event_tx_vector(dev, port, &ev);
148*2eaa37b8SPavan Nikhilesh 			w->processed_pkts += vector_sz;
149*2eaa37b8SPavan Nikhilesh 		} else {
150*2eaa37b8SPavan Nikhilesh 			ev.queue_id++;
151*2eaa37b8SPavan Nikhilesh 			pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC);
152*2eaa37b8SPavan Nikhilesh 			pipeline_event_enqueue(dev, port, &ev);
153*2eaa37b8SPavan Nikhilesh 		}
154*2eaa37b8SPavan Nikhilesh 	}
155*2eaa37b8SPavan Nikhilesh 
156*2eaa37b8SPavan Nikhilesh 	return 0;
157*2eaa37b8SPavan Nikhilesh }
158*2eaa37b8SPavan Nikhilesh 
159*2eaa37b8SPavan Nikhilesh static __rte_noinline int
160*2eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_fwd_vector(void *arg)
161*2eaa37b8SPavan Nikhilesh {
162*2eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_INIT;
163*2eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
164*2eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
165*2eaa37b8SPavan Nikhilesh 
166*2eaa37b8SPavan Nikhilesh 	while (!t->done) {
167*2eaa37b8SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
168*2eaa37b8SPavan Nikhilesh 
169*2eaa37b8SPavan Nikhilesh 		if (!event) {
170*2eaa37b8SPavan Nikhilesh 			rte_pause();
171*2eaa37b8SPavan Nikhilesh 			continue;
172*2eaa37b8SPavan Nikhilesh 		}
173*2eaa37b8SPavan Nikhilesh 
174*2eaa37b8SPavan Nikhilesh 		ev.queue_id = tx_queue[ev.vec->port];
175*2eaa37b8SPavan Nikhilesh 		ev.vec->queue = 0;
176*2eaa37b8SPavan Nikhilesh 		vector_sz = ev.vec->nb_elem;
177*2eaa37b8SPavan Nikhilesh 		pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC);
178*2eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue(dev, port, &ev);
179*2eaa37b8SPavan Nikhilesh 		w->processed_pkts += vector_sz;
180*2eaa37b8SPavan Nikhilesh 	}
181*2eaa37b8SPavan Nikhilesh 
182*2eaa37b8SPavan Nikhilesh 	return 0;
183*2eaa37b8SPavan Nikhilesh }
184*2eaa37b8SPavan Nikhilesh 
185*2eaa37b8SPavan Nikhilesh static __rte_noinline int
186*2eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_tx_vector(void *arg)
187*2eaa37b8SPavan Nikhilesh {
188*2eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT;
189*2eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
190*2eaa37b8SPavan Nikhilesh 
191*2eaa37b8SPavan Nikhilesh 	while (!t->done) {
192*2eaa37b8SPavan Nikhilesh 		uint16_t nb_rx =
193*2eaa37b8SPavan Nikhilesh 			rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);
194*2eaa37b8SPavan Nikhilesh 
195*2eaa37b8SPavan Nikhilesh 		if (!nb_rx) {
196*2eaa37b8SPavan Nikhilesh 			rte_pause();
197*2eaa37b8SPavan Nikhilesh 			continue;
198*2eaa37b8SPavan Nikhilesh 		}
199*2eaa37b8SPavan Nikhilesh 
200*2eaa37b8SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
201*2eaa37b8SPavan Nikhilesh 			if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
202*2eaa37b8SPavan Nikhilesh 				vector_sz = ev[i].vec->nb_elem;
203*2eaa37b8SPavan Nikhilesh 				pipeline_event_tx_vector(dev, port, &ev[i]);
204*2eaa37b8SPavan Nikhilesh 				ev[i].op = RTE_EVENT_OP_RELEASE;
205*2eaa37b8SPavan Nikhilesh 				w->processed_pkts += vector_sz;
206*2eaa37b8SPavan Nikhilesh 			} else {
207*2eaa37b8SPavan Nikhilesh 				ev[i].queue_id++;
208*2eaa37b8SPavan Nikhilesh 				pipeline_fwd_event_vector(
209*2eaa37b8SPavan Nikhilesh 					&ev[i], RTE_SCHED_TYPE_ATOMIC);
210*2eaa37b8SPavan Nikhilesh 			}
211*2eaa37b8SPavan Nikhilesh 		}
212*2eaa37b8SPavan Nikhilesh 
213*2eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
214*2eaa37b8SPavan Nikhilesh 	}
215*2eaa37b8SPavan Nikhilesh 
216*2eaa37b8SPavan Nikhilesh 	return 0;
217*2eaa37b8SPavan Nikhilesh }
218*2eaa37b8SPavan Nikhilesh 
219*2eaa37b8SPavan Nikhilesh static __rte_noinline int
220*2eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_fwd_vector(void *arg)
221*2eaa37b8SPavan Nikhilesh {
222*2eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT;
223*2eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
224*2eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
225*2eaa37b8SPavan Nikhilesh 
226*2eaa37b8SPavan Nikhilesh 	while (!t->done) {
227*2eaa37b8SPavan Nikhilesh 		uint16_t nb_rx =
228*2eaa37b8SPavan Nikhilesh 			rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);
229*2eaa37b8SPavan Nikhilesh 
230*2eaa37b8SPavan Nikhilesh 		if (!nb_rx) {
231*2eaa37b8SPavan Nikhilesh 			rte_pause();
232*2eaa37b8SPavan Nikhilesh 			continue;
233*2eaa37b8SPavan Nikhilesh 		}
234*2eaa37b8SPavan Nikhilesh 
235*2eaa37b8SPavan Nikhilesh 		vector_sz = 0;
236*2eaa37b8SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
237*2eaa37b8SPavan Nikhilesh 			ev[i].queue_id = tx_queue[ev[i].vec->port];
238*2eaa37b8SPavan Nikhilesh 			ev[i].vec->queue = 0;
239*2eaa37b8SPavan Nikhilesh 			vector_sz += ev[i].vec->nb_elem;
240*2eaa37b8SPavan Nikhilesh 			pipeline_fwd_event_vector(&ev[i],
241*2eaa37b8SPavan Nikhilesh 						  RTE_SCHED_TYPE_ATOMIC);
242*2eaa37b8SPavan Nikhilesh 		}
243*2eaa37b8SPavan Nikhilesh 
244*2eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
245*2eaa37b8SPavan Nikhilesh 		w->processed_pkts += vector_sz;
246*2eaa37b8SPavan Nikhilesh 	}
247*2eaa37b8SPavan Nikhilesh 
248*2eaa37b8SPavan Nikhilesh 	return 0;
249*2eaa37b8SPavan Nikhilesh }
250314bcf58SPavan Nikhilesh 
251032a965aSPavan Nikhilesh static __rte_noinline int
252314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_tx(void *arg)
253314bcf58SPavan Nikhilesh {
254f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_INIT;
255032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
256314bcf58SPavan Nikhilesh 
257314bcf58SPavan Nikhilesh 	while (t->done == false) {
258314bcf58SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
259314bcf58SPavan Nikhilesh 
260314bcf58SPavan Nikhilesh 		if (!event) {
261314bcf58SPavan Nikhilesh 			rte_pause();
262314bcf58SPavan Nikhilesh 			continue;
263314bcf58SPavan Nikhilesh 		}
264314bcf58SPavan Nikhilesh 
265314bcf58SPavan Nikhilesh 		cq_id = ev.queue_id % nb_stages;
266314bcf58SPavan Nikhilesh 
267032a965aSPavan Nikhilesh 		if (ev.queue_id == tx_queue[ev.mbuf->port]) {
268032a965aSPavan Nikhilesh 			pipeline_event_tx(dev, port, &ev);
269314bcf58SPavan Nikhilesh 			w->processed_pkts++;
270314bcf58SPavan Nikhilesh 			continue;
271314bcf58SPavan Nikhilesh 		}
272314bcf58SPavan Nikhilesh 
273032a965aSPavan Nikhilesh 		ev.queue_id++;
274032a965aSPavan Nikhilesh 		pipeline_fwd_event(&ev, cq_id != last_queue ?
275032a965aSPavan Nikhilesh 				sched_type_list[cq_id] :
276032a965aSPavan Nikhilesh 				RTE_SCHED_TYPE_ATOMIC);
277314bcf58SPavan Nikhilesh 		pipeline_event_enqueue(dev, port, &ev);
278314bcf58SPavan Nikhilesh 	}
279032a965aSPavan Nikhilesh 
280314bcf58SPavan Nikhilesh 	return 0;
281314bcf58SPavan Nikhilesh }
282314bcf58SPavan Nikhilesh 
283032a965aSPavan Nikhilesh static __rte_noinline int
284314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_fwd(void *arg)
285314bcf58SPavan Nikhilesh {
286f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_INIT;
287032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
288314bcf58SPavan Nikhilesh 
289314bcf58SPavan Nikhilesh 	while (t->done == false) {
290314bcf58SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
291314bcf58SPavan Nikhilesh 
292314bcf58SPavan Nikhilesh 		if (!event) {
293314bcf58SPavan Nikhilesh 			rte_pause();
294314bcf58SPavan Nikhilesh 			continue;
295314bcf58SPavan Nikhilesh 		}
296314bcf58SPavan Nikhilesh 
297314bcf58SPavan Nikhilesh 		cq_id = ev.queue_id % nb_stages;
298314bcf58SPavan Nikhilesh 
299314bcf58SPavan Nikhilesh 		if (cq_id == last_queue) {
300032a965aSPavan Nikhilesh 			ev.queue_id = tx_queue[ev.mbuf->port];
301032a965aSPavan Nikhilesh 			rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0);
302314bcf58SPavan Nikhilesh 			pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
303e0c05737SFeifei Wang 			pipeline_event_enqueue(dev, port, &ev);
304314bcf58SPavan Nikhilesh 			w->processed_pkts++;
305314bcf58SPavan Nikhilesh 		} else {
306314bcf58SPavan Nikhilesh 			ev.queue_id++;
307314bcf58SPavan Nikhilesh 			pipeline_fwd_event(&ev, sched_type_list[cq_id]);
308314bcf58SPavan Nikhilesh 			pipeline_event_enqueue(dev, port, &ev);
309314bcf58SPavan Nikhilesh 		}
310e0c05737SFeifei Wang 	}
311032a965aSPavan Nikhilesh 
312314bcf58SPavan Nikhilesh 	return 0;
313314bcf58SPavan Nikhilesh }
314314bcf58SPavan Nikhilesh 
315032a965aSPavan Nikhilesh static __rte_noinline int
316314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_tx(void *arg)
317314bcf58SPavan Nikhilesh {
318f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
319032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
320314bcf58SPavan Nikhilesh 
321314bcf58SPavan Nikhilesh 	while (t->done == false) {
322314bcf58SPavan Nikhilesh 		uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
323314bcf58SPavan Nikhilesh 				BURST_SIZE, 0);
324314bcf58SPavan Nikhilesh 
325314bcf58SPavan Nikhilesh 		if (!nb_rx) {
326314bcf58SPavan Nikhilesh 			rte_pause();
327314bcf58SPavan Nikhilesh 			continue;
328314bcf58SPavan Nikhilesh 		}
329314bcf58SPavan Nikhilesh 
330314bcf58SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
331314bcf58SPavan Nikhilesh 			rte_prefetch0(ev[i + 1].mbuf);
332314bcf58SPavan Nikhilesh 			cq_id = ev[i].queue_id % nb_stages;
333314bcf58SPavan Nikhilesh 
334032a965aSPavan Nikhilesh 			if (ev[i].queue_id == tx_queue[ev[i].mbuf->port]) {
335032a965aSPavan Nikhilesh 				pipeline_event_tx(dev, port, &ev[i]);
336314bcf58SPavan Nikhilesh 				w->processed_pkts++;
337314bcf58SPavan Nikhilesh 				continue;
338314bcf58SPavan Nikhilesh 			}
339314bcf58SPavan Nikhilesh 
340314bcf58SPavan Nikhilesh 			ev[i].queue_id++;
341032a965aSPavan Nikhilesh 			pipeline_fwd_event(&ev[i], cq_id != last_queue ?
342032a965aSPavan Nikhilesh 					sched_type_list[cq_id] :
343032a965aSPavan Nikhilesh 					RTE_SCHED_TYPE_ATOMIC);
344314bcf58SPavan Nikhilesh 			pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
345314bcf58SPavan Nikhilesh 		}
34621b1ca48SFeifei Wang 	}
347032a965aSPavan Nikhilesh 
348314bcf58SPavan Nikhilesh 	return 0;
349314bcf58SPavan Nikhilesh }
350314bcf58SPavan Nikhilesh 
351032a965aSPavan Nikhilesh static __rte_noinline int
352314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_fwd(void *arg)
353314bcf58SPavan Nikhilesh {
354f26320a6SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
355032a965aSPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
356314bcf58SPavan Nikhilesh 
357314bcf58SPavan Nikhilesh 	while (t->done == false) {
358e0c05737SFeifei Wang 		uint16_t processed_pkts = 0;
359314bcf58SPavan Nikhilesh 		uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
360314bcf58SPavan Nikhilesh 				BURST_SIZE, 0);
361314bcf58SPavan Nikhilesh 
362314bcf58SPavan Nikhilesh 		if (!nb_rx) {
363314bcf58SPavan Nikhilesh 			rte_pause();
364314bcf58SPavan Nikhilesh 			continue;
365314bcf58SPavan Nikhilesh 		}
366314bcf58SPavan Nikhilesh 
367314bcf58SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
368314bcf58SPavan Nikhilesh 			rte_prefetch0(ev[i + 1].mbuf);
369314bcf58SPavan Nikhilesh 			cq_id = ev[i].queue_id % nb_stages;
370314bcf58SPavan Nikhilesh 
371314bcf58SPavan Nikhilesh 			if (cq_id == last_queue) {
372032a965aSPavan Nikhilesh 				ev[i].queue_id = tx_queue[ev[i].mbuf->port];
373032a965aSPavan Nikhilesh 				rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0);
374314bcf58SPavan Nikhilesh 				pipeline_fwd_event(&ev[i],
375314bcf58SPavan Nikhilesh 						RTE_SCHED_TYPE_ATOMIC);
376e0c05737SFeifei Wang 				processed_pkts++;
377314bcf58SPavan Nikhilesh 			} else {
378314bcf58SPavan Nikhilesh 				ev[i].queue_id++;
379314bcf58SPavan Nikhilesh 				pipeline_fwd_event(&ev[i],
380314bcf58SPavan Nikhilesh 						sched_type_list[cq_id]);
381314bcf58SPavan Nikhilesh 			}
382314bcf58SPavan Nikhilesh 		}
383314bcf58SPavan Nikhilesh 
384314bcf58SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
385e0c05737SFeifei Wang 		w->processed_pkts += processed_pkts;
386314bcf58SPavan Nikhilesh 	}
387032a965aSPavan Nikhilesh 
388314bcf58SPavan Nikhilesh 	return 0;
389314bcf58SPavan Nikhilesh }
390314bcf58SPavan Nikhilesh 
391*2eaa37b8SPavan Nikhilesh static __rte_noinline int
392*2eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_tx_vector(void *arg)
393*2eaa37b8SPavan Nikhilesh {
394*2eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_INIT;
395*2eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
396*2eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
397*2eaa37b8SPavan Nikhilesh 
398*2eaa37b8SPavan Nikhilesh 	while (!t->done) {
399*2eaa37b8SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
400*2eaa37b8SPavan Nikhilesh 
401*2eaa37b8SPavan Nikhilesh 		if (!event) {
402*2eaa37b8SPavan Nikhilesh 			rte_pause();
403*2eaa37b8SPavan Nikhilesh 			continue;
404*2eaa37b8SPavan Nikhilesh 		}
405*2eaa37b8SPavan Nikhilesh 
406*2eaa37b8SPavan Nikhilesh 		cq_id = ev.queue_id % nb_stages;
407*2eaa37b8SPavan Nikhilesh 
408*2eaa37b8SPavan Nikhilesh 		if (ev.queue_id == tx_queue[ev.vec->port]) {
409*2eaa37b8SPavan Nikhilesh 			vector_sz = ev.vec->nb_elem;
410*2eaa37b8SPavan Nikhilesh 			pipeline_event_tx_vector(dev, port, &ev);
411*2eaa37b8SPavan Nikhilesh 			w->processed_pkts += vector_sz;
412*2eaa37b8SPavan Nikhilesh 			continue;
413*2eaa37b8SPavan Nikhilesh 		}
414*2eaa37b8SPavan Nikhilesh 
415*2eaa37b8SPavan Nikhilesh 		ev.queue_id++;
416*2eaa37b8SPavan Nikhilesh 		pipeline_fwd_event_vector(&ev, cq_id != last_queue
417*2eaa37b8SPavan Nikhilesh 						       ? sched_type_list[cq_id]
418*2eaa37b8SPavan Nikhilesh 						       : RTE_SCHED_TYPE_ATOMIC);
419*2eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue(dev, port, &ev);
420*2eaa37b8SPavan Nikhilesh 	}
421*2eaa37b8SPavan Nikhilesh 
422*2eaa37b8SPavan Nikhilesh 	return 0;
423*2eaa37b8SPavan Nikhilesh }
424*2eaa37b8SPavan Nikhilesh 
425*2eaa37b8SPavan Nikhilesh static __rte_noinline int
426*2eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_fwd_vector(void *arg)
427*2eaa37b8SPavan Nikhilesh {
428*2eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_INIT;
429*2eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
430*2eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
431*2eaa37b8SPavan Nikhilesh 
432*2eaa37b8SPavan Nikhilesh 	while (!t->done) {
433*2eaa37b8SPavan Nikhilesh 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
434*2eaa37b8SPavan Nikhilesh 
435*2eaa37b8SPavan Nikhilesh 		if (!event) {
436*2eaa37b8SPavan Nikhilesh 			rte_pause();
437*2eaa37b8SPavan Nikhilesh 			continue;
438*2eaa37b8SPavan Nikhilesh 		}
439*2eaa37b8SPavan Nikhilesh 
440*2eaa37b8SPavan Nikhilesh 		cq_id = ev.queue_id % nb_stages;
441*2eaa37b8SPavan Nikhilesh 
442*2eaa37b8SPavan Nikhilesh 		if (cq_id == last_queue) {
443*2eaa37b8SPavan Nikhilesh 			vector_sz = ev.vec->nb_elem;
444*2eaa37b8SPavan Nikhilesh 			ev.queue_id = tx_queue[ev.vec->port];
445*2eaa37b8SPavan Nikhilesh 			pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC);
446*2eaa37b8SPavan Nikhilesh 			w->processed_pkts += vector_sz;
447*2eaa37b8SPavan Nikhilesh 		} else {
448*2eaa37b8SPavan Nikhilesh 			ev.queue_id++;
449*2eaa37b8SPavan Nikhilesh 			pipeline_fwd_event_vector(&ev, sched_type_list[cq_id]);
450*2eaa37b8SPavan Nikhilesh 		}
451*2eaa37b8SPavan Nikhilesh 
452*2eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue(dev, port, &ev);
453*2eaa37b8SPavan Nikhilesh 	}
454*2eaa37b8SPavan Nikhilesh 
455*2eaa37b8SPavan Nikhilesh 	return 0;
456*2eaa37b8SPavan Nikhilesh }
457*2eaa37b8SPavan Nikhilesh 
458*2eaa37b8SPavan Nikhilesh static __rte_noinline int
459*2eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_tx_vector(void *arg)
460*2eaa37b8SPavan Nikhilesh {
461*2eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
462*2eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
463*2eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
464*2eaa37b8SPavan Nikhilesh 
465*2eaa37b8SPavan Nikhilesh 	while (!t->done) {
466*2eaa37b8SPavan Nikhilesh 		uint16_t nb_rx =
467*2eaa37b8SPavan Nikhilesh 			rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);
468*2eaa37b8SPavan Nikhilesh 
469*2eaa37b8SPavan Nikhilesh 		if (!nb_rx) {
470*2eaa37b8SPavan Nikhilesh 			rte_pause();
471*2eaa37b8SPavan Nikhilesh 			continue;
472*2eaa37b8SPavan Nikhilesh 		}
473*2eaa37b8SPavan Nikhilesh 
474*2eaa37b8SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
475*2eaa37b8SPavan Nikhilesh 			cq_id = ev[i].queue_id % nb_stages;
476*2eaa37b8SPavan Nikhilesh 
477*2eaa37b8SPavan Nikhilesh 			if (ev[i].queue_id == tx_queue[ev[i].vec->port]) {
478*2eaa37b8SPavan Nikhilesh 				vector_sz = ev[i].vec->nb_elem;
479*2eaa37b8SPavan Nikhilesh 				pipeline_event_tx_vector(dev, port, &ev[i]);
480*2eaa37b8SPavan Nikhilesh 				ev[i].op = RTE_EVENT_OP_RELEASE;
481*2eaa37b8SPavan Nikhilesh 				w->processed_pkts += vector_sz;
482*2eaa37b8SPavan Nikhilesh 				continue;
483*2eaa37b8SPavan Nikhilesh 			}
484*2eaa37b8SPavan Nikhilesh 
485*2eaa37b8SPavan Nikhilesh 			ev[i].queue_id++;
486*2eaa37b8SPavan Nikhilesh 			pipeline_fwd_event_vector(
487*2eaa37b8SPavan Nikhilesh 				&ev[i], cq_id != last_queue
488*2eaa37b8SPavan Nikhilesh 						? sched_type_list[cq_id]
489*2eaa37b8SPavan Nikhilesh 						: RTE_SCHED_TYPE_ATOMIC);
490*2eaa37b8SPavan Nikhilesh 		}
491*2eaa37b8SPavan Nikhilesh 
492*2eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
493*2eaa37b8SPavan Nikhilesh 	}
494*2eaa37b8SPavan Nikhilesh 
495*2eaa37b8SPavan Nikhilesh 	return 0;
496*2eaa37b8SPavan Nikhilesh }
497*2eaa37b8SPavan Nikhilesh 
498*2eaa37b8SPavan Nikhilesh static __rte_noinline int
499*2eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_fwd_vector(void *arg)
500*2eaa37b8SPavan Nikhilesh {
501*2eaa37b8SPavan Nikhilesh 	PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
502*2eaa37b8SPavan Nikhilesh 	const uint8_t *tx_queue = t->tx_evqueue_id;
503*2eaa37b8SPavan Nikhilesh 	uint16_t vector_sz;
504*2eaa37b8SPavan Nikhilesh 
505*2eaa37b8SPavan Nikhilesh 	while (!t->done) {
506*2eaa37b8SPavan Nikhilesh 		uint16_t nb_rx =
507*2eaa37b8SPavan Nikhilesh 			rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);
508*2eaa37b8SPavan Nikhilesh 
509*2eaa37b8SPavan Nikhilesh 		if (!nb_rx) {
510*2eaa37b8SPavan Nikhilesh 			rte_pause();
511*2eaa37b8SPavan Nikhilesh 			continue;
512*2eaa37b8SPavan Nikhilesh 		}
513*2eaa37b8SPavan Nikhilesh 
514*2eaa37b8SPavan Nikhilesh 		for (i = 0; i < nb_rx; i++) {
515*2eaa37b8SPavan Nikhilesh 			cq_id = ev[i].queue_id % nb_stages;
516*2eaa37b8SPavan Nikhilesh 
517*2eaa37b8SPavan Nikhilesh 			if (cq_id == last_queue) {
518*2eaa37b8SPavan Nikhilesh 				ev[i].queue_id = tx_queue[ev[i].vec->port];
519*2eaa37b8SPavan Nikhilesh 				vector_sz = ev[i].vec->nb_elem;
520*2eaa37b8SPavan Nikhilesh 				pipeline_fwd_event_vector(
521*2eaa37b8SPavan Nikhilesh 					&ev[i], RTE_SCHED_TYPE_ATOMIC);
522*2eaa37b8SPavan Nikhilesh 				w->processed_pkts += vector_sz;
523*2eaa37b8SPavan Nikhilesh 			} else {
524*2eaa37b8SPavan Nikhilesh 				ev[i].queue_id++;
525*2eaa37b8SPavan Nikhilesh 				pipeline_fwd_event_vector(
526*2eaa37b8SPavan Nikhilesh 					&ev[i], sched_type_list[cq_id]);
527*2eaa37b8SPavan Nikhilesh 			}
528*2eaa37b8SPavan Nikhilesh 		}
529*2eaa37b8SPavan Nikhilesh 
530*2eaa37b8SPavan Nikhilesh 		pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
531*2eaa37b8SPavan Nikhilesh 	}
532*2eaa37b8SPavan Nikhilesh 
533*2eaa37b8SPavan Nikhilesh 	return 0;
534*2eaa37b8SPavan Nikhilesh }
535*2eaa37b8SPavan Nikhilesh 
536314bcf58SPavan Nikhilesh static int
537d60b4185SPavan Nikhilesh worker_wrapper(void *arg)
538d60b4185SPavan Nikhilesh {
539314bcf58SPavan Nikhilesh 	struct worker_data *w  = arg;
540314bcf58SPavan Nikhilesh 	struct evt_options *opt = w->t->opt;
541314bcf58SPavan Nikhilesh 	const bool burst = evt_has_burst_mode(w->dev_id);
542032a965aSPavan Nikhilesh 	const bool internal_port = w->t->internal_port;
543314bcf58SPavan Nikhilesh 	const uint8_t nb_stages = opt->nb_stages;
544*2eaa37b8SPavan Nikhilesh 	/*vector/burst/internal_port*/
545*2eaa37b8SPavan Nikhilesh 	const pipeline_queue_worker_t
546*2eaa37b8SPavan Nikhilesh 	pipeline_queue_worker_single_stage[2][2][2] = {
547*2eaa37b8SPavan Nikhilesh 		[0][0][0] = pipeline_queue_worker_single_stage_fwd,
548*2eaa37b8SPavan Nikhilesh 		[0][0][1] = pipeline_queue_worker_single_stage_tx,
549*2eaa37b8SPavan Nikhilesh 		[0][1][0] = pipeline_queue_worker_single_stage_burst_fwd,
550*2eaa37b8SPavan Nikhilesh 		[0][1][1] = pipeline_queue_worker_single_stage_burst_tx,
551*2eaa37b8SPavan Nikhilesh 		[1][0][0] = pipeline_queue_worker_single_stage_fwd_vector,
552*2eaa37b8SPavan Nikhilesh 		[1][0][1] = pipeline_queue_worker_single_stage_tx_vector,
553*2eaa37b8SPavan Nikhilesh 		[1][1][0] = pipeline_queue_worker_single_stage_burst_fwd_vector,
554*2eaa37b8SPavan Nikhilesh 		[1][1][1] = pipeline_queue_worker_single_stage_burst_tx_vector,
555*2eaa37b8SPavan Nikhilesh 	};
556*2eaa37b8SPavan Nikhilesh 	const pipeline_queue_worker_t
557*2eaa37b8SPavan Nikhilesh 	pipeline_queue_worker_multi_stage[2][2][2] = {
558*2eaa37b8SPavan Nikhilesh 		[0][0][0] = pipeline_queue_worker_multi_stage_fwd,
559*2eaa37b8SPavan Nikhilesh 		[0][0][1] = pipeline_queue_worker_multi_stage_tx,
560*2eaa37b8SPavan Nikhilesh 		[0][1][0] = pipeline_queue_worker_multi_stage_burst_fwd,
561*2eaa37b8SPavan Nikhilesh 		[0][1][1] = pipeline_queue_worker_multi_stage_burst_tx,
562*2eaa37b8SPavan Nikhilesh 		[1][0][0] = pipeline_queue_worker_multi_stage_fwd_vector,
563*2eaa37b8SPavan Nikhilesh 		[1][0][1] = pipeline_queue_worker_multi_stage_tx_vector,
564*2eaa37b8SPavan Nikhilesh 		[1][1][0] = pipeline_queue_worker_multi_stage_burst_fwd_vector,
565*2eaa37b8SPavan Nikhilesh 		[1][1][1] = pipeline_queue_worker_multi_stage_burst_tx_vector,
566*2eaa37b8SPavan Nikhilesh 	};
567314bcf58SPavan Nikhilesh 
568*2eaa37b8SPavan Nikhilesh 	if (nb_stages == 1)
569*2eaa37b8SPavan Nikhilesh 		return (pipeline_queue_worker_single_stage[opt->ena_vector]
570*2eaa37b8SPavan Nikhilesh 							  [burst]
571*2eaa37b8SPavan Nikhilesh 							  [internal_port])(arg);
572*2eaa37b8SPavan Nikhilesh 	else
573*2eaa37b8SPavan Nikhilesh 		return (pipeline_queue_worker_multi_stage[opt->ena_vector]
574*2eaa37b8SPavan Nikhilesh 							 [burst]
575*2eaa37b8SPavan Nikhilesh 							 [internal_port])(arg);
576314bcf58SPavan Nikhilesh 
577d60b4185SPavan Nikhilesh 	rte_panic("invalid worker\n");
578d60b4185SPavan Nikhilesh }
579d60b4185SPavan Nikhilesh 
580d60b4185SPavan Nikhilesh static int
581d60b4185SPavan Nikhilesh pipeline_queue_launch_lcores(struct evt_test *test, struct evt_options *opt)
582d60b4185SPavan Nikhilesh {
583d60b4185SPavan Nikhilesh 	return pipeline_launch_lcores(test, opt, worker_wrapper);
584d60b4185SPavan Nikhilesh }
585d60b4185SPavan Nikhilesh 
586d60b4185SPavan Nikhilesh static int
587d60b4185SPavan Nikhilesh pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
588d60b4185SPavan Nikhilesh {
589d60b4185SPavan Nikhilesh 	int ret;
590d60b4185SPavan Nikhilesh 	int nb_ports;
591d60b4185SPavan Nikhilesh 	int nb_queues;
592d60b4185SPavan Nikhilesh 	int nb_stages = opt->nb_stages;
593d60b4185SPavan Nikhilesh 	uint8_t queue;
594032a965aSPavan Nikhilesh 	uint8_t tx_evport_id = 0;
595032a965aSPavan Nikhilesh 	uint8_t tx_evqueue_id[RTE_MAX_ETHPORTS];
596d60b4185SPavan Nikhilesh 	uint8_t queue_arr[RTE_EVENT_MAX_QUEUES_PER_DEV];
597d60b4185SPavan Nikhilesh 	uint8_t nb_worker_queues = 0;
598032a965aSPavan Nikhilesh 	uint16_t prod = 0;
599032a965aSPavan Nikhilesh 	struct rte_event_dev_info info;
600032a965aSPavan Nikhilesh 	struct test_pipeline *t = evt_test_priv(test);
601d60b4185SPavan Nikhilesh 
602d60b4185SPavan Nikhilesh 	nb_ports = evt_nr_active_lcores(opt->wlcores);
603d9a42a69SThomas Monjalon 	nb_queues = rte_eth_dev_count_avail() * (nb_stages);
604d60b4185SPavan Nikhilesh 
605032a965aSPavan Nikhilesh 	/* One queue for Tx adapter per port */
606d9a42a69SThomas Monjalon 	nb_queues += rte_eth_dev_count_avail();
607d60b4185SPavan Nikhilesh 
608032a965aSPavan Nikhilesh 	memset(tx_evqueue_id, 0, sizeof(uint8_t) * RTE_MAX_ETHPORTS);
609032a965aSPavan Nikhilesh 	memset(queue_arr, 0, sizeof(uint8_t) * RTE_EVENT_MAX_QUEUES_PER_DEV);
610d60b4185SPavan Nikhilesh 
611032a965aSPavan Nikhilesh 	rte_event_dev_info_get(opt->dev_id, &info);
612f0959283SPavan Nikhilesh 	ret = evt_configure_eventdev(opt, nb_queues, nb_ports);
613d60b4185SPavan Nikhilesh 	if (ret) {
614d60b4185SPavan Nikhilesh 		evt_err("failed to configure eventdev %d", opt->dev_id);
615d60b4185SPavan Nikhilesh 		return ret;
616d60b4185SPavan Nikhilesh 	}
617d60b4185SPavan Nikhilesh 
618d60b4185SPavan Nikhilesh 	struct rte_event_queue_conf q_conf = {
619d60b4185SPavan Nikhilesh 			.priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
620d60b4185SPavan Nikhilesh 			.nb_atomic_flows = opt->nb_flows,
621d60b4185SPavan Nikhilesh 			.nb_atomic_order_sequences = opt->nb_flows,
622d60b4185SPavan Nikhilesh 	};
623d60b4185SPavan Nikhilesh 	/* queue configurations */
624d60b4185SPavan Nikhilesh 	for (queue = 0; queue < nb_queues; queue++) {
625d60b4185SPavan Nikhilesh 		uint8_t slot;
626d60b4185SPavan Nikhilesh 
627032a965aSPavan Nikhilesh 		q_conf.event_queue_cfg = 0;
628d60b4185SPavan Nikhilesh 		slot = queue % (nb_stages + 1);
629032a965aSPavan Nikhilesh 		if (slot == nb_stages) {
630d60b4185SPavan Nikhilesh 			q_conf.schedule_type = RTE_SCHED_TYPE_ATOMIC;
631032a965aSPavan Nikhilesh 			if (!t->internal_port) {
632d60b4185SPavan Nikhilesh 				q_conf.event_queue_cfg =
633d60b4185SPavan Nikhilesh 					RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
634032a965aSPavan Nikhilesh 			}
635032a965aSPavan Nikhilesh 			tx_evqueue_id[prod++] = queue;
636d60b4185SPavan Nikhilesh 		} else {
637032a965aSPavan Nikhilesh 			q_conf.schedule_type = opt->sched_type_list[slot];
638d60b4185SPavan Nikhilesh 			queue_arr[nb_worker_queues] = queue;
639d60b4185SPavan Nikhilesh 			nb_worker_queues++;
640d60b4185SPavan Nikhilesh 		}
641d60b4185SPavan Nikhilesh 
642d60b4185SPavan Nikhilesh 		ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf);
643d60b4185SPavan Nikhilesh 		if (ret) {
644d60b4185SPavan Nikhilesh 			evt_err("failed to setup queue=%d", queue);
645d60b4185SPavan Nikhilesh 			return ret;
646d60b4185SPavan Nikhilesh 		}
647d60b4185SPavan Nikhilesh 	}
648d60b4185SPavan Nikhilesh 
649535c630cSPavan Nikhilesh 	if (opt->wkr_deq_dep > info.max_event_port_dequeue_depth)
650535c630cSPavan Nikhilesh 		opt->wkr_deq_dep = info.max_event_port_dequeue_depth;
651535c630cSPavan Nikhilesh 
652d60b4185SPavan Nikhilesh 	/* port configuration */
653d60b4185SPavan Nikhilesh 	const struct rte_event_port_conf p_conf = {
654d60b4185SPavan Nikhilesh 			.dequeue_depth = opt->wkr_deq_dep,
655d60b4185SPavan Nikhilesh 			.enqueue_depth = info.max_event_port_dequeue_depth,
656d60b4185SPavan Nikhilesh 			.new_event_threshold = info.max_num_events,
657d60b4185SPavan Nikhilesh 	};
658d60b4185SPavan Nikhilesh 
659032a965aSPavan Nikhilesh 	if (!t->internal_port) {
660d60b4185SPavan Nikhilesh 		ret = pipeline_event_port_setup(test, opt, queue_arr,
661d60b4185SPavan Nikhilesh 				nb_worker_queues, p_conf);
662d60b4185SPavan Nikhilesh 		if (ret)
663d60b4185SPavan Nikhilesh 			return ret;
664d60b4185SPavan Nikhilesh 	} else
665d60b4185SPavan Nikhilesh 		ret = pipeline_event_port_setup(test, opt, NULL, nb_queues,
666d60b4185SPavan Nikhilesh 				p_conf);
667d60b4185SPavan Nikhilesh 
668d60b4185SPavan Nikhilesh 	if (ret)
669d60b4185SPavan Nikhilesh 		return ret;
670d60b4185SPavan Nikhilesh 	/*
671d60b4185SPavan Nikhilesh 	 * The pipelines are setup in the following manner:
672d60b4185SPavan Nikhilesh 	 *
673d60b4185SPavan Nikhilesh 	 * eth_dev_count = 2, nb_stages = 2.
674d60b4185SPavan Nikhilesh 	 *
675d60b4185SPavan Nikhilesh 	 *	queues = 6
676d60b4185SPavan Nikhilesh 	 *	stride = 3
677d60b4185SPavan Nikhilesh 	 *
678d60b4185SPavan Nikhilesh 	 *	event queue pipelines:
679d60b4185SPavan Nikhilesh 	 *	eth0 -> q0 -> q1 -> (q2->tx)
680d60b4185SPavan Nikhilesh 	 *	eth1 -> q3 -> q4 -> (q5->tx)
681d60b4185SPavan Nikhilesh 	 *
682032a965aSPavan Nikhilesh 	 *	q2, q5 configured as ATOMIC | SINGLE_LINK
683d60b4185SPavan Nikhilesh 	 *
684d60b4185SPavan Nikhilesh 	 */
685032a965aSPavan Nikhilesh 	ret = pipeline_event_rx_adapter_setup(opt, nb_stages + 1, p_conf);
686032a965aSPavan Nikhilesh 	if (ret)
687032a965aSPavan Nikhilesh 		return ret;
688032a965aSPavan Nikhilesh 
689032a965aSPavan Nikhilesh 	ret = pipeline_event_tx_adapter_setup(opt, p_conf);
690d60b4185SPavan Nikhilesh 	if (ret)
691d60b4185SPavan Nikhilesh 		return ret;
692d60b4185SPavan Nikhilesh 
693d60b4185SPavan Nikhilesh 	if (!evt_has_distributed_sched(opt->dev_id)) {
694d60b4185SPavan Nikhilesh 		uint32_t service_id;
695d60b4185SPavan Nikhilesh 		rte_event_dev_service_id_get(opt->dev_id, &service_id);
696d60b4185SPavan Nikhilesh 		ret = evt_service_setup(service_id);
697d60b4185SPavan Nikhilesh 		if (ret) {
698d60b4185SPavan Nikhilesh 			evt_err("No service lcore found to run event dev.");
699d60b4185SPavan Nikhilesh 			return ret;
700d60b4185SPavan Nikhilesh 		}
701d60b4185SPavan Nikhilesh 	}
702d60b4185SPavan Nikhilesh 
703032a965aSPavan Nikhilesh 	/* Connect the tx_evqueue_id to the Tx adapter port */
704032a965aSPavan Nikhilesh 	if (!t->internal_port) {
705032a965aSPavan Nikhilesh 		RTE_ETH_FOREACH_DEV(prod) {
706032a965aSPavan Nikhilesh 			ret = rte_event_eth_tx_adapter_event_port_get(prod,
707032a965aSPavan Nikhilesh 					&tx_evport_id);
708032a965aSPavan Nikhilesh 			if (ret) {
709032a965aSPavan Nikhilesh 				evt_err("Unable to get Tx adptr[%d] evprt[%d]",
710032a965aSPavan Nikhilesh 						prod, tx_evport_id);
711032a965aSPavan Nikhilesh 				return ret;
712032a965aSPavan Nikhilesh 			}
713032a965aSPavan Nikhilesh 
714032a965aSPavan Nikhilesh 			if (rte_event_port_link(opt->dev_id, tx_evport_id,
715032a965aSPavan Nikhilesh 						&tx_evqueue_id[prod],
716032a965aSPavan Nikhilesh 						NULL, 1) != 1) {
717032a965aSPavan Nikhilesh 				evt_err("Unable to link Tx adptr[%d] evprt[%d]",
718032a965aSPavan Nikhilesh 						prod, tx_evport_id);
719032a965aSPavan Nikhilesh 				return ret;
720032a965aSPavan Nikhilesh 			}
721032a965aSPavan Nikhilesh 		}
722032a965aSPavan Nikhilesh 	}
723032a965aSPavan Nikhilesh 
72466b82db2SPavan Nikhilesh 	ret = rte_event_dev_start(opt->dev_id);
72566b82db2SPavan Nikhilesh 	if (ret) {
72666b82db2SPavan Nikhilesh 		evt_err("failed to start eventdev %d", opt->dev_id);
72766b82db2SPavan Nikhilesh 		return ret;
72866b82db2SPavan Nikhilesh 	}
72966b82db2SPavan Nikhilesh 
73066b82db2SPavan Nikhilesh 
731032a965aSPavan Nikhilesh 	RTE_ETH_FOREACH_DEV(prod) {
732032a965aSPavan Nikhilesh 		ret = rte_eth_dev_start(prod);
733032a965aSPavan Nikhilesh 		if (ret) {
734032a965aSPavan Nikhilesh 			evt_err("Ethernet dev [%d] failed to start."
735032a965aSPavan Nikhilesh 					" Using synthetic producer", prod);
736032a965aSPavan Nikhilesh 			return ret;
737032a965aSPavan Nikhilesh 		}
738032a965aSPavan Nikhilesh 
739032a965aSPavan Nikhilesh 	}
740032a965aSPavan Nikhilesh 
741032a965aSPavan Nikhilesh 	RTE_ETH_FOREACH_DEV(prod) {
742032a965aSPavan Nikhilesh 		ret = rte_event_eth_rx_adapter_start(prod);
743032a965aSPavan Nikhilesh 		if (ret) {
744032a965aSPavan Nikhilesh 			evt_err("Rx adapter[%d] start failed", prod);
745032a965aSPavan Nikhilesh 			return ret;
746032a965aSPavan Nikhilesh 		}
747032a965aSPavan Nikhilesh 
748032a965aSPavan Nikhilesh 		ret = rte_event_eth_tx_adapter_start(prod);
749032a965aSPavan Nikhilesh 		if (ret) {
750032a965aSPavan Nikhilesh 			evt_err("Tx adapter[%d] start failed", prod);
751032a965aSPavan Nikhilesh 			return ret;
752032a965aSPavan Nikhilesh 		}
753032a965aSPavan Nikhilesh 	}
754032a965aSPavan Nikhilesh 
755032a965aSPavan Nikhilesh 	memcpy(t->tx_evqueue_id, tx_evqueue_id, sizeof(uint8_t) *
756032a965aSPavan Nikhilesh 			RTE_MAX_ETHPORTS);
757032a965aSPavan Nikhilesh 
758d60b4185SPavan Nikhilesh 	return 0;
759d60b4185SPavan Nikhilesh }
760d60b4185SPavan Nikhilesh 
761d60b4185SPavan Nikhilesh static void
762d60b4185SPavan Nikhilesh pipeline_queue_opt_dump(struct evt_options *opt)
763d60b4185SPavan Nikhilesh {
764d60b4185SPavan Nikhilesh 	pipeline_opt_dump(opt, pipeline_queue_nb_event_queues(opt));
765d60b4185SPavan Nikhilesh }
766d60b4185SPavan Nikhilesh 
767d60b4185SPavan Nikhilesh static int
768d60b4185SPavan Nikhilesh pipeline_queue_opt_check(struct evt_options *opt)
769d60b4185SPavan Nikhilesh {
770d60b4185SPavan Nikhilesh 	return pipeline_opt_check(opt, pipeline_queue_nb_event_queues(opt));
771d60b4185SPavan Nikhilesh }
772d60b4185SPavan Nikhilesh 
773d60b4185SPavan Nikhilesh static bool
774d60b4185SPavan Nikhilesh pipeline_queue_capability_check(struct evt_options *opt)
775d60b4185SPavan Nikhilesh {
776d60b4185SPavan Nikhilesh 	struct rte_event_dev_info dev_info;
777d60b4185SPavan Nikhilesh 
778d60b4185SPavan Nikhilesh 	rte_event_dev_info_get(opt->dev_id, &dev_info);
779d60b4185SPavan Nikhilesh 	if (dev_info.max_event_queues < pipeline_queue_nb_event_queues(opt) ||
780d60b4185SPavan Nikhilesh 			dev_info.max_event_ports <
781d60b4185SPavan Nikhilesh 			evt_nr_active_lcores(opt->wlcores)) {
782d60b4185SPavan Nikhilesh 		evt_err("not enough eventdev queues=%d/%d or ports=%d/%d",
783d60b4185SPavan Nikhilesh 			pipeline_queue_nb_event_queues(opt),
784d60b4185SPavan Nikhilesh 			dev_info.max_event_queues,
785d60b4185SPavan Nikhilesh 			evt_nr_active_lcores(opt->wlcores),
786d60b4185SPavan Nikhilesh 			dev_info.max_event_ports);
787d60b4185SPavan Nikhilesh 	}
788d60b4185SPavan Nikhilesh 
789d60b4185SPavan Nikhilesh 	return true;
790d60b4185SPavan Nikhilesh }
791d60b4185SPavan Nikhilesh 
792d60b4185SPavan Nikhilesh static const struct evt_test_ops pipeline_queue =  {
793d60b4185SPavan Nikhilesh 	.cap_check          = pipeline_queue_capability_check,
794d60b4185SPavan Nikhilesh 	.opt_check          = pipeline_queue_opt_check,
795d60b4185SPavan Nikhilesh 	.opt_dump           = pipeline_queue_opt_dump,
796d60b4185SPavan Nikhilesh 	.test_setup         = pipeline_test_setup,
797d60b4185SPavan Nikhilesh 	.mempool_setup      = pipeline_mempool_setup,
798d60b4185SPavan Nikhilesh 	.ethdev_setup	    = pipeline_ethdev_setup,
799d60b4185SPavan Nikhilesh 	.eventdev_setup     = pipeline_queue_eventdev_setup,
800d60b4185SPavan Nikhilesh 	.launch_lcores      = pipeline_queue_launch_lcores,
801d60b4185SPavan Nikhilesh 	.eventdev_destroy   = pipeline_eventdev_destroy,
802d60b4185SPavan Nikhilesh 	.mempool_destroy    = pipeline_mempool_destroy,
803d60b4185SPavan Nikhilesh 	.ethdev_destroy	    = pipeline_ethdev_destroy,
804d60b4185SPavan Nikhilesh 	.test_result        = pipeline_test_result,
805d60b4185SPavan Nikhilesh 	.test_destroy       = pipeline_test_destroy,
806d60b4185SPavan Nikhilesh };
807d60b4185SPavan Nikhilesh 
808d60b4185SPavan Nikhilesh EVT_TEST_REGISTER(pipeline_queue);
809