1d60b4185SPavan Nikhilesh /* 2d60b4185SPavan Nikhilesh * SPDX-License-Identifier: BSD-3-Clause 3d60b4185SPavan Nikhilesh * Copyright 2017 Cavium, Inc. 4d60b4185SPavan Nikhilesh */ 5d60b4185SPavan Nikhilesh 6d60b4185SPavan Nikhilesh #include "test_pipeline_common.h" 7d60b4185SPavan Nikhilesh 843d162bcSThomas Monjalon /* See http://doc.dpdk.org/guides/tools/testeventdev.html for test details */ 9d60b4185SPavan Nikhilesh 10d60b4185SPavan Nikhilesh static __rte_always_inline int 11d60b4185SPavan Nikhilesh pipeline_queue_nb_event_queues(struct evt_options *opt) 12d60b4185SPavan Nikhilesh { 13d9a42a69SThomas Monjalon uint16_t eth_count = rte_eth_dev_count_avail(); 14d60b4185SPavan Nikhilesh 15d60b4185SPavan Nikhilesh return (eth_count * opt->nb_stages) + eth_count; 16d60b4185SPavan Nikhilesh } 17d60b4185SPavan Nikhilesh 182eaa37b8SPavan Nikhilesh typedef int (*pipeline_queue_worker_t)(void *arg); 192eaa37b8SPavan Nikhilesh 20032a965aSPavan Nikhilesh static __rte_noinline int 21314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_tx(void *arg) 22314bcf58SPavan Nikhilesh { 23f26320a6SPavan Nikhilesh PIPELINE_WORKER_SINGLE_STAGE_INIT; 24314bcf58SPavan Nikhilesh 25314bcf58SPavan Nikhilesh while (t->done == false) { 26314bcf58SPavan Nikhilesh uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); 27314bcf58SPavan Nikhilesh 28314bcf58SPavan Nikhilesh if (!event) { 29314bcf58SPavan Nikhilesh rte_pause(); 30314bcf58SPavan Nikhilesh continue; 31314bcf58SPavan Nikhilesh } 32314bcf58SPavan Nikhilesh 33314bcf58SPavan Nikhilesh if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) { 34032a965aSPavan Nikhilesh pipeline_event_tx(dev, port, &ev); 35314bcf58SPavan Nikhilesh w->processed_pkts++; 36314bcf58SPavan Nikhilesh } else { 37314bcf58SPavan Nikhilesh ev.queue_id++; 38314bcf58SPavan Nikhilesh pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); 39314bcf58SPavan Nikhilesh pipeline_event_enqueue(dev, port, &ev); 40314bcf58SPavan Nikhilesh } 41314bcf58SPavan Nikhilesh } 42314bcf58SPavan Nikhilesh 43314bcf58SPavan Nikhilesh return 0; 44314bcf58SPavan Nikhilesh } 45314bcf58SPavan Nikhilesh 46032a965aSPavan Nikhilesh static __rte_noinline int 47314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_fwd(void *arg) 48314bcf58SPavan Nikhilesh { 49f26320a6SPavan Nikhilesh PIPELINE_WORKER_SINGLE_STAGE_INIT; 50032a965aSPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 51314bcf58SPavan Nikhilesh 52314bcf58SPavan Nikhilesh while (t->done == false) { 53314bcf58SPavan Nikhilesh uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); 54314bcf58SPavan Nikhilesh 55314bcf58SPavan Nikhilesh if (!event) { 56314bcf58SPavan Nikhilesh rte_pause(); 57314bcf58SPavan Nikhilesh continue; 58314bcf58SPavan Nikhilesh } 59314bcf58SPavan Nikhilesh 60032a965aSPavan Nikhilesh ev.queue_id = tx_queue[ev.mbuf->port]; 61032a965aSPavan Nikhilesh rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0); 62314bcf58SPavan Nikhilesh pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); 63314bcf58SPavan Nikhilesh pipeline_event_enqueue(dev, port, &ev); 64314bcf58SPavan Nikhilesh w->processed_pkts++; 65314bcf58SPavan Nikhilesh } 66314bcf58SPavan Nikhilesh 67314bcf58SPavan Nikhilesh return 0; 68314bcf58SPavan Nikhilesh } 69314bcf58SPavan Nikhilesh 70032a965aSPavan Nikhilesh static __rte_noinline int 71314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_tx(void *arg) 72314bcf58SPavan Nikhilesh { 73f26320a6SPavan Nikhilesh PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; 74314bcf58SPavan Nikhilesh 75314bcf58SPavan Nikhilesh while (t->done == false) { 76314bcf58SPavan Nikhilesh uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, 77314bcf58SPavan Nikhilesh BURST_SIZE, 0); 78314bcf58SPavan Nikhilesh 79314bcf58SPavan Nikhilesh if (!nb_rx) { 80314bcf58SPavan Nikhilesh rte_pause(); 81314bcf58SPavan Nikhilesh continue; 82314bcf58SPavan Nikhilesh } 83314bcf58SPavan Nikhilesh 84314bcf58SPavan Nikhilesh for (i = 0; i < nb_rx; i++) { 85314bcf58SPavan Nikhilesh rte_prefetch0(ev[i + 1].mbuf); 86314bcf58SPavan Nikhilesh if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) { 87032a965aSPavan Nikhilesh pipeline_event_tx(dev, port, &ev[i]); 88314bcf58SPavan Nikhilesh w->processed_pkts++; 89314bcf58SPavan Nikhilesh } else { 90314bcf58SPavan Nikhilesh ev[i].queue_id++; 91314bcf58SPavan Nikhilesh pipeline_fwd_event(&ev[i], 92314bcf58SPavan Nikhilesh RTE_SCHED_TYPE_ATOMIC); 9321b1ca48SFeifei Wang pipeline_event_enqueue_burst(dev, port, ev, 9421b1ca48SFeifei Wang nb_rx); 95314bcf58SPavan Nikhilesh } 96314bcf58SPavan Nikhilesh } 97314bcf58SPavan Nikhilesh } 98314bcf58SPavan Nikhilesh 99314bcf58SPavan Nikhilesh return 0; 100314bcf58SPavan Nikhilesh } 101314bcf58SPavan Nikhilesh 102032a965aSPavan Nikhilesh static __rte_noinline int 103314bcf58SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_fwd(void *arg) 104314bcf58SPavan Nikhilesh { 105f26320a6SPavan Nikhilesh PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; 106032a965aSPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 107314bcf58SPavan Nikhilesh 108314bcf58SPavan Nikhilesh while (t->done == false) { 109314bcf58SPavan Nikhilesh uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, 110314bcf58SPavan Nikhilesh BURST_SIZE, 0); 111314bcf58SPavan Nikhilesh 112314bcf58SPavan Nikhilesh if (!nb_rx) { 113314bcf58SPavan Nikhilesh rte_pause(); 114314bcf58SPavan Nikhilesh continue; 115314bcf58SPavan Nikhilesh } 116314bcf58SPavan Nikhilesh 117314bcf58SPavan Nikhilesh for (i = 0; i < nb_rx; i++) { 118314bcf58SPavan Nikhilesh rte_prefetch0(ev[i + 1].mbuf); 119032a965aSPavan Nikhilesh ev[i].queue_id = tx_queue[ev[i].mbuf->port]; 120032a965aSPavan Nikhilesh rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0); 121314bcf58SPavan Nikhilesh pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC); 122314bcf58SPavan Nikhilesh } 123314bcf58SPavan Nikhilesh 124314bcf58SPavan Nikhilesh pipeline_event_enqueue_burst(dev, port, ev, nb_rx); 125032a965aSPavan Nikhilesh w->processed_pkts += nb_rx; 126314bcf58SPavan Nikhilesh } 127314bcf58SPavan Nikhilesh 128314bcf58SPavan Nikhilesh return 0; 129314bcf58SPavan Nikhilesh } 130314bcf58SPavan Nikhilesh 1312eaa37b8SPavan Nikhilesh static __rte_noinline int 1322eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_tx_vector(void *arg) 1332eaa37b8SPavan Nikhilesh { 1342eaa37b8SPavan Nikhilesh PIPELINE_WORKER_SINGLE_STAGE_INIT; 1352eaa37b8SPavan Nikhilesh uint16_t vector_sz; 1362eaa37b8SPavan Nikhilesh 1372eaa37b8SPavan Nikhilesh while (!t->done) { 1382eaa37b8SPavan Nikhilesh uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); 1392eaa37b8SPavan Nikhilesh 1402eaa37b8SPavan Nikhilesh if (!event) { 1412eaa37b8SPavan Nikhilesh rte_pause(); 1422eaa37b8SPavan Nikhilesh continue; 1432eaa37b8SPavan Nikhilesh } 1442eaa37b8SPavan Nikhilesh 1452eaa37b8SPavan Nikhilesh if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) { 1462eaa37b8SPavan Nikhilesh vector_sz = ev.vec->nb_elem; 1472eaa37b8SPavan Nikhilesh pipeline_event_tx_vector(dev, port, &ev); 1482eaa37b8SPavan Nikhilesh w->processed_pkts += vector_sz; 1492eaa37b8SPavan Nikhilesh } else { 1502eaa37b8SPavan Nikhilesh ev.queue_id++; 1512eaa37b8SPavan Nikhilesh pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC); 1522eaa37b8SPavan Nikhilesh pipeline_event_enqueue(dev, port, &ev); 1532eaa37b8SPavan Nikhilesh } 1542eaa37b8SPavan Nikhilesh } 1552eaa37b8SPavan Nikhilesh 1562eaa37b8SPavan Nikhilesh return 0; 1572eaa37b8SPavan Nikhilesh } 1582eaa37b8SPavan Nikhilesh 1592eaa37b8SPavan Nikhilesh static __rte_noinline int 1602eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_fwd_vector(void *arg) 1612eaa37b8SPavan Nikhilesh { 1622eaa37b8SPavan Nikhilesh PIPELINE_WORKER_SINGLE_STAGE_INIT; 1632eaa37b8SPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 1642eaa37b8SPavan Nikhilesh uint16_t vector_sz; 1652eaa37b8SPavan Nikhilesh 1662eaa37b8SPavan Nikhilesh while (!t->done) { 1672eaa37b8SPavan Nikhilesh uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); 1682eaa37b8SPavan Nikhilesh 1692eaa37b8SPavan Nikhilesh if (!event) { 1702eaa37b8SPavan Nikhilesh rte_pause(); 1712eaa37b8SPavan Nikhilesh continue; 1722eaa37b8SPavan Nikhilesh } 1732eaa37b8SPavan Nikhilesh 1742eaa37b8SPavan Nikhilesh ev.queue_id = tx_queue[ev.vec->port]; 1752eaa37b8SPavan Nikhilesh ev.vec->queue = 0; 1762eaa37b8SPavan Nikhilesh vector_sz = ev.vec->nb_elem; 1772eaa37b8SPavan Nikhilesh pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC); 1782eaa37b8SPavan Nikhilesh pipeline_event_enqueue(dev, port, &ev); 1792eaa37b8SPavan Nikhilesh w->processed_pkts += vector_sz; 1802eaa37b8SPavan Nikhilesh } 1812eaa37b8SPavan Nikhilesh 1822eaa37b8SPavan Nikhilesh return 0; 1832eaa37b8SPavan Nikhilesh } 1842eaa37b8SPavan Nikhilesh 1852eaa37b8SPavan Nikhilesh static __rte_noinline int 1862eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_tx_vector(void *arg) 1872eaa37b8SPavan Nikhilesh { 1882eaa37b8SPavan Nikhilesh PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; 1892eaa37b8SPavan Nikhilesh uint16_t vector_sz; 1902eaa37b8SPavan Nikhilesh 1912eaa37b8SPavan Nikhilesh while (!t->done) { 1922eaa37b8SPavan Nikhilesh uint16_t nb_rx = 1932eaa37b8SPavan Nikhilesh rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0); 1942eaa37b8SPavan Nikhilesh 1952eaa37b8SPavan Nikhilesh if (!nb_rx) { 1962eaa37b8SPavan Nikhilesh rte_pause(); 1972eaa37b8SPavan Nikhilesh continue; 1982eaa37b8SPavan Nikhilesh } 1992eaa37b8SPavan Nikhilesh 2002eaa37b8SPavan Nikhilesh for (i = 0; i < nb_rx; i++) { 2012eaa37b8SPavan Nikhilesh if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) { 2022eaa37b8SPavan Nikhilesh vector_sz = ev[i].vec->nb_elem; 2032eaa37b8SPavan Nikhilesh pipeline_event_tx_vector(dev, port, &ev[i]); 2042eaa37b8SPavan Nikhilesh ev[i].op = RTE_EVENT_OP_RELEASE; 2052eaa37b8SPavan Nikhilesh w->processed_pkts += vector_sz; 2062eaa37b8SPavan Nikhilesh } else { 2072eaa37b8SPavan Nikhilesh ev[i].queue_id++; 2082eaa37b8SPavan Nikhilesh pipeline_fwd_event_vector( 2092eaa37b8SPavan Nikhilesh &ev[i], RTE_SCHED_TYPE_ATOMIC); 2102eaa37b8SPavan Nikhilesh } 2112eaa37b8SPavan Nikhilesh } 2122eaa37b8SPavan Nikhilesh 2132eaa37b8SPavan Nikhilesh pipeline_event_enqueue_burst(dev, port, ev, nb_rx); 2142eaa37b8SPavan Nikhilesh } 2152eaa37b8SPavan Nikhilesh 2162eaa37b8SPavan Nikhilesh return 0; 2172eaa37b8SPavan Nikhilesh } 2182eaa37b8SPavan Nikhilesh 2192eaa37b8SPavan Nikhilesh static __rte_noinline int 2202eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage_burst_fwd_vector(void *arg) 2212eaa37b8SPavan Nikhilesh { 2222eaa37b8SPavan Nikhilesh PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; 2232eaa37b8SPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 2242eaa37b8SPavan Nikhilesh uint16_t vector_sz; 2252eaa37b8SPavan Nikhilesh 2262eaa37b8SPavan Nikhilesh while (!t->done) { 2272eaa37b8SPavan Nikhilesh uint16_t nb_rx = 2282eaa37b8SPavan Nikhilesh rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0); 2292eaa37b8SPavan Nikhilesh 2302eaa37b8SPavan Nikhilesh if (!nb_rx) { 2312eaa37b8SPavan Nikhilesh rte_pause(); 2322eaa37b8SPavan Nikhilesh continue; 2332eaa37b8SPavan Nikhilesh } 2342eaa37b8SPavan Nikhilesh 2352eaa37b8SPavan Nikhilesh vector_sz = 0; 2362eaa37b8SPavan Nikhilesh for (i = 0; i < nb_rx; i++) { 2372eaa37b8SPavan Nikhilesh ev[i].queue_id = tx_queue[ev[i].vec->port]; 2382eaa37b8SPavan Nikhilesh ev[i].vec->queue = 0; 2392eaa37b8SPavan Nikhilesh vector_sz += ev[i].vec->nb_elem; 2402eaa37b8SPavan Nikhilesh pipeline_fwd_event_vector(&ev[i], 2412eaa37b8SPavan Nikhilesh RTE_SCHED_TYPE_ATOMIC); 2422eaa37b8SPavan Nikhilesh } 2432eaa37b8SPavan Nikhilesh 2442eaa37b8SPavan Nikhilesh pipeline_event_enqueue_burst(dev, port, ev, nb_rx); 2452eaa37b8SPavan Nikhilesh w->processed_pkts += vector_sz; 2462eaa37b8SPavan Nikhilesh } 2472eaa37b8SPavan Nikhilesh 2482eaa37b8SPavan Nikhilesh return 0; 2492eaa37b8SPavan Nikhilesh } 250314bcf58SPavan Nikhilesh 251032a965aSPavan Nikhilesh static __rte_noinline int 252314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_tx(void *arg) 253314bcf58SPavan Nikhilesh { 254f26320a6SPavan Nikhilesh PIPELINE_WORKER_MULTI_STAGE_INIT; 255032a965aSPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 256314bcf58SPavan Nikhilesh 257314bcf58SPavan Nikhilesh while (t->done == false) { 258314bcf58SPavan Nikhilesh uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); 259314bcf58SPavan Nikhilesh 260314bcf58SPavan Nikhilesh if (!event) { 261314bcf58SPavan Nikhilesh rte_pause(); 262314bcf58SPavan Nikhilesh continue; 263314bcf58SPavan Nikhilesh } 264314bcf58SPavan Nikhilesh 265314bcf58SPavan Nikhilesh cq_id = ev.queue_id % nb_stages; 266314bcf58SPavan Nikhilesh 267032a965aSPavan Nikhilesh if (ev.queue_id == tx_queue[ev.mbuf->port]) { 268032a965aSPavan Nikhilesh pipeline_event_tx(dev, port, &ev); 269314bcf58SPavan Nikhilesh w->processed_pkts++; 270314bcf58SPavan Nikhilesh continue; 271314bcf58SPavan Nikhilesh } 272314bcf58SPavan Nikhilesh 273032a965aSPavan Nikhilesh ev.queue_id++; 274032a965aSPavan Nikhilesh pipeline_fwd_event(&ev, cq_id != last_queue ? 275032a965aSPavan Nikhilesh sched_type_list[cq_id] : 276032a965aSPavan Nikhilesh RTE_SCHED_TYPE_ATOMIC); 277314bcf58SPavan Nikhilesh pipeline_event_enqueue(dev, port, &ev); 278314bcf58SPavan Nikhilesh } 279032a965aSPavan Nikhilesh 280314bcf58SPavan Nikhilesh return 0; 281314bcf58SPavan Nikhilesh } 282314bcf58SPavan Nikhilesh 283032a965aSPavan Nikhilesh static __rte_noinline int 284314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_fwd(void *arg) 285314bcf58SPavan Nikhilesh { 286f26320a6SPavan Nikhilesh PIPELINE_WORKER_MULTI_STAGE_INIT; 287032a965aSPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 288314bcf58SPavan Nikhilesh 289314bcf58SPavan Nikhilesh while (t->done == false) { 290314bcf58SPavan Nikhilesh uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); 291314bcf58SPavan Nikhilesh 292314bcf58SPavan Nikhilesh if (!event) { 293314bcf58SPavan Nikhilesh rte_pause(); 294314bcf58SPavan Nikhilesh continue; 295314bcf58SPavan Nikhilesh } 296314bcf58SPavan Nikhilesh 297314bcf58SPavan Nikhilesh cq_id = ev.queue_id % nb_stages; 298314bcf58SPavan Nikhilesh 299314bcf58SPavan Nikhilesh if (cq_id == last_queue) { 300032a965aSPavan Nikhilesh ev.queue_id = tx_queue[ev.mbuf->port]; 301032a965aSPavan Nikhilesh rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0); 302314bcf58SPavan Nikhilesh pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); 303e0c05737SFeifei Wang pipeline_event_enqueue(dev, port, &ev); 304314bcf58SPavan Nikhilesh w->processed_pkts++; 305314bcf58SPavan Nikhilesh } else { 306314bcf58SPavan Nikhilesh ev.queue_id++; 307314bcf58SPavan Nikhilesh pipeline_fwd_event(&ev, sched_type_list[cq_id]); 308314bcf58SPavan Nikhilesh pipeline_event_enqueue(dev, port, &ev); 309314bcf58SPavan Nikhilesh } 310e0c05737SFeifei Wang } 311032a965aSPavan Nikhilesh 312314bcf58SPavan Nikhilesh return 0; 313314bcf58SPavan Nikhilesh } 314314bcf58SPavan Nikhilesh 315032a965aSPavan Nikhilesh static __rte_noinline int 316314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_tx(void *arg) 317314bcf58SPavan Nikhilesh { 318f26320a6SPavan Nikhilesh PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; 319032a965aSPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 320314bcf58SPavan Nikhilesh 321314bcf58SPavan Nikhilesh while (t->done == false) { 322314bcf58SPavan Nikhilesh uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, 323314bcf58SPavan Nikhilesh BURST_SIZE, 0); 324314bcf58SPavan Nikhilesh 325314bcf58SPavan Nikhilesh if (!nb_rx) { 326314bcf58SPavan Nikhilesh rte_pause(); 327314bcf58SPavan Nikhilesh continue; 328314bcf58SPavan Nikhilesh } 329314bcf58SPavan Nikhilesh 330314bcf58SPavan Nikhilesh for (i = 0; i < nb_rx; i++) { 331314bcf58SPavan Nikhilesh rte_prefetch0(ev[i + 1].mbuf); 332314bcf58SPavan Nikhilesh cq_id = ev[i].queue_id % nb_stages; 333314bcf58SPavan Nikhilesh 334032a965aSPavan Nikhilesh if (ev[i].queue_id == tx_queue[ev[i].mbuf->port]) { 335032a965aSPavan Nikhilesh pipeline_event_tx(dev, port, &ev[i]); 336314bcf58SPavan Nikhilesh w->processed_pkts++; 337314bcf58SPavan Nikhilesh continue; 338314bcf58SPavan Nikhilesh } 339314bcf58SPavan Nikhilesh 340314bcf58SPavan Nikhilesh ev[i].queue_id++; 341032a965aSPavan Nikhilesh pipeline_fwd_event(&ev[i], cq_id != last_queue ? 342032a965aSPavan Nikhilesh sched_type_list[cq_id] : 343032a965aSPavan Nikhilesh RTE_SCHED_TYPE_ATOMIC); 344314bcf58SPavan Nikhilesh pipeline_event_enqueue_burst(dev, port, ev, nb_rx); 345314bcf58SPavan Nikhilesh } 34621b1ca48SFeifei Wang } 347032a965aSPavan Nikhilesh 348314bcf58SPavan Nikhilesh return 0; 349314bcf58SPavan Nikhilesh } 350314bcf58SPavan Nikhilesh 351032a965aSPavan Nikhilesh static __rte_noinline int 352314bcf58SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_fwd(void *arg) 353314bcf58SPavan Nikhilesh { 354f26320a6SPavan Nikhilesh PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; 355032a965aSPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 356314bcf58SPavan Nikhilesh 357314bcf58SPavan Nikhilesh while (t->done == false) { 358e0c05737SFeifei Wang uint16_t processed_pkts = 0; 359314bcf58SPavan Nikhilesh uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, 360314bcf58SPavan Nikhilesh BURST_SIZE, 0); 361314bcf58SPavan Nikhilesh 362314bcf58SPavan Nikhilesh if (!nb_rx) { 363314bcf58SPavan Nikhilesh rte_pause(); 364314bcf58SPavan Nikhilesh continue; 365314bcf58SPavan Nikhilesh } 366314bcf58SPavan Nikhilesh 367314bcf58SPavan Nikhilesh for (i = 0; i < nb_rx; i++) { 368314bcf58SPavan Nikhilesh rte_prefetch0(ev[i + 1].mbuf); 369314bcf58SPavan Nikhilesh cq_id = ev[i].queue_id % nb_stages; 370314bcf58SPavan Nikhilesh 371314bcf58SPavan Nikhilesh if (cq_id == last_queue) { 372032a965aSPavan Nikhilesh ev[i].queue_id = tx_queue[ev[i].mbuf->port]; 373032a965aSPavan Nikhilesh rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0); 374314bcf58SPavan Nikhilesh pipeline_fwd_event(&ev[i], 375314bcf58SPavan Nikhilesh RTE_SCHED_TYPE_ATOMIC); 376e0c05737SFeifei Wang processed_pkts++; 377314bcf58SPavan Nikhilesh } else { 378314bcf58SPavan Nikhilesh ev[i].queue_id++; 379314bcf58SPavan Nikhilesh pipeline_fwd_event(&ev[i], 380314bcf58SPavan Nikhilesh sched_type_list[cq_id]); 381314bcf58SPavan Nikhilesh } 382314bcf58SPavan Nikhilesh } 383314bcf58SPavan Nikhilesh 384314bcf58SPavan Nikhilesh pipeline_event_enqueue_burst(dev, port, ev, nb_rx); 385e0c05737SFeifei Wang w->processed_pkts += processed_pkts; 386314bcf58SPavan Nikhilesh } 387032a965aSPavan Nikhilesh 388314bcf58SPavan Nikhilesh return 0; 389314bcf58SPavan Nikhilesh } 390314bcf58SPavan Nikhilesh 3912eaa37b8SPavan Nikhilesh static __rte_noinline int 3922eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_tx_vector(void *arg) 3932eaa37b8SPavan Nikhilesh { 3942eaa37b8SPavan Nikhilesh PIPELINE_WORKER_MULTI_STAGE_INIT; 3952eaa37b8SPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 3962eaa37b8SPavan Nikhilesh uint16_t vector_sz; 3972eaa37b8SPavan Nikhilesh 3982eaa37b8SPavan Nikhilesh while (!t->done) { 3992eaa37b8SPavan Nikhilesh uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); 4002eaa37b8SPavan Nikhilesh 4012eaa37b8SPavan Nikhilesh if (!event) { 4022eaa37b8SPavan Nikhilesh rte_pause(); 4032eaa37b8SPavan Nikhilesh continue; 4042eaa37b8SPavan Nikhilesh } 4052eaa37b8SPavan Nikhilesh 4062eaa37b8SPavan Nikhilesh cq_id = ev.queue_id % nb_stages; 4072eaa37b8SPavan Nikhilesh 4082eaa37b8SPavan Nikhilesh if (ev.queue_id == tx_queue[ev.vec->port]) { 4092eaa37b8SPavan Nikhilesh vector_sz = ev.vec->nb_elem; 4102eaa37b8SPavan Nikhilesh pipeline_event_tx_vector(dev, port, &ev); 4112eaa37b8SPavan Nikhilesh w->processed_pkts += vector_sz; 4122eaa37b8SPavan Nikhilesh continue; 4132eaa37b8SPavan Nikhilesh } 4142eaa37b8SPavan Nikhilesh 4152eaa37b8SPavan Nikhilesh ev.queue_id++; 4162eaa37b8SPavan Nikhilesh pipeline_fwd_event_vector(&ev, cq_id != last_queue 4172eaa37b8SPavan Nikhilesh ? sched_type_list[cq_id] 4182eaa37b8SPavan Nikhilesh : RTE_SCHED_TYPE_ATOMIC); 4192eaa37b8SPavan Nikhilesh pipeline_event_enqueue(dev, port, &ev); 4202eaa37b8SPavan Nikhilesh } 4212eaa37b8SPavan Nikhilesh 4222eaa37b8SPavan Nikhilesh return 0; 4232eaa37b8SPavan Nikhilesh } 4242eaa37b8SPavan Nikhilesh 4252eaa37b8SPavan Nikhilesh static __rte_noinline int 4262eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_fwd_vector(void *arg) 4272eaa37b8SPavan Nikhilesh { 4282eaa37b8SPavan Nikhilesh PIPELINE_WORKER_MULTI_STAGE_INIT; 4292eaa37b8SPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 4302eaa37b8SPavan Nikhilesh uint16_t vector_sz; 4312eaa37b8SPavan Nikhilesh 4322eaa37b8SPavan Nikhilesh while (!t->done) { 4332eaa37b8SPavan Nikhilesh uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); 4342eaa37b8SPavan Nikhilesh 4352eaa37b8SPavan Nikhilesh if (!event) { 4362eaa37b8SPavan Nikhilesh rte_pause(); 4372eaa37b8SPavan Nikhilesh continue; 4382eaa37b8SPavan Nikhilesh } 4392eaa37b8SPavan Nikhilesh 4402eaa37b8SPavan Nikhilesh cq_id = ev.queue_id % nb_stages; 4412eaa37b8SPavan Nikhilesh 4422eaa37b8SPavan Nikhilesh if (cq_id == last_queue) { 4432eaa37b8SPavan Nikhilesh vector_sz = ev.vec->nb_elem; 4442eaa37b8SPavan Nikhilesh ev.queue_id = tx_queue[ev.vec->port]; 4452eaa37b8SPavan Nikhilesh pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC); 4462eaa37b8SPavan Nikhilesh w->processed_pkts += vector_sz; 4472eaa37b8SPavan Nikhilesh } else { 4482eaa37b8SPavan Nikhilesh ev.queue_id++; 4492eaa37b8SPavan Nikhilesh pipeline_fwd_event_vector(&ev, sched_type_list[cq_id]); 4502eaa37b8SPavan Nikhilesh } 4512eaa37b8SPavan Nikhilesh 4522eaa37b8SPavan Nikhilesh pipeline_event_enqueue(dev, port, &ev); 4532eaa37b8SPavan Nikhilesh } 4542eaa37b8SPavan Nikhilesh 4552eaa37b8SPavan Nikhilesh return 0; 4562eaa37b8SPavan Nikhilesh } 4572eaa37b8SPavan Nikhilesh 4582eaa37b8SPavan Nikhilesh static __rte_noinline int 4592eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_tx_vector(void *arg) 4602eaa37b8SPavan Nikhilesh { 4612eaa37b8SPavan Nikhilesh PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; 4622eaa37b8SPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 4632eaa37b8SPavan Nikhilesh uint16_t vector_sz; 4642eaa37b8SPavan Nikhilesh 4652eaa37b8SPavan Nikhilesh while (!t->done) { 4662eaa37b8SPavan Nikhilesh uint16_t nb_rx = 4672eaa37b8SPavan Nikhilesh rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0); 4682eaa37b8SPavan Nikhilesh 4692eaa37b8SPavan Nikhilesh if (!nb_rx) { 4702eaa37b8SPavan Nikhilesh rte_pause(); 4712eaa37b8SPavan Nikhilesh continue; 4722eaa37b8SPavan Nikhilesh } 4732eaa37b8SPavan Nikhilesh 4742eaa37b8SPavan Nikhilesh for (i = 0; i < nb_rx; i++) { 4752eaa37b8SPavan Nikhilesh cq_id = ev[i].queue_id % nb_stages; 4762eaa37b8SPavan Nikhilesh 4772eaa37b8SPavan Nikhilesh if (ev[i].queue_id == tx_queue[ev[i].vec->port]) { 4782eaa37b8SPavan Nikhilesh vector_sz = ev[i].vec->nb_elem; 4792eaa37b8SPavan Nikhilesh pipeline_event_tx_vector(dev, port, &ev[i]); 4802eaa37b8SPavan Nikhilesh ev[i].op = RTE_EVENT_OP_RELEASE; 4812eaa37b8SPavan Nikhilesh w->processed_pkts += vector_sz; 4822eaa37b8SPavan Nikhilesh continue; 4832eaa37b8SPavan Nikhilesh } 4842eaa37b8SPavan Nikhilesh 4852eaa37b8SPavan Nikhilesh ev[i].queue_id++; 4862eaa37b8SPavan Nikhilesh pipeline_fwd_event_vector( 4872eaa37b8SPavan Nikhilesh &ev[i], cq_id != last_queue 4882eaa37b8SPavan Nikhilesh ? sched_type_list[cq_id] 4892eaa37b8SPavan Nikhilesh : RTE_SCHED_TYPE_ATOMIC); 4902eaa37b8SPavan Nikhilesh } 4912eaa37b8SPavan Nikhilesh 4922eaa37b8SPavan Nikhilesh pipeline_event_enqueue_burst(dev, port, ev, nb_rx); 4932eaa37b8SPavan Nikhilesh } 4942eaa37b8SPavan Nikhilesh 4952eaa37b8SPavan Nikhilesh return 0; 4962eaa37b8SPavan Nikhilesh } 4972eaa37b8SPavan Nikhilesh 4982eaa37b8SPavan Nikhilesh static __rte_noinline int 4992eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage_burst_fwd_vector(void *arg) 5002eaa37b8SPavan Nikhilesh { 5012eaa37b8SPavan Nikhilesh PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; 5022eaa37b8SPavan Nikhilesh const uint8_t *tx_queue = t->tx_evqueue_id; 5032eaa37b8SPavan Nikhilesh uint16_t vector_sz; 5042eaa37b8SPavan Nikhilesh 5052eaa37b8SPavan Nikhilesh while (!t->done) { 5062eaa37b8SPavan Nikhilesh uint16_t nb_rx = 5072eaa37b8SPavan Nikhilesh rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0); 5082eaa37b8SPavan Nikhilesh 5092eaa37b8SPavan Nikhilesh if (!nb_rx) { 5102eaa37b8SPavan Nikhilesh rte_pause(); 5112eaa37b8SPavan Nikhilesh continue; 5122eaa37b8SPavan Nikhilesh } 5132eaa37b8SPavan Nikhilesh 5142eaa37b8SPavan Nikhilesh for (i = 0; i < nb_rx; i++) { 5152eaa37b8SPavan Nikhilesh cq_id = ev[i].queue_id % nb_stages; 5162eaa37b8SPavan Nikhilesh 5172eaa37b8SPavan Nikhilesh if (cq_id == last_queue) { 5182eaa37b8SPavan Nikhilesh ev[i].queue_id = tx_queue[ev[i].vec->port]; 5192eaa37b8SPavan Nikhilesh vector_sz = ev[i].vec->nb_elem; 5202eaa37b8SPavan Nikhilesh pipeline_fwd_event_vector( 5212eaa37b8SPavan Nikhilesh &ev[i], RTE_SCHED_TYPE_ATOMIC); 5222eaa37b8SPavan Nikhilesh w->processed_pkts += vector_sz; 5232eaa37b8SPavan Nikhilesh } else { 5242eaa37b8SPavan Nikhilesh ev[i].queue_id++; 5252eaa37b8SPavan Nikhilesh pipeline_fwd_event_vector( 5262eaa37b8SPavan Nikhilesh &ev[i], sched_type_list[cq_id]); 5272eaa37b8SPavan Nikhilesh } 5282eaa37b8SPavan Nikhilesh } 5292eaa37b8SPavan Nikhilesh 5302eaa37b8SPavan Nikhilesh pipeline_event_enqueue_burst(dev, port, ev, nb_rx); 5312eaa37b8SPavan Nikhilesh } 5322eaa37b8SPavan Nikhilesh 5332eaa37b8SPavan Nikhilesh return 0; 5342eaa37b8SPavan Nikhilesh } 5352eaa37b8SPavan Nikhilesh 536314bcf58SPavan Nikhilesh static int 537d60b4185SPavan Nikhilesh worker_wrapper(void *arg) 538d60b4185SPavan Nikhilesh { 539314bcf58SPavan Nikhilesh struct worker_data *w = arg; 540314bcf58SPavan Nikhilesh struct evt_options *opt = w->t->opt; 541314bcf58SPavan Nikhilesh const bool burst = evt_has_burst_mode(w->dev_id); 542032a965aSPavan Nikhilesh const bool internal_port = w->t->internal_port; 543314bcf58SPavan Nikhilesh const uint8_t nb_stages = opt->nb_stages; 5442eaa37b8SPavan Nikhilesh /*vector/burst/internal_port*/ 5452eaa37b8SPavan Nikhilesh const pipeline_queue_worker_t 5462eaa37b8SPavan Nikhilesh pipeline_queue_worker_single_stage[2][2][2] = { 5472eaa37b8SPavan Nikhilesh [0][0][0] = pipeline_queue_worker_single_stage_fwd, 5482eaa37b8SPavan Nikhilesh [0][0][1] = pipeline_queue_worker_single_stage_tx, 5492eaa37b8SPavan Nikhilesh [0][1][0] = pipeline_queue_worker_single_stage_burst_fwd, 5502eaa37b8SPavan Nikhilesh [0][1][1] = pipeline_queue_worker_single_stage_burst_tx, 5512eaa37b8SPavan Nikhilesh [1][0][0] = pipeline_queue_worker_single_stage_fwd_vector, 5522eaa37b8SPavan Nikhilesh [1][0][1] = pipeline_queue_worker_single_stage_tx_vector, 5532eaa37b8SPavan Nikhilesh [1][1][0] = pipeline_queue_worker_single_stage_burst_fwd_vector, 5542eaa37b8SPavan Nikhilesh [1][1][1] = pipeline_queue_worker_single_stage_burst_tx_vector, 5552eaa37b8SPavan Nikhilesh }; 5562eaa37b8SPavan Nikhilesh const pipeline_queue_worker_t 5572eaa37b8SPavan Nikhilesh pipeline_queue_worker_multi_stage[2][2][2] = { 5582eaa37b8SPavan Nikhilesh [0][0][0] = pipeline_queue_worker_multi_stage_fwd, 5592eaa37b8SPavan Nikhilesh [0][0][1] = pipeline_queue_worker_multi_stage_tx, 5602eaa37b8SPavan Nikhilesh [0][1][0] = pipeline_queue_worker_multi_stage_burst_fwd, 5612eaa37b8SPavan Nikhilesh [0][1][1] = pipeline_queue_worker_multi_stage_burst_tx, 5622eaa37b8SPavan Nikhilesh [1][0][0] = pipeline_queue_worker_multi_stage_fwd_vector, 5632eaa37b8SPavan Nikhilesh [1][0][1] = pipeline_queue_worker_multi_stage_tx_vector, 5642eaa37b8SPavan Nikhilesh [1][1][0] = pipeline_queue_worker_multi_stage_burst_fwd_vector, 5652eaa37b8SPavan Nikhilesh [1][1][1] = pipeline_queue_worker_multi_stage_burst_tx_vector, 5662eaa37b8SPavan Nikhilesh }; 567314bcf58SPavan Nikhilesh 5682eaa37b8SPavan Nikhilesh if (nb_stages == 1) 5692eaa37b8SPavan Nikhilesh return (pipeline_queue_worker_single_stage[opt->ena_vector] 5702eaa37b8SPavan Nikhilesh [burst] 5712eaa37b8SPavan Nikhilesh [internal_port])(arg); 5722eaa37b8SPavan Nikhilesh else 5732eaa37b8SPavan Nikhilesh return (pipeline_queue_worker_multi_stage[opt->ena_vector] 5742eaa37b8SPavan Nikhilesh [burst] 5752eaa37b8SPavan Nikhilesh [internal_port])(arg); 576314bcf58SPavan Nikhilesh 577d60b4185SPavan Nikhilesh rte_panic("invalid worker\n"); 578d60b4185SPavan Nikhilesh } 579d60b4185SPavan Nikhilesh 580d60b4185SPavan Nikhilesh static int 581d60b4185SPavan Nikhilesh pipeline_queue_launch_lcores(struct evt_test *test, struct evt_options *opt) 582d60b4185SPavan Nikhilesh { 583d60b4185SPavan Nikhilesh return pipeline_launch_lcores(test, opt, worker_wrapper); 584d60b4185SPavan Nikhilesh } 585d60b4185SPavan Nikhilesh 586d60b4185SPavan Nikhilesh static int 587d60b4185SPavan Nikhilesh pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) 588d60b4185SPavan Nikhilesh { 589d60b4185SPavan Nikhilesh int ret; 590d60b4185SPavan Nikhilesh int nb_ports; 591d60b4185SPavan Nikhilesh int nb_queues; 592d60b4185SPavan Nikhilesh int nb_stages = opt->nb_stages; 593d60b4185SPavan Nikhilesh uint8_t queue; 594032a965aSPavan Nikhilesh uint8_t tx_evport_id = 0; 595032a965aSPavan Nikhilesh uint8_t tx_evqueue_id[RTE_MAX_ETHPORTS]; 596d60b4185SPavan Nikhilesh uint8_t queue_arr[RTE_EVENT_MAX_QUEUES_PER_DEV]; 597d60b4185SPavan Nikhilesh uint8_t nb_worker_queues = 0; 598032a965aSPavan Nikhilesh uint16_t prod = 0; 599032a965aSPavan Nikhilesh struct rte_event_dev_info info; 600032a965aSPavan Nikhilesh struct test_pipeline *t = evt_test_priv(test); 601d60b4185SPavan Nikhilesh 602d60b4185SPavan Nikhilesh nb_ports = evt_nr_active_lcores(opt->wlcores); 603d9a42a69SThomas Monjalon nb_queues = rte_eth_dev_count_avail() * (nb_stages); 604d60b4185SPavan Nikhilesh 605032a965aSPavan Nikhilesh /* One queue for Tx adapter per port */ 606d9a42a69SThomas Monjalon nb_queues += rte_eth_dev_count_avail(); 607d60b4185SPavan Nikhilesh 608032a965aSPavan Nikhilesh memset(tx_evqueue_id, 0, sizeof(uint8_t) * RTE_MAX_ETHPORTS); 609032a965aSPavan Nikhilesh memset(queue_arr, 0, sizeof(uint8_t) * RTE_EVENT_MAX_QUEUES_PER_DEV); 610d60b4185SPavan Nikhilesh 611032a965aSPavan Nikhilesh rte_event_dev_info_get(opt->dev_id, &info); 612f0959283SPavan Nikhilesh ret = evt_configure_eventdev(opt, nb_queues, nb_ports); 613d60b4185SPavan Nikhilesh if (ret) { 614d60b4185SPavan Nikhilesh evt_err("failed to configure eventdev %d", opt->dev_id); 615d60b4185SPavan Nikhilesh return ret; 616d60b4185SPavan Nikhilesh } 617d60b4185SPavan Nikhilesh 618d60b4185SPavan Nikhilesh struct rte_event_queue_conf q_conf = { 619d60b4185SPavan Nikhilesh .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, 620d60b4185SPavan Nikhilesh .nb_atomic_flows = opt->nb_flows, 621d60b4185SPavan Nikhilesh .nb_atomic_order_sequences = opt->nb_flows, 622d60b4185SPavan Nikhilesh }; 623d60b4185SPavan Nikhilesh /* queue configurations */ 624d60b4185SPavan Nikhilesh for (queue = 0; queue < nb_queues; queue++) { 625d60b4185SPavan Nikhilesh uint8_t slot; 626d60b4185SPavan Nikhilesh 627032a965aSPavan Nikhilesh q_conf.event_queue_cfg = 0; 628d60b4185SPavan Nikhilesh slot = queue % (nb_stages + 1); 629032a965aSPavan Nikhilesh if (slot == nb_stages) { 630d60b4185SPavan Nikhilesh q_conf.schedule_type = RTE_SCHED_TYPE_ATOMIC; 631032a965aSPavan Nikhilesh if (!t->internal_port) { 632d60b4185SPavan Nikhilesh q_conf.event_queue_cfg = 633d60b4185SPavan Nikhilesh RTE_EVENT_QUEUE_CFG_SINGLE_LINK; 634032a965aSPavan Nikhilesh } 635032a965aSPavan Nikhilesh tx_evqueue_id[prod++] = queue; 636d60b4185SPavan Nikhilesh } else { 637032a965aSPavan Nikhilesh q_conf.schedule_type = opt->sched_type_list[slot]; 638d60b4185SPavan Nikhilesh queue_arr[nb_worker_queues] = queue; 639d60b4185SPavan Nikhilesh nb_worker_queues++; 640d60b4185SPavan Nikhilesh } 641d60b4185SPavan Nikhilesh 642d60b4185SPavan Nikhilesh ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf); 643d60b4185SPavan Nikhilesh if (ret) { 644d60b4185SPavan Nikhilesh evt_err("failed to setup queue=%d", queue); 645d60b4185SPavan Nikhilesh return ret; 646d60b4185SPavan Nikhilesh } 647d60b4185SPavan Nikhilesh } 648d60b4185SPavan Nikhilesh 649535c630cSPavan Nikhilesh if (opt->wkr_deq_dep > info.max_event_port_dequeue_depth) 650535c630cSPavan Nikhilesh opt->wkr_deq_dep = info.max_event_port_dequeue_depth; 651535c630cSPavan Nikhilesh 652d60b4185SPavan Nikhilesh /* port configuration */ 653d60b4185SPavan Nikhilesh const struct rte_event_port_conf p_conf = { 654d60b4185SPavan Nikhilesh .dequeue_depth = opt->wkr_deq_dep, 655d60b4185SPavan Nikhilesh .enqueue_depth = info.max_event_port_dequeue_depth, 656d60b4185SPavan Nikhilesh .new_event_threshold = info.max_num_events, 657d60b4185SPavan Nikhilesh }; 658d60b4185SPavan Nikhilesh 659032a965aSPavan Nikhilesh if (!t->internal_port) { 660d60b4185SPavan Nikhilesh ret = pipeline_event_port_setup(test, opt, queue_arr, 661d60b4185SPavan Nikhilesh nb_worker_queues, p_conf); 662d60b4185SPavan Nikhilesh if (ret) 663d60b4185SPavan Nikhilesh return ret; 664d60b4185SPavan Nikhilesh } else 665d60b4185SPavan Nikhilesh ret = pipeline_event_port_setup(test, opt, NULL, nb_queues, 666d60b4185SPavan Nikhilesh p_conf); 667d60b4185SPavan Nikhilesh 668d60b4185SPavan Nikhilesh if (ret) 669d60b4185SPavan Nikhilesh return ret; 670d60b4185SPavan Nikhilesh /* 671d60b4185SPavan Nikhilesh * The pipelines are setup in the following manner: 672d60b4185SPavan Nikhilesh * 673d60b4185SPavan Nikhilesh * eth_dev_count = 2, nb_stages = 2. 674d60b4185SPavan Nikhilesh * 675d60b4185SPavan Nikhilesh * queues = 6 676d60b4185SPavan Nikhilesh * stride = 3 677d60b4185SPavan Nikhilesh * 678d60b4185SPavan Nikhilesh * event queue pipelines: 679d60b4185SPavan Nikhilesh * eth0 -> q0 -> q1 -> (q2->tx) 680d60b4185SPavan Nikhilesh * eth1 -> q3 -> q4 -> (q5->tx) 681d60b4185SPavan Nikhilesh * 682032a965aSPavan Nikhilesh * q2, q5 configured as ATOMIC | SINGLE_LINK 683d60b4185SPavan Nikhilesh * 684d60b4185SPavan Nikhilesh */ 685032a965aSPavan Nikhilesh ret = pipeline_event_rx_adapter_setup(opt, nb_stages + 1, p_conf); 686032a965aSPavan Nikhilesh if (ret) 687032a965aSPavan Nikhilesh return ret; 688032a965aSPavan Nikhilesh 689032a965aSPavan Nikhilesh ret = pipeline_event_tx_adapter_setup(opt, p_conf); 690d60b4185SPavan Nikhilesh if (ret) 691d60b4185SPavan Nikhilesh return ret; 692d60b4185SPavan Nikhilesh 693d60b4185SPavan Nikhilesh if (!evt_has_distributed_sched(opt->dev_id)) { 694d60b4185SPavan Nikhilesh uint32_t service_id; 695d60b4185SPavan Nikhilesh rte_event_dev_service_id_get(opt->dev_id, &service_id); 696d60b4185SPavan Nikhilesh ret = evt_service_setup(service_id); 697d60b4185SPavan Nikhilesh if (ret) { 698d60b4185SPavan Nikhilesh evt_err("No service lcore found to run event dev."); 699d60b4185SPavan Nikhilesh return ret; 700d60b4185SPavan Nikhilesh } 701d60b4185SPavan Nikhilesh } 702d60b4185SPavan Nikhilesh 703032a965aSPavan Nikhilesh /* Connect the tx_evqueue_id to the Tx adapter port */ 704032a965aSPavan Nikhilesh if (!t->internal_port) { 705032a965aSPavan Nikhilesh RTE_ETH_FOREACH_DEV(prod) { 706032a965aSPavan Nikhilesh ret = rte_event_eth_tx_adapter_event_port_get(prod, 707032a965aSPavan Nikhilesh &tx_evport_id); 708032a965aSPavan Nikhilesh if (ret) { 709032a965aSPavan Nikhilesh evt_err("Unable to get Tx adptr[%d] evprt[%d]", 710032a965aSPavan Nikhilesh prod, tx_evport_id); 711032a965aSPavan Nikhilesh return ret; 712032a965aSPavan Nikhilesh } 713032a965aSPavan Nikhilesh 714032a965aSPavan Nikhilesh if (rte_event_port_link(opt->dev_id, tx_evport_id, 715032a965aSPavan Nikhilesh &tx_evqueue_id[prod], 716032a965aSPavan Nikhilesh NULL, 1) != 1) { 717032a965aSPavan Nikhilesh evt_err("Unable to link Tx adptr[%d] evprt[%d]", 718032a965aSPavan Nikhilesh prod, tx_evport_id); 719032a965aSPavan Nikhilesh return ret; 720032a965aSPavan Nikhilesh } 721032a965aSPavan Nikhilesh } 722032a965aSPavan Nikhilesh } 723032a965aSPavan Nikhilesh 72466b82db2SPavan Nikhilesh ret = rte_event_dev_start(opt->dev_id); 72566b82db2SPavan Nikhilesh if (ret) { 72666b82db2SPavan Nikhilesh evt_err("failed to start eventdev %d", opt->dev_id); 72766b82db2SPavan Nikhilesh return ret; 72866b82db2SPavan Nikhilesh } 72966b82db2SPavan Nikhilesh 73066b82db2SPavan Nikhilesh 731032a965aSPavan Nikhilesh RTE_ETH_FOREACH_DEV(prod) { 732032a965aSPavan Nikhilesh ret = rte_eth_dev_start(prod); 733032a965aSPavan Nikhilesh if (ret) { 734032a965aSPavan Nikhilesh evt_err("Ethernet dev [%d] failed to start." 735032a965aSPavan Nikhilesh " Using synthetic producer", prod); 736032a965aSPavan Nikhilesh return ret; 737032a965aSPavan Nikhilesh } 738032a965aSPavan Nikhilesh 739032a965aSPavan Nikhilesh } 740032a965aSPavan Nikhilesh 741032a965aSPavan Nikhilesh RTE_ETH_FOREACH_DEV(prod) { 742032a965aSPavan Nikhilesh ret = rte_event_eth_rx_adapter_start(prod); 743032a965aSPavan Nikhilesh if (ret) { 744032a965aSPavan Nikhilesh evt_err("Rx adapter[%d] start failed", prod); 745032a965aSPavan Nikhilesh return ret; 746032a965aSPavan Nikhilesh } 747032a965aSPavan Nikhilesh 748032a965aSPavan Nikhilesh ret = rte_event_eth_tx_adapter_start(prod); 749032a965aSPavan Nikhilesh if (ret) { 750032a965aSPavan Nikhilesh evt_err("Tx adapter[%d] start failed", prod); 751032a965aSPavan Nikhilesh return ret; 752032a965aSPavan Nikhilesh } 753032a965aSPavan Nikhilesh } 754032a965aSPavan Nikhilesh 755032a965aSPavan Nikhilesh memcpy(t->tx_evqueue_id, tx_evqueue_id, sizeof(uint8_t) * 756032a965aSPavan Nikhilesh RTE_MAX_ETHPORTS); 757032a965aSPavan Nikhilesh 758d60b4185SPavan Nikhilesh return 0; 759d60b4185SPavan Nikhilesh } 760d60b4185SPavan Nikhilesh 761d60b4185SPavan Nikhilesh static void 762d60b4185SPavan Nikhilesh pipeline_queue_opt_dump(struct evt_options *opt) 763d60b4185SPavan Nikhilesh { 764d60b4185SPavan Nikhilesh pipeline_opt_dump(opt, pipeline_queue_nb_event_queues(opt)); 765d60b4185SPavan Nikhilesh } 766d60b4185SPavan Nikhilesh 767d60b4185SPavan Nikhilesh static int 768d60b4185SPavan Nikhilesh pipeline_queue_opt_check(struct evt_options *opt) 769d60b4185SPavan Nikhilesh { 770d60b4185SPavan Nikhilesh return pipeline_opt_check(opt, pipeline_queue_nb_event_queues(opt)); 771d60b4185SPavan Nikhilesh } 772d60b4185SPavan Nikhilesh 773d60b4185SPavan Nikhilesh static bool 774d60b4185SPavan Nikhilesh pipeline_queue_capability_check(struct evt_options *opt) 775d60b4185SPavan Nikhilesh { 776d60b4185SPavan Nikhilesh struct rte_event_dev_info dev_info; 777d60b4185SPavan Nikhilesh 778d60b4185SPavan Nikhilesh rte_event_dev_info_get(opt->dev_id, &dev_info); 779d60b4185SPavan Nikhilesh if (dev_info.max_event_queues < pipeline_queue_nb_event_queues(opt) || 780d60b4185SPavan Nikhilesh dev_info.max_event_ports < 781d60b4185SPavan Nikhilesh evt_nr_active_lcores(opt->wlcores)) { 782d60b4185SPavan Nikhilesh evt_err("not enough eventdev queues=%d/%d or ports=%d/%d", 783d60b4185SPavan Nikhilesh pipeline_queue_nb_event_queues(opt), 784d60b4185SPavan Nikhilesh dev_info.max_event_queues, 785d60b4185SPavan Nikhilesh evt_nr_active_lcores(opt->wlcores), 786d60b4185SPavan Nikhilesh dev_info.max_event_ports); 787d60b4185SPavan Nikhilesh } 788d60b4185SPavan Nikhilesh 789d60b4185SPavan Nikhilesh return true; 790d60b4185SPavan Nikhilesh } 791d60b4185SPavan Nikhilesh 792d60b4185SPavan Nikhilesh static const struct evt_test_ops pipeline_queue = { 793d60b4185SPavan Nikhilesh .cap_check = pipeline_queue_capability_check, 794d60b4185SPavan Nikhilesh .opt_check = pipeline_queue_opt_check, 795d60b4185SPavan Nikhilesh .opt_dump = pipeline_queue_opt_dump, 796d60b4185SPavan Nikhilesh .test_setup = pipeline_test_setup, 797d60b4185SPavan Nikhilesh .mempool_setup = pipeline_mempool_setup, 798d60b4185SPavan Nikhilesh .ethdev_setup = pipeline_ethdev_setup, 799d60b4185SPavan Nikhilesh .eventdev_setup = pipeline_queue_eventdev_setup, 800d60b4185SPavan Nikhilesh .launch_lcores = pipeline_queue_launch_lcores, 801*a734e738SPavan Nikhilesh .ethdev_rx_stop = pipeline_ethdev_rx_stop, 802d60b4185SPavan Nikhilesh .eventdev_destroy = pipeline_eventdev_destroy, 803d60b4185SPavan Nikhilesh .mempool_destroy = pipeline_mempool_destroy, 804d60b4185SPavan Nikhilesh .ethdev_destroy = pipeline_ethdev_destroy, 805d60b4185SPavan Nikhilesh .test_result = pipeline_test_result, 806d60b4185SPavan Nikhilesh .test_destroy = pipeline_test_destroy, 807d60b4185SPavan Nikhilesh }; 808d60b4185SPavan Nikhilesh 809d60b4185SPavan Nikhilesh EVT_TEST_REGISTER(pipeline_queue); 810