1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include "test_perf_common.h" 6 7 /* See http://doc.dpdk.org/guides/tools/testeventdev.html for test details */ 8 9 static inline int 10 atq_nb_event_queues(struct evt_options *opt) 11 { 12 /* nb_queues = number of producers */ 13 return opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR ? 14 rte_eth_dev_count_avail() : evt_nr_active_lcores(opt->plcores); 15 } 16 17 static __rte_always_inline void 18 atq_mark_fwd_latency(struct rte_event *const ev) 19 { 20 if (unlikely(ev->sub_event_type == 0)) { 21 struct perf_elt *const m = ev->event_ptr; 22 23 m->timestamp = rte_get_timer_cycles(); 24 } 25 } 26 27 static __rte_always_inline void 28 atq_fwd_event(struct rte_event *const ev, uint8_t *const sched_type_list, 29 const uint8_t nb_stages) 30 { 31 ev->sub_event_type++; 32 ev->sched_type = sched_type_list[ev->sub_event_type % nb_stages]; 33 ev->op = RTE_EVENT_OP_FORWARD; 34 ev->event_type = RTE_EVENT_TYPE_CPU; 35 } 36 37 static int 38 perf_atq_worker(void *arg, const int enable_fwd_latency) 39 { 40 uint16_t enq = 0, deq = 0; 41 struct rte_event ev; 42 PERF_WORKER_INIT; 43 44 while (t->done == false) { 45 deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0); 46 47 if (!deq) { 48 rte_pause(); 49 continue; 50 } 51 52 if (prod_crypto_type && 53 (ev.event_type == RTE_EVENT_TYPE_CRYPTODEV)) { 54 struct rte_crypto_op *op = ev.event_ptr; 55 56 if (op->status == RTE_CRYPTO_OP_STATUS_SUCCESS) { 57 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 58 if (op->sym->m_dst == NULL) 59 ev.event_ptr = op->sym->m_src; 60 else 61 ev.event_ptr = op->sym->m_dst; 62 rte_crypto_op_free(op); 63 } 64 } else { 65 rte_crypto_op_free(op); 66 continue; 67 } 68 } 69 70 if (enable_fwd_latency && !prod_timer_type) 71 /* first stage in pipeline, mark ts to compute fwd latency */ 72 atq_mark_fwd_latency(&ev); 73 74 /* last stage in pipeline */ 75 if (unlikely((ev.sub_event_type % nb_stages) == laststage)) { 76 if (enable_fwd_latency) 77 cnt = perf_process_last_stage_latency(pool, 78 &ev, w, bufs, sz, cnt); 79 else 80 cnt = perf_process_last_stage(pool, &ev, w, 81 bufs, sz, cnt); 82 } else { 83 atq_fwd_event(&ev, sched_type_list, nb_stages); 84 do { 85 enq = rte_event_enqueue_burst(dev, port, &ev, 86 1); 87 } while (!enq && !t->done); 88 } 89 } 90 91 perf_worker_cleanup(pool, dev, port, &ev, enq, deq); 92 93 return 0; 94 } 95 96 static int 97 perf_atq_worker_burst(void *arg, const int enable_fwd_latency) 98 { 99 /* +1 to avoid prefetch out of array check */ 100 struct rte_event ev[BURST_SIZE + 1]; 101 uint16_t enq = 0, nb_rx = 0; 102 PERF_WORKER_INIT; 103 uint16_t i; 104 105 while (t->done == false) { 106 nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0); 107 108 if (!nb_rx) { 109 rte_pause(); 110 continue; 111 } 112 113 for (i = 0; i < nb_rx; i++) { 114 if (prod_crypto_type && 115 (ev[i].event_type == RTE_EVENT_TYPE_CRYPTODEV)) { 116 struct rte_crypto_op *op = ev[i].event_ptr; 117 118 if (op->status == 119 RTE_CRYPTO_OP_STATUS_SUCCESS) { 120 if (op->sym->m_dst == NULL) 121 ev[i].event_ptr = 122 op->sym->m_src; 123 else 124 ev[i].event_ptr = 125 op->sym->m_dst; 126 rte_crypto_op_free(op); 127 } else { 128 rte_crypto_op_free(op); 129 continue; 130 } 131 } 132 133 if (enable_fwd_latency && !prod_timer_type) { 134 rte_prefetch0(ev[i+1].event_ptr); 135 /* first stage in pipeline. 136 * mark time stamp to compute fwd latency 137 */ 138 atq_mark_fwd_latency(&ev[i]); 139 } 140 /* last stage in pipeline */ 141 if (unlikely((ev[i].sub_event_type % nb_stages) 142 == laststage)) { 143 if (enable_fwd_latency) 144 cnt = perf_process_last_stage_latency( 145 pool, &ev[i], w, bufs, sz, cnt); 146 else 147 cnt = perf_process_last_stage(pool, 148 &ev[i], w, bufs, sz, cnt); 149 150 ev[i].op = RTE_EVENT_OP_RELEASE; 151 } else { 152 atq_fwd_event(&ev[i], sched_type_list, 153 nb_stages); 154 } 155 } 156 157 enq = rte_event_enqueue_burst(dev, port, ev, nb_rx); 158 while ((enq < nb_rx) && !t->done) { 159 enq += rte_event_enqueue_burst(dev, port, 160 ev + enq, nb_rx - enq); 161 } 162 } 163 164 perf_worker_cleanup(pool, dev, port, ev, enq, nb_rx); 165 166 return 0; 167 } 168 169 static int 170 worker_wrapper(void *arg) 171 { 172 struct worker_data *w = arg; 173 struct evt_options *opt = w->t->opt; 174 175 const bool burst = evt_has_burst_mode(w->dev_id); 176 const int fwd_latency = opt->fwd_latency; 177 178 /* allow compiler to optimize */ 179 if (!burst && !fwd_latency) 180 return perf_atq_worker(arg, 0); 181 else if (!burst && fwd_latency) 182 return perf_atq_worker(arg, 1); 183 else if (burst && !fwd_latency) 184 return perf_atq_worker_burst(arg, 0); 185 else if (burst && fwd_latency) 186 return perf_atq_worker_burst(arg, 1); 187 188 rte_panic("invalid worker\n"); 189 } 190 191 static int 192 perf_atq_launch_lcores(struct evt_test *test, struct evt_options *opt) 193 { 194 return perf_launch_lcores(test, opt, worker_wrapper); 195 } 196 197 static int 198 perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) 199 { 200 int ret; 201 uint8_t queue; 202 uint8_t nb_queues; 203 uint8_t nb_ports; 204 uint16_t prod; 205 struct rte_event_dev_info dev_info; 206 struct test_perf *t = evt_test_priv(test); 207 208 nb_ports = evt_nr_active_lcores(opt->wlcores); 209 nb_ports += (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR || 210 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) ? 0 : 211 evt_nr_active_lcores(opt->plcores); 212 213 nb_queues = atq_nb_event_queues(opt); 214 215 memset(&dev_info, 0, sizeof(struct rte_event_dev_info)); 216 ret = rte_event_dev_info_get(opt->dev_id, &dev_info); 217 if (ret) { 218 evt_err("failed to get eventdev info %d", opt->dev_id); 219 return ret; 220 } 221 222 ret = evt_configure_eventdev(opt, nb_queues, nb_ports); 223 if (ret) { 224 evt_err("failed to configure eventdev %d", opt->dev_id); 225 return ret; 226 } 227 228 struct rte_event_queue_conf q_conf = { 229 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, 230 .event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES, 231 .nb_atomic_flows = opt->nb_flows, 232 .nb_atomic_order_sequences = opt->nb_flows, 233 }; 234 /* queue configurations */ 235 for (queue = 0; queue < nb_queues; queue++) { 236 ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf); 237 if (ret) { 238 evt_err("failed to setup queue=%d", queue); 239 return ret; 240 } 241 } 242 243 if (opt->wkr_deq_dep > dev_info.max_event_port_dequeue_depth) 244 opt->wkr_deq_dep = dev_info.max_event_port_dequeue_depth; 245 246 /* port configuration */ 247 const struct rte_event_port_conf p_conf = { 248 .dequeue_depth = opt->wkr_deq_dep, 249 .enqueue_depth = dev_info.max_event_port_dequeue_depth, 250 .new_event_threshold = dev_info.max_num_events, 251 }; 252 253 ret = perf_event_dev_port_setup(test, opt, 1 /* stride */, nb_queues, 254 &p_conf); 255 if (ret) 256 return ret; 257 258 if (!evt_has_distributed_sched(opt->dev_id)) { 259 uint32_t service_id; 260 rte_event_dev_service_id_get(opt->dev_id, &service_id); 261 ret = evt_service_setup(service_id); 262 if (ret) { 263 evt_err("No service lcore found to run event dev."); 264 return ret; 265 } 266 } 267 268 ret = rte_event_dev_start(opt->dev_id); 269 if (ret) { 270 evt_err("failed to start eventdev %d", opt->dev_id); 271 return ret; 272 } 273 274 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 275 RTE_ETH_FOREACH_DEV(prod) { 276 ret = rte_eth_dev_start(prod); 277 if (ret) { 278 evt_err("Ethernet dev [%d] failed to start. Using synthetic producer", 279 prod); 280 return ret; 281 } 282 283 ret = rte_event_eth_rx_adapter_start(prod); 284 if (ret) { 285 evt_err("Rx adapter[%d] start failed", prod); 286 return ret; 287 } 288 printf("%s: Port[%d] using Rx adapter[%d] started\n", 289 __func__, prod, prod); 290 } 291 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 292 for (prod = 0; prod < opt->nb_timer_adptrs; prod++) { 293 ret = rte_event_timer_adapter_start( 294 t->timer_adptr[prod]); 295 if (ret) { 296 evt_err("failed to Start event timer adapter %d" 297 , prod); 298 return ret; 299 } 300 } 301 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { 302 uint8_t cdev_id, cdev_count; 303 304 cdev_count = rte_cryptodev_count(); 305 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 306 ret = rte_cryptodev_start(cdev_id); 307 if (ret) { 308 evt_err("Failed to start cryptodev %u", 309 cdev_id); 310 return ret; 311 } 312 } 313 } 314 315 return 0; 316 } 317 318 static void 319 perf_atq_opt_dump(struct evt_options *opt) 320 { 321 perf_opt_dump(opt, atq_nb_event_queues(opt)); 322 } 323 324 static int 325 perf_atq_opt_check(struct evt_options *opt) 326 { 327 return perf_opt_check(opt, atq_nb_event_queues(opt)); 328 } 329 330 static bool 331 perf_atq_capability_check(struct evt_options *opt) 332 { 333 struct rte_event_dev_info dev_info; 334 335 rte_event_dev_info_get(opt->dev_id, &dev_info); 336 if (dev_info.max_event_queues < atq_nb_event_queues(opt) || 337 dev_info.max_event_ports < perf_nb_event_ports(opt)) { 338 evt_err("not enough eventdev queues=%d/%d or ports=%d/%d", 339 atq_nb_event_queues(opt), dev_info.max_event_queues, 340 perf_nb_event_ports(opt), dev_info.max_event_ports); 341 } 342 if (!evt_has_all_types_queue(opt->dev_id)) 343 return false; 344 345 return true; 346 } 347 348 static const struct evt_test_ops perf_atq = { 349 .cap_check = perf_atq_capability_check, 350 .opt_check = perf_atq_opt_check, 351 .opt_dump = perf_atq_opt_dump, 352 .test_setup = perf_test_setup, 353 .ethdev_setup = perf_ethdev_setup, 354 .cryptodev_setup = perf_cryptodev_setup, 355 .ethdev_rx_stop = perf_ethdev_rx_stop, 356 .mempool_setup = perf_mempool_setup, 357 .eventdev_setup = perf_atq_eventdev_setup, 358 .launch_lcores = perf_atq_launch_lcores, 359 .eventdev_destroy = perf_eventdev_destroy, 360 .mempool_destroy = perf_mempool_destroy, 361 .ethdev_destroy = perf_ethdev_destroy, 362 .cryptodev_destroy = perf_cryptodev_destroy, 363 .test_result = perf_test_result, 364 .test_destroy = perf_test_destroy, 365 }; 366 367 EVT_TEST_REGISTER(perf_atq); 368