1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include <stdalign.h> 6 7 #include "test_order_common.h" 8 9 int 10 order_test_result(struct evt_test *test, struct evt_options *opt) 11 { 12 RTE_SET_USED(opt); 13 struct test_order *t = evt_test_priv(test); 14 15 return t->result; 16 } 17 18 static inline int 19 order_producer(void *arg) 20 { 21 struct prod_data *p = arg; 22 struct test_order *t = p->t; 23 struct evt_options *opt = t->opt; 24 const uint8_t dev_id = p->dev_id; 25 const uint8_t port = p->port_id; 26 struct rte_mempool *pool = t->pool; 27 const uint64_t nb_pkts = t->nb_pkts; 28 uint32_t *producer_flow_seq = t->producer_flow_seq; 29 const uint32_t nb_flows = t->nb_flows; 30 uint64_t count = 0; 31 struct rte_mbuf *m; 32 struct rte_event ev; 33 34 if (opt->verbose_level > 1) 35 printf("%s(): lcore %d dev_id %d port=%d queue=%d\n", 36 __func__, rte_lcore_id(), dev_id, port, p->queue_id); 37 38 ev.event = 0; 39 ev.op = RTE_EVENT_OP_NEW; 40 ev.queue_id = p->queue_id; 41 ev.sched_type = RTE_SCHED_TYPE_ORDERED; 42 ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 43 ev.event_type = RTE_EVENT_TYPE_CPU; 44 ev.sub_event_type = 0; /* stage 0 */ 45 46 while (count < nb_pkts && t->err == false) { 47 m = rte_pktmbuf_alloc(pool); 48 if (m == NULL) 49 continue; 50 51 const flow_id_t flow = (uintptr_t)m % nb_flows; 52 /* Maintain seq number per flow */ 53 *order_mbuf_seqn(t, m) = producer_flow_seq[flow]++; 54 order_flow_id_save(t, flow, m, &ev); 55 56 while (rte_event_enqueue_burst(dev_id, port, &ev, 1) != 1) { 57 if (t->err) 58 break; 59 rte_pause(); 60 } 61 62 count++; 63 } 64 return 0; 65 } 66 67 int 68 order_opt_check(struct evt_options *opt) 69 { 70 if (opt->prod_type != EVT_PROD_TYPE_SYNT) { 71 evt_err("Invalid producer type '%s' valid producer '%s'", 72 evt_prod_id_to_name(opt->prod_type), 73 evt_prod_id_to_name(EVT_PROD_TYPE_SYNT)); 74 return -1; 75 } 76 77 /* 1 producer + N workers + main */ 78 if (rte_lcore_count() < 3) { 79 evt_err("test need minimum 3 lcores"); 80 return -1; 81 } 82 83 /* Validate worker lcores */ 84 if (evt_lcores_has_overlap(opt->wlcores, rte_get_main_lcore())) { 85 evt_err("worker lcores overlaps with main lcore"); 86 return -1; 87 } 88 89 if (evt_nr_active_lcores(opt->plcores) == 0) { 90 evt_err("missing the producer lcore"); 91 return -1; 92 } 93 94 if (evt_nr_active_lcores(opt->plcores) != 1) { 95 evt_err("only one producer lcore must be selected"); 96 return -1; 97 } 98 99 int plcore = evt_get_first_active_lcore(opt->plcores); 100 101 if (plcore < 0) { 102 evt_err("failed to find active producer"); 103 return plcore; 104 } 105 106 if (evt_lcores_has_overlap(opt->wlcores, plcore)) { 107 evt_err("worker lcores overlaps producer lcore"); 108 return -1; 109 } 110 if (evt_has_disabled_lcore(opt->wlcores)) { 111 evt_err("one or more workers lcores are not enabled"); 112 return -1; 113 } 114 if (!evt_has_active_lcore(opt->wlcores)) { 115 evt_err("minimum one worker is required"); 116 return -1; 117 } 118 119 /* Validate producer lcore */ 120 if (plcore == (int)rte_get_main_lcore()) { 121 evt_err("producer lcore and main lcore should be different"); 122 return -1; 123 } 124 if (!rte_lcore_is_enabled(plcore)) { 125 evt_err("producer lcore is not enabled"); 126 return -1; 127 } 128 129 /* Fixups */ 130 if (opt->nb_pkts == 0) 131 opt->nb_pkts = INT64_MAX; 132 133 return 0; 134 } 135 136 int 137 order_test_setup(struct evt_test *test, struct evt_options *opt) 138 { 139 void *test_order; 140 struct test_order *t; 141 static const struct rte_mbuf_dynfield flow_id_dynfield_desc = { 142 .name = "test_event_dynfield_flow_id", 143 .size = sizeof(flow_id_t), 144 .align = alignof(flow_id_t), 145 }; 146 static const struct rte_mbuf_dynfield seqn_dynfield_desc = { 147 .name = "test_event_dynfield_seqn", 148 .size = sizeof(seqn_t), 149 .align = alignof(seqn_t), 150 }; 151 152 test_order = rte_zmalloc_socket(test->name, sizeof(struct test_order), 153 RTE_CACHE_LINE_SIZE, opt->socket_id); 154 if (test_order == NULL) { 155 evt_err("failed to allocate test_order memory"); 156 goto nomem; 157 } 158 test->test_priv = test_order; 159 t = evt_test_priv(test); 160 161 t->flow_id_dynfield_offset = 162 rte_mbuf_dynfield_register(&flow_id_dynfield_desc); 163 if (t->flow_id_dynfield_offset < 0) { 164 evt_err("failed to register mbuf field"); 165 return -rte_errno; 166 } 167 168 t->seqn_dynfield_offset = 169 rte_mbuf_dynfield_register(&seqn_dynfield_desc); 170 if (t->seqn_dynfield_offset < 0) { 171 evt_err("failed to register mbuf field"); 172 return -rte_errno; 173 } 174 175 t->producer_flow_seq = rte_zmalloc_socket("test_producer_flow_seq", 176 sizeof(*t->producer_flow_seq) * opt->nb_flows, 177 RTE_CACHE_LINE_SIZE, opt->socket_id); 178 179 if (t->producer_flow_seq == NULL) { 180 evt_err("failed to allocate t->producer_flow_seq memory"); 181 goto prod_nomem; 182 } 183 184 t->expected_flow_seq = rte_zmalloc_socket("test_expected_flow_seq", 185 sizeof(*t->expected_flow_seq) * opt->nb_flows, 186 RTE_CACHE_LINE_SIZE, opt->socket_id); 187 188 if (t->expected_flow_seq == NULL) { 189 evt_err("failed to allocate t->expected_flow_seq memory"); 190 goto exp_nomem; 191 } 192 rte_atomic_store_explicit(&t->outstand_pkts, opt->nb_pkts, rte_memory_order_relaxed); 193 t->err = false; 194 t->nb_pkts = opt->nb_pkts; 195 t->nb_flows = opt->nb_flows; 196 t->result = EVT_TEST_FAILED; 197 t->opt = opt; 198 return 0; 199 200 exp_nomem: 201 rte_free(t->producer_flow_seq); 202 prod_nomem: 203 rte_free(test->test_priv); 204 nomem: 205 return -ENOMEM; 206 } 207 208 void 209 order_test_destroy(struct evt_test *test, struct evt_options *opt) 210 { 211 RTE_SET_USED(opt); 212 struct test_order *t = evt_test_priv(test); 213 214 rte_free(t->expected_flow_seq); 215 rte_free(t->producer_flow_seq); 216 rte_free(test->test_priv); 217 } 218 219 int 220 order_mempool_setup(struct evt_test *test, struct evt_options *opt) 221 { 222 struct test_order *t = evt_test_priv(test); 223 224 t->pool = rte_pktmbuf_pool_create(test->name, opt->pool_sz, 225 256 /* Cache */, 0, 226 512, /* Use very small mbufs */ 227 opt->socket_id); 228 if (t->pool == NULL) { 229 evt_err("failed to create mempool"); 230 return -ENOMEM; 231 } 232 233 return 0; 234 } 235 236 void 237 order_mempool_destroy(struct evt_test *test, struct evt_options *opt) 238 { 239 RTE_SET_USED(opt); 240 struct test_order *t = evt_test_priv(test); 241 242 rte_mempool_free(t->pool); 243 } 244 245 void 246 order_eventdev_destroy(struct evt_test *test, struct evt_options *opt) 247 { 248 RTE_SET_USED(test); 249 250 rte_event_dev_stop(opt->dev_id); 251 rte_event_dev_close(opt->dev_id); 252 } 253 254 void 255 order_opt_dump(struct evt_options *opt) 256 { 257 evt_dump_producer_lcores(opt); 258 evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores)); 259 evt_dump_worker_lcores(opt); 260 evt_dump("nb_evdev_ports", "%d", order_nb_event_ports(opt)); 261 } 262 263 int 264 order_launch_lcores(struct evt_test *test, struct evt_options *opt, 265 int (*worker)(void *)) 266 { 267 int ret, lcore_id; 268 struct test_order *t = evt_test_priv(test); 269 270 int wkr_idx = 0; 271 /* launch workers */ 272 RTE_LCORE_FOREACH_WORKER(lcore_id) { 273 if (!(opt->wlcores[lcore_id])) 274 continue; 275 276 ret = rte_eal_remote_launch(worker, &t->worker[wkr_idx], 277 lcore_id); 278 if (ret) { 279 evt_err("failed to launch worker %d", lcore_id); 280 return ret; 281 } 282 wkr_idx++; 283 } 284 285 /* launch producer */ 286 int plcore = evt_get_first_active_lcore(opt->plcores); 287 288 ret = rte_eal_remote_launch(order_producer, &t->prod, plcore); 289 if (ret) { 290 evt_err("failed to launch order_producer %d", plcore); 291 return ret; 292 } 293 294 uint64_t cycles = rte_get_timer_cycles(); 295 int64_t old_remaining = -1; 296 297 while (t->err == false) { 298 uint64_t new_cycles = rte_get_timer_cycles(); 299 int64_t remaining = rte_atomic_load_explicit(&t->outstand_pkts, 300 rte_memory_order_relaxed); 301 302 if (remaining <= 0) { 303 t->result = EVT_TEST_SUCCESS; 304 break; 305 } 306 307 if (new_cycles - cycles > rte_get_timer_hz() * 1) { 308 printf(CLGRN"\r%"PRId64""CLNRM, remaining); 309 fflush(stdout); 310 if (old_remaining == remaining) { 311 rte_event_dev_dump(opt->dev_id, stdout); 312 evt_err("No schedules for seconds, deadlock"); 313 t->err = true; 314 break; 315 } 316 old_remaining = remaining; 317 cycles = new_cycles; 318 } 319 } 320 printf("\r"); 321 322 return 0; 323 } 324 325 int 326 order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, 327 uint8_t nb_workers, uint8_t nb_queues) 328 { 329 int ret; 330 uint8_t port; 331 struct test_order *t = evt_test_priv(test); 332 struct rte_event_dev_info dev_info; 333 334 ret = rte_event_dev_info_get(opt->dev_id, &dev_info); 335 if (ret) { 336 evt_err("failed to get eventdev info %d", opt->dev_id); 337 return ret; 338 } 339 340 if (opt->wkr_deq_dep > dev_info.max_event_port_dequeue_depth) 341 opt->wkr_deq_dep = dev_info.max_event_port_dequeue_depth; 342 343 /* port configuration */ 344 const struct rte_event_port_conf p_conf = { 345 .dequeue_depth = opt->wkr_deq_dep, 346 .enqueue_depth = dev_info.max_event_port_dequeue_depth, 347 .new_event_threshold = dev_info.max_num_events, 348 }; 349 350 /* setup one port per worker, linking to all queues */ 351 for (port = 0; port < nb_workers; port++) { 352 struct worker_data *w = &t->worker[port]; 353 354 w->dev_id = opt->dev_id; 355 w->port_id = port; 356 w->t = t; 357 358 ret = rte_event_port_setup(opt->dev_id, port, &p_conf); 359 if (ret) { 360 evt_err("failed to setup port %d", port); 361 return ret; 362 } 363 364 ret = rte_event_port_link(opt->dev_id, port, NULL, NULL, 0); 365 if (ret != nb_queues) { 366 evt_err("failed to link all queues to port %d", port); 367 return -EINVAL; 368 } 369 } 370 struct prod_data *p = &t->prod; 371 372 p->dev_id = opt->dev_id; 373 p->port_id = port; /* last port */ 374 p->queue_id = 0; 375 p->t = t; 376 377 ret = rte_event_port_setup(opt->dev_id, port, &p_conf); 378 if (ret) { 379 evt_err("failed to setup producer port %d", port); 380 return ret; 381 } 382 383 return ret; 384 } 385