1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include <math.h> 6 7 #include "test_perf_common.h" 8 9 int 10 perf_test_result(struct evt_test *test, struct evt_options *opt) 11 { 12 RTE_SET_USED(opt); 13 int i; 14 uint64_t total = 0; 15 struct test_perf *t = evt_test_priv(test); 16 17 printf("Packet distribution across worker cores :\n"); 18 for (i = 0; i < t->nb_workers; i++) 19 total += t->worker[i].processed_pkts; 20 for (i = 0; i < t->nb_workers; i++) 21 printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:" 22 CLGRN" %3.2f"CLNRM"\n", i, 23 t->worker[i].processed_pkts, 24 (((double)t->worker[i].processed_pkts)/total) 25 * 100); 26 27 return t->result; 28 } 29 30 static inline int 31 perf_producer(void *arg) 32 { 33 int i; 34 struct prod_data *p = arg; 35 struct test_perf *t = p->t; 36 struct evt_options *opt = t->opt; 37 const uint8_t dev_id = p->dev_id; 38 const uint8_t port = p->port_id; 39 struct rte_mempool *pool = t->pool; 40 const uint64_t nb_pkts = t->nb_pkts; 41 const uint32_t nb_flows = t->nb_flows; 42 uint32_t flow_counter = 0; 43 uint64_t count = 0; 44 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 45 struct rte_event ev; 46 47 if (opt->verbose_level > 1) 48 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__, 49 rte_lcore_id(), dev_id, port, p->queue_id); 50 51 ev.event = 0; 52 ev.op = RTE_EVENT_OP_NEW; 53 ev.queue_id = p->queue_id; 54 ev.sched_type = t->opt->sched_type_list[0]; 55 ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 56 ev.event_type = RTE_EVENT_TYPE_CPU; 57 ev.sub_event_type = 0; /* stage 0 */ 58 59 while (count < nb_pkts && t->done == false) { 60 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 61 continue; 62 for (i = 0; i < BURST_SIZE; i++) { 63 ev.flow_id = flow_counter++ % nb_flows; 64 ev.event_ptr = m[i]; 65 m[i]->timestamp = rte_get_timer_cycles(); 66 while (rte_event_enqueue_burst(dev_id, 67 port, &ev, 1) != 1) { 68 if (t->done) 69 break; 70 rte_pause(); 71 m[i]->timestamp = rte_get_timer_cycles(); 72 } 73 } 74 count += BURST_SIZE; 75 } 76 77 return 0; 78 } 79 80 static inline int 81 perf_producer_burst(void *arg) 82 { 83 uint32_t i; 84 uint64_t timestamp; 85 struct rte_event_dev_info dev_info; 86 struct prod_data *p = arg; 87 struct test_perf *t = p->t; 88 struct evt_options *opt = t->opt; 89 const uint8_t dev_id = p->dev_id; 90 const uint8_t port = p->port_id; 91 struct rte_mempool *pool = t->pool; 92 const uint64_t nb_pkts = t->nb_pkts; 93 const uint32_t nb_flows = t->nb_flows; 94 uint32_t flow_counter = 0; 95 uint16_t enq = 0; 96 uint64_t count = 0; 97 struct perf_elt *m[MAX_PROD_ENQ_BURST_SIZE + 1]; 98 struct rte_event ev[MAX_PROD_ENQ_BURST_SIZE + 1]; 99 uint32_t burst_size = opt->prod_enq_burst_sz; 100 101 memset(m, 0, sizeof(*m) * (MAX_PROD_ENQ_BURST_SIZE + 1)); 102 rte_event_dev_info_get(dev_id, &dev_info); 103 if (dev_info.max_event_port_enqueue_depth < burst_size) 104 burst_size = dev_info.max_event_port_enqueue_depth; 105 106 if (opt->verbose_level > 1) 107 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__, 108 rte_lcore_id(), dev_id, port, p->queue_id); 109 110 for (i = 0; i < burst_size; i++) { 111 ev[i].op = RTE_EVENT_OP_NEW; 112 ev[i].queue_id = p->queue_id; 113 ev[i].sched_type = t->opt->sched_type_list[0]; 114 ev[i].priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 115 ev[i].event_type = RTE_EVENT_TYPE_CPU; 116 ev[i].sub_event_type = 0; /* stage 0 */ 117 } 118 119 while (count < nb_pkts && t->done == false) { 120 if (rte_mempool_get_bulk(pool, (void **)m, burst_size) < 0) 121 continue; 122 timestamp = rte_get_timer_cycles(); 123 for (i = 0; i < burst_size; i++) { 124 ev[i].flow_id = flow_counter++ % nb_flows; 125 ev[i].event_ptr = m[i]; 126 m[i]->timestamp = timestamp; 127 } 128 enq = rte_event_enqueue_burst(dev_id, port, ev, burst_size); 129 while (enq < burst_size) { 130 enq += rte_event_enqueue_burst(dev_id, port, 131 ev + enq, 132 burst_size - enq); 133 if (t->done) 134 break; 135 rte_pause(); 136 timestamp = rte_get_timer_cycles(); 137 for (i = enq; i < burst_size; i++) 138 m[i]->timestamp = timestamp; 139 } 140 count += burst_size; 141 } 142 return 0; 143 } 144 145 static inline int 146 perf_event_timer_producer(void *arg) 147 { 148 int i; 149 struct prod_data *p = arg; 150 struct test_perf *t = p->t; 151 struct evt_options *opt = t->opt; 152 uint32_t flow_counter = 0; 153 uint64_t count = 0; 154 uint64_t arm_latency = 0; 155 const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs; 156 const uint32_t nb_flows = t->nb_flows; 157 const uint64_t nb_timers = opt->nb_timers; 158 struct rte_mempool *pool = t->pool; 159 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 160 struct rte_event_timer_adapter **adptr = t->timer_adptr; 161 struct rte_event_timer tim; 162 uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; 163 164 memset(&tim, 0, sizeof(struct rte_event_timer)); 165 timeout_ticks = 166 opt->optm_timer_tick_nsec 167 ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) / 168 opt->optm_timer_tick_nsec) 169 : timeout_ticks; 170 timeout_ticks += timeout_ticks ? 0 : 1; 171 tim.ev.event_type = RTE_EVENT_TYPE_TIMER; 172 tim.ev.op = RTE_EVENT_OP_NEW; 173 tim.ev.sched_type = t->opt->sched_type_list[0]; 174 tim.ev.queue_id = p->queue_id; 175 tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 176 tim.state = RTE_EVENT_TIMER_NOT_ARMED; 177 tim.timeout_ticks = timeout_ticks; 178 179 if (opt->verbose_level > 1) 180 printf("%s(): lcore %d\n", __func__, rte_lcore_id()); 181 182 while (count < nb_timers && t->done == false) { 183 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 184 continue; 185 for (i = 0; i < BURST_SIZE; i++) { 186 rte_prefetch0(m[i + 1]); 187 m[i]->tim = tim; 188 m[i]->tim.ev.flow_id = flow_counter++ % nb_flows; 189 m[i]->tim.ev.event_ptr = m[i]; 190 m[i]->timestamp = rte_get_timer_cycles(); 191 while (rte_event_timer_arm_burst( 192 adptr[flow_counter % nb_timer_adptrs], 193 (struct rte_event_timer **)&m[i], 1) != 1) { 194 if (t->done) 195 break; 196 m[i]->timestamp = rte_get_timer_cycles(); 197 } 198 arm_latency += rte_get_timer_cycles() - m[i]->timestamp; 199 } 200 count += BURST_SIZE; 201 } 202 fflush(stdout); 203 rte_delay_ms(1000); 204 printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", 205 __func__, rte_lcore_id(), 206 count ? (float)(arm_latency / count) / 207 (rte_get_timer_hz() / 1000000) : 0); 208 return 0; 209 } 210 211 static inline int 212 perf_event_timer_producer_burst(void *arg) 213 { 214 int i; 215 struct prod_data *p = arg; 216 struct test_perf *t = p->t; 217 struct evt_options *opt = t->opt; 218 uint32_t flow_counter = 0; 219 uint64_t count = 0; 220 uint64_t arm_latency = 0; 221 const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs; 222 const uint32_t nb_flows = t->nb_flows; 223 const uint64_t nb_timers = opt->nb_timers; 224 struct rte_mempool *pool = t->pool; 225 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 226 struct rte_event_timer_adapter **adptr = t->timer_adptr; 227 struct rte_event_timer tim; 228 uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; 229 230 memset(&tim, 0, sizeof(struct rte_event_timer)); 231 timeout_ticks = 232 opt->optm_timer_tick_nsec 233 ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) / 234 opt->optm_timer_tick_nsec) 235 : timeout_ticks; 236 timeout_ticks += timeout_ticks ? 0 : 1; 237 tim.ev.event_type = RTE_EVENT_TYPE_TIMER; 238 tim.ev.op = RTE_EVENT_OP_NEW; 239 tim.ev.sched_type = t->opt->sched_type_list[0]; 240 tim.ev.queue_id = p->queue_id; 241 tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 242 tim.state = RTE_EVENT_TIMER_NOT_ARMED; 243 tim.timeout_ticks = timeout_ticks; 244 245 if (opt->verbose_level > 1) 246 printf("%s(): lcore %d\n", __func__, rte_lcore_id()); 247 248 while (count < nb_timers && t->done == false) { 249 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 250 continue; 251 for (i = 0; i < BURST_SIZE; i++) { 252 rte_prefetch0(m[i + 1]); 253 m[i]->tim = tim; 254 m[i]->tim.ev.flow_id = flow_counter++ % nb_flows; 255 m[i]->tim.ev.event_ptr = m[i]; 256 m[i]->timestamp = rte_get_timer_cycles(); 257 } 258 rte_event_timer_arm_tmo_tick_burst( 259 adptr[flow_counter % nb_timer_adptrs], 260 (struct rte_event_timer **)m, 261 tim.timeout_ticks, 262 BURST_SIZE); 263 arm_latency += rte_get_timer_cycles() - m[i - 1]->timestamp; 264 count += BURST_SIZE; 265 } 266 fflush(stdout); 267 rte_delay_ms(1000); 268 printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", 269 __func__, rte_lcore_id(), 270 count ? (float)(arm_latency / count) / 271 (rte_get_timer_hz() / 1000000) : 0); 272 return 0; 273 } 274 275 static int 276 perf_producer_wrapper(void *arg) 277 { 278 struct prod_data *p = arg; 279 struct test_perf *t = p->t; 280 bool burst = evt_has_burst_mode(p->dev_id); 281 282 /* In case of synthetic producer, launch perf_producer or 283 * perf_producer_burst depending on producer enqueue burst size 284 */ 285 if (t->opt->prod_type == EVT_PROD_TYPE_SYNT && 286 t->opt->prod_enq_burst_sz == 1) 287 return perf_producer(arg); 288 else if (t->opt->prod_type == EVT_PROD_TYPE_SYNT && 289 t->opt->prod_enq_burst_sz > 1) { 290 if (!burst) 291 evt_err("This event device does not support burst mode"); 292 else 293 return perf_producer_burst(arg); 294 } 295 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && 296 !t->opt->timdev_use_burst) 297 return perf_event_timer_producer(arg); 298 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && 299 t->opt->timdev_use_burst) 300 return perf_event_timer_producer_burst(arg); 301 return 0; 302 } 303 304 static inline uint64_t 305 processed_pkts(struct test_perf *t) 306 { 307 uint8_t i; 308 uint64_t total = 0; 309 310 for (i = 0; i < t->nb_workers; i++) 311 total += t->worker[i].processed_pkts; 312 313 return total; 314 } 315 316 static inline uint64_t 317 total_latency(struct test_perf *t) 318 { 319 uint8_t i; 320 uint64_t total = 0; 321 322 for (i = 0; i < t->nb_workers; i++) 323 total += t->worker[i].latency; 324 325 return total; 326 } 327 328 329 int 330 perf_launch_lcores(struct evt_test *test, struct evt_options *opt, 331 int (*worker)(void *)) 332 { 333 int ret, lcore_id; 334 struct test_perf *t = evt_test_priv(test); 335 336 int port_idx = 0; 337 /* launch workers */ 338 RTE_LCORE_FOREACH_WORKER(lcore_id) { 339 if (!(opt->wlcores[lcore_id])) 340 continue; 341 342 ret = rte_eal_remote_launch(worker, 343 &t->worker[port_idx], lcore_id); 344 if (ret) { 345 evt_err("failed to launch worker %d", lcore_id); 346 return ret; 347 } 348 port_idx++; 349 } 350 351 /* launch producers */ 352 RTE_LCORE_FOREACH_WORKER(lcore_id) { 353 if (!(opt->plcores[lcore_id])) 354 continue; 355 356 ret = rte_eal_remote_launch(perf_producer_wrapper, 357 &t->prod[port_idx], lcore_id); 358 if (ret) { 359 evt_err("failed to launch perf_producer %d", lcore_id); 360 return ret; 361 } 362 port_idx++; 363 } 364 365 const uint64_t total_pkts = t->outstand_pkts; 366 367 uint64_t dead_lock_cycles = rte_get_timer_cycles(); 368 int64_t dead_lock_remaining = total_pkts; 369 const uint64_t dead_lock_sample = rte_get_timer_hz() * 5; 370 371 uint64_t perf_cycles = rte_get_timer_cycles(); 372 int64_t perf_remaining = total_pkts; 373 const uint64_t perf_sample = rte_get_timer_hz(); 374 375 static float total_mpps; 376 static uint64_t samples; 377 378 const uint64_t freq_mhz = rte_get_timer_hz() / 1000000; 379 int64_t remaining = t->outstand_pkts - processed_pkts(t); 380 381 while (t->done == false) { 382 const uint64_t new_cycles = rte_get_timer_cycles(); 383 384 if ((new_cycles - perf_cycles) > perf_sample) { 385 const uint64_t latency = total_latency(t); 386 const uint64_t pkts = processed_pkts(t); 387 388 remaining = t->outstand_pkts - pkts; 389 float mpps = (float)(perf_remaining-remaining)/1000000; 390 391 perf_remaining = remaining; 392 perf_cycles = new_cycles; 393 total_mpps += mpps; 394 ++samples; 395 if (opt->fwd_latency && pkts > 0) { 396 printf(CLGRN"\r%.3f mpps avg %.3f mpps [avg fwd latency %.3f us] "CLNRM, 397 mpps, total_mpps/samples, 398 (float)(latency/pkts)/freq_mhz); 399 } else { 400 printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM, 401 mpps, total_mpps/samples); 402 } 403 fflush(stdout); 404 405 if (remaining <= 0) { 406 t->result = EVT_TEST_SUCCESS; 407 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 408 opt->prod_type == 409 EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 410 t->done = true; 411 break; 412 } 413 } 414 } 415 416 if (new_cycles - dead_lock_cycles > dead_lock_sample && 417 (opt->prod_type == EVT_PROD_TYPE_SYNT || 418 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR)) { 419 remaining = t->outstand_pkts - processed_pkts(t); 420 if (dead_lock_remaining == remaining) { 421 rte_event_dev_dump(opt->dev_id, stdout); 422 evt_err("No schedules for seconds, deadlock"); 423 t->done = true; 424 break; 425 } 426 dead_lock_remaining = remaining; 427 dead_lock_cycles = new_cycles; 428 } 429 } 430 printf("\n"); 431 return 0; 432 } 433 434 static int 435 perf_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, 436 struct rte_event_port_conf prod_conf) 437 { 438 int ret = 0; 439 uint16_t prod; 440 struct rte_event_eth_rx_adapter_queue_conf queue_conf; 441 442 memset(&queue_conf, 0, 443 sizeof(struct rte_event_eth_rx_adapter_queue_conf)); 444 queue_conf.ev.sched_type = opt->sched_type_list[0]; 445 RTE_ETH_FOREACH_DEV(prod) { 446 uint32_t cap; 447 448 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id, 449 prod, &cap); 450 if (ret) { 451 evt_err("failed to get event rx adapter[%d]" 452 " capabilities", 453 opt->dev_id); 454 return ret; 455 } 456 queue_conf.ev.queue_id = prod * stride; 457 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id, 458 &prod_conf); 459 if (ret) { 460 evt_err("failed to create rx adapter[%d]", prod); 461 return ret; 462 } 463 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1, 464 &queue_conf); 465 if (ret) { 466 evt_err("failed to add rx queues to adapter[%d]", prod); 467 return ret; 468 } 469 470 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) { 471 uint32_t service_id; 472 473 rte_event_eth_rx_adapter_service_id_get(prod, 474 &service_id); 475 ret = evt_service_setup(service_id); 476 if (ret) { 477 evt_err("Failed to setup service core" 478 " for Rx adapter\n"); 479 return ret; 480 } 481 } 482 } 483 484 return ret; 485 } 486 487 static int 488 perf_event_timer_adapter_setup(struct test_perf *t) 489 { 490 int i; 491 int ret; 492 struct rte_event_timer_adapter_info adapter_info; 493 struct rte_event_timer_adapter *wl; 494 uint8_t nb_producers = evt_nr_active_lcores(t->opt->plcores); 495 uint8_t flags = RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES; 496 497 if (nb_producers == 1) 498 flags |= RTE_EVENT_TIMER_ADAPTER_F_SP_PUT; 499 500 for (i = 0; i < t->opt->nb_timer_adptrs; i++) { 501 struct rte_event_timer_adapter_conf config = { 502 .event_dev_id = t->opt->dev_id, 503 .timer_adapter_id = i, 504 .timer_tick_ns = t->opt->timer_tick_nsec, 505 .max_tmo_ns = t->opt->max_tmo_nsec, 506 .nb_timers = t->opt->pool_sz, 507 .flags = flags, 508 }; 509 510 wl = rte_event_timer_adapter_create(&config); 511 if (wl == NULL) { 512 evt_err("failed to create event timer ring %d", i); 513 return rte_errno; 514 } 515 516 memset(&adapter_info, 0, 517 sizeof(struct rte_event_timer_adapter_info)); 518 rte_event_timer_adapter_get_info(wl, &adapter_info); 519 t->opt->optm_timer_tick_nsec = adapter_info.min_resolution_ns; 520 521 if (!(adapter_info.caps & 522 RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) { 523 uint32_t service_id = -1U; 524 525 rte_event_timer_adapter_service_id_get(wl, 526 &service_id); 527 ret = evt_service_setup(service_id); 528 if (ret) { 529 evt_err("Failed to setup service core" 530 " for timer adapter\n"); 531 return ret; 532 } 533 rte_service_runstate_set(service_id, 1); 534 } 535 t->timer_adptr[i] = wl; 536 } 537 return 0; 538 } 539 540 int 541 perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, 542 uint8_t stride, uint8_t nb_queues, 543 const struct rte_event_port_conf *port_conf) 544 { 545 struct test_perf *t = evt_test_priv(test); 546 uint16_t port, prod; 547 int ret = -1; 548 549 /* setup one port per worker, linking to all queues */ 550 for (port = 0; port < evt_nr_active_lcores(opt->wlcores); 551 port++) { 552 struct worker_data *w = &t->worker[port]; 553 554 w->dev_id = opt->dev_id; 555 w->port_id = port; 556 w->t = t; 557 w->processed_pkts = 0; 558 w->latency = 0; 559 560 struct rte_event_port_conf conf = *port_conf; 561 conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_WORKER; 562 563 ret = rte_event_port_setup(opt->dev_id, port, &conf); 564 if (ret) { 565 evt_err("failed to setup port %d", port); 566 return ret; 567 } 568 569 ret = rte_event_port_link(opt->dev_id, port, NULL, NULL, 0); 570 if (ret != nb_queues) { 571 evt_err("failed to link all queues to port %d", port); 572 return -EINVAL; 573 } 574 } 575 576 /* port for producers, no links */ 577 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 578 for ( ; port < perf_nb_event_ports(opt); port++) { 579 struct prod_data *p = &t->prod[port]; 580 p->t = t; 581 } 582 583 struct rte_event_port_conf conf = *port_conf; 584 conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_PRODUCER; 585 586 ret = perf_event_rx_adapter_setup(opt, stride, conf); 587 if (ret) 588 return ret; 589 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 590 prod = 0; 591 for ( ; port < perf_nb_event_ports(opt); port++) { 592 struct prod_data *p = &t->prod[port]; 593 p->queue_id = prod * stride; 594 p->t = t; 595 prod++; 596 } 597 598 ret = perf_event_timer_adapter_setup(t); 599 if (ret) 600 return ret; 601 } else { 602 prod = 0; 603 for ( ; port < perf_nb_event_ports(opt); port++) { 604 struct prod_data *p = &t->prod[port]; 605 606 p->dev_id = opt->dev_id; 607 p->port_id = port; 608 p->queue_id = prod * stride; 609 p->t = t; 610 611 struct rte_event_port_conf conf = *port_conf; 612 conf.event_port_cfg |= 613 RTE_EVENT_PORT_CFG_HINT_PRODUCER | 614 RTE_EVENT_PORT_CFG_HINT_CONSUMER; 615 616 ret = rte_event_port_setup(opt->dev_id, port, &conf); 617 if (ret) { 618 evt_err("failed to setup port %d", port); 619 return ret; 620 } 621 prod++; 622 } 623 } 624 625 return ret; 626 } 627 628 int 629 perf_opt_check(struct evt_options *opt, uint64_t nb_queues) 630 { 631 unsigned int lcores; 632 633 /* N producer + N worker + main when producer cores are used 634 * Else N worker + main when Rx adapter is used 635 */ 636 lcores = opt->prod_type == EVT_PROD_TYPE_SYNT ? 3 : 2; 637 638 if (rte_lcore_count() < lcores) { 639 evt_err("test need minimum %d lcores", lcores); 640 return -1; 641 } 642 643 /* Validate worker lcores */ 644 if (evt_lcores_has_overlap(opt->wlcores, rte_get_main_lcore())) { 645 evt_err("worker lcores overlaps with main lcore"); 646 return -1; 647 } 648 if (evt_lcores_has_overlap_multi(opt->wlcores, opt->plcores)) { 649 evt_err("worker lcores overlaps producer lcores"); 650 return -1; 651 } 652 if (evt_has_disabled_lcore(opt->wlcores)) { 653 evt_err("one or more workers lcores are not enabled"); 654 return -1; 655 } 656 if (!evt_has_active_lcore(opt->wlcores)) { 657 evt_err("minimum one worker is required"); 658 return -1; 659 } 660 661 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 662 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 663 /* Validate producer lcores */ 664 if (evt_lcores_has_overlap(opt->plcores, 665 rte_get_main_lcore())) { 666 evt_err("producer lcores overlaps with main lcore"); 667 return -1; 668 } 669 if (evt_has_disabled_lcore(opt->plcores)) { 670 evt_err("one or more producer lcores are not enabled"); 671 return -1; 672 } 673 if (!evt_has_active_lcore(opt->plcores)) { 674 evt_err("minimum one producer is required"); 675 return -1; 676 } 677 } 678 679 if (evt_has_invalid_stage(opt)) 680 return -1; 681 682 if (evt_has_invalid_sched_type(opt)) 683 return -1; 684 685 if (nb_queues > EVT_MAX_QUEUES) { 686 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES); 687 return -1; 688 } 689 if (perf_nb_event_ports(opt) > EVT_MAX_PORTS) { 690 evt_err("number of ports exceeds %d", EVT_MAX_PORTS); 691 return -1; 692 } 693 694 /* Fixups */ 695 if ((opt->nb_stages == 1 && 696 opt->prod_type != EVT_PROD_TYPE_EVENT_TIMER_ADPTR) && 697 opt->fwd_latency) { 698 evt_info("fwd_latency is valid when nb_stages > 1, disabling"); 699 opt->fwd_latency = 0; 700 } 701 702 if (opt->fwd_latency && !opt->q_priority) { 703 evt_info("enabled queue priority for latency measurement"); 704 opt->q_priority = 1; 705 } 706 if (opt->nb_pkts == 0) 707 opt->nb_pkts = INT64_MAX/evt_nr_active_lcores(opt->plcores); 708 709 return 0; 710 } 711 712 void 713 perf_opt_dump(struct evt_options *opt, uint8_t nb_queues) 714 { 715 evt_dump("nb_prod_lcores", "%d", evt_nr_active_lcores(opt->plcores)); 716 evt_dump_producer_lcores(opt); 717 evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores)); 718 evt_dump_worker_lcores(opt); 719 evt_dump_nb_stages(opt); 720 evt_dump("nb_evdev_ports", "%d", perf_nb_event_ports(opt)); 721 evt_dump("nb_evdev_queues", "%d", nb_queues); 722 evt_dump_queue_priority(opt); 723 evt_dump_sched_type_list(opt); 724 evt_dump_producer_type(opt); 725 evt_dump("prod_enq_burst_sz", "%d", opt->prod_enq_burst_sz); 726 } 727 728 void 729 perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt) 730 { 731 int i; 732 struct test_perf *t = evt_test_priv(test); 733 734 if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 735 for (i = 0; i < opt->nb_timer_adptrs; i++) 736 rte_event_timer_adapter_stop(t->timer_adptr[i]); 737 } 738 rte_event_dev_stop(opt->dev_id); 739 rte_event_dev_close(opt->dev_id); 740 } 741 742 static inline void 743 perf_elt_init(struct rte_mempool *mp, void *arg __rte_unused, 744 void *obj, unsigned i __rte_unused) 745 { 746 memset(obj, 0, mp->elt_size); 747 } 748 749 #define NB_RX_DESC 128 750 #define NB_TX_DESC 512 751 int 752 perf_ethdev_setup(struct evt_test *test, struct evt_options *opt) 753 { 754 uint16_t i; 755 int ret; 756 struct test_perf *t = evt_test_priv(test); 757 struct rte_eth_conf port_conf = { 758 .rxmode = { 759 .mq_mode = RTE_ETH_MQ_RX_RSS, 760 .split_hdr_size = 0, 761 }, 762 .rx_adv_conf = { 763 .rss_conf = { 764 .rss_key = NULL, 765 .rss_hf = RTE_ETH_RSS_IP, 766 }, 767 }, 768 }; 769 770 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 771 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) 772 return 0; 773 774 if (!rte_eth_dev_count_avail()) { 775 evt_err("No ethernet ports found."); 776 return -ENODEV; 777 } 778 779 RTE_ETH_FOREACH_DEV(i) { 780 struct rte_eth_dev_info dev_info; 781 struct rte_eth_conf local_port_conf = port_conf; 782 783 ret = rte_eth_dev_info_get(i, &dev_info); 784 if (ret != 0) { 785 evt_err("Error during getting device (port %u) info: %s\n", 786 i, strerror(-ret)); 787 return ret; 788 } 789 790 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 791 dev_info.flow_type_rss_offloads; 792 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 793 port_conf.rx_adv_conf.rss_conf.rss_hf) { 794 evt_info("Port %u modified RSS hash function based on hardware support," 795 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 796 i, 797 port_conf.rx_adv_conf.rss_conf.rss_hf, 798 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 799 } 800 801 if (rte_eth_dev_configure(i, 1, 1, &local_port_conf) < 0) { 802 evt_err("Failed to configure eth port [%d]", i); 803 return -EINVAL; 804 } 805 806 if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC, 807 rte_socket_id(), NULL, t->pool) < 0) { 808 evt_err("Failed to setup eth port [%d] rx_queue: %d.", 809 i, 0); 810 return -EINVAL; 811 } 812 813 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC, 814 rte_socket_id(), NULL) < 0) { 815 evt_err("Failed to setup eth port [%d] tx_queue: %d.", 816 i, 0); 817 return -EINVAL; 818 } 819 820 ret = rte_eth_promiscuous_enable(i); 821 if (ret != 0) { 822 evt_err("Failed to enable promiscuous mode for eth port [%d]: %s", 823 i, rte_strerror(-ret)); 824 return ret; 825 } 826 } 827 828 return 0; 829 } 830 831 void perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt) 832 { 833 uint16_t i; 834 RTE_SET_USED(test); 835 836 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 837 RTE_ETH_FOREACH_DEV(i) { 838 rte_event_eth_rx_adapter_stop(i); 839 rte_eth_dev_stop(i); 840 } 841 } 842 } 843 844 int 845 perf_mempool_setup(struct evt_test *test, struct evt_options *opt) 846 { 847 struct test_perf *t = evt_test_priv(test); 848 849 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 850 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 851 t->pool = rte_mempool_create(test->name, /* mempool name */ 852 opt->pool_sz, /* number of elements*/ 853 sizeof(struct perf_elt), /* element size*/ 854 512, /* cache size*/ 855 0, NULL, NULL, 856 perf_elt_init, /* obj constructor */ 857 NULL, opt->socket_id, 0); /* flags */ 858 } else { 859 t->pool = rte_pktmbuf_pool_create(test->name, /* mempool name */ 860 opt->pool_sz, /* number of elements*/ 861 512, /* cache size*/ 862 0, 863 RTE_MBUF_DEFAULT_BUF_SIZE, 864 opt->socket_id); /* flags */ 865 866 } 867 868 if (t->pool == NULL) { 869 evt_err("failed to create mempool"); 870 return -ENOMEM; 871 } 872 873 return 0; 874 } 875 876 void 877 perf_mempool_destroy(struct evt_test *test, struct evt_options *opt) 878 { 879 RTE_SET_USED(opt); 880 struct test_perf *t = evt_test_priv(test); 881 882 rte_mempool_free(t->pool); 883 } 884 885 int 886 perf_test_setup(struct evt_test *test, struct evt_options *opt) 887 { 888 void *test_perf; 889 890 test_perf = rte_zmalloc_socket(test->name, sizeof(struct test_perf), 891 RTE_CACHE_LINE_SIZE, opt->socket_id); 892 if (test_perf == NULL) { 893 evt_err("failed to allocate test_perf memory"); 894 goto nomem; 895 } 896 test->test_priv = test_perf; 897 898 struct test_perf *t = evt_test_priv(test); 899 900 if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 901 t->outstand_pkts = opt->nb_timers * 902 evt_nr_active_lcores(opt->plcores); 903 t->nb_pkts = opt->nb_timers; 904 } else { 905 t->outstand_pkts = opt->nb_pkts * 906 evt_nr_active_lcores(opt->plcores); 907 t->nb_pkts = opt->nb_pkts; 908 } 909 910 t->nb_workers = evt_nr_active_lcores(opt->wlcores); 911 t->done = false; 912 t->nb_flows = opt->nb_flows; 913 t->result = EVT_TEST_FAILED; 914 t->opt = opt; 915 memcpy(t->sched_type_list, opt->sched_type_list, 916 sizeof(opt->sched_type_list)); 917 return 0; 918 nomem: 919 return -ENOMEM; 920 } 921 922 void 923 perf_test_destroy(struct evt_test *test, struct evt_options *opt) 924 { 925 RTE_SET_USED(opt); 926 927 rte_free(test->test_priv); 928 } 929