1 /* 2 * SPDX-License-Identifier: BSD-3-Clause 3 * Copyright 2017 Cavium, Inc. 4 */ 5 6 #include "test_pipeline_common.h" 7 8 static int32_t 9 pipeline_event_tx_burst_service_func(void *args) 10 { 11 12 int i; 13 struct tx_service_data *tx = args; 14 const uint8_t dev = tx->dev_id; 15 const uint8_t port = tx->port_id; 16 struct rte_event ev[BURST_SIZE + 1]; 17 18 uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0); 19 20 if (!nb_rx) { 21 for (i = 0; i < tx->nb_ethports; i++) 22 rte_eth_tx_buffer_flush(i, 0, tx->tx_buf[i]); 23 return 0; 24 } 25 26 for (i = 0; i < nb_rx; i++) { 27 struct rte_mbuf *m = ev[i].mbuf; 28 rte_eth_tx_buffer(m->port, 0, tx->tx_buf[m->port], m); 29 } 30 tx->processed_pkts += nb_rx; 31 32 return 0; 33 } 34 35 static int32_t 36 pipeline_event_tx_service_func(void *args) 37 { 38 39 int i; 40 struct tx_service_data *tx = args; 41 const uint8_t dev = tx->dev_id; 42 const uint8_t port = tx->port_id; 43 struct rte_event ev; 44 45 uint16_t nb_rx = rte_event_dequeue_burst(dev, port, &ev, 1, 0); 46 47 if (!nb_rx) { 48 for (i = 0; i < tx->nb_ethports; i++) 49 rte_eth_tx_buffer_flush(i, 0, tx->tx_buf[i]); 50 return 0; 51 } 52 53 struct rte_mbuf *m = ev.mbuf; 54 rte_eth_tx_buffer(m->port, 0, tx->tx_buf[m->port], m); 55 tx->processed_pkts++; 56 57 return 0; 58 } 59 60 int 61 pipeline_test_result(struct evt_test *test, struct evt_options *opt) 62 { 63 RTE_SET_USED(opt); 64 int i; 65 uint64_t total = 0; 66 struct test_pipeline *t = evt_test_priv(test); 67 68 printf("Packet distribution across worker cores :\n"); 69 for (i = 0; i < t->nb_workers; i++) 70 total += t->worker[i].processed_pkts; 71 for (i = 0; i < t->nb_workers; i++) 72 printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:" 73 CLGRN" %3.2f\n"CLNRM, i, 74 t->worker[i].processed_pkts, 75 (((double)t->worker[i].processed_pkts)/total) 76 * 100); 77 return t->result; 78 } 79 80 void 81 pipeline_opt_dump(struct evt_options *opt, uint8_t nb_queues) 82 { 83 evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores)); 84 evt_dump_worker_lcores(opt); 85 evt_dump_nb_stages(opt); 86 evt_dump("nb_evdev_ports", "%d", pipeline_nb_event_ports(opt)); 87 evt_dump("nb_evdev_queues", "%d", nb_queues); 88 evt_dump_queue_priority(opt); 89 evt_dump_sched_type_list(opt); 90 evt_dump_producer_type(opt); 91 } 92 93 static inline uint64_t 94 processed_pkts(struct test_pipeline *t) 95 { 96 uint8_t i; 97 uint64_t total = 0; 98 99 rte_smp_rmb(); 100 if (t->mt_unsafe) 101 total = t->tx_service.processed_pkts; 102 else 103 for (i = 0; i < t->nb_workers; i++) 104 total += t->worker[i].processed_pkts; 105 106 return total; 107 } 108 109 int 110 pipeline_launch_lcores(struct evt_test *test, struct evt_options *opt, 111 int (*worker)(void *)) 112 { 113 int ret, lcore_id; 114 struct test_pipeline *t = evt_test_priv(test); 115 116 int port_idx = 0; 117 /* launch workers */ 118 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 119 if (!(opt->wlcores[lcore_id])) 120 continue; 121 122 ret = rte_eal_remote_launch(worker, 123 &t->worker[port_idx], lcore_id); 124 if (ret) { 125 evt_err("failed to launch worker %d", lcore_id); 126 return ret; 127 } 128 port_idx++; 129 } 130 131 uint64_t perf_cycles = rte_get_timer_cycles(); 132 const uint64_t perf_sample = rte_get_timer_hz(); 133 134 static float total_mpps; 135 static uint64_t samples; 136 137 uint64_t prev_pkts = 0; 138 139 while (t->done == false) { 140 const uint64_t new_cycles = rte_get_timer_cycles(); 141 142 if ((new_cycles - perf_cycles) > perf_sample) { 143 const uint64_t curr_pkts = processed_pkts(t); 144 145 float mpps = (float)(curr_pkts - prev_pkts)/1000000; 146 147 prev_pkts = curr_pkts; 148 perf_cycles = new_cycles; 149 total_mpps += mpps; 150 ++samples; 151 printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM, 152 mpps, total_mpps/samples); 153 fflush(stdout); 154 } 155 } 156 printf("\n"); 157 return 0; 158 } 159 160 int 161 pipeline_opt_check(struct evt_options *opt, uint64_t nb_queues) 162 { 163 unsigned int lcores; 164 /* 165 * N worker + 1 master 166 */ 167 lcores = 2; 168 169 if (!rte_eth_dev_count_avail()) { 170 evt_err("test needs minimum 1 ethernet dev"); 171 return -1; 172 } 173 174 if (rte_lcore_count() < lcores) { 175 evt_err("test need minimum %d lcores", lcores); 176 return -1; 177 } 178 179 /* Validate worker lcores */ 180 if (evt_lcores_has_overlap(opt->wlcores, rte_get_master_lcore())) { 181 evt_err("worker lcores overlaps with master lcore"); 182 return -1; 183 } 184 if (evt_has_disabled_lcore(opt->wlcores)) { 185 evt_err("one or more workers lcores are not enabled"); 186 return -1; 187 } 188 if (!evt_has_active_lcore(opt->wlcores)) { 189 evt_err("minimum one worker is required"); 190 return -1; 191 } 192 193 if (nb_queues > EVT_MAX_QUEUES) { 194 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES); 195 return -1; 196 } 197 if (pipeline_nb_event_ports(opt) > EVT_MAX_PORTS) { 198 evt_err("number of ports exceeds %d", EVT_MAX_PORTS); 199 return -1; 200 } 201 202 if (evt_has_invalid_stage(opt)) 203 return -1; 204 205 if (evt_has_invalid_sched_type(opt)) 206 return -1; 207 208 return 0; 209 } 210 211 #define NB_RX_DESC 128 212 #define NB_TX_DESC 512 213 int 214 pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt) 215 { 216 uint16_t i; 217 uint8_t nb_queues = 1; 218 uint8_t mt_state = 0; 219 struct test_pipeline *t = evt_test_priv(test); 220 struct rte_eth_rxconf rx_conf; 221 struct rte_eth_conf port_conf = { 222 .rxmode = { 223 .mq_mode = ETH_MQ_RX_RSS, 224 .max_rx_pkt_len = ETHER_MAX_LEN, 225 .offloads = DEV_RX_OFFLOAD_CRC_STRIP, 226 }, 227 .rx_adv_conf = { 228 .rss_conf = { 229 .rss_key = NULL, 230 .rss_hf = ETH_RSS_IP, 231 }, 232 }, 233 }; 234 235 RTE_SET_USED(opt); 236 if (!rte_eth_dev_count_avail()) { 237 evt_err("No ethernet ports found.\n"); 238 return -ENODEV; 239 } 240 241 RTE_ETH_FOREACH_DEV(i) { 242 struct rte_eth_dev_info dev_info; 243 struct rte_eth_conf local_port_conf = port_conf; 244 245 rte_eth_dev_info_get(i, &dev_info); 246 mt_state = !(dev_info.tx_offload_capa & 247 DEV_TX_OFFLOAD_MT_LOCKFREE); 248 rx_conf = dev_info.default_rxconf; 249 rx_conf.offloads = port_conf.rxmode.offloads; 250 251 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 252 dev_info.flow_type_rss_offloads; 253 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 254 port_conf.rx_adv_conf.rss_conf.rss_hf) { 255 evt_info("Port %u modified RSS hash function based on hardware support," 256 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 257 i, 258 port_conf.rx_adv_conf.rss_conf.rss_hf, 259 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 260 } 261 262 if (rte_eth_dev_configure(i, nb_queues, nb_queues, 263 &local_port_conf) 264 < 0) { 265 evt_err("Failed to configure eth port [%d]\n", i); 266 return -EINVAL; 267 } 268 269 if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC, 270 rte_socket_id(), &rx_conf, t->pool) < 0) { 271 evt_err("Failed to setup eth port [%d] rx_queue: %d.\n", 272 i, 0); 273 return -EINVAL; 274 } 275 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC, 276 rte_socket_id(), NULL) < 0) { 277 evt_err("Failed to setup eth port [%d] tx_queue: %d.\n", 278 i, 0); 279 return -EINVAL; 280 } 281 282 t->mt_unsafe |= mt_state; 283 t->tx_service.tx_buf[i] = 284 rte_malloc(NULL, RTE_ETH_TX_BUFFER_SIZE(BURST_SIZE), 0); 285 if (t->tx_service.tx_buf[i] == NULL) 286 rte_panic("Unable to allocate Tx buffer memory."); 287 rte_eth_promiscuous_enable(i); 288 } 289 290 return 0; 291 } 292 293 int 294 pipeline_event_port_setup(struct evt_test *test, struct evt_options *opt, 295 uint8_t *queue_arr, uint8_t nb_queues, 296 const struct rte_event_port_conf p_conf) 297 { 298 int i; 299 int ret; 300 uint8_t port; 301 struct test_pipeline *t = evt_test_priv(test); 302 303 304 /* setup one port per worker, linking to all queues */ 305 for (port = 0; port < evt_nr_active_lcores(opt->wlcores); port++) { 306 struct worker_data *w = &t->worker[port]; 307 308 w->dev_id = opt->dev_id; 309 w->port_id = port; 310 w->t = t; 311 w->processed_pkts = 0; 312 313 ret = rte_event_port_setup(opt->dev_id, port, &p_conf); 314 if (ret) { 315 evt_err("failed to setup port %d", port); 316 return ret; 317 } 318 319 if (queue_arr == NULL) { 320 if (rte_event_port_link(opt->dev_id, port, NULL, NULL, 321 0) != nb_queues) 322 goto link_fail; 323 } else { 324 for (i = 0; i < nb_queues; i++) { 325 if (rte_event_port_link(opt->dev_id, port, 326 &queue_arr[i], NULL, 1) != 1) 327 goto link_fail; 328 } 329 } 330 } 331 332 return 0; 333 334 link_fail: 335 evt_err("failed to link all queues to port %d", port); 336 return -EINVAL; 337 } 338 339 int 340 pipeline_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, 341 struct rte_event_port_conf prod_conf) 342 { 343 int ret = 0; 344 uint16_t prod; 345 struct rte_event_eth_rx_adapter_queue_conf queue_conf; 346 347 memset(&queue_conf, 0, 348 sizeof(struct rte_event_eth_rx_adapter_queue_conf)); 349 queue_conf.ev.sched_type = opt->sched_type_list[0]; 350 RTE_ETH_FOREACH_DEV(prod) { 351 uint32_t cap; 352 353 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id, 354 prod, &cap); 355 if (ret) { 356 evt_err("failed to get event rx adapter[%d]" 357 " capabilities", 358 opt->dev_id); 359 return ret; 360 } 361 queue_conf.ev.queue_id = prod * stride; 362 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id, 363 &prod_conf); 364 if (ret) { 365 evt_err("failed to create rx adapter[%d]", prod); 366 return ret; 367 } 368 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1, 369 &queue_conf); 370 if (ret) { 371 evt_err("failed to add rx queues to adapter[%d]", prod); 372 return ret; 373 } 374 375 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) { 376 uint32_t service_id; 377 378 rte_event_eth_rx_adapter_service_id_get(prod, 379 &service_id); 380 ret = evt_service_setup(service_id); 381 if (ret) { 382 evt_err("Failed to setup service core" 383 " for Rx adapter\n"); 384 return ret; 385 } 386 } 387 388 ret = rte_eth_dev_start(prod); 389 if (ret) { 390 evt_err("Ethernet dev [%d] failed to start." 391 " Using synthetic producer", prod); 392 return ret; 393 } 394 395 ret = rte_event_eth_rx_adapter_start(prod); 396 if (ret) { 397 evt_err("Rx adapter[%d] start failed", prod); 398 return ret; 399 } 400 printf("%s: Port[%d] using Rx adapter[%d] started\n", __func__, 401 prod, prod); 402 } 403 404 return ret; 405 } 406 407 int 408 pipeline_event_tx_service_setup(struct evt_test *test, struct evt_options *opt, 409 uint8_t tx_queue_id, uint8_t tx_port_id, 410 const struct rte_event_port_conf p_conf) 411 { 412 int ret; 413 struct rte_service_spec serv; 414 struct test_pipeline *t = evt_test_priv(test); 415 struct tx_service_data *tx = &t->tx_service; 416 417 ret = rte_event_port_setup(opt->dev_id, tx_port_id, &p_conf); 418 if (ret) { 419 evt_err("failed to setup port %d", tx_port_id); 420 return ret; 421 } 422 423 if (rte_event_port_link(opt->dev_id, tx_port_id, &tx_queue_id, 424 NULL, 1) != 1) { 425 evt_err("failed to link queues to port %d", tx_port_id); 426 return -EINVAL; 427 } 428 429 tx->dev_id = opt->dev_id; 430 tx->queue_id = tx_queue_id; 431 tx->port_id = tx_port_id; 432 tx->nb_ethports = rte_eth_dev_count_avail(); 433 tx->t = t; 434 435 /* Register Tx service */ 436 memset(&serv, 0, sizeof(struct rte_service_spec)); 437 snprintf(serv.name, sizeof(serv.name), "Tx_service"); 438 439 if (evt_has_burst_mode(opt->dev_id)) 440 serv.callback = pipeline_event_tx_burst_service_func; 441 else 442 serv.callback = pipeline_event_tx_service_func; 443 444 serv.callback_userdata = (void *)tx; 445 ret = rte_service_component_register(&serv, &tx->service_id); 446 if (ret) { 447 evt_err("failed to register Tx service"); 448 return ret; 449 } 450 451 ret = evt_service_setup(tx->service_id); 452 if (ret) { 453 evt_err("Failed to setup service core for Tx service\n"); 454 return ret; 455 } 456 457 rte_service_runstate_set(tx->service_id, 1); 458 459 return 0; 460 } 461 462 463 void 464 pipeline_ethdev_destroy(struct evt_test *test, struct evt_options *opt) 465 { 466 uint16_t i; 467 RTE_SET_USED(test); 468 RTE_SET_USED(opt); 469 struct test_pipeline *t = evt_test_priv(test); 470 471 if (t->mt_unsafe) { 472 rte_service_component_runstate_set(t->tx_service.service_id, 0); 473 rte_service_runstate_set(t->tx_service.service_id, 0); 474 rte_service_component_unregister(t->tx_service.service_id); 475 } 476 477 RTE_ETH_FOREACH_DEV(i) { 478 rte_event_eth_rx_adapter_stop(i); 479 rte_eth_dev_stop(i); 480 rte_eth_dev_close(i); 481 } 482 } 483 484 void 485 pipeline_eventdev_destroy(struct evt_test *test, struct evt_options *opt) 486 { 487 RTE_SET_USED(test); 488 489 rte_event_dev_stop(opt->dev_id); 490 rte_event_dev_close(opt->dev_id); 491 } 492 493 int 494 pipeline_mempool_setup(struct evt_test *test, struct evt_options *opt) 495 { 496 struct test_pipeline *t = evt_test_priv(test); 497 498 t->pool = rte_pktmbuf_pool_create(test->name, /* mempool name */ 499 opt->pool_sz, /* number of elements*/ 500 512, /* cache size*/ 501 0, 502 RTE_MBUF_DEFAULT_BUF_SIZE, 503 opt->socket_id); /* flags */ 504 505 if (t->pool == NULL) { 506 evt_err("failed to create mempool"); 507 return -ENOMEM; 508 } 509 510 return 0; 511 } 512 513 void 514 pipeline_mempool_destroy(struct evt_test *test, struct evt_options *opt) 515 { 516 RTE_SET_USED(opt); 517 struct test_pipeline *t = evt_test_priv(test); 518 519 rte_mempool_free(t->pool); 520 } 521 522 int 523 pipeline_test_setup(struct evt_test *test, struct evt_options *opt) 524 { 525 void *test_pipeline; 526 527 test_pipeline = rte_zmalloc_socket(test->name, 528 sizeof(struct test_pipeline), RTE_CACHE_LINE_SIZE, 529 opt->socket_id); 530 if (test_pipeline == NULL) { 531 evt_err("failed to allocate test_pipeline memory"); 532 goto nomem; 533 } 534 test->test_priv = test_pipeline; 535 536 struct test_pipeline *t = evt_test_priv(test); 537 538 t->nb_workers = evt_nr_active_lcores(opt->wlcores); 539 t->outstand_pkts = opt->nb_pkts * evt_nr_active_lcores(opt->wlcores); 540 t->done = false; 541 t->nb_flows = opt->nb_flows; 542 t->result = EVT_TEST_FAILED; 543 t->opt = opt; 544 opt->prod_type = EVT_PROD_TYPE_ETH_RX_ADPTR; 545 memcpy(t->sched_type_list, opt->sched_type_list, 546 sizeof(opt->sched_type_list)); 547 return 0; 548 nomem: 549 return -ENOMEM; 550 } 551 552 void 553 pipeline_test_destroy(struct evt_test *test, struct evt_options *opt) 554 { 555 RTE_SET_USED(opt); 556 557 rte_free(test->test_priv); 558 } 559