1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
3 */
4
5 #include <stdalign.h>
6 #include <stdio.h>
7 #include <stdlib.h>
8 #include <string.h>
9 #include <stdint.h>
10 #include <inttypes.h>
11 #include <sys/types.h>
12 #include <unistd.h>
13 #include <sys/queue.h>
14 #include <stdarg.h>
15 #include <ctype.h>
16 #include <errno.h>
17 #include <math.h>
18 #include <assert.h>
19 #include <getopt.h>
20 #include <signal.h>
21
22 #include <rte_common.h>
23 #include <rte_eal.h>
24 #include <rte_cycles.h>
25 #include <rte_ether.h>
26 #include <rte_ethdev.h>
27 #include <rte_ip.h>
28 #include <rte_lcore.h>
29 #include <rte_malloc.h>
30 #include <rte_mbuf.h>
31 #include <rte_mbuf_dyn.h>
32 #include <rte_memory.h>
33 #include <rte_mempool.h>
34 #include <rte_log.h>
35 #include <rte_bbdev.h>
36 #include <rte_bbdev_op.h>
37
38 /* LLR values - negative value for '1' bit */
39 #define LLR_1_BIT 0x81
40 #define LLR_0_BIT 0x7F
41
42 #define MAX_PKT_BURST 32
43 #define NB_MBUF 8191
44 #define MEMPOOL_CACHE_SIZE 256
45
46 /* Hardcoded K value */
47 #define K 40
48 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32))
49
50 #define CRC_24B_LEN 3
51
52 /* Configurable number of RX/TX ring descriptors */
53 #define RX_DESC_DEFAULT 128
54 #define TX_DESC_DEFAULT 512
55
56 #define BBDEV_ASSERT(a) do { \
57 if (!(a)) { \
58 usage(prgname); \
59 return -1; \
60 } \
61 } while (0)
62
63 static int input_dynfield_offset = -1;
64
65 static inline struct rte_mbuf **
mbuf_input(struct rte_mbuf * mbuf)66 mbuf_input(struct rte_mbuf *mbuf)
67 {
68 return RTE_MBUF_DYNFIELD(mbuf,
69 input_dynfield_offset, struct rte_mbuf **);
70 }
71
72 static const struct rte_eth_conf port_conf = {
73 .rxmode = {
74 .mq_mode = RTE_ETH_MQ_RX_NONE,
75 },
76 .txmode = {
77 .mq_mode = RTE_ETH_MQ_TX_NONE,
78 },
79 };
80
81 struct rte_bbdev_op_turbo_enc def_op_enc = {
82 /* These values are arbitrarily put, and does not map to the real
83 * values for the data received from ethdev ports
84 */
85 .rv_index = 0,
86 .code_block_mode = 1,
87 .cb_params = {
88 .k = K,
89 },
90 .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH
91 };
92
93 struct rte_bbdev_op_turbo_dec def_op_dec = {
94 /* These values are arbitrarily put, and does not map to the real
95 * values for the data received from ethdev ports
96 */
97 .code_block_mode = 1,
98 .cb_params = {
99 .k = K,
100 },
101 .rv_index = 0,
102 .iter_max = 8,
103 .iter_min = 4,
104 .ext_scale = 15,
105 .num_maps = 0,
106 .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN
107 };
108
109 struct app_config_params {
110 /* Placeholders for app params */
111 uint16_t port_id;
112 uint16_t bbdev_id;
113 uint64_t enc_core_mask;
114 uint64_t dec_core_mask;
115
116 /* Values filled during init time */
117 uint16_t enc_queue_ids[RTE_MAX_LCORE];
118 uint16_t dec_queue_ids[RTE_MAX_LCORE];
119 uint16_t num_enc_cores;
120 uint16_t num_dec_cores;
121 };
122
123 struct __rte_cache_aligned lcore_statistics {
124 unsigned int enqueued;
125 unsigned int dequeued;
126 unsigned int rx_lost_packets;
127 unsigned int enc_to_dec_lost_packets;
128 unsigned int tx_lost_packets;
129 };
130
131 /** each lcore configuration */
132 struct __rte_cache_aligned lcore_conf {
133 uint64_t core_type;
134
135 unsigned int port_id;
136 unsigned int rx_queue_id;
137 unsigned int tx_queue_id;
138
139 unsigned int bbdev_id;
140 unsigned int enc_queue_id;
141 unsigned int dec_queue_id;
142
143 uint8_t llr_temp_buf[NCB];
144
145 struct rte_mempool *bbdev_dec_op_pool;
146 struct rte_mempool *bbdev_enc_op_pool;
147 struct rte_mempool *enc_out_pool;
148 struct rte_ring *enc_to_dec_ring;
149
150 struct lcore_statistics *lcore_stats;
151 };
152
153 struct stats_lcore_params {
154 struct lcore_conf *lconf;
155 struct app_config_params *app_params;
156 };
157
158
159 static const struct app_config_params def_app_config = {
160 .port_id = 0,
161 .bbdev_id = 0,
162 .enc_core_mask = 0x2,
163 .dec_core_mask = 0x4,
164 .num_enc_cores = 1,
165 .num_dec_cores = 1,
166 };
167
168 static RTE_ATOMIC(uint16_t) global_exit_flag;
169
170 /* display usage */
171 static inline void
usage(const char * prgname)172 usage(const char *prgname)
173 {
174 printf("%s [EAL options] "
175 " --\n"
176 " --enc_cores - number of encoding cores (default = 0x2)\n"
177 " --dec_cores - number of decoding cores (default = 0x4)\n"
178 " --port_id - Ethernet port ID (default = 0)\n"
179 " --bbdev_id - BBDev ID (default = 0)\n"
180 "\n", prgname);
181 }
182
183 /* parse core mask */
184 static inline
bbdev_parse_mask(const char * mask)185 uint16_t bbdev_parse_mask(const char *mask)
186 {
187 char *end = NULL;
188 unsigned long pm;
189
190 /* parse hexadecimal string */
191 pm = strtoul(mask, &end, 16);
192 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
193 return 0;
194
195 return pm;
196 }
197
198 /* parse core mask */
199 static inline
bbdev_parse_number(const char * mask)200 uint16_t bbdev_parse_number(const char *mask)
201 {
202 char *end = NULL;
203 unsigned long pm;
204
205 /* parse hexadecimal string */
206 pm = strtoul(mask, &end, 10);
207 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
208 return 0;
209
210 return pm;
211 }
212
213 static int
bbdev_parse_args(int argc,char ** argv,struct app_config_params * app_params)214 bbdev_parse_args(int argc, char **argv,
215 struct app_config_params *app_params)
216 {
217 int optind = 0;
218 int opt;
219 int opt_indx = 0;
220 char *prgname = argv[0];
221
222 static struct option lgopts[] = {
223 { "enc_core_mask", required_argument, 0, 'e' },
224 { "dec_core_mask", required_argument, 0, 'd' },
225 { "port_id", required_argument, 0, 'p' },
226 { "bbdev_id", required_argument, 0, 'b' },
227 { NULL, 0, 0, 0 }
228 };
229
230 BBDEV_ASSERT(argc != 0);
231 BBDEV_ASSERT(argv != NULL);
232 BBDEV_ASSERT(app_params != NULL);
233
234 while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) !=
235 EOF) {
236 switch (opt) {
237 case 'e':
238 app_params->enc_core_mask =
239 bbdev_parse_mask(optarg);
240 if (app_params->enc_core_mask == 0) {
241 usage(prgname);
242 return -1;
243 }
244 app_params->num_enc_cores =
245 rte_popcount32(app_params->enc_core_mask);
246 break;
247
248 case 'd':
249 app_params->dec_core_mask =
250 bbdev_parse_mask(optarg);
251 if (app_params->dec_core_mask == 0) {
252 usage(prgname);
253 return -1;
254 }
255 app_params->num_dec_cores =
256 rte_popcount32(app_params->dec_core_mask);
257 break;
258
259 case 'p':
260 app_params->port_id = bbdev_parse_number(optarg);
261 break;
262
263 case 'b':
264 app_params->bbdev_id = bbdev_parse_number(optarg);
265 break;
266
267 default:
268 usage(prgname);
269 return -1;
270 }
271 }
272 optind = 0;
273 return optind;
274 }
275
276 static void
signal_handler(int signum)277 signal_handler(int signum)
278 {
279 printf("\nSignal %d received\n", signum);
280 rte_atomic_store_explicit(&global_exit_flag, 1, rte_memory_order_relaxed);
281 }
282
283 static void
print_mac(unsigned int portid,struct rte_ether_addr * bbdev_ports_eth_address)284 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address)
285 {
286 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n",
287 (unsigned int) portid,
288 RTE_ETHER_ADDR_BYTES(bbdev_ports_eth_address));
289 }
290
291 static inline void
pktmbuf_free_bulk(struct rte_mbuf ** mbufs,unsigned int nb_to_free)292 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
293 {
294 unsigned int i;
295 for (i = 0; i < nb_to_free; ++i)
296 rte_pktmbuf_free(mbufs[i]);
297 }
298
299 static inline void
pktmbuf_input_free_bulk(struct rte_mbuf ** mbufs,unsigned int nb_to_free)300 pktmbuf_input_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
301 {
302 unsigned int i;
303 for (i = 0; i < nb_to_free; ++i) {
304 struct rte_mbuf *rx_pkt = *mbuf_input(mbufs[i]);
305 rte_pktmbuf_free(rx_pkt);
306 rte_pktmbuf_free(mbufs[i]);
307 }
308 }
309
310 /* Check the link status of all ports in up to 9s, and print them finally */
311 static int
check_port_link_status(uint16_t port_id)312 check_port_link_status(uint16_t port_id)
313 {
314 #define CHECK_INTERVAL 100 /* 100ms */
315 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
316 uint8_t count;
317 struct rte_eth_link link;
318 int link_get_err = -EINVAL;
319
320 printf("\nChecking link status.");
321 fflush(stdout);
322
323 for (count = 0; count <= MAX_CHECK_TIME &&
324 !rte_atomic_load_explicit(&global_exit_flag,
325 rte_memory_order_relaxed); count++) {
326 memset(&link, 0, sizeof(link));
327 link_get_err = rte_eth_link_get_nowait(port_id, &link);
328
329 if (link_get_err >= 0 && link.link_status) {
330 const char *dp = (link.link_duplex ==
331 RTE_ETH_LINK_FULL_DUPLEX) ?
332 "full-duplex" : "half-duplex";
333 printf("\nPort %u Link Up - speed %s - %s\n",
334 port_id,
335 rte_eth_link_speed_to_str(link.link_speed),
336 dp);
337 return 0;
338 }
339 printf(".");
340 fflush(stdout);
341 rte_delay_ms(CHECK_INTERVAL);
342 }
343
344 if (link_get_err >= 0)
345 printf("\nPort %d Link Down\n", port_id);
346 else
347 printf("\nGet link failed (port %d): %s\n", port_id,
348 rte_strerror(-link_get_err));
349
350 return 0;
351 }
352
353 static inline void
add_ether_hdr(struct rte_mbuf * pkt_src,struct rte_mbuf * pkt_dst)354 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst)
355 {
356 struct rte_ether_hdr *eth_from;
357 struct rte_ether_hdr *eth_to;
358
359 eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *);
360 eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *);
361
362 /* copy header */
363 rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr));
364 }
365
366 static inline void
add_awgn(struct rte_mbuf ** mbufs,uint16_t num_pkts)367 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts)
368 {
369 RTE_SET_USED(mbufs);
370 RTE_SET_USED(num_pkts);
371 }
372
373 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input
374 * so each bit of the encoder output must be translated into one byte of LLR. If
375 * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes
376 * must additionally be inserted at the end of each sub-block.
377 */
378 static inline void
transform_enc_out_dec_in(struct rte_mbuf ** mbufs,uint8_t * temp_buf,uint16_t num_pkts,uint16_t k)379 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf,
380 uint16_t num_pkts, uint16_t k)
381 {
382 uint16_t i, l, j;
383 uint16_t start_bit_idx;
384 uint16_t out_idx;
385 uint16_t d = k + 4;
386 uint16_t kpi = RTE_ALIGN_CEIL(d, 32);
387 uint16_t nd = kpi - d;
388 uint16_t ncb = 3 * kpi;
389
390 for (i = 0; i < num_pkts; ++i) {
391 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) -
392 sizeof(struct rte_ether_hdr);
393
394 /* Resize the packet if needed */
395 if (pkt_data_len < ncb) {
396 char *data = rte_pktmbuf_append(mbufs[i],
397 ncb - pkt_data_len);
398 if (data == NULL)
399 printf(
400 "Not enough space in decoder input packet");
401 }
402
403 /* Translate each bit into 1 LLR byte. */
404 start_bit_idx = 0;
405 out_idx = 0;
406 for (j = 0; j < 3; ++j) {
407 for (l = start_bit_idx; l < start_bit_idx + d; ++l) {
408 uint8_t *data = rte_pktmbuf_mtod_offset(
409 mbufs[i], uint8_t *,
410 sizeof(struct rte_ether_hdr) +
411 (l >> 3));
412 if (*data & (0x80 >> (l & 7)))
413 temp_buf[out_idx] = LLR_1_BIT;
414 else
415 temp_buf[out_idx] = LLR_0_BIT;
416 ++out_idx;
417 }
418 /* Padding bytes should be at the end of the sub-block.
419 */
420 memset(&temp_buf[out_idx], 0, nd);
421 out_idx += nd;
422 start_bit_idx += d;
423 }
424
425 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *,
426 sizeof(struct rte_ether_hdr)), temp_buf, ncb);
427 }
428 }
429
430 static inline void
verify_data(struct rte_mbuf ** mbufs,uint16_t num_pkts)431 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts)
432 {
433 uint16_t i;
434 for (i = 0; i < num_pkts; ++i) {
435 struct rte_mbuf *out = mbufs[i];
436 struct rte_mbuf *in = *mbuf_input(out);
437
438 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *,
439 sizeof(struct rte_ether_hdr)),
440 rte_pktmbuf_mtod_offset(out, uint8_t *,
441 sizeof(struct rte_ether_hdr)),
442 K / 8 - CRC_24B_LEN))
443 printf("Input and output buffers are not equal!\n");
444 }
445 }
446
447 static int
initialize_ports(struct app_config_params * app_params,struct rte_mempool * ethdev_mbuf_mempool)448 initialize_ports(struct app_config_params *app_params,
449 struct rte_mempool *ethdev_mbuf_mempool)
450 {
451 int ret;
452 uint16_t port_id = app_params->port_id;
453 uint16_t q;
454 /* ethernet addresses of ports */
455 struct rte_ether_addr bbdev_port_eth_addr;
456
457 /* initialize ports */
458 printf("\nInitializing port %u...\n", app_params->port_id);
459 ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores,
460 app_params->num_dec_cores, &port_conf);
461
462 if (ret < 0) {
463 printf("Cannot configure device: err=%d, port=%u\n",
464 ret, port_id);
465 return -1;
466 }
467
468 /* initialize RX queues for encoder */
469 for (q = 0; q < app_params->num_enc_cores; q++) {
470 ret = rte_eth_rx_queue_setup(port_id, q,
471 RX_DESC_DEFAULT,
472 rte_eth_dev_socket_id(port_id),
473 NULL, ethdev_mbuf_mempool);
474 if (ret < 0) {
475 printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n",
476 ret, q);
477 return -1;
478 }
479 }
480 /* initialize TX queues for decoder */
481 for (q = 0; q < app_params->num_dec_cores; q++) {
482 ret = rte_eth_tx_queue_setup(port_id, q,
483 TX_DESC_DEFAULT,
484 rte_eth_dev_socket_id(port_id), NULL);
485 if (ret < 0) {
486 printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n",
487 ret, q);
488 return -1;
489 }
490 }
491
492 ret = rte_eth_promiscuous_enable(port_id);
493 if (ret != 0) {
494 printf("Cannot enable promiscuous mode: err=%s, port=%u\n",
495 rte_strerror(-ret), port_id);
496 return ret;
497 }
498
499 ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr);
500 if (ret < 0) {
501 printf("rte_eth_macaddr_get: err=%d, queue=%u\n",
502 ret, q);
503 return -1;
504 }
505
506 print_mac(port_id, &bbdev_port_eth_addr);
507
508 return 0;
509 }
510
511 static void
lcore_conf_init(struct app_config_params * app_params,struct lcore_conf * lcore_conf,struct rte_mempool ** bbdev_op_pools,struct rte_mempool * bbdev_mbuf_mempool,struct rte_ring * enc_to_dec_ring,struct lcore_statistics * lcore_stats)512 lcore_conf_init(struct app_config_params *app_params,
513 struct lcore_conf *lcore_conf,
514 struct rte_mempool **bbdev_op_pools,
515 struct rte_mempool *bbdev_mbuf_mempool,
516 struct rte_ring *enc_to_dec_ring,
517 struct lcore_statistics *lcore_stats)
518 {
519 unsigned int lcore_id;
520 struct lcore_conf *lconf;
521 uint16_t rx_queue_id = 0;
522 uint16_t tx_queue_id = 0;
523 uint16_t enc_q_id = 0;
524 uint16_t dec_q_id = 0;
525
526 /* Configure lcores */
527 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) {
528 lconf = &lcore_conf[lcore_id];
529 lconf->core_type = 0;
530
531 if ((1ULL << lcore_id) & app_params->enc_core_mask) {
532 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC);
533 lconf->rx_queue_id = rx_queue_id++;
534 lconf->enc_queue_id =
535 app_params->enc_queue_ids[enc_q_id++];
536 }
537
538 if ((1ULL << lcore_id) & app_params->dec_core_mask) {
539 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC);
540 lconf->tx_queue_id = tx_queue_id++;
541 lconf->dec_queue_id =
542 app_params->dec_queue_ids[dec_q_id++];
543 }
544
545 lconf->bbdev_enc_op_pool =
546 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC];
547 lconf->bbdev_dec_op_pool =
548 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC];
549 lconf->bbdev_id = app_params->bbdev_id;
550 lconf->port_id = app_params->port_id;
551 lconf->enc_out_pool = bbdev_mbuf_mempool;
552 lconf->enc_to_dec_ring = enc_to_dec_ring;
553 lconf->lcore_stats = &lcore_stats[lcore_id];
554 }
555 }
556
557 static void
print_lcore_stats(struct lcore_statistics * lstats,unsigned int lcore_id)558 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id)
559 {
560 static const char *stats_border = "_______";
561
562 printf("\nLcore %d: %s enqueued count:\t\t%u\n",
563 lcore_id, stats_border, lstats->enqueued);
564 printf("Lcore %d: %s dequeued count:\t\t%u\n",
565 lcore_id, stats_border, lstats->dequeued);
566 printf("Lcore %d: %s RX lost packets count:\t\t%u\n",
567 lcore_id, stats_border, lstats->rx_lost_packets);
568 printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n",
569 lcore_id, stats_border,
570 lstats->enc_to_dec_lost_packets);
571 printf("Lcore %d: %s TX lost packets count:\t\t%u\n",
572 lcore_id, stats_border, lstats->tx_lost_packets);
573 }
574
575 static void
print_stats(struct stats_lcore_params * stats_lcore)576 print_stats(struct stats_lcore_params *stats_lcore)
577 {
578 unsigned int l_id;
579 unsigned int bbdev_id = stats_lcore->app_params->bbdev_id;
580 unsigned int port_id = stats_lcore->app_params->port_id;
581 int len, ret, i;
582
583 struct rte_eth_xstat *xstats;
584 struct rte_eth_xstat_name *xstats_names;
585 struct rte_bbdev_stats bbstats;
586 static const char *stats_border = "_______";
587
588 const char clr[] = { 27, '[', '2', 'J', '\0' };
589 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
590
591 /* Clear screen and move to top left */
592 printf("%s%s", clr, topLeft);
593
594 printf("PORT STATISTICS:\n================\n");
595 len = rte_eth_xstats_get(port_id, NULL, 0);
596 if (len < 0)
597 rte_exit(EXIT_FAILURE,
598 "rte_eth_xstats_get(%u) failed: %d", port_id,
599 len);
600
601 xstats = calloc(len, sizeof(*xstats));
602 if (xstats == NULL)
603 rte_exit(EXIT_FAILURE,
604 "Failed to calloc memory for xstats");
605
606 ret = rte_eth_xstats_get(port_id, xstats, len);
607 if (ret < 0 || ret > len) {
608 free(xstats);
609 rte_exit(EXIT_FAILURE,
610 "rte_eth_xstats_get(%u) len%i failed: %d",
611 port_id, len, ret);
612 }
613
614 xstats_names = calloc(len, sizeof(*xstats_names));
615 if (xstats_names == NULL) {
616 free(xstats);
617 rte_exit(EXIT_FAILURE,
618 "Failed to calloc memory for xstats_names");
619 }
620
621 ret = rte_eth_xstats_get_names(port_id, xstats_names, len);
622 if (ret < 0 || ret > len) {
623 free(xstats);
624 free(xstats_names);
625 rte_exit(EXIT_FAILURE,
626 "rte_eth_xstats_get_names(%u) len%i failed: %d",
627 port_id, len, ret);
628 }
629
630 for (i = 0; i < len; i++) {
631 if (xstats[i].value > 0)
632 printf("Port %u: %s %s:\t\t%"PRIu64"\n",
633 port_id, stats_border,
634 xstats_names[i].name,
635 xstats[i].value);
636 }
637
638 ret = rte_bbdev_stats_get(bbdev_id, &bbstats);
639 if (ret < 0) {
640 free(xstats);
641 free(xstats_names);
642 rte_exit(EXIT_FAILURE,
643 "ERROR(%d): Failure to get BBDEV %u statistics\n",
644 ret, bbdev_id);
645 }
646
647 printf("\nBBDEV STATISTICS:\n=================\n");
648 printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n",
649 bbdev_id, stats_border,
650 bbstats.enqueued_count);
651 printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n",
652 bbdev_id, stats_border,
653 bbstats.dequeued_count);
654 printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n",
655 bbdev_id, stats_border,
656 bbstats.enqueue_err_count);
657 printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n",
658 bbdev_id, stats_border,
659 bbstats.dequeue_err_count);
660
661 printf("LCORE STATISTICS:\n=================\n");
662 for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) {
663 if (stats_lcore->lconf[l_id].core_type == 0)
664 continue;
665 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id);
666 }
667
668 fflush(stdout);
669
670 free(xstats);
671 free(xstats_names);
672 }
673
674 static int
stats_loop(void * arg)675 stats_loop(void *arg)
676 {
677 struct stats_lcore_params *stats_lcore = arg;
678
679 while (!rte_atomic_load_explicit(&global_exit_flag, rte_memory_order_relaxed)) {
680 print_stats(stats_lcore);
681 rte_delay_ms(500);
682 }
683
684 return 0;
685 }
686
687 static inline void
run_encoding(struct lcore_conf * lcore_conf)688 run_encoding(struct lcore_conf *lcore_conf)
689 {
690 uint16_t i;
691 uint16_t port_id, rx_queue_id;
692 uint16_t bbdev_id, enc_queue_id;
693 uint16_t nb_rx, nb_enq, nb_deq, nb_sent;
694 struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST];
695 struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST];
696 struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST];
697 struct lcore_statistics *lcore_stats;
698 struct rte_mempool *bbdev_op_pool, *enc_out_pool;
699 struct rte_ring *enc_to_dec_ring;
700 const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN;
701
702 lcore_stats = lcore_conf->lcore_stats;
703 port_id = lcore_conf->port_id;
704 rx_queue_id = lcore_conf->rx_queue_id;
705 bbdev_id = lcore_conf->bbdev_id;
706 enc_queue_id = lcore_conf->enc_queue_id;
707 bbdev_op_pool = lcore_conf->bbdev_enc_op_pool;
708 enc_out_pool = lcore_conf->enc_out_pool;
709 enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
710
711 /* Read packet from RX queues*/
712 nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst,
713 MAX_PKT_BURST);
714 if (!nb_rx)
715 return;
716
717 if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts,
718 nb_rx) != 0)) {
719 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
720 lcore_stats->rx_lost_packets += nb_rx;
721 return;
722 }
723
724 if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
725 nb_rx) != 0)) {
726 pktmbuf_free_bulk(enc_out_pkts, nb_rx);
727 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
728 lcore_stats->rx_lost_packets += nb_rx;
729 return;
730 }
731
732 for (i = 0; i < nb_rx; i++) {
733 char *data;
734 const uint16_t pkt_data_len =
735 rte_pktmbuf_data_len(rx_pkts_burst[i]) -
736 sizeof(struct rte_ether_hdr);
737 /* save input mbuf pointer for later comparison */
738 *mbuf_input(enc_out_pkts[i]) = rx_pkts_burst[i];
739
740 /* copy ethernet header */
741 rte_pktmbuf_reset(enc_out_pkts[i]);
742 data = rte_pktmbuf_append(enc_out_pkts[i],
743 sizeof(struct rte_ether_hdr));
744 if (data == NULL) {
745 printf(
746 "Not enough space for ethernet header in encoder output mbuf\n");
747 continue;
748 }
749 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]);
750
751 /* set op */
752 bbdev_ops_burst[i]->turbo_enc = def_op_enc;
753
754 bbdev_ops_burst[i]->turbo_enc.input.data =
755 rx_pkts_burst[i];
756 bbdev_ops_burst[i]->turbo_enc.input.offset =
757 sizeof(struct rte_ether_hdr);
758 /* Encoder will attach the CRC24B, adjust the length */
759 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len;
760
761 if (in_data_len < pkt_data_len)
762 rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len -
763 in_data_len);
764 else if (in_data_len > pkt_data_len) {
765 data = rte_pktmbuf_append(rx_pkts_burst[i],
766 in_data_len - pkt_data_len);
767 if (data == NULL)
768 printf(
769 "Not enough storage in mbuf to perform the encoding\n");
770 }
771
772 bbdev_ops_burst[i]->turbo_enc.output.data =
773 enc_out_pkts[i];
774 bbdev_ops_burst[i]->turbo_enc.output.offset =
775 sizeof(struct rte_ether_hdr);
776 }
777
778 /* Enqueue packets on BBDevice */
779 nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id,
780 bbdev_ops_burst, nb_rx);
781 if (unlikely(nb_enq < nb_rx)) {
782 pktmbuf_input_free_bulk(&enc_out_pkts[nb_enq],
783 nb_rx - nb_enq);
784 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq],
785 nb_rx - nb_enq);
786 lcore_stats->rx_lost_packets += nb_rx - nb_enq;
787
788 if (!nb_enq)
789 return;
790 }
791
792 lcore_stats->enqueued += nb_enq;
793
794 /* Dequeue packets from bbdev device*/
795 nb_deq = 0;
796 do {
797 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id,
798 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
799 } while (unlikely(nb_deq < nb_enq));
800
801 lcore_stats->dequeued += nb_deq;
802
803 /* Generate and add AWGN */
804 add_awgn(enc_out_pkts, nb_deq);
805
806 rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq);
807
808 /* Enqueue packets to encoder-to-decoder ring */
809 nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts,
810 nb_deq, NULL);
811 if (unlikely(nb_sent < nb_deq)) {
812 pktmbuf_input_free_bulk(&enc_out_pkts[nb_sent],
813 nb_deq - nb_sent);
814 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent;
815 }
816 }
817
818 static void
run_decoding(struct lcore_conf * lcore_conf)819 run_decoding(struct lcore_conf *lcore_conf)
820 {
821 uint16_t i;
822 uint16_t port_id, tx_queue_id;
823 uint16_t bbdev_id, bbdev_queue_id;
824 uint16_t nb_recv, nb_enq, nb_deq, nb_tx;
825 uint8_t *llr_temp_buf;
826 struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST];
827 struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST];
828 struct lcore_statistics *lcore_stats;
829 struct rte_mempool *bbdev_op_pool;
830 struct rte_ring *enc_to_dec_ring;
831
832 lcore_stats = lcore_conf->lcore_stats;
833 port_id = lcore_conf->port_id;
834 tx_queue_id = lcore_conf->tx_queue_id;
835 bbdev_id = lcore_conf->bbdev_id;
836 bbdev_queue_id = lcore_conf->dec_queue_id;
837 bbdev_op_pool = lcore_conf->bbdev_dec_op_pool;
838 enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
839 llr_temp_buf = lcore_conf->llr_temp_buf;
840
841 /* Dequeue packets from the ring */
842 nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring,
843 (void **)recv_pkts_burst, MAX_PKT_BURST, NULL);
844 if (!nb_recv)
845 return;
846
847 if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
848 nb_recv) != 0)) {
849 pktmbuf_input_free_bulk(recv_pkts_burst, nb_recv);
850 lcore_stats->rx_lost_packets += nb_recv;
851 return;
852 }
853
854 transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv,
855 def_op_dec.cb_params.k);
856
857 for (i = 0; i < nb_recv; i++) {
858 /* set op */
859 bbdev_ops_burst[i]->turbo_dec = def_op_dec;
860
861 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i];
862 bbdev_ops_burst[i]->turbo_dec.input.offset =
863 sizeof(struct rte_ether_hdr);
864 bbdev_ops_burst[i]->turbo_dec.input.length =
865 rte_pktmbuf_data_len(recv_pkts_burst[i])
866 - sizeof(struct rte_ether_hdr);
867
868 bbdev_ops_burst[i]->turbo_dec.hard_output.data =
869 recv_pkts_burst[i];
870 bbdev_ops_burst[i]->turbo_dec.hard_output.offset =
871 sizeof(struct rte_ether_hdr);
872 }
873
874 /* Enqueue packets on BBDevice */
875 nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id,
876 bbdev_ops_burst, nb_recv);
877 if (unlikely(nb_enq < nb_recv)) {
878 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_enq],
879 nb_recv - nb_enq);
880 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq],
881 nb_recv - nb_enq);
882 lcore_stats->rx_lost_packets += nb_recv - nb_enq;
883
884 if (!nb_enq)
885 return;
886 }
887
888 lcore_stats->enqueued += nb_enq;
889
890 /* Dequeue packets from BBDevice */
891 nb_deq = 0;
892 do {
893 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id,
894 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
895 } while (unlikely(nb_deq < nb_enq));
896
897 lcore_stats->dequeued += nb_deq;
898
899 rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq);
900
901 verify_data(recv_pkts_burst, nb_deq);
902
903 /* Free the RX mbufs after verification */
904 for (i = 0; i < nb_deq; ++i)
905 rte_pktmbuf_free(*mbuf_input(recv_pkts_burst[i]));
906
907 /* Transmit the packets */
908 nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq);
909 if (unlikely(nb_tx < nb_deq)) {
910 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_tx],
911 nb_deq - nb_tx);
912 lcore_stats->tx_lost_packets += nb_deq - nb_tx;
913 }
914 }
915
916 static int
processing_loop(void * arg)917 processing_loop(void *arg)
918 {
919 struct lcore_conf *lcore_conf = arg;
920 const bool run_encoder = (lcore_conf->core_type &
921 (1 << RTE_BBDEV_OP_TURBO_ENC));
922 const bool run_decoder = (lcore_conf->core_type &
923 (1 << RTE_BBDEV_OP_TURBO_DEC));
924
925 while (!rte_atomic_load_explicit(&global_exit_flag, rte_memory_order_relaxed)) {
926 if (run_encoder)
927 run_encoding(lcore_conf);
928 if (run_decoder)
929 run_decoding(lcore_conf);
930 }
931
932 return 0;
933 }
934
935 static int
prepare_bbdev_device(unsigned int dev_id,struct rte_bbdev_info * info,struct app_config_params * app_params)936 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info,
937 struct app_config_params *app_params)
938 {
939 int ret;
940 unsigned int q_id, dec_q_id, enc_q_id;
941 struct rte_bbdev_queue_conf qconf = {0};
942 uint16_t dec_qs_nb = app_params->num_dec_cores;
943 uint16_t enc_qs_nb = app_params->num_enc_cores;
944 uint16_t tot_qs = dec_qs_nb + enc_qs_nb;
945
946 ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id);
947 if (ret < 0)
948 rte_exit(EXIT_FAILURE,
949 "ERROR(%d): BBDEV %u not configured properly\n",
950 ret, dev_id);
951
952 /* setup device DEC queues */
953 qconf.socket = info->socket_id;
954 qconf.queue_size = info->drv.queue_size_lim;
955 qconf.op_type = RTE_BBDEV_OP_TURBO_DEC;
956
957 for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) {
958 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
959 if (ret < 0)
960 rte_exit(EXIT_FAILURE,
961 "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n",
962 ret, dev_id, q_id);
963 app_params->dec_queue_ids[dec_q_id++] = q_id;
964 }
965
966 /* setup device ENC queues */
967 qconf.op_type = RTE_BBDEV_OP_TURBO_ENC;
968
969 for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) {
970 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
971 if (ret < 0)
972 rte_exit(EXIT_FAILURE,
973 "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n",
974 ret, dev_id, q_id);
975 app_params->enc_queue_ids[enc_q_id++] = q_id;
976 }
977
978 ret = rte_bbdev_start(dev_id);
979
980 if (ret != 0)
981 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n",
982 ret, dev_id);
983
984 printf("BBdev %u started\n", dev_id);
985
986 return 0;
987 }
988
989 static inline bool
check_matching_capabilities(uint64_t mask,uint64_t required_mask)990 check_matching_capabilities(uint64_t mask, uint64_t required_mask)
991 {
992 return (mask & required_mask) == required_mask;
993 }
994
995 static void
enable_bbdev(struct app_config_params * app_params)996 enable_bbdev(struct app_config_params *app_params)
997 {
998 struct rte_bbdev_info dev_info;
999 const struct rte_bbdev_op_cap *op_cap;
1000 uint16_t bbdev_id = app_params->bbdev_id;
1001 bool encoder_capable = false;
1002 bool decoder_capable = false;
1003
1004 rte_bbdev_info_get(bbdev_id, &dev_info);
1005 op_cap = dev_info.drv.capabilities;
1006
1007 while (op_cap->type != RTE_BBDEV_OP_NONE) {
1008 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
1009 if (check_matching_capabilities(
1010 op_cap->cap.turbo_enc.capability_flags,
1011 def_op_enc.op_flags))
1012 encoder_capable = true;
1013 }
1014
1015 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1016 if (check_matching_capabilities(
1017 op_cap->cap.turbo_dec.capability_flags,
1018 def_op_dec.op_flags))
1019 decoder_capable = true;
1020 }
1021
1022 op_cap++;
1023 }
1024
1025 if (encoder_capable == false)
1026 rte_exit(EXIT_FAILURE,
1027 "The specified BBDev %u doesn't have required encoder capabilities!\n",
1028 bbdev_id);
1029 if (decoder_capable == false)
1030 rte_exit(EXIT_FAILURE,
1031 "The specified BBDev %u doesn't have required decoder capabilities!\n",
1032 bbdev_id);
1033
1034 prepare_bbdev_device(bbdev_id, &dev_info, app_params);
1035 }
1036
1037 int
main(int argc,char ** argv)1038 main(int argc, char **argv)
1039 {
1040 int ret;
1041 unsigned int nb_bbdevs, flags, lcore_id;
1042 void *sigret;
1043 struct app_config_params app_params = def_app_config;
1044 struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool;
1045 struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_SIZE_MAX];
1046 struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} };
1047 struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} };
1048 struct stats_lcore_params stats_lcore;
1049 struct rte_ring *enc_to_dec_ring;
1050 bool stats_thread_started = false;
1051 unsigned int main_lcore_id = rte_get_main_lcore();
1052
1053 static const struct rte_mbuf_dynfield input_dynfield_desc = {
1054 .name = "example_bbdev_dynfield_input",
1055 .size = sizeof(struct rte_mbuf *),
1056 .align = alignof(struct rte_mbuf *),
1057 };
1058
1059 rte_atomic_store_explicit(&global_exit_flag, 0, rte_memory_order_relaxed);
1060
1061 sigret = signal(SIGTERM, signal_handler);
1062 if (sigret == SIG_ERR)
1063 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM);
1064
1065 sigret = signal(SIGINT, signal_handler);
1066 if (sigret == SIG_ERR)
1067 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT);
1068
1069 ret = rte_eal_init(argc, argv);
1070 if (ret < 0)
1071 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
1072
1073 argc -= ret;
1074 argv += ret;
1075
1076 /* parse application arguments (after the EAL ones) */
1077 ret = bbdev_parse_args(argc, argv, &app_params);
1078 if (ret < 0)
1079 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n");
1080
1081 /*create bbdev op pools*/
1082 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] =
1083 rte_bbdev_op_pool_create("bbdev_op_pool_dec",
1084 RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id());
1085 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] =
1086 rte_bbdev_op_pool_create("bbdev_op_pool_enc",
1087 RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id());
1088
1089 if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) ||
1090 (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL))
1091 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n");
1092
1093 /* Create encoder to decoder ring */
1094 flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0;
1095 if (app_params.num_dec_cores == 1)
1096 flags |= RING_F_SC_DEQ;
1097
1098 enc_to_dec_ring = rte_ring_create("enc_to_dec_ring",
1099 rte_align32pow2(NB_MBUF), rte_socket_id(), flags);
1100
1101 /* Get the number of available bbdev devices */
1102 nb_bbdevs = rte_bbdev_count();
1103 if (nb_bbdevs <= app_params.bbdev_id)
1104 rte_exit(EXIT_FAILURE,
1105 "%u BBDevs detected, cannot use BBDev with ID %u!\n",
1106 nb_bbdevs, app_params.bbdev_id);
1107 printf("Number of bbdevs detected: %d\n", nb_bbdevs);
1108
1109 if (!rte_eth_dev_is_valid_port(app_params.port_id))
1110 rte_exit(EXIT_FAILURE,
1111 "cannot use port with ID %u!\n",
1112 app_params.port_id);
1113
1114 /* create the mbuf mempool for ethdev pkts */
1115 ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool",
1116 NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1117 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1118 if (ethdev_mbuf_mempool == NULL)
1119 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1120
1121 /* create the mbuf mempool for encoder output */
1122 bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool",
1123 NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1124 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1125 if (bbdev_mbuf_mempool == NULL)
1126 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1127
1128 /* register mbuf field to store input pointer */
1129 input_dynfield_offset =
1130 rte_mbuf_dynfield_register(&input_dynfield_desc);
1131 if (input_dynfield_offset < 0)
1132 rte_exit(EXIT_FAILURE, "Cannot register mbuf field\n");
1133
1134 /* initialize ports */
1135 ret = initialize_ports(&app_params, ethdev_mbuf_mempool);
1136
1137 /* Check if all requested lcores are available */
1138 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id)
1139 if (((1ULL << lcore_id) & app_params.enc_core_mask) ||
1140 ((1ULL << lcore_id) & app_params.dec_core_mask))
1141 if (!rte_lcore_is_enabled(lcore_id))
1142 rte_exit(EXIT_FAILURE,
1143 "Requested lcore_id %u is not enabled!\n",
1144 lcore_id);
1145
1146 /* Start ethernet port */
1147 ret = rte_eth_dev_start(app_params.port_id);
1148 if (ret < 0)
1149 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
1150 ret, app_params.port_id);
1151
1152 ret = check_port_link_status(app_params.port_id);
1153 if (ret < 0)
1154 exit(EXIT_FAILURE);
1155
1156 /* start BBDevice and save BBDev queue IDs */
1157 enable_bbdev(&app_params);
1158
1159 /* Initialize the port/queue configuration of each logical core */
1160 lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools,
1161 bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats);
1162
1163 stats_lcore.app_params = &app_params;
1164 stats_lcore.lconf = lcore_conf;
1165
1166 RTE_LCORE_FOREACH_WORKER(lcore_id) {
1167 if (lcore_conf[lcore_id].core_type != 0)
1168 /* launch per-lcore processing loop on worker lcores */
1169 rte_eal_remote_launch(processing_loop,
1170 &lcore_conf[lcore_id], lcore_id);
1171 else if (!stats_thread_started) {
1172 /* launch statistics printing loop */
1173 rte_eal_remote_launch(stats_loop, &stats_lcore,
1174 lcore_id);
1175 stats_thread_started = true;
1176 }
1177 }
1178
1179 if (!stats_thread_started &&
1180 lcore_conf[main_lcore_id].core_type != 0)
1181 rte_exit(EXIT_FAILURE,
1182 "Not enough lcores to run the statistics printing loop!");
1183 else if (lcore_conf[main_lcore_id].core_type != 0)
1184 processing_loop(&lcore_conf[main_lcore_id]);
1185 else if (!stats_thread_started)
1186 stats_loop(&stats_lcore);
1187
1188 RTE_LCORE_FOREACH_WORKER(lcore_id) {
1189 ret |= rte_eal_wait_lcore(lcore_id);
1190 }
1191
1192 /* clean up the EAL */
1193 rte_eal_cleanup();
1194
1195 return ret;
1196 }
1197