1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdalign.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <inttypes.h> 11 #include <sys/types.h> 12 #include <unistd.h> 13 #include <sys/queue.h> 14 #include <stdarg.h> 15 #include <ctype.h> 16 #include <errno.h> 17 #include <math.h> 18 #include <assert.h> 19 #include <getopt.h> 20 #include <signal.h> 21 22 #include <rte_common.h> 23 #include <rte_eal.h> 24 #include <rte_cycles.h> 25 #include <rte_ether.h> 26 #include <rte_ethdev.h> 27 #include <rte_ip.h> 28 #include <rte_lcore.h> 29 #include <rte_malloc.h> 30 #include <rte_mbuf.h> 31 #include <rte_mbuf_dyn.h> 32 #include <rte_memory.h> 33 #include <rte_mempool.h> 34 #include <rte_log.h> 35 #include <rte_bbdev.h> 36 #include <rte_bbdev_op.h> 37 38 /* LLR values - negative value for '1' bit */ 39 #define LLR_1_BIT 0x81 40 #define LLR_0_BIT 0x7F 41 42 #define MAX_PKT_BURST 32 43 #define NB_MBUF 8191 44 #define MEMPOOL_CACHE_SIZE 256 45 46 /* Hardcoded K value */ 47 #define K 40 48 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32)) 49 50 #define CRC_24B_LEN 3 51 52 /* Configurable number of RX/TX ring descriptors */ 53 #define RX_DESC_DEFAULT 128 54 #define TX_DESC_DEFAULT 512 55 56 #define BBDEV_ASSERT(a) do { \ 57 if (!(a)) { \ 58 usage(prgname); \ 59 return -1; \ 60 } \ 61 } while (0) 62 63 static int input_dynfield_offset = -1; 64 65 static inline struct rte_mbuf ** 66 mbuf_input(struct rte_mbuf *mbuf) 67 { 68 return RTE_MBUF_DYNFIELD(mbuf, 69 input_dynfield_offset, struct rte_mbuf **); 70 } 71 72 static const struct rte_eth_conf port_conf = { 73 .rxmode = { 74 .mq_mode = RTE_ETH_MQ_RX_NONE, 75 }, 76 .txmode = { 77 .mq_mode = RTE_ETH_MQ_TX_NONE, 78 }, 79 }; 80 81 struct rte_bbdev_op_turbo_enc def_op_enc = { 82 /* These values are arbitrarily put, and does not map to the real 83 * values for the data received from ethdev ports 84 */ 85 .rv_index = 0, 86 .code_block_mode = 1, 87 .cb_params = { 88 .k = K, 89 }, 90 .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH 91 }; 92 93 struct rte_bbdev_op_turbo_dec def_op_dec = { 94 /* These values are arbitrarily put, and does not map to the real 95 * values for the data received from ethdev ports 96 */ 97 .code_block_mode = 1, 98 .cb_params = { 99 .k = K, 100 }, 101 .rv_index = 0, 102 .iter_max = 8, 103 .iter_min = 4, 104 .ext_scale = 15, 105 .num_maps = 0, 106 .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN 107 }; 108 109 struct app_config_params { 110 /* Placeholders for app params */ 111 uint16_t port_id; 112 uint16_t bbdev_id; 113 uint64_t enc_core_mask; 114 uint64_t dec_core_mask; 115 116 /* Values filled during init time */ 117 uint16_t enc_queue_ids[RTE_MAX_LCORE]; 118 uint16_t dec_queue_ids[RTE_MAX_LCORE]; 119 uint16_t num_enc_cores; 120 uint16_t num_dec_cores; 121 }; 122 123 struct __rte_cache_aligned lcore_statistics { 124 unsigned int enqueued; 125 unsigned int dequeued; 126 unsigned int rx_lost_packets; 127 unsigned int enc_to_dec_lost_packets; 128 unsigned int tx_lost_packets; 129 }; 130 131 /** each lcore configuration */ 132 struct __rte_cache_aligned lcore_conf { 133 uint64_t core_type; 134 135 unsigned int port_id; 136 unsigned int rx_queue_id; 137 unsigned int tx_queue_id; 138 139 unsigned int bbdev_id; 140 unsigned int enc_queue_id; 141 unsigned int dec_queue_id; 142 143 uint8_t llr_temp_buf[NCB]; 144 145 struct rte_mempool *bbdev_dec_op_pool; 146 struct rte_mempool *bbdev_enc_op_pool; 147 struct rte_mempool *enc_out_pool; 148 struct rte_ring *enc_to_dec_ring; 149 150 struct lcore_statistics *lcore_stats; 151 }; 152 153 struct stats_lcore_params { 154 struct lcore_conf *lconf; 155 struct app_config_params *app_params; 156 }; 157 158 159 static const struct app_config_params def_app_config = { 160 .port_id = 0, 161 .bbdev_id = 0, 162 .enc_core_mask = 0x2, 163 .dec_core_mask = 0x4, 164 .num_enc_cores = 1, 165 .num_dec_cores = 1, 166 }; 167 168 static RTE_ATOMIC(uint16_t) global_exit_flag; 169 170 /* display usage */ 171 static inline void 172 usage(const char *prgname) 173 { 174 printf("%s [EAL options] " 175 " --\n" 176 " --enc_cores - number of encoding cores (default = 0x2)\n" 177 " --dec_cores - number of decoding cores (default = 0x4)\n" 178 " --port_id - Ethernet port ID (default = 0)\n" 179 " --bbdev_id - BBDev ID (default = 0)\n" 180 "\n", prgname); 181 } 182 183 /* parse core mask */ 184 static inline 185 uint16_t bbdev_parse_mask(const char *mask) 186 { 187 char *end = NULL; 188 unsigned long pm; 189 190 /* parse hexadecimal string */ 191 pm = strtoul(mask, &end, 16); 192 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0')) 193 return 0; 194 195 return pm; 196 } 197 198 /* parse core mask */ 199 static inline 200 uint16_t bbdev_parse_number(const char *mask) 201 { 202 char *end = NULL; 203 unsigned long pm; 204 205 /* parse hexadecimal string */ 206 pm = strtoul(mask, &end, 10); 207 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0')) 208 return 0; 209 210 return pm; 211 } 212 213 static int 214 bbdev_parse_args(int argc, char **argv, 215 struct app_config_params *app_params) 216 { 217 int optind = 0; 218 int opt; 219 int opt_indx = 0; 220 char *prgname = argv[0]; 221 222 static struct option lgopts[] = { 223 { "enc_core_mask", required_argument, 0, 'e' }, 224 { "dec_core_mask", required_argument, 0, 'd' }, 225 { "port_id", required_argument, 0, 'p' }, 226 { "bbdev_id", required_argument, 0, 'b' }, 227 { NULL, 0, 0, 0 } 228 }; 229 230 BBDEV_ASSERT(argc != 0); 231 BBDEV_ASSERT(argv != NULL); 232 BBDEV_ASSERT(app_params != NULL); 233 234 while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) != 235 EOF) { 236 switch (opt) { 237 case 'e': 238 app_params->enc_core_mask = 239 bbdev_parse_mask(optarg); 240 if (app_params->enc_core_mask == 0) { 241 usage(prgname); 242 return -1; 243 } 244 app_params->num_enc_cores = 245 rte_popcount32(app_params->enc_core_mask); 246 break; 247 248 case 'd': 249 app_params->dec_core_mask = 250 bbdev_parse_mask(optarg); 251 if (app_params->dec_core_mask == 0) { 252 usage(prgname); 253 return -1; 254 } 255 app_params->num_dec_cores = 256 rte_popcount32(app_params->dec_core_mask); 257 break; 258 259 case 'p': 260 app_params->port_id = bbdev_parse_number(optarg); 261 break; 262 263 case 'b': 264 app_params->bbdev_id = bbdev_parse_number(optarg); 265 break; 266 267 default: 268 usage(prgname); 269 return -1; 270 } 271 } 272 optind = 0; 273 return optind; 274 } 275 276 static void 277 signal_handler(int signum) 278 { 279 printf("\nSignal %d received\n", signum); 280 rte_atomic_store_explicit(&global_exit_flag, 1, rte_memory_order_relaxed); 281 } 282 283 static void 284 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address) 285 { 286 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n", 287 (unsigned int) portid, 288 RTE_ETHER_ADDR_BYTES(bbdev_ports_eth_address)); 289 } 290 291 static inline void 292 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free) 293 { 294 unsigned int i; 295 for (i = 0; i < nb_to_free; ++i) 296 rte_pktmbuf_free(mbufs[i]); 297 } 298 299 static inline void 300 pktmbuf_input_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free) 301 { 302 unsigned int i; 303 for (i = 0; i < nb_to_free; ++i) { 304 struct rte_mbuf *rx_pkt = *mbuf_input(mbufs[i]); 305 rte_pktmbuf_free(rx_pkt); 306 rte_pktmbuf_free(mbufs[i]); 307 } 308 } 309 310 /* Check the link status of all ports in up to 9s, and print them finally */ 311 static int 312 check_port_link_status(uint16_t port_id) 313 { 314 #define CHECK_INTERVAL 100 /* 100ms */ 315 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 316 uint8_t count; 317 struct rte_eth_link link; 318 int link_get_err = -EINVAL; 319 320 printf("\nChecking link status."); 321 fflush(stdout); 322 323 for (count = 0; count <= MAX_CHECK_TIME && 324 !rte_atomic_load_explicit(&global_exit_flag, 325 rte_memory_order_relaxed); count++) { 326 memset(&link, 0, sizeof(link)); 327 link_get_err = rte_eth_link_get_nowait(port_id, &link); 328 329 if (link_get_err >= 0 && link.link_status) { 330 const char *dp = (link.link_duplex == 331 RTE_ETH_LINK_FULL_DUPLEX) ? 332 "full-duplex" : "half-duplex"; 333 printf("\nPort %u Link Up - speed %s - %s\n", 334 port_id, 335 rte_eth_link_speed_to_str(link.link_speed), 336 dp); 337 return 0; 338 } 339 printf("."); 340 fflush(stdout); 341 rte_delay_ms(CHECK_INTERVAL); 342 } 343 344 if (link_get_err >= 0) 345 printf("\nPort %d Link Down\n", port_id); 346 else 347 printf("\nGet link failed (port %d): %s\n", port_id, 348 rte_strerror(-link_get_err)); 349 350 return 0; 351 } 352 353 static inline void 354 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst) 355 { 356 struct rte_ether_hdr *eth_from; 357 struct rte_ether_hdr *eth_to; 358 359 eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *); 360 eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *); 361 362 /* copy header */ 363 rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr)); 364 } 365 366 static inline void 367 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts) 368 { 369 RTE_SET_USED(mbufs); 370 RTE_SET_USED(num_pkts); 371 } 372 373 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input 374 * so each bit of the encoder output must be translated into one byte of LLR. If 375 * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes 376 * must additionally be inserted at the end of each sub-block. 377 */ 378 static inline void 379 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf, 380 uint16_t num_pkts, uint16_t k) 381 { 382 uint16_t i, l, j; 383 uint16_t start_bit_idx; 384 uint16_t out_idx; 385 uint16_t d = k + 4; 386 uint16_t kpi = RTE_ALIGN_CEIL(d, 32); 387 uint16_t nd = kpi - d; 388 uint16_t ncb = 3 * kpi; 389 390 for (i = 0; i < num_pkts; ++i) { 391 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) - 392 sizeof(struct rte_ether_hdr); 393 394 /* Resize the packet if needed */ 395 if (pkt_data_len < ncb) { 396 char *data = rte_pktmbuf_append(mbufs[i], 397 ncb - pkt_data_len); 398 if (data == NULL) 399 printf( 400 "Not enough space in decoder input packet"); 401 } 402 403 /* Translate each bit into 1 LLR byte. */ 404 start_bit_idx = 0; 405 out_idx = 0; 406 for (j = 0; j < 3; ++j) { 407 for (l = start_bit_idx; l < start_bit_idx + d; ++l) { 408 uint8_t *data = rte_pktmbuf_mtod_offset( 409 mbufs[i], uint8_t *, 410 sizeof(struct rte_ether_hdr) + 411 (l >> 3)); 412 if (*data & (0x80 >> (l & 7))) 413 temp_buf[out_idx] = LLR_1_BIT; 414 else 415 temp_buf[out_idx] = LLR_0_BIT; 416 ++out_idx; 417 } 418 /* Padding bytes should be at the end of the sub-block. 419 */ 420 memset(&temp_buf[out_idx], 0, nd); 421 out_idx += nd; 422 start_bit_idx += d; 423 } 424 425 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *, 426 sizeof(struct rte_ether_hdr)), temp_buf, ncb); 427 } 428 } 429 430 static inline void 431 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts) 432 { 433 uint16_t i; 434 for (i = 0; i < num_pkts; ++i) { 435 struct rte_mbuf *out = mbufs[i]; 436 struct rte_mbuf *in = *mbuf_input(out); 437 438 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *, 439 sizeof(struct rte_ether_hdr)), 440 rte_pktmbuf_mtod_offset(out, uint8_t *, 441 sizeof(struct rte_ether_hdr)), 442 K / 8 - CRC_24B_LEN)) 443 printf("Input and output buffers are not equal!\n"); 444 } 445 } 446 447 static int 448 initialize_ports(struct app_config_params *app_params, 449 struct rte_mempool *ethdev_mbuf_mempool) 450 { 451 int ret; 452 uint16_t port_id = app_params->port_id; 453 uint16_t q; 454 /* ethernet addresses of ports */ 455 struct rte_ether_addr bbdev_port_eth_addr; 456 457 /* initialize ports */ 458 printf("\nInitializing port %u...\n", app_params->port_id); 459 ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores, 460 app_params->num_dec_cores, &port_conf); 461 462 if (ret < 0) { 463 printf("Cannot configure device: err=%d, port=%u\n", 464 ret, port_id); 465 return -1; 466 } 467 468 /* initialize RX queues for encoder */ 469 for (q = 0; q < app_params->num_enc_cores; q++) { 470 ret = rte_eth_rx_queue_setup(port_id, q, 471 RX_DESC_DEFAULT, 472 rte_eth_dev_socket_id(port_id), 473 NULL, ethdev_mbuf_mempool); 474 if (ret < 0) { 475 printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n", 476 ret, q); 477 return -1; 478 } 479 } 480 /* initialize TX queues for decoder */ 481 for (q = 0; q < app_params->num_dec_cores; q++) { 482 ret = rte_eth_tx_queue_setup(port_id, q, 483 TX_DESC_DEFAULT, 484 rte_eth_dev_socket_id(port_id), NULL); 485 if (ret < 0) { 486 printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n", 487 ret, q); 488 return -1; 489 } 490 } 491 492 ret = rte_eth_promiscuous_enable(port_id); 493 if (ret != 0) { 494 printf("Cannot enable promiscuous mode: err=%s, port=%u\n", 495 rte_strerror(-ret), port_id); 496 return ret; 497 } 498 499 ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr); 500 if (ret < 0) { 501 printf("rte_eth_macaddr_get: err=%d, queue=%u\n", 502 ret, q); 503 return -1; 504 } 505 506 print_mac(port_id, &bbdev_port_eth_addr); 507 508 return 0; 509 } 510 511 static void 512 lcore_conf_init(struct app_config_params *app_params, 513 struct lcore_conf *lcore_conf, 514 struct rte_mempool **bbdev_op_pools, 515 struct rte_mempool *bbdev_mbuf_mempool, 516 struct rte_ring *enc_to_dec_ring, 517 struct lcore_statistics *lcore_stats) 518 { 519 unsigned int lcore_id; 520 struct lcore_conf *lconf; 521 uint16_t rx_queue_id = 0; 522 uint16_t tx_queue_id = 0; 523 uint16_t enc_q_id = 0; 524 uint16_t dec_q_id = 0; 525 526 /* Configure lcores */ 527 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) { 528 lconf = &lcore_conf[lcore_id]; 529 lconf->core_type = 0; 530 531 if ((1ULL << lcore_id) & app_params->enc_core_mask) { 532 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC); 533 lconf->rx_queue_id = rx_queue_id++; 534 lconf->enc_queue_id = 535 app_params->enc_queue_ids[enc_q_id++]; 536 } 537 538 if ((1ULL << lcore_id) & app_params->dec_core_mask) { 539 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC); 540 lconf->tx_queue_id = tx_queue_id++; 541 lconf->dec_queue_id = 542 app_params->dec_queue_ids[dec_q_id++]; 543 } 544 545 lconf->bbdev_enc_op_pool = 546 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC]; 547 lconf->bbdev_dec_op_pool = 548 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC]; 549 lconf->bbdev_id = app_params->bbdev_id; 550 lconf->port_id = app_params->port_id; 551 lconf->enc_out_pool = bbdev_mbuf_mempool; 552 lconf->enc_to_dec_ring = enc_to_dec_ring; 553 lconf->lcore_stats = &lcore_stats[lcore_id]; 554 } 555 } 556 557 static void 558 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id) 559 { 560 static const char *stats_border = "_______"; 561 562 printf("\nLcore %d: %s enqueued count:\t\t%u\n", 563 lcore_id, stats_border, lstats->enqueued); 564 printf("Lcore %d: %s dequeued count:\t\t%u\n", 565 lcore_id, stats_border, lstats->dequeued); 566 printf("Lcore %d: %s RX lost packets count:\t\t%u\n", 567 lcore_id, stats_border, lstats->rx_lost_packets); 568 printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n", 569 lcore_id, stats_border, 570 lstats->enc_to_dec_lost_packets); 571 printf("Lcore %d: %s TX lost packets count:\t\t%u\n", 572 lcore_id, stats_border, lstats->tx_lost_packets); 573 } 574 575 static void 576 print_stats(struct stats_lcore_params *stats_lcore) 577 { 578 unsigned int l_id; 579 unsigned int bbdev_id = stats_lcore->app_params->bbdev_id; 580 unsigned int port_id = stats_lcore->app_params->port_id; 581 int len, ret, i; 582 583 struct rte_eth_xstat *xstats; 584 struct rte_eth_xstat_name *xstats_names; 585 struct rte_bbdev_stats bbstats; 586 static const char *stats_border = "_______"; 587 588 const char clr[] = { 27, '[', '2', 'J', '\0' }; 589 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 590 591 /* Clear screen and move to top left */ 592 printf("%s%s", clr, topLeft); 593 594 printf("PORT STATISTICS:\n================\n"); 595 len = rte_eth_xstats_get(port_id, NULL, 0); 596 if (len < 0) 597 rte_exit(EXIT_FAILURE, 598 "rte_eth_xstats_get(%u) failed: %d", port_id, 599 len); 600 601 xstats = calloc(len, sizeof(*xstats)); 602 if (xstats == NULL) 603 rte_exit(EXIT_FAILURE, 604 "Failed to calloc memory for xstats"); 605 606 ret = rte_eth_xstats_get(port_id, xstats, len); 607 if (ret < 0 || ret > len) { 608 free(xstats); 609 rte_exit(EXIT_FAILURE, 610 "rte_eth_xstats_get(%u) len%i failed: %d", 611 port_id, len, ret); 612 } 613 614 xstats_names = calloc(len, sizeof(*xstats_names)); 615 if (xstats_names == NULL) { 616 free(xstats); 617 rte_exit(EXIT_FAILURE, 618 "Failed to calloc memory for xstats_names"); 619 } 620 621 ret = rte_eth_xstats_get_names(port_id, xstats_names, len); 622 if (ret < 0 || ret > len) { 623 free(xstats); 624 free(xstats_names); 625 rte_exit(EXIT_FAILURE, 626 "rte_eth_xstats_get_names(%u) len%i failed: %d", 627 port_id, len, ret); 628 } 629 630 for (i = 0; i < len; i++) { 631 if (xstats[i].value > 0) 632 printf("Port %u: %s %s:\t\t%"PRIu64"\n", 633 port_id, stats_border, 634 xstats_names[i].name, 635 xstats[i].value); 636 } 637 638 ret = rte_bbdev_stats_get(bbdev_id, &bbstats); 639 if (ret < 0) { 640 free(xstats); 641 free(xstats_names); 642 rte_exit(EXIT_FAILURE, 643 "ERROR(%d): Failure to get BBDEV %u statistics\n", 644 ret, bbdev_id); 645 } 646 647 printf("\nBBDEV STATISTICS:\n=================\n"); 648 printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n", 649 bbdev_id, stats_border, 650 bbstats.enqueued_count); 651 printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n", 652 bbdev_id, stats_border, 653 bbstats.dequeued_count); 654 printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n", 655 bbdev_id, stats_border, 656 bbstats.enqueue_err_count); 657 printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n", 658 bbdev_id, stats_border, 659 bbstats.dequeue_err_count); 660 661 printf("LCORE STATISTICS:\n=================\n"); 662 for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) { 663 if (stats_lcore->lconf[l_id].core_type == 0) 664 continue; 665 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id); 666 } 667 668 fflush(stdout); 669 670 free(xstats); 671 free(xstats_names); 672 } 673 674 static int 675 stats_loop(void *arg) 676 { 677 struct stats_lcore_params *stats_lcore = arg; 678 679 while (!rte_atomic_load_explicit(&global_exit_flag, rte_memory_order_relaxed)) { 680 print_stats(stats_lcore); 681 rte_delay_ms(500); 682 } 683 684 return 0; 685 } 686 687 static inline void 688 run_encoding(struct lcore_conf *lcore_conf) 689 { 690 uint16_t i; 691 uint16_t port_id, rx_queue_id; 692 uint16_t bbdev_id, enc_queue_id; 693 uint16_t nb_rx, nb_enq, nb_deq, nb_sent; 694 struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST]; 695 struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST]; 696 struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST]; 697 struct lcore_statistics *lcore_stats; 698 struct rte_mempool *bbdev_op_pool, *enc_out_pool; 699 struct rte_ring *enc_to_dec_ring; 700 const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN; 701 702 lcore_stats = lcore_conf->lcore_stats; 703 port_id = lcore_conf->port_id; 704 rx_queue_id = lcore_conf->rx_queue_id; 705 bbdev_id = lcore_conf->bbdev_id; 706 enc_queue_id = lcore_conf->enc_queue_id; 707 bbdev_op_pool = lcore_conf->bbdev_enc_op_pool; 708 enc_out_pool = lcore_conf->enc_out_pool; 709 enc_to_dec_ring = lcore_conf->enc_to_dec_ring; 710 711 /* Read packet from RX queues*/ 712 nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst, 713 MAX_PKT_BURST); 714 if (!nb_rx) 715 return; 716 717 if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts, 718 nb_rx) != 0)) { 719 pktmbuf_free_bulk(rx_pkts_burst, nb_rx); 720 lcore_stats->rx_lost_packets += nb_rx; 721 return; 722 } 723 724 if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst, 725 nb_rx) != 0)) { 726 pktmbuf_free_bulk(enc_out_pkts, nb_rx); 727 pktmbuf_free_bulk(rx_pkts_burst, nb_rx); 728 lcore_stats->rx_lost_packets += nb_rx; 729 return; 730 } 731 732 for (i = 0; i < nb_rx; i++) { 733 char *data; 734 const uint16_t pkt_data_len = 735 rte_pktmbuf_data_len(rx_pkts_burst[i]) - 736 sizeof(struct rte_ether_hdr); 737 /* save input mbuf pointer for later comparison */ 738 *mbuf_input(enc_out_pkts[i]) = rx_pkts_burst[i]; 739 740 /* copy ethernet header */ 741 rte_pktmbuf_reset(enc_out_pkts[i]); 742 data = rte_pktmbuf_append(enc_out_pkts[i], 743 sizeof(struct rte_ether_hdr)); 744 if (data == NULL) { 745 printf( 746 "Not enough space for ethernet header in encoder output mbuf\n"); 747 continue; 748 } 749 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]); 750 751 /* set op */ 752 bbdev_ops_burst[i]->turbo_enc = def_op_enc; 753 754 bbdev_ops_burst[i]->turbo_enc.input.data = 755 rx_pkts_burst[i]; 756 bbdev_ops_burst[i]->turbo_enc.input.offset = 757 sizeof(struct rte_ether_hdr); 758 /* Encoder will attach the CRC24B, adjust the length */ 759 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len; 760 761 if (in_data_len < pkt_data_len) 762 rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len - 763 in_data_len); 764 else if (in_data_len > pkt_data_len) { 765 data = rte_pktmbuf_append(rx_pkts_burst[i], 766 in_data_len - pkt_data_len); 767 if (data == NULL) 768 printf( 769 "Not enough storage in mbuf to perform the encoding\n"); 770 } 771 772 bbdev_ops_burst[i]->turbo_enc.output.data = 773 enc_out_pkts[i]; 774 bbdev_ops_burst[i]->turbo_enc.output.offset = 775 sizeof(struct rte_ether_hdr); 776 } 777 778 /* Enqueue packets on BBDevice */ 779 nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id, 780 bbdev_ops_burst, nb_rx); 781 if (unlikely(nb_enq < nb_rx)) { 782 pktmbuf_input_free_bulk(&enc_out_pkts[nb_enq], 783 nb_rx - nb_enq); 784 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq], 785 nb_rx - nb_enq); 786 lcore_stats->rx_lost_packets += nb_rx - nb_enq; 787 788 if (!nb_enq) 789 return; 790 } 791 792 lcore_stats->enqueued += nb_enq; 793 794 /* Dequeue packets from bbdev device*/ 795 nb_deq = 0; 796 do { 797 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id, 798 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq); 799 } while (unlikely(nb_deq < nb_enq)); 800 801 lcore_stats->dequeued += nb_deq; 802 803 /* Generate and add AWGN */ 804 add_awgn(enc_out_pkts, nb_deq); 805 806 rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq); 807 808 /* Enqueue packets to encoder-to-decoder ring */ 809 nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts, 810 nb_deq, NULL); 811 if (unlikely(nb_sent < nb_deq)) { 812 pktmbuf_input_free_bulk(&enc_out_pkts[nb_sent], 813 nb_deq - nb_sent); 814 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent; 815 } 816 } 817 818 static void 819 run_decoding(struct lcore_conf *lcore_conf) 820 { 821 uint16_t i; 822 uint16_t port_id, tx_queue_id; 823 uint16_t bbdev_id, bbdev_queue_id; 824 uint16_t nb_recv, nb_enq, nb_deq, nb_tx; 825 uint8_t *llr_temp_buf; 826 struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST]; 827 struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST]; 828 struct lcore_statistics *lcore_stats; 829 struct rte_mempool *bbdev_op_pool; 830 struct rte_ring *enc_to_dec_ring; 831 832 lcore_stats = lcore_conf->lcore_stats; 833 port_id = lcore_conf->port_id; 834 tx_queue_id = lcore_conf->tx_queue_id; 835 bbdev_id = lcore_conf->bbdev_id; 836 bbdev_queue_id = lcore_conf->dec_queue_id; 837 bbdev_op_pool = lcore_conf->bbdev_dec_op_pool; 838 enc_to_dec_ring = lcore_conf->enc_to_dec_ring; 839 llr_temp_buf = lcore_conf->llr_temp_buf; 840 841 /* Dequeue packets from the ring */ 842 nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring, 843 (void **)recv_pkts_burst, MAX_PKT_BURST, NULL); 844 if (!nb_recv) 845 return; 846 847 if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst, 848 nb_recv) != 0)) { 849 pktmbuf_input_free_bulk(recv_pkts_burst, nb_recv); 850 lcore_stats->rx_lost_packets += nb_recv; 851 return; 852 } 853 854 transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv, 855 def_op_dec.cb_params.k); 856 857 for (i = 0; i < nb_recv; i++) { 858 /* set op */ 859 bbdev_ops_burst[i]->turbo_dec = def_op_dec; 860 861 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i]; 862 bbdev_ops_burst[i]->turbo_dec.input.offset = 863 sizeof(struct rte_ether_hdr); 864 bbdev_ops_burst[i]->turbo_dec.input.length = 865 rte_pktmbuf_data_len(recv_pkts_burst[i]) 866 - sizeof(struct rte_ether_hdr); 867 868 bbdev_ops_burst[i]->turbo_dec.hard_output.data = 869 recv_pkts_burst[i]; 870 bbdev_ops_burst[i]->turbo_dec.hard_output.offset = 871 sizeof(struct rte_ether_hdr); 872 } 873 874 /* Enqueue packets on BBDevice */ 875 nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id, 876 bbdev_ops_burst, nb_recv); 877 if (unlikely(nb_enq < nb_recv)) { 878 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_enq], 879 nb_recv - nb_enq); 880 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq], 881 nb_recv - nb_enq); 882 lcore_stats->rx_lost_packets += nb_recv - nb_enq; 883 884 if (!nb_enq) 885 return; 886 } 887 888 lcore_stats->enqueued += nb_enq; 889 890 /* Dequeue packets from BBDevice */ 891 nb_deq = 0; 892 do { 893 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id, 894 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq); 895 } while (unlikely(nb_deq < nb_enq)); 896 897 lcore_stats->dequeued += nb_deq; 898 899 rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq); 900 901 verify_data(recv_pkts_burst, nb_deq); 902 903 /* Free the RX mbufs after verification */ 904 for (i = 0; i < nb_deq; ++i) 905 rte_pktmbuf_free(*mbuf_input(recv_pkts_burst[i])); 906 907 /* Transmit the packets */ 908 nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq); 909 if (unlikely(nb_tx < nb_deq)) { 910 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_tx], 911 nb_deq - nb_tx); 912 lcore_stats->tx_lost_packets += nb_deq - nb_tx; 913 } 914 } 915 916 static int 917 processing_loop(void *arg) 918 { 919 struct lcore_conf *lcore_conf = arg; 920 const bool run_encoder = (lcore_conf->core_type & 921 (1 << RTE_BBDEV_OP_TURBO_ENC)); 922 const bool run_decoder = (lcore_conf->core_type & 923 (1 << RTE_BBDEV_OP_TURBO_DEC)); 924 925 while (!rte_atomic_load_explicit(&global_exit_flag, rte_memory_order_relaxed)) { 926 if (run_encoder) 927 run_encoding(lcore_conf); 928 if (run_decoder) 929 run_decoding(lcore_conf); 930 } 931 932 return 0; 933 } 934 935 static int 936 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info, 937 struct app_config_params *app_params) 938 { 939 int ret; 940 unsigned int q_id, dec_q_id, enc_q_id; 941 struct rte_bbdev_queue_conf qconf = {0}; 942 uint16_t dec_qs_nb = app_params->num_dec_cores; 943 uint16_t enc_qs_nb = app_params->num_enc_cores; 944 uint16_t tot_qs = dec_qs_nb + enc_qs_nb; 945 946 ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id); 947 if (ret < 0) 948 rte_exit(EXIT_FAILURE, 949 "ERROR(%d): BBDEV %u not configured properly\n", 950 ret, dev_id); 951 952 /* setup device DEC queues */ 953 qconf.socket = info->socket_id; 954 qconf.queue_size = info->drv.queue_size_lim; 955 qconf.op_type = RTE_BBDEV_OP_TURBO_DEC; 956 957 for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) { 958 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf); 959 if (ret < 0) 960 rte_exit(EXIT_FAILURE, 961 "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n", 962 ret, dev_id, q_id); 963 app_params->dec_queue_ids[dec_q_id++] = q_id; 964 } 965 966 /* setup device ENC queues */ 967 qconf.op_type = RTE_BBDEV_OP_TURBO_ENC; 968 969 for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) { 970 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf); 971 if (ret < 0) 972 rte_exit(EXIT_FAILURE, 973 "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n", 974 ret, dev_id, q_id); 975 app_params->enc_queue_ids[enc_q_id++] = q_id; 976 } 977 978 ret = rte_bbdev_start(dev_id); 979 980 if (ret != 0) 981 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n", 982 ret, dev_id); 983 984 printf("BBdev %u started\n", dev_id); 985 986 return 0; 987 } 988 989 static inline bool 990 check_matching_capabilities(uint64_t mask, uint64_t required_mask) 991 { 992 return (mask & required_mask) == required_mask; 993 } 994 995 static void 996 enable_bbdev(struct app_config_params *app_params) 997 { 998 struct rte_bbdev_info dev_info; 999 const struct rte_bbdev_op_cap *op_cap; 1000 uint16_t bbdev_id = app_params->bbdev_id; 1001 bool encoder_capable = false; 1002 bool decoder_capable = false; 1003 1004 rte_bbdev_info_get(bbdev_id, &dev_info); 1005 op_cap = dev_info.drv.capabilities; 1006 1007 while (op_cap->type != RTE_BBDEV_OP_NONE) { 1008 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 1009 if (check_matching_capabilities( 1010 op_cap->cap.turbo_enc.capability_flags, 1011 def_op_enc.op_flags)) 1012 encoder_capable = true; 1013 } 1014 1015 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 1016 if (check_matching_capabilities( 1017 op_cap->cap.turbo_dec.capability_flags, 1018 def_op_dec.op_flags)) 1019 decoder_capable = true; 1020 } 1021 1022 op_cap++; 1023 } 1024 1025 if (encoder_capable == false) 1026 rte_exit(EXIT_FAILURE, 1027 "The specified BBDev %u doesn't have required encoder capabilities!\n", 1028 bbdev_id); 1029 if (decoder_capable == false) 1030 rte_exit(EXIT_FAILURE, 1031 "The specified BBDev %u doesn't have required decoder capabilities!\n", 1032 bbdev_id); 1033 1034 prepare_bbdev_device(bbdev_id, &dev_info, app_params); 1035 } 1036 1037 int 1038 main(int argc, char **argv) 1039 { 1040 int ret; 1041 unsigned int nb_bbdevs, flags, lcore_id; 1042 void *sigret; 1043 struct app_config_params app_params = def_app_config; 1044 struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool; 1045 struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_SIZE_MAX]; 1046 struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} }; 1047 struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} }; 1048 struct stats_lcore_params stats_lcore; 1049 struct rte_ring *enc_to_dec_ring; 1050 bool stats_thread_started = false; 1051 unsigned int main_lcore_id = rte_get_main_lcore(); 1052 1053 static const struct rte_mbuf_dynfield input_dynfield_desc = { 1054 .name = "example_bbdev_dynfield_input", 1055 .size = sizeof(struct rte_mbuf *), 1056 .align = alignof(struct rte_mbuf *), 1057 }; 1058 1059 rte_atomic_store_explicit(&global_exit_flag, 0, rte_memory_order_relaxed); 1060 1061 sigret = signal(SIGTERM, signal_handler); 1062 if (sigret == SIG_ERR) 1063 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM); 1064 1065 sigret = signal(SIGINT, signal_handler); 1066 if (sigret == SIG_ERR) 1067 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT); 1068 1069 ret = rte_eal_init(argc, argv); 1070 if (ret < 0) 1071 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); 1072 1073 argc -= ret; 1074 argv += ret; 1075 1076 /* parse application arguments (after the EAL ones) */ 1077 ret = bbdev_parse_args(argc, argv, &app_params); 1078 if (ret < 0) 1079 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n"); 1080 1081 /*create bbdev op pools*/ 1082 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] = 1083 rte_bbdev_op_pool_create("bbdev_op_pool_dec", 1084 RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id()); 1085 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] = 1086 rte_bbdev_op_pool_create("bbdev_op_pool_enc", 1087 RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id()); 1088 1089 if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) || 1090 (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL)) 1091 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n"); 1092 1093 /* Create encoder to decoder ring */ 1094 flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0; 1095 if (app_params.num_dec_cores == 1) 1096 flags |= RING_F_SC_DEQ; 1097 1098 enc_to_dec_ring = rte_ring_create("enc_to_dec_ring", 1099 rte_align32pow2(NB_MBUF), rte_socket_id(), flags); 1100 1101 /* Get the number of available bbdev devices */ 1102 nb_bbdevs = rte_bbdev_count(); 1103 if (nb_bbdevs <= app_params.bbdev_id) 1104 rte_exit(EXIT_FAILURE, 1105 "%u BBDevs detected, cannot use BBDev with ID %u!\n", 1106 nb_bbdevs, app_params.bbdev_id); 1107 printf("Number of bbdevs detected: %d\n", nb_bbdevs); 1108 1109 if (!rte_eth_dev_is_valid_port(app_params.port_id)) 1110 rte_exit(EXIT_FAILURE, 1111 "cannot use port with ID %u!\n", 1112 app_params.port_id); 1113 1114 /* create the mbuf mempool for ethdev pkts */ 1115 ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool", 1116 NB_MBUF, MEMPOOL_CACHE_SIZE, 0, 1117 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 1118 if (ethdev_mbuf_mempool == NULL) 1119 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n"); 1120 1121 /* create the mbuf mempool for encoder output */ 1122 bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool", 1123 NB_MBUF, MEMPOOL_CACHE_SIZE, 0, 1124 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 1125 if (bbdev_mbuf_mempool == NULL) 1126 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n"); 1127 1128 /* register mbuf field to store input pointer */ 1129 input_dynfield_offset = 1130 rte_mbuf_dynfield_register(&input_dynfield_desc); 1131 if (input_dynfield_offset < 0) 1132 rte_exit(EXIT_FAILURE, "Cannot register mbuf field\n"); 1133 1134 /* initialize ports */ 1135 ret = initialize_ports(&app_params, ethdev_mbuf_mempool); 1136 1137 /* Check if all requested lcores are available */ 1138 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) 1139 if (((1ULL << lcore_id) & app_params.enc_core_mask) || 1140 ((1ULL << lcore_id) & app_params.dec_core_mask)) 1141 if (!rte_lcore_is_enabled(lcore_id)) 1142 rte_exit(EXIT_FAILURE, 1143 "Requested lcore_id %u is not enabled!\n", 1144 lcore_id); 1145 1146 /* Start ethernet port */ 1147 ret = rte_eth_dev_start(app_params.port_id); 1148 if (ret < 0) 1149 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", 1150 ret, app_params.port_id); 1151 1152 ret = check_port_link_status(app_params.port_id); 1153 if (ret < 0) 1154 exit(EXIT_FAILURE); 1155 1156 /* start BBDevice and save BBDev queue IDs */ 1157 enable_bbdev(&app_params); 1158 1159 /* Initialize the port/queue configuration of each logical core */ 1160 lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools, 1161 bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats); 1162 1163 stats_lcore.app_params = &app_params; 1164 stats_lcore.lconf = lcore_conf; 1165 1166 RTE_LCORE_FOREACH_WORKER(lcore_id) { 1167 if (lcore_conf[lcore_id].core_type != 0) 1168 /* launch per-lcore processing loop on worker lcores */ 1169 rte_eal_remote_launch(processing_loop, 1170 &lcore_conf[lcore_id], lcore_id); 1171 else if (!stats_thread_started) { 1172 /* launch statistics printing loop */ 1173 rte_eal_remote_launch(stats_loop, &stats_lcore, 1174 lcore_id); 1175 stats_thread_started = true; 1176 } 1177 } 1178 1179 if (!stats_thread_started && 1180 lcore_conf[main_lcore_id].core_type != 0) 1181 rte_exit(EXIT_FAILURE, 1182 "Not enough lcores to run the statistics printing loop!"); 1183 else if (lcore_conf[main_lcore_id].core_type != 0) 1184 processing_loop(&lcore_conf[main_lcore_id]); 1185 else if (!stats_thread_started) 1186 stats_loop(&stats_lcore); 1187 1188 RTE_LCORE_FOREACH_WORKER(lcore_id) { 1189 ret |= rte_eal_wait_lcore(lcore_id); 1190 } 1191 1192 /* clean up the EAL */ 1193 rte_eal_cleanup(); 1194 1195 return ret; 1196 } 1197