1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <stdint.h> 9 #include <inttypes.h> 10 #include <sys/types.h> 11 #include <unistd.h> 12 #include <sys/queue.h> 13 #include <stdarg.h> 14 #include <ctype.h> 15 #include <errno.h> 16 #include <math.h> 17 #include <assert.h> 18 #include <getopt.h> 19 #include <signal.h> 20 21 #include <rte_common.h> 22 #include <rte_eal.h> 23 #include <rte_cycles.h> 24 #include <rte_ether.h> 25 #include <rte_ethdev.h> 26 #include <rte_ip.h> 27 #include <rte_lcore.h> 28 #include <rte_malloc.h> 29 #include <rte_mbuf.h> 30 #include <rte_mbuf_dyn.h> 31 #include <rte_memory.h> 32 #include <rte_mempool.h> 33 #include <rte_log.h> 34 #include <rte_bbdev.h> 35 #include <rte_bbdev_op.h> 36 37 /* LLR values - negative value for '1' bit */ 38 #define LLR_1_BIT 0x81 39 #define LLR_0_BIT 0x7F 40 41 #define MAX_PKT_BURST 32 42 #define NB_MBUF 8191 43 #define MEMPOOL_CACHE_SIZE 256 44 45 /* Hardcoded K value */ 46 #define K 40 47 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32)) 48 49 #define CRC_24B_LEN 3 50 51 /* Configurable number of RX/TX ring descriptors */ 52 #define RX_DESC_DEFAULT 128 53 #define TX_DESC_DEFAULT 512 54 55 #define BBDEV_ASSERT(a) do { \ 56 if (!(a)) { \ 57 usage(prgname); \ 58 return -1; \ 59 } \ 60 } while (0) 61 62 static int input_dynfield_offset = -1; 63 64 static inline struct rte_mbuf ** 65 mbuf_input(struct rte_mbuf *mbuf) 66 { 67 return RTE_MBUF_DYNFIELD(mbuf, 68 input_dynfield_offset, struct rte_mbuf **); 69 } 70 71 static const struct rte_eth_conf port_conf = { 72 .rxmode = { 73 .mq_mode = RTE_ETH_MQ_RX_NONE, 74 }, 75 .txmode = { 76 .mq_mode = RTE_ETH_MQ_TX_NONE, 77 }, 78 }; 79 80 struct rte_bbdev_op_turbo_enc def_op_enc = { 81 /* These values are arbitrarily put, and does not map to the real 82 * values for the data received from ethdev ports 83 */ 84 .rv_index = 0, 85 .code_block_mode = 1, 86 .cb_params = { 87 .k = K, 88 }, 89 .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH 90 }; 91 92 struct rte_bbdev_op_turbo_dec def_op_dec = { 93 /* These values are arbitrarily put, and does not map to the real 94 * values for the data received from ethdev ports 95 */ 96 .code_block_mode = 1, 97 .cb_params = { 98 .k = K, 99 }, 100 .rv_index = 0, 101 .iter_max = 8, 102 .iter_min = 4, 103 .ext_scale = 15, 104 .num_maps = 0, 105 .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN 106 }; 107 108 struct app_config_params { 109 /* Placeholders for app params */ 110 uint16_t port_id; 111 uint16_t bbdev_id; 112 uint64_t enc_core_mask; 113 uint64_t dec_core_mask; 114 115 /* Values filled during init time */ 116 uint16_t enc_queue_ids[RTE_MAX_LCORE]; 117 uint16_t dec_queue_ids[RTE_MAX_LCORE]; 118 uint16_t num_enc_cores; 119 uint16_t num_dec_cores; 120 }; 121 122 struct lcore_statistics { 123 unsigned int enqueued; 124 unsigned int dequeued; 125 unsigned int rx_lost_packets; 126 unsigned int enc_to_dec_lost_packets; 127 unsigned int tx_lost_packets; 128 } __rte_cache_aligned; 129 130 /** each lcore configuration */ 131 struct lcore_conf { 132 uint64_t core_type; 133 134 unsigned int port_id; 135 unsigned int rx_queue_id; 136 unsigned int tx_queue_id; 137 138 unsigned int bbdev_id; 139 unsigned int enc_queue_id; 140 unsigned int dec_queue_id; 141 142 uint8_t llr_temp_buf[NCB]; 143 144 struct rte_mempool *bbdev_dec_op_pool; 145 struct rte_mempool *bbdev_enc_op_pool; 146 struct rte_mempool *enc_out_pool; 147 struct rte_ring *enc_to_dec_ring; 148 149 struct lcore_statistics *lcore_stats; 150 } __rte_cache_aligned; 151 152 struct stats_lcore_params { 153 struct lcore_conf *lconf; 154 struct app_config_params *app_params; 155 }; 156 157 158 static const struct app_config_params def_app_config = { 159 .port_id = 0, 160 .bbdev_id = 0, 161 .enc_core_mask = 0x2, 162 .dec_core_mask = 0x4, 163 .num_enc_cores = 1, 164 .num_dec_cores = 1, 165 }; 166 167 static uint16_t global_exit_flag; 168 169 /* display usage */ 170 static inline void 171 usage(const char *prgname) 172 { 173 printf("%s [EAL options] " 174 " --\n" 175 " --enc_cores - number of encoding cores (default = 0x2)\n" 176 " --dec_cores - number of decoding cores (default = 0x4)\n" 177 " --port_id - Ethernet port ID (default = 0)\n" 178 " --bbdev_id - BBDev ID (default = 0)\n" 179 "\n", prgname); 180 } 181 182 /* parse core mask */ 183 static inline 184 uint16_t bbdev_parse_mask(const char *mask) 185 { 186 char *end = NULL; 187 unsigned long pm; 188 189 /* parse hexadecimal string */ 190 pm = strtoul(mask, &end, 16); 191 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0')) 192 return 0; 193 194 return pm; 195 } 196 197 /* parse core mask */ 198 static inline 199 uint16_t bbdev_parse_number(const char *mask) 200 { 201 char *end = NULL; 202 unsigned long pm; 203 204 /* parse hexadecimal string */ 205 pm = strtoul(mask, &end, 10); 206 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0')) 207 return 0; 208 209 return pm; 210 } 211 212 static int 213 bbdev_parse_args(int argc, char **argv, 214 struct app_config_params *app_params) 215 { 216 int optind = 0; 217 int opt; 218 int opt_indx = 0; 219 char *prgname = argv[0]; 220 221 static struct option lgopts[] = { 222 { "enc_core_mask", required_argument, 0, 'e' }, 223 { "dec_core_mask", required_argument, 0, 'd' }, 224 { "port_id", required_argument, 0, 'p' }, 225 { "bbdev_id", required_argument, 0, 'b' }, 226 { NULL, 0, 0, 0 } 227 }; 228 229 BBDEV_ASSERT(argc != 0); 230 BBDEV_ASSERT(argv != NULL); 231 BBDEV_ASSERT(app_params != NULL); 232 233 while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) != 234 EOF) { 235 switch (opt) { 236 case 'e': 237 app_params->enc_core_mask = 238 bbdev_parse_mask(optarg); 239 if (app_params->enc_core_mask == 0) { 240 usage(prgname); 241 return -1; 242 } 243 app_params->num_enc_cores = 244 rte_popcount32(app_params->enc_core_mask); 245 break; 246 247 case 'd': 248 app_params->dec_core_mask = 249 bbdev_parse_mask(optarg); 250 if (app_params->dec_core_mask == 0) { 251 usage(prgname); 252 return -1; 253 } 254 app_params->num_dec_cores = 255 rte_popcount32(app_params->dec_core_mask); 256 break; 257 258 case 'p': 259 app_params->port_id = bbdev_parse_number(optarg); 260 break; 261 262 case 'b': 263 app_params->bbdev_id = bbdev_parse_number(optarg); 264 break; 265 266 default: 267 usage(prgname); 268 return -1; 269 } 270 } 271 optind = 0; 272 return optind; 273 } 274 275 static void 276 signal_handler(int signum) 277 { 278 printf("\nSignal %d received\n", signum); 279 __atomic_store_n(&global_exit_flag, 1, __ATOMIC_RELAXED); 280 } 281 282 static void 283 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address) 284 { 285 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n", 286 (unsigned int) portid, 287 RTE_ETHER_ADDR_BYTES(bbdev_ports_eth_address)); 288 } 289 290 static inline void 291 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free) 292 { 293 unsigned int i; 294 for (i = 0; i < nb_to_free; ++i) 295 rte_pktmbuf_free(mbufs[i]); 296 } 297 298 static inline void 299 pktmbuf_input_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free) 300 { 301 unsigned int i; 302 for (i = 0; i < nb_to_free; ++i) { 303 struct rte_mbuf *rx_pkt = *mbuf_input(mbufs[i]); 304 rte_pktmbuf_free(rx_pkt); 305 rte_pktmbuf_free(mbufs[i]); 306 } 307 } 308 309 /* Check the link status of all ports in up to 9s, and print them finally */ 310 static int 311 check_port_link_status(uint16_t port_id) 312 { 313 #define CHECK_INTERVAL 100 /* 100ms */ 314 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 315 uint8_t count; 316 struct rte_eth_link link; 317 int link_get_err = -EINVAL; 318 319 printf("\nChecking link status."); 320 fflush(stdout); 321 322 for (count = 0; count <= MAX_CHECK_TIME && 323 !__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED); count++) { 324 memset(&link, 0, sizeof(link)); 325 link_get_err = rte_eth_link_get_nowait(port_id, &link); 326 327 if (link_get_err >= 0 && link.link_status) { 328 const char *dp = (link.link_duplex == 329 RTE_ETH_LINK_FULL_DUPLEX) ? 330 "full-duplex" : "half-duplex"; 331 printf("\nPort %u Link Up - speed %s - %s\n", 332 port_id, 333 rte_eth_link_speed_to_str(link.link_speed), 334 dp); 335 return 0; 336 } 337 printf("."); 338 fflush(stdout); 339 rte_delay_ms(CHECK_INTERVAL); 340 } 341 342 if (link_get_err >= 0) 343 printf("\nPort %d Link Down\n", port_id); 344 else 345 printf("\nGet link failed (port %d): %s\n", port_id, 346 rte_strerror(-link_get_err)); 347 348 return 0; 349 } 350 351 static inline void 352 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst) 353 { 354 struct rte_ether_hdr *eth_from; 355 struct rte_ether_hdr *eth_to; 356 357 eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *); 358 eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *); 359 360 /* copy header */ 361 rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr)); 362 } 363 364 static inline void 365 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts) 366 { 367 RTE_SET_USED(mbufs); 368 RTE_SET_USED(num_pkts); 369 } 370 371 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input 372 * so each bit of the encoder output must be translated into one byte of LLR. If 373 * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes 374 * must additionally be inserted at the end of each sub-block. 375 */ 376 static inline void 377 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf, 378 uint16_t num_pkts, uint16_t k) 379 { 380 uint16_t i, l, j; 381 uint16_t start_bit_idx; 382 uint16_t out_idx; 383 uint16_t d = k + 4; 384 uint16_t kpi = RTE_ALIGN_CEIL(d, 32); 385 uint16_t nd = kpi - d; 386 uint16_t ncb = 3 * kpi; 387 388 for (i = 0; i < num_pkts; ++i) { 389 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) - 390 sizeof(struct rte_ether_hdr); 391 392 /* Resize the packet if needed */ 393 if (pkt_data_len < ncb) { 394 char *data = rte_pktmbuf_append(mbufs[i], 395 ncb - pkt_data_len); 396 if (data == NULL) 397 printf( 398 "Not enough space in decoder input packet"); 399 } 400 401 /* Translate each bit into 1 LLR byte. */ 402 start_bit_idx = 0; 403 out_idx = 0; 404 for (j = 0; j < 3; ++j) { 405 for (l = start_bit_idx; l < start_bit_idx + d; ++l) { 406 uint8_t *data = rte_pktmbuf_mtod_offset( 407 mbufs[i], uint8_t *, 408 sizeof(struct rte_ether_hdr) + 409 (l >> 3)); 410 if (*data & (0x80 >> (l & 7))) 411 temp_buf[out_idx] = LLR_1_BIT; 412 else 413 temp_buf[out_idx] = LLR_0_BIT; 414 ++out_idx; 415 } 416 /* Padding bytes should be at the end of the sub-block. 417 */ 418 memset(&temp_buf[out_idx], 0, nd); 419 out_idx += nd; 420 start_bit_idx += d; 421 } 422 423 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *, 424 sizeof(struct rte_ether_hdr)), temp_buf, ncb); 425 } 426 } 427 428 static inline void 429 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts) 430 { 431 uint16_t i; 432 for (i = 0; i < num_pkts; ++i) { 433 struct rte_mbuf *out = mbufs[i]; 434 struct rte_mbuf *in = *mbuf_input(out); 435 436 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *, 437 sizeof(struct rte_ether_hdr)), 438 rte_pktmbuf_mtod_offset(out, uint8_t *, 439 sizeof(struct rte_ether_hdr)), 440 K / 8 - CRC_24B_LEN)) 441 printf("Input and output buffers are not equal!\n"); 442 } 443 } 444 445 static int 446 initialize_ports(struct app_config_params *app_params, 447 struct rte_mempool *ethdev_mbuf_mempool) 448 { 449 int ret; 450 uint16_t port_id = app_params->port_id; 451 uint16_t q; 452 /* ethernet addresses of ports */ 453 struct rte_ether_addr bbdev_port_eth_addr; 454 455 /* initialize ports */ 456 printf("\nInitializing port %u...\n", app_params->port_id); 457 ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores, 458 app_params->num_dec_cores, &port_conf); 459 460 if (ret < 0) { 461 printf("Cannot configure device: err=%d, port=%u\n", 462 ret, port_id); 463 return -1; 464 } 465 466 /* initialize RX queues for encoder */ 467 for (q = 0; q < app_params->num_enc_cores; q++) { 468 ret = rte_eth_rx_queue_setup(port_id, q, 469 RX_DESC_DEFAULT, 470 rte_eth_dev_socket_id(port_id), 471 NULL, ethdev_mbuf_mempool); 472 if (ret < 0) { 473 printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n", 474 ret, q); 475 return -1; 476 } 477 } 478 /* initialize TX queues for decoder */ 479 for (q = 0; q < app_params->num_dec_cores; q++) { 480 ret = rte_eth_tx_queue_setup(port_id, q, 481 TX_DESC_DEFAULT, 482 rte_eth_dev_socket_id(port_id), NULL); 483 if (ret < 0) { 484 printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n", 485 ret, q); 486 return -1; 487 } 488 } 489 490 ret = rte_eth_promiscuous_enable(port_id); 491 if (ret != 0) { 492 printf("Cannot enable promiscuous mode: err=%s, port=%u\n", 493 rte_strerror(-ret), port_id); 494 return ret; 495 } 496 497 ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr); 498 if (ret < 0) { 499 printf("rte_eth_macaddr_get: err=%d, queue=%u\n", 500 ret, q); 501 return -1; 502 } 503 504 print_mac(port_id, &bbdev_port_eth_addr); 505 506 return 0; 507 } 508 509 static void 510 lcore_conf_init(struct app_config_params *app_params, 511 struct lcore_conf *lcore_conf, 512 struct rte_mempool **bbdev_op_pools, 513 struct rte_mempool *bbdev_mbuf_mempool, 514 struct rte_ring *enc_to_dec_ring, 515 struct lcore_statistics *lcore_stats) 516 { 517 unsigned int lcore_id; 518 struct lcore_conf *lconf; 519 uint16_t rx_queue_id = 0; 520 uint16_t tx_queue_id = 0; 521 uint16_t enc_q_id = 0; 522 uint16_t dec_q_id = 0; 523 524 /* Configure lcores */ 525 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) { 526 lconf = &lcore_conf[lcore_id]; 527 lconf->core_type = 0; 528 529 if ((1ULL << lcore_id) & app_params->enc_core_mask) { 530 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC); 531 lconf->rx_queue_id = rx_queue_id++; 532 lconf->enc_queue_id = 533 app_params->enc_queue_ids[enc_q_id++]; 534 } 535 536 if ((1ULL << lcore_id) & app_params->dec_core_mask) { 537 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC); 538 lconf->tx_queue_id = tx_queue_id++; 539 lconf->dec_queue_id = 540 app_params->dec_queue_ids[dec_q_id++]; 541 } 542 543 lconf->bbdev_enc_op_pool = 544 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC]; 545 lconf->bbdev_dec_op_pool = 546 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC]; 547 lconf->bbdev_id = app_params->bbdev_id; 548 lconf->port_id = app_params->port_id; 549 lconf->enc_out_pool = bbdev_mbuf_mempool; 550 lconf->enc_to_dec_ring = enc_to_dec_ring; 551 lconf->lcore_stats = &lcore_stats[lcore_id]; 552 } 553 } 554 555 static void 556 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id) 557 { 558 static const char *stats_border = "_______"; 559 560 printf("\nLcore %d: %s enqueued count:\t\t%u\n", 561 lcore_id, stats_border, lstats->enqueued); 562 printf("Lcore %d: %s dequeued count:\t\t%u\n", 563 lcore_id, stats_border, lstats->dequeued); 564 printf("Lcore %d: %s RX lost packets count:\t\t%u\n", 565 lcore_id, stats_border, lstats->rx_lost_packets); 566 printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n", 567 lcore_id, stats_border, 568 lstats->enc_to_dec_lost_packets); 569 printf("Lcore %d: %s TX lost packets count:\t\t%u\n", 570 lcore_id, stats_border, lstats->tx_lost_packets); 571 } 572 573 static void 574 print_stats(struct stats_lcore_params *stats_lcore) 575 { 576 unsigned int l_id; 577 unsigned int bbdev_id = stats_lcore->app_params->bbdev_id; 578 unsigned int port_id = stats_lcore->app_params->port_id; 579 int len, ret, i; 580 581 struct rte_eth_xstat *xstats; 582 struct rte_eth_xstat_name *xstats_names; 583 struct rte_bbdev_stats bbstats; 584 static const char *stats_border = "_______"; 585 586 const char clr[] = { 27, '[', '2', 'J', '\0' }; 587 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 588 589 /* Clear screen and move to top left */ 590 printf("%s%s", clr, topLeft); 591 592 printf("PORT STATISTICS:\n================\n"); 593 len = rte_eth_xstats_get(port_id, NULL, 0); 594 if (len < 0) 595 rte_exit(EXIT_FAILURE, 596 "rte_eth_xstats_get(%u) failed: %d", port_id, 597 len); 598 599 xstats = calloc(len, sizeof(*xstats)); 600 if (xstats == NULL) 601 rte_exit(EXIT_FAILURE, 602 "Failed to calloc memory for xstats"); 603 604 ret = rte_eth_xstats_get(port_id, xstats, len); 605 if (ret < 0 || ret > len) { 606 free(xstats); 607 rte_exit(EXIT_FAILURE, 608 "rte_eth_xstats_get(%u) len%i failed: %d", 609 port_id, len, ret); 610 } 611 612 xstats_names = calloc(len, sizeof(*xstats_names)); 613 if (xstats_names == NULL) { 614 free(xstats); 615 rte_exit(EXIT_FAILURE, 616 "Failed to calloc memory for xstats_names"); 617 } 618 619 ret = rte_eth_xstats_get_names(port_id, xstats_names, len); 620 if (ret < 0 || ret > len) { 621 free(xstats); 622 free(xstats_names); 623 rte_exit(EXIT_FAILURE, 624 "rte_eth_xstats_get_names(%u) len%i failed: %d", 625 port_id, len, ret); 626 } 627 628 for (i = 0; i < len; i++) { 629 if (xstats[i].value > 0) 630 printf("Port %u: %s %s:\t\t%"PRIu64"\n", 631 port_id, stats_border, 632 xstats_names[i].name, 633 xstats[i].value); 634 } 635 636 ret = rte_bbdev_stats_get(bbdev_id, &bbstats); 637 if (ret < 0) { 638 free(xstats); 639 free(xstats_names); 640 rte_exit(EXIT_FAILURE, 641 "ERROR(%d): Failure to get BBDEV %u statistics\n", 642 ret, bbdev_id); 643 } 644 645 printf("\nBBDEV STATISTICS:\n=================\n"); 646 printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n", 647 bbdev_id, stats_border, 648 bbstats.enqueued_count); 649 printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n", 650 bbdev_id, stats_border, 651 bbstats.dequeued_count); 652 printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n", 653 bbdev_id, stats_border, 654 bbstats.enqueue_err_count); 655 printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n", 656 bbdev_id, stats_border, 657 bbstats.dequeue_err_count); 658 659 printf("LCORE STATISTICS:\n=================\n"); 660 for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) { 661 if (stats_lcore->lconf[l_id].core_type == 0) 662 continue; 663 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id); 664 } 665 666 fflush(stdout); 667 668 free(xstats); 669 free(xstats_names); 670 } 671 672 static int 673 stats_loop(void *arg) 674 { 675 struct stats_lcore_params *stats_lcore = arg; 676 677 while (!__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED)) { 678 print_stats(stats_lcore); 679 rte_delay_ms(500); 680 } 681 682 return 0; 683 } 684 685 static inline void 686 run_encoding(struct lcore_conf *lcore_conf) 687 { 688 uint16_t i; 689 uint16_t port_id, rx_queue_id; 690 uint16_t bbdev_id, enc_queue_id; 691 uint16_t nb_rx, nb_enq, nb_deq, nb_sent; 692 struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST]; 693 struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST]; 694 struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST]; 695 struct lcore_statistics *lcore_stats; 696 struct rte_mempool *bbdev_op_pool, *enc_out_pool; 697 struct rte_ring *enc_to_dec_ring; 698 const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN; 699 700 lcore_stats = lcore_conf->lcore_stats; 701 port_id = lcore_conf->port_id; 702 rx_queue_id = lcore_conf->rx_queue_id; 703 bbdev_id = lcore_conf->bbdev_id; 704 enc_queue_id = lcore_conf->enc_queue_id; 705 bbdev_op_pool = lcore_conf->bbdev_enc_op_pool; 706 enc_out_pool = lcore_conf->enc_out_pool; 707 enc_to_dec_ring = lcore_conf->enc_to_dec_ring; 708 709 /* Read packet from RX queues*/ 710 nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst, 711 MAX_PKT_BURST); 712 if (!nb_rx) 713 return; 714 715 if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts, 716 nb_rx) != 0)) { 717 pktmbuf_free_bulk(rx_pkts_burst, nb_rx); 718 lcore_stats->rx_lost_packets += nb_rx; 719 return; 720 } 721 722 if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst, 723 nb_rx) != 0)) { 724 pktmbuf_free_bulk(enc_out_pkts, nb_rx); 725 pktmbuf_free_bulk(rx_pkts_burst, nb_rx); 726 lcore_stats->rx_lost_packets += nb_rx; 727 return; 728 } 729 730 for (i = 0; i < nb_rx; i++) { 731 char *data; 732 const uint16_t pkt_data_len = 733 rte_pktmbuf_data_len(rx_pkts_burst[i]) - 734 sizeof(struct rte_ether_hdr); 735 /* save input mbuf pointer for later comparison */ 736 *mbuf_input(enc_out_pkts[i]) = rx_pkts_burst[i]; 737 738 /* copy ethernet header */ 739 rte_pktmbuf_reset(enc_out_pkts[i]); 740 data = rte_pktmbuf_append(enc_out_pkts[i], 741 sizeof(struct rte_ether_hdr)); 742 if (data == NULL) { 743 printf( 744 "Not enough space for ethernet header in encoder output mbuf\n"); 745 continue; 746 } 747 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]); 748 749 /* set op */ 750 bbdev_ops_burst[i]->turbo_enc = def_op_enc; 751 752 bbdev_ops_burst[i]->turbo_enc.input.data = 753 rx_pkts_burst[i]; 754 bbdev_ops_burst[i]->turbo_enc.input.offset = 755 sizeof(struct rte_ether_hdr); 756 /* Encoder will attach the CRC24B, adjust the length */ 757 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len; 758 759 if (in_data_len < pkt_data_len) 760 rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len - 761 in_data_len); 762 else if (in_data_len > pkt_data_len) { 763 data = rte_pktmbuf_append(rx_pkts_burst[i], 764 in_data_len - pkt_data_len); 765 if (data == NULL) 766 printf( 767 "Not enough storage in mbuf to perform the encoding\n"); 768 } 769 770 bbdev_ops_burst[i]->turbo_enc.output.data = 771 enc_out_pkts[i]; 772 bbdev_ops_burst[i]->turbo_enc.output.offset = 773 sizeof(struct rte_ether_hdr); 774 } 775 776 /* Enqueue packets on BBDevice */ 777 nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id, 778 bbdev_ops_burst, nb_rx); 779 if (unlikely(nb_enq < nb_rx)) { 780 pktmbuf_input_free_bulk(&enc_out_pkts[nb_enq], 781 nb_rx - nb_enq); 782 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq], 783 nb_rx - nb_enq); 784 lcore_stats->rx_lost_packets += nb_rx - nb_enq; 785 786 if (!nb_enq) 787 return; 788 } 789 790 lcore_stats->enqueued += nb_enq; 791 792 /* Dequeue packets from bbdev device*/ 793 nb_deq = 0; 794 do { 795 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id, 796 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq); 797 } while (unlikely(nb_deq < nb_enq)); 798 799 lcore_stats->dequeued += nb_deq; 800 801 /* Generate and add AWGN */ 802 add_awgn(enc_out_pkts, nb_deq); 803 804 rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq); 805 806 /* Enqueue packets to encoder-to-decoder ring */ 807 nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts, 808 nb_deq, NULL); 809 if (unlikely(nb_sent < nb_deq)) { 810 pktmbuf_input_free_bulk(&enc_out_pkts[nb_sent], 811 nb_deq - nb_sent); 812 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent; 813 } 814 } 815 816 static void 817 run_decoding(struct lcore_conf *lcore_conf) 818 { 819 uint16_t i; 820 uint16_t port_id, tx_queue_id; 821 uint16_t bbdev_id, bbdev_queue_id; 822 uint16_t nb_recv, nb_enq, nb_deq, nb_tx; 823 uint8_t *llr_temp_buf; 824 struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST]; 825 struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST]; 826 struct lcore_statistics *lcore_stats; 827 struct rte_mempool *bbdev_op_pool; 828 struct rte_ring *enc_to_dec_ring; 829 830 lcore_stats = lcore_conf->lcore_stats; 831 port_id = lcore_conf->port_id; 832 tx_queue_id = lcore_conf->tx_queue_id; 833 bbdev_id = lcore_conf->bbdev_id; 834 bbdev_queue_id = lcore_conf->dec_queue_id; 835 bbdev_op_pool = lcore_conf->bbdev_dec_op_pool; 836 enc_to_dec_ring = lcore_conf->enc_to_dec_ring; 837 llr_temp_buf = lcore_conf->llr_temp_buf; 838 839 /* Dequeue packets from the ring */ 840 nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring, 841 (void **)recv_pkts_burst, MAX_PKT_BURST, NULL); 842 if (!nb_recv) 843 return; 844 845 if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst, 846 nb_recv) != 0)) { 847 pktmbuf_input_free_bulk(recv_pkts_burst, nb_recv); 848 lcore_stats->rx_lost_packets += nb_recv; 849 return; 850 } 851 852 transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv, 853 def_op_dec.cb_params.k); 854 855 for (i = 0; i < nb_recv; i++) { 856 /* set op */ 857 bbdev_ops_burst[i]->turbo_dec = def_op_dec; 858 859 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i]; 860 bbdev_ops_burst[i]->turbo_dec.input.offset = 861 sizeof(struct rte_ether_hdr); 862 bbdev_ops_burst[i]->turbo_dec.input.length = 863 rte_pktmbuf_data_len(recv_pkts_burst[i]) 864 - sizeof(struct rte_ether_hdr); 865 866 bbdev_ops_burst[i]->turbo_dec.hard_output.data = 867 recv_pkts_burst[i]; 868 bbdev_ops_burst[i]->turbo_dec.hard_output.offset = 869 sizeof(struct rte_ether_hdr); 870 } 871 872 /* Enqueue packets on BBDevice */ 873 nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id, 874 bbdev_ops_burst, nb_recv); 875 if (unlikely(nb_enq < nb_recv)) { 876 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_enq], 877 nb_recv - nb_enq); 878 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq], 879 nb_recv - nb_enq); 880 lcore_stats->rx_lost_packets += nb_recv - nb_enq; 881 882 if (!nb_enq) 883 return; 884 } 885 886 lcore_stats->enqueued += nb_enq; 887 888 /* Dequeue packets from BBDevice */ 889 nb_deq = 0; 890 do { 891 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id, 892 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq); 893 } while (unlikely(nb_deq < nb_enq)); 894 895 lcore_stats->dequeued += nb_deq; 896 897 rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq); 898 899 verify_data(recv_pkts_burst, nb_deq); 900 901 /* Free the RX mbufs after verification */ 902 for (i = 0; i < nb_deq; ++i) 903 rte_pktmbuf_free(*mbuf_input(recv_pkts_burst[i])); 904 905 /* Transmit the packets */ 906 nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq); 907 if (unlikely(nb_tx < nb_deq)) { 908 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_tx], 909 nb_deq - nb_tx); 910 lcore_stats->tx_lost_packets += nb_deq - nb_tx; 911 } 912 } 913 914 static int 915 processing_loop(void *arg) 916 { 917 struct lcore_conf *lcore_conf = arg; 918 const bool run_encoder = (lcore_conf->core_type & 919 (1 << RTE_BBDEV_OP_TURBO_ENC)); 920 const bool run_decoder = (lcore_conf->core_type & 921 (1 << RTE_BBDEV_OP_TURBO_DEC)); 922 923 while (!__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED)) { 924 if (run_encoder) 925 run_encoding(lcore_conf); 926 if (run_decoder) 927 run_decoding(lcore_conf); 928 } 929 930 return 0; 931 } 932 933 static int 934 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info, 935 struct app_config_params *app_params) 936 { 937 int ret; 938 unsigned int q_id, dec_q_id, enc_q_id; 939 struct rte_bbdev_queue_conf qconf = {0}; 940 uint16_t dec_qs_nb = app_params->num_dec_cores; 941 uint16_t enc_qs_nb = app_params->num_enc_cores; 942 uint16_t tot_qs = dec_qs_nb + enc_qs_nb; 943 944 ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id); 945 if (ret < 0) 946 rte_exit(EXIT_FAILURE, 947 "ERROR(%d): BBDEV %u not configured properly\n", 948 ret, dev_id); 949 950 /* setup device DEC queues */ 951 qconf.socket = info->socket_id; 952 qconf.queue_size = info->drv.queue_size_lim; 953 qconf.op_type = RTE_BBDEV_OP_TURBO_DEC; 954 955 for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) { 956 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf); 957 if (ret < 0) 958 rte_exit(EXIT_FAILURE, 959 "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n", 960 ret, dev_id, q_id); 961 app_params->dec_queue_ids[dec_q_id++] = q_id; 962 } 963 964 /* setup device ENC queues */ 965 qconf.op_type = RTE_BBDEV_OP_TURBO_ENC; 966 967 for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) { 968 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf); 969 if (ret < 0) 970 rte_exit(EXIT_FAILURE, 971 "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n", 972 ret, dev_id, q_id); 973 app_params->enc_queue_ids[enc_q_id++] = q_id; 974 } 975 976 ret = rte_bbdev_start(dev_id); 977 978 if (ret != 0) 979 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n", 980 ret, dev_id); 981 982 printf("BBdev %u started\n", dev_id); 983 984 return 0; 985 } 986 987 static inline bool 988 check_matching_capabilities(uint64_t mask, uint64_t required_mask) 989 { 990 return (mask & required_mask) == required_mask; 991 } 992 993 static void 994 enable_bbdev(struct app_config_params *app_params) 995 { 996 struct rte_bbdev_info dev_info; 997 const struct rte_bbdev_op_cap *op_cap; 998 uint16_t bbdev_id = app_params->bbdev_id; 999 bool encoder_capable = false; 1000 bool decoder_capable = false; 1001 1002 rte_bbdev_info_get(bbdev_id, &dev_info); 1003 op_cap = dev_info.drv.capabilities; 1004 1005 while (op_cap->type != RTE_BBDEV_OP_NONE) { 1006 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 1007 if (check_matching_capabilities( 1008 op_cap->cap.turbo_enc.capability_flags, 1009 def_op_enc.op_flags)) 1010 encoder_capable = true; 1011 } 1012 1013 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 1014 if (check_matching_capabilities( 1015 op_cap->cap.turbo_dec.capability_flags, 1016 def_op_dec.op_flags)) 1017 decoder_capable = true; 1018 } 1019 1020 op_cap++; 1021 } 1022 1023 if (encoder_capable == false) 1024 rte_exit(EXIT_FAILURE, 1025 "The specified BBDev %u doesn't have required encoder capabilities!\n", 1026 bbdev_id); 1027 if (decoder_capable == false) 1028 rte_exit(EXIT_FAILURE, 1029 "The specified BBDev %u doesn't have required decoder capabilities!\n", 1030 bbdev_id); 1031 1032 prepare_bbdev_device(bbdev_id, &dev_info, app_params); 1033 } 1034 1035 int 1036 main(int argc, char **argv) 1037 { 1038 int ret; 1039 unsigned int nb_bbdevs, flags, lcore_id; 1040 void *sigret; 1041 struct app_config_params app_params = def_app_config; 1042 struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool; 1043 struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_SIZE_MAX]; 1044 struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} }; 1045 struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} }; 1046 struct stats_lcore_params stats_lcore; 1047 struct rte_ring *enc_to_dec_ring; 1048 bool stats_thread_started = false; 1049 unsigned int main_lcore_id = rte_get_main_lcore(); 1050 1051 static const struct rte_mbuf_dynfield input_dynfield_desc = { 1052 .name = "example_bbdev_dynfield_input", 1053 .size = sizeof(struct rte_mbuf *), 1054 .align = __alignof__(struct rte_mbuf *), 1055 }; 1056 1057 __atomic_store_n(&global_exit_flag, 0, __ATOMIC_RELAXED); 1058 1059 sigret = signal(SIGTERM, signal_handler); 1060 if (sigret == SIG_ERR) 1061 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM); 1062 1063 sigret = signal(SIGINT, signal_handler); 1064 if (sigret == SIG_ERR) 1065 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT); 1066 1067 ret = rte_eal_init(argc, argv); 1068 if (ret < 0) 1069 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); 1070 1071 argc -= ret; 1072 argv += ret; 1073 1074 /* parse application arguments (after the EAL ones) */ 1075 ret = bbdev_parse_args(argc, argv, &app_params); 1076 if (ret < 0) 1077 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n"); 1078 1079 /*create bbdev op pools*/ 1080 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] = 1081 rte_bbdev_op_pool_create("bbdev_op_pool_dec", 1082 RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id()); 1083 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] = 1084 rte_bbdev_op_pool_create("bbdev_op_pool_enc", 1085 RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id()); 1086 1087 if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) || 1088 (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL)) 1089 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n"); 1090 1091 /* Create encoder to decoder ring */ 1092 flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0; 1093 if (app_params.num_dec_cores == 1) 1094 flags |= RING_F_SC_DEQ; 1095 1096 enc_to_dec_ring = rte_ring_create("enc_to_dec_ring", 1097 rte_align32pow2(NB_MBUF), rte_socket_id(), flags); 1098 1099 /* Get the number of available bbdev devices */ 1100 nb_bbdevs = rte_bbdev_count(); 1101 if (nb_bbdevs <= app_params.bbdev_id) 1102 rte_exit(EXIT_FAILURE, 1103 "%u BBDevs detected, cannot use BBDev with ID %u!\n", 1104 nb_bbdevs, app_params.bbdev_id); 1105 printf("Number of bbdevs detected: %d\n", nb_bbdevs); 1106 1107 if (!rte_eth_dev_is_valid_port(app_params.port_id)) 1108 rte_exit(EXIT_FAILURE, 1109 "cannot use port with ID %u!\n", 1110 app_params.port_id); 1111 1112 /* create the mbuf mempool for ethdev pkts */ 1113 ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool", 1114 NB_MBUF, MEMPOOL_CACHE_SIZE, 0, 1115 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 1116 if (ethdev_mbuf_mempool == NULL) 1117 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n"); 1118 1119 /* create the mbuf mempool for encoder output */ 1120 bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool", 1121 NB_MBUF, MEMPOOL_CACHE_SIZE, 0, 1122 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 1123 if (bbdev_mbuf_mempool == NULL) 1124 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n"); 1125 1126 /* register mbuf field to store input pointer */ 1127 input_dynfield_offset = 1128 rte_mbuf_dynfield_register(&input_dynfield_desc); 1129 if (input_dynfield_offset < 0) 1130 rte_exit(EXIT_FAILURE, "Cannot register mbuf field\n"); 1131 1132 /* initialize ports */ 1133 ret = initialize_ports(&app_params, ethdev_mbuf_mempool); 1134 1135 /* Check if all requested lcores are available */ 1136 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) 1137 if (((1ULL << lcore_id) & app_params.enc_core_mask) || 1138 ((1ULL << lcore_id) & app_params.dec_core_mask)) 1139 if (!rte_lcore_is_enabled(lcore_id)) 1140 rte_exit(EXIT_FAILURE, 1141 "Requested lcore_id %u is not enabled!\n", 1142 lcore_id); 1143 1144 /* Start ethernet port */ 1145 ret = rte_eth_dev_start(app_params.port_id); 1146 if (ret < 0) 1147 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", 1148 ret, app_params.port_id); 1149 1150 ret = check_port_link_status(app_params.port_id); 1151 if (ret < 0) 1152 exit(EXIT_FAILURE); 1153 1154 /* start BBDevice and save BBDev queue IDs */ 1155 enable_bbdev(&app_params); 1156 1157 /* Initialize the port/queue configuration of each logical core */ 1158 lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools, 1159 bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats); 1160 1161 stats_lcore.app_params = &app_params; 1162 stats_lcore.lconf = lcore_conf; 1163 1164 RTE_LCORE_FOREACH_WORKER(lcore_id) { 1165 if (lcore_conf[lcore_id].core_type != 0) 1166 /* launch per-lcore processing loop on worker lcores */ 1167 rte_eal_remote_launch(processing_loop, 1168 &lcore_conf[lcore_id], lcore_id); 1169 else if (!stats_thread_started) { 1170 /* launch statistics printing loop */ 1171 rte_eal_remote_launch(stats_loop, &stats_lcore, 1172 lcore_id); 1173 stats_thread_started = true; 1174 } 1175 } 1176 1177 if (!stats_thread_started && 1178 lcore_conf[main_lcore_id].core_type != 0) 1179 rte_exit(EXIT_FAILURE, 1180 "Not enough lcores to run the statistics printing loop!"); 1181 else if (lcore_conf[main_lcore_id].core_type != 0) 1182 processing_loop(&lcore_conf[main_lcore_id]); 1183 else if (!stats_thread_started) 1184 stats_loop(&stats_lcore); 1185 1186 RTE_LCORE_FOREACH_WORKER(lcore_id) { 1187 ret |= rte_eal_wait_lcore(lcore_id); 1188 } 1189 1190 /* clean up the EAL */ 1191 rte_eal_cleanup(); 1192 1193 return ret; 1194 } 1195