1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <stdint.h> 9 #include <inttypes.h> 10 #include <sys/types.h> 11 #include <unistd.h> 12 #include <sys/queue.h> 13 #include <stdarg.h> 14 #include <ctype.h> 15 #include <errno.h> 16 #include <math.h> 17 #include <assert.h> 18 #include <getopt.h> 19 #include <signal.h> 20 21 #include <rte_atomic.h> 22 #include <rte_common.h> 23 #include <rte_eal.h> 24 #include <rte_cycles.h> 25 #include <rte_ether.h> 26 #include <rte_ethdev.h> 27 #include <rte_ip.h> 28 #include <rte_lcore.h> 29 #include <rte_malloc.h> 30 #include <rte_mbuf.h> 31 #include <rte_mbuf_dyn.h> 32 #include <rte_memory.h> 33 #include <rte_mempool.h> 34 #include <rte_log.h> 35 #include <rte_bbdev.h> 36 #include <rte_bbdev_op.h> 37 38 /* LLR values - negative value for '1' bit */ 39 #define LLR_1_BIT 0x81 40 #define LLR_0_BIT 0x7F 41 42 #define MAX_PKT_BURST 32 43 #define NB_MBUF 8191 44 #define MEMPOOL_CACHE_SIZE 256 45 46 /* Hardcoded K value */ 47 #define K 40 48 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32)) 49 50 #define CRC_24B_LEN 3 51 52 /* Configurable number of RX/TX ring descriptors */ 53 #define RTE_TEST_RX_DESC_DEFAULT 128 54 #define RTE_TEST_TX_DESC_DEFAULT 512 55 56 #define BBDEV_ASSERT(a) do { \ 57 if (!(a)) { \ 58 usage(prgname); \ 59 return -1; \ 60 } \ 61 } while (0) 62 63 static int input_dynfield_offset = -1; 64 65 static inline struct rte_mbuf ** 66 mbuf_input(struct rte_mbuf *mbuf) 67 { 68 return RTE_MBUF_DYNFIELD(mbuf, 69 input_dynfield_offset, struct rte_mbuf **); 70 } 71 72 static const struct rte_eth_conf port_conf = { 73 .rxmode = { 74 .mq_mode = ETH_MQ_RX_NONE, 75 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 76 .split_hdr_size = 0, 77 }, 78 .txmode = { 79 .mq_mode = ETH_MQ_TX_NONE, 80 }, 81 }; 82 83 struct rte_bbdev_op_turbo_enc def_op_enc = { 84 /* These values are arbitrarily put, and does not map to the real 85 * values for the data received from ethdev ports 86 */ 87 .rv_index = 0, 88 .code_block_mode = 1, 89 .cb_params = { 90 .k = K, 91 }, 92 .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH 93 }; 94 95 struct rte_bbdev_op_turbo_dec def_op_dec = { 96 /* These values are arbitrarily put, and does not map to the real 97 * values for the data received from ethdev ports 98 */ 99 .code_block_mode = 1, 100 .cb_params = { 101 .k = K, 102 }, 103 .rv_index = 0, 104 .iter_max = 8, 105 .iter_min = 4, 106 .ext_scale = 15, 107 .num_maps = 0, 108 .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN 109 }; 110 111 struct app_config_params { 112 /* Placeholders for app params */ 113 uint16_t port_id; 114 uint16_t bbdev_id; 115 uint64_t enc_core_mask; 116 uint64_t dec_core_mask; 117 118 /* Values filled during init time */ 119 uint16_t enc_queue_ids[RTE_MAX_LCORE]; 120 uint16_t dec_queue_ids[RTE_MAX_LCORE]; 121 uint16_t num_enc_cores; 122 uint16_t num_dec_cores; 123 }; 124 125 struct lcore_statistics { 126 unsigned int enqueued; 127 unsigned int dequeued; 128 unsigned int rx_lost_packets; 129 unsigned int enc_to_dec_lost_packets; 130 unsigned int tx_lost_packets; 131 } __rte_cache_aligned; 132 133 /** each lcore configuration */ 134 struct lcore_conf { 135 uint64_t core_type; 136 137 unsigned int port_id; 138 unsigned int rx_queue_id; 139 unsigned int tx_queue_id; 140 141 unsigned int bbdev_id; 142 unsigned int enc_queue_id; 143 unsigned int dec_queue_id; 144 145 uint8_t llr_temp_buf[NCB]; 146 147 struct rte_mempool *bbdev_dec_op_pool; 148 struct rte_mempool *bbdev_enc_op_pool; 149 struct rte_mempool *enc_out_pool; 150 struct rte_ring *enc_to_dec_ring; 151 152 struct lcore_statistics *lcore_stats; 153 } __rte_cache_aligned; 154 155 struct stats_lcore_params { 156 struct lcore_conf *lconf; 157 struct app_config_params *app_params; 158 }; 159 160 161 static const struct app_config_params def_app_config = { 162 .port_id = 0, 163 .bbdev_id = 0, 164 .enc_core_mask = 0x2, 165 .dec_core_mask = 0x4, 166 .num_enc_cores = 1, 167 .num_dec_cores = 1, 168 }; 169 170 static rte_atomic16_t global_exit_flag; 171 172 /* display usage */ 173 static inline void 174 usage(const char *prgname) 175 { 176 printf("%s [EAL options] " 177 " --\n" 178 " --enc_cores - number of encoding cores (default = 0x2)\n" 179 " --dec_cores - number of decoding cores (default = 0x4)\n" 180 " --port_id - Ethernet port ID (default = 0)\n" 181 " --bbdev_id - BBDev ID (default = 0)\n" 182 "\n", prgname); 183 } 184 185 /* parse core mask */ 186 static inline 187 uint16_t bbdev_parse_mask(const char *mask) 188 { 189 char *end = NULL; 190 unsigned long pm; 191 192 /* parse hexadecimal string */ 193 pm = strtoul(mask, &end, 16); 194 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0')) 195 return 0; 196 197 return pm; 198 } 199 200 /* parse core mask */ 201 static inline 202 uint16_t bbdev_parse_number(const char *mask) 203 { 204 char *end = NULL; 205 unsigned long pm; 206 207 /* parse hexadecimal string */ 208 pm = strtoul(mask, &end, 10); 209 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0')) 210 return 0; 211 212 return pm; 213 } 214 215 static int 216 bbdev_parse_args(int argc, char **argv, 217 struct app_config_params *app_params) 218 { 219 int optind = 0; 220 int opt; 221 int opt_indx = 0; 222 char *prgname = argv[0]; 223 224 static struct option lgopts[] = { 225 { "enc_core_mask", required_argument, 0, 'e' }, 226 { "dec_core_mask", required_argument, 0, 'd' }, 227 { "port_id", required_argument, 0, 'p' }, 228 { "bbdev_id", required_argument, 0, 'b' }, 229 { NULL, 0, 0, 0 } 230 }; 231 232 BBDEV_ASSERT(argc != 0); 233 BBDEV_ASSERT(argv != NULL); 234 BBDEV_ASSERT(app_params != NULL); 235 236 while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) != 237 EOF) { 238 switch (opt) { 239 case 'e': 240 app_params->enc_core_mask = 241 bbdev_parse_mask(optarg); 242 if (app_params->enc_core_mask == 0) { 243 usage(prgname); 244 return -1; 245 } 246 app_params->num_enc_cores = 247 __builtin_popcount(app_params->enc_core_mask); 248 break; 249 250 case 'd': 251 app_params->dec_core_mask = 252 bbdev_parse_mask(optarg); 253 if (app_params->dec_core_mask == 0) { 254 usage(prgname); 255 return -1; 256 } 257 app_params->num_dec_cores = 258 __builtin_popcount(app_params->dec_core_mask); 259 break; 260 261 case 'p': 262 app_params->port_id = bbdev_parse_number(optarg); 263 break; 264 265 case 'b': 266 app_params->bbdev_id = bbdev_parse_number(optarg); 267 break; 268 269 default: 270 usage(prgname); 271 return -1; 272 } 273 } 274 optind = 0; 275 return optind; 276 } 277 278 static void 279 signal_handler(int signum) 280 { 281 printf("\nSignal %d received\n", signum); 282 rte_atomic16_set(&global_exit_flag, 1); 283 } 284 285 static void 286 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address) 287 { 288 printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n", 289 (unsigned int) portid, 290 bbdev_ports_eth_address->addr_bytes[0], 291 bbdev_ports_eth_address->addr_bytes[1], 292 bbdev_ports_eth_address->addr_bytes[2], 293 bbdev_ports_eth_address->addr_bytes[3], 294 bbdev_ports_eth_address->addr_bytes[4], 295 bbdev_ports_eth_address->addr_bytes[5]); 296 } 297 298 static inline void 299 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free) 300 { 301 unsigned int i; 302 for (i = 0; i < nb_to_free; ++i) 303 rte_pktmbuf_free(mbufs[i]); 304 } 305 306 static inline void 307 pktmbuf_input_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free) 308 { 309 unsigned int i; 310 for (i = 0; i < nb_to_free; ++i) { 311 struct rte_mbuf *rx_pkt = *mbuf_input(mbufs[i]); 312 rte_pktmbuf_free(rx_pkt); 313 rte_pktmbuf_free(mbufs[i]); 314 } 315 } 316 317 /* Check the link status of all ports in up to 9s, and print them finally */ 318 static int 319 check_port_link_status(uint16_t port_id) 320 { 321 #define CHECK_INTERVAL 100 /* 100ms */ 322 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 323 uint8_t count; 324 struct rte_eth_link link; 325 int link_get_err = -EINVAL; 326 327 printf("\nChecking link status."); 328 fflush(stdout); 329 330 for (count = 0; count <= MAX_CHECK_TIME && 331 !rte_atomic16_read(&global_exit_flag); count++) { 332 memset(&link, 0, sizeof(link)); 333 link_get_err = rte_eth_link_get_nowait(port_id, &link); 334 335 if (link_get_err >= 0 && link.link_status) { 336 const char *dp = (link.link_duplex == 337 ETH_LINK_FULL_DUPLEX) ? 338 "full-duplex" : "half-duplex"; 339 printf("\nPort %u Link Up - speed %s - %s\n", 340 port_id, 341 rte_eth_link_speed_to_str(link.link_speed), 342 dp); 343 return 0; 344 } 345 printf("."); 346 fflush(stdout); 347 rte_delay_ms(CHECK_INTERVAL); 348 } 349 350 if (link_get_err >= 0) 351 printf("\nPort %d Link Down\n", port_id); 352 else 353 printf("\nGet link failed (port %d): %s\n", port_id, 354 rte_strerror(-link_get_err)); 355 356 return 0; 357 } 358 359 static inline void 360 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst) 361 { 362 struct rte_ether_hdr *eth_from; 363 struct rte_ether_hdr *eth_to; 364 365 eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *); 366 eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *); 367 368 /* copy header */ 369 rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr)); 370 } 371 372 static inline void 373 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts) 374 { 375 RTE_SET_USED(mbufs); 376 RTE_SET_USED(num_pkts); 377 } 378 379 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input 380 * so each bit of the encoder output must be translated into one byte of LLR. If 381 * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes 382 * must additionally be insterted at the end of each sub-block. 383 */ 384 static inline void 385 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf, 386 uint16_t num_pkts, uint16_t k) 387 { 388 uint16_t i, l, j; 389 uint16_t start_bit_idx; 390 uint16_t out_idx; 391 uint16_t d = k + 4; 392 uint16_t kpi = RTE_ALIGN_CEIL(d, 32); 393 uint16_t nd = kpi - d; 394 uint16_t ncb = 3 * kpi; 395 396 for (i = 0; i < num_pkts; ++i) { 397 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) - 398 sizeof(struct rte_ether_hdr); 399 400 /* Resize the packet if needed */ 401 if (pkt_data_len < ncb) { 402 char *data = rte_pktmbuf_append(mbufs[i], 403 ncb - pkt_data_len); 404 if (data == NULL) 405 printf( 406 "Not enough space in decoder input packet"); 407 } 408 409 /* Translate each bit into 1 LLR byte. */ 410 start_bit_idx = 0; 411 out_idx = 0; 412 for (j = 0; j < 3; ++j) { 413 for (l = start_bit_idx; l < start_bit_idx + d; ++l) { 414 uint8_t *data = rte_pktmbuf_mtod_offset( 415 mbufs[i], uint8_t *, 416 sizeof(struct rte_ether_hdr) + 417 (l >> 3)); 418 if (*data & (0x80 >> (l & 7))) 419 temp_buf[out_idx] = LLR_1_BIT; 420 else 421 temp_buf[out_idx] = LLR_0_BIT; 422 ++out_idx; 423 } 424 /* Padding bytes should be at the end of the sub-block. 425 */ 426 memset(&temp_buf[out_idx], 0, nd); 427 out_idx += nd; 428 start_bit_idx += d; 429 } 430 431 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *, 432 sizeof(struct rte_ether_hdr)), temp_buf, ncb); 433 } 434 } 435 436 static inline void 437 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts) 438 { 439 uint16_t i; 440 for (i = 0; i < num_pkts; ++i) { 441 struct rte_mbuf *out = mbufs[i]; 442 struct rte_mbuf *in = *mbuf_input(out); 443 444 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *, 445 sizeof(struct rte_ether_hdr)), 446 rte_pktmbuf_mtod_offset(out, uint8_t *, 447 sizeof(struct rte_ether_hdr)), 448 K / 8 - CRC_24B_LEN)) 449 printf("Input and output buffers are not equal!\n"); 450 } 451 } 452 453 static int 454 initialize_ports(struct app_config_params *app_params, 455 struct rte_mempool *ethdev_mbuf_mempool) 456 { 457 int ret; 458 uint16_t port_id = app_params->port_id; 459 uint16_t q; 460 /* ethernet addresses of ports */ 461 struct rte_ether_addr bbdev_port_eth_addr; 462 463 /* initialize ports */ 464 printf("\nInitializing port %u...\n", app_params->port_id); 465 ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores, 466 app_params->num_dec_cores, &port_conf); 467 468 if (ret < 0) { 469 printf("Cannot configure device: err=%d, port=%u\n", 470 ret, port_id); 471 return -1; 472 } 473 474 /* initialize RX queues for encoder */ 475 for (q = 0; q < app_params->num_enc_cores; q++) { 476 ret = rte_eth_rx_queue_setup(port_id, q, 477 RTE_TEST_RX_DESC_DEFAULT, 478 rte_eth_dev_socket_id(port_id), 479 NULL, ethdev_mbuf_mempool); 480 if (ret < 0) { 481 printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n", 482 ret, q); 483 return -1; 484 } 485 } 486 /* initialize TX queues for decoder */ 487 for (q = 0; q < app_params->num_dec_cores; q++) { 488 ret = rte_eth_tx_queue_setup(port_id, q, 489 RTE_TEST_TX_DESC_DEFAULT, 490 rte_eth_dev_socket_id(port_id), NULL); 491 if (ret < 0) { 492 printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n", 493 ret, q); 494 return -1; 495 } 496 } 497 498 ret = rte_eth_promiscuous_enable(port_id); 499 if (ret != 0) { 500 printf("Cannot enable promiscuous mode: err=%s, port=%u\n", 501 rte_strerror(-ret), port_id); 502 return ret; 503 } 504 505 ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr); 506 if (ret < 0) { 507 printf("rte_eth_macaddr_get: err=%d, queue=%u\n", 508 ret, q); 509 return -1; 510 } 511 512 print_mac(port_id, &bbdev_port_eth_addr); 513 514 return 0; 515 } 516 517 static void 518 lcore_conf_init(struct app_config_params *app_params, 519 struct lcore_conf *lcore_conf, 520 struct rte_mempool **bbdev_op_pools, 521 struct rte_mempool *bbdev_mbuf_mempool, 522 struct rte_ring *enc_to_dec_ring, 523 struct lcore_statistics *lcore_stats) 524 { 525 unsigned int lcore_id; 526 struct lcore_conf *lconf; 527 uint16_t rx_queue_id = 0; 528 uint16_t tx_queue_id = 0; 529 uint16_t enc_q_id = 0; 530 uint16_t dec_q_id = 0; 531 532 /* Configure lcores */ 533 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) { 534 lconf = &lcore_conf[lcore_id]; 535 lconf->core_type = 0; 536 537 if ((1ULL << lcore_id) & app_params->enc_core_mask) { 538 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC); 539 lconf->rx_queue_id = rx_queue_id++; 540 lconf->enc_queue_id = 541 app_params->enc_queue_ids[enc_q_id++]; 542 } 543 544 if ((1ULL << lcore_id) & app_params->dec_core_mask) { 545 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC); 546 lconf->tx_queue_id = tx_queue_id++; 547 lconf->dec_queue_id = 548 app_params->dec_queue_ids[dec_q_id++]; 549 } 550 551 lconf->bbdev_enc_op_pool = 552 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC]; 553 lconf->bbdev_dec_op_pool = 554 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC]; 555 lconf->bbdev_id = app_params->bbdev_id; 556 lconf->port_id = app_params->port_id; 557 lconf->enc_out_pool = bbdev_mbuf_mempool; 558 lconf->enc_to_dec_ring = enc_to_dec_ring; 559 lconf->lcore_stats = &lcore_stats[lcore_id]; 560 } 561 } 562 563 static void 564 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id) 565 { 566 static const char *stats_border = "_______"; 567 568 printf("\nLcore %d: %s enqueued count:\t\t%u\n", 569 lcore_id, stats_border, lstats->enqueued); 570 printf("Lcore %d: %s dequeued count:\t\t%u\n", 571 lcore_id, stats_border, lstats->dequeued); 572 printf("Lcore %d: %s RX lost packets count:\t\t%u\n", 573 lcore_id, stats_border, lstats->rx_lost_packets); 574 printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n", 575 lcore_id, stats_border, 576 lstats->enc_to_dec_lost_packets); 577 printf("Lcore %d: %s TX lost packets count:\t\t%u\n", 578 lcore_id, stats_border, lstats->tx_lost_packets); 579 } 580 581 static void 582 print_stats(struct stats_lcore_params *stats_lcore) 583 { 584 unsigned int l_id; 585 unsigned int bbdev_id = stats_lcore->app_params->bbdev_id; 586 unsigned int port_id = stats_lcore->app_params->port_id; 587 int len, ret, i; 588 589 struct rte_eth_xstat *xstats; 590 struct rte_eth_xstat_name *xstats_names; 591 struct rte_bbdev_stats bbstats; 592 static const char *stats_border = "_______"; 593 594 const char clr[] = { 27, '[', '2', 'J', '\0' }; 595 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 596 597 /* Clear screen and move to top left */ 598 printf("%s%s", clr, topLeft); 599 600 printf("PORT STATISTICS:\n================\n"); 601 len = rte_eth_xstats_get(port_id, NULL, 0); 602 if (len < 0) 603 rte_exit(EXIT_FAILURE, 604 "rte_eth_xstats_get(%u) failed: %d", port_id, 605 len); 606 607 xstats = calloc(len, sizeof(*xstats)); 608 if (xstats == NULL) 609 rte_exit(EXIT_FAILURE, 610 "Failed to calloc memory for xstats"); 611 612 ret = rte_eth_xstats_get(port_id, xstats, len); 613 if (ret < 0 || ret > len) { 614 free(xstats); 615 rte_exit(EXIT_FAILURE, 616 "rte_eth_xstats_get(%u) len%i failed: %d", 617 port_id, len, ret); 618 } 619 620 xstats_names = calloc(len, sizeof(*xstats_names)); 621 if (xstats_names == NULL) { 622 free(xstats); 623 rte_exit(EXIT_FAILURE, 624 "Failed to calloc memory for xstats_names"); 625 } 626 627 ret = rte_eth_xstats_get_names(port_id, xstats_names, len); 628 if (ret < 0 || ret > len) { 629 free(xstats); 630 free(xstats_names); 631 rte_exit(EXIT_FAILURE, 632 "rte_eth_xstats_get_names(%u) len%i failed: %d", 633 port_id, len, ret); 634 } 635 636 for (i = 0; i < len; i++) { 637 if (xstats[i].value > 0) 638 printf("Port %u: %s %s:\t\t%"PRIu64"\n", 639 port_id, stats_border, 640 xstats_names[i].name, 641 xstats[i].value); 642 } 643 644 ret = rte_bbdev_stats_get(bbdev_id, &bbstats); 645 if (ret < 0) { 646 free(xstats); 647 free(xstats_names); 648 rte_exit(EXIT_FAILURE, 649 "ERROR(%d): Failure to get BBDEV %u statistics\n", 650 ret, bbdev_id); 651 } 652 653 printf("\nBBDEV STATISTICS:\n=================\n"); 654 printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n", 655 bbdev_id, stats_border, 656 bbstats.enqueued_count); 657 printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n", 658 bbdev_id, stats_border, 659 bbstats.dequeued_count); 660 printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n", 661 bbdev_id, stats_border, 662 bbstats.enqueue_err_count); 663 printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n", 664 bbdev_id, stats_border, 665 bbstats.dequeue_err_count); 666 667 printf("LCORE STATISTICS:\n=================\n"); 668 for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) { 669 if (stats_lcore->lconf[l_id].core_type == 0) 670 continue; 671 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id); 672 } 673 674 fflush(stdout); 675 676 free(xstats); 677 free(xstats_names); 678 } 679 680 static int 681 stats_loop(void *arg) 682 { 683 struct stats_lcore_params *stats_lcore = arg; 684 685 while (!rte_atomic16_read(&global_exit_flag)) { 686 print_stats(stats_lcore); 687 rte_delay_ms(500); 688 } 689 690 return 0; 691 } 692 693 static inline void 694 run_encoding(struct lcore_conf *lcore_conf) 695 { 696 uint16_t i; 697 uint16_t port_id, rx_queue_id; 698 uint16_t bbdev_id, enc_queue_id; 699 uint16_t nb_rx, nb_enq, nb_deq, nb_sent; 700 struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST]; 701 struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST]; 702 struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST]; 703 struct lcore_statistics *lcore_stats; 704 struct rte_mempool *bbdev_op_pool, *enc_out_pool; 705 struct rte_ring *enc_to_dec_ring; 706 const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN; 707 708 lcore_stats = lcore_conf->lcore_stats; 709 port_id = lcore_conf->port_id; 710 rx_queue_id = lcore_conf->rx_queue_id; 711 bbdev_id = lcore_conf->bbdev_id; 712 enc_queue_id = lcore_conf->enc_queue_id; 713 bbdev_op_pool = lcore_conf->bbdev_enc_op_pool; 714 enc_out_pool = lcore_conf->enc_out_pool; 715 enc_to_dec_ring = lcore_conf->enc_to_dec_ring; 716 717 /* Read packet from RX queues*/ 718 nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst, 719 MAX_PKT_BURST); 720 if (!nb_rx) 721 return; 722 723 if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts, 724 nb_rx) != 0)) { 725 pktmbuf_free_bulk(rx_pkts_burst, nb_rx); 726 lcore_stats->rx_lost_packets += nb_rx; 727 return; 728 } 729 730 if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst, 731 nb_rx) != 0)) { 732 pktmbuf_free_bulk(enc_out_pkts, nb_rx); 733 pktmbuf_free_bulk(rx_pkts_burst, nb_rx); 734 lcore_stats->rx_lost_packets += nb_rx; 735 return; 736 } 737 738 for (i = 0; i < nb_rx; i++) { 739 char *data; 740 const uint16_t pkt_data_len = 741 rte_pktmbuf_data_len(rx_pkts_burst[i]) - 742 sizeof(struct rte_ether_hdr); 743 /* save input mbuf pointer for later comparison */ 744 *mbuf_input(enc_out_pkts[i]) = rx_pkts_burst[i]; 745 746 /* copy ethernet header */ 747 rte_pktmbuf_reset(enc_out_pkts[i]); 748 data = rte_pktmbuf_append(enc_out_pkts[i], 749 sizeof(struct rte_ether_hdr)); 750 if (data == NULL) { 751 printf( 752 "Not enough space for ethernet header in encoder output mbuf\n"); 753 continue; 754 } 755 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]); 756 757 /* set op */ 758 bbdev_ops_burst[i]->turbo_enc = def_op_enc; 759 760 bbdev_ops_burst[i]->turbo_enc.input.data = 761 rx_pkts_burst[i]; 762 bbdev_ops_burst[i]->turbo_enc.input.offset = 763 sizeof(struct rte_ether_hdr); 764 /* Encoder will attach the CRC24B, adjust the length */ 765 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len; 766 767 if (in_data_len < pkt_data_len) 768 rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len - 769 in_data_len); 770 else if (in_data_len > pkt_data_len) { 771 data = rte_pktmbuf_append(rx_pkts_burst[i], 772 in_data_len - pkt_data_len); 773 if (data == NULL) 774 printf( 775 "Not enough storage in mbuf to perform the encoding\n"); 776 } 777 778 bbdev_ops_burst[i]->turbo_enc.output.data = 779 enc_out_pkts[i]; 780 bbdev_ops_burst[i]->turbo_enc.output.offset = 781 sizeof(struct rte_ether_hdr); 782 } 783 784 /* Enqueue packets on BBDevice */ 785 nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id, 786 bbdev_ops_burst, nb_rx); 787 if (unlikely(nb_enq < nb_rx)) { 788 pktmbuf_input_free_bulk(&enc_out_pkts[nb_enq], 789 nb_rx - nb_enq); 790 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq], 791 nb_rx - nb_enq); 792 lcore_stats->rx_lost_packets += nb_rx - nb_enq; 793 794 if (!nb_enq) 795 return; 796 } 797 798 lcore_stats->enqueued += nb_enq; 799 800 /* Dequeue packets from bbdev device*/ 801 nb_deq = 0; 802 do { 803 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id, 804 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq); 805 } while (unlikely(nb_deq < nb_enq)); 806 807 lcore_stats->dequeued += nb_deq; 808 809 /* Generate and add AWGN */ 810 add_awgn(enc_out_pkts, nb_deq); 811 812 rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq); 813 814 /* Enqueue packets to encoder-to-decoder ring */ 815 nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts, 816 nb_deq, NULL); 817 if (unlikely(nb_sent < nb_deq)) { 818 pktmbuf_input_free_bulk(&enc_out_pkts[nb_sent], 819 nb_deq - nb_sent); 820 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent; 821 } 822 } 823 824 static void 825 run_decoding(struct lcore_conf *lcore_conf) 826 { 827 uint16_t i; 828 uint16_t port_id, tx_queue_id; 829 uint16_t bbdev_id, bbdev_queue_id; 830 uint16_t nb_recv, nb_enq, nb_deq, nb_tx; 831 uint8_t *llr_temp_buf; 832 struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST]; 833 struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST]; 834 struct lcore_statistics *lcore_stats; 835 struct rte_mempool *bbdev_op_pool; 836 struct rte_ring *enc_to_dec_ring; 837 838 lcore_stats = lcore_conf->lcore_stats; 839 port_id = lcore_conf->port_id; 840 tx_queue_id = lcore_conf->tx_queue_id; 841 bbdev_id = lcore_conf->bbdev_id; 842 bbdev_queue_id = lcore_conf->dec_queue_id; 843 bbdev_op_pool = lcore_conf->bbdev_dec_op_pool; 844 enc_to_dec_ring = lcore_conf->enc_to_dec_ring; 845 llr_temp_buf = lcore_conf->llr_temp_buf; 846 847 /* Dequeue packets from the ring */ 848 nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring, 849 (void **)recv_pkts_burst, MAX_PKT_BURST, NULL); 850 if (!nb_recv) 851 return; 852 853 if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst, 854 nb_recv) != 0)) { 855 pktmbuf_input_free_bulk(recv_pkts_burst, nb_recv); 856 lcore_stats->rx_lost_packets += nb_recv; 857 return; 858 } 859 860 transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv, 861 def_op_dec.cb_params.k); 862 863 for (i = 0; i < nb_recv; i++) { 864 /* set op */ 865 bbdev_ops_burst[i]->turbo_dec = def_op_dec; 866 867 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i]; 868 bbdev_ops_burst[i]->turbo_dec.input.offset = 869 sizeof(struct rte_ether_hdr); 870 bbdev_ops_burst[i]->turbo_dec.input.length = 871 rte_pktmbuf_data_len(recv_pkts_burst[i]) 872 - sizeof(struct rte_ether_hdr); 873 874 bbdev_ops_burst[i]->turbo_dec.hard_output.data = 875 recv_pkts_burst[i]; 876 bbdev_ops_burst[i]->turbo_dec.hard_output.offset = 877 sizeof(struct rte_ether_hdr); 878 } 879 880 /* Enqueue packets on BBDevice */ 881 nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id, 882 bbdev_ops_burst, nb_recv); 883 if (unlikely(nb_enq < nb_recv)) { 884 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_enq], 885 nb_recv - nb_enq); 886 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq], 887 nb_recv - nb_enq); 888 lcore_stats->rx_lost_packets += nb_recv - nb_enq; 889 890 if (!nb_enq) 891 return; 892 } 893 894 lcore_stats->enqueued += nb_enq; 895 896 /* Dequeue packets from BBDevice */ 897 nb_deq = 0; 898 do { 899 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id, 900 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq); 901 } while (unlikely(nb_deq < nb_enq)); 902 903 lcore_stats->dequeued += nb_deq; 904 905 rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq); 906 907 verify_data(recv_pkts_burst, nb_deq); 908 909 /* Free the RX mbufs after verification */ 910 for (i = 0; i < nb_deq; ++i) 911 rte_pktmbuf_free(*mbuf_input(recv_pkts_burst[i])); 912 913 /* Transmit the packets */ 914 nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq); 915 if (unlikely(nb_tx < nb_deq)) { 916 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_tx], 917 nb_deq - nb_tx); 918 lcore_stats->tx_lost_packets += nb_deq - nb_tx; 919 } 920 } 921 922 static int 923 processing_loop(void *arg) 924 { 925 struct lcore_conf *lcore_conf = arg; 926 const bool run_encoder = (lcore_conf->core_type & 927 (1 << RTE_BBDEV_OP_TURBO_ENC)); 928 const bool run_decoder = (lcore_conf->core_type & 929 (1 << RTE_BBDEV_OP_TURBO_DEC)); 930 931 while (!rte_atomic16_read(&global_exit_flag)) { 932 if (run_encoder) 933 run_encoding(lcore_conf); 934 if (run_decoder) 935 run_decoding(lcore_conf); 936 } 937 938 return 0; 939 } 940 941 static int 942 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info, 943 struct app_config_params *app_params) 944 { 945 int ret; 946 unsigned int q_id, dec_q_id, enc_q_id; 947 struct rte_bbdev_queue_conf qconf = {0}; 948 uint16_t dec_qs_nb = app_params->num_dec_cores; 949 uint16_t enc_qs_nb = app_params->num_enc_cores; 950 uint16_t tot_qs = dec_qs_nb + enc_qs_nb; 951 952 ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id); 953 if (ret < 0) 954 rte_exit(EXIT_FAILURE, 955 "ERROR(%d): BBDEV %u not configured properly\n", 956 ret, dev_id); 957 958 /* setup device DEC queues */ 959 qconf.socket = info->socket_id; 960 qconf.queue_size = info->drv.queue_size_lim; 961 qconf.op_type = RTE_BBDEV_OP_TURBO_DEC; 962 963 for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) { 964 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf); 965 if (ret < 0) 966 rte_exit(EXIT_FAILURE, 967 "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n", 968 ret, dev_id, q_id); 969 app_params->dec_queue_ids[dec_q_id++] = q_id; 970 } 971 972 /* setup device ENC queues */ 973 qconf.op_type = RTE_BBDEV_OP_TURBO_ENC; 974 975 for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) { 976 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf); 977 if (ret < 0) 978 rte_exit(EXIT_FAILURE, 979 "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n", 980 ret, dev_id, q_id); 981 app_params->enc_queue_ids[enc_q_id++] = q_id; 982 } 983 984 ret = rte_bbdev_start(dev_id); 985 986 if (ret != 0) 987 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n", 988 ret, dev_id); 989 990 printf("BBdev %u started\n", dev_id); 991 992 return 0; 993 } 994 995 static inline bool 996 check_matching_capabilities(uint64_t mask, uint64_t required_mask) 997 { 998 return (mask & required_mask) == required_mask; 999 } 1000 1001 static void 1002 enable_bbdev(struct app_config_params *app_params) 1003 { 1004 struct rte_bbdev_info dev_info; 1005 const struct rte_bbdev_op_cap *op_cap; 1006 uint16_t bbdev_id = app_params->bbdev_id; 1007 bool encoder_capable = false; 1008 bool decoder_capable = false; 1009 1010 rte_bbdev_info_get(bbdev_id, &dev_info); 1011 op_cap = dev_info.drv.capabilities; 1012 1013 while (op_cap->type != RTE_BBDEV_OP_NONE) { 1014 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 1015 if (check_matching_capabilities( 1016 op_cap->cap.turbo_enc.capability_flags, 1017 def_op_enc.op_flags)) 1018 encoder_capable = true; 1019 } 1020 1021 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 1022 if (check_matching_capabilities( 1023 op_cap->cap.turbo_dec.capability_flags, 1024 def_op_dec.op_flags)) 1025 decoder_capable = true; 1026 } 1027 1028 op_cap++; 1029 } 1030 1031 if (encoder_capable == false) 1032 rte_exit(EXIT_FAILURE, 1033 "The specified BBDev %u doesn't have required encoder capabilities!\n", 1034 bbdev_id); 1035 if (decoder_capable == false) 1036 rte_exit(EXIT_FAILURE, 1037 "The specified BBDev %u doesn't have required decoder capabilities!\n", 1038 bbdev_id); 1039 1040 prepare_bbdev_device(bbdev_id, &dev_info, app_params); 1041 } 1042 1043 int 1044 main(int argc, char **argv) 1045 { 1046 int ret; 1047 unsigned int nb_bbdevs, flags, lcore_id; 1048 void *sigret; 1049 struct app_config_params app_params = def_app_config; 1050 struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool; 1051 struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_COUNT]; 1052 struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} }; 1053 struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} }; 1054 struct stats_lcore_params stats_lcore; 1055 struct rte_ring *enc_to_dec_ring; 1056 bool stats_thread_started = false; 1057 unsigned int main_lcore_id = rte_get_main_lcore(); 1058 1059 static const struct rte_mbuf_dynfield input_dynfield_desc = { 1060 .name = "example_bbdev_dynfield_input", 1061 .size = sizeof(struct rte_mbuf *), 1062 .align = __alignof__(struct rte_mbuf *), 1063 }; 1064 1065 rte_atomic16_init(&global_exit_flag); 1066 1067 sigret = signal(SIGTERM, signal_handler); 1068 if (sigret == SIG_ERR) 1069 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM); 1070 1071 sigret = signal(SIGINT, signal_handler); 1072 if (sigret == SIG_ERR) 1073 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT); 1074 1075 ret = rte_eal_init(argc, argv); 1076 if (ret < 0) 1077 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); 1078 1079 argc -= ret; 1080 argv += ret; 1081 1082 /* parse application arguments (after the EAL ones) */ 1083 ret = bbdev_parse_args(argc, argv, &app_params); 1084 if (ret < 0) 1085 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n"); 1086 1087 /*create bbdev op pools*/ 1088 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] = 1089 rte_bbdev_op_pool_create("bbdev_op_pool_dec", 1090 RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id()); 1091 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] = 1092 rte_bbdev_op_pool_create("bbdev_op_pool_enc", 1093 RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id()); 1094 1095 if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) || 1096 (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL)) 1097 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n"); 1098 1099 /* Create encoder to decoder ring */ 1100 flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0; 1101 if (app_params.num_dec_cores == 1) 1102 flags |= RING_F_SC_DEQ; 1103 1104 enc_to_dec_ring = rte_ring_create("enc_to_dec_ring", 1105 rte_align32pow2(NB_MBUF), rte_socket_id(), flags); 1106 1107 /* Get the number of available bbdev devices */ 1108 nb_bbdevs = rte_bbdev_count(); 1109 if (nb_bbdevs <= app_params.bbdev_id) 1110 rte_exit(EXIT_FAILURE, 1111 "%u BBDevs detected, cannot use BBDev with ID %u!\n", 1112 nb_bbdevs, app_params.bbdev_id); 1113 printf("Number of bbdevs detected: %d\n", nb_bbdevs); 1114 1115 if (!rte_eth_dev_is_valid_port(app_params.port_id)) 1116 rte_exit(EXIT_FAILURE, 1117 "cannot use port with ID %u!\n", 1118 app_params.port_id); 1119 1120 /* create the mbuf mempool for ethdev pkts */ 1121 ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool", 1122 NB_MBUF, MEMPOOL_CACHE_SIZE, 0, 1123 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 1124 if (ethdev_mbuf_mempool == NULL) 1125 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n"); 1126 1127 /* create the mbuf mempool for encoder output */ 1128 bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool", 1129 NB_MBUF, MEMPOOL_CACHE_SIZE, 0, 1130 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 1131 if (bbdev_mbuf_mempool == NULL) 1132 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n"); 1133 1134 /* register mbuf field to store input pointer */ 1135 input_dynfield_offset = 1136 rte_mbuf_dynfield_register(&input_dynfield_desc); 1137 if (input_dynfield_offset < 0) 1138 rte_exit(EXIT_FAILURE, "Cannot register mbuf field\n"); 1139 1140 /* initialize ports */ 1141 ret = initialize_ports(&app_params, ethdev_mbuf_mempool); 1142 1143 /* Check if all requested lcores are available */ 1144 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) 1145 if (((1ULL << lcore_id) & app_params.enc_core_mask) || 1146 ((1ULL << lcore_id) & app_params.dec_core_mask)) 1147 if (!rte_lcore_is_enabled(lcore_id)) 1148 rte_exit(EXIT_FAILURE, 1149 "Requested lcore_id %u is not enabled!\n", 1150 lcore_id); 1151 1152 /* Start ethernet port */ 1153 ret = rte_eth_dev_start(app_params.port_id); 1154 if (ret < 0) 1155 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", 1156 ret, app_params.port_id); 1157 1158 ret = check_port_link_status(app_params.port_id); 1159 if (ret < 0) 1160 exit(EXIT_FAILURE); 1161 1162 /* start BBDevice and save BBDev queue IDs */ 1163 enable_bbdev(&app_params); 1164 1165 /* Initialize the port/queue configuration of each logical core */ 1166 lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools, 1167 bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats); 1168 1169 stats_lcore.app_params = &app_params; 1170 stats_lcore.lconf = lcore_conf; 1171 1172 RTE_LCORE_FOREACH_WORKER(lcore_id) { 1173 if (lcore_conf[lcore_id].core_type != 0) 1174 /* launch per-lcore processing loop on worker lcores */ 1175 rte_eal_remote_launch(processing_loop, 1176 &lcore_conf[lcore_id], lcore_id); 1177 else if (!stats_thread_started) { 1178 /* launch statistics printing loop */ 1179 rte_eal_remote_launch(stats_loop, &stats_lcore, 1180 lcore_id); 1181 stats_thread_started = true; 1182 } 1183 } 1184 1185 if (!stats_thread_started && 1186 lcore_conf[main_lcore_id].core_type != 0) 1187 rte_exit(EXIT_FAILURE, 1188 "Not enough lcores to run the statistics printing loop!"); 1189 else if (lcore_conf[main_lcore_id].core_type != 0) 1190 processing_loop(&lcore_conf[main_lcore_id]); 1191 else if (!stats_thread_started) 1192 stats_loop(&stats_lcore); 1193 1194 RTE_LCORE_FOREACH_WORKER(lcore_id) { 1195 ret |= rte_eal_wait_lcore(lcore_id); 1196 } 1197 1198 return ret; 1199 } 1200