1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <stdint.h> 9 #include <inttypes.h> 10 #include <sys/types.h> 11 #include <sys/unistd.h> 12 #include <sys/queue.h> 13 #include <stdarg.h> 14 #include <ctype.h> 15 #include <errno.h> 16 #include <math.h> 17 #include <assert.h> 18 #include <getopt.h> 19 #include <signal.h> 20 21 #include "rte_atomic.h" 22 #include "rte_common.h" 23 #include "rte_eal.h" 24 #include "rte_cycles.h" 25 #include "rte_ether.h" 26 #include "rte_ethdev.h" 27 #include "rte_ip.h" 28 #include "rte_lcore.h" 29 #include "rte_malloc.h" 30 #include "rte_mbuf.h" 31 #include "rte_memory.h" 32 #include "rte_mempool.h" 33 #include "rte_log.h" 34 #include "rte_bbdev.h" 35 #include "rte_bbdev_op.h" 36 37 /* LLR values - negative value for '1' bit */ 38 #define LLR_1_BIT 0x81 39 #define LLR_0_BIT 0x7F 40 41 #define MAX_PKT_BURST 32 42 #define NB_MBUF 8191 43 #define MEMPOOL_CACHE_SIZE 256 44 45 /* Hardcoded K value */ 46 #define K 40 47 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32)) 48 49 #define CRC_24B_LEN 3 50 51 /* Configurable number of RX/TX ring descriptors */ 52 #define RTE_TEST_RX_DESC_DEFAULT 128 53 #define RTE_TEST_TX_DESC_DEFAULT 512 54 55 #define BBDEV_ASSERT(a) do { \ 56 if (!(a)) { \ 57 usage(prgname); \ 58 return -1; \ 59 } \ 60 } while (0) 61 62 static const struct rte_eth_conf port_conf = { 63 .rxmode = { 64 .mq_mode = ETH_MQ_RX_NONE, 65 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 66 .split_hdr_size = 0, 67 }, 68 .txmode = { 69 .mq_mode = ETH_MQ_TX_NONE, 70 }, 71 }; 72 73 struct rte_bbdev_op_turbo_enc def_op_enc = { 74 /* These values are arbitrarily put, and does not map to the real 75 * values for the data received from ethdev ports 76 */ 77 .rv_index = 0, 78 .code_block_mode = 1, 79 .cb_params = { 80 .k = K, 81 }, 82 .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH 83 }; 84 85 struct rte_bbdev_op_turbo_dec def_op_dec = { 86 /* These values are arbitrarily put, and does not map to the real 87 * values for the data received from ethdev ports 88 */ 89 .code_block_mode = 1, 90 .cb_params = { 91 .k = K, 92 }, 93 .rv_index = 0, 94 .iter_max = 8, 95 .iter_min = 4, 96 .ext_scale = 15, 97 .num_maps = 0, 98 .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN 99 }; 100 101 struct app_config_params { 102 /* Placeholders for app params */ 103 uint16_t port_id; 104 uint16_t bbdev_id; 105 uint64_t enc_core_mask; 106 uint64_t dec_core_mask; 107 108 /* Values filled during init time */ 109 uint16_t enc_queue_ids[RTE_MAX_LCORE]; 110 uint16_t dec_queue_ids[RTE_MAX_LCORE]; 111 uint16_t num_enc_cores; 112 uint16_t num_dec_cores; 113 }; 114 115 struct lcore_statistics { 116 unsigned int enqueued; 117 unsigned int dequeued; 118 unsigned int rx_lost_packets; 119 unsigned int enc_to_dec_lost_packets; 120 unsigned int tx_lost_packets; 121 } __rte_cache_aligned; 122 123 /** each lcore configuration */ 124 struct lcore_conf { 125 uint64_t core_type; 126 127 unsigned int port_id; 128 unsigned int rx_queue_id; 129 unsigned int tx_queue_id; 130 131 unsigned int bbdev_id; 132 unsigned int enc_queue_id; 133 unsigned int dec_queue_id; 134 135 uint8_t llr_temp_buf[NCB]; 136 137 struct rte_mempool *bbdev_dec_op_pool; 138 struct rte_mempool *bbdev_enc_op_pool; 139 struct rte_mempool *enc_out_pool; 140 struct rte_ring *enc_to_dec_ring; 141 142 struct lcore_statistics *lcore_stats; 143 } __rte_cache_aligned; 144 145 struct stats_lcore_params { 146 struct lcore_conf *lconf; 147 struct app_config_params *app_params; 148 }; 149 150 151 static const struct app_config_params def_app_config = { 152 .port_id = 0, 153 .bbdev_id = 0, 154 .enc_core_mask = 0x2, 155 .dec_core_mask = 0x4, 156 .num_enc_cores = 1, 157 .num_dec_cores = 1, 158 }; 159 160 static rte_atomic16_t global_exit_flag; 161 162 /* display usage */ 163 static inline void 164 usage(const char *prgname) 165 { 166 printf("%s [EAL options] " 167 " --\n" 168 " --enc_cores - number of encoding cores (default = 0x2)\n" 169 " --dec_cores - number of decoding cores (default = 0x4)\n" 170 " --port_id - Ethernet port ID (default = 0)\n" 171 " --bbdev_id - BBDev ID (default = 0)\n" 172 "\n", prgname); 173 } 174 175 /* parse core mask */ 176 static inline 177 uint16_t bbdev_parse_mask(const char *mask) 178 { 179 char *end = NULL; 180 unsigned long pm; 181 182 /* parse hexadecimal string */ 183 pm = strtoul(mask, &end, 16); 184 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0')) 185 return 0; 186 187 return pm; 188 } 189 190 /* parse core mask */ 191 static inline 192 uint16_t bbdev_parse_number(const char *mask) 193 { 194 char *end = NULL; 195 unsigned long pm; 196 197 /* parse hexadecimal string */ 198 pm = strtoul(mask, &end, 10); 199 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0')) 200 return 0; 201 202 return pm; 203 } 204 205 static int 206 bbdev_parse_args(int argc, char **argv, 207 struct app_config_params *app_params) 208 { 209 int optind = 0; 210 int opt; 211 int opt_indx = 0; 212 char *prgname = argv[0]; 213 214 static struct option lgopts[] = { 215 { "enc_core_mask", required_argument, 0, 'e' }, 216 { "dec_core_mask", required_argument, 0, 'd' }, 217 { "port_id", required_argument, 0, 'p' }, 218 { "bbdev_id", required_argument, 0, 'b' }, 219 { NULL, 0, 0, 0 } 220 }; 221 222 BBDEV_ASSERT(argc != 0); 223 BBDEV_ASSERT(argv != NULL); 224 BBDEV_ASSERT(app_params != NULL); 225 226 while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) != 227 EOF) { 228 switch (opt) { 229 case 'e': 230 app_params->enc_core_mask = 231 bbdev_parse_mask(optarg); 232 if (app_params->enc_core_mask == 0) { 233 usage(prgname); 234 return -1; 235 } 236 app_params->num_enc_cores = 237 __builtin_popcount(app_params->enc_core_mask); 238 break; 239 240 case 'd': 241 app_params->dec_core_mask = 242 bbdev_parse_mask(optarg); 243 if (app_params->dec_core_mask == 0) { 244 usage(prgname); 245 return -1; 246 } 247 app_params->num_dec_cores = 248 __builtin_popcount(app_params->dec_core_mask); 249 break; 250 251 case 'p': 252 app_params->port_id = bbdev_parse_number(optarg); 253 break; 254 255 case 'b': 256 app_params->bbdev_id = bbdev_parse_number(optarg); 257 break; 258 259 default: 260 usage(prgname); 261 return -1; 262 } 263 } 264 optind = 0; 265 return optind; 266 } 267 268 static void 269 signal_handler(int signum) 270 { 271 printf("\nSignal %d received\n", signum); 272 rte_atomic16_set(&global_exit_flag, 1); 273 } 274 275 static void 276 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address) 277 { 278 printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n", 279 (unsigned int) portid, 280 bbdev_ports_eth_address->addr_bytes[0], 281 bbdev_ports_eth_address->addr_bytes[1], 282 bbdev_ports_eth_address->addr_bytes[2], 283 bbdev_ports_eth_address->addr_bytes[3], 284 bbdev_ports_eth_address->addr_bytes[4], 285 bbdev_ports_eth_address->addr_bytes[5]); 286 } 287 288 static inline void 289 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free) 290 { 291 unsigned int i; 292 for (i = 0; i < nb_to_free; ++i) 293 rte_pktmbuf_free(mbufs[i]); 294 } 295 296 static inline void 297 pktmbuf_userdata_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free) 298 { 299 unsigned int i; 300 for (i = 0; i < nb_to_free; ++i) { 301 struct rte_mbuf *rx_pkt = mbufs[i]->userdata; 302 rte_pktmbuf_free(rx_pkt); 303 rte_pktmbuf_free(mbufs[i]); 304 } 305 } 306 307 /* Check the link status of all ports in up to 9s, and print them finally */ 308 static int 309 check_port_link_status(uint16_t port_id) 310 { 311 #define CHECK_INTERVAL 100 /* 100ms */ 312 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 313 uint8_t count; 314 struct rte_eth_link link; 315 int link_get_err = -EINVAL; 316 317 printf("\nChecking link status."); 318 fflush(stdout); 319 320 for (count = 0; count <= MAX_CHECK_TIME && 321 !rte_atomic16_read(&global_exit_flag); count++) { 322 memset(&link, 0, sizeof(link)); 323 link_get_err = rte_eth_link_get_nowait(port_id, &link); 324 325 if (link_get_err >= 0 && link.link_status) { 326 const char *dp = (link.link_duplex == 327 ETH_LINK_FULL_DUPLEX) ? 328 "full-duplex" : "half-duplex"; 329 printf("\nPort %u Link Up - speed %s - %s\n", 330 port_id, 331 rte_eth_link_speed_to_str(link.link_speed), 332 dp); 333 return 0; 334 } 335 printf("."); 336 fflush(stdout); 337 rte_delay_ms(CHECK_INTERVAL); 338 } 339 340 if (link_get_err >= 0) 341 printf("\nPort %d Link Down\n", port_id); 342 else 343 printf("\nGet link failed (port %d): %s\n", port_id, 344 rte_strerror(-link_get_err)); 345 346 return 0; 347 } 348 349 static inline void 350 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst) 351 { 352 struct rte_ether_hdr *eth_from; 353 struct rte_ether_hdr *eth_to; 354 355 eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *); 356 eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *); 357 358 /* copy header */ 359 rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr)); 360 } 361 362 static inline void 363 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts) 364 { 365 RTE_SET_USED(mbufs); 366 RTE_SET_USED(num_pkts); 367 } 368 369 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input 370 * so each bit of the encoder output must be translated into one byte of LLR. If 371 * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes 372 * must additionally be insterted at the end of each sub-block. 373 */ 374 static inline void 375 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf, 376 uint16_t num_pkts, uint16_t k) 377 { 378 uint16_t i, l, j; 379 uint16_t start_bit_idx; 380 uint16_t out_idx; 381 uint16_t d = k + 4; 382 uint16_t kpi = RTE_ALIGN_CEIL(d, 32); 383 uint16_t nd = kpi - d; 384 uint16_t ncb = 3 * kpi; 385 386 for (i = 0; i < num_pkts; ++i) { 387 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) - 388 sizeof(struct rte_ether_hdr); 389 390 /* Resize the packet if needed */ 391 if (pkt_data_len < ncb) { 392 char *data = rte_pktmbuf_append(mbufs[i], 393 ncb - pkt_data_len); 394 if (data == NULL) 395 printf( 396 "Not enough space in decoder input packet"); 397 } 398 399 /* Translate each bit into 1 LLR byte. */ 400 start_bit_idx = 0; 401 out_idx = 0; 402 for (j = 0; j < 3; ++j) { 403 for (l = start_bit_idx; l < start_bit_idx + d; ++l) { 404 uint8_t *data = rte_pktmbuf_mtod_offset( 405 mbufs[i], uint8_t *, 406 sizeof(struct rte_ether_hdr) + 407 (l >> 3)); 408 if (*data & (0x80 >> (l & 7))) 409 temp_buf[out_idx] = LLR_1_BIT; 410 else 411 temp_buf[out_idx] = LLR_0_BIT; 412 ++out_idx; 413 } 414 /* Padding bytes should be at the end of the sub-block. 415 */ 416 memset(&temp_buf[out_idx], 0, nd); 417 out_idx += nd; 418 start_bit_idx += d; 419 } 420 421 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *, 422 sizeof(struct rte_ether_hdr)), temp_buf, ncb); 423 } 424 } 425 426 static inline void 427 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts) 428 { 429 uint16_t i; 430 for (i = 0; i < num_pkts; ++i) { 431 struct rte_mbuf *out = mbufs[i]; 432 struct rte_mbuf *in = out->userdata; 433 434 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *, 435 sizeof(struct rte_ether_hdr)), 436 rte_pktmbuf_mtod_offset(out, uint8_t *, 437 sizeof(struct rte_ether_hdr)), 438 K / 8 - CRC_24B_LEN)) 439 printf("Input and output buffers are not equal!\n"); 440 } 441 } 442 443 static int 444 initialize_ports(struct app_config_params *app_params, 445 struct rte_mempool *ethdev_mbuf_mempool) 446 { 447 int ret; 448 uint16_t port_id = app_params->port_id; 449 uint16_t q; 450 /* ethernet addresses of ports */ 451 struct rte_ether_addr bbdev_port_eth_addr; 452 453 /* initialize ports */ 454 printf("\nInitializing port %u...\n", app_params->port_id); 455 ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores, 456 app_params->num_dec_cores, &port_conf); 457 458 if (ret < 0) { 459 printf("Cannot configure device: err=%d, port=%u\n", 460 ret, port_id); 461 return -1; 462 } 463 464 /* initialize RX queues for encoder */ 465 for (q = 0; q < app_params->num_enc_cores; q++) { 466 ret = rte_eth_rx_queue_setup(port_id, q, 467 RTE_TEST_RX_DESC_DEFAULT, 468 rte_eth_dev_socket_id(port_id), 469 NULL, ethdev_mbuf_mempool); 470 if (ret < 0) { 471 printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n", 472 ret, q); 473 return -1; 474 } 475 } 476 /* initialize TX queues for decoder */ 477 for (q = 0; q < app_params->num_dec_cores; q++) { 478 ret = rte_eth_tx_queue_setup(port_id, q, 479 RTE_TEST_TX_DESC_DEFAULT, 480 rte_eth_dev_socket_id(port_id), NULL); 481 if (ret < 0) { 482 printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n", 483 ret, q); 484 return -1; 485 } 486 } 487 488 ret = rte_eth_promiscuous_enable(port_id); 489 if (ret != 0) { 490 printf("Cannot enable promiscuous mode: err=%s, port=%u\n", 491 rte_strerror(-ret), port_id); 492 return ret; 493 } 494 495 ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr); 496 if (ret < 0) { 497 printf("rte_eth_macaddr_get: err=%d, queue=%u\n", 498 ret, q); 499 return -1; 500 } 501 502 print_mac(port_id, &bbdev_port_eth_addr); 503 504 return 0; 505 } 506 507 static void 508 lcore_conf_init(struct app_config_params *app_params, 509 struct lcore_conf *lcore_conf, 510 struct rte_mempool **bbdev_op_pools, 511 struct rte_mempool *bbdev_mbuf_mempool, 512 struct rte_ring *enc_to_dec_ring, 513 struct lcore_statistics *lcore_stats) 514 { 515 unsigned int lcore_id; 516 struct lcore_conf *lconf; 517 uint16_t rx_queue_id = 0; 518 uint16_t tx_queue_id = 0; 519 uint16_t enc_q_id = 0; 520 uint16_t dec_q_id = 0; 521 522 /* Configure lcores */ 523 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) { 524 lconf = &lcore_conf[lcore_id]; 525 lconf->core_type = 0; 526 527 if ((1ULL << lcore_id) & app_params->enc_core_mask) { 528 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC); 529 lconf->rx_queue_id = rx_queue_id++; 530 lconf->enc_queue_id = 531 app_params->enc_queue_ids[enc_q_id++]; 532 } 533 534 if ((1ULL << lcore_id) & app_params->dec_core_mask) { 535 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC); 536 lconf->tx_queue_id = tx_queue_id++; 537 lconf->dec_queue_id = 538 app_params->dec_queue_ids[dec_q_id++]; 539 } 540 541 lconf->bbdev_enc_op_pool = 542 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC]; 543 lconf->bbdev_dec_op_pool = 544 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC]; 545 lconf->bbdev_id = app_params->bbdev_id; 546 lconf->port_id = app_params->port_id; 547 lconf->enc_out_pool = bbdev_mbuf_mempool; 548 lconf->enc_to_dec_ring = enc_to_dec_ring; 549 lconf->lcore_stats = &lcore_stats[lcore_id]; 550 } 551 } 552 553 static void 554 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id) 555 { 556 static const char *stats_border = "_______"; 557 558 printf("\nLcore %d: %s enqueued count:\t\t%u\n", 559 lcore_id, stats_border, lstats->enqueued); 560 printf("Lcore %d: %s dequeued count:\t\t%u\n", 561 lcore_id, stats_border, lstats->dequeued); 562 printf("Lcore %d: %s RX lost packets count:\t\t%u\n", 563 lcore_id, stats_border, lstats->rx_lost_packets); 564 printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n", 565 lcore_id, stats_border, 566 lstats->enc_to_dec_lost_packets); 567 printf("Lcore %d: %s TX lost packets count:\t\t%u\n", 568 lcore_id, stats_border, lstats->tx_lost_packets); 569 } 570 571 static void 572 print_stats(struct stats_lcore_params *stats_lcore) 573 { 574 unsigned int l_id; 575 unsigned int bbdev_id = stats_lcore->app_params->bbdev_id; 576 unsigned int port_id = stats_lcore->app_params->port_id; 577 int len, ret, i; 578 579 struct rte_eth_xstat *xstats; 580 struct rte_eth_xstat_name *xstats_names; 581 struct rte_bbdev_stats bbstats; 582 static const char *stats_border = "_______"; 583 584 const char clr[] = { 27, '[', '2', 'J', '\0' }; 585 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 586 587 /* Clear screen and move to top left */ 588 printf("%s%s", clr, topLeft); 589 590 printf("PORT STATISTICS:\n================\n"); 591 len = rte_eth_xstats_get(port_id, NULL, 0); 592 if (len < 0) 593 rte_exit(EXIT_FAILURE, 594 "rte_eth_xstats_get(%u) failed: %d", port_id, 595 len); 596 597 xstats = calloc(len, sizeof(*xstats)); 598 if (xstats == NULL) 599 rte_exit(EXIT_FAILURE, 600 "Failed to calloc memory for xstats"); 601 602 ret = rte_eth_xstats_get(port_id, xstats, len); 603 if (ret < 0 || ret > len) { 604 free(xstats); 605 rte_exit(EXIT_FAILURE, 606 "rte_eth_xstats_get(%u) len%i failed: %d", 607 port_id, len, ret); 608 } 609 610 xstats_names = calloc(len, sizeof(*xstats_names)); 611 if (xstats_names == NULL) { 612 free(xstats); 613 rte_exit(EXIT_FAILURE, 614 "Failed to calloc memory for xstats_names"); 615 } 616 617 ret = rte_eth_xstats_get_names(port_id, xstats_names, len); 618 if (ret < 0 || ret > len) { 619 free(xstats); 620 free(xstats_names); 621 rte_exit(EXIT_FAILURE, 622 "rte_eth_xstats_get_names(%u) len%i failed: %d", 623 port_id, len, ret); 624 } 625 626 for (i = 0; i < len; i++) { 627 if (xstats[i].value > 0) 628 printf("Port %u: %s %s:\t\t%"PRIu64"\n", 629 port_id, stats_border, 630 xstats_names[i].name, 631 xstats[i].value); 632 } 633 634 ret = rte_bbdev_stats_get(bbdev_id, &bbstats); 635 if (ret < 0) { 636 free(xstats); 637 free(xstats_names); 638 rte_exit(EXIT_FAILURE, 639 "ERROR(%d): Failure to get BBDEV %u statistics\n", 640 ret, bbdev_id); 641 } 642 643 printf("\nBBDEV STATISTICS:\n=================\n"); 644 printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n", 645 bbdev_id, stats_border, 646 bbstats.enqueued_count); 647 printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n", 648 bbdev_id, stats_border, 649 bbstats.dequeued_count); 650 printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n", 651 bbdev_id, stats_border, 652 bbstats.enqueue_err_count); 653 printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n", 654 bbdev_id, stats_border, 655 bbstats.dequeue_err_count); 656 657 printf("LCORE STATISTICS:\n=================\n"); 658 for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) { 659 if (stats_lcore->lconf[l_id].core_type == 0) 660 continue; 661 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id); 662 } 663 664 fflush(stdout); 665 666 free(xstats); 667 free(xstats_names); 668 } 669 670 static int 671 stats_loop(void *arg) 672 { 673 struct stats_lcore_params *stats_lcore = arg; 674 675 while (!rte_atomic16_read(&global_exit_flag)) { 676 print_stats(stats_lcore); 677 rte_delay_ms(500); 678 } 679 680 return 0; 681 } 682 683 static inline void 684 run_encoding(struct lcore_conf *lcore_conf) 685 { 686 uint16_t i; 687 uint16_t port_id, rx_queue_id; 688 uint16_t bbdev_id, enc_queue_id; 689 uint16_t nb_rx, nb_enq, nb_deq, nb_sent; 690 struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST]; 691 struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST]; 692 struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST]; 693 struct lcore_statistics *lcore_stats; 694 struct rte_mempool *bbdev_op_pool, *enc_out_pool; 695 struct rte_ring *enc_to_dec_ring; 696 const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN; 697 698 lcore_stats = lcore_conf->lcore_stats; 699 port_id = lcore_conf->port_id; 700 rx_queue_id = lcore_conf->rx_queue_id; 701 bbdev_id = lcore_conf->bbdev_id; 702 enc_queue_id = lcore_conf->enc_queue_id; 703 bbdev_op_pool = lcore_conf->bbdev_enc_op_pool; 704 enc_out_pool = lcore_conf->enc_out_pool; 705 enc_to_dec_ring = lcore_conf->enc_to_dec_ring; 706 707 /* Read packet from RX queues*/ 708 nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst, 709 MAX_PKT_BURST); 710 if (!nb_rx) 711 return; 712 713 if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts, 714 nb_rx) != 0)) { 715 pktmbuf_free_bulk(rx_pkts_burst, nb_rx); 716 lcore_stats->rx_lost_packets += nb_rx; 717 return; 718 } 719 720 if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst, 721 nb_rx) != 0)) { 722 pktmbuf_free_bulk(enc_out_pkts, nb_rx); 723 pktmbuf_free_bulk(rx_pkts_burst, nb_rx); 724 lcore_stats->rx_lost_packets += nb_rx; 725 return; 726 } 727 728 for (i = 0; i < nb_rx; i++) { 729 char *data; 730 const uint16_t pkt_data_len = 731 rte_pktmbuf_data_len(rx_pkts_burst[i]) - 732 sizeof(struct rte_ether_hdr); 733 /* save input mbuf pointer for later comparison */ 734 enc_out_pkts[i]->userdata = rx_pkts_burst[i]; 735 736 /* copy ethernet header */ 737 rte_pktmbuf_reset(enc_out_pkts[i]); 738 data = rte_pktmbuf_append(enc_out_pkts[i], 739 sizeof(struct rte_ether_hdr)); 740 if (data == NULL) { 741 printf( 742 "Not enough space for ethernet header in encoder output mbuf\n"); 743 continue; 744 } 745 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]); 746 747 /* set op */ 748 bbdev_ops_burst[i]->turbo_enc = def_op_enc; 749 750 bbdev_ops_burst[i]->turbo_enc.input.data = 751 rx_pkts_burst[i]; 752 bbdev_ops_burst[i]->turbo_enc.input.offset = 753 sizeof(struct rte_ether_hdr); 754 /* Encoder will attach the CRC24B, adjust the length */ 755 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len; 756 757 if (in_data_len < pkt_data_len) 758 rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len - 759 in_data_len); 760 else if (in_data_len > pkt_data_len) { 761 data = rte_pktmbuf_append(rx_pkts_burst[i], 762 in_data_len - pkt_data_len); 763 if (data == NULL) 764 printf( 765 "Not enough storage in mbuf to perform the encoding\n"); 766 } 767 768 bbdev_ops_burst[i]->turbo_enc.output.data = 769 enc_out_pkts[i]; 770 bbdev_ops_burst[i]->turbo_enc.output.offset = 771 sizeof(struct rte_ether_hdr); 772 } 773 774 /* Enqueue packets on BBDevice */ 775 nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id, 776 bbdev_ops_burst, nb_rx); 777 if (unlikely(nb_enq < nb_rx)) { 778 pktmbuf_userdata_free_bulk(&enc_out_pkts[nb_enq], 779 nb_rx - nb_enq); 780 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq], 781 nb_rx - nb_enq); 782 lcore_stats->rx_lost_packets += nb_rx - nb_enq; 783 784 if (!nb_enq) 785 return; 786 } 787 788 lcore_stats->enqueued += nb_enq; 789 790 /* Dequeue packets from bbdev device*/ 791 nb_deq = 0; 792 do { 793 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id, 794 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq); 795 } while (unlikely(nb_deq < nb_enq)); 796 797 lcore_stats->dequeued += nb_deq; 798 799 /* Generate and add AWGN */ 800 add_awgn(enc_out_pkts, nb_deq); 801 802 rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq); 803 804 /* Enqueue packets to encoder-to-decoder ring */ 805 nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts, 806 nb_deq, NULL); 807 if (unlikely(nb_sent < nb_deq)) { 808 pktmbuf_userdata_free_bulk(&enc_out_pkts[nb_sent], 809 nb_deq - nb_sent); 810 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent; 811 } 812 } 813 814 static void 815 run_decoding(struct lcore_conf *lcore_conf) 816 { 817 uint16_t i; 818 uint16_t port_id, tx_queue_id; 819 uint16_t bbdev_id, bbdev_queue_id; 820 uint16_t nb_recv, nb_enq, nb_deq, nb_tx; 821 uint8_t *llr_temp_buf; 822 struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST]; 823 struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST]; 824 struct lcore_statistics *lcore_stats; 825 struct rte_mempool *bbdev_op_pool; 826 struct rte_ring *enc_to_dec_ring; 827 828 lcore_stats = lcore_conf->lcore_stats; 829 port_id = lcore_conf->port_id; 830 tx_queue_id = lcore_conf->tx_queue_id; 831 bbdev_id = lcore_conf->bbdev_id; 832 bbdev_queue_id = lcore_conf->dec_queue_id; 833 bbdev_op_pool = lcore_conf->bbdev_dec_op_pool; 834 enc_to_dec_ring = lcore_conf->enc_to_dec_ring; 835 llr_temp_buf = lcore_conf->llr_temp_buf; 836 837 /* Dequeue packets from the ring */ 838 nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring, 839 (void **)recv_pkts_burst, MAX_PKT_BURST, NULL); 840 if (!nb_recv) 841 return; 842 843 if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst, 844 nb_recv) != 0)) { 845 pktmbuf_userdata_free_bulk(recv_pkts_burst, nb_recv); 846 lcore_stats->rx_lost_packets += nb_recv; 847 return; 848 } 849 850 transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv, 851 def_op_dec.cb_params.k); 852 853 for (i = 0; i < nb_recv; i++) { 854 /* set op */ 855 bbdev_ops_burst[i]->turbo_dec = def_op_dec; 856 857 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i]; 858 bbdev_ops_burst[i]->turbo_dec.input.offset = 859 sizeof(struct rte_ether_hdr); 860 bbdev_ops_burst[i]->turbo_dec.input.length = 861 rte_pktmbuf_data_len(recv_pkts_burst[i]) 862 - sizeof(struct rte_ether_hdr); 863 864 bbdev_ops_burst[i]->turbo_dec.hard_output.data = 865 recv_pkts_burst[i]; 866 bbdev_ops_burst[i]->turbo_dec.hard_output.offset = 867 sizeof(struct rte_ether_hdr); 868 } 869 870 /* Enqueue packets on BBDevice */ 871 nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id, 872 bbdev_ops_burst, nb_recv); 873 if (unlikely(nb_enq < nb_recv)) { 874 pktmbuf_userdata_free_bulk(&recv_pkts_burst[nb_enq], 875 nb_recv - nb_enq); 876 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq], 877 nb_recv - nb_enq); 878 lcore_stats->rx_lost_packets += nb_recv - nb_enq; 879 880 if (!nb_enq) 881 return; 882 } 883 884 lcore_stats->enqueued += nb_enq; 885 886 /* Dequeue packets from BBDevice */ 887 nb_deq = 0; 888 do { 889 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id, 890 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq); 891 } while (unlikely(nb_deq < nb_enq)); 892 893 lcore_stats->dequeued += nb_deq; 894 895 rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq); 896 897 verify_data(recv_pkts_burst, nb_deq); 898 899 /* Free the RX mbufs after verification */ 900 for (i = 0; i < nb_deq; ++i) 901 rte_pktmbuf_free(recv_pkts_burst[i]->userdata); 902 903 /* Transmit the packets */ 904 nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq); 905 if (unlikely(nb_tx < nb_deq)) { 906 pktmbuf_userdata_free_bulk(&recv_pkts_burst[nb_tx], 907 nb_deq - nb_tx); 908 lcore_stats->tx_lost_packets += nb_deq - nb_tx; 909 } 910 } 911 912 static int 913 processing_loop(void *arg) 914 { 915 struct lcore_conf *lcore_conf = arg; 916 const bool run_encoder = (lcore_conf->core_type & 917 (1 << RTE_BBDEV_OP_TURBO_ENC)); 918 const bool run_decoder = (lcore_conf->core_type & 919 (1 << RTE_BBDEV_OP_TURBO_DEC)); 920 921 while (!rte_atomic16_read(&global_exit_flag)) { 922 if (run_encoder) 923 run_encoding(lcore_conf); 924 if (run_decoder) 925 run_decoding(lcore_conf); 926 } 927 928 return 0; 929 } 930 931 static int 932 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info, 933 struct app_config_params *app_params) 934 { 935 int ret; 936 unsigned int q_id, dec_q_id, enc_q_id; 937 struct rte_bbdev_queue_conf qconf = {0}; 938 uint16_t dec_qs_nb = app_params->num_dec_cores; 939 uint16_t enc_qs_nb = app_params->num_enc_cores; 940 uint16_t tot_qs = dec_qs_nb + enc_qs_nb; 941 942 ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id); 943 if (ret < 0) 944 rte_exit(EXIT_FAILURE, 945 "ERROR(%d): BBDEV %u not configured properly\n", 946 ret, dev_id); 947 948 /* setup device DEC queues */ 949 qconf.socket = info->socket_id; 950 qconf.queue_size = info->drv.queue_size_lim; 951 qconf.op_type = RTE_BBDEV_OP_TURBO_DEC; 952 953 for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) { 954 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf); 955 if (ret < 0) 956 rte_exit(EXIT_FAILURE, 957 "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n", 958 ret, dev_id, q_id); 959 app_params->dec_queue_ids[dec_q_id++] = q_id; 960 } 961 962 /* setup device ENC queues */ 963 qconf.op_type = RTE_BBDEV_OP_TURBO_ENC; 964 965 for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) { 966 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf); 967 if (ret < 0) 968 rte_exit(EXIT_FAILURE, 969 "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n", 970 ret, dev_id, q_id); 971 app_params->enc_queue_ids[enc_q_id++] = q_id; 972 } 973 974 ret = rte_bbdev_start(dev_id); 975 976 if (ret != 0) 977 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n", 978 ret, dev_id); 979 980 printf("BBdev %u started\n", dev_id); 981 982 return 0; 983 } 984 985 static inline bool 986 check_matching_capabilities(uint64_t mask, uint64_t required_mask) 987 { 988 return (mask & required_mask) == required_mask; 989 } 990 991 static void 992 enable_bbdev(struct app_config_params *app_params) 993 { 994 struct rte_bbdev_info dev_info; 995 const struct rte_bbdev_op_cap *op_cap; 996 uint16_t bbdev_id = app_params->bbdev_id; 997 bool encoder_capable = false; 998 bool decoder_capable = false; 999 1000 rte_bbdev_info_get(bbdev_id, &dev_info); 1001 op_cap = dev_info.drv.capabilities; 1002 1003 while (op_cap->type != RTE_BBDEV_OP_NONE) { 1004 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 1005 if (check_matching_capabilities( 1006 op_cap->cap.turbo_enc.capability_flags, 1007 def_op_enc.op_flags)) 1008 encoder_capable = true; 1009 } 1010 1011 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 1012 if (check_matching_capabilities( 1013 op_cap->cap.turbo_dec.capability_flags, 1014 def_op_dec.op_flags)) 1015 decoder_capable = true; 1016 } 1017 1018 op_cap++; 1019 } 1020 1021 if (encoder_capable == false) 1022 rte_exit(EXIT_FAILURE, 1023 "The specified BBDev %u doesn't have required encoder capabilities!\n", 1024 bbdev_id); 1025 if (decoder_capable == false) 1026 rte_exit(EXIT_FAILURE, 1027 "The specified BBDev %u doesn't have required decoder capabilities!\n", 1028 bbdev_id); 1029 1030 prepare_bbdev_device(bbdev_id, &dev_info, app_params); 1031 } 1032 1033 int 1034 main(int argc, char **argv) 1035 { 1036 int ret; 1037 unsigned int nb_bbdevs, flags, lcore_id; 1038 void *sigret; 1039 struct app_config_params app_params = def_app_config; 1040 struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool; 1041 struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_COUNT]; 1042 struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} }; 1043 struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} }; 1044 struct stats_lcore_params stats_lcore; 1045 struct rte_ring *enc_to_dec_ring; 1046 bool stats_thread_started = false; 1047 unsigned int master_lcore_id = rte_get_master_lcore(); 1048 1049 rte_atomic16_init(&global_exit_flag); 1050 1051 sigret = signal(SIGTERM, signal_handler); 1052 if (sigret == SIG_ERR) 1053 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM); 1054 1055 sigret = signal(SIGINT, signal_handler); 1056 if (sigret == SIG_ERR) 1057 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT); 1058 1059 ret = rte_eal_init(argc, argv); 1060 if (ret < 0) 1061 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); 1062 1063 argc -= ret; 1064 argv += ret; 1065 1066 /* parse application arguments (after the EAL ones) */ 1067 ret = bbdev_parse_args(argc, argv, &app_params); 1068 if (ret < 0) 1069 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n"); 1070 1071 /*create bbdev op pools*/ 1072 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] = 1073 rte_bbdev_op_pool_create("bbdev_op_pool_dec", 1074 RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id()); 1075 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] = 1076 rte_bbdev_op_pool_create("bbdev_op_pool_enc", 1077 RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id()); 1078 1079 if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) || 1080 (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL)) 1081 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n"); 1082 1083 /* Create encoder to decoder ring */ 1084 flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0; 1085 if (app_params.num_dec_cores == 1) 1086 flags |= RING_F_SC_DEQ; 1087 1088 enc_to_dec_ring = rte_ring_create("enc_to_dec_ring", 1089 rte_align32pow2(NB_MBUF), rte_socket_id(), flags); 1090 1091 /* Get the number of available bbdev devices */ 1092 nb_bbdevs = rte_bbdev_count(); 1093 if (nb_bbdevs <= app_params.bbdev_id) 1094 rte_exit(EXIT_FAILURE, 1095 "%u BBDevs detected, cannot use BBDev with ID %u!\n", 1096 nb_bbdevs, app_params.bbdev_id); 1097 printf("Number of bbdevs detected: %d\n", nb_bbdevs); 1098 1099 if (!rte_eth_dev_is_valid_port(app_params.port_id)) 1100 rte_exit(EXIT_FAILURE, 1101 "cannot use port with ID %u!\n", 1102 app_params.port_id); 1103 1104 /* create the mbuf mempool for ethdev pkts */ 1105 ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool", 1106 NB_MBUF, MEMPOOL_CACHE_SIZE, 0, 1107 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 1108 if (ethdev_mbuf_mempool == NULL) 1109 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n"); 1110 1111 /* create the mbuf mempool for encoder output */ 1112 bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool", 1113 NB_MBUF, MEMPOOL_CACHE_SIZE, 0, 1114 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 1115 if (bbdev_mbuf_mempool == NULL) 1116 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n"); 1117 1118 /* initialize ports */ 1119 ret = initialize_ports(&app_params, ethdev_mbuf_mempool); 1120 1121 /* Check if all requested lcores are available */ 1122 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) 1123 if (((1ULL << lcore_id) & app_params.enc_core_mask) || 1124 ((1ULL << lcore_id) & app_params.dec_core_mask)) 1125 if (!rte_lcore_is_enabled(lcore_id)) 1126 rte_exit(EXIT_FAILURE, 1127 "Requested lcore_id %u is not enabled!\n", 1128 lcore_id); 1129 1130 /* Start ethernet port */ 1131 ret = rte_eth_dev_start(app_params.port_id); 1132 if (ret < 0) 1133 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", 1134 ret, app_params.port_id); 1135 1136 ret = check_port_link_status(app_params.port_id); 1137 if (ret < 0) 1138 exit(EXIT_FAILURE); 1139 1140 /* start BBDevice and save BBDev queue IDs */ 1141 enable_bbdev(&app_params); 1142 1143 /* Initialize the port/queue configuration of each logical core */ 1144 lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools, 1145 bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats); 1146 1147 stats_lcore.app_params = &app_params; 1148 stats_lcore.lconf = lcore_conf; 1149 1150 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1151 if (lcore_conf[lcore_id].core_type != 0) 1152 /* launch per-lcore processing loop on slave lcores */ 1153 rte_eal_remote_launch(processing_loop, 1154 &lcore_conf[lcore_id], lcore_id); 1155 else if (!stats_thread_started) { 1156 /* launch statistics printing loop */ 1157 rte_eal_remote_launch(stats_loop, &stats_lcore, 1158 lcore_id); 1159 stats_thread_started = true; 1160 } 1161 } 1162 1163 if (!stats_thread_started && 1164 lcore_conf[master_lcore_id].core_type != 0) 1165 rte_exit(EXIT_FAILURE, 1166 "Not enough lcores to run the statistics printing loop!"); 1167 else if (lcore_conf[master_lcore_id].core_type != 0) 1168 processing_loop(&lcore_conf[master_lcore_id]); 1169 else if (!stats_thread_started) 1170 stats_loop(&stats_lcore); 1171 1172 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1173 ret |= rte_eal_wait_lcore(lcore_id); 1174 } 1175 1176 return ret; 1177 } 1178