1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <arpa/inet.h> 6 #include <errno.h> 7 #include <getopt.h> 8 #include <inttypes.h> 9 #include <signal.h> 10 #include <stdarg.h> 11 #include <stdbool.h> 12 #include <stdint.h> 13 #include <stdio.h> 14 #include <stdlib.h> 15 #include <string.h> 16 #include <sys/socket.h> 17 #include <sys/types.h> 18 #include <sys/queue.h> 19 #include <unistd.h> 20 21 #include <rte_branch_prediction.h> 22 #include <rte_common.h> 23 #include <rte_cycles.h> 24 #include <rte_eal.h> 25 #include <rte_ethdev.h> 26 #include <rte_graph_worker.h> 27 #include <rte_launch.h> 28 #include <rte_lcore.h> 29 #include <rte_log.h> 30 #include <rte_mempool.h> 31 #include <rte_node_eth_api.h> 32 #include <rte_node_ip4_api.h> 33 #include <rte_per_lcore.h> 34 #include <rte_string_fns.h> 35 #include <rte_vect.h> 36 37 #include <cmdline_parse.h> 38 #include <cmdline_parse_etheraddr.h> 39 40 /* Log type */ 41 #define RTE_LOGTYPE_L3FWD_GRAPH RTE_LOGTYPE_USER1 42 43 /* 44 * Configurable number of RX/TX ring descriptors 45 */ 46 #define RTE_TEST_RX_DESC_DEFAULT 1024 47 #define RTE_TEST_TX_DESC_DEFAULT 1024 48 49 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 50 #define MAX_RX_QUEUE_PER_PORT 128 51 52 #define MAX_RX_QUEUE_PER_LCORE 16 53 54 #define MAX_LCORE_PARAMS 1024 55 56 #define NB_SOCKETS 8 57 58 /* Static global variables used within this file. */ 59 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 60 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 61 62 /**< Ports set in promiscuous mode off by default. */ 63 static int promiscuous_on; 64 65 static int numa_on = 1; /**< NUMA is enabled by default. */ 66 static int per_port_pool; /**< Use separate buffer pools per port; disabled */ 67 /**< by default */ 68 69 static volatile bool force_quit; 70 71 /* Ethernet addresses of ports */ 72 static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS]; 73 static struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; 74 xmm_t val_eth[RTE_MAX_ETHPORTS]; 75 76 /* Mask of enabled ports */ 77 static uint32_t enabled_port_mask; 78 79 struct lcore_rx_queue { 80 uint16_t port_id; 81 uint8_t queue_id; 82 char node_name[RTE_NODE_NAMESIZE]; 83 }; 84 85 /* Lcore conf */ 86 struct lcore_conf { 87 uint16_t n_rx_queue; 88 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 89 90 struct rte_graph *graph; 91 char name[RTE_GRAPH_NAMESIZE]; 92 rte_graph_t graph_id; 93 } __rte_cache_aligned; 94 95 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 96 97 struct lcore_params { 98 uint16_t port_id; 99 uint8_t queue_id; 100 uint8_t lcore_id; 101 } __rte_cache_aligned; 102 103 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 104 static struct lcore_params lcore_params_array_default[] = { 105 {0, 0, 2}, {0, 1, 2}, {0, 2, 2}, {1, 0, 2}, {1, 1, 2}, 106 {1, 2, 2}, {2, 0, 2}, {3, 0, 3}, {3, 1, 3}, 107 }; 108 109 static struct lcore_params *lcore_params = lcore_params_array_default; 110 static uint16_t nb_lcore_params = RTE_DIM(lcore_params_array_default); 111 112 static struct rte_eth_conf port_conf = { 113 .rxmode = { 114 .mq_mode = RTE_ETH_MQ_RX_RSS, 115 .split_hdr_size = 0, 116 }, 117 .rx_adv_conf = { 118 .rss_conf = { 119 .rss_key = NULL, 120 .rss_hf = RTE_ETH_RSS_IP, 121 }, 122 }, 123 .txmode = { 124 .mq_mode = RTE_ETH_MQ_TX_NONE, 125 }, 126 }; 127 128 static uint32_t max_pkt_len; 129 130 static struct rte_mempool *pktmbuf_pool[RTE_MAX_ETHPORTS][NB_SOCKETS]; 131 132 static struct rte_node_ethdev_config ethdev_conf[RTE_MAX_ETHPORTS]; 133 134 struct ipv4_l3fwd_lpm_route { 135 uint32_t ip; 136 uint8_t depth; 137 uint8_t if_out; 138 }; 139 140 #define IPV4_L3FWD_LPM_NUM_ROUTES \ 141 (sizeof(ipv4_l3fwd_lpm_route_array) / \ 142 sizeof(ipv4_l3fwd_lpm_route_array[0])) 143 /* 198.18.0.0/16 are set aside for RFC2544 benchmarking. */ 144 static struct ipv4_l3fwd_lpm_route ipv4_l3fwd_lpm_route_array[] = { 145 {RTE_IPV4(198, 18, 0, 0), 24, 0}, {RTE_IPV4(198, 18, 1, 0), 24, 1}, 146 {RTE_IPV4(198, 18, 2, 0), 24, 2}, {RTE_IPV4(198, 18, 3, 0), 24, 3}, 147 {RTE_IPV4(198, 18, 4, 0), 24, 4}, {RTE_IPV4(198, 18, 5, 0), 24, 5}, 148 {RTE_IPV4(198, 18, 6, 0), 24, 6}, {RTE_IPV4(198, 18, 7, 0), 24, 7}, 149 }; 150 151 static int 152 check_lcore_params(void) 153 { 154 uint8_t queue, lcore; 155 int socketid; 156 uint16_t i; 157 158 for (i = 0; i < nb_lcore_params; ++i) { 159 queue = lcore_params[i].queue_id; 160 if (queue >= MAX_RX_QUEUE_PER_PORT) { 161 printf("Invalid queue number: %hhu\n", queue); 162 return -1; 163 } 164 lcore = lcore_params[i].lcore_id; 165 if (!rte_lcore_is_enabled(lcore)) { 166 printf("Error: lcore %hhu is not enabled in lcore mask\n", 167 lcore); 168 return -1; 169 } 170 171 if (lcore == rte_get_main_lcore()) { 172 printf("Error: lcore %u is main lcore\n", lcore); 173 return -1; 174 } 175 socketid = rte_lcore_to_socket_id(lcore); 176 if ((socketid != 0) && (numa_on == 0)) { 177 printf("Warning: lcore %hhu is on socket %d with numa off\n", 178 lcore, socketid); 179 } 180 } 181 182 return 0; 183 } 184 185 static int 186 check_port_config(void) 187 { 188 uint16_t portid; 189 uint16_t i; 190 191 for (i = 0; i < nb_lcore_params; ++i) { 192 portid = lcore_params[i].port_id; 193 if ((enabled_port_mask & (1 << portid)) == 0) { 194 printf("Port %u is not enabled in port mask\n", portid); 195 return -1; 196 } 197 if (!rte_eth_dev_is_valid_port(portid)) { 198 printf("Port %u is not present on the board\n", portid); 199 return -1; 200 } 201 } 202 203 return 0; 204 } 205 206 static uint8_t 207 get_port_n_rx_queues(const uint16_t port) 208 { 209 int queue = -1; 210 uint16_t i; 211 212 for (i = 0; i < nb_lcore_params; ++i) { 213 if (lcore_params[i].port_id == port) { 214 if (lcore_params[i].queue_id == queue + 1) 215 queue = lcore_params[i].queue_id; 216 else 217 rte_exit(EXIT_FAILURE, 218 "Queue ids of the port %d must be" 219 " in sequence and must start with 0\n", 220 lcore_params[i].port_id); 221 } 222 } 223 224 return (uint8_t)(++queue); 225 } 226 227 static int 228 init_lcore_rx_queues(void) 229 { 230 uint16_t i, nb_rx_queue; 231 uint8_t lcore; 232 233 for (i = 0; i < nb_lcore_params; ++i) { 234 lcore = lcore_params[i].lcore_id; 235 nb_rx_queue = lcore_conf[lcore].n_rx_queue; 236 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 237 printf("Error: too many queues (%u) for lcore: %u\n", 238 (unsigned int)nb_rx_queue + 1, 239 (unsigned int)lcore); 240 return -1; 241 } 242 243 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 244 lcore_params[i].port_id; 245 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 246 lcore_params[i].queue_id; 247 lcore_conf[lcore].n_rx_queue++; 248 } 249 250 return 0; 251 } 252 253 /* Display usage */ 254 static void 255 print_usage(const char *prgname) 256 { 257 fprintf(stderr, 258 "%s [EAL options] --" 259 " -p PORTMASK" 260 " [-P]" 261 " --config (port,queue,lcore)[,(port,queue,lcore)]" 262 " [--eth-dest=X,MM:MM:MM:MM:MM:MM]" 263 " [--max-pkt-len PKTLEN]" 264 " [--no-numa]" 265 " [--per-port-pool]\n\n" 266 267 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 268 " -P : Enable promiscuous mode\n" 269 " --config (port,queue,lcore): Rx queue configuration\n" 270 " --eth-dest=X,MM:MM:MM:MM:MM:MM: Ethernet destination for " 271 "port X\n" 272 " --max-pkt-len PKTLEN: maximum packet length in decimal (64-9600)\n" 273 " --no-numa: Disable numa awareness\n" 274 " --per-port-pool: Use separate buffer pool per port\n\n", 275 prgname); 276 } 277 278 static int 279 parse_max_pkt_len(const char *pktlen) 280 { 281 unsigned long len; 282 char *end = NULL; 283 284 /* Parse decimal string */ 285 len = strtoul(pktlen, &end, 10); 286 if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) 287 return -1; 288 289 if (len == 0) 290 return -1; 291 292 return len; 293 } 294 295 static int 296 parse_portmask(const char *portmask) 297 { 298 char *end = NULL; 299 unsigned long pm; 300 301 /* Parse hexadecimal string */ 302 pm = strtoul(portmask, &end, 16); 303 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 304 return 0; 305 306 return pm; 307 } 308 309 static int 310 parse_config(const char *q_arg) 311 { 312 enum fieldnames { FLD_PORT = 0, FLD_QUEUE, FLD_LCORE, _NUM_FLD }; 313 unsigned long int_fld[_NUM_FLD]; 314 const char *p, *p0 = q_arg; 315 char *str_fld[_NUM_FLD]; 316 uint32_t size; 317 char s[256]; 318 char *end; 319 int i; 320 321 nb_lcore_params = 0; 322 323 while ((p = strchr(p0, '(')) != NULL) { 324 ++p; 325 p0 = strchr(p, ')'); 326 if (p0 == NULL) 327 return -1; 328 329 size = p0 - p; 330 if (size >= sizeof(s)) 331 return -1; 332 333 memcpy(s, p, size); 334 s[size] = '\0'; 335 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 336 _NUM_FLD) 337 return -1; 338 for (i = 0; i < _NUM_FLD; i++) { 339 errno = 0; 340 int_fld[i] = strtoul(str_fld[i], &end, 0); 341 if (errno != 0 || end == str_fld[i]) 342 return -1; 343 } 344 345 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 346 printf("Exceeded max number of lcore params: %hu\n", 347 nb_lcore_params); 348 return -1; 349 } 350 351 if (int_fld[FLD_PORT] >= RTE_MAX_ETHPORTS || 352 int_fld[FLD_LCORE] >= RTE_MAX_LCORE) { 353 printf("Invalid port/lcore id\n"); 354 return -1; 355 } 356 357 lcore_params_array[nb_lcore_params].port_id = 358 (uint8_t)int_fld[FLD_PORT]; 359 lcore_params_array[nb_lcore_params].queue_id = 360 (uint8_t)int_fld[FLD_QUEUE]; 361 lcore_params_array[nb_lcore_params].lcore_id = 362 (uint8_t)int_fld[FLD_LCORE]; 363 ++nb_lcore_params; 364 } 365 lcore_params = lcore_params_array; 366 367 return 0; 368 } 369 370 static void 371 parse_eth_dest(const char *optarg) 372 { 373 uint8_t c, *dest, peer_addr[6]; 374 uint16_t portid; 375 char *port_end; 376 377 errno = 0; 378 portid = strtoul(optarg, &port_end, 10); 379 if (errno != 0 || port_end == optarg || *port_end++ != ',') 380 rte_exit(EXIT_FAILURE, "Invalid eth-dest: %s", optarg); 381 if (portid >= RTE_MAX_ETHPORTS) 382 rte_exit(EXIT_FAILURE, 383 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n", portid, 384 RTE_MAX_ETHPORTS); 385 386 if (cmdline_parse_etheraddr(NULL, port_end, &peer_addr, 387 sizeof(peer_addr)) < 0) 388 rte_exit(EXIT_FAILURE, "Invalid ethernet address: %s\n", 389 port_end); 390 dest = (uint8_t *)&dest_eth_addr[portid]; 391 for (c = 0; c < 6; c++) 392 dest[c] = peer_addr[c]; 393 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid]; 394 } 395 396 #define MAX_JUMBO_PKT_LEN 9600 397 #define MEMPOOL_CACHE_SIZE 256 398 399 static const char short_options[] = "p:" /* portmask */ 400 "P" /* promiscuous */ 401 ; 402 403 #define CMD_LINE_OPT_CONFIG "config" 404 #define CMD_LINE_OPT_ETH_DEST "eth-dest" 405 #define CMD_LINE_OPT_NO_NUMA "no-numa" 406 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len" 407 #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool" 408 enum { 409 /* Long options mapped to a short option */ 410 411 /* First long only option value must be >= 256, so that we won't 412 * conflict with short options 413 */ 414 CMD_LINE_OPT_MIN_NUM = 256, 415 CMD_LINE_OPT_CONFIG_NUM, 416 CMD_LINE_OPT_ETH_DEST_NUM, 417 CMD_LINE_OPT_NO_NUMA_NUM, 418 CMD_LINE_OPT_MAX_PKT_LEN_NUM, 419 CMD_LINE_OPT_PARSE_PER_PORT_POOL, 420 }; 421 422 static const struct option lgopts[] = { 423 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 424 {CMD_LINE_OPT_ETH_DEST, 1, 0, CMD_LINE_OPT_ETH_DEST_NUM}, 425 {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM}, 426 {CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM}, 427 {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL}, 428 {NULL, 0, 0, 0}, 429 }; 430 431 /* 432 * This expression is used to calculate the number of mbufs needed 433 * depending on user input, taking into account memory for rx and 434 * tx hardware rings, cache per lcore and mtable per port per lcore. 435 * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum 436 * value of 8192 437 */ 438 #define NB_MBUF(nports) \ 439 RTE_MAX((nports * nb_rx_queue * nb_rxd + \ 440 nports * nb_lcores * RTE_GRAPH_BURST_SIZE + \ 441 nports * n_tx_queue * nb_txd + \ 442 nb_lcores * MEMPOOL_CACHE_SIZE), 8192u) 443 444 /* Parse the argument given in the command line of the application */ 445 static int 446 parse_args(int argc, char **argv) 447 { 448 char *prgname = argv[0]; 449 int option_index; 450 char **argvopt; 451 int opt, ret; 452 453 argvopt = argv; 454 455 /* Error or normal output strings. */ 456 while ((opt = getopt_long(argc, argvopt, short_options, lgopts, 457 &option_index)) != EOF) { 458 459 switch (opt) { 460 /* Portmask */ 461 case 'p': 462 enabled_port_mask = parse_portmask(optarg); 463 if (enabled_port_mask == 0) { 464 fprintf(stderr, "Invalid portmask\n"); 465 print_usage(prgname); 466 return -1; 467 } 468 break; 469 470 case 'P': 471 promiscuous_on = 1; 472 break; 473 474 /* Long options */ 475 case CMD_LINE_OPT_CONFIG_NUM: 476 ret = parse_config(optarg); 477 if (ret) { 478 fprintf(stderr, "Invalid config\n"); 479 print_usage(prgname); 480 return -1; 481 } 482 break; 483 484 case CMD_LINE_OPT_ETH_DEST_NUM: 485 parse_eth_dest(optarg); 486 break; 487 488 case CMD_LINE_OPT_NO_NUMA_NUM: 489 numa_on = 0; 490 break; 491 492 case CMD_LINE_OPT_MAX_PKT_LEN_NUM: { 493 max_pkt_len = parse_max_pkt_len(optarg); 494 break; 495 } 496 497 case CMD_LINE_OPT_PARSE_PER_PORT_POOL: 498 printf("Per port buffer pool is enabled\n"); 499 per_port_pool = 1; 500 break; 501 502 default: 503 print_usage(prgname); 504 return -1; 505 } 506 } 507 508 if (optind >= 0) 509 argv[optind - 1] = prgname; 510 ret = optind - 1; 511 optind = 1; /* Reset getopt lib */ 512 513 return ret; 514 } 515 516 static void 517 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 518 { 519 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 520 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 521 printf("%s%s", name, buf); 522 } 523 524 static int 525 init_mem(uint16_t portid, uint32_t nb_mbuf) 526 { 527 uint32_t lcore_id; 528 int socketid; 529 char s[64]; 530 531 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 532 if (rte_lcore_is_enabled(lcore_id) == 0) 533 continue; 534 535 if (numa_on) 536 socketid = rte_lcore_to_socket_id(lcore_id); 537 else 538 socketid = 0; 539 540 if (socketid >= NB_SOCKETS) { 541 rte_exit(EXIT_FAILURE, 542 "Socket %d of lcore %u is out of range %d\n", 543 socketid, lcore_id, NB_SOCKETS); 544 } 545 546 if (pktmbuf_pool[portid][socketid] == NULL) { 547 snprintf(s, sizeof(s), "mbuf_pool_%d:%d", portid, 548 socketid); 549 /* Create a pool with priv size of a cacheline */ 550 pktmbuf_pool[portid][socketid] = 551 rte_pktmbuf_pool_create( 552 s, nb_mbuf, MEMPOOL_CACHE_SIZE, 553 RTE_CACHE_LINE_SIZE, 554 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 555 if (pktmbuf_pool[portid][socketid] == NULL) 556 rte_exit(EXIT_FAILURE, 557 "Cannot init mbuf pool on socket %d\n", 558 socketid); 559 else 560 printf("Allocated mbuf pool on socket %d\n", 561 socketid); 562 } 563 } 564 565 return 0; 566 } 567 568 /* Check the link status of all ports in up to 9s, and print them finally */ 569 static void 570 check_all_ports_link_status(uint32_t port_mask) 571 { 572 #define CHECK_INTERVAL 100 /* 100ms */ 573 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 574 uint8_t count, all_ports_up, print_flag = 0; 575 struct rte_eth_link link; 576 uint16_t portid; 577 int ret; 578 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 579 580 printf("\nChecking link status"); 581 fflush(stdout); 582 for (count = 0; count <= MAX_CHECK_TIME; count++) { 583 if (force_quit) 584 return; 585 all_ports_up = 1; 586 RTE_ETH_FOREACH_DEV(portid) 587 { 588 if (force_quit) 589 return; 590 if ((port_mask & (1 << portid)) == 0) 591 continue; 592 memset(&link, 0, sizeof(link)); 593 ret = rte_eth_link_get_nowait(portid, &link); 594 if (ret < 0) { 595 all_ports_up = 0; 596 if (print_flag == 1) 597 printf("Port %u link get failed: %s\n", 598 portid, rte_strerror(-ret)); 599 continue; 600 } 601 /* Print link status if flag set */ 602 if (print_flag == 1) { 603 rte_eth_link_to_str(link_status_text, 604 sizeof(link_status_text), &link); 605 printf("Port %d %s\n", portid, 606 link_status_text); 607 continue; 608 } 609 /* Clear all_ports_up flag if any link down */ 610 if (link.link_status == RTE_ETH_LINK_DOWN) { 611 all_ports_up = 0; 612 break; 613 } 614 } 615 /* After finally printing all link status, get out */ 616 if (print_flag == 1) 617 break; 618 619 if (all_ports_up == 0) { 620 printf("."); 621 fflush(stdout); 622 rte_delay_ms(CHECK_INTERVAL); 623 } 624 625 /* Set the print_flag if all ports up or timeout */ 626 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 627 print_flag = 1; 628 printf("Done\n"); 629 } 630 } 631 } 632 633 static void 634 signal_handler(int signum) 635 { 636 if (signum == SIGINT || signum == SIGTERM) { 637 printf("\n\nSignal %d received, preparing to exit...\n", 638 signum); 639 force_quit = true; 640 } 641 } 642 643 static void 644 print_stats(void) 645 { 646 const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; 647 const char clr[] = {27, '[', '2', 'J', '\0'}; 648 struct rte_graph_cluster_stats_param s_param; 649 struct rte_graph_cluster_stats *stats; 650 const char *pattern = "worker_*"; 651 652 /* Prepare stats object */ 653 memset(&s_param, 0, sizeof(s_param)); 654 s_param.f = stdout; 655 s_param.socket_id = SOCKET_ID_ANY; 656 s_param.graph_patterns = &pattern; 657 s_param.nb_graph_patterns = 1; 658 659 stats = rte_graph_cluster_stats_create(&s_param); 660 if (stats == NULL) 661 rte_exit(EXIT_FAILURE, "Unable to create stats object\n"); 662 663 while (!force_quit) { 664 /* Clear screen and move to top left */ 665 printf("%s%s", clr, topLeft); 666 rte_graph_cluster_stats_get(stats, 0); 667 rte_delay_ms(1E3); 668 } 669 670 rte_graph_cluster_stats_destroy(stats); 671 } 672 673 /* Main processing loop. 8< */ 674 static int 675 graph_main_loop(void *conf) 676 { 677 struct lcore_conf *qconf; 678 struct rte_graph *graph; 679 uint32_t lcore_id; 680 681 RTE_SET_USED(conf); 682 683 lcore_id = rte_lcore_id(); 684 qconf = &lcore_conf[lcore_id]; 685 graph = qconf->graph; 686 687 if (!graph) { 688 RTE_LOG(INFO, L3FWD_GRAPH, "Lcore %u has nothing to do\n", 689 lcore_id); 690 return 0; 691 } 692 693 RTE_LOG(INFO, L3FWD_GRAPH, 694 "Entering main loop on lcore %u, graph %s(%p)\n", lcore_id, 695 qconf->name, graph); 696 697 while (likely(!force_quit)) 698 rte_graph_walk(graph); 699 700 return 0; 701 } 702 /* >8 End of main processing loop. */ 703 704 static uint32_t 705 eth_dev_get_overhead_len(uint32_t max_rx_pktlen, uint16_t max_mtu) 706 { 707 uint32_t overhead_len; 708 709 if (max_mtu != UINT16_MAX && max_rx_pktlen > max_mtu) 710 overhead_len = max_rx_pktlen - max_mtu; 711 else 712 overhead_len = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 713 714 return overhead_len; 715 } 716 717 static int 718 config_port_max_pkt_len(struct rte_eth_conf *conf, 719 struct rte_eth_dev_info *dev_info) 720 { 721 uint32_t overhead_len; 722 723 if (max_pkt_len == 0) 724 return 0; 725 726 if (max_pkt_len < RTE_ETHER_MIN_LEN || max_pkt_len > MAX_JUMBO_PKT_LEN) 727 return -1; 728 729 overhead_len = eth_dev_get_overhead_len(dev_info->max_rx_pktlen, 730 dev_info->max_mtu); 731 conf->rxmode.mtu = max_pkt_len - overhead_len; 732 733 if (conf->rxmode.mtu > RTE_ETHER_MTU) 734 conf->txmode.offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 735 736 return 0; 737 } 738 739 int 740 main(int argc, char **argv) 741 { 742 /* Rewrite data of src and dst ether addr */ 743 uint8_t rewrite_data[2 * sizeof(struct rte_ether_addr)]; 744 /* Graph initialization. 8< */ 745 static const char * const default_patterns[] = { 746 "ip4*", 747 "ethdev_tx-*", 748 "pkt_drop", 749 }; 750 uint8_t nb_rx_queue, queue, socketid; 751 struct rte_graph_param graph_conf; 752 struct rte_eth_dev_info dev_info; 753 uint32_t nb_ports, nb_conf = 0; 754 uint32_t n_tx_queue, nb_lcores; 755 struct rte_eth_txconf *txconf; 756 uint16_t queueid, portid, i; 757 const char **node_patterns; 758 struct lcore_conf *qconf; 759 uint16_t nb_graphs = 0; 760 uint16_t nb_patterns; 761 uint8_t rewrite_len; 762 uint32_t lcore_id; 763 int ret; 764 765 /* Init EAL */ 766 ret = rte_eal_init(argc, argv); 767 if (ret < 0) 768 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 769 argc -= ret; 770 argv += ret; 771 772 force_quit = false; 773 signal(SIGINT, signal_handler); 774 signal(SIGTERM, signal_handler); 775 776 /* Pre-init dst MACs for all ports to 02:00:00:00:00:xx */ 777 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 778 dest_eth_addr[portid] = 779 RTE_ETHER_LOCAL_ADMIN_ADDR + ((uint64_t)portid << 40); 780 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid]; 781 } 782 783 /* Parse application arguments (after the EAL ones) */ 784 ret = parse_args(argc, argv); 785 if (ret < 0) 786 rte_exit(EXIT_FAILURE, "Invalid L3FWD_GRAPH parameters\n"); 787 788 if (check_lcore_params() < 0) 789 rte_exit(EXIT_FAILURE, "check_lcore_params() failed\n"); 790 791 ret = init_lcore_rx_queues(); 792 if (ret < 0) 793 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues() failed\n"); 794 795 if (check_port_config() < 0) 796 rte_exit(EXIT_FAILURE, "check_port_config() failed\n"); 797 798 nb_ports = rte_eth_dev_count_avail(); 799 nb_lcores = rte_lcore_count(); 800 801 /* Initialize all ports. 8< */ 802 RTE_ETH_FOREACH_DEV(portid) 803 { 804 struct rte_eth_conf local_port_conf = port_conf; 805 806 /* Skip ports that are not enabled */ 807 if ((enabled_port_mask & (1 << portid)) == 0) { 808 printf("\nSkipping disabled port %d\n", portid); 809 continue; 810 } 811 812 /* Init port */ 813 printf("Initializing port %d ... ", portid); 814 fflush(stdout); 815 816 nb_rx_queue = get_port_n_rx_queues(portid); 817 n_tx_queue = nb_lcores; 818 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT) 819 n_tx_queue = MAX_TX_QUEUE_PER_PORT; 820 printf("Creating queues: nb_rxq=%d nb_txq=%u... ", 821 nb_rx_queue, n_tx_queue); 822 823 rte_eth_dev_info_get(portid, &dev_info); 824 825 ret = config_port_max_pkt_len(&local_port_conf, &dev_info); 826 if (ret != 0) 827 rte_exit(EXIT_FAILURE, 828 "Invalid max packet length: %u (port %u)\n", 829 max_pkt_len, portid); 830 831 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 832 local_port_conf.txmode.offloads |= 833 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 834 835 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 836 dev_info.flow_type_rss_offloads; 837 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 838 port_conf.rx_adv_conf.rss_conf.rss_hf) { 839 printf("Port %u modified RSS hash function based on " 840 "hardware support," 841 "requested:%#" PRIx64 " configured:%#" PRIx64 842 "\n", 843 portid, port_conf.rx_adv_conf.rss_conf.rss_hf, 844 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 845 } 846 847 ret = rte_eth_dev_configure(portid, nb_rx_queue, 848 n_tx_queue, &local_port_conf); 849 if (ret < 0) 850 rte_exit(EXIT_FAILURE, 851 "Cannot configure device: err=%d, port=%d\n", 852 ret, portid); 853 854 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 855 &nb_txd); 856 if (ret < 0) 857 rte_exit(EXIT_FAILURE, 858 "Cannot adjust number of descriptors: err=%d, " 859 "port=%d\n", 860 ret, portid); 861 862 rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 863 print_ethaddr(" Address:", &ports_eth_addr[portid]); 864 printf(", "); 865 print_ethaddr( 866 "Destination:", 867 (const struct rte_ether_addr *)&dest_eth_addr[portid]); 868 printf(", "); 869 870 /* 871 * prepare src MACs for each port. 872 */ 873 rte_ether_addr_copy( 874 &ports_eth_addr[portid], 875 (struct rte_ether_addr *)(val_eth + portid) + 1); 876 877 /* Init memory */ 878 if (!per_port_pool) { 879 /* portid = 0; this is *not* signifying the first port, 880 * rather, it signifies that portid is ignored. 881 */ 882 ret = init_mem(0, NB_MBUF(nb_ports)); 883 } else { 884 ret = init_mem(portid, NB_MBUF(1)); 885 } 886 if (ret < 0) 887 rte_exit(EXIT_FAILURE, "init_mem() failed\n"); 888 889 /* Init one TX queue per couple (lcore,port) */ 890 queueid = 0; 891 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 892 if (rte_lcore_is_enabled(lcore_id) == 0) 893 continue; 894 895 qconf = &lcore_conf[lcore_id]; 896 897 if (numa_on) 898 socketid = (uint8_t)rte_lcore_to_socket_id( 899 lcore_id); 900 else 901 socketid = 0; 902 903 printf("txq=%u,%d,%d ", lcore_id, queueid, socketid); 904 fflush(stdout); 905 906 txconf = &dev_info.default_txconf; 907 txconf->offloads = local_port_conf.txmode.offloads; 908 ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, 909 socketid, txconf); 910 if (ret < 0) 911 rte_exit(EXIT_FAILURE, 912 "rte_eth_tx_queue_setup: err=%d, " 913 "port=%d\n", 914 ret, portid); 915 queueid++; 916 } 917 918 /* Setup ethdev node config */ 919 ethdev_conf[nb_conf].port_id = portid; 920 ethdev_conf[nb_conf].num_rx_queues = nb_rx_queue; 921 ethdev_conf[nb_conf].num_tx_queues = n_tx_queue; 922 if (!per_port_pool) 923 ethdev_conf[nb_conf].mp = pktmbuf_pool[0]; 924 925 else 926 ethdev_conf[nb_conf].mp = pktmbuf_pool[portid]; 927 ethdev_conf[nb_conf].mp_count = NB_SOCKETS; 928 929 nb_conf++; 930 printf("\n"); 931 } 932 933 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 934 if (rte_lcore_is_enabled(lcore_id) == 0) 935 continue; 936 qconf = &lcore_conf[lcore_id]; 937 printf("\nInitializing rx queues on lcore %u ... ", lcore_id); 938 fflush(stdout); 939 /* Init RX queues */ 940 for (queue = 0; queue < qconf->n_rx_queue; ++queue) { 941 struct rte_eth_rxconf rxq_conf; 942 943 portid = qconf->rx_queue_list[queue].port_id; 944 queueid = qconf->rx_queue_list[queue].queue_id; 945 946 if (numa_on) 947 socketid = (uint8_t)rte_lcore_to_socket_id( 948 lcore_id); 949 else 950 socketid = 0; 951 952 printf("rxq=%d,%d,%d ", portid, queueid, socketid); 953 fflush(stdout); 954 955 rte_eth_dev_info_get(portid, &dev_info); 956 rxq_conf = dev_info.default_rxconf; 957 rxq_conf.offloads = port_conf.rxmode.offloads; 958 if (!per_port_pool) 959 ret = rte_eth_rx_queue_setup( 960 portid, queueid, nb_rxd, socketid, 961 &rxq_conf, pktmbuf_pool[0][socketid]); 962 else 963 ret = rte_eth_rx_queue_setup( 964 portid, queueid, nb_rxd, socketid, 965 &rxq_conf, 966 pktmbuf_pool[portid][socketid]); 967 if (ret < 0) 968 rte_exit(EXIT_FAILURE, 969 "rte_eth_rx_queue_setup: err=%d, " 970 "port=%d\n", 971 ret, portid); 972 973 /* Add this queue node to its graph */ 974 snprintf(qconf->rx_queue_list[queue].node_name, 975 RTE_NODE_NAMESIZE, "ethdev_rx-%u-%u", portid, 976 queueid); 977 } 978 979 /* Alloc a graph to this lcore only if source exists */ 980 if (qconf->n_rx_queue) 981 nb_graphs++; 982 } 983 984 printf("\n"); 985 986 /* Ethdev node config, skip rx queue mapping */ 987 ret = rte_node_eth_config(ethdev_conf, nb_conf, nb_graphs); 988 /* >8 End of graph creation. */ 989 if (ret) 990 rte_exit(EXIT_FAILURE, "rte_node_eth_config: err=%d\n", ret); 991 992 /* Start ports */ 993 RTE_ETH_FOREACH_DEV(portid) 994 { 995 if ((enabled_port_mask & (1 << portid)) == 0) 996 continue; 997 998 /* Start device */ 999 ret = rte_eth_dev_start(portid); 1000 if (ret < 0) 1001 rte_exit(EXIT_FAILURE, 1002 "rte_eth_dev_start: err=%d, port=%d\n", ret, 1003 portid); 1004 1005 /* 1006 * If enabled, put device in promiscuous mode. 1007 * This allows IO forwarding mode to forward packets 1008 * to itself through 2 cross-connected ports of the 1009 * target machine. 1010 */ 1011 if (promiscuous_on) 1012 rte_eth_promiscuous_enable(portid); 1013 } 1014 1015 printf("\n"); 1016 1017 check_all_ports_link_status(enabled_port_mask); 1018 1019 /* Graph Initialization */ 1020 nb_patterns = RTE_DIM(default_patterns); 1021 node_patterns = malloc((MAX_RX_QUEUE_PER_LCORE + nb_patterns) * 1022 sizeof(*node_patterns)); 1023 if (!node_patterns) 1024 return -ENOMEM; 1025 memcpy(node_patterns, default_patterns, 1026 nb_patterns * sizeof(*node_patterns)); 1027 1028 memset(&graph_conf, 0, sizeof(graph_conf)); 1029 graph_conf.node_patterns = node_patterns; 1030 1031 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1032 rte_graph_t graph_id; 1033 rte_edge_t i; 1034 1035 if (rte_lcore_is_enabled(lcore_id) == 0) 1036 continue; 1037 1038 qconf = &lcore_conf[lcore_id]; 1039 1040 /* Skip graph creation if no source exists */ 1041 if (!qconf->n_rx_queue) 1042 continue; 1043 1044 /* Add rx node patterns of this lcore */ 1045 for (i = 0; i < qconf->n_rx_queue; i++) { 1046 graph_conf.node_patterns[nb_patterns + i] = 1047 qconf->rx_queue_list[i].node_name; 1048 } 1049 1050 graph_conf.nb_node_patterns = nb_patterns + i; 1051 graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id); 1052 1053 snprintf(qconf->name, sizeof(qconf->name), "worker_%u", 1054 lcore_id); 1055 1056 graph_id = rte_graph_create(qconf->name, &graph_conf); 1057 if (graph_id == RTE_GRAPH_ID_INVALID) 1058 rte_exit(EXIT_FAILURE, 1059 "rte_graph_create(): graph_id invalid" 1060 " for lcore %u\n", lcore_id); 1061 1062 qconf->graph_id = graph_id; 1063 qconf->graph = rte_graph_lookup(qconf->name); 1064 /* >8 End of graph initialization. */ 1065 if (!qconf->graph) 1066 rte_exit(EXIT_FAILURE, 1067 "rte_graph_lookup(): graph %s not found\n", 1068 qconf->name); 1069 } 1070 1071 memset(&rewrite_data, 0, sizeof(rewrite_data)); 1072 rewrite_len = sizeof(rewrite_data); 1073 1074 /* Add route to ip4 graph infra. 8< */ 1075 for (i = 0; i < IPV4_L3FWD_LPM_NUM_ROUTES; i++) { 1076 char route_str[INET6_ADDRSTRLEN * 4]; 1077 char abuf[INET6_ADDRSTRLEN]; 1078 struct in_addr in; 1079 uint32_t dst_port; 1080 1081 /* Skip unused ports */ 1082 if ((1 << ipv4_l3fwd_lpm_route_array[i].if_out & 1083 enabled_port_mask) == 0) 1084 continue; 1085 1086 dst_port = ipv4_l3fwd_lpm_route_array[i].if_out; 1087 1088 in.s_addr = htonl(ipv4_l3fwd_lpm_route_array[i].ip); 1089 snprintf(route_str, sizeof(route_str), "%s / %d (%d)", 1090 inet_ntop(AF_INET, &in, abuf, sizeof(abuf)), 1091 ipv4_l3fwd_lpm_route_array[i].depth, 1092 ipv4_l3fwd_lpm_route_array[i].if_out); 1093 1094 /* Use route index 'i' as next hop id */ 1095 ret = rte_node_ip4_route_add( 1096 ipv4_l3fwd_lpm_route_array[i].ip, 1097 ipv4_l3fwd_lpm_route_array[i].depth, i, 1098 RTE_NODE_IP4_LOOKUP_NEXT_REWRITE); 1099 1100 if (ret < 0) 1101 rte_exit(EXIT_FAILURE, 1102 "Unable to add ip4 route %s to graph\n", 1103 route_str); 1104 1105 memcpy(rewrite_data, val_eth + dst_port, rewrite_len); 1106 1107 /* Add next hop rewrite data for id 'i' */ 1108 ret = rte_node_ip4_rewrite_add(i, rewrite_data, 1109 rewrite_len, dst_port); 1110 if (ret < 0) 1111 rte_exit(EXIT_FAILURE, 1112 "Unable to add next hop %u for " 1113 "route %s\n", i, route_str); 1114 1115 RTE_LOG(INFO, L3FWD_GRAPH, "Added route %s, next_hop %u\n", 1116 route_str, i); 1117 } 1118 /* >8 End of adding route to ip4 graph infa. */ 1119 1120 /* Launch per-lcore init on every worker lcore */ 1121 rte_eal_mp_remote_launch(graph_main_loop, NULL, SKIP_MAIN); 1122 1123 /* Accumulate and print stats on main until exit */ 1124 if (rte_graph_has_stats_feature()) 1125 print_stats(); 1126 1127 /* Wait for worker cores to exit */ 1128 ret = 0; 1129 RTE_LCORE_FOREACH_WORKER(lcore_id) { 1130 ret = rte_eal_wait_lcore(lcore_id); 1131 /* Destroy graph */ 1132 if (ret < 0 || rte_graph_destroy( 1133 rte_graph_from_name(lcore_conf[lcore_id].name))) { 1134 ret = -1; 1135 break; 1136 } 1137 } 1138 free(node_patterns); 1139 1140 /* Stop ports */ 1141 RTE_ETH_FOREACH_DEV(portid) { 1142 if ((enabled_port_mask & (1 << portid)) == 0) 1143 continue; 1144 printf("Closing port %d...", portid); 1145 ret = rte_eth_dev_stop(portid); 1146 if (ret != 0) 1147 printf("Failed to stop port %u: %s\n", 1148 portid, rte_strerror(-ret)); 1149 rte_eth_dev_close(portid); 1150 printf(" Done\n"); 1151 } 1152 1153 /* clean up the EAL */ 1154 rte_eal_cleanup(); 1155 printf("Bye...\n"); 1156 1157 return ret; 1158 } 1159