1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <arpa/inet.h> 6 #include <errno.h> 7 #include <getopt.h> 8 #include <inttypes.h> 9 #include <signal.h> 10 #include <stdarg.h> 11 #include <stdbool.h> 12 #include <stdint.h> 13 #include <stdio.h> 14 #include <stdlib.h> 15 #include <string.h> 16 #include <sys/socket.h> 17 #include <sys/types.h> 18 #include <sys/queue.h> 19 #include <unistd.h> 20 21 #include <rte_branch_prediction.h> 22 #include <rte_common.h> 23 #include <rte_cycles.h> 24 #include <rte_eal.h> 25 #include <rte_ethdev.h> 26 #include <rte_graph_worker.h> 27 #include <rte_launch.h> 28 #include <rte_lcore.h> 29 #include <rte_log.h> 30 #include <rte_mempool.h> 31 #include <rte_node_eth_api.h> 32 #include <rte_node_ip4_api.h> 33 #include <rte_per_lcore.h> 34 #include <rte_string_fns.h> 35 #include <rte_vect.h> 36 37 #include <cmdline_parse.h> 38 #include <cmdline_parse_etheraddr.h> 39 40 /* Log type */ 41 #define RTE_LOGTYPE_L3FWD_GRAPH RTE_LOGTYPE_USER1 42 43 /* 44 * Configurable number of RX/TX ring descriptors 45 */ 46 #define RX_DESC_DEFAULT 1024 47 #define TX_DESC_DEFAULT 1024 48 49 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 50 #define MAX_RX_QUEUE_PER_PORT 128 51 52 #define MAX_RX_QUEUE_PER_LCORE 16 53 54 #define MAX_LCORE_PARAMS 1024 55 56 #define NB_SOCKETS 8 57 58 /* Static global variables used within this file. */ 59 static uint16_t nb_rxd = RX_DESC_DEFAULT; 60 static uint16_t nb_txd = TX_DESC_DEFAULT; 61 62 /**< Ports set in promiscuous mode off by default. */ 63 static int promiscuous_on; 64 65 static int numa_on = 1; /**< NUMA is enabled by default. */ 66 static int per_port_pool; /**< Use separate buffer pools per port; disabled */ 67 /**< by default */ 68 69 static volatile bool force_quit; 70 71 /* Ethernet addresses of ports */ 72 static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS]; 73 static struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; 74 xmm_t val_eth[RTE_MAX_ETHPORTS]; 75 76 /* Mask of enabled ports */ 77 static uint32_t enabled_port_mask; 78 79 struct lcore_rx_queue { 80 uint16_t port_id; 81 uint8_t queue_id; 82 char node_name[RTE_NODE_NAMESIZE]; 83 }; 84 85 /* Lcore conf */ 86 struct lcore_conf { 87 uint16_t n_rx_queue; 88 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 89 90 struct rte_graph *graph; 91 char name[RTE_GRAPH_NAMESIZE]; 92 rte_graph_t graph_id; 93 } __rte_cache_aligned; 94 95 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 96 97 struct lcore_params { 98 uint16_t port_id; 99 uint8_t queue_id; 100 uint8_t lcore_id; 101 } __rte_cache_aligned; 102 103 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 104 static struct lcore_params lcore_params_array_default[] = { 105 {0, 0, 2}, {0, 1, 2}, {0, 2, 2}, {1, 0, 2}, {1, 1, 2}, 106 {1, 2, 2}, {2, 0, 2}, {3, 0, 3}, {3, 1, 3}, 107 }; 108 109 static struct lcore_params *lcore_params = lcore_params_array_default; 110 static uint16_t nb_lcore_params = RTE_DIM(lcore_params_array_default); 111 112 static struct rte_eth_conf port_conf = { 113 .rxmode = { 114 .mq_mode = RTE_ETH_MQ_RX_RSS, 115 }, 116 .rx_adv_conf = { 117 .rss_conf = { 118 .rss_key = NULL, 119 .rss_hf = RTE_ETH_RSS_IP, 120 }, 121 }, 122 .txmode = { 123 .mq_mode = RTE_ETH_MQ_TX_NONE, 124 }, 125 }; 126 127 static uint32_t max_pkt_len; 128 129 static struct rte_mempool *pktmbuf_pool[RTE_MAX_ETHPORTS][NB_SOCKETS]; 130 131 static struct rte_node_ethdev_config ethdev_conf[RTE_MAX_ETHPORTS]; 132 133 struct ipv4_l3fwd_lpm_route { 134 uint32_t ip; 135 uint8_t depth; 136 uint8_t if_out; 137 }; 138 139 #define IPV4_L3FWD_LPM_NUM_ROUTES \ 140 (sizeof(ipv4_l3fwd_lpm_route_array) / \ 141 sizeof(ipv4_l3fwd_lpm_route_array[0])) 142 /* 198.18.0.0/16 are set aside for RFC2544 benchmarking. */ 143 static struct ipv4_l3fwd_lpm_route ipv4_l3fwd_lpm_route_array[] = { 144 {RTE_IPV4(198, 18, 0, 0), 24, 0}, {RTE_IPV4(198, 18, 1, 0), 24, 1}, 145 {RTE_IPV4(198, 18, 2, 0), 24, 2}, {RTE_IPV4(198, 18, 3, 0), 24, 3}, 146 {RTE_IPV4(198, 18, 4, 0), 24, 4}, {RTE_IPV4(198, 18, 5, 0), 24, 5}, 147 {RTE_IPV4(198, 18, 6, 0), 24, 6}, {RTE_IPV4(198, 18, 7, 0), 24, 7}, 148 }; 149 150 static int 151 check_lcore_params(void) 152 { 153 uint8_t queue, lcore; 154 int socketid; 155 uint16_t i; 156 157 for (i = 0; i < nb_lcore_params; ++i) { 158 queue = lcore_params[i].queue_id; 159 if (queue >= MAX_RX_QUEUE_PER_PORT) { 160 printf("Invalid queue number: %hhu\n", queue); 161 return -1; 162 } 163 lcore = lcore_params[i].lcore_id; 164 if (!rte_lcore_is_enabled(lcore)) { 165 printf("Error: lcore %hhu is not enabled in lcore mask\n", 166 lcore); 167 return -1; 168 } 169 170 if (lcore == rte_get_main_lcore()) { 171 printf("Error: lcore %u is main lcore\n", lcore); 172 return -1; 173 } 174 socketid = rte_lcore_to_socket_id(lcore); 175 if ((socketid != 0) && (numa_on == 0)) { 176 printf("Warning: lcore %hhu is on socket %d with numa off\n", 177 lcore, socketid); 178 } 179 } 180 181 return 0; 182 } 183 184 static int 185 check_port_config(void) 186 { 187 uint16_t portid; 188 uint16_t i; 189 190 for (i = 0; i < nb_lcore_params; ++i) { 191 portid = lcore_params[i].port_id; 192 if ((enabled_port_mask & (1 << portid)) == 0) { 193 printf("Port %u is not enabled in port mask\n", portid); 194 return -1; 195 } 196 if (!rte_eth_dev_is_valid_port(portid)) { 197 printf("Port %u is not present on the board\n", portid); 198 return -1; 199 } 200 } 201 202 return 0; 203 } 204 205 static uint8_t 206 get_port_n_rx_queues(const uint16_t port) 207 { 208 int queue = -1; 209 uint16_t i; 210 211 for (i = 0; i < nb_lcore_params; ++i) { 212 if (lcore_params[i].port_id == port) { 213 if (lcore_params[i].queue_id == queue + 1) 214 queue = lcore_params[i].queue_id; 215 else 216 rte_exit(EXIT_FAILURE, 217 "Queue ids of the port %d must be" 218 " in sequence and must start with 0\n", 219 lcore_params[i].port_id); 220 } 221 } 222 223 return (uint8_t)(++queue); 224 } 225 226 static int 227 init_lcore_rx_queues(void) 228 { 229 uint16_t i, nb_rx_queue; 230 uint8_t lcore; 231 232 for (i = 0; i < nb_lcore_params; ++i) { 233 lcore = lcore_params[i].lcore_id; 234 nb_rx_queue = lcore_conf[lcore].n_rx_queue; 235 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 236 printf("Error: too many queues (%u) for lcore: %u\n", 237 (unsigned int)nb_rx_queue + 1, 238 (unsigned int)lcore); 239 return -1; 240 } 241 242 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 243 lcore_params[i].port_id; 244 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 245 lcore_params[i].queue_id; 246 lcore_conf[lcore].n_rx_queue++; 247 } 248 249 return 0; 250 } 251 252 /* Display usage */ 253 static void 254 print_usage(const char *prgname) 255 { 256 fprintf(stderr, 257 "%s [EAL options] --" 258 " -p PORTMASK" 259 " [-P]" 260 " --config (port,queue,lcore)[,(port,queue,lcore)]" 261 " [--eth-dest=X,MM:MM:MM:MM:MM:MM]" 262 " [--max-pkt-len PKTLEN]" 263 " [--no-numa]" 264 " [--per-port-pool]\n\n" 265 266 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 267 " -P : Enable promiscuous mode\n" 268 " --config (port,queue,lcore): Rx queue configuration\n" 269 " --eth-dest=X,MM:MM:MM:MM:MM:MM: Ethernet destination for " 270 "port X\n" 271 " --max-pkt-len PKTLEN: maximum packet length in decimal (64-9600)\n" 272 " --no-numa: Disable numa awareness\n" 273 " --per-port-pool: Use separate buffer pool per port\n\n", 274 prgname); 275 } 276 277 static int 278 parse_max_pkt_len(const char *pktlen) 279 { 280 unsigned long len; 281 char *end = NULL; 282 283 /* Parse decimal string */ 284 len = strtoul(pktlen, &end, 10); 285 if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) 286 return -1; 287 288 if (len == 0) 289 return -1; 290 291 return len; 292 } 293 294 static int 295 parse_portmask(const char *portmask) 296 { 297 char *end = NULL; 298 unsigned long pm; 299 300 /* Parse hexadecimal string */ 301 pm = strtoul(portmask, &end, 16); 302 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 303 return 0; 304 305 return pm; 306 } 307 308 static int 309 parse_config(const char *q_arg) 310 { 311 enum fieldnames { FLD_PORT = 0, FLD_QUEUE, FLD_LCORE, _NUM_FLD }; 312 unsigned long int_fld[_NUM_FLD]; 313 const char *p, *p0 = q_arg; 314 char *str_fld[_NUM_FLD]; 315 uint32_t size; 316 char s[256]; 317 char *end; 318 int i; 319 320 nb_lcore_params = 0; 321 322 while ((p = strchr(p0, '(')) != NULL) { 323 ++p; 324 p0 = strchr(p, ')'); 325 if (p0 == NULL) 326 return -1; 327 328 size = p0 - p; 329 if (size >= sizeof(s)) 330 return -1; 331 332 memcpy(s, p, size); 333 s[size] = '\0'; 334 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 335 _NUM_FLD) 336 return -1; 337 for (i = 0; i < _NUM_FLD; i++) { 338 errno = 0; 339 int_fld[i] = strtoul(str_fld[i], &end, 0); 340 if (errno != 0 || end == str_fld[i]) 341 return -1; 342 } 343 344 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 345 printf("Exceeded max number of lcore params: %hu\n", 346 nb_lcore_params); 347 return -1; 348 } 349 350 if (int_fld[FLD_PORT] >= RTE_MAX_ETHPORTS || 351 int_fld[FLD_LCORE] >= RTE_MAX_LCORE) { 352 printf("Invalid port/lcore id\n"); 353 return -1; 354 } 355 356 lcore_params_array[nb_lcore_params].port_id = 357 (uint8_t)int_fld[FLD_PORT]; 358 lcore_params_array[nb_lcore_params].queue_id = 359 (uint8_t)int_fld[FLD_QUEUE]; 360 lcore_params_array[nb_lcore_params].lcore_id = 361 (uint8_t)int_fld[FLD_LCORE]; 362 ++nb_lcore_params; 363 } 364 lcore_params = lcore_params_array; 365 366 return 0; 367 } 368 369 static void 370 parse_eth_dest(const char *optarg) 371 { 372 uint8_t c, *dest, peer_addr[6]; 373 uint16_t portid; 374 char *port_end; 375 376 errno = 0; 377 portid = strtoul(optarg, &port_end, 10); 378 if (errno != 0 || port_end == optarg || *port_end++ != ',') 379 rte_exit(EXIT_FAILURE, "Invalid eth-dest: %s", optarg); 380 if (portid >= RTE_MAX_ETHPORTS) 381 rte_exit(EXIT_FAILURE, 382 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n", portid, 383 RTE_MAX_ETHPORTS); 384 385 if (cmdline_parse_etheraddr(NULL, port_end, &peer_addr, 386 sizeof(peer_addr)) < 0) 387 rte_exit(EXIT_FAILURE, "Invalid ethernet address: %s\n", 388 port_end); 389 dest = (uint8_t *)&dest_eth_addr[portid]; 390 for (c = 0; c < 6; c++) 391 dest[c] = peer_addr[c]; 392 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid]; 393 } 394 395 #define MAX_JUMBO_PKT_LEN 9600 396 #define MEMPOOL_CACHE_SIZE 256 397 398 static const char short_options[] = "p:" /* portmask */ 399 "P" /* promiscuous */ 400 ; 401 402 #define CMD_LINE_OPT_CONFIG "config" 403 #define CMD_LINE_OPT_ETH_DEST "eth-dest" 404 #define CMD_LINE_OPT_NO_NUMA "no-numa" 405 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len" 406 #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool" 407 enum { 408 /* Long options mapped to a short option */ 409 410 /* First long only option value must be >= 256, so that we won't 411 * conflict with short options 412 */ 413 CMD_LINE_OPT_MIN_NUM = 256, 414 CMD_LINE_OPT_CONFIG_NUM, 415 CMD_LINE_OPT_ETH_DEST_NUM, 416 CMD_LINE_OPT_NO_NUMA_NUM, 417 CMD_LINE_OPT_MAX_PKT_LEN_NUM, 418 CMD_LINE_OPT_PARSE_PER_PORT_POOL, 419 }; 420 421 static const struct option lgopts[] = { 422 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 423 {CMD_LINE_OPT_ETH_DEST, 1, 0, CMD_LINE_OPT_ETH_DEST_NUM}, 424 {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM}, 425 {CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM}, 426 {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL}, 427 {NULL, 0, 0, 0}, 428 }; 429 430 /* 431 * This expression is used to calculate the number of mbufs needed 432 * depending on user input, taking into account memory for rx and 433 * tx hardware rings, cache per lcore and mtable per port per lcore. 434 * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum 435 * value of 8192 436 */ 437 #define NB_MBUF(nports) \ 438 RTE_MAX((nports * nb_rx_queue * nb_rxd + \ 439 nports * nb_lcores * RTE_GRAPH_BURST_SIZE + \ 440 nports * n_tx_queue * nb_txd + \ 441 nb_lcores * MEMPOOL_CACHE_SIZE), 8192u) 442 443 /* Parse the argument given in the command line of the application */ 444 static int 445 parse_args(int argc, char **argv) 446 { 447 char *prgname = argv[0]; 448 int option_index; 449 char **argvopt; 450 int opt, ret; 451 452 argvopt = argv; 453 454 /* Error or normal output strings. */ 455 while ((opt = getopt_long(argc, argvopt, short_options, lgopts, 456 &option_index)) != EOF) { 457 458 switch (opt) { 459 /* Portmask */ 460 case 'p': 461 enabled_port_mask = parse_portmask(optarg); 462 if (enabled_port_mask == 0) { 463 fprintf(stderr, "Invalid portmask\n"); 464 print_usage(prgname); 465 return -1; 466 } 467 break; 468 469 case 'P': 470 promiscuous_on = 1; 471 break; 472 473 /* Long options */ 474 case CMD_LINE_OPT_CONFIG_NUM: 475 ret = parse_config(optarg); 476 if (ret) { 477 fprintf(stderr, "Invalid config\n"); 478 print_usage(prgname); 479 return -1; 480 } 481 break; 482 483 case CMD_LINE_OPT_ETH_DEST_NUM: 484 parse_eth_dest(optarg); 485 break; 486 487 case CMD_LINE_OPT_NO_NUMA_NUM: 488 numa_on = 0; 489 break; 490 491 case CMD_LINE_OPT_MAX_PKT_LEN_NUM: { 492 max_pkt_len = parse_max_pkt_len(optarg); 493 break; 494 } 495 496 case CMD_LINE_OPT_PARSE_PER_PORT_POOL: 497 printf("Per port buffer pool is enabled\n"); 498 per_port_pool = 1; 499 break; 500 501 default: 502 print_usage(prgname); 503 return -1; 504 } 505 } 506 507 if (optind >= 0) 508 argv[optind - 1] = prgname; 509 ret = optind - 1; 510 optind = 1; /* Reset getopt lib */ 511 512 return ret; 513 } 514 515 static void 516 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 517 { 518 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 519 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 520 printf("%s%s", name, buf); 521 } 522 523 static int 524 init_mem(uint16_t portid, uint32_t nb_mbuf) 525 { 526 uint32_t lcore_id; 527 int socketid; 528 char s[64]; 529 530 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 531 if (rte_lcore_is_enabled(lcore_id) == 0) 532 continue; 533 534 if (numa_on) 535 socketid = rte_lcore_to_socket_id(lcore_id); 536 else 537 socketid = 0; 538 539 if (socketid >= NB_SOCKETS) { 540 rte_exit(EXIT_FAILURE, 541 "Socket %d of lcore %u is out of range %d\n", 542 socketid, lcore_id, NB_SOCKETS); 543 } 544 545 if (pktmbuf_pool[portid][socketid] == NULL) { 546 snprintf(s, sizeof(s), "mbuf_pool_%d:%d", portid, 547 socketid); 548 /* Create a pool with priv size of a cacheline */ 549 pktmbuf_pool[portid][socketid] = 550 rte_pktmbuf_pool_create( 551 s, nb_mbuf, MEMPOOL_CACHE_SIZE, 552 RTE_CACHE_LINE_SIZE, 553 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 554 if (pktmbuf_pool[portid][socketid] == NULL) 555 rte_exit(EXIT_FAILURE, 556 "Cannot init mbuf pool on socket %d\n", 557 socketid); 558 else 559 printf("Allocated mbuf pool on socket %d\n", 560 socketid); 561 } 562 } 563 564 return 0; 565 } 566 567 /* Check the link status of all ports in up to 9s, and print them finally */ 568 static void 569 check_all_ports_link_status(uint32_t port_mask) 570 { 571 #define CHECK_INTERVAL 100 /* 100ms */ 572 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 573 uint8_t count, all_ports_up, print_flag = 0; 574 struct rte_eth_link link; 575 uint16_t portid; 576 int ret; 577 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 578 579 printf("\nChecking link status"); 580 fflush(stdout); 581 for (count = 0; count <= MAX_CHECK_TIME; count++) { 582 if (force_quit) 583 return; 584 all_ports_up = 1; 585 RTE_ETH_FOREACH_DEV(portid) 586 { 587 if (force_quit) 588 return; 589 if ((port_mask & (1 << portid)) == 0) 590 continue; 591 memset(&link, 0, sizeof(link)); 592 ret = rte_eth_link_get_nowait(portid, &link); 593 if (ret < 0) { 594 all_ports_up = 0; 595 if (print_flag == 1) 596 printf("Port %u link get failed: %s\n", 597 portid, rte_strerror(-ret)); 598 continue; 599 } 600 /* Print link status if flag set */ 601 if (print_flag == 1) { 602 rte_eth_link_to_str(link_status_text, 603 sizeof(link_status_text), &link); 604 printf("Port %d %s\n", portid, 605 link_status_text); 606 continue; 607 } 608 /* Clear all_ports_up flag if any link down */ 609 if (link.link_status == RTE_ETH_LINK_DOWN) { 610 all_ports_up = 0; 611 break; 612 } 613 } 614 /* After finally printing all link status, get out */ 615 if (print_flag == 1) 616 break; 617 618 if (all_ports_up == 0) { 619 printf("."); 620 fflush(stdout); 621 rte_delay_ms(CHECK_INTERVAL); 622 } 623 624 /* Set the print_flag if all ports up or timeout */ 625 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 626 print_flag = 1; 627 printf("Done\n"); 628 } 629 } 630 } 631 632 static void 633 signal_handler(int signum) 634 { 635 if (signum == SIGINT || signum == SIGTERM) { 636 printf("\n\nSignal %d received, preparing to exit...\n", 637 signum); 638 force_quit = true; 639 } 640 } 641 642 static void 643 print_stats(void) 644 { 645 const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; 646 const char clr[] = {27, '[', '2', 'J', '\0'}; 647 struct rte_graph_cluster_stats_param s_param; 648 struct rte_graph_cluster_stats *stats; 649 const char *pattern = "worker_*"; 650 651 /* Prepare stats object */ 652 memset(&s_param, 0, sizeof(s_param)); 653 s_param.f = stdout; 654 s_param.socket_id = SOCKET_ID_ANY; 655 s_param.graph_patterns = &pattern; 656 s_param.nb_graph_patterns = 1; 657 658 stats = rte_graph_cluster_stats_create(&s_param); 659 if (stats == NULL) 660 rte_exit(EXIT_FAILURE, "Unable to create stats object\n"); 661 662 while (!force_quit) { 663 /* Clear screen and move to top left */ 664 printf("%s%s", clr, topLeft); 665 rte_graph_cluster_stats_get(stats, 0); 666 rte_delay_ms(1E3); 667 } 668 669 rte_graph_cluster_stats_destroy(stats); 670 } 671 672 /* Main processing loop. 8< */ 673 static int 674 graph_main_loop(void *conf) 675 { 676 struct lcore_conf *qconf; 677 struct rte_graph *graph; 678 uint32_t lcore_id; 679 680 RTE_SET_USED(conf); 681 682 lcore_id = rte_lcore_id(); 683 qconf = &lcore_conf[lcore_id]; 684 graph = qconf->graph; 685 686 if (!graph) { 687 RTE_LOG(INFO, L3FWD_GRAPH, "Lcore %u has nothing to do\n", 688 lcore_id); 689 return 0; 690 } 691 692 RTE_LOG(INFO, L3FWD_GRAPH, 693 "Entering main loop on lcore %u, graph %s(%p)\n", lcore_id, 694 qconf->name, graph); 695 696 while (likely(!force_quit)) 697 rte_graph_walk(graph); 698 699 return 0; 700 } 701 /* >8 End of main processing loop. */ 702 703 static uint32_t 704 eth_dev_get_overhead_len(uint32_t max_rx_pktlen, uint16_t max_mtu) 705 { 706 uint32_t overhead_len; 707 708 if (max_mtu != UINT16_MAX && max_rx_pktlen > max_mtu) 709 overhead_len = max_rx_pktlen - max_mtu; 710 else 711 overhead_len = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 712 713 return overhead_len; 714 } 715 716 static int 717 config_port_max_pkt_len(struct rte_eth_conf *conf, 718 struct rte_eth_dev_info *dev_info) 719 { 720 uint32_t overhead_len; 721 722 if (max_pkt_len == 0) 723 return 0; 724 725 if (max_pkt_len < RTE_ETHER_MIN_LEN || max_pkt_len > MAX_JUMBO_PKT_LEN) 726 return -1; 727 728 overhead_len = eth_dev_get_overhead_len(dev_info->max_rx_pktlen, 729 dev_info->max_mtu); 730 conf->rxmode.mtu = max_pkt_len - overhead_len; 731 732 if (conf->rxmode.mtu > RTE_ETHER_MTU) 733 conf->txmode.offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 734 735 return 0; 736 } 737 738 int 739 main(int argc, char **argv) 740 { 741 /* Rewrite data of src and dst ether addr */ 742 uint8_t rewrite_data[2 * sizeof(struct rte_ether_addr)]; 743 /* Graph initialization. 8< */ 744 static const char * const default_patterns[] = { 745 "ip4*", 746 "ethdev_tx-*", 747 "pkt_drop", 748 }; 749 uint8_t nb_rx_queue, queue, socketid; 750 struct rte_graph_param graph_conf; 751 struct rte_eth_dev_info dev_info; 752 uint32_t nb_ports, nb_conf = 0; 753 uint32_t n_tx_queue, nb_lcores; 754 struct rte_eth_txconf *txconf; 755 uint16_t queueid, portid, i; 756 const char **node_patterns; 757 struct lcore_conf *qconf; 758 uint16_t nb_graphs = 0; 759 uint16_t nb_patterns; 760 uint8_t rewrite_len; 761 uint32_t lcore_id; 762 int ret; 763 764 /* Init EAL */ 765 ret = rte_eal_init(argc, argv); 766 if (ret < 0) 767 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 768 argc -= ret; 769 argv += ret; 770 771 force_quit = false; 772 signal(SIGINT, signal_handler); 773 signal(SIGTERM, signal_handler); 774 775 /* Pre-init dst MACs for all ports to 02:00:00:00:00:xx */ 776 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 777 dest_eth_addr[portid] = 778 RTE_ETHER_LOCAL_ADMIN_ADDR + ((uint64_t)portid << 40); 779 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid]; 780 } 781 782 /* Parse application arguments (after the EAL ones) */ 783 ret = parse_args(argc, argv); 784 if (ret < 0) 785 rte_exit(EXIT_FAILURE, "Invalid L3FWD_GRAPH parameters\n"); 786 787 if (check_lcore_params() < 0) 788 rte_exit(EXIT_FAILURE, "check_lcore_params() failed\n"); 789 790 ret = init_lcore_rx_queues(); 791 if (ret < 0) 792 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues() failed\n"); 793 794 if (check_port_config() < 0) 795 rte_exit(EXIT_FAILURE, "check_port_config() failed\n"); 796 797 nb_ports = rte_eth_dev_count_avail(); 798 nb_lcores = rte_lcore_count(); 799 800 /* Initialize all ports. 8< */ 801 RTE_ETH_FOREACH_DEV(portid) 802 { 803 struct rte_eth_conf local_port_conf = port_conf; 804 805 /* Skip ports that are not enabled */ 806 if ((enabled_port_mask & (1 << portid)) == 0) { 807 printf("\nSkipping disabled port %d\n", portid); 808 continue; 809 } 810 811 /* Init port */ 812 printf("Initializing port %d ... ", portid); 813 fflush(stdout); 814 815 nb_rx_queue = get_port_n_rx_queues(portid); 816 n_tx_queue = nb_lcores; 817 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT) 818 n_tx_queue = MAX_TX_QUEUE_PER_PORT; 819 printf("Creating queues: nb_rxq=%d nb_txq=%u... ", 820 nb_rx_queue, n_tx_queue); 821 822 rte_eth_dev_info_get(portid, &dev_info); 823 824 ret = config_port_max_pkt_len(&local_port_conf, &dev_info); 825 if (ret != 0) 826 rte_exit(EXIT_FAILURE, 827 "Invalid max packet length: %u (port %u)\n", 828 max_pkt_len, portid); 829 830 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 831 local_port_conf.txmode.offloads |= 832 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 833 834 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 835 dev_info.flow_type_rss_offloads; 836 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 837 port_conf.rx_adv_conf.rss_conf.rss_hf) { 838 printf("Port %u modified RSS hash function based on " 839 "hardware support," 840 "requested:%#" PRIx64 " configured:%#" PRIx64 841 "\n", 842 portid, port_conf.rx_adv_conf.rss_conf.rss_hf, 843 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 844 } 845 846 ret = rte_eth_dev_configure(portid, nb_rx_queue, 847 n_tx_queue, &local_port_conf); 848 if (ret < 0) 849 rte_exit(EXIT_FAILURE, 850 "Cannot configure device: err=%d, port=%d\n", 851 ret, portid); 852 853 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 854 &nb_txd); 855 if (ret < 0) 856 rte_exit(EXIT_FAILURE, 857 "Cannot adjust number of descriptors: err=%d, " 858 "port=%d\n", 859 ret, portid); 860 861 rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 862 print_ethaddr(" Address:", &ports_eth_addr[portid]); 863 printf(", "); 864 print_ethaddr( 865 "Destination:", 866 (const struct rte_ether_addr *)&dest_eth_addr[portid]); 867 printf(", "); 868 869 /* 870 * prepare src MACs for each port. 871 */ 872 rte_ether_addr_copy( 873 &ports_eth_addr[portid], 874 (struct rte_ether_addr *)(val_eth + portid) + 1); 875 876 /* Init memory */ 877 if (!per_port_pool) { 878 /* portid = 0; this is *not* signifying the first port, 879 * rather, it signifies that portid is ignored. 880 */ 881 ret = init_mem(0, NB_MBUF(nb_ports)); 882 } else { 883 ret = init_mem(portid, NB_MBUF(1)); 884 } 885 if (ret < 0) 886 rte_exit(EXIT_FAILURE, "init_mem() failed\n"); 887 888 /* Init one TX queue per couple (lcore,port) */ 889 queueid = 0; 890 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 891 if (rte_lcore_is_enabled(lcore_id) == 0) 892 continue; 893 894 qconf = &lcore_conf[lcore_id]; 895 896 if (numa_on) 897 socketid = (uint8_t)rte_lcore_to_socket_id( 898 lcore_id); 899 else 900 socketid = 0; 901 902 printf("txq=%u,%d,%d ", lcore_id, queueid, socketid); 903 fflush(stdout); 904 905 txconf = &dev_info.default_txconf; 906 txconf->offloads = local_port_conf.txmode.offloads; 907 ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, 908 socketid, txconf); 909 if (ret < 0) 910 rte_exit(EXIT_FAILURE, 911 "rte_eth_tx_queue_setup: err=%d, " 912 "port=%d\n", 913 ret, portid); 914 queueid++; 915 } 916 917 /* Setup ethdev node config */ 918 ethdev_conf[nb_conf].port_id = portid; 919 ethdev_conf[nb_conf].num_rx_queues = nb_rx_queue; 920 ethdev_conf[nb_conf].num_tx_queues = n_tx_queue; 921 if (!per_port_pool) 922 ethdev_conf[nb_conf].mp = pktmbuf_pool[0]; 923 924 else 925 ethdev_conf[nb_conf].mp = pktmbuf_pool[portid]; 926 ethdev_conf[nb_conf].mp_count = NB_SOCKETS; 927 928 nb_conf++; 929 printf("\n"); 930 } 931 932 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 933 if (rte_lcore_is_enabled(lcore_id) == 0) 934 continue; 935 qconf = &lcore_conf[lcore_id]; 936 printf("\nInitializing rx queues on lcore %u ... ", lcore_id); 937 fflush(stdout); 938 /* Init RX queues */ 939 for (queue = 0; queue < qconf->n_rx_queue; ++queue) { 940 struct rte_eth_rxconf rxq_conf; 941 942 portid = qconf->rx_queue_list[queue].port_id; 943 queueid = qconf->rx_queue_list[queue].queue_id; 944 945 if (numa_on) 946 socketid = (uint8_t)rte_lcore_to_socket_id( 947 lcore_id); 948 else 949 socketid = 0; 950 951 printf("rxq=%d,%d,%d ", portid, queueid, socketid); 952 fflush(stdout); 953 954 rte_eth_dev_info_get(portid, &dev_info); 955 rxq_conf = dev_info.default_rxconf; 956 rxq_conf.offloads = port_conf.rxmode.offloads; 957 if (!per_port_pool) 958 ret = rte_eth_rx_queue_setup( 959 portid, queueid, nb_rxd, socketid, 960 &rxq_conf, pktmbuf_pool[0][socketid]); 961 else 962 ret = rte_eth_rx_queue_setup( 963 portid, queueid, nb_rxd, socketid, 964 &rxq_conf, 965 pktmbuf_pool[portid][socketid]); 966 if (ret < 0) 967 rte_exit(EXIT_FAILURE, 968 "rte_eth_rx_queue_setup: err=%d, " 969 "port=%d\n", 970 ret, portid); 971 972 /* Add this queue node to its graph */ 973 snprintf(qconf->rx_queue_list[queue].node_name, 974 RTE_NODE_NAMESIZE, "ethdev_rx-%u-%u", portid, 975 queueid); 976 } 977 978 /* Alloc a graph to this lcore only if source exists */ 979 if (qconf->n_rx_queue) 980 nb_graphs++; 981 } 982 983 printf("\n"); 984 985 /* Ethdev node config, skip rx queue mapping */ 986 ret = rte_node_eth_config(ethdev_conf, nb_conf, nb_graphs); 987 /* >8 End of graph creation. */ 988 if (ret) 989 rte_exit(EXIT_FAILURE, "rte_node_eth_config: err=%d\n", ret); 990 991 /* Start ports */ 992 RTE_ETH_FOREACH_DEV(portid) 993 { 994 if ((enabled_port_mask & (1 << portid)) == 0) 995 continue; 996 997 /* Start device */ 998 ret = rte_eth_dev_start(portid); 999 if (ret < 0) 1000 rte_exit(EXIT_FAILURE, 1001 "rte_eth_dev_start: err=%d, port=%d\n", ret, 1002 portid); 1003 1004 /* 1005 * If enabled, put device in promiscuous mode. 1006 * This allows IO forwarding mode to forward packets 1007 * to itself through 2 cross-connected ports of the 1008 * target machine. 1009 */ 1010 if (promiscuous_on) 1011 rte_eth_promiscuous_enable(portid); 1012 } 1013 1014 printf("\n"); 1015 1016 check_all_ports_link_status(enabled_port_mask); 1017 1018 /* Graph Initialization */ 1019 nb_patterns = RTE_DIM(default_patterns); 1020 node_patterns = malloc((MAX_RX_QUEUE_PER_LCORE + nb_patterns) * 1021 sizeof(*node_patterns)); 1022 if (!node_patterns) 1023 return -ENOMEM; 1024 memcpy(node_patterns, default_patterns, 1025 nb_patterns * sizeof(*node_patterns)); 1026 1027 memset(&graph_conf, 0, sizeof(graph_conf)); 1028 graph_conf.node_patterns = node_patterns; 1029 1030 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1031 rte_graph_t graph_id; 1032 rte_edge_t i; 1033 1034 if (rte_lcore_is_enabled(lcore_id) == 0) 1035 continue; 1036 1037 qconf = &lcore_conf[lcore_id]; 1038 1039 /* Skip graph creation if no source exists */ 1040 if (!qconf->n_rx_queue) 1041 continue; 1042 1043 /* Add rx node patterns of this lcore */ 1044 for (i = 0; i < qconf->n_rx_queue; i++) { 1045 graph_conf.node_patterns[nb_patterns + i] = 1046 qconf->rx_queue_list[i].node_name; 1047 } 1048 1049 graph_conf.nb_node_patterns = nb_patterns + i; 1050 graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id); 1051 1052 snprintf(qconf->name, sizeof(qconf->name), "worker_%u", 1053 lcore_id); 1054 1055 graph_id = rte_graph_create(qconf->name, &graph_conf); 1056 if (graph_id == RTE_GRAPH_ID_INVALID) 1057 rte_exit(EXIT_FAILURE, 1058 "rte_graph_create(): graph_id invalid" 1059 " for lcore %u\n", lcore_id); 1060 1061 qconf->graph_id = graph_id; 1062 qconf->graph = rte_graph_lookup(qconf->name); 1063 /* >8 End of graph initialization. */ 1064 if (!qconf->graph) 1065 rte_exit(EXIT_FAILURE, 1066 "rte_graph_lookup(): graph %s not found\n", 1067 qconf->name); 1068 } 1069 1070 memset(&rewrite_data, 0, sizeof(rewrite_data)); 1071 rewrite_len = sizeof(rewrite_data); 1072 1073 /* Add route to ip4 graph infra. 8< */ 1074 for (i = 0; i < IPV4_L3FWD_LPM_NUM_ROUTES; i++) { 1075 char route_str[INET6_ADDRSTRLEN * 4]; 1076 char abuf[INET6_ADDRSTRLEN]; 1077 struct in_addr in; 1078 uint32_t dst_port; 1079 1080 /* Skip unused ports */ 1081 if ((1 << ipv4_l3fwd_lpm_route_array[i].if_out & 1082 enabled_port_mask) == 0) 1083 continue; 1084 1085 dst_port = ipv4_l3fwd_lpm_route_array[i].if_out; 1086 1087 in.s_addr = htonl(ipv4_l3fwd_lpm_route_array[i].ip); 1088 snprintf(route_str, sizeof(route_str), "%s / %d (%d)", 1089 inet_ntop(AF_INET, &in, abuf, sizeof(abuf)), 1090 ipv4_l3fwd_lpm_route_array[i].depth, 1091 ipv4_l3fwd_lpm_route_array[i].if_out); 1092 1093 /* Use route index 'i' as next hop id */ 1094 ret = rte_node_ip4_route_add( 1095 ipv4_l3fwd_lpm_route_array[i].ip, 1096 ipv4_l3fwd_lpm_route_array[i].depth, i, 1097 RTE_NODE_IP4_LOOKUP_NEXT_REWRITE); 1098 1099 if (ret < 0) 1100 rte_exit(EXIT_FAILURE, 1101 "Unable to add ip4 route %s to graph\n", 1102 route_str); 1103 1104 memcpy(rewrite_data, val_eth + dst_port, rewrite_len); 1105 1106 /* Add next hop rewrite data for id 'i' */ 1107 ret = rte_node_ip4_rewrite_add(i, rewrite_data, 1108 rewrite_len, dst_port); 1109 if (ret < 0) 1110 rte_exit(EXIT_FAILURE, 1111 "Unable to add next hop %u for " 1112 "route %s\n", i, route_str); 1113 1114 RTE_LOG(INFO, L3FWD_GRAPH, "Added route %s, next_hop %u\n", 1115 route_str, i); 1116 } 1117 /* >8 End of adding route to ip4 graph infa. */ 1118 1119 /* Launch per-lcore init on every worker lcore */ 1120 rte_eal_mp_remote_launch(graph_main_loop, NULL, SKIP_MAIN); 1121 1122 /* Accumulate and print stats on main until exit */ 1123 if (rte_graph_has_stats_feature()) 1124 print_stats(); 1125 1126 /* Wait for worker cores to exit */ 1127 ret = 0; 1128 RTE_LCORE_FOREACH_WORKER(lcore_id) { 1129 ret = rte_eal_wait_lcore(lcore_id); 1130 /* Destroy graph */ 1131 if (ret < 0 || rte_graph_destroy( 1132 rte_graph_from_name(lcore_conf[lcore_id].name))) { 1133 ret = -1; 1134 break; 1135 } 1136 } 1137 free(node_patterns); 1138 1139 /* Stop ports */ 1140 RTE_ETH_FOREACH_DEV(portid) { 1141 if ((enabled_port_mask & (1 << portid)) == 0) 1142 continue; 1143 printf("Closing port %d...", portid); 1144 ret = rte_eth_dev_stop(portid); 1145 if (ret != 0) 1146 printf("Failed to stop port %u: %s\n", 1147 portid, rte_strerror(-ret)); 1148 rte_eth_dev_close(portid); 1149 printf(" Done\n"); 1150 } 1151 1152 /* clean up the EAL */ 1153 rte_eal_cleanup(); 1154 printf("Bye...\n"); 1155 1156 return ret; 1157 } 1158