1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <arpa/inet.h> 6 #include <errno.h> 7 #include <getopt.h> 8 #include <inttypes.h> 9 #include <signal.h> 10 #include <stdarg.h> 11 #include <stdbool.h> 12 #include <stdint.h> 13 #include <stdio.h> 14 #include <stdlib.h> 15 #include <string.h> 16 #include <sys/socket.h> 17 #include <sys/types.h> 18 #include <sys/queue.h> 19 #include <unistd.h> 20 21 #include <rte_branch_prediction.h> 22 #include <rte_common.h> 23 #include <rte_cycles.h> 24 #include <rte_eal.h> 25 #include <rte_ethdev.h> 26 #include <rte_graph_worker.h> 27 #include <rte_launch.h> 28 #include <rte_lcore.h> 29 #include <rte_log.h> 30 #include <rte_mempool.h> 31 #include <rte_node_eth_api.h> 32 #include <rte_node_ip4_api.h> 33 #include <rte_per_lcore.h> 34 #include <rte_string_fns.h> 35 #include <rte_vect.h> 36 37 #include <cmdline_parse.h> 38 #include <cmdline_parse_etheraddr.h> 39 40 /* Log type */ 41 #define RTE_LOGTYPE_L3FWD_GRAPH RTE_LOGTYPE_USER1 42 43 /* 44 * Configurable number of RX/TX ring descriptors 45 */ 46 #define RX_DESC_DEFAULT 1024 47 #define TX_DESC_DEFAULT 1024 48 49 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 50 #define MAX_RX_QUEUE_PER_PORT 128 51 52 #define MAX_RX_QUEUE_PER_LCORE 16 53 54 #define MAX_LCORE_PARAMS 1024 55 56 #define NB_SOCKETS 8 57 58 /* Static global variables used within this file. */ 59 static uint16_t nb_rxd = RX_DESC_DEFAULT; 60 static uint16_t nb_txd = TX_DESC_DEFAULT; 61 62 /**< Ports set in promiscuous mode off by default. */ 63 static int promiscuous_on; 64 65 static int numa_on = 1; /**< NUMA is enabled by default. */ 66 static int per_port_pool; /**< Use separate buffer pools per port; disabled */ 67 /**< by default */ 68 69 static volatile bool force_quit; 70 71 /* Ethernet addresses of ports */ 72 static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS]; 73 static struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; 74 xmm_t val_eth[RTE_MAX_ETHPORTS]; 75 76 /* Mask of enabled ports */ 77 static uint32_t enabled_port_mask; 78 79 /* Pcap trace */ 80 static char pcap_filename[RTE_GRAPH_PCAP_FILE_SZ]; 81 static uint64_t packet_to_capture; 82 static int pcap_trace_enable; 83 84 85 struct lcore_rx_queue { 86 uint16_t port_id; 87 uint8_t queue_id; 88 char node_name[RTE_NODE_NAMESIZE]; 89 }; 90 91 /* Lcore conf */ 92 struct lcore_conf { 93 uint16_t n_rx_queue; 94 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 95 96 struct rte_graph *graph; 97 char name[RTE_GRAPH_NAMESIZE]; 98 rte_graph_t graph_id; 99 } __rte_cache_aligned; 100 101 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 102 103 struct lcore_params { 104 uint16_t port_id; 105 uint8_t queue_id; 106 uint8_t lcore_id; 107 } __rte_cache_aligned; 108 109 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 110 static struct lcore_params lcore_params_array_default[] = { 111 {0, 0, 2}, {0, 1, 2}, {0, 2, 2}, {1, 0, 2}, {1, 1, 2}, 112 {1, 2, 2}, {2, 0, 2}, {3, 0, 3}, {3, 1, 3}, 113 }; 114 115 static struct lcore_params *lcore_params = lcore_params_array_default; 116 static uint16_t nb_lcore_params = RTE_DIM(lcore_params_array_default); 117 118 static struct rte_eth_conf port_conf = { 119 .rxmode = { 120 .mq_mode = RTE_ETH_MQ_RX_RSS, 121 }, 122 .rx_adv_conf = { 123 .rss_conf = { 124 .rss_key = NULL, 125 .rss_hf = RTE_ETH_RSS_IP, 126 }, 127 }, 128 .txmode = { 129 .mq_mode = RTE_ETH_MQ_TX_NONE, 130 }, 131 }; 132 133 static uint32_t max_pkt_len; 134 135 static struct rte_mempool *pktmbuf_pool[RTE_MAX_ETHPORTS][NB_SOCKETS]; 136 137 static struct rte_node_ethdev_config ethdev_conf[RTE_MAX_ETHPORTS]; 138 139 struct ipv4_l3fwd_lpm_route { 140 uint32_t ip; 141 uint8_t depth; 142 uint8_t if_out; 143 }; 144 145 #define IPV4_L3FWD_LPM_NUM_ROUTES \ 146 (sizeof(ipv4_l3fwd_lpm_route_array) / \ 147 sizeof(ipv4_l3fwd_lpm_route_array[0])) 148 /* 198.18.0.0/16 are set aside for RFC2544 benchmarking. */ 149 static struct ipv4_l3fwd_lpm_route ipv4_l3fwd_lpm_route_array[] = { 150 {RTE_IPV4(198, 18, 0, 0), 24, 0}, {RTE_IPV4(198, 18, 1, 0), 24, 1}, 151 {RTE_IPV4(198, 18, 2, 0), 24, 2}, {RTE_IPV4(198, 18, 3, 0), 24, 3}, 152 {RTE_IPV4(198, 18, 4, 0), 24, 4}, {RTE_IPV4(198, 18, 5, 0), 24, 5}, 153 {RTE_IPV4(198, 18, 6, 0), 24, 6}, {RTE_IPV4(198, 18, 7, 0), 24, 7}, 154 }; 155 156 static int 157 check_lcore_params(void) 158 { 159 uint8_t queue, lcore; 160 int socketid; 161 uint16_t i; 162 163 for (i = 0; i < nb_lcore_params; ++i) { 164 queue = lcore_params[i].queue_id; 165 if (queue >= MAX_RX_QUEUE_PER_PORT) { 166 printf("Invalid queue number: %hhu\n", queue); 167 return -1; 168 } 169 lcore = lcore_params[i].lcore_id; 170 if (!rte_lcore_is_enabled(lcore)) { 171 printf("Error: lcore %hhu is not enabled in lcore mask\n", 172 lcore); 173 return -1; 174 } 175 176 if (lcore == rte_get_main_lcore()) { 177 printf("Error: lcore %u is main lcore\n", lcore); 178 return -1; 179 } 180 socketid = rte_lcore_to_socket_id(lcore); 181 if ((socketid != 0) && (numa_on == 0)) { 182 printf("Warning: lcore %hhu is on socket %d with numa off\n", 183 lcore, socketid); 184 } 185 } 186 187 return 0; 188 } 189 190 static int 191 check_port_config(void) 192 { 193 uint16_t portid; 194 uint16_t i; 195 196 for (i = 0; i < nb_lcore_params; ++i) { 197 portid = lcore_params[i].port_id; 198 if ((enabled_port_mask & (1 << portid)) == 0) { 199 printf("Port %u is not enabled in port mask\n", portid); 200 return -1; 201 } 202 if (!rte_eth_dev_is_valid_port(portid)) { 203 printf("Port %u is not present on the board\n", portid); 204 return -1; 205 } 206 } 207 208 return 0; 209 } 210 211 static uint8_t 212 get_port_n_rx_queues(const uint16_t port) 213 { 214 int queue = -1; 215 uint16_t i; 216 217 for (i = 0; i < nb_lcore_params; ++i) { 218 if (lcore_params[i].port_id == port) { 219 if (lcore_params[i].queue_id == queue + 1) 220 queue = lcore_params[i].queue_id; 221 else 222 rte_exit(EXIT_FAILURE, 223 "Queue ids of the port %d must be" 224 " in sequence and must start with 0\n", 225 lcore_params[i].port_id); 226 } 227 } 228 229 return (uint8_t)(++queue); 230 } 231 232 static int 233 init_lcore_rx_queues(void) 234 { 235 uint16_t i, nb_rx_queue; 236 uint8_t lcore; 237 238 for (i = 0; i < nb_lcore_params; ++i) { 239 lcore = lcore_params[i].lcore_id; 240 nb_rx_queue = lcore_conf[lcore].n_rx_queue; 241 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 242 printf("Error: too many queues (%u) for lcore: %u\n", 243 (unsigned int)nb_rx_queue + 1, 244 (unsigned int)lcore); 245 return -1; 246 } 247 248 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 249 lcore_params[i].port_id; 250 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 251 lcore_params[i].queue_id; 252 lcore_conf[lcore].n_rx_queue++; 253 } 254 255 return 0; 256 } 257 258 /* Display usage */ 259 static void 260 print_usage(const char *prgname) 261 { 262 fprintf(stderr, 263 "%s [EAL options] --" 264 " -p PORTMASK" 265 " [-P]" 266 " --config (port,queue,lcore)[,(port,queue,lcore)]" 267 " [--eth-dest=X,MM:MM:MM:MM:MM:MM]" 268 " [--max-pkt-len PKTLEN]" 269 " [--no-numa]" 270 " [--per-port-pool]" 271 " [--num-pkt-cap]\n\n" 272 273 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 274 " -P : Enable promiscuous mode\n" 275 " --config (port,queue,lcore): Rx queue configuration\n" 276 " --eth-dest=X,MM:MM:MM:MM:MM:MM: Ethernet destination for " 277 "port X\n" 278 " --max-pkt-len PKTLEN: maximum packet length in decimal (64-9600)\n" 279 " --no-numa: Disable numa awareness\n" 280 " --per-port-pool: Use separate buffer pool per port\n" 281 " --pcap-enable: Enables pcap capture\n" 282 " --pcap-num-cap NUMPKT: Number of packets to capture\n" 283 " --pcap-file-name NAME: Pcap file name\n\n", 284 prgname); 285 } 286 287 static uint64_t 288 parse_num_pkt_cap(const char *num_pkt_cap) 289 { 290 uint64_t num_pkt; 291 char *end = NULL; 292 293 /* Parse decimal string */ 294 num_pkt = strtoull(num_pkt_cap, &end, 10); 295 if ((num_pkt_cap[0] == '\0') || (end == NULL) || (*end != '\0')) 296 return 0; 297 298 if (num_pkt == 0) 299 return 0; 300 301 return num_pkt; 302 } 303 304 static int 305 parse_max_pkt_len(const char *pktlen) 306 { 307 unsigned long len; 308 char *end = NULL; 309 310 /* Parse decimal string */ 311 len = strtoul(pktlen, &end, 10); 312 if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) 313 return -1; 314 315 if (len == 0) 316 return -1; 317 318 return len; 319 } 320 321 static int 322 parse_portmask(const char *portmask) 323 { 324 char *end = NULL; 325 unsigned long pm; 326 327 /* Parse hexadecimal string */ 328 pm = strtoul(portmask, &end, 16); 329 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 330 return 0; 331 332 return pm; 333 } 334 335 static int 336 parse_config(const char *q_arg) 337 { 338 enum fieldnames { FLD_PORT = 0, FLD_QUEUE, FLD_LCORE, _NUM_FLD }; 339 unsigned long int_fld[_NUM_FLD]; 340 const char *p, *p0 = q_arg; 341 char *str_fld[_NUM_FLD]; 342 uint32_t size; 343 char s[256]; 344 char *end; 345 int i; 346 347 nb_lcore_params = 0; 348 349 while ((p = strchr(p0, '(')) != NULL) { 350 ++p; 351 p0 = strchr(p, ')'); 352 if (p0 == NULL) 353 return -1; 354 355 size = p0 - p; 356 if (size >= sizeof(s)) 357 return -1; 358 359 memcpy(s, p, size); 360 s[size] = '\0'; 361 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 362 _NUM_FLD) 363 return -1; 364 for (i = 0; i < _NUM_FLD; i++) { 365 errno = 0; 366 int_fld[i] = strtoul(str_fld[i], &end, 0); 367 if (errno != 0 || end == str_fld[i]) 368 return -1; 369 } 370 371 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 372 printf("Exceeded max number of lcore params: %hu\n", 373 nb_lcore_params); 374 return -1; 375 } 376 377 if (int_fld[FLD_PORT] >= RTE_MAX_ETHPORTS || 378 int_fld[FLD_LCORE] >= RTE_MAX_LCORE) { 379 printf("Invalid port/lcore id\n"); 380 return -1; 381 } 382 383 lcore_params_array[nb_lcore_params].port_id = 384 (uint8_t)int_fld[FLD_PORT]; 385 lcore_params_array[nb_lcore_params].queue_id = 386 (uint8_t)int_fld[FLD_QUEUE]; 387 lcore_params_array[nb_lcore_params].lcore_id = 388 (uint8_t)int_fld[FLD_LCORE]; 389 ++nb_lcore_params; 390 } 391 lcore_params = lcore_params_array; 392 393 return 0; 394 } 395 396 static void 397 parse_eth_dest(const char *optarg) 398 { 399 uint8_t c, *dest, peer_addr[6]; 400 uint16_t portid; 401 char *port_end; 402 403 errno = 0; 404 portid = strtoul(optarg, &port_end, 10); 405 if (errno != 0 || port_end == optarg || *port_end++ != ',') 406 rte_exit(EXIT_FAILURE, "Invalid eth-dest: %s", optarg); 407 if (portid >= RTE_MAX_ETHPORTS) 408 rte_exit(EXIT_FAILURE, 409 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n", portid, 410 RTE_MAX_ETHPORTS); 411 412 if (cmdline_parse_etheraddr(NULL, port_end, &peer_addr, 413 sizeof(peer_addr)) < 0) 414 rte_exit(EXIT_FAILURE, "Invalid ethernet address: %s\n", 415 port_end); 416 dest = (uint8_t *)&dest_eth_addr[portid]; 417 for (c = 0; c < 6; c++) 418 dest[c] = peer_addr[c]; 419 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid]; 420 } 421 422 #define MAX_JUMBO_PKT_LEN 9600 423 #define MEMPOOL_CACHE_SIZE 256 424 425 static const char short_options[] = "p:" /* portmask */ 426 "P" /* promiscuous */ 427 ; 428 429 #define CMD_LINE_OPT_CONFIG "config" 430 #define CMD_LINE_OPT_ETH_DEST "eth-dest" 431 #define CMD_LINE_OPT_NO_NUMA "no-numa" 432 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len" 433 #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool" 434 #define CMD_LINE_OPT_PCAP_ENABLE "pcap-enable" 435 #define CMD_LINE_OPT_NUM_PKT_CAP "pcap-num-cap" 436 #define CMD_LINE_OPT_PCAP_FILENAME "pcap-file-name" 437 enum { 438 /* Long options mapped to a short option */ 439 440 /* First long only option value must be >= 256, so that we won't 441 * conflict with short options 442 */ 443 CMD_LINE_OPT_MIN_NUM = 256, 444 CMD_LINE_OPT_CONFIG_NUM, 445 CMD_LINE_OPT_ETH_DEST_NUM, 446 CMD_LINE_OPT_NO_NUMA_NUM, 447 CMD_LINE_OPT_MAX_PKT_LEN_NUM, 448 CMD_LINE_OPT_PARSE_PER_PORT_POOL, 449 CMD_LINE_OPT_PARSE_PCAP_ENABLE, 450 CMD_LINE_OPT_PARSE_NUM_PKT_CAP, 451 CMD_LINE_OPT_PCAP_FILENAME_CAP, 452 }; 453 454 static const struct option lgopts[] = { 455 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 456 {CMD_LINE_OPT_ETH_DEST, 1, 0, CMD_LINE_OPT_ETH_DEST_NUM}, 457 {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM}, 458 {CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM}, 459 {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL}, 460 {CMD_LINE_OPT_PCAP_ENABLE, 0, 0, CMD_LINE_OPT_PARSE_PCAP_ENABLE}, 461 {CMD_LINE_OPT_NUM_PKT_CAP, 1, 0, CMD_LINE_OPT_PARSE_NUM_PKT_CAP}, 462 {CMD_LINE_OPT_PCAP_FILENAME, 1, 0, CMD_LINE_OPT_PCAP_FILENAME_CAP}, 463 {NULL, 0, 0, 0}, 464 }; 465 466 /* 467 * This expression is used to calculate the number of mbufs needed 468 * depending on user input, taking into account memory for rx and 469 * tx hardware rings, cache per lcore and mtable per port per lcore. 470 * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum 471 * value of 8192 472 */ 473 #define NB_MBUF(nports) \ 474 RTE_MAX((nports * nb_rx_queue * nb_rxd + \ 475 nports * nb_lcores * RTE_GRAPH_BURST_SIZE + \ 476 nports * n_tx_queue * nb_txd + \ 477 nb_lcores * MEMPOOL_CACHE_SIZE), 8192u) 478 479 /* Parse the argument given in the command line of the application */ 480 static int 481 parse_args(int argc, char **argv) 482 { 483 char *prgname = argv[0]; 484 int option_index; 485 char **argvopt; 486 int opt, ret; 487 488 argvopt = argv; 489 490 /* Error or normal output strings. */ 491 while ((opt = getopt_long(argc, argvopt, short_options, lgopts, 492 &option_index)) != EOF) { 493 494 switch (opt) { 495 /* Portmask */ 496 case 'p': 497 enabled_port_mask = parse_portmask(optarg); 498 if (enabled_port_mask == 0) { 499 fprintf(stderr, "Invalid portmask\n"); 500 print_usage(prgname); 501 return -1; 502 } 503 break; 504 505 case 'P': 506 promiscuous_on = 1; 507 break; 508 509 /* Long options */ 510 case CMD_LINE_OPT_CONFIG_NUM: 511 ret = parse_config(optarg); 512 if (ret) { 513 fprintf(stderr, "Invalid config\n"); 514 print_usage(prgname); 515 return -1; 516 } 517 break; 518 519 case CMD_LINE_OPT_ETH_DEST_NUM: 520 parse_eth_dest(optarg); 521 break; 522 523 case CMD_LINE_OPT_NO_NUMA_NUM: 524 numa_on = 0; 525 break; 526 527 case CMD_LINE_OPT_MAX_PKT_LEN_NUM: { 528 max_pkt_len = parse_max_pkt_len(optarg); 529 break; 530 } 531 532 case CMD_LINE_OPT_PARSE_PER_PORT_POOL: 533 printf("Per port buffer pool is enabled\n"); 534 per_port_pool = 1; 535 break; 536 537 case CMD_LINE_OPT_PARSE_PCAP_ENABLE: 538 printf("Packet capture enabled\n"); 539 pcap_trace_enable = 1; 540 break; 541 542 case CMD_LINE_OPT_PARSE_NUM_PKT_CAP: 543 packet_to_capture = parse_num_pkt_cap(optarg); 544 printf("Number of packets to capture: %"PRIu64"\n", 545 packet_to_capture); 546 break; 547 548 case CMD_LINE_OPT_PCAP_FILENAME_CAP: 549 rte_strlcpy(pcap_filename, optarg, 550 sizeof(pcap_filename)); 551 printf("Pcap file name: %s\n", pcap_filename); 552 break; 553 554 default: 555 print_usage(prgname); 556 return -1; 557 } 558 } 559 560 if (optind >= 0) 561 argv[optind - 1] = prgname; 562 ret = optind - 1; 563 optind = 1; /* Reset getopt lib */ 564 565 return ret; 566 } 567 568 static void 569 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 570 { 571 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 572 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 573 printf("%s%s", name, buf); 574 } 575 576 static int 577 init_mem(uint16_t portid, uint32_t nb_mbuf) 578 { 579 uint32_t lcore_id; 580 int socketid; 581 char s[64]; 582 583 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 584 if (rte_lcore_is_enabled(lcore_id) == 0) 585 continue; 586 587 if (numa_on) 588 socketid = rte_lcore_to_socket_id(lcore_id); 589 else 590 socketid = 0; 591 592 if (socketid >= NB_SOCKETS) { 593 rte_exit(EXIT_FAILURE, 594 "Socket %d of lcore %u is out of range %d\n", 595 socketid, lcore_id, NB_SOCKETS); 596 } 597 598 if (pktmbuf_pool[portid][socketid] == NULL) { 599 snprintf(s, sizeof(s), "mbuf_pool_%d:%d", portid, 600 socketid); 601 /* Create a pool with priv size of a cacheline */ 602 pktmbuf_pool[portid][socketid] = 603 rte_pktmbuf_pool_create( 604 s, nb_mbuf, MEMPOOL_CACHE_SIZE, 605 RTE_CACHE_LINE_SIZE, 606 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 607 if (pktmbuf_pool[portid][socketid] == NULL) 608 rte_exit(EXIT_FAILURE, 609 "Cannot init mbuf pool on socket %d\n", 610 socketid); 611 else 612 printf("Allocated mbuf pool on socket %d\n", 613 socketid); 614 } 615 } 616 617 return 0; 618 } 619 620 /* Check the link status of all ports in up to 9s, and print them finally */ 621 static void 622 check_all_ports_link_status(uint32_t port_mask) 623 { 624 #define CHECK_INTERVAL 100 /* 100ms */ 625 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 626 uint8_t count, all_ports_up, print_flag = 0; 627 struct rte_eth_link link; 628 uint16_t portid; 629 int ret; 630 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 631 632 printf("\nChecking link status"); 633 fflush(stdout); 634 for (count = 0; count <= MAX_CHECK_TIME; count++) { 635 if (force_quit) 636 return; 637 all_ports_up = 1; 638 RTE_ETH_FOREACH_DEV(portid) 639 { 640 if (force_quit) 641 return; 642 if ((port_mask & (1 << portid)) == 0) 643 continue; 644 memset(&link, 0, sizeof(link)); 645 ret = rte_eth_link_get_nowait(portid, &link); 646 if (ret < 0) { 647 all_ports_up = 0; 648 if (print_flag == 1) 649 printf("Port %u link get failed: %s\n", 650 portid, rte_strerror(-ret)); 651 continue; 652 } 653 /* Print link status if flag set */ 654 if (print_flag == 1) { 655 rte_eth_link_to_str(link_status_text, 656 sizeof(link_status_text), &link); 657 printf("Port %d %s\n", portid, 658 link_status_text); 659 continue; 660 } 661 /* Clear all_ports_up flag if any link down */ 662 if (link.link_status == RTE_ETH_LINK_DOWN) { 663 all_ports_up = 0; 664 break; 665 } 666 } 667 /* After finally printing all link status, get out */ 668 if (print_flag == 1) 669 break; 670 671 if (all_ports_up == 0) { 672 printf("."); 673 fflush(stdout); 674 rte_delay_ms(CHECK_INTERVAL); 675 } 676 677 /* Set the print_flag if all ports up or timeout */ 678 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 679 print_flag = 1; 680 printf("Done\n"); 681 } 682 } 683 } 684 685 static void 686 signal_handler(int signum) 687 { 688 if (signum == SIGINT || signum == SIGTERM) { 689 printf("\n\nSignal %d received, preparing to exit...\n", 690 signum); 691 force_quit = true; 692 } 693 } 694 695 static void 696 print_stats(void) 697 { 698 const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; 699 const char clr[] = {27, '[', '2', 'J', '\0'}; 700 struct rte_graph_cluster_stats_param s_param; 701 struct rte_graph_cluster_stats *stats; 702 const char *pattern = "worker_*"; 703 704 /* Prepare stats object */ 705 memset(&s_param, 0, sizeof(s_param)); 706 s_param.f = stdout; 707 s_param.socket_id = SOCKET_ID_ANY; 708 s_param.graph_patterns = &pattern; 709 s_param.nb_graph_patterns = 1; 710 711 stats = rte_graph_cluster_stats_create(&s_param); 712 if (stats == NULL) 713 rte_exit(EXIT_FAILURE, "Unable to create stats object\n"); 714 715 while (!force_quit) { 716 /* Clear screen and move to top left */ 717 printf("%s%s", clr, topLeft); 718 rte_graph_cluster_stats_get(stats, 0); 719 rte_delay_ms(1E3); 720 } 721 722 rte_graph_cluster_stats_destroy(stats); 723 } 724 725 /* Main processing loop. 8< */ 726 static int 727 graph_main_loop(void *conf) 728 { 729 struct lcore_conf *qconf; 730 struct rte_graph *graph; 731 uint32_t lcore_id; 732 733 RTE_SET_USED(conf); 734 735 lcore_id = rte_lcore_id(); 736 qconf = &lcore_conf[lcore_id]; 737 graph = qconf->graph; 738 739 if (!graph) { 740 RTE_LOG(INFO, L3FWD_GRAPH, "Lcore %u has nothing to do\n", 741 lcore_id); 742 return 0; 743 } 744 745 RTE_LOG(INFO, L3FWD_GRAPH, 746 "Entering main loop on lcore %u, graph %s(%p)\n", lcore_id, 747 qconf->name, graph); 748 749 while (likely(!force_quit)) 750 rte_graph_walk(graph); 751 752 return 0; 753 } 754 /* >8 End of main processing loop. */ 755 756 static uint32_t 757 eth_dev_get_overhead_len(uint32_t max_rx_pktlen, uint16_t max_mtu) 758 { 759 uint32_t overhead_len; 760 761 if (max_mtu != UINT16_MAX && max_rx_pktlen > max_mtu) 762 overhead_len = max_rx_pktlen - max_mtu; 763 else 764 overhead_len = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 765 766 return overhead_len; 767 } 768 769 static int 770 config_port_max_pkt_len(struct rte_eth_conf *conf, 771 struct rte_eth_dev_info *dev_info) 772 { 773 uint32_t overhead_len; 774 775 if (max_pkt_len == 0) 776 return 0; 777 778 if (max_pkt_len < RTE_ETHER_MIN_LEN || max_pkt_len > MAX_JUMBO_PKT_LEN) 779 return -1; 780 781 overhead_len = eth_dev_get_overhead_len(dev_info->max_rx_pktlen, 782 dev_info->max_mtu); 783 conf->rxmode.mtu = max_pkt_len - overhead_len; 784 785 if (conf->rxmode.mtu > RTE_ETHER_MTU) 786 conf->txmode.offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 787 788 return 0; 789 } 790 791 int 792 main(int argc, char **argv) 793 { 794 /* Rewrite data of src and dst ether addr */ 795 uint8_t rewrite_data[2 * sizeof(struct rte_ether_addr)]; 796 /* Graph initialization. 8< */ 797 static const char * const default_patterns[] = { 798 "ip4*", 799 "ethdev_tx-*", 800 "pkt_drop", 801 }; 802 uint8_t nb_rx_queue, queue, socketid; 803 struct rte_graph_param graph_conf; 804 struct rte_eth_dev_info dev_info; 805 uint32_t nb_ports, nb_conf = 0; 806 uint32_t n_tx_queue, nb_lcores; 807 struct rte_eth_txconf *txconf; 808 uint16_t queueid, portid, i; 809 const char **node_patterns; 810 struct lcore_conf *qconf; 811 uint16_t nb_graphs = 0; 812 uint16_t nb_patterns; 813 uint8_t rewrite_len; 814 uint32_t lcore_id; 815 int ret; 816 817 /* Init EAL */ 818 ret = rte_eal_init(argc, argv); 819 if (ret < 0) 820 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 821 argc -= ret; 822 argv += ret; 823 824 force_quit = false; 825 signal(SIGINT, signal_handler); 826 signal(SIGTERM, signal_handler); 827 828 /* Pre-init dst MACs for all ports to 02:00:00:00:00:xx */ 829 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 830 dest_eth_addr[portid] = 831 RTE_ETHER_LOCAL_ADMIN_ADDR + ((uint64_t)portid << 40); 832 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid]; 833 } 834 835 /* Parse application arguments (after the EAL ones) */ 836 ret = parse_args(argc, argv); 837 if (ret < 0) 838 rte_exit(EXIT_FAILURE, "Invalid L3FWD_GRAPH parameters\n"); 839 840 if (check_lcore_params() < 0) 841 rte_exit(EXIT_FAILURE, "check_lcore_params() failed\n"); 842 843 ret = init_lcore_rx_queues(); 844 if (ret < 0) 845 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues() failed\n"); 846 847 if (check_port_config() < 0) 848 rte_exit(EXIT_FAILURE, "check_port_config() failed\n"); 849 850 nb_ports = rte_eth_dev_count_avail(); 851 nb_lcores = rte_lcore_count(); 852 853 /* Initialize all ports. 8< */ 854 RTE_ETH_FOREACH_DEV(portid) 855 { 856 struct rte_eth_conf local_port_conf = port_conf; 857 858 /* Skip ports that are not enabled */ 859 if ((enabled_port_mask & (1 << portid)) == 0) { 860 printf("\nSkipping disabled port %d\n", portid); 861 continue; 862 } 863 864 /* Init port */ 865 printf("Initializing port %d ... ", portid); 866 fflush(stdout); 867 868 nb_rx_queue = get_port_n_rx_queues(portid); 869 n_tx_queue = nb_lcores; 870 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT) 871 n_tx_queue = MAX_TX_QUEUE_PER_PORT; 872 printf("Creating queues: nb_rxq=%d nb_txq=%u... ", 873 nb_rx_queue, n_tx_queue); 874 875 rte_eth_dev_info_get(portid, &dev_info); 876 877 ret = config_port_max_pkt_len(&local_port_conf, &dev_info); 878 if (ret != 0) 879 rte_exit(EXIT_FAILURE, 880 "Invalid max packet length: %u (port %u)\n", 881 max_pkt_len, portid); 882 883 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 884 local_port_conf.txmode.offloads |= 885 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 886 887 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 888 dev_info.flow_type_rss_offloads; 889 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 890 port_conf.rx_adv_conf.rss_conf.rss_hf) { 891 printf("Port %u modified RSS hash function based on " 892 "hardware support," 893 "requested:%#" PRIx64 " configured:%#" PRIx64 894 "\n", 895 portid, port_conf.rx_adv_conf.rss_conf.rss_hf, 896 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 897 } 898 899 ret = rte_eth_dev_configure(portid, nb_rx_queue, 900 n_tx_queue, &local_port_conf); 901 if (ret < 0) 902 rte_exit(EXIT_FAILURE, 903 "Cannot configure device: err=%d, port=%d\n", 904 ret, portid); 905 906 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 907 &nb_txd); 908 if (ret < 0) 909 rte_exit(EXIT_FAILURE, 910 "Cannot adjust number of descriptors: err=%d, " 911 "port=%d\n", 912 ret, portid); 913 914 rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 915 print_ethaddr(" Address:", &ports_eth_addr[portid]); 916 printf(", "); 917 print_ethaddr( 918 "Destination:", 919 (const struct rte_ether_addr *)&dest_eth_addr[portid]); 920 printf(", "); 921 922 /* 923 * prepare src MACs for each port. 924 */ 925 rte_ether_addr_copy( 926 &ports_eth_addr[portid], 927 (struct rte_ether_addr *)(val_eth + portid) + 1); 928 929 /* Init memory */ 930 if (!per_port_pool) { 931 /* portid = 0; this is *not* signifying the first port, 932 * rather, it signifies that portid is ignored. 933 */ 934 ret = init_mem(0, NB_MBUF(nb_ports)); 935 } else { 936 ret = init_mem(portid, NB_MBUF(1)); 937 } 938 if (ret < 0) 939 rte_exit(EXIT_FAILURE, "init_mem() failed\n"); 940 941 /* Init one TX queue per couple (lcore,port) */ 942 queueid = 0; 943 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 944 if (rte_lcore_is_enabled(lcore_id) == 0) 945 continue; 946 947 qconf = &lcore_conf[lcore_id]; 948 949 if (numa_on) 950 socketid = (uint8_t)rte_lcore_to_socket_id( 951 lcore_id); 952 else 953 socketid = 0; 954 955 printf("txq=%u,%d,%d ", lcore_id, queueid, socketid); 956 fflush(stdout); 957 958 txconf = &dev_info.default_txconf; 959 txconf->offloads = local_port_conf.txmode.offloads; 960 ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, 961 socketid, txconf); 962 if (ret < 0) 963 rte_exit(EXIT_FAILURE, 964 "rte_eth_tx_queue_setup: err=%d, " 965 "port=%d\n", 966 ret, portid); 967 queueid++; 968 } 969 970 /* Setup ethdev node config */ 971 ethdev_conf[nb_conf].port_id = portid; 972 ethdev_conf[nb_conf].num_rx_queues = nb_rx_queue; 973 ethdev_conf[nb_conf].num_tx_queues = n_tx_queue; 974 if (!per_port_pool) 975 ethdev_conf[nb_conf].mp = pktmbuf_pool[0]; 976 977 else 978 ethdev_conf[nb_conf].mp = pktmbuf_pool[portid]; 979 ethdev_conf[nb_conf].mp_count = NB_SOCKETS; 980 981 nb_conf++; 982 printf("\n"); 983 } 984 985 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 986 if (rte_lcore_is_enabled(lcore_id) == 0) 987 continue; 988 qconf = &lcore_conf[lcore_id]; 989 printf("\nInitializing rx queues on lcore %u ... ", lcore_id); 990 fflush(stdout); 991 /* Init RX queues */ 992 for (queue = 0; queue < qconf->n_rx_queue; ++queue) { 993 struct rte_eth_rxconf rxq_conf; 994 995 portid = qconf->rx_queue_list[queue].port_id; 996 queueid = qconf->rx_queue_list[queue].queue_id; 997 998 if (numa_on) 999 socketid = (uint8_t)rte_lcore_to_socket_id( 1000 lcore_id); 1001 else 1002 socketid = 0; 1003 1004 printf("rxq=%d,%d,%d ", portid, queueid, socketid); 1005 fflush(stdout); 1006 1007 rte_eth_dev_info_get(portid, &dev_info); 1008 rxq_conf = dev_info.default_rxconf; 1009 rxq_conf.offloads = port_conf.rxmode.offloads; 1010 if (!per_port_pool) 1011 ret = rte_eth_rx_queue_setup( 1012 portid, queueid, nb_rxd, socketid, 1013 &rxq_conf, pktmbuf_pool[0][socketid]); 1014 else 1015 ret = rte_eth_rx_queue_setup( 1016 portid, queueid, nb_rxd, socketid, 1017 &rxq_conf, 1018 pktmbuf_pool[portid][socketid]); 1019 if (ret < 0) 1020 rte_exit(EXIT_FAILURE, 1021 "rte_eth_rx_queue_setup: err=%d, " 1022 "port=%d\n", 1023 ret, portid); 1024 1025 /* Add this queue node to its graph */ 1026 snprintf(qconf->rx_queue_list[queue].node_name, 1027 RTE_NODE_NAMESIZE, "ethdev_rx-%u-%u", portid, 1028 queueid); 1029 } 1030 1031 /* Alloc a graph to this lcore only if source exists */ 1032 if (qconf->n_rx_queue) 1033 nb_graphs++; 1034 } 1035 1036 printf("\n"); 1037 1038 /* Ethdev node config, skip rx queue mapping */ 1039 ret = rte_node_eth_config(ethdev_conf, nb_conf, nb_graphs); 1040 /* >8 End of graph creation. */ 1041 if (ret) 1042 rte_exit(EXIT_FAILURE, "rte_node_eth_config: err=%d\n", ret); 1043 1044 /* Start ports */ 1045 RTE_ETH_FOREACH_DEV(portid) 1046 { 1047 if ((enabled_port_mask & (1 << portid)) == 0) 1048 continue; 1049 1050 /* Start device */ 1051 ret = rte_eth_dev_start(portid); 1052 if (ret < 0) 1053 rte_exit(EXIT_FAILURE, 1054 "rte_eth_dev_start: err=%d, port=%d\n", ret, 1055 portid); 1056 1057 /* 1058 * If enabled, put device in promiscuous mode. 1059 * This allows IO forwarding mode to forward packets 1060 * to itself through 2 cross-connected ports of the 1061 * target machine. 1062 */ 1063 if (promiscuous_on) 1064 rte_eth_promiscuous_enable(portid); 1065 } 1066 1067 printf("\n"); 1068 1069 check_all_ports_link_status(enabled_port_mask); 1070 1071 /* Graph Initialization */ 1072 nb_patterns = RTE_DIM(default_patterns); 1073 node_patterns = malloc((MAX_RX_QUEUE_PER_LCORE + nb_patterns) * 1074 sizeof(*node_patterns)); 1075 if (!node_patterns) 1076 return -ENOMEM; 1077 memcpy(node_patterns, default_patterns, 1078 nb_patterns * sizeof(*node_patterns)); 1079 1080 memset(&graph_conf, 0, sizeof(graph_conf)); 1081 graph_conf.node_patterns = node_patterns; 1082 1083 /* Pcap config */ 1084 graph_conf.pcap_enable = pcap_trace_enable; 1085 graph_conf.num_pkt_to_capture = packet_to_capture; 1086 graph_conf.pcap_filename = pcap_filename; 1087 1088 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1089 rte_graph_t graph_id; 1090 rte_edge_t i; 1091 1092 if (rte_lcore_is_enabled(lcore_id) == 0) 1093 continue; 1094 1095 qconf = &lcore_conf[lcore_id]; 1096 1097 /* Skip graph creation if no source exists */ 1098 if (!qconf->n_rx_queue) 1099 continue; 1100 1101 /* Add rx node patterns of this lcore */ 1102 for (i = 0; i < qconf->n_rx_queue; i++) { 1103 graph_conf.node_patterns[nb_patterns + i] = 1104 qconf->rx_queue_list[i].node_name; 1105 } 1106 1107 graph_conf.nb_node_patterns = nb_patterns + i; 1108 graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id); 1109 1110 snprintf(qconf->name, sizeof(qconf->name), "worker_%u", 1111 lcore_id); 1112 1113 graph_id = rte_graph_create(qconf->name, &graph_conf); 1114 if (graph_id == RTE_GRAPH_ID_INVALID) 1115 rte_exit(EXIT_FAILURE, 1116 "rte_graph_create(): graph_id invalid" 1117 " for lcore %u\n", lcore_id); 1118 1119 qconf->graph_id = graph_id; 1120 qconf->graph = rte_graph_lookup(qconf->name); 1121 /* >8 End of graph initialization. */ 1122 if (!qconf->graph) 1123 rte_exit(EXIT_FAILURE, 1124 "rte_graph_lookup(): graph %s not found\n", 1125 qconf->name); 1126 } 1127 1128 memset(&rewrite_data, 0, sizeof(rewrite_data)); 1129 rewrite_len = sizeof(rewrite_data); 1130 1131 /* Add route to ip4 graph infra. 8< */ 1132 for (i = 0; i < IPV4_L3FWD_LPM_NUM_ROUTES; i++) { 1133 char route_str[INET6_ADDRSTRLEN * 4]; 1134 char abuf[INET6_ADDRSTRLEN]; 1135 struct in_addr in; 1136 uint32_t dst_port; 1137 1138 /* Skip unused ports */ 1139 if ((1 << ipv4_l3fwd_lpm_route_array[i].if_out & 1140 enabled_port_mask) == 0) 1141 continue; 1142 1143 dst_port = ipv4_l3fwd_lpm_route_array[i].if_out; 1144 1145 in.s_addr = htonl(ipv4_l3fwd_lpm_route_array[i].ip); 1146 snprintf(route_str, sizeof(route_str), "%s / %d (%d)", 1147 inet_ntop(AF_INET, &in, abuf, sizeof(abuf)), 1148 ipv4_l3fwd_lpm_route_array[i].depth, 1149 ipv4_l3fwd_lpm_route_array[i].if_out); 1150 1151 /* Use route index 'i' as next hop id */ 1152 ret = rte_node_ip4_route_add( 1153 ipv4_l3fwd_lpm_route_array[i].ip, 1154 ipv4_l3fwd_lpm_route_array[i].depth, i, 1155 RTE_NODE_IP4_LOOKUP_NEXT_REWRITE); 1156 1157 if (ret < 0) 1158 rte_exit(EXIT_FAILURE, 1159 "Unable to add ip4 route %s to graph\n", 1160 route_str); 1161 1162 memcpy(rewrite_data, val_eth + dst_port, rewrite_len); 1163 1164 /* Add next hop rewrite data for id 'i' */ 1165 ret = rte_node_ip4_rewrite_add(i, rewrite_data, 1166 rewrite_len, dst_port); 1167 if (ret < 0) 1168 rte_exit(EXIT_FAILURE, 1169 "Unable to add next hop %u for " 1170 "route %s\n", i, route_str); 1171 1172 RTE_LOG(INFO, L3FWD_GRAPH, "Added route %s, next_hop %u\n", 1173 route_str, i); 1174 } 1175 /* >8 End of adding route to ip4 graph infa. */ 1176 1177 /* Launch per-lcore init on every worker lcore */ 1178 rte_eal_mp_remote_launch(graph_main_loop, NULL, SKIP_MAIN); 1179 1180 /* Accumulate and print stats on main until exit */ 1181 if (rte_graph_has_stats_feature()) 1182 print_stats(); 1183 1184 /* Wait for worker cores to exit */ 1185 ret = 0; 1186 RTE_LCORE_FOREACH_WORKER(lcore_id) { 1187 ret = rte_eal_wait_lcore(lcore_id); 1188 /* Destroy graph */ 1189 if (ret < 0 || rte_graph_destroy( 1190 rte_graph_from_name(lcore_conf[lcore_id].name))) { 1191 ret = -1; 1192 break; 1193 } 1194 } 1195 free(node_patterns); 1196 1197 /* Stop ports */ 1198 RTE_ETH_FOREACH_DEV(portid) { 1199 if ((enabled_port_mask & (1 << portid)) == 0) 1200 continue; 1201 printf("Closing port %d...", portid); 1202 ret = rte_eth_dev_stop(portid); 1203 if (ret != 0) 1204 printf("Failed to stop port %u: %s\n", 1205 portid, rte_strerror(-ret)); 1206 rte_eth_dev_close(portid); 1207 printf(" Done\n"); 1208 } 1209 1210 /* clean up the EAL */ 1211 rte_eal_cleanup(); 1212 printf("Bye...\n"); 1213 1214 return ret; 1215 } 1216