1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2021 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <string.h> 11 #include <sys/queue.h> 12 #include <stdarg.h> 13 #include <errno.h> 14 #include <getopt.h> 15 #include <signal.h> 16 #include <stdbool.h> 17 18 #include <rte_common.h> 19 #include <rte_vect.h> 20 #include <rte_byteorder.h> 21 #include <rte_log.h> 22 #include <rte_malloc.h> 23 #include <rte_memory.h> 24 #include <rte_memcpy.h> 25 #include <rte_eal.h> 26 #include <rte_launch.h> 27 #include <rte_cycles.h> 28 #include <rte_prefetch.h> 29 #include <rte_lcore.h> 30 #include <rte_per_lcore.h> 31 #include <rte_branch_prediction.h> 32 #include <rte_interrupts.h> 33 #include <rte_random.h> 34 #include <rte_debug.h> 35 #include <rte_ether.h> 36 #include <rte_mempool.h> 37 #include <rte_mbuf.h> 38 #include <rte_ip.h> 39 #include <rte_tcp.h> 40 #include <rte_udp.h> 41 #include <rte_string_fns.h> 42 #include <rte_cpuflags.h> 43 44 #include <cmdline_parse.h> 45 #include <cmdline_parse_etheraddr.h> 46 47 #include "l3fwd.h" 48 #include "l3fwd_event.h" 49 #include "l3fwd_route.h" 50 51 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_LCORE 52 #define MAX_RX_QUEUE_PER_PORT 128 53 54 #define MAX_LCORE_PARAMS 1024 55 56 uint16_t nb_rxd = RX_DESC_DEFAULT; 57 uint16_t nb_txd = TX_DESC_DEFAULT; 58 59 /**< Ports set in promiscuous mode off by default. */ 60 static int promiscuous_on; 61 62 /* Select Longest-Prefix, Exact match, Forwarding Information Base or Access Control. */ 63 enum L3FWD_LOOKUP_MODE { 64 L3FWD_LOOKUP_DEFAULT, 65 L3FWD_LOOKUP_LPM, 66 L3FWD_LOOKUP_EM, 67 L3FWD_LOOKUP_FIB, 68 L3FWD_LOOKUP_ACL 69 }; 70 static enum L3FWD_LOOKUP_MODE lookup_mode; 71 72 /* Global variables. */ 73 static int numa_on = 1; /**< NUMA is enabled by default. */ 74 static int parse_ptype; /**< Parse packet type using rx callback, and */ 75 /**< disabled by default */ 76 static int disable_rss; /**< Disable RSS mode */ 77 static int relax_rx_offload; /**< Relax Rx offload mode, disabled by default */ 78 static int per_port_pool; /**< Use separate buffer pools per port; disabled */ 79 /**< by default */ 80 81 volatile bool force_quit; 82 83 /* ethernet addresses of ports */ 84 uint64_t dest_eth_addr[RTE_MAX_ETHPORTS]; 85 struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; 86 87 xmm_t val_eth[RTE_MAX_ETHPORTS]; 88 89 /* mask of enabled ports */ 90 uint32_t enabled_port_mask; 91 92 /* Used only in exact match mode. */ 93 int ipv6; /**< ipv6 is false by default. */ 94 95 struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 96 97 struct parm_cfg parm_config; 98 99 struct lcore_params { 100 uint16_t port_id; 101 uint8_t queue_id; 102 uint8_t lcore_id; 103 } __rte_cache_aligned; 104 105 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 106 static struct lcore_params lcore_params_array_default[] = { 107 {0, 0, 2}, 108 {0, 1, 2}, 109 {0, 2, 2}, 110 {1, 0, 2}, 111 {1, 1, 2}, 112 {1, 2, 2}, 113 {2, 0, 2}, 114 {3, 0, 3}, 115 {3, 1, 3}, 116 }; 117 118 static struct lcore_params * lcore_params = lcore_params_array_default; 119 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) / 120 sizeof(lcore_params_array_default[0]); 121 122 static struct rte_eth_conf port_conf = { 123 .rxmode = { 124 .mq_mode = RTE_ETH_MQ_RX_RSS, 125 .offloads = RTE_ETH_RX_OFFLOAD_CHECKSUM, 126 }, 127 .rx_adv_conf = { 128 .rss_conf = { 129 .rss_key = NULL, 130 .rss_hf = RTE_ETH_RSS_IP, 131 }, 132 }, 133 .txmode = { 134 .mq_mode = RTE_ETH_MQ_TX_NONE, 135 }, 136 }; 137 138 uint32_t max_pkt_len; 139 140 #ifdef RTE_LIB_EVENTDEV 141 static struct rte_mempool *vector_pool[RTE_MAX_ETHPORTS]; 142 #endif 143 static struct rte_mempool *pktmbuf_pool[RTE_MAX_ETHPORTS][NB_SOCKETS]; 144 static uint8_t lkp_per_socket[NB_SOCKETS]; 145 146 struct l3fwd_lkp_mode { 147 void (*read_config_files)(void); 148 void (*setup)(int); 149 int (*check_ptype)(int); 150 rte_rx_callback_fn cb_parse_ptype; 151 int (*main_loop)(void *); 152 void* (*get_ipv4_lookup_struct)(int); 153 void* (*get_ipv6_lookup_struct)(int); 154 void (*free_routes)(void); 155 }; 156 157 static struct l3fwd_lkp_mode l3fwd_lkp; 158 159 static struct l3fwd_lkp_mode l3fwd_em_lkp = { 160 .read_config_files = read_config_files_em, 161 .setup = setup_hash, 162 .check_ptype = em_check_ptype, 163 .cb_parse_ptype = em_cb_parse_ptype, 164 .main_loop = em_main_loop, 165 .get_ipv4_lookup_struct = em_get_ipv4_l3fwd_lookup_struct, 166 .get_ipv6_lookup_struct = em_get_ipv6_l3fwd_lookup_struct, 167 .free_routes = em_free_routes, 168 }; 169 170 static struct l3fwd_lkp_mode l3fwd_lpm_lkp = { 171 .read_config_files = read_config_files_lpm, 172 .setup = setup_lpm, 173 .check_ptype = lpm_check_ptype, 174 .cb_parse_ptype = lpm_cb_parse_ptype, 175 .main_loop = lpm_main_loop, 176 .get_ipv4_lookup_struct = lpm_get_ipv4_l3fwd_lookup_struct, 177 .get_ipv6_lookup_struct = lpm_get_ipv6_l3fwd_lookup_struct, 178 .free_routes = lpm_free_routes, 179 }; 180 181 static struct l3fwd_lkp_mode l3fwd_fib_lkp = { 182 .read_config_files = read_config_files_lpm, 183 .setup = setup_fib, 184 .check_ptype = lpm_check_ptype, 185 .cb_parse_ptype = lpm_cb_parse_ptype, 186 .main_loop = fib_main_loop, 187 .get_ipv4_lookup_struct = fib_get_ipv4_l3fwd_lookup_struct, 188 .get_ipv6_lookup_struct = fib_get_ipv6_l3fwd_lookup_struct, 189 .free_routes = lpm_free_routes, 190 }; 191 192 static struct l3fwd_lkp_mode l3fwd_acl_lkp = { 193 .read_config_files = read_config_files_acl, 194 .setup = setup_acl, 195 .check_ptype = em_check_ptype, 196 .cb_parse_ptype = em_cb_parse_ptype, 197 .main_loop = acl_main_loop, 198 .get_ipv4_lookup_struct = acl_get_ipv4_l3fwd_lookup_struct, 199 .get_ipv6_lookup_struct = acl_get_ipv6_l3fwd_lookup_struct, 200 .free_routes = acl_free_routes, 201 }; 202 203 /* 204 * 198.18.0.0/16 are set aside for RFC2544 benchmarking (RFC5735). 205 * 198.18.{0-15}.0/24 = Port {0-15} 206 */ 207 const struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { 208 {RTE_IPV4(198, 18, 0, 0), 24, 0}, 209 {RTE_IPV4(198, 18, 1, 0), 24, 1}, 210 {RTE_IPV4(198, 18, 2, 0), 24, 2}, 211 {RTE_IPV4(198, 18, 3, 0), 24, 3}, 212 {RTE_IPV4(198, 18, 4, 0), 24, 4}, 213 {RTE_IPV4(198, 18, 5, 0), 24, 5}, 214 {RTE_IPV4(198, 18, 6, 0), 24, 6}, 215 {RTE_IPV4(198, 18, 7, 0), 24, 7}, 216 {RTE_IPV4(198, 18, 8, 0), 24, 8}, 217 {RTE_IPV4(198, 18, 9, 0), 24, 9}, 218 {RTE_IPV4(198, 18, 10, 0), 24, 10}, 219 {RTE_IPV4(198, 18, 11, 0), 24, 11}, 220 {RTE_IPV4(198, 18, 12, 0), 24, 12}, 221 {RTE_IPV4(198, 18, 13, 0), 24, 13}, 222 {RTE_IPV4(198, 18, 14, 0), 24, 14}, 223 {RTE_IPV4(198, 18, 15, 0), 24, 15}, 224 }; 225 226 /* 227 * 2001:200::/48 is IANA reserved range for IPv6 benchmarking (RFC5180). 228 * 2001:200:0:{0-f}::/64 = Port {0-15} 229 */ 230 const struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = { 231 {{32, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 0}, 232 {{32, 1, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 1}, 233 {{32, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 2}, 234 {{32, 1, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 3}, 235 {{32, 1, 2, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 4}, 236 {{32, 1, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 5}, 237 {{32, 1, 2, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 6}, 238 {{32, 1, 2, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 7}, 239 {{32, 1, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 8}, 240 {{32, 1, 2, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 9}, 241 {{32, 1, 2, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 10}, 242 {{32, 1, 2, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 11}, 243 {{32, 1, 2, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 12}, 244 {{32, 1, 2, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 13}, 245 {{32, 1, 2, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 14}, 246 {{32, 1, 2, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 15}, 247 }; 248 249 /* 250 * API's called during initialization to setup ACL/EM/LPM rules. 251 */ 252 void 253 l3fwd_set_rule_ipv4_name(const char *optarg) 254 { 255 parm_config.rule_ipv4_name = optarg; 256 } 257 258 void 259 l3fwd_set_rule_ipv6_name(const char *optarg) 260 { 261 parm_config.rule_ipv6_name = optarg; 262 } 263 264 void 265 l3fwd_set_alg(const char *optarg) 266 { 267 parm_config.alg = parse_acl_alg(optarg); 268 } 269 270 /* 271 * Setup lookup methods for forwarding. 272 * Currently exact-match, longest-prefix-match and forwarding information 273 * base are the supported ones. 274 */ 275 static void 276 setup_l3fwd_lookup_tables(void) 277 { 278 /* Setup HASH lookup functions. */ 279 if (lookup_mode == L3FWD_LOOKUP_EM) 280 l3fwd_lkp = l3fwd_em_lkp; 281 /* Setup FIB lookup functions. */ 282 else if (lookup_mode == L3FWD_LOOKUP_FIB) 283 l3fwd_lkp = l3fwd_fib_lkp; 284 /* Setup ACL lookup functions. */ 285 else if (lookup_mode == L3FWD_LOOKUP_ACL) 286 l3fwd_lkp = l3fwd_acl_lkp; 287 /* Setup LPM lookup functions. */ 288 else 289 l3fwd_lkp = l3fwd_lpm_lkp; 290 } 291 292 static int 293 check_lcore_params(void) 294 { 295 uint8_t queue, lcore; 296 uint16_t i; 297 int socketid; 298 299 for (i = 0; i < nb_lcore_params; ++i) { 300 queue = lcore_params[i].queue_id; 301 if (queue >= MAX_RX_QUEUE_PER_PORT) { 302 printf("invalid queue number: %hhu\n", queue); 303 return -1; 304 } 305 lcore = lcore_params[i].lcore_id; 306 if (!rte_lcore_is_enabled(lcore)) { 307 printf("error: lcore %hhu is not enabled in lcore mask\n", lcore); 308 return -1; 309 } 310 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) && 311 (numa_on == 0)) { 312 printf("warning: lcore %hhu is on socket %d with numa off \n", 313 lcore, socketid); 314 } 315 } 316 return 0; 317 } 318 319 static int 320 check_port_config(void) 321 { 322 uint16_t portid; 323 uint16_t i; 324 325 for (i = 0; i < nb_lcore_params; ++i) { 326 portid = lcore_params[i].port_id; 327 if ((enabled_port_mask & (1 << portid)) == 0) { 328 printf("port %u is not enabled in port mask\n", portid); 329 return -1; 330 } 331 if (!rte_eth_dev_is_valid_port(portid)) { 332 printf("port %u is not present on the board\n", portid); 333 return -1; 334 } 335 } 336 return 0; 337 } 338 339 static uint8_t 340 get_port_n_rx_queues(const uint16_t port) 341 { 342 int queue = -1; 343 uint16_t i; 344 345 for (i = 0; i < nb_lcore_params; ++i) { 346 if (lcore_params[i].port_id == port) { 347 if (lcore_params[i].queue_id == queue+1) 348 queue = lcore_params[i].queue_id; 349 else 350 rte_exit(EXIT_FAILURE, "queue ids of the port %d must be" 351 " in sequence and must start with 0\n", 352 lcore_params[i].port_id); 353 } 354 } 355 return (uint8_t)(++queue); 356 } 357 358 static int 359 init_lcore_rx_queues(void) 360 { 361 uint16_t i, nb_rx_queue; 362 uint8_t lcore; 363 364 for (i = 0; i < nb_lcore_params; ++i) { 365 lcore = lcore_params[i].lcore_id; 366 nb_rx_queue = lcore_conf[lcore].n_rx_queue; 367 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 368 printf("error: too many queues (%u) for lcore: %u\n", 369 (unsigned)nb_rx_queue + 1, (unsigned)lcore); 370 return -1; 371 } else { 372 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 373 lcore_params[i].port_id; 374 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 375 lcore_params[i].queue_id; 376 lcore_conf[lcore].n_rx_queue++; 377 } 378 } 379 return 0; 380 } 381 382 /* display usage */ 383 static void 384 print_usage(const char *prgname) 385 { 386 char alg[PATH_MAX]; 387 388 usage_acl_alg(alg, sizeof(alg)); 389 fprintf(stderr, "%s [EAL options] --" 390 " -p PORTMASK" 391 " --rule_ipv4=FILE" 392 " --rule_ipv6=FILE" 393 " [-P]" 394 " [--lookup]" 395 " --config (port,queue,lcore)[,(port,queue,lcore)]" 396 " [--rx-queue-size NPKTS]" 397 " [--tx-queue-size NPKTS]" 398 " [--eth-dest=X,MM:MM:MM:MM:MM:MM]" 399 " [--max-pkt-len PKTLEN]" 400 " [--no-numa]" 401 " [--ipv6]" 402 " [--parse-ptype]" 403 " [--per-port-pool]" 404 " [--mode]" 405 #ifdef RTE_LIB_EVENTDEV 406 " [--eventq-sched]" 407 " [--event-vector [--event-vector-size SIZE] [--event-vector-tmo NS]]" 408 #endif 409 " [-E]" 410 " [-L]\n\n" 411 412 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 413 " -P : Enable promiscuous mode\n" 414 " --lookup: Select the lookup method\n" 415 " Default: lpm\n" 416 " Accepted: em (Exact Match), lpm (Longest Prefix Match), fib (Forwarding Information Base),\n" 417 " acl (Access Control List)\n" 418 " --config (port,queue,lcore): Rx queue configuration\n" 419 " --rx-queue-size NPKTS: Rx queue size in decimal\n" 420 " Default: %d\n" 421 " --tx-queue-size NPKTS: Tx queue size in decimal\n" 422 " Default: %d\n" 423 " --eth-dest=X,MM:MM:MM:MM:MM:MM: Ethernet destination for port X\n" 424 " --max-pkt-len PKTLEN: maximum packet length in decimal (64-9600)\n" 425 " --no-numa: Disable numa awareness\n" 426 " --ipv6: Set if running ipv6 packets\n" 427 " --parse-ptype: Set to use software to analyze packet type\n" 428 " --per-port-pool: Use separate buffer pool per port\n" 429 " --mode: Packet transfer mode for I/O, poll or eventdev\n" 430 " Default mode = poll\n" 431 #ifdef RTE_LIB_EVENTDEV 432 " --eventq-sched: Event queue synchronization method\n" 433 " ordered, atomic or parallel.\n" 434 " Default: atomic\n" 435 " Valid only if --mode=eventdev\n" 436 " --event-eth-rxqs: Number of ethernet RX queues per device.\n" 437 " Default: 1\n" 438 " Valid only if --mode=eventdev\n" 439 " --event-vector: Enable event vectorization.\n" 440 " --event-vector-size: Max vector size if event vectorization is enabled.\n" 441 " --event-vector-tmo: Max timeout to form vector in nanoseconds if event vectorization is enabled\n" 442 #endif 443 " -E : Enable exact match, legacy flag please use --lookup=em instead\n" 444 " -L : Enable longest prefix match, legacy flag please use --lookup=lpm instead\n" 445 " --rule_ipv4=FILE: Specify the ipv4 rules entries file.\n" 446 " Each rule occupies one line.\n" 447 " 2 kinds of rules are supported.\n" 448 " One is ACL entry at while line leads with character '%c',\n" 449 " another is route entry at while line leads with character '%c'.\n" 450 " --rule_ipv6=FILE: Specify the ipv6 rules entries file.\n" 451 " --alg: ACL classify method to use, one of: %s.\n\n", 452 prgname, RX_DESC_DEFAULT, TX_DESC_DEFAULT, 453 ACL_LEAD_CHAR, ROUTE_LEAD_CHAR, alg); 454 } 455 456 static int 457 parse_max_pkt_len(const char *pktlen) 458 { 459 char *end = NULL; 460 unsigned long len; 461 462 /* parse decimal string */ 463 len = strtoul(pktlen, &end, 10); 464 if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) 465 return -1; 466 467 if (len == 0) 468 return -1; 469 470 return len; 471 } 472 473 static int 474 parse_portmask(const char *portmask) 475 { 476 char *end = NULL; 477 unsigned long pm; 478 479 /* parse hexadecimal string */ 480 pm = strtoul(portmask, &end, 16); 481 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 482 return 0; 483 484 return pm; 485 } 486 487 static int 488 parse_config(const char *q_arg) 489 { 490 char s[256]; 491 const char *p, *p0 = q_arg; 492 char *end; 493 enum fieldnames { 494 FLD_PORT = 0, 495 FLD_QUEUE, 496 FLD_LCORE, 497 _NUM_FLD 498 }; 499 unsigned long int_fld[_NUM_FLD]; 500 char *str_fld[_NUM_FLD]; 501 int i; 502 unsigned size; 503 504 nb_lcore_params = 0; 505 506 while ((p = strchr(p0,'(')) != NULL) { 507 ++p; 508 if((p0 = strchr(p,')')) == NULL) 509 return -1; 510 511 size = p0 - p; 512 if(size >= sizeof(s)) 513 return -1; 514 515 snprintf(s, sizeof(s), "%.*s", size, p); 516 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD) 517 return -1; 518 for (i = 0; i < _NUM_FLD; i++){ 519 errno = 0; 520 int_fld[i] = strtoul(str_fld[i], &end, 0); 521 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 522 return -1; 523 } 524 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 525 printf("exceeded max number of lcore params: %hu\n", 526 nb_lcore_params); 527 return -1; 528 } 529 lcore_params_array[nb_lcore_params].port_id = 530 (uint8_t)int_fld[FLD_PORT]; 531 lcore_params_array[nb_lcore_params].queue_id = 532 (uint8_t)int_fld[FLD_QUEUE]; 533 lcore_params_array[nb_lcore_params].lcore_id = 534 (uint8_t)int_fld[FLD_LCORE]; 535 ++nb_lcore_params; 536 } 537 lcore_params = lcore_params_array; 538 return 0; 539 } 540 541 static void 542 parse_eth_dest(const char *optarg) 543 { 544 uint16_t portid; 545 char *port_end; 546 uint8_t c, *dest, peer_addr[6]; 547 548 errno = 0; 549 portid = strtoul(optarg, &port_end, 10); 550 if (errno != 0 || port_end == optarg || *port_end++ != ',') 551 rte_exit(EXIT_FAILURE, 552 "Invalid eth-dest: %s", optarg); 553 if (portid >= RTE_MAX_ETHPORTS) 554 rte_exit(EXIT_FAILURE, 555 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n", 556 portid, RTE_MAX_ETHPORTS); 557 558 if (cmdline_parse_etheraddr(NULL, port_end, 559 &peer_addr, sizeof(peer_addr)) < 0) 560 rte_exit(EXIT_FAILURE, 561 "Invalid ethernet address: %s\n", 562 port_end); 563 dest = (uint8_t *)&dest_eth_addr[portid]; 564 for (c = 0; c < 6; c++) 565 dest[c] = peer_addr[c]; 566 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid]; 567 } 568 569 static void 570 parse_mode(const char *optarg __rte_unused) 571 { 572 #ifdef RTE_LIB_EVENTDEV 573 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 574 575 if (!strcmp(optarg, "poll")) 576 evt_rsrc->enabled = false; 577 else if (!strcmp(optarg, "eventdev")) 578 evt_rsrc->enabled = true; 579 #endif 580 } 581 582 static void 583 parse_queue_size(const char *queue_size_arg, uint16_t *queue_size, int rx) 584 { 585 char *end = NULL; 586 unsigned long value; 587 588 /* parse decimal string */ 589 value = strtoul(queue_size_arg, &end, 10); 590 if ((queue_size_arg[0] == '\0') || (end == NULL) || 591 (*end != '\0') || (value == 0)) { 592 if (rx == 1) 593 rte_exit(EXIT_FAILURE, "Invalid rx-queue-size\n"); 594 else 595 rte_exit(EXIT_FAILURE, "Invalid tx-queue-size\n"); 596 597 return; 598 } 599 600 if (value > UINT16_MAX) { 601 if (rx == 1) 602 rte_exit(EXIT_FAILURE, "rx-queue-size %lu > %d\n", 603 value, UINT16_MAX); 604 else 605 rte_exit(EXIT_FAILURE, "tx-queue-size %lu > %d\n", 606 value, UINT16_MAX); 607 608 return; 609 } 610 611 *queue_size = value; 612 } 613 614 #ifdef RTE_LIB_EVENTDEV 615 static void 616 parse_eventq_sched(const char *optarg) 617 { 618 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 619 620 if (!strcmp(optarg, "ordered")) 621 evt_rsrc->sched_type = RTE_SCHED_TYPE_ORDERED; 622 if (!strcmp(optarg, "atomic")) 623 evt_rsrc->sched_type = RTE_SCHED_TYPE_ATOMIC; 624 if (!strcmp(optarg, "parallel")) 625 evt_rsrc->sched_type = RTE_SCHED_TYPE_PARALLEL; 626 } 627 628 static void 629 parse_event_eth_rx_queues(const char *eth_rx_queues) 630 { 631 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 632 char *end = NULL; 633 uint8_t num_eth_rx_queues; 634 635 /* parse decimal string */ 636 num_eth_rx_queues = strtoul(eth_rx_queues, &end, 10); 637 if ((eth_rx_queues[0] == '\0') || (end == NULL) || (*end != '\0')) 638 return; 639 640 if (num_eth_rx_queues == 0) 641 return; 642 643 evt_rsrc->eth_rx_queues = num_eth_rx_queues; 644 } 645 #endif 646 647 static int 648 parse_lookup(const char *optarg) 649 { 650 if (!strcmp(optarg, "em")) 651 lookup_mode = L3FWD_LOOKUP_EM; 652 else if (!strcmp(optarg, "lpm")) 653 lookup_mode = L3FWD_LOOKUP_LPM; 654 else if (!strcmp(optarg, "fib")) 655 lookup_mode = L3FWD_LOOKUP_FIB; 656 else if (!strcmp(optarg, "acl")) 657 lookup_mode = L3FWD_LOOKUP_ACL; 658 else { 659 fprintf(stderr, "Invalid lookup option! Accepted options: acl, em, lpm, fib\n"); 660 return -1; 661 } 662 return 0; 663 } 664 665 #define MAX_JUMBO_PKT_LEN 9600 666 667 static const char short_options[] = 668 "p:" /* portmask */ 669 "P" /* promiscuous */ 670 "L" /* legacy enable long prefix match */ 671 "E" /* legacy enable exact match */ 672 ; 673 674 #define CMD_LINE_OPT_CONFIG "config" 675 #define CMD_LINE_OPT_RX_QUEUE_SIZE "rx-queue-size" 676 #define CMD_LINE_OPT_TX_QUEUE_SIZE "tx-queue-size" 677 #define CMD_LINE_OPT_ETH_DEST "eth-dest" 678 #define CMD_LINE_OPT_NO_NUMA "no-numa" 679 #define CMD_LINE_OPT_IPV6 "ipv6" 680 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len" 681 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num" 682 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype" 683 #define CMD_LINE_OPT_DISABLE_RSS "disable-rss" 684 #define CMD_LINE_OPT_RELAX_RX_OFFLOAD "relax-rx-offload" 685 #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool" 686 #define CMD_LINE_OPT_MODE "mode" 687 #define CMD_LINE_OPT_EVENTQ_SYNC "eventq-sched" 688 #define CMD_LINE_OPT_EVENT_ETH_RX_QUEUES "event-eth-rxqs" 689 #define CMD_LINE_OPT_LOOKUP "lookup" 690 #define CMD_LINE_OPT_ENABLE_VECTOR "event-vector" 691 #define CMD_LINE_OPT_VECTOR_SIZE "event-vector-size" 692 #define CMD_LINE_OPT_VECTOR_TMO_NS "event-vector-tmo" 693 #define CMD_LINE_OPT_RULE_IPV4 "rule_ipv4" 694 #define CMD_LINE_OPT_RULE_IPV6 "rule_ipv6" 695 #define CMD_LINE_OPT_ALG "alg" 696 697 enum { 698 /* long options mapped to a short option */ 699 700 /* first long only option value must be >= 256, so that we won't 701 * conflict with short options */ 702 CMD_LINE_OPT_MIN_NUM = 256, 703 CMD_LINE_OPT_CONFIG_NUM, 704 CMD_LINE_OPT_RX_QUEUE_SIZE_NUM, 705 CMD_LINE_OPT_TX_QUEUE_SIZE_NUM, 706 CMD_LINE_OPT_ETH_DEST_NUM, 707 CMD_LINE_OPT_NO_NUMA_NUM, 708 CMD_LINE_OPT_IPV6_NUM, 709 CMD_LINE_OPT_MAX_PKT_LEN_NUM, 710 CMD_LINE_OPT_HASH_ENTRY_NUM_NUM, 711 CMD_LINE_OPT_PARSE_PTYPE_NUM, 712 CMD_LINE_OPT_DISABLE_RSS_NUM, 713 CMD_LINE_OPT_RELAX_RX_OFFLOAD_NUM, 714 CMD_LINE_OPT_RULE_IPV4_NUM, 715 CMD_LINE_OPT_RULE_IPV6_NUM, 716 CMD_LINE_OPT_ALG_NUM, 717 CMD_LINE_OPT_PARSE_PER_PORT_POOL, 718 CMD_LINE_OPT_MODE_NUM, 719 CMD_LINE_OPT_EVENTQ_SYNC_NUM, 720 CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM, 721 CMD_LINE_OPT_LOOKUP_NUM, 722 CMD_LINE_OPT_ENABLE_VECTOR_NUM, 723 CMD_LINE_OPT_VECTOR_SIZE_NUM, 724 CMD_LINE_OPT_VECTOR_TMO_NS_NUM 725 }; 726 727 static const struct option lgopts[] = { 728 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 729 {CMD_LINE_OPT_RX_QUEUE_SIZE, 1, 0, CMD_LINE_OPT_RX_QUEUE_SIZE_NUM}, 730 {CMD_LINE_OPT_TX_QUEUE_SIZE, 1, 0, CMD_LINE_OPT_TX_QUEUE_SIZE_NUM}, 731 {CMD_LINE_OPT_ETH_DEST, 1, 0, CMD_LINE_OPT_ETH_DEST_NUM}, 732 {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM}, 733 {CMD_LINE_OPT_IPV6, 0, 0, CMD_LINE_OPT_IPV6_NUM}, 734 {CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM}, 735 {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, CMD_LINE_OPT_HASH_ENTRY_NUM_NUM}, 736 {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, CMD_LINE_OPT_PARSE_PTYPE_NUM}, 737 {CMD_LINE_OPT_RELAX_RX_OFFLOAD, 0, 0, CMD_LINE_OPT_RELAX_RX_OFFLOAD_NUM}, 738 {CMD_LINE_OPT_DISABLE_RSS, 0, 0, CMD_LINE_OPT_DISABLE_RSS_NUM}, 739 {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL}, 740 {CMD_LINE_OPT_MODE, 1, 0, CMD_LINE_OPT_MODE_NUM}, 741 {CMD_LINE_OPT_EVENTQ_SYNC, 1, 0, CMD_LINE_OPT_EVENTQ_SYNC_NUM}, 742 {CMD_LINE_OPT_EVENT_ETH_RX_QUEUES, 1, 0, 743 CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM}, 744 {CMD_LINE_OPT_LOOKUP, 1, 0, CMD_LINE_OPT_LOOKUP_NUM}, 745 {CMD_LINE_OPT_ENABLE_VECTOR, 0, 0, CMD_LINE_OPT_ENABLE_VECTOR_NUM}, 746 {CMD_LINE_OPT_VECTOR_SIZE, 1, 0, CMD_LINE_OPT_VECTOR_SIZE_NUM}, 747 {CMD_LINE_OPT_VECTOR_TMO_NS, 1, 0, CMD_LINE_OPT_VECTOR_TMO_NS_NUM}, 748 {CMD_LINE_OPT_RULE_IPV4, 1, 0, CMD_LINE_OPT_RULE_IPV4_NUM}, 749 {CMD_LINE_OPT_RULE_IPV6, 1, 0, CMD_LINE_OPT_RULE_IPV6_NUM}, 750 {CMD_LINE_OPT_ALG, 1, 0, CMD_LINE_OPT_ALG_NUM}, 751 {NULL, 0, 0, 0} 752 }; 753 754 /* 755 * This expression is used to calculate the number of mbufs needed 756 * depending on user input, taking into account memory for rx and 757 * tx hardware rings, cache per lcore and mtable per port per lcore. 758 * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum 759 * value of 8192 760 */ 761 #define NB_MBUF(nports) RTE_MAX( \ 762 (nports*nb_rx_queue*nb_rxd + \ 763 nports*nb_lcores*MAX_PKT_BURST + \ 764 nports*n_tx_queue*nb_txd + \ 765 nb_lcores*MEMPOOL_CACHE_SIZE), \ 766 (unsigned)8192) 767 768 /* Parse the argument given in the command line of the application */ 769 static int 770 parse_args(int argc, char **argv) 771 { 772 int opt, ret; 773 char **argvopt; 774 int option_index; 775 char *prgname = argv[0]; 776 uint8_t lcore_params = 0; 777 #ifdef RTE_LIB_EVENTDEV 778 uint8_t eventq_sched = 0; 779 uint8_t eth_rx_q = 0; 780 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 781 #endif 782 783 argvopt = argv; 784 785 /* Error or normal output strings. */ 786 while ((opt = getopt_long(argc, argvopt, short_options, 787 lgopts, &option_index)) != EOF) { 788 789 switch (opt) { 790 /* portmask */ 791 case 'p': 792 enabled_port_mask = parse_portmask(optarg); 793 if (enabled_port_mask == 0) { 794 fprintf(stderr, "Invalid portmask\n"); 795 print_usage(prgname); 796 return -1; 797 } 798 break; 799 800 case 'P': 801 promiscuous_on = 1; 802 break; 803 804 case 'E': 805 if (lookup_mode != L3FWD_LOOKUP_DEFAULT) { 806 fprintf(stderr, "Only one lookup mode is allowed at a time!\n"); 807 return -1; 808 } 809 lookup_mode = L3FWD_LOOKUP_EM; 810 break; 811 812 case 'L': 813 if (lookup_mode != L3FWD_LOOKUP_DEFAULT) { 814 fprintf(stderr, "Only one lookup mode is allowed at a time!\n"); 815 return -1; 816 } 817 lookup_mode = L3FWD_LOOKUP_LPM; 818 break; 819 820 /* long options */ 821 case CMD_LINE_OPT_CONFIG_NUM: 822 ret = parse_config(optarg); 823 if (ret) { 824 fprintf(stderr, "Invalid config\n"); 825 print_usage(prgname); 826 return -1; 827 } 828 lcore_params = 1; 829 break; 830 831 case CMD_LINE_OPT_RX_QUEUE_SIZE_NUM: 832 parse_queue_size(optarg, &nb_rxd, 1); 833 break; 834 835 case CMD_LINE_OPT_TX_QUEUE_SIZE_NUM: 836 parse_queue_size(optarg, &nb_txd, 0); 837 break; 838 839 case CMD_LINE_OPT_ETH_DEST_NUM: 840 parse_eth_dest(optarg); 841 break; 842 843 case CMD_LINE_OPT_NO_NUMA_NUM: 844 numa_on = 0; 845 break; 846 847 case CMD_LINE_OPT_IPV6_NUM: 848 ipv6 = 1; 849 break; 850 851 case CMD_LINE_OPT_MAX_PKT_LEN_NUM: 852 max_pkt_len = parse_max_pkt_len(optarg); 853 break; 854 855 case CMD_LINE_OPT_HASH_ENTRY_NUM_NUM: 856 fprintf(stderr, "Hash entry number will be ignored\n"); 857 break; 858 859 case CMD_LINE_OPT_PARSE_PTYPE_NUM: 860 printf("soft parse-ptype is enabled\n"); 861 parse_ptype = 1; 862 break; 863 864 case CMD_LINE_OPT_RELAX_RX_OFFLOAD_NUM: 865 printf("Rx offload is relaxed\n"); 866 relax_rx_offload = 1; 867 break; 868 869 case CMD_LINE_OPT_DISABLE_RSS_NUM: 870 printf("RSS is disabled\n"); 871 disable_rss = 1; 872 break; 873 874 case CMD_LINE_OPT_PARSE_PER_PORT_POOL: 875 printf("per port buffer pool is enabled\n"); 876 per_port_pool = 1; 877 break; 878 879 case CMD_LINE_OPT_MODE_NUM: 880 parse_mode(optarg); 881 break; 882 883 #ifdef RTE_LIB_EVENTDEV 884 case CMD_LINE_OPT_EVENTQ_SYNC_NUM: 885 parse_eventq_sched(optarg); 886 eventq_sched = 1; 887 break; 888 889 case CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM: 890 parse_event_eth_rx_queues(optarg); 891 eth_rx_q = 1; 892 break; 893 894 case CMD_LINE_OPT_ENABLE_VECTOR_NUM: 895 printf("event vectorization is enabled\n"); 896 evt_rsrc->vector_enabled = 1; 897 break; 898 899 case CMD_LINE_OPT_VECTOR_SIZE_NUM: 900 evt_rsrc->vector_size = strtol(optarg, NULL, 10); 901 break; 902 903 case CMD_LINE_OPT_VECTOR_TMO_NS_NUM: 904 evt_rsrc->vector_tmo_ns = strtoull(optarg, NULL, 10); 905 break; 906 #endif 907 908 case CMD_LINE_OPT_LOOKUP_NUM: 909 if (lookup_mode != L3FWD_LOOKUP_DEFAULT) { 910 fprintf(stderr, "Only one lookup mode is allowed at a time!\n"); 911 return -1; 912 } 913 ret = parse_lookup(optarg); 914 /* 915 * If parse_lookup was passed an invalid lookup type 916 * then return -1. Error log included within 917 * parse_lookup for simplicity. 918 */ 919 if (ret) 920 return -1; 921 break; 922 923 case CMD_LINE_OPT_RULE_IPV4_NUM: 924 l3fwd_set_rule_ipv4_name(optarg); 925 break; 926 case CMD_LINE_OPT_RULE_IPV6_NUM: 927 l3fwd_set_rule_ipv6_name(optarg); 928 break; 929 case CMD_LINE_OPT_ALG_NUM: 930 l3fwd_set_alg(optarg); 931 break; 932 default: 933 print_usage(prgname); 934 return -1; 935 } 936 } 937 938 RTE_SET_USED(lcore_params); /* needed if no eventdev block */ 939 #ifdef RTE_LIB_EVENTDEV 940 if (evt_rsrc->enabled && lcore_params) { 941 fprintf(stderr, "lcore config is not valid when event mode is selected\n"); 942 return -1; 943 } 944 945 if (!evt_rsrc->enabled && eth_rx_q) { 946 fprintf(stderr, "eth_rx_queues is valid only when event mode is selected\n"); 947 return -1; 948 } 949 950 if (!evt_rsrc->enabled && eventq_sched) { 951 fprintf(stderr, "eventq_sched is valid only when event mode is selected\n"); 952 return -1; 953 } 954 955 if (evt_rsrc->vector_enabled && !evt_rsrc->vector_size) { 956 evt_rsrc->vector_size = VECTOR_SIZE_DEFAULT; 957 fprintf(stderr, "vector size set to default (%" PRIu16 ")\n", 958 evt_rsrc->vector_size); 959 } 960 961 if (evt_rsrc->vector_enabled && !evt_rsrc->vector_tmo_ns) { 962 evt_rsrc->vector_tmo_ns = VECTOR_TMO_NS_DEFAULT; 963 fprintf(stderr, 964 "vector timeout set to default (%" PRIu64 " ns)\n", 965 evt_rsrc->vector_tmo_ns); 966 } 967 #endif 968 969 /* 970 * Nothing is selected, pick longest-prefix match 971 * as default match. 972 */ 973 if (lookup_mode == L3FWD_LOOKUP_DEFAULT) { 974 fprintf(stderr, "Neither ACL, LPM, EM, or FIB selected, defaulting to LPM\n"); 975 lookup_mode = L3FWD_LOOKUP_LPM; 976 } 977 978 /* For ACL, update port config rss hash filter */ 979 if (lookup_mode == L3FWD_LOOKUP_ACL) { 980 port_conf.rx_adv_conf.rss_conf.rss_hf |= 981 RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP | RTE_ETH_RSS_SCTP; 982 } 983 984 if (optind >= 0) 985 argv[optind-1] = prgname; 986 987 ret = optind-1; 988 optind = 1; /* reset getopt lib */ 989 return ret; 990 } 991 992 static void 993 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 994 { 995 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 996 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 997 printf("%s%s", name, buf); 998 } 999 1000 int 1001 init_mem(uint16_t portid, unsigned int nb_mbuf) 1002 { 1003 #ifdef RTE_LIB_EVENTDEV 1004 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 1005 #endif 1006 struct lcore_conf *qconf; 1007 int socketid; 1008 unsigned lcore_id; 1009 char s[64]; 1010 1011 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1012 if (rte_lcore_is_enabled(lcore_id) == 0) 1013 continue; 1014 1015 if (numa_on) 1016 socketid = rte_lcore_to_socket_id(lcore_id); 1017 else 1018 socketid = 0; 1019 1020 if (socketid >= NB_SOCKETS) { 1021 rte_exit(EXIT_FAILURE, 1022 "Socket %d of lcore %u is out of range %d\n", 1023 socketid, lcore_id, NB_SOCKETS); 1024 } 1025 1026 if (pktmbuf_pool[portid][socketid] == NULL) { 1027 snprintf(s, sizeof(s), "mbuf_pool_%d:%d", 1028 portid, socketid); 1029 pktmbuf_pool[portid][socketid] = 1030 rte_pktmbuf_pool_create(s, nb_mbuf, 1031 MEMPOOL_CACHE_SIZE, 0, 1032 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 1033 if (pktmbuf_pool[portid][socketid] == NULL) 1034 rte_exit(EXIT_FAILURE, 1035 "Cannot init mbuf pool on socket %d\n", 1036 socketid); 1037 else 1038 printf("Allocated mbuf pool on socket %d\n", 1039 socketid); 1040 1041 /* Setup ACL, LPM, EM(f.e Hash) or FIB. But, only once per 1042 * available socket. 1043 */ 1044 if (!lkp_per_socket[socketid]) { 1045 l3fwd_lkp.setup(socketid); 1046 lkp_per_socket[socketid] = 1; 1047 } 1048 } 1049 1050 #ifdef RTE_LIB_EVENTDEV 1051 if (evt_rsrc->vector_enabled && vector_pool[portid] == NULL) { 1052 unsigned int nb_vec; 1053 1054 nb_vec = (nb_mbuf + evt_rsrc->vector_size - 1) / 1055 evt_rsrc->vector_size; 1056 nb_vec = RTE_MAX(512U, nb_vec); 1057 nb_vec += rte_lcore_count() * 32; 1058 snprintf(s, sizeof(s), "vector_pool_%d", portid); 1059 vector_pool[portid] = rte_event_vector_pool_create( 1060 s, nb_vec, 32, evt_rsrc->vector_size, socketid); 1061 if (vector_pool[portid] == NULL) 1062 rte_exit(EXIT_FAILURE, 1063 "Failed to create vector pool for port %d\n", 1064 portid); 1065 else 1066 printf("Allocated vector pool for port %d\n", 1067 portid); 1068 } 1069 #endif 1070 1071 qconf = &lcore_conf[lcore_id]; 1072 qconf->ipv4_lookup_struct = 1073 l3fwd_lkp.get_ipv4_lookup_struct(socketid); 1074 qconf->ipv6_lookup_struct = 1075 l3fwd_lkp.get_ipv6_lookup_struct(socketid); 1076 } 1077 return 0; 1078 } 1079 1080 /* Check the link status of all ports in up to 9s, and print them finally */ 1081 static void 1082 check_all_ports_link_status(uint32_t port_mask) 1083 { 1084 #define CHECK_INTERVAL 100 /* 100ms */ 1085 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1086 uint16_t portid; 1087 uint8_t count, all_ports_up, print_flag = 0; 1088 struct rte_eth_link link; 1089 int ret; 1090 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 1091 1092 printf("\nChecking link status"); 1093 fflush(stdout); 1094 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1095 if (force_quit) 1096 return; 1097 all_ports_up = 1; 1098 RTE_ETH_FOREACH_DEV(portid) { 1099 if (force_quit) 1100 return; 1101 if ((port_mask & (1 << portid)) == 0) 1102 continue; 1103 memset(&link, 0, sizeof(link)); 1104 ret = rte_eth_link_get_nowait(portid, &link); 1105 if (ret < 0) { 1106 all_ports_up = 0; 1107 if (print_flag == 1) 1108 printf("Port %u link get failed: %s\n", 1109 portid, rte_strerror(-ret)); 1110 continue; 1111 } 1112 /* print link status if flag set */ 1113 if (print_flag == 1) { 1114 rte_eth_link_to_str(link_status_text, 1115 sizeof(link_status_text), &link); 1116 printf("Port %d %s\n", portid, 1117 link_status_text); 1118 continue; 1119 } 1120 /* clear all_ports_up flag if any link down */ 1121 if (link.link_status == RTE_ETH_LINK_DOWN) { 1122 all_ports_up = 0; 1123 break; 1124 } 1125 } 1126 /* after finally printing all link status, get out */ 1127 if (print_flag == 1) 1128 break; 1129 1130 if (all_ports_up == 0) { 1131 printf("."); 1132 fflush(stdout); 1133 rte_delay_ms(CHECK_INTERVAL); 1134 } 1135 1136 /* set the print_flag if all ports up or timeout */ 1137 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1138 print_flag = 1; 1139 printf("done\n"); 1140 } 1141 } 1142 } 1143 1144 static void 1145 signal_handler(int signum) 1146 { 1147 if (signum == SIGINT || signum == SIGTERM) { 1148 printf("\n\nSignal %d received, preparing to exit...\n", 1149 signum); 1150 force_quit = true; 1151 } 1152 } 1153 1154 static int 1155 prepare_ptype_parser(uint16_t portid, uint16_t queueid) 1156 { 1157 if (parse_ptype) { 1158 printf("Port %d: softly parse packet type info\n", portid); 1159 if (rte_eth_add_rx_callback(portid, queueid, 1160 l3fwd_lkp.cb_parse_ptype, 1161 NULL)) 1162 return 1; 1163 1164 printf("Failed to add rx callback: port=%d\n", portid); 1165 return 0; 1166 } 1167 1168 if (l3fwd_lkp.check_ptype(portid)) 1169 return 1; 1170 1171 printf("port %d cannot parse packet type, please add --%s\n", 1172 portid, CMD_LINE_OPT_PARSE_PTYPE); 1173 return 0; 1174 } 1175 1176 static uint32_t 1177 eth_dev_get_overhead_len(uint32_t max_rx_pktlen, uint16_t max_mtu) 1178 { 1179 uint32_t overhead_len; 1180 1181 if (max_mtu != UINT16_MAX && max_rx_pktlen > max_mtu) 1182 overhead_len = max_rx_pktlen - max_mtu; 1183 else 1184 overhead_len = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1185 1186 return overhead_len; 1187 } 1188 1189 int 1190 config_port_max_pkt_len(struct rte_eth_conf *conf, 1191 struct rte_eth_dev_info *dev_info) 1192 { 1193 uint32_t overhead_len; 1194 1195 if (max_pkt_len == 0) 1196 return 0; 1197 1198 if (max_pkt_len < RTE_ETHER_MIN_LEN || max_pkt_len > MAX_JUMBO_PKT_LEN) 1199 return -1; 1200 1201 overhead_len = eth_dev_get_overhead_len(dev_info->max_rx_pktlen, 1202 dev_info->max_mtu); 1203 conf->rxmode.mtu = max_pkt_len - overhead_len; 1204 1205 if (conf->rxmode.mtu > RTE_ETHER_MTU) 1206 conf->txmode.offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 1207 1208 return 0; 1209 } 1210 1211 static void 1212 l3fwd_poll_resource_setup(void) 1213 { 1214 uint8_t nb_rx_queue, queue, socketid; 1215 struct rte_eth_dev_info dev_info; 1216 uint32_t n_tx_queue, nb_lcores; 1217 struct rte_eth_txconf *txconf; 1218 struct lcore_conf *qconf; 1219 uint16_t queueid, portid; 1220 unsigned int nb_ports; 1221 unsigned int lcore_id; 1222 int ret; 1223 1224 if (check_lcore_params() < 0) 1225 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n"); 1226 1227 ret = init_lcore_rx_queues(); 1228 if (ret < 0) 1229 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 1230 1231 nb_ports = rte_eth_dev_count_avail(); 1232 1233 if (check_port_config() < 0) 1234 rte_exit(EXIT_FAILURE, "check_port_config failed\n"); 1235 1236 nb_lcores = rte_lcore_count(); 1237 1238 /* initialize all ports */ 1239 RTE_ETH_FOREACH_DEV(portid) { 1240 struct rte_eth_conf local_port_conf = port_conf; 1241 1242 /* skip ports that are not enabled */ 1243 if ((enabled_port_mask & (1 << portid)) == 0) { 1244 printf("\nSkipping disabled port %d\n", portid); 1245 continue; 1246 } 1247 1248 /* init port */ 1249 printf("Initializing port %d ... ", portid ); 1250 fflush(stdout); 1251 1252 nb_rx_queue = get_port_n_rx_queues(portid); 1253 n_tx_queue = nb_lcores; 1254 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT) 1255 n_tx_queue = MAX_TX_QUEUE_PER_PORT; 1256 printf("Creating queues: nb_rxq=%d nb_txq=%u... ", 1257 nb_rx_queue, (unsigned)n_tx_queue ); 1258 1259 ret = rte_eth_dev_info_get(portid, &dev_info); 1260 if (ret != 0) 1261 rte_exit(EXIT_FAILURE, 1262 "Error during getting device (port %u) info: %s\n", 1263 portid, strerror(-ret)); 1264 1265 ret = config_port_max_pkt_len(&local_port_conf, &dev_info); 1266 if (ret != 0) 1267 rte_exit(EXIT_FAILURE, 1268 "Invalid max packet length: %u (port %u)\n", 1269 max_pkt_len, portid); 1270 1271 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 1272 local_port_conf.txmode.offloads |= 1273 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1274 1275 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 1276 dev_info.flow_type_rss_offloads; 1277 1278 if (disable_rss == 1 || dev_info.max_rx_queues == 1) 1279 local_port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 1280 1281 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 1282 port_conf.rx_adv_conf.rss_conf.rss_hf) { 1283 printf("Port %u modified RSS hash function based on hardware support," 1284 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 1285 portid, 1286 port_conf.rx_adv_conf.rss_conf.rss_hf, 1287 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 1288 } 1289 1290 /* Relax Rx offload requirement */ 1291 if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) != 1292 local_port_conf.rxmode.offloads) { 1293 printf("Port %u requested Rx offloads 0x%"PRIx64 1294 " does not match Rx offloads capabilities 0x%"PRIx64"\n", 1295 portid, local_port_conf.rxmode.offloads, 1296 dev_info.rx_offload_capa); 1297 if (relax_rx_offload) { 1298 local_port_conf.rxmode.offloads &= dev_info.rx_offload_capa; 1299 printf("Warning: modified Rx offload to 0x%"PRIx64 1300 " based on device capability\n", 1301 local_port_conf.rxmode.offloads); 1302 } 1303 } 1304 1305 ret = rte_eth_dev_configure(portid, nb_rx_queue, 1306 (uint16_t)n_tx_queue, &local_port_conf); 1307 if (ret < 0) 1308 rte_exit(EXIT_FAILURE, 1309 "Cannot configure device: err=%d, port=%d\n", 1310 ret, portid); 1311 1312 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 1313 &nb_txd); 1314 if (ret < 0) 1315 rte_exit(EXIT_FAILURE, 1316 "Cannot adjust number of descriptors: err=%d, " 1317 "port=%d\n", ret, portid); 1318 1319 ret = rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 1320 if (ret < 0) 1321 rte_exit(EXIT_FAILURE, 1322 "Cannot get MAC address: err=%d, port=%d\n", 1323 ret, portid); 1324 1325 print_ethaddr(" Address:", &ports_eth_addr[portid]); 1326 printf(", "); 1327 print_ethaddr("Destination:", 1328 (const struct rte_ether_addr *)&dest_eth_addr[portid]); 1329 printf(", "); 1330 1331 /* 1332 * prepare src MACs for each port. 1333 */ 1334 rte_ether_addr_copy(&ports_eth_addr[portid], 1335 (struct rte_ether_addr *)(val_eth + portid) + 1); 1336 1337 /* init memory */ 1338 if (!per_port_pool) { 1339 /* portid = 0; this is *not* signifying the first port, 1340 * rather, it signifies that portid is ignored. 1341 */ 1342 ret = init_mem(0, NB_MBUF(nb_ports)); 1343 } else { 1344 ret = init_mem(portid, NB_MBUF(1)); 1345 } 1346 if (ret < 0) 1347 rte_exit(EXIT_FAILURE, "init_mem failed\n"); 1348 1349 /* init one TX queue per couple (lcore,port) */ 1350 queueid = 0; 1351 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1352 if (rte_lcore_is_enabled(lcore_id) == 0) 1353 continue; 1354 1355 if (numa_on) 1356 socketid = 1357 (uint8_t)rte_lcore_to_socket_id(lcore_id); 1358 else 1359 socketid = 0; 1360 1361 printf("txq=%u,%d,%d ", lcore_id, queueid, socketid); 1362 fflush(stdout); 1363 1364 txconf = &dev_info.default_txconf; 1365 txconf->offloads = local_port_conf.txmode.offloads; 1366 ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, 1367 socketid, txconf); 1368 if (ret < 0) 1369 rte_exit(EXIT_FAILURE, 1370 "rte_eth_tx_queue_setup: err=%d, " 1371 "port=%d\n", ret, portid); 1372 1373 qconf = &lcore_conf[lcore_id]; 1374 qconf->tx_queue_id[portid] = queueid; 1375 queueid++; 1376 1377 qconf->tx_port_id[qconf->n_tx_port] = portid; 1378 qconf->n_tx_port++; 1379 } 1380 printf("\n"); 1381 } 1382 1383 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1384 if (rte_lcore_is_enabled(lcore_id) == 0) 1385 continue; 1386 qconf = &lcore_conf[lcore_id]; 1387 printf("\nInitializing rx queues on lcore %u ... ", lcore_id ); 1388 fflush(stdout); 1389 /* init RX queues */ 1390 for(queue = 0; queue < qconf->n_rx_queue; ++queue) { 1391 struct rte_eth_rxconf rxq_conf; 1392 1393 portid = qconf->rx_queue_list[queue].port_id; 1394 queueid = qconf->rx_queue_list[queue].queue_id; 1395 1396 if (numa_on) 1397 socketid = 1398 (uint8_t)rte_lcore_to_socket_id(lcore_id); 1399 else 1400 socketid = 0; 1401 1402 printf("rxq=%d,%d,%d ", portid, queueid, socketid); 1403 fflush(stdout); 1404 1405 ret = rte_eth_dev_info_get(portid, &dev_info); 1406 if (ret != 0) 1407 rte_exit(EXIT_FAILURE, 1408 "Error during getting device (port %u) info: %s\n", 1409 portid, strerror(-ret)); 1410 1411 rxq_conf = dev_info.default_rxconf; 1412 rxq_conf.offloads = port_conf.rxmode.offloads; 1413 if (!per_port_pool) 1414 ret = rte_eth_rx_queue_setup(portid, queueid, 1415 nb_rxd, socketid, 1416 &rxq_conf, 1417 pktmbuf_pool[0][socketid]); 1418 else 1419 ret = rte_eth_rx_queue_setup(portid, queueid, 1420 nb_rxd, socketid, 1421 &rxq_conf, 1422 pktmbuf_pool[portid][socketid]); 1423 if (ret < 0) 1424 rte_exit(EXIT_FAILURE, 1425 "rte_eth_rx_queue_setup: err=%d, port=%d\n", 1426 ret, portid); 1427 } 1428 } 1429 } 1430 1431 static inline int 1432 l3fwd_service_enable(uint32_t service_id) 1433 { 1434 uint8_t min_service_count = UINT8_MAX; 1435 uint32_t slcore_array[RTE_MAX_LCORE]; 1436 unsigned int slcore = 0; 1437 uint8_t service_count; 1438 int32_t slcore_count; 1439 1440 if (!rte_service_lcore_count()) 1441 return -ENOENT; 1442 1443 slcore_count = rte_service_lcore_list(slcore_array, RTE_MAX_LCORE); 1444 if (slcore_count < 0) 1445 return -ENOENT; 1446 /* Get the core which has least number of services running. */ 1447 while (slcore_count--) { 1448 /* Reset default mapping */ 1449 if (rte_service_map_lcore_set(service_id, 1450 slcore_array[slcore_count], 0) != 0) 1451 return -ENOENT; 1452 service_count = rte_service_lcore_count_services( 1453 slcore_array[slcore_count]); 1454 if (service_count < min_service_count) { 1455 slcore = slcore_array[slcore_count]; 1456 min_service_count = service_count; 1457 } 1458 } 1459 if (rte_service_map_lcore_set(service_id, slcore, 1)) 1460 return -ENOENT; 1461 rte_service_lcore_start(slcore); 1462 1463 return 0; 1464 } 1465 1466 #ifdef RTE_LIB_EVENTDEV 1467 static void 1468 l3fwd_event_service_setup(void) 1469 { 1470 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 1471 struct rte_event_dev_info evdev_info; 1472 uint32_t service_id, caps; 1473 int ret, i; 1474 1475 rte_event_dev_info_get(evt_rsrc->event_d_id, &evdev_info); 1476 if (!(evdev_info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED)) { 1477 ret = rte_event_dev_service_id_get(evt_rsrc->event_d_id, 1478 &service_id); 1479 if (ret != -ESRCH && ret != 0) 1480 rte_exit(EXIT_FAILURE, 1481 "Error in starting eventdev service\n"); 1482 l3fwd_service_enable(service_id); 1483 } 1484 1485 for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++) { 1486 ret = rte_event_eth_rx_adapter_caps_get(evt_rsrc->event_d_id, 1487 evt_rsrc->rx_adptr.rx_adptr[i], &caps); 1488 if (ret < 0) 1489 rte_exit(EXIT_FAILURE, 1490 "Failed to get Rx adapter[%d] caps\n", 1491 evt_rsrc->rx_adptr.rx_adptr[i]); 1492 ret = rte_event_eth_rx_adapter_service_id_get( 1493 evt_rsrc->event_d_id, 1494 &service_id); 1495 if (ret != -ESRCH && ret != 0) 1496 rte_exit(EXIT_FAILURE, 1497 "Error in starting Rx adapter[%d] service\n", 1498 evt_rsrc->rx_adptr.rx_adptr[i]); 1499 l3fwd_service_enable(service_id); 1500 } 1501 1502 for (i = 0; i < evt_rsrc->tx_adptr.nb_tx_adptr; i++) { 1503 ret = rte_event_eth_tx_adapter_caps_get(evt_rsrc->event_d_id, 1504 evt_rsrc->tx_adptr.tx_adptr[i], &caps); 1505 if (ret < 0) 1506 rte_exit(EXIT_FAILURE, 1507 "Failed to get Rx adapter[%d] caps\n", 1508 evt_rsrc->tx_adptr.tx_adptr[i]); 1509 ret = rte_event_eth_tx_adapter_service_id_get( 1510 evt_rsrc->event_d_id, 1511 &service_id); 1512 if (ret != -ESRCH && ret != 0) 1513 rte_exit(EXIT_FAILURE, 1514 "Error in starting Rx adapter[%d] service\n", 1515 evt_rsrc->tx_adptr.tx_adptr[i]); 1516 l3fwd_service_enable(service_id); 1517 } 1518 } 1519 #endif 1520 1521 int 1522 main(int argc, char **argv) 1523 { 1524 #ifdef RTE_LIB_EVENTDEV 1525 struct l3fwd_event_resources *evt_rsrc; 1526 int i; 1527 #endif 1528 struct lcore_conf *qconf; 1529 uint16_t queueid, portid; 1530 unsigned int lcore_id; 1531 uint8_t queue; 1532 int ret; 1533 1534 /* init EAL */ 1535 ret = rte_eal_init(argc, argv); 1536 if (ret < 0) 1537 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 1538 argc -= ret; 1539 argv += ret; 1540 1541 force_quit = false; 1542 signal(SIGINT, signal_handler); 1543 signal(SIGTERM, signal_handler); 1544 1545 /* pre-init dst MACs for all ports to 02:00:00:00:00:xx */ 1546 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 1547 dest_eth_addr[portid] = 1548 RTE_ETHER_LOCAL_ADMIN_ADDR + ((uint64_t)portid << 40); 1549 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid]; 1550 } 1551 1552 #ifdef RTE_LIB_EVENTDEV 1553 evt_rsrc = l3fwd_get_eventdev_rsrc(); 1554 #endif 1555 /* parse application arguments (after the EAL ones) */ 1556 ret = parse_args(argc, argv); 1557 if (ret < 0) 1558 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n"); 1559 1560 /* Setup function pointers for lookup method. */ 1561 setup_l3fwd_lookup_tables(); 1562 1563 /* Add the config file rules */ 1564 l3fwd_lkp.read_config_files(); 1565 1566 #ifdef RTE_LIB_EVENTDEV 1567 evt_rsrc->per_port_pool = per_port_pool; 1568 evt_rsrc->pkt_pool = pktmbuf_pool; 1569 evt_rsrc->vec_pool = vector_pool; 1570 evt_rsrc->port_mask = enabled_port_mask; 1571 /* Configure eventdev parameters if user has requested */ 1572 if (evt_rsrc->enabled) { 1573 l3fwd_event_resource_setup(&port_conf); 1574 if (lookup_mode == L3FWD_LOOKUP_EM) 1575 l3fwd_lkp.main_loop = evt_rsrc->ops.em_event_loop; 1576 else if (lookup_mode == L3FWD_LOOKUP_FIB) 1577 l3fwd_lkp.main_loop = evt_rsrc->ops.fib_event_loop; 1578 else 1579 l3fwd_lkp.main_loop = evt_rsrc->ops.lpm_event_loop; 1580 l3fwd_event_service_setup(); 1581 } else 1582 #endif 1583 l3fwd_poll_resource_setup(); 1584 1585 /* start ports */ 1586 RTE_ETH_FOREACH_DEV(portid) { 1587 if ((enabled_port_mask & (1 << portid)) == 0) { 1588 continue; 1589 } 1590 /* Start device */ 1591 ret = rte_eth_dev_start(portid); 1592 if (ret < 0) 1593 rte_exit(EXIT_FAILURE, 1594 "rte_eth_dev_start: err=%d, port=%d\n", 1595 ret, portid); 1596 1597 /* 1598 * If enabled, put device in promiscuous mode. 1599 * This allows IO forwarding mode to forward packets 1600 * to itself through 2 cross-connected ports of the 1601 * target machine. 1602 */ 1603 if (promiscuous_on) { 1604 ret = rte_eth_promiscuous_enable(portid); 1605 if (ret != 0) 1606 rte_exit(EXIT_FAILURE, 1607 "rte_eth_promiscuous_enable: err=%s, port=%u\n", 1608 rte_strerror(-ret), portid); 1609 } 1610 } 1611 1612 printf("\n"); 1613 1614 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1615 if (rte_lcore_is_enabled(lcore_id) == 0) 1616 continue; 1617 qconf = &lcore_conf[lcore_id]; 1618 for (queue = 0; queue < qconf->n_rx_queue; ++queue) { 1619 portid = qconf->rx_queue_list[queue].port_id; 1620 queueid = qconf->rx_queue_list[queue].queue_id; 1621 if (prepare_ptype_parser(portid, queueid) == 0) 1622 rte_exit(EXIT_FAILURE, "ptype check fails\n"); 1623 } 1624 } 1625 1626 check_all_ports_link_status(enabled_port_mask); 1627 1628 ret = 0; 1629 /* launch per-lcore init on every lcore */ 1630 rte_eal_mp_remote_launch(l3fwd_lkp.main_loop, NULL, CALL_MAIN); 1631 1632 #ifdef RTE_LIB_EVENTDEV 1633 if (evt_rsrc->enabled) { 1634 for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++) 1635 rte_event_eth_rx_adapter_stop( 1636 evt_rsrc->rx_adptr.rx_adptr[i]); 1637 for (i = 0; i < evt_rsrc->tx_adptr.nb_tx_adptr; i++) 1638 rte_event_eth_tx_adapter_stop( 1639 evt_rsrc->tx_adptr.tx_adptr[i]); 1640 1641 RTE_ETH_FOREACH_DEV(portid) { 1642 if ((enabled_port_mask & (1 << portid)) == 0) 1643 continue; 1644 ret = rte_eth_dev_stop(portid); 1645 if (ret != 0) 1646 printf("rte_eth_dev_stop: err=%d, port=%u\n", 1647 ret, portid); 1648 } 1649 1650 rte_eal_mp_wait_lcore(); 1651 RTE_ETH_FOREACH_DEV(portid) { 1652 if ((enabled_port_mask & (1 << portid)) == 0) 1653 continue; 1654 rte_eth_dev_close(portid); 1655 } 1656 1657 rte_event_dev_stop(evt_rsrc->event_d_id); 1658 rte_event_dev_close(evt_rsrc->event_d_id); 1659 1660 } else 1661 #endif 1662 { 1663 rte_eal_mp_wait_lcore(); 1664 1665 RTE_ETH_FOREACH_DEV(portid) { 1666 if ((enabled_port_mask & (1 << portid)) == 0) 1667 continue; 1668 printf("Closing port %d...", portid); 1669 ret = rte_eth_dev_stop(portid); 1670 if (ret != 0) 1671 printf("rte_eth_dev_stop: err=%d, port=%u\n", 1672 ret, portid); 1673 rte_eth_dev_close(portid); 1674 printf(" Done\n"); 1675 } 1676 } 1677 1678 /* clean up config file routes */ 1679 l3fwd_lkp.free_routes(); 1680 1681 /* clean up the EAL */ 1682 rte_eal_cleanup(); 1683 1684 printf("Bye...\n"); 1685 1686 return ret; 1687 } 1688