1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2021 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <string.h> 11 #include <sys/queue.h> 12 #include <stdarg.h> 13 #include <errno.h> 14 #include <getopt.h> 15 #include <signal.h> 16 #include <stdbool.h> 17 18 #include <rte_common.h> 19 #include <rte_vect.h> 20 #include <rte_byteorder.h> 21 #include <rte_log.h> 22 #include <rte_malloc.h> 23 #include <rte_memory.h> 24 #include <rte_memcpy.h> 25 #include <rte_eal.h> 26 #include <rte_launch.h> 27 #include <rte_cycles.h> 28 #include <rte_prefetch.h> 29 #include <rte_lcore.h> 30 #include <rte_per_lcore.h> 31 #include <rte_branch_prediction.h> 32 #include <rte_interrupts.h> 33 #include <rte_random.h> 34 #include <rte_debug.h> 35 #include <rte_ether.h> 36 #include <rte_mempool.h> 37 #include <rte_mbuf.h> 38 #include <rte_ip.h> 39 #include <rte_tcp.h> 40 #include <rte_udp.h> 41 #include <rte_string_fns.h> 42 #include <rte_cpuflags.h> 43 44 #include <cmdline_parse.h> 45 #include <cmdline_parse_etheraddr.h> 46 47 #include "l3fwd.h" 48 #include "l3fwd_event.h" 49 #include "l3fwd_route.h" 50 51 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_LCORE 52 #define MAX_RX_QUEUE_PER_PORT 128 53 54 #define MAX_LCORE_PARAMS 1024 55 56 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 57 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 58 59 /**< Ports set in promiscuous mode off by default. */ 60 static int promiscuous_on; 61 62 /* Select Longest-Prefix, Exact match, Forwarding Information Base or Access Control. */ 63 enum L3FWD_LOOKUP_MODE { 64 L3FWD_LOOKUP_DEFAULT, 65 L3FWD_LOOKUP_LPM, 66 L3FWD_LOOKUP_EM, 67 L3FWD_LOOKUP_FIB, 68 L3FWD_LOOKUP_ACL 69 }; 70 static enum L3FWD_LOOKUP_MODE lookup_mode; 71 72 /* Global variables. */ 73 static int numa_on = 1; /**< NUMA is enabled by default. */ 74 static int parse_ptype; /**< Parse packet type using rx callback, and */ 75 /**< disabled by default */ 76 static int per_port_pool; /**< Use separate buffer pools per port; disabled */ 77 /**< by default */ 78 79 volatile bool force_quit; 80 81 /* ethernet addresses of ports */ 82 uint64_t dest_eth_addr[RTE_MAX_ETHPORTS]; 83 struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; 84 85 xmm_t val_eth[RTE_MAX_ETHPORTS]; 86 87 /* mask of enabled ports */ 88 uint32_t enabled_port_mask; 89 90 /* Used only in exact match mode. */ 91 int ipv6; /**< ipv6 is false by default. */ 92 uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT; 93 94 struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 95 96 struct parm_cfg parm_config; 97 98 struct lcore_params { 99 uint16_t port_id; 100 uint8_t queue_id; 101 uint8_t lcore_id; 102 } __rte_cache_aligned; 103 104 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 105 static struct lcore_params lcore_params_array_default[] = { 106 {0, 0, 2}, 107 {0, 1, 2}, 108 {0, 2, 2}, 109 {1, 0, 2}, 110 {1, 1, 2}, 111 {1, 2, 2}, 112 {2, 0, 2}, 113 {3, 0, 3}, 114 {3, 1, 3}, 115 }; 116 117 static struct lcore_params * lcore_params = lcore_params_array_default; 118 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) / 119 sizeof(lcore_params_array_default[0]); 120 121 static struct rte_eth_conf port_conf = { 122 .rxmode = { 123 .mq_mode = RTE_ETH_MQ_RX_RSS, 124 .split_hdr_size = 0, 125 .offloads = RTE_ETH_RX_OFFLOAD_CHECKSUM, 126 }, 127 .rx_adv_conf = { 128 .rss_conf = { 129 .rss_key = NULL, 130 .rss_hf = RTE_ETH_RSS_IP, 131 }, 132 }, 133 .txmode = { 134 .mq_mode = RTE_ETH_MQ_TX_NONE, 135 }, 136 }; 137 138 static uint32_t max_pkt_len; 139 140 static struct rte_mempool *pktmbuf_pool[RTE_MAX_ETHPORTS][NB_SOCKETS]; 141 static struct rte_mempool *vector_pool[RTE_MAX_ETHPORTS]; 142 static uint8_t lkp_per_socket[NB_SOCKETS]; 143 144 struct l3fwd_lkp_mode { 145 void (*read_config_files)(void); 146 void (*setup)(int); 147 int (*check_ptype)(int); 148 rte_rx_callback_fn cb_parse_ptype; 149 int (*main_loop)(void *); 150 void* (*get_ipv4_lookup_struct)(int); 151 void* (*get_ipv6_lookup_struct)(int); 152 void (*free_routes)(void); 153 }; 154 155 static struct l3fwd_lkp_mode l3fwd_lkp; 156 157 static struct l3fwd_lkp_mode l3fwd_em_lkp = { 158 .read_config_files = read_config_files_em, 159 .setup = setup_hash, 160 .check_ptype = em_check_ptype, 161 .cb_parse_ptype = em_cb_parse_ptype, 162 .main_loop = em_main_loop, 163 .get_ipv4_lookup_struct = em_get_ipv4_l3fwd_lookup_struct, 164 .get_ipv6_lookup_struct = em_get_ipv6_l3fwd_lookup_struct, 165 .free_routes = em_free_routes, 166 }; 167 168 static struct l3fwd_lkp_mode l3fwd_lpm_lkp = { 169 .read_config_files = read_config_files_lpm, 170 .setup = setup_lpm, 171 .check_ptype = lpm_check_ptype, 172 .cb_parse_ptype = lpm_cb_parse_ptype, 173 .main_loop = lpm_main_loop, 174 .get_ipv4_lookup_struct = lpm_get_ipv4_l3fwd_lookup_struct, 175 .get_ipv6_lookup_struct = lpm_get_ipv6_l3fwd_lookup_struct, 176 .free_routes = lpm_free_routes, 177 }; 178 179 static struct l3fwd_lkp_mode l3fwd_fib_lkp = { 180 .read_config_files = read_config_files_lpm, 181 .setup = setup_fib, 182 .check_ptype = lpm_check_ptype, 183 .cb_parse_ptype = lpm_cb_parse_ptype, 184 .main_loop = fib_main_loop, 185 .get_ipv4_lookup_struct = fib_get_ipv4_l3fwd_lookup_struct, 186 .get_ipv6_lookup_struct = fib_get_ipv6_l3fwd_lookup_struct, 187 .free_routes = lpm_free_routes, 188 }; 189 190 static struct l3fwd_lkp_mode l3fwd_acl_lkp = { 191 .read_config_files = read_config_files_acl, 192 .setup = setup_acl, 193 .check_ptype = em_check_ptype, 194 .cb_parse_ptype = em_cb_parse_ptype, 195 .main_loop = acl_main_loop, 196 .get_ipv4_lookup_struct = acl_get_ipv4_l3fwd_lookup_struct, 197 .get_ipv6_lookup_struct = acl_get_ipv6_l3fwd_lookup_struct, 198 .free_routes = acl_free_routes, 199 }; 200 201 /* 202 * 198.18.0.0/16 are set aside for RFC2544 benchmarking (RFC5735). 203 * 198.18.{0-15}.0/24 = Port {0-15} 204 */ 205 const struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { 206 {RTE_IPV4(198, 18, 0, 0), 24, 0}, 207 {RTE_IPV4(198, 18, 1, 0), 24, 1}, 208 {RTE_IPV4(198, 18, 2, 0), 24, 2}, 209 {RTE_IPV4(198, 18, 3, 0), 24, 3}, 210 {RTE_IPV4(198, 18, 4, 0), 24, 4}, 211 {RTE_IPV4(198, 18, 5, 0), 24, 5}, 212 {RTE_IPV4(198, 18, 6, 0), 24, 6}, 213 {RTE_IPV4(198, 18, 7, 0), 24, 7}, 214 {RTE_IPV4(198, 18, 8, 0), 24, 8}, 215 {RTE_IPV4(198, 18, 9, 0), 24, 9}, 216 {RTE_IPV4(198, 18, 10, 0), 24, 10}, 217 {RTE_IPV4(198, 18, 11, 0), 24, 11}, 218 {RTE_IPV4(198, 18, 12, 0), 24, 12}, 219 {RTE_IPV4(198, 18, 13, 0), 24, 13}, 220 {RTE_IPV4(198, 18, 14, 0), 24, 14}, 221 {RTE_IPV4(198, 18, 15, 0), 24, 15}, 222 }; 223 224 /* 225 * 2001:200::/48 is IANA reserved range for IPv6 benchmarking (RFC5180). 226 * 2001:200:0:{0-f}::/64 = Port {0-15} 227 */ 228 const struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = { 229 {{32, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 0}, 230 {{32, 1, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 1}, 231 {{32, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 2}, 232 {{32, 1, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 3}, 233 {{32, 1, 2, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 4}, 234 {{32, 1, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 5}, 235 {{32, 1, 2, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 6}, 236 {{32, 1, 2, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 7}, 237 {{32, 1, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 8}, 238 {{32, 1, 2, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 9}, 239 {{32, 1, 2, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 10}, 240 {{32, 1, 2, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 11}, 241 {{32, 1, 2, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 12}, 242 {{32, 1, 2, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 13}, 243 {{32, 1, 2, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 14}, 244 {{32, 1, 2, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 15}, 245 }; 246 247 /* 248 * API's called during initialization to setup ACL/EM/LPM rules. 249 */ 250 void 251 l3fwd_set_rule_ipv4_name(const char *optarg) 252 { 253 parm_config.rule_ipv4_name = optarg; 254 } 255 256 void 257 l3fwd_set_rule_ipv6_name(const char *optarg) 258 { 259 parm_config.rule_ipv6_name = optarg; 260 } 261 262 void 263 l3fwd_set_alg(const char *optarg) 264 { 265 parm_config.alg = parse_acl_alg(optarg); 266 } 267 268 /* 269 * Setup lookup methods for forwarding. 270 * Currently exact-match, longest-prefix-match and forwarding information 271 * base are the supported ones. 272 */ 273 static void 274 setup_l3fwd_lookup_tables(void) 275 { 276 /* Setup HASH lookup functions. */ 277 if (lookup_mode == L3FWD_LOOKUP_EM) 278 l3fwd_lkp = l3fwd_em_lkp; 279 /* Setup FIB lookup functions. */ 280 else if (lookup_mode == L3FWD_LOOKUP_FIB) 281 l3fwd_lkp = l3fwd_fib_lkp; 282 /* Setup ACL lookup functions. */ 283 else if (lookup_mode == L3FWD_LOOKUP_ACL) 284 l3fwd_lkp = l3fwd_acl_lkp; 285 /* Setup LPM lookup functions. */ 286 else 287 l3fwd_lkp = l3fwd_lpm_lkp; 288 } 289 290 static int 291 check_lcore_params(void) 292 { 293 uint8_t queue, lcore; 294 uint16_t i; 295 int socketid; 296 297 for (i = 0; i < nb_lcore_params; ++i) { 298 queue = lcore_params[i].queue_id; 299 if (queue >= MAX_RX_QUEUE_PER_PORT) { 300 printf("invalid queue number: %hhu\n", queue); 301 return -1; 302 } 303 lcore = lcore_params[i].lcore_id; 304 if (!rte_lcore_is_enabled(lcore)) { 305 printf("error: lcore %hhu is not enabled in lcore mask\n", lcore); 306 return -1; 307 } 308 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) && 309 (numa_on == 0)) { 310 printf("warning: lcore %hhu is on socket %d with numa off \n", 311 lcore, socketid); 312 } 313 } 314 return 0; 315 } 316 317 static int 318 check_port_config(void) 319 { 320 uint16_t portid; 321 uint16_t i; 322 323 for (i = 0; i < nb_lcore_params; ++i) { 324 portid = lcore_params[i].port_id; 325 if ((enabled_port_mask & (1 << portid)) == 0) { 326 printf("port %u is not enabled in port mask\n", portid); 327 return -1; 328 } 329 if (!rte_eth_dev_is_valid_port(portid)) { 330 printf("port %u is not present on the board\n", portid); 331 return -1; 332 } 333 } 334 return 0; 335 } 336 337 static uint8_t 338 get_port_n_rx_queues(const uint16_t port) 339 { 340 int queue = -1; 341 uint16_t i; 342 343 for (i = 0; i < nb_lcore_params; ++i) { 344 if (lcore_params[i].port_id == port) { 345 if (lcore_params[i].queue_id == queue+1) 346 queue = lcore_params[i].queue_id; 347 else 348 rte_exit(EXIT_FAILURE, "queue ids of the port %d must be" 349 " in sequence and must start with 0\n", 350 lcore_params[i].port_id); 351 } 352 } 353 return (uint8_t)(++queue); 354 } 355 356 static int 357 init_lcore_rx_queues(void) 358 { 359 uint16_t i, nb_rx_queue; 360 uint8_t lcore; 361 362 for (i = 0; i < nb_lcore_params; ++i) { 363 lcore = lcore_params[i].lcore_id; 364 nb_rx_queue = lcore_conf[lcore].n_rx_queue; 365 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 366 printf("error: too many queues (%u) for lcore: %u\n", 367 (unsigned)nb_rx_queue + 1, (unsigned)lcore); 368 return -1; 369 } else { 370 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 371 lcore_params[i].port_id; 372 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 373 lcore_params[i].queue_id; 374 lcore_conf[lcore].n_rx_queue++; 375 } 376 } 377 return 0; 378 } 379 380 /* display usage */ 381 static void 382 print_usage(const char *prgname) 383 { 384 char alg[PATH_MAX]; 385 386 usage_acl_alg(alg, sizeof(alg)); 387 fprintf(stderr, "%s [EAL options] --" 388 " -p PORTMASK" 389 " --rule_ipv4=FILE" 390 " --rule_ipv6=FILE" 391 " [-P]" 392 " [--lookup]" 393 " --config (port,queue,lcore)[,(port,queue,lcore)]" 394 " [--rx-queue-size NPKTS]" 395 " [--tx-queue-size NPKTS]" 396 " [--eth-dest=X,MM:MM:MM:MM:MM:MM]" 397 " [--max-pkt-len PKTLEN]" 398 " [--no-numa]" 399 " [--hash-entry-num]" 400 " [--ipv6]" 401 " [--parse-ptype]" 402 " [--per-port-pool]" 403 " [--mode]" 404 " [--eventq-sched]" 405 " [--event-vector [--event-vector-size SIZE] [--event-vector-tmo NS]]" 406 " [-E]" 407 " [-L]\n\n" 408 409 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 410 " -P : Enable promiscuous mode\n" 411 " --lookup: Select the lookup method\n" 412 " Default: lpm\n" 413 " Accepted: em (Exact Match), lpm (Longest Prefix Match), fib (Forwarding Information Base),\n" 414 " acl (Access Control List)\n" 415 " --config (port,queue,lcore): Rx queue configuration\n" 416 " --rx-queue-size NPKTS: Rx queue size in decimal\n" 417 " Default: %d\n" 418 " --tx-queue-size NPKTS: Tx queue size in decimal\n" 419 " Default: %d\n" 420 " --eth-dest=X,MM:MM:MM:MM:MM:MM: Ethernet destination for port X\n" 421 " --max-pkt-len PKTLEN: maximum packet length in decimal (64-9600)\n" 422 " --no-numa: Disable numa awareness\n" 423 " --hash-entry-num: Specify the hash entry number in hexadecimal to be setup\n" 424 " --ipv6: Set if running ipv6 packets\n" 425 " --parse-ptype: Set to use software to analyze packet type\n" 426 " --per-port-pool: Use separate buffer pool per port\n" 427 " --mode: Packet transfer mode for I/O, poll or eventdev\n" 428 " Default mode = poll\n" 429 " --eventq-sched: Event queue synchronization method\n" 430 " ordered, atomic or parallel.\n" 431 " Default: atomic\n" 432 " Valid only if --mode=eventdev\n" 433 " --event-eth-rxqs: Number of ethernet RX queues per device.\n" 434 " Default: 1\n" 435 " Valid only if --mode=eventdev\n" 436 " --event-vector: Enable event vectorization.\n" 437 " --event-vector-size: Max vector size if event vectorization is enabled.\n" 438 " --event-vector-tmo: Max timeout to form vector in nanoseconds if event vectorization is enabled\n" 439 " -E : Enable exact match, legacy flag please use --lookup=em instead\n" 440 " -L : Enable longest prefix match, legacy flag please use --lookup=lpm instead\n" 441 " --rule_ipv4=FILE: Specify the ipv4 rules entries file.\n" 442 " Each rule occupies one line.\n" 443 " 2 kinds of rules are supported.\n" 444 " One is ACL entry at while line leads with character '%c',\n" 445 " another is route entry at while line leads with character '%c'.\n" 446 " --rule_ipv6=FILE: Specify the ipv6 rules entries file.\n" 447 " --alg: ACL classify method to use, one of: %s.\n\n", 448 prgname, RTE_TEST_RX_DESC_DEFAULT, RTE_TEST_TX_DESC_DEFAULT, 449 ACL_LEAD_CHAR, ROUTE_LEAD_CHAR, alg); 450 } 451 452 static int 453 parse_max_pkt_len(const char *pktlen) 454 { 455 char *end = NULL; 456 unsigned long len; 457 458 /* parse decimal string */ 459 len = strtoul(pktlen, &end, 10); 460 if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) 461 return -1; 462 463 if (len == 0) 464 return -1; 465 466 return len; 467 } 468 469 static int 470 parse_portmask(const char *portmask) 471 { 472 char *end = NULL; 473 unsigned long pm; 474 475 /* parse hexadecimal string */ 476 pm = strtoul(portmask, &end, 16); 477 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 478 return 0; 479 480 return pm; 481 } 482 483 static int 484 parse_hash_entry_number(const char *hash_entry_num) 485 { 486 char *end = NULL; 487 unsigned long hash_en; 488 /* parse hexadecimal string */ 489 hash_en = strtoul(hash_entry_num, &end, 16); 490 if ((hash_entry_num[0] == '\0') || (end == NULL) || (*end != '\0')) 491 return -1; 492 493 if (hash_en == 0) 494 return -1; 495 496 return hash_en; 497 } 498 499 static int 500 parse_config(const char *q_arg) 501 { 502 char s[256]; 503 const char *p, *p0 = q_arg; 504 char *end; 505 enum fieldnames { 506 FLD_PORT = 0, 507 FLD_QUEUE, 508 FLD_LCORE, 509 _NUM_FLD 510 }; 511 unsigned long int_fld[_NUM_FLD]; 512 char *str_fld[_NUM_FLD]; 513 int i; 514 unsigned size; 515 516 nb_lcore_params = 0; 517 518 while ((p = strchr(p0,'(')) != NULL) { 519 ++p; 520 if((p0 = strchr(p,')')) == NULL) 521 return -1; 522 523 size = p0 - p; 524 if(size >= sizeof(s)) 525 return -1; 526 527 snprintf(s, sizeof(s), "%.*s", size, p); 528 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD) 529 return -1; 530 for (i = 0; i < _NUM_FLD; i++){ 531 errno = 0; 532 int_fld[i] = strtoul(str_fld[i], &end, 0); 533 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 534 return -1; 535 } 536 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 537 printf("exceeded max number of lcore params: %hu\n", 538 nb_lcore_params); 539 return -1; 540 } 541 lcore_params_array[nb_lcore_params].port_id = 542 (uint8_t)int_fld[FLD_PORT]; 543 lcore_params_array[nb_lcore_params].queue_id = 544 (uint8_t)int_fld[FLD_QUEUE]; 545 lcore_params_array[nb_lcore_params].lcore_id = 546 (uint8_t)int_fld[FLD_LCORE]; 547 ++nb_lcore_params; 548 } 549 lcore_params = lcore_params_array; 550 return 0; 551 } 552 553 static void 554 parse_eth_dest(const char *optarg) 555 { 556 uint16_t portid; 557 char *port_end; 558 uint8_t c, *dest, peer_addr[6]; 559 560 errno = 0; 561 portid = strtoul(optarg, &port_end, 10); 562 if (errno != 0 || port_end == optarg || *port_end++ != ',') 563 rte_exit(EXIT_FAILURE, 564 "Invalid eth-dest: %s", optarg); 565 if (portid >= RTE_MAX_ETHPORTS) 566 rte_exit(EXIT_FAILURE, 567 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n", 568 portid, RTE_MAX_ETHPORTS); 569 570 if (cmdline_parse_etheraddr(NULL, port_end, 571 &peer_addr, sizeof(peer_addr)) < 0) 572 rte_exit(EXIT_FAILURE, 573 "Invalid ethernet address: %s\n", 574 port_end); 575 dest = (uint8_t *)&dest_eth_addr[portid]; 576 for (c = 0; c < 6; c++) 577 dest[c] = peer_addr[c]; 578 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid]; 579 } 580 581 static void 582 parse_mode(const char *optarg) 583 { 584 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 585 586 if (!strcmp(optarg, "poll")) 587 evt_rsrc->enabled = false; 588 else if (!strcmp(optarg, "eventdev")) 589 evt_rsrc->enabled = true; 590 } 591 592 static void 593 parse_queue_size(const char *queue_size_arg, uint16_t *queue_size, int rx) 594 { 595 char *end = NULL; 596 unsigned long value; 597 598 /* parse decimal string */ 599 value = strtoul(queue_size_arg, &end, 10); 600 if ((queue_size_arg[0] == '\0') || (end == NULL) || 601 (*end != '\0') || (value == 0)) { 602 if (rx == 1) 603 rte_exit(EXIT_FAILURE, "Invalid rx-queue-size\n"); 604 else 605 rte_exit(EXIT_FAILURE, "Invalid tx-queue-size\n"); 606 607 return; 608 } 609 610 if (value > UINT16_MAX) { 611 if (rx == 1) 612 rte_exit(EXIT_FAILURE, "rx-queue-size %lu > %d\n", 613 value, UINT16_MAX); 614 else 615 rte_exit(EXIT_FAILURE, "tx-queue-size %lu > %d\n", 616 value, UINT16_MAX); 617 618 return; 619 } 620 621 *queue_size = value; 622 } 623 624 static void 625 parse_eventq_sched(const char *optarg) 626 { 627 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 628 629 if (!strcmp(optarg, "ordered")) 630 evt_rsrc->sched_type = RTE_SCHED_TYPE_ORDERED; 631 if (!strcmp(optarg, "atomic")) 632 evt_rsrc->sched_type = RTE_SCHED_TYPE_ATOMIC; 633 if (!strcmp(optarg, "parallel")) 634 evt_rsrc->sched_type = RTE_SCHED_TYPE_PARALLEL; 635 } 636 637 static void 638 parse_event_eth_rx_queues(const char *eth_rx_queues) 639 { 640 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 641 char *end = NULL; 642 uint8_t num_eth_rx_queues; 643 644 /* parse decimal string */ 645 num_eth_rx_queues = strtoul(eth_rx_queues, &end, 10); 646 if ((eth_rx_queues[0] == '\0') || (end == NULL) || (*end != '\0')) 647 return; 648 649 if (num_eth_rx_queues == 0) 650 return; 651 652 evt_rsrc->eth_rx_queues = num_eth_rx_queues; 653 } 654 655 static int 656 parse_lookup(const char *optarg) 657 { 658 if (!strcmp(optarg, "em")) 659 lookup_mode = L3FWD_LOOKUP_EM; 660 else if (!strcmp(optarg, "lpm")) 661 lookup_mode = L3FWD_LOOKUP_LPM; 662 else if (!strcmp(optarg, "fib")) 663 lookup_mode = L3FWD_LOOKUP_FIB; 664 else if (!strcmp(optarg, "acl")) 665 lookup_mode = L3FWD_LOOKUP_ACL; 666 else { 667 fprintf(stderr, "Invalid lookup option! Accepted options: acl, em, lpm, fib\n"); 668 return -1; 669 } 670 return 0; 671 } 672 673 #define MAX_JUMBO_PKT_LEN 9600 674 675 static const char short_options[] = 676 "p:" /* portmask */ 677 "P" /* promiscuous */ 678 "L" /* legacy enable long prefix match */ 679 "E" /* legacy enable exact match */ 680 ; 681 682 #define CMD_LINE_OPT_CONFIG "config" 683 #define CMD_LINE_OPT_RX_QUEUE_SIZE "rx-queue-size" 684 #define CMD_LINE_OPT_TX_QUEUE_SIZE "tx-queue-size" 685 #define CMD_LINE_OPT_ETH_DEST "eth-dest" 686 #define CMD_LINE_OPT_NO_NUMA "no-numa" 687 #define CMD_LINE_OPT_IPV6 "ipv6" 688 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len" 689 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num" 690 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype" 691 #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool" 692 #define CMD_LINE_OPT_MODE "mode" 693 #define CMD_LINE_OPT_EVENTQ_SYNC "eventq-sched" 694 #define CMD_LINE_OPT_EVENT_ETH_RX_QUEUES "event-eth-rxqs" 695 #define CMD_LINE_OPT_LOOKUP "lookup" 696 #define CMD_LINE_OPT_ENABLE_VECTOR "event-vector" 697 #define CMD_LINE_OPT_VECTOR_SIZE "event-vector-size" 698 #define CMD_LINE_OPT_VECTOR_TMO_NS "event-vector-tmo" 699 #define CMD_LINE_OPT_RULE_IPV4 "rule_ipv4" 700 #define CMD_LINE_OPT_RULE_IPV6 "rule_ipv6" 701 #define CMD_LINE_OPT_ALG "alg" 702 703 enum { 704 /* long options mapped to a short option */ 705 706 /* first long only option value must be >= 256, so that we won't 707 * conflict with short options */ 708 CMD_LINE_OPT_MIN_NUM = 256, 709 CMD_LINE_OPT_CONFIG_NUM, 710 CMD_LINE_OPT_RX_QUEUE_SIZE_NUM, 711 CMD_LINE_OPT_TX_QUEUE_SIZE_NUM, 712 CMD_LINE_OPT_ETH_DEST_NUM, 713 CMD_LINE_OPT_NO_NUMA_NUM, 714 CMD_LINE_OPT_IPV6_NUM, 715 CMD_LINE_OPT_MAX_PKT_LEN_NUM, 716 CMD_LINE_OPT_HASH_ENTRY_NUM_NUM, 717 CMD_LINE_OPT_PARSE_PTYPE_NUM, 718 CMD_LINE_OPT_RULE_IPV4_NUM, 719 CMD_LINE_OPT_RULE_IPV6_NUM, 720 CMD_LINE_OPT_ALG_NUM, 721 CMD_LINE_OPT_PARSE_PER_PORT_POOL, 722 CMD_LINE_OPT_MODE_NUM, 723 CMD_LINE_OPT_EVENTQ_SYNC_NUM, 724 CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM, 725 CMD_LINE_OPT_LOOKUP_NUM, 726 CMD_LINE_OPT_ENABLE_VECTOR_NUM, 727 CMD_LINE_OPT_VECTOR_SIZE_NUM, 728 CMD_LINE_OPT_VECTOR_TMO_NS_NUM 729 }; 730 731 static const struct option lgopts[] = { 732 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 733 {CMD_LINE_OPT_RX_QUEUE_SIZE, 1, 0, CMD_LINE_OPT_RX_QUEUE_SIZE_NUM}, 734 {CMD_LINE_OPT_TX_QUEUE_SIZE, 1, 0, CMD_LINE_OPT_TX_QUEUE_SIZE_NUM}, 735 {CMD_LINE_OPT_ETH_DEST, 1, 0, CMD_LINE_OPT_ETH_DEST_NUM}, 736 {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM}, 737 {CMD_LINE_OPT_IPV6, 0, 0, CMD_LINE_OPT_IPV6_NUM}, 738 {CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM}, 739 {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, CMD_LINE_OPT_HASH_ENTRY_NUM_NUM}, 740 {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, CMD_LINE_OPT_PARSE_PTYPE_NUM}, 741 {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL}, 742 {CMD_LINE_OPT_MODE, 1, 0, CMD_LINE_OPT_MODE_NUM}, 743 {CMD_LINE_OPT_EVENTQ_SYNC, 1, 0, CMD_LINE_OPT_EVENTQ_SYNC_NUM}, 744 {CMD_LINE_OPT_EVENT_ETH_RX_QUEUES, 1, 0, 745 CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM}, 746 {CMD_LINE_OPT_LOOKUP, 1, 0, CMD_LINE_OPT_LOOKUP_NUM}, 747 {CMD_LINE_OPT_ENABLE_VECTOR, 0, 0, CMD_LINE_OPT_ENABLE_VECTOR_NUM}, 748 {CMD_LINE_OPT_VECTOR_SIZE, 1, 0, CMD_LINE_OPT_VECTOR_SIZE_NUM}, 749 {CMD_LINE_OPT_VECTOR_TMO_NS, 1, 0, CMD_LINE_OPT_VECTOR_TMO_NS_NUM}, 750 {CMD_LINE_OPT_RULE_IPV4, 1, 0, CMD_LINE_OPT_RULE_IPV4_NUM}, 751 {CMD_LINE_OPT_RULE_IPV6, 1, 0, CMD_LINE_OPT_RULE_IPV6_NUM}, 752 {CMD_LINE_OPT_ALG, 1, 0, CMD_LINE_OPT_ALG_NUM}, 753 {NULL, 0, 0, 0} 754 }; 755 756 /* 757 * This expression is used to calculate the number of mbufs needed 758 * depending on user input, taking into account memory for rx and 759 * tx hardware rings, cache per lcore and mtable per port per lcore. 760 * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum 761 * value of 8192 762 */ 763 #define NB_MBUF(nports) RTE_MAX( \ 764 (nports*nb_rx_queue*nb_rxd + \ 765 nports*nb_lcores*MAX_PKT_BURST + \ 766 nports*n_tx_queue*nb_txd + \ 767 nb_lcores*MEMPOOL_CACHE_SIZE), \ 768 (unsigned)8192) 769 770 /* Parse the argument given in the command line of the application */ 771 static int 772 parse_args(int argc, char **argv) 773 { 774 int opt, ret; 775 char **argvopt; 776 int option_index; 777 char *prgname = argv[0]; 778 uint8_t lcore_params = 0; 779 uint8_t eventq_sched = 0; 780 uint8_t eth_rx_q = 0; 781 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 782 783 argvopt = argv; 784 785 /* Error or normal output strings. */ 786 while ((opt = getopt_long(argc, argvopt, short_options, 787 lgopts, &option_index)) != EOF) { 788 789 switch (opt) { 790 /* portmask */ 791 case 'p': 792 enabled_port_mask = parse_portmask(optarg); 793 if (enabled_port_mask == 0) { 794 fprintf(stderr, "Invalid portmask\n"); 795 print_usage(prgname); 796 return -1; 797 } 798 break; 799 800 case 'P': 801 promiscuous_on = 1; 802 break; 803 804 case 'E': 805 if (lookup_mode != L3FWD_LOOKUP_DEFAULT) { 806 fprintf(stderr, "Only one lookup mode is allowed at a time!\n"); 807 return -1; 808 } 809 lookup_mode = L3FWD_LOOKUP_EM; 810 break; 811 812 case 'L': 813 if (lookup_mode != L3FWD_LOOKUP_DEFAULT) { 814 fprintf(stderr, "Only one lookup mode is allowed at a time!\n"); 815 return -1; 816 } 817 lookup_mode = L3FWD_LOOKUP_LPM; 818 break; 819 820 /* long options */ 821 case CMD_LINE_OPT_CONFIG_NUM: 822 ret = parse_config(optarg); 823 if (ret) { 824 fprintf(stderr, "Invalid config\n"); 825 print_usage(prgname); 826 return -1; 827 } 828 lcore_params = 1; 829 break; 830 831 case CMD_LINE_OPT_RX_QUEUE_SIZE_NUM: 832 parse_queue_size(optarg, &nb_rxd, 1); 833 break; 834 835 case CMD_LINE_OPT_TX_QUEUE_SIZE_NUM: 836 parse_queue_size(optarg, &nb_txd, 0); 837 break; 838 839 case CMD_LINE_OPT_ETH_DEST_NUM: 840 parse_eth_dest(optarg); 841 break; 842 843 case CMD_LINE_OPT_NO_NUMA_NUM: 844 numa_on = 0; 845 break; 846 847 case CMD_LINE_OPT_IPV6_NUM: 848 ipv6 = 1; 849 break; 850 851 case CMD_LINE_OPT_MAX_PKT_LEN_NUM: 852 max_pkt_len = parse_max_pkt_len(optarg); 853 break; 854 855 case CMD_LINE_OPT_HASH_ENTRY_NUM_NUM: 856 ret = parse_hash_entry_number(optarg); 857 if ((ret > 0) && (ret <= L3FWD_HASH_ENTRIES)) { 858 hash_entry_number = ret; 859 } else { 860 fprintf(stderr, "invalid hash entry number\n"); 861 print_usage(prgname); 862 return -1; 863 } 864 break; 865 866 case CMD_LINE_OPT_PARSE_PTYPE_NUM: 867 printf("soft parse-ptype is enabled\n"); 868 parse_ptype = 1; 869 break; 870 871 case CMD_LINE_OPT_PARSE_PER_PORT_POOL: 872 printf("per port buffer pool is enabled\n"); 873 per_port_pool = 1; 874 break; 875 876 case CMD_LINE_OPT_MODE_NUM: 877 parse_mode(optarg); 878 break; 879 880 case CMD_LINE_OPT_EVENTQ_SYNC_NUM: 881 parse_eventq_sched(optarg); 882 eventq_sched = 1; 883 break; 884 885 case CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM: 886 parse_event_eth_rx_queues(optarg); 887 eth_rx_q = 1; 888 break; 889 890 case CMD_LINE_OPT_LOOKUP_NUM: 891 if (lookup_mode != L3FWD_LOOKUP_DEFAULT) { 892 fprintf(stderr, "Only one lookup mode is allowed at a time!\n"); 893 return -1; 894 } 895 ret = parse_lookup(optarg); 896 /* 897 * If parse_lookup was passed an invalid lookup type 898 * then return -1. Error log included within 899 * parse_lookup for simplicity. 900 */ 901 if (ret) 902 return -1; 903 break; 904 905 case CMD_LINE_OPT_ENABLE_VECTOR_NUM: 906 printf("event vectorization is enabled\n"); 907 evt_rsrc->vector_enabled = 1; 908 break; 909 case CMD_LINE_OPT_VECTOR_SIZE_NUM: 910 evt_rsrc->vector_size = strtol(optarg, NULL, 10); 911 break; 912 case CMD_LINE_OPT_VECTOR_TMO_NS_NUM: 913 evt_rsrc->vector_tmo_ns = strtoull(optarg, NULL, 10); 914 break; 915 case CMD_LINE_OPT_RULE_IPV4_NUM: 916 l3fwd_set_rule_ipv4_name(optarg); 917 break; 918 case CMD_LINE_OPT_RULE_IPV6_NUM: 919 l3fwd_set_rule_ipv6_name(optarg); 920 break; 921 case CMD_LINE_OPT_ALG_NUM: 922 l3fwd_set_alg(optarg); 923 break; 924 default: 925 print_usage(prgname); 926 return -1; 927 } 928 } 929 930 if (evt_rsrc->enabled && lcore_params) { 931 fprintf(stderr, "lcore config is not valid when event mode is selected\n"); 932 return -1; 933 } 934 935 if (!evt_rsrc->enabled && eth_rx_q) { 936 fprintf(stderr, "eth_rx_queues is valid only when event mode is selected\n"); 937 return -1; 938 } 939 940 if (!evt_rsrc->enabled && eventq_sched) { 941 fprintf(stderr, "eventq_sched is valid only when event mode is selected\n"); 942 return -1; 943 } 944 945 if (evt_rsrc->vector_enabled && !evt_rsrc->vector_size) { 946 evt_rsrc->vector_size = VECTOR_SIZE_DEFAULT; 947 fprintf(stderr, "vector size set to default (%" PRIu16 ")\n", 948 evt_rsrc->vector_size); 949 } 950 951 if (evt_rsrc->vector_enabled && !evt_rsrc->vector_tmo_ns) { 952 evt_rsrc->vector_tmo_ns = VECTOR_TMO_NS_DEFAULT; 953 fprintf(stderr, 954 "vector timeout set to default (%" PRIu64 " ns)\n", 955 evt_rsrc->vector_tmo_ns); 956 } 957 958 /* 959 * Nothing is selected, pick longest-prefix match 960 * as default match. 961 */ 962 if (lookup_mode == L3FWD_LOOKUP_DEFAULT) { 963 fprintf(stderr, "Neither ACL, LPM, EM, or FIB selected, defaulting to LPM\n"); 964 lookup_mode = L3FWD_LOOKUP_LPM; 965 } 966 967 /* 968 * ipv6 and hash flags are valid only for 969 * exact match, reset them to default for 970 * longest-prefix match. 971 */ 972 if (lookup_mode == L3FWD_LOOKUP_LPM) { 973 ipv6 = 0; 974 hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT; 975 } 976 977 /* For ACL, update port config rss hash filter */ 978 if (lookup_mode == L3FWD_LOOKUP_ACL) { 979 port_conf.rx_adv_conf.rss_conf.rss_hf |= 980 RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP | RTE_ETH_RSS_SCTP; 981 } 982 983 if (optind >= 0) 984 argv[optind-1] = prgname; 985 986 ret = optind-1; 987 optind = 1; /* reset getopt lib */ 988 return ret; 989 } 990 991 static void 992 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 993 { 994 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 995 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 996 printf("%s%s", name, buf); 997 } 998 999 int 1000 init_mem(uint16_t portid, unsigned int nb_mbuf) 1001 { 1002 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 1003 struct lcore_conf *qconf; 1004 int socketid; 1005 unsigned lcore_id; 1006 char s[64]; 1007 1008 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1009 if (rte_lcore_is_enabled(lcore_id) == 0) 1010 continue; 1011 1012 if (numa_on) 1013 socketid = rte_lcore_to_socket_id(lcore_id); 1014 else 1015 socketid = 0; 1016 1017 if (socketid >= NB_SOCKETS) { 1018 rte_exit(EXIT_FAILURE, 1019 "Socket %d of lcore %u is out of range %d\n", 1020 socketid, lcore_id, NB_SOCKETS); 1021 } 1022 1023 if (pktmbuf_pool[portid][socketid] == NULL) { 1024 snprintf(s, sizeof(s), "mbuf_pool_%d:%d", 1025 portid, socketid); 1026 pktmbuf_pool[portid][socketid] = 1027 rte_pktmbuf_pool_create(s, nb_mbuf, 1028 MEMPOOL_CACHE_SIZE, 0, 1029 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 1030 if (pktmbuf_pool[portid][socketid] == NULL) 1031 rte_exit(EXIT_FAILURE, 1032 "Cannot init mbuf pool on socket %d\n", 1033 socketid); 1034 else 1035 printf("Allocated mbuf pool on socket %d\n", 1036 socketid); 1037 1038 /* Setup ACL, LPM, EM(f.e Hash) or FIB. But, only once per 1039 * available socket. 1040 */ 1041 if (!lkp_per_socket[socketid]) { 1042 l3fwd_lkp.setup(socketid); 1043 lkp_per_socket[socketid] = 1; 1044 } 1045 } 1046 1047 if (evt_rsrc->vector_enabled && vector_pool[portid] == NULL) { 1048 unsigned int nb_vec; 1049 1050 nb_vec = (nb_mbuf + evt_rsrc->vector_size - 1) / 1051 evt_rsrc->vector_size; 1052 nb_vec = RTE_MAX(512U, nb_vec); 1053 nb_vec += rte_lcore_count() * 32; 1054 snprintf(s, sizeof(s), "vector_pool_%d", portid); 1055 vector_pool[portid] = rte_event_vector_pool_create( 1056 s, nb_vec, 32, evt_rsrc->vector_size, socketid); 1057 if (vector_pool[portid] == NULL) 1058 rte_exit(EXIT_FAILURE, 1059 "Failed to create vector pool for port %d\n", 1060 portid); 1061 else 1062 printf("Allocated vector pool for port %d\n", 1063 portid); 1064 } 1065 1066 qconf = &lcore_conf[lcore_id]; 1067 qconf->ipv4_lookup_struct = 1068 l3fwd_lkp.get_ipv4_lookup_struct(socketid); 1069 qconf->ipv6_lookup_struct = 1070 l3fwd_lkp.get_ipv6_lookup_struct(socketid); 1071 } 1072 return 0; 1073 } 1074 1075 /* Check the link status of all ports in up to 9s, and print them finally */ 1076 static void 1077 check_all_ports_link_status(uint32_t port_mask) 1078 { 1079 #define CHECK_INTERVAL 100 /* 100ms */ 1080 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1081 uint16_t portid; 1082 uint8_t count, all_ports_up, print_flag = 0; 1083 struct rte_eth_link link; 1084 int ret; 1085 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 1086 1087 printf("\nChecking link status"); 1088 fflush(stdout); 1089 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1090 if (force_quit) 1091 return; 1092 all_ports_up = 1; 1093 RTE_ETH_FOREACH_DEV(portid) { 1094 if (force_quit) 1095 return; 1096 if ((port_mask & (1 << portid)) == 0) 1097 continue; 1098 memset(&link, 0, sizeof(link)); 1099 ret = rte_eth_link_get_nowait(portid, &link); 1100 if (ret < 0) { 1101 all_ports_up = 0; 1102 if (print_flag == 1) 1103 printf("Port %u link get failed: %s\n", 1104 portid, rte_strerror(-ret)); 1105 continue; 1106 } 1107 /* print link status if flag set */ 1108 if (print_flag == 1) { 1109 rte_eth_link_to_str(link_status_text, 1110 sizeof(link_status_text), &link); 1111 printf("Port %d %s\n", portid, 1112 link_status_text); 1113 continue; 1114 } 1115 /* clear all_ports_up flag if any link down */ 1116 if (link.link_status == RTE_ETH_LINK_DOWN) { 1117 all_ports_up = 0; 1118 break; 1119 } 1120 } 1121 /* after finally printing all link status, get out */ 1122 if (print_flag == 1) 1123 break; 1124 1125 if (all_ports_up == 0) { 1126 printf("."); 1127 fflush(stdout); 1128 rte_delay_ms(CHECK_INTERVAL); 1129 } 1130 1131 /* set the print_flag if all ports up or timeout */ 1132 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1133 print_flag = 1; 1134 printf("done\n"); 1135 } 1136 } 1137 } 1138 1139 static void 1140 signal_handler(int signum) 1141 { 1142 if (signum == SIGINT || signum == SIGTERM) { 1143 printf("\n\nSignal %d received, preparing to exit...\n", 1144 signum); 1145 force_quit = true; 1146 } 1147 } 1148 1149 static int 1150 prepare_ptype_parser(uint16_t portid, uint16_t queueid) 1151 { 1152 if (parse_ptype) { 1153 printf("Port %d: softly parse packet type info\n", portid); 1154 if (rte_eth_add_rx_callback(portid, queueid, 1155 l3fwd_lkp.cb_parse_ptype, 1156 NULL)) 1157 return 1; 1158 1159 printf("Failed to add rx callback: port=%d\n", portid); 1160 return 0; 1161 } 1162 1163 if (l3fwd_lkp.check_ptype(portid)) 1164 return 1; 1165 1166 printf("port %d cannot parse packet type, please add --%s\n", 1167 portid, CMD_LINE_OPT_PARSE_PTYPE); 1168 return 0; 1169 } 1170 1171 static uint32_t 1172 eth_dev_get_overhead_len(uint32_t max_rx_pktlen, uint16_t max_mtu) 1173 { 1174 uint32_t overhead_len; 1175 1176 if (max_mtu != UINT16_MAX && max_rx_pktlen > max_mtu) 1177 overhead_len = max_rx_pktlen - max_mtu; 1178 else 1179 overhead_len = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1180 1181 return overhead_len; 1182 } 1183 1184 static int 1185 config_port_max_pkt_len(struct rte_eth_conf *conf, 1186 struct rte_eth_dev_info *dev_info) 1187 { 1188 uint32_t overhead_len; 1189 1190 if (max_pkt_len == 0) 1191 return 0; 1192 1193 if (max_pkt_len < RTE_ETHER_MIN_LEN || max_pkt_len > MAX_JUMBO_PKT_LEN) 1194 return -1; 1195 1196 overhead_len = eth_dev_get_overhead_len(dev_info->max_rx_pktlen, 1197 dev_info->max_mtu); 1198 conf->rxmode.mtu = max_pkt_len - overhead_len; 1199 1200 if (conf->rxmode.mtu > RTE_ETHER_MTU) 1201 conf->txmode.offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 1202 1203 return 0; 1204 } 1205 1206 static void 1207 l3fwd_poll_resource_setup(void) 1208 { 1209 uint8_t nb_rx_queue, queue, socketid; 1210 struct rte_eth_dev_info dev_info; 1211 uint32_t n_tx_queue, nb_lcores; 1212 struct rte_eth_txconf *txconf; 1213 struct lcore_conf *qconf; 1214 uint16_t queueid, portid; 1215 unsigned int nb_ports; 1216 unsigned int lcore_id; 1217 int ret; 1218 1219 if (check_lcore_params() < 0) 1220 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n"); 1221 1222 ret = init_lcore_rx_queues(); 1223 if (ret < 0) 1224 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 1225 1226 nb_ports = rte_eth_dev_count_avail(); 1227 1228 if (check_port_config() < 0) 1229 rte_exit(EXIT_FAILURE, "check_port_config failed\n"); 1230 1231 nb_lcores = rte_lcore_count(); 1232 1233 /* initialize all ports */ 1234 RTE_ETH_FOREACH_DEV(portid) { 1235 struct rte_eth_conf local_port_conf = port_conf; 1236 1237 /* skip ports that are not enabled */ 1238 if ((enabled_port_mask & (1 << portid)) == 0) { 1239 printf("\nSkipping disabled port %d\n", portid); 1240 continue; 1241 } 1242 1243 /* init port */ 1244 printf("Initializing port %d ... ", portid ); 1245 fflush(stdout); 1246 1247 nb_rx_queue = get_port_n_rx_queues(portid); 1248 n_tx_queue = nb_lcores; 1249 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT) 1250 n_tx_queue = MAX_TX_QUEUE_PER_PORT; 1251 printf("Creating queues: nb_rxq=%d nb_txq=%u... ", 1252 nb_rx_queue, (unsigned)n_tx_queue ); 1253 1254 ret = rte_eth_dev_info_get(portid, &dev_info); 1255 if (ret != 0) 1256 rte_exit(EXIT_FAILURE, 1257 "Error during getting device (port %u) info: %s\n", 1258 portid, strerror(-ret)); 1259 1260 ret = config_port_max_pkt_len(&local_port_conf, &dev_info); 1261 if (ret != 0) 1262 rte_exit(EXIT_FAILURE, 1263 "Invalid max packet length: %u (port %u)\n", 1264 max_pkt_len, portid); 1265 1266 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 1267 local_port_conf.txmode.offloads |= 1268 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1269 1270 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 1271 dev_info.flow_type_rss_offloads; 1272 1273 if (dev_info.max_rx_queues == 1) 1274 local_port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 1275 1276 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 1277 port_conf.rx_adv_conf.rss_conf.rss_hf) { 1278 printf("Port %u modified RSS hash function based on hardware support," 1279 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 1280 portid, 1281 port_conf.rx_adv_conf.rss_conf.rss_hf, 1282 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 1283 } 1284 1285 ret = rte_eth_dev_configure(portid, nb_rx_queue, 1286 (uint16_t)n_tx_queue, &local_port_conf); 1287 if (ret < 0) 1288 rte_exit(EXIT_FAILURE, 1289 "Cannot configure device: err=%d, port=%d\n", 1290 ret, portid); 1291 1292 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 1293 &nb_txd); 1294 if (ret < 0) 1295 rte_exit(EXIT_FAILURE, 1296 "Cannot adjust number of descriptors: err=%d, " 1297 "port=%d\n", ret, portid); 1298 1299 ret = rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 1300 if (ret < 0) 1301 rte_exit(EXIT_FAILURE, 1302 "Cannot get MAC address: err=%d, port=%d\n", 1303 ret, portid); 1304 1305 print_ethaddr(" Address:", &ports_eth_addr[portid]); 1306 printf(", "); 1307 print_ethaddr("Destination:", 1308 (const struct rte_ether_addr *)&dest_eth_addr[portid]); 1309 printf(", "); 1310 1311 /* 1312 * prepare src MACs for each port. 1313 */ 1314 rte_ether_addr_copy(&ports_eth_addr[portid], 1315 (struct rte_ether_addr *)(val_eth + portid) + 1); 1316 1317 /* init memory */ 1318 if (!per_port_pool) { 1319 /* portid = 0; this is *not* signifying the first port, 1320 * rather, it signifies that portid is ignored. 1321 */ 1322 ret = init_mem(0, NB_MBUF(nb_ports)); 1323 } else { 1324 ret = init_mem(portid, NB_MBUF(1)); 1325 } 1326 if (ret < 0) 1327 rte_exit(EXIT_FAILURE, "init_mem failed\n"); 1328 1329 /* init one TX queue per couple (lcore,port) */ 1330 queueid = 0; 1331 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1332 if (rte_lcore_is_enabled(lcore_id) == 0) 1333 continue; 1334 1335 if (numa_on) 1336 socketid = 1337 (uint8_t)rte_lcore_to_socket_id(lcore_id); 1338 else 1339 socketid = 0; 1340 1341 printf("txq=%u,%d,%d ", lcore_id, queueid, socketid); 1342 fflush(stdout); 1343 1344 txconf = &dev_info.default_txconf; 1345 txconf->offloads = local_port_conf.txmode.offloads; 1346 ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, 1347 socketid, txconf); 1348 if (ret < 0) 1349 rte_exit(EXIT_FAILURE, 1350 "rte_eth_tx_queue_setup: err=%d, " 1351 "port=%d\n", ret, portid); 1352 1353 qconf = &lcore_conf[lcore_id]; 1354 qconf->tx_queue_id[portid] = queueid; 1355 queueid++; 1356 1357 qconf->tx_port_id[qconf->n_tx_port] = portid; 1358 qconf->n_tx_port++; 1359 } 1360 printf("\n"); 1361 } 1362 1363 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1364 if (rte_lcore_is_enabled(lcore_id) == 0) 1365 continue; 1366 qconf = &lcore_conf[lcore_id]; 1367 printf("\nInitializing rx queues on lcore %u ... ", lcore_id ); 1368 fflush(stdout); 1369 /* init RX queues */ 1370 for(queue = 0; queue < qconf->n_rx_queue; ++queue) { 1371 struct rte_eth_rxconf rxq_conf; 1372 1373 portid = qconf->rx_queue_list[queue].port_id; 1374 queueid = qconf->rx_queue_list[queue].queue_id; 1375 1376 if (numa_on) 1377 socketid = 1378 (uint8_t)rte_lcore_to_socket_id(lcore_id); 1379 else 1380 socketid = 0; 1381 1382 printf("rxq=%d,%d,%d ", portid, queueid, socketid); 1383 fflush(stdout); 1384 1385 ret = rte_eth_dev_info_get(portid, &dev_info); 1386 if (ret != 0) 1387 rte_exit(EXIT_FAILURE, 1388 "Error during getting device (port %u) info: %s\n", 1389 portid, strerror(-ret)); 1390 1391 rxq_conf = dev_info.default_rxconf; 1392 rxq_conf.offloads = port_conf.rxmode.offloads; 1393 if (!per_port_pool) 1394 ret = rte_eth_rx_queue_setup(portid, queueid, 1395 nb_rxd, socketid, 1396 &rxq_conf, 1397 pktmbuf_pool[0][socketid]); 1398 else 1399 ret = rte_eth_rx_queue_setup(portid, queueid, 1400 nb_rxd, socketid, 1401 &rxq_conf, 1402 pktmbuf_pool[portid][socketid]); 1403 if (ret < 0) 1404 rte_exit(EXIT_FAILURE, 1405 "rte_eth_rx_queue_setup: err=%d, port=%d\n", 1406 ret, portid); 1407 } 1408 } 1409 } 1410 1411 static inline int 1412 l3fwd_service_enable(uint32_t service_id) 1413 { 1414 uint8_t min_service_count = UINT8_MAX; 1415 uint32_t slcore_array[RTE_MAX_LCORE]; 1416 unsigned int slcore = 0; 1417 uint8_t service_count; 1418 int32_t slcore_count; 1419 1420 if (!rte_service_lcore_count()) 1421 return -ENOENT; 1422 1423 slcore_count = rte_service_lcore_list(slcore_array, RTE_MAX_LCORE); 1424 if (slcore_count < 0) 1425 return -ENOENT; 1426 /* Get the core which has least number of services running. */ 1427 while (slcore_count--) { 1428 /* Reset default mapping */ 1429 if (rte_service_map_lcore_set(service_id, 1430 slcore_array[slcore_count], 0) != 0) 1431 return -ENOENT; 1432 service_count = rte_service_lcore_count_services( 1433 slcore_array[slcore_count]); 1434 if (service_count < min_service_count) { 1435 slcore = slcore_array[slcore_count]; 1436 min_service_count = service_count; 1437 } 1438 } 1439 if (rte_service_map_lcore_set(service_id, slcore, 1)) 1440 return -ENOENT; 1441 rte_service_lcore_start(slcore); 1442 1443 return 0; 1444 } 1445 1446 static void 1447 l3fwd_event_service_setup(void) 1448 { 1449 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc(); 1450 struct rte_event_dev_info evdev_info; 1451 uint32_t service_id, caps; 1452 int ret, i; 1453 1454 rte_event_dev_info_get(evt_rsrc->event_d_id, &evdev_info); 1455 if (!(evdev_info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED)) { 1456 ret = rte_event_dev_service_id_get(evt_rsrc->event_d_id, 1457 &service_id); 1458 if (ret != -ESRCH && ret != 0) 1459 rte_exit(EXIT_FAILURE, 1460 "Error in starting eventdev service\n"); 1461 l3fwd_service_enable(service_id); 1462 } 1463 1464 for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++) { 1465 ret = rte_event_eth_rx_adapter_caps_get(evt_rsrc->event_d_id, 1466 evt_rsrc->rx_adptr.rx_adptr[i], &caps); 1467 if (ret < 0) 1468 rte_exit(EXIT_FAILURE, 1469 "Failed to get Rx adapter[%d] caps\n", 1470 evt_rsrc->rx_adptr.rx_adptr[i]); 1471 ret = rte_event_eth_rx_adapter_service_id_get( 1472 evt_rsrc->event_d_id, 1473 &service_id); 1474 if (ret != -ESRCH && ret != 0) 1475 rte_exit(EXIT_FAILURE, 1476 "Error in starting Rx adapter[%d] service\n", 1477 evt_rsrc->rx_adptr.rx_adptr[i]); 1478 l3fwd_service_enable(service_id); 1479 } 1480 1481 for (i = 0; i < evt_rsrc->tx_adptr.nb_tx_adptr; i++) { 1482 ret = rte_event_eth_tx_adapter_caps_get(evt_rsrc->event_d_id, 1483 evt_rsrc->tx_adptr.tx_adptr[i], &caps); 1484 if (ret < 0) 1485 rte_exit(EXIT_FAILURE, 1486 "Failed to get Rx adapter[%d] caps\n", 1487 evt_rsrc->tx_adptr.tx_adptr[i]); 1488 ret = rte_event_eth_tx_adapter_service_id_get( 1489 evt_rsrc->event_d_id, 1490 &service_id); 1491 if (ret != -ESRCH && ret != 0) 1492 rte_exit(EXIT_FAILURE, 1493 "Error in starting Rx adapter[%d] service\n", 1494 evt_rsrc->tx_adptr.tx_adptr[i]); 1495 l3fwd_service_enable(service_id); 1496 } 1497 } 1498 1499 int 1500 main(int argc, char **argv) 1501 { 1502 struct l3fwd_event_resources *evt_rsrc; 1503 struct lcore_conf *qconf; 1504 uint16_t queueid, portid; 1505 unsigned int lcore_id; 1506 uint8_t queue; 1507 int i, ret; 1508 1509 /* init EAL */ 1510 ret = rte_eal_init(argc, argv); 1511 if (ret < 0) 1512 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 1513 argc -= ret; 1514 argv += ret; 1515 1516 force_quit = false; 1517 signal(SIGINT, signal_handler); 1518 signal(SIGTERM, signal_handler); 1519 1520 /* pre-init dst MACs for all ports to 02:00:00:00:00:xx */ 1521 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 1522 dest_eth_addr[portid] = 1523 RTE_ETHER_LOCAL_ADMIN_ADDR + ((uint64_t)portid << 40); 1524 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid]; 1525 } 1526 1527 evt_rsrc = l3fwd_get_eventdev_rsrc(); 1528 /* parse application arguments (after the EAL ones) */ 1529 ret = parse_args(argc, argv); 1530 if (ret < 0) 1531 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n"); 1532 1533 /* Setup function pointers for lookup method. */ 1534 setup_l3fwd_lookup_tables(); 1535 1536 /* Add the config file rules */ 1537 l3fwd_lkp.read_config_files(); 1538 1539 evt_rsrc->per_port_pool = per_port_pool; 1540 evt_rsrc->pkt_pool = pktmbuf_pool; 1541 evt_rsrc->vec_pool = vector_pool; 1542 evt_rsrc->port_mask = enabled_port_mask; 1543 /* Configure eventdev parameters if user has requested */ 1544 if (evt_rsrc->enabled) { 1545 l3fwd_event_resource_setup(&port_conf); 1546 if (lookup_mode == L3FWD_LOOKUP_EM) 1547 l3fwd_lkp.main_loop = evt_rsrc->ops.em_event_loop; 1548 else if (lookup_mode == L3FWD_LOOKUP_FIB) 1549 l3fwd_lkp.main_loop = evt_rsrc->ops.fib_event_loop; 1550 else 1551 l3fwd_lkp.main_loop = evt_rsrc->ops.lpm_event_loop; 1552 l3fwd_event_service_setup(); 1553 } else 1554 l3fwd_poll_resource_setup(); 1555 1556 /* start ports */ 1557 RTE_ETH_FOREACH_DEV(portid) { 1558 if ((enabled_port_mask & (1 << portid)) == 0) { 1559 continue; 1560 } 1561 /* Start device */ 1562 ret = rte_eth_dev_start(portid); 1563 if (ret < 0) 1564 rte_exit(EXIT_FAILURE, 1565 "rte_eth_dev_start: err=%d, port=%d\n", 1566 ret, portid); 1567 1568 /* 1569 * If enabled, put device in promiscuous mode. 1570 * This allows IO forwarding mode to forward packets 1571 * to itself through 2 cross-connected ports of the 1572 * target machine. 1573 */ 1574 if (promiscuous_on) { 1575 ret = rte_eth_promiscuous_enable(portid); 1576 if (ret != 0) 1577 rte_exit(EXIT_FAILURE, 1578 "rte_eth_promiscuous_enable: err=%s, port=%u\n", 1579 rte_strerror(-ret), portid); 1580 } 1581 } 1582 1583 printf("\n"); 1584 1585 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1586 if (rte_lcore_is_enabled(lcore_id) == 0) 1587 continue; 1588 qconf = &lcore_conf[lcore_id]; 1589 for (queue = 0; queue < qconf->n_rx_queue; ++queue) { 1590 portid = qconf->rx_queue_list[queue].port_id; 1591 queueid = qconf->rx_queue_list[queue].queue_id; 1592 if (prepare_ptype_parser(portid, queueid) == 0) 1593 rte_exit(EXIT_FAILURE, "ptype check fails\n"); 1594 } 1595 } 1596 1597 check_all_ports_link_status(enabled_port_mask); 1598 1599 ret = 0; 1600 /* launch per-lcore init on every lcore */ 1601 rte_eal_mp_remote_launch(l3fwd_lkp.main_loop, NULL, CALL_MAIN); 1602 if (evt_rsrc->enabled) { 1603 for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++) 1604 rte_event_eth_rx_adapter_stop( 1605 evt_rsrc->rx_adptr.rx_adptr[i]); 1606 for (i = 0; i < evt_rsrc->tx_adptr.nb_tx_adptr; i++) 1607 rte_event_eth_tx_adapter_stop( 1608 evt_rsrc->tx_adptr.tx_adptr[i]); 1609 1610 RTE_ETH_FOREACH_DEV(portid) { 1611 if ((enabled_port_mask & (1 << portid)) == 0) 1612 continue; 1613 ret = rte_eth_dev_stop(portid); 1614 if (ret != 0) 1615 printf("rte_eth_dev_stop: err=%d, port=%u\n", 1616 ret, portid); 1617 } 1618 1619 rte_eal_mp_wait_lcore(); 1620 RTE_ETH_FOREACH_DEV(portid) { 1621 if ((enabled_port_mask & (1 << portid)) == 0) 1622 continue; 1623 rte_eth_dev_close(portid); 1624 } 1625 1626 rte_event_dev_stop(evt_rsrc->event_d_id); 1627 rte_event_dev_close(evt_rsrc->event_d_id); 1628 1629 } else { 1630 rte_eal_mp_wait_lcore(); 1631 1632 RTE_ETH_FOREACH_DEV(portid) { 1633 if ((enabled_port_mask & (1 << portid)) == 0) 1634 continue; 1635 printf("Closing port %d...", portid); 1636 ret = rte_eth_dev_stop(portid); 1637 if (ret != 0) 1638 printf("rte_eth_dev_stop: err=%d, port=%u\n", 1639 ret, portid); 1640 rte_eth_dev_close(portid); 1641 printf(" Done\n"); 1642 } 1643 } 1644 1645 /* clean up config file routes */ 1646 l3fwd_lkp.free_routes(); 1647 1648 /* clean up the EAL */ 1649 rte_eal_cleanup(); 1650 1651 printf("Bye...\n"); 1652 1653 return ret; 1654 } 1655