1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <stdint.h> 37 #include <inttypes.h> 38 #include <sys/types.h> 39 #include <string.h> 40 #include <sys/queue.h> 41 #include <stdarg.h> 42 #include <errno.h> 43 #include <getopt.h> 44 #include <unistd.h> 45 #include <signal.h> 46 47 #include <rte_common.h> 48 #include <rte_byteorder.h> 49 #include <rte_log.h> 50 #include <rte_malloc.h> 51 #include <rte_memory.h> 52 #include <rte_memcpy.h> 53 #include <rte_memzone.h> 54 #include <rte_eal.h> 55 #include <rte_launch.h> 56 #include <rte_atomic.h> 57 #include <rte_cycles.h> 58 #include <rte_prefetch.h> 59 #include <rte_lcore.h> 60 #include <rte_per_lcore.h> 61 #include <rte_branch_prediction.h> 62 #include <rte_interrupts.h> 63 #include <rte_pci.h> 64 #include <rte_random.h> 65 #include <rte_debug.h> 66 #include <rte_ether.h> 67 #include <rte_ethdev.h> 68 #include <rte_mempool.h> 69 #include <rte_mbuf.h> 70 #include <rte_ip.h> 71 #include <rte_tcp.h> 72 #include <rte_udp.h> 73 #include <rte_string_fns.h> 74 #include <rte_timer.h> 75 #include <rte_power.h> 76 #include <rte_spinlock.h> 77 78 #define RTE_LOGTYPE_L3FWD_POWER RTE_LOGTYPE_USER1 79 80 #define MAX_PKT_BURST 32 81 82 #define MIN_ZERO_POLL_COUNT 10 83 84 /* around 100ms at 2 Ghz */ 85 #define TIMER_RESOLUTION_CYCLES 200000000ULL 86 /* 100 ms interval */ 87 #define TIMER_NUMBER_PER_SECOND 10 88 /* 100000 us */ 89 #define SCALING_PERIOD (1000000/TIMER_NUMBER_PER_SECOND) 90 #define SCALING_DOWN_TIME_RATIO_THRESHOLD 0.25 91 92 #define APP_LOOKUP_EXACT_MATCH 0 93 #define APP_LOOKUP_LPM 1 94 #define DO_RFC_1812_CHECKS 95 96 #ifndef APP_LOOKUP_METHOD 97 #define APP_LOOKUP_METHOD APP_LOOKUP_LPM 98 #endif 99 100 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 101 #include <rte_hash.h> 102 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 103 #include <rte_lpm.h> 104 #else 105 #error "APP_LOOKUP_METHOD set to incorrect value" 106 #endif 107 108 #ifndef IPv6_BYTES 109 #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\ 110 "%02x%02x:%02x%02x:%02x%02x:%02x%02x" 111 #define IPv6_BYTES(addr) \ 112 addr[0], addr[1], addr[2], addr[3], \ 113 addr[4], addr[5], addr[6], addr[7], \ 114 addr[8], addr[9], addr[10], addr[11],\ 115 addr[12], addr[13],addr[14], addr[15] 116 #endif 117 118 #define MAX_JUMBO_PKT_LEN 9600 119 120 #define IPV6_ADDR_LEN 16 121 122 #define MEMPOOL_CACHE_SIZE 256 123 124 /* 125 * This expression is used to calculate the number of mbufs needed depending on 126 * user input, taking into account memory for rx and tx hardware rings, cache 127 * per lcore and mtable per port per lcore. RTE_MAX is used to ensure that 128 * NB_MBUF never goes below a minimum value of 8192. 129 */ 130 131 #define NB_MBUF RTE_MAX ( \ 132 (nb_ports*nb_rx_queue*nb_rxd + \ 133 nb_ports*nb_lcores*MAX_PKT_BURST + \ 134 nb_ports*n_tx_queue*nb_txd + \ 135 nb_lcores*MEMPOOL_CACHE_SIZE), \ 136 (unsigned)8192) 137 138 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 139 140 #define NB_SOCKETS 8 141 142 /* Configure how many packets ahead to prefetch, when reading packets */ 143 #define PREFETCH_OFFSET 3 144 145 /* 146 * Configurable number of RX/TX ring descriptors 147 */ 148 #define RTE_TEST_RX_DESC_DEFAULT 512 149 #define RTE_TEST_TX_DESC_DEFAULT 512 150 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 151 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 152 153 /* ethernet addresses of ports */ 154 static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; 155 156 /* ethernet addresses of ports */ 157 static rte_spinlock_t locks[RTE_MAX_ETHPORTS]; 158 159 /* mask of enabled ports */ 160 static uint32_t enabled_port_mask = 0; 161 /* Ports set in promiscuous mode off by default. */ 162 static int promiscuous_on = 0; 163 /* NUMA is enabled by default. */ 164 static int numa_on = 1; 165 static int parse_ptype; /**< Parse packet type using rx callback, and */ 166 /**< disabled by default */ 167 168 enum freq_scale_hint_t 169 { 170 FREQ_LOWER = -1, 171 FREQ_CURRENT = 0, 172 FREQ_HIGHER = 1, 173 FREQ_HIGHEST = 2 174 }; 175 176 struct lcore_rx_queue { 177 uint16_t port_id; 178 uint8_t queue_id; 179 enum freq_scale_hint_t freq_up_hint; 180 uint32_t zero_rx_packet_count; 181 uint32_t idle_hint; 182 } __rte_cache_aligned; 183 184 #define MAX_RX_QUEUE_PER_LCORE 16 185 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 186 #define MAX_RX_QUEUE_PER_PORT 128 187 188 #define MAX_RX_QUEUE_INTERRUPT_PER_PORT 16 189 190 191 #define MAX_LCORE_PARAMS 1024 192 struct lcore_params { 193 uint16_t port_id; 194 uint8_t queue_id; 195 uint8_t lcore_id; 196 } __rte_cache_aligned; 197 198 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 199 static struct lcore_params lcore_params_array_default[] = { 200 {0, 0, 2}, 201 {0, 1, 2}, 202 {0, 2, 2}, 203 {1, 0, 2}, 204 {1, 1, 2}, 205 {1, 2, 2}, 206 {2, 0, 2}, 207 {3, 0, 3}, 208 {3, 1, 3}, 209 }; 210 211 static struct lcore_params * lcore_params = lcore_params_array_default; 212 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) / 213 sizeof(lcore_params_array_default[0]); 214 215 static struct rte_eth_conf port_conf = { 216 .rxmode = { 217 .mq_mode = ETH_MQ_RX_RSS, 218 .max_rx_pkt_len = ETHER_MAX_LEN, 219 .split_hdr_size = 0, 220 .header_split = 0, /**< Header Split disabled */ 221 .hw_ip_checksum = 1, /**< IP checksum offload enabled */ 222 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 223 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 224 .hw_strip_crc = 1, /**< CRC stripped by hardware */ 225 }, 226 .rx_adv_conf = { 227 .rss_conf = { 228 .rss_key = NULL, 229 .rss_hf = ETH_RSS_UDP, 230 }, 231 }, 232 .txmode = { 233 .mq_mode = ETH_MQ_TX_NONE, 234 }, 235 .intr_conf = { 236 .lsc = 1, 237 .rxq = 1, 238 }, 239 }; 240 241 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS]; 242 243 244 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 245 246 #ifdef RTE_ARCH_X86 247 #include <rte_hash_crc.h> 248 #define DEFAULT_HASH_FUNC rte_hash_crc 249 #else 250 #include <rte_jhash.h> 251 #define DEFAULT_HASH_FUNC rte_jhash 252 #endif 253 254 struct ipv4_5tuple { 255 uint32_t ip_dst; 256 uint32_t ip_src; 257 uint16_t port_dst; 258 uint16_t port_src; 259 uint8_t proto; 260 } __attribute__((__packed__)); 261 262 struct ipv6_5tuple { 263 uint8_t ip_dst[IPV6_ADDR_LEN]; 264 uint8_t ip_src[IPV6_ADDR_LEN]; 265 uint16_t port_dst; 266 uint16_t port_src; 267 uint8_t proto; 268 } __attribute__((__packed__)); 269 270 struct ipv4_l3fwd_route { 271 struct ipv4_5tuple key; 272 uint8_t if_out; 273 }; 274 275 struct ipv6_l3fwd_route { 276 struct ipv6_5tuple key; 277 uint8_t if_out; 278 }; 279 280 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { 281 {{IPv4(100,10,0,1), IPv4(200,10,0,1), 101, 11, IPPROTO_TCP}, 0}, 282 {{IPv4(100,20,0,2), IPv4(200,20,0,2), 102, 12, IPPROTO_TCP}, 1}, 283 {{IPv4(100,30,0,3), IPv4(200,30,0,3), 103, 13, IPPROTO_TCP}, 2}, 284 {{IPv4(100,40,0,4), IPv4(200,40,0,4), 104, 14, IPPROTO_TCP}, 3}, 285 }; 286 287 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = { 288 { 289 { 290 {0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 291 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05}, 292 {0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 293 0x02, 0x1e, 0x67, 0xff, 0xfe, 0x0d, 0xb6, 0x0a}, 294 1, 10, IPPROTO_UDP 295 }, 4 296 }, 297 }; 298 299 typedef struct rte_hash lookup_struct_t; 300 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS]; 301 static lookup_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS]; 302 303 #define L3FWD_HASH_ENTRIES 1024 304 305 #define IPV4_L3FWD_NUM_ROUTES \ 306 (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0])) 307 308 #define IPV6_L3FWD_NUM_ROUTES \ 309 (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0])) 310 311 static uint16_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; 312 static uint16_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; 313 #endif 314 315 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 316 struct ipv4_l3fwd_route { 317 uint32_t ip; 318 uint8_t depth; 319 uint8_t if_out; 320 }; 321 322 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { 323 {IPv4(1,1,1,0), 24, 0}, 324 {IPv4(2,1,1,0), 24, 1}, 325 {IPv4(3,1,1,0), 24, 2}, 326 {IPv4(4,1,1,0), 24, 3}, 327 {IPv4(5,1,1,0), 24, 4}, 328 {IPv4(6,1,1,0), 24, 5}, 329 {IPv4(7,1,1,0), 24, 6}, 330 {IPv4(8,1,1,0), 24, 7}, 331 }; 332 333 #define IPV4_L3FWD_NUM_ROUTES \ 334 (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0])) 335 336 #define IPV4_L3FWD_LPM_MAX_RULES 1024 337 338 typedef struct rte_lpm lookup_struct_t; 339 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS]; 340 #endif 341 342 struct lcore_conf { 343 uint16_t n_rx_queue; 344 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 345 uint16_t n_tx_port; 346 uint16_t tx_port_id[RTE_MAX_ETHPORTS]; 347 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 348 struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS]; 349 lookup_struct_t * ipv4_lookup_struct; 350 lookup_struct_t * ipv6_lookup_struct; 351 } __rte_cache_aligned; 352 353 struct lcore_stats { 354 /* total sleep time in ms since last frequency scaling down */ 355 uint32_t sleep_time; 356 /* number of long sleep recently */ 357 uint32_t nb_long_sleep; 358 /* freq. scaling up trend */ 359 uint32_t trend; 360 /* total packet processed recently */ 361 uint64_t nb_rx_processed; 362 /* total iterations looped recently */ 363 uint64_t nb_iteration_looped; 364 uint32_t padding[9]; 365 } __rte_cache_aligned; 366 367 static struct lcore_conf lcore_conf[RTE_MAX_LCORE] __rte_cache_aligned; 368 static struct lcore_stats stats[RTE_MAX_LCORE] __rte_cache_aligned; 369 static struct rte_timer power_timers[RTE_MAX_LCORE]; 370 371 static inline uint32_t power_idle_heuristic(uint32_t zero_rx_packet_count); 372 static inline enum freq_scale_hint_t power_freq_scaleup_heuristic( \ 373 unsigned int lcore_id, uint16_t port_id, uint16_t queue_id); 374 375 /* exit signal handler */ 376 static void 377 signal_exit_now(int sigtype) 378 { 379 unsigned lcore_id; 380 unsigned int portid, nb_ports; 381 int ret; 382 383 if (sigtype == SIGINT) { 384 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 385 if (rte_lcore_is_enabled(lcore_id) == 0) 386 continue; 387 388 /* init power management library */ 389 ret = rte_power_exit(lcore_id); 390 if (ret) 391 rte_exit(EXIT_FAILURE, "Power management " 392 "library de-initialization failed on " 393 "core%u\n", lcore_id); 394 } 395 396 nb_ports = rte_eth_dev_count(); 397 for (portid = 0; portid < nb_ports; portid++) { 398 if ((enabled_port_mask & (1 << portid)) == 0) 399 continue; 400 401 rte_eth_dev_stop(portid); 402 rte_eth_dev_close(portid); 403 } 404 } 405 406 rte_exit(EXIT_SUCCESS, "User forced exit\n"); 407 } 408 409 /* Freqency scale down timer callback */ 410 static void 411 power_timer_cb(__attribute__((unused)) struct rte_timer *tim, 412 __attribute__((unused)) void *arg) 413 { 414 uint64_t hz; 415 float sleep_time_ratio; 416 unsigned lcore_id = rte_lcore_id(); 417 418 /* accumulate total execution time in us when callback is invoked */ 419 sleep_time_ratio = (float)(stats[lcore_id].sleep_time) / 420 (float)SCALING_PERIOD; 421 /** 422 * check whether need to scale down frequency a step if it sleep a lot. 423 */ 424 if (sleep_time_ratio >= SCALING_DOWN_TIME_RATIO_THRESHOLD) { 425 if (rte_power_freq_down) 426 rte_power_freq_down(lcore_id); 427 } 428 else if ( (unsigned)(stats[lcore_id].nb_rx_processed / 429 stats[lcore_id].nb_iteration_looped) < MAX_PKT_BURST) { 430 /** 431 * scale down a step if average packet per iteration less 432 * than expectation. 433 */ 434 if (rte_power_freq_down) 435 rte_power_freq_down(lcore_id); 436 } 437 438 /** 439 * initialize another timer according to current frequency to ensure 440 * timer interval is relatively fixed. 441 */ 442 hz = rte_get_timer_hz(); 443 rte_timer_reset(&power_timers[lcore_id], hz/TIMER_NUMBER_PER_SECOND, 444 SINGLE, lcore_id, power_timer_cb, NULL); 445 446 stats[lcore_id].nb_rx_processed = 0; 447 stats[lcore_id].nb_iteration_looped = 0; 448 449 stats[lcore_id].sleep_time = 0; 450 } 451 452 /* Enqueue a single packet, and send burst if queue is filled */ 453 static inline int 454 send_single_packet(struct rte_mbuf *m, uint8_t port) 455 { 456 uint32_t lcore_id; 457 struct lcore_conf *qconf; 458 459 lcore_id = rte_lcore_id(); 460 qconf = &lcore_conf[lcore_id]; 461 462 rte_eth_tx_buffer(port, qconf->tx_queue_id[port], 463 qconf->tx_buffer[port], m); 464 465 return 0; 466 } 467 468 #ifdef DO_RFC_1812_CHECKS 469 static inline int 470 is_valid_ipv4_pkt(struct ipv4_hdr *pkt, uint32_t link_len) 471 { 472 /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */ 473 /* 474 * 1. The packet length reported by the Link Layer must be large 475 * enough to hold the minimum length legal IP datagram (20 bytes). 476 */ 477 if (link_len < sizeof(struct ipv4_hdr)) 478 return -1; 479 480 /* 2. The IP checksum must be correct. */ 481 /* this is checked in H/W */ 482 483 /* 484 * 3. The IP version number must be 4. If the version number is not 4 485 * then the packet may be another version of IP, such as IPng or 486 * ST-II. 487 */ 488 if (((pkt->version_ihl) >> 4) != 4) 489 return -3; 490 /* 491 * 4. The IP header length field must be large enough to hold the 492 * minimum length legal IP datagram (20 bytes = 5 words). 493 */ 494 if ((pkt->version_ihl & 0xf) < 5) 495 return -4; 496 497 /* 498 * 5. The IP total length field must be large enough to hold the IP 499 * datagram header, whose length is specified in the IP header length 500 * field. 501 */ 502 if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct ipv4_hdr)) 503 return -5; 504 505 return 0; 506 } 507 #endif 508 509 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 510 static void 511 print_ipv4_key(struct ipv4_5tuple key) 512 { 513 printf("IP dst = %08x, IP src = %08x, port dst = %d, port src = %d, " 514 "proto = %d\n", (unsigned)key.ip_dst, (unsigned)key.ip_src, 515 key.port_dst, key.port_src, key.proto); 516 } 517 static void 518 print_ipv6_key(struct ipv6_5tuple key) 519 { 520 printf( "IP dst = " IPv6_BYTES_FMT ", IP src = " IPv6_BYTES_FMT ", " 521 "port dst = %d, port src = %d, proto = %d\n", 522 IPv6_BYTES(key.ip_dst), IPv6_BYTES(key.ip_src), 523 key.port_dst, key.port_src, key.proto); 524 } 525 526 static inline uint16_t 527 get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint16_t portid, 528 lookup_struct_t * ipv4_l3fwd_lookup_struct) 529 { 530 struct ipv4_5tuple key; 531 struct tcp_hdr *tcp; 532 struct udp_hdr *udp; 533 int ret = 0; 534 535 key.ip_dst = rte_be_to_cpu_32(ipv4_hdr->dst_addr); 536 key.ip_src = rte_be_to_cpu_32(ipv4_hdr->src_addr); 537 key.proto = ipv4_hdr->next_proto_id; 538 539 switch (ipv4_hdr->next_proto_id) { 540 case IPPROTO_TCP: 541 tcp = (struct tcp_hdr *)((unsigned char *)ipv4_hdr + 542 sizeof(struct ipv4_hdr)); 543 key.port_dst = rte_be_to_cpu_16(tcp->dst_port); 544 key.port_src = rte_be_to_cpu_16(tcp->src_port); 545 break; 546 547 case IPPROTO_UDP: 548 udp = (struct udp_hdr *)((unsigned char *)ipv4_hdr + 549 sizeof(struct ipv4_hdr)); 550 key.port_dst = rte_be_to_cpu_16(udp->dst_port); 551 key.port_src = rte_be_to_cpu_16(udp->src_port); 552 break; 553 554 default: 555 key.port_dst = 0; 556 key.port_src = 0; 557 break; 558 } 559 560 /* Find destination port */ 561 ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key); 562 return ((ret < 0) ? portid : ipv4_l3fwd_out_if[ret]); 563 } 564 565 static inline uint16_t 566 get_ipv6_dst_port(struct ipv6_hdr *ipv6_hdr, uint16_t portid, 567 lookup_struct_t *ipv6_l3fwd_lookup_struct) 568 { 569 struct ipv6_5tuple key; 570 struct tcp_hdr *tcp; 571 struct udp_hdr *udp; 572 int ret = 0; 573 574 memcpy(key.ip_dst, ipv6_hdr->dst_addr, IPV6_ADDR_LEN); 575 memcpy(key.ip_src, ipv6_hdr->src_addr, IPV6_ADDR_LEN); 576 577 key.proto = ipv6_hdr->proto; 578 579 switch (ipv6_hdr->proto) { 580 case IPPROTO_TCP: 581 tcp = (struct tcp_hdr *)((unsigned char *) ipv6_hdr + 582 sizeof(struct ipv6_hdr)); 583 key.port_dst = rte_be_to_cpu_16(tcp->dst_port); 584 key.port_src = rte_be_to_cpu_16(tcp->src_port); 585 break; 586 587 case IPPROTO_UDP: 588 udp = (struct udp_hdr *)((unsigned char *) ipv6_hdr + 589 sizeof(struct ipv6_hdr)); 590 key.port_dst = rte_be_to_cpu_16(udp->dst_port); 591 key.port_src = rte_be_to_cpu_16(udp->src_port); 592 break; 593 594 default: 595 key.port_dst = 0; 596 key.port_src = 0; 597 break; 598 } 599 600 /* Find destination port */ 601 ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key); 602 return ((ret < 0) ? portid : ipv6_l3fwd_out_if[ret]); 603 } 604 #endif 605 606 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 607 static inline uint16_t 608 get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint16_t portid, 609 lookup_struct_t *ipv4_l3fwd_lookup_struct) 610 { 611 uint32_t next_hop; 612 613 return ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, 614 rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop) == 0)? 615 next_hop : portid); 616 } 617 #endif 618 619 static inline void 620 parse_ptype_one(struct rte_mbuf *m) 621 { 622 struct ether_hdr *eth_hdr; 623 uint32_t packet_type = RTE_PTYPE_UNKNOWN; 624 uint16_t ether_type; 625 626 eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); 627 ether_type = eth_hdr->ether_type; 628 if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) 629 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 630 else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) 631 packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; 632 633 m->packet_type = packet_type; 634 } 635 636 static uint16_t 637 cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused, 638 struct rte_mbuf *pkts[], uint16_t nb_pkts, 639 uint16_t max_pkts __rte_unused, 640 void *user_param __rte_unused) 641 { 642 unsigned int i; 643 644 for (i = 0; i < nb_pkts; ++i) 645 parse_ptype_one(pkts[i]); 646 647 return nb_pkts; 648 } 649 650 static int 651 add_cb_parse_ptype(uint16_t portid, uint16_t queueid) 652 { 653 printf("Port %d: softly parse packet type info\n", portid); 654 if (rte_eth_add_rx_callback(portid, queueid, cb_parse_ptype, NULL)) 655 return 0; 656 657 printf("Failed to add rx callback: port=%d\n", portid); 658 return -1; 659 } 660 661 static inline void 662 l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, 663 struct lcore_conf *qconf) 664 { 665 struct ether_hdr *eth_hdr; 666 struct ipv4_hdr *ipv4_hdr; 667 void *d_addr_bytes; 668 uint16_t dst_port; 669 670 eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); 671 672 if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) { 673 /* Handle IPv4 headers.*/ 674 ipv4_hdr = 675 rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, 676 sizeof(struct ether_hdr)); 677 678 #ifdef DO_RFC_1812_CHECKS 679 /* Check to make sure the packet is valid (RFC1812) */ 680 if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len) < 0) { 681 rte_pktmbuf_free(m); 682 return; 683 } 684 #endif 685 686 dst_port = get_ipv4_dst_port(ipv4_hdr, portid, 687 qconf->ipv4_lookup_struct); 688 if (dst_port >= RTE_MAX_ETHPORTS || 689 (enabled_port_mask & 1 << dst_port) == 0) 690 dst_port = portid; 691 692 /* 02:00:00:00:00:xx */ 693 d_addr_bytes = ð_hdr->d_addr.addr_bytes[0]; 694 *((uint64_t *)d_addr_bytes) = 695 0x000000000002 + ((uint64_t)dst_port << 40); 696 697 #ifdef DO_RFC_1812_CHECKS 698 /* Update time to live and header checksum */ 699 --(ipv4_hdr->time_to_live); 700 ++(ipv4_hdr->hdr_checksum); 701 #endif 702 703 /* src addr */ 704 ether_addr_copy(&ports_eth_addr[dst_port], ð_hdr->s_addr); 705 706 send_single_packet(m, dst_port); 707 } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) { 708 /* Handle IPv6 headers.*/ 709 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 710 struct ipv6_hdr *ipv6_hdr; 711 712 ipv6_hdr = 713 rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *, 714 sizeof(struct ether_hdr)); 715 716 dst_port = get_ipv6_dst_port(ipv6_hdr, portid, 717 qconf->ipv6_lookup_struct); 718 719 if (dst_port >= RTE_MAX_ETHPORTS || 720 (enabled_port_mask & 1 << dst_port) == 0) 721 dst_port = portid; 722 723 /* 02:00:00:00:00:xx */ 724 d_addr_bytes = ð_hdr->d_addr.addr_bytes[0]; 725 *((uint64_t *)d_addr_bytes) = 726 0x000000000002 + ((uint64_t)dst_port << 40); 727 728 /* src addr */ 729 ether_addr_copy(&ports_eth_addr[dst_port], ð_hdr->s_addr); 730 731 send_single_packet(m, dst_port); 732 #else 733 /* We don't currently handle IPv6 packets in LPM mode. */ 734 rte_pktmbuf_free(m); 735 #endif 736 } else 737 rte_pktmbuf_free(m); 738 739 } 740 741 #define MINIMUM_SLEEP_TIME 1 742 #define SUSPEND_THRESHOLD 300 743 744 static inline uint32_t 745 power_idle_heuristic(uint32_t zero_rx_packet_count) 746 { 747 /* If zero count is less than 100, sleep 1us */ 748 if (zero_rx_packet_count < SUSPEND_THRESHOLD) 749 return MINIMUM_SLEEP_TIME; 750 /* If zero count is less than 1000, sleep 100 us which is the 751 minimum latency switching from C3/C6 to C0 752 */ 753 else 754 return SUSPEND_THRESHOLD; 755 756 return 0; 757 } 758 759 static inline enum freq_scale_hint_t 760 power_freq_scaleup_heuristic(unsigned lcore_id, 761 uint16_t port_id, 762 uint16_t queue_id) 763 { 764 /** 765 * HW Rx queue size is 128 by default, Rx burst read at maximum 32 entries 766 * per iteration 767 */ 768 #define FREQ_GEAR1_RX_PACKET_THRESHOLD MAX_PKT_BURST 769 #define FREQ_GEAR2_RX_PACKET_THRESHOLD (MAX_PKT_BURST*2) 770 #define FREQ_GEAR3_RX_PACKET_THRESHOLD (MAX_PKT_BURST*3) 771 #define FREQ_UP_TREND1_ACC 1 772 #define FREQ_UP_TREND2_ACC 100 773 #define FREQ_UP_THRESHOLD 10000 774 775 if (likely(rte_eth_rx_descriptor_done(port_id, queue_id, 776 FREQ_GEAR3_RX_PACKET_THRESHOLD) > 0)) { 777 stats[lcore_id].trend = 0; 778 return FREQ_HIGHEST; 779 } else if (likely(rte_eth_rx_descriptor_done(port_id, queue_id, 780 FREQ_GEAR2_RX_PACKET_THRESHOLD) > 0)) 781 stats[lcore_id].trend += FREQ_UP_TREND2_ACC; 782 else if (likely(rte_eth_rx_descriptor_done(port_id, queue_id, 783 FREQ_GEAR1_RX_PACKET_THRESHOLD) > 0)) 784 stats[lcore_id].trend += FREQ_UP_TREND1_ACC; 785 786 if (likely(stats[lcore_id].trend > FREQ_UP_THRESHOLD)) { 787 stats[lcore_id].trend = 0; 788 return FREQ_HIGHER; 789 } 790 791 return FREQ_CURRENT; 792 } 793 794 /** 795 * force polling thread sleep until one-shot rx interrupt triggers 796 * @param port_id 797 * Port id. 798 * @param queue_id 799 * Rx queue id. 800 * @return 801 * 0 on success 802 */ 803 static int 804 sleep_until_rx_interrupt(int num) 805 { 806 struct rte_epoll_event event[num]; 807 int n, i; 808 uint16_t port_id; 809 uint8_t queue_id; 810 void *data; 811 812 RTE_LOG(INFO, L3FWD_POWER, 813 "lcore %u sleeps until interrupt triggers\n", 814 rte_lcore_id()); 815 816 n = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, num, -1); 817 for (i = 0; i < n; i++) { 818 data = event[i].epdata.data; 819 port_id = ((uintptr_t)data) >> CHAR_BIT; 820 queue_id = ((uintptr_t)data) & 821 RTE_LEN2MASK(CHAR_BIT, uint8_t); 822 rte_eth_dev_rx_intr_disable(port_id, queue_id); 823 RTE_LOG(INFO, L3FWD_POWER, 824 "lcore %u is waked up from rx interrupt on" 825 " port %d queue %d\n", 826 rte_lcore_id(), port_id, queue_id); 827 } 828 829 return 0; 830 } 831 832 static void turn_on_intr(struct lcore_conf *qconf) 833 { 834 int i; 835 struct lcore_rx_queue *rx_queue; 836 uint8_t queue_id; 837 uint16_t port_id; 838 839 for (i = 0; i < qconf->n_rx_queue; ++i) { 840 rx_queue = &(qconf->rx_queue_list[i]); 841 port_id = rx_queue->port_id; 842 queue_id = rx_queue->queue_id; 843 844 rte_spinlock_lock(&(locks[port_id])); 845 rte_eth_dev_rx_intr_enable(port_id, queue_id); 846 rte_spinlock_unlock(&(locks[port_id])); 847 } 848 } 849 850 static int event_register(struct lcore_conf *qconf) 851 { 852 struct lcore_rx_queue *rx_queue; 853 uint8_t queueid; 854 uint16_t portid; 855 uint32_t data; 856 int ret; 857 int i; 858 859 for (i = 0; i < qconf->n_rx_queue; ++i) { 860 rx_queue = &(qconf->rx_queue_list[i]); 861 portid = rx_queue->port_id; 862 queueid = rx_queue->queue_id; 863 data = portid << CHAR_BIT | queueid; 864 865 ret = rte_eth_dev_rx_intr_ctl_q(portid, queueid, 866 RTE_EPOLL_PER_THREAD, 867 RTE_INTR_EVENT_ADD, 868 (void *)((uintptr_t)data)); 869 if (ret) 870 return ret; 871 } 872 873 return 0; 874 } 875 876 /* main processing loop */ 877 static int 878 main_loop(__attribute__((unused)) void *dummy) 879 { 880 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 881 unsigned lcore_id; 882 uint64_t prev_tsc, diff_tsc, cur_tsc; 883 uint64_t prev_tsc_power = 0, cur_tsc_power, diff_tsc_power; 884 int i, j, nb_rx; 885 uint8_t queueid; 886 uint16_t portid; 887 struct lcore_conf *qconf; 888 struct lcore_rx_queue *rx_queue; 889 enum freq_scale_hint_t lcore_scaleup_hint; 890 uint32_t lcore_rx_idle_count = 0; 891 uint32_t lcore_idle_hint = 0; 892 int intr_en = 0; 893 894 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; 895 896 prev_tsc = 0; 897 898 lcore_id = rte_lcore_id(); 899 qconf = &lcore_conf[lcore_id]; 900 901 if (qconf->n_rx_queue == 0) { 902 RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", lcore_id); 903 return 0; 904 } 905 906 RTE_LOG(INFO, L3FWD_POWER, "entering main loop on lcore %u\n", lcore_id); 907 908 for (i = 0; i < qconf->n_rx_queue; i++) { 909 portid = qconf->rx_queue_list[i].port_id; 910 queueid = qconf->rx_queue_list[i].queue_id; 911 RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u " 912 "rxqueueid=%hhu\n", lcore_id, portid, queueid); 913 } 914 915 /* add into event wait list */ 916 if (event_register(qconf) == 0) 917 intr_en = 1; 918 else 919 RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n"); 920 921 while (1) { 922 stats[lcore_id].nb_iteration_looped++; 923 924 cur_tsc = rte_rdtsc(); 925 cur_tsc_power = cur_tsc; 926 927 /* 928 * TX burst queue drain 929 */ 930 diff_tsc = cur_tsc - prev_tsc; 931 if (unlikely(diff_tsc > drain_tsc)) { 932 for (i = 0; i < qconf->n_tx_port; ++i) { 933 portid = qconf->tx_port_id[i]; 934 rte_eth_tx_buffer_flush(portid, 935 qconf->tx_queue_id[portid], 936 qconf->tx_buffer[portid]); 937 } 938 prev_tsc = cur_tsc; 939 } 940 941 diff_tsc_power = cur_tsc_power - prev_tsc_power; 942 if (diff_tsc_power > TIMER_RESOLUTION_CYCLES) { 943 rte_timer_manage(); 944 prev_tsc_power = cur_tsc_power; 945 } 946 947 start_rx: 948 /* 949 * Read packet from RX queues 950 */ 951 lcore_scaleup_hint = FREQ_CURRENT; 952 lcore_rx_idle_count = 0; 953 for (i = 0; i < qconf->n_rx_queue; ++i) { 954 rx_queue = &(qconf->rx_queue_list[i]); 955 rx_queue->idle_hint = 0; 956 portid = rx_queue->port_id; 957 queueid = rx_queue->queue_id; 958 959 nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, 960 MAX_PKT_BURST); 961 962 stats[lcore_id].nb_rx_processed += nb_rx; 963 if (unlikely(nb_rx == 0)) { 964 /** 965 * no packet received from rx queue, try to 966 * sleep for a while forcing CPU enter deeper 967 * C states. 968 */ 969 rx_queue->zero_rx_packet_count++; 970 971 if (rx_queue->zero_rx_packet_count <= 972 MIN_ZERO_POLL_COUNT) 973 continue; 974 975 rx_queue->idle_hint = power_idle_heuristic(\ 976 rx_queue->zero_rx_packet_count); 977 lcore_rx_idle_count++; 978 } else { 979 rx_queue->zero_rx_packet_count = 0; 980 981 /** 982 * do not scale up frequency immediately as 983 * user to kernel space communication is costly 984 * which might impact packet I/O for received 985 * packets. 986 */ 987 rx_queue->freq_up_hint = 988 power_freq_scaleup_heuristic(lcore_id, 989 portid, queueid); 990 } 991 992 /* Prefetch first packets */ 993 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 994 rte_prefetch0(rte_pktmbuf_mtod( 995 pkts_burst[j], void *)); 996 } 997 998 /* Prefetch and forward already prefetched packets */ 999 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1000 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1001 j + PREFETCH_OFFSET], void *)); 1002 l3fwd_simple_forward(pkts_burst[j], portid, 1003 qconf); 1004 } 1005 1006 /* Forward remaining prefetched packets */ 1007 for (; j < nb_rx; j++) { 1008 l3fwd_simple_forward(pkts_burst[j], portid, 1009 qconf); 1010 } 1011 } 1012 1013 if (likely(lcore_rx_idle_count != qconf->n_rx_queue)) { 1014 for (i = 1, lcore_scaleup_hint = 1015 qconf->rx_queue_list[0].freq_up_hint; 1016 i < qconf->n_rx_queue; ++i) { 1017 rx_queue = &(qconf->rx_queue_list[i]); 1018 if (rx_queue->freq_up_hint > 1019 lcore_scaleup_hint) 1020 lcore_scaleup_hint = 1021 rx_queue->freq_up_hint; 1022 } 1023 1024 if (lcore_scaleup_hint == FREQ_HIGHEST) { 1025 if (rte_power_freq_max) 1026 rte_power_freq_max(lcore_id); 1027 } else if (lcore_scaleup_hint == FREQ_HIGHER) { 1028 if (rte_power_freq_up) 1029 rte_power_freq_up(lcore_id); 1030 } 1031 } else { 1032 /** 1033 * All Rx queues empty in recent consecutive polls, 1034 * sleep in a conservative manner, meaning sleep as 1035 * less as possible. 1036 */ 1037 for (i = 1, lcore_idle_hint = 1038 qconf->rx_queue_list[0].idle_hint; 1039 i < qconf->n_rx_queue; ++i) { 1040 rx_queue = &(qconf->rx_queue_list[i]); 1041 if (rx_queue->idle_hint < lcore_idle_hint) 1042 lcore_idle_hint = rx_queue->idle_hint; 1043 } 1044 1045 if (lcore_idle_hint < SUSPEND_THRESHOLD) 1046 /** 1047 * execute "pause" instruction to avoid context 1048 * switch which generally take hundred of 1049 * microseconds for short sleep. 1050 */ 1051 rte_delay_us(lcore_idle_hint); 1052 else { 1053 /* suspend until rx interrupt trigges */ 1054 if (intr_en) { 1055 turn_on_intr(qconf); 1056 sleep_until_rx_interrupt( 1057 qconf->n_rx_queue); 1058 } 1059 /* start receiving packets immediately */ 1060 goto start_rx; 1061 } 1062 stats[lcore_id].sleep_time += lcore_idle_hint; 1063 } 1064 } 1065 } 1066 1067 static int 1068 check_lcore_params(void) 1069 { 1070 uint8_t queue, lcore; 1071 uint16_t i; 1072 int socketid; 1073 1074 for (i = 0; i < nb_lcore_params; ++i) { 1075 queue = lcore_params[i].queue_id; 1076 if (queue >= MAX_RX_QUEUE_PER_PORT) { 1077 printf("invalid queue number: %hhu\n", queue); 1078 return -1; 1079 } 1080 lcore = lcore_params[i].lcore_id; 1081 if (!rte_lcore_is_enabled(lcore)) { 1082 printf("error: lcore %hhu is not enabled in lcore " 1083 "mask\n", lcore); 1084 return -1; 1085 } 1086 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) && 1087 (numa_on == 0)) { 1088 printf("warning: lcore %hhu is on socket %d with numa " 1089 "off\n", lcore, socketid); 1090 } 1091 } 1092 return 0; 1093 } 1094 1095 static int 1096 check_port_config(const unsigned nb_ports) 1097 { 1098 unsigned portid; 1099 uint16_t i; 1100 1101 for (i = 0; i < nb_lcore_params; ++i) { 1102 portid = lcore_params[i].port_id; 1103 if ((enabled_port_mask & (1 << portid)) == 0) { 1104 printf("port %u is not enabled in port mask\n", 1105 portid); 1106 return -1; 1107 } 1108 if (portid >= nb_ports) { 1109 printf("port %u is not present on the board\n", 1110 portid); 1111 return -1; 1112 } 1113 } 1114 return 0; 1115 } 1116 1117 static uint8_t 1118 get_port_n_rx_queues(const uint8_t port) 1119 { 1120 int queue = -1; 1121 uint16_t i; 1122 1123 for (i = 0; i < nb_lcore_params; ++i) { 1124 if (lcore_params[i].port_id == port && 1125 lcore_params[i].queue_id > queue) 1126 queue = lcore_params[i].queue_id; 1127 } 1128 return (uint8_t)(++queue); 1129 } 1130 1131 static int 1132 init_lcore_rx_queues(void) 1133 { 1134 uint16_t i, nb_rx_queue; 1135 uint8_t lcore; 1136 1137 for (i = 0; i < nb_lcore_params; ++i) { 1138 lcore = lcore_params[i].lcore_id; 1139 nb_rx_queue = lcore_conf[lcore].n_rx_queue; 1140 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1141 printf("error: too many queues (%u) for lcore: %u\n", 1142 (unsigned)nb_rx_queue + 1, (unsigned)lcore); 1143 return -1; 1144 } else { 1145 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1146 lcore_params[i].port_id; 1147 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1148 lcore_params[i].queue_id; 1149 lcore_conf[lcore].n_rx_queue++; 1150 } 1151 } 1152 return 0; 1153 } 1154 1155 /* display usage */ 1156 static void 1157 print_usage(const char *prgname) 1158 { 1159 printf ("%s [EAL options] -- -p PORTMASK -P" 1160 " [--config (port,queue,lcore)[,(port,queue,lcore]]" 1161 " [--enable-jumbo [--max-pkt-len PKTLEN]]\n" 1162 " -p PORTMASK: hexadecimal bitmask of ports to configure\n" 1163 " -P : enable promiscuous mode\n" 1164 " --config (port,queue,lcore): rx queues configuration\n" 1165 " --no-numa: optional, disable numa awareness\n" 1166 " --enable-jumbo: enable jumbo frame" 1167 " which max packet len is PKTLEN in decimal (64-9600)\n" 1168 " --parse-ptype: parse packet type by software\n", 1169 prgname); 1170 } 1171 1172 static int parse_max_pkt_len(const char *pktlen) 1173 { 1174 char *end = NULL; 1175 unsigned long len; 1176 1177 /* parse decimal string */ 1178 len = strtoul(pktlen, &end, 10); 1179 if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) 1180 return -1; 1181 1182 if (len == 0) 1183 return -1; 1184 1185 return len; 1186 } 1187 1188 static int 1189 parse_portmask(const char *portmask) 1190 { 1191 char *end = NULL; 1192 unsigned long pm; 1193 1194 /* parse hexadecimal string */ 1195 pm = strtoul(portmask, &end, 16); 1196 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1197 return -1; 1198 1199 if (pm == 0) 1200 return -1; 1201 1202 return pm; 1203 } 1204 1205 static int 1206 parse_config(const char *q_arg) 1207 { 1208 char s[256]; 1209 const char *p, *p0 = q_arg; 1210 char *end; 1211 enum fieldnames { 1212 FLD_PORT = 0, 1213 FLD_QUEUE, 1214 FLD_LCORE, 1215 _NUM_FLD 1216 }; 1217 unsigned long int_fld[_NUM_FLD]; 1218 char *str_fld[_NUM_FLD]; 1219 int i; 1220 unsigned size; 1221 1222 nb_lcore_params = 0; 1223 1224 while ((p = strchr(p0,'(')) != NULL) { 1225 ++p; 1226 if((p0 = strchr(p,')')) == NULL) 1227 return -1; 1228 1229 size = p0 - p; 1230 if(size >= sizeof(s)) 1231 return -1; 1232 1233 snprintf(s, sizeof(s), "%.*s", size, p); 1234 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1235 _NUM_FLD) 1236 return -1; 1237 for (i = 0; i < _NUM_FLD; i++){ 1238 errno = 0; 1239 int_fld[i] = strtoul(str_fld[i], &end, 0); 1240 if (errno != 0 || end == str_fld[i] || int_fld[i] > 1241 255) 1242 return -1; 1243 } 1244 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1245 printf("exceeded max number of lcore params: %hu\n", 1246 nb_lcore_params); 1247 return -1; 1248 } 1249 lcore_params_array[nb_lcore_params].port_id = 1250 (uint8_t)int_fld[FLD_PORT]; 1251 lcore_params_array[nb_lcore_params].queue_id = 1252 (uint8_t)int_fld[FLD_QUEUE]; 1253 lcore_params_array[nb_lcore_params].lcore_id = 1254 (uint8_t)int_fld[FLD_LCORE]; 1255 ++nb_lcore_params; 1256 } 1257 lcore_params = lcore_params_array; 1258 1259 return 0; 1260 } 1261 1262 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype" 1263 1264 /* Parse the argument given in the command line of the application */ 1265 static int 1266 parse_args(int argc, char **argv) 1267 { 1268 int opt, ret; 1269 char **argvopt; 1270 int option_index; 1271 char *prgname = argv[0]; 1272 static struct option lgopts[] = { 1273 {"config", 1, 0, 0}, 1274 {"no-numa", 0, 0, 0}, 1275 {"enable-jumbo", 0, 0, 0}, 1276 {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0}, 1277 {NULL, 0, 0, 0} 1278 }; 1279 1280 argvopt = argv; 1281 1282 while ((opt = getopt_long(argc, argvopt, "p:P", 1283 lgopts, &option_index)) != EOF) { 1284 1285 switch (opt) { 1286 /* portmask */ 1287 case 'p': 1288 enabled_port_mask = parse_portmask(optarg); 1289 if (enabled_port_mask == 0) { 1290 printf("invalid portmask\n"); 1291 print_usage(prgname); 1292 return -1; 1293 } 1294 break; 1295 case 'P': 1296 printf("Promiscuous mode selected\n"); 1297 promiscuous_on = 1; 1298 break; 1299 1300 /* long options */ 1301 case 0: 1302 if (!strncmp(lgopts[option_index].name, "config", 6)) { 1303 ret = parse_config(optarg); 1304 if (ret) { 1305 printf("invalid config\n"); 1306 print_usage(prgname); 1307 return -1; 1308 } 1309 } 1310 1311 if (!strncmp(lgopts[option_index].name, 1312 "no-numa", 7)) { 1313 printf("numa is disabled \n"); 1314 numa_on = 0; 1315 } 1316 1317 if (!strncmp(lgopts[option_index].name, 1318 "enable-jumbo", 12)) { 1319 struct option lenopts = 1320 {"max-pkt-len", required_argument, \ 1321 0, 0}; 1322 1323 printf("jumbo frame is enabled \n"); 1324 port_conf.rxmode.jumbo_frame = 1; 1325 1326 /** 1327 * if no max-pkt-len set, use the default value 1328 * ETHER_MAX_LEN 1329 */ 1330 if (0 == getopt_long(argc, argvopt, "", 1331 &lenopts, &option_index)) { 1332 ret = parse_max_pkt_len(optarg); 1333 if ((ret < 64) || 1334 (ret > MAX_JUMBO_PKT_LEN)){ 1335 printf("invalid packet " 1336 "length\n"); 1337 print_usage(prgname); 1338 return -1; 1339 } 1340 port_conf.rxmode.max_rx_pkt_len = ret; 1341 } 1342 printf("set jumbo frame " 1343 "max packet length to %u\n", 1344 (unsigned int)port_conf.rxmode.max_rx_pkt_len); 1345 } 1346 1347 if (!strncmp(lgopts[option_index].name, 1348 CMD_LINE_OPT_PARSE_PTYPE, 1349 sizeof(CMD_LINE_OPT_PARSE_PTYPE))) { 1350 printf("soft parse-ptype is enabled\n"); 1351 parse_ptype = 1; 1352 } 1353 1354 break; 1355 1356 default: 1357 print_usage(prgname); 1358 return -1; 1359 } 1360 } 1361 1362 if (optind >= 0) 1363 argv[optind-1] = prgname; 1364 1365 ret = optind-1; 1366 optind = 1; /* reset getopt lib */ 1367 return ret; 1368 } 1369 1370 static void 1371 print_ethaddr(const char *name, const struct ether_addr *eth_addr) 1372 { 1373 char buf[ETHER_ADDR_FMT_SIZE]; 1374 ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); 1375 printf("%s%s", name, buf); 1376 } 1377 1378 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1379 static void 1380 setup_hash(int socketid) 1381 { 1382 struct rte_hash_parameters ipv4_l3fwd_hash_params = { 1383 .name = NULL, 1384 .entries = L3FWD_HASH_ENTRIES, 1385 .key_len = sizeof(struct ipv4_5tuple), 1386 .hash_func = DEFAULT_HASH_FUNC, 1387 .hash_func_init_val = 0, 1388 }; 1389 1390 struct rte_hash_parameters ipv6_l3fwd_hash_params = { 1391 .name = NULL, 1392 .entries = L3FWD_HASH_ENTRIES, 1393 .key_len = sizeof(struct ipv6_5tuple), 1394 .hash_func = DEFAULT_HASH_FUNC, 1395 .hash_func_init_val = 0, 1396 }; 1397 1398 unsigned i; 1399 int ret; 1400 char s[64]; 1401 1402 /* create ipv4 hash */ 1403 snprintf(s, sizeof(s), "ipv4_l3fwd_hash_%d", socketid); 1404 ipv4_l3fwd_hash_params.name = s; 1405 ipv4_l3fwd_hash_params.socket_id = socketid; 1406 ipv4_l3fwd_lookup_struct[socketid] = 1407 rte_hash_create(&ipv4_l3fwd_hash_params); 1408 if (ipv4_l3fwd_lookup_struct[socketid] == NULL) 1409 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on " 1410 "socket %d\n", socketid); 1411 1412 /* create ipv6 hash */ 1413 snprintf(s, sizeof(s), "ipv6_l3fwd_hash_%d", socketid); 1414 ipv6_l3fwd_hash_params.name = s; 1415 ipv6_l3fwd_hash_params.socket_id = socketid; 1416 ipv6_l3fwd_lookup_struct[socketid] = 1417 rte_hash_create(&ipv6_l3fwd_hash_params); 1418 if (ipv6_l3fwd_lookup_struct[socketid] == NULL) 1419 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on " 1420 "socket %d\n", socketid); 1421 1422 1423 /* populate the ipv4 hash */ 1424 for (i = 0; i < IPV4_L3FWD_NUM_ROUTES; i++) { 1425 ret = rte_hash_add_key (ipv4_l3fwd_lookup_struct[socketid], 1426 (void *) &ipv4_l3fwd_route_array[i].key); 1427 if (ret < 0) { 1428 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the" 1429 "l3fwd hash on socket %d\n", i, socketid); 1430 } 1431 ipv4_l3fwd_out_if[ret] = ipv4_l3fwd_route_array[i].if_out; 1432 printf("Hash: Adding key\n"); 1433 print_ipv4_key(ipv4_l3fwd_route_array[i].key); 1434 } 1435 1436 /* populate the ipv6 hash */ 1437 for (i = 0; i < IPV6_L3FWD_NUM_ROUTES; i++) { 1438 ret = rte_hash_add_key (ipv6_l3fwd_lookup_struct[socketid], 1439 (void *) &ipv6_l3fwd_route_array[i].key); 1440 if (ret < 0) { 1441 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the" 1442 "l3fwd hash on socket %d\n", i, socketid); 1443 } 1444 ipv6_l3fwd_out_if[ret] = ipv6_l3fwd_route_array[i].if_out; 1445 printf("Hash: Adding key\n"); 1446 print_ipv6_key(ipv6_l3fwd_route_array[i].key); 1447 } 1448 } 1449 #endif 1450 1451 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 1452 static void 1453 setup_lpm(int socketid) 1454 { 1455 unsigned i; 1456 int ret; 1457 char s[64]; 1458 1459 /* create the LPM table */ 1460 struct rte_lpm_config lpm_ipv4_config; 1461 1462 lpm_ipv4_config.max_rules = IPV4_L3FWD_LPM_MAX_RULES; 1463 lpm_ipv4_config.number_tbl8s = 256; 1464 lpm_ipv4_config.flags = 0; 1465 1466 snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socketid); 1467 ipv4_l3fwd_lookup_struct[socketid] = 1468 rte_lpm_create(s, socketid, &lpm_ipv4_config); 1469 if (ipv4_l3fwd_lookup_struct[socketid] == NULL) 1470 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table" 1471 " on socket %d\n", socketid); 1472 1473 /* populate the LPM table */ 1474 for (i = 0; i < IPV4_L3FWD_NUM_ROUTES; i++) { 1475 ret = rte_lpm_add(ipv4_l3fwd_lookup_struct[socketid], 1476 ipv4_l3fwd_route_array[i].ip, 1477 ipv4_l3fwd_route_array[i].depth, 1478 ipv4_l3fwd_route_array[i].if_out); 1479 1480 if (ret < 0) { 1481 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the " 1482 "l3fwd LPM table on socket %d\n", 1483 i, socketid); 1484 } 1485 1486 printf("LPM: Adding route 0x%08x / %d (%d)\n", 1487 (unsigned)ipv4_l3fwd_route_array[i].ip, 1488 ipv4_l3fwd_route_array[i].depth, 1489 ipv4_l3fwd_route_array[i].if_out); 1490 } 1491 } 1492 #endif 1493 1494 static int 1495 init_mem(unsigned nb_mbuf) 1496 { 1497 struct lcore_conf *qconf; 1498 int socketid; 1499 unsigned lcore_id; 1500 char s[64]; 1501 1502 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1503 if (rte_lcore_is_enabled(lcore_id) == 0) 1504 continue; 1505 1506 if (numa_on) 1507 socketid = rte_lcore_to_socket_id(lcore_id); 1508 else 1509 socketid = 0; 1510 1511 if (socketid >= NB_SOCKETS) { 1512 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is " 1513 "out of range %d\n", socketid, 1514 lcore_id, NB_SOCKETS); 1515 } 1516 if (pktmbuf_pool[socketid] == NULL) { 1517 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 1518 pktmbuf_pool[socketid] = 1519 rte_pktmbuf_pool_create(s, nb_mbuf, 1520 MEMPOOL_CACHE_SIZE, 0, 1521 RTE_MBUF_DEFAULT_BUF_SIZE, 1522 socketid); 1523 if (pktmbuf_pool[socketid] == NULL) 1524 rte_exit(EXIT_FAILURE, 1525 "Cannot init mbuf pool on socket %d\n", 1526 socketid); 1527 else 1528 printf("Allocated mbuf pool on socket %d\n", 1529 socketid); 1530 1531 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 1532 setup_lpm(socketid); 1533 #else 1534 setup_hash(socketid); 1535 #endif 1536 } 1537 qconf = &lcore_conf[lcore_id]; 1538 qconf->ipv4_lookup_struct = ipv4_l3fwd_lookup_struct[socketid]; 1539 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1540 qconf->ipv6_lookup_struct = ipv6_l3fwd_lookup_struct[socketid]; 1541 #endif 1542 } 1543 return 0; 1544 } 1545 1546 /* Check the link status of all ports in up to 9s, and print them finally */ 1547 static void 1548 check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) 1549 { 1550 #define CHECK_INTERVAL 100 /* 100ms */ 1551 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1552 uint8_t count, all_ports_up, print_flag = 0; 1553 uint16_t portid; 1554 struct rte_eth_link link; 1555 1556 printf("\nChecking link status"); 1557 fflush(stdout); 1558 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1559 all_ports_up = 1; 1560 for (portid = 0; portid < port_num; portid++) { 1561 if ((port_mask & (1 << portid)) == 0) 1562 continue; 1563 memset(&link, 0, sizeof(link)); 1564 rte_eth_link_get_nowait(portid, &link); 1565 /* print link status if flag set */ 1566 if (print_flag == 1) { 1567 if (link.link_status) 1568 printf("Port %d Link Up - speed %u " 1569 "Mbps - %s\n", (uint8_t)portid, 1570 (unsigned)link.link_speed, 1571 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1572 ("full-duplex") : ("half-duplex\n")); 1573 else 1574 printf("Port %d Link Down\n", 1575 (uint8_t)portid); 1576 continue; 1577 } 1578 /* clear all_ports_up flag if any link down */ 1579 if (link.link_status == ETH_LINK_DOWN) { 1580 all_ports_up = 0; 1581 break; 1582 } 1583 } 1584 /* after finally printing all link status, get out */ 1585 if (print_flag == 1) 1586 break; 1587 1588 if (all_ports_up == 0) { 1589 printf("."); 1590 fflush(stdout); 1591 rte_delay_ms(CHECK_INTERVAL); 1592 } 1593 1594 /* set the print_flag if all ports up or timeout */ 1595 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1596 print_flag = 1; 1597 printf("done\n"); 1598 } 1599 } 1600 } 1601 1602 static int check_ptype(uint8_t portid) 1603 { 1604 int i, ret; 1605 int ptype_l3_ipv4 = 0; 1606 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1607 int ptype_l3_ipv6 = 0; 1608 #endif 1609 uint32_t ptype_mask = RTE_PTYPE_L3_MASK; 1610 1611 ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, NULL, 0); 1612 if (ret <= 0) 1613 return 0; 1614 1615 uint32_t ptypes[ret]; 1616 1617 ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, ptypes, ret); 1618 for (i = 0; i < ret; ++i) { 1619 if (ptypes[i] & RTE_PTYPE_L3_IPV4) 1620 ptype_l3_ipv4 = 1; 1621 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1622 if (ptypes[i] & RTE_PTYPE_L3_IPV6) 1623 ptype_l3_ipv6 = 1; 1624 #endif 1625 } 1626 1627 if (ptype_l3_ipv4 == 0) 1628 printf("port %d cannot parse RTE_PTYPE_L3_IPV4\n", portid); 1629 1630 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1631 if (ptype_l3_ipv6 == 0) 1632 printf("port %d cannot parse RTE_PTYPE_L3_IPV6\n", portid); 1633 #endif 1634 1635 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 1636 if (ptype_l3_ipv4) 1637 #else /* APP_LOOKUP_EXACT_MATCH */ 1638 if (ptype_l3_ipv4 && ptype_l3_ipv6) 1639 #endif 1640 return 1; 1641 1642 return 0; 1643 1644 } 1645 1646 int 1647 main(int argc, char **argv) 1648 { 1649 struct lcore_conf *qconf; 1650 struct rte_eth_dev_info dev_info; 1651 struct rte_eth_txconf *txconf; 1652 int ret; 1653 unsigned nb_ports; 1654 uint16_t queueid; 1655 unsigned lcore_id; 1656 uint64_t hz; 1657 uint32_t n_tx_queue, nb_lcores; 1658 uint32_t dev_rxq_num, dev_txq_num; 1659 uint8_t nb_rx_queue, queue, socketid; 1660 uint16_t portid; 1661 uint16_t org_rxq_intr = port_conf.intr_conf.rxq; 1662 1663 /* catch SIGINT and restore cpufreq governor to ondemand */ 1664 signal(SIGINT, signal_exit_now); 1665 1666 /* init EAL */ 1667 ret = rte_eal_init(argc, argv); 1668 if (ret < 0) 1669 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 1670 argc -= ret; 1671 argv += ret; 1672 1673 /* init RTE timer library to be used late */ 1674 rte_timer_subsystem_init(); 1675 1676 /* parse application arguments (after the EAL ones) */ 1677 ret = parse_args(argc, argv); 1678 if (ret < 0) 1679 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n"); 1680 1681 if (check_lcore_params() < 0) 1682 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n"); 1683 1684 ret = init_lcore_rx_queues(); 1685 if (ret < 0) 1686 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 1687 1688 nb_ports = rte_eth_dev_count(); 1689 1690 if (check_port_config(nb_ports) < 0) 1691 rte_exit(EXIT_FAILURE, "check_port_config failed\n"); 1692 1693 nb_lcores = rte_lcore_count(); 1694 1695 /* initialize all ports */ 1696 for (portid = 0; portid < nb_ports; portid++) { 1697 /* skip ports that are not enabled */ 1698 if ((enabled_port_mask & (1 << portid)) == 0) { 1699 printf("\nSkipping disabled port %d\n", portid); 1700 continue; 1701 } 1702 1703 /* init port */ 1704 printf("Initializing port %d ... ", portid ); 1705 fflush(stdout); 1706 1707 rte_eth_dev_info_get(portid, &dev_info); 1708 dev_rxq_num = dev_info.max_rx_queues; 1709 dev_txq_num = dev_info.max_tx_queues; 1710 1711 nb_rx_queue = get_port_n_rx_queues(portid); 1712 if (nb_rx_queue > dev_rxq_num) 1713 rte_exit(EXIT_FAILURE, 1714 "Cannot configure not existed rxq: " 1715 "port=%d\n", portid); 1716 1717 n_tx_queue = nb_lcores; 1718 if (n_tx_queue > dev_txq_num) 1719 n_tx_queue = dev_txq_num; 1720 printf("Creating queues: nb_rxq=%d nb_txq=%u... ", 1721 nb_rx_queue, (unsigned)n_tx_queue ); 1722 /* If number of Rx queue is 0, no need to enable Rx interrupt */ 1723 if (nb_rx_queue == 0) 1724 port_conf.intr_conf.rxq = 0; 1725 ret = rte_eth_dev_configure(portid, nb_rx_queue, 1726 (uint16_t)n_tx_queue, &port_conf); 1727 /* Revert to original value */ 1728 port_conf.intr_conf.rxq = org_rxq_intr; 1729 if (ret < 0) 1730 rte_exit(EXIT_FAILURE, "Cannot configure device: " 1731 "err=%d, port=%d\n", ret, portid); 1732 1733 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 1734 &nb_txd); 1735 if (ret < 0) 1736 rte_exit(EXIT_FAILURE, 1737 "Cannot adjust number of descriptors: err=%d, port=%d\n", 1738 ret, portid); 1739 1740 rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 1741 print_ethaddr(" Address:", &ports_eth_addr[portid]); 1742 printf(", "); 1743 1744 /* init memory */ 1745 ret = init_mem(NB_MBUF); 1746 if (ret < 0) 1747 rte_exit(EXIT_FAILURE, "init_mem failed\n"); 1748 1749 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1750 if (rte_lcore_is_enabled(lcore_id) == 0) 1751 continue; 1752 1753 /* Initialize TX buffers */ 1754 qconf = &lcore_conf[lcore_id]; 1755 qconf->tx_buffer[portid] = rte_zmalloc_socket("tx_buffer", 1756 RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0, 1757 rte_eth_dev_socket_id(portid)); 1758 if (qconf->tx_buffer[portid] == NULL) 1759 rte_exit(EXIT_FAILURE, "Can't allocate tx buffer for port %u\n", 1760 portid); 1761 1762 rte_eth_tx_buffer_init(qconf->tx_buffer[portid], MAX_PKT_BURST); 1763 } 1764 1765 /* init one TX queue per couple (lcore,port) */ 1766 queueid = 0; 1767 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1768 if (rte_lcore_is_enabled(lcore_id) == 0) 1769 continue; 1770 1771 if (queueid >= dev_txq_num) 1772 continue; 1773 1774 if (numa_on) 1775 socketid = \ 1776 (uint8_t)rte_lcore_to_socket_id(lcore_id); 1777 else 1778 socketid = 0; 1779 1780 printf("txq=%u,%d,%d ", lcore_id, queueid, socketid); 1781 fflush(stdout); 1782 1783 rte_eth_dev_info_get(portid, &dev_info); 1784 txconf = &dev_info.default_txconf; 1785 if (port_conf.rxmode.jumbo_frame) 1786 txconf->txq_flags = 0; 1787 ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, 1788 socketid, txconf); 1789 if (ret < 0) 1790 rte_exit(EXIT_FAILURE, 1791 "rte_eth_tx_queue_setup: err=%d, " 1792 "port=%d\n", ret, portid); 1793 1794 qconf = &lcore_conf[lcore_id]; 1795 qconf->tx_queue_id[portid] = queueid; 1796 queueid++; 1797 1798 qconf->tx_port_id[qconf->n_tx_port] = portid; 1799 qconf->n_tx_port++; 1800 } 1801 printf("\n"); 1802 } 1803 1804 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1805 if (rte_lcore_is_enabled(lcore_id) == 0) 1806 continue; 1807 1808 /* init power management library */ 1809 ret = rte_power_init(lcore_id); 1810 if (ret) 1811 RTE_LOG(ERR, POWER, 1812 "Library initialization failed on core %u\n", lcore_id); 1813 1814 /* init timer structures for each enabled lcore */ 1815 rte_timer_init(&power_timers[lcore_id]); 1816 hz = rte_get_timer_hz(); 1817 rte_timer_reset(&power_timers[lcore_id], 1818 hz/TIMER_NUMBER_PER_SECOND, SINGLE, lcore_id, 1819 power_timer_cb, NULL); 1820 1821 qconf = &lcore_conf[lcore_id]; 1822 printf("\nInitializing rx queues on lcore %u ... ", lcore_id ); 1823 fflush(stdout); 1824 /* init RX queues */ 1825 for(queue = 0; queue < qconf->n_rx_queue; ++queue) { 1826 portid = qconf->rx_queue_list[queue].port_id; 1827 queueid = qconf->rx_queue_list[queue].queue_id; 1828 1829 if (numa_on) 1830 socketid = \ 1831 (uint8_t)rte_lcore_to_socket_id(lcore_id); 1832 else 1833 socketid = 0; 1834 1835 printf("rxq=%d,%d,%d ", portid, queueid, socketid); 1836 fflush(stdout); 1837 1838 ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd, 1839 socketid, NULL, 1840 pktmbuf_pool[socketid]); 1841 if (ret < 0) 1842 rte_exit(EXIT_FAILURE, 1843 "rte_eth_rx_queue_setup: err=%d, " 1844 "port=%d\n", ret, portid); 1845 1846 if (parse_ptype) { 1847 if (add_cb_parse_ptype(portid, queueid) < 0) 1848 rte_exit(EXIT_FAILURE, 1849 "Fail to add ptype cb\n"); 1850 } else if (!check_ptype(portid)) 1851 rte_exit(EXIT_FAILURE, 1852 "PMD can not provide needed ptypes\n"); 1853 } 1854 } 1855 1856 printf("\n"); 1857 1858 /* start ports */ 1859 for (portid = 0; portid < nb_ports; portid++) { 1860 if ((enabled_port_mask & (1 << portid)) == 0) { 1861 continue; 1862 } 1863 /* Start device */ 1864 ret = rte_eth_dev_start(portid); 1865 if (ret < 0) 1866 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, " 1867 "port=%d\n", ret, portid); 1868 /* 1869 * If enabled, put device in promiscuous mode. 1870 * This allows IO forwarding mode to forward packets 1871 * to itself through 2 cross-connected ports of the 1872 * target machine. 1873 */ 1874 if (promiscuous_on) 1875 rte_eth_promiscuous_enable(portid); 1876 /* initialize spinlock for each port */ 1877 rte_spinlock_init(&(locks[portid])); 1878 } 1879 1880 check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); 1881 1882 /* launch per-lcore init on every lcore */ 1883 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 1884 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1885 if (rte_eal_wait_lcore(lcore_id) < 0) 1886 return -1; 1887 } 1888 1889 return 0; 1890 } 1891