1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2018 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <string.h> 11 #include <sys/queue.h> 12 #include <stdarg.h> 13 #include <errno.h> 14 #include <getopt.h> 15 #include <unistd.h> 16 #include <signal.h> 17 #include <math.h> 18 19 #include <rte_common.h> 20 #include <rte_byteorder.h> 21 #include <rte_log.h> 22 #include <rte_malloc.h> 23 #include <rte_memory.h> 24 #include <rte_memcpy.h> 25 #include <rte_eal.h> 26 #include <rte_launch.h> 27 #include <rte_atomic.h> 28 #include <rte_cycles.h> 29 #include <rte_prefetch.h> 30 #include <rte_lcore.h> 31 #include <rte_per_lcore.h> 32 #include <rte_branch_prediction.h> 33 #include <rte_interrupts.h> 34 #include <rte_random.h> 35 #include <rte_debug.h> 36 #include <rte_ether.h> 37 #include <rte_ethdev.h> 38 #include <rte_mempool.h> 39 #include <rte_mbuf.h> 40 #include <rte_ip.h> 41 #include <rte_tcp.h> 42 #include <rte_udp.h> 43 #include <rte_string_fns.h> 44 #include <rte_timer.h> 45 #include <rte_power.h> 46 #include <rte_spinlock.h> 47 #include <rte_power_empty_poll.h> 48 #include <rte_metrics.h> 49 #include <rte_telemetry.h> 50 51 #include "perf_core.h" 52 #include "main.h" 53 54 #define RTE_LOGTYPE_L3FWD_POWER RTE_LOGTYPE_USER1 55 56 #define MAX_PKT_BURST 32 57 58 #define MIN_ZERO_POLL_COUNT 10 59 60 /* 100 ms interval */ 61 #define TIMER_NUMBER_PER_SECOND 10 62 /* (10ms) */ 63 #define INTERVALS_PER_SECOND 100 64 /* 100000 us */ 65 #define SCALING_PERIOD (1000000/TIMER_NUMBER_PER_SECOND) 66 #define SCALING_DOWN_TIME_RATIO_THRESHOLD 0.25 67 68 #define APP_LOOKUP_EXACT_MATCH 0 69 #define APP_LOOKUP_LPM 1 70 #define DO_RFC_1812_CHECKS 71 72 #ifndef APP_LOOKUP_METHOD 73 #define APP_LOOKUP_METHOD APP_LOOKUP_LPM 74 #endif 75 76 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 77 #include <rte_hash.h> 78 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 79 #include <rte_lpm.h> 80 #else 81 #error "APP_LOOKUP_METHOD set to incorrect value" 82 #endif 83 84 #ifndef IPv6_BYTES 85 #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\ 86 "%02x%02x:%02x%02x:%02x%02x:%02x%02x" 87 #define IPv6_BYTES(addr) \ 88 addr[0], addr[1], addr[2], addr[3], \ 89 addr[4], addr[5], addr[6], addr[7], \ 90 addr[8], addr[9], addr[10], addr[11],\ 91 addr[12], addr[13],addr[14], addr[15] 92 #endif 93 94 #define MAX_JUMBO_PKT_LEN 9600 95 96 #define IPV6_ADDR_LEN 16 97 98 #define MEMPOOL_CACHE_SIZE 256 99 100 /* 101 * This expression is used to calculate the number of mbufs needed depending on 102 * user input, taking into account memory for rx and tx hardware rings, cache 103 * per lcore and mtable per port per lcore. RTE_MAX is used to ensure that 104 * NB_MBUF never goes below a minimum value of 8192. 105 */ 106 107 #define NB_MBUF RTE_MAX ( \ 108 (nb_ports*nb_rx_queue*nb_rxd + \ 109 nb_ports*nb_lcores*MAX_PKT_BURST + \ 110 nb_ports*n_tx_queue*nb_txd + \ 111 nb_lcores*MEMPOOL_CACHE_SIZE), \ 112 (unsigned)8192) 113 114 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 115 116 #define NB_SOCKETS 8 117 118 /* Configure how many packets ahead to prefetch, when reading packets */ 119 #define PREFETCH_OFFSET 3 120 121 /* 122 * Configurable number of RX/TX ring descriptors 123 */ 124 #define RTE_TEST_RX_DESC_DEFAULT 1024 125 #define RTE_TEST_TX_DESC_DEFAULT 1024 126 127 /* 128 * These two thresholds were decided on by running the training algorithm on 129 * a 2.5GHz Xeon. These defaults can be overridden by supplying non-zero values 130 * for the med_threshold and high_threshold parameters on the command line. 131 */ 132 #define EMPTY_POLL_MED_THRESHOLD 350000UL 133 #define EMPTY_POLL_HGH_THRESHOLD 580000UL 134 135 #define NUM_TELSTATS RTE_DIM(telstats_strings) 136 137 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 138 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 139 140 /* ethernet addresses of ports */ 141 static struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; 142 143 /* ethernet addresses of ports */ 144 static rte_spinlock_t locks[RTE_MAX_ETHPORTS]; 145 146 /* mask of enabled ports */ 147 static uint32_t enabled_port_mask = 0; 148 /* Ports set in promiscuous mode off by default. */ 149 static int promiscuous_on = 0; 150 /* NUMA is enabled by default. */ 151 static int numa_on = 1; 152 static bool empty_poll_stop; 153 static bool empty_poll_train; 154 volatile bool quit_signal; 155 static struct ep_params *ep_params; 156 static struct ep_policy policy; 157 static long ep_med_edpi, ep_hgh_edpi; 158 /* timer to update telemetry every 500ms */ 159 static struct rte_timer telemetry_timer; 160 161 /* stats index returned by metrics lib */ 162 int telstats_index; 163 164 struct telstats_name { 165 char name[RTE_ETH_XSTATS_NAME_SIZE]; 166 }; 167 168 /* telemetry stats to be reported */ 169 const struct telstats_name telstats_strings[] = { 170 {"empty_poll"}, 171 {"full_poll"}, 172 {"busy_percent"} 173 }; 174 175 /* core busyness in percentage */ 176 enum busy_rate { 177 ZERO = 0, 178 PARTIAL = 50, 179 FULL = 100 180 }; 181 182 /* reference poll count to measure core busyness */ 183 #define DEFAULT_COUNT 10000 184 /* 185 * reference CYCLES to be used to 186 * measure core busyness based on poll count 187 */ 188 #define MIN_CYCLES 1500000ULL 189 #define MAX_CYCLES 22000000ULL 190 191 /* (500ms) */ 192 #define TELEMETRY_INTERVALS_PER_SEC 2 193 194 static int parse_ptype; /**< Parse packet type using rx callback, and */ 195 /**< disabled by default */ 196 197 enum appmode { 198 APP_MODE_LEGACY = 0, 199 APP_MODE_EMPTY_POLL, 200 APP_MODE_TELEMETRY 201 }; 202 203 enum appmode app_mode; 204 205 enum freq_scale_hint_t 206 { 207 FREQ_LOWER = -1, 208 FREQ_CURRENT = 0, 209 FREQ_HIGHER = 1, 210 FREQ_HIGHEST = 2 211 }; 212 213 struct lcore_rx_queue { 214 uint16_t port_id; 215 uint8_t queue_id; 216 enum freq_scale_hint_t freq_up_hint; 217 uint32_t zero_rx_packet_count; 218 uint32_t idle_hint; 219 } __rte_cache_aligned; 220 221 #define MAX_RX_QUEUE_PER_LCORE 16 222 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 223 #define MAX_RX_QUEUE_PER_PORT 128 224 225 #define MAX_RX_QUEUE_INTERRUPT_PER_PORT 16 226 227 228 struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 229 static struct lcore_params lcore_params_array_default[] = { 230 {0, 0, 2}, 231 {0, 1, 2}, 232 {0, 2, 2}, 233 {1, 0, 2}, 234 {1, 1, 2}, 235 {1, 2, 2}, 236 {2, 0, 2}, 237 {3, 0, 3}, 238 {3, 1, 3}, 239 }; 240 241 struct lcore_params *lcore_params = lcore_params_array_default; 242 uint16_t nb_lcore_params = RTE_DIM(lcore_params_array_default); 243 244 static struct rte_eth_conf port_conf = { 245 .rxmode = { 246 .mq_mode = ETH_MQ_RX_RSS, 247 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 248 .split_hdr_size = 0, 249 .offloads = DEV_RX_OFFLOAD_CHECKSUM, 250 }, 251 .rx_adv_conf = { 252 .rss_conf = { 253 .rss_key = NULL, 254 .rss_hf = ETH_RSS_UDP, 255 }, 256 }, 257 .txmode = { 258 .mq_mode = ETH_MQ_TX_NONE, 259 }, 260 .intr_conf = { 261 .rxq = 1, 262 }, 263 }; 264 265 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS]; 266 267 268 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 269 270 #ifdef RTE_ARCH_X86 271 #include <rte_hash_crc.h> 272 #define DEFAULT_HASH_FUNC rte_hash_crc 273 #else 274 #include <rte_jhash.h> 275 #define DEFAULT_HASH_FUNC rte_jhash 276 #endif 277 278 struct ipv4_5tuple { 279 uint32_t ip_dst; 280 uint32_t ip_src; 281 uint16_t port_dst; 282 uint16_t port_src; 283 uint8_t proto; 284 } __rte_packed; 285 286 struct ipv6_5tuple { 287 uint8_t ip_dst[IPV6_ADDR_LEN]; 288 uint8_t ip_src[IPV6_ADDR_LEN]; 289 uint16_t port_dst; 290 uint16_t port_src; 291 uint8_t proto; 292 } __rte_packed; 293 294 struct ipv4_l3fwd_route { 295 struct ipv4_5tuple key; 296 uint8_t if_out; 297 }; 298 299 struct ipv6_l3fwd_route { 300 struct ipv6_5tuple key; 301 uint8_t if_out; 302 }; 303 304 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { 305 {{RTE_IPV4(100,10,0,1), RTE_IPV4(200,10,0,1), 101, 11, IPPROTO_TCP}, 0}, 306 {{RTE_IPV4(100,20,0,2), RTE_IPV4(200,20,0,2), 102, 12, IPPROTO_TCP}, 1}, 307 {{RTE_IPV4(100,30,0,3), RTE_IPV4(200,30,0,3), 103, 13, IPPROTO_TCP}, 2}, 308 {{RTE_IPV4(100,40,0,4), RTE_IPV4(200,40,0,4), 104, 14, IPPROTO_TCP}, 3}, 309 }; 310 311 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = { 312 { 313 { 314 {0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 315 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05}, 316 {0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 317 0x02, 0x1e, 0x67, 0xff, 0xfe, 0x0d, 0xb6, 0x0a}, 318 1, 10, IPPROTO_UDP 319 }, 4 320 }, 321 }; 322 323 typedef struct rte_hash lookup_struct_t; 324 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS]; 325 static lookup_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS]; 326 327 #define L3FWD_HASH_ENTRIES 1024 328 329 static uint16_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; 330 static uint16_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; 331 #endif 332 333 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 334 struct ipv4_l3fwd_route { 335 uint32_t ip; 336 uint8_t depth; 337 uint8_t if_out; 338 }; 339 340 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { 341 {RTE_IPV4(1,1,1,0), 24, 0}, 342 {RTE_IPV4(2,1,1,0), 24, 1}, 343 {RTE_IPV4(3,1,1,0), 24, 2}, 344 {RTE_IPV4(4,1,1,0), 24, 3}, 345 {RTE_IPV4(5,1,1,0), 24, 4}, 346 {RTE_IPV4(6,1,1,0), 24, 5}, 347 {RTE_IPV4(7,1,1,0), 24, 6}, 348 {RTE_IPV4(8,1,1,0), 24, 7}, 349 }; 350 351 #define IPV4_L3FWD_LPM_MAX_RULES 1024 352 353 typedef struct rte_lpm lookup_struct_t; 354 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS]; 355 #endif 356 357 struct lcore_conf { 358 uint16_t n_rx_queue; 359 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 360 uint16_t n_tx_port; 361 uint16_t tx_port_id[RTE_MAX_ETHPORTS]; 362 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 363 struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS]; 364 lookup_struct_t * ipv4_lookup_struct; 365 lookup_struct_t * ipv6_lookup_struct; 366 } __rte_cache_aligned; 367 368 struct lcore_stats { 369 /* total sleep time in ms since last frequency scaling down */ 370 uint32_t sleep_time; 371 /* number of long sleep recently */ 372 uint32_t nb_long_sleep; 373 /* freq. scaling up trend */ 374 uint32_t trend; 375 /* total packet processed recently */ 376 uint64_t nb_rx_processed; 377 /* total iterations looped recently */ 378 uint64_t nb_iteration_looped; 379 /* 380 * Represents empty and non empty polls 381 * of rte_eth_rx_burst(); 382 * ep_nep[0] holds non empty polls 383 * i.e. 0 < nb_rx <= MAX_BURST 384 * ep_nep[1] holds empty polls. 385 * i.e. nb_rx == 0 386 */ 387 uint64_t ep_nep[2]; 388 /* 389 * Represents full and empty+partial 390 * polls of rte_eth_rx_burst(); 391 * ep_nep[0] holds empty+partial polls. 392 * i.e. 0 <= nb_rx < MAX_BURST 393 * ep_nep[1] holds full polls 394 * i.e. nb_rx == MAX_BURST 395 */ 396 uint64_t fp_nfp[2]; 397 enum busy_rate br; 398 rte_spinlock_t telemetry_lock; 399 } __rte_cache_aligned; 400 401 static struct lcore_conf lcore_conf[RTE_MAX_LCORE] __rte_cache_aligned; 402 static struct lcore_stats stats[RTE_MAX_LCORE] __rte_cache_aligned; 403 static struct rte_timer power_timers[RTE_MAX_LCORE]; 404 405 static inline uint32_t power_idle_heuristic(uint32_t zero_rx_packet_count); 406 static inline enum freq_scale_hint_t power_freq_scaleup_heuristic( \ 407 unsigned int lcore_id, uint16_t port_id, uint16_t queue_id); 408 409 410 /* 411 * These defaults are using the max frequency index (1), a medium index (9) 412 * and a typical low frequency index (14). These can be adjusted to use 413 * different indexes using the relevant command line parameters. 414 */ 415 static uint8_t freq_tlb[] = {14, 9, 1}; 416 417 static int is_done(void) 418 { 419 return quit_signal; 420 } 421 422 /* exit signal handler */ 423 static void 424 signal_exit_now(int sigtype) 425 { 426 427 if (sigtype == SIGINT) 428 quit_signal = true; 429 430 } 431 432 /* Freqency scale down timer callback */ 433 static void 434 power_timer_cb(__rte_unused struct rte_timer *tim, 435 __rte_unused void *arg) 436 { 437 uint64_t hz; 438 float sleep_time_ratio; 439 unsigned lcore_id = rte_lcore_id(); 440 441 /* accumulate total execution time in us when callback is invoked */ 442 sleep_time_ratio = (float)(stats[lcore_id].sleep_time) / 443 (float)SCALING_PERIOD; 444 /** 445 * check whether need to scale down frequency a step if it sleep a lot. 446 */ 447 if (sleep_time_ratio >= SCALING_DOWN_TIME_RATIO_THRESHOLD) { 448 if (rte_power_freq_down) 449 rte_power_freq_down(lcore_id); 450 } 451 else if ( (unsigned)(stats[lcore_id].nb_rx_processed / 452 stats[lcore_id].nb_iteration_looped) < MAX_PKT_BURST) { 453 /** 454 * scale down a step if average packet per iteration less 455 * than expectation. 456 */ 457 if (rte_power_freq_down) 458 rte_power_freq_down(lcore_id); 459 } 460 461 /** 462 * initialize another timer according to current frequency to ensure 463 * timer interval is relatively fixed. 464 */ 465 hz = rte_get_timer_hz(); 466 rte_timer_reset(&power_timers[lcore_id], hz/TIMER_NUMBER_PER_SECOND, 467 SINGLE, lcore_id, power_timer_cb, NULL); 468 469 stats[lcore_id].nb_rx_processed = 0; 470 stats[lcore_id].nb_iteration_looped = 0; 471 472 stats[lcore_id].sleep_time = 0; 473 } 474 475 /* Enqueue a single packet, and send burst if queue is filled */ 476 static inline int 477 send_single_packet(struct rte_mbuf *m, uint16_t port) 478 { 479 uint32_t lcore_id; 480 struct lcore_conf *qconf; 481 482 lcore_id = rte_lcore_id(); 483 qconf = &lcore_conf[lcore_id]; 484 485 rte_eth_tx_buffer(port, qconf->tx_queue_id[port], 486 qconf->tx_buffer[port], m); 487 488 return 0; 489 } 490 491 #ifdef DO_RFC_1812_CHECKS 492 static inline int 493 is_valid_ipv4_pkt(struct rte_ipv4_hdr *pkt, uint32_t link_len) 494 { 495 /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */ 496 /* 497 * 1. The packet length reported by the Link Layer must be large 498 * enough to hold the minimum length legal IP datagram (20 bytes). 499 */ 500 if (link_len < sizeof(struct rte_ipv4_hdr)) 501 return -1; 502 503 /* 2. The IP checksum must be correct. */ 504 /* this is checked in H/W */ 505 506 /* 507 * 3. The IP version number must be 4. If the version number is not 4 508 * then the packet may be another version of IP, such as IPng or 509 * ST-II. 510 */ 511 if (((pkt->version_ihl) >> 4) != 4) 512 return -3; 513 /* 514 * 4. The IP header length field must be large enough to hold the 515 * minimum length legal IP datagram (20 bytes = 5 words). 516 */ 517 if ((pkt->version_ihl & 0xf) < 5) 518 return -4; 519 520 /* 521 * 5. The IP total length field must be large enough to hold the IP 522 * datagram header, whose length is specified in the IP header length 523 * field. 524 */ 525 if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct rte_ipv4_hdr)) 526 return -5; 527 528 return 0; 529 } 530 #endif 531 532 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 533 static void 534 print_ipv4_key(struct ipv4_5tuple key) 535 { 536 printf("IP dst = %08x, IP src = %08x, port dst = %d, port src = %d, " 537 "proto = %d\n", (unsigned)key.ip_dst, (unsigned)key.ip_src, 538 key.port_dst, key.port_src, key.proto); 539 } 540 static void 541 print_ipv6_key(struct ipv6_5tuple key) 542 { 543 printf( "IP dst = " IPv6_BYTES_FMT ", IP src = " IPv6_BYTES_FMT ", " 544 "port dst = %d, port src = %d, proto = %d\n", 545 IPv6_BYTES(key.ip_dst), IPv6_BYTES(key.ip_src), 546 key.port_dst, key.port_src, key.proto); 547 } 548 549 static inline uint16_t 550 get_ipv4_dst_port(struct rte_ipv4_hdr *ipv4_hdr, uint16_t portid, 551 lookup_struct_t * ipv4_l3fwd_lookup_struct) 552 { 553 struct ipv4_5tuple key; 554 struct rte_tcp_hdr *tcp; 555 struct rte_udp_hdr *udp; 556 int ret = 0; 557 558 key.ip_dst = rte_be_to_cpu_32(ipv4_hdr->dst_addr); 559 key.ip_src = rte_be_to_cpu_32(ipv4_hdr->src_addr); 560 key.proto = ipv4_hdr->next_proto_id; 561 562 switch (ipv4_hdr->next_proto_id) { 563 case IPPROTO_TCP: 564 tcp = (struct rte_tcp_hdr *)((unsigned char *)ipv4_hdr + 565 sizeof(struct rte_ipv4_hdr)); 566 key.port_dst = rte_be_to_cpu_16(tcp->dst_port); 567 key.port_src = rte_be_to_cpu_16(tcp->src_port); 568 break; 569 570 case IPPROTO_UDP: 571 udp = (struct rte_udp_hdr *)((unsigned char *)ipv4_hdr + 572 sizeof(struct rte_ipv4_hdr)); 573 key.port_dst = rte_be_to_cpu_16(udp->dst_port); 574 key.port_src = rte_be_to_cpu_16(udp->src_port); 575 break; 576 577 default: 578 key.port_dst = 0; 579 key.port_src = 0; 580 break; 581 } 582 583 /* Find destination port */ 584 ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key); 585 return ((ret < 0) ? portid : ipv4_l3fwd_out_if[ret]); 586 } 587 588 static inline uint16_t 589 get_ipv6_dst_port(struct rte_ipv6_hdr *ipv6_hdr, uint16_t portid, 590 lookup_struct_t *ipv6_l3fwd_lookup_struct) 591 { 592 struct ipv6_5tuple key; 593 struct rte_tcp_hdr *tcp; 594 struct rte_udp_hdr *udp; 595 int ret = 0; 596 597 memcpy(key.ip_dst, ipv6_hdr->dst_addr, IPV6_ADDR_LEN); 598 memcpy(key.ip_src, ipv6_hdr->src_addr, IPV6_ADDR_LEN); 599 600 key.proto = ipv6_hdr->proto; 601 602 switch (ipv6_hdr->proto) { 603 case IPPROTO_TCP: 604 tcp = (struct rte_tcp_hdr *)((unsigned char *) ipv6_hdr + 605 sizeof(struct rte_ipv6_hdr)); 606 key.port_dst = rte_be_to_cpu_16(tcp->dst_port); 607 key.port_src = rte_be_to_cpu_16(tcp->src_port); 608 break; 609 610 case IPPROTO_UDP: 611 udp = (struct rte_udp_hdr *)((unsigned char *) ipv6_hdr + 612 sizeof(struct rte_ipv6_hdr)); 613 key.port_dst = rte_be_to_cpu_16(udp->dst_port); 614 key.port_src = rte_be_to_cpu_16(udp->src_port); 615 break; 616 617 default: 618 key.port_dst = 0; 619 key.port_src = 0; 620 break; 621 } 622 623 /* Find destination port */ 624 ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key); 625 return ((ret < 0) ? portid : ipv6_l3fwd_out_if[ret]); 626 } 627 #endif 628 629 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 630 static inline uint16_t 631 get_ipv4_dst_port(struct rte_ipv4_hdr *ipv4_hdr, uint16_t portid, 632 lookup_struct_t *ipv4_l3fwd_lookup_struct) 633 { 634 uint32_t next_hop; 635 636 return ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, 637 rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop) == 0)? 638 next_hop : portid); 639 } 640 #endif 641 642 static inline void 643 parse_ptype_one(struct rte_mbuf *m) 644 { 645 struct rte_ether_hdr *eth_hdr; 646 uint32_t packet_type = RTE_PTYPE_UNKNOWN; 647 uint16_t ether_type; 648 649 eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 650 ether_type = eth_hdr->ether_type; 651 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) 652 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 653 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) 654 packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; 655 656 m->packet_type = packet_type; 657 } 658 659 static uint16_t 660 cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused, 661 struct rte_mbuf *pkts[], uint16_t nb_pkts, 662 uint16_t max_pkts __rte_unused, 663 void *user_param __rte_unused) 664 { 665 unsigned int i; 666 667 for (i = 0; i < nb_pkts; ++i) 668 parse_ptype_one(pkts[i]); 669 670 return nb_pkts; 671 } 672 673 static int 674 add_cb_parse_ptype(uint16_t portid, uint16_t queueid) 675 { 676 printf("Port %d: softly parse packet type info\n", portid); 677 if (rte_eth_add_rx_callback(portid, queueid, cb_parse_ptype, NULL)) 678 return 0; 679 680 printf("Failed to add rx callback: port=%d\n", portid); 681 return -1; 682 } 683 684 static inline void 685 l3fwd_simple_forward(struct rte_mbuf *m, uint16_t portid, 686 struct lcore_conf *qconf) 687 { 688 struct rte_ether_hdr *eth_hdr; 689 struct rte_ipv4_hdr *ipv4_hdr; 690 void *d_addr_bytes; 691 uint16_t dst_port; 692 693 eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 694 695 if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) { 696 /* Handle IPv4 headers.*/ 697 ipv4_hdr = 698 rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, 699 sizeof(struct rte_ether_hdr)); 700 701 #ifdef DO_RFC_1812_CHECKS 702 /* Check to make sure the packet is valid (RFC1812) */ 703 if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len) < 0) { 704 rte_pktmbuf_free(m); 705 return; 706 } 707 #endif 708 709 dst_port = get_ipv4_dst_port(ipv4_hdr, portid, 710 qconf->ipv4_lookup_struct); 711 if (dst_port >= RTE_MAX_ETHPORTS || 712 (enabled_port_mask & 1 << dst_port) == 0) 713 dst_port = portid; 714 715 /* 02:00:00:00:00:xx */ 716 d_addr_bytes = ð_hdr->d_addr.addr_bytes[0]; 717 *((uint64_t *)d_addr_bytes) = 718 0x000000000002 + ((uint64_t)dst_port << 40); 719 720 #ifdef DO_RFC_1812_CHECKS 721 /* Update time to live and header checksum */ 722 --(ipv4_hdr->time_to_live); 723 ++(ipv4_hdr->hdr_checksum); 724 #endif 725 726 /* src addr */ 727 rte_ether_addr_copy(&ports_eth_addr[dst_port], 728 ð_hdr->s_addr); 729 730 send_single_packet(m, dst_port); 731 } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) { 732 /* Handle IPv6 headers.*/ 733 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 734 struct rte_ipv6_hdr *ipv6_hdr; 735 736 ipv6_hdr = 737 rte_pktmbuf_mtod_offset(m, struct rte_ipv6_hdr *, 738 sizeof(struct rte_ether_hdr)); 739 740 dst_port = get_ipv6_dst_port(ipv6_hdr, portid, 741 qconf->ipv6_lookup_struct); 742 743 if (dst_port >= RTE_MAX_ETHPORTS || 744 (enabled_port_mask & 1 << dst_port) == 0) 745 dst_port = portid; 746 747 /* 02:00:00:00:00:xx */ 748 d_addr_bytes = ð_hdr->d_addr.addr_bytes[0]; 749 *((uint64_t *)d_addr_bytes) = 750 0x000000000002 + ((uint64_t)dst_port << 40); 751 752 /* src addr */ 753 rte_ether_addr_copy(&ports_eth_addr[dst_port], 754 ð_hdr->s_addr); 755 756 send_single_packet(m, dst_port); 757 #else 758 /* We don't currently handle IPv6 packets in LPM mode. */ 759 rte_pktmbuf_free(m); 760 #endif 761 } else 762 rte_pktmbuf_free(m); 763 764 } 765 766 #define MINIMUM_SLEEP_TIME 1 767 #define SUSPEND_THRESHOLD 300 768 769 static inline uint32_t 770 power_idle_heuristic(uint32_t zero_rx_packet_count) 771 { 772 /* If zero count is less than 100, sleep 1us */ 773 if (zero_rx_packet_count < SUSPEND_THRESHOLD) 774 return MINIMUM_SLEEP_TIME; 775 /* If zero count is less than 1000, sleep 100 us which is the 776 minimum latency switching from C3/C6 to C0 777 */ 778 else 779 return SUSPEND_THRESHOLD; 780 } 781 782 static inline enum freq_scale_hint_t 783 power_freq_scaleup_heuristic(unsigned lcore_id, 784 uint16_t port_id, 785 uint16_t queue_id) 786 { 787 uint32_t rxq_count = rte_eth_rx_queue_count(port_id, queue_id); 788 /** 789 * HW Rx queue size is 128 by default, Rx burst read at maximum 32 entries 790 * per iteration 791 */ 792 #define FREQ_GEAR1_RX_PACKET_THRESHOLD MAX_PKT_BURST 793 #define FREQ_GEAR2_RX_PACKET_THRESHOLD (MAX_PKT_BURST*2) 794 #define FREQ_GEAR3_RX_PACKET_THRESHOLD (MAX_PKT_BURST*3) 795 #define FREQ_UP_TREND1_ACC 1 796 #define FREQ_UP_TREND2_ACC 100 797 #define FREQ_UP_THRESHOLD 10000 798 799 if (likely(rxq_count > FREQ_GEAR3_RX_PACKET_THRESHOLD)) { 800 stats[lcore_id].trend = 0; 801 return FREQ_HIGHEST; 802 } else if (likely(rxq_count > FREQ_GEAR2_RX_PACKET_THRESHOLD)) 803 stats[lcore_id].trend += FREQ_UP_TREND2_ACC; 804 else if (likely(rxq_count > FREQ_GEAR1_RX_PACKET_THRESHOLD)) 805 stats[lcore_id].trend += FREQ_UP_TREND1_ACC; 806 807 if (likely(stats[lcore_id].trend > FREQ_UP_THRESHOLD)) { 808 stats[lcore_id].trend = 0; 809 return FREQ_HIGHER; 810 } 811 812 return FREQ_CURRENT; 813 } 814 815 /** 816 * force polling thread sleep until one-shot rx interrupt triggers 817 * @param port_id 818 * Port id. 819 * @param queue_id 820 * Rx queue id. 821 * @return 822 * 0 on success 823 */ 824 static int 825 sleep_until_rx_interrupt(int num) 826 { 827 /* 828 * we want to track when we are woken up by traffic so that we can go 829 * back to sleep again without log spamming. 830 */ 831 static bool timeout; 832 struct rte_epoll_event event[num]; 833 int n, i; 834 uint16_t port_id; 835 uint8_t queue_id; 836 void *data; 837 838 if (!timeout) { 839 RTE_LOG(INFO, L3FWD_POWER, 840 "lcore %u sleeps until interrupt triggers\n", 841 rte_lcore_id()); 842 } 843 844 n = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, num, 10); 845 for (i = 0; i < n; i++) { 846 data = event[i].epdata.data; 847 port_id = ((uintptr_t)data) >> CHAR_BIT; 848 queue_id = ((uintptr_t)data) & 849 RTE_LEN2MASK(CHAR_BIT, uint8_t); 850 RTE_LOG(INFO, L3FWD_POWER, 851 "lcore %u is waked up from rx interrupt on" 852 " port %d queue %d\n", 853 rte_lcore_id(), port_id, queue_id); 854 } 855 timeout = n == 0; 856 857 return 0; 858 } 859 860 static void turn_on_off_intr(struct lcore_conf *qconf, bool on) 861 { 862 int i; 863 struct lcore_rx_queue *rx_queue; 864 uint8_t queue_id; 865 uint16_t port_id; 866 867 for (i = 0; i < qconf->n_rx_queue; ++i) { 868 rx_queue = &(qconf->rx_queue_list[i]); 869 port_id = rx_queue->port_id; 870 queue_id = rx_queue->queue_id; 871 872 rte_spinlock_lock(&(locks[port_id])); 873 if (on) 874 rte_eth_dev_rx_intr_enable(port_id, queue_id); 875 else 876 rte_eth_dev_rx_intr_disable(port_id, queue_id); 877 rte_spinlock_unlock(&(locks[port_id])); 878 } 879 } 880 881 static int event_register(struct lcore_conf *qconf) 882 { 883 struct lcore_rx_queue *rx_queue; 884 uint8_t queueid; 885 uint16_t portid; 886 uint32_t data; 887 int ret; 888 int i; 889 890 for (i = 0; i < qconf->n_rx_queue; ++i) { 891 rx_queue = &(qconf->rx_queue_list[i]); 892 portid = rx_queue->port_id; 893 queueid = rx_queue->queue_id; 894 data = portid << CHAR_BIT | queueid; 895 896 ret = rte_eth_dev_rx_intr_ctl_q(portid, queueid, 897 RTE_EPOLL_PER_THREAD, 898 RTE_INTR_EVENT_ADD, 899 (void *)((uintptr_t)data)); 900 if (ret) 901 return ret; 902 } 903 904 return 0; 905 } 906 /* main processing loop */ 907 static int 908 main_telemetry_loop(__rte_unused void *dummy) 909 { 910 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 911 unsigned int lcore_id; 912 uint64_t prev_tsc, diff_tsc, cur_tsc, prev_tel_tsc; 913 int i, j, nb_rx; 914 uint8_t queueid; 915 uint16_t portid; 916 struct lcore_conf *qconf; 917 struct lcore_rx_queue *rx_queue; 918 uint64_t ep_nep[2] = {0}, fp_nfp[2] = {0}; 919 uint64_t poll_count; 920 enum busy_rate br; 921 922 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 923 US_PER_S * BURST_TX_DRAIN_US; 924 925 poll_count = 0; 926 prev_tsc = 0; 927 prev_tel_tsc = 0; 928 929 lcore_id = rte_lcore_id(); 930 qconf = &lcore_conf[lcore_id]; 931 932 if (qconf->n_rx_queue == 0) { 933 RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", 934 lcore_id); 935 return 0; 936 } 937 938 RTE_LOG(INFO, L3FWD_POWER, "entering main telemetry loop on lcore %u\n", 939 lcore_id); 940 941 for (i = 0; i < qconf->n_rx_queue; i++) { 942 portid = qconf->rx_queue_list[i].port_id; 943 queueid = qconf->rx_queue_list[i].queue_id; 944 RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u " 945 "rxqueueid=%hhu\n", lcore_id, portid, queueid); 946 } 947 948 while (!is_done()) { 949 950 cur_tsc = rte_rdtsc(); 951 /* 952 * TX burst queue drain 953 */ 954 diff_tsc = cur_tsc - prev_tsc; 955 if (unlikely(diff_tsc > drain_tsc)) { 956 for (i = 0; i < qconf->n_tx_port; ++i) { 957 portid = qconf->tx_port_id[i]; 958 rte_eth_tx_buffer_flush(portid, 959 qconf->tx_queue_id[portid], 960 qconf->tx_buffer[portid]); 961 } 962 prev_tsc = cur_tsc; 963 } 964 965 /* 966 * Read packet from RX queues 967 */ 968 for (i = 0; i < qconf->n_rx_queue; ++i) { 969 rx_queue = &(qconf->rx_queue_list[i]); 970 portid = rx_queue->port_id; 971 queueid = rx_queue->queue_id; 972 973 nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, 974 MAX_PKT_BURST); 975 ep_nep[nb_rx == 0]++; 976 fp_nfp[nb_rx == MAX_PKT_BURST]++; 977 poll_count++; 978 if (unlikely(nb_rx == 0)) 979 continue; 980 981 /* Prefetch first packets */ 982 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 983 rte_prefetch0(rte_pktmbuf_mtod( 984 pkts_burst[j], void *)); 985 } 986 987 /* Prefetch and forward already prefetched packets */ 988 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 989 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 990 j + PREFETCH_OFFSET], void *)); 991 l3fwd_simple_forward(pkts_burst[j], portid, 992 qconf); 993 } 994 995 /* Forward remaining prefetched packets */ 996 for (; j < nb_rx; j++) { 997 l3fwd_simple_forward(pkts_burst[j], portid, 998 qconf); 999 } 1000 } 1001 if (unlikely(poll_count >= DEFAULT_COUNT)) { 1002 diff_tsc = cur_tsc - prev_tel_tsc; 1003 if (diff_tsc >= MAX_CYCLES) { 1004 br = FULL; 1005 } else if (diff_tsc > MIN_CYCLES && 1006 diff_tsc < MAX_CYCLES) { 1007 br = (diff_tsc * 100) / MAX_CYCLES; 1008 } else { 1009 br = ZERO; 1010 } 1011 poll_count = 0; 1012 prev_tel_tsc = cur_tsc; 1013 /* update stats for telemetry */ 1014 rte_spinlock_lock(&stats[lcore_id].telemetry_lock); 1015 stats[lcore_id].ep_nep[0] = ep_nep[0]; 1016 stats[lcore_id].ep_nep[1] = ep_nep[1]; 1017 stats[lcore_id].fp_nfp[0] = fp_nfp[0]; 1018 stats[lcore_id].fp_nfp[1] = fp_nfp[1]; 1019 stats[lcore_id].br = br; 1020 rte_spinlock_unlock(&stats[lcore_id].telemetry_lock); 1021 } 1022 } 1023 1024 return 0; 1025 } 1026 /* main processing loop */ 1027 static int 1028 main_empty_poll_loop(__rte_unused void *dummy) 1029 { 1030 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1031 unsigned int lcore_id; 1032 uint64_t prev_tsc, diff_tsc, cur_tsc; 1033 int i, j, nb_rx; 1034 uint8_t queueid; 1035 uint16_t portid; 1036 struct lcore_conf *qconf; 1037 struct lcore_rx_queue *rx_queue; 1038 1039 const uint64_t drain_tsc = 1040 (rte_get_tsc_hz() + US_PER_S - 1) / 1041 US_PER_S * BURST_TX_DRAIN_US; 1042 1043 prev_tsc = 0; 1044 1045 lcore_id = rte_lcore_id(); 1046 qconf = &lcore_conf[lcore_id]; 1047 1048 if (qconf->n_rx_queue == 0) { 1049 RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", 1050 lcore_id); 1051 return 0; 1052 } 1053 1054 for (i = 0; i < qconf->n_rx_queue; i++) { 1055 portid = qconf->rx_queue_list[i].port_id; 1056 queueid = qconf->rx_queue_list[i].queue_id; 1057 RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u " 1058 "rxqueueid=%hhu\n", lcore_id, portid, queueid); 1059 } 1060 1061 while (!is_done()) { 1062 stats[lcore_id].nb_iteration_looped++; 1063 1064 cur_tsc = rte_rdtsc(); 1065 /* 1066 * TX burst queue drain 1067 */ 1068 diff_tsc = cur_tsc - prev_tsc; 1069 if (unlikely(diff_tsc > drain_tsc)) { 1070 for (i = 0; i < qconf->n_tx_port; ++i) { 1071 portid = qconf->tx_port_id[i]; 1072 rte_eth_tx_buffer_flush(portid, 1073 qconf->tx_queue_id[portid], 1074 qconf->tx_buffer[portid]); 1075 } 1076 prev_tsc = cur_tsc; 1077 } 1078 1079 /* 1080 * Read packet from RX queues 1081 */ 1082 for (i = 0; i < qconf->n_rx_queue; ++i) { 1083 rx_queue = &(qconf->rx_queue_list[i]); 1084 rx_queue->idle_hint = 0; 1085 portid = rx_queue->port_id; 1086 queueid = rx_queue->queue_id; 1087 1088 nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, 1089 MAX_PKT_BURST); 1090 1091 stats[lcore_id].nb_rx_processed += nb_rx; 1092 1093 if (nb_rx == 0) { 1094 1095 rte_power_empty_poll_stat_update(lcore_id); 1096 1097 continue; 1098 } else { 1099 rte_power_poll_stat_update(lcore_id, nb_rx); 1100 } 1101 1102 1103 /* Prefetch first packets */ 1104 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1105 rte_prefetch0(rte_pktmbuf_mtod( 1106 pkts_burst[j], void *)); 1107 } 1108 1109 /* Prefetch and forward already prefetched packets */ 1110 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1111 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1112 j + PREFETCH_OFFSET], 1113 void *)); 1114 l3fwd_simple_forward(pkts_burst[j], portid, 1115 qconf); 1116 } 1117 1118 /* Forward remaining prefetched packets */ 1119 for (; j < nb_rx; j++) { 1120 l3fwd_simple_forward(pkts_burst[j], portid, 1121 qconf); 1122 } 1123 1124 } 1125 1126 } 1127 1128 return 0; 1129 } 1130 /* main processing loop */ 1131 static int 1132 main_loop(__rte_unused void *dummy) 1133 { 1134 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1135 unsigned lcore_id; 1136 uint64_t prev_tsc, diff_tsc, cur_tsc, tim_res_tsc, hz; 1137 uint64_t prev_tsc_power = 0, cur_tsc_power, diff_tsc_power; 1138 int i, j, nb_rx; 1139 uint8_t queueid; 1140 uint16_t portid; 1141 struct lcore_conf *qconf; 1142 struct lcore_rx_queue *rx_queue; 1143 enum freq_scale_hint_t lcore_scaleup_hint; 1144 uint32_t lcore_rx_idle_count = 0; 1145 uint32_t lcore_idle_hint = 0; 1146 int intr_en = 0; 1147 1148 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; 1149 1150 prev_tsc = 0; 1151 hz = rte_get_timer_hz(); 1152 tim_res_tsc = hz/TIMER_NUMBER_PER_SECOND; 1153 1154 lcore_id = rte_lcore_id(); 1155 qconf = &lcore_conf[lcore_id]; 1156 1157 if (qconf->n_rx_queue == 0) { 1158 RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", lcore_id); 1159 return 0; 1160 } 1161 1162 RTE_LOG(INFO, L3FWD_POWER, "entering main loop on lcore %u\n", lcore_id); 1163 1164 for (i = 0; i < qconf->n_rx_queue; i++) { 1165 portid = qconf->rx_queue_list[i].port_id; 1166 queueid = qconf->rx_queue_list[i].queue_id; 1167 RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u " 1168 "rxqueueid=%hhu\n", lcore_id, portid, queueid); 1169 } 1170 1171 /* add into event wait list */ 1172 if (event_register(qconf) == 0) 1173 intr_en = 1; 1174 else 1175 RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n"); 1176 1177 while (!is_done()) { 1178 stats[lcore_id].nb_iteration_looped++; 1179 1180 cur_tsc = rte_rdtsc(); 1181 cur_tsc_power = cur_tsc; 1182 1183 /* 1184 * TX burst queue drain 1185 */ 1186 diff_tsc = cur_tsc - prev_tsc; 1187 if (unlikely(diff_tsc > drain_tsc)) { 1188 for (i = 0; i < qconf->n_tx_port; ++i) { 1189 portid = qconf->tx_port_id[i]; 1190 rte_eth_tx_buffer_flush(portid, 1191 qconf->tx_queue_id[portid], 1192 qconf->tx_buffer[portid]); 1193 } 1194 prev_tsc = cur_tsc; 1195 } 1196 1197 diff_tsc_power = cur_tsc_power - prev_tsc_power; 1198 if (diff_tsc_power > tim_res_tsc) { 1199 rte_timer_manage(); 1200 prev_tsc_power = cur_tsc_power; 1201 } 1202 1203 start_rx: 1204 /* 1205 * Read packet from RX queues 1206 */ 1207 lcore_scaleup_hint = FREQ_CURRENT; 1208 lcore_rx_idle_count = 0; 1209 for (i = 0; i < qconf->n_rx_queue; ++i) { 1210 rx_queue = &(qconf->rx_queue_list[i]); 1211 rx_queue->idle_hint = 0; 1212 portid = rx_queue->port_id; 1213 queueid = rx_queue->queue_id; 1214 1215 nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, 1216 MAX_PKT_BURST); 1217 1218 stats[lcore_id].nb_rx_processed += nb_rx; 1219 if (unlikely(nb_rx == 0)) { 1220 /** 1221 * no packet received from rx queue, try to 1222 * sleep for a while forcing CPU enter deeper 1223 * C states. 1224 */ 1225 rx_queue->zero_rx_packet_count++; 1226 1227 if (rx_queue->zero_rx_packet_count <= 1228 MIN_ZERO_POLL_COUNT) 1229 continue; 1230 1231 rx_queue->idle_hint = power_idle_heuristic(\ 1232 rx_queue->zero_rx_packet_count); 1233 lcore_rx_idle_count++; 1234 } else { 1235 rx_queue->zero_rx_packet_count = 0; 1236 1237 /** 1238 * do not scale up frequency immediately as 1239 * user to kernel space communication is costly 1240 * which might impact packet I/O for received 1241 * packets. 1242 */ 1243 rx_queue->freq_up_hint = 1244 power_freq_scaleup_heuristic(lcore_id, 1245 portid, queueid); 1246 } 1247 1248 /* Prefetch first packets */ 1249 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1250 rte_prefetch0(rte_pktmbuf_mtod( 1251 pkts_burst[j], void *)); 1252 } 1253 1254 /* Prefetch and forward already prefetched packets */ 1255 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1256 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1257 j + PREFETCH_OFFSET], void *)); 1258 l3fwd_simple_forward(pkts_burst[j], portid, 1259 qconf); 1260 } 1261 1262 /* Forward remaining prefetched packets */ 1263 for (; j < nb_rx; j++) { 1264 l3fwd_simple_forward(pkts_burst[j], portid, 1265 qconf); 1266 } 1267 } 1268 1269 if (likely(lcore_rx_idle_count != qconf->n_rx_queue)) { 1270 for (i = 1, lcore_scaleup_hint = 1271 qconf->rx_queue_list[0].freq_up_hint; 1272 i < qconf->n_rx_queue; ++i) { 1273 rx_queue = &(qconf->rx_queue_list[i]); 1274 if (rx_queue->freq_up_hint > 1275 lcore_scaleup_hint) 1276 lcore_scaleup_hint = 1277 rx_queue->freq_up_hint; 1278 } 1279 1280 if (lcore_scaleup_hint == FREQ_HIGHEST) { 1281 if (rte_power_freq_max) 1282 rte_power_freq_max(lcore_id); 1283 } else if (lcore_scaleup_hint == FREQ_HIGHER) { 1284 if (rte_power_freq_up) 1285 rte_power_freq_up(lcore_id); 1286 } 1287 } else { 1288 /** 1289 * All Rx queues empty in recent consecutive polls, 1290 * sleep in a conservative manner, meaning sleep as 1291 * less as possible. 1292 */ 1293 for (i = 1, lcore_idle_hint = 1294 qconf->rx_queue_list[0].idle_hint; 1295 i < qconf->n_rx_queue; ++i) { 1296 rx_queue = &(qconf->rx_queue_list[i]); 1297 if (rx_queue->idle_hint < lcore_idle_hint) 1298 lcore_idle_hint = rx_queue->idle_hint; 1299 } 1300 1301 if (lcore_idle_hint < SUSPEND_THRESHOLD) 1302 /** 1303 * execute "pause" instruction to avoid context 1304 * switch which generally take hundred of 1305 * microseconds for short sleep. 1306 */ 1307 rte_delay_us(lcore_idle_hint); 1308 else { 1309 /* suspend until rx interrupt triggers */ 1310 if (intr_en) { 1311 turn_on_off_intr(qconf, 1); 1312 sleep_until_rx_interrupt( 1313 qconf->n_rx_queue); 1314 turn_on_off_intr(qconf, 0); 1315 /** 1316 * start receiving packets immediately 1317 */ 1318 if (likely(!is_done())) 1319 goto start_rx; 1320 } 1321 } 1322 stats[lcore_id].sleep_time += lcore_idle_hint; 1323 } 1324 } 1325 1326 return 0; 1327 } 1328 1329 static int 1330 check_lcore_params(void) 1331 { 1332 uint8_t queue, lcore; 1333 uint16_t i; 1334 int socketid; 1335 1336 for (i = 0; i < nb_lcore_params; ++i) { 1337 queue = lcore_params[i].queue_id; 1338 if (queue >= MAX_RX_QUEUE_PER_PORT) { 1339 printf("invalid queue number: %hhu\n", queue); 1340 return -1; 1341 } 1342 lcore = lcore_params[i].lcore_id; 1343 if (!rte_lcore_is_enabled(lcore)) { 1344 printf("error: lcore %hhu is not enabled in lcore " 1345 "mask\n", lcore); 1346 return -1; 1347 } 1348 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) && 1349 (numa_on == 0)) { 1350 printf("warning: lcore %hhu is on socket %d with numa " 1351 "off\n", lcore, socketid); 1352 } 1353 if (app_mode == APP_MODE_TELEMETRY && lcore == rte_lcore_id()) { 1354 printf("cannot enable master core %d in config for telemetry mode\n", 1355 rte_lcore_id()); 1356 return -1; 1357 } 1358 } 1359 return 0; 1360 } 1361 1362 static int 1363 check_port_config(void) 1364 { 1365 unsigned portid; 1366 uint16_t i; 1367 1368 for (i = 0; i < nb_lcore_params; ++i) { 1369 portid = lcore_params[i].port_id; 1370 if ((enabled_port_mask & (1 << portid)) == 0) { 1371 printf("port %u is not enabled in port mask\n", 1372 portid); 1373 return -1; 1374 } 1375 if (!rte_eth_dev_is_valid_port(portid)) { 1376 printf("port %u is not present on the board\n", 1377 portid); 1378 return -1; 1379 } 1380 } 1381 return 0; 1382 } 1383 1384 static uint8_t 1385 get_port_n_rx_queues(const uint16_t port) 1386 { 1387 int queue = -1; 1388 uint16_t i; 1389 1390 for (i = 0; i < nb_lcore_params; ++i) { 1391 if (lcore_params[i].port_id == port && 1392 lcore_params[i].queue_id > queue) 1393 queue = lcore_params[i].queue_id; 1394 } 1395 return (uint8_t)(++queue); 1396 } 1397 1398 static int 1399 init_lcore_rx_queues(void) 1400 { 1401 uint16_t i, nb_rx_queue; 1402 uint8_t lcore; 1403 1404 for (i = 0; i < nb_lcore_params; ++i) { 1405 lcore = lcore_params[i].lcore_id; 1406 nb_rx_queue = lcore_conf[lcore].n_rx_queue; 1407 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1408 printf("error: too many queues (%u) for lcore: %u\n", 1409 (unsigned)nb_rx_queue + 1, (unsigned)lcore); 1410 return -1; 1411 } else { 1412 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1413 lcore_params[i].port_id; 1414 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1415 lcore_params[i].queue_id; 1416 lcore_conf[lcore].n_rx_queue++; 1417 } 1418 } 1419 return 0; 1420 } 1421 1422 /* display usage */ 1423 static void 1424 print_usage(const char *prgname) 1425 { 1426 printf ("%s [EAL options] -- -p PORTMASK -P" 1427 " [--config (port,queue,lcore)[,(port,queue,lcore]]" 1428 " [--high-perf-cores CORELIST" 1429 " [--perf-config (port,queue,hi_perf,lcore_index)[,(port,queue,hi_perf,lcore_index]]" 1430 " [--enable-jumbo [--max-pkt-len PKTLEN]]\n" 1431 " -p PORTMASK: hexadecimal bitmask of ports to configure\n" 1432 " -P : enable promiscuous mode\n" 1433 " --config (port,queue,lcore): rx queues configuration\n" 1434 " --high-perf-cores CORELIST: list of high performance cores\n" 1435 " --perf-config: similar as config, cores specified as indices" 1436 " for bins containing high or regular performance cores\n" 1437 " --no-numa: optional, disable numa awareness\n" 1438 " --enable-jumbo: enable jumbo frame" 1439 " which max packet len is PKTLEN in decimal (64-9600)\n" 1440 " --parse-ptype: parse packet type by software\n" 1441 " --empty-poll: enable empty poll detection" 1442 " follow (training_flag, high_threshold, med_threshold)\n" 1443 " --telemetry: enable telemetry mode, to update" 1444 " empty polls, full polls, and core busyness to telemetry\n", 1445 prgname); 1446 } 1447 1448 static int parse_max_pkt_len(const char *pktlen) 1449 { 1450 char *end = NULL; 1451 unsigned long len; 1452 1453 /* parse decimal string */ 1454 len = strtoul(pktlen, &end, 10); 1455 if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) 1456 return -1; 1457 1458 if (len == 0) 1459 return -1; 1460 1461 return len; 1462 } 1463 1464 static int 1465 parse_portmask(const char *portmask) 1466 { 1467 char *end = NULL; 1468 unsigned long pm; 1469 1470 /* parse hexadecimal string */ 1471 pm = strtoul(portmask, &end, 16); 1472 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1473 return -1; 1474 1475 if (pm == 0) 1476 return -1; 1477 1478 return pm; 1479 } 1480 1481 static int 1482 parse_config(const char *q_arg) 1483 { 1484 char s[256]; 1485 const char *p, *p0 = q_arg; 1486 char *end; 1487 enum fieldnames { 1488 FLD_PORT = 0, 1489 FLD_QUEUE, 1490 FLD_LCORE, 1491 _NUM_FLD 1492 }; 1493 unsigned long int_fld[_NUM_FLD]; 1494 char *str_fld[_NUM_FLD]; 1495 int i; 1496 unsigned size; 1497 1498 nb_lcore_params = 0; 1499 1500 while ((p = strchr(p0,'(')) != NULL) { 1501 ++p; 1502 if((p0 = strchr(p,')')) == NULL) 1503 return -1; 1504 1505 size = p0 - p; 1506 if(size >= sizeof(s)) 1507 return -1; 1508 1509 snprintf(s, sizeof(s), "%.*s", size, p); 1510 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1511 _NUM_FLD) 1512 return -1; 1513 for (i = 0; i < _NUM_FLD; i++){ 1514 errno = 0; 1515 int_fld[i] = strtoul(str_fld[i], &end, 0); 1516 if (errno != 0 || end == str_fld[i] || int_fld[i] > 1517 255) 1518 return -1; 1519 } 1520 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1521 printf("exceeded max number of lcore params: %hu\n", 1522 nb_lcore_params); 1523 return -1; 1524 } 1525 lcore_params_array[nb_lcore_params].port_id = 1526 (uint8_t)int_fld[FLD_PORT]; 1527 lcore_params_array[nb_lcore_params].queue_id = 1528 (uint8_t)int_fld[FLD_QUEUE]; 1529 lcore_params_array[nb_lcore_params].lcore_id = 1530 (uint8_t)int_fld[FLD_LCORE]; 1531 ++nb_lcore_params; 1532 } 1533 lcore_params = lcore_params_array; 1534 1535 return 0; 1536 } 1537 static int 1538 parse_ep_config(const char *q_arg) 1539 { 1540 char s[256]; 1541 const char *p = q_arg; 1542 char *end; 1543 int num_arg; 1544 1545 char *str_fld[3]; 1546 1547 int training_flag; 1548 int med_edpi; 1549 int hgh_edpi; 1550 1551 ep_med_edpi = EMPTY_POLL_MED_THRESHOLD; 1552 ep_hgh_edpi = EMPTY_POLL_MED_THRESHOLD; 1553 1554 strlcpy(s, p, sizeof(s)); 1555 1556 num_arg = rte_strsplit(s, sizeof(s), str_fld, 3, ','); 1557 1558 empty_poll_train = false; 1559 1560 if (num_arg == 0) 1561 return 0; 1562 1563 if (num_arg == 3) { 1564 1565 training_flag = strtoul(str_fld[0], &end, 0); 1566 med_edpi = strtoul(str_fld[1], &end, 0); 1567 hgh_edpi = strtoul(str_fld[2], &end, 0); 1568 1569 if (training_flag == 1) 1570 empty_poll_train = true; 1571 1572 if (med_edpi > 0) 1573 ep_med_edpi = med_edpi; 1574 1575 if (med_edpi > 0) 1576 ep_hgh_edpi = hgh_edpi; 1577 1578 } else { 1579 1580 return -1; 1581 } 1582 1583 return 0; 1584 1585 } 1586 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype" 1587 #define CMD_LINE_OPT_TELEMETRY "telemetry" 1588 1589 /* Parse the argument given in the command line of the application */ 1590 static int 1591 parse_args(int argc, char **argv) 1592 { 1593 int opt, ret; 1594 char **argvopt; 1595 int option_index; 1596 uint32_t limit; 1597 char *prgname = argv[0]; 1598 static struct option lgopts[] = { 1599 {"config", 1, 0, 0}, 1600 {"perf-config", 1, 0, 0}, 1601 {"high-perf-cores", 1, 0, 0}, 1602 {"no-numa", 0, 0, 0}, 1603 {"enable-jumbo", 0, 0, 0}, 1604 {"empty-poll", 1, 0, 0}, 1605 {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0}, 1606 {CMD_LINE_OPT_TELEMETRY, 0, 0, 0}, 1607 {NULL, 0, 0, 0} 1608 }; 1609 1610 argvopt = argv; 1611 1612 while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P", 1613 lgopts, &option_index)) != EOF) { 1614 1615 switch (opt) { 1616 /* portmask */ 1617 case 'p': 1618 enabled_port_mask = parse_portmask(optarg); 1619 if (enabled_port_mask == 0) { 1620 printf("invalid portmask\n"); 1621 print_usage(prgname); 1622 return -1; 1623 } 1624 break; 1625 case 'P': 1626 printf("Promiscuous mode selected\n"); 1627 promiscuous_on = 1; 1628 break; 1629 case 'l': 1630 limit = parse_max_pkt_len(optarg); 1631 freq_tlb[LOW] = limit; 1632 break; 1633 case 'm': 1634 limit = parse_max_pkt_len(optarg); 1635 freq_tlb[MED] = limit; 1636 break; 1637 case 'h': 1638 limit = parse_max_pkt_len(optarg); 1639 freq_tlb[HGH] = limit; 1640 break; 1641 /* long options */ 1642 case 0: 1643 if (!strncmp(lgopts[option_index].name, "config", 6)) { 1644 ret = parse_config(optarg); 1645 if (ret) { 1646 printf("invalid config\n"); 1647 print_usage(prgname); 1648 return -1; 1649 } 1650 } 1651 1652 if (!strncmp(lgopts[option_index].name, 1653 "perf-config", 11)) { 1654 ret = parse_perf_config(optarg); 1655 if (ret) { 1656 printf("invalid perf-config\n"); 1657 print_usage(prgname); 1658 return -1; 1659 } 1660 } 1661 1662 if (!strncmp(lgopts[option_index].name, 1663 "high-perf-cores", 15)) { 1664 ret = parse_perf_core_list(optarg); 1665 if (ret) { 1666 printf("invalid high-perf-cores\n"); 1667 print_usage(prgname); 1668 return -1; 1669 } 1670 } 1671 1672 if (!strncmp(lgopts[option_index].name, 1673 "no-numa", 7)) { 1674 printf("numa is disabled \n"); 1675 numa_on = 0; 1676 } 1677 1678 if (!strncmp(lgopts[option_index].name, 1679 "empty-poll", 10)) { 1680 if (app_mode == APP_MODE_TELEMETRY) { 1681 printf(" empty-poll cannot be enabled as telemetry mode is enabled\n"); 1682 return -1; 1683 } 1684 app_mode = APP_MODE_EMPTY_POLL; 1685 ret = parse_ep_config(optarg); 1686 1687 if (ret) { 1688 printf("invalid empty poll config\n"); 1689 print_usage(prgname); 1690 return -1; 1691 } 1692 printf("empty-poll is enabled\n"); 1693 } 1694 1695 if (!strncmp(lgopts[option_index].name, 1696 CMD_LINE_OPT_TELEMETRY, 1697 sizeof(CMD_LINE_OPT_TELEMETRY))) { 1698 if (app_mode == APP_MODE_EMPTY_POLL) { 1699 printf("telemetry mode cannot be enabled as empty poll mode is enabled\n"); 1700 return -1; 1701 } 1702 app_mode = APP_MODE_TELEMETRY; 1703 printf("telemetry mode is enabled\n"); 1704 } 1705 1706 if (!strncmp(lgopts[option_index].name, 1707 "enable-jumbo", 12)) { 1708 struct option lenopts = 1709 {"max-pkt-len", required_argument, \ 1710 0, 0}; 1711 1712 printf("jumbo frame is enabled \n"); 1713 port_conf.rxmode.offloads |= 1714 DEV_RX_OFFLOAD_JUMBO_FRAME; 1715 port_conf.txmode.offloads |= 1716 DEV_TX_OFFLOAD_MULTI_SEGS; 1717 1718 /** 1719 * if no max-pkt-len set, use the default value 1720 * RTE_ETHER_MAX_LEN 1721 */ 1722 if (0 == getopt_long(argc, argvopt, "", 1723 &lenopts, &option_index)) { 1724 ret = parse_max_pkt_len(optarg); 1725 if ((ret < 64) || 1726 (ret > MAX_JUMBO_PKT_LEN)){ 1727 printf("invalid packet " 1728 "length\n"); 1729 print_usage(prgname); 1730 return -1; 1731 } 1732 port_conf.rxmode.max_rx_pkt_len = ret; 1733 } 1734 printf("set jumbo frame " 1735 "max packet length to %u\n", 1736 (unsigned int)port_conf.rxmode.max_rx_pkt_len); 1737 } 1738 1739 if (!strncmp(lgopts[option_index].name, 1740 CMD_LINE_OPT_PARSE_PTYPE, 1741 sizeof(CMD_LINE_OPT_PARSE_PTYPE))) { 1742 printf("soft parse-ptype is enabled\n"); 1743 parse_ptype = 1; 1744 } 1745 1746 break; 1747 1748 default: 1749 print_usage(prgname); 1750 return -1; 1751 } 1752 } 1753 1754 if (optind >= 0) 1755 argv[optind-1] = prgname; 1756 1757 ret = optind-1; 1758 optind = 1; /* reset getopt lib */ 1759 return ret; 1760 } 1761 1762 static void 1763 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 1764 { 1765 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 1766 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 1767 printf("%s%s", name, buf); 1768 } 1769 1770 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1771 static void 1772 setup_hash(int socketid) 1773 { 1774 struct rte_hash_parameters ipv4_l3fwd_hash_params = { 1775 .name = NULL, 1776 .entries = L3FWD_HASH_ENTRIES, 1777 .key_len = sizeof(struct ipv4_5tuple), 1778 .hash_func = DEFAULT_HASH_FUNC, 1779 .hash_func_init_val = 0, 1780 }; 1781 1782 struct rte_hash_parameters ipv6_l3fwd_hash_params = { 1783 .name = NULL, 1784 .entries = L3FWD_HASH_ENTRIES, 1785 .key_len = sizeof(struct ipv6_5tuple), 1786 .hash_func = DEFAULT_HASH_FUNC, 1787 .hash_func_init_val = 0, 1788 }; 1789 1790 unsigned i; 1791 int ret; 1792 char s[64]; 1793 1794 /* create ipv4 hash */ 1795 snprintf(s, sizeof(s), "ipv4_l3fwd_hash_%d", socketid); 1796 ipv4_l3fwd_hash_params.name = s; 1797 ipv4_l3fwd_hash_params.socket_id = socketid; 1798 ipv4_l3fwd_lookup_struct[socketid] = 1799 rte_hash_create(&ipv4_l3fwd_hash_params); 1800 if (ipv4_l3fwd_lookup_struct[socketid] == NULL) 1801 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on " 1802 "socket %d\n", socketid); 1803 1804 /* create ipv6 hash */ 1805 snprintf(s, sizeof(s), "ipv6_l3fwd_hash_%d", socketid); 1806 ipv6_l3fwd_hash_params.name = s; 1807 ipv6_l3fwd_hash_params.socket_id = socketid; 1808 ipv6_l3fwd_lookup_struct[socketid] = 1809 rte_hash_create(&ipv6_l3fwd_hash_params); 1810 if (ipv6_l3fwd_lookup_struct[socketid] == NULL) 1811 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on " 1812 "socket %d\n", socketid); 1813 1814 1815 /* populate the ipv4 hash */ 1816 for (i = 0; i < RTE_DIM(ipv4_l3fwd_route_array); i++) { 1817 ret = rte_hash_add_key (ipv4_l3fwd_lookup_struct[socketid], 1818 (void *) &ipv4_l3fwd_route_array[i].key); 1819 if (ret < 0) { 1820 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the" 1821 "l3fwd hash on socket %d\n", i, socketid); 1822 } 1823 ipv4_l3fwd_out_if[ret] = ipv4_l3fwd_route_array[i].if_out; 1824 printf("Hash: Adding key\n"); 1825 print_ipv4_key(ipv4_l3fwd_route_array[i].key); 1826 } 1827 1828 /* populate the ipv6 hash */ 1829 for (i = 0; i < RTE_DIM(ipv6_l3fwd_route_array); i++) { 1830 ret = rte_hash_add_key (ipv6_l3fwd_lookup_struct[socketid], 1831 (void *) &ipv6_l3fwd_route_array[i].key); 1832 if (ret < 0) { 1833 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the" 1834 "l3fwd hash on socket %d\n", i, socketid); 1835 } 1836 ipv6_l3fwd_out_if[ret] = ipv6_l3fwd_route_array[i].if_out; 1837 printf("Hash: Adding key\n"); 1838 print_ipv6_key(ipv6_l3fwd_route_array[i].key); 1839 } 1840 } 1841 #endif 1842 1843 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 1844 static void 1845 setup_lpm(int socketid) 1846 { 1847 unsigned i; 1848 int ret; 1849 char s[64]; 1850 1851 /* create the LPM table */ 1852 struct rte_lpm_config lpm_ipv4_config; 1853 1854 lpm_ipv4_config.max_rules = IPV4_L3FWD_LPM_MAX_RULES; 1855 lpm_ipv4_config.number_tbl8s = 256; 1856 lpm_ipv4_config.flags = 0; 1857 1858 snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socketid); 1859 ipv4_l3fwd_lookup_struct[socketid] = 1860 rte_lpm_create(s, socketid, &lpm_ipv4_config); 1861 if (ipv4_l3fwd_lookup_struct[socketid] == NULL) 1862 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table" 1863 " on socket %d\n", socketid); 1864 1865 /* populate the LPM table */ 1866 for (i = 0; i < RTE_DIM(ipv4_l3fwd_route_array); i++) { 1867 ret = rte_lpm_add(ipv4_l3fwd_lookup_struct[socketid], 1868 ipv4_l3fwd_route_array[i].ip, 1869 ipv4_l3fwd_route_array[i].depth, 1870 ipv4_l3fwd_route_array[i].if_out); 1871 1872 if (ret < 0) { 1873 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the " 1874 "l3fwd LPM table on socket %d\n", 1875 i, socketid); 1876 } 1877 1878 printf("LPM: Adding route 0x%08x / %d (%d)\n", 1879 (unsigned)ipv4_l3fwd_route_array[i].ip, 1880 ipv4_l3fwd_route_array[i].depth, 1881 ipv4_l3fwd_route_array[i].if_out); 1882 } 1883 } 1884 #endif 1885 1886 static int 1887 init_mem(unsigned nb_mbuf) 1888 { 1889 struct lcore_conf *qconf; 1890 int socketid; 1891 unsigned lcore_id; 1892 char s[64]; 1893 1894 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1895 if (rte_lcore_is_enabled(lcore_id) == 0) 1896 continue; 1897 1898 if (numa_on) 1899 socketid = rte_lcore_to_socket_id(lcore_id); 1900 else 1901 socketid = 0; 1902 1903 if (socketid >= NB_SOCKETS) { 1904 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is " 1905 "out of range %d\n", socketid, 1906 lcore_id, NB_SOCKETS); 1907 } 1908 if (pktmbuf_pool[socketid] == NULL) { 1909 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 1910 pktmbuf_pool[socketid] = 1911 rte_pktmbuf_pool_create(s, nb_mbuf, 1912 MEMPOOL_CACHE_SIZE, 0, 1913 RTE_MBUF_DEFAULT_BUF_SIZE, 1914 socketid); 1915 if (pktmbuf_pool[socketid] == NULL) 1916 rte_exit(EXIT_FAILURE, 1917 "Cannot init mbuf pool on socket %d\n", 1918 socketid); 1919 else 1920 printf("Allocated mbuf pool on socket %d\n", 1921 socketid); 1922 1923 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 1924 setup_lpm(socketid); 1925 #else 1926 setup_hash(socketid); 1927 #endif 1928 } 1929 qconf = &lcore_conf[lcore_id]; 1930 qconf->ipv4_lookup_struct = ipv4_l3fwd_lookup_struct[socketid]; 1931 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1932 qconf->ipv6_lookup_struct = ipv6_l3fwd_lookup_struct[socketid]; 1933 #endif 1934 } 1935 return 0; 1936 } 1937 1938 /* Check the link status of all ports in up to 9s, and print them finally */ 1939 static void 1940 check_all_ports_link_status(uint32_t port_mask) 1941 { 1942 #define CHECK_INTERVAL 100 /* 100ms */ 1943 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1944 uint8_t count, all_ports_up, print_flag = 0; 1945 uint16_t portid; 1946 struct rte_eth_link link; 1947 int ret; 1948 1949 printf("\nChecking link status"); 1950 fflush(stdout); 1951 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1952 all_ports_up = 1; 1953 RTE_ETH_FOREACH_DEV(portid) { 1954 if ((port_mask & (1 << portid)) == 0) 1955 continue; 1956 memset(&link, 0, sizeof(link)); 1957 ret = rte_eth_link_get_nowait(portid, &link); 1958 if (ret < 0) { 1959 all_ports_up = 0; 1960 if (print_flag == 1) 1961 printf("Port %u link get failed: %s\n", 1962 portid, rte_strerror(-ret)); 1963 continue; 1964 } 1965 /* print link status if flag set */ 1966 if (print_flag == 1) { 1967 if (link.link_status) 1968 printf("Port %d Link Up - speed %u " 1969 "Mbps - %s\n", (uint8_t)portid, 1970 (unsigned)link.link_speed, 1971 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1972 ("full-duplex") : ("half-duplex\n")); 1973 else 1974 printf("Port %d Link Down\n", 1975 (uint8_t)portid); 1976 continue; 1977 } 1978 /* clear all_ports_up flag if any link down */ 1979 if (link.link_status == ETH_LINK_DOWN) { 1980 all_ports_up = 0; 1981 break; 1982 } 1983 } 1984 /* after finally printing all link status, get out */ 1985 if (print_flag == 1) 1986 break; 1987 1988 if (all_ports_up == 0) { 1989 printf("."); 1990 fflush(stdout); 1991 rte_delay_ms(CHECK_INTERVAL); 1992 } 1993 1994 /* set the print_flag if all ports up or timeout */ 1995 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1996 print_flag = 1; 1997 printf("done\n"); 1998 } 1999 } 2000 } 2001 2002 static int check_ptype(uint16_t portid) 2003 { 2004 int i, ret; 2005 int ptype_l3_ipv4 = 0; 2006 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 2007 int ptype_l3_ipv6 = 0; 2008 #endif 2009 uint32_t ptype_mask = RTE_PTYPE_L3_MASK; 2010 2011 ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, NULL, 0); 2012 if (ret <= 0) 2013 return 0; 2014 2015 uint32_t ptypes[ret]; 2016 2017 ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, ptypes, ret); 2018 for (i = 0; i < ret; ++i) { 2019 if (ptypes[i] & RTE_PTYPE_L3_IPV4) 2020 ptype_l3_ipv4 = 1; 2021 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 2022 if (ptypes[i] & RTE_PTYPE_L3_IPV6) 2023 ptype_l3_ipv6 = 1; 2024 #endif 2025 } 2026 2027 if (ptype_l3_ipv4 == 0) 2028 printf("port %d cannot parse RTE_PTYPE_L3_IPV4\n", portid); 2029 2030 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 2031 if (ptype_l3_ipv6 == 0) 2032 printf("port %d cannot parse RTE_PTYPE_L3_IPV6\n", portid); 2033 #endif 2034 2035 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 2036 if (ptype_l3_ipv4) 2037 #else /* APP_LOOKUP_EXACT_MATCH */ 2038 if (ptype_l3_ipv4 && ptype_l3_ipv6) 2039 #endif 2040 return 1; 2041 2042 return 0; 2043 2044 } 2045 2046 static int 2047 init_power_library(void) 2048 { 2049 unsigned int lcore_id; 2050 int ret = 0; 2051 2052 RTE_LCORE_FOREACH(lcore_id) { 2053 /* init power management library */ 2054 ret = rte_power_init(lcore_id); 2055 if (ret) { 2056 RTE_LOG(ERR, POWER, 2057 "Library initialization failed on core %u\n", 2058 lcore_id); 2059 return ret; 2060 } 2061 } 2062 return ret; 2063 } 2064 2065 static int 2066 deinit_power_library(void) 2067 { 2068 unsigned int lcore_id; 2069 int ret = 0; 2070 2071 RTE_LCORE_FOREACH(lcore_id) { 2072 /* deinit power management library */ 2073 ret = rte_power_exit(lcore_id); 2074 if (ret) { 2075 RTE_LOG(ERR, POWER, 2076 "Library deinitialization failed on core %u\n", 2077 lcore_id); 2078 return ret; 2079 } 2080 } 2081 return ret; 2082 } 2083 2084 static void 2085 get_current_stat_values(uint64_t *values) 2086 { 2087 unsigned int lcore_id = rte_lcore_id(); 2088 struct lcore_conf *qconf; 2089 uint64_t app_eps = 0, app_fps = 0, app_br = 0; 2090 uint64_t count = 0; 2091 2092 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2093 qconf = &lcore_conf[lcore_id]; 2094 if (qconf->n_rx_queue == 0) 2095 continue; 2096 count++; 2097 rte_spinlock_lock(&stats[lcore_id].telemetry_lock); 2098 app_eps += stats[lcore_id].ep_nep[1]; 2099 app_fps += stats[lcore_id].fp_nfp[1]; 2100 app_br += stats[lcore_id].br; 2101 rte_spinlock_unlock(&stats[lcore_id].telemetry_lock); 2102 } 2103 2104 if (count > 0) { 2105 values[0] = app_eps/count; 2106 values[1] = app_fps/count; 2107 values[2] = app_br/count; 2108 } else 2109 memset(values, 0, sizeof(uint64_t) * NUM_TELSTATS); 2110 2111 } 2112 2113 static void 2114 update_telemetry(__rte_unused struct rte_timer *tim, 2115 __rte_unused void *arg) 2116 { 2117 int ret; 2118 uint64_t values[NUM_TELSTATS] = {0}; 2119 2120 get_current_stat_values(values); 2121 ret = rte_metrics_update_values(RTE_METRICS_GLOBAL, telstats_index, 2122 values, RTE_DIM(values)); 2123 if (ret < 0) 2124 RTE_LOG(WARNING, POWER, "failed to update metrcis\n"); 2125 } 2126 2127 static int 2128 handle_app_stats(const char *cmd __rte_unused, 2129 const char *params __rte_unused, 2130 struct rte_tel_data *d) 2131 { 2132 uint64_t values[NUM_TELSTATS] = {0}; 2133 uint32_t i; 2134 2135 rte_tel_data_start_dict(d); 2136 get_current_stat_values(values); 2137 for (i = 0; i < NUM_TELSTATS; i++) 2138 rte_tel_data_add_dict_u64(d, telstats_strings[i].name, 2139 values[i]); 2140 return 0; 2141 } 2142 2143 static void 2144 telemetry_setup_timer(void) 2145 { 2146 int lcore_id = rte_lcore_id(); 2147 uint64_t hz = rte_get_timer_hz(); 2148 uint64_t ticks; 2149 2150 ticks = hz / TELEMETRY_INTERVALS_PER_SEC; 2151 rte_timer_reset_sync(&telemetry_timer, 2152 ticks, 2153 PERIODICAL, 2154 lcore_id, 2155 update_telemetry, 2156 NULL); 2157 } 2158 static void 2159 empty_poll_setup_timer(void) 2160 { 2161 int lcore_id = rte_lcore_id(); 2162 uint64_t hz = rte_get_timer_hz(); 2163 2164 struct ep_params *ep_ptr = ep_params; 2165 2166 ep_ptr->interval_ticks = hz / INTERVALS_PER_SECOND; 2167 2168 rte_timer_reset_sync(&ep_ptr->timer0, 2169 ep_ptr->interval_ticks, 2170 PERIODICAL, 2171 lcore_id, 2172 rte_empty_poll_detection, 2173 (void *)ep_ptr); 2174 2175 } 2176 static int 2177 launch_timer(unsigned int lcore_id) 2178 { 2179 int64_t prev_tsc = 0, cur_tsc, diff_tsc, cycles_10ms; 2180 2181 RTE_SET_USED(lcore_id); 2182 2183 2184 if (rte_get_master_lcore() != lcore_id) { 2185 rte_panic("timer on lcore:%d which is not master core:%d\n", 2186 lcore_id, 2187 rte_get_master_lcore()); 2188 } 2189 2190 RTE_LOG(INFO, POWER, "Bring up the Timer\n"); 2191 2192 if (app_mode == APP_MODE_EMPTY_POLL) 2193 empty_poll_setup_timer(); 2194 else 2195 telemetry_setup_timer(); 2196 2197 cycles_10ms = rte_get_timer_hz() / 100; 2198 2199 while (!is_done()) { 2200 cur_tsc = rte_rdtsc(); 2201 diff_tsc = cur_tsc - prev_tsc; 2202 if (diff_tsc > cycles_10ms) { 2203 rte_timer_manage(); 2204 prev_tsc = cur_tsc; 2205 cycles_10ms = rte_get_timer_hz() / 100; 2206 } 2207 } 2208 2209 RTE_LOG(INFO, POWER, "Timer_subsystem is done\n"); 2210 2211 return 0; 2212 } 2213 2214 2215 int 2216 main(int argc, char **argv) 2217 { 2218 struct lcore_conf *qconf; 2219 struct rte_eth_dev_info dev_info; 2220 struct rte_eth_txconf *txconf; 2221 int ret; 2222 uint16_t nb_ports; 2223 uint16_t queueid; 2224 unsigned lcore_id; 2225 uint64_t hz; 2226 uint32_t n_tx_queue, nb_lcores; 2227 uint32_t dev_rxq_num, dev_txq_num; 2228 uint8_t nb_rx_queue, queue, socketid; 2229 uint16_t portid; 2230 const char *ptr_strings[NUM_TELSTATS]; 2231 2232 /* catch SIGINT and restore cpufreq governor to ondemand */ 2233 signal(SIGINT, signal_exit_now); 2234 2235 /* init EAL */ 2236 ret = rte_eal_init(argc, argv); 2237 if (ret < 0) 2238 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 2239 argc -= ret; 2240 argv += ret; 2241 2242 /* init RTE timer library to be used late */ 2243 rte_timer_subsystem_init(); 2244 2245 /* parse application arguments (after the EAL ones) */ 2246 ret = parse_args(argc, argv); 2247 if (ret < 0) 2248 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n"); 2249 2250 if (app_mode != APP_MODE_TELEMETRY && init_power_library()) 2251 rte_exit(EXIT_FAILURE, "init_power_library failed\n"); 2252 2253 if (update_lcore_params() < 0) 2254 rte_exit(EXIT_FAILURE, "update_lcore_params failed\n"); 2255 2256 if (check_lcore_params() < 0) 2257 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n"); 2258 2259 ret = init_lcore_rx_queues(); 2260 if (ret < 0) 2261 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 2262 2263 nb_ports = rte_eth_dev_count_avail(); 2264 2265 if (check_port_config() < 0) 2266 rte_exit(EXIT_FAILURE, "check_port_config failed\n"); 2267 2268 nb_lcores = rte_lcore_count(); 2269 2270 /* initialize all ports */ 2271 RTE_ETH_FOREACH_DEV(portid) { 2272 struct rte_eth_conf local_port_conf = port_conf; 2273 2274 /* skip ports that are not enabled */ 2275 if ((enabled_port_mask & (1 << portid)) == 0) { 2276 printf("\nSkipping disabled port %d\n", portid); 2277 continue; 2278 } 2279 2280 /* init port */ 2281 printf("Initializing port %d ... ", portid ); 2282 fflush(stdout); 2283 2284 ret = rte_eth_dev_info_get(portid, &dev_info); 2285 if (ret != 0) 2286 rte_exit(EXIT_FAILURE, 2287 "Error during getting device (port %u) info: %s\n", 2288 portid, strerror(-ret)); 2289 2290 dev_rxq_num = dev_info.max_rx_queues; 2291 dev_txq_num = dev_info.max_tx_queues; 2292 2293 nb_rx_queue = get_port_n_rx_queues(portid); 2294 if (nb_rx_queue > dev_rxq_num) 2295 rte_exit(EXIT_FAILURE, 2296 "Cannot configure not existed rxq: " 2297 "port=%d\n", portid); 2298 2299 n_tx_queue = nb_lcores; 2300 if (n_tx_queue > dev_txq_num) 2301 n_tx_queue = dev_txq_num; 2302 printf("Creating queues: nb_rxq=%d nb_txq=%u... ", 2303 nb_rx_queue, (unsigned)n_tx_queue ); 2304 /* If number of Rx queue is 0, no need to enable Rx interrupt */ 2305 if (nb_rx_queue == 0) 2306 local_port_conf.intr_conf.rxq = 0; 2307 2308 ret = rte_eth_dev_info_get(portid, &dev_info); 2309 if (ret != 0) 2310 rte_exit(EXIT_FAILURE, 2311 "Error during getting device (port %u) info: %s\n", 2312 portid, strerror(-ret)); 2313 2314 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 2315 local_port_conf.txmode.offloads |= 2316 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 2317 2318 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 2319 dev_info.flow_type_rss_offloads; 2320 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 2321 port_conf.rx_adv_conf.rss_conf.rss_hf) { 2322 printf("Port %u modified RSS hash function based on hardware support," 2323 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 2324 portid, 2325 port_conf.rx_adv_conf.rss_conf.rss_hf, 2326 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 2327 } 2328 2329 ret = rte_eth_dev_configure(portid, nb_rx_queue, 2330 (uint16_t)n_tx_queue, &local_port_conf); 2331 if (ret < 0) 2332 rte_exit(EXIT_FAILURE, "Cannot configure device: " 2333 "err=%d, port=%d\n", ret, portid); 2334 2335 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 2336 &nb_txd); 2337 if (ret < 0) 2338 rte_exit(EXIT_FAILURE, 2339 "Cannot adjust number of descriptors: err=%d, port=%d\n", 2340 ret, portid); 2341 2342 ret = rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 2343 if (ret < 0) 2344 rte_exit(EXIT_FAILURE, 2345 "Cannot get MAC address: err=%d, port=%d\n", 2346 ret, portid); 2347 2348 print_ethaddr(" Address:", &ports_eth_addr[portid]); 2349 printf(", "); 2350 2351 /* init memory */ 2352 ret = init_mem(NB_MBUF); 2353 if (ret < 0) 2354 rte_exit(EXIT_FAILURE, "init_mem failed\n"); 2355 2356 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2357 if (rte_lcore_is_enabled(lcore_id) == 0) 2358 continue; 2359 2360 /* Initialize TX buffers */ 2361 qconf = &lcore_conf[lcore_id]; 2362 qconf->tx_buffer[portid] = rte_zmalloc_socket("tx_buffer", 2363 RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0, 2364 rte_eth_dev_socket_id(portid)); 2365 if (qconf->tx_buffer[portid] == NULL) 2366 rte_exit(EXIT_FAILURE, "Can't allocate tx buffer for port %u\n", 2367 portid); 2368 2369 rte_eth_tx_buffer_init(qconf->tx_buffer[portid], MAX_PKT_BURST); 2370 } 2371 2372 /* init one TX queue per couple (lcore,port) */ 2373 queueid = 0; 2374 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2375 if (rte_lcore_is_enabled(lcore_id) == 0) 2376 continue; 2377 2378 if (queueid >= dev_txq_num) 2379 continue; 2380 2381 if (numa_on) 2382 socketid = \ 2383 (uint8_t)rte_lcore_to_socket_id(lcore_id); 2384 else 2385 socketid = 0; 2386 2387 printf("txq=%u,%d,%d ", lcore_id, queueid, socketid); 2388 fflush(stdout); 2389 2390 txconf = &dev_info.default_txconf; 2391 txconf->offloads = local_port_conf.txmode.offloads; 2392 ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, 2393 socketid, txconf); 2394 if (ret < 0) 2395 rte_exit(EXIT_FAILURE, 2396 "rte_eth_tx_queue_setup: err=%d, " 2397 "port=%d\n", ret, portid); 2398 2399 qconf = &lcore_conf[lcore_id]; 2400 qconf->tx_queue_id[portid] = queueid; 2401 queueid++; 2402 2403 qconf->tx_port_id[qconf->n_tx_port] = portid; 2404 qconf->n_tx_port++; 2405 } 2406 printf("\n"); 2407 } 2408 2409 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2410 if (rte_lcore_is_enabled(lcore_id) == 0) 2411 continue; 2412 2413 if (app_mode == APP_MODE_LEGACY) { 2414 /* init timer structures for each enabled lcore */ 2415 rte_timer_init(&power_timers[lcore_id]); 2416 hz = rte_get_timer_hz(); 2417 rte_timer_reset(&power_timers[lcore_id], 2418 hz/TIMER_NUMBER_PER_SECOND, 2419 SINGLE, lcore_id, 2420 power_timer_cb, NULL); 2421 } 2422 qconf = &lcore_conf[lcore_id]; 2423 printf("\nInitializing rx queues on lcore %u ... ", lcore_id ); 2424 fflush(stdout); 2425 /* init RX queues */ 2426 for(queue = 0; queue < qconf->n_rx_queue; ++queue) { 2427 struct rte_eth_rxconf rxq_conf; 2428 2429 portid = qconf->rx_queue_list[queue].port_id; 2430 queueid = qconf->rx_queue_list[queue].queue_id; 2431 2432 if (numa_on) 2433 socketid = \ 2434 (uint8_t)rte_lcore_to_socket_id(lcore_id); 2435 else 2436 socketid = 0; 2437 2438 printf("rxq=%d,%d,%d ", portid, queueid, socketid); 2439 fflush(stdout); 2440 2441 ret = rte_eth_dev_info_get(portid, &dev_info); 2442 if (ret != 0) 2443 rte_exit(EXIT_FAILURE, 2444 "Error during getting device (port %u) info: %s\n", 2445 portid, strerror(-ret)); 2446 2447 rxq_conf = dev_info.default_rxconf; 2448 rxq_conf.offloads = port_conf.rxmode.offloads; 2449 ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd, 2450 socketid, &rxq_conf, 2451 pktmbuf_pool[socketid]); 2452 if (ret < 0) 2453 rte_exit(EXIT_FAILURE, 2454 "rte_eth_rx_queue_setup: err=%d, " 2455 "port=%d\n", ret, portid); 2456 2457 if (parse_ptype) { 2458 if (add_cb_parse_ptype(portid, queueid) < 0) 2459 rte_exit(EXIT_FAILURE, 2460 "Fail to add ptype cb\n"); 2461 } else if (!check_ptype(portid)) 2462 rte_exit(EXIT_FAILURE, 2463 "PMD can not provide needed ptypes\n"); 2464 } 2465 } 2466 2467 printf("\n"); 2468 2469 /* start ports */ 2470 RTE_ETH_FOREACH_DEV(portid) { 2471 if ((enabled_port_mask & (1 << portid)) == 0) { 2472 continue; 2473 } 2474 /* Start device */ 2475 ret = rte_eth_dev_start(portid); 2476 if (ret < 0) 2477 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, " 2478 "port=%d\n", ret, portid); 2479 /* 2480 * If enabled, put device in promiscuous mode. 2481 * This allows IO forwarding mode to forward packets 2482 * to itself through 2 cross-connected ports of the 2483 * target machine. 2484 */ 2485 if (promiscuous_on) { 2486 ret = rte_eth_promiscuous_enable(portid); 2487 if (ret != 0) 2488 rte_exit(EXIT_FAILURE, 2489 "rte_eth_promiscuous_enable: err=%s, port=%u\n", 2490 rte_strerror(-ret), portid); 2491 } 2492 /* initialize spinlock for each port */ 2493 rte_spinlock_init(&(locks[portid])); 2494 } 2495 2496 check_all_ports_link_status(enabled_port_mask); 2497 2498 if (app_mode == APP_MODE_EMPTY_POLL) { 2499 2500 if (empty_poll_train) { 2501 policy.state = TRAINING; 2502 } else { 2503 policy.state = MED_NORMAL; 2504 policy.med_base_edpi = ep_med_edpi; 2505 policy.hgh_base_edpi = ep_hgh_edpi; 2506 } 2507 2508 ret = rte_power_empty_poll_stat_init(&ep_params, 2509 freq_tlb, 2510 &policy); 2511 if (ret < 0) 2512 rte_exit(EXIT_FAILURE, "empty poll init failed"); 2513 } 2514 2515 2516 /* launch per-lcore init on every lcore */ 2517 if (app_mode == APP_MODE_LEGACY) { 2518 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 2519 } else if (app_mode == APP_MODE_EMPTY_POLL) { 2520 empty_poll_stop = false; 2521 rte_eal_mp_remote_launch(main_empty_poll_loop, NULL, 2522 SKIP_MASTER); 2523 } else { 2524 unsigned int i; 2525 2526 /* Init metrics library */ 2527 rte_metrics_init(rte_socket_id()); 2528 /** Register stats with metrics library */ 2529 for (i = 0; i < NUM_TELSTATS; i++) 2530 ptr_strings[i] = telstats_strings[i].name; 2531 2532 ret = rte_metrics_reg_names(ptr_strings, NUM_TELSTATS); 2533 if (ret >= 0) 2534 telstats_index = ret; 2535 else 2536 rte_exit(EXIT_FAILURE, "failed to register metrics names"); 2537 2538 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2539 rte_spinlock_init(&stats[lcore_id].telemetry_lock); 2540 } 2541 rte_timer_init(&telemetry_timer); 2542 rte_telemetry_register_cmd("/l3fwd-power/stats", 2543 handle_app_stats, 2544 "Returns global power stats. Parameters: None"); 2545 rte_eal_mp_remote_launch(main_telemetry_loop, NULL, 2546 SKIP_MASTER); 2547 } 2548 2549 if (app_mode == APP_MODE_EMPTY_POLL || app_mode == APP_MODE_TELEMETRY) 2550 launch_timer(rte_lcore_id()); 2551 2552 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2553 if (rte_eal_wait_lcore(lcore_id) < 0) 2554 return -1; 2555 } 2556 2557 RTE_ETH_FOREACH_DEV(portid) 2558 { 2559 if ((enabled_port_mask & (1 << portid)) == 0) 2560 continue; 2561 2562 rte_eth_dev_stop(portid); 2563 rte_eth_dev_close(portid); 2564 } 2565 2566 if (app_mode == APP_MODE_EMPTY_POLL) 2567 rte_power_empty_poll_stat_free(); 2568 2569 if (app_mode != APP_MODE_TELEMETRY && deinit_power_library()) 2570 rte_exit(EXIT_FAILURE, "deinit_power_library failed\n"); 2571 2572 if (rte_eal_cleanup() < 0) 2573 RTE_LOG(ERR, L3FWD_POWER, "EAL cleanup failed\n"); 2574 2575 return 0; 2576 } 2577