1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <stdint.h> 37 #include <inttypes.h> 38 #include <sys/types.h> 39 #include <string.h> 40 #include <sys/queue.h> 41 #include <stdarg.h> 42 #include <errno.h> 43 #include <getopt.h> 44 45 #include <tmmintrin.h> 46 #include <rte_common.h> 47 #include <rte_byteorder.h> 48 #include <rte_log.h> 49 #include <rte_memory.h> 50 #include <rte_memcpy.h> 51 #include <rte_memzone.h> 52 #include <rte_tailq.h> 53 #include <rte_eal.h> 54 #include <rte_per_lcore.h> 55 #include <rte_launch.h> 56 #include <rte_atomic.h> 57 #include <rte_cycles.h> 58 #include <rte_prefetch.h> 59 #include <rte_lcore.h> 60 #include <rte_per_lcore.h> 61 #include <rte_branch_prediction.h> 62 #include <rte_interrupts.h> 63 #include <rte_pci.h> 64 #include <rte_random.h> 65 #include <rte_debug.h> 66 #include <rte_ether.h> 67 #include <rte_ethdev.h> 68 #include <rte_ring.h> 69 #include <rte_mempool.h> 70 #include <rte_mbuf.h> 71 #include <rte_ip.h> 72 #include <rte_tcp.h> 73 #include <rte_udp.h> 74 #include <rte_string_fns.h> 75 76 #include "main.h" 77 78 #define APP_LOOKUP_EXACT_MATCH 0 79 #define APP_LOOKUP_LPM 1 80 #define DO_RFC_1812_CHECKS 81 82 #ifndef APP_LOOKUP_METHOD 83 #define APP_LOOKUP_METHOD APP_LOOKUP_LPM 84 #endif 85 86 #define ENABLE_MULTI_BUFFER_OPTIMIZE 1 87 88 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 89 #include <rte_hash.h> 90 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 91 #include <rte_lpm.h> 92 #include <rte_lpm6.h> 93 #else 94 #error "APP_LOOKUP_METHOD set to incorrect value" 95 #endif 96 97 #ifndef IPv6_BYTES 98 #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\ 99 "%02x%02x:%02x%02x:%02x%02x:%02x%02x" 100 #define IPv6_BYTES(addr) \ 101 addr[0], addr[1], addr[2], addr[3], \ 102 addr[4], addr[5], addr[6], addr[7], \ 103 addr[8], addr[9], addr[10], addr[11],\ 104 addr[12], addr[13],addr[14], addr[15] 105 #endif 106 107 108 #define RTE_LOGTYPE_L3FWD RTE_LOGTYPE_USER1 109 110 #define MAX_JUMBO_PKT_LEN 9600 111 112 #define IPV6_ADDR_LEN 16 113 114 #define MEMPOOL_CACHE_SIZE 256 115 116 #define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM) 117 118 /* 119 * This expression is used to calculate the number of mbufs needed depending on user input, taking 120 * into account memory for rx and tx hardware rings, cache per lcore and mtable per port per lcore. 121 * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum value of 8192 122 */ 123 124 #define NB_MBUF RTE_MAX ( \ 125 (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \ 126 nb_ports*nb_lcores*MAX_PKT_BURST + \ 127 nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT + \ 128 nb_lcores*MEMPOOL_CACHE_SIZE), \ 129 (unsigned)8192) 130 131 /* 132 * RX and TX Prefetch, Host, and Write-back threshold values should be 133 * carefully set for optimal performance. Consult the network 134 * controller's datasheet and supporting DPDK documentation for guidance 135 * on how these parameters should be set. 136 */ 137 #define RX_PTHRESH 8 /**< Default values of RX prefetch threshold reg. */ 138 #define RX_HTHRESH 8 /**< Default values of RX host threshold reg. */ 139 #define RX_WTHRESH 4 /**< Default values of RX write-back threshold reg. */ 140 141 /* 142 * These default values are optimized for use with the Intel(R) 82599 10 GbE 143 * Controller and the DPDK ixgbe PMD. Consider using other values for other 144 * network controllers and/or network drivers. 145 */ 146 #define TX_PTHRESH 36 /**< Default values of TX prefetch threshold reg. */ 147 #define TX_HTHRESH 0 /**< Default values of TX host threshold reg. */ 148 #define TX_WTHRESH 0 /**< Default values of TX write-back threshold reg. */ 149 150 #define MAX_PKT_BURST 32 151 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 152 153 #define NB_SOCKETS 8 154 155 /* Configure how many packets ahead to prefetch, when reading packets */ 156 #define PREFETCH_OFFSET 3 157 158 /* 159 * Configurable number of RX/TX ring descriptors 160 */ 161 #define RTE_TEST_RX_DESC_DEFAULT 128 162 #define RTE_TEST_TX_DESC_DEFAULT 512 163 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 164 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 165 166 /* ethernet addresses of ports */ 167 static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; 168 169 /* mask of enabled ports */ 170 static uint32_t enabled_port_mask = 0; 171 static int promiscuous_on = 0; /**< Ports set in promiscuous mode off by default. */ 172 static int numa_on = 1; /**< NUMA is enabled by default. */ 173 174 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 175 static int ipv6 = 0; /**< ipv6 is false by default. */ 176 #endif 177 178 struct mbuf_table { 179 uint16_t len; 180 struct rte_mbuf *m_table[MAX_PKT_BURST]; 181 }; 182 183 struct lcore_rx_queue { 184 uint8_t port_id; 185 uint8_t queue_id; 186 } __rte_cache_aligned; 187 188 #define MAX_RX_QUEUE_PER_LCORE 16 189 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 190 #define MAX_RX_QUEUE_PER_PORT 128 191 192 #define MAX_LCORE_PARAMS 1024 193 struct lcore_params { 194 uint8_t port_id; 195 uint8_t queue_id; 196 uint8_t lcore_id; 197 } __rte_cache_aligned; 198 199 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 200 static struct lcore_params lcore_params_array_default[] = { 201 {0, 0, 2}, 202 {0, 1, 2}, 203 {0, 2, 2}, 204 {1, 0, 2}, 205 {1, 1, 2}, 206 {1, 2, 2}, 207 {2, 0, 2}, 208 {3, 0, 3}, 209 {3, 1, 3}, 210 }; 211 212 static struct lcore_params * lcore_params = lcore_params_array_default; 213 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) / 214 sizeof(lcore_params_array_default[0]); 215 216 static struct rte_eth_conf port_conf = { 217 .rxmode = { 218 .max_rx_pkt_len = ETHER_MAX_LEN, 219 .split_hdr_size = 0, 220 .header_split = 0, /**< Header Split disabled */ 221 .hw_ip_checksum = 1, /**< IP checksum offload enabled */ 222 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 223 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 224 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 225 }, 226 .rx_adv_conf = { 227 .rss_conf = { 228 .rss_key = NULL, 229 .rss_hf = ETH_RSS_IPV4 | ETH_RSS_IPV6, 230 }, 231 }, 232 .txmode = { 233 .mq_mode = ETH_MQ_TX_NONE, 234 }, 235 }; 236 237 static const struct rte_eth_rxconf rx_conf = { 238 .rx_thresh = { 239 .pthresh = RX_PTHRESH, 240 .hthresh = RX_HTHRESH, 241 .wthresh = RX_WTHRESH, 242 }, 243 .rx_free_thresh = 32, 244 }; 245 246 static struct rte_eth_txconf tx_conf = { 247 .tx_thresh = { 248 .pthresh = TX_PTHRESH, 249 .hthresh = TX_HTHRESH, 250 .wthresh = TX_WTHRESH, 251 }, 252 .tx_free_thresh = 0, /* Use PMD default values */ 253 .tx_rs_thresh = 0, /* Use PMD default values */ 254 .txq_flags = (ETH_TXQ_FLAGS_NOMULTSEGS | 255 ETH_TXQ_FLAGS_NOVLANOFFL | 256 ETH_TXQ_FLAGS_NOXSUMSCTP | 257 ETH_TXQ_FLAGS_NOXSUMUDP | 258 ETH_TXQ_FLAGS_NOXSUMTCP) 259 260 }; 261 262 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS]; 263 264 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 265 266 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2 267 #include <rte_hash_crc.h> 268 #define DEFAULT_HASH_FUNC rte_hash_crc 269 #else 270 #include <rte_jhash.h> 271 #define DEFAULT_HASH_FUNC rte_jhash 272 #endif 273 274 struct ipv4_5tuple { 275 uint32_t ip_dst; 276 uint32_t ip_src; 277 uint16_t port_dst; 278 uint16_t port_src; 279 uint8_t proto; 280 } __attribute__((__packed__)); 281 282 union ipv4_5tuple_host { 283 struct { 284 uint8_t pad0; 285 uint8_t proto; 286 uint16_t pad1; 287 uint32_t ip_src; 288 uint32_t ip_dst; 289 uint16_t port_src; 290 uint16_t port_dst; 291 }; 292 __m128i xmm; 293 }; 294 295 #define XMM_NUM_IN_IPV6_5TUPLE 3 296 297 struct ipv6_5tuple { 298 uint8_t ip_dst[IPV6_ADDR_LEN]; 299 uint8_t ip_src[IPV6_ADDR_LEN]; 300 uint16_t port_dst; 301 uint16_t port_src; 302 uint8_t proto; 303 } __attribute__((__packed__)); 304 305 union ipv6_5tuple_host { 306 struct { 307 uint16_t pad0; 308 uint8_t proto; 309 uint8_t pad1; 310 uint8_t ip_src[IPV6_ADDR_LEN]; 311 uint8_t ip_dst[IPV6_ADDR_LEN]; 312 uint16_t port_src; 313 uint16_t port_dst; 314 uint64_t reserve; 315 }; 316 __m128i xmm[XMM_NUM_IN_IPV6_5TUPLE]; 317 }; 318 319 struct ipv4_l3fwd_route { 320 struct ipv4_5tuple key; 321 uint8_t if_out; 322 }; 323 324 struct ipv6_l3fwd_route { 325 struct ipv6_5tuple key; 326 uint8_t if_out; 327 }; 328 329 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { 330 {{IPv4(101,0,0,0), IPv4(100,10,0,1), 101, 11, IPPROTO_TCP}, 0}, 331 {{IPv4(201,0,0,0), IPv4(200,20,0,1), 102, 12, IPPROTO_TCP}, 1}, 332 {{IPv4(111,0,0,0), IPv4(100,30,0,1), 101, 11, IPPROTO_TCP}, 2}, 333 {{IPv4(211,0,0,0), IPv4(200,40,0,1), 102, 12, IPPROTO_TCP}, 3}, 334 }; 335 336 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = { 337 {{ 338 {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0}, 339 {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05}, 340 101, 11, IPPROTO_TCP}, 0}, 341 342 {{ 343 {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0}, 344 {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05}, 345 102, 12, IPPROTO_TCP}, 1}, 346 347 {{ 348 {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0}, 349 {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05}, 350 101, 11, IPPROTO_TCP}, 2}, 351 352 {{ 353 {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0}, 354 {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05}, 355 102, 12, IPPROTO_TCP}, 3}, 356 }; 357 358 typedef struct rte_hash lookup_struct_t; 359 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS]; 360 static lookup_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS]; 361 362 #ifdef RTE_ARCH_X86_64 363 /* default to 4 million hash entries (approx) */ 364 #define L3FWD_HASH_ENTRIES 1024*1024*4 365 #else 366 /* 32-bit has less address-space for hugepage memory, limit to 1M entries */ 367 #define L3FWD_HASH_ENTRIES 1024*1024*1 368 #endif 369 #define HASH_ENTRY_NUMBER_DEFAULT 4 370 371 static uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT; 372 373 static inline uint32_t 374 ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len, 375 uint32_t init_val) 376 { 377 const union ipv4_5tuple_host *k; 378 uint32_t t; 379 const uint32_t *p; 380 381 k = data; 382 t = k->proto; 383 p = (const uint32_t *)&k->port_src; 384 385 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2 386 init_val = rte_hash_crc_4byte(t, init_val); 387 init_val = rte_hash_crc_4byte(k->ip_src, init_val); 388 init_val = rte_hash_crc_4byte(k->ip_dst, init_val); 389 init_val = rte_hash_crc_4byte(*p, init_val); 390 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */ 391 init_val = rte_jhash_1word(t, init_val); 392 init_val = rte_jhash_1word(k->ip_src, init_val); 393 init_val = rte_jhash_1word(k->ip_dst, init_val); 394 init_val = rte_jhash_1word(*p, init_val); 395 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ 396 return (init_val); 397 } 398 399 static inline uint32_t 400 ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, uint32_t init_val) 401 { 402 const union ipv6_5tuple_host *k; 403 uint32_t t; 404 const uint32_t *p; 405 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2 406 const uint32_t *ip_src0, *ip_src1, *ip_src2, *ip_src3; 407 const uint32_t *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3; 408 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ 409 410 k = data; 411 t = k->proto; 412 p = (const uint32_t *)&k->port_src; 413 414 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2 415 ip_src0 = (const uint32_t *) k->ip_src; 416 ip_src1 = (const uint32_t *)(k->ip_src+4); 417 ip_src2 = (const uint32_t *)(k->ip_src+8); 418 ip_src3 = (const uint32_t *)(k->ip_src+12); 419 ip_dst0 = (const uint32_t *) k->ip_dst; 420 ip_dst1 = (const uint32_t *)(k->ip_dst+4); 421 ip_dst2 = (const uint32_t *)(k->ip_dst+8); 422 ip_dst3 = (const uint32_t *)(k->ip_dst+12); 423 init_val = rte_hash_crc_4byte(t, init_val); 424 init_val = rte_hash_crc_4byte(*ip_src0, init_val); 425 init_val = rte_hash_crc_4byte(*ip_src1, init_val); 426 init_val = rte_hash_crc_4byte(*ip_src2, init_val); 427 init_val = rte_hash_crc_4byte(*ip_src3, init_val); 428 init_val = rte_hash_crc_4byte(*ip_dst0, init_val); 429 init_val = rte_hash_crc_4byte(*ip_dst1, init_val); 430 init_val = rte_hash_crc_4byte(*ip_dst2, init_val); 431 init_val = rte_hash_crc_4byte(*ip_dst3, init_val); 432 init_val = rte_hash_crc_4byte(*p, init_val); 433 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */ 434 init_val = rte_jhash_1word(t, init_val); 435 init_val = rte_jhash(k->ip_src, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val); 436 init_val = rte_jhash(k->ip_dst, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val); 437 init_val = rte_jhash_1word(*p, init_val); 438 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ 439 return (init_val); 440 } 441 442 #define IPV4_L3FWD_NUM_ROUTES \ 443 (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0])) 444 445 #define IPV6_L3FWD_NUM_ROUTES \ 446 (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0])) 447 448 static uint8_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; 449 static uint8_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; 450 451 #endif 452 453 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 454 struct ipv4_l3fwd_route { 455 uint32_t ip; 456 uint8_t depth; 457 uint8_t if_out; 458 }; 459 460 struct ipv6_l3fwd_route { 461 uint8_t ip[16]; 462 uint8_t depth; 463 uint8_t if_out; 464 }; 465 466 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { 467 {IPv4(1,1,1,0), 24, 0}, 468 {IPv4(2,1,1,0), 24, 1}, 469 {IPv4(3,1,1,0), 24, 2}, 470 {IPv4(4,1,1,0), 24, 3}, 471 {IPv4(5,1,1,0), 24, 4}, 472 {IPv4(6,1,1,0), 24, 5}, 473 {IPv4(7,1,1,0), 24, 6}, 474 {IPv4(8,1,1,0), 24, 7}, 475 }; 476 477 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = { 478 {{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 0}, 479 {{2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 1}, 480 {{3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 2}, 481 {{4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 3}, 482 {{5,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 4}, 483 {{6,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 5}, 484 {{7,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 6}, 485 {{8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 7}, 486 }; 487 488 #define IPV4_L3FWD_NUM_ROUTES \ 489 (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0])) 490 #define IPV6_L3FWD_NUM_ROUTES \ 491 (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0])) 492 493 #define IPV4_L3FWD_LPM_MAX_RULES 1024 494 #define IPV6_L3FWD_LPM_MAX_RULES 1024 495 #define IPV6_L3FWD_LPM_NUMBER_TBL8S (1 << 16) 496 497 typedef struct rte_lpm lookup_struct_t; 498 typedef struct rte_lpm6 lookup6_struct_t; 499 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS]; 500 static lookup6_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS]; 501 #endif 502 503 struct lcore_conf { 504 uint16_t n_rx_queue; 505 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 506 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 507 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; 508 lookup_struct_t * ipv4_lookup_struct; 509 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 510 lookup6_struct_t * ipv6_lookup_struct; 511 #else 512 lookup_struct_t * ipv6_lookup_struct; 513 #endif 514 } __rte_cache_aligned; 515 516 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 517 518 /* Send burst of packets on an output interface */ 519 static inline int 520 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 521 { 522 struct rte_mbuf **m_table; 523 int ret; 524 uint16_t queueid; 525 526 queueid = qconf->tx_queue_id[port]; 527 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 528 529 ret = rte_eth_tx_burst(port, queueid, m_table, n); 530 if (unlikely(ret < n)) { 531 do { 532 rte_pktmbuf_free(m_table[ret]); 533 } while (++ret < n); 534 } 535 536 return 0; 537 } 538 539 /* Enqueue a single packet, and send burst if queue is filled */ 540 static inline int 541 send_single_packet(struct rte_mbuf *m, uint8_t port) 542 { 543 uint32_t lcore_id; 544 uint16_t len; 545 struct lcore_conf *qconf; 546 547 lcore_id = rte_lcore_id(); 548 549 qconf = &lcore_conf[lcore_id]; 550 len = qconf->tx_mbufs[port].len; 551 qconf->tx_mbufs[port].m_table[len] = m; 552 len++; 553 554 /* enough pkts to be sent */ 555 if (unlikely(len == MAX_PKT_BURST)) { 556 send_burst(qconf, MAX_PKT_BURST, port); 557 len = 0; 558 } 559 560 qconf->tx_mbufs[port].len = len; 561 return 0; 562 } 563 564 #ifdef DO_RFC_1812_CHECKS 565 static inline int 566 is_valid_ipv4_pkt(struct ipv4_hdr *pkt, uint32_t link_len) 567 { 568 /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */ 569 /* 570 * 1. The packet length reported by the Link Layer must be large 571 * enough to hold the minimum length legal IP datagram (20 bytes). 572 */ 573 if (link_len < sizeof(struct ipv4_hdr)) 574 return -1; 575 576 /* 2. The IP checksum must be correct. */ 577 /* this is checked in H/W */ 578 579 /* 580 * 3. The IP version number must be 4. If the version number is not 4 581 * then the packet may be another version of IP, such as IPng or 582 * ST-II. 583 */ 584 if (((pkt->version_ihl) >> 4) != 4) 585 return -3; 586 /* 587 * 4. The IP header length field must be large enough to hold the 588 * minimum length legal IP datagram (20 bytes = 5 words). 589 */ 590 if ((pkt->version_ihl & 0xf) < 5) 591 return -4; 592 593 /* 594 * 5. The IP total length field must be large enough to hold the IP 595 * datagram header, whose length is specified in the IP header length 596 * field. 597 */ 598 if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct ipv4_hdr)) 599 return -5; 600 601 return 0; 602 } 603 #endif 604 605 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 606 607 static __m128i mask0; 608 static __m128i mask1; 609 static __m128i mask2; 610 static inline uint8_t 611 get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, lookup_struct_t * ipv4_l3fwd_lookup_struct) 612 { 613 int ret = 0; 614 union ipv4_5tuple_host key; 615 616 ipv4_hdr = (uint8_t *)ipv4_hdr + offsetof(struct ipv4_hdr, time_to_live); 617 __m128i data = _mm_loadu_si128((__m128i*)(ipv4_hdr)); 618 /* Get 5 tuple: dst port, src port, dst IP address, src IP address and protocol */ 619 key.xmm = _mm_and_si128(data, mask0); 620 /* Find destination port */ 621 ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key); 622 return (uint8_t)((ret < 0)? portid : ipv4_l3fwd_out_if[ret]); 623 } 624 625 static inline uint8_t 626 get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, lookup_struct_t * ipv6_l3fwd_lookup_struct) 627 { 628 int ret = 0; 629 union ipv6_5tuple_host key; 630 631 ipv6_hdr = (uint8_t *)ipv6_hdr + offsetof(struct ipv6_hdr, payload_len); 632 __m128i data0 = _mm_loadu_si128((__m128i*)(ipv6_hdr)); 633 __m128i data1 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i))); 634 __m128i data2 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i)+sizeof(__m128i))); 635 /* Get part of 5 tuple: src IP address lower 96 bits and protocol */ 636 key.xmm[0] = _mm_and_si128(data0, mask1); 637 /* Get part of 5 tuple: dst IP address lower 96 bits and src IP address higher 32 bits */ 638 key.xmm[1] = data1; 639 /* Get part of 5 tuple: dst port and src port and dst IP address higher 32 bits */ 640 key.xmm[2] = _mm_and_si128(data2, mask2); 641 642 /* Find destination port */ 643 ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key); 644 return (uint8_t)((ret < 0)? portid : ipv6_l3fwd_out_if[ret]); 645 } 646 #endif 647 648 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 649 static inline uint8_t 650 get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, lookup_struct_t * ipv4_l3fwd_lookup_struct) 651 { 652 uint8_t next_hop; 653 654 return (uint8_t) ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, 655 rte_be_to_cpu_32(((struct ipv4_hdr*)ipv4_hdr)->dst_addr), &next_hop) == 0)? 656 next_hop : portid); 657 } 658 659 static inline uint8_t 660 get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, lookup6_struct_t * ipv6_l3fwd_lookup_struct) 661 { 662 uint8_t next_hop; 663 return (uint8_t) ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct, 664 ((struct ipv6_hdr*)ipv6_hdr)->dst_addr, &next_hop) == 0)? 665 next_hop : portid); 666 } 667 #endif 668 669 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) & (ENABLE_MULTI_BUFFER_OPTIMIZE == 1) 670 static inline void l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, struct lcore_conf *qconf); 671 672 #define MASK_ALL_PKTS 0xf 673 #define EXECLUDE_1ST_PKT 0xe 674 #define EXECLUDE_2ND_PKT 0xd 675 #define EXECLUDE_3RD_PKT 0xb 676 #define EXECLUDE_4TH_PKT 0x7 677 678 static inline void 679 simple_ipv4_fwd_4pkts(struct rte_mbuf* m[4], uint8_t portid, struct lcore_conf *qconf) 680 { 681 struct ether_hdr *eth_hdr[4]; 682 struct ipv4_hdr *ipv4_hdr[4]; 683 void *d_addr_bytes[4]; 684 uint8_t dst_port[4]; 685 int32_t ret[4]; 686 union ipv4_5tuple_host key[4]; 687 __m128i data[4]; 688 689 eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *); 690 eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *); 691 eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *); 692 eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *); 693 694 /* Handle IPv4 headers.*/ 695 ipv4_hdr[0] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[0], unsigned char *) + 696 sizeof(struct ether_hdr)); 697 ipv4_hdr[1] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[1], unsigned char *) + 698 sizeof(struct ether_hdr)); 699 ipv4_hdr[2] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[2], unsigned char *) + 700 sizeof(struct ether_hdr)); 701 ipv4_hdr[3] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[3], unsigned char *) + 702 sizeof(struct ether_hdr)); 703 704 #ifdef DO_RFC_1812_CHECKS 705 /* Check to make sure the packet is valid (RFC1812) */ 706 uint8_t valid_mask = MASK_ALL_PKTS; 707 if (is_valid_ipv4_pkt(ipv4_hdr[0], m[0]->pkt.pkt_len) < 0) { 708 rte_pktmbuf_free(m[0]); 709 valid_mask &= EXECLUDE_1ST_PKT; 710 } 711 if (is_valid_ipv4_pkt(ipv4_hdr[1], m[1]->pkt.pkt_len) < 0) { 712 rte_pktmbuf_free(m[1]); 713 valid_mask &= EXECLUDE_2ND_PKT; 714 } 715 if (is_valid_ipv4_pkt(ipv4_hdr[2], m[2]->pkt.pkt_len) < 0) { 716 rte_pktmbuf_free(m[2]); 717 valid_mask &= EXECLUDE_3RD_PKT; 718 } 719 if (is_valid_ipv4_pkt(ipv4_hdr[3], m[3]->pkt.pkt_len) < 0) { 720 rte_pktmbuf_free(m[3]); 721 valid_mask &= EXECLUDE_4TH_PKT; 722 } 723 if (unlikely(valid_mask != MASK_ALL_PKTS)) { 724 if (valid_mask == 0){ 725 return; 726 } else { 727 uint8_t i = 0; 728 for (i = 0; i < 4; i++) { 729 if ((0x1 << i) & valid_mask) { 730 l3fwd_simple_forward(m[i], portid, qconf); 731 } 732 } 733 return; 734 } 735 } 736 #endif // End of #ifdef DO_RFC_1812_CHECKS 737 738 data[0] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[0], unsigned char *) + 739 sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live))); 740 data[1] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[1], unsigned char *) + 741 sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live))); 742 data[2] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[2], unsigned char *) + 743 sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live))); 744 data[3] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[3], unsigned char *) + 745 sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live))); 746 747 key[0].xmm = _mm_and_si128(data[0], mask0); 748 key[1].xmm = _mm_and_si128(data[1], mask0); 749 key[2].xmm = _mm_and_si128(data[2], mask0); 750 key[3].xmm = _mm_and_si128(data[3], mask0); 751 752 const void *key_array[4] = {&key[0], &key[1], &key[2],&key[3]}; 753 rte_hash_lookup_multi(qconf->ipv4_lookup_struct, &key_array[0], 4, ret); 754 dst_port[0] = (uint8_t) ((ret[0] < 0) ? portid : ipv4_l3fwd_out_if[ret[0]]); 755 dst_port[1] = (uint8_t) ((ret[1] < 0) ? portid : ipv4_l3fwd_out_if[ret[1]]); 756 dst_port[2] = (uint8_t) ((ret[2] < 0) ? portid : ipv4_l3fwd_out_if[ret[2]]); 757 dst_port[3] = (uint8_t) ((ret[3] < 0) ? portid : ipv4_l3fwd_out_if[ret[3]]); 758 759 if (dst_port[0] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[0]) == 0) 760 dst_port[0] = portid; 761 if (dst_port[1] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[1]) == 0) 762 dst_port[1] = portid; 763 if (dst_port[2] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[2]) == 0) 764 dst_port[2] = portid; 765 if (dst_port[3] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[3]) == 0) 766 dst_port[3] = portid; 767 768 /* 02:00:00:00:00:xx */ 769 d_addr_bytes[0] = ð_hdr[0]->d_addr.addr_bytes[0]; 770 d_addr_bytes[1] = ð_hdr[1]->d_addr.addr_bytes[0]; 771 d_addr_bytes[2] = ð_hdr[2]->d_addr.addr_bytes[0]; 772 d_addr_bytes[3] = ð_hdr[3]->d_addr.addr_bytes[0]; 773 *((uint64_t *)d_addr_bytes[0]) = 0x000000000002 + ((uint64_t)dst_port[0] << 40); 774 *((uint64_t *)d_addr_bytes[1]) = 0x000000000002 + ((uint64_t)dst_port[1] << 40); 775 *((uint64_t *)d_addr_bytes[2]) = 0x000000000002 + ((uint64_t)dst_port[2] << 40); 776 *((uint64_t *)d_addr_bytes[3]) = 0x000000000002 + ((uint64_t)dst_port[3] << 40); 777 778 #ifdef DO_RFC_1812_CHECKS 779 /* Update time to live and header checksum */ 780 --(ipv4_hdr[0]->time_to_live); 781 --(ipv4_hdr[1]->time_to_live); 782 --(ipv4_hdr[2]->time_to_live); 783 --(ipv4_hdr[3]->time_to_live); 784 ++(ipv4_hdr[0]->hdr_checksum); 785 ++(ipv4_hdr[1]->hdr_checksum); 786 ++(ipv4_hdr[2]->hdr_checksum); 787 ++(ipv4_hdr[3]->hdr_checksum); 788 #endif 789 790 /* src addr */ 791 ether_addr_copy(&ports_eth_addr[dst_port[0]], ð_hdr[0]->s_addr); 792 ether_addr_copy(&ports_eth_addr[dst_port[1]], ð_hdr[1]->s_addr); 793 ether_addr_copy(&ports_eth_addr[dst_port[2]], ð_hdr[2]->s_addr); 794 ether_addr_copy(&ports_eth_addr[dst_port[3]], ð_hdr[3]->s_addr); 795 796 send_single_packet(m[0], (uint8_t)dst_port[0]); 797 send_single_packet(m[1], (uint8_t)dst_port[1]); 798 send_single_packet(m[2], (uint8_t)dst_port[2]); 799 send_single_packet(m[3], (uint8_t)dst_port[3]); 800 801 } 802 803 static inline void get_ipv6_5tuple(struct rte_mbuf* m0, __m128i mask0, __m128i mask1, 804 union ipv6_5tuple_host * key) 805 { 806 __m128i tmpdata0 = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m0, unsigned char *) 807 + sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len))); 808 __m128i tmpdata1 = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m0, unsigned char *) 809 + sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) 810 + sizeof(__m128i))); 811 __m128i tmpdata2 = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m0, unsigned char *) 812 + sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) 813 + sizeof(__m128i) + sizeof(__m128i))); 814 key->xmm[0] = _mm_and_si128(tmpdata0, mask0); 815 key->xmm[1] = tmpdata1; 816 key->xmm[2] = _mm_and_si128(tmpdata2, mask1); 817 return; 818 } 819 820 static inline void 821 simple_ipv6_fwd_4pkts(struct rte_mbuf* m[4], uint8_t portid, struct lcore_conf *qconf) 822 { 823 struct ether_hdr *eth_hdr[4]; 824 __attribute__((unused)) struct ipv6_hdr *ipv6_hdr[4]; 825 void *d_addr_bytes[4]; 826 uint8_t dst_port[4]; 827 int32_t ret[4]; 828 union ipv6_5tuple_host key[4]; 829 830 eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *); 831 eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *); 832 eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *); 833 eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *); 834 835 /* Handle IPv6 headers.*/ 836 ipv6_hdr[0] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[0], unsigned char *) + 837 sizeof(struct ether_hdr)); 838 ipv6_hdr[1] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[1], unsigned char *) + 839 sizeof(struct ether_hdr)); 840 ipv6_hdr[2] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[2], unsigned char *) + 841 sizeof(struct ether_hdr)); 842 ipv6_hdr[3] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[3], unsigned char *) + 843 sizeof(struct ether_hdr)); 844 845 get_ipv6_5tuple(m[0], mask1, mask2, &key[0]); 846 get_ipv6_5tuple(m[1], mask1, mask2, &key[1]); 847 get_ipv6_5tuple(m[2], mask1, mask2, &key[2]); 848 get_ipv6_5tuple(m[3], mask1, mask2, &key[3]); 849 850 const void *key_array[4] = {&key[0], &key[1], &key[2],&key[3]}; 851 rte_hash_lookup_multi(qconf->ipv6_lookup_struct, &key_array[0], 4, ret); 852 dst_port[0] = (uint8_t) ((ret[0] < 0)? portid:ipv6_l3fwd_out_if[ret[0]]); 853 dst_port[1] = (uint8_t) ((ret[1] < 0)? portid:ipv6_l3fwd_out_if[ret[1]]); 854 dst_port[2] = (uint8_t) ((ret[2] < 0)? portid:ipv6_l3fwd_out_if[ret[2]]); 855 dst_port[3] = (uint8_t) ((ret[3] < 0)? portid:ipv6_l3fwd_out_if[ret[3]]); 856 857 if (dst_port[0] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[0]) == 0) 858 dst_port[0] = portid; 859 if (dst_port[1] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[1]) == 0) 860 dst_port[1] = portid; 861 if (dst_port[2] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[2]) == 0) 862 dst_port[2] = portid; 863 if (dst_port[3] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[3]) == 0) 864 dst_port[3] = portid; 865 866 /* 02:00:00:00:00:xx */ 867 d_addr_bytes[0] = ð_hdr[0]->d_addr.addr_bytes[0]; 868 d_addr_bytes[1] = ð_hdr[1]->d_addr.addr_bytes[0]; 869 d_addr_bytes[2] = ð_hdr[2]->d_addr.addr_bytes[0]; 870 d_addr_bytes[3] = ð_hdr[3]->d_addr.addr_bytes[0]; 871 *((uint64_t *)d_addr_bytes[0]) = 0x000000000002 + ((uint64_t)dst_port[0] << 40); 872 *((uint64_t *)d_addr_bytes[1]) = 0x000000000002 + ((uint64_t)dst_port[1] << 40); 873 *((uint64_t *)d_addr_bytes[2]) = 0x000000000002 + ((uint64_t)dst_port[2] << 40); 874 *((uint64_t *)d_addr_bytes[3]) = 0x000000000002 + ((uint64_t)dst_port[3] << 40); 875 876 /* src addr */ 877 ether_addr_copy(&ports_eth_addr[dst_port[0]], ð_hdr[0]->s_addr); 878 ether_addr_copy(&ports_eth_addr[dst_port[1]], ð_hdr[1]->s_addr); 879 ether_addr_copy(&ports_eth_addr[dst_port[2]], ð_hdr[2]->s_addr); 880 ether_addr_copy(&ports_eth_addr[dst_port[3]], ð_hdr[3]->s_addr); 881 882 send_single_packet(m[0], (uint8_t)dst_port[0]); 883 send_single_packet(m[1], (uint8_t)dst_port[1]); 884 send_single_packet(m[2], (uint8_t)dst_port[2]); 885 send_single_packet(m[3], (uint8_t)dst_port[3]); 886 887 } 888 #endif // End of #if(APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)&(ENABLE_MULTI_BUFFER_OPTIMIZE == 1) 889 890 static inline __attribute__((always_inline)) void 891 l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, struct lcore_conf *qconf) 892 { 893 struct ether_hdr *eth_hdr; 894 struct ipv4_hdr *ipv4_hdr; 895 void *d_addr_bytes; 896 uint8_t dst_port; 897 898 eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); 899 900 if (m->ol_flags & PKT_RX_IPV4_HDR) { 901 /* Handle IPv4 headers.*/ 902 ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m, unsigned char *) + 903 sizeof(struct ether_hdr)); 904 905 #ifdef DO_RFC_1812_CHECKS 906 /* Check to make sure the packet is valid (RFC1812) */ 907 if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt.pkt_len) < 0) { 908 rte_pktmbuf_free(m); 909 return; 910 } 911 #endif 912 913 dst_port = get_ipv4_dst_port(ipv4_hdr, portid, qconf->ipv4_lookup_struct); 914 if (dst_port >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port) == 0) 915 dst_port = portid; 916 917 /* 02:00:00:00:00:xx */ 918 d_addr_bytes = ð_hdr->d_addr.addr_bytes[0]; 919 *((uint64_t *)d_addr_bytes) = 0x000000000002 + ((uint64_t)dst_port << 40); 920 921 #ifdef DO_RFC_1812_CHECKS 922 /* Update time to live and header checksum */ 923 --(ipv4_hdr->time_to_live); 924 ++(ipv4_hdr->hdr_checksum); 925 #endif 926 927 /* src addr */ 928 ether_addr_copy(&ports_eth_addr[dst_port], ð_hdr->s_addr); 929 930 send_single_packet(m, dst_port); 931 932 } else { 933 /* Handle IPv6 headers.*/ 934 struct ipv6_hdr *ipv6_hdr; 935 936 ipv6_hdr = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m, unsigned char *) + 937 sizeof(struct ether_hdr)); 938 939 dst_port = get_ipv6_dst_port(ipv6_hdr, portid, qconf->ipv6_lookup_struct); 940 941 if (dst_port >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port) == 0) 942 dst_port = portid; 943 944 /* 02:00:00:00:00:xx */ 945 d_addr_bytes = ð_hdr->d_addr.addr_bytes[0]; 946 *((uint64_t *)d_addr_bytes) = 0x000000000002 + ((uint64_t)dst_port << 40); 947 948 /* src addr */ 949 ether_addr_copy(&ports_eth_addr[dst_port], ð_hdr->s_addr); 950 951 send_single_packet(m, dst_port); 952 } 953 954 } 955 956 /* main processing loop */ 957 static int 958 main_loop(__attribute__((unused)) void *dummy) 959 { 960 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 961 unsigned lcore_id; 962 uint64_t prev_tsc, diff_tsc, cur_tsc; 963 int i, j, nb_rx; 964 uint8_t portid, queueid; 965 struct lcore_conf *qconf; 966 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; 967 968 prev_tsc = 0; 969 970 lcore_id = rte_lcore_id(); 971 qconf = &lcore_conf[lcore_id]; 972 973 if (qconf->n_rx_queue == 0) { 974 RTE_LOG(INFO, L3FWD, "lcore %u has nothing to do\n", lcore_id); 975 return 0; 976 } 977 978 RTE_LOG(INFO, L3FWD, "entering main loop on lcore %u\n", lcore_id); 979 980 for (i = 0; i < qconf->n_rx_queue; i++) { 981 982 portid = qconf->rx_queue_list[i].port_id; 983 queueid = qconf->rx_queue_list[i].queue_id; 984 RTE_LOG(INFO, L3FWD, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", lcore_id, 985 portid, queueid); 986 } 987 988 while (1) { 989 990 cur_tsc = rte_rdtsc(); 991 992 /* 993 * TX burst queue drain 994 */ 995 diff_tsc = cur_tsc - prev_tsc; 996 if (unlikely(diff_tsc > drain_tsc)) { 997 998 /* 999 * This could be optimized (use queueid instead of 1000 * portid), but it is not called so often 1001 */ 1002 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 1003 if (qconf->tx_mbufs[portid].len == 0) 1004 continue; 1005 send_burst(&lcore_conf[lcore_id], 1006 qconf->tx_mbufs[portid].len, 1007 portid); 1008 qconf->tx_mbufs[portid].len = 0; 1009 } 1010 1011 prev_tsc = cur_tsc; 1012 } 1013 1014 /* 1015 * Read packet from RX queues 1016 */ 1017 for (i = 0; i < qconf->n_rx_queue; ++i) { 1018 portid = qconf->rx_queue_list[i].port_id; 1019 queueid = qconf->rx_queue_list[i].queue_id; 1020 nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, MAX_PKT_BURST); 1021 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) & (ENABLE_MULTI_BUFFER_OPTIMIZE == 1) 1022 { 1023 /* Send nb_rx - nb_rx%4 packets in groups of 4.*/ 1024 int32_t n = RTE_ALIGN_FLOOR(nb_rx, 4); 1025 for (j = 0; j < n ; j+=4) { 1026 uint32_t ol_flag = pkts_burst[j]->ol_flags 1027 & pkts_burst[j+1]->ol_flags 1028 & pkts_burst[j+2]->ol_flags 1029 & pkts_burst[j+3]->ol_flags; 1030 if (ol_flag & PKT_RX_IPV4_HDR ) { 1031 simple_ipv4_fwd_4pkts(&pkts_burst[j], 1032 portid, qconf); 1033 } else if (ol_flag & PKT_RX_IPV6_HDR) { 1034 simple_ipv6_fwd_4pkts(&pkts_burst[j], 1035 portid, qconf); 1036 } else { 1037 l3fwd_simple_forward(pkts_burst[j], 1038 portid, qconf); 1039 l3fwd_simple_forward(pkts_burst[j+1], 1040 portid, qconf); 1041 l3fwd_simple_forward(pkts_burst[j+2], 1042 portid, qconf); 1043 l3fwd_simple_forward(pkts_burst[j+3], 1044 portid, qconf); 1045 } 1046 } 1047 for (; j < nb_rx ; j++) { 1048 l3fwd_simple_forward(pkts_burst[j], 1049 portid, qconf); 1050 } 1051 } 1052 #else 1053 /* Prefetch first packets */ 1054 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1055 rte_prefetch0(rte_pktmbuf_mtod( 1056 pkts_burst[j], void *)); 1057 } 1058 1059 /* Prefetch and forward already prefetched packets */ 1060 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1061 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1062 j + PREFETCH_OFFSET], void *)); 1063 l3fwd_simple_forward(pkts_burst[j], portid, qconf); 1064 } 1065 1066 /* Forward remaining prefetched packets */ 1067 for (; j < nb_rx; j++) { 1068 l3fwd_simple_forward(pkts_burst[j], portid, qconf); 1069 } 1070 #endif // End of #if((ENABLE_MULTI_BUFFER_OPTIMIZE == 1)&(APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)) 1071 } 1072 } 1073 } 1074 1075 static int 1076 check_lcore_params(void) 1077 { 1078 uint8_t queue, lcore; 1079 uint16_t i; 1080 int socketid; 1081 1082 for (i = 0; i < nb_lcore_params; ++i) { 1083 queue = lcore_params[i].queue_id; 1084 if (queue >= MAX_RX_QUEUE_PER_PORT) { 1085 printf("invalid queue number: %hhu\n", queue); 1086 return -1; 1087 } 1088 lcore = lcore_params[i].lcore_id; 1089 if (!rte_lcore_is_enabled(lcore)) { 1090 printf("error: lcore %hhu is not enabled in lcore mask\n", lcore); 1091 return -1; 1092 } 1093 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) && 1094 (numa_on == 0)) { 1095 printf("warning: lcore %hhu is on socket %d with numa off \n", 1096 lcore, socketid); 1097 } 1098 } 1099 return 0; 1100 } 1101 1102 static int 1103 check_port_config(const unsigned nb_ports) 1104 { 1105 unsigned portid; 1106 uint16_t i; 1107 1108 for (i = 0; i < nb_lcore_params; ++i) { 1109 portid = lcore_params[i].port_id; 1110 if ((enabled_port_mask & (1 << portid)) == 0) { 1111 printf("port %u is not enabled in port mask\n", portid); 1112 return -1; 1113 } 1114 if (portid >= nb_ports) { 1115 printf("port %u is not present on the board\n", portid); 1116 return -1; 1117 } 1118 } 1119 return 0; 1120 } 1121 1122 static uint8_t 1123 get_port_n_rx_queues(const uint8_t port) 1124 { 1125 int queue = -1; 1126 uint16_t i; 1127 1128 for (i = 0; i < nb_lcore_params; ++i) { 1129 if (lcore_params[i].port_id == port && lcore_params[i].queue_id > queue) 1130 queue = lcore_params[i].queue_id; 1131 } 1132 return (uint8_t)(++queue); 1133 } 1134 1135 static int 1136 init_lcore_rx_queues(void) 1137 { 1138 uint16_t i, nb_rx_queue; 1139 uint8_t lcore; 1140 1141 for (i = 0; i < nb_lcore_params; ++i) { 1142 lcore = lcore_params[i].lcore_id; 1143 nb_rx_queue = lcore_conf[lcore].n_rx_queue; 1144 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1145 printf("error: too many queues (%u) for lcore: %u\n", 1146 (unsigned)nb_rx_queue + 1, (unsigned)lcore); 1147 return -1; 1148 } else { 1149 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1150 lcore_params[i].port_id; 1151 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1152 lcore_params[i].queue_id; 1153 lcore_conf[lcore].n_rx_queue++; 1154 } 1155 } 1156 return 0; 1157 } 1158 1159 /* display usage */ 1160 static void 1161 print_usage(const char *prgname) 1162 { 1163 printf ("%s [EAL options] -- -p PORTMASK -P" 1164 " [--config (port,queue,lcore)[,(port,queue,lcore]]" 1165 " [--enable-jumbo [--max-pkt-len PKTLEN]]\n" 1166 " -p PORTMASK: hexadecimal bitmask of ports to configure\n" 1167 " -P : enable promiscuous mode\n" 1168 " --config (port,queue,lcore): rx queues configuration\n" 1169 " --no-numa: optional, disable numa awareness\n" 1170 " --ipv6: optional, specify it if running ipv6 packets\n" 1171 " --enable-jumbo: enable jumbo frame" 1172 " which max packet len is PKTLEN in decimal (64-9600)\n" 1173 " --hash-entry-num: specify the hash entry number in hexadecimal to be setup\n", 1174 prgname); 1175 } 1176 1177 static int parse_max_pkt_len(const char *pktlen) 1178 { 1179 char *end = NULL; 1180 unsigned long len; 1181 1182 /* parse decimal string */ 1183 len = strtoul(pktlen, &end, 10); 1184 if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) 1185 return -1; 1186 1187 if (len == 0) 1188 return -1; 1189 1190 return len; 1191 } 1192 1193 static int 1194 parse_portmask(const char *portmask) 1195 { 1196 char *end = NULL; 1197 unsigned long pm; 1198 1199 /* parse hexadecimal string */ 1200 pm = strtoul(portmask, &end, 16); 1201 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1202 return -1; 1203 1204 if (pm == 0) 1205 return -1; 1206 1207 return pm; 1208 } 1209 1210 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1211 static int 1212 parse_hash_entry_number(const char *hash_entry_num) 1213 { 1214 char *end = NULL; 1215 unsigned long hash_en; 1216 /* parse hexadecimal string */ 1217 hash_en = strtoul(hash_entry_num, &end, 16); 1218 if ((hash_entry_num[0] == '\0') || (end == NULL) || (*end != '\0')) 1219 return -1; 1220 1221 if (hash_en == 0) 1222 return -1; 1223 1224 return hash_en; 1225 } 1226 #endif 1227 1228 static int 1229 parse_config(const char *q_arg) 1230 { 1231 char s[256]; 1232 const char *p, *p0 = q_arg; 1233 char *end; 1234 enum fieldnames { 1235 FLD_PORT = 0, 1236 FLD_QUEUE, 1237 FLD_LCORE, 1238 _NUM_FLD 1239 }; 1240 unsigned long int_fld[_NUM_FLD]; 1241 char *str_fld[_NUM_FLD]; 1242 int i; 1243 unsigned size; 1244 1245 nb_lcore_params = 0; 1246 1247 while ((p = strchr(p0,'(')) != NULL) { 1248 ++p; 1249 if((p0 = strchr(p,')')) == NULL) 1250 return -1; 1251 1252 size = p0 - p; 1253 if(size >= sizeof(s)) 1254 return -1; 1255 1256 rte_snprintf(s, sizeof(s), "%.*s", size, p); 1257 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD) 1258 return -1; 1259 for (i = 0; i < _NUM_FLD; i++){ 1260 errno = 0; 1261 int_fld[i] = strtoul(str_fld[i], &end, 0); 1262 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1263 return -1; 1264 } 1265 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1266 printf("exceeded max number of lcore params: %hu\n", 1267 nb_lcore_params); 1268 return -1; 1269 } 1270 lcore_params_array[nb_lcore_params].port_id = (uint8_t)int_fld[FLD_PORT]; 1271 lcore_params_array[nb_lcore_params].queue_id = (uint8_t)int_fld[FLD_QUEUE]; 1272 lcore_params_array[nb_lcore_params].lcore_id = (uint8_t)int_fld[FLD_LCORE]; 1273 ++nb_lcore_params; 1274 } 1275 lcore_params = lcore_params_array; 1276 return 0; 1277 } 1278 1279 #define CMD_LINE_OPT_CONFIG "config" 1280 #define CMD_LINE_OPT_NO_NUMA "no-numa" 1281 #define CMD_LINE_OPT_IPV6 "ipv6" 1282 #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo" 1283 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num" 1284 1285 /* Parse the argument given in the command line of the application */ 1286 static int 1287 parse_args(int argc, char **argv) 1288 { 1289 int opt, ret; 1290 char **argvopt; 1291 int option_index; 1292 char *prgname = argv[0]; 1293 static struct option lgopts[] = { 1294 {CMD_LINE_OPT_CONFIG, 1, 0, 0}, 1295 {CMD_LINE_OPT_NO_NUMA, 0, 0, 0}, 1296 {CMD_LINE_OPT_IPV6, 0, 0, 0}, 1297 {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, 0}, 1298 {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, 0}, 1299 {NULL, 0, 0, 0} 1300 }; 1301 1302 argvopt = argv; 1303 1304 while ((opt = getopt_long(argc, argvopt, "p:P", 1305 lgopts, &option_index)) != EOF) { 1306 1307 switch (opt) { 1308 /* portmask */ 1309 case 'p': 1310 enabled_port_mask = parse_portmask(optarg); 1311 if (enabled_port_mask == 0) { 1312 printf("invalid portmask\n"); 1313 print_usage(prgname); 1314 return -1; 1315 } 1316 break; 1317 case 'P': 1318 printf("Promiscuous mode selected\n"); 1319 promiscuous_on = 1; 1320 break; 1321 1322 /* long options */ 1323 case 0: 1324 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_CONFIG, 1325 sizeof (CMD_LINE_OPT_CONFIG))) { 1326 ret = parse_config(optarg); 1327 if (ret) { 1328 printf("invalid config\n"); 1329 print_usage(prgname); 1330 return -1; 1331 } 1332 } 1333 1334 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_NUMA, 1335 sizeof(CMD_LINE_OPT_NO_NUMA))) { 1336 printf("numa is disabled \n"); 1337 numa_on = 0; 1338 } 1339 1340 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1341 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_IPV6, 1342 sizeof(CMD_LINE_OPT_IPV6))) { 1343 printf("ipv6 is specified \n"); 1344 ipv6 = 1; 1345 } 1346 #endif 1347 1348 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ENABLE_JUMBO, 1349 sizeof (CMD_LINE_OPT_ENABLE_JUMBO))) { 1350 struct option lenopts = {"max-pkt-len", required_argument, 0, 0}; 1351 1352 printf("jumbo frame is enabled - disabling simple TX path\n"); 1353 port_conf.rxmode.jumbo_frame = 1; 1354 tx_conf.txq_flags = 0; 1355 1356 /* if no max-pkt-len set, use the default value ETHER_MAX_LEN */ 1357 if (0 == getopt_long(argc, argvopt, "", &lenopts, &option_index)) { 1358 ret = parse_max_pkt_len(optarg); 1359 if ((ret < 64) || (ret > MAX_JUMBO_PKT_LEN)){ 1360 printf("invalid packet length\n"); 1361 print_usage(prgname); 1362 return -1; 1363 } 1364 port_conf.rxmode.max_rx_pkt_len = ret; 1365 } 1366 printf("set jumbo frame max packet length to %u\n", 1367 (unsigned int)port_conf.rxmode.max_rx_pkt_len); 1368 } 1369 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1370 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_HASH_ENTRY_NUM, 1371 sizeof(CMD_LINE_OPT_HASH_ENTRY_NUM))) { 1372 ret = parse_hash_entry_number(optarg); 1373 if ((ret > 0) && (ret <= L3FWD_HASH_ENTRIES)) { 1374 hash_entry_number = ret; 1375 } else { 1376 printf("invalid hash entry number\n"); 1377 print_usage(prgname); 1378 return -1; 1379 } 1380 } 1381 #endif 1382 break; 1383 1384 default: 1385 print_usage(prgname); 1386 return -1; 1387 } 1388 } 1389 1390 if (optind >= 0) 1391 argv[optind-1] = prgname; 1392 1393 ret = optind-1; 1394 optind = 0; /* reset getopt lib */ 1395 return ret; 1396 } 1397 1398 static void 1399 print_ethaddr(const char *name, const struct ether_addr *eth_addr) 1400 { 1401 printf ("%s%02X:%02X:%02X:%02X:%02X:%02X", name, 1402 eth_addr->addr_bytes[0], 1403 eth_addr->addr_bytes[1], 1404 eth_addr->addr_bytes[2], 1405 eth_addr->addr_bytes[3], 1406 eth_addr->addr_bytes[4], 1407 eth_addr->addr_bytes[5]); 1408 } 1409 1410 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1411 1412 static void convert_ipv4_5tuple(struct ipv4_5tuple* key1, 1413 union ipv4_5tuple_host* key2) 1414 { 1415 key2->ip_dst = rte_cpu_to_be_32(key1->ip_dst); 1416 key2->ip_src = rte_cpu_to_be_32(key1->ip_src); 1417 key2->port_dst = rte_cpu_to_be_16(key1->port_dst); 1418 key2->port_src = rte_cpu_to_be_16(key1->port_src); 1419 key2->proto = key1->proto; 1420 key2->pad0 = 0; 1421 key2->pad1 = 0; 1422 return; 1423 } 1424 1425 static void convert_ipv6_5tuple(struct ipv6_5tuple* key1, 1426 union ipv6_5tuple_host* key2) 1427 { 1428 uint32_t i; 1429 for (i = 0; i < 16; i++) 1430 { 1431 key2->ip_dst[i] = key1->ip_dst[i]; 1432 key2->ip_src[i] = key1->ip_src[i]; 1433 } 1434 key2->port_dst = rte_cpu_to_be_16(key1->port_dst); 1435 key2->port_src = rte_cpu_to_be_16(key1->port_src); 1436 key2->proto = key1->proto; 1437 key2->pad0 = 0; 1438 key2->pad1 = 0; 1439 key2->reserve = 0; 1440 return; 1441 } 1442 1443 #define BYTE_VALUE_MAX 256 1444 #define ALL_32_BITS 0xffffffff 1445 #define BIT_8_TO_15 0x0000ff00 1446 static inline void 1447 populate_ipv4_few_flow_into_table(const struct rte_hash* h) 1448 { 1449 uint32_t i; 1450 int32_t ret; 1451 uint32_t array_len = sizeof(ipv4_l3fwd_route_array)/sizeof(ipv4_l3fwd_route_array[0]); 1452 1453 mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15); 1454 for (i = 0; i < array_len; i++) { 1455 struct ipv4_l3fwd_route entry; 1456 union ipv4_5tuple_host newkey; 1457 entry = ipv4_l3fwd_route_array[i]; 1458 convert_ipv4_5tuple(&entry.key, &newkey); 1459 ret = rte_hash_add_key (h,(void *) &newkey); 1460 if (ret < 0) { 1461 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the" 1462 "l3fwd hash.\n", i); 1463 } 1464 ipv4_l3fwd_out_if[ret] = entry.if_out; 1465 } 1466 printf("Hash: Adding 0x%x keys\n", array_len); 1467 } 1468 1469 #define BIT_16_TO_23 0x00ff0000 1470 static inline void 1471 populate_ipv6_few_flow_into_table(const struct rte_hash* h) 1472 { 1473 uint32_t i; 1474 int32_t ret; 1475 uint32_t array_len = sizeof(ipv6_l3fwd_route_array)/sizeof(ipv6_l3fwd_route_array[0]); 1476 1477 mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23); 1478 mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS); 1479 for (i = 0; i < array_len; i++) { 1480 struct ipv6_l3fwd_route entry; 1481 union ipv6_5tuple_host newkey; 1482 entry = ipv6_l3fwd_route_array[i]; 1483 convert_ipv6_5tuple(&entry.key, &newkey); 1484 ret = rte_hash_add_key (h, (void *) &newkey); 1485 if (ret < 0) { 1486 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the" 1487 "l3fwd hash.\n", i); 1488 } 1489 ipv6_l3fwd_out_if[ret] = entry.if_out; 1490 } 1491 printf("Hash: Adding 0x%xkeys\n", array_len); 1492 } 1493 1494 #define NUMBER_PORT_USED 4 1495 static inline void 1496 populate_ipv4_many_flow_into_table(const struct rte_hash* h, 1497 unsigned int nr_flow) 1498 { 1499 unsigned i; 1500 mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15); 1501 for (i = 0; i < nr_flow; i++) { 1502 struct ipv4_l3fwd_route entry; 1503 union ipv4_5tuple_host newkey; 1504 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX); 1505 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX); 1506 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX)); 1507 /* Create the ipv4 exact match flow */ 1508 switch (i & (NUMBER_PORT_USED -1)) { 1509 case 0: 1510 entry = ipv4_l3fwd_route_array[0]; 1511 entry.key.ip_dst = IPv4(101,c,b,a); 1512 break; 1513 case 1: 1514 entry = ipv4_l3fwd_route_array[1]; 1515 entry.key.ip_dst = IPv4(201,c,b,a); 1516 break; 1517 case 2: 1518 entry = ipv4_l3fwd_route_array[2]; 1519 entry.key.ip_dst = IPv4(111,c,b,a); 1520 break; 1521 case 3: 1522 entry = ipv4_l3fwd_route_array[3]; 1523 entry.key.ip_dst = IPv4(211,c,b,a); 1524 break; 1525 }; 1526 convert_ipv4_5tuple(&entry.key, &newkey); 1527 int32_t ret = rte_hash_add_key(h,(void *) &newkey); 1528 if (ret < 0) { 1529 rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i); 1530 } 1531 ipv4_l3fwd_out_if[ret] = (uint8_t) entry.if_out; 1532 1533 } 1534 printf("Hash: Adding 0x%x keys\n", nr_flow); 1535 } 1536 1537 static inline void 1538 populate_ipv6_many_flow_into_table(const struct rte_hash* h, 1539 unsigned int nr_flow) 1540 { 1541 unsigned i; 1542 mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23); 1543 mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS); 1544 for (i = 0; i < nr_flow; i++) { 1545 struct ipv6_l3fwd_route entry; 1546 union ipv6_5tuple_host newkey; 1547 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX); 1548 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX); 1549 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX)); 1550 /* Create the ipv6 exact match flow */ 1551 switch (i & (NUMBER_PORT_USED - 1)) { 1552 case 0: entry = ipv6_l3fwd_route_array[0]; break; 1553 case 1: entry = ipv6_l3fwd_route_array[1]; break; 1554 case 2: entry = ipv6_l3fwd_route_array[2]; break; 1555 case 3: entry = ipv6_l3fwd_route_array[3]; break; 1556 }; 1557 entry.key.ip_dst[13] = c; 1558 entry.key.ip_dst[14] = b; 1559 entry.key.ip_dst[15] = a; 1560 convert_ipv6_5tuple(&entry.key, &newkey); 1561 int32_t ret = rte_hash_add_key(h,(void *) &newkey); 1562 if (ret < 0) { 1563 rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i); 1564 } 1565 ipv6_l3fwd_out_if[ret] = (uint8_t) entry.if_out; 1566 1567 } 1568 printf("Hash: Adding 0x%x keys\n", nr_flow); 1569 } 1570 1571 static void 1572 setup_hash(int socketid) 1573 { 1574 struct rte_hash_parameters ipv4_l3fwd_hash_params = { 1575 .name = NULL, 1576 .entries = L3FWD_HASH_ENTRIES, 1577 .bucket_entries = 4, 1578 .key_len = sizeof(union ipv4_5tuple_host), 1579 .hash_func = ipv4_hash_crc, 1580 .hash_func_init_val = 0, 1581 }; 1582 1583 struct rte_hash_parameters ipv6_l3fwd_hash_params = { 1584 .name = NULL, 1585 .entries = L3FWD_HASH_ENTRIES, 1586 .bucket_entries = 4, 1587 .key_len = sizeof(union ipv6_5tuple_host), 1588 .hash_func = ipv6_hash_crc, 1589 .hash_func_init_val = 0, 1590 }; 1591 1592 char s[64]; 1593 1594 /* create ipv4 hash */ 1595 rte_snprintf(s, sizeof(s), "ipv4_l3fwd_hash_%d", socketid); 1596 ipv4_l3fwd_hash_params.name = s; 1597 ipv4_l3fwd_hash_params.socket_id = socketid; 1598 ipv4_l3fwd_lookup_struct[socketid] = rte_hash_create(&ipv4_l3fwd_hash_params); 1599 if (ipv4_l3fwd_lookup_struct[socketid] == NULL) 1600 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on " 1601 "socket %d\n", socketid); 1602 1603 /* create ipv6 hash */ 1604 rte_snprintf(s, sizeof(s), "ipv6_l3fwd_hash_%d", socketid); 1605 ipv6_l3fwd_hash_params.name = s; 1606 ipv6_l3fwd_hash_params.socket_id = socketid; 1607 ipv6_l3fwd_lookup_struct[socketid] = rte_hash_create(&ipv6_l3fwd_hash_params); 1608 if (ipv6_l3fwd_lookup_struct[socketid] == NULL) 1609 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on " 1610 "socket %d\n", socketid); 1611 1612 if (hash_entry_number != HASH_ENTRY_NUMBER_DEFAULT) { 1613 /* For testing hash matching with a large number of flows we 1614 * generate millions of IP 5-tuples with an incremented dst 1615 * address to initialize the hash table. */ 1616 if (ipv6 == 0) { 1617 /* populate the ipv4 hash */ 1618 populate_ipv4_many_flow_into_table( 1619 ipv4_l3fwd_lookup_struct[socketid], hash_entry_number); 1620 } else { 1621 /* populate the ipv6 hash */ 1622 populate_ipv6_many_flow_into_table( 1623 ipv6_l3fwd_lookup_struct[socketid], hash_entry_number); 1624 } 1625 } else { 1626 /* Use data in ipv4/ipv6 l3fwd lookup table directly to initialize the hash table */ 1627 if (ipv6 == 0) { 1628 /* populate the ipv4 hash */ 1629 populate_ipv4_few_flow_into_table(ipv4_l3fwd_lookup_struct[socketid]); 1630 } else { 1631 /* populate the ipv6 hash */ 1632 populate_ipv6_few_flow_into_table(ipv6_l3fwd_lookup_struct[socketid]); 1633 } 1634 } 1635 } 1636 #endif 1637 1638 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 1639 static void 1640 setup_lpm(int socketid) 1641 { 1642 struct rte_lpm6_config config; 1643 unsigned i; 1644 int ret; 1645 char s[64]; 1646 1647 /* create the LPM table */ 1648 rte_snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socketid); 1649 ipv4_l3fwd_lookup_struct[socketid] = rte_lpm_create(s, socketid, 1650 IPV4_L3FWD_LPM_MAX_RULES, 0); 1651 if (ipv4_l3fwd_lookup_struct[socketid] == NULL) 1652 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table" 1653 " on socket %d\n", socketid); 1654 1655 /* populate the LPM table */ 1656 for (i = 0; i < IPV4_L3FWD_NUM_ROUTES; i++) { 1657 ret = rte_lpm_add(ipv4_l3fwd_lookup_struct[socketid], 1658 ipv4_l3fwd_route_array[i].ip, 1659 ipv4_l3fwd_route_array[i].depth, 1660 ipv4_l3fwd_route_array[i].if_out); 1661 1662 if (ret < 0) { 1663 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the " 1664 "l3fwd LPM table on socket %d\n", 1665 i, socketid); 1666 } 1667 1668 printf("LPM: Adding route 0x%08x / %d (%d)\n", 1669 (unsigned)ipv4_l3fwd_route_array[i].ip, 1670 ipv4_l3fwd_route_array[i].depth, 1671 ipv4_l3fwd_route_array[i].if_out); 1672 } 1673 1674 /* create the LPM6 table */ 1675 rte_snprintf(s, sizeof(s), "IPV6_L3FWD_LPM_%d", socketid); 1676 1677 config.max_rules = IPV6_L3FWD_LPM_MAX_RULES; 1678 config.number_tbl8s = IPV6_L3FWD_LPM_NUMBER_TBL8S; 1679 config.flags = 0; 1680 ipv6_l3fwd_lookup_struct[socketid] = rte_lpm6_create(s, socketid, 1681 &config); 1682 if (ipv6_l3fwd_lookup_struct[socketid] == NULL) 1683 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table" 1684 " on socket %d\n", socketid); 1685 1686 /* populate the LPM table */ 1687 for (i = 0; i < IPV6_L3FWD_NUM_ROUTES; i++) { 1688 ret = rte_lpm6_add(ipv6_l3fwd_lookup_struct[socketid], 1689 ipv6_l3fwd_route_array[i].ip, 1690 ipv6_l3fwd_route_array[i].depth, 1691 ipv6_l3fwd_route_array[i].if_out); 1692 1693 if (ret < 0) { 1694 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the " 1695 "l3fwd LPM table on socket %d\n", 1696 i, socketid); 1697 } 1698 1699 printf("LPM: Adding route %s / %d (%d)\n", 1700 "IPV6", 1701 ipv6_l3fwd_route_array[i].depth, 1702 ipv6_l3fwd_route_array[i].if_out); 1703 } 1704 } 1705 #endif 1706 1707 static int 1708 init_mem(unsigned nb_mbuf) 1709 { 1710 struct lcore_conf *qconf; 1711 int socketid; 1712 unsigned lcore_id; 1713 char s[64]; 1714 1715 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1716 if (rte_lcore_is_enabled(lcore_id) == 0) 1717 continue; 1718 1719 if (numa_on) 1720 socketid = rte_lcore_to_socket_id(lcore_id); 1721 else 1722 socketid = 0; 1723 1724 if (socketid >= NB_SOCKETS) { 1725 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 1726 socketid, lcore_id, NB_SOCKETS); 1727 } 1728 if (pktmbuf_pool[socketid] == NULL) { 1729 rte_snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 1730 pktmbuf_pool[socketid] = 1731 rte_mempool_create(s, nb_mbuf, MBUF_SIZE, MEMPOOL_CACHE_SIZE, 1732 sizeof(struct rte_pktmbuf_pool_private), 1733 rte_pktmbuf_pool_init, NULL, 1734 rte_pktmbuf_init, NULL, 1735 socketid, 0); 1736 if (pktmbuf_pool[socketid] == NULL) 1737 rte_exit(EXIT_FAILURE, 1738 "Cannot init mbuf pool on socket %d\n", socketid); 1739 else 1740 printf("Allocated mbuf pool on socket %d\n", socketid); 1741 1742 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 1743 setup_lpm(socketid); 1744 #else 1745 setup_hash(socketid); 1746 #endif 1747 } 1748 qconf = &lcore_conf[lcore_id]; 1749 qconf->ipv4_lookup_struct = ipv4_l3fwd_lookup_struct[socketid]; 1750 qconf->ipv6_lookup_struct = ipv6_l3fwd_lookup_struct[socketid]; 1751 } 1752 return 0; 1753 } 1754 1755 /* Check the link status of all ports in up to 9s, and print them finally */ 1756 static void 1757 check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) 1758 { 1759 #define CHECK_INTERVAL 100 /* 100ms */ 1760 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1761 uint8_t portid, count, all_ports_up, print_flag = 0; 1762 struct rte_eth_link link; 1763 1764 printf("\nChecking link status"); 1765 fflush(stdout); 1766 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1767 all_ports_up = 1; 1768 for (portid = 0; portid < port_num; portid++) { 1769 if ((port_mask & (1 << portid)) == 0) 1770 continue; 1771 memset(&link, 0, sizeof(link)); 1772 rte_eth_link_get_nowait(portid, &link); 1773 /* print link status if flag set */ 1774 if (print_flag == 1) { 1775 if (link.link_status) 1776 printf("Port %d Link Up - speed %u " 1777 "Mbps - %s\n", (uint8_t)portid, 1778 (unsigned)link.link_speed, 1779 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1780 ("full-duplex") : ("half-duplex\n")); 1781 else 1782 printf("Port %d Link Down\n", 1783 (uint8_t)portid); 1784 continue; 1785 } 1786 /* clear all_ports_up flag if any link down */ 1787 if (link.link_status == 0) { 1788 all_ports_up = 0; 1789 break; 1790 } 1791 } 1792 /* after finally printing all link status, get out */ 1793 if (print_flag == 1) 1794 break; 1795 1796 if (all_ports_up == 0) { 1797 printf("."); 1798 fflush(stdout); 1799 rte_delay_ms(CHECK_INTERVAL); 1800 } 1801 1802 /* set the print_flag if all ports up or timeout */ 1803 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1804 print_flag = 1; 1805 printf("done\n"); 1806 } 1807 } 1808 } 1809 1810 int 1811 MAIN(int argc, char **argv) 1812 { 1813 struct lcore_conf *qconf; 1814 int ret; 1815 unsigned nb_ports; 1816 uint16_t queueid; 1817 unsigned lcore_id; 1818 uint32_t n_tx_queue, nb_lcores; 1819 uint8_t portid, nb_rx_queue, queue, socketid; 1820 1821 /* init EAL */ 1822 ret = rte_eal_init(argc, argv); 1823 if (ret < 0) 1824 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 1825 argc -= ret; 1826 argv += ret; 1827 1828 /* parse application arguments (after the EAL ones) */ 1829 ret = parse_args(argc, argv); 1830 if (ret < 0) 1831 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n"); 1832 1833 if (check_lcore_params() < 0) 1834 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n"); 1835 1836 ret = init_lcore_rx_queues(); 1837 if (ret < 0) 1838 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 1839 1840 1841 /* init driver(s) */ 1842 if (rte_pmd_init_all() < 0) 1843 rte_exit(EXIT_FAILURE, "Cannot init pmd\n"); 1844 1845 if (rte_eal_pci_probe() < 0) 1846 rte_exit(EXIT_FAILURE, "Cannot probe PCI\n"); 1847 1848 nb_ports = rte_eth_dev_count(); 1849 if (nb_ports > RTE_MAX_ETHPORTS) 1850 nb_ports = RTE_MAX_ETHPORTS; 1851 1852 if (check_port_config(nb_ports) < 0) 1853 rte_exit(EXIT_FAILURE, "check_port_config failed\n"); 1854 1855 nb_lcores = rte_lcore_count(); 1856 1857 /* initialize all ports */ 1858 for (portid = 0; portid < nb_ports; portid++) { 1859 /* skip ports that are not enabled */ 1860 if ((enabled_port_mask & (1 << portid)) == 0) { 1861 printf("\nSkipping disabled port %d\n", portid); 1862 continue; 1863 } 1864 1865 /* init port */ 1866 printf("Initializing port %d ... ", portid ); 1867 fflush(stdout); 1868 1869 nb_rx_queue = get_port_n_rx_queues(portid); 1870 n_tx_queue = nb_lcores; 1871 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT) 1872 n_tx_queue = MAX_TX_QUEUE_PER_PORT; 1873 printf("Creating queues: nb_rxq=%d nb_txq=%u... ", 1874 nb_rx_queue, (unsigned)n_tx_queue ); 1875 ret = rte_eth_dev_configure(portid, nb_rx_queue, 1876 (uint16_t)n_tx_queue, &port_conf); 1877 if (ret < 0) 1878 rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%d\n", 1879 ret, portid); 1880 1881 rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 1882 print_ethaddr(" Address:", &ports_eth_addr[portid]); 1883 printf(", "); 1884 1885 /* init memory */ 1886 ret = init_mem(NB_MBUF); 1887 if (ret < 0) 1888 rte_exit(EXIT_FAILURE, "init_mem failed\n"); 1889 1890 /* init one TX queue per couple (lcore,port) */ 1891 queueid = 0; 1892 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1893 if (rte_lcore_is_enabled(lcore_id) == 0) 1894 continue; 1895 1896 if (numa_on) 1897 socketid = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1898 else 1899 socketid = 0; 1900 1901 printf("txq=%u,%d,%d ", lcore_id, queueid, socketid); 1902 fflush(stdout); 1903 ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, 1904 socketid, &tx_conf); 1905 if (ret < 0) 1906 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: err=%d, " 1907 "port=%d\n", ret, portid); 1908 1909 qconf = &lcore_conf[lcore_id]; 1910 qconf->tx_queue_id[portid] = queueid; 1911 queueid++; 1912 } 1913 printf("\n"); 1914 } 1915 1916 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1917 if (rte_lcore_is_enabled(lcore_id) == 0) 1918 continue; 1919 qconf = &lcore_conf[lcore_id]; 1920 printf("\nInitializing rx queues on lcore %u ... ", lcore_id ); 1921 fflush(stdout); 1922 /* init RX queues */ 1923 for(queue = 0; queue < qconf->n_rx_queue; ++queue) { 1924 portid = qconf->rx_queue_list[queue].port_id; 1925 queueid = qconf->rx_queue_list[queue].queue_id; 1926 1927 if (numa_on) 1928 socketid = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1929 else 1930 socketid = 0; 1931 1932 printf("rxq=%d,%d,%d ", portid, queueid, socketid); 1933 fflush(stdout); 1934 1935 ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd, 1936 socketid, &rx_conf, pktmbuf_pool[socketid]); 1937 if (ret < 0) 1938 rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup: err=%d," 1939 "port=%d\n", ret, portid); 1940 } 1941 } 1942 1943 printf("\n"); 1944 1945 /* start ports */ 1946 for (portid = 0; portid < nb_ports; portid++) { 1947 if ((enabled_port_mask & (1 << portid)) == 0) { 1948 continue; 1949 } 1950 /* Start device */ 1951 ret = rte_eth_dev_start(portid); 1952 if (ret < 0) 1953 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, port=%d\n", 1954 ret, portid); 1955 1956 /* 1957 * If enabled, put device in promiscuous mode. 1958 * This allows IO forwarding mode to forward packets 1959 * to itself through 2 cross-connected ports of the 1960 * target machine. 1961 */ 1962 if (promiscuous_on) 1963 rte_eth_promiscuous_enable(portid); 1964 } 1965 1966 check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); 1967 1968 /* launch per-lcore init on every lcore */ 1969 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 1970 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1971 if (rte_eal_wait_lcore(lcore_id) < 0) 1972 return -1; 1973 } 1974 1975 return 0; 1976 } 1977