1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <stdint.h> 37 #include <inttypes.h> 38 #include <sys/types.h> 39 #include <string.h> 40 #include <sys/queue.h> 41 #include <stdarg.h> 42 #include <errno.h> 43 #include <getopt.h> 44 45 #include <rte_common.h> 46 #include <rte_common_vect.h> 47 #include <rte_byteorder.h> 48 #include <rte_log.h> 49 #include <rte_memory.h> 50 #include <rte_memcpy.h> 51 #include <rte_memzone.h> 52 #include <rte_tailq.h> 53 #include <rte_eal.h> 54 #include <rte_per_lcore.h> 55 #include <rte_launch.h> 56 #include <rte_atomic.h> 57 #include <rte_cycles.h> 58 #include <rte_prefetch.h> 59 #include <rte_lcore.h> 60 #include <rte_per_lcore.h> 61 #include <rte_branch_prediction.h> 62 #include <rte_interrupts.h> 63 #include <rte_pci.h> 64 #include <rte_random.h> 65 #include <rte_debug.h> 66 #include <rte_ether.h> 67 #include <rte_ethdev.h> 68 #include <rte_ring.h> 69 #include <rte_mempool.h> 70 #include <rte_mbuf.h> 71 #include <rte_ip.h> 72 #include <rte_tcp.h> 73 #include <rte_udp.h> 74 #include <rte_string_fns.h> 75 76 #include "main.h" 77 78 #define APP_LOOKUP_EXACT_MATCH 0 79 #define APP_LOOKUP_LPM 1 80 #define DO_RFC_1812_CHECKS 81 82 #ifndef APP_LOOKUP_METHOD 83 #define APP_LOOKUP_METHOD APP_LOOKUP_LPM 84 #endif 85 86 /* 87 * When set to zero, simple forwaring path is eanbled. 88 * When set to one, optimized forwarding path is enabled. 89 * Note that LPM optimisation path uses SSE4.1 instructions. 90 */ 91 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && !defined(__SSE4_1__)) 92 #define ENABLE_MULTI_BUFFER_OPTIMIZE 0 93 #else 94 #define ENABLE_MULTI_BUFFER_OPTIMIZE 1 95 #endif 96 97 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 98 #include <rte_hash.h> 99 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 100 #include <rte_lpm.h> 101 #include <rte_lpm6.h> 102 #else 103 #error "APP_LOOKUP_METHOD set to incorrect value" 104 #endif 105 106 #ifndef IPv6_BYTES 107 #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\ 108 "%02x%02x:%02x%02x:%02x%02x:%02x%02x" 109 #define IPv6_BYTES(addr) \ 110 addr[0], addr[1], addr[2], addr[3], \ 111 addr[4], addr[5], addr[6], addr[7], \ 112 addr[8], addr[9], addr[10], addr[11],\ 113 addr[12], addr[13],addr[14], addr[15] 114 #endif 115 116 117 #define RTE_LOGTYPE_L3FWD RTE_LOGTYPE_USER1 118 119 #define MAX_JUMBO_PKT_LEN 9600 120 121 #define IPV6_ADDR_LEN 16 122 123 #define MEMPOOL_CACHE_SIZE 256 124 125 #define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM) 126 127 /* 128 * This expression is used to calculate the number of mbufs needed depending on user input, taking 129 * into account memory for rx and tx hardware rings, cache per lcore and mtable per port per lcore. 130 * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum value of 8192 131 */ 132 133 #define NB_MBUF RTE_MAX ( \ 134 (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \ 135 nb_ports*nb_lcores*MAX_PKT_BURST + \ 136 nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT + \ 137 nb_lcores*MEMPOOL_CACHE_SIZE), \ 138 (unsigned)8192) 139 140 #define MAX_PKT_BURST 32 141 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 142 143 /* 144 * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send. 145 */ 146 #define MAX_TX_BURST (MAX_PKT_BURST / 2) 147 148 #define NB_SOCKETS 8 149 150 /* Configure how many packets ahead to prefetch, when reading packets */ 151 #define PREFETCH_OFFSET 3 152 153 /* Used to mark destination port as 'invalid'. */ 154 #define BAD_PORT ((uint16_t)-1) 155 156 #define FWDSTEP 4 157 158 /* 159 * Configurable number of RX/TX ring descriptors 160 */ 161 #define RTE_TEST_RX_DESC_DEFAULT 128 162 #define RTE_TEST_TX_DESC_DEFAULT 512 163 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 164 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 165 166 /* ethernet addresses of ports */ 167 static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; 168 169 static __m128i val_eth[RTE_MAX_ETHPORTS]; 170 171 /* replace first 12B of the ethernet header. */ 172 #define MASK_ETH 0x3f 173 174 /* mask of enabled ports */ 175 static uint32_t enabled_port_mask = 0; 176 static int promiscuous_on = 0; /**< Ports set in promiscuous mode off by default. */ 177 static int numa_on = 1; /**< NUMA is enabled by default. */ 178 179 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 180 static int ipv6 = 0; /**< ipv6 is false by default. */ 181 #endif 182 183 struct mbuf_table { 184 uint16_t len; 185 struct rte_mbuf *m_table[MAX_PKT_BURST]; 186 }; 187 188 struct lcore_rx_queue { 189 uint8_t port_id; 190 uint8_t queue_id; 191 } __rte_cache_aligned; 192 193 #define MAX_RX_QUEUE_PER_LCORE 16 194 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 195 #define MAX_RX_QUEUE_PER_PORT 128 196 197 #define MAX_LCORE_PARAMS 1024 198 struct lcore_params { 199 uint8_t port_id; 200 uint8_t queue_id; 201 uint8_t lcore_id; 202 } __rte_cache_aligned; 203 204 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 205 static struct lcore_params lcore_params_array_default[] = { 206 {0, 0, 2}, 207 {0, 1, 2}, 208 {0, 2, 2}, 209 {1, 0, 2}, 210 {1, 1, 2}, 211 {1, 2, 2}, 212 {2, 0, 2}, 213 {3, 0, 3}, 214 {3, 1, 3}, 215 }; 216 217 static struct lcore_params * lcore_params = lcore_params_array_default; 218 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) / 219 sizeof(lcore_params_array_default[0]); 220 221 static struct rte_eth_conf port_conf = { 222 .rxmode = { 223 .mq_mode = ETH_MQ_RX_RSS, 224 .max_rx_pkt_len = ETHER_MAX_LEN, 225 .split_hdr_size = 0, 226 .header_split = 0, /**< Header Split disabled */ 227 .hw_ip_checksum = 1, /**< IP checksum offload enabled */ 228 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 229 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 230 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 231 }, 232 .rx_adv_conf = { 233 .rss_conf = { 234 .rss_key = NULL, 235 .rss_hf = ETH_RSS_IP, 236 }, 237 }, 238 .txmode = { 239 .mq_mode = ETH_MQ_TX_NONE, 240 }, 241 }; 242 243 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS]; 244 245 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 246 247 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2 248 #include <rte_hash_crc.h> 249 #define DEFAULT_HASH_FUNC rte_hash_crc 250 #else 251 #include <rte_jhash.h> 252 #define DEFAULT_HASH_FUNC rte_jhash 253 #endif 254 255 struct ipv4_5tuple { 256 uint32_t ip_dst; 257 uint32_t ip_src; 258 uint16_t port_dst; 259 uint16_t port_src; 260 uint8_t proto; 261 } __attribute__((__packed__)); 262 263 union ipv4_5tuple_host { 264 struct { 265 uint8_t pad0; 266 uint8_t proto; 267 uint16_t pad1; 268 uint32_t ip_src; 269 uint32_t ip_dst; 270 uint16_t port_src; 271 uint16_t port_dst; 272 }; 273 __m128i xmm; 274 }; 275 276 #define XMM_NUM_IN_IPV6_5TUPLE 3 277 278 struct ipv6_5tuple { 279 uint8_t ip_dst[IPV6_ADDR_LEN]; 280 uint8_t ip_src[IPV6_ADDR_LEN]; 281 uint16_t port_dst; 282 uint16_t port_src; 283 uint8_t proto; 284 } __attribute__((__packed__)); 285 286 union ipv6_5tuple_host { 287 struct { 288 uint16_t pad0; 289 uint8_t proto; 290 uint8_t pad1; 291 uint8_t ip_src[IPV6_ADDR_LEN]; 292 uint8_t ip_dst[IPV6_ADDR_LEN]; 293 uint16_t port_src; 294 uint16_t port_dst; 295 uint64_t reserve; 296 }; 297 __m128i xmm[XMM_NUM_IN_IPV6_5TUPLE]; 298 }; 299 300 struct ipv4_l3fwd_route { 301 struct ipv4_5tuple key; 302 uint8_t if_out; 303 }; 304 305 struct ipv6_l3fwd_route { 306 struct ipv6_5tuple key; 307 uint8_t if_out; 308 }; 309 310 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { 311 {{IPv4(101,0,0,0), IPv4(100,10,0,1), 101, 11, IPPROTO_TCP}, 0}, 312 {{IPv4(201,0,0,0), IPv4(200,20,0,1), 102, 12, IPPROTO_TCP}, 1}, 313 {{IPv4(111,0,0,0), IPv4(100,30,0,1), 101, 11, IPPROTO_TCP}, 2}, 314 {{IPv4(211,0,0,0), IPv4(200,40,0,1), 102, 12, IPPROTO_TCP}, 3}, 315 }; 316 317 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = { 318 {{ 319 {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0}, 320 {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05}, 321 101, 11, IPPROTO_TCP}, 0}, 322 323 {{ 324 {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0}, 325 {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05}, 326 102, 12, IPPROTO_TCP}, 1}, 327 328 {{ 329 {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0}, 330 {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05}, 331 101, 11, IPPROTO_TCP}, 2}, 332 333 {{ 334 {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0}, 335 {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05}, 336 102, 12, IPPROTO_TCP}, 3}, 337 }; 338 339 typedef struct rte_hash lookup_struct_t; 340 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS]; 341 static lookup_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS]; 342 343 #ifdef RTE_ARCH_X86_64 344 /* default to 4 million hash entries (approx) */ 345 #define L3FWD_HASH_ENTRIES 1024*1024*4 346 #else 347 /* 32-bit has less address-space for hugepage memory, limit to 1M entries */ 348 #define L3FWD_HASH_ENTRIES 1024*1024*1 349 #endif 350 #define HASH_ENTRY_NUMBER_DEFAULT 4 351 352 static uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT; 353 354 static inline uint32_t 355 ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len, 356 uint32_t init_val) 357 { 358 const union ipv4_5tuple_host *k; 359 uint32_t t; 360 const uint32_t *p; 361 362 k = data; 363 t = k->proto; 364 p = (const uint32_t *)&k->port_src; 365 366 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2 367 init_val = rte_hash_crc_4byte(t, init_val); 368 init_val = rte_hash_crc_4byte(k->ip_src, init_val); 369 init_val = rte_hash_crc_4byte(k->ip_dst, init_val); 370 init_val = rte_hash_crc_4byte(*p, init_val); 371 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */ 372 init_val = rte_jhash_1word(t, init_val); 373 init_val = rte_jhash_1word(k->ip_src, init_val); 374 init_val = rte_jhash_1word(k->ip_dst, init_val); 375 init_val = rte_jhash_1word(*p, init_val); 376 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ 377 return (init_val); 378 } 379 380 static inline uint32_t 381 ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, uint32_t init_val) 382 { 383 const union ipv6_5tuple_host *k; 384 uint32_t t; 385 const uint32_t *p; 386 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2 387 const uint32_t *ip_src0, *ip_src1, *ip_src2, *ip_src3; 388 const uint32_t *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3; 389 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ 390 391 k = data; 392 t = k->proto; 393 p = (const uint32_t *)&k->port_src; 394 395 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2 396 ip_src0 = (const uint32_t *) k->ip_src; 397 ip_src1 = (const uint32_t *)(k->ip_src+4); 398 ip_src2 = (const uint32_t *)(k->ip_src+8); 399 ip_src3 = (const uint32_t *)(k->ip_src+12); 400 ip_dst0 = (const uint32_t *) k->ip_dst; 401 ip_dst1 = (const uint32_t *)(k->ip_dst+4); 402 ip_dst2 = (const uint32_t *)(k->ip_dst+8); 403 ip_dst3 = (const uint32_t *)(k->ip_dst+12); 404 init_val = rte_hash_crc_4byte(t, init_val); 405 init_val = rte_hash_crc_4byte(*ip_src0, init_val); 406 init_val = rte_hash_crc_4byte(*ip_src1, init_val); 407 init_val = rte_hash_crc_4byte(*ip_src2, init_val); 408 init_val = rte_hash_crc_4byte(*ip_src3, init_val); 409 init_val = rte_hash_crc_4byte(*ip_dst0, init_val); 410 init_val = rte_hash_crc_4byte(*ip_dst1, init_val); 411 init_val = rte_hash_crc_4byte(*ip_dst2, init_val); 412 init_val = rte_hash_crc_4byte(*ip_dst3, init_val); 413 init_val = rte_hash_crc_4byte(*p, init_val); 414 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */ 415 init_val = rte_jhash_1word(t, init_val); 416 init_val = rte_jhash(k->ip_src, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val); 417 init_val = rte_jhash(k->ip_dst, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val); 418 init_val = rte_jhash_1word(*p, init_val); 419 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ 420 return (init_val); 421 } 422 423 #define IPV4_L3FWD_NUM_ROUTES \ 424 (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0])) 425 426 #define IPV6_L3FWD_NUM_ROUTES \ 427 (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0])) 428 429 static uint8_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; 430 static uint8_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; 431 432 #endif 433 434 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 435 struct ipv4_l3fwd_route { 436 uint32_t ip; 437 uint8_t depth; 438 uint8_t if_out; 439 }; 440 441 struct ipv6_l3fwd_route { 442 uint8_t ip[16]; 443 uint8_t depth; 444 uint8_t if_out; 445 }; 446 447 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { 448 {IPv4(1,1,1,0), 24, 0}, 449 {IPv4(2,1,1,0), 24, 1}, 450 {IPv4(3,1,1,0), 24, 2}, 451 {IPv4(4,1,1,0), 24, 3}, 452 {IPv4(5,1,1,0), 24, 4}, 453 {IPv4(6,1,1,0), 24, 5}, 454 {IPv4(7,1,1,0), 24, 6}, 455 {IPv4(8,1,1,0), 24, 7}, 456 }; 457 458 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = { 459 {{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 0}, 460 {{2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 1}, 461 {{3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 2}, 462 {{4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 3}, 463 {{5,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 4}, 464 {{6,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 5}, 465 {{7,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 6}, 466 {{8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 7}, 467 }; 468 469 #define IPV4_L3FWD_NUM_ROUTES \ 470 (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0])) 471 #define IPV6_L3FWD_NUM_ROUTES \ 472 (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0])) 473 474 #define IPV4_L3FWD_LPM_MAX_RULES 1024 475 #define IPV6_L3FWD_LPM_MAX_RULES 1024 476 #define IPV6_L3FWD_LPM_NUMBER_TBL8S (1 << 16) 477 478 typedef struct rte_lpm lookup_struct_t; 479 typedef struct rte_lpm6 lookup6_struct_t; 480 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS]; 481 static lookup6_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS]; 482 #endif 483 484 struct lcore_conf { 485 uint16_t n_rx_queue; 486 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 487 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 488 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; 489 lookup_struct_t * ipv4_lookup_struct; 490 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 491 lookup6_struct_t * ipv6_lookup_struct; 492 #else 493 lookup_struct_t * ipv6_lookup_struct; 494 #endif 495 } __rte_cache_aligned; 496 497 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 498 499 /* Send burst of packets on an output interface */ 500 static inline int 501 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 502 { 503 struct rte_mbuf **m_table; 504 int ret; 505 uint16_t queueid; 506 507 queueid = qconf->tx_queue_id[port]; 508 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 509 510 ret = rte_eth_tx_burst(port, queueid, m_table, n); 511 if (unlikely(ret < n)) { 512 do { 513 rte_pktmbuf_free(m_table[ret]); 514 } while (++ret < n); 515 } 516 517 return 0; 518 } 519 520 /* Enqueue a single packet, and send burst if queue is filled */ 521 static inline int 522 send_single_packet(struct rte_mbuf *m, uint8_t port) 523 { 524 uint32_t lcore_id; 525 uint16_t len; 526 struct lcore_conf *qconf; 527 528 lcore_id = rte_lcore_id(); 529 530 qconf = &lcore_conf[lcore_id]; 531 len = qconf->tx_mbufs[port].len; 532 qconf->tx_mbufs[port].m_table[len] = m; 533 len++; 534 535 /* enough pkts to be sent */ 536 if (unlikely(len == MAX_PKT_BURST)) { 537 send_burst(qconf, MAX_PKT_BURST, port); 538 len = 0; 539 } 540 541 qconf->tx_mbufs[port].len = len; 542 return 0; 543 } 544 545 static inline __attribute__((always_inline)) void 546 send_packetsx4(struct lcore_conf *qconf, uint8_t port, 547 struct rte_mbuf *m[], uint32_t num) 548 { 549 uint32_t len, j, n; 550 551 len = qconf->tx_mbufs[port].len; 552 553 /* 554 * If TX buffer for that queue is empty, and we have enough packets, 555 * then send them straightway. 556 */ 557 if (num >= MAX_TX_BURST && len == 0) { 558 n = rte_eth_tx_burst(port, qconf->tx_queue_id[port], m, num); 559 if (unlikely(n < num)) { 560 do { 561 rte_pktmbuf_free(m[n]); 562 } while (++n < num); 563 } 564 return; 565 } 566 567 /* 568 * Put packets into TX buffer for that queue. 569 */ 570 571 n = len + num; 572 n = (n > MAX_PKT_BURST) ? MAX_PKT_BURST - len : num; 573 574 j = 0; 575 switch (n % FWDSTEP) { 576 while (j < n) { 577 case 0: 578 qconf->tx_mbufs[port].m_table[len + j] = m[j]; 579 j++; 580 case 3: 581 qconf->tx_mbufs[port].m_table[len + j] = m[j]; 582 j++; 583 case 2: 584 qconf->tx_mbufs[port].m_table[len + j] = m[j]; 585 j++; 586 case 1: 587 qconf->tx_mbufs[port].m_table[len + j] = m[j]; 588 j++; 589 } 590 } 591 592 len += n; 593 594 /* enough pkts to be sent */ 595 if (unlikely(len == MAX_PKT_BURST)) { 596 597 send_burst(qconf, MAX_PKT_BURST, port); 598 599 /* copy rest of the packets into the TX buffer. */ 600 len = num - n; 601 j = 0; 602 switch (len % FWDSTEP) { 603 while (j < len) { 604 case 0: 605 qconf->tx_mbufs[port].m_table[j] = m[n + j]; 606 j++; 607 case 3: 608 qconf->tx_mbufs[port].m_table[j] = m[n + j]; 609 j++; 610 case 2: 611 qconf->tx_mbufs[port].m_table[j] = m[n + j]; 612 j++; 613 case 1: 614 qconf->tx_mbufs[port].m_table[j] = m[n + j]; 615 j++; 616 } 617 } 618 } 619 620 qconf->tx_mbufs[port].len = len; 621 } 622 623 #ifdef DO_RFC_1812_CHECKS 624 static inline int 625 is_valid_ipv4_pkt(struct ipv4_hdr *pkt, uint32_t link_len) 626 { 627 /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */ 628 /* 629 * 1. The packet length reported by the Link Layer must be large 630 * enough to hold the minimum length legal IP datagram (20 bytes). 631 */ 632 if (link_len < sizeof(struct ipv4_hdr)) 633 return -1; 634 635 /* 2. The IP checksum must be correct. */ 636 /* this is checked in H/W */ 637 638 /* 639 * 3. The IP version number must be 4. If the version number is not 4 640 * then the packet may be another version of IP, such as IPng or 641 * ST-II. 642 */ 643 if (((pkt->version_ihl) >> 4) != 4) 644 return -3; 645 /* 646 * 4. The IP header length field must be large enough to hold the 647 * minimum length legal IP datagram (20 bytes = 5 words). 648 */ 649 if ((pkt->version_ihl & 0xf) < 5) 650 return -4; 651 652 /* 653 * 5. The IP total length field must be large enough to hold the IP 654 * datagram header, whose length is specified in the IP header length 655 * field. 656 */ 657 if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct ipv4_hdr)) 658 return -5; 659 660 return 0; 661 } 662 #endif 663 664 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 665 666 static __m128i mask0; 667 static __m128i mask1; 668 static __m128i mask2; 669 static inline uint8_t 670 get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, lookup_struct_t * ipv4_l3fwd_lookup_struct) 671 { 672 int ret = 0; 673 union ipv4_5tuple_host key; 674 675 ipv4_hdr = (uint8_t *)ipv4_hdr + offsetof(struct ipv4_hdr, time_to_live); 676 __m128i data = _mm_loadu_si128((__m128i*)(ipv4_hdr)); 677 /* Get 5 tuple: dst port, src port, dst IP address, src IP address and protocol */ 678 key.xmm = _mm_and_si128(data, mask0); 679 /* Find destination port */ 680 ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key); 681 return (uint8_t)((ret < 0)? portid : ipv4_l3fwd_out_if[ret]); 682 } 683 684 static inline uint8_t 685 get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, lookup_struct_t * ipv6_l3fwd_lookup_struct) 686 { 687 int ret = 0; 688 union ipv6_5tuple_host key; 689 690 ipv6_hdr = (uint8_t *)ipv6_hdr + offsetof(struct ipv6_hdr, payload_len); 691 __m128i data0 = _mm_loadu_si128((__m128i*)(ipv6_hdr)); 692 __m128i data1 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i))); 693 __m128i data2 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i)+sizeof(__m128i))); 694 /* Get part of 5 tuple: src IP address lower 96 bits and protocol */ 695 key.xmm[0] = _mm_and_si128(data0, mask1); 696 /* Get part of 5 tuple: dst IP address lower 96 bits and src IP address higher 32 bits */ 697 key.xmm[1] = data1; 698 /* Get part of 5 tuple: dst port and src port and dst IP address higher 32 bits */ 699 key.xmm[2] = _mm_and_si128(data2, mask2); 700 701 /* Find destination port */ 702 ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key); 703 return (uint8_t)((ret < 0)? portid : ipv6_l3fwd_out_if[ret]); 704 } 705 #endif 706 707 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 708 709 static inline uint8_t 710 get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, lookup_struct_t * ipv4_l3fwd_lookup_struct) 711 { 712 uint8_t next_hop; 713 714 return (uint8_t) ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, 715 rte_be_to_cpu_32(((struct ipv4_hdr *)ipv4_hdr)->dst_addr), 716 &next_hop) == 0) ? next_hop : portid); 717 } 718 719 static inline uint8_t 720 get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, lookup6_struct_t * ipv6_l3fwd_lookup_struct) 721 { 722 uint8_t next_hop; 723 return (uint8_t) ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct, 724 ((struct ipv6_hdr*)ipv6_hdr)->dst_addr, &next_hop) == 0)? 725 next_hop : portid); 726 } 727 #endif 728 729 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) && \ 730 (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)) 731 static inline void l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, struct lcore_conf *qconf); 732 733 #define MASK_ALL_PKTS 0xf 734 #define EXECLUDE_1ST_PKT 0xe 735 #define EXECLUDE_2ND_PKT 0xd 736 #define EXECLUDE_3RD_PKT 0xb 737 #define EXECLUDE_4TH_PKT 0x7 738 739 static inline void 740 simple_ipv4_fwd_4pkts(struct rte_mbuf* m[4], uint8_t portid, struct lcore_conf *qconf) 741 { 742 struct ether_hdr *eth_hdr[4]; 743 struct ipv4_hdr *ipv4_hdr[4]; 744 void *d_addr_bytes[4]; 745 uint8_t dst_port[4]; 746 int32_t ret[4]; 747 union ipv4_5tuple_host key[4]; 748 __m128i data[4]; 749 750 eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *); 751 eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *); 752 eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *); 753 eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *); 754 755 /* Handle IPv4 headers.*/ 756 ipv4_hdr[0] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[0], unsigned char *) + 757 sizeof(struct ether_hdr)); 758 ipv4_hdr[1] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[1], unsigned char *) + 759 sizeof(struct ether_hdr)); 760 ipv4_hdr[2] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[2], unsigned char *) + 761 sizeof(struct ether_hdr)); 762 ipv4_hdr[3] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[3], unsigned char *) + 763 sizeof(struct ether_hdr)); 764 765 #ifdef DO_RFC_1812_CHECKS 766 /* Check to make sure the packet is valid (RFC1812) */ 767 uint8_t valid_mask = MASK_ALL_PKTS; 768 if (is_valid_ipv4_pkt(ipv4_hdr[0], m[0]->pkt_len) < 0) { 769 rte_pktmbuf_free(m[0]); 770 valid_mask &= EXECLUDE_1ST_PKT; 771 } 772 if (is_valid_ipv4_pkt(ipv4_hdr[1], m[1]->pkt_len) < 0) { 773 rte_pktmbuf_free(m[1]); 774 valid_mask &= EXECLUDE_2ND_PKT; 775 } 776 if (is_valid_ipv4_pkt(ipv4_hdr[2], m[2]->pkt_len) < 0) { 777 rte_pktmbuf_free(m[2]); 778 valid_mask &= EXECLUDE_3RD_PKT; 779 } 780 if (is_valid_ipv4_pkt(ipv4_hdr[3], m[3]->pkt_len) < 0) { 781 rte_pktmbuf_free(m[3]); 782 valid_mask &= EXECLUDE_4TH_PKT; 783 } 784 if (unlikely(valid_mask != MASK_ALL_PKTS)) { 785 if (valid_mask == 0){ 786 return; 787 } else { 788 uint8_t i = 0; 789 for (i = 0; i < 4; i++) { 790 if ((0x1 << i) & valid_mask) { 791 l3fwd_simple_forward(m[i], portid, qconf); 792 } 793 } 794 return; 795 } 796 } 797 #endif // End of #ifdef DO_RFC_1812_CHECKS 798 799 data[0] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[0], unsigned char *) + 800 sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live))); 801 data[1] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[1], unsigned char *) + 802 sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live))); 803 data[2] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[2], unsigned char *) + 804 sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live))); 805 data[3] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[3], unsigned char *) + 806 sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live))); 807 808 key[0].xmm = _mm_and_si128(data[0], mask0); 809 key[1].xmm = _mm_and_si128(data[1], mask0); 810 key[2].xmm = _mm_and_si128(data[2], mask0); 811 key[3].xmm = _mm_and_si128(data[3], mask0); 812 813 const void *key_array[4] = {&key[0], &key[1], &key[2],&key[3]}; 814 rte_hash_lookup_multi(qconf->ipv4_lookup_struct, &key_array[0], 4, ret); 815 dst_port[0] = (uint8_t) ((ret[0] < 0) ? portid : ipv4_l3fwd_out_if[ret[0]]); 816 dst_port[1] = (uint8_t) ((ret[1] < 0) ? portid : ipv4_l3fwd_out_if[ret[1]]); 817 dst_port[2] = (uint8_t) ((ret[2] < 0) ? portid : ipv4_l3fwd_out_if[ret[2]]); 818 dst_port[3] = (uint8_t) ((ret[3] < 0) ? portid : ipv4_l3fwd_out_if[ret[3]]); 819 820 if (dst_port[0] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[0]) == 0) 821 dst_port[0] = portid; 822 if (dst_port[1] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[1]) == 0) 823 dst_port[1] = portid; 824 if (dst_port[2] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[2]) == 0) 825 dst_port[2] = portid; 826 if (dst_port[3] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[3]) == 0) 827 dst_port[3] = portid; 828 829 /* 02:00:00:00:00:xx */ 830 d_addr_bytes[0] = ð_hdr[0]->d_addr.addr_bytes[0]; 831 d_addr_bytes[1] = ð_hdr[1]->d_addr.addr_bytes[0]; 832 d_addr_bytes[2] = ð_hdr[2]->d_addr.addr_bytes[0]; 833 d_addr_bytes[3] = ð_hdr[3]->d_addr.addr_bytes[0]; 834 *((uint64_t *)d_addr_bytes[0]) = 0x000000000002 + ((uint64_t)dst_port[0] << 40); 835 *((uint64_t *)d_addr_bytes[1]) = 0x000000000002 + ((uint64_t)dst_port[1] << 40); 836 *((uint64_t *)d_addr_bytes[2]) = 0x000000000002 + ((uint64_t)dst_port[2] << 40); 837 *((uint64_t *)d_addr_bytes[3]) = 0x000000000002 + ((uint64_t)dst_port[3] << 40); 838 839 #ifdef DO_RFC_1812_CHECKS 840 /* Update time to live and header checksum */ 841 --(ipv4_hdr[0]->time_to_live); 842 --(ipv4_hdr[1]->time_to_live); 843 --(ipv4_hdr[2]->time_to_live); 844 --(ipv4_hdr[3]->time_to_live); 845 ++(ipv4_hdr[0]->hdr_checksum); 846 ++(ipv4_hdr[1]->hdr_checksum); 847 ++(ipv4_hdr[2]->hdr_checksum); 848 ++(ipv4_hdr[3]->hdr_checksum); 849 #endif 850 851 /* src addr */ 852 ether_addr_copy(&ports_eth_addr[dst_port[0]], ð_hdr[0]->s_addr); 853 ether_addr_copy(&ports_eth_addr[dst_port[1]], ð_hdr[1]->s_addr); 854 ether_addr_copy(&ports_eth_addr[dst_port[2]], ð_hdr[2]->s_addr); 855 ether_addr_copy(&ports_eth_addr[dst_port[3]], ð_hdr[3]->s_addr); 856 857 send_single_packet(m[0], (uint8_t)dst_port[0]); 858 send_single_packet(m[1], (uint8_t)dst_port[1]); 859 send_single_packet(m[2], (uint8_t)dst_port[2]); 860 send_single_packet(m[3], (uint8_t)dst_port[3]); 861 862 } 863 864 static inline void get_ipv6_5tuple(struct rte_mbuf* m0, __m128i mask0, __m128i mask1, 865 union ipv6_5tuple_host * key) 866 { 867 __m128i tmpdata0 = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m0, unsigned char *) 868 + sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len))); 869 __m128i tmpdata1 = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m0, unsigned char *) 870 + sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) 871 + sizeof(__m128i))); 872 __m128i tmpdata2 = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m0, unsigned char *) 873 + sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) 874 + sizeof(__m128i) + sizeof(__m128i))); 875 key->xmm[0] = _mm_and_si128(tmpdata0, mask0); 876 key->xmm[1] = tmpdata1; 877 key->xmm[2] = _mm_and_si128(tmpdata2, mask1); 878 return; 879 } 880 881 static inline void 882 simple_ipv6_fwd_4pkts(struct rte_mbuf* m[4], uint8_t portid, struct lcore_conf *qconf) 883 { 884 struct ether_hdr *eth_hdr[4]; 885 __attribute__((unused)) struct ipv6_hdr *ipv6_hdr[4]; 886 void *d_addr_bytes[4]; 887 uint8_t dst_port[4]; 888 int32_t ret[4]; 889 union ipv6_5tuple_host key[4]; 890 891 eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *); 892 eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *); 893 eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *); 894 eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *); 895 896 /* Handle IPv6 headers.*/ 897 ipv6_hdr[0] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[0], unsigned char *) + 898 sizeof(struct ether_hdr)); 899 ipv6_hdr[1] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[1], unsigned char *) + 900 sizeof(struct ether_hdr)); 901 ipv6_hdr[2] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[2], unsigned char *) + 902 sizeof(struct ether_hdr)); 903 ipv6_hdr[3] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[3], unsigned char *) + 904 sizeof(struct ether_hdr)); 905 906 get_ipv6_5tuple(m[0], mask1, mask2, &key[0]); 907 get_ipv6_5tuple(m[1], mask1, mask2, &key[1]); 908 get_ipv6_5tuple(m[2], mask1, mask2, &key[2]); 909 get_ipv6_5tuple(m[3], mask1, mask2, &key[3]); 910 911 const void *key_array[4] = {&key[0], &key[1], &key[2],&key[3]}; 912 rte_hash_lookup_multi(qconf->ipv6_lookup_struct, &key_array[0], 4, ret); 913 dst_port[0] = (uint8_t) ((ret[0] < 0)? portid:ipv6_l3fwd_out_if[ret[0]]); 914 dst_port[1] = (uint8_t) ((ret[1] < 0)? portid:ipv6_l3fwd_out_if[ret[1]]); 915 dst_port[2] = (uint8_t) ((ret[2] < 0)? portid:ipv6_l3fwd_out_if[ret[2]]); 916 dst_port[3] = (uint8_t) ((ret[3] < 0)? portid:ipv6_l3fwd_out_if[ret[3]]); 917 918 if (dst_port[0] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[0]) == 0) 919 dst_port[0] = portid; 920 if (dst_port[1] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[1]) == 0) 921 dst_port[1] = portid; 922 if (dst_port[2] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[2]) == 0) 923 dst_port[2] = portid; 924 if (dst_port[3] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[3]) == 0) 925 dst_port[3] = portid; 926 927 /* 02:00:00:00:00:xx */ 928 d_addr_bytes[0] = ð_hdr[0]->d_addr.addr_bytes[0]; 929 d_addr_bytes[1] = ð_hdr[1]->d_addr.addr_bytes[0]; 930 d_addr_bytes[2] = ð_hdr[2]->d_addr.addr_bytes[0]; 931 d_addr_bytes[3] = ð_hdr[3]->d_addr.addr_bytes[0]; 932 *((uint64_t *)d_addr_bytes[0]) = 0x000000000002 + ((uint64_t)dst_port[0] << 40); 933 *((uint64_t *)d_addr_bytes[1]) = 0x000000000002 + ((uint64_t)dst_port[1] << 40); 934 *((uint64_t *)d_addr_bytes[2]) = 0x000000000002 + ((uint64_t)dst_port[2] << 40); 935 *((uint64_t *)d_addr_bytes[3]) = 0x000000000002 + ((uint64_t)dst_port[3] << 40); 936 937 /* src addr */ 938 ether_addr_copy(&ports_eth_addr[dst_port[0]], ð_hdr[0]->s_addr); 939 ether_addr_copy(&ports_eth_addr[dst_port[1]], ð_hdr[1]->s_addr); 940 ether_addr_copy(&ports_eth_addr[dst_port[2]], ð_hdr[2]->s_addr); 941 ether_addr_copy(&ports_eth_addr[dst_port[3]], ð_hdr[3]->s_addr); 942 943 send_single_packet(m[0], (uint8_t)dst_port[0]); 944 send_single_packet(m[1], (uint8_t)dst_port[1]); 945 send_single_packet(m[2], (uint8_t)dst_port[2]); 946 send_single_packet(m[3], (uint8_t)dst_port[3]); 947 948 } 949 #endif /* APP_LOOKUP_METHOD */ 950 951 static inline __attribute__((always_inline)) void 952 l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, struct lcore_conf *qconf) 953 { 954 struct ether_hdr *eth_hdr; 955 struct ipv4_hdr *ipv4_hdr; 956 void *d_addr_bytes; 957 uint8_t dst_port; 958 959 eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); 960 961 if (m->ol_flags & PKT_RX_IPV4_HDR) { 962 /* Handle IPv4 headers.*/ 963 ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m, unsigned char *) + 964 sizeof(struct ether_hdr)); 965 966 #ifdef DO_RFC_1812_CHECKS 967 /* Check to make sure the packet is valid (RFC1812) */ 968 if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len) < 0) { 969 rte_pktmbuf_free(m); 970 return; 971 } 972 #endif 973 974 dst_port = get_ipv4_dst_port(ipv4_hdr, portid, 975 qconf->ipv4_lookup_struct); 976 if (dst_port >= RTE_MAX_ETHPORTS || 977 (enabled_port_mask & 1 << dst_port) == 0) 978 dst_port = portid; 979 980 /* 02:00:00:00:00:xx */ 981 d_addr_bytes = ð_hdr->d_addr.addr_bytes[0]; 982 *((uint64_t *)d_addr_bytes) = ETHER_LOCAL_ADMIN_ADDR + 983 ((uint64_t)dst_port << 40); 984 985 #ifdef DO_RFC_1812_CHECKS 986 /* Update time to live and header checksum */ 987 --(ipv4_hdr->time_to_live); 988 ++(ipv4_hdr->hdr_checksum); 989 #endif 990 991 /* src addr */ 992 ether_addr_copy(&ports_eth_addr[dst_port], ð_hdr->s_addr); 993 994 send_single_packet(m, dst_port); 995 996 } else { 997 /* Handle IPv6 headers.*/ 998 struct ipv6_hdr *ipv6_hdr; 999 1000 ipv6_hdr = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m, unsigned char *) + 1001 sizeof(struct ether_hdr)); 1002 1003 dst_port = get_ipv6_dst_port(ipv6_hdr, portid, qconf->ipv6_lookup_struct); 1004 1005 if (dst_port >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port) == 0) 1006 dst_port = portid; 1007 1008 /* 02:00:00:00:00:xx */ 1009 d_addr_bytes = ð_hdr->d_addr.addr_bytes[0]; 1010 *((uint64_t *)d_addr_bytes) = ETHER_LOCAL_ADMIN_ADDR + 1011 ((uint64_t)dst_port << 40); 1012 1013 /* src addr */ 1014 ether_addr_copy(&ports_eth_addr[dst_port], ð_hdr->s_addr); 1015 1016 send_single_packet(m, dst_port); 1017 } 1018 1019 } 1020 1021 #ifdef DO_RFC_1812_CHECKS 1022 1023 #define IPV4_MIN_VER_IHL 0x45 1024 #define IPV4_MAX_VER_IHL 0x4f 1025 #define IPV4_MAX_VER_IHL_DIFF (IPV4_MAX_VER_IHL - IPV4_MIN_VER_IHL) 1026 1027 /* Minimum value of IPV4 total length (20B) in network byte order. */ 1028 #define IPV4_MIN_LEN_BE (sizeof(struct ipv4_hdr) << 8) 1029 1030 /* 1031 * From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2: 1032 * - The IP version number must be 4. 1033 * - The IP header length field must be large enough to hold the 1034 * minimum length legal IP datagram (20 bytes = 5 words). 1035 * - The IP total length field must be large enough to hold the IP 1036 * datagram header, whose length is specified in the IP header length 1037 * field. 1038 * If we encounter invalid IPV4 packet, then set destination port for it 1039 * to BAD_PORT value. 1040 */ 1041 static inline __attribute__((always_inline)) void 1042 rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t flags) 1043 { 1044 uint8_t ihl; 1045 1046 if ((flags & PKT_RX_IPV4_HDR) != 0) { 1047 1048 ihl = ipv4_hdr->version_ihl - IPV4_MIN_VER_IHL; 1049 1050 ipv4_hdr->time_to_live--; 1051 ipv4_hdr->hdr_checksum++; 1052 1053 if (ihl > IPV4_MAX_VER_IHL_DIFF || 1054 ((uint8_t)ipv4_hdr->total_length == 0 && 1055 ipv4_hdr->total_length < IPV4_MIN_LEN_BE)) { 1056 dp[0] = BAD_PORT; 1057 } 1058 } 1059 } 1060 1061 #else 1062 #define rfc1812_process(mb, dp) do { } while (0) 1063 #endif /* DO_RFC_1812_CHECKS */ 1064 1065 1066 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \ 1067 (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)) 1068 1069 static inline __attribute__((always_inline)) uint16_t 1070 get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt, 1071 uint32_t dst_ipv4, uint8_t portid) 1072 { 1073 uint8_t next_hop; 1074 struct ipv6_hdr *ipv6_hdr; 1075 struct ether_hdr *eth_hdr; 1076 1077 if (pkt->ol_flags & PKT_RX_IPV4_HDR) { 1078 if (rte_lpm_lookup(qconf->ipv4_lookup_struct, dst_ipv4, 1079 &next_hop) != 0) 1080 next_hop = portid; 1081 } else if (pkt->ol_flags & PKT_RX_IPV6_HDR) { 1082 eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); 1083 ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1); 1084 if (rte_lpm6_lookup(qconf->ipv6_lookup_struct, 1085 ipv6_hdr->dst_addr, &next_hop) != 0) 1086 next_hop = portid; 1087 } else { 1088 next_hop = portid; 1089 } 1090 1091 return next_hop; 1092 } 1093 1094 static inline void 1095 process_packet(struct lcore_conf *qconf, struct rte_mbuf *pkt, 1096 uint16_t *dst_port, uint8_t portid) 1097 { 1098 struct ether_hdr *eth_hdr; 1099 struct ipv4_hdr *ipv4_hdr; 1100 uint32_t dst_ipv4; 1101 uint16_t dp; 1102 __m128i te, ve; 1103 1104 eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); 1105 ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); 1106 1107 dst_ipv4 = ipv4_hdr->dst_addr; 1108 dst_ipv4 = rte_be_to_cpu_32(dst_ipv4); 1109 dp = get_dst_port(qconf, pkt, dst_ipv4, portid); 1110 1111 te = _mm_load_si128((__m128i *)eth_hdr); 1112 ve = val_eth[dp]; 1113 1114 dst_port[0] = dp; 1115 rfc1812_process(ipv4_hdr, dst_port, pkt->ol_flags); 1116 1117 te = _mm_blend_epi16(te, ve, MASK_ETH); 1118 _mm_store_si128((__m128i *)eth_hdr, te); 1119 } 1120 1121 /* 1122 * Read ol_flags and destination IPV4 addresses from 4 mbufs. 1123 */ 1124 static inline void 1125 processx4_step1(struct rte_mbuf *pkt[FWDSTEP], __m128i *dip, uint32_t *flag) 1126 { 1127 struct ipv4_hdr *ipv4_hdr; 1128 struct ether_hdr *eth_hdr; 1129 uint32_t x0, x1, x2, x3; 1130 1131 eth_hdr = rte_pktmbuf_mtod(pkt[0], struct ether_hdr *); 1132 ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); 1133 x0 = ipv4_hdr->dst_addr; 1134 flag[0] = pkt[0]->ol_flags & PKT_RX_IPV4_HDR; 1135 1136 eth_hdr = rte_pktmbuf_mtod(pkt[1], struct ether_hdr *); 1137 ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); 1138 x1 = ipv4_hdr->dst_addr; 1139 flag[0] &= pkt[1]->ol_flags; 1140 1141 eth_hdr = rte_pktmbuf_mtod(pkt[2], struct ether_hdr *); 1142 ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); 1143 x2 = ipv4_hdr->dst_addr; 1144 flag[0] &= pkt[2]->ol_flags; 1145 1146 eth_hdr = rte_pktmbuf_mtod(pkt[3], struct ether_hdr *); 1147 ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); 1148 x3 = ipv4_hdr->dst_addr; 1149 flag[0] &= pkt[3]->ol_flags; 1150 1151 dip[0] = _mm_set_epi32(x3, x2, x1, x0); 1152 } 1153 1154 /* 1155 * Lookup into LPM for destination port. 1156 * If lookup fails, use incoming port (portid) as destination port. 1157 */ 1158 static inline void 1159 processx4_step2(const struct lcore_conf *qconf, __m128i dip, uint32_t flag, 1160 uint8_t portid, struct rte_mbuf *pkt[FWDSTEP], uint16_t dprt[FWDSTEP]) 1161 { 1162 rte_xmm_t dst; 1163 const __m128i bswap_mask = _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 1164 4, 5, 6, 7, 0, 1, 2, 3); 1165 1166 /* Byte swap 4 IPV4 addresses. */ 1167 dip = _mm_shuffle_epi8(dip, bswap_mask); 1168 1169 /* if all 4 packets are IPV4. */ 1170 if (likely(flag != 0)) { 1171 rte_lpm_lookupx4(qconf->ipv4_lookup_struct, dip, dprt, portid); 1172 } else { 1173 dst.m = dip; 1174 dprt[0] = get_dst_port(qconf, pkt[0], dst.u32[0], portid); 1175 dprt[1] = get_dst_port(qconf, pkt[1], dst.u32[1], portid); 1176 dprt[2] = get_dst_port(qconf, pkt[2], dst.u32[2], portid); 1177 dprt[3] = get_dst_port(qconf, pkt[3], dst.u32[3], portid); 1178 } 1179 } 1180 1181 /* 1182 * Update source and destination MAC addresses in the ethernet header. 1183 * Perform RFC1812 checks and updates for IPV4 packets. 1184 */ 1185 static inline void 1186 processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP]) 1187 { 1188 __m128i te[FWDSTEP]; 1189 __m128i ve[FWDSTEP]; 1190 __m128i *p[FWDSTEP]; 1191 1192 p[0] = (rte_pktmbuf_mtod(pkt[0], __m128i *)); 1193 p[1] = (rte_pktmbuf_mtod(pkt[1], __m128i *)); 1194 p[2] = (rte_pktmbuf_mtod(pkt[2], __m128i *)); 1195 p[3] = (rte_pktmbuf_mtod(pkt[3], __m128i *)); 1196 1197 ve[0] = val_eth[dst_port[0]]; 1198 te[0] = _mm_load_si128(p[0]); 1199 1200 ve[1] = val_eth[dst_port[1]]; 1201 te[1] = _mm_load_si128(p[1]); 1202 1203 ve[2] = val_eth[dst_port[2]]; 1204 te[2] = _mm_load_si128(p[2]); 1205 1206 ve[3] = val_eth[dst_port[3]]; 1207 te[3] = _mm_load_si128(p[3]); 1208 1209 /* Update first 12 bytes, keep rest bytes intact. */ 1210 te[0] = _mm_blend_epi16(te[0], ve[0], MASK_ETH); 1211 te[1] = _mm_blend_epi16(te[1], ve[1], MASK_ETH); 1212 te[2] = _mm_blend_epi16(te[2], ve[2], MASK_ETH); 1213 te[3] = _mm_blend_epi16(te[3], ve[3], MASK_ETH); 1214 1215 _mm_store_si128(p[0], te[0]); 1216 _mm_store_si128(p[1], te[1]); 1217 _mm_store_si128(p[2], te[2]); 1218 _mm_store_si128(p[3], te[3]); 1219 1220 rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[0] + 1), 1221 &dst_port[0], pkt[0]->ol_flags); 1222 rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[1] + 1), 1223 &dst_port[1], pkt[1]->ol_flags); 1224 rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[2] + 1), 1225 &dst_port[2], pkt[2]->ol_flags); 1226 rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[3] + 1), 1227 &dst_port[3], pkt[3]->ol_flags); 1228 } 1229 1230 /* 1231 * We group consecutive packets with the same destionation port into one burst. 1232 * To avoid extra latency this is done together with some other packet 1233 * processing, but after we made a final decision about packet's destination. 1234 * To do this we maintain: 1235 * pnum - array of number of consecutive packets with the same dest port for 1236 * each packet in the input burst. 1237 * lp - pointer to the last updated element in the pnum. 1238 * dlp - dest port value lp corresponds to. 1239 */ 1240 1241 #define GRPSZ (1 << FWDSTEP) 1242 #define GRPMSK (GRPSZ - 1) 1243 1244 #define GROUP_PORT_STEP(dlp, dcp, lp, pn, idx) do { \ 1245 if (likely((dlp) == (dcp)[(idx)])) { \ 1246 (lp)[0]++; \ 1247 } else { \ 1248 (dlp) = (dcp)[idx]; \ 1249 (lp) = (pn) + (idx); \ 1250 (lp)[0] = 1; \ 1251 } \ 1252 } while (0) 1253 1254 /* 1255 * Group consecutive packets with the same destination port in bursts of 4. 1256 * Suppose we have array of destionation ports: 1257 * dst_port[] = {a, b, c, d,, e, ... } 1258 * dp1 should contain: <a, b, c, d>, dp2: <b, c, d, e>. 1259 * We doing 4 comparisions at once and the result is 4 bit mask. 1260 * This mask is used as an index into prebuild array of pnum values. 1261 */ 1262 static inline uint16_t * 1263 port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, __m128i dp1, __m128i dp2) 1264 { 1265 static const struct { 1266 uint64_t pnum; /* prebuild 4 values for pnum[]. */ 1267 int32_t idx; /* index for new last updated elemnet. */ 1268 uint16_t lpv; /* add value to the last updated element. */ 1269 } gptbl[GRPSZ] = { 1270 { 1271 /* 0: a != b, b != c, c != d, d != e */ 1272 .pnum = UINT64_C(0x0001000100010001), 1273 .idx = 4, 1274 .lpv = 0, 1275 }, 1276 { 1277 /* 1: a == b, b != c, c != d, d != e */ 1278 .pnum = UINT64_C(0x0001000100010002), 1279 .idx = 4, 1280 .lpv = 1, 1281 }, 1282 { 1283 /* 2: a != b, b == c, c != d, d != e */ 1284 .pnum = UINT64_C(0x0001000100020001), 1285 .idx = 4, 1286 .lpv = 0, 1287 }, 1288 { 1289 /* 3: a == b, b == c, c != d, d != e */ 1290 .pnum = UINT64_C(0x0001000100020003), 1291 .idx = 4, 1292 .lpv = 2, 1293 }, 1294 { 1295 /* 4: a != b, b != c, c == d, d != e */ 1296 .pnum = UINT64_C(0x0001000200010001), 1297 .idx = 4, 1298 .lpv = 0, 1299 }, 1300 { 1301 /* 5: a == b, b != c, c == d, d != e */ 1302 .pnum = UINT64_C(0x0001000200010002), 1303 .idx = 4, 1304 .lpv = 1, 1305 }, 1306 { 1307 /* 6: a != b, b == c, c == d, d != e */ 1308 .pnum = UINT64_C(0x0001000200030001), 1309 .idx = 4, 1310 .lpv = 0, 1311 }, 1312 { 1313 /* 7: a == b, b == c, c == d, d != e */ 1314 .pnum = UINT64_C(0x0001000200030004), 1315 .idx = 4, 1316 .lpv = 3, 1317 }, 1318 { 1319 /* 8: a != b, b != c, c != d, d == e */ 1320 .pnum = UINT64_C(0x0002000100010001), 1321 .idx = 3, 1322 .lpv = 0, 1323 }, 1324 { 1325 /* 9: a == b, b != c, c != d, d == e */ 1326 .pnum = UINT64_C(0x0002000100010002), 1327 .idx = 3, 1328 .lpv = 1, 1329 }, 1330 { 1331 /* 0xa: a != b, b == c, c != d, d == e */ 1332 .pnum = UINT64_C(0x0002000100020001), 1333 .idx = 3, 1334 .lpv = 0, 1335 }, 1336 { 1337 /* 0xb: a == b, b == c, c != d, d == e */ 1338 .pnum = UINT64_C(0x0002000100020003), 1339 .idx = 3, 1340 .lpv = 2, 1341 }, 1342 { 1343 /* 0xc: a != b, b != c, c == d, d == e */ 1344 .pnum = UINT64_C(0x0002000300010001), 1345 .idx = 2, 1346 .lpv = 0, 1347 }, 1348 { 1349 /* 0xd: a == b, b != c, c == d, d == e */ 1350 .pnum = UINT64_C(0x0002000300010002), 1351 .idx = 2, 1352 .lpv = 1, 1353 }, 1354 { 1355 /* 0xe: a != b, b == c, c == d, d == e */ 1356 .pnum = UINT64_C(0x0002000300040001), 1357 .idx = 1, 1358 .lpv = 0, 1359 }, 1360 { 1361 /* 0xf: a == b, b == c, c == d, d == e */ 1362 .pnum = UINT64_C(0x0002000300040005), 1363 .idx = 0, 1364 .lpv = 4, 1365 }, 1366 }; 1367 1368 union { 1369 uint16_t u16[FWDSTEP + 1]; 1370 uint64_t u64; 1371 } *pnum = (void *)pn; 1372 1373 int32_t v; 1374 1375 dp1 = _mm_cmpeq_epi16(dp1, dp2); 1376 dp1 = _mm_unpacklo_epi16(dp1, dp1); 1377 v = _mm_movemask_ps((__m128)dp1); 1378 1379 /* update last port counter. */ 1380 lp[0] += gptbl[v].lpv; 1381 1382 /* if dest port value has changed. */ 1383 if (v != GRPMSK) { 1384 lp = pnum->u16 + gptbl[v].idx; 1385 lp[0] = 1; 1386 pnum->u64 = gptbl[v].pnum; 1387 } 1388 1389 return lp; 1390 } 1391 1392 #endif /* APP_LOOKUP_METHOD */ 1393 1394 /* main processing loop */ 1395 static int 1396 main_loop(__attribute__((unused)) void *dummy) 1397 { 1398 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1399 unsigned lcore_id; 1400 uint64_t prev_tsc, diff_tsc, cur_tsc; 1401 int i, j, nb_rx; 1402 uint8_t portid, queueid; 1403 struct lcore_conf *qconf; 1404 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 1405 US_PER_S * BURST_TX_DRAIN_US; 1406 1407 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \ 1408 (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)) 1409 int32_t k; 1410 uint16_t dlp; 1411 uint16_t *lp; 1412 uint16_t dst_port[MAX_PKT_BURST]; 1413 __m128i dip[MAX_PKT_BURST / FWDSTEP]; 1414 uint32_t flag[MAX_PKT_BURST / FWDSTEP]; 1415 uint16_t pnum[MAX_PKT_BURST + 1]; 1416 #endif 1417 1418 prev_tsc = 0; 1419 1420 lcore_id = rte_lcore_id(); 1421 qconf = &lcore_conf[lcore_id]; 1422 1423 if (qconf->n_rx_queue == 0) { 1424 RTE_LOG(INFO, L3FWD, "lcore %u has nothing to do\n", lcore_id); 1425 return 0; 1426 } 1427 1428 RTE_LOG(INFO, L3FWD, "entering main loop on lcore %u\n", lcore_id); 1429 1430 for (i = 0; i < qconf->n_rx_queue; i++) { 1431 1432 portid = qconf->rx_queue_list[i].port_id; 1433 queueid = qconf->rx_queue_list[i].queue_id; 1434 RTE_LOG(INFO, L3FWD, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", lcore_id, 1435 portid, queueid); 1436 } 1437 1438 while (1) { 1439 1440 cur_tsc = rte_rdtsc(); 1441 1442 /* 1443 * TX burst queue drain 1444 */ 1445 diff_tsc = cur_tsc - prev_tsc; 1446 if (unlikely(diff_tsc > drain_tsc)) { 1447 1448 /* 1449 * This could be optimized (use queueid instead of 1450 * portid), but it is not called so often 1451 */ 1452 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 1453 if (qconf->tx_mbufs[portid].len == 0) 1454 continue; 1455 send_burst(qconf, 1456 qconf->tx_mbufs[portid].len, 1457 portid); 1458 qconf->tx_mbufs[portid].len = 0; 1459 } 1460 1461 prev_tsc = cur_tsc; 1462 } 1463 1464 /* 1465 * Read packet from RX queues 1466 */ 1467 for (i = 0; i < qconf->n_rx_queue; ++i) { 1468 portid = qconf->rx_queue_list[i].port_id; 1469 queueid = qconf->rx_queue_list[i].queue_id; 1470 nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, 1471 MAX_PKT_BURST); 1472 if (nb_rx == 0) 1473 continue; 1474 1475 #if (ENABLE_MULTI_BUFFER_OPTIMIZE == 1) 1476 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1477 { 1478 /* 1479 * Send nb_rx - nb_rx%4 packets 1480 * in groups of 4. 1481 */ 1482 int32_t n = RTE_ALIGN_FLOOR(nb_rx, 4); 1483 for (j = 0; j < n ; j+=4) { 1484 uint32_t ol_flag = pkts_burst[j]->ol_flags 1485 & pkts_burst[j+1]->ol_flags 1486 & pkts_burst[j+2]->ol_flags 1487 & pkts_burst[j+3]->ol_flags; 1488 if (ol_flag & PKT_RX_IPV4_HDR ) { 1489 simple_ipv4_fwd_4pkts(&pkts_burst[j], 1490 portid, qconf); 1491 } else if (ol_flag & PKT_RX_IPV6_HDR) { 1492 simple_ipv6_fwd_4pkts(&pkts_burst[j], 1493 portid, qconf); 1494 } else { 1495 l3fwd_simple_forward(pkts_burst[j], 1496 portid, qconf); 1497 l3fwd_simple_forward(pkts_burst[j+1], 1498 portid, qconf); 1499 l3fwd_simple_forward(pkts_burst[j+2], 1500 portid, qconf); 1501 l3fwd_simple_forward(pkts_burst[j+3], 1502 portid, qconf); 1503 } 1504 } 1505 for (; j < nb_rx ; j++) { 1506 l3fwd_simple_forward(pkts_burst[j], 1507 portid, qconf); 1508 } 1509 } 1510 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 1511 1512 k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP); 1513 for (j = 0; j != k; j += FWDSTEP) { 1514 processx4_step1(&pkts_burst[j], 1515 &dip[j / FWDSTEP], 1516 &flag[j / FWDSTEP]); 1517 } 1518 1519 k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP); 1520 for (j = 0; j != k; j += FWDSTEP) { 1521 processx4_step2(qconf, dip[j / FWDSTEP], 1522 flag[j / FWDSTEP], portid, 1523 &pkts_burst[j], &dst_port[j]); 1524 } 1525 1526 /* 1527 * Finish packet processing and group consecutive 1528 * packets with the same destination port. 1529 */ 1530 k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP); 1531 if (k != 0) { 1532 __m128i dp1, dp2; 1533 1534 lp = pnum; 1535 lp[0] = 1; 1536 1537 processx4_step3(pkts_burst, dst_port); 1538 1539 /* dp1: <d[0], d[1], d[2], d[3], ... > */ 1540 dp1 = _mm_loadu_si128((__m128i *)dst_port); 1541 1542 for (j = FWDSTEP; j != k; j += FWDSTEP) { 1543 processx4_step3(&pkts_burst[j], 1544 &dst_port[j]); 1545 1546 /* 1547 * dp2: 1548 * <d[j-3], d[j-2], d[j-1], d[j], ... > 1549 */ 1550 dp2 = _mm_loadu_si128((__m128i *) 1551 &dst_port[j - FWDSTEP + 1]); 1552 lp = port_groupx4(&pnum[j - FWDSTEP], 1553 lp, dp1, dp2); 1554 1555 /* 1556 * dp1: 1557 * <d[j], d[j+1], d[j+2], d[j+3], ... > 1558 */ 1559 dp1 = _mm_srli_si128(dp2, 1560 (FWDSTEP - 1) * 1561 sizeof(dst_port[0])); 1562 } 1563 1564 /* 1565 * dp2: <d[j-3], d[j-2], d[j-1], d[j-1], ... > 1566 */ 1567 dp2 = _mm_shufflelo_epi16(dp1, 0xf9); 1568 lp = port_groupx4(&pnum[j - FWDSTEP], lp, 1569 dp1, dp2); 1570 1571 /* 1572 * remove values added by the last repeated 1573 * dst port. 1574 */ 1575 lp[0]--; 1576 dlp = dst_port[j - 1]; 1577 } else { 1578 /* set dlp and lp to the never used values. */ 1579 dlp = BAD_PORT - 1; 1580 lp = pnum + MAX_PKT_BURST; 1581 } 1582 1583 /* Process up to last 3 packets one by one. */ 1584 switch (nb_rx % FWDSTEP) { 1585 case 3: 1586 process_packet(qconf, pkts_burst[j], 1587 dst_port + j, portid); 1588 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); 1589 j++; 1590 case 2: 1591 process_packet(qconf, pkts_burst[j], 1592 dst_port + j, portid); 1593 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); 1594 j++; 1595 case 1: 1596 process_packet(qconf, pkts_burst[j], 1597 dst_port + j, portid); 1598 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); 1599 j++; 1600 } 1601 1602 /* 1603 * Send packets out, through destination port. 1604 * Consecuteve pacekts with the same destination port 1605 * are already grouped together. 1606 * If destination port for the packet equals BAD_PORT, 1607 * then free the packet without sending it out. 1608 */ 1609 for (j = 0; j < nb_rx; j += k) { 1610 1611 int32_t m; 1612 uint16_t pn; 1613 1614 pn = dst_port[j]; 1615 k = pnum[j]; 1616 1617 if (likely(pn != BAD_PORT)) { 1618 send_packetsx4(qconf, pn, 1619 pkts_burst + j, k); 1620 } else { 1621 for (m = j; m != j + k; m++) 1622 rte_pktmbuf_free(pkts_burst[m]); 1623 } 1624 } 1625 1626 #endif /* APP_LOOKUP_METHOD */ 1627 #else /* ENABLE_MULTI_BUFFER_OPTIMIZE == 0 */ 1628 1629 /* Prefetch first packets */ 1630 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1631 rte_prefetch0(rte_pktmbuf_mtod( 1632 pkts_burst[j], void *)); 1633 } 1634 1635 /* Prefetch and forward already prefetched packets */ 1636 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1637 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1638 j + PREFETCH_OFFSET], void *)); 1639 l3fwd_simple_forward(pkts_burst[j], portid, 1640 qconf); 1641 } 1642 1643 /* Forward remaining prefetched packets */ 1644 for (; j < nb_rx; j++) { 1645 l3fwd_simple_forward(pkts_burst[j], portid, 1646 qconf); 1647 } 1648 #endif /* ENABLE_MULTI_BUFFER_OPTIMIZE */ 1649 1650 } 1651 } 1652 } 1653 1654 static int 1655 check_lcore_params(void) 1656 { 1657 uint8_t queue, lcore; 1658 uint16_t i; 1659 int socketid; 1660 1661 for (i = 0; i < nb_lcore_params; ++i) { 1662 queue = lcore_params[i].queue_id; 1663 if (queue >= MAX_RX_QUEUE_PER_PORT) { 1664 printf("invalid queue number: %hhu\n", queue); 1665 return -1; 1666 } 1667 lcore = lcore_params[i].lcore_id; 1668 if (!rte_lcore_is_enabled(lcore)) { 1669 printf("error: lcore %hhu is not enabled in lcore mask\n", lcore); 1670 return -1; 1671 } 1672 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) && 1673 (numa_on == 0)) { 1674 printf("warning: lcore %hhu is on socket %d with numa off \n", 1675 lcore, socketid); 1676 } 1677 } 1678 return 0; 1679 } 1680 1681 static int 1682 check_port_config(const unsigned nb_ports) 1683 { 1684 unsigned portid; 1685 uint16_t i; 1686 1687 for (i = 0; i < nb_lcore_params; ++i) { 1688 portid = lcore_params[i].port_id; 1689 if ((enabled_port_mask & (1 << portid)) == 0) { 1690 printf("port %u is not enabled in port mask\n", portid); 1691 return -1; 1692 } 1693 if (portid >= nb_ports) { 1694 printf("port %u is not present on the board\n", portid); 1695 return -1; 1696 } 1697 } 1698 return 0; 1699 } 1700 1701 static uint8_t 1702 get_port_n_rx_queues(const uint8_t port) 1703 { 1704 int queue = -1; 1705 uint16_t i; 1706 1707 for (i = 0; i < nb_lcore_params; ++i) { 1708 if (lcore_params[i].port_id == port && lcore_params[i].queue_id > queue) 1709 queue = lcore_params[i].queue_id; 1710 } 1711 return (uint8_t)(++queue); 1712 } 1713 1714 static int 1715 init_lcore_rx_queues(void) 1716 { 1717 uint16_t i, nb_rx_queue; 1718 uint8_t lcore; 1719 1720 for (i = 0; i < nb_lcore_params; ++i) { 1721 lcore = lcore_params[i].lcore_id; 1722 nb_rx_queue = lcore_conf[lcore].n_rx_queue; 1723 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1724 printf("error: too many queues (%u) for lcore: %u\n", 1725 (unsigned)nb_rx_queue + 1, (unsigned)lcore); 1726 return -1; 1727 } else { 1728 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1729 lcore_params[i].port_id; 1730 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1731 lcore_params[i].queue_id; 1732 lcore_conf[lcore].n_rx_queue++; 1733 } 1734 } 1735 return 0; 1736 } 1737 1738 /* display usage */ 1739 static void 1740 print_usage(const char *prgname) 1741 { 1742 printf ("%s [EAL options] -- -p PORTMASK -P" 1743 " [--config (port,queue,lcore)[,(port,queue,lcore]]" 1744 " [--enable-jumbo [--max-pkt-len PKTLEN]]\n" 1745 " -p PORTMASK: hexadecimal bitmask of ports to configure\n" 1746 " -P : enable promiscuous mode\n" 1747 " --config (port,queue,lcore): rx queues configuration\n" 1748 " --no-numa: optional, disable numa awareness\n" 1749 " --ipv6: optional, specify it if running ipv6 packets\n" 1750 " --enable-jumbo: enable jumbo frame" 1751 " which max packet len is PKTLEN in decimal (64-9600)\n" 1752 " --hash-entry-num: specify the hash entry number in hexadecimal to be setup\n", 1753 prgname); 1754 } 1755 1756 static int parse_max_pkt_len(const char *pktlen) 1757 { 1758 char *end = NULL; 1759 unsigned long len; 1760 1761 /* parse decimal string */ 1762 len = strtoul(pktlen, &end, 10); 1763 if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) 1764 return -1; 1765 1766 if (len == 0) 1767 return -1; 1768 1769 return len; 1770 } 1771 1772 static int 1773 parse_portmask(const char *portmask) 1774 { 1775 char *end = NULL; 1776 unsigned long pm; 1777 1778 /* parse hexadecimal string */ 1779 pm = strtoul(portmask, &end, 16); 1780 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1781 return -1; 1782 1783 if (pm == 0) 1784 return -1; 1785 1786 return pm; 1787 } 1788 1789 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1790 static int 1791 parse_hash_entry_number(const char *hash_entry_num) 1792 { 1793 char *end = NULL; 1794 unsigned long hash_en; 1795 /* parse hexadecimal string */ 1796 hash_en = strtoul(hash_entry_num, &end, 16); 1797 if ((hash_entry_num[0] == '\0') || (end == NULL) || (*end != '\0')) 1798 return -1; 1799 1800 if (hash_en == 0) 1801 return -1; 1802 1803 return hash_en; 1804 } 1805 #endif 1806 1807 static int 1808 parse_config(const char *q_arg) 1809 { 1810 char s[256]; 1811 const char *p, *p0 = q_arg; 1812 char *end; 1813 enum fieldnames { 1814 FLD_PORT = 0, 1815 FLD_QUEUE, 1816 FLD_LCORE, 1817 _NUM_FLD 1818 }; 1819 unsigned long int_fld[_NUM_FLD]; 1820 char *str_fld[_NUM_FLD]; 1821 int i; 1822 unsigned size; 1823 1824 nb_lcore_params = 0; 1825 1826 while ((p = strchr(p0,'(')) != NULL) { 1827 ++p; 1828 if((p0 = strchr(p,')')) == NULL) 1829 return -1; 1830 1831 size = p0 - p; 1832 if(size >= sizeof(s)) 1833 return -1; 1834 1835 snprintf(s, sizeof(s), "%.*s", size, p); 1836 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD) 1837 return -1; 1838 for (i = 0; i < _NUM_FLD; i++){ 1839 errno = 0; 1840 int_fld[i] = strtoul(str_fld[i], &end, 0); 1841 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1842 return -1; 1843 } 1844 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1845 printf("exceeded max number of lcore params: %hu\n", 1846 nb_lcore_params); 1847 return -1; 1848 } 1849 lcore_params_array[nb_lcore_params].port_id = (uint8_t)int_fld[FLD_PORT]; 1850 lcore_params_array[nb_lcore_params].queue_id = (uint8_t)int_fld[FLD_QUEUE]; 1851 lcore_params_array[nb_lcore_params].lcore_id = (uint8_t)int_fld[FLD_LCORE]; 1852 ++nb_lcore_params; 1853 } 1854 lcore_params = lcore_params_array; 1855 return 0; 1856 } 1857 1858 #define CMD_LINE_OPT_CONFIG "config" 1859 #define CMD_LINE_OPT_NO_NUMA "no-numa" 1860 #define CMD_LINE_OPT_IPV6 "ipv6" 1861 #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo" 1862 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num" 1863 1864 /* Parse the argument given in the command line of the application */ 1865 static int 1866 parse_args(int argc, char **argv) 1867 { 1868 int opt, ret; 1869 char **argvopt; 1870 int option_index; 1871 char *prgname = argv[0]; 1872 static struct option lgopts[] = { 1873 {CMD_LINE_OPT_CONFIG, 1, 0, 0}, 1874 {CMD_LINE_OPT_NO_NUMA, 0, 0, 0}, 1875 {CMD_LINE_OPT_IPV6, 0, 0, 0}, 1876 {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, 0}, 1877 {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, 0}, 1878 {NULL, 0, 0, 0} 1879 }; 1880 1881 argvopt = argv; 1882 1883 while ((opt = getopt_long(argc, argvopt, "p:P", 1884 lgopts, &option_index)) != EOF) { 1885 1886 switch (opt) { 1887 /* portmask */ 1888 case 'p': 1889 enabled_port_mask = parse_portmask(optarg); 1890 if (enabled_port_mask == 0) { 1891 printf("invalid portmask\n"); 1892 print_usage(prgname); 1893 return -1; 1894 } 1895 break; 1896 case 'P': 1897 printf("Promiscuous mode selected\n"); 1898 promiscuous_on = 1; 1899 break; 1900 1901 /* long options */ 1902 case 0: 1903 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_CONFIG, 1904 sizeof (CMD_LINE_OPT_CONFIG))) { 1905 ret = parse_config(optarg); 1906 if (ret) { 1907 printf("invalid config\n"); 1908 print_usage(prgname); 1909 return -1; 1910 } 1911 } 1912 1913 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_NUMA, 1914 sizeof(CMD_LINE_OPT_NO_NUMA))) { 1915 printf("numa is disabled \n"); 1916 numa_on = 0; 1917 } 1918 1919 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1920 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_IPV6, 1921 sizeof(CMD_LINE_OPT_IPV6))) { 1922 printf("ipv6 is specified \n"); 1923 ipv6 = 1; 1924 } 1925 #endif 1926 1927 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ENABLE_JUMBO, 1928 sizeof (CMD_LINE_OPT_ENABLE_JUMBO))) { 1929 struct option lenopts = {"max-pkt-len", required_argument, 0, 0}; 1930 1931 printf("jumbo frame is enabled - disabling simple TX path\n"); 1932 port_conf.rxmode.jumbo_frame = 1; 1933 1934 /* if no max-pkt-len set, use the default value ETHER_MAX_LEN */ 1935 if (0 == getopt_long(argc, argvopt, "", &lenopts, &option_index)) { 1936 ret = parse_max_pkt_len(optarg); 1937 if ((ret < 64) || (ret > MAX_JUMBO_PKT_LEN)){ 1938 printf("invalid packet length\n"); 1939 print_usage(prgname); 1940 return -1; 1941 } 1942 port_conf.rxmode.max_rx_pkt_len = ret; 1943 } 1944 printf("set jumbo frame max packet length to %u\n", 1945 (unsigned int)port_conf.rxmode.max_rx_pkt_len); 1946 } 1947 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1948 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_HASH_ENTRY_NUM, 1949 sizeof(CMD_LINE_OPT_HASH_ENTRY_NUM))) { 1950 ret = parse_hash_entry_number(optarg); 1951 if ((ret > 0) && (ret <= L3FWD_HASH_ENTRIES)) { 1952 hash_entry_number = ret; 1953 } else { 1954 printf("invalid hash entry number\n"); 1955 print_usage(prgname); 1956 return -1; 1957 } 1958 } 1959 #endif 1960 break; 1961 1962 default: 1963 print_usage(prgname); 1964 return -1; 1965 } 1966 } 1967 1968 if (optind >= 0) 1969 argv[optind-1] = prgname; 1970 1971 ret = optind-1; 1972 optind = 0; /* reset getopt lib */ 1973 return ret; 1974 } 1975 1976 static void 1977 print_ethaddr(const char *name, const struct ether_addr *eth_addr) 1978 { 1979 char buf[ETHER_ADDR_FMT_SIZE]; 1980 ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); 1981 printf("%s%s", name, buf); 1982 } 1983 1984 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) 1985 1986 static void convert_ipv4_5tuple(struct ipv4_5tuple* key1, 1987 union ipv4_5tuple_host* key2) 1988 { 1989 key2->ip_dst = rte_cpu_to_be_32(key1->ip_dst); 1990 key2->ip_src = rte_cpu_to_be_32(key1->ip_src); 1991 key2->port_dst = rte_cpu_to_be_16(key1->port_dst); 1992 key2->port_src = rte_cpu_to_be_16(key1->port_src); 1993 key2->proto = key1->proto; 1994 key2->pad0 = 0; 1995 key2->pad1 = 0; 1996 return; 1997 } 1998 1999 static void convert_ipv6_5tuple(struct ipv6_5tuple* key1, 2000 union ipv6_5tuple_host* key2) 2001 { 2002 uint32_t i; 2003 for (i = 0; i < 16; i++) 2004 { 2005 key2->ip_dst[i] = key1->ip_dst[i]; 2006 key2->ip_src[i] = key1->ip_src[i]; 2007 } 2008 key2->port_dst = rte_cpu_to_be_16(key1->port_dst); 2009 key2->port_src = rte_cpu_to_be_16(key1->port_src); 2010 key2->proto = key1->proto; 2011 key2->pad0 = 0; 2012 key2->pad1 = 0; 2013 key2->reserve = 0; 2014 return; 2015 } 2016 2017 #define BYTE_VALUE_MAX 256 2018 #define ALL_32_BITS 0xffffffff 2019 #define BIT_8_TO_15 0x0000ff00 2020 static inline void 2021 populate_ipv4_few_flow_into_table(const struct rte_hash* h) 2022 { 2023 uint32_t i; 2024 int32_t ret; 2025 uint32_t array_len = sizeof(ipv4_l3fwd_route_array)/sizeof(ipv4_l3fwd_route_array[0]); 2026 2027 mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15); 2028 for (i = 0; i < array_len; i++) { 2029 struct ipv4_l3fwd_route entry; 2030 union ipv4_5tuple_host newkey; 2031 entry = ipv4_l3fwd_route_array[i]; 2032 convert_ipv4_5tuple(&entry.key, &newkey); 2033 ret = rte_hash_add_key (h,(void *) &newkey); 2034 if (ret < 0) { 2035 rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32 2036 " to the l3fwd hash.\n", i); 2037 } 2038 ipv4_l3fwd_out_if[ret] = entry.if_out; 2039 } 2040 printf("Hash: Adding 0x%" PRIx32 " keys\n", array_len); 2041 } 2042 2043 #define BIT_16_TO_23 0x00ff0000 2044 static inline void 2045 populate_ipv6_few_flow_into_table(const struct rte_hash* h) 2046 { 2047 uint32_t i; 2048 int32_t ret; 2049 uint32_t array_len = sizeof(ipv6_l3fwd_route_array)/sizeof(ipv6_l3fwd_route_array[0]); 2050 2051 mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23); 2052 mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS); 2053 for (i = 0; i < array_len; i++) { 2054 struct ipv6_l3fwd_route entry; 2055 union ipv6_5tuple_host newkey; 2056 entry = ipv6_l3fwd_route_array[i]; 2057 convert_ipv6_5tuple(&entry.key, &newkey); 2058 ret = rte_hash_add_key (h, (void *) &newkey); 2059 if (ret < 0) { 2060 rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32 2061 " to the l3fwd hash.\n", i); 2062 } 2063 ipv6_l3fwd_out_if[ret] = entry.if_out; 2064 } 2065 printf("Hash: Adding 0x%" PRIx32 "keys\n", array_len); 2066 } 2067 2068 #define NUMBER_PORT_USED 4 2069 static inline void 2070 populate_ipv4_many_flow_into_table(const struct rte_hash* h, 2071 unsigned int nr_flow) 2072 { 2073 unsigned i; 2074 mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15); 2075 for (i = 0; i < nr_flow; i++) { 2076 struct ipv4_l3fwd_route entry; 2077 union ipv4_5tuple_host newkey; 2078 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX); 2079 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX); 2080 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX)); 2081 /* Create the ipv4 exact match flow */ 2082 memset(&entry, 0, sizeof(entry)); 2083 switch (i & (NUMBER_PORT_USED -1)) { 2084 case 0: 2085 entry = ipv4_l3fwd_route_array[0]; 2086 entry.key.ip_dst = IPv4(101,c,b,a); 2087 break; 2088 case 1: 2089 entry = ipv4_l3fwd_route_array[1]; 2090 entry.key.ip_dst = IPv4(201,c,b,a); 2091 break; 2092 case 2: 2093 entry = ipv4_l3fwd_route_array[2]; 2094 entry.key.ip_dst = IPv4(111,c,b,a); 2095 break; 2096 case 3: 2097 entry = ipv4_l3fwd_route_array[3]; 2098 entry.key.ip_dst = IPv4(211,c,b,a); 2099 break; 2100 }; 2101 convert_ipv4_5tuple(&entry.key, &newkey); 2102 int32_t ret = rte_hash_add_key(h,(void *) &newkey); 2103 if (ret < 0) { 2104 rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i); 2105 } 2106 ipv4_l3fwd_out_if[ret] = (uint8_t) entry.if_out; 2107 2108 } 2109 printf("Hash: Adding 0x%x keys\n", nr_flow); 2110 } 2111 2112 static inline void 2113 populate_ipv6_many_flow_into_table(const struct rte_hash* h, 2114 unsigned int nr_flow) 2115 { 2116 unsigned i; 2117 mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23); 2118 mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS); 2119 for (i = 0; i < nr_flow; i++) { 2120 struct ipv6_l3fwd_route entry; 2121 union ipv6_5tuple_host newkey; 2122 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX); 2123 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX); 2124 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX)); 2125 /* Create the ipv6 exact match flow */ 2126 memset(&entry, 0, sizeof(entry)); 2127 switch (i & (NUMBER_PORT_USED - 1)) { 2128 case 0: entry = ipv6_l3fwd_route_array[0]; break; 2129 case 1: entry = ipv6_l3fwd_route_array[1]; break; 2130 case 2: entry = ipv6_l3fwd_route_array[2]; break; 2131 case 3: entry = ipv6_l3fwd_route_array[3]; break; 2132 }; 2133 entry.key.ip_dst[13] = c; 2134 entry.key.ip_dst[14] = b; 2135 entry.key.ip_dst[15] = a; 2136 convert_ipv6_5tuple(&entry.key, &newkey); 2137 int32_t ret = rte_hash_add_key(h,(void *) &newkey); 2138 if (ret < 0) { 2139 rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i); 2140 } 2141 ipv6_l3fwd_out_if[ret] = (uint8_t) entry.if_out; 2142 2143 } 2144 printf("Hash: Adding 0x%x keys\n", nr_flow); 2145 } 2146 2147 static void 2148 setup_hash(int socketid) 2149 { 2150 struct rte_hash_parameters ipv4_l3fwd_hash_params = { 2151 .name = NULL, 2152 .entries = L3FWD_HASH_ENTRIES, 2153 .bucket_entries = 4, 2154 .key_len = sizeof(union ipv4_5tuple_host), 2155 .hash_func = ipv4_hash_crc, 2156 .hash_func_init_val = 0, 2157 }; 2158 2159 struct rte_hash_parameters ipv6_l3fwd_hash_params = { 2160 .name = NULL, 2161 .entries = L3FWD_HASH_ENTRIES, 2162 .bucket_entries = 4, 2163 .key_len = sizeof(union ipv6_5tuple_host), 2164 .hash_func = ipv6_hash_crc, 2165 .hash_func_init_val = 0, 2166 }; 2167 2168 char s[64]; 2169 2170 /* create ipv4 hash */ 2171 snprintf(s, sizeof(s), "ipv4_l3fwd_hash_%d", socketid); 2172 ipv4_l3fwd_hash_params.name = s; 2173 ipv4_l3fwd_hash_params.socket_id = socketid; 2174 ipv4_l3fwd_lookup_struct[socketid] = rte_hash_create(&ipv4_l3fwd_hash_params); 2175 if (ipv4_l3fwd_lookup_struct[socketid] == NULL) 2176 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on " 2177 "socket %d\n", socketid); 2178 2179 /* create ipv6 hash */ 2180 snprintf(s, sizeof(s), "ipv6_l3fwd_hash_%d", socketid); 2181 ipv6_l3fwd_hash_params.name = s; 2182 ipv6_l3fwd_hash_params.socket_id = socketid; 2183 ipv6_l3fwd_lookup_struct[socketid] = rte_hash_create(&ipv6_l3fwd_hash_params); 2184 if (ipv6_l3fwd_lookup_struct[socketid] == NULL) 2185 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on " 2186 "socket %d\n", socketid); 2187 2188 if (hash_entry_number != HASH_ENTRY_NUMBER_DEFAULT) { 2189 /* For testing hash matching with a large number of flows we 2190 * generate millions of IP 5-tuples with an incremented dst 2191 * address to initialize the hash table. */ 2192 if (ipv6 == 0) { 2193 /* populate the ipv4 hash */ 2194 populate_ipv4_many_flow_into_table( 2195 ipv4_l3fwd_lookup_struct[socketid], hash_entry_number); 2196 } else { 2197 /* populate the ipv6 hash */ 2198 populate_ipv6_many_flow_into_table( 2199 ipv6_l3fwd_lookup_struct[socketid], hash_entry_number); 2200 } 2201 } else { 2202 /* Use data in ipv4/ipv6 l3fwd lookup table directly to initialize the hash table */ 2203 if (ipv6 == 0) { 2204 /* populate the ipv4 hash */ 2205 populate_ipv4_few_flow_into_table(ipv4_l3fwd_lookup_struct[socketid]); 2206 } else { 2207 /* populate the ipv6 hash */ 2208 populate_ipv6_few_flow_into_table(ipv6_l3fwd_lookup_struct[socketid]); 2209 } 2210 } 2211 } 2212 #endif 2213 2214 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 2215 static void 2216 setup_lpm(int socketid) 2217 { 2218 struct rte_lpm6_config config; 2219 unsigned i; 2220 int ret; 2221 char s[64]; 2222 2223 /* create the LPM table */ 2224 snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socketid); 2225 ipv4_l3fwd_lookup_struct[socketid] = rte_lpm_create(s, socketid, 2226 IPV4_L3FWD_LPM_MAX_RULES, 0); 2227 if (ipv4_l3fwd_lookup_struct[socketid] == NULL) 2228 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table" 2229 " on socket %d\n", socketid); 2230 2231 /* populate the LPM table */ 2232 for (i = 0; i < IPV4_L3FWD_NUM_ROUTES; i++) { 2233 2234 /* skip unused ports */ 2235 if ((1 << ipv4_l3fwd_route_array[i].if_out & 2236 enabled_port_mask) == 0) 2237 continue; 2238 2239 ret = rte_lpm_add(ipv4_l3fwd_lookup_struct[socketid], 2240 ipv4_l3fwd_route_array[i].ip, 2241 ipv4_l3fwd_route_array[i].depth, 2242 ipv4_l3fwd_route_array[i].if_out); 2243 2244 if (ret < 0) { 2245 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the " 2246 "l3fwd LPM table on socket %d\n", 2247 i, socketid); 2248 } 2249 2250 printf("LPM: Adding route 0x%08x / %d (%d)\n", 2251 (unsigned)ipv4_l3fwd_route_array[i].ip, 2252 ipv4_l3fwd_route_array[i].depth, 2253 ipv4_l3fwd_route_array[i].if_out); 2254 } 2255 2256 /* create the LPM6 table */ 2257 snprintf(s, sizeof(s), "IPV6_L3FWD_LPM_%d", socketid); 2258 2259 config.max_rules = IPV6_L3FWD_LPM_MAX_RULES; 2260 config.number_tbl8s = IPV6_L3FWD_LPM_NUMBER_TBL8S; 2261 config.flags = 0; 2262 ipv6_l3fwd_lookup_struct[socketid] = rte_lpm6_create(s, socketid, 2263 &config); 2264 if (ipv6_l3fwd_lookup_struct[socketid] == NULL) 2265 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table" 2266 " on socket %d\n", socketid); 2267 2268 /* populate the LPM table */ 2269 for (i = 0; i < IPV6_L3FWD_NUM_ROUTES; i++) { 2270 2271 /* skip unused ports */ 2272 if ((1 << ipv6_l3fwd_route_array[i].if_out & 2273 enabled_port_mask) == 0) 2274 continue; 2275 2276 ret = rte_lpm6_add(ipv6_l3fwd_lookup_struct[socketid], 2277 ipv6_l3fwd_route_array[i].ip, 2278 ipv6_l3fwd_route_array[i].depth, 2279 ipv6_l3fwd_route_array[i].if_out); 2280 2281 if (ret < 0) { 2282 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the " 2283 "l3fwd LPM table on socket %d\n", 2284 i, socketid); 2285 } 2286 2287 printf("LPM: Adding route %s / %d (%d)\n", 2288 "IPV6", 2289 ipv6_l3fwd_route_array[i].depth, 2290 ipv6_l3fwd_route_array[i].if_out); 2291 } 2292 } 2293 #endif 2294 2295 static int 2296 init_mem(unsigned nb_mbuf) 2297 { 2298 struct lcore_conf *qconf; 2299 int socketid; 2300 unsigned lcore_id; 2301 char s[64]; 2302 2303 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2304 if (rte_lcore_is_enabled(lcore_id) == 0) 2305 continue; 2306 2307 if (numa_on) 2308 socketid = rte_lcore_to_socket_id(lcore_id); 2309 else 2310 socketid = 0; 2311 2312 if (socketid >= NB_SOCKETS) { 2313 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 2314 socketid, lcore_id, NB_SOCKETS); 2315 } 2316 if (pktmbuf_pool[socketid] == NULL) { 2317 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 2318 pktmbuf_pool[socketid] = 2319 rte_mempool_create(s, nb_mbuf, MBUF_SIZE, MEMPOOL_CACHE_SIZE, 2320 sizeof(struct rte_pktmbuf_pool_private), 2321 rte_pktmbuf_pool_init, NULL, 2322 rte_pktmbuf_init, NULL, 2323 socketid, 0); 2324 if (pktmbuf_pool[socketid] == NULL) 2325 rte_exit(EXIT_FAILURE, 2326 "Cannot init mbuf pool on socket %d\n", socketid); 2327 else 2328 printf("Allocated mbuf pool on socket %d\n", socketid); 2329 2330 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) 2331 setup_lpm(socketid); 2332 #else 2333 setup_hash(socketid); 2334 #endif 2335 } 2336 qconf = &lcore_conf[lcore_id]; 2337 qconf->ipv4_lookup_struct = ipv4_l3fwd_lookup_struct[socketid]; 2338 qconf->ipv6_lookup_struct = ipv6_l3fwd_lookup_struct[socketid]; 2339 } 2340 return 0; 2341 } 2342 2343 /* Check the link status of all ports in up to 9s, and print them finally */ 2344 static void 2345 check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) 2346 { 2347 #define CHECK_INTERVAL 100 /* 100ms */ 2348 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2349 uint8_t portid, count, all_ports_up, print_flag = 0; 2350 struct rte_eth_link link; 2351 2352 printf("\nChecking link status"); 2353 fflush(stdout); 2354 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2355 all_ports_up = 1; 2356 for (portid = 0; portid < port_num; portid++) { 2357 if ((port_mask & (1 << portid)) == 0) 2358 continue; 2359 memset(&link, 0, sizeof(link)); 2360 rte_eth_link_get_nowait(portid, &link); 2361 /* print link status if flag set */ 2362 if (print_flag == 1) { 2363 if (link.link_status) 2364 printf("Port %d Link Up - speed %u " 2365 "Mbps - %s\n", (uint8_t)portid, 2366 (unsigned)link.link_speed, 2367 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2368 ("full-duplex") : ("half-duplex\n")); 2369 else 2370 printf("Port %d Link Down\n", 2371 (uint8_t)portid); 2372 continue; 2373 } 2374 /* clear all_ports_up flag if any link down */ 2375 if (link.link_status == 0) { 2376 all_ports_up = 0; 2377 break; 2378 } 2379 } 2380 /* after finally printing all link status, get out */ 2381 if (print_flag == 1) 2382 break; 2383 2384 if (all_ports_up == 0) { 2385 printf("."); 2386 fflush(stdout); 2387 rte_delay_ms(CHECK_INTERVAL); 2388 } 2389 2390 /* set the print_flag if all ports up or timeout */ 2391 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2392 print_flag = 1; 2393 printf("done\n"); 2394 } 2395 } 2396 } 2397 2398 int 2399 MAIN(int argc, char **argv) 2400 { 2401 struct lcore_conf *qconf; 2402 struct rte_eth_dev_info dev_info; 2403 struct rte_eth_txconf *txconf; 2404 int ret; 2405 unsigned nb_ports; 2406 uint16_t queueid; 2407 unsigned lcore_id; 2408 uint32_t n_tx_queue, nb_lcores; 2409 uint8_t portid, nb_rx_queue, queue, socketid; 2410 2411 /* init EAL */ 2412 ret = rte_eal_init(argc, argv); 2413 if (ret < 0) 2414 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 2415 argc -= ret; 2416 argv += ret; 2417 2418 /* parse application arguments (after the EAL ones) */ 2419 ret = parse_args(argc, argv); 2420 if (ret < 0) 2421 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n"); 2422 2423 if (check_lcore_params() < 0) 2424 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n"); 2425 2426 ret = init_lcore_rx_queues(); 2427 if (ret < 0) 2428 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 2429 2430 nb_ports = rte_eth_dev_count(); 2431 if (nb_ports > RTE_MAX_ETHPORTS) 2432 nb_ports = RTE_MAX_ETHPORTS; 2433 2434 if (check_port_config(nb_ports) < 0) 2435 rte_exit(EXIT_FAILURE, "check_port_config failed\n"); 2436 2437 nb_lcores = rte_lcore_count(); 2438 2439 /* initialize all ports */ 2440 for (portid = 0; portid < nb_ports; portid++) { 2441 /* skip ports that are not enabled */ 2442 if ((enabled_port_mask & (1 << portid)) == 0) { 2443 printf("\nSkipping disabled port %d\n", portid); 2444 continue; 2445 } 2446 2447 /* init port */ 2448 printf("Initializing port %d ... ", portid ); 2449 fflush(stdout); 2450 2451 nb_rx_queue = get_port_n_rx_queues(portid); 2452 n_tx_queue = nb_lcores; 2453 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT) 2454 n_tx_queue = MAX_TX_QUEUE_PER_PORT; 2455 printf("Creating queues: nb_rxq=%d nb_txq=%u... ", 2456 nb_rx_queue, (unsigned)n_tx_queue ); 2457 ret = rte_eth_dev_configure(portid, nb_rx_queue, 2458 (uint16_t)n_tx_queue, &port_conf); 2459 if (ret < 0) 2460 rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%d\n", 2461 ret, portid); 2462 2463 rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 2464 print_ethaddr(" Address:", &ports_eth_addr[portid]); 2465 printf(", "); 2466 2467 /* 2468 * prepare dst and src MACs for each port. 2469 */ 2470 *(uint64_t *)(val_eth + portid) = 2471 ETHER_LOCAL_ADMIN_ADDR + ((uint64_t)portid << 40); 2472 ether_addr_copy(&ports_eth_addr[portid], 2473 (struct ether_addr *)(val_eth + portid) + 1); 2474 2475 /* init memory */ 2476 ret = init_mem(NB_MBUF); 2477 if (ret < 0) 2478 rte_exit(EXIT_FAILURE, "init_mem failed\n"); 2479 2480 /* init one TX queue per couple (lcore,port) */ 2481 queueid = 0; 2482 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2483 if (rte_lcore_is_enabled(lcore_id) == 0) 2484 continue; 2485 2486 if (numa_on) 2487 socketid = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2488 else 2489 socketid = 0; 2490 2491 printf("txq=%u,%d,%d ", lcore_id, queueid, socketid); 2492 fflush(stdout); 2493 2494 rte_eth_dev_info_get(portid, &dev_info); 2495 txconf = &dev_info.default_txconf; 2496 if (port_conf.rxmode.jumbo_frame) 2497 txconf->txq_flags = 0; 2498 ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, 2499 socketid, txconf); 2500 if (ret < 0) 2501 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: err=%d, " 2502 "port=%d\n", ret, portid); 2503 2504 qconf = &lcore_conf[lcore_id]; 2505 qconf->tx_queue_id[portid] = queueid; 2506 queueid++; 2507 } 2508 printf("\n"); 2509 } 2510 2511 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2512 if (rte_lcore_is_enabled(lcore_id) == 0) 2513 continue; 2514 qconf = &lcore_conf[lcore_id]; 2515 printf("\nInitializing rx queues on lcore %u ... ", lcore_id ); 2516 fflush(stdout); 2517 /* init RX queues */ 2518 for(queue = 0; queue < qconf->n_rx_queue; ++queue) { 2519 portid = qconf->rx_queue_list[queue].port_id; 2520 queueid = qconf->rx_queue_list[queue].queue_id; 2521 2522 if (numa_on) 2523 socketid = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2524 else 2525 socketid = 0; 2526 2527 printf("rxq=%d,%d,%d ", portid, queueid, socketid); 2528 fflush(stdout); 2529 2530 ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd, 2531 socketid, 2532 NULL, 2533 pktmbuf_pool[socketid]); 2534 if (ret < 0) 2535 rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup: err=%d," 2536 "port=%d\n", ret, portid); 2537 } 2538 } 2539 2540 printf("\n"); 2541 2542 /* start ports */ 2543 for (portid = 0; portid < nb_ports; portid++) { 2544 if ((enabled_port_mask & (1 << portid)) == 0) { 2545 continue; 2546 } 2547 /* Start device */ 2548 ret = rte_eth_dev_start(portid); 2549 if (ret < 0) 2550 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, port=%d\n", 2551 ret, portid); 2552 2553 /* 2554 * If enabled, put device in promiscuous mode. 2555 * This allows IO forwarding mode to forward packets 2556 * to itself through 2 cross-connected ports of the 2557 * target machine. 2558 */ 2559 if (promiscuous_on) 2560 rte_eth_promiscuous_enable(portid); 2561 } 2562 2563 check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); 2564 2565 /* launch per-lcore init on every lcore */ 2566 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 2567 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2568 if (rte_eal_wait_lcore(lcore_id) < 0) 2569 return -1; 2570 } 2571 2572 return 0; 2573 } 2574