1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdarg.h> 35 #include <stdio.h> 36 #include <errno.h> 37 #include <stdint.h> 38 #include <unistd.h> 39 #include <inttypes.h> 40 41 #include <sys/queue.h> 42 #include <sys/stat.h> 43 44 #include <rte_common.h> 45 #include <rte_byteorder.h> 46 #include <rte_log.h> 47 #include <rte_debug.h> 48 #include <rte_cycles.h> 49 #include <rte_memory.h> 50 #include <rte_memcpy.h> 51 #include <rte_memzone.h> 52 #include <rte_launch.h> 53 #include <rte_tailq.h> 54 #include <rte_eal.h> 55 #include <rte_per_lcore.h> 56 #include <rte_lcore.h> 57 #include <rte_atomic.h> 58 #include <rte_branch_prediction.h> 59 #include <rte_ring.h> 60 #include <rte_memory.h> 61 #include <rte_mempool.h> 62 #include <rte_mbuf.h> 63 #include <rte_memcpy.h> 64 #include <rte_interrupts.h> 65 #include <rte_pci.h> 66 #include <rte_ether.h> 67 #include <rte_ethdev.h> 68 #include <rte_ip.h> 69 #include <rte_tcp.h> 70 #include <rte_udp.h> 71 #include <rte_sctp.h> 72 #include <rte_prefetch.h> 73 #include <rte_string_fns.h> 74 #include "testpmd.h" 75 76 77 78 #define IP_DEFTTL 64 /* from RFC 1340. */ 79 #define IP_VERSION 0x40 80 #define IP_HDRLEN 0x05 /* default IP header length == five 32-bits words. */ 81 #define IP_VHL_DEF (IP_VERSION | IP_HDRLEN) 82 83 static inline uint16_t 84 get_16b_sum(uint16_t *ptr16, uint32_t nr) 85 { 86 uint32_t sum = 0; 87 while (nr > 1) 88 { 89 sum +=*ptr16; 90 nr -= sizeof(uint16_t); 91 ptr16++; 92 if (sum > UINT16_MAX) 93 sum -= UINT16_MAX; 94 } 95 96 /* If length is in odd bytes */ 97 if (nr) 98 sum += *((uint8_t*)ptr16); 99 100 sum = ((sum & 0xffff0000) >> 16) + (sum & 0xffff); 101 sum &= 0x0ffff; 102 return (uint16_t)sum; 103 } 104 105 static inline uint16_t 106 get_ipv4_cksum(struct ipv4_hdr *ipv4_hdr) 107 { 108 uint16_t cksum; 109 cksum = get_16b_sum((uint16_t*)ipv4_hdr, sizeof(struct ipv4_hdr)); 110 return (uint16_t)((cksum == 0xffff)?cksum:~cksum); 111 } 112 113 114 static inline uint16_t 115 get_ipv4_psd_sum (struct ipv4_hdr * ip_hdr) 116 { 117 /* Pseudo Header for IPv4/UDP/TCP checksum */ 118 union ipv4_psd_header { 119 struct { 120 uint32_t src_addr; /* IP address of source host. */ 121 uint32_t dst_addr; /* IP address of destination host(s). */ 122 uint8_t zero; /* zero. */ 123 uint8_t proto; /* L4 protocol type. */ 124 uint16_t len; /* L4 length. */ 125 } __attribute__((__packed__)); 126 uint16_t u16_arr[0]; 127 } psd_hdr; 128 129 psd_hdr.src_addr = ip_hdr->src_addr; 130 psd_hdr.dst_addr = ip_hdr->dst_addr; 131 psd_hdr.zero = 0; 132 psd_hdr.proto = ip_hdr->next_proto_id; 133 psd_hdr.len = rte_cpu_to_be_16((uint16_t)(rte_be_to_cpu_16(ip_hdr->total_length) 134 - sizeof(struct ipv4_hdr))); 135 return get_16b_sum(psd_hdr.u16_arr, sizeof(psd_hdr)); 136 } 137 138 static inline uint16_t 139 get_ipv6_psd_sum (struct ipv6_hdr * ip_hdr) 140 { 141 /* Pseudo Header for IPv6/UDP/TCP checksum */ 142 union ipv6_psd_header { 143 struct { 144 uint8_t src_addr[16]; /* IP address of source host. */ 145 uint8_t dst_addr[16]; /* IP address of destination host(s). */ 146 uint32_t len; /* L4 length. */ 147 uint32_t proto; /* L4 protocol - top 3 bytes must be zero */ 148 } __attribute__((__packed__)); 149 150 uint16_t u16_arr[0]; /* allow use as 16-bit values with safe aliasing */ 151 } psd_hdr; 152 153 rte_memcpy(&psd_hdr.src_addr, ip_hdr->src_addr, 154 sizeof(ip_hdr->src_addr) + sizeof(ip_hdr->dst_addr)); 155 psd_hdr.len = ip_hdr->payload_len; 156 psd_hdr.proto = (ip_hdr->proto << 24); 157 158 return get_16b_sum(psd_hdr.u16_arr, sizeof(psd_hdr)); 159 } 160 161 static inline uint16_t 162 get_ipv4_udptcp_checksum(struct ipv4_hdr *ipv4_hdr, uint16_t *l4_hdr) 163 { 164 uint32_t cksum; 165 uint32_t l4_len; 166 167 l4_len = rte_be_to_cpu_16(ipv4_hdr->total_length) - sizeof(struct ipv4_hdr); 168 169 cksum = get_16b_sum(l4_hdr, l4_len); 170 cksum += get_ipv4_psd_sum(ipv4_hdr); 171 172 cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff); 173 cksum = (~cksum) & 0xffff; 174 if (cksum == 0) 175 cksum = 0xffff; 176 return (uint16_t)cksum; 177 178 } 179 180 static inline uint16_t 181 get_ipv6_udptcp_checksum(struct ipv6_hdr *ipv6_hdr, uint16_t *l4_hdr) 182 { 183 uint32_t cksum; 184 uint32_t l4_len; 185 186 l4_len = rte_be_to_cpu_16(ipv6_hdr->payload_len); 187 188 cksum = get_16b_sum(l4_hdr, l4_len); 189 cksum += get_ipv6_psd_sum(ipv6_hdr); 190 191 cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff); 192 cksum = (~cksum) & 0xffff; 193 if (cksum == 0) 194 cksum = 0xffff; 195 196 return (uint16_t)cksum; 197 } 198 199 200 /* 201 * Forwarding of packets. Change the checksum field with HW or SW methods 202 * The HW/SW method selection depends on the ol_flags on every packet 203 */ 204 static void 205 pkt_burst_checksum_forward(struct fwd_stream *fs) 206 { 207 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 208 struct rte_port *txp; 209 struct rte_mbuf *mb; 210 struct ether_hdr *eth_hdr; 211 struct ipv4_hdr *ipv4_hdr; 212 struct ipv6_hdr *ipv6_hdr; 213 struct udp_hdr *udp_hdr; 214 struct tcp_hdr *tcp_hdr; 215 struct sctp_hdr *sctp_hdr; 216 217 uint16_t nb_rx; 218 uint16_t nb_tx; 219 uint16_t i; 220 uint16_t ol_flags; 221 uint16_t pkt_ol_flags; 222 uint16_t tx_ol_flags; 223 uint16_t l4_proto; 224 uint16_t eth_type; 225 uint8_t l2_len; 226 uint8_t l3_len; 227 228 uint32_t rx_bad_ip_csum; 229 uint32_t rx_bad_l4_csum; 230 231 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 232 uint64_t start_tsc; 233 uint64_t end_tsc; 234 uint64_t core_cycles; 235 #endif 236 237 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 238 start_tsc = rte_rdtsc(); 239 #endif 240 241 /* 242 * Receive a burst of packets and forward them. 243 */ 244 nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue, pkts_burst, 245 nb_pkt_per_burst); 246 if (unlikely(nb_rx == 0)) 247 return; 248 249 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 250 fs->rx_burst_stats.pkt_burst_spread[nb_rx]++; 251 #endif 252 fs->rx_packets += nb_rx; 253 rx_bad_ip_csum = 0; 254 rx_bad_l4_csum = 0; 255 256 txp = &ports[fs->tx_port]; 257 tx_ol_flags = txp->tx_ol_flags; 258 259 for (i = 0; i < nb_rx; i++) { 260 261 mb = pkts_burst[i]; 262 l2_len = sizeof(struct ether_hdr); 263 pkt_ol_flags = mb->ol_flags; 264 ol_flags = (uint16_t) (pkt_ol_flags & (~PKT_TX_L4_MASK)); 265 266 eth_hdr = (struct ether_hdr *) mb->pkt.data; 267 eth_type = rte_be_to_cpu_16(eth_hdr->ether_type); 268 if (eth_type == ETHER_TYPE_VLAN) { 269 /* Only allow single VLAN label here */ 270 l2_len += sizeof(struct vlan_hdr); 271 eth_type = rte_be_to_cpu_16(*(uint16_t *) 272 ((uintptr_t)ð_hdr->ether_type + 273 sizeof(struct vlan_hdr))); 274 } 275 276 /* Update the L3/L4 checksum error packet count */ 277 rx_bad_ip_csum += (uint16_t) ((pkt_ol_flags & PKT_RX_IP_CKSUM_BAD) != 0); 278 rx_bad_l4_csum += (uint16_t) ((pkt_ol_flags & PKT_RX_L4_CKSUM_BAD) != 0); 279 280 /* 281 * Try to figure out L3 packet type by SW. 282 */ 283 if ((pkt_ol_flags & (PKT_RX_IPV4_HDR | PKT_RX_IPV4_HDR_EXT | 284 PKT_RX_IPV6_HDR | PKT_RX_IPV6_HDR_EXT)) == 0) { 285 if (eth_type == ETHER_TYPE_IPv4) 286 pkt_ol_flags |= PKT_RX_IPV4_HDR; 287 else if (eth_type == ETHER_TYPE_IPv6) 288 pkt_ol_flags |= PKT_RX_IPV6_HDR; 289 } 290 291 /* 292 * Simplify the protocol parsing 293 * Assuming the incoming packets format as 294 * Ethernet2 + optional single VLAN 295 * + ipv4 or ipv6 296 * + udp or tcp or sctp or others 297 */ 298 if (pkt_ol_flags & PKT_RX_IPV4_HDR) { 299 300 /* Do not support ipv4 option field */ 301 l3_len = sizeof(struct ipv4_hdr) ; 302 303 ipv4_hdr = (struct ipv4_hdr *) (rte_pktmbuf_mtod(mb, 304 unsigned char *) + l2_len); 305 306 l4_proto = ipv4_hdr->next_proto_id; 307 308 /* Do not delete, this is required by HW*/ 309 ipv4_hdr->hdr_checksum = 0; 310 311 if (tx_ol_flags & 0x1) { 312 /* HW checksum */ 313 ol_flags |= PKT_TX_IP_CKSUM; 314 } 315 else { 316 /* SW checksum calculation */ 317 ipv4_hdr->src_addr++; 318 ipv4_hdr->hdr_checksum = get_ipv4_cksum(ipv4_hdr); 319 } 320 321 if (l4_proto == IPPROTO_UDP) { 322 udp_hdr = (struct udp_hdr*) (rte_pktmbuf_mtod(mb, 323 unsigned char *) + l2_len + l3_len); 324 if (tx_ol_flags & 0x2) { 325 /* HW Offload */ 326 ol_flags |= PKT_TX_UDP_CKSUM; 327 /* Pseudo header sum need be set properly */ 328 udp_hdr->dgram_cksum = get_ipv4_psd_sum(ipv4_hdr); 329 } 330 else { 331 /* SW Implementation, clear checksum field first */ 332 udp_hdr->dgram_cksum = 0; 333 udp_hdr->dgram_cksum = get_ipv4_udptcp_checksum(ipv4_hdr, 334 (uint16_t*)udp_hdr); 335 } 336 } 337 else if (l4_proto == IPPROTO_TCP){ 338 tcp_hdr = (struct tcp_hdr*) (rte_pktmbuf_mtod(mb, 339 unsigned char *) + l2_len + l3_len); 340 if (tx_ol_flags & 0x4) { 341 ol_flags |= PKT_TX_TCP_CKSUM; 342 tcp_hdr->cksum = get_ipv4_psd_sum(ipv4_hdr); 343 } 344 else { 345 tcp_hdr->cksum = 0; 346 tcp_hdr->cksum = get_ipv4_udptcp_checksum(ipv4_hdr, 347 (uint16_t*)tcp_hdr); 348 } 349 } 350 else if (l4_proto == IPPROTO_SCTP) { 351 sctp_hdr = (struct sctp_hdr*) (rte_pktmbuf_mtod(mb, 352 unsigned char *) + l2_len + l3_len); 353 354 if (tx_ol_flags & 0x8) { 355 ol_flags |= PKT_TX_SCTP_CKSUM; 356 sctp_hdr->cksum = 0; 357 358 /* Sanity check, only number of 4 bytes supported */ 359 if ((rte_be_to_cpu_16(ipv4_hdr->total_length) % 4) != 0) 360 printf("sctp payload must be a multiple " 361 "of 4 bytes for checksum offload"); 362 } 363 else { 364 sctp_hdr->cksum = 0; 365 /* CRC32c sample code available in RFC3309 */ 366 } 367 } 368 /* End of L4 Handling*/ 369 } 370 else if (pkt_ol_flags & PKT_RX_IPV6_HDR) { 371 372 ipv6_hdr = (struct ipv6_hdr *) (rte_pktmbuf_mtod(mb, 373 unsigned char *) + l2_len); 374 l3_len = sizeof(struct ipv6_hdr) ; 375 l4_proto = ipv6_hdr->proto; 376 377 if (l4_proto == IPPROTO_UDP) { 378 udp_hdr = (struct udp_hdr*) (rte_pktmbuf_mtod(mb, 379 unsigned char *) + l2_len + l3_len); 380 if (tx_ol_flags & 0x2) { 381 /* HW Offload */ 382 ol_flags |= PKT_TX_UDP_CKSUM; 383 udp_hdr->dgram_cksum = get_ipv6_psd_sum(ipv6_hdr); 384 } 385 else { 386 /* SW Implementation */ 387 /* checksum field need be clear first */ 388 udp_hdr->dgram_cksum = 0; 389 udp_hdr->dgram_cksum = get_ipv6_udptcp_checksum(ipv6_hdr, 390 (uint16_t*)udp_hdr); 391 } 392 } 393 else if (l4_proto == IPPROTO_TCP) { 394 tcp_hdr = (struct tcp_hdr*) (rte_pktmbuf_mtod(mb, 395 unsigned char *) + l2_len + l3_len); 396 if (tx_ol_flags & 0x4) { 397 ol_flags |= PKT_TX_TCP_CKSUM; 398 tcp_hdr->cksum = get_ipv6_psd_sum(ipv6_hdr); 399 } 400 else { 401 tcp_hdr->cksum = 0; 402 tcp_hdr->cksum = get_ipv6_udptcp_checksum(ipv6_hdr, 403 (uint16_t*)tcp_hdr); 404 } 405 } 406 else if (l4_proto == IPPROTO_SCTP) { 407 sctp_hdr = (struct sctp_hdr*) (rte_pktmbuf_mtod(mb, 408 unsigned char *) + l2_len + l3_len); 409 410 if (tx_ol_flags & 0x8) { 411 ol_flags |= PKT_TX_SCTP_CKSUM; 412 sctp_hdr->cksum = 0; 413 /* Sanity check, only number of 4 bytes supported by HW */ 414 if ((rte_be_to_cpu_16(ipv6_hdr->payload_len) % 4) != 0) 415 printf("sctp payload must be a multiple " 416 "of 4 bytes for checksum offload"); 417 } 418 else { 419 /* CRC32c sample code available in RFC3309 */ 420 sctp_hdr->cksum = 0; 421 } 422 } else { 423 printf("Test flow control for 1G PMD \n"); 424 } 425 /* End of L6 Handling*/ 426 } 427 else { 428 l3_len = 0; 429 printf("Unhandled packet type: %#hx\n", eth_type); 430 } 431 432 /* Combine the packet header write. VLAN is not consider here */ 433 mb->pkt.vlan_macip.f.l2_len = l2_len; 434 mb->pkt.vlan_macip.f.l3_len = l3_len; 435 mb->ol_flags = ol_flags; 436 } 437 nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_rx); 438 fs->tx_packets += nb_tx; 439 fs->rx_bad_ip_csum += rx_bad_ip_csum; 440 fs->rx_bad_l4_csum += rx_bad_l4_csum; 441 442 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 443 fs->tx_burst_stats.pkt_burst_spread[nb_tx]++; 444 #endif 445 if (unlikely(nb_tx < nb_rx)) { 446 fs->fwd_dropped += (nb_rx - nb_tx); 447 do { 448 rte_pktmbuf_free(pkts_burst[nb_tx]); 449 } while (++nb_tx < nb_rx); 450 } 451 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 452 end_tsc = rte_rdtsc(); 453 core_cycles = (end_tsc - start_tsc); 454 fs->core_cycles = (uint64_t) (fs->core_cycles + core_cycles); 455 #endif 456 } 457 458 459 struct fwd_engine csum_fwd_engine = { 460 .fwd_mode_name = "csum", 461 .port_fwd_begin = NULL, 462 .port_fwd_end = NULL, 463 .packet_fwd = pkt_burst_checksum_forward, 464 }; 465 466