1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 6 #ifndef _L3FWD_SSE_H_ 7 #define _L3FWD_SSE_H_ 8 9 #include "l3fwd.h" 10 #include "sse/port_group.h" 11 #include "l3fwd_common.h" 12 13 #undef SENDM_PORT_OVERHEAD 14 #define SENDM_PORT_OVERHEAD(x) ((x) + 2 * FWDSTEP) 15 16 /* 17 * Update source and destination MAC addresses in the ethernet header. 18 * Perform RFC1812 checks and updates for IPV4 packets. 19 */ 20 static inline void 21 processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP]) 22 { 23 __m128i te[FWDSTEP]; 24 __m128i ve[FWDSTEP]; 25 __m128i *p[FWDSTEP]; 26 27 p[0] = rte_pktmbuf_mtod(pkt[0], __m128i *); 28 p[1] = rte_pktmbuf_mtod(pkt[1], __m128i *); 29 p[2] = rte_pktmbuf_mtod(pkt[2], __m128i *); 30 p[3] = rte_pktmbuf_mtod(pkt[3], __m128i *); 31 32 ve[0] = val_eth[dst_port[0]]; 33 te[0] = _mm_loadu_si128(p[0]); 34 35 ve[1] = val_eth[dst_port[1]]; 36 te[1] = _mm_loadu_si128(p[1]); 37 38 ve[2] = val_eth[dst_port[2]]; 39 te[2] = _mm_loadu_si128(p[2]); 40 41 ve[3] = val_eth[dst_port[3]]; 42 te[3] = _mm_loadu_si128(p[3]); 43 44 /* Update first 12 bytes, keep rest bytes intact. */ 45 te[0] = _mm_blend_epi16(te[0], ve[0], MASK_ETH); 46 te[1] = _mm_blend_epi16(te[1], ve[1], MASK_ETH); 47 te[2] = _mm_blend_epi16(te[2], ve[2], MASK_ETH); 48 te[3] = _mm_blend_epi16(te[3], ve[3], MASK_ETH); 49 50 _mm_storeu_si128(p[0], te[0]); 51 _mm_storeu_si128(p[1], te[1]); 52 _mm_storeu_si128(p[2], te[2]); 53 _mm_storeu_si128(p[3], te[3]); 54 55 rfc1812_process((struct rte_ipv4_hdr *) 56 ((struct rte_ether_hdr *)p[0] + 1), 57 &dst_port[0], pkt[0]->packet_type); 58 rfc1812_process((struct rte_ipv4_hdr *) 59 ((struct rte_ether_hdr *)p[1] + 1), 60 &dst_port[1], pkt[1]->packet_type); 61 rfc1812_process((struct rte_ipv4_hdr *) 62 ((struct rte_ether_hdr *)p[2] + 1), 63 &dst_port[2], pkt[2]->packet_type); 64 rfc1812_process((struct rte_ipv4_hdr *) 65 ((struct rte_ether_hdr *)p[3] + 1), 66 &dst_port[3], pkt[3]->packet_type); 67 } 68 69 /** 70 * Process one packet: 71 * Update source and destination MAC addresses in the ethernet header. 72 * Perform RFC1812 checks and updates for IPV4 packets. 73 */ 74 static inline void 75 process_packet(struct rte_mbuf *pkt, uint16_t *dst_port) 76 { 77 struct rte_ether_hdr *eth_hdr; 78 __m128i te, ve; 79 80 eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *); 81 82 te = _mm_loadu_si128((__m128i *)eth_hdr); 83 ve = val_eth[dst_port[0]]; 84 85 rfc1812_process((struct rte_ipv4_hdr *)(eth_hdr + 1), dst_port, 86 pkt->packet_type); 87 88 te = _mm_blend_epi16(te, ve, MASK_ETH); 89 _mm_storeu_si128((__m128i *)eth_hdr, te); 90 } 91 92 /** 93 * Send packets burst from pkts_burst to the ports in dst_port array 94 */ 95 static __rte_always_inline void 96 send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst, 97 uint16_t dst_port[SENDM_PORT_OVERHEAD(MAX_PKT_BURST)], 98 int nb_rx) 99 { 100 int32_t k; 101 int j = 0; 102 uint16_t dlp; 103 uint16_t *lp; 104 uint16_t pnum[MAX_PKT_BURST + 1]; 105 106 /* 107 * Finish packet processing and group consecutive 108 * packets with the same destination port. 109 */ 110 k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP); 111 if (k != 0) { 112 __m128i dp1, dp2; 113 114 lp = pnum; 115 lp[0] = 1; 116 117 processx4_step3(pkts_burst, dst_port); 118 119 /* dp1: <d[0], d[1], d[2], d[3], ... > */ 120 dp1 = _mm_loadu_si128((__m128i *)dst_port); 121 122 for (j = FWDSTEP; j != k; j += FWDSTEP) { 123 processx4_step3(&pkts_burst[j], &dst_port[j]); 124 125 /* 126 * dp2: 127 * <d[j-3], d[j-2], d[j-1], d[j], ... > 128 */ 129 dp2 = _mm_loadu_si128((__m128i *) 130 &dst_port[j - FWDSTEP + 1]); 131 lp = port_groupx4(&pnum[j - FWDSTEP], lp, dp1, dp2); 132 133 /* 134 * dp1: 135 * <d[j], d[j+1], d[j+2], d[j+3], ... > 136 */ 137 dp1 = _mm_srli_si128(dp2, (FWDSTEP - 1) * 138 sizeof(dst_port[0])); 139 } 140 141 /* 142 * dp2: <d[j-3], d[j-2], d[j-1], d[j-1], ... > 143 */ 144 dp2 = _mm_shufflelo_epi16(dp1, 0xf9); 145 lp = port_groupx4(&pnum[j - FWDSTEP], lp, dp1, dp2); 146 147 /* 148 * remove values added by the last repeated 149 * dst port. 150 */ 151 lp[0]--; 152 dlp = dst_port[j - 1]; 153 } else { 154 /* set dlp and lp to the never used values. */ 155 dlp = BAD_PORT - 1; 156 lp = pnum + MAX_PKT_BURST; 157 } 158 159 /* Process up to last 3 packets one by one. */ 160 switch (nb_rx % FWDSTEP) { 161 case 3: 162 process_packet(pkts_burst[j], dst_port + j); 163 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); 164 j++; 165 /* fall-through */ 166 case 2: 167 process_packet(pkts_burst[j], dst_port + j); 168 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); 169 j++; 170 /* fall-through */ 171 case 1: 172 process_packet(pkts_burst[j], dst_port + j); 173 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); 174 j++; 175 } 176 177 /* 178 * Send packets out, through destination port. 179 * Consecutive packets with the same destination port 180 * are already grouped together. 181 * If destination port for the packet equals BAD_PORT, 182 * then free the packet without sending it out. 183 */ 184 for (j = 0; j < nb_rx; j += k) { 185 186 int32_t m; 187 uint16_t pn; 188 189 pn = dst_port[j]; 190 k = pnum[j]; 191 192 if (likely(pn != BAD_PORT)) 193 send_packetsx4(qconf, pn, pkts_burst + j, k); 194 else 195 for (m = j; m != j + k; m++) 196 rte_pktmbuf_free(pkts_burst[m]); 197 198 } 199 } 200 201 static __rte_always_inline uint16_t 202 process_dst_port(uint16_t *dst_ports, uint16_t nb_elem) 203 { 204 uint16_t i = 0, res; 205 206 while (nb_elem > 7) { 207 __m128i dp = _mm_set1_epi16(dst_ports[0]); 208 __m128i dp1; 209 210 dp1 = _mm_loadu_si128((__m128i *)&dst_ports[i]); 211 dp1 = _mm_cmpeq_epi16(dp1, dp); 212 res = _mm_movemask_epi8(dp1); 213 if (res != 0xFFFF) 214 return BAD_PORT; 215 216 nb_elem -= 8; 217 i += 8; 218 } 219 220 while (nb_elem > 3) { 221 __m128i dp = _mm_set1_epi16(dst_ports[0]); 222 __m128i dp1; 223 224 dp1 = _mm_loadu_si128((__m128i *)&dst_ports[i]); 225 dp1 = _mm_cmpeq_epi16(dp1, dp); 226 dp1 = _mm_unpacklo_epi16(dp1, dp1); 227 res = _mm_movemask_ps((__m128)dp1); 228 if (res != 0xF) 229 return BAD_PORT; 230 231 nb_elem -= 4; 232 i += 4; 233 } 234 235 while (nb_elem) { 236 if (dst_ports[i] != dst_ports[0]) 237 return BAD_PORT; 238 nb_elem--; 239 i++; 240 } 241 242 return dst_ports[0]; 243 } 244 245 #endif /* _L3FWD_SSE_H_ */ 246