1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2018 Intel Corporation. 3 * Copyright(c) 2017-2018 Linaro Limited. 4 */ 5 6 #ifndef __L3FWD_EM_HLM_H__ 7 #define __L3FWD_EM_HLM_H__ 8 9 #if defined RTE_ARCH_X86 10 #include "l3fwd_sse.h" 11 #include "l3fwd_em_hlm_sse.h" 12 #elif defined __ARM_NEON 13 #include "l3fwd_neon.h" 14 #include "l3fwd_em_hlm_neon.h" 15 #endif 16 17 #ifdef RTE_ARCH_ARM64 18 #define EM_HASH_LOOKUP_COUNT 16 19 #else 20 #define EM_HASH_LOOKUP_COUNT 8 21 #endif 22 23 24 static __rte_always_inline void 25 em_get_dst_port_ipv4xN(struct lcore_conf *qconf, struct rte_mbuf *m[], 26 uint16_t portid, uint16_t dst_port[]) 27 { 28 int i; 29 int32_t ret[EM_HASH_LOOKUP_COUNT]; 30 union ipv4_5tuple_host key[EM_HASH_LOOKUP_COUNT]; 31 const void *key_array[EM_HASH_LOOKUP_COUNT]; 32 33 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { 34 get_ipv4_5tuple(m[i], mask0.x, &key[i]); 35 key_array[i] = &key[i]; 36 } 37 38 rte_hash_lookup_bulk(qconf->ipv4_lookup_struct, &key_array[0], 39 EM_HASH_LOOKUP_COUNT, ret); 40 41 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { 42 dst_port[i] = ((ret[i] < 0) ? 43 portid : ipv4_l3fwd_out_if[ret[i]]); 44 45 if (dst_port[i] >= RTE_MAX_ETHPORTS || 46 (enabled_port_mask & 1 << dst_port[i]) == 0) 47 dst_port[i] = portid; 48 } 49 } 50 51 static __rte_always_inline void 52 em_get_dst_port_ipv6xN(struct lcore_conf *qconf, struct rte_mbuf *m[], 53 uint16_t portid, uint16_t dst_port[]) 54 { 55 int i; 56 int32_t ret[EM_HASH_LOOKUP_COUNT]; 57 union ipv6_5tuple_host key[EM_HASH_LOOKUP_COUNT]; 58 const void *key_array[EM_HASH_LOOKUP_COUNT]; 59 60 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { 61 get_ipv6_5tuple(m[i], mask1.x, mask2.x, &key[i]); 62 key_array[i] = &key[i]; 63 } 64 65 rte_hash_lookup_bulk(qconf->ipv6_lookup_struct, &key_array[0], 66 EM_HASH_LOOKUP_COUNT, ret); 67 68 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { 69 dst_port[i] = ((ret[i] < 0) ? 70 portid : ipv6_l3fwd_out_if[ret[i]]); 71 72 if (dst_port[i] >= RTE_MAX_ETHPORTS || 73 (enabled_port_mask & 1 << dst_port[i]) == 0) 74 dst_port[i] = portid; 75 } 76 } 77 78 static __rte_always_inline void 79 em_get_dst_port_ipv4xN_events(struct lcore_conf *qconf, struct rte_mbuf *m[], 80 uint16_t dst_port[]) 81 { 82 int i; 83 int32_t ret[EM_HASH_LOOKUP_COUNT]; 84 union ipv4_5tuple_host key[EM_HASH_LOOKUP_COUNT]; 85 const void *key_array[EM_HASH_LOOKUP_COUNT]; 86 87 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { 88 get_ipv4_5tuple(m[i], mask0.x, &key[i]); 89 key_array[i] = &key[i]; 90 } 91 92 rte_hash_lookup_bulk(qconf->ipv4_lookup_struct, &key_array[0], 93 EM_HASH_LOOKUP_COUNT, ret); 94 95 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { 96 dst_port[i] = ((ret[i] < 0) ? 97 m[i]->port : ipv4_l3fwd_out_if[ret[i]]); 98 99 if (dst_port[i] >= RTE_MAX_ETHPORTS || 100 (enabled_port_mask & 1 << dst_port[i]) == 0) 101 dst_port[i] = m[i]->port; 102 } 103 } 104 105 static __rte_always_inline void 106 em_get_dst_port_ipv6xN_events(struct lcore_conf *qconf, struct rte_mbuf *m[], 107 uint16_t dst_port[]) 108 { 109 int i; 110 int32_t ret[EM_HASH_LOOKUP_COUNT]; 111 union ipv6_5tuple_host key[EM_HASH_LOOKUP_COUNT]; 112 const void *key_array[EM_HASH_LOOKUP_COUNT]; 113 114 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { 115 get_ipv6_5tuple(m[i], mask1.x, mask2.x, &key[i]); 116 key_array[i] = &key[i]; 117 } 118 119 rte_hash_lookup_bulk(qconf->ipv6_lookup_struct, &key_array[0], 120 EM_HASH_LOOKUP_COUNT, ret); 121 122 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { 123 dst_port[i] = ((ret[i] < 0) ? 124 m[i]->port : ipv6_l3fwd_out_if[ret[i]]); 125 126 if (dst_port[i] >= RTE_MAX_ETHPORTS || 127 (enabled_port_mask & 1 << dst_port[i]) == 0) 128 dst_port[i] = m[i]->port; 129 } 130 } 131 132 static __rte_always_inline uint16_t 133 em_get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt, 134 uint16_t portid) 135 { 136 uint16_t next_hop; 137 struct rte_ipv4_hdr *ipv4_hdr; 138 struct rte_ipv6_hdr *ipv6_hdr; 139 uint32_t tcp_or_udp; 140 uint32_t l3_ptypes; 141 142 tcp_or_udp = pkt->packet_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); 143 l3_ptypes = pkt->packet_type & RTE_PTYPE_L3_MASK; 144 145 if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV4)) { 146 147 /* Handle IPv4 headers.*/ 148 ipv4_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_ipv4_hdr *, 149 sizeof(struct rte_ether_hdr)); 150 151 next_hop = em_get_ipv4_dst_port(ipv4_hdr, portid, 152 qconf->ipv4_lookup_struct); 153 154 if (next_hop >= RTE_MAX_ETHPORTS || 155 (enabled_port_mask & 1 << next_hop) == 0) 156 next_hop = portid; 157 158 return next_hop; 159 160 } else if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV6)) { 161 162 /* Handle IPv6 headers.*/ 163 ipv6_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_ipv6_hdr *, 164 sizeof(struct rte_ether_hdr)); 165 166 next_hop = em_get_ipv6_dst_port(ipv6_hdr, portid, 167 qconf->ipv6_lookup_struct); 168 169 if (next_hop >= RTE_MAX_ETHPORTS || 170 (enabled_port_mask & 1 << next_hop) == 0) 171 next_hop = portid; 172 173 return next_hop; 174 175 } 176 177 return portid; 178 } 179 180 static inline void 181 l3fwd_em_process_packets(int nb_rx, struct rte_mbuf **pkts_burst, 182 uint16_t *dst_port, uint16_t portid, 183 struct lcore_conf *qconf, const uint8_t do_step3) 184 { 185 int32_t i, j, pos; 186 187 /* 188 * Send nb_rx - nb_rx % EM_HASH_LOOKUP_COUNT packets 189 * in groups of EM_HASH_LOOKUP_COUNT. 190 */ 191 int32_t n = RTE_ALIGN_FLOOR(nb_rx, EM_HASH_LOOKUP_COUNT); 192 193 for (j = 0; j < EM_HASH_LOOKUP_COUNT && j < nb_rx; j++) { 194 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[j], 195 struct rte_ether_hdr *) + 1); 196 } 197 198 for (j = 0; j < n; j += EM_HASH_LOOKUP_COUNT) { 199 200 uint32_t pkt_type = RTE_PTYPE_L3_MASK | 201 RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP; 202 uint32_t l3_type, tcp_or_udp; 203 204 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) 205 pkt_type &= pkts_burst[j + i]->packet_type; 206 207 l3_type = pkt_type & RTE_PTYPE_L3_MASK; 208 tcp_or_udp = pkt_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); 209 210 for (i = 0, pos = j + EM_HASH_LOOKUP_COUNT; 211 i < EM_HASH_LOOKUP_COUNT && pos < nb_rx; i++, pos++) { 212 rte_prefetch0(rte_pktmbuf_mtod( 213 pkts_burst[pos], 214 struct rte_ether_hdr *) + 1); 215 } 216 217 if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV4)) { 218 219 em_get_dst_port_ipv4xN(qconf, &pkts_burst[j], portid, 220 &dst_port[j]); 221 222 } else if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV6)) { 223 224 em_get_dst_port_ipv6xN(qconf, &pkts_burst[j], portid, 225 &dst_port[j]); 226 227 } else { 228 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) 229 dst_port[j + i] = em_get_dst_port(qconf, 230 pkts_burst[j + i], portid); 231 } 232 233 for (i = 0; i < EM_HASH_LOOKUP_COUNT && do_step3; i += FWDSTEP) 234 processx4_step3(&pkts_burst[j + i], &dst_port[j + i]); 235 } 236 237 for (; j < nb_rx; j++) { 238 dst_port[j] = em_get_dst_port(qconf, pkts_burst[j], portid); 239 if (do_step3) 240 process_packet(pkts_burst[j], &pkts_burst[j]->port); 241 } 242 } 243 244 /* 245 * Buffer optimized handling of packets, invoked 246 * from main_loop. 247 */ 248 static inline void 249 l3fwd_em_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, uint16_t portid, 250 struct lcore_conf *qconf) 251 { 252 uint16_t dst_port[SENDM_PORT_OVERHEAD(MAX_PKT_BURST)]; 253 254 l3fwd_em_process_packets(nb_rx, pkts_burst, dst_port, portid, qconf, 0); 255 send_packets_multi(qconf, pkts_burst, dst_port, nb_rx); 256 } 257 258 #ifdef RTE_LIB_EVENTDEV 259 /* 260 * Buffer optimized handling of events, invoked 261 * from main_loop. 262 */ 263 static inline void 264 l3fwd_em_process_events(int nb_rx, struct rte_event **ev, 265 struct lcore_conf *qconf) 266 { 267 int32_t i, j, pos; 268 uint16_t dst_port[MAX_PKT_BURST]; 269 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 270 271 /* 272 * Send nb_rx - nb_rx % EM_HASH_LOOKUP_COUNT packets 273 * in groups of EM_HASH_LOOKUP_COUNT. 274 */ 275 int32_t n = RTE_ALIGN_FLOOR(nb_rx, EM_HASH_LOOKUP_COUNT); 276 277 for (j = 0; j < nb_rx; j++) 278 pkts_burst[j] = ev[j]->mbuf; 279 280 for (j = 0; j < n; j += EM_HASH_LOOKUP_COUNT) { 281 282 uint32_t pkt_type = RTE_PTYPE_L3_MASK | 283 RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP; 284 uint32_t l3_type, tcp_or_udp; 285 286 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) 287 pkt_type &= pkts_burst[j + i]->packet_type; 288 289 l3_type = pkt_type & RTE_PTYPE_L3_MASK; 290 tcp_or_udp = pkt_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); 291 292 for (i = 0, pos = j + EM_HASH_LOOKUP_COUNT; 293 i < EM_HASH_LOOKUP_COUNT && pos < nb_rx; i++, pos++) { 294 rte_prefetch0(rte_pktmbuf_mtod( 295 pkts_burst[pos], 296 struct rte_ether_hdr *) + 1); 297 } 298 299 if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV4)) { 300 301 em_get_dst_port_ipv4xN_events(qconf, &pkts_burst[j], 302 &dst_port[j]); 303 304 } else if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV6)) { 305 306 em_get_dst_port_ipv6xN_events(qconf, &pkts_burst[j], 307 &dst_port[j]); 308 309 } else { 310 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { 311 pkts_burst[j + i]->port = em_get_dst_port(qconf, 312 pkts_burst[j + i], 313 pkts_burst[j + i]->port); 314 process_packet(pkts_burst[j + i], 315 &pkts_burst[j + i]->port); 316 } 317 continue; 318 } 319 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i += FWDSTEP) 320 processx4_step3(&pkts_burst[j + i], &dst_port[j + i]); 321 322 for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) 323 pkts_burst[j + i]->port = dst_port[j + i]; 324 325 } 326 327 for (; j < nb_rx; j++) { 328 pkts_burst[j]->port = em_get_dst_port(qconf, pkts_burst[j], 329 pkts_burst[j]->port); 330 process_packet(pkts_burst[j], &pkts_burst[j]->port); 331 } 332 } 333 334 static inline void 335 l3fwd_em_process_event_vector(struct rte_event_vector *vec, 336 struct lcore_conf *qconf, uint16_t *dst_port) 337 { 338 uint16_t i; 339 340 if (vec->attr_valid) 341 l3fwd_em_process_packets(vec->nb_elem, vec->mbufs, dst_port, 342 vec->port, qconf, 1); 343 else 344 for (i = 0; i < vec->nb_elem; i++) 345 l3fwd_em_process_packets(1, &vec->mbufs[i], 346 &dst_port[i], 347 vec->mbufs[i]->port, qconf, 1); 348 349 process_event_vector(vec, dst_port); 350 } 351 #endif /* RTE_LIB_EVENTDEV */ 352 353 #endif /* __L3FWD_EM_HLM_H__ */ 354