1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <rte_debug.h> 6 #include <rte_ethdev.h> 7 #include <rte_ether.h> 8 #include <rte_graph.h> 9 #include <rte_graph_worker.h> 10 #include <rte_ip.h> 11 #include <rte_malloc.h> 12 #include <rte_mbuf.h> 13 #include <rte_tcp.h> 14 #include <rte_udp.h> 15 #include <rte_vect.h> 16 17 #include "rte_node_ip4_api.h" 18 19 #include "ip4_rewrite_priv.h" 20 #include "node_private.h" 21 22 struct ip4_rewrite_node_ctx { 23 /* Dynamic offset to mbuf priv1 */ 24 int mbuf_priv1_off; 25 /* Cached next index */ 26 uint16_t next_index; 27 }; 28 29 static struct ip4_rewrite_node_main *ip4_rewrite_nm; 30 31 #define IP4_REWRITE_NODE_LAST_NEXT(ctx) \ 32 (((struct ip4_rewrite_node_ctx *)ctx)->next_index) 33 34 #define IP4_REWRITE_NODE_PRIV1_OFF(ctx) \ 35 (((struct ip4_rewrite_node_ctx *)ctx)->mbuf_priv1_off) 36 37 static uint16_t 38 ip4_rewrite_node_process(struct rte_graph *graph, struct rte_node *node, 39 void **objs, uint16_t nb_objs) 40 { 41 struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts; 42 struct ip4_rewrite_nh_header *nh = ip4_rewrite_nm->nh; 43 const int dyn = IP4_REWRITE_NODE_PRIV1_OFF(node->ctx); 44 uint16_t next0, next1, next2, next3, next_index; 45 struct rte_ipv4_hdr *ip0, *ip1, *ip2, *ip3; 46 uint16_t n_left_from, held = 0, last_spec = 0; 47 void *d0, *d1, *d2, *d3; 48 void **to_next, **from; 49 rte_xmm_t priv01; 50 rte_xmm_t priv23; 51 int i; 52 53 /* Speculative next as last next */ 54 next_index = IP4_REWRITE_NODE_LAST_NEXT(node->ctx); 55 rte_prefetch0(nh); 56 57 pkts = (struct rte_mbuf **)objs; 58 from = objs; 59 n_left_from = nb_objs; 60 61 for (i = 0; i < 4 && i < n_left_from; i++) 62 rte_prefetch0(pkts[i]); 63 64 /* Get stream for the speculated next node */ 65 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs); 66 /* Update Ethernet header of pkts */ 67 while (n_left_from >= 4) { 68 if (likely(n_left_from > 7)) { 69 /* Prefetch only next-mbuf struct and priv area. 70 * Data need not be prefetched as we only write. 71 */ 72 rte_prefetch0(pkts[4]); 73 rte_prefetch0(pkts[5]); 74 rte_prefetch0(pkts[6]); 75 rte_prefetch0(pkts[7]); 76 } 77 78 mbuf0 = pkts[0]; 79 mbuf1 = pkts[1]; 80 mbuf2 = pkts[2]; 81 mbuf3 = pkts[3]; 82 83 pkts += 4; 84 n_left_from -= 4; 85 priv01.u64[0] = node_mbuf_priv1(mbuf0, dyn)->u; 86 priv01.u64[1] = node_mbuf_priv1(mbuf1, dyn)->u; 87 priv23.u64[0] = node_mbuf_priv1(mbuf2, dyn)->u; 88 priv23.u64[1] = node_mbuf_priv1(mbuf3, dyn)->u; 89 90 /* Increment checksum by one. */ 91 priv01.u32[1] += rte_cpu_to_be_16(0x0100); 92 priv01.u32[3] += rte_cpu_to_be_16(0x0100); 93 priv23.u32[1] += rte_cpu_to_be_16(0x0100); 94 priv23.u32[3] += rte_cpu_to_be_16(0x0100); 95 96 /* Update ttl,cksum rewrite ethernet hdr on mbuf0 */ 97 d0 = rte_pktmbuf_mtod(mbuf0, void *); 98 rte_memcpy(d0, nh[priv01.u16[0]].rewrite_data, 99 nh[priv01.u16[0]].rewrite_len); 100 101 next0 = nh[priv01.u16[0]].tx_node; 102 ip0 = (struct rte_ipv4_hdr *)((uint8_t *)d0 + 103 sizeof(struct rte_ether_hdr)); 104 ip0->time_to_live = priv01.u16[1] - 1; 105 ip0->hdr_checksum = priv01.u16[2] + priv01.u16[3]; 106 107 /* Update ttl,cksum rewrite ethernet hdr on mbuf1 */ 108 d1 = rte_pktmbuf_mtod(mbuf1, void *); 109 rte_memcpy(d1, nh[priv01.u16[4]].rewrite_data, 110 nh[priv01.u16[4]].rewrite_len); 111 112 next1 = nh[priv01.u16[4]].tx_node; 113 ip1 = (struct rte_ipv4_hdr *)((uint8_t *)d1 + 114 sizeof(struct rte_ether_hdr)); 115 ip1->time_to_live = priv01.u16[5] - 1; 116 ip1->hdr_checksum = priv01.u16[6] + priv01.u16[7]; 117 118 /* Update ttl,cksum rewrite ethernet hdr on mbuf2 */ 119 d2 = rte_pktmbuf_mtod(mbuf2, void *); 120 rte_memcpy(d2, nh[priv23.u16[0]].rewrite_data, 121 nh[priv23.u16[0]].rewrite_len); 122 next2 = nh[priv23.u16[0]].tx_node; 123 ip2 = (struct rte_ipv4_hdr *)((uint8_t *)d2 + 124 sizeof(struct rte_ether_hdr)); 125 ip2->time_to_live = priv23.u16[1] - 1; 126 ip2->hdr_checksum = priv23.u16[2] + priv23.u16[3]; 127 128 /* Update ttl,cksum rewrite ethernet hdr on mbuf3 */ 129 d3 = rte_pktmbuf_mtod(mbuf3, void *); 130 rte_memcpy(d3, nh[priv23.u16[4]].rewrite_data, 131 nh[priv23.u16[4]].rewrite_len); 132 133 next3 = nh[priv23.u16[4]].tx_node; 134 ip3 = (struct rte_ipv4_hdr *)((uint8_t *)d3 + 135 sizeof(struct rte_ether_hdr)); 136 ip3->time_to_live = priv23.u16[5] - 1; 137 ip3->hdr_checksum = priv23.u16[6] + priv23.u16[7]; 138 139 /* Enqueue four to next node */ 140 rte_edge_t fix_spec = 141 ((next_index == next0) && (next0 == next1) && 142 (next1 == next2) && (next2 == next3)); 143 144 if (unlikely(fix_spec == 0)) { 145 /* Copy things successfully speculated till now */ 146 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 147 from += last_spec; 148 to_next += last_spec; 149 held += last_spec; 150 last_spec = 0; 151 152 /* next0 */ 153 if (next_index == next0) { 154 to_next[0] = from[0]; 155 to_next++; 156 held++; 157 } else { 158 rte_node_enqueue_x1(graph, node, next0, 159 from[0]); 160 } 161 162 /* next1 */ 163 if (next_index == next1) { 164 to_next[0] = from[1]; 165 to_next++; 166 held++; 167 } else { 168 rte_node_enqueue_x1(graph, node, next1, 169 from[1]); 170 } 171 172 /* next2 */ 173 if (next_index == next2) { 174 to_next[0] = from[2]; 175 to_next++; 176 held++; 177 } else { 178 rte_node_enqueue_x1(graph, node, next2, 179 from[2]); 180 } 181 182 /* next3 */ 183 if (next_index == next3) { 184 to_next[0] = from[3]; 185 to_next++; 186 held++; 187 } else { 188 rte_node_enqueue_x1(graph, node, next3, 189 from[3]); 190 } 191 192 from += 4; 193 194 /* Change speculation if last two are same */ 195 if ((next_index != next3) && (next2 == next3)) { 196 /* Put the current speculated node */ 197 rte_node_next_stream_put(graph, node, 198 next_index, held); 199 held = 0; 200 201 /* Get next speculated stream */ 202 next_index = next3; 203 to_next = rte_node_next_stream_get( 204 graph, node, next_index, nb_objs); 205 } 206 } else { 207 last_spec += 4; 208 } 209 } 210 211 while (n_left_from > 0) { 212 uint16_t chksum; 213 214 mbuf0 = pkts[0]; 215 216 pkts += 1; 217 n_left_from -= 1; 218 219 d0 = rte_pktmbuf_mtod(mbuf0, void *); 220 rte_memcpy(d0, nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_data, 221 nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_len); 222 223 next0 = nh[node_mbuf_priv1(mbuf0, dyn)->nh].tx_node; 224 ip0 = (struct rte_ipv4_hdr *)((uint8_t *)d0 + 225 sizeof(struct rte_ether_hdr)); 226 chksum = node_mbuf_priv1(mbuf0, dyn)->cksum + 227 rte_cpu_to_be_16(0x0100); 228 chksum += chksum >= 0xffff; 229 ip0->hdr_checksum = chksum; 230 ip0->time_to_live = node_mbuf_priv1(mbuf0, dyn)->ttl - 1; 231 232 if (unlikely(next_index ^ next0)) { 233 /* Copy things successfully speculated till now */ 234 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 235 from += last_spec; 236 to_next += last_spec; 237 held += last_spec; 238 last_spec = 0; 239 240 rte_node_enqueue_x1(graph, node, next0, from[0]); 241 from += 1; 242 } else { 243 last_spec += 1; 244 } 245 } 246 247 /* !!! Home run !!! */ 248 if (likely(last_spec == nb_objs)) { 249 rte_node_next_stream_move(graph, node, next_index); 250 return nb_objs; 251 } 252 253 held += last_spec; 254 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 255 rte_node_next_stream_put(graph, node, next_index, held); 256 /* Save the last next used */ 257 IP4_REWRITE_NODE_LAST_NEXT(node->ctx) = next_index; 258 259 return nb_objs; 260 } 261 262 static int 263 ip4_rewrite_node_init(const struct rte_graph *graph, struct rte_node *node) 264 { 265 static bool init_once; 266 267 RTE_SET_USED(graph); 268 RTE_BUILD_BUG_ON(sizeof(struct ip4_rewrite_node_ctx) > RTE_NODE_CTX_SZ); 269 270 if (!init_once) { 271 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register( 272 &node_mbuf_priv1_dynfield_desc); 273 if (node_mbuf_priv1_dynfield_offset < 0) 274 return -rte_errno; 275 init_once = true; 276 } 277 IP4_REWRITE_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset; 278 279 node_dbg("ip4_rewrite", "Initialized ip4_rewrite node initialized"); 280 281 return 0; 282 } 283 284 int 285 ip4_rewrite_set_next(uint16_t port_id, uint16_t next_index) 286 { 287 if (ip4_rewrite_nm == NULL) { 288 ip4_rewrite_nm = rte_zmalloc( 289 "ip4_rewrite", sizeof(struct ip4_rewrite_node_main), 290 RTE_CACHE_LINE_SIZE); 291 if (ip4_rewrite_nm == NULL) 292 return -ENOMEM; 293 } 294 ip4_rewrite_nm->next_index[port_id] = next_index; 295 296 return 0; 297 } 298 299 int 300 rte_node_ip4_rewrite_add(uint16_t next_hop, uint8_t *rewrite_data, 301 uint8_t rewrite_len, uint16_t dst_port) 302 { 303 struct ip4_rewrite_nh_header *nh; 304 305 if (next_hop >= RTE_GRAPH_IP4_REWRITE_MAX_NH) 306 return -EINVAL; 307 308 if (rewrite_len > RTE_GRAPH_IP4_REWRITE_MAX_LEN) 309 return -EINVAL; 310 311 if (ip4_rewrite_nm == NULL) { 312 ip4_rewrite_nm = rte_zmalloc( 313 "ip4_rewrite", sizeof(struct ip4_rewrite_node_main), 314 RTE_CACHE_LINE_SIZE); 315 if (ip4_rewrite_nm == NULL) 316 return -ENOMEM; 317 } 318 319 /* Check if dst port doesn't exist as edge */ 320 if (!ip4_rewrite_nm->next_index[dst_port]) 321 return -EINVAL; 322 323 /* Update next hop */ 324 nh = &ip4_rewrite_nm->nh[next_hop]; 325 326 memcpy(nh->rewrite_data, rewrite_data, rewrite_len); 327 nh->tx_node = ip4_rewrite_nm->next_index[dst_port]; 328 nh->rewrite_len = rewrite_len; 329 nh->enabled = true; 330 331 return 0; 332 } 333 334 static struct rte_node_register ip4_rewrite_node = { 335 .process = ip4_rewrite_node_process, 336 .name = "ip4_rewrite", 337 /* Default edge i.e '0' is pkt drop */ 338 .nb_edges = 1, 339 .next_nodes = { 340 [0] = "pkt_drop", 341 }, 342 .init = ip4_rewrite_node_init, 343 }; 344 345 struct rte_node_register * 346 ip4_rewrite_node_get(void) 347 { 348 return &ip4_rewrite_node; 349 } 350 351 RTE_NODE_REGISTER(ip4_rewrite_node); 352