1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <rte_ethdev.h> 6 #include <rte_ether.h> 7 #include <rte_graph.h> 8 #include <rte_graph_worker.h> 9 #include <rte_ip.h> 10 #include <rte_malloc.h> 11 #include <rte_vect.h> 12 13 #include "rte_node_ip4_api.h" 14 15 #include "ip4_rewrite_priv.h" 16 #include "node_private.h" 17 18 struct ip4_rewrite_node_ctx { 19 /* Dynamic offset to mbuf priv1 */ 20 int mbuf_priv1_off; 21 /* Cached next index */ 22 uint16_t next_index; 23 }; 24 25 static struct ip4_rewrite_node_main *ip4_rewrite_nm; 26 27 #define IP4_REWRITE_NODE_LAST_NEXT(ctx) \ 28 (((struct ip4_rewrite_node_ctx *)ctx)->next_index) 29 30 #define IP4_REWRITE_NODE_PRIV1_OFF(ctx) \ 31 (((struct ip4_rewrite_node_ctx *)ctx)->mbuf_priv1_off) 32 33 static uint16_t 34 ip4_rewrite_node_process(struct rte_graph *graph, struct rte_node *node, 35 void **objs, uint16_t nb_objs) 36 { 37 struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts; 38 struct ip4_rewrite_nh_header *nh = ip4_rewrite_nm->nh; 39 const int dyn = IP4_REWRITE_NODE_PRIV1_OFF(node->ctx); 40 uint16_t next0, next1, next2, next3, next_index; 41 struct rte_ipv4_hdr *ip0, *ip1, *ip2, *ip3; 42 uint16_t n_left_from, held = 0, last_spec = 0; 43 void *d0, *d1, *d2, *d3; 44 void **to_next, **from; 45 rte_xmm_t priv01; 46 rte_xmm_t priv23; 47 int i; 48 49 /* Speculative next as last next */ 50 next_index = IP4_REWRITE_NODE_LAST_NEXT(node->ctx); 51 rte_prefetch0(nh); 52 53 pkts = (struct rte_mbuf **)objs; 54 from = objs; 55 n_left_from = nb_objs; 56 57 for (i = 0; i < 4 && i < n_left_from; i++) 58 rte_prefetch0(pkts[i]); 59 60 /* Get stream for the speculated next node */ 61 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs); 62 /* Update Ethernet header of pkts */ 63 while (n_left_from >= 4) { 64 if (likely(n_left_from > 7)) { 65 /* Prefetch only next-mbuf struct and priv area. 66 * Data need not be prefetched as we only write. 67 */ 68 rte_prefetch0(pkts[4]); 69 rte_prefetch0(pkts[5]); 70 rte_prefetch0(pkts[6]); 71 rte_prefetch0(pkts[7]); 72 } 73 74 mbuf0 = pkts[0]; 75 mbuf1 = pkts[1]; 76 mbuf2 = pkts[2]; 77 mbuf3 = pkts[3]; 78 79 pkts += 4; 80 n_left_from -= 4; 81 priv01.u64[0] = node_mbuf_priv1(mbuf0, dyn)->u; 82 priv01.u64[1] = node_mbuf_priv1(mbuf1, dyn)->u; 83 priv23.u64[0] = node_mbuf_priv1(mbuf2, dyn)->u; 84 priv23.u64[1] = node_mbuf_priv1(mbuf3, dyn)->u; 85 86 /* Increment checksum by one. */ 87 priv01.u32[1] += rte_cpu_to_be_16(0x0100); 88 priv01.u32[3] += rte_cpu_to_be_16(0x0100); 89 priv23.u32[1] += rte_cpu_to_be_16(0x0100); 90 priv23.u32[3] += rte_cpu_to_be_16(0x0100); 91 92 /* Update ttl,cksum rewrite ethernet hdr on mbuf0 */ 93 d0 = rte_pktmbuf_mtod(mbuf0, void *); 94 rte_memcpy(d0, nh[priv01.u16[0]].rewrite_data, 95 nh[priv01.u16[0]].rewrite_len); 96 97 next0 = nh[priv01.u16[0]].tx_node; 98 ip0 = (struct rte_ipv4_hdr *)((uint8_t *)d0 + 99 sizeof(struct rte_ether_hdr)); 100 ip0->time_to_live = priv01.u16[1] - 1; 101 ip0->hdr_checksum = priv01.u16[2] + priv01.u16[3]; 102 103 /* Update ttl,cksum rewrite ethernet hdr on mbuf1 */ 104 d1 = rte_pktmbuf_mtod(mbuf1, void *); 105 rte_memcpy(d1, nh[priv01.u16[4]].rewrite_data, 106 nh[priv01.u16[4]].rewrite_len); 107 108 next1 = nh[priv01.u16[4]].tx_node; 109 ip1 = (struct rte_ipv4_hdr *)((uint8_t *)d1 + 110 sizeof(struct rte_ether_hdr)); 111 ip1->time_to_live = priv01.u16[5] - 1; 112 ip1->hdr_checksum = priv01.u16[6] + priv01.u16[7]; 113 114 /* Update ttl,cksum rewrite ethernet hdr on mbuf2 */ 115 d2 = rte_pktmbuf_mtod(mbuf2, void *); 116 rte_memcpy(d2, nh[priv23.u16[0]].rewrite_data, 117 nh[priv23.u16[0]].rewrite_len); 118 next2 = nh[priv23.u16[0]].tx_node; 119 ip2 = (struct rte_ipv4_hdr *)((uint8_t *)d2 + 120 sizeof(struct rte_ether_hdr)); 121 ip2->time_to_live = priv23.u16[1] - 1; 122 ip2->hdr_checksum = priv23.u16[2] + priv23.u16[3]; 123 124 /* Update ttl,cksum rewrite ethernet hdr on mbuf3 */ 125 d3 = rte_pktmbuf_mtod(mbuf3, void *); 126 rte_memcpy(d3, nh[priv23.u16[4]].rewrite_data, 127 nh[priv23.u16[4]].rewrite_len); 128 129 next3 = nh[priv23.u16[4]].tx_node; 130 ip3 = (struct rte_ipv4_hdr *)((uint8_t *)d3 + 131 sizeof(struct rte_ether_hdr)); 132 ip3->time_to_live = priv23.u16[5] - 1; 133 ip3->hdr_checksum = priv23.u16[6] + priv23.u16[7]; 134 135 /* Enqueue four to next node */ 136 rte_edge_t fix_spec = 137 ((next_index == next0) && (next0 == next1) && 138 (next1 == next2) && (next2 == next3)); 139 140 if (unlikely(fix_spec == 0)) { 141 /* Copy things successfully speculated till now */ 142 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 143 from += last_spec; 144 to_next += last_spec; 145 held += last_spec; 146 last_spec = 0; 147 148 /* next0 */ 149 if (next_index == next0) { 150 to_next[0] = from[0]; 151 to_next++; 152 held++; 153 } else { 154 rte_node_enqueue_x1(graph, node, next0, 155 from[0]); 156 } 157 158 /* next1 */ 159 if (next_index == next1) { 160 to_next[0] = from[1]; 161 to_next++; 162 held++; 163 } else { 164 rte_node_enqueue_x1(graph, node, next1, 165 from[1]); 166 } 167 168 /* next2 */ 169 if (next_index == next2) { 170 to_next[0] = from[2]; 171 to_next++; 172 held++; 173 } else { 174 rte_node_enqueue_x1(graph, node, next2, 175 from[2]); 176 } 177 178 /* next3 */ 179 if (next_index == next3) { 180 to_next[0] = from[3]; 181 to_next++; 182 held++; 183 } else { 184 rte_node_enqueue_x1(graph, node, next3, 185 from[3]); 186 } 187 188 from += 4; 189 190 /* Change speculation if last two are same */ 191 if ((next_index != next3) && (next2 == next3)) { 192 /* Put the current speculated node */ 193 rte_node_next_stream_put(graph, node, 194 next_index, held); 195 held = 0; 196 197 /* Get next speculated stream */ 198 next_index = next3; 199 to_next = rte_node_next_stream_get( 200 graph, node, next_index, nb_objs); 201 } 202 } else { 203 last_spec += 4; 204 } 205 } 206 207 while (n_left_from > 0) { 208 uint16_t chksum; 209 210 mbuf0 = pkts[0]; 211 212 pkts += 1; 213 n_left_from -= 1; 214 215 d0 = rte_pktmbuf_mtod(mbuf0, void *); 216 rte_memcpy(d0, nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_data, 217 nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_len); 218 219 next0 = nh[node_mbuf_priv1(mbuf0, dyn)->nh].tx_node; 220 ip0 = (struct rte_ipv4_hdr *)((uint8_t *)d0 + 221 sizeof(struct rte_ether_hdr)); 222 chksum = node_mbuf_priv1(mbuf0, dyn)->cksum + 223 rte_cpu_to_be_16(0x0100); 224 chksum += chksum >= 0xffff; 225 ip0->hdr_checksum = chksum; 226 ip0->time_to_live = node_mbuf_priv1(mbuf0, dyn)->ttl - 1; 227 228 if (unlikely(next_index ^ next0)) { 229 /* Copy things successfully speculated till now */ 230 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 231 from += last_spec; 232 to_next += last_spec; 233 held += last_spec; 234 last_spec = 0; 235 236 rte_node_enqueue_x1(graph, node, next0, from[0]); 237 from += 1; 238 } else { 239 last_spec += 1; 240 } 241 } 242 243 /* !!! Home run !!! */ 244 if (likely(last_spec == nb_objs)) { 245 rte_node_next_stream_move(graph, node, next_index); 246 return nb_objs; 247 } 248 249 held += last_spec; 250 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 251 rte_node_next_stream_put(graph, node, next_index, held); 252 /* Save the last next used */ 253 IP4_REWRITE_NODE_LAST_NEXT(node->ctx) = next_index; 254 255 return nb_objs; 256 } 257 258 static int 259 ip4_rewrite_node_init(const struct rte_graph *graph, struct rte_node *node) 260 { 261 static bool init_once; 262 263 RTE_SET_USED(graph); 264 RTE_BUILD_BUG_ON(sizeof(struct ip4_rewrite_node_ctx) > RTE_NODE_CTX_SZ); 265 266 if (!init_once) { 267 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register( 268 &node_mbuf_priv1_dynfield_desc); 269 if (node_mbuf_priv1_dynfield_offset < 0) 270 return -rte_errno; 271 init_once = true; 272 } 273 IP4_REWRITE_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset; 274 275 node_dbg("ip4_rewrite", "Initialized ip4_rewrite node initialized"); 276 277 return 0; 278 } 279 280 int 281 ip4_rewrite_set_next(uint16_t port_id, uint16_t next_index) 282 { 283 if (ip4_rewrite_nm == NULL) { 284 ip4_rewrite_nm = rte_zmalloc( 285 "ip4_rewrite", sizeof(struct ip4_rewrite_node_main), 286 RTE_CACHE_LINE_SIZE); 287 if (ip4_rewrite_nm == NULL) 288 return -ENOMEM; 289 } 290 ip4_rewrite_nm->next_index[port_id] = next_index; 291 292 return 0; 293 } 294 295 int 296 rte_node_ip4_rewrite_add(uint16_t next_hop, uint8_t *rewrite_data, 297 uint8_t rewrite_len, uint16_t dst_port) 298 { 299 struct ip4_rewrite_nh_header *nh; 300 301 if (next_hop >= RTE_GRAPH_IP4_REWRITE_MAX_NH) 302 return -EINVAL; 303 304 if (rewrite_len > RTE_GRAPH_IP4_REWRITE_MAX_LEN) 305 return -EINVAL; 306 307 if (ip4_rewrite_nm == NULL) { 308 ip4_rewrite_nm = rte_zmalloc( 309 "ip4_rewrite", sizeof(struct ip4_rewrite_node_main), 310 RTE_CACHE_LINE_SIZE); 311 if (ip4_rewrite_nm == NULL) 312 return -ENOMEM; 313 } 314 315 /* Check if dst port doesn't exist as edge */ 316 if (!ip4_rewrite_nm->next_index[dst_port]) 317 return -EINVAL; 318 319 /* Update next hop */ 320 nh = &ip4_rewrite_nm->nh[next_hop]; 321 322 memcpy(nh->rewrite_data, rewrite_data, rewrite_len); 323 nh->tx_node = ip4_rewrite_nm->next_index[dst_port]; 324 nh->rewrite_len = rewrite_len; 325 nh->enabled = true; 326 327 return 0; 328 } 329 330 static struct rte_node_register ip4_rewrite_node = { 331 .process = ip4_rewrite_node_process, 332 .name = "ip4_rewrite", 333 /* Default edge i.e '0' is pkt drop */ 334 .nb_edges = 1, 335 .next_nodes = { 336 [0] = "pkt_drop", 337 }, 338 .init = ip4_rewrite_node_init, 339 }; 340 341 struct rte_node_register * 342 ip4_rewrite_node_get(void) 343 { 344 return &ip4_rewrite_node; 345 } 346 347 RTE_NODE_REGISTER(ip4_rewrite_node); 348