1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2023 Marvell. 3 */ 4 5 #include <rte_ethdev.h> 6 #include <rte_ether.h> 7 #include <rte_graph.h> 8 #include <rte_graph_worker.h> 9 #include <rte_ip.h> 10 #include <rte_malloc.h> 11 #include <rte_vect.h> 12 13 #include "rte_node_ip6_api.h" 14 15 #include "ip6_rewrite_priv.h" 16 #include "node_private.h" 17 18 struct ip6_rewrite_node_ctx { 19 /* Dynamic offset to mbuf priv1 */ 20 int mbuf_priv1_off; 21 /* Cached next index */ 22 uint16_t next_index; 23 }; 24 25 static struct ip6_rewrite_node_main *ip6_rewrite_nm; 26 27 #define IP6_REWRITE_NODE_LAST_NEXT(ctx) \ 28 (((struct ip6_rewrite_node_ctx *)ctx)->next_index) 29 30 #define IP6_REWRITE_NODE_PRIV1_OFF(ctx) \ 31 (((struct ip6_rewrite_node_ctx *)ctx)->mbuf_priv1_off) 32 33 static uint16_t 34 ip6_rewrite_node_process(struct rte_graph *graph, struct rte_node *node, 35 void **objs, uint16_t nb_objs) 36 { 37 struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts; 38 struct ip6_rewrite_nh_header *nh = ip6_rewrite_nm->nh; 39 const int dyn = IP6_REWRITE_NODE_PRIV1_OFF(node->ctx); 40 uint16_t next0, next1, next2, next3, next_index; 41 uint16_t n_left_from, held = 0, last_spec = 0; 42 struct rte_ipv6_hdr *ip0, *ip1, *ip2, *ip3; 43 void *d0, *d1, *d2, *d3; 44 void **to_next, **from; 45 rte_xmm_t priv01; 46 rte_xmm_t priv23; 47 int i; 48 49 /* Speculative next as last next */ 50 next_index = IP6_REWRITE_NODE_LAST_NEXT(node->ctx); 51 rte_prefetch0(nh); 52 53 pkts = (struct rte_mbuf **)objs; 54 from = objs; 55 n_left_from = nb_objs; 56 57 for (i = 0; i < 4 && i < n_left_from; i++) 58 rte_prefetch0(pkts[i]); 59 60 /* Get stream for the speculated next node */ 61 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs); 62 /* Update Ethernet header of pkts */ 63 while (n_left_from >= 4) { 64 if (likely(n_left_from > 7)) { 65 /* Prefetch only next-mbuf struct and priv area. 66 * Data need not be prefetched as we only write. 67 */ 68 rte_prefetch0(pkts[4]); 69 rte_prefetch0(pkts[5]); 70 rte_prefetch0(pkts[6]); 71 rte_prefetch0(pkts[7]); 72 } 73 74 mbuf0 = pkts[0]; 75 mbuf1 = pkts[1]; 76 mbuf2 = pkts[2]; 77 mbuf3 = pkts[3]; 78 79 pkts += 4; 80 n_left_from -= 4; 81 priv01.u64[0] = node_mbuf_priv1(mbuf0, dyn)->u; 82 priv01.u64[1] = node_mbuf_priv1(mbuf1, dyn)->u; 83 priv23.u64[0] = node_mbuf_priv1(mbuf2, dyn)->u; 84 priv23.u64[1] = node_mbuf_priv1(mbuf3, dyn)->u; 85 86 /* Update next_hop rewrite ethernet hdr on mbuf0 */ 87 d0 = rte_pktmbuf_mtod(mbuf0, void *); 88 rte_memcpy(d0, nh[priv01.u16[0]].rewrite_data, 89 nh[priv01.u16[0]].rewrite_len); 90 91 next0 = nh[priv01.u16[0]].tx_node; 92 ip0 = (struct rte_ipv6_hdr *)((uint8_t *)d0 + 93 sizeof(struct rte_ether_hdr)); 94 ip0->hop_limits = priv01.u16[1] - 1; 95 96 /* Update next_hop rewrite ethernet hdr on mbuf1 */ 97 d1 = rte_pktmbuf_mtod(mbuf1, void *); 98 rte_memcpy(d1, nh[priv01.u16[4]].rewrite_data, 99 nh[priv01.u16[4]].rewrite_len); 100 101 next1 = nh[priv01.u16[4]].tx_node; 102 ip1 = (struct rte_ipv6_hdr *)((uint8_t *)d1 + 103 sizeof(struct rte_ether_hdr)); 104 ip1->hop_limits = priv01.u16[5] - 1; 105 106 /* Update next_hop rewrite ethernet hdr on mbuf2 */ 107 d2 = rte_pktmbuf_mtod(mbuf2, void *); 108 rte_memcpy(d2, nh[priv23.u16[0]].rewrite_data, 109 nh[priv23.u16[0]].rewrite_len); 110 next2 = nh[priv23.u16[0]].tx_node; 111 ip2 = (struct rte_ipv6_hdr *)((uint8_t *)d2 + 112 sizeof(struct rte_ether_hdr)); 113 ip2->hop_limits = priv23.u16[1] - 1; 114 115 /* Update next_hop rewrite ethernet hdr on mbuf3 */ 116 d3 = rte_pktmbuf_mtod(mbuf3, void *); 117 rte_memcpy(d3, nh[priv23.u16[4]].rewrite_data, 118 nh[priv23.u16[4]].rewrite_len); 119 120 next3 = nh[priv23.u16[4]].tx_node; 121 ip3 = (struct rte_ipv6_hdr *)((uint8_t *)d3 + 122 sizeof(struct rte_ether_hdr)); 123 ip3->hop_limits = priv23.u16[5] - 1; 124 125 /* Enqueue four packets to next node */ 126 rte_edge_t fix_spec = 127 ((next_index == next0) && (next0 == next1) && 128 (next1 == next2) && (next2 == next3)); 129 130 if (unlikely(fix_spec == 0)) { 131 /* Copy things successfully speculated till now */ 132 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 133 from += last_spec; 134 to_next += last_spec; 135 held += last_spec; 136 last_spec = 0; 137 138 /* next0 */ 139 if (next_index == next0) { 140 to_next[0] = from[0]; 141 to_next++; 142 held++; 143 } else { 144 rte_node_enqueue_x1(graph, node, next0, 145 from[0]); 146 } 147 148 /* next1 */ 149 if (next_index == next1) { 150 to_next[0] = from[1]; 151 to_next++; 152 held++; 153 } else { 154 rte_node_enqueue_x1(graph, node, next1, 155 from[1]); 156 } 157 158 /* next2 */ 159 if (next_index == next2) { 160 to_next[0] = from[2]; 161 to_next++; 162 held++; 163 } else { 164 rte_node_enqueue_x1(graph, node, next2, 165 from[2]); 166 } 167 168 /* next3 */ 169 if (next_index == next3) { 170 to_next[0] = from[3]; 171 to_next++; 172 held++; 173 } else { 174 rte_node_enqueue_x1(graph, node, next3, 175 from[3]); 176 } 177 178 from += 4; 179 180 /* Change speculation if last two are same */ 181 if ((next_index != next3) && (next2 == next3)) { 182 /* Put the current speculated node */ 183 rte_node_next_stream_put(graph, node, 184 next_index, held); 185 held = 0; 186 187 /* Get next speculated stream */ 188 next_index = next3; 189 to_next = rte_node_next_stream_get( 190 graph, node, next_index, nb_objs); 191 } 192 } else { 193 last_spec += 4; 194 } 195 } 196 197 while (n_left_from > 0) { 198 mbuf0 = pkts[0]; 199 200 pkts += 1; 201 n_left_from -= 1; 202 203 d0 = rte_pktmbuf_mtod(mbuf0, void *); 204 rte_memcpy(d0, nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_data, 205 nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_len); 206 207 next0 = nh[node_mbuf_priv1(mbuf0, dyn)->nh].tx_node; 208 ip0 = (struct rte_ipv6_hdr *)((uint8_t *)d0 + 209 sizeof(struct rte_ether_hdr)); 210 ip0->hop_limits = node_mbuf_priv1(mbuf0, dyn)->ttl - 1; 211 212 if (unlikely(next_index ^ next0)) { 213 /* Copy things successfully speculated till now */ 214 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 215 from += last_spec; 216 to_next += last_spec; 217 held += last_spec; 218 last_spec = 0; 219 220 rte_node_enqueue_x1(graph, node, next0, from[0]); 221 from += 1; 222 } else { 223 last_spec += 1; 224 } 225 } 226 227 /* !!! Home run !!! */ 228 if (likely(last_spec == nb_objs)) { 229 rte_node_next_stream_move(graph, node, next_index); 230 return nb_objs; 231 } 232 233 held += last_spec; 234 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 235 rte_node_next_stream_put(graph, node, next_index, held); 236 /* Save the last next used */ 237 IP6_REWRITE_NODE_LAST_NEXT(node->ctx) = next_index; 238 239 return nb_objs; 240 } 241 242 static int 243 ip6_rewrite_node_init(const struct rte_graph *graph, struct rte_node *node) 244 { 245 static bool init_once; 246 247 RTE_SET_USED(graph); 248 RTE_BUILD_BUG_ON(sizeof(struct ip6_rewrite_node_ctx) > RTE_NODE_CTX_SZ); 249 250 if (!init_once) { 251 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register( 252 &node_mbuf_priv1_dynfield_desc); 253 if (node_mbuf_priv1_dynfield_offset < 0) 254 return -rte_errno; 255 init_once = true; 256 } 257 IP6_REWRITE_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset; 258 259 node_dbg("ip6_rewrite", "Initialized ip6_rewrite node"); 260 261 return 0; 262 } 263 264 int 265 ip6_rewrite_set_next(uint16_t port_id, uint16_t next_index) 266 { 267 if (ip6_rewrite_nm == NULL) { 268 ip6_rewrite_nm = rte_zmalloc( 269 "ip6_rewrite", sizeof(struct ip6_rewrite_node_main), 270 RTE_CACHE_LINE_SIZE); 271 if (ip6_rewrite_nm == NULL) 272 return -ENOMEM; 273 } 274 ip6_rewrite_nm->next_index[port_id] = next_index; 275 276 return 0; 277 } 278 279 int 280 rte_node_ip6_rewrite_add(uint16_t next_hop, uint8_t *rewrite_data, 281 uint8_t rewrite_len, uint16_t dst_port) 282 { 283 struct ip6_rewrite_nh_header *nh; 284 285 if (next_hop >= RTE_GRAPH_IP6_REWRITE_MAX_NH) 286 return -EINVAL; 287 288 if (rewrite_len > RTE_GRAPH_IP6_REWRITE_MAX_LEN) 289 return -EINVAL; 290 291 if (ip6_rewrite_nm == NULL) { 292 ip6_rewrite_nm = rte_zmalloc( 293 "ip6_rewrite", sizeof(struct ip6_rewrite_node_main), 294 RTE_CACHE_LINE_SIZE); 295 if (ip6_rewrite_nm == NULL) 296 return -ENOMEM; 297 } 298 299 /* Check if dst port doesn't exist as edge */ 300 if (!ip6_rewrite_nm->next_index[dst_port]) 301 return -EINVAL; 302 303 /* Update next hop */ 304 nh = &ip6_rewrite_nm->nh[next_hop]; 305 306 memcpy(nh->rewrite_data, rewrite_data, rewrite_len); 307 nh->tx_node = ip6_rewrite_nm->next_index[dst_port]; 308 nh->rewrite_len = rewrite_len; 309 nh->enabled = true; 310 311 return 0; 312 } 313 314 static struct rte_node_register ip6_rewrite_node = { 315 .process = ip6_rewrite_node_process, 316 .name = "ip6_rewrite", 317 /* Default edge i.e '0' is pkt drop */ 318 .nb_edges = 1, 319 .next_nodes = { 320 [0] = "pkt_drop", 321 }, 322 .init = ip6_rewrite_node_init, 323 }; 324 325 struct rte_node_register * 326 ip6_rewrite_node_get(void) 327 { 328 return &ip6_rewrite_node; 329 } 330 331 RTE_NODE_REGISTER(ip6_rewrite_node); 332