1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2023 Marvell.
3 */
4
5 #include <rte_ethdev.h>
6 #include <rte_ether.h>
7 #include <rte_graph.h>
8 #include <rte_graph_worker.h>
9 #include <rte_ip.h>
10 #include <rte_malloc.h>
11 #include <rte_vect.h>
12
13 #include "rte_node_ip6_api.h"
14
15 #include "ip6_rewrite_priv.h"
16 #include "node_private.h"
17
18 struct ip6_rewrite_node_ctx {
19 /* Dynamic offset to mbuf priv1 */
20 int mbuf_priv1_off;
21 /* Cached next index */
22 uint16_t next_index;
23 };
24
25 static struct ip6_rewrite_node_main *ip6_rewrite_nm;
26
27 #define IP6_REWRITE_NODE_LAST_NEXT(ctx) \
28 (((struct ip6_rewrite_node_ctx *)ctx)->next_index)
29
30 #define IP6_REWRITE_NODE_PRIV1_OFF(ctx) \
31 (((struct ip6_rewrite_node_ctx *)ctx)->mbuf_priv1_off)
32
33 static uint16_t
ip6_rewrite_node_process(struct rte_graph * graph,struct rte_node * node,void ** objs,uint16_t nb_objs)34 ip6_rewrite_node_process(struct rte_graph *graph, struct rte_node *node,
35 void **objs, uint16_t nb_objs)
36 {
37 struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts;
38 struct ip6_rewrite_nh_header *nh = ip6_rewrite_nm->nh;
39 const int dyn = IP6_REWRITE_NODE_PRIV1_OFF(node->ctx);
40 uint16_t next0, next1, next2, next3, next_index;
41 uint16_t n_left_from, held = 0, last_spec = 0;
42 struct rte_ipv6_hdr *ip0, *ip1, *ip2, *ip3;
43 void *d0, *d1, *d2, *d3;
44 void **to_next, **from;
45 rte_xmm_t priv01;
46 rte_xmm_t priv23;
47 int i;
48
49 /* Speculative next as last next */
50 next_index = IP6_REWRITE_NODE_LAST_NEXT(node->ctx);
51 rte_prefetch0(nh);
52
53 pkts = (struct rte_mbuf **)objs;
54 from = objs;
55 n_left_from = nb_objs;
56
57 for (i = 0; i < 4 && i < n_left_from; i++)
58 rte_prefetch0(pkts[i]);
59
60 /* Get stream for the speculated next node */
61 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs);
62 /* Update Ethernet header of pkts */
63 while (n_left_from >= 4) {
64 if (likely(n_left_from > 7)) {
65 /* Prefetch only next-mbuf struct and priv area.
66 * Data need not be prefetched as we only write.
67 */
68 rte_prefetch0(pkts[4]);
69 rte_prefetch0(pkts[5]);
70 rte_prefetch0(pkts[6]);
71 rte_prefetch0(pkts[7]);
72 }
73
74 mbuf0 = pkts[0];
75 mbuf1 = pkts[1];
76 mbuf2 = pkts[2];
77 mbuf3 = pkts[3];
78
79 pkts += 4;
80 n_left_from -= 4;
81 priv01.u64[0] = node_mbuf_priv1(mbuf0, dyn)->u;
82 priv01.u64[1] = node_mbuf_priv1(mbuf1, dyn)->u;
83 priv23.u64[0] = node_mbuf_priv1(mbuf2, dyn)->u;
84 priv23.u64[1] = node_mbuf_priv1(mbuf3, dyn)->u;
85
86 /* Update next_hop rewrite ethernet hdr on mbuf0 */
87 d0 = rte_pktmbuf_mtod(mbuf0, void *);
88 rte_memcpy(d0, nh[priv01.u16[0]].rewrite_data,
89 nh[priv01.u16[0]].rewrite_len);
90
91 next0 = nh[priv01.u16[0]].tx_node;
92 ip0 = (struct rte_ipv6_hdr *)((uint8_t *)d0 +
93 sizeof(struct rte_ether_hdr));
94 ip0->hop_limits = priv01.u16[1] - 1;
95
96 /* Update next_hop rewrite ethernet hdr on mbuf1 */
97 d1 = rte_pktmbuf_mtod(mbuf1, void *);
98 rte_memcpy(d1, nh[priv01.u16[4]].rewrite_data,
99 nh[priv01.u16[4]].rewrite_len);
100
101 next1 = nh[priv01.u16[4]].tx_node;
102 ip1 = (struct rte_ipv6_hdr *)((uint8_t *)d1 +
103 sizeof(struct rte_ether_hdr));
104 ip1->hop_limits = priv01.u16[5] - 1;
105
106 /* Update next_hop rewrite ethernet hdr on mbuf2 */
107 d2 = rte_pktmbuf_mtod(mbuf2, void *);
108 rte_memcpy(d2, nh[priv23.u16[0]].rewrite_data,
109 nh[priv23.u16[0]].rewrite_len);
110 next2 = nh[priv23.u16[0]].tx_node;
111 ip2 = (struct rte_ipv6_hdr *)((uint8_t *)d2 +
112 sizeof(struct rte_ether_hdr));
113 ip2->hop_limits = priv23.u16[1] - 1;
114
115 /* Update next_hop rewrite ethernet hdr on mbuf3 */
116 d3 = rte_pktmbuf_mtod(mbuf3, void *);
117 rte_memcpy(d3, nh[priv23.u16[4]].rewrite_data,
118 nh[priv23.u16[4]].rewrite_len);
119
120 next3 = nh[priv23.u16[4]].tx_node;
121 ip3 = (struct rte_ipv6_hdr *)((uint8_t *)d3 +
122 sizeof(struct rte_ether_hdr));
123 ip3->hop_limits = priv23.u16[5] - 1;
124
125 /* Enqueue four packets to next node */
126 rte_edge_t fix_spec =
127 ((next_index == next0) && (next0 == next1) &&
128 (next1 == next2) && (next2 == next3));
129
130 if (unlikely(fix_spec == 0)) {
131 /* Copy things successfully speculated till now */
132 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
133 from += last_spec;
134 to_next += last_spec;
135 held += last_spec;
136 last_spec = 0;
137
138 /* next0 */
139 if (next_index == next0) {
140 to_next[0] = from[0];
141 to_next++;
142 held++;
143 } else {
144 rte_node_enqueue_x1(graph, node, next0,
145 from[0]);
146 }
147
148 /* next1 */
149 if (next_index == next1) {
150 to_next[0] = from[1];
151 to_next++;
152 held++;
153 } else {
154 rte_node_enqueue_x1(graph, node, next1,
155 from[1]);
156 }
157
158 /* next2 */
159 if (next_index == next2) {
160 to_next[0] = from[2];
161 to_next++;
162 held++;
163 } else {
164 rte_node_enqueue_x1(graph, node, next2,
165 from[2]);
166 }
167
168 /* next3 */
169 if (next_index == next3) {
170 to_next[0] = from[3];
171 to_next++;
172 held++;
173 } else {
174 rte_node_enqueue_x1(graph, node, next3,
175 from[3]);
176 }
177
178 from += 4;
179
180 /* Change speculation if last two are same */
181 if ((next_index != next3) && (next2 == next3)) {
182 /* Put the current speculated node */
183 rte_node_next_stream_put(graph, node,
184 next_index, held);
185 held = 0;
186
187 /* Get next speculated stream */
188 next_index = next3;
189 to_next = rte_node_next_stream_get(
190 graph, node, next_index, nb_objs);
191 }
192 } else {
193 last_spec += 4;
194 }
195 }
196
197 while (n_left_from > 0) {
198 mbuf0 = pkts[0];
199
200 pkts += 1;
201 n_left_from -= 1;
202
203 d0 = rte_pktmbuf_mtod(mbuf0, void *);
204 rte_memcpy(d0, nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_data,
205 nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_len);
206
207 next0 = nh[node_mbuf_priv1(mbuf0, dyn)->nh].tx_node;
208 ip0 = (struct rte_ipv6_hdr *)((uint8_t *)d0 +
209 sizeof(struct rte_ether_hdr));
210 ip0->hop_limits = node_mbuf_priv1(mbuf0, dyn)->ttl - 1;
211
212 if (unlikely(next_index ^ next0)) {
213 /* Copy things successfully speculated till now */
214 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
215 from += last_spec;
216 to_next += last_spec;
217 held += last_spec;
218 last_spec = 0;
219
220 rte_node_enqueue_x1(graph, node, next0, from[0]);
221 from += 1;
222 } else {
223 last_spec += 1;
224 }
225 }
226
227 /* !!! Home run !!! */
228 if (likely(last_spec == nb_objs)) {
229 rte_node_next_stream_move(graph, node, next_index);
230 return nb_objs;
231 }
232
233 held += last_spec;
234 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
235 rte_node_next_stream_put(graph, node, next_index, held);
236 /* Save the last next used */
237 IP6_REWRITE_NODE_LAST_NEXT(node->ctx) = next_index;
238
239 return nb_objs;
240 }
241
242 static int
ip6_rewrite_node_init(const struct rte_graph * graph,struct rte_node * node)243 ip6_rewrite_node_init(const struct rte_graph *graph, struct rte_node *node)
244 {
245 static bool init_once;
246
247 RTE_SET_USED(graph);
248 RTE_BUILD_BUG_ON(sizeof(struct ip6_rewrite_node_ctx) > RTE_NODE_CTX_SZ);
249
250 if (!init_once) {
251 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register(
252 &node_mbuf_priv1_dynfield_desc);
253 if (node_mbuf_priv1_dynfield_offset < 0)
254 return -rte_errno;
255 init_once = true;
256 }
257 IP6_REWRITE_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset;
258
259 node_dbg("ip6_rewrite", "Initialized ip6_rewrite node");
260
261 return 0;
262 }
263
264 int
ip6_rewrite_set_next(uint16_t port_id,uint16_t next_index)265 ip6_rewrite_set_next(uint16_t port_id, uint16_t next_index)
266 {
267 if (ip6_rewrite_nm == NULL) {
268 ip6_rewrite_nm = rte_zmalloc(
269 "ip6_rewrite", sizeof(struct ip6_rewrite_node_main),
270 RTE_CACHE_LINE_SIZE);
271 if (ip6_rewrite_nm == NULL)
272 return -ENOMEM;
273 }
274 ip6_rewrite_nm->next_index[port_id] = next_index;
275
276 return 0;
277 }
278
279 int
rte_node_ip6_rewrite_add(uint16_t next_hop,uint8_t * rewrite_data,uint8_t rewrite_len,uint16_t dst_port)280 rte_node_ip6_rewrite_add(uint16_t next_hop, uint8_t *rewrite_data,
281 uint8_t rewrite_len, uint16_t dst_port)
282 {
283 struct ip6_rewrite_nh_header *nh;
284
285 if (next_hop >= RTE_GRAPH_IP6_REWRITE_MAX_NH)
286 return -EINVAL;
287
288 if (rewrite_len > RTE_GRAPH_IP6_REWRITE_MAX_LEN)
289 return -EINVAL;
290
291 if (ip6_rewrite_nm == NULL) {
292 ip6_rewrite_nm = rte_zmalloc(
293 "ip6_rewrite", sizeof(struct ip6_rewrite_node_main),
294 RTE_CACHE_LINE_SIZE);
295 if (ip6_rewrite_nm == NULL)
296 return -ENOMEM;
297 }
298
299 /* Check if dst port doesn't exist as edge */
300 if (!ip6_rewrite_nm->next_index[dst_port])
301 return -EINVAL;
302
303 /* Update next hop */
304 nh = &ip6_rewrite_nm->nh[next_hop];
305
306 memcpy(nh->rewrite_data, rewrite_data, rewrite_len);
307 nh->tx_node = ip6_rewrite_nm->next_index[dst_port];
308 nh->rewrite_len = rewrite_len;
309 nh->enabled = true;
310
311 return 0;
312 }
313
314 static struct rte_node_register ip6_rewrite_node = {
315 .process = ip6_rewrite_node_process,
316 .name = "ip6_rewrite",
317 /* Default edge i.e '0' is pkt drop */
318 .nb_edges = 1,
319 .next_nodes = {
320 [0] = "pkt_drop",
321 },
322 .init = ip6_rewrite_node_init,
323 };
324
325 struct rte_node_register *
ip6_rewrite_node_get(void)326 ip6_rewrite_node_get(void)
327 {
328 return &ip6_rewrite_node;
329 }
330
331 RTE_NODE_REGISTER(ip6_rewrite_node);
332