1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2020 Marvell International Ltd.
3 */
4
5 #include <rte_ethdev.h>
6 #include <rte_ether.h>
7 #include <rte_graph.h>
8 #include <rte_graph_worker.h>
9 #include <rte_ip.h>
10 #include <rte_malloc.h>
11 #include <rte_vect.h>
12
13 #include "rte_node_ip4_api.h"
14
15 #include "ip4_rewrite_priv.h"
16 #include "node_private.h"
17
18 struct ip4_rewrite_node_ctx {
19 /* Dynamic offset to mbuf priv1 */
20 int mbuf_priv1_off;
21 /* Cached next index */
22 uint16_t next_index;
23 };
24
25 static struct ip4_rewrite_node_main *ip4_rewrite_nm;
26
27 #define IP4_REWRITE_NODE_LAST_NEXT(ctx) \
28 (((struct ip4_rewrite_node_ctx *)ctx)->next_index)
29
30 #define IP4_REWRITE_NODE_PRIV1_OFF(ctx) \
31 (((struct ip4_rewrite_node_ctx *)ctx)->mbuf_priv1_off)
32
33 static uint16_t
ip4_rewrite_node_process(struct rte_graph * graph,struct rte_node * node,void ** objs,uint16_t nb_objs)34 ip4_rewrite_node_process(struct rte_graph *graph, struct rte_node *node,
35 void **objs, uint16_t nb_objs)
36 {
37 struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts;
38 struct ip4_rewrite_nh_header *nh = ip4_rewrite_nm->nh;
39 const int dyn = IP4_REWRITE_NODE_PRIV1_OFF(node->ctx);
40 uint16_t next0, next1, next2, next3, next_index;
41 struct rte_ipv4_hdr *ip0, *ip1, *ip2, *ip3;
42 uint16_t n_left_from, held = 0, last_spec = 0;
43 void *d0, *d1, *d2, *d3;
44 void **to_next, **from;
45 rte_xmm_t priv01;
46 rte_xmm_t priv23;
47 int i;
48
49 /* Speculative next as last next */
50 next_index = IP4_REWRITE_NODE_LAST_NEXT(node->ctx);
51 rte_prefetch0(nh);
52
53 pkts = (struct rte_mbuf **)objs;
54 from = objs;
55 n_left_from = nb_objs;
56
57 for (i = 0; i < 4 && i < n_left_from; i++)
58 rte_prefetch0(pkts[i]);
59
60 /* Get stream for the speculated next node */
61 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs);
62 /* Update Ethernet header of pkts */
63 while (n_left_from >= 4) {
64 if (likely(n_left_from > 7)) {
65 /* Prefetch only next-mbuf struct and priv area.
66 * Data need not be prefetched as we only write.
67 */
68 rte_prefetch0(pkts[4]);
69 rte_prefetch0(pkts[5]);
70 rte_prefetch0(pkts[6]);
71 rte_prefetch0(pkts[7]);
72 }
73
74 mbuf0 = pkts[0];
75 mbuf1 = pkts[1];
76 mbuf2 = pkts[2];
77 mbuf3 = pkts[3];
78
79 pkts += 4;
80 n_left_from -= 4;
81 priv01.u64[0] = node_mbuf_priv1(mbuf0, dyn)->u;
82 priv01.u64[1] = node_mbuf_priv1(mbuf1, dyn)->u;
83 priv23.u64[0] = node_mbuf_priv1(mbuf2, dyn)->u;
84 priv23.u64[1] = node_mbuf_priv1(mbuf3, dyn)->u;
85
86 /* Increment checksum by one. */
87 priv01.u32[1] += rte_cpu_to_be_16(0x0100);
88 priv01.u32[3] += rte_cpu_to_be_16(0x0100);
89 priv23.u32[1] += rte_cpu_to_be_16(0x0100);
90 priv23.u32[3] += rte_cpu_to_be_16(0x0100);
91
92 /* Update ttl,cksum rewrite ethernet hdr on mbuf0 */
93 d0 = rte_pktmbuf_mtod(mbuf0, void *);
94 rte_memcpy(d0, nh[priv01.u16[0]].rewrite_data,
95 nh[priv01.u16[0]].rewrite_len);
96
97 next0 = nh[priv01.u16[0]].tx_node;
98 ip0 = (struct rte_ipv4_hdr *)((uint8_t *)d0 +
99 sizeof(struct rte_ether_hdr));
100 ip0->time_to_live = priv01.u16[1] - 1;
101 ip0->hdr_checksum = priv01.u16[2] + priv01.u16[3];
102
103 /* Update ttl,cksum rewrite ethernet hdr on mbuf1 */
104 d1 = rte_pktmbuf_mtod(mbuf1, void *);
105 rte_memcpy(d1, nh[priv01.u16[4]].rewrite_data,
106 nh[priv01.u16[4]].rewrite_len);
107
108 next1 = nh[priv01.u16[4]].tx_node;
109 ip1 = (struct rte_ipv4_hdr *)((uint8_t *)d1 +
110 sizeof(struct rte_ether_hdr));
111 ip1->time_to_live = priv01.u16[5] - 1;
112 ip1->hdr_checksum = priv01.u16[6] + priv01.u16[7];
113
114 /* Update ttl,cksum rewrite ethernet hdr on mbuf2 */
115 d2 = rte_pktmbuf_mtod(mbuf2, void *);
116 rte_memcpy(d2, nh[priv23.u16[0]].rewrite_data,
117 nh[priv23.u16[0]].rewrite_len);
118 next2 = nh[priv23.u16[0]].tx_node;
119 ip2 = (struct rte_ipv4_hdr *)((uint8_t *)d2 +
120 sizeof(struct rte_ether_hdr));
121 ip2->time_to_live = priv23.u16[1] - 1;
122 ip2->hdr_checksum = priv23.u16[2] + priv23.u16[3];
123
124 /* Update ttl,cksum rewrite ethernet hdr on mbuf3 */
125 d3 = rte_pktmbuf_mtod(mbuf3, void *);
126 rte_memcpy(d3, nh[priv23.u16[4]].rewrite_data,
127 nh[priv23.u16[4]].rewrite_len);
128
129 next3 = nh[priv23.u16[4]].tx_node;
130 ip3 = (struct rte_ipv4_hdr *)((uint8_t *)d3 +
131 sizeof(struct rte_ether_hdr));
132 ip3->time_to_live = priv23.u16[5] - 1;
133 ip3->hdr_checksum = priv23.u16[6] + priv23.u16[7];
134
135 /* Enqueue four to next node */
136 rte_edge_t fix_spec =
137 ((next_index == next0) && (next0 == next1) &&
138 (next1 == next2) && (next2 == next3));
139
140 if (unlikely(fix_spec == 0)) {
141 /* Copy things successfully speculated till now */
142 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
143 from += last_spec;
144 to_next += last_spec;
145 held += last_spec;
146 last_spec = 0;
147
148 /* next0 */
149 if (next_index == next0) {
150 to_next[0] = from[0];
151 to_next++;
152 held++;
153 } else {
154 rte_node_enqueue_x1(graph, node, next0,
155 from[0]);
156 }
157
158 /* next1 */
159 if (next_index == next1) {
160 to_next[0] = from[1];
161 to_next++;
162 held++;
163 } else {
164 rte_node_enqueue_x1(graph, node, next1,
165 from[1]);
166 }
167
168 /* next2 */
169 if (next_index == next2) {
170 to_next[0] = from[2];
171 to_next++;
172 held++;
173 } else {
174 rte_node_enqueue_x1(graph, node, next2,
175 from[2]);
176 }
177
178 /* next3 */
179 if (next_index == next3) {
180 to_next[0] = from[3];
181 to_next++;
182 held++;
183 } else {
184 rte_node_enqueue_x1(graph, node, next3,
185 from[3]);
186 }
187
188 from += 4;
189
190 /* Change speculation if last two are same */
191 if ((next_index != next3) && (next2 == next3)) {
192 /* Put the current speculated node */
193 rte_node_next_stream_put(graph, node,
194 next_index, held);
195 held = 0;
196
197 /* Get next speculated stream */
198 next_index = next3;
199 to_next = rte_node_next_stream_get(
200 graph, node, next_index, nb_objs);
201 }
202 } else {
203 last_spec += 4;
204 }
205 }
206
207 while (n_left_from > 0) {
208 uint16_t chksum;
209
210 mbuf0 = pkts[0];
211
212 pkts += 1;
213 n_left_from -= 1;
214
215 d0 = rte_pktmbuf_mtod(mbuf0, void *);
216 rte_memcpy(d0, nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_data,
217 nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_len);
218
219 next0 = nh[node_mbuf_priv1(mbuf0, dyn)->nh].tx_node;
220 ip0 = (struct rte_ipv4_hdr *)((uint8_t *)d0 +
221 sizeof(struct rte_ether_hdr));
222 chksum = node_mbuf_priv1(mbuf0, dyn)->cksum +
223 rte_cpu_to_be_16(0x0100);
224 chksum += chksum >= 0xffff;
225 ip0->hdr_checksum = chksum;
226 ip0->time_to_live = node_mbuf_priv1(mbuf0, dyn)->ttl - 1;
227
228 if (unlikely(next_index ^ next0)) {
229 /* Copy things successfully speculated till now */
230 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
231 from += last_spec;
232 to_next += last_spec;
233 held += last_spec;
234 last_spec = 0;
235
236 rte_node_enqueue_x1(graph, node, next0, from[0]);
237 from += 1;
238 } else {
239 last_spec += 1;
240 }
241 }
242
243 /* !!! Home run !!! */
244 if (likely(last_spec == nb_objs)) {
245 rte_node_next_stream_move(graph, node, next_index);
246 return nb_objs;
247 }
248
249 held += last_spec;
250 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
251 rte_node_next_stream_put(graph, node, next_index, held);
252 /* Save the last next used */
253 IP4_REWRITE_NODE_LAST_NEXT(node->ctx) = next_index;
254
255 return nb_objs;
256 }
257
258 static int
ip4_rewrite_node_init(const struct rte_graph * graph,struct rte_node * node)259 ip4_rewrite_node_init(const struct rte_graph *graph, struct rte_node *node)
260 {
261 static bool init_once;
262
263 RTE_SET_USED(graph);
264 RTE_BUILD_BUG_ON(sizeof(struct ip4_rewrite_node_ctx) > RTE_NODE_CTX_SZ);
265
266 if (!init_once) {
267 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register(
268 &node_mbuf_priv1_dynfield_desc);
269 if (node_mbuf_priv1_dynfield_offset < 0)
270 return -rte_errno;
271 init_once = true;
272 }
273 IP4_REWRITE_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset;
274
275 node_dbg("ip4_rewrite", "Initialized ip4_rewrite node initialized");
276
277 return 0;
278 }
279
280 int
ip4_rewrite_set_next(uint16_t port_id,uint16_t next_index)281 ip4_rewrite_set_next(uint16_t port_id, uint16_t next_index)
282 {
283 if (ip4_rewrite_nm == NULL) {
284 ip4_rewrite_nm = rte_zmalloc(
285 "ip4_rewrite", sizeof(struct ip4_rewrite_node_main),
286 RTE_CACHE_LINE_SIZE);
287 if (ip4_rewrite_nm == NULL)
288 return -ENOMEM;
289 }
290 ip4_rewrite_nm->next_index[port_id] = next_index;
291
292 return 0;
293 }
294
295 int
rte_node_ip4_rewrite_add(uint16_t next_hop,uint8_t * rewrite_data,uint8_t rewrite_len,uint16_t dst_port)296 rte_node_ip4_rewrite_add(uint16_t next_hop, uint8_t *rewrite_data,
297 uint8_t rewrite_len, uint16_t dst_port)
298 {
299 struct ip4_rewrite_nh_header *nh;
300
301 if (next_hop >= RTE_GRAPH_IP4_REWRITE_MAX_NH)
302 return -EINVAL;
303
304 if (rewrite_len > RTE_GRAPH_IP4_REWRITE_MAX_LEN)
305 return -EINVAL;
306
307 if (ip4_rewrite_nm == NULL) {
308 ip4_rewrite_nm = rte_zmalloc(
309 "ip4_rewrite", sizeof(struct ip4_rewrite_node_main),
310 RTE_CACHE_LINE_SIZE);
311 if (ip4_rewrite_nm == NULL)
312 return -ENOMEM;
313 }
314
315 /* Check if dst port doesn't exist as edge */
316 if (!ip4_rewrite_nm->next_index[dst_port])
317 return -EINVAL;
318
319 /* Update next hop */
320 nh = &ip4_rewrite_nm->nh[next_hop];
321
322 memcpy(nh->rewrite_data, rewrite_data, rewrite_len);
323 nh->tx_node = ip4_rewrite_nm->next_index[dst_port];
324 nh->rewrite_len = rewrite_len;
325 nh->enabled = true;
326
327 return 0;
328 }
329
330 static struct rte_node_register ip4_rewrite_node = {
331 .process = ip4_rewrite_node_process,
332 .name = "ip4_rewrite",
333 /* Default edge i.e '0' is pkt drop */
334 .nb_edges = 1,
335 .next_nodes = {
336 [0] = "pkt_drop",
337 },
338 .init = ip4_rewrite_node_init,
339 };
340
341 struct rte_node_register *
ip4_rewrite_node_get(void)342 ip4_rewrite_node_get(void)
343 {
344 return &ip4_rewrite_node;
345 }
346
347 RTE_NODE_REGISTER(ip4_rewrite_node);
348