1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (C) 2020 Marvell.
3 */
4
5 #include <stdalign.h>
6
7 #include <rte_graph.h>
8 #include <rte_graph_worker.h>
9
10 #include "pkt_cls_priv.h"
11 #include "node_private.h"
12
13 /* Next node for each ptype, default is '0' is "pkt_drop" */
14 static const alignas(RTE_CACHE_LINE_SIZE) uint8_t p_nxt[256] = {
15 [RTE_PTYPE_L3_IPV4] = PKT_CLS_NEXT_IP4_LOOKUP,
16
17 [RTE_PTYPE_L3_IPV4_EXT] = PKT_CLS_NEXT_IP4_LOOKUP,
18
19 [RTE_PTYPE_L3_IPV4_EXT_UNKNOWN] = PKT_CLS_NEXT_IP4_LOOKUP,
20
21 [RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER] =
22 PKT_CLS_NEXT_IP4_LOOKUP,
23
24 [RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L2_ETHER] =
25 PKT_CLS_NEXT_IP4_LOOKUP,
26
27 [RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER] =
28 PKT_CLS_NEXT_IP4_LOOKUP,
29
30 [RTE_PTYPE_L3_IPV6] = PKT_CLS_NEXT_IP6_LOOKUP,
31
32 [RTE_PTYPE_L3_IPV6_EXT] = PKT_CLS_NEXT_IP6_LOOKUP,
33
34 [RTE_PTYPE_L3_IPV6_EXT_UNKNOWN] = PKT_CLS_NEXT_IP6_LOOKUP,
35
36 [RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L2_ETHER] = PKT_CLS_NEXT_IP6_LOOKUP,
37
38 [RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L2_ETHER] = PKT_CLS_NEXT_IP6_LOOKUP,
39
40 [RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER] =
41 PKT_CLS_NEXT_IP6_LOOKUP,
42 };
43
44 static uint16_t
pkt_cls_node_process(struct rte_graph * graph,struct rte_node * node,void ** objs,uint16_t nb_objs)45 pkt_cls_node_process(struct rte_graph *graph, struct rte_node *node,
46 void **objs, uint16_t nb_objs)
47 {
48 struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts;
49 uint8_t l0, l1, l2, l3, last_type;
50 uint16_t next_index, n_left_from;
51 uint16_t held = 0, last_spec = 0;
52 struct pkt_cls_node_ctx *ctx;
53 void **to_next, **from;
54 uint32_t i;
55
56 pkts = (struct rte_mbuf **)objs;
57 from = objs;
58 n_left_from = nb_objs;
59
60 for (i = OBJS_PER_CLINE; i < RTE_GRAPH_BURST_SIZE; i += OBJS_PER_CLINE)
61 rte_prefetch0(&objs[i]);
62
63 #if RTE_GRAPH_BURST_SIZE > 64
64 for (i = 0; i < 4 && i < n_left_from; i++)
65 rte_prefetch0(pkts[i]);
66 #endif
67
68 ctx = (struct pkt_cls_node_ctx *)node->ctx;
69 last_type = ctx->l2l3_type;
70 next_index = p_nxt[last_type];
71
72 /* Get stream for the speculated next node */
73 to_next = rte_node_next_stream_get(graph, node,
74 next_index, nb_objs);
75 while (n_left_from >= 4) {
76 #if RTE_GRAPH_BURST_SIZE > 64
77 if (likely(n_left_from > 7)) {
78 rte_prefetch0(pkts[4]);
79 rte_prefetch0(pkts[5]);
80 rte_prefetch0(pkts[6]);
81 rte_prefetch0(pkts[7]);
82 }
83 #endif
84
85 mbuf0 = pkts[0];
86 mbuf1 = pkts[1];
87 mbuf2 = pkts[2];
88 mbuf3 = pkts[3];
89 pkts += 4;
90 n_left_from -= 4;
91
92 l0 = mbuf0->packet_type &
93 (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
94 l1 = mbuf1->packet_type &
95 (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
96 l2 = mbuf2->packet_type &
97 (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
98 l3 = mbuf3->packet_type &
99 (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
100
101 /* Check if they are destined to same
102 * next node based on l2l3 packet type.
103 */
104 uint8_t fix_spec = (last_type ^ l0) | (last_type ^ l1) |
105 (last_type ^ l2) | (last_type ^ l3);
106
107 if (unlikely(fix_spec)) {
108 /* Copy things successfully speculated till now */
109 rte_memcpy(to_next, from,
110 last_spec * sizeof(from[0]));
111 from += last_spec;
112 to_next += last_spec;
113 held += last_spec;
114 last_spec = 0;
115
116 /* l0 */
117 if (p_nxt[l0] == next_index) {
118 to_next[0] = from[0];
119 to_next++;
120 held++;
121 } else {
122 rte_node_enqueue_x1(graph, node,
123 p_nxt[l0], from[0]);
124 }
125
126 /* l1 */
127 if (p_nxt[l1] == next_index) {
128 to_next[0] = from[1];
129 to_next++;
130 held++;
131 } else {
132 rte_node_enqueue_x1(graph, node,
133 p_nxt[l1], from[1]);
134 }
135
136 /* l2 */
137 if (p_nxt[l2] == next_index) {
138 to_next[0] = from[2];
139 to_next++;
140 held++;
141 } else {
142 rte_node_enqueue_x1(graph, node,
143 p_nxt[l2], from[2]);
144 }
145
146 /* l3 */
147 if (p_nxt[l3] == next_index) {
148 to_next[0] = from[3];
149 to_next++;
150 held++;
151 } else {
152 rte_node_enqueue_x1(graph, node,
153 p_nxt[l3], from[3]);
154 }
155
156 /* Update speculated ptype */
157 if ((last_type != l3) && (l2 == l3) &&
158 (next_index != p_nxt[l3])) {
159 /* Put the current stream for
160 * speculated ltype.
161 */
162 rte_node_next_stream_put(graph, node,
163 next_index, held);
164
165 held = 0;
166
167 /* Get next stream for new ltype */
168 next_index = p_nxt[l3];
169 last_type = l3;
170 to_next = rte_node_next_stream_get(graph, node,
171 next_index,
172 nb_objs);
173 } else if (next_index == p_nxt[l3]) {
174 last_type = l3;
175 }
176
177 from += 4;
178 } else {
179 last_spec += 4;
180 }
181 }
182
183 while (n_left_from > 0) {
184 mbuf0 = pkts[0];
185
186 pkts += 1;
187 n_left_from -= 1;
188
189 l0 = mbuf0->packet_type &
190 (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
191 if (unlikely((l0 != last_type) &&
192 (p_nxt[l0] != next_index))) {
193 /* Copy things successfully speculated till now */
194 rte_memcpy(to_next, from,
195 last_spec * sizeof(from[0]));
196 from += last_spec;
197 to_next += last_spec;
198 held += last_spec;
199 last_spec = 0;
200
201 rte_node_enqueue_x1(graph, node,
202 p_nxt[l0], from[0]);
203 from += 1;
204 } else {
205 last_spec += 1;
206 }
207 }
208
209 /* !!! Home run !!! */
210 if (likely(last_spec == nb_objs)) {
211 rte_node_next_stream_move(graph, node, next_index);
212 return nb_objs;
213 }
214
215 held += last_spec;
216 /* Copy things successfully speculated till now */
217 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
218 rte_node_next_stream_put(graph, node, next_index, held);
219
220 ctx->l2l3_type = last_type;
221 return nb_objs;
222 }
223
224 /* Packet Classification Node */
225 struct rte_node_register pkt_cls_node = {
226 .process = pkt_cls_node_process,
227 .name = "pkt_cls",
228
229 .nb_edges = PKT_CLS_NEXT_MAX,
230 .next_nodes = {
231 /* Pkt drop node starts at '0' */
232 [PKT_CLS_NEXT_PKT_DROP] = "pkt_drop",
233 [PKT_CLS_NEXT_IP4_LOOKUP] = "ip4_lookup",
234 [PKT_CLS_NEXT_IP6_LOOKUP] = "ip6_lookup",
235 },
236 };
237 RTE_NODE_REGISTER(pkt_cls_node);
238