1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
3 */
4
5 #ifndef _ACL_RUN_H_
6 #define _ACL_RUN_H_
7
8 #include <rte_acl.h>
9 #include "acl.h"
10
11 #define MAX_SEARCHES_AVX16 16
12 #define MAX_SEARCHES_SSE8 8
13 #define MAX_SEARCHES_ALTIVEC8 8
14 #define MAX_SEARCHES_SSE4 4
15 #define MAX_SEARCHES_ALTIVEC4 4
16 #define MAX_SEARCHES_SCALAR 2
17
18 #define GET_NEXT_4BYTES(prm, idx) \
19 (*((const int32_t *)((prm)[(idx)].data + *(prm)[idx].data_index++)))
20
21
22 #define RTE_ACL_NODE_INDEX ((uint32_t)~RTE_ACL_NODE_TYPE)
23
24 #define SCALAR_QRANGE_MULT 0x01010101
25 #define SCALAR_QRANGE_MASK 0x7f7f7f7f
26 #define SCALAR_QRANGE_MIN 0x80808080
27
28 /*
29 * Structure to manage N parallel trie traversals.
30 * The runtime trie traversal routines can process 8, 4, or 2 tries
31 * in parallel. Each packet may require multiple trie traversals (up to 4).
32 * This structure is used to fill the slots (0 to n-1) for parallel processing
33 * with the trie traversals needed for each packet.
34 */
35 struct acl_flow_data {
36 uint32_t num_packets;
37 /* number of packets processed */
38 uint32_t started;
39 /* number of trie traversals in progress */
40 uint32_t trie;
41 /* current trie index (0 to N-1) */
42 uint32_t cmplt_size;
43 /* maximum number of packets to process */
44 uint32_t total_packets;
45 /* number of result categories per packet. */
46 uint32_t categories;
47 const uint64_t *trans;
48 const uint8_t **data;
49 uint32_t *results;
50 struct completion *last_cmplt;
51 struct completion *cmplt_array;
52 };
53
54 /*
55 * Structure to maintain running results for
56 * a single packet (up to 4 tries).
57 */
__rte_aligned(XMM_SIZE)58 struct __rte_aligned(XMM_SIZE) completion {
59 uint32_t *results; /* running results. */
60 int32_t priority[RTE_ACL_MAX_CATEGORIES]; /* running priorities. */
61 uint32_t count; /* num of remaining tries */
62 /* true for allocated struct */
63 };
64
65 /*
66 * One parms structure for each slot in the search engine.
67 */
68 struct parms {
69 const uint8_t *data;
70 /* input data for this packet */
71 const uint32_t *data_index;
72 /* data indirection for this trie */
73 struct completion *cmplt;
74 /* completion data for this packet */
75 };
76
77 /*
78 * Define an global idle node for unused engine slots
79 */
80 static const uint32_t idle[UINT8_MAX + 1];
81
82 /*
83 * Allocate a completion structure to manage the tries for a packet.
84 */
85 static inline struct completion *
alloc_completion(struct completion * p,uint32_t size,uint32_t tries,uint32_t * results)86 alloc_completion(struct completion *p, uint32_t size, uint32_t tries,
87 uint32_t *results)
88 {
89 uint32_t n;
90
91 for (n = 0; n < size; n++) {
92
93 if (p[n].count == 0) {
94
95 /* mark as allocated and set number of tries. */
96 p[n].count = tries;
97 p[n].results = results;
98 return &(p[n]);
99 }
100 }
101
102 /* should never get here */
103 return NULL;
104 }
105
106 /*
107 * Resolve priority for a single result trie.
108 */
109 static inline void
resolve_single_priority(uint64_t transition,int n,const struct rte_acl_ctx * ctx,struct parms * parms,const struct rte_acl_match_results * p)110 resolve_single_priority(uint64_t transition, int n,
111 const struct rte_acl_ctx *ctx, struct parms *parms,
112 const struct rte_acl_match_results *p)
113 {
114 if (parms[n].cmplt->count == ctx->num_tries ||
115 parms[n].cmplt->priority[0] <=
116 p[transition].priority[0]) {
117
118 parms[n].cmplt->priority[0] = p[transition].priority[0];
119 parms[n].cmplt->results[0] = p[transition].results[0];
120 }
121 }
122
123 /*
124 * Routine to fill a slot in the parallel trie traversal array (parms) from
125 * the list of packets (flows).
126 */
127 static inline uint64_t
acl_start_next_trie(struct acl_flow_data * flows,struct parms * parms,int n,const struct rte_acl_ctx * ctx)128 acl_start_next_trie(struct acl_flow_data *flows, struct parms *parms, int n,
129 const struct rte_acl_ctx *ctx)
130 {
131 uint64_t transition;
132
133 /* if there are any more packets to process */
134 if (flows->num_packets < flows->total_packets) {
135 parms[n].data = flows->data[flows->num_packets];
136 parms[n].data_index = ctx->trie[flows->trie].data_index;
137
138 /* if this is the first trie for this packet */
139 if (flows->trie == 0) {
140 flows->last_cmplt = alloc_completion(flows->cmplt_array,
141 flows->cmplt_size, ctx->num_tries,
142 flows->results +
143 flows->num_packets * flows->categories);
144 }
145
146 /* set completion parameters and starting index for this slot */
147 parms[n].cmplt = flows->last_cmplt;
148 transition =
149 flows->trans[parms[n].data[*parms[n].data_index++] +
150 ctx->trie[flows->trie].root_index];
151
152 /*
153 * if this is the last trie for this packet,
154 * then setup next packet.
155 */
156 flows->trie++;
157 if (flows->trie >= ctx->num_tries) {
158 flows->trie = 0;
159 flows->num_packets++;
160 }
161
162 /* keep track of number of active trie traversals */
163 flows->started++;
164
165 /* no more tries to process, set slot to an idle position */
166 } else {
167 transition = ctx->idle;
168 parms[n].data = (const uint8_t *)idle;
169 parms[n].data_index = idle;
170 }
171 return transition;
172 }
173
174 static inline void
acl_set_flow(struct acl_flow_data * flows,struct completion * cmplt,uint32_t cmplt_size,const uint8_t ** data,uint32_t * results,uint32_t data_num,uint32_t categories,const uint64_t * trans)175 acl_set_flow(struct acl_flow_data *flows, struct completion *cmplt,
176 uint32_t cmplt_size, const uint8_t **data, uint32_t *results,
177 uint32_t data_num, uint32_t categories, const uint64_t *trans)
178 {
179 flows->num_packets = 0;
180 flows->started = 0;
181 flows->trie = 0;
182 flows->last_cmplt = NULL;
183 flows->cmplt_array = cmplt;
184 flows->total_packets = data_num;
185 flows->categories = categories;
186 flows->cmplt_size = cmplt_size;
187 flows->data = data;
188 flows->results = results;
189 flows->trans = trans;
190 }
191
192 typedef void (*resolve_priority_t)
193 (uint64_t transition, int n, const struct rte_acl_ctx *ctx,
194 struct parms *parms, const struct rte_acl_match_results *p,
195 uint32_t categories);
196
197 /*
198 * Detect matches. If a match node transition is found, then this trie
199 * traversal is complete and fill the slot with the next trie
200 * to be processed.
201 */
202 static inline uint64_t
acl_match_check(uint64_t transition,int slot,const struct rte_acl_ctx * ctx,struct parms * parms,struct acl_flow_data * flows,resolve_priority_t resolve_priority)203 acl_match_check(uint64_t transition, int slot,
204 const struct rte_acl_ctx *ctx, struct parms *parms,
205 struct acl_flow_data *flows, resolve_priority_t resolve_priority)
206 {
207 const struct rte_acl_match_results *p;
208
209 p = (const struct rte_acl_match_results *)
210 (flows->trans + ctx->match_index);
211
212 if (transition & RTE_ACL_NODE_MATCH) {
213
214 /* Remove flags from index and decrement active traversals */
215 transition &= RTE_ACL_NODE_INDEX;
216 flows->started--;
217
218 /* Resolve priorities for this trie and running results */
219 if (flows->categories == 1)
220 resolve_single_priority(transition, slot, ctx,
221 parms, p);
222 else
223 resolve_priority(transition, slot, ctx, parms,
224 p, flows->categories);
225
226 /* Count down completed tries for this search request */
227 parms[slot].cmplt->count--;
228
229 /* Fill the slot with the next trie or idle trie */
230 transition = acl_start_next_trie(flows, parms, slot, ctx);
231 }
232
233 return transition;
234 }
235
236 #endif /* _ACL_RUN_H_ */
237