1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #ifndef _ACL_RUN_H_ 6 #define _ACL_RUN_H_ 7 8 #include <rte_acl.h> 9 #include "acl.h" 10 11 #define MAX_SEARCHES_AVX16 16 12 #define MAX_SEARCHES_SSE8 8 13 #define MAX_SEARCHES_ALTIVEC8 8 14 #define MAX_SEARCHES_SSE4 4 15 #define MAX_SEARCHES_ALTIVEC4 4 16 #define MAX_SEARCHES_SCALAR 2 17 18 #define GET_NEXT_4BYTES(prm, idx) \ 19 (*((const int32_t *)((prm)[(idx)].data + *(prm)[idx].data_index++))) 20 21 22 #define RTE_ACL_NODE_INDEX ((uint32_t)~RTE_ACL_NODE_TYPE) 23 24 #define SCALAR_QRANGE_MULT 0x01010101 25 #define SCALAR_QRANGE_MASK 0x7f7f7f7f 26 #define SCALAR_QRANGE_MIN 0x80808080 27 28 /* 29 * Structure to manage N parallel trie traversals. 30 * The runtime trie traversal routines can process 8, 4, or 2 tries 31 * in parallel. Each packet may require multiple trie traversals (up to 4). 32 * This structure is used to fill the slots (0 to n-1) for parallel processing 33 * with the trie traversals needed for each packet. 34 */ 35 struct acl_flow_data { 36 uint32_t num_packets; 37 /* number of packets processed */ 38 uint32_t started; 39 /* number of trie traversals in progress */ 40 uint32_t trie; 41 /* current trie index (0 to N-1) */ 42 uint32_t cmplt_size; 43 /* maximum number of packets to process */ 44 uint32_t total_packets; 45 /* number of result categories per packet. */ 46 uint32_t categories; 47 const uint64_t *trans; 48 const uint8_t **data; 49 uint32_t *results; 50 struct completion *last_cmplt; 51 struct completion *cmplt_array; 52 }; 53 54 /* 55 * Structure to maintain running results for 56 * a single packet (up to 4 tries). 57 */ 58 struct completion { 59 uint32_t *results; /* running results. */ 60 int32_t priority[RTE_ACL_MAX_CATEGORIES]; /* running priorities. */ 61 uint32_t count; /* num of remaining tries */ 62 /* true for allocated struct */ 63 } __rte_aligned(XMM_SIZE); 64 65 /* 66 * One parms structure for each slot in the search engine. 67 */ 68 struct parms { 69 const uint8_t *data; 70 /* input data for this packet */ 71 const uint32_t *data_index; 72 /* data indirection for this trie */ 73 struct completion *cmplt; 74 /* completion data for this packet */ 75 }; 76 77 /* 78 * Define an global idle node for unused engine slots 79 */ 80 static const uint32_t idle[UINT8_MAX + 1]; 81 82 /* 83 * Allocate a completion structure to manage the tries for a packet. 84 */ 85 static inline struct completion * 86 alloc_completion(struct completion *p, uint32_t size, uint32_t tries, 87 uint32_t *results) 88 { 89 uint32_t n; 90 91 for (n = 0; n < size; n++) { 92 93 if (p[n].count == 0) { 94 95 /* mark as allocated and set number of tries. */ 96 p[n].count = tries; 97 p[n].results = results; 98 return &(p[n]); 99 } 100 } 101 102 /* should never get here */ 103 return NULL; 104 } 105 106 /* 107 * Resolve priority for a single result trie. 108 */ 109 static inline void 110 resolve_single_priority(uint64_t transition, int n, 111 const struct rte_acl_ctx *ctx, struct parms *parms, 112 const struct rte_acl_match_results *p) 113 { 114 if (parms[n].cmplt->count == ctx->num_tries || 115 parms[n].cmplt->priority[0] <= 116 p[transition].priority[0]) { 117 118 parms[n].cmplt->priority[0] = p[transition].priority[0]; 119 parms[n].cmplt->results[0] = p[transition].results[0]; 120 } 121 } 122 123 /* 124 * Routine to fill a slot in the parallel trie traversal array (parms) from 125 * the list of packets (flows). 126 */ 127 static inline uint64_t 128 acl_start_next_trie(struct acl_flow_data *flows, struct parms *parms, int n, 129 const struct rte_acl_ctx *ctx) 130 { 131 uint64_t transition; 132 133 /* if there are any more packets to process */ 134 if (flows->num_packets < flows->total_packets) { 135 parms[n].data = flows->data[flows->num_packets]; 136 parms[n].data_index = ctx->trie[flows->trie].data_index; 137 138 /* if this is the first trie for this packet */ 139 if (flows->trie == 0) { 140 flows->last_cmplt = alloc_completion(flows->cmplt_array, 141 flows->cmplt_size, ctx->num_tries, 142 flows->results + 143 flows->num_packets * flows->categories); 144 } 145 146 /* set completion parameters and starting index for this slot */ 147 parms[n].cmplt = flows->last_cmplt; 148 transition = 149 flows->trans[parms[n].data[*parms[n].data_index++] + 150 ctx->trie[flows->trie].root_index]; 151 152 /* 153 * if this is the last trie for this packet, 154 * then setup next packet. 155 */ 156 flows->trie++; 157 if (flows->trie >= ctx->num_tries) { 158 flows->trie = 0; 159 flows->num_packets++; 160 } 161 162 /* keep track of number of active trie traversals */ 163 flows->started++; 164 165 /* no more tries to process, set slot to an idle position */ 166 } else { 167 transition = ctx->idle; 168 parms[n].data = (const uint8_t *)idle; 169 parms[n].data_index = idle; 170 } 171 return transition; 172 } 173 174 static inline void 175 acl_set_flow(struct acl_flow_data *flows, struct completion *cmplt, 176 uint32_t cmplt_size, const uint8_t **data, uint32_t *results, 177 uint32_t data_num, uint32_t categories, const uint64_t *trans) 178 { 179 flows->num_packets = 0; 180 flows->started = 0; 181 flows->trie = 0; 182 flows->last_cmplt = NULL; 183 flows->cmplt_array = cmplt; 184 flows->total_packets = data_num; 185 flows->categories = categories; 186 flows->cmplt_size = cmplt_size; 187 flows->data = data; 188 flows->results = results; 189 flows->trans = trans; 190 } 191 192 typedef void (*resolve_priority_t) 193 (uint64_t transition, int n, const struct rte_acl_ctx *ctx, 194 struct parms *parms, const struct rte_acl_match_results *p, 195 uint32_t categories); 196 197 /* 198 * Detect matches. If a match node transition is found, then this trie 199 * traversal is complete and fill the slot with the next trie 200 * to be processed. 201 */ 202 static inline uint64_t 203 acl_match_check(uint64_t transition, int slot, 204 const struct rte_acl_ctx *ctx, struct parms *parms, 205 struct acl_flow_data *flows, resolve_priority_t resolve_priority) 206 { 207 const struct rte_acl_match_results *p; 208 209 p = (const struct rte_acl_match_results *) 210 (flows->trans + ctx->match_index); 211 212 if (transition & RTE_ACL_NODE_MATCH) { 213 214 /* Remove flags from index and decrement active traversals */ 215 transition &= RTE_ACL_NODE_INDEX; 216 flows->started--; 217 218 /* Resolve priorities for this trie and running results */ 219 if (flows->categories == 1) 220 resolve_single_priority(transition, slot, ctx, 221 parms, p); 222 else 223 resolve_priority(transition, slot, ctx, parms, 224 p, flows->categories); 225 226 /* Count down completed tries for this search request */ 227 parms[slot].cmplt->count--; 228 229 /* Fill the slot with the next trie or idle trie */ 230 transition = acl_start_next_trie(flows, parms, slot, ctx); 231 } 232 233 return transition; 234 } 235 236 #endif /* _ACL_RUN_H_ */ 237