1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies, Ltd
3 */
4
5 #include <rte_log.h>
6 #include <rte_errno.h>
7 #include <rte_malloc.h>
8 #include <rte_regexdev.h>
9 #include <rte_regexdev_core.h>
10 #include <rte_regexdev_driver.h>
11 #include <sys/mman.h>
12
13 #include <mlx5_glue.h>
14 #include <mlx5_devx_cmds.h>
15 #include <mlx5_prm.h>
16 #include <mlx5_common_os.h>
17
18 #include "mlx5_regex.h"
19 #include "mlx5_regex_utils.h"
20 #include "mlx5_rxp.h"
21
22 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES
23 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH
24 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX
25 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS
26
27 #define MLX5_REGEX_RXP_ROF2_LINE_LEN 34
28
29 const uint64_t combined_rof_tag = 0xff52544424a52475;
30
31 /* Private Declarations */
32 static int
33 rxp_create_mkey(struct mlx5_regex_priv *priv, void *ptr, size_t size,
34 uint32_t access, struct mlx5_regex_mkey *mkey);
35 static inline void
36 rxp_destroy_mkey(struct mlx5_regex_mkey *mkey);
37
38 int
mlx5_regex_info_get(struct rte_regexdev * dev __rte_unused,struct rte_regexdev_info * info)39 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused,
40 struct rte_regexdev_info *info)
41 {
42 info->max_matches = MLX5_REGEX_MAX_MATCHES;
43 info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE;
44 info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP;
45 info->max_groups = MLX5_REGEX_MAX_GROUPS;
46 info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F |
47 RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F;
48 info->rule_flags = 0;
49 info->max_queue_pairs = UINT16_MAX;
50 info->max_segs = mlx5_regexdev_max_segs_get();
51 return 0;
52 }
53
54 static int
rxp_create_mkey(struct mlx5_regex_priv * priv,void * ptr,size_t size,uint32_t access,struct mlx5_regex_mkey * mkey)55 rxp_create_mkey(struct mlx5_regex_priv *priv, void *ptr, size_t size,
56 uint32_t access, struct mlx5_regex_mkey *mkey)
57 {
58 struct mlx5_devx_mkey_attr mkey_attr;
59
60 /* Register the memory. */
61 mkey->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx, ptr, size, access);
62 if (!mkey->umem) {
63 DRV_LOG(ERR, "Failed to register memory!");
64 return -ENODEV;
65 }
66 /* Create mkey */
67 mkey_attr = (struct mlx5_devx_mkey_attr) {
68 .addr = (uintptr_t)ptr,
69 .size = (uint32_t)size,
70 .umem_id = mlx5_os_get_umem_id(mkey->umem),
71 .pg_access = 1,
72 .umr_en = 0,
73 };
74 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
75 mkey_attr.pd = priv->cdev->pdn;
76 #endif
77 mkey->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &mkey_attr);
78 if (!mkey->mkey) {
79 DRV_LOG(ERR, "Failed to create direct mkey!");
80 return -ENODEV;
81 }
82 return 0;
83 }
84
85 static inline void
rxp_destroy_mkey(struct mlx5_regex_mkey * mkey)86 rxp_destroy_mkey(struct mlx5_regex_mkey *mkey)
87 {
88 if (mkey->mkey)
89 claim_zero(mlx5_devx_cmd_destroy(mkey->mkey));
90 if (mkey->umem)
91 claim_zero(mlx5_glue->devx_umem_dereg(mkey->umem));
92 }
93
94 int
mlx5_regex_get_rxp_vers(uint32_t regexp_version,uint32_t * target_rxp_vers)95 mlx5_regex_get_rxp_vers(uint32_t regexp_version, uint32_t *target_rxp_vers)
96 {
97 int ret = 0;
98 switch (regexp_version) {
99 case MLX5_RXP_BF2_IDENTIFIER:
100 *target_rxp_vers = MLX5_RXP_BF2_ROF_VERSION_STRING;
101 break;
102 case MLX5_RXP_BF3_IDENTIFIER:
103 *target_rxp_vers = MLX5_RXP_BF3_ROF_VERSION_STRING;
104 break;
105 default:
106 DRV_LOG(ERR, "Unsupported rxp version: %u", regexp_version);
107 ret = -EINVAL;
108 break;
109 }
110 return ret;
111 }
112
113 int
mlx5_regex_check_rof_version(uint32_t combined_rof_vers)114 mlx5_regex_check_rof_version(uint32_t combined_rof_vers)
115 {
116 int ret = 0;
117 /* Check if combined rof version is supported */
118 switch (combined_rof_vers) {
119 case 1:
120 break;
121 default:
122 DRV_LOG(ERR, "Unsupported combined rof version: %u",
123 combined_rof_vers);
124 ret = -EINVAL;
125 break;
126 }
127 return ret;
128 }
129
130 int
mlx5_regex_parse_rules_db(struct mlx5_regex_priv * priv,const char ** rules_db,uint32_t * rules_db_len)131 mlx5_regex_parse_rules_db(struct mlx5_regex_priv *priv,
132 const char **rules_db, uint32_t *rules_db_len)
133 {
134 int i = 0;
135 uint32_t j = 0;
136 int ret = 0;
137 bool combined_rof = true;
138 const char *rof_ptr = *rules_db;
139 uint32_t combined_rof_vers = 0;
140 uint32_t num_rof_blocks = 0;
141 uint32_t rxpc_vers = 0;
142 uint32_t target_rxp_vers = 0;
143 uint32_t byte_count = 0;
144 uint32_t rof_bytes_read = 0;
145 bool rof_binary_found = false;
146 struct mlx5_hca_attr *attr = &priv->cdev->config.hca_attr;
147
148 /* Need minimum of 8 bytes to process single or combined rof */
149 if (*rules_db_len < 8)
150 return -EINVAL;
151
152 for (i = 0; i < 8; i++) {
153 if ((char) *rof_ptr !=
154 (char)((combined_rof_tag >> (i * 8)) & 0xFF)) {
155 combined_rof = false;
156 break;
157 }
158 rof_ptr++;
159 }
160 rof_bytes_read += 8;
161
162 if (combined_rof == true) {
163 /* Need at least 24 bytes of header info: 16 byte combined */
164 /* rof header and 8 byte binary rof blob header. */
165 if (*rules_db_len < 24)
166 return -EINVAL;
167
168 /* Read the combined rof version and number of rof blocks */
169 for (i = 0; i < 4; i++) {
170 combined_rof_vers |= *rof_ptr << (i * 8);
171 rof_ptr++;
172 }
173
174 rof_bytes_read += 4;
175 ret = mlx5_regex_check_rof_version(combined_rof_vers);
176 if (ret < 0)
177 return ret;
178
179 for (i = 0; i < 4; i++) {
180 num_rof_blocks |= *rof_ptr << (i * 8);
181 rof_ptr++;
182 }
183 rof_bytes_read += 4;
184
185 if (num_rof_blocks == 0)
186 return -EINVAL;
187
188 /* Get the version of rxp we need the rof for */
189 ret = mlx5_regex_get_rxp_vers(attr->regexp_version, &target_rxp_vers);
190 if (ret < 0)
191 return ret;
192
193 /* Try to find the rof binary blob for this version of rxp */
194 for (j = 0; j < num_rof_blocks; j++) {
195 rxpc_vers = 0;
196 byte_count = 0;
197 for (i = 0; i < 4; i++) {
198 rxpc_vers |= (*rof_ptr & 0xFF) << (i * 8);
199 rof_ptr++;
200 }
201 for (i = 0; i < 4; i++) {
202 byte_count |= (*rof_ptr & 0xFF) << (i * 8);
203 rof_ptr++;
204 }
205 rof_bytes_read += 8;
206
207 if (rxpc_vers == target_rxp_vers) {
208 /* Found corresponding binary rof entry */
209 if (rof_bytes_read + byte_count <= (*rules_db_len))
210 rof_binary_found = true;
211 else
212 DRV_LOG(ERR, "Compatible rof file found - invalid length!");
213 break;
214 }
215 /* Move on to next rof blob */
216 if (rof_bytes_read + byte_count + 8 < (*rules_db_len)) {
217 rof_ptr += byte_count;
218 rof_bytes_read += byte_count;
219 } else {
220 /* Cannot parse any more of combined rof file */
221 break;
222 }
223 }
224 if (rof_binary_found == true) {
225 *rules_db = rof_ptr;
226 *rules_db_len = byte_count;
227 } else {
228 DRV_LOG(ERR, "Compatible rof file not found!");
229 return -EINVAL;
230 }
231 }
232 return 0;
233 }
234
235 int
mlx5_regex_rules_db_import(struct rte_regexdev * dev,const char * rule_db,uint32_t rule_db_len)236 mlx5_regex_rules_db_import(struct rte_regexdev *dev,
237 const char *rule_db, uint32_t rule_db_len)
238 {
239 struct mlx5_regex_priv *priv = dev->data->dev_private;
240 struct mlx5_regex_mkey mkey;
241 uint32_t id;
242 int ret;
243 void *ptr;
244
245 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) {
246 DRV_LOG(ERR, "RXP programming mode not set!");
247 return -1;
248 }
249 if (rule_db == NULL) {
250 DRV_LOG(ERR, "Database empty!");
251 return -ENODEV;
252 }
253 if (rule_db_len == 0)
254 return -EINVAL;
255
256 ret = mlx5_regex_parse_rules_db(priv, &rule_db, &rule_db_len);
257 if (ret < 0)
258 return ret;
259
260 /* copy rules - rules have to be 4KB aligned. */
261 ptr = rte_malloc("", rule_db_len, 1 << 12);
262 if (!ptr) {
263 DRV_LOG(ERR, "Failed to allocate rules file memory.");
264 return -ENOMEM;
265 }
266 rte_memcpy(ptr, rule_db, rule_db_len);
267 /* Register umem and create rof mkey. */
268 ret = rxp_create_mkey(priv, ptr, rule_db_len, /*access=*/7, &mkey);
269 if (ret < 0)
270 return ret;
271
272 for (id = 0; id < priv->nb_engines; id++) {
273 ret = mlx5_devx_regex_rules_program(priv->cdev->ctx, id,
274 mkey.mkey->id, rule_db_len, (uintptr_t)ptr);
275 if (ret < 0) {
276 DRV_LOG(ERR, "Failed to program rxp rules.");
277 ret = -ENODEV;
278 break;
279 }
280 ret = 0;
281 }
282 rxp_destroy_mkey(&mkey);
283 rte_free(ptr);
284 return ret;
285 }
286
287 int
mlx5_regex_configure(struct rte_regexdev * dev,const struct rte_regexdev_config * cfg)288 mlx5_regex_configure(struct rte_regexdev *dev,
289 const struct rte_regexdev_config *cfg)
290 {
291 struct mlx5_regex_priv *priv = dev->data->dev_private;
292 int ret;
293
294 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
295 return -1;
296 if (cfg->nb_max_matches != MLX5_REGEX_MAX_MATCHES) {
297 DRV_LOG(ERR, "nb_max_matches is not configurable.");
298 rte_errno = EINVAL;
299 return -rte_errno;
300 }
301 priv->nb_queues = cfg->nb_queue_pairs;
302 dev->data->dev_conf.nb_queue_pairs = priv->nb_queues;
303 priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) *
304 priv->nb_queues, 0);
305 if (!priv->qps) {
306 DRV_LOG(ERR, "can't allocate qps memory");
307 rte_errno = ENOMEM;
308 return -rte_errno;
309 }
310 priv->nb_max_matches = cfg->nb_max_matches;
311 if (cfg->rule_db != NULL) {
312 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db,
313 cfg->rule_db_len);
314 if (ret < 0) {
315 DRV_LOG(ERR, "Failed to program rxp rules.");
316 rte_errno = ENODEV;
317 goto configure_error;
318 }
319 } else
320 DRV_LOG(DEBUG, "Regex config without rules programming!");
321 return 0;
322 configure_error:
323 rte_free(priv->qps);
324 return -rte_errno;
325 }
326