1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <rte_log.h> 6 #include <rte_errno.h> 7 #include <rte_malloc.h> 8 #include <rte_regexdev.h> 9 #include <rte_regexdev_core.h> 10 #include <rte_regexdev_driver.h> 11 #include <sys/mman.h> 12 13 #include <mlx5_glue.h> 14 #include <mlx5_devx_cmds.h> 15 #include <mlx5_prm.h> 16 #include <mlx5_common_os.h> 17 18 #include "mlx5_regex.h" 19 #include "mlx5_regex_utils.h" 20 #include "mlx5_rxp.h" 21 22 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES 23 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH 24 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX 25 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS 26 27 #define MLX5_REGEX_RXP_ROF2_LINE_LEN 34 28 29 const uint64_t combined_rof_tag = 0xff52544424a52475; 30 31 /* Private Declarations */ 32 static int 33 rxp_create_mkey(struct mlx5_regex_priv *priv, void *ptr, size_t size, 34 uint32_t access, struct mlx5_regex_mkey *mkey); 35 static inline void 36 rxp_destroy_mkey(struct mlx5_regex_mkey *mkey); 37 38 int 39 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused, 40 struct rte_regexdev_info *info) 41 { 42 info->max_matches = MLX5_REGEX_MAX_MATCHES; 43 info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE; 44 info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP; 45 info->max_groups = MLX5_REGEX_MAX_GROUPS; 46 info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F | 47 RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F; 48 info->rule_flags = 0; 49 info->max_queue_pairs = UINT16_MAX; 50 info->max_segs = mlx5_regexdev_max_segs_get(); 51 return 0; 52 } 53 54 static int 55 rxp_create_mkey(struct mlx5_regex_priv *priv, void *ptr, size_t size, 56 uint32_t access, struct mlx5_regex_mkey *mkey) 57 { 58 struct mlx5_devx_mkey_attr mkey_attr; 59 60 /* Register the memory. */ 61 mkey->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx, ptr, size, access); 62 if (!mkey->umem) { 63 DRV_LOG(ERR, "Failed to register memory!"); 64 return -ENODEV; 65 } 66 /* Create mkey */ 67 mkey_attr = (struct mlx5_devx_mkey_attr) { 68 .addr = (uintptr_t)ptr, 69 .size = (uint32_t)size, 70 .umem_id = mlx5_os_get_umem_id(mkey->umem), 71 .pg_access = 1, 72 .umr_en = 0, 73 }; 74 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 75 mkey_attr.pd = priv->cdev->pdn; 76 #endif 77 mkey->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &mkey_attr); 78 if (!mkey->mkey) { 79 DRV_LOG(ERR, "Failed to create direct mkey!"); 80 return -ENODEV; 81 } 82 return 0; 83 } 84 85 static inline void 86 rxp_destroy_mkey(struct mlx5_regex_mkey *mkey) 87 { 88 if (mkey->mkey) 89 claim_zero(mlx5_devx_cmd_destroy(mkey->mkey)); 90 if (mkey->umem) 91 claim_zero(mlx5_glue->devx_umem_dereg(mkey->umem)); 92 } 93 94 int 95 mlx5_regex_get_rxp_vers(uint32_t regexp_version, uint32_t *target_rxp_vers) 96 { 97 int ret = 0; 98 switch (regexp_version) { 99 case MLX5_RXP_BF2_IDENTIFIER: 100 *target_rxp_vers = MLX5_RXP_BF2_ROF_VERSION_STRING; 101 break; 102 case MLX5_RXP_BF3_IDENTIFIER: 103 *target_rxp_vers = MLX5_RXP_BF3_ROF_VERSION_STRING; 104 break; 105 default: 106 DRV_LOG(ERR, "Unsupported rxp version: %u", regexp_version); 107 ret = -EINVAL; 108 break; 109 } 110 return ret; 111 } 112 113 int 114 mlx5_regex_check_rof_version(uint32_t combined_rof_vers) 115 { 116 int ret = 0; 117 /* Check if combined rof version is supported */ 118 switch (combined_rof_vers) { 119 case 1: 120 break; 121 default: 122 DRV_LOG(ERR, "Unsupported combined rof version: %u", 123 combined_rof_vers); 124 ret = -EINVAL; 125 break; 126 } 127 return ret; 128 } 129 130 int 131 mlx5_regex_parse_rules_db(struct mlx5_regex_priv *priv, 132 const char **rules_db, uint32_t *rules_db_len) 133 { 134 int i = 0; 135 uint32_t j = 0; 136 int ret = 0; 137 bool combined_rof = true; 138 const char *rof_ptr = *rules_db; 139 uint32_t combined_rof_vers = 0; 140 uint32_t num_rof_blocks = 0; 141 uint32_t rxpc_vers = 0; 142 uint32_t target_rxp_vers = 0; 143 uint32_t byte_count = 0; 144 uint32_t rof_bytes_read = 0; 145 bool rof_binary_found = false; 146 struct mlx5_hca_attr *attr = &priv->cdev->config.hca_attr; 147 148 /* Need minimum of 8 bytes to process single or combined rof */ 149 if (*rules_db_len < 8) 150 return -EINVAL; 151 152 for (i = 0; i < 8; i++) { 153 if ((char) *rof_ptr != 154 (char)((combined_rof_tag >> (i * 8)) & 0xFF)) { 155 combined_rof = false; 156 break; 157 } 158 rof_ptr++; 159 } 160 rof_bytes_read += 8; 161 162 if (combined_rof == true) { 163 /* Need at least 24 bytes of header info: 16 byte combined */ 164 /* rof header and 8 byte binary rof blob header. */ 165 if (*rules_db_len < 24) 166 return -EINVAL; 167 168 /* Read the combined rof version and number of rof blocks */ 169 for (i = 0; i < 4; i++) { 170 combined_rof_vers |= *rof_ptr << (i * 8); 171 rof_ptr++; 172 } 173 174 rof_bytes_read += 4; 175 ret = mlx5_regex_check_rof_version(combined_rof_vers); 176 if (ret < 0) 177 return ret; 178 179 for (i = 0; i < 4; i++) { 180 num_rof_blocks |= *rof_ptr << (i * 8); 181 rof_ptr++; 182 } 183 rof_bytes_read += 4; 184 185 if (num_rof_blocks == 0) 186 return -EINVAL; 187 188 /* Get the version of rxp we need the rof for */ 189 ret = mlx5_regex_get_rxp_vers(attr->regexp_version, &target_rxp_vers); 190 if (ret < 0) 191 return ret; 192 193 /* Try to find the rof binary blob for this version of rxp */ 194 for (j = 0; j < num_rof_blocks; j++) { 195 rxpc_vers = 0; 196 byte_count = 0; 197 for (i = 0; i < 4; i++) { 198 rxpc_vers |= (*rof_ptr & 0xFF) << (i * 8); 199 rof_ptr++; 200 } 201 for (i = 0; i < 4; i++) { 202 byte_count |= (*rof_ptr & 0xFF) << (i * 8); 203 rof_ptr++; 204 } 205 rof_bytes_read += 8; 206 207 if (rxpc_vers == target_rxp_vers) { 208 /* Found corresponding binary rof entry */ 209 if (rof_bytes_read + byte_count <= (*rules_db_len)) 210 rof_binary_found = true; 211 else 212 DRV_LOG(ERR, "Compatible rof file found - invalid length!"); 213 break; 214 } 215 /* Move on to next rof blob */ 216 if (rof_bytes_read + byte_count + 8 < (*rules_db_len)) { 217 rof_ptr += byte_count; 218 rof_bytes_read += byte_count; 219 } else { 220 /* Cannot parse any more of combined rof file */ 221 break; 222 } 223 } 224 if (rof_binary_found == true) { 225 *rules_db = rof_ptr; 226 *rules_db_len = byte_count; 227 } else { 228 DRV_LOG(ERR, "Compatible rof file not found!"); 229 return -EINVAL; 230 } 231 } 232 return 0; 233 } 234 235 int 236 mlx5_regex_rules_db_import(struct rte_regexdev *dev, 237 const char *rule_db, uint32_t rule_db_len) 238 { 239 struct mlx5_regex_priv *priv = dev->data->dev_private; 240 struct mlx5_regex_mkey mkey; 241 uint32_t id; 242 int ret; 243 void *ptr; 244 245 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) { 246 DRV_LOG(ERR, "RXP programming mode not set!"); 247 return -1; 248 } 249 if (rule_db == NULL) { 250 DRV_LOG(ERR, "Database empty!"); 251 return -ENODEV; 252 } 253 if (rule_db_len == 0) 254 return -EINVAL; 255 256 ret = mlx5_regex_parse_rules_db(priv, &rule_db, &rule_db_len); 257 if (ret < 0) 258 return ret; 259 260 /* copy rules - rules have to be 4KB aligned. */ 261 ptr = rte_malloc("", rule_db_len, 1 << 12); 262 if (!ptr) { 263 DRV_LOG(ERR, "Failed to allocate rules file memory."); 264 return -ENOMEM; 265 } 266 rte_memcpy(ptr, rule_db, rule_db_len); 267 /* Register umem and create rof mkey. */ 268 ret = rxp_create_mkey(priv, ptr, rule_db_len, /*access=*/7, &mkey); 269 if (ret < 0) 270 return ret; 271 272 for (id = 0; id < priv->nb_engines; id++) { 273 ret = mlx5_devx_regex_rules_program(priv->cdev->ctx, id, 274 mkey.mkey->id, rule_db_len, (uintptr_t)ptr); 275 if (ret < 0) { 276 DRV_LOG(ERR, "Failed to program rxp rules."); 277 ret = -ENODEV; 278 break; 279 } 280 ret = 0; 281 } 282 rxp_destroy_mkey(&mkey); 283 rte_free(ptr); 284 return ret; 285 } 286 287 int 288 mlx5_regex_configure(struct rte_regexdev *dev, 289 const struct rte_regexdev_config *cfg) 290 { 291 struct mlx5_regex_priv *priv = dev->data->dev_private; 292 int ret; 293 294 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) 295 return -1; 296 if (cfg->nb_max_matches != MLX5_REGEX_MAX_MATCHES) { 297 DRV_LOG(ERR, "nb_max_matches is not configurable."); 298 rte_errno = EINVAL; 299 return -rte_errno; 300 } 301 priv->nb_queues = cfg->nb_queue_pairs; 302 dev->data->dev_conf.nb_queue_pairs = priv->nb_queues; 303 priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) * 304 priv->nb_queues, 0); 305 if (!priv->qps) { 306 DRV_LOG(ERR, "can't allocate qps memory"); 307 rte_errno = ENOMEM; 308 return -rte_errno; 309 } 310 priv->nb_max_matches = cfg->nb_max_matches; 311 if (cfg->rule_db != NULL) { 312 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db, 313 cfg->rule_db_len); 314 if (ret < 0) { 315 DRV_LOG(ERR, "Failed to program rxp rules."); 316 rte_errno = ENODEV; 317 goto configure_error; 318 } 319 } else 320 DRV_LOG(DEBUG, "Regex config without rules programming!"); 321 return 0; 322 configure_error: 323 rte_free(priv->qps); 324 return -rte_errno; 325 } 326