1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <rte_log.h> 6 #include <rte_errno.h> 7 #include <rte_malloc.h> 8 #include <rte_regexdev.h> 9 #include <rte_regexdev_core.h> 10 #include <rte_regexdev_driver.h> 11 12 #include <mlx5_glue.h> 13 #include <mlx5_devx_cmds.h> 14 #include <mlx5_prm.h> 15 #include <mlx5_common_os.h> 16 17 #include "mlx5_regex.h" 18 #include "mlx5_regex_utils.h" 19 #include "mlx5_rxp_csrs.h" 20 #include "mlx5_rxp.h" 21 22 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES 23 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH 24 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX 25 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS 26 27 /* Private Declarations */ 28 static int 29 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value, 30 uint32_t address, uint32_t expected_value, 31 uint32_t expected_mask, uint32_t timeout_ms, uint8_t id); 32 static int 33 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use); 34 static int 35 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id); 36 static int 37 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id); 38 static int 39 program_rxp_rules(struct mlx5_regex_priv *priv, 40 struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id); 41 static int 42 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id); 43 static int 44 write_private_rules(struct mlx5_regex_priv *priv, 45 struct mlx5_rxp_ctl_rules_pgm *rules, 46 uint8_t id); 47 static int 48 write_shared_rules(struct mlx5_regex_priv *priv, 49 struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count, 50 uint8_t db_to_program); 51 static int 52 rxp_db_setup(struct mlx5_regex_priv *priv); 53 static void 54 rxp_dump_csrs(struct ibv_context *ctx, uint8_t id); 55 static int 56 rxp_write_rules_via_cp(struct ibv_context *ctx, 57 struct mlx5_rxp_rof_entry *rules, 58 int count, uint8_t id); 59 static int 60 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules, 61 int count, uint8_t id); 62 static int 63 rxp_start_engine(struct ibv_context *ctx, uint8_t id); 64 static int 65 rxp_stop_engine(struct ibv_context *ctx, uint8_t id); 66 67 static void __rte_unused 68 rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused) 69 { 70 uint32_t reg, i; 71 72 /* Main CSRs*/ 73 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { 74 if (mlx5_devx_regex_register_read(ctx, id, 75 (MLX5_RXP_CSR_WIDTH * i) + 76 MLX5_RXP_CSR_BASE_ADDRESS, 77 ®)) { 78 DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id); 79 return; 80 } 81 DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x", 82 id, i, reg); 83 } 84 /* RTRU CSRs*/ 85 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { 86 if (mlx5_devx_regex_register_read(ctx, id, 87 (MLX5_RXP_CSR_WIDTH * i) + 88 MLX5_RXP_RTRU_CSR_BASE_ADDRESS, 89 ®)) { 90 DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id); 91 return; 92 } 93 DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x", 94 id, i, reg); 95 } 96 /* STAT CSRs */ 97 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { 98 if (mlx5_devx_regex_register_read(ctx, id, 99 (MLX5_RXP_CSR_WIDTH * i) + 100 MLX5_RXP_STATS_CSR_BASE_ADDRESS, 101 ®)) { 102 DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id); 103 return; 104 } 105 DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x", 106 id, i, reg); 107 } 108 } 109 110 int 111 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused, 112 struct rte_regexdev_info *info) 113 { 114 info->max_matches = MLX5_REGEX_MAX_MATCHES; 115 info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE; 116 info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP; 117 info->max_groups = MLX5_REGEX_MAX_GROUPS; 118 info->max_queue_pairs = 1; 119 info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F; 120 info->rule_flags = 0; 121 info->max_queue_pairs = 10; 122 return 0; 123 } 124 125 /** 126 * Actual writing of RXP instructions to RXP via CSRs. 127 */ 128 static int 129 rxp_write_rules_via_cp(struct ibv_context *ctx, 130 struct mlx5_rxp_rof_entry *rules, 131 int count, uint8_t id) 132 { 133 int i, ret = 0; 134 uint32_t tmp; 135 136 for (i = 0; i < count; i++) { 137 tmp = (uint32_t)rules[i].value; 138 ret |= mlx5_devx_regex_register_write(ctx, id, 139 MLX5_RXP_RTRU_CSR_DATA_0, 140 tmp); 141 tmp = (uint32_t)(rules[i].value >> 32); 142 ret |= mlx5_devx_regex_register_write(ctx, id, 143 MLX5_RXP_RTRU_CSR_DATA_0 + 144 MLX5_RXP_CSR_WIDTH, tmp); 145 tmp = rules[i].addr; 146 ret |= mlx5_devx_regex_register_write(ctx, id, 147 MLX5_RXP_RTRU_CSR_ADDR, 148 tmp); 149 if (ret) { 150 DRV_LOG(ERR, "Failed to copy instructions to RXP."); 151 return -1; 152 } 153 } 154 DRV_LOG(DEBUG, "Written %d instructions", count); 155 return 0; 156 } 157 158 static int 159 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules, 160 int count, uint8_t id) 161 { 162 uint32_t val, fifo_depth; 163 int ret; 164 165 ret = rxp_write_rules_via_cp(ctx, rules, count, id); 166 if (ret < 0) { 167 DRV_LOG(ERR, "Failed to write rules via CSRs."); 168 return -1; 169 } 170 ret = mlx5_devx_regex_register_read(ctx, id, 171 MLX5_RXP_RTRU_CSR_CAPABILITY, 172 &fifo_depth); 173 if (ret) { 174 DRV_LOG(ERR, "CSR read failed!"); 175 return -1; 176 } 177 ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_FIFO_STAT, 178 count, ~0, 179 MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id); 180 if (ret < 0) { 181 DRV_LOG(ERR, "Rules not rx by RXP: credit: %d, depth: %d", val, 182 fifo_depth); 183 return ret; 184 } 185 DRV_LOG(DEBUG, "RTRU FIFO depth: 0x%x", fifo_depth); 186 DRV_LOG(DEBUG, "Rules flush took %d cycles.", ret); 187 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 188 &val); 189 if (ret) { 190 DRV_LOG(ERR, "CSR read failed!"); 191 return -1; 192 } 193 val |= MLX5_RXP_RTRU_CSR_CTRL_GO; 194 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 195 val); 196 ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_STATUS, 197 MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE, 198 MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE, 199 MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id); 200 if (ret < 0) { 201 DRV_LOG(ERR, "Rules update timeout: 0x%08X", val); 202 return ret; 203 } 204 DRV_LOG(DEBUG, "Rules update took %d cycles", ret); 205 if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 206 &val)) { 207 DRV_LOG(ERR, "CSR read failed!"); 208 return -1; 209 } 210 val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO); 211 if (mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 212 val)) { 213 DRV_LOG(ERR, "CSR write write failed!"); 214 return -1; 215 } 216 217 DRV_LOG(DEBUG, "RXP Flush rules finished."); 218 return 0; 219 } 220 221 static int 222 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value, 223 uint32_t address, uint32_t expected_value, 224 uint32_t expected_mask, uint32_t timeout_ms, uint8_t id) 225 { 226 unsigned int i; 227 int ret = 0; 228 229 ret = -EBUSY; 230 for (i = 0; i < timeout_ms; i++) { 231 if (mlx5_devx_regex_register_read(ctx, id, address, value)) 232 return -1; 233 if ((*value & expected_mask) == expected_value) { 234 ret = 0; 235 break; 236 } 237 rte_delay_us(1000); 238 } 239 return ret; 240 } 241 242 static int 243 rxp_start_engine(struct ibv_context *ctx, uint8_t id) 244 { 245 uint32_t ctrl; 246 int ret; 247 248 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 249 if (ret) 250 return ret; 251 ctrl |= MLX5_RXP_CSR_CTRL_GO; 252 ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C; 253 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 254 return ret; 255 } 256 257 static int 258 rxp_stop_engine(struct ibv_context *ctx, uint8_t id) 259 { 260 uint32_t ctrl; 261 int ret; 262 263 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 264 if (ret) 265 return ret; 266 ctrl &= ~MLX5_RXP_CSR_CTRL_GO; 267 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 268 return ret; 269 } 270 271 static int 272 rxp_init_rtru(struct ibv_context *ctx, uint8_t id, uint32_t init_bits) 273 { 274 uint32_t ctrl_value; 275 uint32_t poll_value; 276 uint32_t expected_value; 277 uint32_t expected_mask; 278 int ret = 0; 279 280 /* Read the rtru ctrl CSR. */ 281 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 282 &ctrl_value); 283 if (ret) 284 return -1; 285 /* Clear any previous init modes. */ 286 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK); 287 if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) { 288 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT); 289 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 290 ctrl_value); 291 } 292 /* Set the init_mode bits in the rtru ctrl CSR. */ 293 ctrl_value |= init_bits; 294 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 295 ctrl_value); 296 /* Need to sleep for a short period after pulsing the rtru init bit. */ 297 rte_delay_us(20000); 298 /* Poll the rtru status CSR until all the init done bits are set. */ 299 DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init"); 300 /* Set the init bit in the rtru ctrl CSR. */ 301 ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT; 302 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 303 ctrl_value); 304 /* Clear the init bit in the rtru ctrl CSR */ 305 ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT; 306 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 307 ctrl_value); 308 /* Check that the following bits are set in the RTRU_CSR. */ 309 if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) { 310 /* Must be incremental mode */ 311 expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE | 312 MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE; 313 } else { 314 expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE | 315 MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE | 316 MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE; 317 } 318 expected_mask = expected_value; 319 ret = rxp_poll_csr_for_value(ctx, &poll_value, 320 MLX5_RXP_RTRU_CSR_STATUS, 321 expected_value, expected_mask, 322 MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id); 323 if (ret) 324 return ret; 325 DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value); 326 /* Clear the init bit in the rtru ctrl CSR */ 327 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT); 328 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 329 ctrl_value); 330 return 0; 331 } 332 333 static int 334 rxp_parse_rof(const char *buf, uint32_t len, 335 struct mlx5_rxp_ctl_rules_pgm **rules) 336 { 337 static const char del[] = "\n\r"; 338 char *line; 339 char *tmp; 340 char *cur_pos; 341 uint32_t lines = 0; 342 uint32_t entries; 343 struct mlx5_rxp_rof_entry *curentry; 344 345 tmp = rte_malloc("", len, 0); 346 if (!tmp) 347 return -ENOMEM; 348 memcpy(tmp, buf, len); 349 line = strtok(tmp, del); 350 while (line) { 351 if (line[0] != '#' && line[0] != '\0') 352 lines++; 353 line = strtok(NULL, del); 354 } 355 *rules = rte_malloc("", lines * sizeof(*curentry) + sizeof(**rules), 0); 356 if (!(*rules)) { 357 rte_free(tmp); 358 return -ENOMEM; 359 } 360 memset(*rules, 0, lines * sizeof(curentry) + sizeof(**rules)); 361 curentry = (*rules)->rules; 362 (*rules)->hdr.cmd = MLX5_RXP_CTL_RULES_PGM; 363 entries = 0; 364 memcpy(tmp, buf, len); 365 line = strtok(tmp, del); 366 while (line) { 367 if (line[0] == '#' || line[0] == '\0') { 368 line = strtok(NULL, del); 369 continue; 370 } 371 curentry->type = strtoul(line, &cur_pos, 10); 372 if (cur_pos == line || cur_pos[0] != ',') 373 goto parse_error; 374 cur_pos++; 375 curentry->addr = strtoul(cur_pos, &cur_pos, 16); 376 if (cur_pos[0] != ',') 377 goto parse_error; 378 cur_pos++; 379 curentry->value = strtoull(cur_pos, &cur_pos, 16); 380 if (cur_pos[0] != '\0' && cur_pos[0] != '\n') 381 goto parse_error; 382 curentry++; 383 entries++; 384 if (entries > lines) 385 goto parse_error; 386 line = strtok(NULL, del); 387 } 388 (*rules)->count = entries; 389 (*rules)->hdr.len = entries * sizeof(*curentry) + sizeof(**rules); 390 rte_free(tmp); 391 return 0; 392 parse_error: 393 rte_free(tmp); 394 if (*rules) 395 rte_free(*rules); 396 return -EINVAL; 397 } 398 399 static int 400 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use) 401 { 402 int ret; 403 uint32_t umem_id; 404 405 ret = mlx5_devx_regex_database_stop(priv->ctx, id); 406 if (ret < 0) { 407 DRV_LOG(ERR, "stop engine failed!"); 408 return ret; 409 } 410 umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem); 411 ret = mlx5_devx_regex_database_program(priv->ctx, id, umem_id, 0); 412 if (ret < 0) { 413 DRV_LOG(ERR, "program db failed!"); 414 return ret; 415 } 416 return 0; 417 } 418 419 static int 420 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id) 421 { 422 mlx5_devx_regex_database_resume(priv->ctx, id); 423 return 0; 424 } 425 426 /* 427 * Assign db memory for RXP programming. 428 */ 429 static int 430 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id) 431 { 432 unsigned int i; 433 uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED; 434 uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED; 435 436 /* Check which database rxp_eng is currently located if any? */ 437 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); 438 i++) { 439 if (priv->db[i].db_assigned_to_eng_num == id) { 440 eng_assigned = i; 441 break; 442 } 443 } 444 /* 445 * If private mode then, we can keep the same db ptr as RXP will be 446 * programming EM itself if necessary, however need to see if 447 * programmed yet. 448 */ 449 if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) && 450 (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED)) 451 return eng_assigned; 452 /* Check for inactive db memory to use. */ 453 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); 454 i++) { 455 if (priv->db[i].active == true) 456 continue; /* Already in use, so skip db. */ 457 /* Set this db to active now as free to use. */ 458 priv->db[i].active = true; 459 /* Now unassign last db index in use by RXP Eng. */ 460 if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) { 461 priv->db[eng_assigned].active = false; 462 priv->db[eng_assigned].db_assigned_to_eng_num = 463 MLX5_RXP_DB_NOT_ASSIGNED; 464 465 /* Set all DB memory to 0's before setting up DB. */ 466 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE); 467 } 468 /* Now reassign new db index with RXP Engine. */ 469 priv->db[i].db_assigned_to_eng_num = id; 470 db_free = i; 471 break; 472 } 473 if (db_free == MLX5_RXP_DB_NOT_ASSIGNED) 474 return -1; 475 return db_free; 476 } 477 478 /* 479 * Program RXP instruction db to RXP engine/s. 480 */ 481 static int 482 program_rxp_rules(struct mlx5_regex_priv *priv, 483 struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id) 484 { 485 int ret, db_free; 486 uint32_t rule_cnt; 487 488 rule_cnt = rules->count; 489 db_free = mlnx_update_database(priv, id); 490 if (db_free < 0) { 491 DRV_LOG(ERR, "Failed to setup db memory!"); 492 return db_free; 493 } 494 if (priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) { 495 /* Register early to ensure RXP writes to EM use valid addr. */ 496 ret = mlnx_set_database(priv, id, db_free); 497 if (ret < 0) { 498 DRV_LOG(ERR, "Failed to register db memory!"); 499 return ret; 500 } 501 } 502 ret = write_private_rules(priv, rules, id); 503 if (ret < 0) { 504 DRV_LOG(ERR, "Failed to write rules!"); 505 return ret; 506 } 507 if (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) { 508 /* Write external rules directly to EM. */ 509 rules->count = rule_cnt; 510 /* Now write external instructions to EM. */ 511 ret = write_shared_rules(priv, rules, rules->hdr.len, db_free); 512 if (ret < 0) { 513 DRV_LOG(ERR, "Failed to write EM rules!"); 514 return ret; 515 } 516 ret = mlnx_set_database(priv, id, db_free); 517 if (ret < 0) { 518 DRV_LOG(ERR, "Failed to register db memory!"); 519 return ret; 520 } 521 } 522 ret = mlnx_resume_database(priv, id); 523 if (ret < 0) { 524 DRV_LOG(ERR, "Failed to resume engine!"); 525 return ret; 526 } 527 DRV_LOG(DEBUG, "Programmed RXP Engine %d\n", id); 528 rules->count = rule_cnt; 529 return 0; 530 } 531 532 static int 533 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id) 534 { 535 uint32_t ctrl; 536 uint32_t reg; 537 struct ibv_context *ctx = priv->ctx; 538 int ret; 539 540 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 541 if (ret) 542 return ret; 543 if (ctrl & MLX5_RXP_CSR_CTRL_INIT) { 544 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT; 545 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, 546 ctrl); 547 if (ret) 548 return ret; 549 } 550 ctrl |= MLX5_RXP_CSR_CTRL_INIT; 551 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 552 if (ret) 553 return ret; 554 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT; 555 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 556 rte_delay_us(20000); 557 ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS, 558 MLX5_RXP_CSR_STATUS_INIT_DONE, 559 MLX5_RXP_CSR_STATUS_INIT_DONE, 560 MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id); 561 if (ret) 562 return ret; 563 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 564 if (ret) 565 return ret; 566 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT; 567 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, 568 ctrl); 569 if (ret) 570 return ret; 571 ret = rxp_init_rtru(ctx, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2); 572 if (ret) 573 return ret; 574 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5, 575 ®); 576 if (ret) 577 return ret; 578 DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16, 579 reg & 0xffff); 580 if ((reg >> 16) >= priv->nb_max_matches) 581 ret = mlx5_devx_regex_register_write(ctx, id, 582 MLX5_RXP_CSR_MAX_MATCH, 583 priv->nb_max_matches); 584 else 585 ret = mlx5_devx_regex_register_write(ctx, id, 586 MLX5_RXP_CSR_MAX_MATCH, 587 (reg >> 16)); 588 ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX, 589 (reg & 0xFFFF)); 590 ret |= mlx5_devx_regex_register_write(ctx, id, 591 MLX5_RXP_CSR_MAX_LATENCY, 0); 592 ret |= mlx5_devx_regex_register_write(ctx, id, 593 MLX5_RXP_CSR_MAX_PRI_THREAD, 0); 594 return ret; 595 } 596 597 static int 598 write_private_rules(struct mlx5_regex_priv *priv, 599 struct mlx5_rxp_ctl_rules_pgm *rules, 600 uint8_t id) 601 { 602 unsigned int pending; 603 uint32_t block, reg, val, rule_cnt, rule_offset, rtru_max_num_entries; 604 int ret = 1; 605 606 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) 607 return -EINVAL; 608 if (rules->hdr.len == 0 || rules->hdr.cmd < MLX5_RXP_CTL_RULES_PGM || 609 rules->hdr.cmd > MLX5_RXP_CTL_RULES_PGM_INCR) 610 return -EINVAL; 611 /* For a non-incremental rules program, re-init the RXP. */ 612 if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM) { 613 ret = rxp_init_eng(priv, id); 614 if (ret < 0) 615 return ret; 616 } else if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM_INCR) { 617 /* Flush RXP L1 and L2 cache by using MODE_L1_L2. */ 618 ret = rxp_init_rtru(priv->ctx, id, 619 MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2); 620 if (ret < 0) 621 return ret; 622 } 623 if (rules->count == 0) 624 return -EINVAL; 625 /* Confirm the RXP is initialised. */ 626 if (mlx5_devx_regex_register_read(priv->ctx, id, 627 MLX5_RXP_CSR_STATUS, &val)) { 628 DRV_LOG(ERR, "Failed to read from RXP!"); 629 return -ENODEV; 630 } 631 if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) { 632 DRV_LOG(ERR, "RXP not initialised..."); 633 return -EBUSY; 634 } 635 /* Get the RTRU maximum number of entries allowed. */ 636 if (mlx5_devx_regex_register_read(priv->ctx, id, 637 MLX5_RXP_RTRU_CSR_CAPABILITY, &rtru_max_num_entries)) { 638 DRV_LOG(ERR, "Failed to read RTRU capability!"); 639 return -ENODEV; 640 } 641 rtru_max_num_entries = (rtru_max_num_entries & 0x00FF); 642 rule_cnt = 0; 643 pending = 0; 644 while (rules->count > 0) { 645 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_INST) || 646 (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_IM) || 647 (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM)) { 648 if ((rules->rules[rule_cnt].type == 649 MLX5_RXP_ROF_ENTRY_EM) && 650 (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) { 651 /* Skip EM rules programming. */ 652 if (pending > 0) { 653 /* Flush any rules that are pending. */ 654 rule_offset = (rule_cnt - pending); 655 ret = rxp_flush_rules(priv->ctx, 656 &rules->rules[rule_offset], 657 pending, id); 658 if (ret < 0) { 659 DRV_LOG(ERR, "Flushing rules."); 660 return -ENODEV; 661 } 662 pending = 0; 663 } 664 rule_cnt++; 665 } else { 666 pending++; 667 rule_cnt++; 668 /* 669 * If parsing the last rule, or if reached the 670 * maximum number of rules for this batch, then 671 * flush the rules batch to the RXP. 672 */ 673 if ((rules->count == 1) || 674 (pending == rtru_max_num_entries)) { 675 rule_offset = (rule_cnt - pending); 676 ret = rxp_flush_rules(priv->ctx, 677 &rules->rules[rule_offset], 678 pending, id); 679 if (ret < 0) { 680 DRV_LOG(ERR, "Flushing rules."); 681 return -ENODEV; 682 } 683 pending = 0; 684 } 685 } 686 } else if ((rules->rules[rule_cnt].type == 687 MLX5_RXP_ROF_ENTRY_EQ) || 688 (rules->rules[rule_cnt].type == 689 MLX5_RXP_ROF_ENTRY_GTE) || 690 (rules->rules[rule_cnt].type == 691 MLX5_RXP_ROF_ENTRY_LTE) || 692 (rules->rules[rule_cnt].type == 693 MLX5_RXP_ROF_ENTRY_CHECKSUM) || 694 (rules->rules[rule_cnt].type == 695 MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM)) { 696 if (pending) { 697 /* Flush rules before checking reg values. */ 698 rule_offset = (rule_cnt - pending); 699 ret = rxp_flush_rules(priv->ctx, 700 &rules->rules[rule_offset], 701 pending, id); 702 if (ret < 0) { 703 DRV_LOG(ERR, "Failed to flush rules."); 704 return -ENODEV; 705 } 706 } 707 block = (rules->rules[rule_cnt].addr >> 16) & 0xFFFF; 708 if (block == 0) 709 reg = MLX5_RXP_CSR_BASE_ADDRESS; 710 else if (block == 1) 711 reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS; 712 else { 713 DRV_LOG(ERR, "Invalid ROF register 0x%08X!", 714 rules->rules[rule_cnt].addr); 715 return -EINVAL; 716 } 717 reg += (rules->rules[rule_cnt].addr & 0xFFFF) * 718 MLX5_RXP_CSR_WIDTH; 719 ret = mlx5_devx_regex_register_read(priv->ctx, id, 720 reg, &val); 721 if (ret) { 722 DRV_LOG(ERR, "RXP CSR read failed!"); 723 return ret; 724 } 725 if ((priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) && 726 ((rules->rules[rule_cnt].type == 727 MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM) && 728 (val != rules->rules[rule_cnt].value))) { 729 DRV_LOG(ERR, "Unexpected value for register:"); 730 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32, 731 rules->rules[rule_cnt].addr, val); 732 DRV_LOG(ERR, "expected %" PRIx64 ".", 733 rules->rules[rule_cnt].value); 734 return -EINVAL; 735 } else if ((priv->prog_mode == 736 MLX5_RXP_PRIVATE_PROG_MODE) && 737 (rules->rules[rule_cnt].type == 738 MLX5_RXP_ROF_ENTRY_CHECKSUM) && 739 (val != rules->rules[rule_cnt].value)) { 740 DRV_LOG(ERR, "Unexpected value for register:"); 741 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32, 742 rules->rules[rule_cnt].addr, val); 743 DRV_LOG(ERR, "expected %" PRIx64 ".", 744 rules->rules[rule_cnt].value); 745 return -EINVAL; 746 } else if ((rules->rules[rule_cnt].type == 747 MLX5_RXP_ROF_ENTRY_EQ) && 748 (val != rules->rules[rule_cnt].value)) { 749 DRV_LOG(ERR, "Unexpected value for register:"); 750 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32, 751 rules->rules[rule_cnt].addr, val); 752 DRV_LOG(ERR, "expected %" PRIx64 ".", 753 rules->rules[rule_cnt].value); 754 return -EINVAL; 755 } else if ((rules->rules[rule_cnt].type == 756 MLX5_RXP_ROF_ENTRY_GTE) && 757 (val < rules->rules[rule_cnt].value)) { 758 DRV_LOG(ERR, "Unexpected value reg 0x%08X,", 759 rules->rules[rule_cnt].addr); 760 DRV_LOG(ERR, "got %X, expected >= %" PRIx64 ".", 761 val, rules->rules[rule_cnt].value); 762 return -EINVAL; 763 } else if ((rules->rules[rule_cnt].type == 764 MLX5_RXP_ROF_ENTRY_LTE) && 765 (val > rules->rules[rule_cnt].value)) { 766 DRV_LOG(ERR, "Unexpected value reg 0x%08X,", 767 rules->rules[rule_cnt].addr); 768 DRV_LOG(ERR, "got %08X expected <= %" PRIx64, 769 val, rules->rules[rule_cnt].value); 770 return -EINVAL; 771 } 772 rule_cnt++; 773 pending = 0; 774 } else { 775 DRV_LOG(ERR, "Error: Invalid rule type %d!", 776 rules->rules[rule_cnt].type); 777 return -EINVAL; 778 } 779 rules->count--; 780 } 781 return ret; 782 } 783 784 /* 785 * Shared memory programming mode, here all external db instructions are written 786 * to EM via the host. 787 */ 788 static int 789 write_shared_rules(struct mlx5_regex_priv *priv, 790 struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count, 791 uint8_t db_to_program) 792 { 793 uint32_t rule_cnt, rof_rule_addr; 794 uint64_t tmp_write_swap[4]; 795 796 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) 797 return -EINVAL; 798 if ((rules->count == 0) || (count == 0)) 799 return -EINVAL; 800 rule_cnt = 0; 801 /* 802 * Note the following section of code carries out a 32byte swap of 803 * instruction to coincide with HW 32byte swap. This may need removed 804 * in new variants of this programming function! 805 */ 806 while (rule_cnt < rules->count) { 807 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM) && 808 (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) { 809 /* 810 * Note there are always blocks of 8 instructions for 811 * 7's written sequentially. However there is no 812 * guarantee that all blocks are sequential! 813 */ 814 if (count >= (rule_cnt + MLX5_RXP_INST_BLOCK_SIZE)) { 815 /* 816 * Ensure memory write not exceeding boundary 817 * Check essential to ensure 0x10000 offset 818 * accounted for! 819 */ 820 if ((uint8_t *)((uint8_t *) 821 priv->db[db_to_program].ptr + 822 ((rules->rules[rule_cnt + 7].addr << 823 MLX5_RXP_INST_OFFSET))) >= 824 ((uint8_t *)((uint8_t *) 825 priv->db[db_to_program].ptr + 826 MLX5_MAX_DB_SIZE))) { 827 DRV_LOG(ERR, "DB exceeded memory!"); 828 return -ENODEV; 829 } 830 /* 831 * Rule address Offset to align with RXP 832 * external instruction offset. 833 */ 834 rof_rule_addr = (rules->rules[rule_cnt].addr << 835 MLX5_RXP_INST_OFFSET); 836 /* 32 byte instruction swap (sw work around)! */ 837 tmp_write_swap[0] = le64toh( 838 rules->rules[(rule_cnt + 4)].value); 839 tmp_write_swap[1] = le64toh( 840 rules->rules[(rule_cnt + 5)].value); 841 tmp_write_swap[2] = le64toh( 842 rules->rules[(rule_cnt + 6)].value); 843 tmp_write_swap[3] = le64toh( 844 rules->rules[(rule_cnt + 7)].value); 845 /* Write only 4 of the 8 instructions. */ 846 memcpy((uint8_t *)((uint8_t *) 847 priv->db[db_to_program].ptr + 848 rof_rule_addr), &tmp_write_swap, 849 (sizeof(uint64_t) * 4)); 850 /* Write 1st 4 rules of block after last 4. */ 851 rof_rule_addr = (rules->rules[ 852 (rule_cnt + 4)].addr << 853 MLX5_RXP_INST_OFFSET); 854 tmp_write_swap[0] = le64toh( 855 rules->rules[(rule_cnt + 0)].value); 856 tmp_write_swap[1] = le64toh( 857 rules->rules[(rule_cnt + 1)].value); 858 tmp_write_swap[2] = le64toh( 859 rules->rules[(rule_cnt + 2)].value); 860 tmp_write_swap[3] = le64toh( 861 rules->rules[(rule_cnt + 3)].value); 862 memcpy((uint8_t *)((uint8_t *) 863 priv->db[db_to_program].ptr + 864 rof_rule_addr), &tmp_write_swap, 865 (sizeof(uint64_t) * 4)); 866 } else 867 return -1; 868 /* Fast forward as already handled block of 8. */ 869 rule_cnt += MLX5_RXP_INST_BLOCK_SIZE; 870 } else 871 rule_cnt++; /* Must be something other than EM rule. */ 872 } 873 return 0; 874 } 875 876 static int 877 rxp_db_setup(struct mlx5_regex_priv *priv) 878 { 879 int ret; 880 uint8_t i; 881 882 /* Setup database memories for both RXP engines + reprogram memory. */ 883 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) { 884 priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 0); 885 if (!priv->db[i].ptr) { 886 DRV_LOG(ERR, "Failed to alloc db memory!"); 887 ret = ENODEV; 888 goto tidyup_error; 889 } 890 /* Register the memory. */ 891 priv->db[i].umem.umem = mlx5_glue->devx_umem_reg(priv->ctx, 892 priv->db[i].ptr, 893 MLX5_MAX_DB_SIZE, 7); 894 if (!priv->db[i].umem.umem) { 895 DRV_LOG(ERR, "Failed to register memory!"); 896 ret = ENODEV; 897 goto tidyup_error; 898 } 899 /* Ensure set all DB memory to 0's before setting up DB. */ 900 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE); 901 /* No data currently in database. */ 902 priv->db[i].len = 0; 903 priv->db[i].active = false; 904 priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED; 905 } 906 return 0; 907 tidyup_error: 908 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) { 909 if (priv->db[i].ptr) 910 rte_free(priv->db[i].ptr); 911 if (priv->db[i].umem.umem) 912 mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem); 913 } 914 return -ret; 915 } 916 917 int 918 mlx5_regex_rules_db_import(struct rte_regexdev *dev, 919 const char *rule_db, uint32_t rule_db_len) 920 { 921 struct mlx5_regex_priv *priv = dev->data->dev_private; 922 struct mlx5_rxp_ctl_rules_pgm *rules = NULL; 923 uint8_t id; 924 int ret; 925 926 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) { 927 DRV_LOG(ERR, "RXP programming mode not set!"); 928 return -1; 929 } 930 if (rule_db == NULL) { 931 DRV_LOG(ERR, "Database empty!"); 932 return -ENODEV; 933 } 934 if (rule_db_len == 0) 935 return -EINVAL; 936 ret = rxp_parse_rof(rule_db, rule_db_len, &rules); 937 if (ret) { 938 DRV_LOG(ERR, "Can't parse ROF file."); 939 return ret; 940 } 941 /* Need to ensure RXP not busy before stop! */ 942 for (id = 0; id < priv->nb_engines; id++) { 943 ret = rxp_stop_engine(priv->ctx, id); 944 if (ret) { 945 DRV_LOG(ERR, "Can't stop engine."); 946 ret = -ENODEV; 947 goto tidyup_error; 948 } 949 ret = program_rxp_rules(priv, rules, id); 950 if (ret < 0) { 951 DRV_LOG(ERR, "Failed to program rxp rules."); 952 ret = -ENODEV; 953 goto tidyup_error; 954 } 955 ret = rxp_start_engine(priv->ctx, id); 956 if (ret) { 957 DRV_LOG(ERR, "Can't start engine."); 958 ret = -ENODEV; 959 goto tidyup_error; 960 } 961 } 962 rte_free(rules); 963 return 0; 964 tidyup_error: 965 rte_free(rules); 966 return ret; 967 } 968 969 int 970 mlx5_regex_configure(struct rte_regexdev *dev, 971 const struct rte_regexdev_config *cfg) 972 { 973 struct mlx5_regex_priv *priv = dev->data->dev_private; 974 int ret; 975 976 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) 977 return -1; 978 priv->nb_queues = cfg->nb_queue_pairs; 979 dev->data->dev_conf.nb_queue_pairs = priv->nb_queues; 980 priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) * 981 priv->nb_queues, 0); 982 if (!priv->nb_queues) { 983 DRV_LOG(ERR, "can't allocate qps memory"); 984 rte_errno = ENOMEM; 985 return -rte_errno; 986 } 987 priv->nb_max_matches = cfg->nb_max_matches; 988 /* Setup rxp db memories. */ 989 if (rxp_db_setup(priv)) { 990 DRV_LOG(ERR, "Failed to setup RXP db memory"); 991 rte_errno = ENOMEM; 992 return -rte_errno; 993 } 994 if (cfg->rule_db != NULL) { 995 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db, 996 cfg->rule_db_len); 997 if (ret < 0) { 998 DRV_LOG(ERR, "Failed to program rxp rules."); 999 rte_errno = ENODEV; 1000 goto configure_error; 1001 } 1002 } else 1003 DRV_LOG(DEBUG, "Regex config without rules programming!"); 1004 return 0; 1005 configure_error: 1006 if (priv->qps) 1007 rte_free(priv->qps); 1008 return -rte_errno; 1009 } 1010