1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <rte_log.h> 6 #include <rte_errno.h> 7 #include <rte_malloc.h> 8 #include <rte_regexdev.h> 9 #include <rte_regexdev_core.h> 10 #include <rte_regexdev_driver.h> 11 12 #include <mlx5_glue.h> 13 #include <mlx5_devx_cmds.h> 14 #include <mlx5_prm.h> 15 #include <mlx5_common_os.h> 16 17 #include "mlx5_regex.h" 18 #include "mlx5_regex_utils.h" 19 #include "mlx5_rxp_csrs.h" 20 #include "mlx5_rxp.h" 21 22 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES 23 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH 24 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX 25 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS 26 27 /* Private Declarations */ 28 static int 29 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value, 30 uint32_t address, uint32_t expected_value, 31 uint32_t expected_mask, uint32_t timeout_ms, uint8_t id); 32 static int 33 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use); 34 static int 35 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id); 36 static int 37 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id); 38 static int 39 program_rxp_rules(struct mlx5_regex_priv *priv, 40 struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id); 41 static int 42 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id); 43 static int 44 write_private_rules(struct mlx5_regex_priv *priv, 45 struct mlx5_rxp_ctl_rules_pgm *rules, 46 uint8_t id); 47 static int 48 write_shared_rules(struct mlx5_regex_priv *priv, 49 struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count, 50 uint8_t db_to_program); 51 static int 52 rxp_db_setup(struct mlx5_regex_priv *priv); 53 static void 54 rxp_dump_csrs(struct ibv_context *ctx, uint8_t id); 55 static int 56 rxp_write_rules_via_cp(struct ibv_context *ctx, 57 struct mlx5_rxp_rof_entry *rules, 58 int count, uint8_t id); 59 static int 60 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules, 61 int count, uint8_t id); 62 static int 63 rxp_start_engine(struct ibv_context *ctx, uint8_t id); 64 static int 65 rxp_stop_engine(struct ibv_context *ctx, uint8_t id); 66 67 static void __rte_unused 68 rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused) 69 { 70 uint32_t reg, i; 71 72 /* Main CSRs*/ 73 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { 74 if (mlx5_devx_regex_register_read(ctx, id, 75 (MLX5_RXP_CSR_WIDTH * i) + 76 MLX5_RXP_CSR_BASE_ADDRESS, 77 ®)) { 78 DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id); 79 return; 80 } 81 DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x", 82 id, i, reg); 83 } 84 /* RTRU CSRs*/ 85 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { 86 if (mlx5_devx_regex_register_read(ctx, id, 87 (MLX5_RXP_CSR_WIDTH * i) + 88 MLX5_RXP_RTRU_CSR_BASE_ADDRESS, 89 ®)) { 90 DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id); 91 return; 92 } 93 DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x", 94 id, i, reg); 95 } 96 /* STAT CSRs */ 97 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { 98 if (mlx5_devx_regex_register_read(ctx, id, 99 (MLX5_RXP_CSR_WIDTH * i) + 100 MLX5_RXP_STATS_CSR_BASE_ADDRESS, 101 ®)) { 102 DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id); 103 return; 104 } 105 DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x", 106 id, i, reg); 107 } 108 } 109 110 int 111 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused, 112 struct rte_regexdev_info *info) 113 { 114 info->max_matches = MLX5_REGEX_MAX_MATCHES; 115 info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE; 116 info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP; 117 info->max_groups = MLX5_REGEX_MAX_GROUPS; 118 info->max_queue_pairs = 1; 119 info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F | 120 RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F; 121 info->rule_flags = 0; 122 info->max_queue_pairs = 10; 123 return 0; 124 } 125 126 /** 127 * Actual writing of RXP instructions to RXP via CSRs. 128 */ 129 static int 130 rxp_write_rules_via_cp(struct ibv_context *ctx, 131 struct mlx5_rxp_rof_entry *rules, 132 int count, uint8_t id) 133 { 134 int i, ret = 0; 135 uint32_t tmp; 136 137 for (i = 0; i < count; i++) { 138 tmp = (uint32_t)rules[i].value; 139 ret |= mlx5_devx_regex_register_write(ctx, id, 140 MLX5_RXP_RTRU_CSR_DATA_0, 141 tmp); 142 tmp = (uint32_t)(rules[i].value >> 32); 143 ret |= mlx5_devx_regex_register_write(ctx, id, 144 MLX5_RXP_RTRU_CSR_DATA_0 + 145 MLX5_RXP_CSR_WIDTH, tmp); 146 tmp = rules[i].addr; 147 ret |= mlx5_devx_regex_register_write(ctx, id, 148 MLX5_RXP_RTRU_CSR_ADDR, 149 tmp); 150 if (ret) { 151 DRV_LOG(ERR, "Failed to copy instructions to RXP."); 152 return -1; 153 } 154 } 155 DRV_LOG(DEBUG, "Written %d instructions", count); 156 return 0; 157 } 158 159 static int 160 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules, 161 int count, uint8_t id) 162 { 163 uint32_t val, fifo_depth; 164 int ret; 165 166 ret = rxp_write_rules_via_cp(ctx, rules, count, id); 167 if (ret < 0) { 168 DRV_LOG(ERR, "Failed to write rules via CSRs."); 169 return -1; 170 } 171 ret = mlx5_devx_regex_register_read(ctx, id, 172 MLX5_RXP_RTRU_CSR_CAPABILITY, 173 &fifo_depth); 174 if (ret) { 175 DRV_LOG(ERR, "CSR read failed!"); 176 return -1; 177 } 178 ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_FIFO_STAT, 179 count, ~0, 180 MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id); 181 if (ret < 0) { 182 DRV_LOG(ERR, "Rules not rx by RXP: credit: %d, depth: %d", val, 183 fifo_depth); 184 return ret; 185 } 186 DRV_LOG(DEBUG, "RTRU FIFO depth: 0x%x", fifo_depth); 187 DRV_LOG(DEBUG, "Rules flush took %d cycles.", ret); 188 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 189 &val); 190 if (ret) { 191 DRV_LOG(ERR, "CSR read failed!"); 192 return -1; 193 } 194 val |= MLX5_RXP_RTRU_CSR_CTRL_GO; 195 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 196 val); 197 ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_STATUS, 198 MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE, 199 MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE, 200 MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id); 201 if (ret < 0) { 202 DRV_LOG(ERR, "Rules update timeout: 0x%08X", val); 203 return ret; 204 } 205 DRV_LOG(DEBUG, "Rules update took %d cycles", ret); 206 if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 207 &val)) { 208 DRV_LOG(ERR, "CSR read failed!"); 209 return -1; 210 } 211 val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO); 212 if (mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 213 val)) { 214 DRV_LOG(ERR, "CSR write write failed!"); 215 return -1; 216 } 217 218 DRV_LOG(DEBUG, "RXP Flush rules finished."); 219 return 0; 220 } 221 222 static int 223 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value, 224 uint32_t address, uint32_t expected_value, 225 uint32_t expected_mask, uint32_t timeout_ms, uint8_t id) 226 { 227 unsigned int i; 228 int ret = 0; 229 230 ret = -EBUSY; 231 for (i = 0; i < timeout_ms; i++) { 232 if (mlx5_devx_regex_register_read(ctx, id, address, value)) 233 return -1; 234 if ((*value & expected_mask) == expected_value) { 235 ret = 0; 236 break; 237 } 238 rte_delay_us(1000); 239 } 240 return ret; 241 } 242 243 static int 244 rxp_start_engine(struct ibv_context *ctx, uint8_t id) 245 { 246 uint32_t ctrl; 247 int ret; 248 249 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 250 if (ret) 251 return ret; 252 ctrl |= MLX5_RXP_CSR_CTRL_GO; 253 ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C; 254 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 255 return ret; 256 } 257 258 static int 259 rxp_stop_engine(struct ibv_context *ctx, uint8_t id) 260 { 261 uint32_t ctrl; 262 int ret; 263 264 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 265 if (ret) 266 return ret; 267 ctrl &= ~MLX5_RXP_CSR_CTRL_GO; 268 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 269 return ret; 270 } 271 272 static int 273 rxp_init_rtru(struct ibv_context *ctx, uint8_t id, uint32_t init_bits) 274 { 275 uint32_t ctrl_value; 276 uint32_t poll_value; 277 uint32_t expected_value; 278 uint32_t expected_mask; 279 int ret = 0; 280 281 /* Read the rtru ctrl CSR. */ 282 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 283 &ctrl_value); 284 if (ret) 285 return -1; 286 /* Clear any previous init modes. */ 287 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK); 288 if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) { 289 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT); 290 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 291 ctrl_value); 292 } 293 /* Set the init_mode bits in the rtru ctrl CSR. */ 294 ctrl_value |= init_bits; 295 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 296 ctrl_value); 297 /* Need to sleep for a short period after pulsing the rtru init bit. */ 298 rte_delay_us(20000); 299 /* Poll the rtru status CSR until all the init done bits are set. */ 300 DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init"); 301 /* Set the init bit in the rtru ctrl CSR. */ 302 ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT; 303 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 304 ctrl_value); 305 /* Clear the init bit in the rtru ctrl CSR */ 306 ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT; 307 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 308 ctrl_value); 309 /* Check that the following bits are set in the RTRU_CSR. */ 310 if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) { 311 /* Must be incremental mode */ 312 expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE | 313 MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE; 314 } else { 315 expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE | 316 MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE | 317 MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE; 318 } 319 expected_mask = expected_value; 320 ret = rxp_poll_csr_for_value(ctx, &poll_value, 321 MLX5_RXP_RTRU_CSR_STATUS, 322 expected_value, expected_mask, 323 MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id); 324 if (ret) 325 return ret; 326 DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value); 327 /* Clear the init bit in the rtru ctrl CSR */ 328 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT); 329 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 330 ctrl_value); 331 return 0; 332 } 333 334 static int 335 rxp_parse_rof(const char *buf, uint32_t len, 336 struct mlx5_rxp_ctl_rules_pgm **rules) 337 { 338 static const char del[] = "\n\r"; 339 char *line; 340 char *tmp; 341 char *cur_pos; 342 uint32_t lines = 0; 343 uint32_t entries; 344 struct mlx5_rxp_rof_entry *curentry; 345 346 tmp = rte_malloc("", len, 0); 347 if (!tmp) 348 return -ENOMEM; 349 memcpy(tmp, buf, len); 350 line = strtok(tmp, del); 351 while (line) { 352 if (line[0] != '#' && line[0] != '\0') 353 lines++; 354 line = strtok(NULL, del); 355 } 356 *rules = rte_malloc("", lines * sizeof(*curentry) + sizeof(**rules), 0); 357 if (!(*rules)) { 358 rte_free(tmp); 359 return -ENOMEM; 360 } 361 memset(*rules, 0, lines * sizeof(curentry) + sizeof(**rules)); 362 curentry = (*rules)->rules; 363 (*rules)->hdr.cmd = MLX5_RXP_CTL_RULES_PGM; 364 entries = 0; 365 memcpy(tmp, buf, len); 366 line = strtok(tmp, del); 367 while (line) { 368 if (line[0] == '#' || line[0] == '\0') { 369 line = strtok(NULL, del); 370 continue; 371 } 372 curentry->type = strtoul(line, &cur_pos, 10); 373 if (cur_pos == line || cur_pos[0] != ',') 374 goto parse_error; 375 cur_pos++; 376 curentry->addr = strtoul(cur_pos, &cur_pos, 16); 377 if (cur_pos[0] != ',') 378 goto parse_error; 379 cur_pos++; 380 curentry->value = strtoull(cur_pos, &cur_pos, 16); 381 if (cur_pos[0] != '\0' && cur_pos[0] != '\n') 382 goto parse_error; 383 curentry++; 384 entries++; 385 if (entries > lines) 386 goto parse_error; 387 line = strtok(NULL, del); 388 } 389 (*rules)->count = entries; 390 (*rules)->hdr.len = entries * sizeof(*curentry) + sizeof(**rules); 391 rte_free(tmp); 392 return 0; 393 parse_error: 394 rte_free(tmp); 395 if (*rules) 396 rte_free(*rules); 397 return -EINVAL; 398 } 399 400 static int 401 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use) 402 { 403 int ret; 404 uint32_t umem_id; 405 406 ret = mlx5_devx_regex_database_stop(priv->ctx, id); 407 if (ret < 0) { 408 DRV_LOG(ERR, "stop engine failed!"); 409 return ret; 410 } 411 umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem); 412 ret = mlx5_devx_regex_database_program(priv->ctx, id, umem_id, 0); 413 if (ret < 0) { 414 DRV_LOG(ERR, "program db failed!"); 415 return ret; 416 } 417 return 0; 418 } 419 420 static int 421 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id) 422 { 423 mlx5_devx_regex_database_resume(priv->ctx, id); 424 return 0; 425 } 426 427 /* 428 * Assign db memory for RXP programming. 429 */ 430 static int 431 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id) 432 { 433 unsigned int i; 434 uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED; 435 uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED; 436 437 /* Check which database rxp_eng is currently located if any? */ 438 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); 439 i++) { 440 if (priv->db[i].db_assigned_to_eng_num == id) { 441 eng_assigned = i; 442 break; 443 } 444 } 445 /* 446 * If private mode then, we can keep the same db ptr as RXP will be 447 * programming EM itself if necessary, however need to see if 448 * programmed yet. 449 */ 450 if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) && 451 (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED)) 452 return eng_assigned; 453 /* Check for inactive db memory to use. */ 454 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); 455 i++) { 456 if (priv->db[i].active == true) 457 continue; /* Already in use, so skip db. */ 458 /* Set this db to active now as free to use. */ 459 priv->db[i].active = true; 460 /* Now unassign last db index in use by RXP Eng. */ 461 if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) { 462 priv->db[eng_assigned].active = false; 463 priv->db[eng_assigned].db_assigned_to_eng_num = 464 MLX5_RXP_DB_NOT_ASSIGNED; 465 466 /* Set all DB memory to 0's before setting up DB. */ 467 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE); 468 } 469 /* Now reassign new db index with RXP Engine. */ 470 priv->db[i].db_assigned_to_eng_num = id; 471 db_free = i; 472 break; 473 } 474 if (db_free == MLX5_RXP_DB_NOT_ASSIGNED) 475 return -1; 476 return db_free; 477 } 478 479 /* 480 * Program RXP instruction db to RXP engine/s. 481 */ 482 static int 483 program_rxp_rules(struct mlx5_regex_priv *priv, 484 struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id) 485 { 486 int ret, db_free; 487 uint32_t rule_cnt; 488 489 rule_cnt = rules->count; 490 db_free = mlnx_update_database(priv, id); 491 if (db_free < 0) { 492 DRV_LOG(ERR, "Failed to setup db memory!"); 493 return db_free; 494 } 495 if (priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) { 496 /* Register early to ensure RXP writes to EM use valid addr. */ 497 ret = mlnx_set_database(priv, id, db_free); 498 if (ret < 0) { 499 DRV_LOG(ERR, "Failed to register db memory!"); 500 return ret; 501 } 502 } 503 ret = write_private_rules(priv, rules, id); 504 if (ret < 0) { 505 DRV_LOG(ERR, "Failed to write rules!"); 506 return ret; 507 } 508 if (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) { 509 /* Write external rules directly to EM. */ 510 rules->count = rule_cnt; 511 /* Now write external instructions to EM. */ 512 ret = write_shared_rules(priv, rules, rules->hdr.len, db_free); 513 if (ret < 0) { 514 DRV_LOG(ERR, "Failed to write EM rules!"); 515 return ret; 516 } 517 ret = mlnx_set_database(priv, id, db_free); 518 if (ret < 0) { 519 DRV_LOG(ERR, "Failed to register db memory!"); 520 return ret; 521 } 522 } 523 ret = mlnx_resume_database(priv, id); 524 if (ret < 0) { 525 DRV_LOG(ERR, "Failed to resume engine!"); 526 return ret; 527 } 528 DRV_LOG(DEBUG, "Programmed RXP Engine %d\n", id); 529 rules->count = rule_cnt; 530 return 0; 531 } 532 533 static int 534 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id) 535 { 536 uint32_t ctrl; 537 uint32_t reg; 538 struct ibv_context *ctx = priv->ctx; 539 int ret; 540 541 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 542 if (ret) 543 return ret; 544 if (ctrl & MLX5_RXP_CSR_CTRL_INIT) { 545 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT; 546 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, 547 ctrl); 548 if (ret) 549 return ret; 550 } 551 ctrl |= MLX5_RXP_CSR_CTRL_INIT; 552 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 553 if (ret) 554 return ret; 555 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT; 556 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 557 rte_delay_us(20000); 558 ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS, 559 MLX5_RXP_CSR_STATUS_INIT_DONE, 560 MLX5_RXP_CSR_STATUS_INIT_DONE, 561 MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id); 562 if (ret) 563 return ret; 564 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 565 if (ret) 566 return ret; 567 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT; 568 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, 569 ctrl); 570 if (ret) 571 return ret; 572 ret = rxp_init_rtru(ctx, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2); 573 if (ret) 574 return ret; 575 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5, 576 ®); 577 if (ret) 578 return ret; 579 DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16, 580 reg & 0xffff); 581 if ((reg >> 16) >= priv->nb_max_matches) 582 ret = mlx5_devx_regex_register_write(ctx, id, 583 MLX5_RXP_CSR_MAX_MATCH, 584 priv->nb_max_matches); 585 else 586 ret = mlx5_devx_regex_register_write(ctx, id, 587 MLX5_RXP_CSR_MAX_MATCH, 588 (reg >> 16)); 589 ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX, 590 (reg & 0xFFFF)); 591 ret |= mlx5_devx_regex_register_write(ctx, id, 592 MLX5_RXP_CSR_MAX_LATENCY, 0); 593 ret |= mlx5_devx_regex_register_write(ctx, id, 594 MLX5_RXP_CSR_MAX_PRI_THREAD, 0); 595 return ret; 596 } 597 598 static int 599 write_private_rules(struct mlx5_regex_priv *priv, 600 struct mlx5_rxp_ctl_rules_pgm *rules, 601 uint8_t id) 602 { 603 unsigned int pending; 604 uint32_t block, reg, val, rule_cnt, rule_offset, rtru_max_num_entries; 605 int ret = 1; 606 607 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) 608 return -EINVAL; 609 if (rules->hdr.len == 0 || rules->hdr.cmd < MLX5_RXP_CTL_RULES_PGM || 610 rules->hdr.cmd > MLX5_RXP_CTL_RULES_PGM_INCR) 611 return -EINVAL; 612 /* For a non-incremental rules program, re-init the RXP. */ 613 if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM) { 614 ret = rxp_init_eng(priv, id); 615 if (ret < 0) 616 return ret; 617 } else if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM_INCR) { 618 /* Flush RXP L1 and L2 cache by using MODE_L1_L2. */ 619 ret = rxp_init_rtru(priv->ctx, id, 620 MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2); 621 if (ret < 0) 622 return ret; 623 } 624 if (rules->count == 0) 625 return -EINVAL; 626 /* Confirm the RXP is initialised. */ 627 if (mlx5_devx_regex_register_read(priv->ctx, id, 628 MLX5_RXP_CSR_STATUS, &val)) { 629 DRV_LOG(ERR, "Failed to read from RXP!"); 630 return -ENODEV; 631 } 632 if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) { 633 DRV_LOG(ERR, "RXP not initialised..."); 634 return -EBUSY; 635 } 636 /* Get the RTRU maximum number of entries allowed. */ 637 if (mlx5_devx_regex_register_read(priv->ctx, id, 638 MLX5_RXP_RTRU_CSR_CAPABILITY, &rtru_max_num_entries)) { 639 DRV_LOG(ERR, "Failed to read RTRU capability!"); 640 return -ENODEV; 641 } 642 rtru_max_num_entries = (rtru_max_num_entries & 0x00FF); 643 rule_cnt = 0; 644 pending = 0; 645 while (rules->count > 0) { 646 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_INST) || 647 (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_IM) || 648 (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM)) { 649 if ((rules->rules[rule_cnt].type == 650 MLX5_RXP_ROF_ENTRY_EM) && 651 (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) { 652 /* Skip EM rules programming. */ 653 if (pending > 0) { 654 /* Flush any rules that are pending. */ 655 rule_offset = (rule_cnt - pending); 656 ret = rxp_flush_rules(priv->ctx, 657 &rules->rules[rule_offset], 658 pending, id); 659 if (ret < 0) { 660 DRV_LOG(ERR, "Flushing rules."); 661 return -ENODEV; 662 } 663 pending = 0; 664 } 665 rule_cnt++; 666 } else { 667 pending++; 668 rule_cnt++; 669 /* 670 * If parsing the last rule, or if reached the 671 * maximum number of rules for this batch, then 672 * flush the rules batch to the RXP. 673 */ 674 if ((rules->count == 1) || 675 (pending == rtru_max_num_entries)) { 676 rule_offset = (rule_cnt - pending); 677 ret = rxp_flush_rules(priv->ctx, 678 &rules->rules[rule_offset], 679 pending, id); 680 if (ret < 0) { 681 DRV_LOG(ERR, "Flushing rules."); 682 return -ENODEV; 683 } 684 pending = 0; 685 } 686 } 687 } else if ((rules->rules[rule_cnt].type == 688 MLX5_RXP_ROF_ENTRY_EQ) || 689 (rules->rules[rule_cnt].type == 690 MLX5_RXP_ROF_ENTRY_GTE) || 691 (rules->rules[rule_cnt].type == 692 MLX5_RXP_ROF_ENTRY_LTE) || 693 (rules->rules[rule_cnt].type == 694 MLX5_RXP_ROF_ENTRY_CHECKSUM) || 695 (rules->rules[rule_cnt].type == 696 MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM)) { 697 if (pending) { 698 /* Flush rules before checking reg values. */ 699 rule_offset = (rule_cnt - pending); 700 ret = rxp_flush_rules(priv->ctx, 701 &rules->rules[rule_offset], 702 pending, id); 703 if (ret < 0) { 704 DRV_LOG(ERR, "Failed to flush rules."); 705 return -ENODEV; 706 } 707 } 708 block = (rules->rules[rule_cnt].addr >> 16) & 0xFFFF; 709 if (block == 0) 710 reg = MLX5_RXP_CSR_BASE_ADDRESS; 711 else if (block == 1) 712 reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS; 713 else { 714 DRV_LOG(ERR, "Invalid ROF register 0x%08X!", 715 rules->rules[rule_cnt].addr); 716 return -EINVAL; 717 } 718 reg += (rules->rules[rule_cnt].addr & 0xFFFF) * 719 MLX5_RXP_CSR_WIDTH; 720 ret = mlx5_devx_regex_register_read(priv->ctx, id, 721 reg, &val); 722 if (ret) { 723 DRV_LOG(ERR, "RXP CSR read failed!"); 724 return ret; 725 } 726 if ((priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) && 727 ((rules->rules[rule_cnt].type == 728 MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM) && 729 (val != rules->rules[rule_cnt].value))) { 730 DRV_LOG(ERR, "Unexpected value for register:"); 731 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32, 732 rules->rules[rule_cnt].addr, val); 733 DRV_LOG(ERR, "expected %" PRIx64 ".", 734 rules->rules[rule_cnt].value); 735 return -EINVAL; 736 } else if ((priv->prog_mode == 737 MLX5_RXP_PRIVATE_PROG_MODE) && 738 (rules->rules[rule_cnt].type == 739 MLX5_RXP_ROF_ENTRY_CHECKSUM) && 740 (val != rules->rules[rule_cnt].value)) { 741 DRV_LOG(ERR, "Unexpected value for register:"); 742 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32, 743 rules->rules[rule_cnt].addr, val); 744 DRV_LOG(ERR, "expected %" PRIx64 ".", 745 rules->rules[rule_cnt].value); 746 return -EINVAL; 747 } else if ((rules->rules[rule_cnt].type == 748 MLX5_RXP_ROF_ENTRY_EQ) && 749 (val != rules->rules[rule_cnt].value)) { 750 DRV_LOG(ERR, "Unexpected value for register:"); 751 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32, 752 rules->rules[rule_cnt].addr, val); 753 DRV_LOG(ERR, "expected %" PRIx64 ".", 754 rules->rules[rule_cnt].value); 755 return -EINVAL; 756 } else if ((rules->rules[rule_cnt].type == 757 MLX5_RXP_ROF_ENTRY_GTE) && 758 (val < rules->rules[rule_cnt].value)) { 759 DRV_LOG(ERR, "Unexpected value reg 0x%08X,", 760 rules->rules[rule_cnt].addr); 761 DRV_LOG(ERR, "got %X, expected >= %" PRIx64 ".", 762 val, rules->rules[rule_cnt].value); 763 return -EINVAL; 764 } else if ((rules->rules[rule_cnt].type == 765 MLX5_RXP_ROF_ENTRY_LTE) && 766 (val > rules->rules[rule_cnt].value)) { 767 DRV_LOG(ERR, "Unexpected value reg 0x%08X,", 768 rules->rules[rule_cnt].addr); 769 DRV_LOG(ERR, "got %08X expected <= %" PRIx64, 770 val, rules->rules[rule_cnt].value); 771 return -EINVAL; 772 } 773 rule_cnt++; 774 pending = 0; 775 } else { 776 DRV_LOG(ERR, "Error: Invalid rule type %d!", 777 rules->rules[rule_cnt].type); 778 return -EINVAL; 779 } 780 rules->count--; 781 } 782 return ret; 783 } 784 785 /* 786 * Shared memory programming mode, here all external db instructions are written 787 * to EM via the host. 788 */ 789 static int 790 write_shared_rules(struct mlx5_regex_priv *priv, 791 struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count, 792 uint8_t db_to_program) 793 { 794 uint32_t rule_cnt, rof_rule_addr; 795 uint64_t tmp_write_swap[4]; 796 797 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) 798 return -EINVAL; 799 if ((rules->count == 0) || (count == 0)) 800 return -EINVAL; 801 rule_cnt = 0; 802 /* 803 * Note the following section of code carries out a 32byte swap of 804 * instruction to coincide with HW 32byte swap. This may need removed 805 * in new variants of this programming function! 806 */ 807 while (rule_cnt < rules->count) { 808 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM) && 809 (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) { 810 /* 811 * Note there are always blocks of 8 instructions for 812 * 7's written sequentially. However there is no 813 * guarantee that all blocks are sequential! 814 */ 815 if (count >= (rule_cnt + MLX5_RXP_INST_BLOCK_SIZE)) { 816 /* 817 * Ensure memory write not exceeding boundary 818 * Check essential to ensure 0x10000 offset 819 * accounted for! 820 */ 821 if ((uint8_t *)((uint8_t *) 822 priv->db[db_to_program].ptr + 823 ((rules->rules[rule_cnt + 7].addr << 824 MLX5_RXP_INST_OFFSET))) >= 825 ((uint8_t *)((uint8_t *) 826 priv->db[db_to_program].ptr + 827 MLX5_MAX_DB_SIZE))) { 828 DRV_LOG(ERR, "DB exceeded memory!"); 829 return -ENODEV; 830 } 831 /* 832 * Rule address Offset to align with RXP 833 * external instruction offset. 834 */ 835 rof_rule_addr = (rules->rules[rule_cnt].addr << 836 MLX5_RXP_INST_OFFSET); 837 /* 32 byte instruction swap (sw work around)! */ 838 tmp_write_swap[0] = le64toh( 839 rules->rules[(rule_cnt + 4)].value); 840 tmp_write_swap[1] = le64toh( 841 rules->rules[(rule_cnt + 5)].value); 842 tmp_write_swap[2] = le64toh( 843 rules->rules[(rule_cnt + 6)].value); 844 tmp_write_swap[3] = le64toh( 845 rules->rules[(rule_cnt + 7)].value); 846 /* Write only 4 of the 8 instructions. */ 847 memcpy((uint8_t *)((uint8_t *) 848 priv->db[db_to_program].ptr + 849 rof_rule_addr), &tmp_write_swap, 850 (sizeof(uint64_t) * 4)); 851 /* Write 1st 4 rules of block after last 4. */ 852 rof_rule_addr = (rules->rules[ 853 (rule_cnt + 4)].addr << 854 MLX5_RXP_INST_OFFSET); 855 tmp_write_swap[0] = le64toh( 856 rules->rules[(rule_cnt + 0)].value); 857 tmp_write_swap[1] = le64toh( 858 rules->rules[(rule_cnt + 1)].value); 859 tmp_write_swap[2] = le64toh( 860 rules->rules[(rule_cnt + 2)].value); 861 tmp_write_swap[3] = le64toh( 862 rules->rules[(rule_cnt + 3)].value); 863 memcpy((uint8_t *)((uint8_t *) 864 priv->db[db_to_program].ptr + 865 rof_rule_addr), &tmp_write_swap, 866 (sizeof(uint64_t) * 4)); 867 } else 868 return -1; 869 /* Fast forward as already handled block of 8. */ 870 rule_cnt += MLX5_RXP_INST_BLOCK_SIZE; 871 } else 872 rule_cnt++; /* Must be something other than EM rule. */ 873 } 874 return 0; 875 } 876 877 static int 878 rxp_db_setup(struct mlx5_regex_priv *priv) 879 { 880 int ret; 881 uint8_t i; 882 883 /* Setup database memories for both RXP engines + reprogram memory. */ 884 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) { 885 priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 0); 886 if (!priv->db[i].ptr) { 887 DRV_LOG(ERR, "Failed to alloc db memory!"); 888 ret = ENODEV; 889 goto tidyup_error; 890 } 891 /* Register the memory. */ 892 priv->db[i].umem.umem = mlx5_glue->devx_umem_reg(priv->ctx, 893 priv->db[i].ptr, 894 MLX5_MAX_DB_SIZE, 7); 895 if (!priv->db[i].umem.umem) { 896 DRV_LOG(ERR, "Failed to register memory!"); 897 ret = ENODEV; 898 goto tidyup_error; 899 } 900 /* Ensure set all DB memory to 0's before setting up DB. */ 901 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE); 902 /* No data currently in database. */ 903 priv->db[i].len = 0; 904 priv->db[i].active = false; 905 priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED; 906 } 907 return 0; 908 tidyup_error: 909 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) { 910 if (priv->db[i].ptr) 911 rte_free(priv->db[i].ptr); 912 if (priv->db[i].umem.umem) 913 mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem); 914 } 915 return -ret; 916 } 917 918 int 919 mlx5_regex_rules_db_import(struct rte_regexdev *dev, 920 const char *rule_db, uint32_t rule_db_len) 921 { 922 struct mlx5_regex_priv *priv = dev->data->dev_private; 923 struct mlx5_rxp_ctl_rules_pgm *rules = NULL; 924 uint8_t id; 925 int ret; 926 927 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) { 928 DRV_LOG(ERR, "RXP programming mode not set!"); 929 return -1; 930 } 931 if (rule_db == NULL) { 932 DRV_LOG(ERR, "Database empty!"); 933 return -ENODEV; 934 } 935 if (rule_db_len == 0) 936 return -EINVAL; 937 ret = rxp_parse_rof(rule_db, rule_db_len, &rules); 938 if (ret) { 939 DRV_LOG(ERR, "Can't parse ROF file."); 940 return ret; 941 } 942 /* Need to ensure RXP not busy before stop! */ 943 for (id = 0; id < priv->nb_engines; id++) { 944 ret = rxp_stop_engine(priv->ctx, id); 945 if (ret) { 946 DRV_LOG(ERR, "Can't stop engine."); 947 ret = -ENODEV; 948 goto tidyup_error; 949 } 950 ret = program_rxp_rules(priv, rules, id); 951 if (ret < 0) { 952 DRV_LOG(ERR, "Failed to program rxp rules."); 953 ret = -ENODEV; 954 goto tidyup_error; 955 } 956 ret = rxp_start_engine(priv->ctx, id); 957 if (ret) { 958 DRV_LOG(ERR, "Can't start engine."); 959 ret = -ENODEV; 960 goto tidyup_error; 961 } 962 } 963 rte_free(rules); 964 return 0; 965 tidyup_error: 966 rte_free(rules); 967 return ret; 968 } 969 970 int 971 mlx5_regex_configure(struct rte_regexdev *dev, 972 const struct rte_regexdev_config *cfg) 973 { 974 struct mlx5_regex_priv *priv = dev->data->dev_private; 975 int ret; 976 977 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) 978 return -1; 979 priv->nb_queues = cfg->nb_queue_pairs; 980 dev->data->dev_conf.nb_queue_pairs = priv->nb_queues; 981 priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) * 982 priv->nb_queues, 0); 983 if (!priv->nb_queues) { 984 DRV_LOG(ERR, "can't allocate qps memory"); 985 rte_errno = ENOMEM; 986 return -rte_errno; 987 } 988 priv->nb_max_matches = cfg->nb_max_matches; 989 /* Setup rxp db memories. */ 990 if (rxp_db_setup(priv)) { 991 DRV_LOG(ERR, "Failed to setup RXP db memory"); 992 rte_errno = ENOMEM; 993 return -rte_errno; 994 } 995 if (cfg->rule_db != NULL) { 996 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db, 997 cfg->rule_db_len); 998 if (ret < 0) { 999 DRV_LOG(ERR, "Failed to program rxp rules."); 1000 rte_errno = ENODEV; 1001 goto configure_error; 1002 } 1003 } else 1004 DRV_LOG(DEBUG, "Regex config without rules programming!"); 1005 return 0; 1006 configure_error: 1007 if (priv->qps) 1008 rte_free(priv->qps); 1009 return -rte_errno; 1010 } 1011