1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <rte_log.h> 6 #include <rte_errno.h> 7 #include <rte_malloc.h> 8 #include <rte_regexdev.h> 9 #include <rte_regexdev_core.h> 10 #include <rte_regexdev_driver.h> 11 #include <sys/mman.h> 12 13 #include <mlx5_glue.h> 14 #include <mlx5_devx_cmds.h> 15 #include <mlx5_prm.h> 16 #include <mlx5_common_os.h> 17 18 #include "mlx5_regex.h" 19 #include "mlx5_regex_utils.h" 20 #include "mlx5_rxp_csrs.h" 21 #include "mlx5_rxp.h" 22 23 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES 24 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH 25 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX 26 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS 27 28 #define MLX5_REGEX_RXP_ROF2_LINE_LEN 34 29 30 /* Private Declarations */ 31 static int 32 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value, 33 uint32_t address, uint32_t expected_value, 34 uint32_t expected_mask, uint32_t timeout_ms, uint8_t id); 35 static int 36 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use); 37 static int 38 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id); 39 static int 40 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id); 41 static int 42 program_rxp_rules(struct mlx5_regex_priv *priv, const char *buf, uint32_t len, 43 uint8_t id); 44 static int 45 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id); 46 static int 47 rxp_db_setup(struct mlx5_regex_priv *priv); 48 static void 49 rxp_dump_csrs(struct ibv_context *ctx, uint8_t id); 50 static int 51 rxp_start_engine(struct ibv_context *ctx, uint8_t id); 52 static int 53 rxp_stop_engine(struct ibv_context *ctx, uint8_t id); 54 55 static void __rte_unused 56 rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused) 57 { 58 uint32_t reg, i; 59 60 /* Main CSRs*/ 61 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { 62 if (mlx5_devx_regex_register_read(ctx, id, 63 (MLX5_RXP_CSR_WIDTH * i) + 64 MLX5_RXP_CSR_BASE_ADDRESS, 65 ®)) { 66 DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id); 67 return; 68 } 69 DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x", 70 id, i, reg); 71 } 72 /* RTRU CSRs*/ 73 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { 74 if (mlx5_devx_regex_register_read(ctx, id, 75 (MLX5_RXP_CSR_WIDTH * i) + 76 MLX5_RXP_RTRU_CSR_BASE_ADDRESS, 77 ®)) { 78 DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id); 79 return; 80 } 81 DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x", 82 id, i, reg); 83 } 84 /* STAT CSRs */ 85 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { 86 if (mlx5_devx_regex_register_read(ctx, id, 87 (MLX5_RXP_CSR_WIDTH * i) + 88 MLX5_RXP_STATS_CSR_BASE_ADDRESS, 89 ®)) { 90 DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id); 91 return; 92 } 93 DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x", 94 id, i, reg); 95 } 96 } 97 98 int 99 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused, 100 struct rte_regexdev_info *info) 101 { 102 info->max_matches = MLX5_REGEX_MAX_MATCHES; 103 info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE; 104 info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP; 105 info->max_groups = MLX5_REGEX_MAX_GROUPS; 106 info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F | 107 RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F; 108 info->rule_flags = 0; 109 info->max_queue_pairs = UINT16_MAX; 110 return 0; 111 } 112 113 static int 114 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value, 115 uint32_t address, uint32_t expected_value, 116 uint32_t expected_mask, uint32_t timeout_ms, uint8_t id) 117 { 118 unsigned int i; 119 int ret; 120 121 ret = -EBUSY; 122 for (i = 0; i < timeout_ms; i++) { 123 if (mlx5_devx_regex_register_read(ctx, id, address, value)) 124 return -1; 125 if ((*value & expected_mask) == expected_value) { 126 ret = 0; 127 break; 128 } 129 rte_delay_us(1000); 130 } 131 return ret; 132 } 133 134 static int 135 rxp_start_engine(struct ibv_context *ctx, uint8_t id) 136 { 137 uint32_t ctrl; 138 int ret; 139 140 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 141 if (ret) 142 return ret; 143 ctrl |= MLX5_RXP_CSR_CTRL_GO; 144 ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C; 145 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 146 return ret; 147 } 148 149 static int 150 rxp_stop_engine(struct ibv_context *ctx, uint8_t id) 151 { 152 uint32_t ctrl; 153 int ret; 154 155 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 156 if (ret) 157 return ret; 158 ctrl &= ~MLX5_RXP_CSR_CTRL_GO; 159 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 160 return ret; 161 } 162 163 static int 164 rxp_init_rtru(struct mlx5_regex_priv *priv, uint8_t id, uint32_t init_bits) 165 { 166 uint32_t ctrl_value; 167 uint32_t poll_value; 168 uint32_t expected_value; 169 uint32_t expected_mask; 170 struct ibv_context *ctx = priv->cdev->ctx; 171 int ret = 0; 172 173 /* Read the rtru ctrl CSR. */ 174 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 175 &ctrl_value); 176 if (ret) 177 return -1; 178 /* Clear any previous init modes. */ 179 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK); 180 if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) { 181 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT); 182 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 183 ctrl_value); 184 } 185 /* Set the init_mode bits in the rtru ctrl CSR. */ 186 ctrl_value |= init_bits; 187 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 188 ctrl_value); 189 /* Need to sleep for a short period after pulsing the rtru init bit. */ 190 rte_delay_us(20000); 191 /* Poll the rtru status CSR until all the init done bits are set. */ 192 DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init"); 193 /* Set the init bit in the rtru ctrl CSR. */ 194 ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT; 195 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 196 ctrl_value); 197 /* Clear the init bit in the rtru ctrl CSR */ 198 ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT; 199 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 200 ctrl_value); 201 /* Check that the following bits are set in the RTRU_CSR. */ 202 if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) { 203 /* Must be incremental mode */ 204 expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE; 205 } else { 206 expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE | 207 MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE; 208 } 209 if (priv->is_bf2) 210 expected_value |= MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE; 211 212 213 expected_mask = expected_value; 214 ret = rxp_poll_csr_for_value(ctx, &poll_value, 215 MLX5_RXP_RTRU_CSR_STATUS, 216 expected_value, expected_mask, 217 MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id); 218 if (ret) 219 return ret; 220 DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value); 221 /* Clear the init bit in the rtru ctrl CSR */ 222 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT); 223 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 224 ctrl_value); 225 return 0; 226 } 227 228 static int 229 rxp_parse_line(char *line, uint32_t *type, uint32_t *address, uint64_t *value) 230 { 231 char *cur_pos; 232 233 if (*line == '\0' || *line == '#') 234 return 1; 235 *type = strtoul(line, &cur_pos, 10); 236 if (*cur_pos != ',' && *cur_pos != '\0') 237 return -1; 238 *address = strtoul(cur_pos+1, &cur_pos, 16); 239 if (*cur_pos != ',' && *cur_pos != '\0') 240 return -1; 241 *value = strtoul(cur_pos+1, &cur_pos, 16); 242 if (*cur_pos != ',' && *cur_pos != '\0') 243 return -1; 244 return 0; 245 } 246 247 static uint32_t 248 rxp_get_reg_address(uint32_t address) 249 { 250 uint32_t block; 251 uint32_t reg; 252 253 block = (address >> 16) & 0xFFFF; 254 if (block == 0) 255 reg = MLX5_RXP_CSR_BASE_ADDRESS; 256 else if (block == 1) 257 reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS; 258 else { 259 DRV_LOG(ERR, "Invalid ROF register 0x%08X!", address); 260 return UINT32_MAX; 261 } 262 reg += (address & 0xFFFF) * MLX5_RXP_CSR_WIDTH; 263 return reg; 264 } 265 266 #define MLX5_RXP_NUM_LINES_PER_BLOCK 8 267 268 static int 269 rxp_program_rof(struct mlx5_regex_priv *priv, const char *buf, uint32_t len, 270 uint8_t id) 271 { 272 static const char del[] = "\n\r"; 273 char *line; 274 char *tmp; 275 uint32_t type = 0; 276 uint32_t address; 277 uint64_t val; 278 uint32_t reg_val; 279 int ret; 280 int skip = -1; 281 int last = 0; 282 uint32_t temp; 283 uint32_t tmp_addr; 284 uint32_t rof_rule_addr; 285 uint64_t tmp_write_swap[4]; 286 struct mlx5_rxp_rof_entry rules[8]; 287 int i; 288 int db_free; 289 int j; 290 291 tmp = rte_malloc("", len, 0); 292 if (!tmp) 293 return -ENOMEM; 294 memcpy(tmp, buf, len); 295 db_free = mlnx_update_database(priv, id); 296 if (db_free < 0) { 297 DRV_LOG(ERR, "Failed to setup db memory!"); 298 rte_free(tmp); 299 return db_free; 300 } 301 for (line = strtok(tmp, del), j = 0; line; line = strtok(NULL, del), 302 j++, last = type) { 303 ret = rxp_parse_line(line, &type, &address, &val); 304 if (ret != 0) { 305 if (ret < 0) 306 goto parse_error; 307 continue; 308 } 309 switch (type) { 310 case MLX5_RXP_ROF_ENTRY_EQ: 311 if (skip == 0 && address == 0) 312 skip = 1; 313 tmp_addr = rxp_get_reg_address(address); 314 if (tmp_addr == UINT32_MAX) 315 goto parse_error; 316 ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id, 317 tmp_addr, ®_val); 318 if (ret) 319 goto parse_error; 320 if (skip == -1 && address == 0) { 321 if (val == reg_val) { 322 skip = 0; 323 continue; 324 } 325 } else if (skip == 0) { 326 if (val != reg_val) { 327 DRV_LOG(ERR, 328 "got %08X expected == %" PRIx64, 329 reg_val, val); 330 goto parse_error; 331 } 332 } 333 break; 334 case MLX5_RXP_ROF_ENTRY_GTE: 335 if (skip == 0 && address == 0) 336 skip = 1; 337 tmp_addr = rxp_get_reg_address(address); 338 if (tmp_addr == UINT32_MAX) 339 goto parse_error; 340 ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id, 341 tmp_addr, ®_val); 342 if (ret) 343 goto parse_error; 344 if (skip == -1 && address == 0) { 345 if (reg_val >= val) { 346 skip = 0; 347 continue; 348 } 349 } else if (skip == 0) { 350 if (reg_val < val) { 351 DRV_LOG(ERR, 352 "got %08X expected >= %" PRIx64, 353 reg_val, val); 354 goto parse_error; 355 } 356 } 357 break; 358 case MLX5_RXP_ROF_ENTRY_LTE: 359 tmp_addr = rxp_get_reg_address(address); 360 if (tmp_addr == UINT32_MAX) 361 goto parse_error; 362 ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id, 363 tmp_addr, ®_val); 364 if (ret) 365 goto parse_error; 366 if (skip == 0 && address == 0 && 367 last != MLX5_RXP_ROF_ENTRY_GTE) { 368 skip = 1; 369 } else if (skip == 0 && address == 0 && 370 last == MLX5_RXP_ROF_ENTRY_GTE) { 371 if (reg_val > val) 372 skip = -1; 373 continue; 374 } 375 if (skip == -1 && address == 0) { 376 if (reg_val <= val) { 377 skip = 0; 378 continue; 379 } 380 } else if (skip == 0) { 381 if (reg_val > val) { 382 DRV_LOG(ERR, 383 "got %08X expected <= %" PRIx64, 384 reg_val, val); 385 goto parse_error; 386 } 387 } 388 break; 389 case MLX5_RXP_ROF_ENTRY_CHECKSUM: 390 break; 391 case MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM: 392 if (skip) 393 continue; 394 tmp_addr = rxp_get_reg_address(address); 395 if (tmp_addr == UINT32_MAX) 396 goto parse_error; 397 398 ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id, 399 tmp_addr, ®_val); 400 if (ret) { 401 DRV_LOG(ERR, "RXP CSR read failed!"); 402 return ret; 403 } 404 if (reg_val != val) { 405 DRV_LOG(ERR, "got %08X expected <= %" PRIx64, 406 reg_val, val); 407 goto parse_error; 408 } 409 break; 410 case MLX5_RXP_ROF_ENTRY_IM: 411 if (skip) 412 continue; 413 /* 414 * NOTE: All rules written to RXP must be carried out in 415 * triplets of: 2xData + 1xAddr. 416 * No optimisation is currently allowed in this 417 * sequence to perform less writes. 418 */ 419 temp = val; 420 ret |= mlx5_devx_regex_register_write 421 (priv->cdev->ctx, id, 422 MLX5_RXP_RTRU_CSR_DATA_0, temp); 423 temp = (uint32_t)(val >> 32); 424 ret |= mlx5_devx_regex_register_write 425 (priv->cdev->ctx, id, 426 MLX5_RXP_RTRU_CSR_DATA_0 + 427 MLX5_RXP_CSR_WIDTH, temp); 428 temp = address; 429 ret |= mlx5_devx_regex_register_write 430 (priv->cdev->ctx, id, 431 MLX5_RXP_RTRU_CSR_ADDR, temp); 432 if (ret) { 433 DRV_LOG(ERR, 434 "Failed to copy instructions to RXP."); 435 goto parse_error; 436 } 437 break; 438 case MLX5_RXP_ROF_ENTRY_EM: 439 if (skip) 440 continue; 441 for (i = 0; i < MLX5_RXP_NUM_LINES_PER_BLOCK; i++) { 442 ret = rxp_parse_line(line, &type, 443 &rules[i].addr, 444 &rules[i].value); 445 if (ret != 0) 446 goto parse_error; 447 if (i < (MLX5_RXP_NUM_LINES_PER_BLOCK - 1)) { 448 line = strtok(NULL, del); 449 if (!line) 450 goto parse_error; 451 } 452 } 453 if ((uint8_t *)((uint8_t *) 454 priv->db[id].ptr + 455 ((rules[7].addr << 456 MLX5_RXP_INST_OFFSET))) >= 457 ((uint8_t *)((uint8_t *) 458 priv->db[id].ptr + MLX5_MAX_DB_SIZE))) { 459 DRV_LOG(ERR, "DB exceeded memory!"); 460 goto parse_error; 461 } 462 /* 463 * Rule address Offset to align with RXP 464 * external instruction offset. 465 */ 466 rof_rule_addr = (rules[0].addr << MLX5_RXP_INST_OFFSET); 467 /* 32 byte instruction swap (sw work around)! */ 468 tmp_write_swap[0] = le64toh(rules[4].value); 469 tmp_write_swap[1] = le64toh(rules[5].value); 470 tmp_write_swap[2] = le64toh(rules[6].value); 471 tmp_write_swap[3] = le64toh(rules[7].value); 472 /* Write only 4 of the 8 instructions. */ 473 memcpy((uint8_t *)((uint8_t *) 474 priv->db[id].ptr + rof_rule_addr), 475 &tmp_write_swap, (sizeof(uint64_t) * 4)); 476 /* Write 1st 4 rules of block after last 4. */ 477 rof_rule_addr = (rules[4].addr << MLX5_RXP_INST_OFFSET); 478 tmp_write_swap[0] = le64toh(rules[0].value); 479 tmp_write_swap[1] = le64toh(rules[1].value); 480 tmp_write_swap[2] = le64toh(rules[2].value); 481 tmp_write_swap[3] = le64toh(rules[3].value); 482 memcpy((uint8_t *)((uint8_t *) 483 priv->db[id].ptr + rof_rule_addr), 484 &tmp_write_swap, (sizeof(uint64_t) * 4)); 485 break; 486 default: 487 break; 488 } 489 490 } 491 ret = mlnx_set_database(priv, id, db_free); 492 if (ret < 0) { 493 DRV_LOG(ERR, "Failed to register db memory!"); 494 goto parse_error; 495 } 496 rte_free(tmp); 497 return 0; 498 parse_error: 499 rte_free(tmp); 500 return ret; 501 } 502 503 static int 504 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use) 505 { 506 int ret; 507 uint32_t umem_id; 508 509 ret = mlx5_devx_regex_database_stop(priv->cdev->ctx, id); 510 if (ret < 0) { 511 DRV_LOG(ERR, "stop engine failed!"); 512 return ret; 513 } 514 umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem); 515 ret = mlx5_devx_regex_database_program(priv->cdev->ctx, id, umem_id, 0); 516 if (ret < 0) { 517 DRV_LOG(ERR, "program db failed!"); 518 return ret; 519 } 520 return 0; 521 } 522 523 static int 524 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id) 525 { 526 mlx5_devx_regex_database_resume(priv->cdev->ctx, id); 527 return 0; 528 } 529 530 /* 531 * Assign db memory for RXP programming. 532 */ 533 static int 534 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id) 535 { 536 unsigned int i; 537 uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED; 538 uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED; 539 540 /* Check which database rxp_eng is currently located if any? */ 541 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); 542 i++) { 543 if (priv->db[i].db_assigned_to_eng_num == id) { 544 eng_assigned = i; 545 break; 546 } 547 } 548 /* 549 * If private mode then, we can keep the same db ptr as RXP will be 550 * programming EM itself if necessary, however need to see if 551 * programmed yet. 552 */ 553 if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) && 554 (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED)) 555 return eng_assigned; 556 /* Check for inactive db memory to use. */ 557 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); 558 i++) { 559 if (priv->db[i].active == true) 560 continue; /* Already in use, so skip db. */ 561 /* Set this db to active now as free to use. */ 562 priv->db[i].active = true; 563 /* Now unassign last db index in use by RXP Eng. */ 564 if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) { 565 priv->db[eng_assigned].active = false; 566 priv->db[eng_assigned].db_assigned_to_eng_num = 567 MLX5_RXP_DB_NOT_ASSIGNED; 568 569 /* Set all DB memory to 0's before setting up DB. */ 570 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE); 571 } 572 /* Now reassign new db index with RXP Engine. */ 573 priv->db[i].db_assigned_to_eng_num = id; 574 db_free = i; 575 break; 576 } 577 if (db_free == MLX5_RXP_DB_NOT_ASSIGNED) 578 return -1; 579 return db_free; 580 } 581 582 /* 583 * Program RXP instruction db to RXP engine/s. 584 */ 585 static int 586 program_rxp_rules(struct mlx5_regex_priv *priv, const char *buf, uint32_t len, 587 uint8_t id) 588 { 589 int ret; 590 uint32_t val; 591 struct ibv_context *ctx = priv->cdev->ctx; 592 593 ret = rxp_init_eng(priv, id); 594 if (ret < 0) 595 return ret; 596 /* Confirm the RXP is initialised. */ 597 if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_STATUS, &val)) { 598 DRV_LOG(ERR, "Failed to read from RXP!"); 599 return -ENODEV; 600 } 601 if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) { 602 DRV_LOG(ERR, "RXP not initialised..."); 603 return -EBUSY; 604 } 605 ret = mlx5_devx_regex_register_read(ctx, id, 606 MLX5_RXP_RTRU_CSR_CTRL, &val); 607 if (ret) { 608 DRV_LOG(ERR, "CSR read failed!"); 609 return -1; 610 } 611 val |= MLX5_RXP_RTRU_CSR_CTRL_GO; 612 ret = mlx5_devx_regex_register_write(ctx, id, 613 MLX5_RXP_RTRU_CSR_CTRL, val); 614 if (ret) { 615 DRV_LOG(ERR, "Can't program rof file!"); 616 return -1; 617 } 618 ret = rxp_program_rof(priv, buf, len, id); 619 if (ret) { 620 DRV_LOG(ERR, "Can't program rof file!"); 621 return -1; 622 } 623 if (priv->is_bf2) { 624 ret = rxp_poll_csr_for_value 625 (ctx, &val, MLX5_RXP_RTRU_CSR_STATUS, 626 MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE, 627 MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE, 628 MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id); 629 if (ret < 0) { 630 DRV_LOG(ERR, "Rules update timeout: 0x%08X", val); 631 return ret; 632 } 633 DRV_LOG(DEBUG, "Rules update took %d cycles", ret); 634 } 635 if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, 636 &val)) { 637 DRV_LOG(ERR, "CSR read failed!"); 638 return -1; 639 } 640 val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO); 641 if (mlx5_devx_regex_register_write(ctx, id, 642 MLX5_RXP_RTRU_CSR_CTRL, val)) { 643 DRV_LOG(ERR, "CSR write failed!"); 644 return -1; 645 } 646 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &val); 647 if (ret) 648 return ret; 649 val &= ~MLX5_RXP_CSR_CTRL_INIT; 650 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, val); 651 if (ret) 652 return ret; 653 rxp_init_rtru(priv, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2); 654 if (priv->is_bf2) { 655 ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_CSR_STATUS, 656 MLX5_RXP_CSR_STATUS_INIT_DONE, 657 MLX5_RXP_CSR_STATUS_INIT_DONE, 658 MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, 659 id); 660 if (ret) { 661 DRV_LOG(ERR, "Device init failed!"); 662 return ret; 663 } 664 } 665 ret = mlnx_resume_database(priv, id); 666 if (ret < 0) { 667 DRV_LOG(ERR, "Failed to resume engine!"); 668 return ret; 669 } 670 671 return ret; 672 673 } 674 675 static int 676 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id) 677 { 678 uint32_t ctrl; 679 uint32_t reg; 680 struct ibv_context *ctx = priv->cdev->ctx; 681 int ret; 682 683 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 684 if (ret) 685 return ret; 686 if (ctrl & MLX5_RXP_CSR_CTRL_INIT) { 687 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT; 688 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, 689 ctrl); 690 if (ret) 691 return ret; 692 } 693 ctrl |= MLX5_RXP_CSR_CTRL_INIT; 694 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 695 if (ret) 696 return ret; 697 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT; 698 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); 699 if (ret) 700 return ret; 701 rte_delay_us(20000); 702 ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS, 703 MLX5_RXP_CSR_STATUS_INIT_DONE, 704 MLX5_RXP_CSR_STATUS_INIT_DONE, 705 MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id); 706 if (ret) 707 return ret; 708 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl); 709 if (ret) 710 return ret; 711 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT; 712 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, 713 ctrl); 714 if (ret) 715 return ret; 716 ret = rxp_init_rtru(priv, id, 717 MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2); 718 if (ret) 719 return ret; 720 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5, 721 ®); 722 if (ret) 723 return ret; 724 DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16, 725 reg & 0xffff); 726 if ((reg >> 16) >= priv->nb_max_matches) 727 ret = mlx5_devx_regex_register_write(ctx, id, 728 MLX5_RXP_CSR_MAX_MATCH, 729 priv->nb_max_matches); 730 else 731 ret = mlx5_devx_regex_register_write(ctx, id, 732 MLX5_RXP_CSR_MAX_MATCH, 733 (reg >> 16)); 734 ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX, 735 (reg & 0xFFFF)); 736 ret |= mlx5_devx_regex_register_write(ctx, id, 737 MLX5_RXP_CSR_MAX_LATENCY, 0); 738 ret |= mlx5_devx_regex_register_write(ctx, id, 739 MLX5_RXP_CSR_MAX_PRI_THREAD, 0); 740 return ret; 741 } 742 743 static int 744 rxp_db_setup(struct mlx5_regex_priv *priv) 745 { 746 int ret; 747 uint8_t i; 748 749 /* Setup database memories for both RXP engines + reprogram memory. */ 750 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) { 751 priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 1 << 21); 752 if (!priv->db[i].ptr) { 753 DRV_LOG(ERR, "Failed to alloc db memory!"); 754 ret = ENODEV; 755 goto tidyup_error; 756 } 757 /* Register the memory. */ 758 priv->db[i].umem.umem = mlx5_glue->devx_umem_reg 759 (priv->cdev->ctx, 760 priv->db[i].ptr, 761 MLX5_MAX_DB_SIZE, 7); 762 if (!priv->db[i].umem.umem) { 763 DRV_LOG(ERR, "Failed to register memory!"); 764 ret = ENODEV; 765 goto tidyup_error; 766 } 767 /* Ensure set all DB memory to 0's before setting up DB. */ 768 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE); 769 /* No data currently in database. */ 770 priv->db[i].len = 0; 771 priv->db[i].active = false; 772 priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED; 773 } 774 return 0; 775 tidyup_error: 776 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) { 777 if (priv->db[i].ptr) 778 rte_free(priv->db[i].ptr); 779 if (priv->db[i].umem.umem) 780 mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem); 781 } 782 return -ret; 783 } 784 785 int 786 mlx5_regex_rules_db_import(struct rte_regexdev *dev, 787 const char *rule_db, uint32_t rule_db_len) 788 { 789 struct mlx5_regex_priv *priv = dev->data->dev_private; 790 struct mlx5_rxp_ctl_rules_pgm *rules = NULL; 791 uint32_t id; 792 int ret; 793 uint32_t ver; 794 795 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) { 796 DRV_LOG(ERR, "RXP programming mode not set!"); 797 return -1; 798 } 799 if (rule_db == NULL) { 800 DRV_LOG(ERR, "Database empty!"); 801 return -ENODEV; 802 } 803 if (rule_db_len == 0) 804 return -EINVAL; 805 if (mlx5_devx_regex_register_read(priv->cdev->ctx, 0, 806 MLX5_RXP_CSR_BASE_ADDRESS, &ver)) { 807 DRV_LOG(ERR, "Failed to read Main CSRs Engine 0!"); 808 return -1; 809 } 810 /* Need to ensure RXP not busy before stop! */ 811 for (id = 0; id < priv->nb_engines; id++) { 812 ret = rxp_stop_engine(priv->cdev->ctx, id); 813 if (ret) { 814 DRV_LOG(ERR, "Can't stop engine."); 815 ret = -ENODEV; 816 goto tidyup_error; 817 } 818 ret = program_rxp_rules(priv, rule_db, rule_db_len, id); 819 if (ret < 0) { 820 DRV_LOG(ERR, "Failed to program rxp rules."); 821 ret = -ENODEV; 822 goto tidyup_error; 823 } 824 ret = rxp_start_engine(priv->cdev->ctx, id); 825 if (ret) { 826 DRV_LOG(ERR, "Can't start engine."); 827 ret = -ENODEV; 828 goto tidyup_error; 829 } 830 } 831 rte_free(rules); 832 return 0; 833 tidyup_error: 834 rte_free(rules); 835 return ret; 836 } 837 838 int 839 mlx5_regex_configure(struct rte_regexdev *dev, 840 const struct rte_regexdev_config *cfg) 841 { 842 struct mlx5_regex_priv *priv = dev->data->dev_private; 843 int ret; 844 845 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) 846 return -1; 847 priv->nb_queues = cfg->nb_queue_pairs; 848 dev->data->dev_conf.nb_queue_pairs = priv->nb_queues; 849 priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) * 850 priv->nb_queues, 0); 851 if (!priv->nb_queues) { 852 DRV_LOG(ERR, "can't allocate qps memory"); 853 rte_errno = ENOMEM; 854 return -rte_errno; 855 } 856 priv->nb_max_matches = cfg->nb_max_matches; 857 /* Setup rxp db memories. */ 858 if (rxp_db_setup(priv)) { 859 DRV_LOG(ERR, "Failed to setup RXP db memory"); 860 rte_errno = ENOMEM; 861 return -rte_errno; 862 } 863 if (cfg->rule_db != NULL) { 864 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db, 865 cfg->rule_db_len); 866 if (ret < 0) { 867 DRV_LOG(ERR, "Failed to program rxp rules."); 868 rte_errno = ENODEV; 869 goto configure_error; 870 } 871 } else 872 DRV_LOG(DEBUG, "Regex config without rules programming!"); 873 return 0; 874 configure_error: 875 if (priv->qps) 876 rte_free(priv->qps); 877 return -rte_errno; 878 } 879