1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022 NVIDIA Corporation & Affiliates 3 */ 4 5 #include "mlx5dr_internal.h" 6 7 static bool mlx5dr_matcher_requires_col_tbl(uint8_t log_num_of_rules) 8 { 9 /* Collision table concatenation is done only for large rule tables */ 10 return log_num_of_rules > MLX5DR_MATCHER_ASSURED_RULES_TH; 11 } 12 13 static uint8_t mlx5dr_matcher_rules_to_tbl_depth(uint8_t log_num_of_rules) 14 { 15 if (mlx5dr_matcher_requires_col_tbl(log_num_of_rules)) 16 return MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH; 17 18 /* For small rule tables we use a single deep table to assure insertion */ 19 return RTE_MIN(log_num_of_rules, MLX5DR_MATCHER_ASSURED_COL_TBL_DEPTH); 20 } 21 22 static int mlx5dr_matcher_create_end_ft(struct mlx5dr_matcher *matcher) 23 { 24 struct mlx5dr_table *tbl = matcher->tbl; 25 26 matcher->end_ft = mlx5dr_table_create_default_ft(tbl); 27 if (!matcher->end_ft) { 28 DR_LOG(ERR, "Failed to create matcher end flow table"); 29 return rte_errno; 30 } 31 return 0; 32 } 33 34 static void mlx5dr_matcher_destroy_end_ft(struct mlx5dr_matcher *matcher) 35 { 36 mlx5dr_table_destroy_default_ft(matcher->tbl, matcher->end_ft); 37 } 38 39 static int mlx5dr_matcher_free_rtc_pointing(uint32_t fw_ft_type, 40 enum mlx5dr_table_type type, 41 struct mlx5dr_devx_obj *devx_obj) 42 { 43 struct mlx5dr_cmd_ft_modify_attr ft_attr = {0}; 44 int ret; 45 46 if (type != MLX5DR_TABLE_TYPE_FDB) 47 return 0; 48 49 ft_attr.modify_fs = MLX5_IFC_MODIFY_FLOW_TABLE_RTC_ID; 50 ft_attr.type = fw_ft_type; 51 ft_attr.rtc_id_0 = 0; 52 ft_attr.rtc_id_1 = 0; 53 54 ret = mlx5dr_cmd_flow_table_modify(devx_obj, &ft_attr); 55 if (ret) { 56 DR_LOG(ERR, "Failed to disconnect previous RTC"); 57 return ret; 58 } 59 60 return 0; 61 } 62 63 static int mlx5dr_matcher_connect(struct mlx5dr_matcher *matcher) 64 { 65 struct mlx5dr_cmd_ft_modify_attr ft_attr = {0}; 66 struct mlx5dr_table *tbl = matcher->tbl; 67 struct mlx5dr_matcher *prev = NULL; 68 struct mlx5dr_matcher *next = NULL; 69 struct mlx5dr_matcher *tmp_matcher; 70 struct mlx5dr_devx_obj *ft; 71 int ret; 72 73 /* Find location in matcher list */ 74 if (LIST_EMPTY(&tbl->head)) { 75 LIST_INSERT_HEAD(&tbl->head, matcher, next); 76 goto connect; 77 } 78 79 LIST_FOREACH(tmp_matcher, &tbl->head, next) { 80 if (tmp_matcher->attr.priority > matcher->attr.priority) { 81 next = tmp_matcher; 82 break; 83 } 84 prev = tmp_matcher; 85 } 86 87 if (next) 88 LIST_INSERT_BEFORE(next, matcher, next); 89 else 90 LIST_INSERT_AFTER(prev, matcher, next); 91 92 connect: 93 ft_attr.modify_fs = MLX5_IFC_MODIFY_FLOW_TABLE_RTC_ID; 94 ft_attr.type = tbl->fw_ft_type; 95 96 /* Connect to next */ 97 if (next) { 98 if (next->match_ste.rtc_0) 99 ft_attr.rtc_id_0 = next->match_ste.rtc_0->id; 100 if (next->match_ste.rtc_1) 101 ft_attr.rtc_id_1 = next->match_ste.rtc_1->id; 102 103 ret = mlx5dr_cmd_flow_table_modify(matcher->end_ft, &ft_attr); 104 if (ret) { 105 DR_LOG(ERR, "Failed to connect new matcher to next RTC"); 106 goto remove_from_list; 107 } 108 } 109 110 /* Connect to previous */ 111 ft = prev ? prev->end_ft : tbl->ft; 112 113 if (matcher->match_ste.rtc_0) 114 ft_attr.rtc_id_0 = matcher->match_ste.rtc_0->id; 115 if (matcher->match_ste.rtc_1) 116 ft_attr.rtc_id_1 = matcher->match_ste.rtc_1->id; 117 118 ret = mlx5dr_cmd_flow_table_modify(ft, &ft_attr); 119 if (ret) { 120 DR_LOG(ERR, "Failed to connect new matcher to previous FT"); 121 goto remove_from_list; 122 } 123 124 return 0; 125 126 remove_from_list: 127 LIST_REMOVE(matcher, next); 128 return ret; 129 } 130 131 static int mlx5dr_matcher_disconnect(struct mlx5dr_matcher *matcher) 132 { 133 struct mlx5dr_cmd_ft_modify_attr ft_attr = {0}; 134 struct mlx5dr_table *tbl = matcher->tbl; 135 struct mlx5dr_matcher *tmp_matcher; 136 struct mlx5dr_devx_obj *prev_ft; 137 struct mlx5dr_matcher *next; 138 int ret; 139 140 prev_ft = matcher->tbl->ft; 141 LIST_FOREACH(tmp_matcher, &tbl->head, next) { 142 if (tmp_matcher == matcher) 143 break; 144 145 prev_ft = tmp_matcher->end_ft; 146 } 147 148 next = matcher->next.le_next; 149 150 ft_attr.modify_fs = MLX5_IFC_MODIFY_FLOW_TABLE_RTC_ID; 151 ft_attr.type = matcher->tbl->fw_ft_type; 152 153 if (next) { 154 /* Connect previous end FT to next RTC if exists */ 155 if (next->match_ste.rtc_0) 156 ft_attr.rtc_id_0 = next->match_ste.rtc_0->id; 157 if (next->match_ste.rtc_1) 158 ft_attr.rtc_id_1 = next->match_ste.rtc_1->id; 159 } else { 160 /* Matcher is last, point prev end FT to default miss */ 161 mlx5dr_cmd_set_attr_connect_miss_tbl(tbl->ctx, 162 tbl->fw_ft_type, 163 tbl->type, 164 &ft_attr); 165 } 166 167 ret = mlx5dr_cmd_flow_table_modify(prev_ft, &ft_attr); 168 if (ret) { 169 DR_LOG(ERR, "Failed to disconnect matcher"); 170 return ret; 171 } 172 173 LIST_REMOVE(matcher, next); 174 175 if (!next) { 176 /* ft no longer points to any RTC, drop refcount */ 177 ret = mlx5dr_matcher_free_rtc_pointing(tbl->fw_ft_type, 178 tbl->type, 179 prev_ft); 180 if (ret) { 181 DR_LOG(ERR, "Failed to reset last RTC refcount"); 182 return ret; 183 } 184 } 185 186 return 0; 187 } 188 189 static void mlx5dr_matcher_set_rtc_attr_sz(struct mlx5dr_matcher *matcher, 190 struct mlx5dr_cmd_rtc_create_attr *rtc_attr, 191 bool is_match_rtc, 192 bool is_mirror) 193 { 194 enum mlx5dr_matcher_flow_src flow_src = matcher->attr.optimize_flow_src; 195 struct mlx5dr_pool_chunk *ste = &matcher->action_ste.ste; 196 197 if ((flow_src == MLX5DR_MATCHER_FLOW_SRC_VPORT && !is_mirror) || 198 (flow_src == MLX5DR_MATCHER_FLOW_SRC_WIRE && is_mirror)) { 199 /* Optimize FDB RTC */ 200 rtc_attr->log_size = 0; 201 rtc_attr->log_depth = 0; 202 } else { 203 /* Keep original values */ 204 rtc_attr->log_size = is_match_rtc ? matcher->attr.table.sz_row_log : ste->order; 205 rtc_attr->log_depth = is_match_rtc ? matcher->attr.table.sz_col_log : 0; 206 } 207 } 208 209 static int mlx5dr_matcher_create_rtc(struct mlx5dr_matcher *matcher, 210 bool is_match_rtc) 211 { 212 const char *rtc_type_str = is_match_rtc ? "match" : "action"; 213 struct mlx5dr_cmd_rtc_create_attr rtc_attr = {0}; 214 struct mlx5dr_context *ctx = matcher->tbl->ctx; 215 struct mlx5dr_action_default_stc *default_stc; 216 struct mlx5dr_table *tbl = matcher->tbl; 217 struct mlx5dr_devx_obj **rtc_0, **rtc_1; 218 struct mlx5dr_pool *ste_pool, *stc_pool; 219 struct mlx5dr_devx_obj *devx_obj; 220 struct mlx5dr_pool_chunk *ste; 221 int ret; 222 223 if (is_match_rtc) { 224 rtc_0 = &matcher->match_ste.rtc_0; 225 rtc_1 = &matcher->match_ste.rtc_1; 226 ste_pool = matcher->match_ste.pool; 227 ste = &matcher->match_ste.ste; 228 ste->order = matcher->attr.table.sz_col_log + 229 matcher->attr.table.sz_row_log; 230 rtc_attr.log_size = matcher->attr.table.sz_row_log; 231 rtc_attr.log_depth = matcher->attr.table.sz_col_log; 232 rtc_attr.update_index_mode = MLX5_IFC_RTC_STE_UPDATE_MODE_BY_HASH; 233 /* The first match template is used since all share the same definer */ 234 rtc_attr.definer_id = mlx5dr_definer_get_id(matcher->mt[0]->definer); 235 rtc_attr.is_jumbo = mlx5dr_definer_is_jumbo(matcher->mt[0]->definer); 236 rtc_attr.miss_ft_id = matcher->end_ft->id; 237 /* Match pool requires implicit allocation */ 238 ret = mlx5dr_pool_chunk_alloc(ste_pool, ste); 239 if (ret) { 240 DR_LOG(ERR, "Failed to allocate STE for %s RTC", rtc_type_str); 241 return ret; 242 } 243 } else { 244 rtc_0 = &matcher->action_ste.rtc_0; 245 rtc_1 = &matcher->action_ste.rtc_1; 246 ste_pool = matcher->action_ste.pool; 247 ste = &matcher->action_ste.ste; 248 ste->order = rte_log2_u32(matcher->action_ste.max_stes) + 249 matcher->attr.table.sz_row_log; 250 rtc_attr.log_size = ste->order; 251 rtc_attr.log_depth = 0; 252 rtc_attr.update_index_mode = MLX5_IFC_RTC_STE_UPDATE_MODE_BY_OFFSET; 253 /* The action STEs use the default always hit definer */ 254 rtc_attr.definer_id = ctx->caps->trivial_match_definer; 255 rtc_attr.is_jumbo = false; 256 rtc_attr.miss_ft_id = 0; 257 } 258 259 devx_obj = mlx5dr_pool_chunk_get_base_devx_obj(ste_pool, ste); 260 261 rtc_attr.pd = ctx->pd_num; 262 rtc_attr.ste_base = devx_obj->id; 263 rtc_attr.ste_offset = ste->offset; 264 rtc_attr.table_type = mlx5dr_table_get_res_fw_ft_type(tbl->type, false); 265 mlx5dr_matcher_set_rtc_attr_sz(matcher, &rtc_attr, is_match_rtc, false); 266 267 /* STC is a single resource (devx_obj), use any STC for the ID */ 268 stc_pool = ctx->stc_pool[tbl->type]; 269 default_stc = ctx->common_res[tbl->type].default_stc; 270 devx_obj = mlx5dr_pool_chunk_get_base_devx_obj(stc_pool, &default_stc->default_hit); 271 rtc_attr.stc_base = devx_obj->id; 272 273 *rtc_0 = mlx5dr_cmd_rtc_create(ctx->ibv_ctx, &rtc_attr); 274 if (!*rtc_0) { 275 DR_LOG(ERR, "Failed to create matcher %s RTC", rtc_type_str); 276 goto free_ste; 277 } 278 279 if (tbl->type == MLX5DR_TABLE_TYPE_FDB) { 280 devx_obj = mlx5dr_pool_chunk_get_base_devx_obj_mirror(ste_pool, ste); 281 rtc_attr.ste_base = devx_obj->id; 282 rtc_attr.table_type = mlx5dr_table_get_res_fw_ft_type(tbl->type, true); 283 284 devx_obj = mlx5dr_pool_chunk_get_base_devx_obj_mirror(stc_pool, &default_stc->default_hit); 285 rtc_attr.stc_base = devx_obj->id; 286 mlx5dr_matcher_set_rtc_attr_sz(matcher, &rtc_attr, is_match_rtc, true); 287 288 *rtc_1 = mlx5dr_cmd_rtc_create(ctx->ibv_ctx, &rtc_attr); 289 if (!*rtc_1) { 290 DR_LOG(ERR, "Failed to create peer matcher %s RTC0", rtc_type_str); 291 goto destroy_rtc_0; 292 } 293 } 294 295 return 0; 296 297 destroy_rtc_0: 298 mlx5dr_cmd_destroy_obj(*rtc_0); 299 free_ste: 300 if (is_match_rtc) 301 mlx5dr_pool_chunk_free(ste_pool, ste); 302 return rte_errno; 303 } 304 305 static void mlx5dr_matcher_destroy_rtc(struct mlx5dr_matcher *matcher, 306 bool is_match_rtc) 307 { 308 struct mlx5dr_table *tbl = matcher->tbl; 309 struct mlx5dr_devx_obj *rtc_0, *rtc_1; 310 struct mlx5dr_pool_chunk *ste; 311 struct mlx5dr_pool *ste_pool; 312 313 if (is_match_rtc) { 314 rtc_0 = matcher->match_ste.rtc_0; 315 rtc_1 = matcher->match_ste.rtc_1; 316 ste_pool = matcher->match_ste.pool; 317 ste = &matcher->match_ste.ste; 318 } else { 319 rtc_0 = matcher->action_ste.rtc_0; 320 rtc_1 = matcher->action_ste.rtc_1; 321 ste_pool = matcher->action_ste.pool; 322 ste = &matcher->action_ste.ste; 323 } 324 325 if (tbl->type == MLX5DR_TABLE_TYPE_FDB) 326 mlx5dr_cmd_destroy_obj(rtc_1); 327 328 mlx5dr_cmd_destroy_obj(rtc_0); 329 if (is_match_rtc) 330 mlx5dr_pool_chunk_free(ste_pool, ste); 331 } 332 333 static void mlx5dr_matcher_set_pool_attr(struct mlx5dr_pool_attr *attr, 334 struct mlx5dr_matcher *matcher) 335 { 336 switch (matcher->attr.optimize_flow_src) { 337 case MLX5DR_MATCHER_FLOW_SRC_VPORT: 338 attr->opt_type = MLX5DR_POOL_OPTIMIZE_ORIG; 339 break; 340 case MLX5DR_MATCHER_FLOW_SRC_WIRE: 341 attr->opt_type = MLX5DR_POOL_OPTIMIZE_MIRROR; 342 break; 343 default: 344 break; 345 } 346 } 347 348 static int mlx5dr_matcher_bind_at(struct mlx5dr_matcher *matcher) 349 { 350 bool is_jumbo = mlx5dr_definer_is_jumbo(matcher->mt[0]->definer); 351 struct mlx5dr_cmd_stc_modify_attr stc_attr = {0}; 352 struct mlx5dr_table *tbl = matcher->tbl; 353 struct mlx5dr_pool_attr pool_attr = {0}; 354 struct mlx5dr_context *ctx = tbl->ctx; 355 uint32_t required_stes; 356 int i, ret; 357 bool valid; 358 359 for (i = 0; i < matcher->num_of_at; i++) { 360 struct mlx5dr_action_template *at = matcher->at[i]; 361 362 /* Check if action combinabtion is valid */ 363 valid = mlx5dr_action_check_combo(at->action_type_arr, matcher->tbl->type); 364 if (!valid) { 365 DR_LOG(ERR, "Invalid combination in action template %d", i); 366 return rte_errno; 367 } 368 369 /* Process action template to setters */ 370 ret = mlx5dr_action_template_process(at); 371 if (ret) { 372 DR_LOG(ERR, "Failed to process action template %d", i); 373 return rte_errno; 374 } 375 376 required_stes = at->num_of_action_stes - (!is_jumbo || at->only_term); 377 matcher->action_ste.max_stes = RTE_MAX(matcher->action_ste.max_stes, required_stes); 378 379 /* Future: Optimize reparse */ 380 } 381 382 /* There are no additioanl STEs required for matcher */ 383 if (!matcher->action_ste.max_stes) 384 return 0; 385 386 /* Allocate action STE mempool */ 387 pool_attr.table_type = tbl->type; 388 pool_attr.pool_type = MLX5DR_POOL_TYPE_STE; 389 pool_attr.flags = MLX5DR_POOL_FLAGS_FOR_STE_ACTION_POOL; 390 pool_attr.alloc_log_sz = rte_log2_u32(matcher->action_ste.max_stes) + 391 matcher->attr.table.sz_row_log; 392 mlx5dr_matcher_set_pool_attr(&pool_attr, matcher); 393 matcher->action_ste.pool = mlx5dr_pool_create(ctx, &pool_attr); 394 if (!matcher->action_ste.pool) { 395 DR_LOG(ERR, "Failed to create action ste pool"); 396 return rte_errno; 397 } 398 399 /* Allocate action RTC */ 400 ret = mlx5dr_matcher_create_rtc(matcher, false); 401 if (ret) { 402 DR_LOG(ERR, "Failed to create action RTC"); 403 goto free_ste_pool; 404 } 405 406 /* Allocate STC for jumps to STE */ 407 stc_attr.action_offset = MLX5DR_ACTION_OFFSET_HIT; 408 stc_attr.action_type = MLX5_IFC_STC_ACTION_TYPE_JUMP_TO_STE_TABLE; 409 stc_attr.ste_table.ste = matcher->action_ste.ste; 410 stc_attr.ste_table.ste_pool = matcher->action_ste.pool; 411 stc_attr.ste_table.match_definer_id = ctx->caps->trivial_match_definer; 412 413 ret = mlx5dr_action_alloc_single_stc(ctx, &stc_attr, tbl->type, 414 &matcher->action_ste.stc); 415 if (ret) { 416 DR_LOG(ERR, "Failed to create action jump to table STC"); 417 goto free_rtc; 418 } 419 420 return 0; 421 422 free_rtc: 423 mlx5dr_matcher_destroy_rtc(matcher, false); 424 free_ste_pool: 425 mlx5dr_pool_destroy(matcher->action_ste.pool); 426 return rte_errno; 427 } 428 429 static void mlx5dr_matcher_unbind_at(struct mlx5dr_matcher *matcher) 430 { 431 struct mlx5dr_table *tbl = matcher->tbl; 432 433 if (!matcher->action_ste.max_stes) 434 return; 435 436 mlx5dr_action_free_single_stc(tbl->ctx, tbl->type, &matcher->action_ste.stc); 437 mlx5dr_matcher_destroy_rtc(matcher, false); 438 mlx5dr_pool_destroy(matcher->action_ste.pool); 439 } 440 441 static int mlx5dr_matcher_bind_mt(struct mlx5dr_matcher *matcher) 442 { 443 struct mlx5dr_context *ctx = matcher->tbl->ctx; 444 struct mlx5dr_pool_attr pool_attr = {0}; 445 int i, created = 0; 446 int ret = -1; 447 448 for (i = 0; i < matcher->num_of_mt; i++) { 449 /* Get a definer for each match template */ 450 ret = mlx5dr_definer_get(ctx, matcher->mt[i]); 451 if (ret) 452 goto definer_put; 453 454 created++; 455 456 /* Verify all templates produce the same definer */ 457 if (i == 0) 458 continue; 459 460 ret = mlx5dr_definer_compare(matcher->mt[i]->definer, 461 matcher->mt[i - 1]->definer); 462 if (ret) { 463 DR_LOG(ERR, "Match templates cannot be used on the same matcher"); 464 rte_errno = ENOTSUP; 465 goto definer_put; 466 } 467 } 468 469 /* Create an STE pool per matcher*/ 470 pool_attr.pool_type = MLX5DR_POOL_TYPE_STE; 471 pool_attr.flags = MLX5DR_POOL_FLAGS_FOR_MATCHER_STE_POOL; 472 pool_attr.table_type = matcher->tbl->type; 473 pool_attr.alloc_log_sz = matcher->attr.table.sz_col_log + 474 matcher->attr.table.sz_row_log; 475 mlx5dr_matcher_set_pool_attr(&pool_attr, matcher); 476 477 matcher->match_ste.pool = mlx5dr_pool_create(ctx, &pool_attr); 478 if (!matcher->match_ste.pool) { 479 DR_LOG(ERR, "Failed to allocate matcher STE pool"); 480 goto definer_put; 481 } 482 483 return 0; 484 485 definer_put: 486 while (created--) 487 mlx5dr_definer_put(matcher->mt[created]); 488 489 return ret; 490 } 491 492 static void mlx5dr_matcher_unbind_mt(struct mlx5dr_matcher *matcher) 493 { 494 int i; 495 496 for (i = 0; i < matcher->num_of_mt; i++) 497 mlx5dr_definer_put(matcher->mt[i]); 498 499 mlx5dr_pool_destroy(matcher->match_ste.pool); 500 } 501 502 static int 503 mlx5dr_matcher_process_attr(struct mlx5dr_cmd_query_caps *caps, 504 struct mlx5dr_matcher *matcher, 505 bool is_root) 506 { 507 struct mlx5dr_matcher_attr *attr = &matcher->attr; 508 509 if (matcher->tbl->type != MLX5DR_TABLE_TYPE_FDB && attr->optimize_flow_src) { 510 DR_LOG(ERR, "NIC domain doesn't support flow_src"); 511 goto not_supported; 512 } 513 514 if (is_root) { 515 if (attr->mode != MLX5DR_MATCHER_RESOURCE_MODE_RULE) { 516 DR_LOG(ERR, "Root matcher supports only rule resource mode"); 517 goto not_supported; 518 } 519 if (attr->optimize_flow_src) { 520 DR_LOG(ERR, "Root matcher can't specify FDB direction"); 521 goto not_supported; 522 } 523 return 0; 524 } 525 526 /* Convert number of rules to the required depth */ 527 if (attr->mode == MLX5DR_MATCHER_RESOURCE_MODE_RULE) 528 attr->table.sz_col_log = mlx5dr_matcher_rules_to_tbl_depth(attr->rule.num_log); 529 530 if (attr->table.sz_col_log > caps->rtc_log_depth_max) { 531 DR_LOG(ERR, "Matcher depth exceeds limit %d", caps->rtc_log_depth_max); 532 goto not_supported; 533 } 534 535 if (attr->table.sz_col_log + attr->table.sz_row_log > caps->ste_alloc_log_max) { 536 DR_LOG(ERR, "Total matcher size exceeds limit %d", caps->ste_alloc_log_max); 537 goto not_supported; 538 } 539 540 if (attr->table.sz_col_log + attr->table.sz_row_log < caps->ste_alloc_log_gran) { 541 DR_LOG(ERR, "Total matcher size below limit %d", caps->ste_alloc_log_gran); 542 goto not_supported; 543 } 544 545 return 0; 546 547 not_supported: 548 rte_errno = EOPNOTSUPP; 549 return rte_errno; 550 } 551 552 static int mlx5dr_matcher_create_and_connect(struct mlx5dr_matcher *matcher) 553 { 554 int ret; 555 556 /* Select and create the definers for current matcher */ 557 ret = mlx5dr_matcher_bind_mt(matcher); 558 if (ret) 559 return ret; 560 561 /* Calculate and verify action combination */ 562 ret = mlx5dr_matcher_bind_at(matcher); 563 if (ret) 564 goto unbind_mt; 565 566 /* Create matcher end flow table anchor */ 567 ret = mlx5dr_matcher_create_end_ft(matcher); 568 if (ret) 569 goto unbind_at; 570 571 /* Allocate the RTC for the new matcher */ 572 ret = mlx5dr_matcher_create_rtc(matcher, true); 573 if (ret) 574 goto destroy_end_ft; 575 576 /* Connect the matcher to the matcher list */ 577 ret = mlx5dr_matcher_connect(matcher); 578 if (ret) 579 goto destroy_rtc; 580 581 return 0; 582 583 destroy_rtc: 584 mlx5dr_matcher_destroy_rtc(matcher, true); 585 destroy_end_ft: 586 mlx5dr_matcher_destroy_end_ft(matcher); 587 unbind_at: 588 mlx5dr_matcher_unbind_at(matcher); 589 unbind_mt: 590 mlx5dr_matcher_unbind_mt(matcher); 591 return ret; 592 } 593 594 static void mlx5dr_matcher_destroy_and_disconnect(struct mlx5dr_matcher *matcher) 595 { 596 mlx5dr_matcher_disconnect(matcher); 597 mlx5dr_matcher_destroy_rtc(matcher, true); 598 mlx5dr_matcher_destroy_end_ft(matcher); 599 mlx5dr_matcher_unbind_at(matcher); 600 mlx5dr_matcher_unbind_mt(matcher); 601 } 602 603 static int 604 mlx5dr_matcher_create_col_matcher(struct mlx5dr_matcher *matcher) 605 { 606 struct mlx5dr_context *ctx = matcher->tbl->ctx; 607 struct mlx5dr_matcher *col_matcher; 608 int ret; 609 610 if (matcher->attr.mode != MLX5DR_MATCHER_RESOURCE_MODE_RULE) 611 return 0; 612 613 if (!mlx5dr_matcher_requires_col_tbl(matcher->attr.rule.num_log)) 614 return 0; 615 616 col_matcher = simple_calloc(1, sizeof(*matcher)); 617 if (!col_matcher) { 618 rte_errno = ENOMEM; 619 return rte_errno; 620 } 621 622 col_matcher->tbl = matcher->tbl; 623 col_matcher->num_of_mt = matcher->num_of_mt; 624 memcpy(col_matcher->mt, matcher->mt, matcher->num_of_mt * sizeof(*matcher->mt)); 625 col_matcher->num_of_at = matcher->num_of_at; 626 memcpy(col_matcher->at, matcher->at, matcher->num_of_at * sizeof(*matcher->at)); 627 628 col_matcher->attr.priority = matcher->attr.priority; 629 col_matcher->attr.mode = MLX5DR_MATCHER_RESOURCE_MODE_HTABLE; 630 col_matcher->attr.optimize_flow_src = matcher->attr.optimize_flow_src; 631 col_matcher->attr.table.sz_row_log = matcher->attr.rule.num_log; 632 col_matcher->attr.table.sz_col_log = MLX5DR_MATCHER_ASSURED_COL_TBL_DEPTH; 633 if (col_matcher->attr.table.sz_row_log > MLX5DR_MATCHER_ASSURED_ROW_RATIO) 634 col_matcher->attr.table.sz_row_log -= MLX5DR_MATCHER_ASSURED_ROW_RATIO; 635 636 ret = mlx5dr_matcher_process_attr(ctx->caps, col_matcher, false); 637 if (ret) 638 goto free_col_matcher; 639 640 ret = mlx5dr_matcher_create_and_connect(col_matcher); 641 if (ret) 642 goto free_col_matcher; 643 644 matcher->col_matcher = col_matcher; 645 646 return 0; 647 648 free_col_matcher: 649 simple_free(col_matcher); 650 DR_LOG(ERR, "Failed to create assured collision matcher"); 651 return ret; 652 } 653 654 static void 655 mlx5dr_matcher_destroy_col_matcher(struct mlx5dr_matcher *matcher) 656 { 657 if (matcher->attr.mode != MLX5DR_MATCHER_RESOURCE_MODE_RULE) 658 return; 659 660 if (matcher->col_matcher) { 661 mlx5dr_matcher_destroy_and_disconnect(matcher->col_matcher); 662 simple_free(matcher->col_matcher); 663 } 664 } 665 666 static int mlx5dr_matcher_init(struct mlx5dr_matcher *matcher) 667 { 668 struct mlx5dr_context *ctx = matcher->tbl->ctx; 669 int ret; 670 671 pthread_spin_lock(&ctx->ctrl_lock); 672 673 /* Allocate matcher resource and connect to the packet pipe */ 674 ret = mlx5dr_matcher_create_and_connect(matcher); 675 if (ret) 676 goto unlock_err; 677 678 /* Create additional matcher for collision handling */ 679 ret = mlx5dr_matcher_create_col_matcher(matcher); 680 if (ret) 681 goto destory_and_disconnect; 682 683 pthread_spin_unlock(&ctx->ctrl_lock); 684 685 return 0; 686 687 destory_and_disconnect: 688 mlx5dr_matcher_destroy_and_disconnect(matcher); 689 unlock_err: 690 pthread_spin_unlock(&ctx->ctrl_lock); 691 return ret; 692 } 693 694 static int mlx5dr_matcher_uninit(struct mlx5dr_matcher *matcher) 695 { 696 struct mlx5dr_context *ctx = matcher->tbl->ctx; 697 698 pthread_spin_lock(&ctx->ctrl_lock); 699 mlx5dr_matcher_destroy_col_matcher(matcher); 700 mlx5dr_matcher_destroy_and_disconnect(matcher); 701 pthread_spin_unlock(&ctx->ctrl_lock); 702 703 return 0; 704 } 705 706 static int mlx5dr_matcher_init_root(struct mlx5dr_matcher *matcher) 707 { 708 enum mlx5dr_table_type type = matcher->tbl->type; 709 struct mlx5dr_context *ctx = matcher->tbl->ctx; 710 struct mlx5dv_flow_matcher_attr attr = {0}; 711 struct mlx5dv_flow_match_parameters *mask; 712 struct mlx5_flow_attr flow_attr = {0}; 713 struct rte_flow_error rte_error; 714 uint8_t match_criteria; 715 int ret; 716 717 #ifdef HAVE_MLX5DV_FLOW_MATCHER_FT_TYPE 718 attr.comp_mask = MLX5DV_FLOW_MATCHER_MASK_FT_TYPE; 719 720 switch (type) { 721 case MLX5DR_TABLE_TYPE_NIC_RX: 722 attr.ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 723 break; 724 case MLX5DR_TABLE_TYPE_NIC_TX: 725 attr.ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX; 726 break; 727 case MLX5DR_TABLE_TYPE_FDB: 728 attr.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 729 break; 730 default: 731 assert(0); 732 break; 733 } 734 #endif 735 736 if (matcher->attr.priority > UINT16_MAX) { 737 DR_LOG(ERR, "Root matcher priority exceeds allowed limit"); 738 rte_errno = EINVAL; 739 return rte_errno; 740 } 741 742 mask = simple_calloc(1, MLX5_ST_SZ_BYTES(fte_match_param) + 743 offsetof(struct mlx5dv_flow_match_parameters, match_buf)); 744 if (!mask) { 745 rte_errno = ENOMEM; 746 return rte_errno; 747 } 748 749 flow_attr.tbl_type = type; 750 751 /* On root table matcher, only a single match template is supported */ 752 ret = flow_dv_translate_items_hws(matcher->mt[0]->items, 753 &flow_attr, mask->match_buf, 754 MLX5_SET_MATCHER_HS_M, NULL, 755 &match_criteria, 756 &rte_error); 757 if (ret) { 758 DR_LOG(ERR, "Failed to convert items to PRM [%s]", rte_error.message); 759 goto free_mask; 760 } 761 762 mask->match_sz = MLX5_ST_SZ_BYTES(fte_match_param); 763 attr.match_mask = mask; 764 attr.match_criteria_enable = match_criteria; 765 attr.type = IBV_FLOW_ATTR_NORMAL; 766 attr.priority = matcher->attr.priority; 767 768 matcher->dv_matcher = 769 mlx5_glue->dv_create_flow_matcher_root(ctx->ibv_ctx, &attr); 770 if (!matcher->dv_matcher) { 771 DR_LOG(ERR, "Failed to create DV flow matcher"); 772 rte_errno = errno; 773 goto free_mask; 774 } 775 776 simple_free(mask); 777 778 pthread_spin_lock(&ctx->ctrl_lock); 779 LIST_INSERT_HEAD(&matcher->tbl->head, matcher, next); 780 pthread_spin_unlock(&ctx->ctrl_lock); 781 782 return 0; 783 784 free_mask: 785 simple_free(mask); 786 return rte_errno; 787 } 788 789 static int mlx5dr_matcher_uninit_root(struct mlx5dr_matcher *matcher) 790 { 791 struct mlx5dr_context *ctx = matcher->tbl->ctx; 792 int ret; 793 794 pthread_spin_lock(&ctx->ctrl_lock); 795 LIST_REMOVE(matcher, next); 796 pthread_spin_unlock(&ctx->ctrl_lock); 797 798 ret = mlx5_glue->dv_destroy_flow_matcher_root(matcher->dv_matcher); 799 if (ret) { 800 DR_LOG(ERR, "Failed to Destroy DV flow matcher"); 801 rte_errno = errno; 802 } 803 804 return ret; 805 } 806 807 static int 808 mlx5dr_matcher_check_template(uint8_t num_of_mt, uint8_t num_of_at, bool is_root) 809 { 810 uint8_t max_num_of_mt; 811 812 max_num_of_mt = is_root ? 813 MLX5DR_MATCHER_MAX_MT_ROOT : 814 MLX5DR_MATCHER_MAX_MT; 815 816 if (!num_of_mt || !num_of_at) { 817 DR_LOG(ERR, "Number of action/match template cannot be zero"); 818 goto out_not_sup; 819 } 820 821 if (num_of_at > MLX5DR_MATCHER_MAX_AT) { 822 DR_LOG(ERR, "Number of action templates exceeds limit"); 823 goto out_not_sup; 824 } 825 826 if (num_of_mt > max_num_of_mt) { 827 DR_LOG(ERR, "Number of match templates exceeds limit"); 828 goto out_not_sup; 829 } 830 831 return 0; 832 833 out_not_sup: 834 rte_errno = ENOTSUP; 835 return rte_errno; 836 } 837 838 struct mlx5dr_matcher * 839 mlx5dr_matcher_create(struct mlx5dr_table *tbl, 840 struct mlx5dr_match_template *mt[], 841 uint8_t num_of_mt, 842 struct mlx5dr_action_template *at[], 843 uint8_t num_of_at, 844 struct mlx5dr_matcher_attr *attr) 845 { 846 bool is_root = mlx5dr_table_is_root(tbl); 847 struct mlx5dr_matcher *matcher; 848 int ret; 849 850 ret = mlx5dr_matcher_check_template(num_of_mt, num_of_at, is_root); 851 if (ret) 852 return NULL; 853 854 matcher = simple_calloc(1, sizeof(*matcher)); 855 if (!matcher) { 856 rte_errno = ENOMEM; 857 return NULL; 858 } 859 860 matcher->tbl = tbl; 861 matcher->attr = *attr; 862 matcher->num_of_mt = num_of_mt; 863 memcpy(matcher->mt, mt, num_of_mt * sizeof(*mt)); 864 matcher->num_of_at = num_of_at; 865 memcpy(matcher->at, at, num_of_at * sizeof(*at)); 866 867 ret = mlx5dr_matcher_process_attr(tbl->ctx->caps, matcher, is_root); 868 if (ret) 869 goto free_matcher; 870 871 if (is_root) 872 ret = mlx5dr_matcher_init_root(matcher); 873 else 874 ret = mlx5dr_matcher_init(matcher); 875 876 if (ret) { 877 DR_LOG(ERR, "Failed to initialise matcher: %d", ret); 878 goto free_matcher; 879 } 880 881 return matcher; 882 883 free_matcher: 884 simple_free(matcher); 885 return NULL; 886 } 887 888 int mlx5dr_matcher_destroy(struct mlx5dr_matcher *matcher) 889 { 890 if (mlx5dr_table_is_root(matcher->tbl)) 891 mlx5dr_matcher_uninit_root(matcher); 892 else 893 mlx5dr_matcher_uninit(matcher); 894 895 simple_free(matcher); 896 return 0; 897 } 898 899 struct mlx5dr_match_template * 900 mlx5dr_match_template_create(const struct rte_flow_item items[], 901 enum mlx5dr_match_template_flags flags) 902 { 903 struct mlx5dr_match_template *mt; 904 struct rte_flow_error error; 905 int ret, len; 906 907 if (flags > MLX5DR_MATCH_TEMPLATE_FLAG_RELAXED_MATCH) { 908 DR_LOG(ERR, "Unsupported match template flag provided"); 909 rte_errno = EINVAL; 910 return NULL; 911 } 912 913 mt = simple_calloc(1, sizeof(*mt)); 914 if (!mt) { 915 DR_LOG(ERR, "Failed to allocate match template"); 916 rte_errno = ENOMEM; 917 return NULL; 918 } 919 920 mt->flags = flags; 921 922 /* Duplicate the user given items */ 923 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error); 924 if (ret <= 0) { 925 DR_LOG(ERR, "Unable to process items (%s): %s", 926 error.message ? error.message : "unspecified", 927 strerror(rte_errno)); 928 goto free_template; 929 } 930 931 len = RTE_ALIGN(ret, 16); 932 mt->items = simple_calloc(1, len); 933 if (!mt->items) { 934 DR_LOG(ERR, "Failed to allocate item copy"); 935 rte_errno = ENOMEM; 936 goto free_template; 937 } 938 939 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, mt->items, ret, items, &error); 940 if (ret <= 0) 941 goto free_dst; 942 943 return mt; 944 945 free_dst: 946 simple_free(mt->items); 947 free_template: 948 simple_free(mt); 949 return NULL; 950 } 951 952 int mlx5dr_match_template_destroy(struct mlx5dr_match_template *mt) 953 { 954 assert(!mt->refcount); 955 simple_free(mt->items); 956 simple_free(mt); 957 return 0; 958 } 959