1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2024 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_flow.h> 6 7 #include <mlx5_malloc.h> 8 #include "mlx5.h" 9 #include "mlx5_defs.h" 10 #include "mlx5_flow.h" 11 #include "mlx5_rx.h" 12 #include "rte_common.h" 13 14 #ifdef HAVE_MLX5_HWS_SUPPORT 15 16 struct mlx5_nta_rss_ctx { 17 struct rte_eth_dev *dev; 18 struct rte_flow_attr *attr; 19 struct rte_flow_item *pattern; 20 struct rte_flow_action *actions; 21 const struct rte_flow_action_rss *rss_conf; 22 struct rte_flow_error *error; 23 struct mlx5_nta_rss_flow_head *head; 24 uint64_t pattern_flags; 25 enum mlx5_flow_type flow_type; 26 bool external; 27 }; 28 29 #define MLX5_RSS_PTYPE_ITEM_INDEX 0 30 #ifdef MLX5_RSS_PTYPE_DEBUG 31 #define MLX5_RSS_PTYPE_ACTION_INDEX 1 32 #else 33 #define MLX5_RSS_PTYPE_ACTION_INDEX 0 34 #endif 35 36 #define MLX5_RSS_PTYPE_ITEMS_NUM (MLX5_RSS_PTYPE_ITEM_INDEX + 2) 37 #define MLX5_RSS_PTYPE_ACTIONS_NUM (MLX5_RSS_PTYPE_ACTION_INDEX + 2) 38 39 static int 40 mlx5_nta_ptype_rss_flow_create(struct mlx5_nta_rss_ctx *ctx, 41 uint32_t ptype, uint64_t rss_type) 42 { 43 int ret; 44 struct rte_flow_hw *flow; 45 struct rte_flow_item_ptype *ptype_spec = (void *)(uintptr_t) 46 ctx->pattern[MLX5_RSS_PTYPE_ITEM_INDEX].spec; 47 struct rte_flow_action_rss *rss_conf = (void *)(uintptr_t) 48 ctx->actions[MLX5_RSS_PTYPE_ACTION_INDEX].conf; 49 bool dbg_log = rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG); 50 uint32_t mark_id = 0; 51 #ifdef MLX5_RSS_PTYPE_DEBUG 52 struct rte_flow_action_mark *mark = (void *)(uintptr_t) 53 ctx->actions[MLX5_RSS_PTYPE_ACTION_INDEX - 1].conf; 54 55 /* 56 * Inner L3 and L4 ptype values are too large for 24bit mark 57 */ 58 mark->id = 59 ((ptype & (RTE_PTYPE_INNER_L3_MASK | RTE_PTYPE_INNER_L4_MASK)) == ptype) ? 60 ptype >> 20 : ptype; 61 mark_id = mark->id; 62 dbg_log = true; 63 #endif 64 ptype_spec->packet_type = ptype; 65 rss_conf->types = rss_type; 66 ret = flow_hw_create_flow(ctx->dev, MLX5_FLOW_TYPE_GEN, ctx->attr, 67 ctx->pattern, ctx->actions, 68 MLX5_FLOW_ITEM_PTYPE, MLX5_FLOW_ACTION_RSS, 69 ctx->external, &flow, ctx->error); 70 if (flow) { 71 SLIST_INSERT_HEAD(ctx->head, flow, nt2hws->next); 72 if (dbg_log) { 73 DRV_LOG(NOTICE, 74 "PTYPE RSS: group %u ptype spec %#x rss types %#lx mark %#x\n", 75 ctx->attr->group, ptype_spec->packet_type, 76 (unsigned long)rss_conf->types, mark_id); 77 } 78 } 79 return ret; 80 } 81 82 /* 83 * Call conditions: 84 * * Flow pattern did not include outer L3 and L4 items. 85 * * RSS configuration had L3 hash types. 86 */ 87 static struct rte_flow_hw * 88 mlx5_hw_rss_expand_l3(struct mlx5_nta_rss_ctx *rss_ctx) 89 { 90 int ret; 91 int ptype_ip4, ptype_ip6; 92 uint64_t rss_types = rte_eth_rss_hf_refine(rss_ctx->rss_conf->types); 93 94 if (rss_ctx->rss_conf->level < 2) { 95 ptype_ip4 = RTE_PTYPE_L3_IPV4; 96 ptype_ip6 = RTE_PTYPE_L3_IPV6; 97 } else { 98 ptype_ip4 = RTE_PTYPE_INNER_L3_IPV4; 99 ptype_ip6 = RTE_PTYPE_INNER_L3_IPV6; 100 } 101 if (rss_types & MLX5_IPV4_LAYER_TYPES) { 102 ret = mlx5_nta_ptype_rss_flow_create 103 (rss_ctx, ptype_ip4, (rss_types & ~MLX5_IPV6_LAYER_TYPES)); 104 if (ret) 105 goto error; 106 } 107 if (rss_types & MLX5_IPV6_LAYER_TYPES) { 108 ret = mlx5_nta_ptype_rss_flow_create 109 (rss_ctx, ptype_ip6, rss_types & ~MLX5_IPV4_LAYER_TYPES); 110 if (ret) 111 goto error; 112 } 113 return SLIST_FIRST(rss_ctx->head); 114 115 error: 116 flow_hw_list_destroy(rss_ctx->dev, rss_ctx->flow_type, 117 (uintptr_t)SLIST_FIRST(rss_ctx->head)); 118 return NULL; 119 } 120 121 static void 122 mlx5_nta_rss_expand_l3_l4(struct mlx5_nta_rss_ctx *rss_ctx, 123 uint64_t rss_types, uint64_t rss_l3_types) 124 { 125 int ret; 126 int ptype_l3, ptype_l4_udp, ptype_l4_tcp, ptype_l4_esp = 0; 127 uint64_t rss = rss_types & 128 ~(rss_l3_types == MLX5_IPV4_LAYER_TYPES ? 129 MLX5_IPV6_LAYER_TYPES : MLX5_IPV4_LAYER_TYPES); 130 131 132 if (rss_ctx->rss_conf->level < 2) { 133 ptype_l3 = rss_l3_types == MLX5_IPV4_LAYER_TYPES ? 134 RTE_PTYPE_L3_IPV4 : RTE_PTYPE_L3_IPV6; 135 ptype_l4_esp = RTE_PTYPE_TUNNEL_ESP; 136 ptype_l4_udp = RTE_PTYPE_L4_UDP; 137 ptype_l4_tcp = RTE_PTYPE_L4_TCP; 138 } else { 139 ptype_l3 = rss_l3_types == MLX5_IPV4_LAYER_TYPES ? 140 RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_INNER_L3_IPV6; 141 ptype_l4_udp = RTE_PTYPE_INNER_L4_UDP; 142 ptype_l4_tcp = RTE_PTYPE_INNER_L4_TCP; 143 } 144 if (rss_types & RTE_ETH_RSS_ESP) { 145 ret = mlx5_nta_ptype_rss_flow_create 146 (rss_ctx, ptype_l3 | ptype_l4_esp, 147 rss & ~(RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP)); 148 if (ret) 149 goto error; 150 } 151 if (rss_types & RTE_ETH_RSS_UDP) { 152 ret = mlx5_nta_ptype_rss_flow_create(rss_ctx, 153 ptype_l3 | ptype_l4_udp, 154 rss & ~(RTE_ETH_RSS_ESP | RTE_ETH_RSS_TCP)); 155 if (ret) 156 goto error; 157 } 158 if (rss_types & RTE_ETH_RSS_TCP) { 159 ret = mlx5_nta_ptype_rss_flow_create(rss_ctx, 160 ptype_l3 | ptype_l4_tcp, 161 rss & ~(RTE_ETH_RSS_ESP | RTE_ETH_RSS_UDP)); 162 if (ret) 163 goto error; 164 } 165 return; 166 error: 167 flow_hw_list_destroy(rss_ctx->dev, rss_ctx->flow_type, 168 (uintptr_t)SLIST_FIRST(rss_ctx->head)); 169 } 170 171 /* 172 * Call conditions: 173 * * Flow pattern did not include L4 item. 174 * * RSS configuration had L4 hash types. 175 */ 176 static struct rte_flow_hw * 177 mlx5_hw_rss_expand_l4(struct mlx5_nta_rss_ctx *rss_ctx) 178 { 179 uint64_t rss_types = rte_eth_rss_hf_refine(rss_ctx->rss_conf->types); 180 uint64_t l3_item = rss_ctx->pattern_flags & 181 (rss_ctx->rss_conf->level < 2 ? 182 MLX5_FLOW_LAYER_OUTER_L3 : MLX5_FLOW_LAYER_INNER_L3); 183 184 if (l3_item) { 185 /* 186 * Outer L3 header was present in the original pattern. 187 * Expand L4 level only. 188 */ 189 if (l3_item & MLX5_FLOW_LAYER_L3_IPV4) 190 mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types, MLX5_IPV4_LAYER_TYPES); 191 else 192 mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types, MLX5_IPV6_LAYER_TYPES); 193 } else { 194 if (rss_types & (MLX5_IPV4_LAYER_TYPES | MLX5_IPV6_LAYER_TYPES)) { 195 mlx5_hw_rss_expand_l3(rss_ctx); 196 /* 197 * No outer L3 item in application flow pattern. 198 * RSS hash types are L3 and L4. 199 * ** Expand L3 according to RSS configuration and L4. 200 */ 201 if (rss_types & MLX5_IPV4_LAYER_TYPES) 202 mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types, 203 MLX5_IPV4_LAYER_TYPES); 204 if (rss_types & MLX5_IPV6_LAYER_TYPES) 205 mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types, 206 MLX5_IPV6_LAYER_TYPES); 207 } else { 208 /* 209 * No outer L3 item in application flow pattern, 210 * RSS hash type is L4 only. 211 */ 212 mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types, 213 MLX5_IPV4_LAYER_TYPES); 214 mlx5_nta_rss_expand_l3_l4(rss_ctx, rss_types, 215 MLX5_IPV6_LAYER_TYPES); 216 } 217 } 218 return SLIST_EMPTY(rss_ctx->head) ? NULL : SLIST_FIRST(rss_ctx->head); 219 } 220 221 static struct mlx5_indexed_pool * 222 mlx5_nta_ptype_ipool_create(struct rte_eth_dev *dev) 223 { 224 struct mlx5_priv *priv = dev->data->dev_private; 225 struct mlx5_indexed_pool_config ipool_cfg = { 226 .size = 1, 227 .trunk_size = 32, 228 .grow_trunk = 5, 229 .grow_shift = 1, 230 .need_lock = 1, 231 .release_mem_en = !!priv->sh->config.reclaim_mode, 232 .malloc = mlx5_malloc, 233 .max_idx = MLX5_FLOW_TABLE_PTYPE_RSS_NUM, 234 .free = mlx5_free, 235 .type = "mlx5_nta_ptype_rss" 236 }; 237 return mlx5_ipool_create(&ipool_cfg); 238 } 239 240 static void 241 mlx5_hw_release_rss_ptype_group(struct rte_eth_dev *dev, uint32_t group) 242 { 243 struct mlx5_priv *priv = dev->data->dev_private; 244 245 if (!priv->ptype_rss_groups) 246 return; 247 mlx5_ipool_free(priv->ptype_rss_groups, group); 248 } 249 250 static uint32_t 251 mlx5_hw_get_rss_ptype_group(struct rte_eth_dev *dev) 252 { 253 void *obj; 254 uint32_t idx = 0; 255 struct mlx5_priv *priv = dev->data->dev_private; 256 257 if (!priv->ptype_rss_groups) { 258 priv->ptype_rss_groups = mlx5_nta_ptype_ipool_create(dev); 259 if (!priv->ptype_rss_groups) { 260 DRV_LOG(DEBUG, "PTYPE RSS: failed to allocate groups pool"); 261 return 0; 262 } 263 } 264 obj = mlx5_ipool_malloc(priv->ptype_rss_groups, &idx); 265 if (!obj) { 266 DRV_LOG(DEBUG, "PTYPE RSS: failed to fetch ptype group from the pool"); 267 return 0; 268 } 269 return idx + MLX5_FLOW_TABLE_PTYPE_RSS_BASE; 270 } 271 272 static struct rte_flow_hw * 273 mlx5_hw_rss_ptype_create_miss_flow(struct rte_eth_dev *dev, 274 const struct rte_flow_action_rss *rss_conf, 275 uint32_t ptype_group, bool external, 276 struct rte_flow_error *error) 277 { 278 struct rte_flow_hw *flow = NULL; 279 const struct rte_flow_attr miss_attr = { 280 .ingress = 1, 281 .group = ptype_group, 282 .priority = 3 283 }; 284 const struct rte_flow_item miss_pattern[2] = { 285 [0] = { .type = RTE_FLOW_ITEM_TYPE_ETH }, 286 [1] = { .type = RTE_FLOW_ITEM_TYPE_END } 287 }; 288 struct rte_flow_action miss_actions[] = { 289 #ifdef MLX5_RSS_PTYPE_DEBUG 290 [MLX5_RSS_PTYPE_ACTION_INDEX - 1] = { 291 .type = RTE_FLOW_ACTION_TYPE_MARK, 292 .conf = &(const struct rte_flow_action_mark){.id = 0xfac} 293 }, 294 #endif 295 [MLX5_RSS_PTYPE_ACTION_INDEX] = { 296 .type = RTE_FLOW_ACTION_TYPE_RSS, 297 .conf = rss_conf 298 }, 299 [MLX5_RSS_PTYPE_ACTION_INDEX + 1] = { .type = RTE_FLOW_ACTION_TYPE_END } 300 }; 301 302 flow_hw_create_flow(dev, MLX5_FLOW_TYPE_GEN, &miss_attr, 303 miss_pattern, miss_actions, 0, MLX5_FLOW_ACTION_RSS, 304 external, &flow, error); 305 return flow; 306 } 307 308 static struct rte_flow_hw * 309 mlx5_hw_rss_ptype_create_base_flow(struct rte_eth_dev *dev, 310 const struct rte_flow_attr *attr, 311 const struct rte_flow_item pattern[], 312 const struct rte_flow_action orig_actions[], 313 uint32_t ptype_group, uint64_t item_flags, 314 uint64_t action_flags, bool external, 315 enum mlx5_flow_type flow_type, 316 struct rte_flow_error *error) 317 { 318 int i = 0; 319 struct rte_flow_hw *flow = NULL; 320 struct rte_flow_action actions[MLX5_HW_MAX_ACTS]; 321 enum mlx5_indirect_type indirect_type; 322 323 do { 324 switch (orig_actions[i].type) { 325 case RTE_FLOW_ACTION_TYPE_INDIRECT: 326 indirect_type = (typeof(indirect_type)) 327 MLX5_INDIRECT_ACTION_TYPE_GET 328 (orig_actions[i].conf); 329 if (indirect_type != MLX5_INDIRECT_ACTION_TYPE_RSS) { 330 actions[i] = orig_actions[i]; 331 break; 332 } 333 /* Fall through */ 334 case RTE_FLOW_ACTION_TYPE_RSS: 335 actions[i].type = RTE_FLOW_ACTION_TYPE_JUMP; 336 actions[i].conf = &(const struct rte_flow_action_jump) { 337 .group = ptype_group 338 }; 339 break; 340 default: 341 actions[i] = orig_actions[i]; 342 } 343 344 } while (actions[i++].type != RTE_FLOW_ACTION_TYPE_END); 345 action_flags &= ~MLX5_FLOW_ACTION_RSS; 346 action_flags |= MLX5_FLOW_ACTION_JUMP; 347 flow_hw_create_flow(dev, flow_type, attr, pattern, actions, 348 item_flags, action_flags, external, &flow, error); 349 return flow; 350 } 351 352 const struct rte_flow_action_rss * 353 flow_nta_locate_rss(struct rte_eth_dev *dev, 354 const struct rte_flow_action actions[], 355 struct rte_flow_error *error) 356 { 357 const struct rte_flow_action *a; 358 const struct rte_flow_action_rss *rss_conf = NULL; 359 360 for (a = actions; a->type != RTE_FLOW_ACTION_TYPE_END; a++) { 361 if (a->type == RTE_FLOW_ACTION_TYPE_RSS) { 362 rss_conf = a->conf; 363 break; 364 } 365 if (a->type == RTE_FLOW_ACTION_TYPE_INDIRECT && 366 MLX5_INDIRECT_ACTION_TYPE_GET(a->conf) == 367 MLX5_INDIRECT_ACTION_TYPE_RSS) { 368 struct mlx5_priv *priv = dev->data->dev_private; 369 struct mlx5_shared_action_rss *shared_rss; 370 uint32_t handle = (uint32_t)(uintptr_t)a->conf; 371 372 shared_rss = mlx5_ipool_get 373 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 374 MLX5_INDIRECT_ACTION_IDX_GET(handle)); 375 if (!shared_rss) { 376 rte_flow_error_set(error, EINVAL, 377 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 378 a->conf, "invalid shared RSS handle"); 379 return NULL; 380 } 381 rss_conf = &shared_rss->origin; 382 break; 383 } 384 } 385 if (a->type == RTE_FLOW_ACTION_TYPE_END) { 386 rte_flow_error_set(error, 0, RTE_FLOW_ERROR_TYPE_NONE, NULL, NULL); 387 return NULL; 388 } 389 return rss_conf; 390 } 391 392 static __rte_always_inline void 393 mlx5_nta_rss_init_ptype_ctx(struct mlx5_nta_rss_ctx *rss_ctx, 394 struct rte_eth_dev *dev, 395 struct rte_flow_attr *ptype_attr, 396 struct rte_flow_item *ptype_pattern, 397 struct rte_flow_action *ptype_actions, 398 const struct rte_flow_action_rss *rss_conf, 399 struct mlx5_nta_rss_flow_head *head, 400 struct rte_flow_error *error, 401 uint64_t item_flags, 402 enum mlx5_flow_type flow_type, bool external) 403 { 404 rss_ctx->dev = dev; 405 rss_ctx->attr = ptype_attr; 406 rss_ctx->pattern = ptype_pattern; 407 rss_ctx->actions = ptype_actions; 408 rss_ctx->rss_conf = rss_conf; 409 rss_ctx->error = error; 410 rss_ctx->head = head; 411 rss_ctx->pattern_flags = item_flags; 412 rss_ctx->flow_type = flow_type; 413 rss_ctx->external = external; 414 } 415 416 static struct rte_flow_hw * 417 flow_nta_create_single(struct rte_eth_dev *dev, 418 const struct rte_flow_attr *attr, 419 const struct rte_flow_item items[], 420 const struct rte_flow_action actions[], 421 struct rte_flow_action_rss *rss_conf, 422 int64_t item_flags, uint64_t action_flags, 423 bool external, bool copy_actions, 424 enum mlx5_flow_type flow_type, 425 struct rte_flow_error *error) 426 { 427 struct rte_flow_hw *flow = NULL; 428 struct rte_flow_action copy[MLX5_HW_MAX_ACTS]; 429 const struct rte_flow_action *_actions; 430 431 if (copy_actions) { 432 int i; 433 434 _actions = copy; 435 for (i = 0; ; i++) { 436 copy[i] = actions[i]; 437 switch (actions[i].type) { 438 case RTE_FLOW_ACTION_TYPE_RSS: 439 copy[i].conf = rss_conf; 440 break; 441 case RTE_FLOW_ACTION_TYPE_INDIRECT: 442 if (MLX5_INDIRECT_ACTION_TYPE_GET(actions[i].conf) == 443 MLX5_INDIRECT_ACTION_TYPE_RSS) { 444 copy[i].type = RTE_FLOW_ACTION_TYPE_RSS; 445 copy[i].conf = rss_conf; 446 } 447 break; 448 case RTE_FLOW_ACTION_TYPE_END: 449 goto end; 450 default: 451 break; 452 } 453 } 454 } else { 455 _actions = actions; 456 } 457 end: 458 flow_hw_create_flow(dev, flow_type, attr, items, 459 _actions, item_flags, action_flags, 460 external, &flow, error); 461 return flow; 462 } 463 464 /* 465 * MLX5 HW hashes IPv4 and IPv6 L3 headers and UDP, TCP, ESP L4 headers. 466 * RSS expansion is required when RSS action was configured to hash 467 * network protocol that was not mentioned in flow pattern. 468 * 469 */ 470 #define MLX5_PTYPE_RSS_OUTER_MASK (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV6 | \ 471 RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP | \ 472 RTE_PTYPE_TUNNEL_ESP) 473 #define MLX5_PTYPE_RSS_INNER_MASK (RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L3_IPV6 | \ 474 RTE_PTYPE_INNER_L4_TCP | RTE_PTYPE_INNER_L4_UDP) 475 476 struct rte_flow_hw * 477 flow_nta_handle_rss(struct rte_eth_dev *dev, 478 const struct rte_flow_attr *attr, 479 const struct rte_flow_item items[], 480 const struct rte_flow_action actions[], 481 const struct rte_flow_action_rss *rss_conf, 482 uint64_t item_flags, uint64_t action_flags, 483 bool external, enum mlx5_flow_type flow_type, 484 struct rte_flow_error *error) 485 { 486 struct rte_flow_hw *rss_base = NULL, *rss_next = NULL, *rss_miss = NULL; 487 struct rte_flow_action_rss ptype_rss_conf = *rss_conf; 488 struct mlx5_nta_rss_ctx rss_ctx; 489 uint64_t rss_types = rte_eth_rss_hf_refine(rss_conf->types); 490 bool expand = true; 491 bool copy_actions = false; 492 bool inner_rss = rss_conf->level > 1; 493 bool outer_rss = !inner_rss; 494 bool l3_item = (outer_rss && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 495 (inner_rss && (item_flags & MLX5_FLOW_LAYER_INNER_L3)); 496 bool l4_item = (outer_rss && (item_flags & MLX5_FLOW_LAYER_OUTER_L4)) || 497 (inner_rss && (item_flags & MLX5_FLOW_LAYER_INNER_L4)); 498 bool l3_hash = rss_types & (MLX5_IPV4_LAYER_TYPES | MLX5_IPV6_LAYER_TYPES); 499 bool l4_hash = rss_types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP | RTE_ETH_RSS_ESP); 500 struct mlx5_nta_rss_flow_head expansion_head = SLIST_HEAD_INITIALIZER(0); 501 struct rte_flow_attr ptype_attr = { 502 .ingress = 1 503 }; 504 struct rte_flow_item_ptype ptype_spec = { .packet_type = 0 }; 505 const struct rte_flow_item_ptype ptype_mask = { 506 .packet_type = outer_rss ? 507 MLX5_PTYPE_RSS_OUTER_MASK : MLX5_PTYPE_RSS_INNER_MASK 508 }; 509 struct rte_flow_item ptype_pattern[MLX5_RSS_PTYPE_ITEMS_NUM] = { 510 [MLX5_RSS_PTYPE_ITEM_INDEX] = { 511 .type = RTE_FLOW_ITEM_TYPE_PTYPE, 512 .spec = &ptype_spec, 513 .mask = &ptype_mask 514 }, 515 [MLX5_RSS_PTYPE_ITEM_INDEX + 1] = { .type = RTE_FLOW_ITEM_TYPE_END } 516 }; 517 struct rte_flow_action ptype_actions[MLX5_RSS_PTYPE_ACTIONS_NUM] = { 518 #ifdef MLX5_RSS_PTYPE_DEBUG 519 [MLX5_RSS_PTYPE_ACTION_INDEX - 1] = { 520 .type = RTE_FLOW_ACTION_TYPE_MARK, 521 .conf = &(const struct rte_flow_action_mark) {.id = 101} 522 }, 523 #endif 524 [MLX5_RSS_PTYPE_ACTION_INDEX] = { 525 .type = RTE_FLOW_ACTION_TYPE_RSS, 526 .conf = &ptype_rss_conf 527 }, 528 [MLX5_RSS_PTYPE_ACTION_INDEX + 1] = { .type = RTE_FLOW_ACTION_TYPE_END } 529 }; 530 531 ptype_rss_conf.types = rss_types; 532 if (l4_item) { 533 /* 534 * Original flow pattern extended up to L4 level. 535 * L4 is the maximal expansion level. 536 * Original pattern does not need expansion. 537 */ 538 expand = false; 539 } else if (!l4_hash) { 540 if (!l3_hash) { 541 /* 542 * RSS action was not configured to hash L3 or L4. 543 * No expansion needed. 544 */ 545 expand = false; 546 } else if (l3_item) { 547 /* 548 * Original flow pattern extended up to L3 level. 549 * RSS action was not set for L4 hash. 550 */ 551 bool ip4_item = 552 (outer_rss && (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)) || 553 (inner_rss && (item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV4)); 554 bool ip6_item = 555 (outer_rss && (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)) || 556 (inner_rss && (item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)); 557 bool ip4_hash = rss_types & MLX5_IPV4_LAYER_TYPES; 558 bool ip6_hash = rss_types & MLX5_IPV6_LAYER_TYPES; 559 560 expand = false; 561 if (ip4_item && ip4_hash) { 562 ptype_rss_conf.types &= ~MLX5_IPV6_LAYER_TYPES; 563 copy_actions = true; 564 } else if (ip6_item && ip6_hash) { 565 /* 566 * MLX5 HW will not activate TIR IPv6 hash 567 * if that TIR has also IPv4 hash 568 */ 569 ptype_rss_conf.types &= ~MLX5_IPV4_LAYER_TYPES; 570 copy_actions = true; 571 } 572 } 573 } 574 if (!expand) 575 return flow_nta_create_single(dev, attr, items, actions, 576 &ptype_rss_conf, item_flags, 577 action_flags, external, 578 copy_actions, flow_type, error); 579 /* Create RSS expansions in dedicated PTYPE flow group */ 580 ptype_attr.group = mlx5_hw_get_rss_ptype_group(dev); 581 if (!ptype_attr.group) { 582 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 583 NULL, "cannot get RSS PTYPE group"); 584 return NULL; 585 } 586 mlx5_nta_rss_init_ptype_ctx(&rss_ctx, dev, &ptype_attr, ptype_pattern, 587 ptype_actions, &ptype_rss_conf, &expansion_head, 588 error, item_flags, flow_type, external); 589 rss_miss = mlx5_hw_rss_ptype_create_miss_flow(dev, &ptype_rss_conf, ptype_attr.group, 590 external, error); 591 if (!rss_miss) 592 goto error; 593 if (l4_hash) { 594 rss_next = mlx5_hw_rss_expand_l4(&rss_ctx); 595 if (!rss_next) 596 goto error; 597 } else if (l3_hash) { 598 rss_next = mlx5_hw_rss_expand_l3(&rss_ctx); 599 if (!rss_next) 600 goto error; 601 } 602 rss_base = mlx5_hw_rss_ptype_create_base_flow(dev, attr, items, actions, 603 ptype_attr.group, item_flags, 604 action_flags, external, 605 flow_type, error); 606 if (!rss_base) 607 goto error; 608 SLIST_INSERT_HEAD(&expansion_head, rss_miss, nt2hws->next); 609 SLIST_INSERT_HEAD(&expansion_head, rss_base, nt2hws->next); 610 /** 611 * PMD must return to application a reference to the base flow. 612 * This way RSS expansion could work with counter, meter and other 613 * flow actions. 614 */ 615 MLX5_ASSERT(rss_base == SLIST_FIRST(&expansion_head)); 616 rss_next = SLIST_NEXT(rss_base, nt2hws->next); 617 while (rss_next) { 618 rss_next->nt2hws->chaned_flow = 1; 619 rss_next = SLIST_NEXT(rss_next, nt2hws->next); 620 } 621 return SLIST_FIRST(&expansion_head); 622 623 error: 624 if (rss_miss) 625 flow_hw_list_destroy(dev, flow_type, (uintptr_t)rss_miss); 626 if (rss_next) 627 flow_hw_list_destroy(dev, flow_type, (uintptr_t)rss_next); 628 mlx5_hw_release_rss_ptype_group(dev, ptype_attr.group); 629 return NULL; 630 } 631 632 #endif 633 634