1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see RTE_ETH_RSS_* definitions). 102 */ 103 uint64_t node_flags; 104 /**< 105 * Bit-fields that define how the node is used in the expansion. 106 * (see MLX5_EXPANSION_NODE_* definitions). 107 */ 108 }; 109 110 /* Optional expand field. The expansion alg will not go deeper. */ 111 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 112 113 /* The node is not added implicitly as expansion to the flow pattern. 114 * If the node type does not match the flow pattern item type, the 115 * expansion alg will go deeper to its next items. 116 * In the current implementation, the list of next nodes indexes can 117 * have up to one node with this flag set and it has to be the last 118 * node index (before the list terminator). 119 */ 120 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 121 122 /** Object returned by mlx5_flow_expand_rss(). */ 123 struct mlx5_flow_expand_rss { 124 uint32_t entries; 125 /**< Number of entries @p patterns and @p priorities. */ 126 struct { 127 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 128 uint32_t priority; /**< Priority offset for each expansion. */ 129 } entry[]; 130 }; 131 132 static void 133 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 134 135 static const struct mlx5_flow_expand_node * 136 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 137 unsigned int item_idx, 138 const struct mlx5_flow_expand_node graph[], 139 const struct mlx5_flow_expand_node *node); 140 141 static bool 142 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 143 { 144 switch (item->type) { 145 case RTE_FLOW_ITEM_TYPE_ETH: 146 case RTE_FLOW_ITEM_TYPE_VLAN: 147 case RTE_FLOW_ITEM_TYPE_IPV4: 148 case RTE_FLOW_ITEM_TYPE_IPV6: 149 case RTE_FLOW_ITEM_TYPE_UDP: 150 case RTE_FLOW_ITEM_TYPE_TCP: 151 case RTE_FLOW_ITEM_TYPE_VXLAN: 152 case RTE_FLOW_ITEM_TYPE_NVGRE: 153 case RTE_FLOW_ITEM_TYPE_GRE: 154 case RTE_FLOW_ITEM_TYPE_GENEVE: 155 case RTE_FLOW_ITEM_TYPE_MPLS: 156 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 157 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 158 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 159 case RTE_FLOW_ITEM_TYPE_GTP: 160 return true; 161 default: 162 break; 163 } 164 return false; 165 } 166 167 /** 168 * Network Service Header (NSH) and its next protocol values 169 * are described in RFC-8393. 170 */ 171 static enum rte_flow_item_type 172 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 173 { 174 enum rte_flow_item_type type; 175 176 switch (proto_mask & proto_spec) { 177 case RTE_VXLAN_GPE_TYPE_IPV4: 178 type = RTE_FLOW_ITEM_TYPE_IPV4; 179 break; 180 case RTE_VXLAN_GPE_TYPE_IPV6: 181 type = RTE_VXLAN_GPE_TYPE_IPV6; 182 break; 183 case RTE_VXLAN_GPE_TYPE_ETH: 184 type = RTE_FLOW_ITEM_TYPE_ETH; 185 break; 186 default: 187 type = RTE_FLOW_ITEM_TYPE_END; 188 } 189 return type; 190 } 191 192 static enum rte_flow_item_type 193 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 194 { 195 enum rte_flow_item_type type; 196 197 switch (proto_mask & proto_spec) { 198 case IPPROTO_UDP: 199 type = RTE_FLOW_ITEM_TYPE_UDP; 200 break; 201 case IPPROTO_TCP: 202 type = RTE_FLOW_ITEM_TYPE_TCP; 203 break; 204 case IPPROTO_IP: 205 type = RTE_FLOW_ITEM_TYPE_IPV4; 206 break; 207 case IPPROTO_IPV6: 208 type = RTE_FLOW_ITEM_TYPE_IPV6; 209 break; 210 default: 211 type = RTE_FLOW_ITEM_TYPE_END; 212 } 213 return type; 214 } 215 216 static enum rte_flow_item_type 217 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 218 rte_be16_t type_mask, bool is_tunnel) 219 { 220 enum rte_flow_item_type type; 221 222 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 223 case RTE_ETHER_TYPE_TEB: 224 type = is_tunnel ? 225 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 226 break; 227 case RTE_ETHER_TYPE_VLAN: 228 type = !is_tunnel ? 229 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 230 break; 231 case RTE_ETHER_TYPE_IPV4: 232 type = RTE_FLOW_ITEM_TYPE_IPV4; 233 break; 234 case RTE_ETHER_TYPE_IPV6: 235 type = RTE_FLOW_ITEM_TYPE_IPV6; 236 break; 237 default: 238 type = RTE_FLOW_ITEM_TYPE_END; 239 } 240 return type; 241 } 242 243 static enum rte_flow_item_type 244 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 245 { 246 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 247 do { \ 248 const void *m = item->mask; \ 249 const void *s = item->spec; \ 250 mask = m ? \ 251 ((const struct rte_flow_item_##type *)m)->fld : \ 252 rte_flow_item_##type##_mask.fld; \ 253 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 254 } while (0) 255 256 enum rte_flow_item_type ret; 257 uint16_t spec, mask; 258 259 if (item == NULL || item->spec == NULL) 260 return RTE_FLOW_ITEM_TYPE_VOID; 261 switch (item->type) { 262 case RTE_FLOW_ITEM_TYPE_ETH: 263 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 264 if (!mask) 265 return RTE_FLOW_ITEM_TYPE_VOID; 266 ret = mlx5_ethertype_to_item_type(spec, mask, false); 267 break; 268 case RTE_FLOW_ITEM_TYPE_VLAN: 269 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 270 if (!mask) 271 return RTE_FLOW_ITEM_TYPE_VOID; 272 ret = mlx5_ethertype_to_item_type(spec, mask, false); 273 break; 274 case RTE_FLOW_ITEM_TYPE_IPV4: 275 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 276 if (!mask) 277 return RTE_FLOW_ITEM_TYPE_VOID; 278 ret = mlx5_inet_proto_to_item_type(spec, mask); 279 break; 280 case RTE_FLOW_ITEM_TYPE_IPV6: 281 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 282 if (!mask) 283 return RTE_FLOW_ITEM_TYPE_VOID; 284 ret = mlx5_inet_proto_to_item_type(spec, mask); 285 break; 286 case RTE_FLOW_ITEM_TYPE_GENEVE: 287 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 288 ret = mlx5_ethertype_to_item_type(spec, mask, true); 289 break; 290 case RTE_FLOW_ITEM_TYPE_GRE: 291 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 292 ret = mlx5_ethertype_to_item_type(spec, mask, true); 293 break; 294 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 295 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 296 ret = mlx5_nsh_proto_to_item_type(spec, mask); 297 break; 298 default: 299 ret = RTE_FLOW_ITEM_TYPE_VOID; 300 break; 301 } 302 return ret; 303 #undef MLX5_XSET_ITEM_MASK_SPEC 304 } 305 306 static const int * 307 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 308 const int *next_node) 309 { 310 const struct mlx5_flow_expand_node *node = NULL; 311 const int *next = next_node; 312 313 while (next && *next) { 314 /* 315 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 316 * flag set, because they were not found in the flow pattern. 317 */ 318 node = &graph[*next]; 319 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 320 break; 321 next = node->next; 322 } 323 return next; 324 } 325 326 #define MLX5_RSS_EXP_ELT_N 16 327 328 /** 329 * Expand RSS flows into several possible flows according to the RSS hash 330 * fields requested and the driver capabilities. 331 * 332 * @param[out] buf 333 * Buffer to store the result expansion. 334 * @param[in] size 335 * Buffer size in bytes. If 0, @p buf can be NULL. 336 * @param[in] pattern 337 * User flow pattern. 338 * @param[in] types 339 * RSS types to expand (see RTE_ETH_RSS_* definitions). 340 * @param[in] graph 341 * Input graph to expand @p pattern according to @p types. 342 * @param[in] graph_root_index 343 * Index of root node in @p graph, typically 0. 344 * 345 * @return 346 * A positive value representing the size of @p buf in bytes regardless of 347 * @p size on success, a negative errno value otherwise and rte_errno is 348 * set, the following errors are defined: 349 * 350 * -E2BIG: graph-depth @p graph is too deep. 351 * -EINVAL: @p size has not enough space for expanded pattern. 352 */ 353 static int 354 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 355 const struct rte_flow_item *pattern, uint64_t types, 356 const struct mlx5_flow_expand_node graph[], 357 int graph_root_index) 358 { 359 const struct rte_flow_item *item; 360 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 361 const int *next_node; 362 const int *stack[MLX5_RSS_EXP_ELT_N]; 363 int stack_pos = 0; 364 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 365 unsigned int i, item_idx, last_expand_item_idx = 0; 366 size_t lsize; 367 size_t user_pattern_size = 0; 368 void *addr = NULL; 369 const struct mlx5_flow_expand_node *next = NULL; 370 struct rte_flow_item missed_item; 371 int missed = 0; 372 int elt = 0; 373 const struct rte_flow_item *last_expand_item = NULL; 374 375 memset(&missed_item, 0, sizeof(missed_item)); 376 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 377 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 378 if (lsize > size) 379 return -EINVAL; 380 buf->entry[0].priority = 0; 381 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 382 buf->entries = 0; 383 addr = buf->entry[0].pattern; 384 for (item = pattern, item_idx = 0; 385 item->type != RTE_FLOW_ITEM_TYPE_END; 386 item++, item_idx++) { 387 if (!mlx5_flow_is_rss_expandable_item(item)) { 388 user_pattern_size += sizeof(*item); 389 continue; 390 } 391 last_expand_item = item; 392 last_expand_item_idx = item_idx; 393 i = 0; 394 while (node->next && node->next[i]) { 395 next = &graph[node->next[i]]; 396 if (next->type == item->type) 397 break; 398 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 399 node = next; 400 i = 0; 401 } else { 402 ++i; 403 } 404 } 405 if (next) 406 node = next; 407 user_pattern_size += sizeof(*item); 408 } 409 user_pattern_size += sizeof(*item); /* Handle END item. */ 410 lsize += user_pattern_size; 411 if (lsize > size) 412 return -EINVAL; 413 /* Copy the user pattern in the first entry of the buffer. */ 414 rte_memcpy(addr, pattern, user_pattern_size); 415 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 416 buf->entries = 1; 417 /* Start expanding. */ 418 memset(flow_items, 0, sizeof(flow_items)); 419 user_pattern_size -= sizeof(*item); 420 /* 421 * Check if the last valid item has spec set, need complete pattern, 422 * and the pattern can be used for expansion. 423 */ 424 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 425 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 426 /* Item type END indicates expansion is not required. */ 427 return lsize; 428 } 429 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 430 next = NULL; 431 missed = 1; 432 i = 0; 433 while (node->next && node->next[i]) { 434 next = &graph[node->next[i]]; 435 if (next->type == missed_item.type) { 436 flow_items[0].type = missed_item.type; 437 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 438 break; 439 } 440 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 441 node = next; 442 i = 0; 443 } else { 444 ++i; 445 } 446 next = NULL; 447 } 448 } 449 if (next && missed) { 450 elt = 2; /* missed item + item end. */ 451 node = next; 452 lsize += elt * sizeof(*item) + user_pattern_size; 453 if (lsize > size) 454 return -EINVAL; 455 if (node->rss_types & types) { 456 buf->entry[buf->entries].priority = 1; 457 buf->entry[buf->entries].pattern = addr; 458 buf->entries++; 459 rte_memcpy(addr, buf->entry[0].pattern, 460 user_pattern_size); 461 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 462 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 463 addr = (void *)(((uintptr_t)addr) + 464 elt * sizeof(*item)); 465 } 466 } else if (last_expand_item != NULL) { 467 node = mlx5_flow_expand_rss_adjust_node(pattern, 468 last_expand_item_idx, graph, node); 469 } 470 memset(flow_items, 0, sizeof(flow_items)); 471 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 472 node->next); 473 stack[stack_pos] = next_node; 474 node = next_node ? &graph[*next_node] : NULL; 475 while (node) { 476 flow_items[stack_pos].type = node->type; 477 if (node->rss_types & types) { 478 size_t n; 479 /* 480 * compute the number of items to copy from the 481 * expansion and copy it. 482 * When the stack_pos is 0, there are 1 element in it, 483 * plus the addition END item. 484 */ 485 elt = stack_pos + 2; 486 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 487 lsize += elt * sizeof(*item) + user_pattern_size; 488 if (lsize > size) 489 return -EINVAL; 490 n = elt * sizeof(*item); 491 buf->entry[buf->entries].priority = 492 stack_pos + 1 + missed; 493 buf->entry[buf->entries].pattern = addr; 494 buf->entries++; 495 rte_memcpy(addr, buf->entry[0].pattern, 496 user_pattern_size); 497 addr = (void *)(((uintptr_t)addr) + 498 user_pattern_size); 499 rte_memcpy(addr, &missed_item, 500 missed * sizeof(*item)); 501 addr = (void *)(((uintptr_t)addr) + 502 missed * sizeof(*item)); 503 rte_memcpy(addr, flow_items, n); 504 addr = (void *)(((uintptr_t)addr) + n); 505 } 506 /* Go deeper. */ 507 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 508 node->next) { 509 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 510 node->next); 511 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 512 rte_errno = E2BIG; 513 return -rte_errno; 514 } 515 stack[stack_pos] = next_node; 516 } else if (*(next_node + 1)) { 517 /* Follow up with the next possibility. */ 518 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 519 ++next_node); 520 } else if (!stack_pos) { 521 /* 522 * Completing the traverse over the different paths. 523 * The next_node is advanced to the terminator. 524 */ 525 ++next_node; 526 } else { 527 /* Move to the next path. */ 528 while (stack_pos) { 529 next_node = stack[--stack_pos]; 530 next_node++; 531 if (*next_node) 532 break; 533 } 534 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 535 next_node); 536 stack[stack_pos] = next_node; 537 } 538 node = next_node && *next_node ? &graph[*next_node] : NULL; 539 }; 540 return lsize; 541 } 542 543 enum mlx5_expansion { 544 MLX5_EXPANSION_ROOT, 545 MLX5_EXPANSION_ROOT_OUTER, 546 MLX5_EXPANSION_OUTER_ETH, 547 MLX5_EXPANSION_OUTER_VLAN, 548 MLX5_EXPANSION_OUTER_IPV4, 549 MLX5_EXPANSION_OUTER_IPV4_UDP, 550 MLX5_EXPANSION_OUTER_IPV4_TCP, 551 MLX5_EXPANSION_OUTER_IPV6, 552 MLX5_EXPANSION_OUTER_IPV6_UDP, 553 MLX5_EXPANSION_OUTER_IPV6_TCP, 554 MLX5_EXPANSION_VXLAN, 555 MLX5_EXPANSION_STD_VXLAN, 556 MLX5_EXPANSION_L3_VXLAN, 557 MLX5_EXPANSION_VXLAN_GPE, 558 MLX5_EXPANSION_GRE, 559 MLX5_EXPANSION_NVGRE, 560 MLX5_EXPANSION_GRE_KEY, 561 MLX5_EXPANSION_MPLS, 562 MLX5_EXPANSION_ETH, 563 MLX5_EXPANSION_VLAN, 564 MLX5_EXPANSION_IPV4, 565 MLX5_EXPANSION_IPV4_UDP, 566 MLX5_EXPANSION_IPV4_TCP, 567 MLX5_EXPANSION_IPV6, 568 MLX5_EXPANSION_IPV6_UDP, 569 MLX5_EXPANSION_IPV6_TCP, 570 MLX5_EXPANSION_IPV6_FRAG_EXT, 571 MLX5_EXPANSION_GTP, 572 MLX5_EXPANSION_GENEVE, 573 }; 574 575 /** Supported expansion of items. */ 576 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 577 [MLX5_EXPANSION_ROOT] = { 578 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 579 MLX5_EXPANSION_IPV4, 580 MLX5_EXPANSION_IPV6), 581 .type = RTE_FLOW_ITEM_TYPE_END, 582 }, 583 [MLX5_EXPANSION_ROOT_OUTER] = { 584 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 585 MLX5_EXPANSION_OUTER_IPV4, 586 MLX5_EXPANSION_OUTER_IPV6), 587 .type = RTE_FLOW_ITEM_TYPE_END, 588 }, 589 [MLX5_EXPANSION_OUTER_ETH] = { 590 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 591 .type = RTE_FLOW_ITEM_TYPE_ETH, 592 .rss_types = 0, 593 }, 594 [MLX5_EXPANSION_OUTER_VLAN] = { 595 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 596 MLX5_EXPANSION_OUTER_IPV6), 597 .type = RTE_FLOW_ITEM_TYPE_VLAN, 598 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 599 }, 600 [MLX5_EXPANSION_OUTER_IPV4] = { 601 .next = MLX5_FLOW_EXPAND_RSS_NEXT 602 (MLX5_EXPANSION_OUTER_IPV4_UDP, 603 MLX5_EXPANSION_OUTER_IPV4_TCP, 604 MLX5_EXPANSION_GRE, 605 MLX5_EXPANSION_NVGRE, 606 MLX5_EXPANSION_IPV4, 607 MLX5_EXPANSION_IPV6), 608 .type = RTE_FLOW_ITEM_TYPE_IPV4, 609 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 610 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 611 }, 612 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 613 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 614 MLX5_EXPANSION_VXLAN_GPE, 615 MLX5_EXPANSION_MPLS, 616 MLX5_EXPANSION_GENEVE, 617 MLX5_EXPANSION_GTP), 618 .type = RTE_FLOW_ITEM_TYPE_UDP, 619 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 620 }, 621 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 622 .type = RTE_FLOW_ITEM_TYPE_TCP, 623 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 624 }, 625 [MLX5_EXPANSION_OUTER_IPV6] = { 626 .next = MLX5_FLOW_EXPAND_RSS_NEXT 627 (MLX5_EXPANSION_OUTER_IPV6_UDP, 628 MLX5_EXPANSION_OUTER_IPV6_TCP, 629 MLX5_EXPANSION_IPV4, 630 MLX5_EXPANSION_IPV6, 631 MLX5_EXPANSION_GRE, 632 MLX5_EXPANSION_NVGRE), 633 .type = RTE_FLOW_ITEM_TYPE_IPV6, 634 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 635 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 636 }, 637 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 638 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 639 MLX5_EXPANSION_VXLAN_GPE, 640 MLX5_EXPANSION_MPLS, 641 MLX5_EXPANSION_GENEVE, 642 MLX5_EXPANSION_GTP), 643 .type = RTE_FLOW_ITEM_TYPE_UDP, 644 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 645 }, 646 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 647 .type = RTE_FLOW_ITEM_TYPE_TCP, 648 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 649 }, 650 [MLX5_EXPANSION_VXLAN] = { 651 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 652 MLX5_EXPANSION_IPV4, 653 MLX5_EXPANSION_IPV6), 654 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 655 }, 656 [MLX5_EXPANSION_STD_VXLAN] = { 657 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 658 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 659 }, 660 [MLX5_EXPANSION_L3_VXLAN] = { 661 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 662 MLX5_EXPANSION_IPV6), 663 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 664 }, 665 [MLX5_EXPANSION_VXLAN_GPE] = { 666 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 667 MLX5_EXPANSION_IPV4, 668 MLX5_EXPANSION_IPV6), 669 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 670 }, 671 [MLX5_EXPANSION_GRE] = { 672 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 673 MLX5_EXPANSION_IPV4, 674 MLX5_EXPANSION_IPV6, 675 MLX5_EXPANSION_GRE_KEY, 676 MLX5_EXPANSION_MPLS), 677 .type = RTE_FLOW_ITEM_TYPE_GRE, 678 }, 679 [MLX5_EXPANSION_GRE_KEY] = { 680 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 681 MLX5_EXPANSION_IPV6, 682 MLX5_EXPANSION_MPLS), 683 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 684 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 685 }, 686 [MLX5_EXPANSION_NVGRE] = { 687 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 688 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 689 }, 690 [MLX5_EXPANSION_MPLS] = { 691 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 692 MLX5_EXPANSION_IPV6, 693 MLX5_EXPANSION_ETH), 694 .type = RTE_FLOW_ITEM_TYPE_MPLS, 695 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 696 }, 697 [MLX5_EXPANSION_ETH] = { 698 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 699 .type = RTE_FLOW_ITEM_TYPE_ETH, 700 }, 701 [MLX5_EXPANSION_VLAN] = { 702 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 703 MLX5_EXPANSION_IPV6), 704 .type = RTE_FLOW_ITEM_TYPE_VLAN, 705 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 706 }, 707 [MLX5_EXPANSION_IPV4] = { 708 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 709 MLX5_EXPANSION_IPV4_TCP), 710 .type = RTE_FLOW_ITEM_TYPE_IPV4, 711 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 712 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 713 }, 714 [MLX5_EXPANSION_IPV4_UDP] = { 715 .type = RTE_FLOW_ITEM_TYPE_UDP, 716 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 717 }, 718 [MLX5_EXPANSION_IPV4_TCP] = { 719 .type = RTE_FLOW_ITEM_TYPE_TCP, 720 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 721 }, 722 [MLX5_EXPANSION_IPV6] = { 723 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 724 MLX5_EXPANSION_IPV6_TCP, 725 MLX5_EXPANSION_IPV6_FRAG_EXT), 726 .type = RTE_FLOW_ITEM_TYPE_IPV6, 727 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 728 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 729 }, 730 [MLX5_EXPANSION_IPV6_UDP] = { 731 .type = RTE_FLOW_ITEM_TYPE_UDP, 732 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 733 }, 734 [MLX5_EXPANSION_IPV6_TCP] = { 735 .type = RTE_FLOW_ITEM_TYPE_TCP, 736 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 737 }, 738 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 739 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 740 }, 741 [MLX5_EXPANSION_GTP] = { 742 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 743 MLX5_EXPANSION_IPV6), 744 .type = RTE_FLOW_ITEM_TYPE_GTP, 745 }, 746 [MLX5_EXPANSION_GENEVE] = { 747 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 748 MLX5_EXPANSION_IPV4, 749 MLX5_EXPANSION_IPV6), 750 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 751 }, 752 }; 753 754 static struct rte_flow_action_handle * 755 mlx5_action_handle_create(struct rte_eth_dev *dev, 756 const struct rte_flow_indir_action_conf *conf, 757 const struct rte_flow_action *action, 758 struct rte_flow_error *error); 759 static int mlx5_action_handle_destroy 760 (struct rte_eth_dev *dev, 761 struct rte_flow_action_handle *handle, 762 struct rte_flow_error *error); 763 static int mlx5_action_handle_update 764 (struct rte_eth_dev *dev, 765 struct rte_flow_action_handle *handle, 766 const void *update, 767 struct rte_flow_error *error); 768 static int mlx5_action_handle_query 769 (struct rte_eth_dev *dev, 770 const struct rte_flow_action_handle *handle, 771 void *data, 772 struct rte_flow_error *error); 773 static int 774 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 775 struct rte_flow_tunnel *app_tunnel, 776 struct rte_flow_action **actions, 777 uint32_t *num_of_actions, 778 struct rte_flow_error *error); 779 static int 780 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 781 struct rte_flow_tunnel *app_tunnel, 782 struct rte_flow_item **items, 783 uint32_t *num_of_items, 784 struct rte_flow_error *error); 785 static int 786 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 787 struct rte_flow_item *pmd_items, 788 uint32_t num_items, struct rte_flow_error *err); 789 static int 790 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 791 struct rte_flow_action *pmd_actions, 792 uint32_t num_actions, 793 struct rte_flow_error *err); 794 static int 795 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 796 struct rte_mbuf *m, 797 struct rte_flow_restore_info *info, 798 struct rte_flow_error *err); 799 static struct rte_flow_item_flex_handle * 800 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 801 const struct rte_flow_item_flex_conf *conf, 802 struct rte_flow_error *error); 803 static int 804 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 805 const struct rte_flow_item_flex_handle *handle, 806 struct rte_flow_error *error); 807 808 static const struct rte_flow_ops mlx5_flow_ops = { 809 .validate = mlx5_flow_validate, 810 .create = mlx5_flow_create, 811 .destroy = mlx5_flow_destroy, 812 .flush = mlx5_flow_flush, 813 .isolate = mlx5_flow_isolate, 814 .query = mlx5_flow_query, 815 .dev_dump = mlx5_flow_dev_dump, 816 .get_aged_flows = mlx5_flow_get_aged_flows, 817 .action_handle_create = mlx5_action_handle_create, 818 .action_handle_destroy = mlx5_action_handle_destroy, 819 .action_handle_update = mlx5_action_handle_update, 820 .action_handle_query = mlx5_action_handle_query, 821 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 822 .tunnel_match = mlx5_flow_tunnel_match, 823 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 824 .tunnel_item_release = mlx5_flow_tunnel_item_release, 825 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 826 .flex_item_create = mlx5_flow_flex_item_create, 827 .flex_item_release = mlx5_flow_flex_item_release, 828 }; 829 830 /* Tunnel information. */ 831 struct mlx5_flow_tunnel_info { 832 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 833 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 834 }; 835 836 static struct mlx5_flow_tunnel_info tunnels_info[] = { 837 { 838 .tunnel = MLX5_FLOW_LAYER_VXLAN, 839 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 840 }, 841 { 842 .tunnel = MLX5_FLOW_LAYER_GENEVE, 843 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 844 }, 845 { 846 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 847 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 848 }, 849 { 850 .tunnel = MLX5_FLOW_LAYER_GRE, 851 .ptype = RTE_PTYPE_TUNNEL_GRE, 852 }, 853 { 854 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 855 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 856 }, 857 { 858 .tunnel = MLX5_FLOW_LAYER_MPLS, 859 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 860 }, 861 { 862 .tunnel = MLX5_FLOW_LAYER_NVGRE, 863 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 864 }, 865 { 866 .tunnel = MLX5_FLOW_LAYER_IPIP, 867 .ptype = RTE_PTYPE_TUNNEL_IP, 868 }, 869 { 870 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 871 .ptype = RTE_PTYPE_TUNNEL_IP, 872 }, 873 { 874 .tunnel = MLX5_FLOW_LAYER_GTP, 875 .ptype = RTE_PTYPE_TUNNEL_GTPU, 876 }, 877 }; 878 879 880 881 /** 882 * Translate tag ID to register. 883 * 884 * @param[in] dev 885 * Pointer to the Ethernet device structure. 886 * @param[in] feature 887 * The feature that request the register. 888 * @param[in] id 889 * The request register ID. 890 * @param[out] error 891 * Error description in case of any. 892 * 893 * @return 894 * The request register on success, a negative errno 895 * value otherwise and rte_errno is set. 896 */ 897 int 898 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 899 enum mlx5_feature_name feature, 900 uint32_t id, 901 struct rte_flow_error *error) 902 { 903 struct mlx5_priv *priv = dev->data->dev_private; 904 struct mlx5_dev_config *config = &priv->config; 905 enum modify_reg start_reg; 906 bool skip_mtr_reg = false; 907 908 switch (feature) { 909 case MLX5_HAIRPIN_RX: 910 return REG_B; 911 case MLX5_HAIRPIN_TX: 912 return REG_A; 913 case MLX5_METADATA_RX: 914 switch (config->dv_xmeta_en) { 915 case MLX5_XMETA_MODE_LEGACY: 916 return REG_B; 917 case MLX5_XMETA_MODE_META16: 918 return REG_C_0; 919 case MLX5_XMETA_MODE_META32: 920 return REG_C_1; 921 } 922 break; 923 case MLX5_METADATA_TX: 924 return REG_A; 925 case MLX5_METADATA_FDB: 926 switch (config->dv_xmeta_en) { 927 case MLX5_XMETA_MODE_LEGACY: 928 return REG_NON; 929 case MLX5_XMETA_MODE_META16: 930 return REG_C_0; 931 case MLX5_XMETA_MODE_META32: 932 return REG_C_1; 933 } 934 break; 935 case MLX5_FLOW_MARK: 936 switch (config->dv_xmeta_en) { 937 case MLX5_XMETA_MODE_LEGACY: 938 return REG_NON; 939 case MLX5_XMETA_MODE_META16: 940 return REG_C_1; 941 case MLX5_XMETA_MODE_META32: 942 return REG_C_0; 943 } 944 break; 945 case MLX5_MTR_ID: 946 /* 947 * If meter color and meter id share one register, flow match 948 * should use the meter color register for match. 949 */ 950 if (priv->mtr_reg_share) 951 return priv->mtr_color_reg; 952 else 953 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 954 REG_C_3; 955 case MLX5_MTR_COLOR: 956 case MLX5_ASO_FLOW_HIT: 957 case MLX5_ASO_CONNTRACK: 958 case MLX5_SAMPLE_ID: 959 /* All features use the same REG_C. */ 960 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 961 return priv->mtr_color_reg; 962 case MLX5_COPY_MARK: 963 /* 964 * Metadata COPY_MARK register using is in meter suffix sub 965 * flow while with meter. It's safe to share the same register. 966 */ 967 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 968 case MLX5_APP_TAG: 969 /* 970 * If meter is enable, it will engage the register for color 971 * match and flow match. If meter color match is not using the 972 * REG_C_2, need to skip the REG_C_x be used by meter color 973 * match. 974 * If meter is disable, free to use all available registers. 975 */ 976 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 977 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 978 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 979 if (id > (uint32_t)(REG_C_7 - start_reg)) 980 return rte_flow_error_set(error, EINVAL, 981 RTE_FLOW_ERROR_TYPE_ITEM, 982 NULL, "invalid tag id"); 983 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 984 return rte_flow_error_set(error, ENOTSUP, 985 RTE_FLOW_ERROR_TYPE_ITEM, 986 NULL, "unsupported tag id"); 987 /* 988 * This case means meter is using the REG_C_x great than 2. 989 * Take care not to conflict with meter color REG_C_x. 990 * If the available index REG_C_y >= REG_C_x, skip the 991 * color register. 992 */ 993 if (skip_mtr_reg && priv->sh->flow_mreg_c 994 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 995 if (id >= (uint32_t)(REG_C_7 - start_reg)) 996 return rte_flow_error_set(error, EINVAL, 997 RTE_FLOW_ERROR_TYPE_ITEM, 998 NULL, "invalid tag id"); 999 if (priv->sh->flow_mreg_c 1000 [id + 1 + start_reg - REG_C_0] != REG_NON) 1001 return priv->sh->flow_mreg_c 1002 [id + 1 + start_reg - REG_C_0]; 1003 return rte_flow_error_set(error, ENOTSUP, 1004 RTE_FLOW_ERROR_TYPE_ITEM, 1005 NULL, "unsupported tag id"); 1006 } 1007 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1008 } 1009 MLX5_ASSERT(false); 1010 return rte_flow_error_set(error, EINVAL, 1011 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1012 NULL, "invalid feature name"); 1013 } 1014 1015 /** 1016 * Check extensive flow metadata register support. 1017 * 1018 * @param dev 1019 * Pointer to rte_eth_dev structure. 1020 * 1021 * @return 1022 * True if device supports extensive flow metadata register, otherwise false. 1023 */ 1024 bool 1025 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1026 { 1027 struct mlx5_priv *priv = dev->data->dev_private; 1028 1029 /* 1030 * Having available reg_c can be regarded inclusively as supporting 1031 * extensive flow metadata register, which could mean, 1032 * - metadata register copy action by modify header. 1033 * - 16 modify header actions is supported. 1034 * - reg_c's are preserved across different domain (FDB and NIC) on 1035 * packet loopback by flow lookup miss. 1036 */ 1037 return priv->sh->flow_mreg_c[2] != REG_NON; 1038 } 1039 1040 /** 1041 * Get the lowest priority. 1042 * 1043 * @param[in] dev 1044 * Pointer to the Ethernet device structure. 1045 * @param[in] attributes 1046 * Pointer to device flow rule attributes. 1047 * 1048 * @return 1049 * The value of lowest priority of flow. 1050 */ 1051 uint32_t 1052 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1053 const struct rte_flow_attr *attr) 1054 { 1055 struct mlx5_priv *priv = dev->data->dev_private; 1056 1057 if (!attr->group && !attr->transfer) 1058 return priv->sh->flow_max_priority - 2; 1059 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1060 } 1061 1062 /** 1063 * Calculate matcher priority of the flow. 1064 * 1065 * @param[in] dev 1066 * Pointer to the Ethernet device structure. 1067 * @param[in] attr 1068 * Pointer to device flow rule attributes. 1069 * @param[in] subpriority 1070 * The priority based on the items. 1071 * @param[in] external 1072 * Flow is user flow. 1073 * @return 1074 * The matcher priority of the flow. 1075 */ 1076 uint16_t 1077 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1078 const struct rte_flow_attr *attr, 1079 uint32_t subpriority, bool external) 1080 { 1081 uint16_t priority = (uint16_t)attr->priority; 1082 struct mlx5_priv *priv = dev->data->dev_private; 1083 1084 if (!attr->group && !attr->transfer) { 1085 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1086 priority = priv->sh->flow_max_priority - 1; 1087 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1088 } else if (!external && attr->transfer && attr->group == 0 && 1089 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1090 return (priv->sh->flow_max_priority - 1) * 3; 1091 } 1092 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1093 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1094 return priority * 3 + subpriority; 1095 } 1096 1097 /** 1098 * Verify the @p item specifications (spec, last, mask) are compatible with the 1099 * NIC capabilities. 1100 * 1101 * @param[in] item 1102 * Item specification. 1103 * @param[in] mask 1104 * @p item->mask or flow default bit-masks. 1105 * @param[in] nic_mask 1106 * Bit-masks covering supported fields by the NIC to compare with user mask. 1107 * @param[in] size 1108 * Bit-masks size in bytes. 1109 * @param[in] range_accepted 1110 * True if range of values is accepted for specific fields, false otherwise. 1111 * @param[out] error 1112 * Pointer to error structure. 1113 * 1114 * @return 1115 * 0 on success, a negative errno value otherwise and rte_errno is set. 1116 */ 1117 int 1118 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1119 const uint8_t *mask, 1120 const uint8_t *nic_mask, 1121 unsigned int size, 1122 bool range_accepted, 1123 struct rte_flow_error *error) 1124 { 1125 unsigned int i; 1126 1127 MLX5_ASSERT(nic_mask); 1128 for (i = 0; i < size; ++i) 1129 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1130 return rte_flow_error_set(error, ENOTSUP, 1131 RTE_FLOW_ERROR_TYPE_ITEM, 1132 item, 1133 "mask enables non supported" 1134 " bits"); 1135 if (!item->spec && (item->mask || item->last)) 1136 return rte_flow_error_set(error, EINVAL, 1137 RTE_FLOW_ERROR_TYPE_ITEM, item, 1138 "mask/last without a spec is not" 1139 " supported"); 1140 if (item->spec && item->last && !range_accepted) { 1141 uint8_t spec[size]; 1142 uint8_t last[size]; 1143 unsigned int i; 1144 int ret; 1145 1146 for (i = 0; i < size; ++i) { 1147 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1148 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1149 } 1150 ret = memcmp(spec, last, size); 1151 if (ret != 0) 1152 return rte_flow_error_set(error, EINVAL, 1153 RTE_FLOW_ERROR_TYPE_ITEM, 1154 item, 1155 "range is not valid"); 1156 } 1157 return 0; 1158 } 1159 1160 /** 1161 * Adjust the hash fields according to the @p flow information. 1162 * 1163 * @param[in] dev_flow. 1164 * Pointer to the mlx5_flow. 1165 * @param[in] tunnel 1166 * 1 when the hash field is for a tunnel item. 1167 * @param[in] layer_types 1168 * RTE_ETH_RSS_* types. 1169 * @param[in] hash_fields 1170 * Item hash fields. 1171 * 1172 * @return 1173 * The hash fields that should be used. 1174 */ 1175 uint64_t 1176 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1177 int tunnel __rte_unused, uint64_t layer_types, 1178 uint64_t hash_fields) 1179 { 1180 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1181 int rss_request_inner = rss_desc->level >= 2; 1182 1183 /* Check RSS hash level for tunnel. */ 1184 if (tunnel && rss_request_inner) 1185 hash_fields |= IBV_RX_HASH_INNER; 1186 else if (tunnel || rss_request_inner) 1187 return 0; 1188 #endif 1189 /* Check if requested layer matches RSS hash fields. */ 1190 if (!(rss_desc->types & layer_types)) 1191 return 0; 1192 return hash_fields; 1193 } 1194 1195 /** 1196 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1197 * if several tunnel rules are used on this queue, the tunnel ptype will be 1198 * cleared. 1199 * 1200 * @param rxq_ctrl 1201 * Rx queue to update. 1202 */ 1203 static void 1204 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1205 { 1206 unsigned int i; 1207 uint32_t tunnel_ptype = 0; 1208 1209 /* Look up for the ptype to use. */ 1210 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1211 if (!rxq_ctrl->flow_tunnels_n[i]) 1212 continue; 1213 if (!tunnel_ptype) { 1214 tunnel_ptype = tunnels_info[i].ptype; 1215 } else { 1216 tunnel_ptype = 0; 1217 break; 1218 } 1219 } 1220 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1221 } 1222 1223 /** 1224 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1225 * flow. 1226 * 1227 * @param[in] dev 1228 * Pointer to the Ethernet device structure. 1229 * @param[in] dev_handle 1230 * Pointer to device flow handle structure. 1231 */ 1232 void 1233 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1234 struct mlx5_flow_handle *dev_handle) 1235 { 1236 struct mlx5_priv *priv = dev->data->dev_private; 1237 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1238 struct mlx5_ind_table_obj *ind_tbl = NULL; 1239 unsigned int i; 1240 1241 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1242 struct mlx5_hrxq *hrxq; 1243 1244 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1245 dev_handle->rix_hrxq); 1246 if (hrxq) 1247 ind_tbl = hrxq->ind_table; 1248 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1249 struct mlx5_shared_action_rss *shared_rss; 1250 1251 shared_rss = mlx5_ipool_get 1252 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1253 dev_handle->rix_srss); 1254 if (shared_rss) 1255 ind_tbl = shared_rss->ind_tbl; 1256 } 1257 if (!ind_tbl) 1258 return; 1259 for (i = 0; i != ind_tbl->queues_n; ++i) { 1260 int idx = ind_tbl->queues[i]; 1261 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1262 1263 MLX5_ASSERT(rxq_ctrl != NULL); 1264 if (rxq_ctrl == NULL) 1265 continue; 1266 /* 1267 * To support metadata register copy on Tx loopback, 1268 * this must be always enabled (metadata may arive 1269 * from other port - not from local flows only. 1270 */ 1271 if (tunnel) { 1272 unsigned int j; 1273 1274 /* Increase the counter matching the flow. */ 1275 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1276 if ((tunnels_info[j].tunnel & 1277 dev_handle->layers) == 1278 tunnels_info[j].tunnel) { 1279 rxq_ctrl->flow_tunnels_n[j]++; 1280 break; 1281 } 1282 } 1283 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1284 } 1285 } 1286 } 1287 1288 static void 1289 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1290 { 1291 struct mlx5_priv *priv = dev->data->dev_private; 1292 struct mlx5_rxq_ctrl *rxq_ctrl; 1293 1294 if (priv->mark_enabled) 1295 return; 1296 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1297 rxq_ctrl->rxq.mark = 1; 1298 } 1299 priv->mark_enabled = 1; 1300 } 1301 1302 /** 1303 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1304 * 1305 * @param[in] dev 1306 * Pointer to the Ethernet device structure. 1307 * @param[in] flow 1308 * Pointer to flow structure. 1309 */ 1310 static void 1311 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1312 { 1313 struct mlx5_priv *priv = dev->data->dev_private; 1314 uint32_t handle_idx; 1315 struct mlx5_flow_handle *dev_handle; 1316 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1317 1318 MLX5_ASSERT(wks); 1319 if (wks->mark) 1320 flow_rxq_mark_flag_set(dev); 1321 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1322 handle_idx, dev_handle, next) 1323 flow_drv_rxq_flags_set(dev, dev_handle); 1324 } 1325 1326 /** 1327 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1328 * device flow if no other flow uses it with the same kind of request. 1329 * 1330 * @param dev 1331 * Pointer to Ethernet device. 1332 * @param[in] dev_handle 1333 * Pointer to the device flow handle structure. 1334 */ 1335 static void 1336 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1337 struct mlx5_flow_handle *dev_handle) 1338 { 1339 struct mlx5_priv *priv = dev->data->dev_private; 1340 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1341 struct mlx5_ind_table_obj *ind_tbl = NULL; 1342 unsigned int i; 1343 1344 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1345 struct mlx5_hrxq *hrxq; 1346 1347 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1348 dev_handle->rix_hrxq); 1349 if (hrxq) 1350 ind_tbl = hrxq->ind_table; 1351 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1352 struct mlx5_shared_action_rss *shared_rss; 1353 1354 shared_rss = mlx5_ipool_get 1355 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1356 dev_handle->rix_srss); 1357 if (shared_rss) 1358 ind_tbl = shared_rss->ind_tbl; 1359 } 1360 if (!ind_tbl) 1361 return; 1362 MLX5_ASSERT(dev->data->dev_started); 1363 for (i = 0; i != ind_tbl->queues_n; ++i) { 1364 int idx = ind_tbl->queues[i]; 1365 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1366 1367 MLX5_ASSERT(rxq_ctrl != NULL); 1368 if (rxq_ctrl == NULL) 1369 continue; 1370 if (tunnel) { 1371 unsigned int j; 1372 1373 /* Decrease the counter matching the flow. */ 1374 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1375 if ((tunnels_info[j].tunnel & 1376 dev_handle->layers) == 1377 tunnels_info[j].tunnel) { 1378 rxq_ctrl->flow_tunnels_n[j]--; 1379 break; 1380 } 1381 } 1382 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1383 } 1384 } 1385 } 1386 1387 /** 1388 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1389 * @p flow if no other flow uses it with the same kind of request. 1390 * 1391 * @param dev 1392 * Pointer to Ethernet device. 1393 * @param[in] flow 1394 * Pointer to the flow. 1395 */ 1396 static void 1397 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1398 { 1399 struct mlx5_priv *priv = dev->data->dev_private; 1400 uint32_t handle_idx; 1401 struct mlx5_flow_handle *dev_handle; 1402 1403 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1404 handle_idx, dev_handle, next) 1405 flow_drv_rxq_flags_trim(dev, dev_handle); 1406 } 1407 1408 /** 1409 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1410 * 1411 * @param dev 1412 * Pointer to Ethernet device. 1413 */ 1414 static void 1415 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1416 { 1417 struct mlx5_priv *priv = dev->data->dev_private; 1418 unsigned int i; 1419 1420 for (i = 0; i != priv->rxqs_n; ++i) { 1421 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1422 unsigned int j; 1423 1424 if (rxq == NULL || rxq->ctrl == NULL) 1425 continue; 1426 rxq->ctrl->rxq.mark = 0; 1427 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1428 rxq->ctrl->flow_tunnels_n[j] = 0; 1429 rxq->ctrl->rxq.tunnel = 0; 1430 } 1431 priv->mark_enabled = 0; 1432 } 1433 1434 /** 1435 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1436 * 1437 * @param[in] dev 1438 * Pointer to the Ethernet device structure. 1439 */ 1440 void 1441 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1442 { 1443 struct mlx5_priv *priv = dev->data->dev_private; 1444 unsigned int i; 1445 1446 for (i = 0; i != priv->rxqs_n; ++i) { 1447 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1448 struct mlx5_rxq_data *data; 1449 1450 if (rxq == NULL || rxq->ctrl == NULL) 1451 continue; 1452 data = &rxq->ctrl->rxq; 1453 if (!rte_flow_dynf_metadata_avail()) { 1454 data->dynf_meta = 0; 1455 data->flow_meta_mask = 0; 1456 data->flow_meta_offset = -1; 1457 data->flow_meta_port_mask = 0; 1458 } else { 1459 data->dynf_meta = 1; 1460 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1461 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1462 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1463 } 1464 } 1465 } 1466 1467 /* 1468 * return a pointer to the desired action in the list of actions. 1469 * 1470 * @param[in] actions 1471 * The list of actions to search the action in. 1472 * @param[in] action 1473 * The action to find. 1474 * 1475 * @return 1476 * Pointer to the action in the list, if found. NULL otherwise. 1477 */ 1478 const struct rte_flow_action * 1479 mlx5_flow_find_action(const struct rte_flow_action *actions, 1480 enum rte_flow_action_type action) 1481 { 1482 if (actions == NULL) 1483 return NULL; 1484 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1485 if (actions->type == action) 1486 return actions; 1487 return NULL; 1488 } 1489 1490 /* 1491 * Validate the flag action. 1492 * 1493 * @param[in] action_flags 1494 * Bit-fields that holds the actions detected until now. 1495 * @param[in] attr 1496 * Attributes of flow that includes this action. 1497 * @param[out] error 1498 * Pointer to error structure. 1499 * 1500 * @return 1501 * 0 on success, a negative errno value otherwise and rte_errno is set. 1502 */ 1503 int 1504 mlx5_flow_validate_action_flag(uint64_t action_flags, 1505 const struct rte_flow_attr *attr, 1506 struct rte_flow_error *error) 1507 { 1508 if (action_flags & MLX5_FLOW_ACTION_MARK) 1509 return rte_flow_error_set(error, EINVAL, 1510 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1511 "can't mark and flag in same flow"); 1512 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1513 return rte_flow_error_set(error, EINVAL, 1514 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1515 "can't have 2 flag" 1516 " actions in same flow"); 1517 if (attr->egress) 1518 return rte_flow_error_set(error, ENOTSUP, 1519 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1520 "flag action not supported for " 1521 "egress"); 1522 return 0; 1523 } 1524 1525 /* 1526 * Validate the mark action. 1527 * 1528 * @param[in] action 1529 * Pointer to the queue action. 1530 * @param[in] action_flags 1531 * Bit-fields that holds the actions detected until now. 1532 * @param[in] attr 1533 * Attributes of flow that includes this action. 1534 * @param[out] error 1535 * Pointer to error structure. 1536 * 1537 * @return 1538 * 0 on success, a negative errno value otherwise and rte_errno is set. 1539 */ 1540 int 1541 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1542 uint64_t action_flags, 1543 const struct rte_flow_attr *attr, 1544 struct rte_flow_error *error) 1545 { 1546 const struct rte_flow_action_mark *mark = action->conf; 1547 1548 if (!mark) 1549 return rte_flow_error_set(error, EINVAL, 1550 RTE_FLOW_ERROR_TYPE_ACTION, 1551 action, 1552 "configuration cannot be null"); 1553 if (mark->id >= MLX5_FLOW_MARK_MAX) 1554 return rte_flow_error_set(error, EINVAL, 1555 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1556 &mark->id, 1557 "mark id must in 0 <= id < " 1558 RTE_STR(MLX5_FLOW_MARK_MAX)); 1559 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1560 return rte_flow_error_set(error, EINVAL, 1561 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1562 "can't flag and mark in same flow"); 1563 if (action_flags & MLX5_FLOW_ACTION_MARK) 1564 return rte_flow_error_set(error, EINVAL, 1565 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1566 "can't have 2 mark actions in same" 1567 " flow"); 1568 if (attr->egress) 1569 return rte_flow_error_set(error, ENOTSUP, 1570 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1571 "mark action not supported for " 1572 "egress"); 1573 return 0; 1574 } 1575 1576 /* 1577 * Validate the drop action. 1578 * 1579 * @param[in] action_flags 1580 * Bit-fields that holds the actions detected until now. 1581 * @param[in] attr 1582 * Attributes of flow that includes this action. 1583 * @param[out] error 1584 * Pointer to error structure. 1585 * 1586 * @return 1587 * 0 on success, a negative errno value otherwise and rte_errno is set. 1588 */ 1589 int 1590 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1591 const struct rte_flow_attr *attr, 1592 struct rte_flow_error *error) 1593 { 1594 if (attr->egress) 1595 return rte_flow_error_set(error, ENOTSUP, 1596 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1597 "drop action not supported for " 1598 "egress"); 1599 return 0; 1600 } 1601 1602 /* 1603 * Validate the queue action. 1604 * 1605 * @param[in] action 1606 * Pointer to the queue action. 1607 * @param[in] action_flags 1608 * Bit-fields that holds the actions detected until now. 1609 * @param[in] dev 1610 * Pointer to the Ethernet device structure. 1611 * @param[in] attr 1612 * Attributes of flow that includes this action. 1613 * @param[out] error 1614 * Pointer to error structure. 1615 * 1616 * @return 1617 * 0 on success, a negative errno value otherwise and rte_errno is set. 1618 */ 1619 int 1620 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1621 uint64_t action_flags, 1622 struct rte_eth_dev *dev, 1623 const struct rte_flow_attr *attr, 1624 struct rte_flow_error *error) 1625 { 1626 struct mlx5_priv *priv = dev->data->dev_private; 1627 const struct rte_flow_action_queue *queue = action->conf; 1628 1629 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1630 return rte_flow_error_set(error, EINVAL, 1631 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1632 "can't have 2 fate actions in" 1633 " same flow"); 1634 if (!priv->rxqs_n) 1635 return rte_flow_error_set(error, EINVAL, 1636 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1637 NULL, "No Rx queues configured"); 1638 if (queue->index >= priv->rxqs_n) 1639 return rte_flow_error_set(error, EINVAL, 1640 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1641 &queue->index, 1642 "queue index out of range"); 1643 if (mlx5_rxq_get(dev, queue->index) == NULL) 1644 return rte_flow_error_set(error, EINVAL, 1645 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1646 &queue->index, 1647 "queue is not configured"); 1648 if (attr->egress) 1649 return rte_flow_error_set(error, ENOTSUP, 1650 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1651 "queue action not supported for " 1652 "egress"); 1653 return 0; 1654 } 1655 1656 /** 1657 * Validate queue numbers for device RSS. 1658 * 1659 * @param[in] dev 1660 * Configured device. 1661 * @param[in] queues 1662 * Array of queue numbers. 1663 * @param[in] queues_n 1664 * Size of the @p queues array. 1665 * @param[out] error 1666 * On error, filled with a textual error description. 1667 * @param[out] queue 1668 * On error, filled with an offending queue index in @p queues array. 1669 * 1670 * @return 1671 * 0 on success, a negative errno code on error. 1672 */ 1673 static int 1674 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1675 const uint16_t *queues, uint32_t queues_n, 1676 const char **error, uint32_t *queue_idx) 1677 { 1678 const struct mlx5_priv *priv = dev->data->dev_private; 1679 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1680 uint32_t i; 1681 1682 for (i = 0; i != queues_n; ++i) { 1683 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, 1684 queues[i]); 1685 1686 if (queues[i] >= priv->rxqs_n) { 1687 *error = "queue index out of range"; 1688 *queue_idx = i; 1689 return -EINVAL; 1690 } 1691 if (rxq_ctrl == NULL) { 1692 *error = "queue is not configured"; 1693 *queue_idx = i; 1694 return -EINVAL; 1695 } 1696 if (i == 0) 1697 rxq_type = rxq_ctrl->type; 1698 if (rxq_type != rxq_ctrl->type) { 1699 *error = "combining hairpin and regular RSS queues is not supported"; 1700 *queue_idx = i; 1701 return -ENOTSUP; 1702 } 1703 } 1704 return 0; 1705 } 1706 1707 /* 1708 * Validate the rss action. 1709 * 1710 * @param[in] dev 1711 * Pointer to the Ethernet device structure. 1712 * @param[in] action 1713 * Pointer to the queue action. 1714 * @param[out] error 1715 * Pointer to error structure. 1716 * 1717 * @return 1718 * 0 on success, a negative errno value otherwise and rte_errno is set. 1719 */ 1720 int 1721 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1722 const struct rte_flow_action *action, 1723 struct rte_flow_error *error) 1724 { 1725 struct mlx5_priv *priv = dev->data->dev_private; 1726 const struct rte_flow_action_rss *rss = action->conf; 1727 int ret; 1728 const char *message; 1729 uint32_t queue_idx; 1730 1731 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1732 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1733 return rte_flow_error_set(error, ENOTSUP, 1734 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1735 &rss->func, 1736 "RSS hash function not supported"); 1737 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1738 if (rss->level > 2) 1739 #else 1740 if (rss->level > 1) 1741 #endif 1742 return rte_flow_error_set(error, ENOTSUP, 1743 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1744 &rss->level, 1745 "tunnel RSS is not supported"); 1746 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1747 if (rss->key_len == 0 && rss->key != NULL) 1748 return rte_flow_error_set(error, ENOTSUP, 1749 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1750 &rss->key_len, 1751 "RSS hash key length 0"); 1752 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1753 return rte_flow_error_set(error, ENOTSUP, 1754 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1755 &rss->key_len, 1756 "RSS hash key too small"); 1757 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1758 return rte_flow_error_set(error, ENOTSUP, 1759 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1760 &rss->key_len, 1761 "RSS hash key too large"); 1762 if (rss->queue_num > priv->config.ind_table_max_size) 1763 return rte_flow_error_set(error, ENOTSUP, 1764 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1765 &rss->queue_num, 1766 "number of queues too large"); 1767 if (rss->types & MLX5_RSS_HF_MASK) 1768 return rte_flow_error_set(error, ENOTSUP, 1769 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1770 &rss->types, 1771 "some RSS protocols are not" 1772 " supported"); 1773 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1774 !(rss->types & RTE_ETH_RSS_IP)) 1775 return rte_flow_error_set(error, EINVAL, 1776 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1777 "L3 partial RSS requested but L3 RSS" 1778 " type not specified"); 1779 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1780 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1781 return rte_flow_error_set(error, EINVAL, 1782 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1783 "L4 partial RSS requested but L4 RSS" 1784 " type not specified"); 1785 if (!priv->rxqs_n) 1786 return rte_flow_error_set(error, EINVAL, 1787 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1788 NULL, "No Rx queues configured"); 1789 if (!rss->queue_num) 1790 return rte_flow_error_set(error, EINVAL, 1791 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1792 NULL, "No queues configured"); 1793 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1794 &message, &queue_idx); 1795 if (ret != 0) { 1796 return rte_flow_error_set(error, -ret, 1797 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1798 &rss->queue[queue_idx], message); 1799 } 1800 return 0; 1801 } 1802 1803 /* 1804 * Validate the rss action. 1805 * 1806 * @param[in] action 1807 * Pointer to the queue action. 1808 * @param[in] action_flags 1809 * Bit-fields that holds the actions detected until now. 1810 * @param[in] dev 1811 * Pointer to the Ethernet device structure. 1812 * @param[in] attr 1813 * Attributes of flow that includes this action. 1814 * @param[in] item_flags 1815 * Items that were detected. 1816 * @param[out] error 1817 * Pointer to error structure. 1818 * 1819 * @return 1820 * 0 on success, a negative errno value otherwise and rte_errno is set. 1821 */ 1822 int 1823 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1824 uint64_t action_flags, 1825 struct rte_eth_dev *dev, 1826 const struct rte_flow_attr *attr, 1827 uint64_t item_flags, 1828 struct rte_flow_error *error) 1829 { 1830 const struct rte_flow_action_rss *rss = action->conf; 1831 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1832 int ret; 1833 1834 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1835 return rte_flow_error_set(error, EINVAL, 1836 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1837 "can't have 2 fate actions" 1838 " in same flow"); 1839 ret = mlx5_validate_action_rss(dev, action, error); 1840 if (ret) 1841 return ret; 1842 if (attr->egress) 1843 return rte_flow_error_set(error, ENOTSUP, 1844 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1845 "rss action not supported for " 1846 "egress"); 1847 if (rss->level > 1 && !tunnel) 1848 return rte_flow_error_set(error, EINVAL, 1849 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1850 "inner RSS is not supported for " 1851 "non-tunnel flows"); 1852 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1853 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1854 return rte_flow_error_set(error, EINVAL, 1855 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1856 "RSS on eCPRI is not supported now"); 1857 } 1858 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1859 !(item_flags & 1860 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1861 rss->level > 1) 1862 return rte_flow_error_set(error, EINVAL, 1863 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1864 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1865 return 0; 1866 } 1867 1868 /* 1869 * Validate the default miss action. 1870 * 1871 * @param[in] action_flags 1872 * Bit-fields that holds the actions detected until now. 1873 * @param[out] error 1874 * Pointer to error structure. 1875 * 1876 * @return 1877 * 0 on success, a negative errno value otherwise and rte_errno is set. 1878 */ 1879 int 1880 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1881 const struct rte_flow_attr *attr, 1882 struct rte_flow_error *error) 1883 { 1884 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1885 return rte_flow_error_set(error, EINVAL, 1886 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1887 "can't have 2 fate actions in" 1888 " same flow"); 1889 if (attr->egress) 1890 return rte_flow_error_set(error, ENOTSUP, 1891 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1892 "default miss action not supported " 1893 "for egress"); 1894 if (attr->group) 1895 return rte_flow_error_set(error, ENOTSUP, 1896 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1897 "only group 0 is supported"); 1898 if (attr->transfer) 1899 return rte_flow_error_set(error, ENOTSUP, 1900 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1901 NULL, "transfer is not supported"); 1902 return 0; 1903 } 1904 1905 /* 1906 * Validate the count action. 1907 * 1908 * @param[in] dev 1909 * Pointer to the Ethernet device structure. 1910 * @param[in] attr 1911 * Attributes of flow that includes this action. 1912 * @param[out] error 1913 * Pointer to error structure. 1914 * 1915 * @return 1916 * 0 on success, a negative errno value otherwise and rte_errno is set. 1917 */ 1918 int 1919 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1920 const struct rte_flow_attr *attr, 1921 struct rte_flow_error *error) 1922 { 1923 if (attr->egress) 1924 return rte_flow_error_set(error, ENOTSUP, 1925 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1926 "count action not supported for " 1927 "egress"); 1928 return 0; 1929 } 1930 1931 /* 1932 * Validate the ASO CT action. 1933 * 1934 * @param[in] dev 1935 * Pointer to the Ethernet device structure. 1936 * @param[in] conntrack 1937 * Pointer to the CT action profile. 1938 * @param[out] error 1939 * Pointer to error structure. 1940 * 1941 * @return 1942 * 0 on success, a negative errno value otherwise and rte_errno is set. 1943 */ 1944 int 1945 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1946 const struct rte_flow_action_conntrack *conntrack, 1947 struct rte_flow_error *error) 1948 { 1949 RTE_SET_USED(dev); 1950 1951 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1952 return rte_flow_error_set(error, EINVAL, 1953 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1954 "Invalid CT state"); 1955 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1956 return rte_flow_error_set(error, EINVAL, 1957 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1958 "Invalid last TCP packet flag"); 1959 return 0; 1960 } 1961 1962 /** 1963 * Verify the @p attributes will be correctly understood by the NIC and store 1964 * them in the @p flow if everything is correct. 1965 * 1966 * @param[in] dev 1967 * Pointer to the Ethernet device structure. 1968 * @param[in] attributes 1969 * Pointer to flow attributes 1970 * @param[out] error 1971 * Pointer to error structure. 1972 * 1973 * @return 1974 * 0 on success, a negative errno value otherwise and rte_errno is set. 1975 */ 1976 int 1977 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1978 const struct rte_flow_attr *attributes, 1979 struct rte_flow_error *error) 1980 { 1981 struct mlx5_priv *priv = dev->data->dev_private; 1982 uint32_t priority_max = priv->sh->flow_max_priority - 1; 1983 1984 if (attributes->group) 1985 return rte_flow_error_set(error, ENOTSUP, 1986 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1987 NULL, "groups is not supported"); 1988 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1989 attributes->priority >= priority_max) 1990 return rte_flow_error_set(error, ENOTSUP, 1991 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1992 NULL, "priority out of range"); 1993 if (attributes->egress) 1994 return rte_flow_error_set(error, ENOTSUP, 1995 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1996 "egress is not supported"); 1997 if (attributes->transfer && !priv->config.dv_esw_en) 1998 return rte_flow_error_set(error, ENOTSUP, 1999 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2000 NULL, "transfer is not supported"); 2001 if (!attributes->ingress) 2002 return rte_flow_error_set(error, EINVAL, 2003 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2004 NULL, 2005 "ingress attribute is mandatory"); 2006 return 0; 2007 } 2008 2009 /** 2010 * Validate ICMP6 item. 2011 * 2012 * @param[in] item 2013 * Item specification. 2014 * @param[in] item_flags 2015 * Bit-fields that holds the items detected until now. 2016 * @param[in] ext_vlan_sup 2017 * Whether extended VLAN features are supported or not. 2018 * @param[out] error 2019 * Pointer to error structure. 2020 * 2021 * @return 2022 * 0 on success, a negative errno value otherwise and rte_errno is set. 2023 */ 2024 int 2025 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2026 uint64_t item_flags, 2027 uint8_t target_protocol, 2028 struct rte_flow_error *error) 2029 { 2030 const struct rte_flow_item_icmp6 *mask = item->mask; 2031 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2032 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2033 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2034 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2035 MLX5_FLOW_LAYER_OUTER_L4; 2036 int ret; 2037 2038 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2039 return rte_flow_error_set(error, EINVAL, 2040 RTE_FLOW_ERROR_TYPE_ITEM, item, 2041 "protocol filtering not compatible" 2042 " with ICMP6 layer"); 2043 if (!(item_flags & l3m)) 2044 return rte_flow_error_set(error, EINVAL, 2045 RTE_FLOW_ERROR_TYPE_ITEM, item, 2046 "IPv6 is mandatory to filter on" 2047 " ICMP6"); 2048 if (item_flags & l4m) 2049 return rte_flow_error_set(error, EINVAL, 2050 RTE_FLOW_ERROR_TYPE_ITEM, item, 2051 "multiple L4 layers not supported"); 2052 if (!mask) 2053 mask = &rte_flow_item_icmp6_mask; 2054 ret = mlx5_flow_item_acceptable 2055 (item, (const uint8_t *)mask, 2056 (const uint8_t *)&rte_flow_item_icmp6_mask, 2057 sizeof(struct rte_flow_item_icmp6), 2058 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2059 if (ret < 0) 2060 return ret; 2061 return 0; 2062 } 2063 2064 /** 2065 * Validate ICMP item. 2066 * 2067 * @param[in] item 2068 * Item specification. 2069 * @param[in] item_flags 2070 * Bit-fields that holds the items detected until now. 2071 * @param[out] error 2072 * Pointer to error structure. 2073 * 2074 * @return 2075 * 0 on success, a negative errno value otherwise and rte_errno is set. 2076 */ 2077 int 2078 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2079 uint64_t item_flags, 2080 uint8_t target_protocol, 2081 struct rte_flow_error *error) 2082 { 2083 const struct rte_flow_item_icmp *mask = item->mask; 2084 const struct rte_flow_item_icmp nic_mask = { 2085 .hdr.icmp_type = 0xff, 2086 .hdr.icmp_code = 0xff, 2087 .hdr.icmp_ident = RTE_BE16(0xffff), 2088 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2089 }; 2090 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2091 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2092 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2093 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2094 MLX5_FLOW_LAYER_OUTER_L4; 2095 int ret; 2096 2097 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2098 return rte_flow_error_set(error, EINVAL, 2099 RTE_FLOW_ERROR_TYPE_ITEM, item, 2100 "protocol filtering not compatible" 2101 " with ICMP layer"); 2102 if (!(item_flags & l3m)) 2103 return rte_flow_error_set(error, EINVAL, 2104 RTE_FLOW_ERROR_TYPE_ITEM, item, 2105 "IPv4 is mandatory to filter" 2106 " on ICMP"); 2107 if (item_flags & l4m) 2108 return rte_flow_error_set(error, EINVAL, 2109 RTE_FLOW_ERROR_TYPE_ITEM, item, 2110 "multiple L4 layers not supported"); 2111 if (!mask) 2112 mask = &nic_mask; 2113 ret = mlx5_flow_item_acceptable 2114 (item, (const uint8_t *)mask, 2115 (const uint8_t *)&nic_mask, 2116 sizeof(struct rte_flow_item_icmp), 2117 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2118 if (ret < 0) 2119 return ret; 2120 return 0; 2121 } 2122 2123 /** 2124 * Validate Ethernet item. 2125 * 2126 * @param[in] item 2127 * Item specification. 2128 * @param[in] item_flags 2129 * Bit-fields that holds the items detected until now. 2130 * @param[out] error 2131 * Pointer to error structure. 2132 * 2133 * @return 2134 * 0 on success, a negative errno value otherwise and rte_errno is set. 2135 */ 2136 int 2137 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2138 uint64_t item_flags, bool ext_vlan_sup, 2139 struct rte_flow_error *error) 2140 { 2141 const struct rte_flow_item_eth *mask = item->mask; 2142 const struct rte_flow_item_eth nic_mask = { 2143 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2144 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2145 .type = RTE_BE16(0xffff), 2146 .has_vlan = ext_vlan_sup ? 1 : 0, 2147 }; 2148 int ret; 2149 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2150 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2151 MLX5_FLOW_LAYER_OUTER_L2; 2152 2153 if (item_flags & ethm) 2154 return rte_flow_error_set(error, ENOTSUP, 2155 RTE_FLOW_ERROR_TYPE_ITEM, item, 2156 "multiple L2 layers not supported"); 2157 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2158 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2159 return rte_flow_error_set(error, EINVAL, 2160 RTE_FLOW_ERROR_TYPE_ITEM, item, 2161 "L2 layer should not follow " 2162 "L3 layers"); 2163 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2164 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2165 return rte_flow_error_set(error, EINVAL, 2166 RTE_FLOW_ERROR_TYPE_ITEM, item, 2167 "L2 layer should not follow VLAN"); 2168 if (item_flags & MLX5_FLOW_LAYER_GTP) 2169 return rte_flow_error_set(error, EINVAL, 2170 RTE_FLOW_ERROR_TYPE_ITEM, item, 2171 "L2 layer should not follow GTP"); 2172 if (!mask) 2173 mask = &rte_flow_item_eth_mask; 2174 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2175 (const uint8_t *)&nic_mask, 2176 sizeof(struct rte_flow_item_eth), 2177 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2178 return ret; 2179 } 2180 2181 /** 2182 * Validate VLAN item. 2183 * 2184 * @param[in] item 2185 * Item specification. 2186 * @param[in] item_flags 2187 * Bit-fields that holds the items detected until now. 2188 * @param[in] dev 2189 * Ethernet device flow is being created on. 2190 * @param[out] error 2191 * Pointer to error structure. 2192 * 2193 * @return 2194 * 0 on success, a negative errno value otherwise and rte_errno is set. 2195 */ 2196 int 2197 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2198 uint64_t item_flags, 2199 struct rte_eth_dev *dev, 2200 struct rte_flow_error *error) 2201 { 2202 const struct rte_flow_item_vlan *spec = item->spec; 2203 const struct rte_flow_item_vlan *mask = item->mask; 2204 const struct rte_flow_item_vlan nic_mask = { 2205 .tci = RTE_BE16(UINT16_MAX), 2206 .inner_type = RTE_BE16(UINT16_MAX), 2207 }; 2208 uint16_t vlan_tag = 0; 2209 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2210 int ret; 2211 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2212 MLX5_FLOW_LAYER_INNER_L4) : 2213 (MLX5_FLOW_LAYER_OUTER_L3 | 2214 MLX5_FLOW_LAYER_OUTER_L4); 2215 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2216 MLX5_FLOW_LAYER_OUTER_VLAN; 2217 2218 if (item_flags & vlanm) 2219 return rte_flow_error_set(error, EINVAL, 2220 RTE_FLOW_ERROR_TYPE_ITEM, item, 2221 "multiple VLAN layers not supported"); 2222 else if ((item_flags & l34m) != 0) 2223 return rte_flow_error_set(error, EINVAL, 2224 RTE_FLOW_ERROR_TYPE_ITEM, item, 2225 "VLAN cannot follow L3/L4 layer"); 2226 if (!mask) 2227 mask = &rte_flow_item_vlan_mask; 2228 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2229 (const uint8_t *)&nic_mask, 2230 sizeof(struct rte_flow_item_vlan), 2231 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2232 if (ret) 2233 return ret; 2234 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2235 struct mlx5_priv *priv = dev->data->dev_private; 2236 2237 if (priv->vmwa_context) { 2238 /* 2239 * Non-NULL context means we have a virtual machine 2240 * and SR-IOV enabled, we have to create VLAN interface 2241 * to make hypervisor to setup E-Switch vport 2242 * context correctly. We avoid creating the multiple 2243 * VLAN interfaces, so we cannot support VLAN tag mask. 2244 */ 2245 return rte_flow_error_set(error, EINVAL, 2246 RTE_FLOW_ERROR_TYPE_ITEM, 2247 item, 2248 "VLAN tag mask is not" 2249 " supported in virtual" 2250 " environment"); 2251 } 2252 } 2253 if (spec) { 2254 vlan_tag = spec->tci; 2255 vlan_tag &= mask->tci; 2256 } 2257 /* 2258 * From verbs perspective an empty VLAN is equivalent 2259 * to a packet without VLAN layer. 2260 */ 2261 if (!vlan_tag) 2262 return rte_flow_error_set(error, EINVAL, 2263 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2264 item->spec, 2265 "VLAN cannot be empty"); 2266 return 0; 2267 } 2268 2269 /** 2270 * Validate IPV4 item. 2271 * 2272 * @param[in] item 2273 * Item specification. 2274 * @param[in] item_flags 2275 * Bit-fields that holds the items detected until now. 2276 * @param[in] last_item 2277 * Previous validated item in the pattern items. 2278 * @param[in] ether_type 2279 * Type in the ethernet layer header (including dot1q). 2280 * @param[in] acc_mask 2281 * Acceptable mask, if NULL default internal default mask 2282 * will be used to check whether item fields are supported. 2283 * @param[in] range_accepted 2284 * True if range of values is accepted for specific fields, false otherwise. 2285 * @param[out] error 2286 * Pointer to error structure. 2287 * 2288 * @return 2289 * 0 on success, a negative errno value otherwise and rte_errno is set. 2290 */ 2291 int 2292 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2293 uint64_t item_flags, 2294 uint64_t last_item, 2295 uint16_t ether_type, 2296 const struct rte_flow_item_ipv4 *acc_mask, 2297 bool range_accepted, 2298 struct rte_flow_error *error) 2299 { 2300 const struct rte_flow_item_ipv4 *mask = item->mask; 2301 const struct rte_flow_item_ipv4 *spec = item->spec; 2302 const struct rte_flow_item_ipv4 nic_mask = { 2303 .hdr = { 2304 .src_addr = RTE_BE32(0xffffffff), 2305 .dst_addr = RTE_BE32(0xffffffff), 2306 .type_of_service = 0xff, 2307 .next_proto_id = 0xff, 2308 }, 2309 }; 2310 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2311 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2312 MLX5_FLOW_LAYER_OUTER_L3; 2313 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2314 MLX5_FLOW_LAYER_OUTER_L4; 2315 int ret; 2316 uint8_t next_proto = 0xFF; 2317 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2318 MLX5_FLOW_LAYER_OUTER_VLAN | 2319 MLX5_FLOW_LAYER_INNER_VLAN); 2320 2321 if ((last_item & l2_vlan) && ether_type && 2322 ether_type != RTE_ETHER_TYPE_IPV4) 2323 return rte_flow_error_set(error, EINVAL, 2324 RTE_FLOW_ERROR_TYPE_ITEM, item, 2325 "IPv4 cannot follow L2/VLAN layer " 2326 "which ether type is not IPv4"); 2327 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2328 if (mask && spec) 2329 next_proto = mask->hdr.next_proto_id & 2330 spec->hdr.next_proto_id; 2331 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2332 return rte_flow_error_set(error, EINVAL, 2333 RTE_FLOW_ERROR_TYPE_ITEM, 2334 item, 2335 "multiple tunnel " 2336 "not supported"); 2337 } 2338 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2339 return rte_flow_error_set(error, EINVAL, 2340 RTE_FLOW_ERROR_TYPE_ITEM, item, 2341 "wrong tunnel type - IPv6 specified " 2342 "but IPv4 item provided"); 2343 if (item_flags & l3m) 2344 return rte_flow_error_set(error, ENOTSUP, 2345 RTE_FLOW_ERROR_TYPE_ITEM, item, 2346 "multiple L3 layers not supported"); 2347 else if (item_flags & l4m) 2348 return rte_flow_error_set(error, EINVAL, 2349 RTE_FLOW_ERROR_TYPE_ITEM, item, 2350 "L3 cannot follow an L4 layer."); 2351 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2352 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2353 return rte_flow_error_set(error, EINVAL, 2354 RTE_FLOW_ERROR_TYPE_ITEM, item, 2355 "L3 cannot follow an NVGRE layer."); 2356 if (!mask) 2357 mask = &rte_flow_item_ipv4_mask; 2358 else if (mask->hdr.next_proto_id != 0 && 2359 mask->hdr.next_proto_id != 0xff) 2360 return rte_flow_error_set(error, EINVAL, 2361 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2362 "partial mask is not supported" 2363 " for protocol"); 2364 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2365 acc_mask ? (const uint8_t *)acc_mask 2366 : (const uint8_t *)&nic_mask, 2367 sizeof(struct rte_flow_item_ipv4), 2368 range_accepted, error); 2369 if (ret < 0) 2370 return ret; 2371 return 0; 2372 } 2373 2374 /** 2375 * Validate IPV6 item. 2376 * 2377 * @param[in] item 2378 * Item specification. 2379 * @param[in] item_flags 2380 * Bit-fields that holds the items detected until now. 2381 * @param[in] last_item 2382 * Previous validated item in the pattern items. 2383 * @param[in] ether_type 2384 * Type in the ethernet layer header (including dot1q). 2385 * @param[in] acc_mask 2386 * Acceptable mask, if NULL default internal default mask 2387 * will be used to check whether item fields are supported. 2388 * @param[out] error 2389 * Pointer to error structure. 2390 * 2391 * @return 2392 * 0 on success, a negative errno value otherwise and rte_errno is set. 2393 */ 2394 int 2395 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2396 uint64_t item_flags, 2397 uint64_t last_item, 2398 uint16_t ether_type, 2399 const struct rte_flow_item_ipv6 *acc_mask, 2400 struct rte_flow_error *error) 2401 { 2402 const struct rte_flow_item_ipv6 *mask = item->mask; 2403 const struct rte_flow_item_ipv6 *spec = item->spec; 2404 const struct rte_flow_item_ipv6 nic_mask = { 2405 .hdr = { 2406 .src_addr = 2407 "\xff\xff\xff\xff\xff\xff\xff\xff" 2408 "\xff\xff\xff\xff\xff\xff\xff\xff", 2409 .dst_addr = 2410 "\xff\xff\xff\xff\xff\xff\xff\xff" 2411 "\xff\xff\xff\xff\xff\xff\xff\xff", 2412 .vtc_flow = RTE_BE32(0xffffffff), 2413 .proto = 0xff, 2414 }, 2415 }; 2416 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2417 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2418 MLX5_FLOW_LAYER_OUTER_L3; 2419 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2420 MLX5_FLOW_LAYER_OUTER_L4; 2421 int ret; 2422 uint8_t next_proto = 0xFF; 2423 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2424 MLX5_FLOW_LAYER_OUTER_VLAN | 2425 MLX5_FLOW_LAYER_INNER_VLAN); 2426 2427 if ((last_item & l2_vlan) && ether_type && 2428 ether_type != RTE_ETHER_TYPE_IPV6) 2429 return rte_flow_error_set(error, EINVAL, 2430 RTE_FLOW_ERROR_TYPE_ITEM, item, 2431 "IPv6 cannot follow L2/VLAN layer " 2432 "which ether type is not IPv6"); 2433 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2434 next_proto = spec->hdr.proto; 2435 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2436 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2437 return rte_flow_error_set(error, EINVAL, 2438 RTE_FLOW_ERROR_TYPE_ITEM, 2439 item, 2440 "multiple tunnel " 2441 "not supported"); 2442 } 2443 if (next_proto == IPPROTO_HOPOPTS || 2444 next_proto == IPPROTO_ROUTING || 2445 next_proto == IPPROTO_FRAGMENT || 2446 next_proto == IPPROTO_ESP || 2447 next_proto == IPPROTO_AH || 2448 next_proto == IPPROTO_DSTOPTS) 2449 return rte_flow_error_set(error, EINVAL, 2450 RTE_FLOW_ERROR_TYPE_ITEM, item, 2451 "IPv6 proto (next header) should " 2452 "not be set as extension header"); 2453 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2454 return rte_flow_error_set(error, EINVAL, 2455 RTE_FLOW_ERROR_TYPE_ITEM, item, 2456 "wrong tunnel type - IPv4 specified " 2457 "but IPv6 item provided"); 2458 if (item_flags & l3m) 2459 return rte_flow_error_set(error, ENOTSUP, 2460 RTE_FLOW_ERROR_TYPE_ITEM, item, 2461 "multiple L3 layers not supported"); 2462 else if (item_flags & l4m) 2463 return rte_flow_error_set(error, EINVAL, 2464 RTE_FLOW_ERROR_TYPE_ITEM, item, 2465 "L3 cannot follow an L4 layer."); 2466 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2467 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2468 return rte_flow_error_set(error, EINVAL, 2469 RTE_FLOW_ERROR_TYPE_ITEM, item, 2470 "L3 cannot follow an NVGRE layer."); 2471 if (!mask) 2472 mask = &rte_flow_item_ipv6_mask; 2473 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2474 acc_mask ? (const uint8_t *)acc_mask 2475 : (const uint8_t *)&nic_mask, 2476 sizeof(struct rte_flow_item_ipv6), 2477 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2478 if (ret < 0) 2479 return ret; 2480 return 0; 2481 } 2482 2483 /** 2484 * Validate UDP item. 2485 * 2486 * @param[in] item 2487 * Item specification. 2488 * @param[in] item_flags 2489 * Bit-fields that holds the items detected until now. 2490 * @param[in] target_protocol 2491 * The next protocol in the previous item. 2492 * @param[in] flow_mask 2493 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2494 * @param[out] error 2495 * Pointer to error structure. 2496 * 2497 * @return 2498 * 0 on success, a negative errno value otherwise and rte_errno is set. 2499 */ 2500 int 2501 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2502 uint64_t item_flags, 2503 uint8_t target_protocol, 2504 struct rte_flow_error *error) 2505 { 2506 const struct rte_flow_item_udp *mask = item->mask; 2507 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2508 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2509 MLX5_FLOW_LAYER_OUTER_L3; 2510 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2511 MLX5_FLOW_LAYER_OUTER_L4; 2512 int ret; 2513 2514 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2515 return rte_flow_error_set(error, EINVAL, 2516 RTE_FLOW_ERROR_TYPE_ITEM, item, 2517 "protocol filtering not compatible" 2518 " with UDP layer"); 2519 if (!(item_flags & l3m)) 2520 return rte_flow_error_set(error, EINVAL, 2521 RTE_FLOW_ERROR_TYPE_ITEM, item, 2522 "L3 is mandatory to filter on L4"); 2523 if (item_flags & l4m) 2524 return rte_flow_error_set(error, EINVAL, 2525 RTE_FLOW_ERROR_TYPE_ITEM, item, 2526 "multiple L4 layers not supported"); 2527 if (!mask) 2528 mask = &rte_flow_item_udp_mask; 2529 ret = mlx5_flow_item_acceptable 2530 (item, (const uint8_t *)mask, 2531 (const uint8_t *)&rte_flow_item_udp_mask, 2532 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2533 error); 2534 if (ret < 0) 2535 return ret; 2536 return 0; 2537 } 2538 2539 /** 2540 * Validate TCP item. 2541 * 2542 * @param[in] item 2543 * Item specification. 2544 * @param[in] item_flags 2545 * Bit-fields that holds the items detected until now. 2546 * @param[in] target_protocol 2547 * The next protocol in the previous item. 2548 * @param[out] error 2549 * Pointer to error structure. 2550 * 2551 * @return 2552 * 0 on success, a negative errno value otherwise and rte_errno is set. 2553 */ 2554 int 2555 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2556 uint64_t item_flags, 2557 uint8_t target_protocol, 2558 const struct rte_flow_item_tcp *flow_mask, 2559 struct rte_flow_error *error) 2560 { 2561 const struct rte_flow_item_tcp *mask = item->mask; 2562 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2563 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2564 MLX5_FLOW_LAYER_OUTER_L3; 2565 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2566 MLX5_FLOW_LAYER_OUTER_L4; 2567 int ret; 2568 2569 MLX5_ASSERT(flow_mask); 2570 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2571 return rte_flow_error_set(error, EINVAL, 2572 RTE_FLOW_ERROR_TYPE_ITEM, item, 2573 "protocol filtering not compatible" 2574 " with TCP layer"); 2575 if (!(item_flags & l3m)) 2576 return rte_flow_error_set(error, EINVAL, 2577 RTE_FLOW_ERROR_TYPE_ITEM, item, 2578 "L3 is mandatory to filter on L4"); 2579 if (item_flags & l4m) 2580 return rte_flow_error_set(error, EINVAL, 2581 RTE_FLOW_ERROR_TYPE_ITEM, item, 2582 "multiple L4 layers not supported"); 2583 if (!mask) 2584 mask = &rte_flow_item_tcp_mask; 2585 ret = mlx5_flow_item_acceptable 2586 (item, (const uint8_t *)mask, 2587 (const uint8_t *)flow_mask, 2588 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2589 error); 2590 if (ret < 0) 2591 return ret; 2592 return 0; 2593 } 2594 2595 /** 2596 * Validate VXLAN item. 2597 * 2598 * @param[in] dev 2599 * Pointer to the Ethernet device structure. 2600 * @param[in] udp_dport 2601 * UDP destination port 2602 * @param[in] item 2603 * Item specification. 2604 * @param[in] item_flags 2605 * Bit-fields that holds the items detected until now. 2606 * @param[in] attr 2607 * Flow rule attributes. 2608 * @param[out] error 2609 * Pointer to error structure. 2610 * 2611 * @return 2612 * 0 on success, a negative errno value otherwise and rte_errno is set. 2613 */ 2614 int 2615 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2616 uint16_t udp_dport, 2617 const struct rte_flow_item *item, 2618 uint64_t item_flags, 2619 const struct rte_flow_attr *attr, 2620 struct rte_flow_error *error) 2621 { 2622 const struct rte_flow_item_vxlan *spec = item->spec; 2623 const struct rte_flow_item_vxlan *mask = item->mask; 2624 int ret; 2625 struct mlx5_priv *priv = dev->data->dev_private; 2626 union vni { 2627 uint32_t vlan_id; 2628 uint8_t vni[4]; 2629 } id = { .vlan_id = 0, }; 2630 const struct rte_flow_item_vxlan nic_mask = { 2631 .vni = "\xff\xff\xff", 2632 .rsvd1 = 0xff, 2633 }; 2634 const struct rte_flow_item_vxlan *valid_mask; 2635 2636 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2637 return rte_flow_error_set(error, ENOTSUP, 2638 RTE_FLOW_ERROR_TYPE_ITEM, item, 2639 "multiple tunnel layers not" 2640 " supported"); 2641 valid_mask = &rte_flow_item_vxlan_mask; 2642 /* 2643 * Verify only UDPv4 is present as defined in 2644 * https://tools.ietf.org/html/rfc7348 2645 */ 2646 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2647 return rte_flow_error_set(error, EINVAL, 2648 RTE_FLOW_ERROR_TYPE_ITEM, item, 2649 "no outer UDP layer found"); 2650 if (!mask) 2651 mask = &rte_flow_item_vxlan_mask; 2652 2653 if (priv->sh->steering_format_version != 2654 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2655 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2656 /* FDB domain & NIC domain non-zero group */ 2657 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2658 valid_mask = &nic_mask; 2659 /* Group zero in NIC domain */ 2660 if (!attr->group && !attr->transfer && 2661 priv->sh->tunnel_header_0_1) 2662 valid_mask = &nic_mask; 2663 } 2664 ret = mlx5_flow_item_acceptable 2665 (item, (const uint8_t *)mask, 2666 (const uint8_t *)valid_mask, 2667 sizeof(struct rte_flow_item_vxlan), 2668 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2669 if (ret < 0) 2670 return ret; 2671 if (spec) { 2672 memcpy(&id.vni[1], spec->vni, 3); 2673 memcpy(&id.vni[1], mask->vni, 3); 2674 } 2675 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2676 return rte_flow_error_set(error, ENOTSUP, 2677 RTE_FLOW_ERROR_TYPE_ITEM, item, 2678 "VXLAN tunnel must be fully defined"); 2679 return 0; 2680 } 2681 2682 /** 2683 * Validate VXLAN_GPE item. 2684 * 2685 * @param[in] item 2686 * Item specification. 2687 * @param[in] item_flags 2688 * Bit-fields that holds the items detected until now. 2689 * @param[in] priv 2690 * Pointer to the private data structure. 2691 * @param[in] target_protocol 2692 * The next protocol in the previous item. 2693 * @param[out] error 2694 * Pointer to error structure. 2695 * 2696 * @return 2697 * 0 on success, a negative errno value otherwise and rte_errno is set. 2698 */ 2699 int 2700 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2701 uint64_t item_flags, 2702 struct rte_eth_dev *dev, 2703 struct rte_flow_error *error) 2704 { 2705 struct mlx5_priv *priv = dev->data->dev_private; 2706 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2707 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2708 int ret; 2709 union vni { 2710 uint32_t vlan_id; 2711 uint8_t vni[4]; 2712 } id = { .vlan_id = 0, }; 2713 2714 if (!priv->config.l3_vxlan_en) 2715 return rte_flow_error_set(error, ENOTSUP, 2716 RTE_FLOW_ERROR_TYPE_ITEM, item, 2717 "L3 VXLAN is not enabled by device" 2718 " parameter and/or not configured in" 2719 " firmware"); 2720 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2721 return rte_flow_error_set(error, ENOTSUP, 2722 RTE_FLOW_ERROR_TYPE_ITEM, item, 2723 "multiple tunnel layers not" 2724 " supported"); 2725 /* 2726 * Verify only UDPv4 is present as defined in 2727 * https://tools.ietf.org/html/rfc7348 2728 */ 2729 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2730 return rte_flow_error_set(error, EINVAL, 2731 RTE_FLOW_ERROR_TYPE_ITEM, item, 2732 "no outer UDP layer found"); 2733 if (!mask) 2734 mask = &rte_flow_item_vxlan_gpe_mask; 2735 ret = mlx5_flow_item_acceptable 2736 (item, (const uint8_t *)mask, 2737 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2738 sizeof(struct rte_flow_item_vxlan_gpe), 2739 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2740 if (ret < 0) 2741 return ret; 2742 if (spec) { 2743 if (spec->protocol) 2744 return rte_flow_error_set(error, ENOTSUP, 2745 RTE_FLOW_ERROR_TYPE_ITEM, 2746 item, 2747 "VxLAN-GPE protocol" 2748 " not supported"); 2749 memcpy(&id.vni[1], spec->vni, 3); 2750 memcpy(&id.vni[1], mask->vni, 3); 2751 } 2752 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2753 return rte_flow_error_set(error, ENOTSUP, 2754 RTE_FLOW_ERROR_TYPE_ITEM, item, 2755 "VXLAN-GPE tunnel must be fully" 2756 " defined"); 2757 return 0; 2758 } 2759 /** 2760 * Validate GRE Key item. 2761 * 2762 * @param[in] item 2763 * Item specification. 2764 * @param[in] item_flags 2765 * Bit flags to mark detected items. 2766 * @param[in] gre_item 2767 * Pointer to gre_item 2768 * @param[out] error 2769 * Pointer to error structure. 2770 * 2771 * @return 2772 * 0 on success, a negative errno value otherwise and rte_errno is set. 2773 */ 2774 int 2775 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2776 uint64_t item_flags, 2777 const struct rte_flow_item *gre_item, 2778 struct rte_flow_error *error) 2779 { 2780 const rte_be32_t *mask = item->mask; 2781 int ret = 0; 2782 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2783 const struct rte_flow_item_gre *gre_spec; 2784 const struct rte_flow_item_gre *gre_mask; 2785 2786 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2787 return rte_flow_error_set(error, ENOTSUP, 2788 RTE_FLOW_ERROR_TYPE_ITEM, item, 2789 "Multiple GRE key not support"); 2790 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2791 return rte_flow_error_set(error, ENOTSUP, 2792 RTE_FLOW_ERROR_TYPE_ITEM, item, 2793 "No preceding GRE header"); 2794 if (item_flags & MLX5_FLOW_LAYER_INNER) 2795 return rte_flow_error_set(error, ENOTSUP, 2796 RTE_FLOW_ERROR_TYPE_ITEM, item, 2797 "GRE key following a wrong item"); 2798 gre_mask = gre_item->mask; 2799 if (!gre_mask) 2800 gre_mask = &rte_flow_item_gre_mask; 2801 gre_spec = gre_item->spec; 2802 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2803 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2804 return rte_flow_error_set(error, EINVAL, 2805 RTE_FLOW_ERROR_TYPE_ITEM, item, 2806 "Key bit must be on"); 2807 2808 if (!mask) 2809 mask = &gre_key_default_mask; 2810 ret = mlx5_flow_item_acceptable 2811 (item, (const uint8_t *)mask, 2812 (const uint8_t *)&gre_key_default_mask, 2813 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2814 return ret; 2815 } 2816 2817 /** 2818 * Validate GRE item. 2819 * 2820 * @param[in] item 2821 * Item specification. 2822 * @param[in] item_flags 2823 * Bit flags to mark detected items. 2824 * @param[in] target_protocol 2825 * The next protocol in the previous item. 2826 * @param[out] error 2827 * Pointer to error structure. 2828 * 2829 * @return 2830 * 0 on success, a negative errno value otherwise and rte_errno is set. 2831 */ 2832 int 2833 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2834 uint64_t item_flags, 2835 uint8_t target_protocol, 2836 struct rte_flow_error *error) 2837 { 2838 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2839 const struct rte_flow_item_gre *mask = item->mask; 2840 int ret; 2841 const struct rte_flow_item_gre nic_mask = { 2842 .c_rsvd0_ver = RTE_BE16(0xB000), 2843 .protocol = RTE_BE16(UINT16_MAX), 2844 }; 2845 2846 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2847 return rte_flow_error_set(error, EINVAL, 2848 RTE_FLOW_ERROR_TYPE_ITEM, item, 2849 "protocol filtering not compatible" 2850 " with this GRE layer"); 2851 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2852 return rte_flow_error_set(error, ENOTSUP, 2853 RTE_FLOW_ERROR_TYPE_ITEM, item, 2854 "multiple tunnel layers not" 2855 " supported"); 2856 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2857 return rte_flow_error_set(error, ENOTSUP, 2858 RTE_FLOW_ERROR_TYPE_ITEM, item, 2859 "L3 Layer is missing"); 2860 if (!mask) 2861 mask = &rte_flow_item_gre_mask; 2862 ret = mlx5_flow_item_acceptable 2863 (item, (const uint8_t *)mask, 2864 (const uint8_t *)&nic_mask, 2865 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2866 error); 2867 if (ret < 0) 2868 return ret; 2869 #ifndef HAVE_MLX5DV_DR 2870 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2871 if (spec && (spec->protocol & mask->protocol)) 2872 return rte_flow_error_set(error, ENOTSUP, 2873 RTE_FLOW_ERROR_TYPE_ITEM, item, 2874 "without MPLS support the" 2875 " specification cannot be used for" 2876 " filtering"); 2877 #endif 2878 #endif 2879 return 0; 2880 } 2881 2882 /** 2883 * Validate Geneve item. 2884 * 2885 * @param[in] item 2886 * Item specification. 2887 * @param[in] itemFlags 2888 * Bit-fields that holds the items detected until now. 2889 * @param[in] enPriv 2890 * Pointer to the private data structure. 2891 * @param[out] error 2892 * Pointer to error structure. 2893 * 2894 * @return 2895 * 0 on success, a negative errno value otherwise and rte_errno is set. 2896 */ 2897 2898 int 2899 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2900 uint64_t item_flags, 2901 struct rte_eth_dev *dev, 2902 struct rte_flow_error *error) 2903 { 2904 struct mlx5_priv *priv = dev->data->dev_private; 2905 const struct rte_flow_item_geneve *spec = item->spec; 2906 const struct rte_flow_item_geneve *mask = item->mask; 2907 int ret; 2908 uint16_t gbhdr; 2909 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2910 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2911 const struct rte_flow_item_geneve nic_mask = { 2912 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2913 .vni = "\xff\xff\xff", 2914 .protocol = RTE_BE16(UINT16_MAX), 2915 }; 2916 2917 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2918 return rte_flow_error_set(error, ENOTSUP, 2919 RTE_FLOW_ERROR_TYPE_ITEM, item, 2920 "L3 Geneve is not enabled by device" 2921 " parameter and/or not configured in" 2922 " firmware"); 2923 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2924 return rte_flow_error_set(error, ENOTSUP, 2925 RTE_FLOW_ERROR_TYPE_ITEM, item, 2926 "multiple tunnel layers not" 2927 " supported"); 2928 /* 2929 * Verify only UDPv4 is present as defined in 2930 * https://tools.ietf.org/html/rfc7348 2931 */ 2932 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2933 return rte_flow_error_set(error, EINVAL, 2934 RTE_FLOW_ERROR_TYPE_ITEM, item, 2935 "no outer UDP layer found"); 2936 if (!mask) 2937 mask = &rte_flow_item_geneve_mask; 2938 ret = mlx5_flow_item_acceptable 2939 (item, (const uint8_t *)mask, 2940 (const uint8_t *)&nic_mask, 2941 sizeof(struct rte_flow_item_geneve), 2942 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2943 if (ret) 2944 return ret; 2945 if (spec) { 2946 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2947 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2948 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2949 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2950 return rte_flow_error_set(error, ENOTSUP, 2951 RTE_FLOW_ERROR_TYPE_ITEM, 2952 item, 2953 "Geneve protocol unsupported" 2954 " fields are being used"); 2955 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2956 return rte_flow_error_set 2957 (error, ENOTSUP, 2958 RTE_FLOW_ERROR_TYPE_ITEM, 2959 item, 2960 "Unsupported Geneve options length"); 2961 } 2962 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2963 return rte_flow_error_set 2964 (error, ENOTSUP, 2965 RTE_FLOW_ERROR_TYPE_ITEM, item, 2966 "Geneve tunnel must be fully defined"); 2967 return 0; 2968 } 2969 2970 /** 2971 * Validate Geneve TLV option item. 2972 * 2973 * @param[in] item 2974 * Item specification. 2975 * @param[in] last_item 2976 * Previous validated item in the pattern items. 2977 * @param[in] geneve_item 2978 * Previous GENEVE item specification. 2979 * @param[in] dev 2980 * Pointer to the rte_eth_dev structure. 2981 * @param[out] error 2982 * Pointer to error structure. 2983 * 2984 * @return 2985 * 0 on success, a negative errno value otherwise and rte_errno is set. 2986 */ 2987 int 2988 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2989 uint64_t last_item, 2990 const struct rte_flow_item *geneve_item, 2991 struct rte_eth_dev *dev, 2992 struct rte_flow_error *error) 2993 { 2994 struct mlx5_priv *priv = dev->data->dev_private; 2995 struct mlx5_dev_ctx_shared *sh = priv->sh; 2996 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2997 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2998 uint8_t data_max_supported = 2999 hca_attr->max_geneve_tlv_option_data_len * 4; 3000 struct mlx5_dev_config *config = &priv->config; 3001 const struct rte_flow_item_geneve *geneve_spec; 3002 const struct rte_flow_item_geneve *geneve_mask; 3003 const struct rte_flow_item_geneve_opt *spec = item->spec; 3004 const struct rte_flow_item_geneve_opt *mask = item->mask; 3005 unsigned int i; 3006 unsigned int data_len; 3007 uint8_t tlv_option_len; 3008 uint16_t optlen_m, optlen_v; 3009 const struct rte_flow_item_geneve_opt full_mask = { 3010 .option_class = RTE_BE16(0xffff), 3011 .option_type = 0xff, 3012 .option_len = 0x1f, 3013 }; 3014 3015 if (!mask) 3016 mask = &rte_flow_item_geneve_opt_mask; 3017 if (!spec) 3018 return rte_flow_error_set 3019 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3020 "Geneve TLV opt class/type/length must be specified"); 3021 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3022 return rte_flow_error_set 3023 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3024 "Geneve TLV opt length exceeds the limit (31)"); 3025 /* Check if class type and length masks are full. */ 3026 if (full_mask.option_class != mask->option_class || 3027 full_mask.option_type != mask->option_type || 3028 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3029 return rte_flow_error_set 3030 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3031 "Geneve TLV opt class/type/length masks must be full"); 3032 /* Check if length is supported */ 3033 if ((uint32_t)spec->option_len > 3034 config->hca_attr.max_geneve_tlv_option_data_len) 3035 return rte_flow_error_set 3036 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3037 "Geneve TLV opt length not supported"); 3038 if (config->hca_attr.max_geneve_tlv_options > 1) 3039 DRV_LOG(DEBUG, 3040 "max_geneve_tlv_options supports more than 1 option"); 3041 /* Check GENEVE item preceding. */ 3042 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3043 return rte_flow_error_set 3044 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3045 "Geneve opt item must be preceded with Geneve item"); 3046 geneve_spec = geneve_item->spec; 3047 geneve_mask = geneve_item->mask ? geneve_item->mask : 3048 &rte_flow_item_geneve_mask; 3049 /* Check if GENEVE TLV option size doesn't exceed option length */ 3050 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3051 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3052 tlv_option_len = spec->option_len & mask->option_len; 3053 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3054 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3055 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3056 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3057 if ((optlen_v & optlen_m) <= tlv_option_len) 3058 return rte_flow_error_set 3059 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3060 "GENEVE TLV option length exceeds optlen"); 3061 } 3062 /* Check if length is 0 or data is 0. */ 3063 if (spec->data == NULL || spec->option_len == 0) 3064 return rte_flow_error_set 3065 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3066 "Geneve TLV opt with zero data/length not supported"); 3067 /* Check not all data & mask are 0. */ 3068 data_len = spec->option_len * 4; 3069 if (mask->data == NULL) { 3070 for (i = 0; i < data_len; i++) 3071 if (spec->data[i]) 3072 break; 3073 if (i == data_len) 3074 return rte_flow_error_set(error, ENOTSUP, 3075 RTE_FLOW_ERROR_TYPE_ITEM, item, 3076 "Can't match on Geneve option data 0"); 3077 } else { 3078 for (i = 0; i < data_len; i++) 3079 if (spec->data[i] & mask->data[i]) 3080 break; 3081 if (i == data_len) 3082 return rte_flow_error_set(error, ENOTSUP, 3083 RTE_FLOW_ERROR_TYPE_ITEM, item, 3084 "Can't match on Geneve option data and mask 0"); 3085 /* Check data mask supported. */ 3086 for (i = data_max_supported; i < data_len ; i++) 3087 if (mask->data[i]) 3088 return rte_flow_error_set(error, ENOTSUP, 3089 RTE_FLOW_ERROR_TYPE_ITEM, item, 3090 "Data mask is of unsupported size"); 3091 } 3092 /* Check GENEVE option is supported in NIC. */ 3093 if (!config->hca_attr.geneve_tlv_opt) 3094 return rte_flow_error_set 3095 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3096 "Geneve TLV opt not supported"); 3097 /* Check if we already have geneve option with different type/class. */ 3098 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3099 geneve_opt_resource = sh->geneve_tlv_option_resource; 3100 if (geneve_opt_resource != NULL) 3101 if (geneve_opt_resource->option_class != spec->option_class || 3102 geneve_opt_resource->option_type != spec->option_type || 3103 geneve_opt_resource->length != spec->option_len) { 3104 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3105 return rte_flow_error_set(error, ENOTSUP, 3106 RTE_FLOW_ERROR_TYPE_ITEM, item, 3107 "Only one Geneve TLV option supported"); 3108 } 3109 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3110 return 0; 3111 } 3112 3113 /** 3114 * Validate MPLS item. 3115 * 3116 * @param[in] dev 3117 * Pointer to the rte_eth_dev structure. 3118 * @param[in] item 3119 * Item specification. 3120 * @param[in] item_flags 3121 * Bit-fields that holds the items detected until now. 3122 * @param[in] prev_layer 3123 * The protocol layer indicated in previous item. 3124 * @param[out] error 3125 * Pointer to error structure. 3126 * 3127 * @return 3128 * 0 on success, a negative errno value otherwise and rte_errno is set. 3129 */ 3130 int 3131 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3132 const struct rte_flow_item *item __rte_unused, 3133 uint64_t item_flags __rte_unused, 3134 uint64_t prev_layer __rte_unused, 3135 struct rte_flow_error *error) 3136 { 3137 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3138 const struct rte_flow_item_mpls *mask = item->mask; 3139 struct mlx5_priv *priv = dev->data->dev_private; 3140 int ret; 3141 3142 if (!priv->config.mpls_en) 3143 return rte_flow_error_set(error, ENOTSUP, 3144 RTE_FLOW_ERROR_TYPE_ITEM, item, 3145 "MPLS not supported or" 3146 " disabled in firmware" 3147 " configuration."); 3148 /* MPLS over UDP, GRE is allowed */ 3149 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3150 MLX5_FLOW_LAYER_GRE | 3151 MLX5_FLOW_LAYER_GRE_KEY))) 3152 return rte_flow_error_set(error, EINVAL, 3153 RTE_FLOW_ERROR_TYPE_ITEM, item, 3154 "protocol filtering not compatible" 3155 " with MPLS layer"); 3156 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3157 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3158 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3159 return rte_flow_error_set(error, ENOTSUP, 3160 RTE_FLOW_ERROR_TYPE_ITEM, item, 3161 "multiple tunnel layers not" 3162 " supported"); 3163 if (!mask) 3164 mask = &rte_flow_item_mpls_mask; 3165 ret = mlx5_flow_item_acceptable 3166 (item, (const uint8_t *)mask, 3167 (const uint8_t *)&rte_flow_item_mpls_mask, 3168 sizeof(struct rte_flow_item_mpls), 3169 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3170 if (ret < 0) 3171 return ret; 3172 return 0; 3173 #else 3174 return rte_flow_error_set(error, ENOTSUP, 3175 RTE_FLOW_ERROR_TYPE_ITEM, item, 3176 "MPLS is not supported by Verbs, please" 3177 " update."); 3178 #endif 3179 } 3180 3181 /** 3182 * Validate NVGRE item. 3183 * 3184 * @param[in] item 3185 * Item specification. 3186 * @param[in] item_flags 3187 * Bit flags to mark detected items. 3188 * @param[in] target_protocol 3189 * The next protocol in the previous item. 3190 * @param[out] error 3191 * Pointer to error structure. 3192 * 3193 * @return 3194 * 0 on success, a negative errno value otherwise and rte_errno is set. 3195 */ 3196 int 3197 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3198 uint64_t item_flags, 3199 uint8_t target_protocol, 3200 struct rte_flow_error *error) 3201 { 3202 const struct rte_flow_item_nvgre *mask = item->mask; 3203 int ret; 3204 3205 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3206 return rte_flow_error_set(error, EINVAL, 3207 RTE_FLOW_ERROR_TYPE_ITEM, item, 3208 "protocol filtering not compatible" 3209 " with this GRE layer"); 3210 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3211 return rte_flow_error_set(error, ENOTSUP, 3212 RTE_FLOW_ERROR_TYPE_ITEM, item, 3213 "multiple tunnel layers not" 3214 " supported"); 3215 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3216 return rte_flow_error_set(error, ENOTSUP, 3217 RTE_FLOW_ERROR_TYPE_ITEM, item, 3218 "L3 Layer is missing"); 3219 if (!mask) 3220 mask = &rte_flow_item_nvgre_mask; 3221 ret = mlx5_flow_item_acceptable 3222 (item, (const uint8_t *)mask, 3223 (const uint8_t *)&rte_flow_item_nvgre_mask, 3224 sizeof(struct rte_flow_item_nvgre), 3225 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3226 if (ret < 0) 3227 return ret; 3228 return 0; 3229 } 3230 3231 /** 3232 * Validate eCPRI item. 3233 * 3234 * @param[in] item 3235 * Item specification. 3236 * @param[in] item_flags 3237 * Bit-fields that holds the items detected until now. 3238 * @param[in] last_item 3239 * Previous validated item in the pattern items. 3240 * @param[in] ether_type 3241 * Type in the ethernet layer header (including dot1q). 3242 * @param[in] acc_mask 3243 * Acceptable mask, if NULL default internal default mask 3244 * will be used to check whether item fields are supported. 3245 * @param[out] error 3246 * Pointer to error structure. 3247 * 3248 * @return 3249 * 0 on success, a negative errno value otherwise and rte_errno is set. 3250 */ 3251 int 3252 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3253 uint64_t item_flags, 3254 uint64_t last_item, 3255 uint16_t ether_type, 3256 const struct rte_flow_item_ecpri *acc_mask, 3257 struct rte_flow_error *error) 3258 { 3259 const struct rte_flow_item_ecpri *mask = item->mask; 3260 const struct rte_flow_item_ecpri nic_mask = { 3261 .hdr = { 3262 .common = { 3263 .u32 = 3264 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3265 .type = 0xFF, 3266 }).u32), 3267 }, 3268 .dummy[0] = 0xFFFFFFFF, 3269 }, 3270 }; 3271 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3272 MLX5_FLOW_LAYER_OUTER_VLAN); 3273 struct rte_flow_item_ecpri mask_lo; 3274 3275 if (!(last_item & outer_l2_vlan) && 3276 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3277 return rte_flow_error_set(error, EINVAL, 3278 RTE_FLOW_ERROR_TYPE_ITEM, item, 3279 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3280 if ((last_item & outer_l2_vlan) && ether_type && 3281 ether_type != RTE_ETHER_TYPE_ECPRI) 3282 return rte_flow_error_set(error, EINVAL, 3283 RTE_FLOW_ERROR_TYPE_ITEM, item, 3284 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3285 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3286 return rte_flow_error_set(error, EINVAL, 3287 RTE_FLOW_ERROR_TYPE_ITEM, item, 3288 "eCPRI with tunnel is not supported right now"); 3289 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3290 return rte_flow_error_set(error, ENOTSUP, 3291 RTE_FLOW_ERROR_TYPE_ITEM, item, 3292 "multiple L3 layers not supported"); 3293 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3294 return rte_flow_error_set(error, EINVAL, 3295 RTE_FLOW_ERROR_TYPE_ITEM, item, 3296 "eCPRI cannot coexist with a TCP layer"); 3297 /* In specification, eCPRI could be over UDP layer. */ 3298 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3299 return rte_flow_error_set(error, EINVAL, 3300 RTE_FLOW_ERROR_TYPE_ITEM, item, 3301 "eCPRI over UDP layer is not yet supported right now"); 3302 /* Mask for type field in common header could be zero. */ 3303 if (!mask) 3304 mask = &rte_flow_item_ecpri_mask; 3305 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3306 /* Input mask is in big-endian format. */ 3307 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3308 return rte_flow_error_set(error, EINVAL, 3309 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3310 "partial mask is not supported for protocol"); 3311 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3312 return rte_flow_error_set(error, EINVAL, 3313 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3314 "message header mask must be after a type mask"); 3315 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3316 acc_mask ? (const uint8_t *)acc_mask 3317 : (const uint8_t *)&nic_mask, 3318 sizeof(struct rte_flow_item_ecpri), 3319 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3320 } 3321 3322 static int 3323 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3324 const struct rte_flow_attr *attr __rte_unused, 3325 const struct rte_flow_item items[] __rte_unused, 3326 const struct rte_flow_action actions[] __rte_unused, 3327 bool external __rte_unused, 3328 int hairpin __rte_unused, 3329 struct rte_flow_error *error) 3330 { 3331 return rte_flow_error_set(error, ENOTSUP, 3332 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3333 } 3334 3335 static struct mlx5_flow * 3336 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3337 const struct rte_flow_attr *attr __rte_unused, 3338 const struct rte_flow_item items[] __rte_unused, 3339 const struct rte_flow_action actions[] __rte_unused, 3340 struct rte_flow_error *error) 3341 { 3342 rte_flow_error_set(error, ENOTSUP, 3343 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3344 return NULL; 3345 } 3346 3347 static int 3348 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3349 struct mlx5_flow *dev_flow __rte_unused, 3350 const struct rte_flow_attr *attr __rte_unused, 3351 const struct rte_flow_item items[] __rte_unused, 3352 const struct rte_flow_action actions[] __rte_unused, 3353 struct rte_flow_error *error) 3354 { 3355 return rte_flow_error_set(error, ENOTSUP, 3356 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3357 } 3358 3359 static int 3360 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3361 struct rte_flow *flow __rte_unused, 3362 struct rte_flow_error *error) 3363 { 3364 return rte_flow_error_set(error, ENOTSUP, 3365 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3366 } 3367 3368 static void 3369 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3370 struct rte_flow *flow __rte_unused) 3371 { 3372 } 3373 3374 static void 3375 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3376 struct rte_flow *flow __rte_unused) 3377 { 3378 } 3379 3380 static int 3381 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3382 struct rte_flow *flow __rte_unused, 3383 const struct rte_flow_action *actions __rte_unused, 3384 void *data __rte_unused, 3385 struct rte_flow_error *error) 3386 { 3387 return rte_flow_error_set(error, ENOTSUP, 3388 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3389 } 3390 3391 static int 3392 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3393 uint32_t domains __rte_unused, 3394 uint32_t flags __rte_unused) 3395 { 3396 return 0; 3397 } 3398 3399 /* Void driver to protect from null pointer reference. */ 3400 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3401 .validate = flow_null_validate, 3402 .prepare = flow_null_prepare, 3403 .translate = flow_null_translate, 3404 .apply = flow_null_apply, 3405 .remove = flow_null_remove, 3406 .destroy = flow_null_destroy, 3407 .query = flow_null_query, 3408 .sync_domain = flow_null_sync_domain, 3409 }; 3410 3411 /** 3412 * Select flow driver type according to flow attributes and device 3413 * configuration. 3414 * 3415 * @param[in] dev 3416 * Pointer to the dev structure. 3417 * @param[in] attr 3418 * Pointer to the flow attributes. 3419 * 3420 * @return 3421 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3422 */ 3423 static enum mlx5_flow_drv_type 3424 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3425 { 3426 struct mlx5_priv *priv = dev->data->dev_private; 3427 /* The OS can determine first a specific flow type (DV, VERBS) */ 3428 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3429 3430 if (type != MLX5_FLOW_TYPE_MAX) 3431 return type; 3432 /* If no OS specific type - continue with DV/VERBS selection */ 3433 if (attr->transfer && priv->config.dv_esw_en) 3434 type = MLX5_FLOW_TYPE_DV; 3435 if (!attr->transfer) 3436 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3437 MLX5_FLOW_TYPE_VERBS; 3438 return type; 3439 } 3440 3441 #define flow_get_drv_ops(type) flow_drv_ops[type] 3442 3443 /** 3444 * Flow driver validation API. This abstracts calling driver specific functions. 3445 * The type of flow driver is determined according to flow attributes. 3446 * 3447 * @param[in] dev 3448 * Pointer to the dev structure. 3449 * @param[in] attr 3450 * Pointer to the flow attributes. 3451 * @param[in] items 3452 * Pointer to the list of items. 3453 * @param[in] actions 3454 * Pointer to the list of actions. 3455 * @param[in] external 3456 * This flow rule is created by request external to PMD. 3457 * @param[in] hairpin 3458 * Number of hairpin TX actions, 0 means classic flow. 3459 * @param[out] error 3460 * Pointer to the error structure. 3461 * 3462 * @return 3463 * 0 on success, a negative errno value otherwise and rte_errno is set. 3464 */ 3465 static inline int 3466 flow_drv_validate(struct rte_eth_dev *dev, 3467 const struct rte_flow_attr *attr, 3468 const struct rte_flow_item items[], 3469 const struct rte_flow_action actions[], 3470 bool external, int hairpin, struct rte_flow_error *error) 3471 { 3472 const struct mlx5_flow_driver_ops *fops; 3473 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3474 3475 fops = flow_get_drv_ops(type); 3476 return fops->validate(dev, attr, items, actions, external, 3477 hairpin, error); 3478 } 3479 3480 /** 3481 * Flow driver preparation API. This abstracts calling driver specific 3482 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3483 * calculates the size of memory required for device flow, allocates the memory, 3484 * initializes the device flow and returns the pointer. 3485 * 3486 * @note 3487 * This function initializes device flow structure such as dv or verbs in 3488 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3489 * rest. For example, adding returning device flow to flow->dev_flow list and 3490 * setting backward reference to the flow should be done out of this function. 3491 * layers field is not filled either. 3492 * 3493 * @param[in] dev 3494 * Pointer to the dev structure. 3495 * @param[in] attr 3496 * Pointer to the flow attributes. 3497 * @param[in] items 3498 * Pointer to the list of items. 3499 * @param[in] actions 3500 * Pointer to the list of actions. 3501 * @param[in] flow_idx 3502 * This memory pool index to the flow. 3503 * @param[out] error 3504 * Pointer to the error structure. 3505 * 3506 * @return 3507 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3508 */ 3509 static inline struct mlx5_flow * 3510 flow_drv_prepare(struct rte_eth_dev *dev, 3511 const struct rte_flow *flow, 3512 const struct rte_flow_attr *attr, 3513 const struct rte_flow_item items[], 3514 const struct rte_flow_action actions[], 3515 uint32_t flow_idx, 3516 struct rte_flow_error *error) 3517 { 3518 const struct mlx5_flow_driver_ops *fops; 3519 enum mlx5_flow_drv_type type = flow->drv_type; 3520 struct mlx5_flow *mlx5_flow = NULL; 3521 3522 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3523 fops = flow_get_drv_ops(type); 3524 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3525 if (mlx5_flow) 3526 mlx5_flow->flow_idx = flow_idx; 3527 return mlx5_flow; 3528 } 3529 3530 /** 3531 * Flow driver translation API. This abstracts calling driver specific 3532 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3533 * translates a generic flow into a driver flow. flow_drv_prepare() must 3534 * precede. 3535 * 3536 * @note 3537 * dev_flow->layers could be filled as a result of parsing during translation 3538 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3539 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3540 * flow->actions could be overwritten even though all the expanded dev_flows 3541 * have the same actions. 3542 * 3543 * @param[in] dev 3544 * Pointer to the rte dev structure. 3545 * @param[in, out] dev_flow 3546 * Pointer to the mlx5 flow. 3547 * @param[in] attr 3548 * Pointer to the flow attributes. 3549 * @param[in] items 3550 * Pointer to the list of items. 3551 * @param[in] actions 3552 * Pointer to the list of actions. 3553 * @param[out] error 3554 * Pointer to the error structure. 3555 * 3556 * @return 3557 * 0 on success, a negative errno value otherwise and rte_errno is set. 3558 */ 3559 static inline int 3560 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3561 const struct rte_flow_attr *attr, 3562 const struct rte_flow_item items[], 3563 const struct rte_flow_action actions[], 3564 struct rte_flow_error *error) 3565 { 3566 const struct mlx5_flow_driver_ops *fops; 3567 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3568 3569 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3570 fops = flow_get_drv_ops(type); 3571 return fops->translate(dev, dev_flow, attr, items, actions, error); 3572 } 3573 3574 /** 3575 * Flow driver apply API. This abstracts calling driver specific functions. 3576 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3577 * translated driver flows on to device. flow_drv_translate() must precede. 3578 * 3579 * @param[in] dev 3580 * Pointer to Ethernet device structure. 3581 * @param[in, out] flow 3582 * Pointer to flow structure. 3583 * @param[out] error 3584 * Pointer to error structure. 3585 * 3586 * @return 3587 * 0 on success, a negative errno value otherwise and rte_errno is set. 3588 */ 3589 static inline int 3590 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3591 struct rte_flow_error *error) 3592 { 3593 const struct mlx5_flow_driver_ops *fops; 3594 enum mlx5_flow_drv_type type = flow->drv_type; 3595 3596 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3597 fops = flow_get_drv_ops(type); 3598 return fops->apply(dev, flow, error); 3599 } 3600 3601 /** 3602 * Flow driver destroy API. This abstracts calling driver specific functions. 3603 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3604 * on device and releases resources of the flow. 3605 * 3606 * @param[in] dev 3607 * Pointer to Ethernet device. 3608 * @param[in, out] flow 3609 * Pointer to flow structure. 3610 */ 3611 static inline void 3612 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3613 { 3614 const struct mlx5_flow_driver_ops *fops; 3615 enum mlx5_flow_drv_type type = flow->drv_type; 3616 3617 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3618 fops = flow_get_drv_ops(type); 3619 fops->destroy(dev, flow); 3620 } 3621 3622 /** 3623 * Flow driver find RSS policy tbl API. This abstracts calling driver 3624 * specific functions. Parent flow (rte_flow) should have driver 3625 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3626 * 3627 * @param[in] dev 3628 * Pointer to Ethernet device. 3629 * @param[in, out] flow 3630 * Pointer to flow structure. 3631 * @param[in] policy 3632 * Pointer to meter policy table. 3633 * @param[in] rss_desc 3634 * Pointer to rss_desc 3635 */ 3636 static struct mlx5_flow_meter_sub_policy * 3637 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3638 struct rte_flow *flow, 3639 struct mlx5_flow_meter_policy *policy, 3640 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3641 { 3642 const struct mlx5_flow_driver_ops *fops; 3643 enum mlx5_flow_drv_type type = flow->drv_type; 3644 3645 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3646 fops = flow_get_drv_ops(type); 3647 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3648 } 3649 3650 /** 3651 * Flow driver color tag rule API. This abstracts calling driver 3652 * specific functions. Parent flow (rte_flow) should have driver 3653 * type (drv_type). It will create the color tag rules in hierarchy meter. 3654 * 3655 * @param[in] dev 3656 * Pointer to Ethernet device. 3657 * @param[in, out] flow 3658 * Pointer to flow structure. 3659 * @param[in] fm 3660 * Pointer to flow meter structure. 3661 * @param[in] src_port 3662 * The src port this extra rule should use. 3663 * @param[in] item 3664 * The src port id match item. 3665 * @param[out] error 3666 * Pointer to error structure. 3667 */ 3668 static int 3669 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3670 struct rte_flow *flow, 3671 struct mlx5_flow_meter_info *fm, 3672 int32_t src_port, 3673 const struct rte_flow_item *item, 3674 struct rte_flow_error *error) 3675 { 3676 const struct mlx5_flow_driver_ops *fops; 3677 enum mlx5_flow_drv_type type = flow->drv_type; 3678 3679 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3680 fops = flow_get_drv_ops(type); 3681 return fops->meter_hierarchy_rule_create(dev, fm, 3682 src_port, item, error); 3683 } 3684 3685 /** 3686 * Get RSS action from the action list. 3687 * 3688 * @param[in] dev 3689 * Pointer to Ethernet device. 3690 * @param[in] actions 3691 * Pointer to the list of actions. 3692 * @param[in] flow 3693 * Parent flow structure pointer. 3694 * 3695 * @return 3696 * Pointer to the RSS action if exist, else return NULL. 3697 */ 3698 static const struct rte_flow_action_rss* 3699 flow_get_rss_action(struct rte_eth_dev *dev, 3700 const struct rte_flow_action actions[]) 3701 { 3702 struct mlx5_priv *priv = dev->data->dev_private; 3703 const struct rte_flow_action_rss *rss = NULL; 3704 struct mlx5_meter_policy_action_container *acg; 3705 struct mlx5_meter_policy_action_container *acy; 3706 3707 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3708 switch (actions->type) { 3709 case RTE_FLOW_ACTION_TYPE_RSS: 3710 rss = actions->conf; 3711 break; 3712 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3713 { 3714 const struct rte_flow_action_sample *sample = 3715 actions->conf; 3716 const struct rte_flow_action *act = sample->actions; 3717 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3718 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3719 rss = act->conf; 3720 break; 3721 } 3722 case RTE_FLOW_ACTION_TYPE_METER: 3723 { 3724 uint32_t mtr_idx; 3725 struct mlx5_flow_meter_info *fm; 3726 struct mlx5_flow_meter_policy *policy; 3727 const struct rte_flow_action_meter *mtr = actions->conf; 3728 3729 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3730 if (fm && !fm->def_policy) { 3731 policy = mlx5_flow_meter_policy_find(dev, 3732 fm->policy_id, NULL); 3733 MLX5_ASSERT(policy); 3734 if (policy->is_hierarchy) { 3735 policy = 3736 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3737 policy); 3738 if (!policy) 3739 return NULL; 3740 } 3741 if (policy->is_rss) { 3742 acg = 3743 &policy->act_cnt[RTE_COLOR_GREEN]; 3744 acy = 3745 &policy->act_cnt[RTE_COLOR_YELLOW]; 3746 if (acg->fate_action == 3747 MLX5_FLOW_FATE_SHARED_RSS) 3748 rss = acg->rss->conf; 3749 else if (acy->fate_action == 3750 MLX5_FLOW_FATE_SHARED_RSS) 3751 rss = acy->rss->conf; 3752 } 3753 } 3754 break; 3755 } 3756 default: 3757 break; 3758 } 3759 } 3760 return rss; 3761 } 3762 3763 /** 3764 * Get ASO age action by index. 3765 * 3766 * @param[in] dev 3767 * Pointer to the Ethernet device structure. 3768 * @param[in] age_idx 3769 * Index to the ASO age action. 3770 * 3771 * @return 3772 * The specified ASO age action. 3773 */ 3774 struct mlx5_aso_age_action* 3775 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3776 { 3777 uint16_t pool_idx = age_idx & UINT16_MAX; 3778 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3779 struct mlx5_priv *priv = dev->data->dev_private; 3780 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3781 struct mlx5_aso_age_pool *pool; 3782 3783 rte_rwlock_read_lock(&mng->resize_rwl); 3784 pool = mng->pools[pool_idx]; 3785 rte_rwlock_read_unlock(&mng->resize_rwl); 3786 return &pool->actions[offset - 1]; 3787 } 3788 3789 /* maps indirect action to translated direct in some actions array */ 3790 struct mlx5_translated_action_handle { 3791 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3792 int index; /**< Index in related array of rte_flow_action. */ 3793 }; 3794 3795 /** 3796 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3797 * direct action if translation possible. 3798 * This functionality used to run same execution path for both direct and 3799 * indirect actions on flow create. All necessary preparations for indirect 3800 * action handling should be performed on *handle* actions list returned 3801 * from this call. 3802 * 3803 * @param[in] dev 3804 * Pointer to Ethernet device. 3805 * @param[in] actions 3806 * List of actions to translate. 3807 * @param[out] handle 3808 * List to store translated indirect action object handles. 3809 * @param[in, out] indir_n 3810 * Size of *handle* array. On return should be updated with number of 3811 * indirect actions retrieved from the *actions* list. 3812 * @param[out] translated_actions 3813 * List of actions where all indirect actions were translated to direct 3814 * if possible. NULL if no translation took place. 3815 * @param[out] error 3816 * Pointer to the error structure. 3817 * 3818 * @return 3819 * 0 on success, a negative errno value otherwise and rte_errno is set. 3820 */ 3821 static int 3822 flow_action_handles_translate(struct rte_eth_dev *dev, 3823 const struct rte_flow_action actions[], 3824 struct mlx5_translated_action_handle *handle, 3825 int *indir_n, 3826 struct rte_flow_action **translated_actions, 3827 struct rte_flow_error *error) 3828 { 3829 struct mlx5_priv *priv = dev->data->dev_private; 3830 struct rte_flow_action *translated = NULL; 3831 size_t actions_size; 3832 int n; 3833 int copied_n = 0; 3834 struct mlx5_translated_action_handle *handle_end = NULL; 3835 3836 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3837 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3838 continue; 3839 if (copied_n == *indir_n) { 3840 return rte_flow_error_set 3841 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3842 NULL, "too many shared actions"); 3843 } 3844 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3845 sizeof(actions[n].conf)); 3846 handle[copied_n].index = n; 3847 copied_n++; 3848 } 3849 n++; 3850 *indir_n = copied_n; 3851 if (!copied_n) 3852 return 0; 3853 actions_size = sizeof(struct rte_flow_action) * n; 3854 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3855 if (!translated) { 3856 rte_errno = ENOMEM; 3857 return -ENOMEM; 3858 } 3859 memcpy(translated, actions, actions_size); 3860 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3861 struct mlx5_shared_action_rss *shared_rss; 3862 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3863 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3864 uint32_t idx = act_idx & 3865 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3866 3867 switch (type) { 3868 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3869 shared_rss = mlx5_ipool_get 3870 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3871 translated[handle->index].type = 3872 RTE_FLOW_ACTION_TYPE_RSS; 3873 translated[handle->index].conf = 3874 &shared_rss->origin; 3875 break; 3876 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3877 translated[handle->index].type = 3878 (enum rte_flow_action_type) 3879 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3880 translated[handle->index].conf = (void *)(uintptr_t)idx; 3881 break; 3882 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3883 if (priv->sh->flow_hit_aso_en) { 3884 translated[handle->index].type = 3885 (enum rte_flow_action_type) 3886 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3887 translated[handle->index].conf = 3888 (void *)(uintptr_t)idx; 3889 break; 3890 } 3891 /* Fall-through */ 3892 case MLX5_INDIRECT_ACTION_TYPE_CT: 3893 if (priv->sh->ct_aso_en) { 3894 translated[handle->index].type = 3895 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3896 translated[handle->index].conf = 3897 (void *)(uintptr_t)idx; 3898 break; 3899 } 3900 /* Fall-through */ 3901 default: 3902 mlx5_free(translated); 3903 return rte_flow_error_set 3904 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3905 NULL, "invalid indirect action type"); 3906 } 3907 } 3908 *translated_actions = translated; 3909 return 0; 3910 } 3911 3912 /** 3913 * Get Shared RSS action from the action list. 3914 * 3915 * @param[in] dev 3916 * Pointer to Ethernet device. 3917 * @param[in] shared 3918 * Pointer to the list of actions. 3919 * @param[in] shared_n 3920 * Actions list length. 3921 * 3922 * @return 3923 * The MLX5 RSS action ID if exists, otherwise return 0. 3924 */ 3925 static uint32_t 3926 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3927 struct mlx5_translated_action_handle *handle, 3928 int shared_n) 3929 { 3930 struct mlx5_translated_action_handle *handle_end; 3931 struct mlx5_priv *priv = dev->data->dev_private; 3932 struct mlx5_shared_action_rss *shared_rss; 3933 3934 3935 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3936 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3937 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3938 uint32_t idx = act_idx & 3939 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3940 switch (type) { 3941 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3942 shared_rss = mlx5_ipool_get 3943 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3944 idx); 3945 __atomic_add_fetch(&shared_rss->refcnt, 1, 3946 __ATOMIC_RELAXED); 3947 return idx; 3948 default: 3949 break; 3950 } 3951 } 3952 return 0; 3953 } 3954 3955 static unsigned int 3956 find_graph_root(uint32_t rss_level) 3957 { 3958 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3959 MLX5_EXPANSION_ROOT_OUTER; 3960 } 3961 3962 /** 3963 * Get layer flags from the prefix flow. 3964 * 3965 * Some flows may be split to several subflows, the prefix subflow gets the 3966 * match items and the suffix sub flow gets the actions. 3967 * Some actions need the user defined match item flags to get the detail for 3968 * the action. 3969 * This function helps the suffix flow to get the item layer flags from prefix 3970 * subflow. 3971 * 3972 * @param[in] dev_flow 3973 * Pointer the created prefix subflow. 3974 * 3975 * @return 3976 * The layers get from prefix subflow. 3977 */ 3978 static inline uint64_t 3979 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3980 { 3981 uint64_t layers = 0; 3982 3983 /* 3984 * Layers bits could be localization, but usually the compiler will 3985 * help to do the optimization work for source code. 3986 * If no decap actions, use the layers directly. 3987 */ 3988 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3989 return dev_flow->handle->layers; 3990 /* Convert L3 layers with decap action. */ 3991 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3992 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3993 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3994 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3995 /* Convert L4 layers with decap action. */ 3996 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3997 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3998 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3999 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4000 return layers; 4001 } 4002 4003 /** 4004 * Get metadata split action information. 4005 * 4006 * @param[in] actions 4007 * Pointer to the list of actions. 4008 * @param[out] qrss 4009 * Pointer to the return pointer. 4010 * @param[out] qrss_type 4011 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4012 * if no QUEUE/RSS is found. 4013 * @param[out] encap_idx 4014 * Pointer to the index of the encap action if exists, otherwise the last 4015 * action index. 4016 * 4017 * @return 4018 * Total number of actions. 4019 */ 4020 static int 4021 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4022 const struct rte_flow_action **qrss, 4023 int *encap_idx) 4024 { 4025 const struct rte_flow_action_raw_encap *raw_encap; 4026 int actions_n = 0; 4027 int raw_decap_idx = -1; 4028 4029 *encap_idx = -1; 4030 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4031 switch (actions->type) { 4032 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4033 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4034 *encap_idx = actions_n; 4035 break; 4036 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4037 raw_decap_idx = actions_n; 4038 break; 4039 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4040 raw_encap = actions->conf; 4041 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4042 *encap_idx = raw_decap_idx != -1 ? 4043 raw_decap_idx : actions_n; 4044 break; 4045 case RTE_FLOW_ACTION_TYPE_QUEUE: 4046 case RTE_FLOW_ACTION_TYPE_RSS: 4047 *qrss = actions; 4048 break; 4049 default: 4050 break; 4051 } 4052 actions_n++; 4053 } 4054 if (*encap_idx == -1) 4055 *encap_idx = actions_n; 4056 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4057 return actions_n + 1; 4058 } 4059 4060 /** 4061 * Check if the action will change packet. 4062 * 4063 * @param dev 4064 * Pointer to Ethernet device. 4065 * @param[in] type 4066 * action type. 4067 * 4068 * @return 4069 * true if action will change packet, false otherwise. 4070 */ 4071 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4072 enum rte_flow_action_type type) 4073 { 4074 struct mlx5_priv *priv = dev->data->dev_private; 4075 4076 switch (type) { 4077 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4078 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4079 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4080 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4081 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4082 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4083 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4084 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4085 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4086 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4087 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4088 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4089 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4090 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4091 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4092 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4093 case RTE_FLOW_ACTION_TYPE_SET_META: 4094 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4095 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4096 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4097 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4098 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4099 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4100 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4101 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4102 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4103 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4104 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4105 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4106 return true; 4107 case RTE_FLOW_ACTION_TYPE_FLAG: 4108 case RTE_FLOW_ACTION_TYPE_MARK: 4109 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4110 return true; 4111 else 4112 return false; 4113 default: 4114 return false; 4115 } 4116 } 4117 4118 /** 4119 * Check meter action from the action list. 4120 * 4121 * @param dev 4122 * Pointer to Ethernet device. 4123 * @param[in] actions 4124 * Pointer to the list of actions. 4125 * @param[out] has_mtr 4126 * Pointer to the meter exist flag. 4127 * @param[out] has_modify 4128 * Pointer to the flag showing there's packet change action. 4129 * @param[out] meter_id 4130 * Pointer to the meter id. 4131 * 4132 * @return 4133 * Total number of actions. 4134 */ 4135 static int 4136 flow_check_meter_action(struct rte_eth_dev *dev, 4137 const struct rte_flow_action actions[], 4138 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4139 { 4140 const struct rte_flow_action_meter *mtr = NULL; 4141 int actions_n = 0; 4142 4143 MLX5_ASSERT(has_mtr); 4144 *has_mtr = false; 4145 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4146 switch (actions->type) { 4147 case RTE_FLOW_ACTION_TYPE_METER: 4148 mtr = actions->conf; 4149 *meter_id = mtr->mtr_id; 4150 *has_mtr = true; 4151 break; 4152 default: 4153 break; 4154 } 4155 if (!*has_mtr) 4156 *has_modify |= flow_check_modify_action_type(dev, 4157 actions->type); 4158 actions_n++; 4159 } 4160 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4161 return actions_n + 1; 4162 } 4163 4164 /** 4165 * Check if the flow should be split due to hairpin. 4166 * The reason for the split is that in current HW we can't 4167 * support encap and push-vlan on Rx, so if a flow contains 4168 * these actions we move it to Tx. 4169 * 4170 * @param dev 4171 * Pointer to Ethernet device. 4172 * @param[in] attr 4173 * Flow rule attributes. 4174 * @param[in] actions 4175 * Associated actions (list terminated by the END action). 4176 * 4177 * @return 4178 * > 0 the number of actions and the flow should be split, 4179 * 0 when no split required. 4180 */ 4181 static int 4182 flow_check_hairpin_split(struct rte_eth_dev *dev, 4183 const struct rte_flow_attr *attr, 4184 const struct rte_flow_action actions[]) 4185 { 4186 int queue_action = 0; 4187 int action_n = 0; 4188 int split = 0; 4189 const struct rte_flow_action_queue *queue; 4190 const struct rte_flow_action_rss *rss; 4191 const struct rte_flow_action_raw_encap *raw_encap; 4192 const struct rte_eth_hairpin_conf *conf; 4193 4194 if (!attr->ingress) 4195 return 0; 4196 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4197 switch (actions->type) { 4198 case RTE_FLOW_ACTION_TYPE_QUEUE: 4199 queue = actions->conf; 4200 if (queue == NULL) 4201 return 0; 4202 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4203 if (conf == NULL || conf->tx_explicit != 0) 4204 return 0; 4205 queue_action = 1; 4206 action_n++; 4207 break; 4208 case RTE_FLOW_ACTION_TYPE_RSS: 4209 rss = actions->conf; 4210 if (rss == NULL || rss->queue_num == 0) 4211 return 0; 4212 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4213 if (conf == NULL || conf->tx_explicit != 0) 4214 return 0; 4215 queue_action = 1; 4216 action_n++; 4217 break; 4218 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4219 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4220 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4221 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4222 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4223 split++; 4224 action_n++; 4225 break; 4226 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4227 raw_encap = actions->conf; 4228 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4229 split++; 4230 action_n++; 4231 break; 4232 default: 4233 action_n++; 4234 break; 4235 } 4236 } 4237 if (split && queue_action) 4238 return action_n; 4239 return 0; 4240 } 4241 4242 /* Declare flow create/destroy prototype in advance. */ 4243 static uint32_t 4244 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4245 const struct rte_flow_attr *attr, 4246 const struct rte_flow_item items[], 4247 const struct rte_flow_action actions[], 4248 bool external, struct rte_flow_error *error); 4249 4250 static void 4251 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4252 uint32_t flow_idx); 4253 4254 int 4255 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4256 struct mlx5_list_entry *entry, void *cb_ctx) 4257 { 4258 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4259 struct mlx5_flow_mreg_copy_resource *mcp_res = 4260 container_of(entry, typeof(*mcp_res), hlist_ent); 4261 4262 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4263 } 4264 4265 struct mlx5_list_entry * 4266 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4267 { 4268 struct rte_eth_dev *dev = tool_ctx; 4269 struct mlx5_priv *priv = dev->data->dev_private; 4270 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4271 struct mlx5_flow_mreg_copy_resource *mcp_res; 4272 struct rte_flow_error *error = ctx->error; 4273 uint32_t idx = 0; 4274 int ret; 4275 uint32_t mark_id = *(uint32_t *)(ctx->data); 4276 struct rte_flow_attr attr = { 4277 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4278 .ingress = 1, 4279 }; 4280 struct mlx5_rte_flow_item_tag tag_spec = { 4281 .data = mark_id, 4282 }; 4283 struct rte_flow_item items[] = { 4284 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4285 }; 4286 struct rte_flow_action_mark ftag = { 4287 .id = mark_id, 4288 }; 4289 struct mlx5_flow_action_copy_mreg cp_mreg = { 4290 .dst = REG_B, 4291 .src = REG_NON, 4292 }; 4293 struct rte_flow_action_jump jump = { 4294 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4295 }; 4296 struct rte_flow_action actions[] = { 4297 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4298 }; 4299 4300 /* Fill the register fields in the flow. */ 4301 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4302 if (ret < 0) 4303 return NULL; 4304 tag_spec.id = ret; 4305 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4306 if (ret < 0) 4307 return NULL; 4308 cp_mreg.src = ret; 4309 /* Provide the full width of FLAG specific value. */ 4310 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4311 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4312 /* Build a new flow. */ 4313 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4314 items[0] = (struct rte_flow_item){ 4315 .type = (enum rte_flow_item_type) 4316 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4317 .spec = &tag_spec, 4318 }; 4319 items[1] = (struct rte_flow_item){ 4320 .type = RTE_FLOW_ITEM_TYPE_END, 4321 }; 4322 actions[0] = (struct rte_flow_action){ 4323 .type = (enum rte_flow_action_type) 4324 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4325 .conf = &ftag, 4326 }; 4327 actions[1] = (struct rte_flow_action){ 4328 .type = (enum rte_flow_action_type) 4329 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4330 .conf = &cp_mreg, 4331 }; 4332 actions[2] = (struct rte_flow_action){ 4333 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4334 .conf = &jump, 4335 }; 4336 actions[3] = (struct rte_flow_action){ 4337 .type = RTE_FLOW_ACTION_TYPE_END, 4338 }; 4339 } else { 4340 /* Default rule, wildcard match. */ 4341 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4342 items[0] = (struct rte_flow_item){ 4343 .type = RTE_FLOW_ITEM_TYPE_END, 4344 }; 4345 actions[0] = (struct rte_flow_action){ 4346 .type = (enum rte_flow_action_type) 4347 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4348 .conf = &cp_mreg, 4349 }; 4350 actions[1] = (struct rte_flow_action){ 4351 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4352 .conf = &jump, 4353 }; 4354 actions[2] = (struct rte_flow_action){ 4355 .type = RTE_FLOW_ACTION_TYPE_END, 4356 }; 4357 } 4358 /* Build a new entry. */ 4359 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4360 if (!mcp_res) { 4361 rte_errno = ENOMEM; 4362 return NULL; 4363 } 4364 mcp_res->idx = idx; 4365 mcp_res->mark_id = mark_id; 4366 /* 4367 * The copy Flows are not included in any list. There 4368 * ones are referenced from other Flows and can not 4369 * be applied, removed, deleted in arbitrary order 4370 * by list traversing. 4371 */ 4372 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4373 &attr, items, actions, false, error); 4374 if (!mcp_res->rix_flow) { 4375 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4376 return NULL; 4377 } 4378 return &mcp_res->hlist_ent; 4379 } 4380 4381 struct mlx5_list_entry * 4382 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4383 void *cb_ctx __rte_unused) 4384 { 4385 struct rte_eth_dev *dev = tool_ctx; 4386 struct mlx5_priv *priv = dev->data->dev_private; 4387 struct mlx5_flow_mreg_copy_resource *mcp_res; 4388 uint32_t idx = 0; 4389 4390 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4391 if (!mcp_res) { 4392 rte_errno = ENOMEM; 4393 return NULL; 4394 } 4395 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4396 mcp_res->idx = idx; 4397 return &mcp_res->hlist_ent; 4398 } 4399 4400 void 4401 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4402 { 4403 struct mlx5_flow_mreg_copy_resource *mcp_res = 4404 container_of(entry, typeof(*mcp_res), hlist_ent); 4405 struct rte_eth_dev *dev = tool_ctx; 4406 struct mlx5_priv *priv = dev->data->dev_private; 4407 4408 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4409 } 4410 4411 /** 4412 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4413 * 4414 * As mark_id is unique, if there's already a registered flow for the mark_id, 4415 * return by increasing the reference counter of the resource. Otherwise, create 4416 * the resource (mcp_res) and flow. 4417 * 4418 * Flow looks like, 4419 * - If ingress port is ANY and reg_c[1] is mark_id, 4420 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4421 * 4422 * For default flow (zero mark_id), flow is like, 4423 * - If ingress port is ANY, 4424 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4425 * 4426 * @param dev 4427 * Pointer to Ethernet device. 4428 * @param mark_id 4429 * ID of MARK action, zero means default flow for META. 4430 * @param[out] error 4431 * Perform verbose error reporting if not NULL. 4432 * 4433 * @return 4434 * Associated resource on success, NULL otherwise and rte_errno is set. 4435 */ 4436 static struct mlx5_flow_mreg_copy_resource * 4437 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4438 struct rte_flow_error *error) 4439 { 4440 struct mlx5_priv *priv = dev->data->dev_private; 4441 struct mlx5_list_entry *entry; 4442 struct mlx5_flow_cb_ctx ctx = { 4443 .dev = dev, 4444 .error = error, 4445 .data = &mark_id, 4446 }; 4447 4448 /* Check if already registered. */ 4449 MLX5_ASSERT(priv->mreg_cp_tbl); 4450 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4451 if (!entry) 4452 return NULL; 4453 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4454 hlist_ent); 4455 } 4456 4457 void 4458 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4459 { 4460 struct mlx5_flow_mreg_copy_resource *mcp_res = 4461 container_of(entry, typeof(*mcp_res), hlist_ent); 4462 struct rte_eth_dev *dev = tool_ctx; 4463 struct mlx5_priv *priv = dev->data->dev_private; 4464 4465 MLX5_ASSERT(mcp_res->rix_flow); 4466 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4467 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4468 } 4469 4470 /** 4471 * Release flow in RX_CP_TBL. 4472 * 4473 * @param dev 4474 * Pointer to Ethernet device. 4475 * @flow 4476 * Parent flow for wich copying is provided. 4477 */ 4478 static void 4479 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4480 struct rte_flow *flow) 4481 { 4482 struct mlx5_flow_mreg_copy_resource *mcp_res; 4483 struct mlx5_priv *priv = dev->data->dev_private; 4484 4485 if (!flow->rix_mreg_copy) 4486 return; 4487 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4488 flow->rix_mreg_copy); 4489 if (!mcp_res || !priv->mreg_cp_tbl) 4490 return; 4491 MLX5_ASSERT(mcp_res->rix_flow); 4492 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4493 flow->rix_mreg_copy = 0; 4494 } 4495 4496 /** 4497 * Remove the default copy action from RX_CP_TBL. 4498 * 4499 * This functions is called in the mlx5_dev_start(). No thread safe 4500 * is guaranteed. 4501 * 4502 * @param dev 4503 * Pointer to Ethernet device. 4504 */ 4505 static void 4506 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4507 { 4508 struct mlx5_list_entry *entry; 4509 struct mlx5_priv *priv = dev->data->dev_private; 4510 struct mlx5_flow_cb_ctx ctx; 4511 uint32_t mark_id; 4512 4513 /* Check if default flow is registered. */ 4514 if (!priv->mreg_cp_tbl) 4515 return; 4516 mark_id = MLX5_DEFAULT_COPY_ID; 4517 ctx.data = &mark_id; 4518 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4519 if (!entry) 4520 return; 4521 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4522 } 4523 4524 /** 4525 * Add the default copy action in in RX_CP_TBL. 4526 * 4527 * This functions is called in the mlx5_dev_start(). No thread safe 4528 * is guaranteed. 4529 * 4530 * @param dev 4531 * Pointer to Ethernet device. 4532 * @param[out] error 4533 * Perform verbose error reporting if not NULL. 4534 * 4535 * @return 4536 * 0 for success, negative value otherwise and rte_errno is set. 4537 */ 4538 static int 4539 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4540 struct rte_flow_error *error) 4541 { 4542 struct mlx5_priv *priv = dev->data->dev_private; 4543 struct mlx5_flow_mreg_copy_resource *mcp_res; 4544 struct mlx5_flow_cb_ctx ctx; 4545 uint32_t mark_id; 4546 4547 /* Check whether extensive metadata feature is engaged. */ 4548 if (!priv->config.dv_flow_en || 4549 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4550 !mlx5_flow_ext_mreg_supported(dev) || 4551 !priv->sh->dv_regc0_mask) 4552 return 0; 4553 /* 4554 * Add default mreg copy flow may be called multiple time, but 4555 * only be called once in stop. Avoid register it twice. 4556 */ 4557 mark_id = MLX5_DEFAULT_COPY_ID; 4558 ctx.data = &mark_id; 4559 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4560 return 0; 4561 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4562 if (!mcp_res) 4563 return -rte_errno; 4564 return 0; 4565 } 4566 4567 /** 4568 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4569 * 4570 * All the flow having Q/RSS action should be split by 4571 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4572 * performs the following, 4573 * - CQE->flow_tag := reg_c[1] (MARK) 4574 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4575 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4576 * but there should be a flow per each MARK ID set by MARK action. 4577 * 4578 * For the aforementioned reason, if there's a MARK action in flow's action 4579 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4580 * the MARK ID to CQE's flow_tag like, 4581 * - If reg_c[1] is mark_id, 4582 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4583 * 4584 * For SET_META action which stores value in reg_c[0], as the destination is 4585 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4586 * MARK ID means the default flow. The default flow looks like, 4587 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4588 * 4589 * @param dev 4590 * Pointer to Ethernet device. 4591 * @param flow 4592 * Pointer to flow structure. 4593 * @param[in] actions 4594 * Pointer to the list of actions. 4595 * @param[out] error 4596 * Perform verbose error reporting if not NULL. 4597 * 4598 * @return 4599 * 0 on success, negative value otherwise and rte_errno is set. 4600 */ 4601 static int 4602 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4603 struct rte_flow *flow, 4604 const struct rte_flow_action *actions, 4605 struct rte_flow_error *error) 4606 { 4607 struct mlx5_priv *priv = dev->data->dev_private; 4608 struct mlx5_dev_config *config = &priv->config; 4609 struct mlx5_flow_mreg_copy_resource *mcp_res; 4610 const struct rte_flow_action_mark *mark; 4611 4612 /* Check whether extensive metadata feature is engaged. */ 4613 if (!config->dv_flow_en || 4614 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4615 !mlx5_flow_ext_mreg_supported(dev) || 4616 !priv->sh->dv_regc0_mask) 4617 return 0; 4618 /* Find MARK action. */ 4619 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4620 switch (actions->type) { 4621 case RTE_FLOW_ACTION_TYPE_FLAG: 4622 mcp_res = flow_mreg_add_copy_action 4623 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4624 if (!mcp_res) 4625 return -rte_errno; 4626 flow->rix_mreg_copy = mcp_res->idx; 4627 return 0; 4628 case RTE_FLOW_ACTION_TYPE_MARK: 4629 mark = (const struct rte_flow_action_mark *) 4630 actions->conf; 4631 mcp_res = 4632 flow_mreg_add_copy_action(dev, mark->id, error); 4633 if (!mcp_res) 4634 return -rte_errno; 4635 flow->rix_mreg_copy = mcp_res->idx; 4636 return 0; 4637 default: 4638 break; 4639 } 4640 } 4641 return 0; 4642 } 4643 4644 #define MLX5_MAX_SPLIT_ACTIONS 24 4645 #define MLX5_MAX_SPLIT_ITEMS 24 4646 4647 /** 4648 * Split the hairpin flow. 4649 * Since HW can't support encap and push-vlan on Rx, we move these 4650 * actions to Tx. 4651 * If the count action is after the encap then we also 4652 * move the count action. in this case the count will also measure 4653 * the outer bytes. 4654 * 4655 * @param dev 4656 * Pointer to Ethernet device. 4657 * @param[in] actions 4658 * Associated actions (list terminated by the END action). 4659 * @param[out] actions_rx 4660 * Rx flow actions. 4661 * @param[out] actions_tx 4662 * Tx flow actions.. 4663 * @param[out] pattern_tx 4664 * The pattern items for the Tx flow. 4665 * @param[out] flow_id 4666 * The flow ID connected to this flow. 4667 * 4668 * @return 4669 * 0 on success. 4670 */ 4671 static int 4672 flow_hairpin_split(struct rte_eth_dev *dev, 4673 const struct rte_flow_action actions[], 4674 struct rte_flow_action actions_rx[], 4675 struct rte_flow_action actions_tx[], 4676 struct rte_flow_item pattern_tx[], 4677 uint32_t flow_id) 4678 { 4679 const struct rte_flow_action_raw_encap *raw_encap; 4680 const struct rte_flow_action_raw_decap *raw_decap; 4681 struct mlx5_rte_flow_action_set_tag *set_tag; 4682 struct rte_flow_action *tag_action; 4683 struct mlx5_rte_flow_item_tag *tag_item; 4684 struct rte_flow_item *item; 4685 char *addr; 4686 int encap = 0; 4687 4688 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4689 switch (actions->type) { 4690 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4691 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4692 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4693 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4694 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4695 rte_memcpy(actions_tx, actions, 4696 sizeof(struct rte_flow_action)); 4697 actions_tx++; 4698 break; 4699 case RTE_FLOW_ACTION_TYPE_COUNT: 4700 if (encap) { 4701 rte_memcpy(actions_tx, actions, 4702 sizeof(struct rte_flow_action)); 4703 actions_tx++; 4704 } else { 4705 rte_memcpy(actions_rx, actions, 4706 sizeof(struct rte_flow_action)); 4707 actions_rx++; 4708 } 4709 break; 4710 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4711 raw_encap = actions->conf; 4712 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4713 memcpy(actions_tx, actions, 4714 sizeof(struct rte_flow_action)); 4715 actions_tx++; 4716 encap = 1; 4717 } else { 4718 rte_memcpy(actions_rx, actions, 4719 sizeof(struct rte_flow_action)); 4720 actions_rx++; 4721 } 4722 break; 4723 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4724 raw_decap = actions->conf; 4725 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4726 memcpy(actions_tx, actions, 4727 sizeof(struct rte_flow_action)); 4728 actions_tx++; 4729 } else { 4730 rte_memcpy(actions_rx, actions, 4731 sizeof(struct rte_flow_action)); 4732 actions_rx++; 4733 } 4734 break; 4735 default: 4736 rte_memcpy(actions_rx, actions, 4737 sizeof(struct rte_flow_action)); 4738 actions_rx++; 4739 break; 4740 } 4741 } 4742 /* Add set meta action and end action for the Rx flow. */ 4743 tag_action = actions_rx; 4744 tag_action->type = (enum rte_flow_action_type) 4745 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4746 actions_rx++; 4747 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4748 actions_rx++; 4749 set_tag = (void *)actions_rx; 4750 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4751 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4752 .data = flow_id, 4753 }; 4754 MLX5_ASSERT(set_tag->id > REG_NON); 4755 tag_action->conf = set_tag; 4756 /* Create Tx item list. */ 4757 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4758 addr = (void *)&pattern_tx[2]; 4759 item = pattern_tx; 4760 item->type = (enum rte_flow_item_type) 4761 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4762 tag_item = (void *)addr; 4763 tag_item->data = flow_id; 4764 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4765 MLX5_ASSERT(set_tag->id > REG_NON); 4766 item->spec = tag_item; 4767 addr += sizeof(struct mlx5_rte_flow_item_tag); 4768 tag_item = (void *)addr; 4769 tag_item->data = UINT32_MAX; 4770 tag_item->id = UINT16_MAX; 4771 item->mask = tag_item; 4772 item->last = NULL; 4773 item++; 4774 item->type = RTE_FLOW_ITEM_TYPE_END; 4775 return 0; 4776 } 4777 4778 /** 4779 * The last stage of splitting chain, just creates the subflow 4780 * without any modification. 4781 * 4782 * @param[in] dev 4783 * Pointer to Ethernet device. 4784 * @param[in] flow 4785 * Parent flow structure pointer. 4786 * @param[in, out] sub_flow 4787 * Pointer to return the created subflow, may be NULL. 4788 * @param[in] attr 4789 * Flow rule attributes. 4790 * @param[in] items 4791 * Pattern specification (list terminated by the END pattern item). 4792 * @param[in] actions 4793 * Associated actions (list terminated by the END action). 4794 * @param[in] flow_split_info 4795 * Pointer to flow split info structure. 4796 * @param[out] error 4797 * Perform verbose error reporting if not NULL. 4798 * @return 4799 * 0 on success, negative value otherwise 4800 */ 4801 static int 4802 flow_create_split_inner(struct rte_eth_dev *dev, 4803 struct rte_flow *flow, 4804 struct mlx5_flow **sub_flow, 4805 const struct rte_flow_attr *attr, 4806 const struct rte_flow_item items[], 4807 const struct rte_flow_action actions[], 4808 struct mlx5_flow_split_info *flow_split_info, 4809 struct rte_flow_error *error) 4810 { 4811 struct mlx5_flow *dev_flow; 4812 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 4813 4814 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4815 flow_split_info->flow_idx, error); 4816 if (!dev_flow) 4817 return -rte_errno; 4818 dev_flow->flow = flow; 4819 dev_flow->external = flow_split_info->external; 4820 dev_flow->skip_scale = flow_split_info->skip_scale; 4821 /* Subflow object was created, we must include one in the list. */ 4822 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4823 dev_flow->handle, next); 4824 /* 4825 * If dev_flow is as one of the suffix flow, some actions in suffix 4826 * flow may need some user defined item layer flags, and pass the 4827 * Metadata rxq mark flag to suffix flow as well. 4828 */ 4829 if (flow_split_info->prefix_layers) 4830 dev_flow->handle->layers = flow_split_info->prefix_layers; 4831 if (flow_split_info->prefix_mark) { 4832 MLX5_ASSERT(wks); 4833 wks->mark = 1; 4834 } 4835 if (sub_flow) 4836 *sub_flow = dev_flow; 4837 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4838 dev_flow->dv.table_id = flow_split_info->table_id; 4839 #endif 4840 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4841 } 4842 4843 /** 4844 * Get the sub policy of a meter. 4845 * 4846 * @param[in] dev 4847 * Pointer to Ethernet device. 4848 * @param[in] flow 4849 * Parent flow structure pointer. 4850 * @param wks 4851 * Pointer to thread flow work space. 4852 * @param[in] attr 4853 * Flow rule attributes. 4854 * @param[in] items 4855 * Pattern specification (list terminated by the END pattern item). 4856 * @param[out] error 4857 * Perform verbose error reporting if not NULL. 4858 * 4859 * @return 4860 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4861 */ 4862 static struct mlx5_flow_meter_sub_policy * 4863 get_meter_sub_policy(struct rte_eth_dev *dev, 4864 struct rte_flow *flow, 4865 struct mlx5_flow_workspace *wks, 4866 const struct rte_flow_attr *attr, 4867 const struct rte_flow_item items[], 4868 struct rte_flow_error *error) 4869 { 4870 struct mlx5_flow_meter_policy *policy; 4871 struct mlx5_flow_meter_policy *final_policy; 4872 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4873 4874 policy = wks->policy; 4875 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4876 if (final_policy->is_rss || final_policy->is_queue) { 4877 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4878 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4879 uint32_t i; 4880 4881 /* 4882 * This is a tmp dev_flow, 4883 * no need to register any matcher for it in translate. 4884 */ 4885 wks->skip_matcher_reg = 1; 4886 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4887 struct mlx5_flow dev_flow = {0}; 4888 struct mlx5_flow_handle dev_handle = { {0} }; 4889 uint8_t fate = final_policy->act_cnt[i].fate_action; 4890 4891 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4892 const struct rte_flow_action_rss *rss_act = 4893 final_policy->act_cnt[i].rss->conf; 4894 struct rte_flow_action rss_actions[2] = { 4895 [0] = { 4896 .type = RTE_FLOW_ACTION_TYPE_RSS, 4897 .conf = rss_act, 4898 }, 4899 [1] = { 4900 .type = RTE_FLOW_ACTION_TYPE_END, 4901 .conf = NULL, 4902 } 4903 }; 4904 4905 dev_flow.handle = &dev_handle; 4906 dev_flow.ingress = attr->ingress; 4907 dev_flow.flow = flow; 4908 dev_flow.external = 0; 4909 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4910 dev_flow.dv.transfer = attr->transfer; 4911 #endif 4912 /** 4913 * Translate RSS action to get rss hash fields. 4914 */ 4915 if (flow_drv_translate(dev, &dev_flow, attr, 4916 items, rss_actions, error)) 4917 goto exit; 4918 rss_desc_v[i] = wks->rss_desc; 4919 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4920 rss_desc_v[i].hash_fields = 4921 dev_flow.hash_fields; 4922 rss_desc_v[i].queue_num = 4923 rss_desc_v[i].hash_fields ? 4924 rss_desc_v[i].queue_num : 1; 4925 rss_desc_v[i].tunnel = 4926 !!(dev_flow.handle->layers & 4927 MLX5_FLOW_LAYER_TUNNEL); 4928 /* Use the RSS queues in the containers. */ 4929 rss_desc_v[i].queue = 4930 (uint16_t *)(uintptr_t)rss_act->queue; 4931 rss_desc[i] = &rss_desc_v[i]; 4932 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 4933 /* This is queue action. */ 4934 rss_desc_v[i] = wks->rss_desc; 4935 rss_desc_v[i].key_len = 0; 4936 rss_desc_v[i].hash_fields = 0; 4937 rss_desc_v[i].queue = 4938 &final_policy->act_cnt[i].queue; 4939 rss_desc_v[i].queue_num = 1; 4940 rss_desc[i] = &rss_desc_v[i]; 4941 } else { 4942 rss_desc[i] = NULL; 4943 } 4944 } 4945 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4946 flow, policy, rss_desc); 4947 } else { 4948 enum mlx5_meter_domain mtr_domain = 4949 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4950 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4951 MLX5_MTR_DOMAIN_INGRESS); 4952 sub_policy = policy->sub_policys[mtr_domain][0]; 4953 } 4954 if (!sub_policy) 4955 rte_flow_error_set(error, EINVAL, 4956 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4957 "Failed to get meter sub-policy."); 4958 exit: 4959 return sub_policy; 4960 } 4961 4962 /** 4963 * Split the meter flow. 4964 * 4965 * As meter flow will split to three sub flow, other than meter 4966 * action, the other actions make sense to only meter accepts 4967 * the packet. If it need to be dropped, no other additional 4968 * actions should be take. 4969 * 4970 * One kind of special action which decapsulates the L3 tunnel 4971 * header will be in the prefix sub flow, as not to take the 4972 * L3 tunnel header into account. 4973 * 4974 * @param[in] dev 4975 * Pointer to Ethernet device. 4976 * @param[in] flow 4977 * Parent flow structure pointer. 4978 * @param wks 4979 * Pointer to thread flow work space. 4980 * @param[in] attr 4981 * Flow rule attributes. 4982 * @param[in] items 4983 * Pattern specification (list terminated by the END pattern item). 4984 * @param[out] sfx_items 4985 * Suffix flow match items (list terminated by the END pattern item). 4986 * @param[in] actions 4987 * Associated actions (list terminated by the END action). 4988 * @param[out] actions_sfx 4989 * Suffix flow actions. 4990 * @param[out] actions_pre 4991 * Prefix flow actions. 4992 * @param[out] mtr_flow_id 4993 * Pointer to meter flow id. 4994 * @param[out] error 4995 * Perform verbose error reporting if not NULL. 4996 * 4997 * @return 4998 * 0 on success, a negative errno value otherwise and rte_errno is set. 4999 */ 5000 static int 5001 flow_meter_split_prep(struct rte_eth_dev *dev, 5002 struct rte_flow *flow, 5003 struct mlx5_flow_workspace *wks, 5004 const struct rte_flow_attr *attr, 5005 const struct rte_flow_item items[], 5006 struct rte_flow_item sfx_items[], 5007 const struct rte_flow_action actions[], 5008 struct rte_flow_action actions_sfx[], 5009 struct rte_flow_action actions_pre[], 5010 uint32_t *mtr_flow_id, 5011 struct rte_flow_error *error) 5012 { 5013 struct mlx5_priv *priv = dev->data->dev_private; 5014 struct mlx5_flow_meter_info *fm = wks->fm; 5015 struct rte_flow_action *tag_action = NULL; 5016 struct rte_flow_item *tag_item; 5017 struct mlx5_rte_flow_action_set_tag *set_tag; 5018 const struct rte_flow_action_raw_encap *raw_encap; 5019 const struct rte_flow_action_raw_decap *raw_decap; 5020 struct mlx5_rte_flow_item_tag *tag_item_spec; 5021 struct mlx5_rte_flow_item_tag *tag_item_mask; 5022 uint32_t tag_id = 0; 5023 struct rte_flow_item *vlan_item_dst = NULL; 5024 const struct rte_flow_item *vlan_item_src = NULL; 5025 struct rte_flow_action *hw_mtr_action; 5026 struct rte_flow_action *action_pre_head = NULL; 5027 int32_t flow_src_port = priv->representor_id; 5028 bool mtr_first; 5029 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5030 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5031 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5032 uint32_t flow_id = 0; 5033 uint32_t flow_id_reversed = 0; 5034 uint8_t flow_id_bits = 0; 5035 int shift; 5036 5037 /* Prepare the suffix subflow items. */ 5038 tag_item = sfx_items++; 5039 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5040 struct mlx5_priv *port_priv; 5041 const struct rte_flow_item_port_id *pid_v; 5042 int item_type = items->type; 5043 5044 switch (item_type) { 5045 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5046 pid_v = items->spec; 5047 MLX5_ASSERT(pid_v); 5048 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5049 if (!port_priv) 5050 return rte_flow_error_set(error, 5051 rte_errno, 5052 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5053 pid_v, 5054 "Failed to get port info."); 5055 flow_src_port = port_priv->representor_id; 5056 if (!fm->def_policy && wks->policy->is_hierarchy && 5057 flow_src_port != priv->representor_id) { 5058 if (flow_drv_mtr_hierarchy_rule_create(dev, 5059 flow, fm, 5060 flow_src_port, 5061 items, 5062 error)) 5063 return -rte_errno; 5064 } 5065 memcpy(sfx_items, items, sizeof(*sfx_items)); 5066 sfx_items++; 5067 break; 5068 case RTE_FLOW_ITEM_TYPE_VLAN: 5069 /* Determine if copy vlan item below. */ 5070 vlan_item_src = items; 5071 vlan_item_dst = sfx_items++; 5072 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5073 break; 5074 default: 5075 break; 5076 } 5077 } 5078 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5079 sfx_items++; 5080 mtr_first = priv->sh->meter_aso_en && 5081 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5082 /* For ASO meter, meter must be before tag in TX direction. */ 5083 if (mtr_first) { 5084 action_pre_head = actions_pre++; 5085 /* Leave space for tag action. */ 5086 tag_action = actions_pre++; 5087 } 5088 /* Prepare the actions for prefix and suffix flow. */ 5089 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5090 struct rte_flow_action *action_cur = NULL; 5091 5092 switch (actions->type) { 5093 case RTE_FLOW_ACTION_TYPE_METER: 5094 if (mtr_first) { 5095 action_cur = action_pre_head; 5096 } else { 5097 /* Leave space for tag action. */ 5098 tag_action = actions_pre++; 5099 action_cur = actions_pre++; 5100 } 5101 break; 5102 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5103 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5104 action_cur = actions_pre++; 5105 break; 5106 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5107 raw_encap = actions->conf; 5108 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5109 action_cur = actions_pre++; 5110 break; 5111 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5112 raw_decap = actions->conf; 5113 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5114 action_cur = actions_pre++; 5115 break; 5116 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5117 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5118 if (vlan_item_dst && vlan_item_src) { 5119 memcpy(vlan_item_dst, vlan_item_src, 5120 sizeof(*vlan_item_dst)); 5121 /* 5122 * Convert to internal match item, it is used 5123 * for vlan push and set vid. 5124 */ 5125 vlan_item_dst->type = (enum rte_flow_item_type) 5126 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5127 } 5128 break; 5129 default: 5130 break; 5131 } 5132 if (!action_cur) 5133 action_cur = (fm->def_policy) ? 5134 actions_sfx++ : actions_pre++; 5135 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5136 } 5137 /* Add end action to the actions. */ 5138 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5139 if (priv->sh->meter_aso_en) { 5140 /** 5141 * For ASO meter, need to add an extra jump action explicitly, 5142 * to jump from meter to policer table. 5143 */ 5144 struct mlx5_flow_meter_sub_policy *sub_policy; 5145 struct mlx5_flow_tbl_data_entry *tbl_data; 5146 5147 if (!fm->def_policy) { 5148 sub_policy = get_meter_sub_policy(dev, flow, wks, 5149 attr, items, error); 5150 if (!sub_policy) 5151 return -rte_errno; 5152 } else { 5153 enum mlx5_meter_domain mtr_domain = 5154 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5155 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5156 MLX5_MTR_DOMAIN_INGRESS); 5157 5158 sub_policy = 5159 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5160 } 5161 tbl_data = container_of(sub_policy->tbl_rsc, 5162 struct mlx5_flow_tbl_data_entry, tbl); 5163 hw_mtr_action = actions_pre++; 5164 hw_mtr_action->type = (enum rte_flow_action_type) 5165 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5166 hw_mtr_action->conf = tbl_data->jump.action; 5167 } 5168 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5169 actions_pre++; 5170 if (!tag_action) 5171 return rte_flow_error_set(error, ENOMEM, 5172 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5173 NULL, "No tag action space."); 5174 if (!mtr_flow_id) { 5175 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5176 goto exit; 5177 } 5178 /* Only default-policy Meter creates mtr flow id. */ 5179 if (fm->def_policy) { 5180 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5181 if (!tag_id) 5182 return rte_flow_error_set(error, ENOMEM, 5183 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5184 "Failed to allocate meter flow id."); 5185 flow_id = tag_id - 1; 5186 flow_id_bits = (!flow_id) ? 1 : 5187 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5188 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5189 mtr_reg_bits) { 5190 mlx5_ipool_free(fm->flow_ipool, tag_id); 5191 return rte_flow_error_set(error, EINVAL, 5192 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5193 "Meter flow id exceeds max limit."); 5194 } 5195 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5196 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5197 } 5198 /* Build tag actions and items for meter_id/meter flow_id. */ 5199 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5200 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5201 tag_item_mask = tag_item_spec + 1; 5202 /* Both flow_id and meter_id share the same register. */ 5203 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5204 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5205 0, error), 5206 .offset = mtr_id_offset, 5207 .length = mtr_reg_bits, 5208 .data = flow->meter, 5209 }; 5210 /* 5211 * The color Reg bits used by flow_id are growing from 5212 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5213 */ 5214 for (shift = 0; shift < flow_id_bits; shift++) 5215 flow_id_reversed = (flow_id_reversed << 1) | 5216 ((flow_id >> shift) & 0x1); 5217 set_tag->data |= 5218 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5219 tag_item_spec->id = set_tag->id; 5220 tag_item_spec->data = set_tag->data << mtr_id_offset; 5221 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5222 tag_action->type = (enum rte_flow_action_type) 5223 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5224 tag_action->conf = set_tag; 5225 tag_item->type = (enum rte_flow_item_type) 5226 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5227 tag_item->spec = tag_item_spec; 5228 tag_item->last = NULL; 5229 tag_item->mask = tag_item_mask; 5230 exit: 5231 if (mtr_flow_id) 5232 *mtr_flow_id = tag_id; 5233 return 0; 5234 } 5235 5236 /** 5237 * Split action list having QUEUE/RSS for metadata register copy. 5238 * 5239 * Once Q/RSS action is detected in user's action list, the flow action 5240 * should be split in order to copy metadata registers, which will happen in 5241 * RX_CP_TBL like, 5242 * - CQE->flow_tag := reg_c[1] (MARK) 5243 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5244 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5245 * This is because the last action of each flow must be a terminal action 5246 * (QUEUE, RSS or DROP). 5247 * 5248 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5249 * stored and kept in the mlx5_flow structure per each sub_flow. 5250 * 5251 * The Q/RSS action is replaced with, 5252 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5253 * And the following JUMP action is added at the end, 5254 * - JUMP, to RX_CP_TBL. 5255 * 5256 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5257 * flow_create_split_metadata() routine. The flow will look like, 5258 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5259 * 5260 * @param dev 5261 * Pointer to Ethernet device. 5262 * @param[out] split_actions 5263 * Pointer to store split actions to jump to CP_TBL. 5264 * @param[in] actions 5265 * Pointer to the list of original flow actions. 5266 * @param[in] qrss 5267 * Pointer to the Q/RSS action. 5268 * @param[in] actions_n 5269 * Number of original actions. 5270 * @param[in] mtr_sfx 5271 * Check if it is in meter suffix table. 5272 * @param[out] error 5273 * Perform verbose error reporting if not NULL. 5274 * 5275 * @return 5276 * non-zero unique flow_id on success, otherwise 0 and 5277 * error/rte_error are set. 5278 */ 5279 static uint32_t 5280 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5281 struct rte_flow_action *split_actions, 5282 const struct rte_flow_action *actions, 5283 const struct rte_flow_action *qrss, 5284 int actions_n, int mtr_sfx, 5285 struct rte_flow_error *error) 5286 { 5287 struct mlx5_priv *priv = dev->data->dev_private; 5288 struct mlx5_rte_flow_action_set_tag *set_tag; 5289 struct rte_flow_action_jump *jump; 5290 const int qrss_idx = qrss - actions; 5291 uint32_t flow_id = 0; 5292 int ret = 0; 5293 5294 /* 5295 * Given actions will be split 5296 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5297 * - Add jump to mreg CP_TBL. 5298 * As a result, there will be one more action. 5299 */ 5300 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5301 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5302 ++actions_n; 5303 set_tag = (void *)(split_actions + actions_n); 5304 /* 5305 * If we are not the meter suffix flow, add the tag action. 5306 * Since meter suffix flow already has the tag added. 5307 */ 5308 if (!mtr_sfx) { 5309 /* 5310 * Allocate the new subflow ID. This one is unique within 5311 * device and not shared with representors. Otherwise, 5312 * we would have to resolve multi-thread access synch 5313 * issue. Each flow on the shared device is appended 5314 * with source vport identifier, so the resulting 5315 * flows will be unique in the shared (by master and 5316 * representors) domain even if they have coinciding 5317 * IDs. 5318 */ 5319 mlx5_ipool_malloc(priv->sh->ipool 5320 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5321 if (!flow_id) 5322 return rte_flow_error_set(error, ENOMEM, 5323 RTE_FLOW_ERROR_TYPE_ACTION, 5324 NULL, "can't allocate id " 5325 "for split Q/RSS subflow"); 5326 /* Internal SET_TAG action to set flow ID. */ 5327 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5328 .data = flow_id, 5329 }; 5330 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5331 if (ret < 0) 5332 return ret; 5333 set_tag->id = ret; 5334 /* Construct new actions array. */ 5335 /* Replace QUEUE/RSS action. */ 5336 split_actions[qrss_idx] = (struct rte_flow_action){ 5337 .type = (enum rte_flow_action_type) 5338 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5339 .conf = set_tag, 5340 }; 5341 } else { 5342 /* 5343 * If we are the suffix flow of meter, tag already exist. 5344 * Set the QUEUE/RSS action to void. 5345 */ 5346 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5347 } 5348 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5349 jump = (void *)(set_tag + 1); 5350 *jump = (struct rte_flow_action_jump){ 5351 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5352 }; 5353 split_actions[actions_n - 2] = (struct rte_flow_action){ 5354 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5355 .conf = jump, 5356 }; 5357 split_actions[actions_n - 1] = (struct rte_flow_action){ 5358 .type = RTE_FLOW_ACTION_TYPE_END, 5359 }; 5360 return flow_id; 5361 } 5362 5363 /** 5364 * Extend the given action list for Tx metadata copy. 5365 * 5366 * Copy the given action list to the ext_actions and add flow metadata register 5367 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5368 * 5369 * @param[out] ext_actions 5370 * Pointer to the extended action list. 5371 * @param[in] actions 5372 * Pointer to the list of actions. 5373 * @param[in] actions_n 5374 * Number of actions in the list. 5375 * @param[out] error 5376 * Perform verbose error reporting if not NULL. 5377 * @param[in] encap_idx 5378 * The encap action index. 5379 * 5380 * @return 5381 * 0 on success, negative value otherwise 5382 */ 5383 static int 5384 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5385 struct rte_flow_action *ext_actions, 5386 const struct rte_flow_action *actions, 5387 int actions_n, struct rte_flow_error *error, 5388 int encap_idx) 5389 { 5390 struct mlx5_flow_action_copy_mreg *cp_mreg = 5391 (struct mlx5_flow_action_copy_mreg *) 5392 (ext_actions + actions_n + 1); 5393 int ret; 5394 5395 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5396 if (ret < 0) 5397 return ret; 5398 cp_mreg->dst = ret; 5399 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5400 if (ret < 0) 5401 return ret; 5402 cp_mreg->src = ret; 5403 if (encap_idx != 0) 5404 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5405 if (encap_idx == actions_n - 1) { 5406 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5407 .type = (enum rte_flow_action_type) 5408 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5409 .conf = cp_mreg, 5410 }; 5411 ext_actions[actions_n] = (struct rte_flow_action){ 5412 .type = RTE_FLOW_ACTION_TYPE_END, 5413 }; 5414 } else { 5415 ext_actions[encap_idx] = (struct rte_flow_action){ 5416 .type = (enum rte_flow_action_type) 5417 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5418 .conf = cp_mreg, 5419 }; 5420 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5421 sizeof(*ext_actions) * (actions_n - encap_idx)); 5422 } 5423 return 0; 5424 } 5425 5426 /** 5427 * Check the match action from the action list. 5428 * 5429 * @param[in] actions 5430 * Pointer to the list of actions. 5431 * @param[in] attr 5432 * Flow rule attributes. 5433 * @param[in] action 5434 * The action to be check if exist. 5435 * @param[out] match_action_pos 5436 * Pointer to the position of the matched action if exists, otherwise is -1. 5437 * @param[out] qrss_action_pos 5438 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5439 * @param[out] modify_after_mirror 5440 * Pointer to the flag of modify action after FDB mirroring. 5441 * 5442 * @return 5443 * > 0 the total number of actions. 5444 * 0 if not found match action in action list. 5445 */ 5446 static int 5447 flow_check_match_action(const struct rte_flow_action actions[], 5448 const struct rte_flow_attr *attr, 5449 enum rte_flow_action_type action, 5450 int *match_action_pos, int *qrss_action_pos, 5451 int *modify_after_mirror) 5452 { 5453 const struct rte_flow_action_sample *sample; 5454 const struct rte_flow_action_raw_decap *decap; 5455 int actions_n = 0; 5456 uint32_t ratio = 0; 5457 int sub_type = 0; 5458 int flag = 0; 5459 int fdb_mirror = 0; 5460 5461 *match_action_pos = -1; 5462 *qrss_action_pos = -1; 5463 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5464 if (actions->type == action) { 5465 flag = 1; 5466 *match_action_pos = actions_n; 5467 } 5468 switch (actions->type) { 5469 case RTE_FLOW_ACTION_TYPE_QUEUE: 5470 case RTE_FLOW_ACTION_TYPE_RSS: 5471 *qrss_action_pos = actions_n; 5472 break; 5473 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5474 sample = actions->conf; 5475 ratio = sample->ratio; 5476 sub_type = ((const struct rte_flow_action *) 5477 (sample->actions))->type; 5478 if (ratio == 1 && attr->transfer) 5479 fdb_mirror = 1; 5480 break; 5481 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5482 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5483 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5484 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5485 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5486 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5487 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5488 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5489 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5490 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5491 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5492 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5493 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5494 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5495 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5496 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5497 case RTE_FLOW_ACTION_TYPE_FLAG: 5498 case RTE_FLOW_ACTION_TYPE_MARK: 5499 case RTE_FLOW_ACTION_TYPE_SET_META: 5500 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5501 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5502 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5503 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5504 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5505 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5506 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5507 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5508 case RTE_FLOW_ACTION_TYPE_METER: 5509 if (fdb_mirror) 5510 *modify_after_mirror = 1; 5511 break; 5512 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5513 decap = actions->conf; 5514 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5515 ; 5516 actions_n++; 5517 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5518 const struct rte_flow_action_raw_encap *encap = 5519 actions->conf; 5520 if (decap->size <= 5521 MLX5_ENCAPSULATION_DECISION_SIZE && 5522 encap->size > 5523 MLX5_ENCAPSULATION_DECISION_SIZE) 5524 /* L3 encap. */ 5525 break; 5526 } 5527 if (fdb_mirror) 5528 *modify_after_mirror = 1; 5529 break; 5530 default: 5531 break; 5532 } 5533 actions_n++; 5534 } 5535 if (flag && fdb_mirror && !*modify_after_mirror) { 5536 /* FDB mirroring uses the destination array to implement 5537 * instead of FLOW_SAMPLER object. 5538 */ 5539 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5540 flag = 0; 5541 } 5542 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5543 return flag ? actions_n + 1 : 0; 5544 } 5545 5546 #define SAMPLE_SUFFIX_ITEM 2 5547 5548 /** 5549 * Split the sample flow. 5550 * 5551 * As sample flow will split to two sub flow, sample flow with 5552 * sample action, the other actions will move to new suffix flow. 5553 * 5554 * Also add unique tag id with tag action in the sample flow, 5555 * the same tag id will be as match in the suffix flow. 5556 * 5557 * @param dev 5558 * Pointer to Ethernet device. 5559 * @param[in] add_tag 5560 * Add extra tag action flag. 5561 * @param[out] sfx_items 5562 * Suffix flow match items (list terminated by the END pattern item). 5563 * @param[in] actions 5564 * Associated actions (list terminated by the END action). 5565 * @param[out] actions_sfx 5566 * Suffix flow actions. 5567 * @param[out] actions_pre 5568 * Prefix flow actions. 5569 * @param[in] actions_n 5570 * The total number of actions. 5571 * @param[in] sample_action_pos 5572 * The sample action position. 5573 * @param[in] qrss_action_pos 5574 * The Queue/RSS action position. 5575 * @param[in] jump_table 5576 * Add extra jump action flag. 5577 * @param[out] error 5578 * Perform verbose error reporting if not NULL. 5579 * 5580 * @return 5581 * 0 on success, or unique flow_id, a negative errno value 5582 * otherwise and rte_errno is set. 5583 */ 5584 static int 5585 flow_sample_split_prep(struct rte_eth_dev *dev, 5586 int add_tag, 5587 struct rte_flow_item sfx_items[], 5588 const struct rte_flow_action actions[], 5589 struct rte_flow_action actions_sfx[], 5590 struct rte_flow_action actions_pre[], 5591 int actions_n, 5592 int sample_action_pos, 5593 int qrss_action_pos, 5594 int jump_table, 5595 struct rte_flow_error *error) 5596 { 5597 struct mlx5_priv *priv = dev->data->dev_private; 5598 struct mlx5_rte_flow_action_set_tag *set_tag; 5599 struct mlx5_rte_flow_item_tag *tag_spec; 5600 struct mlx5_rte_flow_item_tag *tag_mask; 5601 struct rte_flow_action_jump *jump_action; 5602 uint32_t tag_id = 0; 5603 int index; 5604 int append_index = 0; 5605 int ret; 5606 5607 if (sample_action_pos < 0) 5608 return rte_flow_error_set(error, EINVAL, 5609 RTE_FLOW_ERROR_TYPE_ACTION, 5610 NULL, "invalid position of sample " 5611 "action in list"); 5612 /* Prepare the actions for prefix and suffix flow. */ 5613 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5614 index = qrss_action_pos; 5615 /* Put the preceding the Queue/RSS action into prefix flow. */ 5616 if (index != 0) 5617 memcpy(actions_pre, actions, 5618 sizeof(struct rte_flow_action) * index); 5619 /* Put others preceding the sample action into prefix flow. */ 5620 if (sample_action_pos > index + 1) 5621 memcpy(actions_pre + index, actions + index + 1, 5622 sizeof(struct rte_flow_action) * 5623 (sample_action_pos - index - 1)); 5624 index = sample_action_pos - 1; 5625 /* Put Queue/RSS action into Suffix flow. */ 5626 memcpy(actions_sfx, actions + qrss_action_pos, 5627 sizeof(struct rte_flow_action)); 5628 actions_sfx++; 5629 } else { 5630 index = sample_action_pos; 5631 if (index != 0) 5632 memcpy(actions_pre, actions, 5633 sizeof(struct rte_flow_action) * index); 5634 } 5635 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5636 * For CX6DX and above, metadata registers Cx preserve their value, 5637 * add an extra tag action for NIC-RX and E-Switch Domain. 5638 */ 5639 if (add_tag) { 5640 /* Prepare the prefix tag action. */ 5641 append_index++; 5642 set_tag = (void *)(actions_pre + actions_n + append_index); 5643 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5644 if (ret < 0) 5645 return ret; 5646 mlx5_ipool_malloc(priv->sh->ipool 5647 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5648 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5649 .id = ret, 5650 .data = tag_id, 5651 }; 5652 /* Prepare the suffix subflow items. */ 5653 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5654 tag_spec->data = tag_id; 5655 tag_spec->id = set_tag->id; 5656 tag_mask = tag_spec + 1; 5657 tag_mask->data = UINT32_MAX; 5658 sfx_items[0] = (struct rte_flow_item){ 5659 .type = (enum rte_flow_item_type) 5660 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5661 .spec = tag_spec, 5662 .last = NULL, 5663 .mask = tag_mask, 5664 }; 5665 sfx_items[1] = (struct rte_flow_item){ 5666 .type = (enum rte_flow_item_type) 5667 RTE_FLOW_ITEM_TYPE_END, 5668 }; 5669 /* Prepare the tag action in prefix subflow. */ 5670 actions_pre[index++] = 5671 (struct rte_flow_action){ 5672 .type = (enum rte_flow_action_type) 5673 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5674 .conf = set_tag, 5675 }; 5676 } 5677 memcpy(actions_pre + index, actions + sample_action_pos, 5678 sizeof(struct rte_flow_action)); 5679 index += 1; 5680 /* For the modify action after the sample action in E-Switch mirroring, 5681 * Add the extra jump action in prefix subflow and jump into the next 5682 * table, then do the modify action in the new table. 5683 */ 5684 if (jump_table) { 5685 /* Prepare the prefix jump action. */ 5686 append_index++; 5687 jump_action = (void *)(actions_pre + actions_n + append_index); 5688 jump_action->group = jump_table; 5689 actions_pre[index++] = 5690 (struct rte_flow_action){ 5691 .type = (enum rte_flow_action_type) 5692 RTE_FLOW_ACTION_TYPE_JUMP, 5693 .conf = jump_action, 5694 }; 5695 } 5696 actions_pre[index] = (struct rte_flow_action){ 5697 .type = (enum rte_flow_action_type) 5698 RTE_FLOW_ACTION_TYPE_END, 5699 }; 5700 /* Put the actions after sample into Suffix flow. */ 5701 memcpy(actions_sfx, actions + sample_action_pos + 1, 5702 sizeof(struct rte_flow_action) * 5703 (actions_n - sample_action_pos - 1)); 5704 return tag_id; 5705 } 5706 5707 /** 5708 * The splitting for metadata feature. 5709 * 5710 * - Q/RSS action on NIC Rx should be split in order to pass by 5711 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5712 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5713 * 5714 * - All the actions on NIC Tx should have a mreg copy action to 5715 * copy reg_a from WQE to reg_c[0]. 5716 * 5717 * @param dev 5718 * Pointer to Ethernet device. 5719 * @param[in] flow 5720 * Parent flow structure pointer. 5721 * @param[in] attr 5722 * Flow rule attributes. 5723 * @param[in] items 5724 * Pattern specification (list terminated by the END pattern item). 5725 * @param[in] actions 5726 * Associated actions (list terminated by the END action). 5727 * @param[in] flow_split_info 5728 * Pointer to flow split info structure. 5729 * @param[out] error 5730 * Perform verbose error reporting if not NULL. 5731 * @return 5732 * 0 on success, negative value otherwise 5733 */ 5734 static int 5735 flow_create_split_metadata(struct rte_eth_dev *dev, 5736 struct rte_flow *flow, 5737 const struct rte_flow_attr *attr, 5738 const struct rte_flow_item items[], 5739 const struct rte_flow_action actions[], 5740 struct mlx5_flow_split_info *flow_split_info, 5741 struct rte_flow_error *error) 5742 { 5743 struct mlx5_priv *priv = dev->data->dev_private; 5744 struct mlx5_dev_config *config = &priv->config; 5745 const struct rte_flow_action *qrss = NULL; 5746 struct rte_flow_action *ext_actions = NULL; 5747 struct mlx5_flow *dev_flow = NULL; 5748 uint32_t qrss_id = 0; 5749 int mtr_sfx = 0; 5750 size_t act_size; 5751 int actions_n; 5752 int encap_idx; 5753 int ret; 5754 5755 /* Check whether extensive metadata feature is engaged. */ 5756 if (!config->dv_flow_en || 5757 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5758 !mlx5_flow_ext_mreg_supported(dev)) 5759 return flow_create_split_inner(dev, flow, NULL, attr, items, 5760 actions, flow_split_info, error); 5761 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5762 &encap_idx); 5763 if (qrss) { 5764 /* Exclude hairpin flows from splitting. */ 5765 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5766 const struct rte_flow_action_queue *queue; 5767 5768 queue = qrss->conf; 5769 if (mlx5_rxq_get_type(dev, queue->index) == 5770 MLX5_RXQ_TYPE_HAIRPIN) 5771 qrss = NULL; 5772 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5773 const struct rte_flow_action_rss *rss; 5774 5775 rss = qrss->conf; 5776 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5777 MLX5_RXQ_TYPE_HAIRPIN) 5778 qrss = NULL; 5779 } 5780 } 5781 if (qrss) { 5782 /* Check if it is in meter suffix table. */ 5783 mtr_sfx = attr->group == (attr->transfer ? 5784 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5785 MLX5_FLOW_TABLE_LEVEL_METER); 5786 /* 5787 * Q/RSS action on NIC Rx should be split in order to pass by 5788 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5789 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5790 */ 5791 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5792 sizeof(struct rte_flow_action_set_tag) + 5793 sizeof(struct rte_flow_action_jump); 5794 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5795 SOCKET_ID_ANY); 5796 if (!ext_actions) 5797 return rte_flow_error_set(error, ENOMEM, 5798 RTE_FLOW_ERROR_TYPE_ACTION, 5799 NULL, "no memory to split " 5800 "metadata flow"); 5801 /* 5802 * Create the new actions list with removed Q/RSS action 5803 * and appended set tag and jump to register copy table 5804 * (RX_CP_TBL). We should preallocate unique tag ID here 5805 * in advance, because it is needed for set tag action. 5806 */ 5807 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5808 qrss, actions_n, 5809 mtr_sfx, error); 5810 if (!mtr_sfx && !qrss_id) { 5811 ret = -rte_errno; 5812 goto exit; 5813 } 5814 } else if (attr->egress && !attr->transfer) { 5815 /* 5816 * All the actions on NIC Tx should have a metadata register 5817 * copy action to copy reg_a from WQE to reg_c[meta] 5818 */ 5819 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5820 sizeof(struct mlx5_flow_action_copy_mreg); 5821 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5822 SOCKET_ID_ANY); 5823 if (!ext_actions) 5824 return rte_flow_error_set(error, ENOMEM, 5825 RTE_FLOW_ERROR_TYPE_ACTION, 5826 NULL, "no memory to split " 5827 "metadata flow"); 5828 /* Create the action list appended with copy register. */ 5829 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5830 actions_n, error, encap_idx); 5831 if (ret < 0) 5832 goto exit; 5833 } 5834 /* Add the unmodified original or prefix subflow. */ 5835 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5836 items, ext_actions ? ext_actions : 5837 actions, flow_split_info, error); 5838 if (ret < 0) 5839 goto exit; 5840 MLX5_ASSERT(dev_flow); 5841 if (qrss) { 5842 const struct rte_flow_attr q_attr = { 5843 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5844 .ingress = 1, 5845 }; 5846 /* Internal PMD action to set register. */ 5847 struct mlx5_rte_flow_item_tag q_tag_spec = { 5848 .data = qrss_id, 5849 .id = REG_NON, 5850 }; 5851 struct rte_flow_item q_items[] = { 5852 { 5853 .type = (enum rte_flow_item_type) 5854 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5855 .spec = &q_tag_spec, 5856 .last = NULL, 5857 .mask = NULL, 5858 }, 5859 { 5860 .type = RTE_FLOW_ITEM_TYPE_END, 5861 }, 5862 }; 5863 struct rte_flow_action q_actions[] = { 5864 { 5865 .type = qrss->type, 5866 .conf = qrss->conf, 5867 }, 5868 { 5869 .type = RTE_FLOW_ACTION_TYPE_END, 5870 }, 5871 }; 5872 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5873 5874 /* 5875 * Configure the tag item only if there is no meter subflow. 5876 * Since tag is already marked in the meter suffix subflow 5877 * we can just use the meter suffix items as is. 5878 */ 5879 if (qrss_id) { 5880 /* Not meter subflow. */ 5881 MLX5_ASSERT(!mtr_sfx); 5882 /* 5883 * Put unique id in prefix flow due to it is destroyed 5884 * after suffix flow and id will be freed after there 5885 * is no actual flows with this id and identifier 5886 * reallocation becomes possible (for example, for 5887 * other flows in other threads). 5888 */ 5889 dev_flow->handle->split_flow_id = qrss_id; 5890 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5891 error); 5892 if (ret < 0) 5893 goto exit; 5894 q_tag_spec.id = ret; 5895 } 5896 dev_flow = NULL; 5897 /* Add suffix subflow to execute Q/RSS. */ 5898 flow_split_info->prefix_layers = layers; 5899 flow_split_info->prefix_mark = 0; 5900 flow_split_info->table_id = 0; 5901 ret = flow_create_split_inner(dev, flow, &dev_flow, 5902 &q_attr, mtr_sfx ? items : 5903 q_items, q_actions, 5904 flow_split_info, error); 5905 if (ret < 0) 5906 goto exit; 5907 /* qrss ID should be freed if failed. */ 5908 qrss_id = 0; 5909 MLX5_ASSERT(dev_flow); 5910 } 5911 5912 exit: 5913 /* 5914 * We do not destroy the partially created sub_flows in case of error. 5915 * These ones are included into parent flow list and will be destroyed 5916 * by flow_drv_destroy. 5917 */ 5918 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5919 qrss_id); 5920 mlx5_free(ext_actions); 5921 return ret; 5922 } 5923 5924 /** 5925 * Create meter internal drop flow with the original pattern. 5926 * 5927 * @param dev 5928 * Pointer to Ethernet device. 5929 * @param[in] flow 5930 * Parent flow structure pointer. 5931 * @param[in] attr 5932 * Flow rule attributes. 5933 * @param[in] items 5934 * Pattern specification (list terminated by the END pattern item). 5935 * @param[in] flow_split_info 5936 * Pointer to flow split info structure. 5937 * @param[in] fm 5938 * Pointer to flow meter structure. 5939 * @param[out] error 5940 * Perform verbose error reporting if not NULL. 5941 * @return 5942 * 0 on success, negative value otherwise 5943 */ 5944 static uint32_t 5945 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5946 struct rte_flow *flow, 5947 const struct rte_flow_attr *attr, 5948 const struct rte_flow_item items[], 5949 struct mlx5_flow_split_info *flow_split_info, 5950 struct mlx5_flow_meter_info *fm, 5951 struct rte_flow_error *error) 5952 { 5953 struct mlx5_flow *dev_flow = NULL; 5954 struct rte_flow_attr drop_attr = *attr; 5955 struct rte_flow_action drop_actions[3]; 5956 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5957 5958 MLX5_ASSERT(fm->drop_cnt); 5959 drop_actions[0].type = 5960 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5961 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5962 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5963 drop_actions[1].conf = NULL; 5964 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5965 drop_actions[2].conf = NULL; 5966 drop_split_info.external = false; 5967 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5968 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5969 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5970 return flow_create_split_inner(dev, flow, &dev_flow, 5971 &drop_attr, items, drop_actions, 5972 &drop_split_info, error); 5973 } 5974 5975 /** 5976 * The splitting for meter feature. 5977 * 5978 * - The meter flow will be split to two flows as prefix and 5979 * suffix flow. The packets make sense only it pass the prefix 5980 * meter action. 5981 * 5982 * - Reg_C_5 is used for the packet to match betweend prefix and 5983 * suffix flow. 5984 * 5985 * @param dev 5986 * Pointer to Ethernet device. 5987 * @param[in] flow 5988 * Parent flow structure pointer. 5989 * @param[in] attr 5990 * Flow rule attributes. 5991 * @param[in] items 5992 * Pattern specification (list terminated by the END pattern item). 5993 * @param[in] actions 5994 * Associated actions (list terminated by the END action). 5995 * @param[in] flow_split_info 5996 * Pointer to flow split info structure. 5997 * @param[out] error 5998 * Perform verbose error reporting if not NULL. 5999 * @return 6000 * 0 on success, negative value otherwise 6001 */ 6002 static int 6003 flow_create_split_meter(struct rte_eth_dev *dev, 6004 struct rte_flow *flow, 6005 const struct rte_flow_attr *attr, 6006 const struct rte_flow_item items[], 6007 const struct rte_flow_action actions[], 6008 struct mlx5_flow_split_info *flow_split_info, 6009 struct rte_flow_error *error) 6010 { 6011 struct mlx5_priv *priv = dev->data->dev_private; 6012 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6013 struct rte_flow_action *sfx_actions = NULL; 6014 struct rte_flow_action *pre_actions = NULL; 6015 struct rte_flow_item *sfx_items = NULL; 6016 struct mlx5_flow *dev_flow = NULL; 6017 struct rte_flow_attr sfx_attr = *attr; 6018 struct mlx5_flow_meter_info *fm = NULL; 6019 uint8_t skip_scale_restore; 6020 bool has_mtr = false; 6021 bool has_modify = false; 6022 bool set_mtr_reg = true; 6023 bool is_mtr_hierarchy = false; 6024 uint32_t meter_id = 0; 6025 uint32_t mtr_idx = 0; 6026 uint32_t mtr_flow_id = 0; 6027 size_t act_size; 6028 size_t item_size; 6029 int actions_n = 0; 6030 int ret = 0; 6031 6032 if (priv->mtr_en) 6033 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6034 &has_modify, &meter_id); 6035 if (has_mtr) { 6036 if (flow->meter) { 6037 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6038 if (!fm) 6039 return rte_flow_error_set(error, EINVAL, 6040 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6041 NULL, "Meter not found."); 6042 } else { 6043 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6044 if (!fm) 6045 return rte_flow_error_set(error, EINVAL, 6046 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6047 NULL, "Meter not found."); 6048 ret = mlx5_flow_meter_attach(priv, fm, 6049 &sfx_attr, error); 6050 if (ret) 6051 return -rte_errno; 6052 flow->meter = mtr_idx; 6053 } 6054 MLX5_ASSERT(wks); 6055 wks->fm = fm; 6056 if (!fm->def_policy) { 6057 wks->policy = mlx5_flow_meter_policy_find(dev, 6058 fm->policy_id, 6059 NULL); 6060 MLX5_ASSERT(wks->policy); 6061 if (wks->policy->is_hierarchy) { 6062 wks->final_policy = 6063 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6064 wks->policy); 6065 if (!wks->final_policy) 6066 return rte_flow_error_set(error, 6067 EINVAL, 6068 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6069 "Failed to find terminal policy of hierarchy."); 6070 is_mtr_hierarchy = true; 6071 } 6072 } 6073 /* 6074 * If it isn't default-policy Meter, and 6075 * 1. There's no action in flow to change 6076 * packet (modify/encap/decap etc.), OR 6077 * 2. No drop count needed for this meter. 6078 * 3. It's not meter hierarchy. 6079 * Then no need to use regC to save meter id anymore. 6080 */ 6081 if (!fm->def_policy && !is_mtr_hierarchy && 6082 (!has_modify || !fm->drop_cnt)) 6083 set_mtr_reg = false; 6084 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6085 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6086 sizeof(struct mlx5_rte_flow_action_set_tag); 6087 /* Suffix items: tag, vlan, port id, end. */ 6088 #define METER_SUFFIX_ITEM 4 6089 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6090 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6091 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6092 0, SOCKET_ID_ANY); 6093 if (!sfx_actions) 6094 return rte_flow_error_set(error, ENOMEM, 6095 RTE_FLOW_ERROR_TYPE_ACTION, 6096 NULL, "no memory to split " 6097 "meter flow"); 6098 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6099 act_size); 6100 /* There's no suffix flow for meter of non-default policy. */ 6101 if (!fm->def_policy) 6102 pre_actions = sfx_actions + 1; 6103 else 6104 pre_actions = sfx_actions + actions_n; 6105 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6106 items, sfx_items, actions, 6107 sfx_actions, pre_actions, 6108 (set_mtr_reg ? &mtr_flow_id : NULL), 6109 error); 6110 if (ret) { 6111 ret = -rte_errno; 6112 goto exit; 6113 } 6114 /* Add the prefix subflow. */ 6115 skip_scale_restore = flow_split_info->skip_scale; 6116 flow_split_info->skip_scale |= 6117 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6118 ret = flow_create_split_inner(dev, flow, &dev_flow, 6119 attr, items, pre_actions, 6120 flow_split_info, error); 6121 flow_split_info->skip_scale = skip_scale_restore; 6122 if (ret) { 6123 if (mtr_flow_id) 6124 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6125 ret = -rte_errno; 6126 goto exit; 6127 } 6128 if (mtr_flow_id) { 6129 dev_flow->handle->split_flow_id = mtr_flow_id; 6130 dev_flow->handle->is_meter_flow_id = 1; 6131 } 6132 if (!fm->def_policy) { 6133 if (!set_mtr_reg && fm->drop_cnt) 6134 ret = 6135 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6136 &sfx_attr, items, 6137 flow_split_info, 6138 fm, error); 6139 goto exit; 6140 } 6141 /* Setting the sfx group atrr. */ 6142 sfx_attr.group = sfx_attr.transfer ? 6143 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6144 MLX5_FLOW_TABLE_LEVEL_METER; 6145 flow_split_info->prefix_layers = 6146 flow_get_prefix_layer_flags(dev_flow); 6147 flow_split_info->prefix_mark |= wks->mark; 6148 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6149 } 6150 /* Add the prefix subflow. */ 6151 ret = flow_create_split_metadata(dev, flow, 6152 &sfx_attr, sfx_items ? 6153 sfx_items : items, 6154 sfx_actions ? sfx_actions : actions, 6155 flow_split_info, error); 6156 exit: 6157 if (sfx_actions) 6158 mlx5_free(sfx_actions); 6159 return ret; 6160 } 6161 6162 /** 6163 * The splitting for sample feature. 6164 * 6165 * Once Sample action is detected in the action list, the flow actions should 6166 * be split into prefix sub flow and suffix sub flow. 6167 * 6168 * The original items remain in the prefix sub flow, all actions preceding the 6169 * sample action and the sample action itself will be copied to the prefix 6170 * sub flow, the actions following the sample action will be copied to the 6171 * suffix sub flow, Queue action always be located in the suffix sub flow. 6172 * 6173 * In order to make the packet from prefix sub flow matches with suffix sub 6174 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6175 * flow uses tag item with the unique flow id. 6176 * 6177 * @param dev 6178 * Pointer to Ethernet device. 6179 * @param[in] flow 6180 * Parent flow structure pointer. 6181 * @param[in] attr 6182 * Flow rule attributes. 6183 * @param[in] items 6184 * Pattern specification (list terminated by the END pattern item). 6185 * @param[in] actions 6186 * Associated actions (list terminated by the END action). 6187 * @param[in] flow_split_info 6188 * Pointer to flow split info structure. 6189 * @param[out] error 6190 * Perform verbose error reporting if not NULL. 6191 * @return 6192 * 0 on success, negative value otherwise 6193 */ 6194 static int 6195 flow_create_split_sample(struct rte_eth_dev *dev, 6196 struct rte_flow *flow, 6197 const struct rte_flow_attr *attr, 6198 const struct rte_flow_item items[], 6199 const struct rte_flow_action actions[], 6200 struct mlx5_flow_split_info *flow_split_info, 6201 struct rte_flow_error *error) 6202 { 6203 struct mlx5_priv *priv = dev->data->dev_private; 6204 struct rte_flow_action *sfx_actions = NULL; 6205 struct rte_flow_action *pre_actions = NULL; 6206 struct rte_flow_item *sfx_items = NULL; 6207 struct mlx5_flow *dev_flow = NULL; 6208 struct rte_flow_attr sfx_attr = *attr; 6209 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6210 struct mlx5_flow_dv_sample_resource *sample_res; 6211 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6212 struct mlx5_flow_tbl_resource *sfx_tbl; 6213 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6214 #endif 6215 size_t act_size; 6216 size_t item_size; 6217 uint32_t fdb_tx = 0; 6218 int32_t tag_id = 0; 6219 int actions_n = 0; 6220 int sample_action_pos; 6221 int qrss_action_pos; 6222 int add_tag = 0; 6223 int modify_after_mirror = 0; 6224 uint16_t jump_table = 0; 6225 const uint32_t next_ft_step = 1; 6226 int ret = 0; 6227 6228 if (priv->sampler_en) 6229 actions_n = flow_check_match_action(actions, attr, 6230 RTE_FLOW_ACTION_TYPE_SAMPLE, 6231 &sample_action_pos, &qrss_action_pos, 6232 &modify_after_mirror); 6233 if (actions_n) { 6234 /* The prefix actions must includes sample, tag, end. */ 6235 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6236 + sizeof(struct mlx5_rte_flow_action_set_tag); 6237 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6238 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6239 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6240 item_size), 0, SOCKET_ID_ANY); 6241 if (!sfx_actions) 6242 return rte_flow_error_set(error, ENOMEM, 6243 RTE_FLOW_ERROR_TYPE_ACTION, 6244 NULL, "no memory to split " 6245 "sample flow"); 6246 /* The representor_id is UINT16_MAX for uplink. */ 6247 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6248 /* 6249 * When reg_c_preserve is set, metadata registers Cx preserve 6250 * their value even through packet duplication. 6251 */ 6252 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 6253 if (add_tag) 6254 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6255 + act_size); 6256 if (modify_after_mirror) 6257 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6258 next_ft_step; 6259 pre_actions = sfx_actions + actions_n; 6260 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6261 actions, sfx_actions, 6262 pre_actions, actions_n, 6263 sample_action_pos, 6264 qrss_action_pos, jump_table, 6265 error); 6266 if (tag_id < 0 || (add_tag && !tag_id)) { 6267 ret = -rte_errno; 6268 goto exit; 6269 } 6270 if (modify_after_mirror) 6271 flow_split_info->skip_scale = 6272 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6273 /* Add the prefix subflow. */ 6274 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6275 items, pre_actions, 6276 flow_split_info, error); 6277 if (ret) { 6278 ret = -rte_errno; 6279 goto exit; 6280 } 6281 dev_flow->handle->split_flow_id = tag_id; 6282 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6283 if (!modify_after_mirror) { 6284 /* Set the sfx group attr. */ 6285 sample_res = (struct mlx5_flow_dv_sample_resource *) 6286 dev_flow->dv.sample_res; 6287 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6288 sample_res->normal_path_tbl; 6289 sfx_tbl_data = container_of(sfx_tbl, 6290 struct mlx5_flow_tbl_data_entry, 6291 tbl); 6292 sfx_attr.group = sfx_attr.transfer ? 6293 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6294 } else { 6295 MLX5_ASSERT(attr->transfer); 6296 sfx_attr.group = jump_table; 6297 } 6298 flow_split_info->prefix_layers = 6299 flow_get_prefix_layer_flags(dev_flow); 6300 MLX5_ASSERT(wks); 6301 flow_split_info->prefix_mark |= wks->mark; 6302 /* Suffix group level already be scaled with factor, set 6303 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6304 * again in translation. 6305 */ 6306 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6307 #endif 6308 } 6309 /* Add the suffix subflow. */ 6310 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6311 sfx_items ? sfx_items : items, 6312 sfx_actions ? sfx_actions : actions, 6313 flow_split_info, error); 6314 exit: 6315 if (sfx_actions) 6316 mlx5_free(sfx_actions); 6317 return ret; 6318 } 6319 6320 /** 6321 * Split the flow to subflow set. The splitters might be linked 6322 * in the chain, like this: 6323 * flow_create_split_outer() calls: 6324 * flow_create_split_meter() calls: 6325 * flow_create_split_metadata(meter_subflow_0) calls: 6326 * flow_create_split_inner(metadata_subflow_0) 6327 * flow_create_split_inner(metadata_subflow_1) 6328 * flow_create_split_inner(metadata_subflow_2) 6329 * flow_create_split_metadata(meter_subflow_1) calls: 6330 * flow_create_split_inner(metadata_subflow_0) 6331 * flow_create_split_inner(metadata_subflow_1) 6332 * flow_create_split_inner(metadata_subflow_2) 6333 * 6334 * This provide flexible way to add new levels of flow splitting. 6335 * The all of successfully created subflows are included to the 6336 * parent flow dev_flow list. 6337 * 6338 * @param dev 6339 * Pointer to Ethernet device. 6340 * @param[in] flow 6341 * Parent flow structure pointer. 6342 * @param[in] attr 6343 * Flow rule attributes. 6344 * @param[in] items 6345 * Pattern specification (list terminated by the END pattern item). 6346 * @param[in] actions 6347 * Associated actions (list terminated by the END action). 6348 * @param[in] flow_split_info 6349 * Pointer to flow split info structure. 6350 * @param[out] error 6351 * Perform verbose error reporting if not NULL. 6352 * @return 6353 * 0 on success, negative value otherwise 6354 */ 6355 static int 6356 flow_create_split_outer(struct rte_eth_dev *dev, 6357 struct rte_flow *flow, 6358 const struct rte_flow_attr *attr, 6359 const struct rte_flow_item items[], 6360 const struct rte_flow_action actions[], 6361 struct mlx5_flow_split_info *flow_split_info, 6362 struct rte_flow_error *error) 6363 { 6364 int ret; 6365 6366 ret = flow_create_split_sample(dev, flow, attr, items, 6367 actions, flow_split_info, error); 6368 MLX5_ASSERT(ret <= 0); 6369 return ret; 6370 } 6371 6372 static inline struct mlx5_flow_tunnel * 6373 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6374 { 6375 struct mlx5_flow_tunnel *tunnel; 6376 6377 #pragma GCC diagnostic push 6378 #pragma GCC diagnostic ignored "-Wcast-qual" 6379 tunnel = (typeof(tunnel))flow->tunnel; 6380 #pragma GCC diagnostic pop 6381 6382 return tunnel; 6383 } 6384 6385 /** 6386 * Adjust flow RSS workspace if needed. 6387 * 6388 * @param wks 6389 * Pointer to thread flow work space. 6390 * @param rss_desc 6391 * Pointer to RSS descriptor. 6392 * @param[in] nrssq_num 6393 * New RSS queue number. 6394 * 6395 * @return 6396 * 0 on success, -1 otherwise and rte_errno is set. 6397 */ 6398 static int 6399 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6400 struct mlx5_flow_rss_desc *rss_desc, 6401 uint32_t nrssq_num) 6402 { 6403 if (likely(nrssq_num <= wks->rssq_num)) 6404 return 0; 6405 rss_desc->queue = realloc(rss_desc->queue, 6406 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6407 if (!rss_desc->queue) { 6408 rte_errno = ENOMEM; 6409 return -1; 6410 } 6411 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6412 return 0; 6413 } 6414 6415 /** 6416 * Create a flow and add it to @p list. 6417 * 6418 * @param dev 6419 * Pointer to Ethernet device. 6420 * @param list 6421 * Pointer to a TAILQ flow list. If this parameter NULL, 6422 * no list insertion occurred, flow is just created, 6423 * this is caller's responsibility to track the 6424 * created flow. 6425 * @param[in] attr 6426 * Flow rule attributes. 6427 * @param[in] items 6428 * Pattern specification (list terminated by the END pattern item). 6429 * @param[in] actions 6430 * Associated actions (list terminated by the END action). 6431 * @param[in] external 6432 * This flow rule is created by request external to PMD. 6433 * @param[out] error 6434 * Perform verbose error reporting if not NULL. 6435 * 6436 * @return 6437 * A flow index on success, 0 otherwise and rte_errno is set. 6438 */ 6439 static uint32_t 6440 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6441 const struct rte_flow_attr *attr, 6442 const struct rte_flow_item items[], 6443 const struct rte_flow_action original_actions[], 6444 bool external, struct rte_flow_error *error) 6445 { 6446 struct mlx5_priv *priv = dev->data->dev_private; 6447 struct rte_flow *flow = NULL; 6448 struct mlx5_flow *dev_flow; 6449 const struct rte_flow_action_rss *rss = NULL; 6450 struct mlx5_translated_action_handle 6451 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6452 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6453 union { 6454 struct mlx5_flow_expand_rss buf; 6455 uint8_t buffer[4096]; 6456 } expand_buffer; 6457 union { 6458 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6459 uint8_t buffer[2048]; 6460 } actions_rx; 6461 union { 6462 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6463 uint8_t buffer[2048]; 6464 } actions_hairpin_tx; 6465 union { 6466 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6467 uint8_t buffer[2048]; 6468 } items_tx; 6469 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6470 struct mlx5_flow_rss_desc *rss_desc; 6471 const struct rte_flow_action *p_actions_rx; 6472 uint32_t i; 6473 uint32_t idx = 0; 6474 int hairpin_flow; 6475 struct rte_flow_attr attr_tx = { .priority = 0 }; 6476 const struct rte_flow_action *actions; 6477 struct rte_flow_action *translated_actions = NULL; 6478 struct mlx5_flow_tunnel *tunnel; 6479 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6480 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6481 struct mlx5_flow_split_info flow_split_info = { 6482 .external = !!external, 6483 .skip_scale = 0, 6484 .flow_idx = 0, 6485 .prefix_mark = 0, 6486 .prefix_layers = 0, 6487 .table_id = 0 6488 }; 6489 int ret; 6490 6491 MLX5_ASSERT(wks); 6492 rss_desc = &wks->rss_desc; 6493 ret = flow_action_handles_translate(dev, original_actions, 6494 indir_actions, 6495 &indir_actions_n, 6496 &translated_actions, error); 6497 if (ret < 0) { 6498 MLX5_ASSERT(translated_actions == NULL); 6499 return 0; 6500 } 6501 actions = translated_actions ? translated_actions : original_actions; 6502 p_actions_rx = actions; 6503 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6504 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6505 external, hairpin_flow, error); 6506 if (ret < 0) 6507 goto error_before_hairpin_split; 6508 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6509 if (!flow) { 6510 rte_errno = ENOMEM; 6511 goto error_before_hairpin_split; 6512 } 6513 if (hairpin_flow > 0) { 6514 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6515 rte_errno = EINVAL; 6516 goto error_before_hairpin_split; 6517 } 6518 flow_hairpin_split(dev, actions, actions_rx.actions, 6519 actions_hairpin_tx.actions, items_tx.items, 6520 idx); 6521 p_actions_rx = actions_rx.actions; 6522 } 6523 flow_split_info.flow_idx = idx; 6524 flow->drv_type = flow_get_drv_type(dev, attr); 6525 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6526 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6527 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6528 /* RSS Action only works on NIC RX domain */ 6529 if (attr->ingress && !attr->transfer) 6530 rss = flow_get_rss_action(dev, p_actions_rx); 6531 if (rss) { 6532 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6533 return 0; 6534 /* 6535 * The following information is required by 6536 * mlx5_flow_hashfields_adjust() in advance. 6537 */ 6538 rss_desc->level = rss->level; 6539 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6540 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6541 } 6542 flow->dev_handles = 0; 6543 if (rss && rss->types) { 6544 unsigned int graph_root; 6545 6546 graph_root = find_graph_root(rss->level); 6547 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6548 items, rss->types, 6549 mlx5_support_expansion, graph_root); 6550 MLX5_ASSERT(ret > 0 && 6551 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6552 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6553 for (i = 0; i < buf->entries; ++i) 6554 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6555 } 6556 } else { 6557 buf->entries = 1; 6558 buf->entry[0].pattern = (void *)(uintptr_t)items; 6559 } 6560 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6561 indir_actions_n); 6562 for (i = 0; i < buf->entries; ++i) { 6563 /* Initialize flow split data. */ 6564 flow_split_info.prefix_layers = 0; 6565 flow_split_info.prefix_mark = 0; 6566 flow_split_info.skip_scale = 0; 6567 /* 6568 * The splitter may create multiple dev_flows, 6569 * depending on configuration. In the simplest 6570 * case it just creates unmodified original flow. 6571 */ 6572 ret = flow_create_split_outer(dev, flow, attr, 6573 buf->entry[i].pattern, 6574 p_actions_rx, &flow_split_info, 6575 error); 6576 if (ret < 0) 6577 goto error; 6578 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6579 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6580 p_actions_rx, 6581 idx, 6582 wks->flows[0].tunnel, 6583 &default_miss_ctx, 6584 error); 6585 if (ret < 0) { 6586 mlx5_free(default_miss_ctx.queue); 6587 goto error; 6588 } 6589 } 6590 } 6591 /* Create the tx flow. */ 6592 if (hairpin_flow) { 6593 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6594 attr_tx.ingress = 0; 6595 attr_tx.egress = 1; 6596 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6597 actions_hairpin_tx.actions, 6598 idx, error); 6599 if (!dev_flow) 6600 goto error; 6601 dev_flow->flow = flow; 6602 dev_flow->external = 0; 6603 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6604 dev_flow->handle, next); 6605 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6606 items_tx.items, 6607 actions_hairpin_tx.actions, error); 6608 if (ret < 0) 6609 goto error; 6610 } 6611 /* 6612 * Update the metadata register copy table. If extensive 6613 * metadata feature is enabled and registers are supported 6614 * we might create the extra rte_flow for each unique 6615 * MARK/FLAG action ID. 6616 * 6617 * The table is updated for ingress Flows only, because 6618 * the egress Flows belong to the different device and 6619 * copy table should be updated in peer NIC Rx domain. 6620 */ 6621 if (attr->ingress && 6622 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6623 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6624 if (ret) 6625 goto error; 6626 } 6627 /* 6628 * If the flow is external (from application) OR device is started, 6629 * OR mreg discover, then apply immediately. 6630 */ 6631 if (external || dev->data->dev_started || 6632 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6633 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6634 ret = flow_drv_apply(dev, flow, error); 6635 if (ret < 0) 6636 goto error; 6637 } 6638 flow->type = type; 6639 flow_rxq_flags_set(dev, flow); 6640 rte_free(translated_actions); 6641 tunnel = flow_tunnel_from_rule(wks->flows); 6642 if (tunnel) { 6643 flow->tunnel = 1; 6644 flow->tunnel_id = tunnel->tunnel_id; 6645 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6646 mlx5_free(default_miss_ctx.queue); 6647 } 6648 mlx5_flow_pop_thread_workspace(); 6649 return idx; 6650 error: 6651 MLX5_ASSERT(flow); 6652 ret = rte_errno; /* Save rte_errno before cleanup. */ 6653 flow_mreg_del_copy_action(dev, flow); 6654 flow_drv_destroy(dev, flow); 6655 if (rss_desc->shared_rss) 6656 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6657 mlx5_ipool_get 6658 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6659 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6660 mlx5_ipool_free(priv->flows[type], idx); 6661 rte_errno = ret; /* Restore rte_errno. */ 6662 ret = rte_errno; 6663 rte_errno = ret; 6664 mlx5_flow_pop_thread_workspace(); 6665 error_before_hairpin_split: 6666 rte_free(translated_actions); 6667 return 0; 6668 } 6669 6670 /** 6671 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6672 * incoming packets to table 1. 6673 * 6674 * Other flow rules, requested for group n, will be created in 6675 * e-switch table n+1. 6676 * Jump action to e-switch group n will be created to group n+1. 6677 * 6678 * Used when working in switchdev mode, to utilise advantages of table 1 6679 * and above. 6680 * 6681 * @param dev 6682 * Pointer to Ethernet device. 6683 * 6684 * @return 6685 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6686 */ 6687 struct rte_flow * 6688 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6689 { 6690 const struct rte_flow_attr attr = { 6691 .group = 0, 6692 .priority = 0, 6693 .ingress = 1, 6694 .egress = 0, 6695 .transfer = 1, 6696 }; 6697 const struct rte_flow_item pattern = { 6698 .type = RTE_FLOW_ITEM_TYPE_END, 6699 }; 6700 struct rte_flow_action_jump jump = { 6701 .group = 1, 6702 }; 6703 const struct rte_flow_action actions[] = { 6704 { 6705 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6706 .conf = &jump, 6707 }, 6708 { 6709 .type = RTE_FLOW_ACTION_TYPE_END, 6710 }, 6711 }; 6712 struct rte_flow_error error; 6713 6714 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6715 &attr, &pattern, 6716 actions, false, &error); 6717 } 6718 6719 /** 6720 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6721 * and sq number, directs all packets to peer vport. 6722 * 6723 * @param dev 6724 * Pointer to Ethernet device. 6725 * @param txq 6726 * Txq index. 6727 * 6728 * @return 6729 * Flow ID on success, 0 otherwise and rte_errno is set. 6730 */ 6731 uint32_t 6732 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6733 { 6734 struct rte_flow_attr attr = { 6735 .group = 0, 6736 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6737 .ingress = 1, 6738 .egress = 0, 6739 .transfer = 1, 6740 }; 6741 struct rte_flow_item_port_id port_spec = { 6742 .id = MLX5_PORT_ESW_MGR, 6743 }; 6744 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6745 .queue = txq, 6746 }; 6747 struct rte_flow_item pattern[] = { 6748 { 6749 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6750 .spec = &port_spec, 6751 }, 6752 { 6753 .type = (enum rte_flow_item_type) 6754 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6755 .spec = &txq_spec, 6756 }, 6757 { 6758 .type = RTE_FLOW_ITEM_TYPE_END, 6759 }, 6760 }; 6761 struct rte_flow_action_jump jump = { 6762 .group = 1, 6763 }; 6764 struct rte_flow_action_port_id port = { 6765 .id = dev->data->port_id, 6766 }; 6767 struct rte_flow_action actions[] = { 6768 { 6769 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6770 .conf = &jump, 6771 }, 6772 { 6773 .type = RTE_FLOW_ACTION_TYPE_END, 6774 }, 6775 }; 6776 struct rte_flow_error error; 6777 6778 /* 6779 * Creates group 0, highest priority jump flow. 6780 * Matches txq to bypass kernel packets. 6781 */ 6782 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 6783 false, &error) == 0) 6784 return 0; 6785 /* Create group 1, lowest priority redirect flow for txq. */ 6786 attr.group = 1; 6787 actions[0].conf = &port; 6788 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 6789 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 6790 actions, false, &error); 6791 } 6792 6793 /** 6794 * Validate a flow supported by the NIC. 6795 * 6796 * @see rte_flow_validate() 6797 * @see rte_flow_ops 6798 */ 6799 int 6800 mlx5_flow_validate(struct rte_eth_dev *dev, 6801 const struct rte_flow_attr *attr, 6802 const struct rte_flow_item items[], 6803 const struct rte_flow_action original_actions[], 6804 struct rte_flow_error *error) 6805 { 6806 int hairpin_flow; 6807 struct mlx5_translated_action_handle 6808 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6809 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6810 const struct rte_flow_action *actions; 6811 struct rte_flow_action *translated_actions = NULL; 6812 int ret = flow_action_handles_translate(dev, original_actions, 6813 indir_actions, 6814 &indir_actions_n, 6815 &translated_actions, error); 6816 6817 if (ret) 6818 return ret; 6819 actions = translated_actions ? translated_actions : original_actions; 6820 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6821 ret = flow_drv_validate(dev, attr, items, actions, 6822 true, hairpin_flow, error); 6823 rte_free(translated_actions); 6824 return ret; 6825 } 6826 6827 /** 6828 * Create a flow. 6829 * 6830 * @see rte_flow_create() 6831 * @see rte_flow_ops 6832 */ 6833 struct rte_flow * 6834 mlx5_flow_create(struct rte_eth_dev *dev, 6835 const struct rte_flow_attr *attr, 6836 const struct rte_flow_item items[], 6837 const struct rte_flow_action actions[], 6838 struct rte_flow_error *error) 6839 { 6840 /* 6841 * If the device is not started yet, it is not allowed to created a 6842 * flow from application. PMD default flows and traffic control flows 6843 * are not affected. 6844 */ 6845 if (unlikely(!dev->data->dev_started)) { 6846 DRV_LOG(DEBUG, "port %u is not started when " 6847 "inserting a flow", dev->data->port_id); 6848 rte_flow_error_set(error, ENODEV, 6849 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6850 NULL, 6851 "port not started"); 6852 return NULL; 6853 } 6854 6855 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6856 attr, items, actions, 6857 true, error); 6858 } 6859 6860 /** 6861 * Destroy a flow in a list. 6862 * 6863 * @param dev 6864 * Pointer to Ethernet device. 6865 * @param[in] flow_idx 6866 * Index of flow to destroy. 6867 */ 6868 static void 6869 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6870 uint32_t flow_idx) 6871 { 6872 struct mlx5_priv *priv = dev->data->dev_private; 6873 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6874 6875 if (!flow) 6876 return; 6877 MLX5_ASSERT(flow->type == type); 6878 /* 6879 * Update RX queue flags only if port is started, otherwise it is 6880 * already clean. 6881 */ 6882 if (dev->data->dev_started) 6883 flow_rxq_flags_trim(dev, flow); 6884 flow_drv_destroy(dev, flow); 6885 if (flow->tunnel) { 6886 struct mlx5_flow_tunnel *tunnel; 6887 6888 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6889 RTE_VERIFY(tunnel); 6890 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6891 mlx5_flow_tunnel_free(dev, tunnel); 6892 } 6893 flow_mreg_del_copy_action(dev, flow); 6894 mlx5_ipool_free(priv->flows[type], flow_idx); 6895 } 6896 6897 /** 6898 * Destroy all flows. 6899 * 6900 * @param dev 6901 * Pointer to Ethernet device. 6902 * @param type 6903 * Flow type to be flushed. 6904 * @param active 6905 * If flushing is called actively. 6906 */ 6907 void 6908 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6909 bool active) 6910 { 6911 struct mlx5_priv *priv = dev->data->dev_private; 6912 uint32_t num_flushed = 0, fidx = 1; 6913 struct rte_flow *flow; 6914 6915 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6916 flow_list_destroy(dev, type, fidx); 6917 num_flushed++; 6918 } 6919 if (active) { 6920 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6921 dev->data->port_id, num_flushed); 6922 } 6923 } 6924 6925 /** 6926 * Stop all default actions for flows. 6927 * 6928 * @param dev 6929 * Pointer to Ethernet device. 6930 */ 6931 void 6932 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6933 { 6934 flow_mreg_del_default_copy_action(dev); 6935 flow_rxq_flags_clear(dev); 6936 } 6937 6938 /** 6939 * Start all default actions for flows. 6940 * 6941 * @param dev 6942 * Pointer to Ethernet device. 6943 * @return 6944 * 0 on success, a negative errno value otherwise and rte_errno is set. 6945 */ 6946 int 6947 mlx5_flow_start_default(struct rte_eth_dev *dev) 6948 { 6949 struct rte_flow_error error; 6950 6951 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6952 return flow_mreg_add_default_copy_action(dev, &error); 6953 } 6954 6955 /** 6956 * Release key of thread specific flow workspace data. 6957 */ 6958 void 6959 flow_release_workspace(void *data) 6960 { 6961 struct mlx5_flow_workspace *wks = data; 6962 struct mlx5_flow_workspace *next; 6963 6964 while (wks) { 6965 next = wks->next; 6966 free(wks->rss_desc.queue); 6967 free(wks); 6968 wks = next; 6969 } 6970 } 6971 6972 /** 6973 * Get thread specific current flow workspace. 6974 * 6975 * @return pointer to thread specific flow workspace data, NULL on error. 6976 */ 6977 struct mlx5_flow_workspace* 6978 mlx5_flow_get_thread_workspace(void) 6979 { 6980 struct mlx5_flow_workspace *data; 6981 6982 data = mlx5_flow_os_get_specific_workspace(); 6983 MLX5_ASSERT(data && data->inuse); 6984 if (!data || !data->inuse) 6985 DRV_LOG(ERR, "flow workspace not initialized."); 6986 return data; 6987 } 6988 6989 /** 6990 * Allocate and init new flow workspace. 6991 * 6992 * @return pointer to flow workspace data, NULL on error. 6993 */ 6994 static struct mlx5_flow_workspace* 6995 flow_alloc_thread_workspace(void) 6996 { 6997 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6998 6999 if (!data) { 7000 DRV_LOG(ERR, "Failed to allocate flow workspace " 7001 "memory."); 7002 return NULL; 7003 } 7004 data->rss_desc.queue = calloc(1, 7005 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7006 if (!data->rss_desc.queue) 7007 goto err; 7008 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7009 return data; 7010 err: 7011 free(data->rss_desc.queue); 7012 free(data); 7013 return NULL; 7014 } 7015 7016 /** 7017 * Get new thread specific flow workspace. 7018 * 7019 * If current workspace inuse, create new one and set as current. 7020 * 7021 * @return pointer to thread specific flow workspace data, NULL on error. 7022 */ 7023 static struct mlx5_flow_workspace* 7024 mlx5_flow_push_thread_workspace(void) 7025 { 7026 struct mlx5_flow_workspace *curr; 7027 struct mlx5_flow_workspace *data; 7028 7029 curr = mlx5_flow_os_get_specific_workspace(); 7030 if (!curr) { 7031 data = flow_alloc_thread_workspace(); 7032 if (!data) 7033 return NULL; 7034 } else if (!curr->inuse) { 7035 data = curr; 7036 } else if (curr->next) { 7037 data = curr->next; 7038 } else { 7039 data = flow_alloc_thread_workspace(); 7040 if (!data) 7041 return NULL; 7042 curr->next = data; 7043 data->prev = curr; 7044 } 7045 data->inuse = 1; 7046 data->flow_idx = 0; 7047 /* Set as current workspace */ 7048 if (mlx5_flow_os_set_specific_workspace(data)) 7049 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7050 return data; 7051 } 7052 7053 /** 7054 * Close current thread specific flow workspace. 7055 * 7056 * If previous workspace available, set it as current. 7057 * 7058 * @return pointer to thread specific flow workspace data, NULL on error. 7059 */ 7060 static void 7061 mlx5_flow_pop_thread_workspace(void) 7062 { 7063 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7064 7065 if (!data) 7066 return; 7067 if (!data->inuse) { 7068 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7069 return; 7070 } 7071 data->inuse = 0; 7072 if (!data->prev) 7073 return; 7074 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7075 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7076 } 7077 7078 /** 7079 * Verify the flow list is empty 7080 * 7081 * @param dev 7082 * Pointer to Ethernet device. 7083 * 7084 * @return the number of flows not released. 7085 */ 7086 int 7087 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7088 { 7089 struct mlx5_priv *priv = dev->data->dev_private; 7090 struct rte_flow *flow; 7091 uint32_t idx = 0; 7092 int ret = 0, i; 7093 7094 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7095 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7096 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7097 dev->data->port_id, (void *)flow); 7098 ret++; 7099 } 7100 } 7101 return ret; 7102 } 7103 7104 /** 7105 * Enable default hairpin egress flow. 7106 * 7107 * @param dev 7108 * Pointer to Ethernet device. 7109 * @param queue 7110 * The queue index. 7111 * 7112 * @return 7113 * 0 on success, a negative errno value otherwise and rte_errno is set. 7114 */ 7115 int 7116 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7117 uint32_t queue) 7118 { 7119 const struct rte_flow_attr attr = { 7120 .egress = 1, 7121 .priority = 0, 7122 }; 7123 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7124 .queue = queue, 7125 }; 7126 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7127 .queue = UINT32_MAX, 7128 }; 7129 struct rte_flow_item items[] = { 7130 { 7131 .type = (enum rte_flow_item_type) 7132 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7133 .spec = &queue_spec, 7134 .last = NULL, 7135 .mask = &queue_mask, 7136 }, 7137 { 7138 .type = RTE_FLOW_ITEM_TYPE_END, 7139 }, 7140 }; 7141 struct rte_flow_action_jump jump = { 7142 .group = MLX5_HAIRPIN_TX_TABLE, 7143 }; 7144 struct rte_flow_action actions[2]; 7145 uint32_t flow_idx; 7146 struct rte_flow_error error; 7147 7148 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7149 actions[0].conf = &jump; 7150 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7151 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7152 &attr, items, actions, false, &error); 7153 if (!flow_idx) { 7154 DRV_LOG(DEBUG, 7155 "Failed to create ctrl flow: rte_errno(%d)," 7156 " type(%d), message(%s)", 7157 rte_errno, error.type, 7158 error.message ? error.message : " (no stated reason)"); 7159 return -rte_errno; 7160 } 7161 return 0; 7162 } 7163 7164 /** 7165 * Enable a control flow configured from the control plane. 7166 * 7167 * @param dev 7168 * Pointer to Ethernet device. 7169 * @param eth_spec 7170 * An Ethernet flow spec to apply. 7171 * @param eth_mask 7172 * An Ethernet flow mask to apply. 7173 * @param vlan_spec 7174 * A VLAN flow spec to apply. 7175 * @param vlan_mask 7176 * A VLAN flow mask to apply. 7177 * 7178 * @return 7179 * 0 on success, a negative errno value otherwise and rte_errno is set. 7180 */ 7181 int 7182 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7183 struct rte_flow_item_eth *eth_spec, 7184 struct rte_flow_item_eth *eth_mask, 7185 struct rte_flow_item_vlan *vlan_spec, 7186 struct rte_flow_item_vlan *vlan_mask) 7187 { 7188 struct mlx5_priv *priv = dev->data->dev_private; 7189 const struct rte_flow_attr attr = { 7190 .ingress = 1, 7191 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7192 }; 7193 struct rte_flow_item items[] = { 7194 { 7195 .type = RTE_FLOW_ITEM_TYPE_ETH, 7196 .spec = eth_spec, 7197 .last = NULL, 7198 .mask = eth_mask, 7199 }, 7200 { 7201 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7202 RTE_FLOW_ITEM_TYPE_END, 7203 .spec = vlan_spec, 7204 .last = NULL, 7205 .mask = vlan_mask, 7206 }, 7207 { 7208 .type = RTE_FLOW_ITEM_TYPE_END, 7209 }, 7210 }; 7211 uint16_t queue[priv->reta_idx_n]; 7212 struct rte_flow_action_rss action_rss = { 7213 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7214 .level = 0, 7215 .types = priv->rss_conf.rss_hf, 7216 .key_len = priv->rss_conf.rss_key_len, 7217 .queue_num = priv->reta_idx_n, 7218 .key = priv->rss_conf.rss_key, 7219 .queue = queue, 7220 }; 7221 struct rte_flow_action actions[] = { 7222 { 7223 .type = RTE_FLOW_ACTION_TYPE_RSS, 7224 .conf = &action_rss, 7225 }, 7226 { 7227 .type = RTE_FLOW_ACTION_TYPE_END, 7228 }, 7229 }; 7230 uint32_t flow_idx; 7231 struct rte_flow_error error; 7232 unsigned int i; 7233 7234 if (!priv->reta_idx_n || !priv->rxqs_n) { 7235 return 0; 7236 } 7237 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7238 action_rss.types = 0; 7239 for (i = 0; i != priv->reta_idx_n; ++i) 7240 queue[i] = (*priv->reta_idx)[i]; 7241 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7242 &attr, items, actions, false, &error); 7243 if (!flow_idx) 7244 return -rte_errno; 7245 return 0; 7246 } 7247 7248 /** 7249 * Enable a flow control configured from the control plane. 7250 * 7251 * @param dev 7252 * Pointer to Ethernet device. 7253 * @param eth_spec 7254 * An Ethernet flow spec to apply. 7255 * @param eth_mask 7256 * An Ethernet flow mask to apply. 7257 * 7258 * @return 7259 * 0 on success, a negative errno value otherwise and rte_errno is set. 7260 */ 7261 int 7262 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7263 struct rte_flow_item_eth *eth_spec, 7264 struct rte_flow_item_eth *eth_mask) 7265 { 7266 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7267 } 7268 7269 /** 7270 * Create default miss flow rule matching lacp traffic 7271 * 7272 * @param dev 7273 * Pointer to Ethernet device. 7274 * @param eth_spec 7275 * An Ethernet flow spec to apply. 7276 * 7277 * @return 7278 * 0 on success, a negative errno value otherwise and rte_errno is set. 7279 */ 7280 int 7281 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7282 { 7283 /* 7284 * The LACP matching is done by only using ether type since using 7285 * a multicast dst mac causes kernel to give low priority to this flow. 7286 */ 7287 static const struct rte_flow_item_eth lacp_spec = { 7288 .type = RTE_BE16(0x8809), 7289 }; 7290 static const struct rte_flow_item_eth lacp_mask = { 7291 .type = 0xffff, 7292 }; 7293 const struct rte_flow_attr attr = { 7294 .ingress = 1, 7295 }; 7296 struct rte_flow_item items[] = { 7297 { 7298 .type = RTE_FLOW_ITEM_TYPE_ETH, 7299 .spec = &lacp_spec, 7300 .mask = &lacp_mask, 7301 }, 7302 { 7303 .type = RTE_FLOW_ITEM_TYPE_END, 7304 }, 7305 }; 7306 struct rte_flow_action actions[] = { 7307 { 7308 .type = (enum rte_flow_action_type) 7309 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7310 }, 7311 { 7312 .type = RTE_FLOW_ACTION_TYPE_END, 7313 }, 7314 }; 7315 struct rte_flow_error error; 7316 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7317 &attr, items, actions, 7318 false, &error); 7319 7320 if (!flow_idx) 7321 return -rte_errno; 7322 return 0; 7323 } 7324 7325 /** 7326 * Destroy a flow. 7327 * 7328 * @see rte_flow_destroy() 7329 * @see rte_flow_ops 7330 */ 7331 int 7332 mlx5_flow_destroy(struct rte_eth_dev *dev, 7333 struct rte_flow *flow, 7334 struct rte_flow_error *error __rte_unused) 7335 { 7336 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7337 (uintptr_t)(void *)flow); 7338 return 0; 7339 } 7340 7341 /** 7342 * Destroy all flows. 7343 * 7344 * @see rte_flow_flush() 7345 * @see rte_flow_ops 7346 */ 7347 int 7348 mlx5_flow_flush(struct rte_eth_dev *dev, 7349 struct rte_flow_error *error __rte_unused) 7350 { 7351 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7352 return 0; 7353 } 7354 7355 /** 7356 * Isolated mode. 7357 * 7358 * @see rte_flow_isolate() 7359 * @see rte_flow_ops 7360 */ 7361 int 7362 mlx5_flow_isolate(struct rte_eth_dev *dev, 7363 int enable, 7364 struct rte_flow_error *error) 7365 { 7366 struct mlx5_priv *priv = dev->data->dev_private; 7367 7368 if (dev->data->dev_started) { 7369 rte_flow_error_set(error, EBUSY, 7370 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7371 NULL, 7372 "port must be stopped first"); 7373 return -rte_errno; 7374 } 7375 priv->isolated = !!enable; 7376 if (enable) 7377 dev->dev_ops = &mlx5_dev_ops_isolate; 7378 else 7379 dev->dev_ops = &mlx5_dev_ops; 7380 7381 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7382 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7383 7384 return 0; 7385 } 7386 7387 /** 7388 * Query a flow. 7389 * 7390 * @see rte_flow_query() 7391 * @see rte_flow_ops 7392 */ 7393 static int 7394 flow_drv_query(struct rte_eth_dev *dev, 7395 uint32_t flow_idx, 7396 const struct rte_flow_action *actions, 7397 void *data, 7398 struct rte_flow_error *error) 7399 { 7400 struct mlx5_priv *priv = dev->data->dev_private; 7401 const struct mlx5_flow_driver_ops *fops; 7402 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7403 flow_idx); 7404 enum mlx5_flow_drv_type ftype; 7405 7406 if (!flow) { 7407 return rte_flow_error_set(error, ENOENT, 7408 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7409 NULL, 7410 "invalid flow handle"); 7411 } 7412 ftype = flow->drv_type; 7413 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7414 fops = flow_get_drv_ops(ftype); 7415 7416 return fops->query(dev, flow, actions, data, error); 7417 } 7418 7419 /** 7420 * Query a flow. 7421 * 7422 * @see rte_flow_query() 7423 * @see rte_flow_ops 7424 */ 7425 int 7426 mlx5_flow_query(struct rte_eth_dev *dev, 7427 struct rte_flow *flow, 7428 const struct rte_flow_action *actions, 7429 void *data, 7430 struct rte_flow_error *error) 7431 { 7432 int ret; 7433 7434 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7435 error); 7436 if (ret < 0) 7437 return ret; 7438 return 0; 7439 } 7440 7441 /** 7442 * Get rte_flow callbacks. 7443 * 7444 * @param dev 7445 * Pointer to Ethernet device structure. 7446 * @param ops 7447 * Pointer to operation-specific structure. 7448 * 7449 * @return 0 7450 */ 7451 int 7452 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7453 const struct rte_flow_ops **ops) 7454 { 7455 *ops = &mlx5_flow_ops; 7456 return 0; 7457 } 7458 7459 /** 7460 * Validate meter policy actions. 7461 * Dispatcher for action type specific validation. 7462 * 7463 * @param[in] dev 7464 * Pointer to the Ethernet device structure. 7465 * @param[in] action 7466 * The meter policy action object to validate. 7467 * @param[in] attr 7468 * Attributes of flow to determine steering domain. 7469 * @param[out] is_rss 7470 * Is RSS or not. 7471 * @param[out] domain_bitmap 7472 * Domain bitmap. 7473 * @param[out] is_def_policy 7474 * Is default policy or not. 7475 * @param[out] error 7476 * Perform verbose error reporting if not NULL. Initialized in case of 7477 * error only. 7478 * 7479 * @return 7480 * 0 on success, otherwise negative errno value. 7481 */ 7482 int 7483 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7484 const struct rte_flow_action *actions[RTE_COLORS], 7485 struct rte_flow_attr *attr, 7486 bool *is_rss, 7487 uint8_t *domain_bitmap, 7488 uint8_t *policy_mode, 7489 struct rte_mtr_error *error) 7490 { 7491 const struct mlx5_flow_driver_ops *fops; 7492 7493 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7494 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7495 domain_bitmap, policy_mode, error); 7496 } 7497 7498 /** 7499 * Destroy the meter table set. 7500 * 7501 * @param[in] dev 7502 * Pointer to Ethernet device. 7503 * @param[in] mtr_policy 7504 * Meter policy struct. 7505 */ 7506 void 7507 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7508 struct mlx5_flow_meter_policy *mtr_policy) 7509 { 7510 const struct mlx5_flow_driver_ops *fops; 7511 7512 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7513 fops->destroy_mtr_acts(dev, mtr_policy); 7514 } 7515 7516 /** 7517 * Create policy action, lock free, 7518 * (mutex should be acquired by caller). 7519 * Dispatcher for action type specific call. 7520 * 7521 * @param[in] dev 7522 * Pointer to the Ethernet device structure. 7523 * @param[in] mtr_policy 7524 * Meter policy struct. 7525 * @param[in] action 7526 * Action specification used to create meter actions. 7527 * @param[out] error 7528 * Perform verbose error reporting if not NULL. Initialized in case of 7529 * error only. 7530 * 7531 * @return 7532 * 0 on success, otherwise negative errno value. 7533 */ 7534 int 7535 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7536 struct mlx5_flow_meter_policy *mtr_policy, 7537 const struct rte_flow_action *actions[RTE_COLORS], 7538 struct rte_mtr_error *error) 7539 { 7540 const struct mlx5_flow_driver_ops *fops; 7541 7542 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7543 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7544 } 7545 7546 /** 7547 * Create policy rules, lock free, 7548 * (mutex should be acquired by caller). 7549 * Dispatcher for action type specific call. 7550 * 7551 * @param[in] dev 7552 * Pointer to the Ethernet device structure. 7553 * @param[in] mtr_policy 7554 * Meter policy struct. 7555 * 7556 * @return 7557 * 0 on success, -1 otherwise. 7558 */ 7559 int 7560 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7561 struct mlx5_flow_meter_policy *mtr_policy) 7562 { 7563 const struct mlx5_flow_driver_ops *fops; 7564 7565 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7566 return fops->create_policy_rules(dev, mtr_policy); 7567 } 7568 7569 /** 7570 * Destroy policy rules, lock free, 7571 * (mutex should be acquired by caller). 7572 * Dispatcher for action type specific call. 7573 * 7574 * @param[in] dev 7575 * Pointer to the Ethernet device structure. 7576 * @param[in] mtr_policy 7577 * Meter policy struct. 7578 */ 7579 void 7580 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7581 struct mlx5_flow_meter_policy *mtr_policy) 7582 { 7583 const struct mlx5_flow_driver_ops *fops; 7584 7585 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7586 fops->destroy_policy_rules(dev, mtr_policy); 7587 } 7588 7589 /** 7590 * Destroy the default policy table set. 7591 * 7592 * @param[in] dev 7593 * Pointer to Ethernet device. 7594 */ 7595 void 7596 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7597 { 7598 const struct mlx5_flow_driver_ops *fops; 7599 7600 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7601 fops->destroy_def_policy(dev); 7602 } 7603 7604 /** 7605 * Destroy the default policy table set. 7606 * 7607 * @param[in] dev 7608 * Pointer to Ethernet device. 7609 * 7610 * @return 7611 * 0 on success, -1 otherwise. 7612 */ 7613 int 7614 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7615 { 7616 const struct mlx5_flow_driver_ops *fops; 7617 7618 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7619 return fops->create_def_policy(dev); 7620 } 7621 7622 /** 7623 * Create the needed meter and suffix tables. 7624 * 7625 * @param[in] dev 7626 * Pointer to Ethernet device. 7627 * 7628 * @return 7629 * 0 on success, -1 otherwise. 7630 */ 7631 int 7632 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7633 struct mlx5_flow_meter_info *fm, 7634 uint32_t mtr_idx, 7635 uint8_t domain_bitmap) 7636 { 7637 const struct mlx5_flow_driver_ops *fops; 7638 7639 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7640 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7641 } 7642 7643 /** 7644 * Destroy the meter table set. 7645 * 7646 * @param[in] dev 7647 * Pointer to Ethernet device. 7648 * @param[in] tbl 7649 * Pointer to the meter table set. 7650 */ 7651 void 7652 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7653 struct mlx5_flow_meter_info *fm) 7654 { 7655 const struct mlx5_flow_driver_ops *fops; 7656 7657 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7658 fops->destroy_mtr_tbls(dev, fm); 7659 } 7660 7661 /** 7662 * Destroy the global meter drop table. 7663 * 7664 * @param[in] dev 7665 * Pointer to Ethernet device. 7666 */ 7667 void 7668 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7669 { 7670 const struct mlx5_flow_driver_ops *fops; 7671 7672 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7673 fops->destroy_mtr_drop_tbls(dev); 7674 } 7675 7676 /** 7677 * Destroy the sub policy table with RX queue. 7678 * 7679 * @param[in] dev 7680 * Pointer to Ethernet device. 7681 * @param[in] mtr_policy 7682 * Pointer to meter policy table. 7683 */ 7684 void 7685 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7686 struct mlx5_flow_meter_policy *mtr_policy) 7687 { 7688 const struct mlx5_flow_driver_ops *fops; 7689 7690 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7691 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7692 } 7693 7694 /** 7695 * Allocate the needed aso flow meter id. 7696 * 7697 * @param[in] dev 7698 * Pointer to Ethernet device. 7699 * 7700 * @return 7701 * Index to aso flow meter on success, NULL otherwise. 7702 */ 7703 uint32_t 7704 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7705 { 7706 const struct mlx5_flow_driver_ops *fops; 7707 7708 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7709 return fops->create_meter(dev); 7710 } 7711 7712 /** 7713 * Free the aso flow meter id. 7714 * 7715 * @param[in] dev 7716 * Pointer to Ethernet device. 7717 * @param[in] mtr_idx 7718 * Index to aso flow meter to be free. 7719 * 7720 * @return 7721 * 0 on success. 7722 */ 7723 void 7724 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7725 { 7726 const struct mlx5_flow_driver_ops *fops; 7727 7728 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7729 fops->free_meter(dev, mtr_idx); 7730 } 7731 7732 /** 7733 * Allocate a counter. 7734 * 7735 * @param[in] dev 7736 * Pointer to Ethernet device structure. 7737 * 7738 * @return 7739 * Index to allocated counter on success, 0 otherwise. 7740 */ 7741 uint32_t 7742 mlx5_counter_alloc(struct rte_eth_dev *dev) 7743 { 7744 const struct mlx5_flow_driver_ops *fops; 7745 struct rte_flow_attr attr = { .transfer = 0 }; 7746 7747 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7748 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7749 return fops->counter_alloc(dev); 7750 } 7751 DRV_LOG(ERR, 7752 "port %u counter allocate is not supported.", 7753 dev->data->port_id); 7754 return 0; 7755 } 7756 7757 /** 7758 * Free a counter. 7759 * 7760 * @param[in] dev 7761 * Pointer to Ethernet device structure. 7762 * @param[in] cnt 7763 * Index to counter to be free. 7764 */ 7765 void 7766 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7767 { 7768 const struct mlx5_flow_driver_ops *fops; 7769 struct rte_flow_attr attr = { .transfer = 0 }; 7770 7771 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7772 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7773 fops->counter_free(dev, cnt); 7774 return; 7775 } 7776 DRV_LOG(ERR, 7777 "port %u counter free is not supported.", 7778 dev->data->port_id); 7779 } 7780 7781 /** 7782 * Query counter statistics. 7783 * 7784 * @param[in] dev 7785 * Pointer to Ethernet device structure. 7786 * @param[in] cnt 7787 * Index to counter to query. 7788 * @param[in] clear 7789 * Set to clear counter statistics. 7790 * @param[out] pkts 7791 * The counter hits packets number to save. 7792 * @param[out] bytes 7793 * The counter hits bytes number to save. 7794 * 7795 * @return 7796 * 0 on success, a negative errno value otherwise. 7797 */ 7798 int 7799 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7800 bool clear, uint64_t *pkts, uint64_t *bytes) 7801 { 7802 const struct mlx5_flow_driver_ops *fops; 7803 struct rte_flow_attr attr = { .transfer = 0 }; 7804 7805 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7806 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7807 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7808 } 7809 DRV_LOG(ERR, 7810 "port %u counter query is not supported.", 7811 dev->data->port_id); 7812 return -ENOTSUP; 7813 } 7814 7815 /** 7816 * Allocate a new memory for the counter values wrapped by all the needed 7817 * management. 7818 * 7819 * @param[in] sh 7820 * Pointer to mlx5_dev_ctx_shared object. 7821 * 7822 * @return 7823 * 0 on success, a negative errno value otherwise. 7824 */ 7825 static int 7826 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7827 { 7828 struct mlx5_counter_stats_mem_mng *mem_mng; 7829 volatile struct flow_counter_stats *raw_data; 7830 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7831 int size = (sizeof(struct flow_counter_stats) * 7832 MLX5_COUNTERS_PER_POOL + 7833 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7834 sizeof(struct mlx5_counter_stats_mem_mng); 7835 size_t pgsize = rte_mem_page_size(); 7836 uint8_t *mem; 7837 int ret; 7838 int i; 7839 7840 if (pgsize == (size_t)-1) { 7841 DRV_LOG(ERR, "Failed to get mem page size"); 7842 rte_errno = ENOMEM; 7843 return -ENOMEM; 7844 } 7845 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7846 if (!mem) { 7847 rte_errno = ENOMEM; 7848 return -ENOMEM; 7849 } 7850 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7851 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7852 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 7853 sh->cdev->pdn, mem, size, 7854 &mem_mng->wm); 7855 if (ret) { 7856 rte_errno = errno; 7857 mlx5_free(mem); 7858 return -rte_errno; 7859 } 7860 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7861 raw_data = (volatile struct flow_counter_stats *)mem; 7862 for (i = 0; i < raws_n; ++i) { 7863 mem_mng->raws[i].mem_mng = mem_mng; 7864 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7865 } 7866 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7867 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7868 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7869 next); 7870 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7871 sh->cmng.mem_mng = mem_mng; 7872 return 0; 7873 } 7874 7875 /** 7876 * Set the statistic memory to the new counter pool. 7877 * 7878 * @param[in] sh 7879 * Pointer to mlx5_dev_ctx_shared object. 7880 * @param[in] pool 7881 * Pointer to the pool to set the statistic memory. 7882 * 7883 * @return 7884 * 0 on success, a negative errno value otherwise. 7885 */ 7886 static int 7887 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7888 struct mlx5_flow_counter_pool *pool) 7889 { 7890 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7891 /* Resize statistic memory once used out. */ 7892 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7893 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7894 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7895 return -1; 7896 } 7897 rte_spinlock_lock(&pool->sl); 7898 pool->raw = cmng->mem_mng->raws + pool->index % 7899 MLX5_CNT_CONTAINER_RESIZE; 7900 rte_spinlock_unlock(&pool->sl); 7901 pool->raw_hw = NULL; 7902 return 0; 7903 } 7904 7905 #define MLX5_POOL_QUERY_FREQ_US 1000000 7906 7907 /** 7908 * Set the periodic procedure for triggering asynchronous batch queries for all 7909 * the counter pools. 7910 * 7911 * @param[in] sh 7912 * Pointer to mlx5_dev_ctx_shared object. 7913 */ 7914 void 7915 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7916 { 7917 uint32_t pools_n, us; 7918 7919 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7920 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7921 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7922 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7923 sh->cmng.query_thread_on = 0; 7924 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7925 } else { 7926 sh->cmng.query_thread_on = 1; 7927 } 7928 } 7929 7930 /** 7931 * The periodic procedure for triggering asynchronous batch queries for all the 7932 * counter pools. This function is probably called by the host thread. 7933 * 7934 * @param[in] arg 7935 * The parameter for the alarm process. 7936 */ 7937 void 7938 mlx5_flow_query_alarm(void *arg) 7939 { 7940 struct mlx5_dev_ctx_shared *sh = arg; 7941 int ret; 7942 uint16_t pool_index = sh->cmng.pool_index; 7943 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7944 struct mlx5_flow_counter_pool *pool; 7945 uint16_t n_valid; 7946 7947 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7948 goto set_alarm; 7949 rte_spinlock_lock(&cmng->pool_update_sl); 7950 pool = cmng->pools[pool_index]; 7951 n_valid = cmng->n_valid; 7952 rte_spinlock_unlock(&cmng->pool_update_sl); 7953 /* Set the statistic memory to the new created pool. */ 7954 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7955 goto set_alarm; 7956 if (pool->raw_hw) 7957 /* There is a pool query in progress. */ 7958 goto set_alarm; 7959 pool->raw_hw = 7960 LIST_FIRST(&sh->cmng.free_stat_raws); 7961 if (!pool->raw_hw) 7962 /* No free counter statistics raw memory. */ 7963 goto set_alarm; 7964 /* 7965 * Identify the counters released between query trigger and query 7966 * handle more efficiently. The counter released in this gap period 7967 * should wait for a new round of query as the new arrived packets 7968 * will not be taken into account. 7969 */ 7970 pool->query_gen++; 7971 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7972 MLX5_COUNTERS_PER_POOL, 7973 NULL, NULL, 7974 pool->raw_hw->mem_mng->wm.lkey, 7975 (void *)(uintptr_t) 7976 pool->raw_hw->data, 7977 sh->devx_comp, 7978 (uint64_t)(uintptr_t)pool); 7979 if (ret) { 7980 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7981 " %d", pool->min_dcs->id); 7982 pool->raw_hw = NULL; 7983 goto set_alarm; 7984 } 7985 LIST_REMOVE(pool->raw_hw, next); 7986 sh->cmng.pending_queries++; 7987 pool_index++; 7988 if (pool_index >= n_valid) 7989 pool_index = 0; 7990 set_alarm: 7991 sh->cmng.pool_index = pool_index; 7992 mlx5_set_query_alarm(sh); 7993 } 7994 7995 /** 7996 * Check and callback event for new aged flow in the counter pool 7997 * 7998 * @param[in] sh 7999 * Pointer to mlx5_dev_ctx_shared object. 8000 * @param[in] pool 8001 * Pointer to Current counter pool. 8002 */ 8003 static void 8004 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8005 struct mlx5_flow_counter_pool *pool) 8006 { 8007 struct mlx5_priv *priv; 8008 struct mlx5_flow_counter *cnt; 8009 struct mlx5_age_info *age_info; 8010 struct mlx5_age_param *age_param; 8011 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8012 struct mlx5_counter_stats_raw *prev = pool->raw; 8013 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8014 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8015 uint16_t expected = AGE_CANDIDATE; 8016 uint32_t i; 8017 8018 pool->time_of_last_age_check = curr_time; 8019 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8020 cnt = MLX5_POOL_GET_CNT(pool, i); 8021 age_param = MLX5_CNT_TO_AGE(cnt); 8022 if (__atomic_load_n(&age_param->state, 8023 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8024 continue; 8025 if (cur->data[i].hits != prev->data[i].hits) { 8026 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8027 __ATOMIC_RELAXED); 8028 continue; 8029 } 8030 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8031 time_delta, 8032 __ATOMIC_RELAXED) <= age_param->timeout) 8033 continue; 8034 /** 8035 * Hold the lock first, or if between the 8036 * state AGE_TMOUT and tailq operation the 8037 * release happened, the release procedure 8038 * may delete a non-existent tailq node. 8039 */ 8040 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8041 age_info = GET_PORT_AGE_INFO(priv); 8042 rte_spinlock_lock(&age_info->aged_sl); 8043 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8044 AGE_TMOUT, false, 8045 __ATOMIC_RELAXED, 8046 __ATOMIC_RELAXED)) { 8047 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8048 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8049 } 8050 rte_spinlock_unlock(&age_info->aged_sl); 8051 } 8052 mlx5_age_event_prepare(sh); 8053 } 8054 8055 /** 8056 * Handler for the HW respond about ready values from an asynchronous batch 8057 * query. This function is probably called by the host thread. 8058 * 8059 * @param[in] sh 8060 * The pointer to the shared device context. 8061 * @param[in] async_id 8062 * The Devx async ID. 8063 * @param[in] status 8064 * The status of the completion. 8065 */ 8066 void 8067 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8068 uint64_t async_id, int status) 8069 { 8070 struct mlx5_flow_counter_pool *pool = 8071 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8072 struct mlx5_counter_stats_raw *raw_to_free; 8073 uint8_t query_gen = pool->query_gen ^ 1; 8074 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8075 enum mlx5_counter_type cnt_type = 8076 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8077 MLX5_COUNTER_TYPE_ORIGIN; 8078 8079 if (unlikely(status)) { 8080 raw_to_free = pool->raw_hw; 8081 } else { 8082 raw_to_free = pool->raw; 8083 if (pool->is_aged) 8084 mlx5_flow_aging_check(sh, pool); 8085 rte_spinlock_lock(&pool->sl); 8086 pool->raw = pool->raw_hw; 8087 rte_spinlock_unlock(&pool->sl); 8088 /* Be sure the new raw counters data is updated in memory. */ 8089 rte_io_wmb(); 8090 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8091 rte_spinlock_lock(&cmng->csl[cnt_type]); 8092 TAILQ_CONCAT(&cmng->counters[cnt_type], 8093 &pool->counters[query_gen], next); 8094 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8095 } 8096 } 8097 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8098 pool->raw_hw = NULL; 8099 sh->cmng.pending_queries--; 8100 } 8101 8102 static int 8103 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8104 const struct flow_grp_info *grp_info, 8105 struct rte_flow_error *error) 8106 { 8107 if (grp_info->transfer && grp_info->external && 8108 grp_info->fdb_def_rule) { 8109 if (group == UINT32_MAX) 8110 return rte_flow_error_set 8111 (error, EINVAL, 8112 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8113 NULL, 8114 "group index not supported"); 8115 *table = group + 1; 8116 } else { 8117 *table = group; 8118 } 8119 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8120 return 0; 8121 } 8122 8123 /** 8124 * Translate the rte_flow group index to HW table value. 8125 * 8126 * If tunnel offload is disabled, all group ids converted to flow table 8127 * id using the standard method. 8128 * If tunnel offload is enabled, group id can be converted using the 8129 * standard or tunnel conversion method. Group conversion method 8130 * selection depends on flags in `grp_info` parameter: 8131 * - Internal (grp_info.external == 0) groups conversion uses the 8132 * standard method. 8133 * - Group ids in JUMP action converted with the tunnel conversion. 8134 * - Group id in rule attribute conversion depends on a rule type and 8135 * group id value: 8136 * ** non zero group attributes converted with the tunnel method 8137 * ** zero group attribute in non-tunnel rule is converted using the 8138 * standard method - there's only one root table 8139 * ** zero group attribute in steer tunnel rule is converted with the 8140 * standard method - single root table 8141 * ** zero group attribute in match tunnel rule is a special OvS 8142 * case: that value is used for portability reasons. That group 8143 * id is converted with the tunnel conversion method. 8144 * 8145 * @param[in] dev 8146 * Port device 8147 * @param[in] tunnel 8148 * PMD tunnel offload object 8149 * @param[in] group 8150 * rte_flow group index value. 8151 * @param[out] table 8152 * HW table value. 8153 * @param[in] grp_info 8154 * flags used for conversion 8155 * @param[out] error 8156 * Pointer to error structure. 8157 * 8158 * @return 8159 * 0 on success, a negative errno value otherwise and rte_errno is set. 8160 */ 8161 int 8162 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8163 const struct mlx5_flow_tunnel *tunnel, 8164 uint32_t group, uint32_t *table, 8165 const struct flow_grp_info *grp_info, 8166 struct rte_flow_error *error) 8167 { 8168 int ret; 8169 bool standard_translation; 8170 8171 if (!grp_info->skip_scale && grp_info->external && 8172 group < MLX5_MAX_TABLES_EXTERNAL) 8173 group *= MLX5_FLOW_TABLE_FACTOR; 8174 if (is_tunnel_offload_active(dev)) { 8175 standard_translation = !grp_info->external || 8176 grp_info->std_tbl_fix; 8177 } else { 8178 standard_translation = true; 8179 } 8180 DRV_LOG(DEBUG, 8181 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8182 dev->data->port_id, group, grp_info->transfer, 8183 grp_info->external, grp_info->fdb_def_rule, 8184 standard_translation ? "STANDARD" : "TUNNEL"); 8185 if (standard_translation) 8186 ret = flow_group_to_table(dev->data->port_id, group, table, 8187 grp_info, error); 8188 else 8189 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8190 table, error); 8191 8192 return ret; 8193 } 8194 8195 /** 8196 * Discover availability of metadata reg_c's. 8197 * 8198 * Iteratively use test flows to check availability. 8199 * 8200 * @param[in] dev 8201 * Pointer to the Ethernet device structure. 8202 * 8203 * @return 8204 * 0 on success, a negative errno value otherwise and rte_errno is set. 8205 */ 8206 int 8207 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8208 { 8209 struct mlx5_priv *priv = dev->data->dev_private; 8210 enum modify_reg idx; 8211 int n = 0; 8212 8213 /* reg_c[0] and reg_c[1] are reserved. */ 8214 priv->sh->flow_mreg_c[n++] = REG_C_0; 8215 priv->sh->flow_mreg_c[n++] = REG_C_1; 8216 /* Discover availability of other reg_c's. */ 8217 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 8218 struct rte_flow_attr attr = { 8219 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 8220 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8221 .ingress = 1, 8222 }; 8223 struct rte_flow_item items[] = { 8224 [0] = { 8225 .type = RTE_FLOW_ITEM_TYPE_END, 8226 }, 8227 }; 8228 struct rte_flow_action actions[] = { 8229 [0] = { 8230 .type = (enum rte_flow_action_type) 8231 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8232 .conf = &(struct mlx5_flow_action_copy_mreg){ 8233 .src = REG_C_1, 8234 .dst = idx, 8235 }, 8236 }, 8237 [1] = { 8238 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8239 .conf = &(struct rte_flow_action_jump){ 8240 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8241 }, 8242 }, 8243 [2] = { 8244 .type = RTE_FLOW_ACTION_TYPE_END, 8245 }, 8246 }; 8247 uint32_t flow_idx; 8248 struct rte_flow *flow; 8249 struct rte_flow_error error; 8250 8251 if (!priv->config.dv_flow_en) 8252 break; 8253 /* Create internal flow, validation skips copy action. */ 8254 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8255 items, actions, false, &error); 8256 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8257 flow_idx); 8258 if (!flow) 8259 continue; 8260 priv->sh->flow_mreg_c[n++] = idx; 8261 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8262 } 8263 for (; n < MLX5_MREG_C_NUM; ++n) 8264 priv->sh->flow_mreg_c[n] = REG_NON; 8265 priv->sh->metadata_regc_check_flag = 1; 8266 return 0; 8267 } 8268 8269 int 8270 save_dump_file(const uint8_t *data, uint32_t size, 8271 uint32_t type, uint64_t id, void *arg, FILE *file) 8272 { 8273 char line[BUF_SIZE]; 8274 uint32_t out = 0; 8275 uint32_t k; 8276 uint32_t actions_num; 8277 struct rte_flow_query_count *count; 8278 8279 memset(line, 0, BUF_SIZE); 8280 switch (type) { 8281 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8282 actions_num = *(uint32_t *)(arg); 8283 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 8284 type, id, actions_num); 8285 break; 8286 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8287 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 8288 type, id); 8289 break; 8290 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8291 count = (struct rte_flow_query_count *)arg; 8292 fprintf(file, 8293 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 8294 type, id, count->hits, count->bytes); 8295 return 0; 8296 default: 8297 return -1; 8298 } 8299 8300 for (k = 0; k < size; k++) { 8301 /* Make sure we do not overrun the line buffer length. */ 8302 if (out >= BUF_SIZE - 4) { 8303 line[out] = '\0'; 8304 break; 8305 } 8306 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8307 (data[k]) & 0xff); 8308 } 8309 fprintf(file, "%s\n", line); 8310 return 0; 8311 } 8312 8313 int 8314 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8315 struct rte_flow_query_count *count, struct rte_flow_error *error) 8316 { 8317 struct rte_flow_action action[2]; 8318 enum mlx5_flow_drv_type ftype; 8319 const struct mlx5_flow_driver_ops *fops; 8320 8321 if (!flow) { 8322 return rte_flow_error_set(error, ENOENT, 8323 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8324 NULL, 8325 "invalid flow handle"); 8326 } 8327 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8328 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8329 if (flow->counter) { 8330 memset(count, 0, sizeof(struct rte_flow_query_count)); 8331 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8332 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8333 ftype < MLX5_FLOW_TYPE_MAX); 8334 fops = flow_get_drv_ops(ftype); 8335 return fops->query(dev, flow, action, count, error); 8336 } 8337 return -1; 8338 } 8339 8340 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8341 /** 8342 * Dump flow ipool data to file 8343 * 8344 * @param[in] dev 8345 * The pointer to Ethernet device. 8346 * @param[in] file 8347 * A pointer to a file for output. 8348 * @param[out] error 8349 * Perform verbose error reporting if not NULL. PMDs initialize this 8350 * structure in case of error only. 8351 * @return 8352 * 0 on success, a negative value otherwise. 8353 */ 8354 int 8355 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8356 struct rte_flow *flow, FILE *file, 8357 struct rte_flow_error *error) 8358 { 8359 struct mlx5_priv *priv = dev->data->dev_private; 8360 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8361 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8362 uint32_t handle_idx; 8363 struct mlx5_flow_handle *dh; 8364 struct rte_flow_query_count count; 8365 uint32_t actions_num; 8366 const uint8_t *data; 8367 size_t size; 8368 uint64_t id; 8369 uint32_t type; 8370 void *action = NULL; 8371 8372 if (!flow) { 8373 return rte_flow_error_set(error, ENOENT, 8374 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8375 NULL, 8376 "invalid flow handle"); 8377 } 8378 handle_idx = flow->dev_handles; 8379 while (handle_idx) { 8380 dh = mlx5_ipool_get(priv->sh->ipool 8381 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8382 if (!dh) 8383 continue; 8384 handle_idx = dh->next.next; 8385 8386 /* query counter */ 8387 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8388 flow_dv_query_count_ptr(dev, flow->counter, 8389 &action, error); 8390 if (action) { 8391 id = (uint64_t)(uintptr_t)action; 8392 if (!mlx5_flow_query_counter(dev, flow, &count, error)) 8393 save_dump_file(NULL, 0, type, 8394 id, (void *)&count, file); 8395 } 8396 /* Get modify_hdr and encap_decap buf from ipools. */ 8397 encap_decap = NULL; 8398 modify_hdr = dh->dvh.modify_hdr; 8399 8400 if (dh->dvh.rix_encap_decap) { 8401 encap_decap = mlx5_ipool_get(priv->sh->ipool 8402 [MLX5_IPOOL_DECAP_ENCAP], 8403 dh->dvh.rix_encap_decap); 8404 } 8405 if (modify_hdr) { 8406 data = (const uint8_t *)modify_hdr->actions; 8407 size = (size_t)(modify_hdr->actions_num) * 8; 8408 id = (uint64_t)(uintptr_t)modify_hdr->action; 8409 actions_num = modify_hdr->actions_num; 8410 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8411 save_dump_file(data, size, type, id, 8412 (void *)(&actions_num), file); 8413 } 8414 if (encap_decap) { 8415 data = encap_decap->buf; 8416 size = encap_decap->size; 8417 id = (uint64_t)(uintptr_t)encap_decap->action; 8418 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8419 save_dump_file(data, size, type, 8420 id, NULL, file); 8421 } 8422 } 8423 return 0; 8424 } 8425 8426 /** 8427 * Dump all flow's encap_decap/modify_hdr/counter data to file 8428 * 8429 * @param[in] dev 8430 * The pointer to Ethernet device. 8431 * @param[in] file 8432 * A pointer to a file for output. 8433 * @param[out] error 8434 * Perform verbose error reporting if not NULL. PMDs initialize this 8435 * structure in case of error only. 8436 * @return 8437 * 0 on success, a negative value otherwise. 8438 */ 8439 static int 8440 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 8441 FILE *file, struct rte_flow_error *error) 8442 { 8443 struct mlx5_priv *priv = dev->data->dev_private; 8444 struct mlx5_dev_ctx_shared *sh = priv->sh; 8445 struct mlx5_hlist *h; 8446 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8447 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8448 struct rte_flow_query_count count; 8449 uint32_t actions_num; 8450 const uint8_t *data; 8451 size_t size; 8452 uint64_t id; 8453 uint32_t type; 8454 uint32_t i; 8455 uint32_t j; 8456 struct mlx5_list_inconst *l_inconst; 8457 struct mlx5_list_entry *e; 8458 int lcore_index; 8459 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 8460 uint32_t max; 8461 void *action; 8462 8463 /* encap_decap hlist is lcore_share, get global core cache. */ 8464 i = MLX5_LIST_GLOBAL; 8465 h = sh->encaps_decaps; 8466 if (h) { 8467 for (j = 0; j <= h->mask; j++) { 8468 l_inconst = &h->buckets[j].l; 8469 if (!l_inconst || !l_inconst->cache[i]) 8470 continue; 8471 8472 e = LIST_FIRST(&l_inconst->cache[i]->h); 8473 while (e) { 8474 encap_decap = 8475 (struct mlx5_flow_dv_encap_decap_resource *)e; 8476 data = encap_decap->buf; 8477 size = encap_decap->size; 8478 id = (uint64_t)(uintptr_t)encap_decap->action; 8479 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8480 save_dump_file(data, size, type, 8481 id, NULL, file); 8482 e = LIST_NEXT(e, next); 8483 } 8484 } 8485 } 8486 8487 /* get modify_hdr */ 8488 h = sh->modify_cmds; 8489 if (h) { 8490 lcore_index = rte_lcore_index(rte_lcore_id()); 8491 if (unlikely(lcore_index == -1)) { 8492 lcore_index = MLX5_LIST_NLCORE; 8493 rte_spinlock_lock(&h->l_const.lcore_lock); 8494 } 8495 i = lcore_index; 8496 8497 for (j = 0; j <= h->mask; j++) { 8498 l_inconst = &h->buckets[j].l; 8499 if (!l_inconst || !l_inconst->cache[i]) 8500 continue; 8501 8502 e = LIST_FIRST(&l_inconst->cache[i]->h); 8503 while (e) { 8504 modify_hdr = 8505 (struct mlx5_flow_dv_modify_hdr_resource *)e; 8506 data = (const uint8_t *)modify_hdr->actions; 8507 size = (size_t)(modify_hdr->actions_num) * 8; 8508 actions_num = modify_hdr->actions_num; 8509 id = (uint64_t)(uintptr_t)modify_hdr->action; 8510 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8511 save_dump_file(data, size, type, id, 8512 (void *)(&actions_num), file); 8513 e = LIST_NEXT(e, next); 8514 } 8515 } 8516 8517 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 8518 rte_spinlock_unlock(&h->l_const.lcore_lock); 8519 } 8520 8521 /* get counter */ 8522 MLX5_ASSERT(cmng->n_valid <= cmng->n); 8523 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 8524 for (j = 1; j <= max; j++) { 8525 action = NULL; 8526 flow_dv_query_count_ptr(dev, j, &action, error); 8527 if (action) { 8528 if (!flow_dv_query_count(dev, j, &count, error)) { 8529 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8530 id = (uint64_t)(uintptr_t)action; 8531 save_dump_file(NULL, 0, type, 8532 id, (void *)&count, file); 8533 } 8534 } 8535 } 8536 return 0; 8537 } 8538 #endif 8539 8540 /** 8541 * Dump flow raw hw data to file 8542 * 8543 * @param[in] dev 8544 * The pointer to Ethernet device. 8545 * @param[in] file 8546 * A pointer to a file for output. 8547 * @param[out] error 8548 * Perform verbose error reporting if not NULL. PMDs initialize this 8549 * structure in case of error only. 8550 * @return 8551 * 0 on success, a negative value otherwise. 8552 */ 8553 int 8554 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8555 FILE *file, 8556 struct rte_flow_error *error __rte_unused) 8557 { 8558 struct mlx5_priv *priv = dev->data->dev_private; 8559 struct mlx5_dev_ctx_shared *sh = priv->sh; 8560 uint32_t handle_idx; 8561 int ret; 8562 struct mlx5_flow_handle *dh; 8563 struct rte_flow *flow; 8564 8565 if (!priv->config.dv_flow_en) { 8566 if (fputs("device dv flow disabled\n", file) <= 0) 8567 return -errno; 8568 return -ENOTSUP; 8569 } 8570 8571 /* dump all */ 8572 if (!flow_idx) { 8573 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8574 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 8575 return -EINVAL; 8576 #endif 8577 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8578 sh->rx_domain, 8579 sh->tx_domain, file); 8580 } 8581 /* dump one */ 8582 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8583 (uintptr_t)(void *)flow_idx); 8584 if (!flow) 8585 return -EINVAL; 8586 8587 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8588 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8589 #endif 8590 handle_idx = flow->dev_handles; 8591 while (handle_idx) { 8592 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8593 handle_idx); 8594 if (!dh) 8595 return -ENOENT; 8596 if (dh->drv_flow) { 8597 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8598 file); 8599 if (ret) 8600 return -ENOENT; 8601 } 8602 handle_idx = dh->next.next; 8603 } 8604 return 0; 8605 } 8606 8607 /** 8608 * Get aged-out flows. 8609 * 8610 * @param[in] dev 8611 * Pointer to the Ethernet device structure. 8612 * @param[in] context 8613 * The address of an array of pointers to the aged-out flows contexts. 8614 * @param[in] nb_countexts 8615 * The length of context array pointers. 8616 * @param[out] error 8617 * Perform verbose error reporting if not NULL. Initialized in case of 8618 * error only. 8619 * 8620 * @return 8621 * how many contexts get in success, otherwise negative errno value. 8622 * if nb_contexts is 0, return the amount of all aged contexts. 8623 * if nb_contexts is not 0 , return the amount of aged flows reported 8624 * in the context array. 8625 */ 8626 int 8627 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8628 uint32_t nb_contexts, struct rte_flow_error *error) 8629 { 8630 const struct mlx5_flow_driver_ops *fops; 8631 struct rte_flow_attr attr = { .transfer = 0 }; 8632 8633 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8634 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8635 return fops->get_aged_flows(dev, contexts, nb_contexts, 8636 error); 8637 } 8638 DRV_LOG(ERR, 8639 "port %u get aged flows is not supported.", 8640 dev->data->port_id); 8641 return -ENOTSUP; 8642 } 8643 8644 /* Wrapper for driver action_validate op callback */ 8645 static int 8646 flow_drv_action_validate(struct rte_eth_dev *dev, 8647 const struct rte_flow_indir_action_conf *conf, 8648 const struct rte_flow_action *action, 8649 const struct mlx5_flow_driver_ops *fops, 8650 struct rte_flow_error *error) 8651 { 8652 static const char err_msg[] = "indirect action validation unsupported"; 8653 8654 if (!fops->action_validate) { 8655 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8656 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8657 NULL, err_msg); 8658 return -rte_errno; 8659 } 8660 return fops->action_validate(dev, conf, action, error); 8661 } 8662 8663 /** 8664 * Destroys the shared action by handle. 8665 * 8666 * @param dev 8667 * Pointer to Ethernet device structure. 8668 * @param[in] handle 8669 * Handle for the indirect action object to be destroyed. 8670 * @param[out] error 8671 * Perform verbose error reporting if not NULL. PMDs initialize this 8672 * structure in case of error only. 8673 * 8674 * @return 8675 * 0 on success, a negative errno value otherwise and rte_errno is set. 8676 * 8677 * @note: wrapper for driver action_create op callback. 8678 */ 8679 static int 8680 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8681 struct rte_flow_action_handle *handle, 8682 struct rte_flow_error *error) 8683 { 8684 static const char err_msg[] = "indirect action destruction unsupported"; 8685 struct rte_flow_attr attr = { .transfer = 0 }; 8686 const struct mlx5_flow_driver_ops *fops = 8687 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8688 8689 if (!fops->action_destroy) { 8690 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8691 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8692 NULL, err_msg); 8693 return -rte_errno; 8694 } 8695 return fops->action_destroy(dev, handle, error); 8696 } 8697 8698 /* Wrapper for driver action_destroy op callback */ 8699 static int 8700 flow_drv_action_update(struct rte_eth_dev *dev, 8701 struct rte_flow_action_handle *handle, 8702 const void *update, 8703 const struct mlx5_flow_driver_ops *fops, 8704 struct rte_flow_error *error) 8705 { 8706 static const char err_msg[] = "indirect action update unsupported"; 8707 8708 if (!fops->action_update) { 8709 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8710 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8711 NULL, err_msg); 8712 return -rte_errno; 8713 } 8714 return fops->action_update(dev, handle, update, error); 8715 } 8716 8717 /* Wrapper for driver action_destroy op callback */ 8718 static int 8719 flow_drv_action_query(struct rte_eth_dev *dev, 8720 const struct rte_flow_action_handle *handle, 8721 void *data, 8722 const struct mlx5_flow_driver_ops *fops, 8723 struct rte_flow_error *error) 8724 { 8725 static const char err_msg[] = "indirect action query unsupported"; 8726 8727 if (!fops->action_query) { 8728 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8729 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8730 NULL, err_msg); 8731 return -rte_errno; 8732 } 8733 return fops->action_query(dev, handle, data, error); 8734 } 8735 8736 /** 8737 * Create indirect action for reuse in multiple flow rules. 8738 * 8739 * @param dev 8740 * Pointer to Ethernet device structure. 8741 * @param conf 8742 * Pointer to indirect action object configuration. 8743 * @param[in] action 8744 * Action configuration for indirect action object creation. 8745 * @param[out] error 8746 * Perform verbose error reporting if not NULL. PMDs initialize this 8747 * structure in case of error only. 8748 * @return 8749 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8750 */ 8751 static struct rte_flow_action_handle * 8752 mlx5_action_handle_create(struct rte_eth_dev *dev, 8753 const struct rte_flow_indir_action_conf *conf, 8754 const struct rte_flow_action *action, 8755 struct rte_flow_error *error) 8756 { 8757 static const char err_msg[] = "indirect action creation unsupported"; 8758 struct rte_flow_attr attr = { .transfer = 0 }; 8759 const struct mlx5_flow_driver_ops *fops = 8760 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8761 8762 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8763 return NULL; 8764 if (!fops->action_create) { 8765 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8766 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8767 NULL, err_msg); 8768 return NULL; 8769 } 8770 return fops->action_create(dev, conf, action, error); 8771 } 8772 8773 /** 8774 * Updates inplace the indirect action configuration pointed by *handle* 8775 * with the configuration provided as *update* argument. 8776 * The update of the indirect action configuration effects all flow rules 8777 * reusing the action via handle. 8778 * 8779 * @param dev 8780 * Pointer to Ethernet device structure. 8781 * @param[in] handle 8782 * Handle for the indirect action to be updated. 8783 * @param[in] update 8784 * Action specification used to modify the action pointed by handle. 8785 * *update* could be of same type with the action pointed by the *handle* 8786 * handle argument, or some other structures like a wrapper, depending on 8787 * the indirect action type. 8788 * @param[out] error 8789 * Perform verbose error reporting if not NULL. PMDs initialize this 8790 * structure in case of error only. 8791 * 8792 * @return 8793 * 0 on success, a negative errno value otherwise and rte_errno is set. 8794 */ 8795 static int 8796 mlx5_action_handle_update(struct rte_eth_dev *dev, 8797 struct rte_flow_action_handle *handle, 8798 const void *update, 8799 struct rte_flow_error *error) 8800 { 8801 struct rte_flow_attr attr = { .transfer = 0 }; 8802 const struct mlx5_flow_driver_ops *fops = 8803 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8804 int ret; 8805 8806 ret = flow_drv_action_validate(dev, NULL, 8807 (const struct rte_flow_action *)update, fops, error); 8808 if (ret) 8809 return ret; 8810 return flow_drv_action_update(dev, handle, update, fops, 8811 error); 8812 } 8813 8814 /** 8815 * Query the indirect action by handle. 8816 * 8817 * This function allows retrieving action-specific data such as counters. 8818 * Data is gathered by special action which may be present/referenced in 8819 * more than one flow rule definition. 8820 * 8821 * see @RTE_FLOW_ACTION_TYPE_COUNT 8822 * 8823 * @param dev 8824 * Pointer to Ethernet device structure. 8825 * @param[in] handle 8826 * Handle for the indirect action to query. 8827 * @param[in, out] data 8828 * Pointer to storage for the associated query data type. 8829 * @param[out] error 8830 * Perform verbose error reporting if not NULL. PMDs initialize this 8831 * structure in case of error only. 8832 * 8833 * @return 8834 * 0 on success, a negative errno value otherwise and rte_errno is set. 8835 */ 8836 static int 8837 mlx5_action_handle_query(struct rte_eth_dev *dev, 8838 const struct rte_flow_action_handle *handle, 8839 void *data, 8840 struct rte_flow_error *error) 8841 { 8842 struct rte_flow_attr attr = { .transfer = 0 }; 8843 const struct mlx5_flow_driver_ops *fops = 8844 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8845 8846 return flow_drv_action_query(dev, handle, data, fops, error); 8847 } 8848 8849 /** 8850 * Destroy all indirect actions (shared RSS). 8851 * 8852 * @param dev 8853 * Pointer to Ethernet device. 8854 * 8855 * @return 8856 * 0 on success, a negative errno value otherwise and rte_errno is set. 8857 */ 8858 int 8859 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8860 { 8861 struct rte_flow_error error; 8862 struct mlx5_priv *priv = dev->data->dev_private; 8863 struct mlx5_shared_action_rss *shared_rss; 8864 int ret = 0; 8865 uint32_t idx; 8866 8867 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8868 priv->rss_shared_actions, idx, shared_rss, next) { 8869 ret |= mlx5_action_handle_destroy(dev, 8870 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8871 } 8872 return ret; 8873 } 8874 8875 /** 8876 * Validate existing indirect actions against current device configuration 8877 * and attach them to device resources. 8878 * 8879 * @param dev 8880 * Pointer to Ethernet device. 8881 * 8882 * @return 8883 * 0 on success, a negative errno value otherwise and rte_errno is set. 8884 */ 8885 int 8886 mlx5_action_handle_attach(struct rte_eth_dev *dev) 8887 { 8888 struct mlx5_priv *priv = dev->data->dev_private; 8889 struct mlx5_indexed_pool *ipool = 8890 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 8891 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 8892 int ret = 0; 8893 uint32_t idx; 8894 8895 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8896 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8897 const char *message; 8898 uint32_t queue_idx; 8899 8900 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 8901 ind_tbl->queues_n, 8902 &message, &queue_idx); 8903 if (ret != 0) { 8904 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 8905 dev->data->port_id, ind_tbl->queues[queue_idx], 8906 message); 8907 break; 8908 } 8909 } 8910 if (ret != 0) 8911 return ret; 8912 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8913 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8914 8915 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 8916 if (ret != 0) { 8917 DRV_LOG(ERR, "Port %u could not attach " 8918 "indirection table obj %p", 8919 dev->data->port_id, (void *)ind_tbl); 8920 goto error; 8921 } 8922 } 8923 return 0; 8924 error: 8925 shared_rss_last = shared_rss; 8926 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8927 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8928 8929 if (shared_rss == shared_rss_last) 8930 break; 8931 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 8932 DRV_LOG(CRIT, "Port %u could not detach " 8933 "indirection table obj %p on rollback", 8934 dev->data->port_id, (void *)ind_tbl); 8935 } 8936 return ret; 8937 } 8938 8939 /** 8940 * Detach indirect actions of the device from its resources. 8941 * 8942 * @param dev 8943 * Pointer to Ethernet device. 8944 * 8945 * @return 8946 * 0 on success, a negative errno value otherwise and rte_errno is set. 8947 */ 8948 int 8949 mlx5_action_handle_detach(struct rte_eth_dev *dev) 8950 { 8951 struct mlx5_priv *priv = dev->data->dev_private; 8952 struct mlx5_indexed_pool *ipool = 8953 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 8954 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 8955 int ret = 0; 8956 uint32_t idx; 8957 8958 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8959 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8960 8961 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 8962 if (ret != 0) { 8963 DRV_LOG(ERR, "Port %u could not detach " 8964 "indirection table obj %p", 8965 dev->data->port_id, (void *)ind_tbl); 8966 goto error; 8967 } 8968 } 8969 return 0; 8970 error: 8971 shared_rss_last = shared_rss; 8972 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8973 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8974 8975 if (shared_rss == shared_rss_last) 8976 break; 8977 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 8978 DRV_LOG(CRIT, "Port %u could not attach " 8979 "indirection table obj %p on rollback", 8980 dev->data->port_id, (void *)ind_tbl); 8981 } 8982 return ret; 8983 } 8984 8985 #ifndef HAVE_MLX5DV_DR 8986 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8987 #else 8988 #define MLX5_DOMAIN_SYNC_FLOW \ 8989 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8990 #endif 8991 8992 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8993 { 8994 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8995 const struct mlx5_flow_driver_ops *fops; 8996 int ret; 8997 struct rte_flow_attr attr = { .transfer = 0 }; 8998 8999 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9000 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9001 if (ret > 0) 9002 ret = -ret; 9003 return ret; 9004 } 9005 9006 const struct mlx5_flow_tunnel * 9007 mlx5_get_tof(const struct rte_flow_item *item, 9008 const struct rte_flow_action *action, 9009 enum mlx5_tof_rule_type *rule_type) 9010 { 9011 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9012 if (item->type == (typeof(item->type)) 9013 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9014 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9015 return flow_items_to_tunnel(item); 9016 } 9017 } 9018 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9019 if (action->type == (typeof(action->type)) 9020 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9021 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9022 return flow_actions_to_tunnel(action); 9023 } 9024 } 9025 return NULL; 9026 } 9027 9028 /** 9029 * tunnel offload functionality is defined for DV environment only 9030 */ 9031 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9032 __extension__ 9033 union tunnel_offload_mark { 9034 uint32_t val; 9035 struct { 9036 uint32_t app_reserve:8; 9037 uint32_t table_id:15; 9038 uint32_t transfer:1; 9039 uint32_t _unused_:8; 9040 }; 9041 }; 9042 9043 static bool 9044 mlx5_access_tunnel_offload_db 9045 (struct rte_eth_dev *dev, 9046 bool (*match)(struct rte_eth_dev *, 9047 struct mlx5_flow_tunnel *, const void *), 9048 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9049 void (*miss)(struct rte_eth_dev *, void *), 9050 void *ctx, bool lock_op); 9051 9052 static int 9053 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9054 struct rte_flow *flow, 9055 const struct rte_flow_attr *attr, 9056 const struct rte_flow_action *app_actions, 9057 uint32_t flow_idx, 9058 const struct mlx5_flow_tunnel *tunnel, 9059 struct tunnel_default_miss_ctx *ctx, 9060 struct rte_flow_error *error) 9061 { 9062 struct mlx5_priv *priv = dev->data->dev_private; 9063 struct mlx5_flow *dev_flow; 9064 struct rte_flow_attr miss_attr = *attr; 9065 const struct rte_flow_item miss_items[2] = { 9066 { 9067 .type = RTE_FLOW_ITEM_TYPE_ETH, 9068 .spec = NULL, 9069 .last = NULL, 9070 .mask = NULL 9071 }, 9072 { 9073 .type = RTE_FLOW_ITEM_TYPE_END, 9074 .spec = NULL, 9075 .last = NULL, 9076 .mask = NULL 9077 } 9078 }; 9079 union tunnel_offload_mark mark_id; 9080 struct rte_flow_action_mark miss_mark; 9081 struct rte_flow_action miss_actions[3] = { 9082 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9083 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9084 }; 9085 const struct rte_flow_action_jump *jump_data; 9086 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9087 struct flow_grp_info grp_info = { 9088 .external = 1, 9089 .transfer = attr->transfer, 9090 .fdb_def_rule = !!priv->fdb_def_rule, 9091 .std_tbl_fix = 0, 9092 }; 9093 int ret; 9094 9095 if (!attr->transfer) { 9096 uint32_t q_size; 9097 9098 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9099 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9100 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9101 0, SOCKET_ID_ANY); 9102 if (!ctx->queue) 9103 return rte_flow_error_set 9104 (error, ENOMEM, 9105 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9106 NULL, "invalid default miss RSS"); 9107 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9108 ctx->action_rss.level = 0, 9109 ctx->action_rss.types = priv->rss_conf.rss_hf, 9110 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9111 ctx->action_rss.queue_num = priv->reta_idx_n, 9112 ctx->action_rss.key = priv->rss_conf.rss_key, 9113 ctx->action_rss.queue = ctx->queue; 9114 if (!priv->reta_idx_n || !priv->rxqs_n) 9115 return rte_flow_error_set 9116 (error, EINVAL, 9117 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9118 NULL, "invalid port configuration"); 9119 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9120 ctx->action_rss.types = 0; 9121 for (i = 0; i != priv->reta_idx_n; ++i) 9122 ctx->queue[i] = (*priv->reta_idx)[i]; 9123 } else { 9124 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9125 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9126 } 9127 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9128 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9129 jump_data = app_actions->conf; 9130 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9131 miss_attr.group = jump_data->group; 9132 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9133 &flow_table, &grp_info, error); 9134 if (ret) 9135 return rte_flow_error_set(error, EINVAL, 9136 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9137 NULL, "invalid tunnel id"); 9138 mark_id.app_reserve = 0; 9139 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9140 mark_id.transfer = !!attr->transfer; 9141 mark_id._unused_ = 0; 9142 miss_mark.id = mark_id.val; 9143 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9144 miss_items, miss_actions, flow_idx, error); 9145 if (!dev_flow) 9146 return -rte_errno; 9147 dev_flow->flow = flow; 9148 dev_flow->external = true; 9149 dev_flow->tunnel = tunnel; 9150 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9151 /* Subflow object was created, we must include one in the list. */ 9152 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9153 dev_flow->handle, next); 9154 DRV_LOG(DEBUG, 9155 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9156 dev->data->port_id, tunnel->app_tunnel.type, 9157 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9158 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9159 miss_actions, error); 9160 if (!ret) 9161 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9162 error); 9163 9164 return ret; 9165 } 9166 9167 static const struct mlx5_flow_tbl_data_entry * 9168 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9169 { 9170 struct mlx5_priv *priv = dev->data->dev_private; 9171 struct mlx5_dev_ctx_shared *sh = priv->sh; 9172 struct mlx5_list_entry *he; 9173 union tunnel_offload_mark mbits = { .val = mark }; 9174 union mlx5_flow_tbl_key table_key = { 9175 { 9176 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9177 .id = 0, 9178 .reserved = 0, 9179 .dummy = 0, 9180 .is_fdb = !!mbits.transfer, 9181 .is_egress = 0, 9182 } 9183 }; 9184 struct mlx5_flow_cb_ctx ctx = { 9185 .data = &table_key.v64, 9186 }; 9187 9188 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9189 return he ? 9190 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9191 } 9192 9193 static void 9194 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9195 struct mlx5_list_entry *entry) 9196 { 9197 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9198 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9199 9200 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9201 tunnel_flow_tbl_to_id(tte->flow_table)); 9202 mlx5_free(tte); 9203 } 9204 9205 static int 9206 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9207 struct mlx5_list_entry *entry, void *cb_ctx) 9208 { 9209 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9210 union tunnel_tbl_key tbl = { 9211 .val = *(uint64_t *)(ctx->data), 9212 }; 9213 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9214 9215 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9216 } 9217 9218 static struct mlx5_list_entry * 9219 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 9220 { 9221 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9222 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9223 struct tunnel_tbl_entry *tte; 9224 union tunnel_tbl_key tbl = { 9225 .val = *(uint64_t *)(ctx->data), 9226 }; 9227 9228 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 9229 sizeof(*tte), 0, 9230 SOCKET_ID_ANY); 9231 if (!tte) 9232 goto err; 9233 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9234 &tte->flow_table); 9235 if (tte->flow_table >= MLX5_MAX_TABLES) { 9236 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 9237 tte->flow_table); 9238 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9239 tte->flow_table); 9240 goto err; 9241 } else if (!tte->flow_table) { 9242 goto err; 9243 } 9244 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 9245 tte->tunnel_id = tbl.tunnel_id; 9246 tte->group = tbl.group; 9247 return &tte->hash; 9248 err: 9249 if (tte) 9250 mlx5_free(tte); 9251 return NULL; 9252 } 9253 9254 static struct mlx5_list_entry * 9255 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 9256 struct mlx5_list_entry *oentry, 9257 void *cb_ctx __rte_unused) 9258 { 9259 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 9260 0, SOCKET_ID_ANY); 9261 9262 if (!tte) 9263 return NULL; 9264 memcpy(tte, oentry, sizeof(*tte)); 9265 return &tte->hash; 9266 } 9267 9268 static void 9269 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 9270 struct mlx5_list_entry *entry) 9271 { 9272 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9273 9274 mlx5_free(tte); 9275 } 9276 9277 static uint32_t 9278 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 9279 const struct mlx5_flow_tunnel *tunnel, 9280 uint32_t group, uint32_t *table, 9281 struct rte_flow_error *error) 9282 { 9283 struct mlx5_list_entry *he; 9284 struct tunnel_tbl_entry *tte; 9285 union tunnel_tbl_key key = { 9286 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 9287 .group = group 9288 }; 9289 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9290 struct mlx5_hlist *group_hash; 9291 struct mlx5_flow_cb_ctx ctx = { 9292 .data = &key.val, 9293 }; 9294 9295 group_hash = tunnel ? tunnel->groups : thub->groups; 9296 he = mlx5_hlist_register(group_hash, key.val, &ctx); 9297 if (!he) 9298 return rte_flow_error_set(error, EINVAL, 9299 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9300 NULL, 9301 "tunnel group index not supported"); 9302 tte = container_of(he, typeof(*tte), hash); 9303 *table = tte->flow_table; 9304 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 9305 dev->data->port_id, key.tunnel_id, group, *table); 9306 return 0; 9307 } 9308 9309 static void 9310 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 9311 struct mlx5_flow_tunnel *tunnel) 9312 { 9313 struct mlx5_priv *priv = dev->data->dev_private; 9314 struct mlx5_indexed_pool *ipool; 9315 9316 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 9317 dev->data->port_id, tunnel->tunnel_id); 9318 LIST_REMOVE(tunnel, chain); 9319 mlx5_hlist_destroy(tunnel->groups); 9320 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9321 mlx5_ipool_free(ipool, tunnel->tunnel_id); 9322 } 9323 9324 static bool 9325 mlx5_access_tunnel_offload_db 9326 (struct rte_eth_dev *dev, 9327 bool (*match)(struct rte_eth_dev *, 9328 struct mlx5_flow_tunnel *, const void *), 9329 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9330 void (*miss)(struct rte_eth_dev *, void *), 9331 void *ctx, bool lock_op) 9332 { 9333 bool verdict = false; 9334 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9335 struct mlx5_flow_tunnel *tunnel; 9336 9337 rte_spinlock_lock(&thub->sl); 9338 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 9339 verdict = match(dev, tunnel, (const void *)ctx); 9340 if (verdict) 9341 break; 9342 } 9343 if (!lock_op) 9344 rte_spinlock_unlock(&thub->sl); 9345 if (verdict && hit) 9346 hit(dev, tunnel, ctx); 9347 if (!verdict && miss) 9348 miss(dev, ctx); 9349 if (lock_op) 9350 rte_spinlock_unlock(&thub->sl); 9351 9352 return verdict; 9353 } 9354 9355 struct tunnel_db_find_tunnel_id_ctx { 9356 uint32_t tunnel_id; 9357 struct mlx5_flow_tunnel *tunnel; 9358 }; 9359 9360 static bool 9361 find_tunnel_id_match(struct rte_eth_dev *dev, 9362 struct mlx5_flow_tunnel *tunnel, const void *x) 9363 { 9364 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9365 9366 RTE_SET_USED(dev); 9367 return tunnel->tunnel_id == ctx->tunnel_id; 9368 } 9369 9370 static void 9371 find_tunnel_id_hit(struct rte_eth_dev *dev, 9372 struct mlx5_flow_tunnel *tunnel, void *x) 9373 { 9374 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9375 RTE_SET_USED(dev); 9376 ctx->tunnel = tunnel; 9377 } 9378 9379 static struct mlx5_flow_tunnel * 9380 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 9381 { 9382 struct tunnel_db_find_tunnel_id_ctx ctx = { 9383 .tunnel_id = id, 9384 }; 9385 9386 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 9387 find_tunnel_id_hit, NULL, &ctx, true); 9388 9389 return ctx.tunnel; 9390 } 9391 9392 static struct mlx5_flow_tunnel * 9393 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 9394 const struct rte_flow_tunnel *app_tunnel) 9395 { 9396 struct mlx5_priv *priv = dev->data->dev_private; 9397 struct mlx5_indexed_pool *ipool; 9398 struct mlx5_flow_tunnel *tunnel; 9399 uint32_t id; 9400 9401 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9402 tunnel = mlx5_ipool_zmalloc(ipool, &id); 9403 if (!tunnel) 9404 return NULL; 9405 if (id >= MLX5_MAX_TUNNELS) { 9406 mlx5_ipool_free(ipool, id); 9407 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 9408 return NULL; 9409 } 9410 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 9411 priv->sh, 9412 mlx5_flow_tunnel_grp2tbl_create_cb, 9413 mlx5_flow_tunnel_grp2tbl_match_cb, 9414 mlx5_flow_tunnel_grp2tbl_remove_cb, 9415 mlx5_flow_tunnel_grp2tbl_clone_cb, 9416 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9417 if (!tunnel->groups) { 9418 mlx5_ipool_free(ipool, id); 9419 return NULL; 9420 } 9421 /* initiate new PMD tunnel */ 9422 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 9423 tunnel->tunnel_id = id; 9424 tunnel->action.type = (typeof(tunnel->action.type)) 9425 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 9426 tunnel->action.conf = tunnel; 9427 tunnel->item.type = (typeof(tunnel->item.type)) 9428 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 9429 tunnel->item.spec = tunnel; 9430 tunnel->item.last = NULL; 9431 tunnel->item.mask = NULL; 9432 9433 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 9434 dev->data->port_id, tunnel->tunnel_id); 9435 9436 return tunnel; 9437 } 9438 9439 struct tunnel_db_get_tunnel_ctx { 9440 const struct rte_flow_tunnel *app_tunnel; 9441 struct mlx5_flow_tunnel *tunnel; 9442 }; 9443 9444 static bool get_tunnel_match(struct rte_eth_dev *dev, 9445 struct mlx5_flow_tunnel *tunnel, const void *x) 9446 { 9447 const struct tunnel_db_get_tunnel_ctx *ctx = x; 9448 9449 RTE_SET_USED(dev); 9450 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 9451 sizeof(*ctx->app_tunnel)); 9452 } 9453 9454 static void get_tunnel_hit(struct rte_eth_dev *dev, 9455 struct mlx5_flow_tunnel *tunnel, void *x) 9456 { 9457 /* called under tunnel spinlock protection */ 9458 struct tunnel_db_get_tunnel_ctx *ctx = x; 9459 9460 RTE_SET_USED(dev); 9461 tunnel->refctn++; 9462 ctx->tunnel = tunnel; 9463 } 9464 9465 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 9466 { 9467 /* called under tunnel spinlock protection */ 9468 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9469 struct tunnel_db_get_tunnel_ctx *ctx = x; 9470 9471 rte_spinlock_unlock(&thub->sl); 9472 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 9473 rte_spinlock_lock(&thub->sl); 9474 if (ctx->tunnel) { 9475 ctx->tunnel->refctn = 1; 9476 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 9477 } 9478 } 9479 9480 9481 static int 9482 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 9483 const struct rte_flow_tunnel *app_tunnel, 9484 struct mlx5_flow_tunnel **tunnel) 9485 { 9486 struct tunnel_db_get_tunnel_ctx ctx = { 9487 .app_tunnel = app_tunnel, 9488 }; 9489 9490 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 9491 get_tunnel_miss, &ctx, true); 9492 *tunnel = ctx.tunnel; 9493 return ctx.tunnel ? 0 : -ENOMEM; 9494 } 9495 9496 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 9497 { 9498 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 9499 9500 if (!thub) 9501 return; 9502 if (!LIST_EMPTY(&thub->tunnels)) 9503 DRV_LOG(WARNING, "port %u tunnels present", port_id); 9504 mlx5_hlist_destroy(thub->groups); 9505 mlx5_free(thub); 9506 } 9507 9508 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 9509 { 9510 int err; 9511 struct mlx5_flow_tunnel_hub *thub; 9512 9513 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 9514 0, SOCKET_ID_ANY); 9515 if (!thub) 9516 return -ENOMEM; 9517 LIST_INIT(&thub->tunnels); 9518 rte_spinlock_init(&thub->sl); 9519 thub->groups = mlx5_hlist_create("flow groups", 64, 9520 false, true, sh, 9521 mlx5_flow_tunnel_grp2tbl_create_cb, 9522 mlx5_flow_tunnel_grp2tbl_match_cb, 9523 mlx5_flow_tunnel_grp2tbl_remove_cb, 9524 mlx5_flow_tunnel_grp2tbl_clone_cb, 9525 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9526 if (!thub->groups) { 9527 err = -rte_errno; 9528 goto err; 9529 } 9530 sh->tunnel_hub = thub; 9531 9532 return 0; 9533 9534 err: 9535 if (thub->groups) 9536 mlx5_hlist_destroy(thub->groups); 9537 if (thub) 9538 mlx5_free(thub); 9539 return err; 9540 } 9541 9542 static inline int 9543 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 9544 struct rte_flow_tunnel *tunnel, 9545 struct rte_flow_error *error) 9546 { 9547 struct mlx5_priv *priv = dev->data->dev_private; 9548 9549 if (!priv->config.dv_flow_en) 9550 return rte_flow_error_set(error, ENOTSUP, 9551 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9552 "flow DV interface is off"); 9553 if (!is_tunnel_offload_active(dev)) 9554 return rte_flow_error_set(error, ENOTSUP, 9555 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9556 "tunnel offload was not activated"); 9557 if (!tunnel) 9558 return rte_flow_error_set(error, EINVAL, 9559 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9560 "no application tunnel"); 9561 switch (tunnel->type) { 9562 default: 9563 return rte_flow_error_set(error, EINVAL, 9564 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9565 "unsupported tunnel type"); 9566 case RTE_FLOW_ITEM_TYPE_VXLAN: 9567 case RTE_FLOW_ITEM_TYPE_GRE: 9568 case RTE_FLOW_ITEM_TYPE_NVGRE: 9569 case RTE_FLOW_ITEM_TYPE_GENEVE: 9570 break; 9571 } 9572 return 0; 9573 } 9574 9575 static int 9576 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 9577 struct rte_flow_tunnel *app_tunnel, 9578 struct rte_flow_action **actions, 9579 uint32_t *num_of_actions, 9580 struct rte_flow_error *error) 9581 { 9582 struct mlx5_flow_tunnel *tunnel; 9583 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9584 9585 if (ret) 9586 return ret; 9587 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9588 if (ret < 0) { 9589 return rte_flow_error_set(error, ret, 9590 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9591 "failed to initialize pmd tunnel"); 9592 } 9593 *actions = &tunnel->action; 9594 *num_of_actions = 1; 9595 return 0; 9596 } 9597 9598 static int 9599 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9600 struct rte_flow_tunnel *app_tunnel, 9601 struct rte_flow_item **items, 9602 uint32_t *num_of_items, 9603 struct rte_flow_error *error) 9604 { 9605 struct mlx5_flow_tunnel *tunnel; 9606 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9607 9608 if (ret) 9609 return ret; 9610 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9611 if (ret < 0) { 9612 return rte_flow_error_set(error, ret, 9613 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9614 "failed to initialize pmd tunnel"); 9615 } 9616 *items = &tunnel->item; 9617 *num_of_items = 1; 9618 return 0; 9619 } 9620 9621 struct tunnel_db_element_release_ctx { 9622 struct rte_flow_item *items; 9623 struct rte_flow_action *actions; 9624 uint32_t num_elements; 9625 struct rte_flow_error *error; 9626 int ret; 9627 }; 9628 9629 static bool 9630 tunnel_element_release_match(struct rte_eth_dev *dev, 9631 struct mlx5_flow_tunnel *tunnel, const void *x) 9632 { 9633 const struct tunnel_db_element_release_ctx *ctx = x; 9634 9635 RTE_SET_USED(dev); 9636 if (ctx->num_elements != 1) 9637 return false; 9638 else if (ctx->items) 9639 return ctx->items == &tunnel->item; 9640 else if (ctx->actions) 9641 return ctx->actions == &tunnel->action; 9642 9643 return false; 9644 } 9645 9646 static void 9647 tunnel_element_release_hit(struct rte_eth_dev *dev, 9648 struct mlx5_flow_tunnel *tunnel, void *x) 9649 { 9650 struct tunnel_db_element_release_ctx *ctx = x; 9651 ctx->ret = 0; 9652 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9653 mlx5_flow_tunnel_free(dev, tunnel); 9654 } 9655 9656 static void 9657 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9658 { 9659 struct tunnel_db_element_release_ctx *ctx = x; 9660 RTE_SET_USED(dev); 9661 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9662 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9663 "invalid argument"); 9664 } 9665 9666 static int 9667 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9668 struct rte_flow_item *pmd_items, 9669 uint32_t num_items, struct rte_flow_error *err) 9670 { 9671 struct tunnel_db_element_release_ctx ctx = { 9672 .items = pmd_items, 9673 .actions = NULL, 9674 .num_elements = num_items, 9675 .error = err, 9676 }; 9677 9678 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9679 tunnel_element_release_hit, 9680 tunnel_element_release_miss, &ctx, false); 9681 9682 return ctx.ret; 9683 } 9684 9685 static int 9686 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9687 struct rte_flow_action *pmd_actions, 9688 uint32_t num_actions, struct rte_flow_error *err) 9689 { 9690 struct tunnel_db_element_release_ctx ctx = { 9691 .items = NULL, 9692 .actions = pmd_actions, 9693 .num_elements = num_actions, 9694 .error = err, 9695 }; 9696 9697 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9698 tunnel_element_release_hit, 9699 tunnel_element_release_miss, &ctx, false); 9700 9701 return ctx.ret; 9702 } 9703 9704 static int 9705 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9706 struct rte_mbuf *m, 9707 struct rte_flow_restore_info *info, 9708 struct rte_flow_error *err) 9709 { 9710 uint64_t ol_flags = m->ol_flags; 9711 const struct mlx5_flow_tbl_data_entry *tble; 9712 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 9713 9714 if (!is_tunnel_offload_active(dev)) { 9715 info->flags = 0; 9716 return 0; 9717 } 9718 9719 if ((ol_flags & mask) != mask) 9720 goto err; 9721 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9722 if (!tble) { 9723 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9724 dev->data->port_id, m->hash.fdir.hi); 9725 goto err; 9726 } 9727 MLX5_ASSERT(tble->tunnel); 9728 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9729 info->group_id = tble->group_id; 9730 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9731 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9732 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9733 9734 return 0; 9735 9736 err: 9737 return rte_flow_error_set(err, EINVAL, 9738 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9739 "failed to get restore info"); 9740 } 9741 9742 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9743 static int 9744 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9745 __rte_unused struct rte_flow_tunnel *app_tunnel, 9746 __rte_unused struct rte_flow_action **actions, 9747 __rte_unused uint32_t *num_of_actions, 9748 __rte_unused struct rte_flow_error *error) 9749 { 9750 return -ENOTSUP; 9751 } 9752 9753 static int 9754 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9755 __rte_unused struct rte_flow_tunnel *app_tunnel, 9756 __rte_unused struct rte_flow_item **items, 9757 __rte_unused uint32_t *num_of_items, 9758 __rte_unused struct rte_flow_error *error) 9759 { 9760 return -ENOTSUP; 9761 } 9762 9763 static int 9764 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9765 __rte_unused struct rte_flow_item *pmd_items, 9766 __rte_unused uint32_t num_items, 9767 __rte_unused struct rte_flow_error *err) 9768 { 9769 return -ENOTSUP; 9770 } 9771 9772 static int 9773 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9774 __rte_unused struct rte_flow_action *pmd_action, 9775 __rte_unused uint32_t num_actions, 9776 __rte_unused struct rte_flow_error *err) 9777 { 9778 return -ENOTSUP; 9779 } 9780 9781 static int 9782 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9783 __rte_unused struct rte_mbuf *m, 9784 __rte_unused struct rte_flow_restore_info *i, 9785 __rte_unused struct rte_flow_error *err) 9786 { 9787 return -ENOTSUP; 9788 } 9789 9790 static int 9791 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9792 __rte_unused struct rte_flow *flow, 9793 __rte_unused const struct rte_flow_attr *attr, 9794 __rte_unused const struct rte_flow_action *actions, 9795 __rte_unused uint32_t flow_idx, 9796 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9797 __rte_unused struct tunnel_default_miss_ctx *ctx, 9798 __rte_unused struct rte_flow_error *error) 9799 { 9800 return -ENOTSUP; 9801 } 9802 9803 static struct mlx5_flow_tunnel * 9804 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9805 __rte_unused uint32_t id) 9806 { 9807 return NULL; 9808 } 9809 9810 static void 9811 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9812 __rte_unused struct mlx5_flow_tunnel *tunnel) 9813 { 9814 } 9815 9816 static uint32_t 9817 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9818 __rte_unused const struct mlx5_flow_tunnel *t, 9819 __rte_unused uint32_t group, 9820 __rte_unused uint32_t *table, 9821 struct rte_flow_error *error) 9822 { 9823 return rte_flow_error_set(error, ENOTSUP, 9824 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9825 "tunnel offload requires DV support"); 9826 } 9827 9828 void 9829 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9830 __rte_unused uint16_t port_id) 9831 { 9832 } 9833 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9834 9835 /* Flex flow item API */ 9836 static struct rte_flow_item_flex_handle * 9837 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 9838 const struct rte_flow_item_flex_conf *conf, 9839 struct rte_flow_error *error) 9840 { 9841 static const char err_msg[] = "flex item creation unsupported"; 9842 struct rte_flow_attr attr = { .transfer = 0 }; 9843 const struct mlx5_flow_driver_ops *fops = 9844 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9845 9846 if (!fops->item_create) { 9847 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9848 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9849 NULL, err_msg); 9850 return NULL; 9851 } 9852 return fops->item_create(dev, conf, error); 9853 } 9854 9855 static int 9856 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 9857 const struct rte_flow_item_flex_handle *handle, 9858 struct rte_flow_error *error) 9859 { 9860 static const char err_msg[] = "flex item release unsupported"; 9861 struct rte_flow_attr attr = { .transfer = 0 }; 9862 const struct mlx5_flow_driver_ops *fops = 9863 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9864 9865 if (!fops->item_release) { 9866 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9867 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9868 NULL, err_msg); 9869 return -rte_errno; 9870 } 9871 return fops->item_release(dev, handle, error); 9872 } 9873 9874 static void 9875 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9876 { 9877 int ret; 9878 struct rte_flow_error error; 9879 9880 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9881 char *item_name; 9882 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9883 sizeof(item_name), 9884 (void *)(uintptr_t)item->type, &error); 9885 if (ret > 0) 9886 printf("%s ", item_name); 9887 else 9888 printf("%d\n", (int)item->type); 9889 } 9890 printf("END\n"); 9891 } 9892 9893 static int 9894 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 9895 { 9896 const struct rte_flow_item_udp *spec = udp_item->spec; 9897 const struct rte_flow_item_udp *mask = udp_item->mask; 9898 uint16_t udp_dport = 0; 9899 9900 if (spec != NULL) { 9901 if (!mask) 9902 mask = &rte_flow_item_udp_mask; 9903 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 9904 mask->hdr.dst_port); 9905 } 9906 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 9907 } 9908 9909 static const struct mlx5_flow_expand_node * 9910 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 9911 unsigned int item_idx, 9912 const struct mlx5_flow_expand_node graph[], 9913 const struct mlx5_flow_expand_node *node) 9914 { 9915 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 9916 9917 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 9918 node != NULL && 9919 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 9920 /* 9921 * The expansion node is VXLAN and it is also the last 9922 * expandable item in the pattern, so need to continue 9923 * expansion of the inner tunnel. 9924 */ 9925 MLX5_ASSERT(item_idx > 0); 9926 prev_item = pattern + item_idx - 1; 9927 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 9928 if (mlx5_flow_is_std_vxlan_port(prev_item)) 9929 return &graph[MLX5_EXPANSION_STD_VXLAN]; 9930 return &graph[MLX5_EXPANSION_L3_VXLAN]; 9931 } 9932 return node; 9933 } 9934 9935 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 9936 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 9937 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 9938 }; 9939 9940 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 9941 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 9942 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 9943 { 9, 10, 11 }, { 12, 13, 14 }, 9944 }; 9945 9946 /** 9947 * Discover the number of available flow priorities. 9948 * 9949 * @param dev 9950 * Ethernet device. 9951 * 9952 * @return 9953 * On success, number of available flow priorities. 9954 * On failure, a negative errno-style code and rte_errno is set. 9955 */ 9956 int 9957 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 9958 { 9959 static const uint16_t vprio[] = {8, 16}; 9960 const struct mlx5_priv *priv = dev->data->dev_private; 9961 const struct mlx5_flow_driver_ops *fops; 9962 enum mlx5_flow_drv_type type; 9963 int ret; 9964 9965 type = mlx5_flow_os_get_type(); 9966 if (type == MLX5_FLOW_TYPE_MAX) { 9967 type = MLX5_FLOW_TYPE_VERBS; 9968 if (priv->sh->devx && priv->config.dv_flow_en) 9969 type = MLX5_FLOW_TYPE_DV; 9970 } 9971 fops = flow_get_drv_ops(type); 9972 if (fops->discover_priorities == NULL) { 9973 DRV_LOG(ERR, "Priority discovery not supported"); 9974 rte_errno = ENOTSUP; 9975 return -rte_errno; 9976 } 9977 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 9978 if (ret < 0) 9979 return ret; 9980 switch (ret) { 9981 case 8: 9982 ret = RTE_DIM(priority_map_3); 9983 break; 9984 case 16: 9985 ret = RTE_DIM(priority_map_5); 9986 break; 9987 default: 9988 rte_errno = ENOTSUP; 9989 DRV_LOG(ERR, 9990 "port %u maximum priority: %d expected 8/16", 9991 dev->data->port_id, ret); 9992 return -rte_errno; 9993 } 9994 DRV_LOG(INFO, "port %u supported flow priorities:" 9995 " 0-%d for ingress or egress root table," 9996 " 0-%d for non-root table or transfer root table.", 9997 dev->data->port_id, ret - 2, 9998 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 9999 return ret; 10000 } 10001 10002 /** 10003 * Adjust flow priority based on the highest layer and the request priority. 10004 * 10005 * @param[in] dev 10006 * Pointer to the Ethernet device structure. 10007 * @param[in] priority 10008 * The rule base priority. 10009 * @param[in] subpriority 10010 * The priority based on the items. 10011 * 10012 * @return 10013 * The new priority. 10014 */ 10015 uint32_t 10016 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10017 uint32_t subpriority) 10018 { 10019 uint32_t res = 0; 10020 struct mlx5_priv *priv = dev->data->dev_private; 10021 10022 switch (priv->sh->flow_max_priority) { 10023 case RTE_DIM(priority_map_3): 10024 res = priority_map_3[priority][subpriority]; 10025 break; 10026 case RTE_DIM(priority_map_5): 10027 res = priority_map_5[priority][subpriority]; 10028 break; 10029 } 10030 return res; 10031 } 10032