1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 80 #endif 81 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 82 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 83 }; 84 85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 87 (const int []){ \ 88 __VA_ARGS__, 0, \ 89 } 90 91 /** Node object of input graph for mlx5_flow_expand_rss(). */ 92 struct mlx5_flow_expand_node { 93 const int *const next; 94 /**< 95 * List of next node indexes. Index 0 is interpreted as a terminator. 96 */ 97 const enum rte_flow_item_type type; 98 /**< Pattern item type of current node. */ 99 uint64_t rss_types; 100 /**< 101 * RSS types bit-field associated with this node 102 * (see RTE_ETH_RSS_* definitions). 103 */ 104 uint64_t node_flags; 105 /**< 106 * Bit-fields that define how the node is used in the expansion. 107 * (see MLX5_EXPANSION_NODE_* definitions). 108 */ 109 }; 110 111 /* Optional expand field. The expansion alg will not go deeper. */ 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 113 114 /* The node is not added implicitly as expansion to the flow pattern. 115 * If the node type does not match the flow pattern item type, the 116 * expansion alg will go deeper to its next items. 117 * In the current implementation, the list of next nodes indexes can 118 * have up to one node with this flag set and it has to be the last 119 * node index (before the list terminator). 120 */ 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 122 123 /** Object returned by mlx5_flow_expand_rss(). */ 124 struct mlx5_flow_expand_rss { 125 uint32_t entries; 126 /**< Number of entries @p patterns and @p priorities. */ 127 struct { 128 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 129 uint32_t priority; /**< Priority offset for each expansion. */ 130 } entry[]; 131 }; 132 133 static void 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 135 136 static const struct mlx5_flow_expand_node * 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 138 unsigned int item_idx, 139 const struct mlx5_flow_expand_node graph[], 140 const struct mlx5_flow_expand_node *node); 141 142 static bool 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 144 { 145 switch (item->type) { 146 case RTE_FLOW_ITEM_TYPE_ETH: 147 case RTE_FLOW_ITEM_TYPE_VLAN: 148 case RTE_FLOW_ITEM_TYPE_IPV4: 149 case RTE_FLOW_ITEM_TYPE_IPV6: 150 case RTE_FLOW_ITEM_TYPE_UDP: 151 case RTE_FLOW_ITEM_TYPE_TCP: 152 case RTE_FLOW_ITEM_TYPE_VXLAN: 153 case RTE_FLOW_ITEM_TYPE_NVGRE: 154 case RTE_FLOW_ITEM_TYPE_GRE: 155 case RTE_FLOW_ITEM_TYPE_GENEVE: 156 case RTE_FLOW_ITEM_TYPE_MPLS: 157 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 158 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 159 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 160 case RTE_FLOW_ITEM_TYPE_GTP: 161 return true; 162 default: 163 break; 164 } 165 return false; 166 } 167 168 /** 169 * Network Service Header (NSH) and its next protocol values 170 * are described in RFC-8393. 171 */ 172 static enum rte_flow_item_type 173 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 174 { 175 enum rte_flow_item_type type; 176 177 switch (proto_mask & proto_spec) { 178 case RTE_VXLAN_GPE_TYPE_IPV4: 179 type = RTE_FLOW_ITEM_TYPE_IPV4; 180 break; 181 case RTE_VXLAN_GPE_TYPE_IPV6: 182 type = RTE_VXLAN_GPE_TYPE_IPV6; 183 break; 184 case RTE_VXLAN_GPE_TYPE_ETH: 185 type = RTE_FLOW_ITEM_TYPE_ETH; 186 break; 187 default: 188 type = RTE_FLOW_ITEM_TYPE_END; 189 } 190 return type; 191 } 192 193 static enum rte_flow_item_type 194 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 195 { 196 enum rte_flow_item_type type; 197 198 switch (proto_mask & proto_spec) { 199 case IPPROTO_UDP: 200 type = RTE_FLOW_ITEM_TYPE_UDP; 201 break; 202 case IPPROTO_TCP: 203 type = RTE_FLOW_ITEM_TYPE_TCP; 204 break; 205 case IPPROTO_IP: 206 type = RTE_FLOW_ITEM_TYPE_IPV4; 207 break; 208 case IPPROTO_IPV6: 209 type = RTE_FLOW_ITEM_TYPE_IPV6; 210 break; 211 default: 212 type = RTE_FLOW_ITEM_TYPE_END; 213 } 214 return type; 215 } 216 217 static enum rte_flow_item_type 218 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 219 rte_be16_t type_mask, bool is_tunnel) 220 { 221 enum rte_flow_item_type type; 222 223 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 224 case RTE_ETHER_TYPE_TEB: 225 type = is_tunnel ? 226 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 227 break; 228 case RTE_ETHER_TYPE_VLAN: 229 type = !is_tunnel ? 230 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 231 break; 232 case RTE_ETHER_TYPE_IPV4: 233 type = RTE_FLOW_ITEM_TYPE_IPV4; 234 break; 235 case RTE_ETHER_TYPE_IPV6: 236 type = RTE_FLOW_ITEM_TYPE_IPV6; 237 break; 238 default: 239 type = RTE_FLOW_ITEM_TYPE_END; 240 } 241 return type; 242 } 243 244 static enum rte_flow_item_type 245 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 246 { 247 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 248 do { \ 249 const void *m = item->mask; \ 250 const void *s = item->spec; \ 251 mask = m ? \ 252 ((const struct rte_flow_item_##type *)m)->fld : \ 253 rte_flow_item_##type##_mask.fld; \ 254 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 255 } while (0) 256 257 enum rte_flow_item_type ret; 258 uint16_t spec, mask; 259 260 if (item == NULL || item->spec == NULL) 261 return RTE_FLOW_ITEM_TYPE_VOID; 262 switch (item->type) { 263 case RTE_FLOW_ITEM_TYPE_ETH: 264 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 265 if (!mask) 266 return RTE_FLOW_ITEM_TYPE_VOID; 267 ret = mlx5_ethertype_to_item_type(spec, mask, false); 268 break; 269 case RTE_FLOW_ITEM_TYPE_VLAN: 270 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 271 if (!mask) 272 return RTE_FLOW_ITEM_TYPE_VOID; 273 ret = mlx5_ethertype_to_item_type(spec, mask, false); 274 break; 275 case RTE_FLOW_ITEM_TYPE_IPV4: 276 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 277 if (!mask) 278 return RTE_FLOW_ITEM_TYPE_VOID; 279 ret = mlx5_inet_proto_to_item_type(spec, mask); 280 break; 281 case RTE_FLOW_ITEM_TYPE_IPV6: 282 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 283 if (!mask) 284 return RTE_FLOW_ITEM_TYPE_VOID; 285 ret = mlx5_inet_proto_to_item_type(spec, mask); 286 break; 287 case RTE_FLOW_ITEM_TYPE_GENEVE: 288 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 289 ret = mlx5_ethertype_to_item_type(spec, mask, true); 290 break; 291 case RTE_FLOW_ITEM_TYPE_GRE: 292 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 293 ret = mlx5_ethertype_to_item_type(spec, mask, true); 294 break; 295 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 296 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 297 ret = mlx5_nsh_proto_to_item_type(spec, mask); 298 break; 299 default: 300 ret = RTE_FLOW_ITEM_TYPE_VOID; 301 break; 302 } 303 return ret; 304 #undef MLX5_XSET_ITEM_MASK_SPEC 305 } 306 307 static const int * 308 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 309 const int *next_node) 310 { 311 const struct mlx5_flow_expand_node *node = NULL; 312 const int *next = next_node; 313 314 while (next && *next) { 315 /* 316 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 317 * flag set, because they were not found in the flow pattern. 318 */ 319 node = &graph[*next]; 320 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 321 break; 322 next = node->next; 323 } 324 return next; 325 } 326 327 #define MLX5_RSS_EXP_ELT_N 16 328 329 /** 330 * Expand RSS flows into several possible flows according to the RSS hash 331 * fields requested and the driver capabilities. 332 * 333 * @param[out] buf 334 * Buffer to store the result expansion. 335 * @param[in] size 336 * Buffer size in bytes. If 0, @p buf can be NULL. 337 * @param[in] pattern 338 * User flow pattern. 339 * @param[in] types 340 * RSS types to expand (see RTE_ETH_RSS_* definitions). 341 * @param[in] graph 342 * Input graph to expand @p pattern according to @p types. 343 * @param[in] graph_root_index 344 * Index of root node in @p graph, typically 0. 345 * 346 * @return 347 * A positive value representing the size of @p buf in bytes regardless of 348 * @p size on success, a negative errno value otherwise and rte_errno is 349 * set, the following errors are defined: 350 * 351 * -E2BIG: graph-depth @p graph is too deep. 352 * -EINVAL: @p size has not enough space for expanded pattern. 353 */ 354 static int 355 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 356 const struct rte_flow_item *pattern, uint64_t types, 357 const struct mlx5_flow_expand_node graph[], 358 int graph_root_index) 359 { 360 const struct rte_flow_item *item; 361 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 362 const int *next_node; 363 const int *stack[MLX5_RSS_EXP_ELT_N]; 364 int stack_pos = 0; 365 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 366 unsigned int i, item_idx, last_expand_item_idx = 0; 367 size_t lsize; 368 size_t user_pattern_size = 0; 369 void *addr = NULL; 370 const struct mlx5_flow_expand_node *next = NULL; 371 struct rte_flow_item missed_item; 372 int missed = 0; 373 int elt = 0; 374 const struct rte_flow_item *last_expand_item = NULL; 375 376 memset(&missed_item, 0, sizeof(missed_item)); 377 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 378 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 379 if (lsize > size) 380 return -EINVAL; 381 buf->entry[0].priority = 0; 382 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 383 buf->entries = 0; 384 addr = buf->entry[0].pattern; 385 for (item = pattern, item_idx = 0; 386 item->type != RTE_FLOW_ITEM_TYPE_END; 387 item++, item_idx++) { 388 if (!mlx5_flow_is_rss_expandable_item(item)) { 389 user_pattern_size += sizeof(*item); 390 continue; 391 } 392 last_expand_item = item; 393 last_expand_item_idx = item_idx; 394 i = 0; 395 while (node->next && node->next[i]) { 396 next = &graph[node->next[i]]; 397 if (next->type == item->type) 398 break; 399 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 400 node = next; 401 i = 0; 402 } else { 403 ++i; 404 } 405 } 406 if (next) 407 node = next; 408 user_pattern_size += sizeof(*item); 409 } 410 user_pattern_size += sizeof(*item); /* Handle END item. */ 411 lsize += user_pattern_size; 412 if (lsize > size) 413 return -EINVAL; 414 /* Copy the user pattern in the first entry of the buffer. */ 415 rte_memcpy(addr, pattern, user_pattern_size); 416 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 417 buf->entries = 1; 418 /* Start expanding. */ 419 memset(flow_items, 0, sizeof(flow_items)); 420 user_pattern_size -= sizeof(*item); 421 /* 422 * Check if the last valid item has spec set, need complete pattern, 423 * and the pattern can be used for expansion. 424 */ 425 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 426 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 427 /* Item type END indicates expansion is not required. */ 428 return lsize; 429 } 430 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 431 next = NULL; 432 missed = 1; 433 i = 0; 434 while (node->next && node->next[i]) { 435 next = &graph[node->next[i]]; 436 if (next->type == missed_item.type) { 437 flow_items[0].type = missed_item.type; 438 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 439 break; 440 } 441 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 442 node = next; 443 i = 0; 444 } else { 445 ++i; 446 } 447 next = NULL; 448 } 449 } 450 if (next && missed) { 451 elt = 2; /* missed item + item end. */ 452 node = next; 453 lsize += elt * sizeof(*item) + user_pattern_size; 454 if (lsize > size) 455 return -EINVAL; 456 if (node->rss_types & types) { 457 buf->entry[buf->entries].priority = 1; 458 buf->entry[buf->entries].pattern = addr; 459 buf->entries++; 460 rte_memcpy(addr, buf->entry[0].pattern, 461 user_pattern_size); 462 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 463 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 464 addr = (void *)(((uintptr_t)addr) + 465 elt * sizeof(*item)); 466 } 467 } else if (last_expand_item != NULL) { 468 node = mlx5_flow_expand_rss_adjust_node(pattern, 469 last_expand_item_idx, graph, node); 470 } 471 memset(flow_items, 0, sizeof(flow_items)); 472 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 473 node->next); 474 stack[stack_pos] = next_node; 475 node = next_node ? &graph[*next_node] : NULL; 476 while (node) { 477 flow_items[stack_pos].type = node->type; 478 if (node->rss_types & types) { 479 size_t n; 480 /* 481 * compute the number of items to copy from the 482 * expansion and copy it. 483 * When the stack_pos is 0, there are 1 element in it, 484 * plus the addition END item. 485 */ 486 elt = stack_pos + 2; 487 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 488 lsize += elt * sizeof(*item) + user_pattern_size; 489 if (lsize > size) 490 return -EINVAL; 491 n = elt * sizeof(*item); 492 buf->entry[buf->entries].priority = 493 stack_pos + 1 + missed; 494 buf->entry[buf->entries].pattern = addr; 495 buf->entries++; 496 rte_memcpy(addr, buf->entry[0].pattern, 497 user_pattern_size); 498 addr = (void *)(((uintptr_t)addr) + 499 user_pattern_size); 500 rte_memcpy(addr, &missed_item, 501 missed * sizeof(*item)); 502 addr = (void *)(((uintptr_t)addr) + 503 missed * sizeof(*item)); 504 rte_memcpy(addr, flow_items, n); 505 addr = (void *)(((uintptr_t)addr) + n); 506 } 507 /* Go deeper. */ 508 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 509 node->next) { 510 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 511 node->next); 512 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 513 rte_errno = E2BIG; 514 return -rte_errno; 515 } 516 stack[stack_pos] = next_node; 517 } else if (*(next_node + 1)) { 518 /* Follow up with the next possibility. */ 519 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 520 ++next_node); 521 } else if (!stack_pos) { 522 /* 523 * Completing the traverse over the different paths. 524 * The next_node is advanced to the terminator. 525 */ 526 ++next_node; 527 } else { 528 /* Move to the next path. */ 529 while (stack_pos) { 530 next_node = stack[--stack_pos]; 531 next_node++; 532 if (*next_node) 533 break; 534 } 535 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 536 next_node); 537 stack[stack_pos] = next_node; 538 } 539 node = next_node && *next_node ? &graph[*next_node] : NULL; 540 }; 541 return lsize; 542 } 543 544 enum mlx5_expansion { 545 MLX5_EXPANSION_ROOT, 546 MLX5_EXPANSION_ROOT_OUTER, 547 MLX5_EXPANSION_OUTER_ETH, 548 MLX5_EXPANSION_OUTER_VLAN, 549 MLX5_EXPANSION_OUTER_IPV4, 550 MLX5_EXPANSION_OUTER_IPV4_UDP, 551 MLX5_EXPANSION_OUTER_IPV4_TCP, 552 MLX5_EXPANSION_OUTER_IPV6, 553 MLX5_EXPANSION_OUTER_IPV6_UDP, 554 MLX5_EXPANSION_OUTER_IPV6_TCP, 555 MLX5_EXPANSION_VXLAN, 556 MLX5_EXPANSION_STD_VXLAN, 557 MLX5_EXPANSION_L3_VXLAN, 558 MLX5_EXPANSION_VXLAN_GPE, 559 MLX5_EXPANSION_GRE, 560 MLX5_EXPANSION_NVGRE, 561 MLX5_EXPANSION_GRE_KEY, 562 MLX5_EXPANSION_MPLS, 563 MLX5_EXPANSION_ETH, 564 MLX5_EXPANSION_VLAN, 565 MLX5_EXPANSION_IPV4, 566 MLX5_EXPANSION_IPV4_UDP, 567 MLX5_EXPANSION_IPV4_TCP, 568 MLX5_EXPANSION_IPV6, 569 MLX5_EXPANSION_IPV6_UDP, 570 MLX5_EXPANSION_IPV6_TCP, 571 MLX5_EXPANSION_IPV6_FRAG_EXT, 572 MLX5_EXPANSION_GTP, 573 MLX5_EXPANSION_GENEVE, 574 }; 575 576 /** Supported expansion of items. */ 577 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 578 [MLX5_EXPANSION_ROOT] = { 579 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 580 MLX5_EXPANSION_IPV4, 581 MLX5_EXPANSION_IPV6), 582 .type = RTE_FLOW_ITEM_TYPE_END, 583 }, 584 [MLX5_EXPANSION_ROOT_OUTER] = { 585 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 586 MLX5_EXPANSION_OUTER_IPV4, 587 MLX5_EXPANSION_OUTER_IPV6), 588 .type = RTE_FLOW_ITEM_TYPE_END, 589 }, 590 [MLX5_EXPANSION_OUTER_ETH] = { 591 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 592 .type = RTE_FLOW_ITEM_TYPE_ETH, 593 .rss_types = 0, 594 }, 595 [MLX5_EXPANSION_OUTER_VLAN] = { 596 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 597 MLX5_EXPANSION_OUTER_IPV6), 598 .type = RTE_FLOW_ITEM_TYPE_VLAN, 599 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 600 }, 601 [MLX5_EXPANSION_OUTER_IPV4] = { 602 .next = MLX5_FLOW_EXPAND_RSS_NEXT 603 (MLX5_EXPANSION_OUTER_IPV4_UDP, 604 MLX5_EXPANSION_OUTER_IPV4_TCP, 605 MLX5_EXPANSION_GRE, 606 MLX5_EXPANSION_NVGRE, 607 MLX5_EXPANSION_IPV4, 608 MLX5_EXPANSION_IPV6), 609 .type = RTE_FLOW_ITEM_TYPE_IPV4, 610 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 611 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 612 }, 613 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 614 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 615 MLX5_EXPANSION_VXLAN_GPE, 616 MLX5_EXPANSION_MPLS, 617 MLX5_EXPANSION_GENEVE, 618 MLX5_EXPANSION_GTP), 619 .type = RTE_FLOW_ITEM_TYPE_UDP, 620 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 621 }, 622 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 623 .type = RTE_FLOW_ITEM_TYPE_TCP, 624 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 625 }, 626 [MLX5_EXPANSION_OUTER_IPV6] = { 627 .next = MLX5_FLOW_EXPAND_RSS_NEXT 628 (MLX5_EXPANSION_OUTER_IPV6_UDP, 629 MLX5_EXPANSION_OUTER_IPV6_TCP, 630 MLX5_EXPANSION_IPV4, 631 MLX5_EXPANSION_IPV6, 632 MLX5_EXPANSION_GRE, 633 MLX5_EXPANSION_NVGRE), 634 .type = RTE_FLOW_ITEM_TYPE_IPV6, 635 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 636 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 637 }, 638 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 639 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 640 MLX5_EXPANSION_VXLAN_GPE, 641 MLX5_EXPANSION_MPLS, 642 MLX5_EXPANSION_GENEVE, 643 MLX5_EXPANSION_GTP), 644 .type = RTE_FLOW_ITEM_TYPE_UDP, 645 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 646 }, 647 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 648 .type = RTE_FLOW_ITEM_TYPE_TCP, 649 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 650 }, 651 [MLX5_EXPANSION_VXLAN] = { 652 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 653 MLX5_EXPANSION_IPV4, 654 MLX5_EXPANSION_IPV6), 655 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 656 }, 657 [MLX5_EXPANSION_STD_VXLAN] = { 658 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 659 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 660 }, 661 [MLX5_EXPANSION_L3_VXLAN] = { 662 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 663 MLX5_EXPANSION_IPV6), 664 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 665 }, 666 [MLX5_EXPANSION_VXLAN_GPE] = { 667 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 668 MLX5_EXPANSION_IPV4, 669 MLX5_EXPANSION_IPV6), 670 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 671 }, 672 [MLX5_EXPANSION_GRE] = { 673 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 674 MLX5_EXPANSION_IPV4, 675 MLX5_EXPANSION_IPV6, 676 MLX5_EXPANSION_GRE_KEY, 677 MLX5_EXPANSION_MPLS), 678 .type = RTE_FLOW_ITEM_TYPE_GRE, 679 }, 680 [MLX5_EXPANSION_GRE_KEY] = { 681 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 682 MLX5_EXPANSION_IPV6, 683 MLX5_EXPANSION_MPLS), 684 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 685 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 686 }, 687 [MLX5_EXPANSION_NVGRE] = { 688 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 689 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 690 }, 691 [MLX5_EXPANSION_MPLS] = { 692 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 693 MLX5_EXPANSION_IPV6, 694 MLX5_EXPANSION_ETH), 695 .type = RTE_FLOW_ITEM_TYPE_MPLS, 696 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 697 }, 698 [MLX5_EXPANSION_ETH] = { 699 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 700 .type = RTE_FLOW_ITEM_TYPE_ETH, 701 }, 702 [MLX5_EXPANSION_VLAN] = { 703 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 704 MLX5_EXPANSION_IPV6), 705 .type = RTE_FLOW_ITEM_TYPE_VLAN, 706 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 707 }, 708 [MLX5_EXPANSION_IPV4] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 710 MLX5_EXPANSION_IPV4_TCP), 711 .type = RTE_FLOW_ITEM_TYPE_IPV4, 712 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 713 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 714 }, 715 [MLX5_EXPANSION_IPV4_UDP] = { 716 .type = RTE_FLOW_ITEM_TYPE_UDP, 717 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 718 }, 719 [MLX5_EXPANSION_IPV4_TCP] = { 720 .type = RTE_FLOW_ITEM_TYPE_TCP, 721 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 722 }, 723 [MLX5_EXPANSION_IPV6] = { 724 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 725 MLX5_EXPANSION_IPV6_TCP, 726 MLX5_EXPANSION_IPV6_FRAG_EXT), 727 .type = RTE_FLOW_ITEM_TYPE_IPV6, 728 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 729 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 730 }, 731 [MLX5_EXPANSION_IPV6_UDP] = { 732 .type = RTE_FLOW_ITEM_TYPE_UDP, 733 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 734 }, 735 [MLX5_EXPANSION_IPV6_TCP] = { 736 .type = RTE_FLOW_ITEM_TYPE_TCP, 737 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 738 }, 739 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 740 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 741 }, 742 [MLX5_EXPANSION_GTP] = { 743 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_GTP, 746 }, 747 [MLX5_EXPANSION_GENEVE] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 749 MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 752 }, 753 }; 754 755 static struct rte_flow_action_handle * 756 mlx5_action_handle_create(struct rte_eth_dev *dev, 757 const struct rte_flow_indir_action_conf *conf, 758 const struct rte_flow_action *action, 759 struct rte_flow_error *error); 760 static int mlx5_action_handle_destroy 761 (struct rte_eth_dev *dev, 762 struct rte_flow_action_handle *handle, 763 struct rte_flow_error *error); 764 static int mlx5_action_handle_update 765 (struct rte_eth_dev *dev, 766 struct rte_flow_action_handle *handle, 767 const void *update, 768 struct rte_flow_error *error); 769 static int mlx5_action_handle_query 770 (struct rte_eth_dev *dev, 771 const struct rte_flow_action_handle *handle, 772 void *data, 773 struct rte_flow_error *error); 774 static int 775 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 776 struct rte_flow_tunnel *app_tunnel, 777 struct rte_flow_action **actions, 778 uint32_t *num_of_actions, 779 struct rte_flow_error *error); 780 static int 781 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 782 struct rte_flow_tunnel *app_tunnel, 783 struct rte_flow_item **items, 784 uint32_t *num_of_items, 785 struct rte_flow_error *error); 786 static int 787 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 788 struct rte_flow_item *pmd_items, 789 uint32_t num_items, struct rte_flow_error *err); 790 static int 791 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 792 struct rte_flow_action *pmd_actions, 793 uint32_t num_actions, 794 struct rte_flow_error *err); 795 static int 796 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 797 struct rte_mbuf *m, 798 struct rte_flow_restore_info *info, 799 struct rte_flow_error *err); 800 static struct rte_flow_item_flex_handle * 801 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 802 const struct rte_flow_item_flex_conf *conf, 803 struct rte_flow_error *error); 804 static int 805 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 806 const struct rte_flow_item_flex_handle *handle, 807 struct rte_flow_error *error); 808 809 static const struct rte_flow_ops mlx5_flow_ops = { 810 .validate = mlx5_flow_validate, 811 .create = mlx5_flow_create, 812 .destroy = mlx5_flow_destroy, 813 .flush = mlx5_flow_flush, 814 .isolate = mlx5_flow_isolate, 815 .query = mlx5_flow_query, 816 .dev_dump = mlx5_flow_dev_dump, 817 .get_aged_flows = mlx5_flow_get_aged_flows, 818 .action_handle_create = mlx5_action_handle_create, 819 .action_handle_destroy = mlx5_action_handle_destroy, 820 .action_handle_update = mlx5_action_handle_update, 821 .action_handle_query = mlx5_action_handle_query, 822 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 823 .tunnel_match = mlx5_flow_tunnel_match, 824 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 825 .tunnel_item_release = mlx5_flow_tunnel_item_release, 826 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 827 .flex_item_create = mlx5_flow_flex_item_create, 828 .flex_item_release = mlx5_flow_flex_item_release, 829 }; 830 831 /* Tunnel information. */ 832 struct mlx5_flow_tunnel_info { 833 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 834 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 835 }; 836 837 static struct mlx5_flow_tunnel_info tunnels_info[] = { 838 { 839 .tunnel = MLX5_FLOW_LAYER_VXLAN, 840 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 841 }, 842 { 843 .tunnel = MLX5_FLOW_LAYER_GENEVE, 844 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 845 }, 846 { 847 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 848 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 849 }, 850 { 851 .tunnel = MLX5_FLOW_LAYER_GRE, 852 .ptype = RTE_PTYPE_TUNNEL_GRE, 853 }, 854 { 855 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 856 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 857 }, 858 { 859 .tunnel = MLX5_FLOW_LAYER_MPLS, 860 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 861 }, 862 { 863 .tunnel = MLX5_FLOW_LAYER_NVGRE, 864 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 865 }, 866 { 867 .tunnel = MLX5_FLOW_LAYER_IPIP, 868 .ptype = RTE_PTYPE_TUNNEL_IP, 869 }, 870 { 871 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 872 .ptype = RTE_PTYPE_TUNNEL_IP, 873 }, 874 { 875 .tunnel = MLX5_FLOW_LAYER_GTP, 876 .ptype = RTE_PTYPE_TUNNEL_GTPU, 877 }, 878 }; 879 880 881 882 /** 883 * Translate tag ID to register. 884 * 885 * @param[in] dev 886 * Pointer to the Ethernet device structure. 887 * @param[in] feature 888 * The feature that request the register. 889 * @param[in] id 890 * The request register ID. 891 * @param[out] error 892 * Error description in case of any. 893 * 894 * @return 895 * The request register on success, a negative errno 896 * value otherwise and rte_errno is set. 897 */ 898 int 899 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 900 enum mlx5_feature_name feature, 901 uint32_t id, 902 struct rte_flow_error *error) 903 { 904 struct mlx5_priv *priv = dev->data->dev_private; 905 struct mlx5_sh_config *config = &priv->sh->config; 906 enum modify_reg start_reg; 907 bool skip_mtr_reg = false; 908 909 switch (feature) { 910 case MLX5_HAIRPIN_RX: 911 return REG_B; 912 case MLX5_HAIRPIN_TX: 913 return REG_A; 914 case MLX5_METADATA_RX: 915 switch (config->dv_xmeta_en) { 916 case MLX5_XMETA_MODE_LEGACY: 917 return REG_B; 918 case MLX5_XMETA_MODE_META16: 919 return REG_C_0; 920 case MLX5_XMETA_MODE_META32: 921 return REG_C_1; 922 } 923 break; 924 case MLX5_METADATA_TX: 925 return REG_A; 926 case MLX5_METADATA_FDB: 927 switch (config->dv_xmeta_en) { 928 case MLX5_XMETA_MODE_LEGACY: 929 return REG_NON; 930 case MLX5_XMETA_MODE_META16: 931 return REG_C_0; 932 case MLX5_XMETA_MODE_META32: 933 return REG_C_1; 934 } 935 break; 936 case MLX5_FLOW_MARK: 937 switch (config->dv_xmeta_en) { 938 case MLX5_XMETA_MODE_LEGACY: 939 return REG_NON; 940 case MLX5_XMETA_MODE_META16: 941 return REG_C_1; 942 case MLX5_XMETA_MODE_META32: 943 return REG_C_0; 944 } 945 break; 946 case MLX5_MTR_ID: 947 /* 948 * If meter color and meter id share one register, flow match 949 * should use the meter color register for match. 950 */ 951 if (priv->mtr_reg_share) 952 return priv->mtr_color_reg; 953 else 954 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 955 REG_C_3; 956 case MLX5_MTR_COLOR: 957 case MLX5_ASO_FLOW_HIT: 958 case MLX5_ASO_CONNTRACK: 959 case MLX5_SAMPLE_ID: 960 /* All features use the same REG_C. */ 961 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 962 return priv->mtr_color_reg; 963 case MLX5_COPY_MARK: 964 /* 965 * Metadata COPY_MARK register using is in meter suffix sub 966 * flow while with meter. It's safe to share the same register. 967 */ 968 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 969 case MLX5_APP_TAG: 970 /* 971 * If meter is enable, it will engage the register for color 972 * match and flow match. If meter color match is not using the 973 * REG_C_2, need to skip the REG_C_x be used by meter color 974 * match. 975 * If meter is disable, free to use all available registers. 976 */ 977 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 978 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 979 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 980 if (id > (uint32_t)(REG_C_7 - start_reg)) 981 return rte_flow_error_set(error, EINVAL, 982 RTE_FLOW_ERROR_TYPE_ITEM, 983 NULL, "invalid tag id"); 984 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 985 return rte_flow_error_set(error, ENOTSUP, 986 RTE_FLOW_ERROR_TYPE_ITEM, 987 NULL, "unsupported tag id"); 988 /* 989 * This case means meter is using the REG_C_x great than 2. 990 * Take care not to conflict with meter color REG_C_x. 991 * If the available index REG_C_y >= REG_C_x, skip the 992 * color register. 993 */ 994 if (skip_mtr_reg && priv->sh->flow_mreg_c 995 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 996 if (id >= (uint32_t)(REG_C_7 - start_reg)) 997 return rte_flow_error_set(error, EINVAL, 998 RTE_FLOW_ERROR_TYPE_ITEM, 999 NULL, "invalid tag id"); 1000 if (priv->sh->flow_mreg_c 1001 [id + 1 + start_reg - REG_C_0] != REG_NON) 1002 return priv->sh->flow_mreg_c 1003 [id + 1 + start_reg - REG_C_0]; 1004 return rte_flow_error_set(error, ENOTSUP, 1005 RTE_FLOW_ERROR_TYPE_ITEM, 1006 NULL, "unsupported tag id"); 1007 } 1008 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1009 } 1010 MLX5_ASSERT(false); 1011 return rte_flow_error_set(error, EINVAL, 1012 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1013 NULL, "invalid feature name"); 1014 } 1015 1016 /** 1017 * Check extensive flow metadata register support. 1018 * 1019 * @param dev 1020 * Pointer to rte_eth_dev structure. 1021 * 1022 * @return 1023 * True if device supports extensive flow metadata register, otherwise false. 1024 */ 1025 bool 1026 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1027 { 1028 struct mlx5_priv *priv = dev->data->dev_private; 1029 1030 /* 1031 * Having available reg_c can be regarded inclusively as supporting 1032 * extensive flow metadata register, which could mean, 1033 * - metadata register copy action by modify header. 1034 * - 16 modify header actions is supported. 1035 * - reg_c's are preserved across different domain (FDB and NIC) on 1036 * packet loopback by flow lookup miss. 1037 */ 1038 return priv->sh->flow_mreg_c[2] != REG_NON; 1039 } 1040 1041 /** 1042 * Get the lowest priority. 1043 * 1044 * @param[in] dev 1045 * Pointer to the Ethernet device structure. 1046 * @param[in] attributes 1047 * Pointer to device flow rule attributes. 1048 * 1049 * @return 1050 * The value of lowest priority of flow. 1051 */ 1052 uint32_t 1053 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1054 const struct rte_flow_attr *attr) 1055 { 1056 struct mlx5_priv *priv = dev->data->dev_private; 1057 1058 if (!attr->group && !attr->transfer) 1059 return priv->sh->flow_max_priority - 2; 1060 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1061 } 1062 1063 /** 1064 * Calculate matcher priority of the flow. 1065 * 1066 * @param[in] dev 1067 * Pointer to the Ethernet device structure. 1068 * @param[in] attr 1069 * Pointer to device flow rule attributes. 1070 * @param[in] subpriority 1071 * The priority based on the items. 1072 * @param[in] external 1073 * Flow is user flow. 1074 * @return 1075 * The matcher priority of the flow. 1076 */ 1077 uint16_t 1078 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1079 const struct rte_flow_attr *attr, 1080 uint32_t subpriority, bool external) 1081 { 1082 uint16_t priority = (uint16_t)attr->priority; 1083 struct mlx5_priv *priv = dev->data->dev_private; 1084 1085 if (!attr->group && !attr->transfer) { 1086 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1087 priority = priv->sh->flow_max_priority - 1; 1088 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1089 } else if (!external && attr->transfer && attr->group == 0 && 1090 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1091 return (priv->sh->flow_max_priority - 1) * 3; 1092 } 1093 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1094 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1095 return priority * 3 + subpriority; 1096 } 1097 1098 /** 1099 * Verify the @p item specifications (spec, last, mask) are compatible with the 1100 * NIC capabilities. 1101 * 1102 * @param[in] item 1103 * Item specification. 1104 * @param[in] mask 1105 * @p item->mask or flow default bit-masks. 1106 * @param[in] nic_mask 1107 * Bit-masks covering supported fields by the NIC to compare with user mask. 1108 * @param[in] size 1109 * Bit-masks size in bytes. 1110 * @param[in] range_accepted 1111 * True if range of values is accepted for specific fields, false otherwise. 1112 * @param[out] error 1113 * Pointer to error structure. 1114 * 1115 * @return 1116 * 0 on success, a negative errno value otherwise and rte_errno is set. 1117 */ 1118 int 1119 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1120 const uint8_t *mask, 1121 const uint8_t *nic_mask, 1122 unsigned int size, 1123 bool range_accepted, 1124 struct rte_flow_error *error) 1125 { 1126 unsigned int i; 1127 1128 MLX5_ASSERT(nic_mask); 1129 for (i = 0; i < size; ++i) 1130 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1131 return rte_flow_error_set(error, ENOTSUP, 1132 RTE_FLOW_ERROR_TYPE_ITEM, 1133 item, 1134 "mask enables non supported" 1135 " bits"); 1136 if (!item->spec && (item->mask || item->last)) 1137 return rte_flow_error_set(error, EINVAL, 1138 RTE_FLOW_ERROR_TYPE_ITEM, item, 1139 "mask/last without a spec is not" 1140 " supported"); 1141 if (item->spec && item->last && !range_accepted) { 1142 uint8_t spec[size]; 1143 uint8_t last[size]; 1144 unsigned int i; 1145 int ret; 1146 1147 for (i = 0; i < size; ++i) { 1148 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1149 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1150 } 1151 ret = memcmp(spec, last, size); 1152 if (ret != 0) 1153 return rte_flow_error_set(error, EINVAL, 1154 RTE_FLOW_ERROR_TYPE_ITEM, 1155 item, 1156 "range is not valid"); 1157 } 1158 return 0; 1159 } 1160 1161 /** 1162 * Adjust the hash fields according to the @p flow information. 1163 * 1164 * @param[in] dev_flow. 1165 * Pointer to the mlx5_flow. 1166 * @param[in] tunnel 1167 * 1 when the hash field is for a tunnel item. 1168 * @param[in] layer_types 1169 * RTE_ETH_RSS_* types. 1170 * @param[in] hash_fields 1171 * Item hash fields. 1172 * 1173 * @return 1174 * The hash fields that should be used. 1175 */ 1176 uint64_t 1177 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1178 int tunnel __rte_unused, uint64_t layer_types, 1179 uint64_t hash_fields) 1180 { 1181 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1182 int rss_request_inner = rss_desc->level >= 2; 1183 1184 /* Check RSS hash level for tunnel. */ 1185 if (tunnel && rss_request_inner) 1186 hash_fields |= IBV_RX_HASH_INNER; 1187 else if (tunnel || rss_request_inner) 1188 return 0; 1189 #endif 1190 /* Check if requested layer matches RSS hash fields. */ 1191 if (!(rss_desc->types & layer_types)) 1192 return 0; 1193 return hash_fields; 1194 } 1195 1196 /** 1197 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1198 * if several tunnel rules are used on this queue, the tunnel ptype will be 1199 * cleared. 1200 * 1201 * @param rxq_ctrl 1202 * Rx queue to update. 1203 */ 1204 static void 1205 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1206 { 1207 unsigned int i; 1208 uint32_t tunnel_ptype = 0; 1209 1210 /* Look up for the ptype to use. */ 1211 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1212 if (!rxq_ctrl->flow_tunnels_n[i]) 1213 continue; 1214 if (!tunnel_ptype) { 1215 tunnel_ptype = tunnels_info[i].ptype; 1216 } else { 1217 tunnel_ptype = 0; 1218 break; 1219 } 1220 } 1221 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1222 } 1223 1224 /** 1225 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1226 * flow. 1227 * 1228 * @param[in] dev 1229 * Pointer to the Ethernet device structure. 1230 * @param[in] dev_handle 1231 * Pointer to device flow handle structure. 1232 */ 1233 void 1234 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1235 struct mlx5_flow_handle *dev_handle) 1236 { 1237 struct mlx5_priv *priv = dev->data->dev_private; 1238 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1239 struct mlx5_ind_table_obj *ind_tbl = NULL; 1240 unsigned int i; 1241 1242 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1243 struct mlx5_hrxq *hrxq; 1244 1245 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1246 dev_handle->rix_hrxq); 1247 if (hrxq) 1248 ind_tbl = hrxq->ind_table; 1249 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1250 struct mlx5_shared_action_rss *shared_rss; 1251 1252 shared_rss = mlx5_ipool_get 1253 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1254 dev_handle->rix_srss); 1255 if (shared_rss) 1256 ind_tbl = shared_rss->ind_tbl; 1257 } 1258 if (!ind_tbl) 1259 return; 1260 for (i = 0; i != ind_tbl->queues_n; ++i) { 1261 int idx = ind_tbl->queues[i]; 1262 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1263 1264 MLX5_ASSERT(rxq_ctrl != NULL); 1265 if (rxq_ctrl == NULL) 1266 continue; 1267 /* 1268 * To support metadata register copy on Tx loopback, 1269 * this must be always enabled (metadata may arive 1270 * from other port - not from local flows only. 1271 */ 1272 if (tunnel) { 1273 unsigned int j; 1274 1275 /* Increase the counter matching the flow. */ 1276 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1277 if ((tunnels_info[j].tunnel & 1278 dev_handle->layers) == 1279 tunnels_info[j].tunnel) { 1280 rxq_ctrl->flow_tunnels_n[j]++; 1281 break; 1282 } 1283 } 1284 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1285 } 1286 } 1287 } 1288 1289 static void 1290 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1291 { 1292 struct mlx5_priv *priv = dev->data->dev_private; 1293 struct mlx5_rxq_ctrl *rxq_ctrl; 1294 1295 if (priv->mark_enabled) 1296 return; 1297 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1298 rxq_ctrl->rxq.mark = 1; 1299 } 1300 priv->mark_enabled = 1; 1301 } 1302 1303 /** 1304 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1305 * 1306 * @param[in] dev 1307 * Pointer to the Ethernet device structure. 1308 * @param[in] flow 1309 * Pointer to flow structure. 1310 */ 1311 static void 1312 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1313 { 1314 struct mlx5_priv *priv = dev->data->dev_private; 1315 uint32_t handle_idx; 1316 struct mlx5_flow_handle *dev_handle; 1317 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1318 1319 MLX5_ASSERT(wks); 1320 if (wks->mark) 1321 flow_rxq_mark_flag_set(dev); 1322 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1323 handle_idx, dev_handle, next) 1324 flow_drv_rxq_flags_set(dev, dev_handle); 1325 } 1326 1327 /** 1328 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1329 * device flow if no other flow uses it with the same kind of request. 1330 * 1331 * @param dev 1332 * Pointer to Ethernet device. 1333 * @param[in] dev_handle 1334 * Pointer to the device flow handle structure. 1335 */ 1336 static void 1337 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1338 struct mlx5_flow_handle *dev_handle) 1339 { 1340 struct mlx5_priv *priv = dev->data->dev_private; 1341 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1342 struct mlx5_ind_table_obj *ind_tbl = NULL; 1343 unsigned int i; 1344 1345 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1346 struct mlx5_hrxq *hrxq; 1347 1348 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1349 dev_handle->rix_hrxq); 1350 if (hrxq) 1351 ind_tbl = hrxq->ind_table; 1352 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1353 struct mlx5_shared_action_rss *shared_rss; 1354 1355 shared_rss = mlx5_ipool_get 1356 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1357 dev_handle->rix_srss); 1358 if (shared_rss) 1359 ind_tbl = shared_rss->ind_tbl; 1360 } 1361 if (!ind_tbl) 1362 return; 1363 MLX5_ASSERT(dev->data->dev_started); 1364 for (i = 0; i != ind_tbl->queues_n; ++i) { 1365 int idx = ind_tbl->queues[i]; 1366 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1367 1368 MLX5_ASSERT(rxq_ctrl != NULL); 1369 if (rxq_ctrl == NULL) 1370 continue; 1371 if (tunnel) { 1372 unsigned int j; 1373 1374 /* Decrease the counter matching the flow. */ 1375 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1376 if ((tunnels_info[j].tunnel & 1377 dev_handle->layers) == 1378 tunnels_info[j].tunnel) { 1379 rxq_ctrl->flow_tunnels_n[j]--; 1380 break; 1381 } 1382 } 1383 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1384 } 1385 } 1386 } 1387 1388 /** 1389 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1390 * @p flow if no other flow uses it with the same kind of request. 1391 * 1392 * @param dev 1393 * Pointer to Ethernet device. 1394 * @param[in] flow 1395 * Pointer to the flow. 1396 */ 1397 static void 1398 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1399 { 1400 struct mlx5_priv *priv = dev->data->dev_private; 1401 uint32_t handle_idx; 1402 struct mlx5_flow_handle *dev_handle; 1403 1404 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1405 handle_idx, dev_handle, next) 1406 flow_drv_rxq_flags_trim(dev, dev_handle); 1407 } 1408 1409 /** 1410 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1411 * 1412 * @param dev 1413 * Pointer to Ethernet device. 1414 */ 1415 static void 1416 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1417 { 1418 struct mlx5_priv *priv = dev->data->dev_private; 1419 unsigned int i; 1420 1421 for (i = 0; i != priv->rxqs_n; ++i) { 1422 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1423 unsigned int j; 1424 1425 if (rxq == NULL || rxq->ctrl == NULL) 1426 continue; 1427 rxq->ctrl->rxq.mark = 0; 1428 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1429 rxq->ctrl->flow_tunnels_n[j] = 0; 1430 rxq->ctrl->rxq.tunnel = 0; 1431 } 1432 priv->mark_enabled = 0; 1433 } 1434 1435 /** 1436 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1437 * 1438 * @param[in] dev 1439 * Pointer to the Ethernet device structure. 1440 */ 1441 void 1442 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1443 { 1444 struct mlx5_priv *priv = dev->data->dev_private; 1445 unsigned int i; 1446 1447 for (i = 0; i != priv->rxqs_n; ++i) { 1448 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1449 struct mlx5_rxq_data *data; 1450 1451 if (rxq == NULL || rxq->ctrl == NULL) 1452 continue; 1453 data = &rxq->ctrl->rxq; 1454 if (!rte_flow_dynf_metadata_avail()) { 1455 data->dynf_meta = 0; 1456 data->flow_meta_mask = 0; 1457 data->flow_meta_offset = -1; 1458 data->flow_meta_port_mask = 0; 1459 } else { 1460 data->dynf_meta = 1; 1461 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1462 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1463 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1464 } 1465 } 1466 } 1467 1468 /* 1469 * return a pointer to the desired action in the list of actions. 1470 * 1471 * @param[in] actions 1472 * The list of actions to search the action in. 1473 * @param[in] action 1474 * The action to find. 1475 * 1476 * @return 1477 * Pointer to the action in the list, if found. NULL otherwise. 1478 */ 1479 const struct rte_flow_action * 1480 mlx5_flow_find_action(const struct rte_flow_action *actions, 1481 enum rte_flow_action_type action) 1482 { 1483 if (actions == NULL) 1484 return NULL; 1485 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1486 if (actions->type == action) 1487 return actions; 1488 return NULL; 1489 } 1490 1491 /* 1492 * Validate the flag action. 1493 * 1494 * @param[in] action_flags 1495 * Bit-fields that holds the actions detected until now. 1496 * @param[in] attr 1497 * Attributes of flow that includes this action. 1498 * @param[out] error 1499 * Pointer to error structure. 1500 * 1501 * @return 1502 * 0 on success, a negative errno value otherwise and rte_errno is set. 1503 */ 1504 int 1505 mlx5_flow_validate_action_flag(uint64_t action_flags, 1506 const struct rte_flow_attr *attr, 1507 struct rte_flow_error *error) 1508 { 1509 if (action_flags & MLX5_FLOW_ACTION_MARK) 1510 return rte_flow_error_set(error, EINVAL, 1511 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1512 "can't mark and flag in same flow"); 1513 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1514 return rte_flow_error_set(error, EINVAL, 1515 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1516 "can't have 2 flag" 1517 " actions in same flow"); 1518 if (attr->egress) 1519 return rte_flow_error_set(error, ENOTSUP, 1520 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1521 "flag action not supported for " 1522 "egress"); 1523 return 0; 1524 } 1525 1526 /* 1527 * Validate the mark action. 1528 * 1529 * @param[in] action 1530 * Pointer to the queue action. 1531 * @param[in] action_flags 1532 * Bit-fields that holds the actions detected until now. 1533 * @param[in] attr 1534 * Attributes of flow that includes this action. 1535 * @param[out] error 1536 * Pointer to error structure. 1537 * 1538 * @return 1539 * 0 on success, a negative errno value otherwise and rte_errno is set. 1540 */ 1541 int 1542 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1543 uint64_t action_flags, 1544 const struct rte_flow_attr *attr, 1545 struct rte_flow_error *error) 1546 { 1547 const struct rte_flow_action_mark *mark = action->conf; 1548 1549 if (!mark) 1550 return rte_flow_error_set(error, EINVAL, 1551 RTE_FLOW_ERROR_TYPE_ACTION, 1552 action, 1553 "configuration cannot be null"); 1554 if (mark->id >= MLX5_FLOW_MARK_MAX) 1555 return rte_flow_error_set(error, EINVAL, 1556 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1557 &mark->id, 1558 "mark id must in 0 <= id < " 1559 RTE_STR(MLX5_FLOW_MARK_MAX)); 1560 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1561 return rte_flow_error_set(error, EINVAL, 1562 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1563 "can't flag and mark in same flow"); 1564 if (action_flags & MLX5_FLOW_ACTION_MARK) 1565 return rte_flow_error_set(error, EINVAL, 1566 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1567 "can't have 2 mark actions in same" 1568 " flow"); 1569 if (attr->egress) 1570 return rte_flow_error_set(error, ENOTSUP, 1571 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1572 "mark action not supported for " 1573 "egress"); 1574 return 0; 1575 } 1576 1577 /* 1578 * Validate the drop action. 1579 * 1580 * @param[in] action_flags 1581 * Bit-fields that holds the actions detected until now. 1582 * @param[in] attr 1583 * Attributes of flow that includes this action. 1584 * @param[out] error 1585 * Pointer to error structure. 1586 * 1587 * @return 1588 * 0 on success, a negative errno value otherwise and rte_errno is set. 1589 */ 1590 int 1591 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1592 const struct rte_flow_attr *attr, 1593 struct rte_flow_error *error) 1594 { 1595 if (attr->egress) 1596 return rte_flow_error_set(error, ENOTSUP, 1597 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1598 "drop action not supported for " 1599 "egress"); 1600 return 0; 1601 } 1602 1603 /* 1604 * Validate the queue action. 1605 * 1606 * @param[in] action 1607 * Pointer to the queue action. 1608 * @param[in] action_flags 1609 * Bit-fields that holds the actions detected until now. 1610 * @param[in] dev 1611 * Pointer to the Ethernet device structure. 1612 * @param[in] attr 1613 * Attributes of flow that includes this action. 1614 * @param[out] error 1615 * Pointer to error structure. 1616 * 1617 * @return 1618 * 0 on success, a negative errno value otherwise and rte_errno is set. 1619 */ 1620 int 1621 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1622 uint64_t action_flags, 1623 struct rte_eth_dev *dev, 1624 const struct rte_flow_attr *attr, 1625 struct rte_flow_error *error) 1626 { 1627 struct mlx5_priv *priv = dev->data->dev_private; 1628 const struct rte_flow_action_queue *queue = action->conf; 1629 1630 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1631 return rte_flow_error_set(error, EINVAL, 1632 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1633 "can't have 2 fate actions in" 1634 " same flow"); 1635 if (!priv->rxqs_n) 1636 return rte_flow_error_set(error, EINVAL, 1637 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1638 NULL, "No Rx queues configured"); 1639 if (queue->index >= priv->rxqs_n) 1640 return rte_flow_error_set(error, EINVAL, 1641 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1642 &queue->index, 1643 "queue index out of range"); 1644 if (mlx5_rxq_get(dev, queue->index) == NULL) 1645 return rte_flow_error_set(error, EINVAL, 1646 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1647 &queue->index, 1648 "queue is not configured"); 1649 if (attr->egress) 1650 return rte_flow_error_set(error, ENOTSUP, 1651 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1652 "queue action not supported for " 1653 "egress"); 1654 return 0; 1655 } 1656 1657 /** 1658 * Validate queue numbers for device RSS. 1659 * 1660 * @param[in] dev 1661 * Configured device. 1662 * @param[in] queues 1663 * Array of queue numbers. 1664 * @param[in] queues_n 1665 * Size of the @p queues array. 1666 * @param[out] error 1667 * On error, filled with a textual error description. 1668 * @param[out] queue 1669 * On error, filled with an offending queue index in @p queues array. 1670 * 1671 * @return 1672 * 0 on success, a negative errno code on error. 1673 */ 1674 static int 1675 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1676 const uint16_t *queues, uint32_t queues_n, 1677 const char **error, uint32_t *queue_idx) 1678 { 1679 const struct mlx5_priv *priv = dev->data->dev_private; 1680 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1681 uint32_t i; 1682 1683 for (i = 0; i != queues_n; ++i) { 1684 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, 1685 queues[i]); 1686 1687 if (queues[i] >= priv->rxqs_n) { 1688 *error = "queue index out of range"; 1689 *queue_idx = i; 1690 return -EINVAL; 1691 } 1692 if (rxq_ctrl == NULL) { 1693 *error = "queue is not configured"; 1694 *queue_idx = i; 1695 return -EINVAL; 1696 } 1697 if (i == 0) 1698 rxq_type = rxq_ctrl->type; 1699 if (rxq_type != rxq_ctrl->type) { 1700 *error = "combining hairpin and regular RSS queues is not supported"; 1701 *queue_idx = i; 1702 return -ENOTSUP; 1703 } 1704 } 1705 return 0; 1706 } 1707 1708 /* 1709 * Validate the rss action. 1710 * 1711 * @param[in] dev 1712 * Pointer to the Ethernet device structure. 1713 * @param[in] action 1714 * Pointer to the queue action. 1715 * @param[out] error 1716 * Pointer to error structure. 1717 * 1718 * @return 1719 * 0 on success, a negative errno value otherwise and rte_errno is set. 1720 */ 1721 int 1722 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1723 const struct rte_flow_action *action, 1724 struct rte_flow_error *error) 1725 { 1726 struct mlx5_priv *priv = dev->data->dev_private; 1727 const struct rte_flow_action_rss *rss = action->conf; 1728 int ret; 1729 const char *message; 1730 uint32_t queue_idx; 1731 1732 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1733 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1734 return rte_flow_error_set(error, ENOTSUP, 1735 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1736 &rss->func, 1737 "RSS hash function not supported"); 1738 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1739 if (rss->level > 2) 1740 #else 1741 if (rss->level > 1) 1742 #endif 1743 return rte_flow_error_set(error, ENOTSUP, 1744 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1745 &rss->level, 1746 "tunnel RSS is not supported"); 1747 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1748 if (rss->key_len == 0 && rss->key != NULL) 1749 return rte_flow_error_set(error, ENOTSUP, 1750 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1751 &rss->key_len, 1752 "RSS hash key length 0"); 1753 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1754 return rte_flow_error_set(error, ENOTSUP, 1755 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1756 &rss->key_len, 1757 "RSS hash key too small"); 1758 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1759 return rte_flow_error_set(error, ENOTSUP, 1760 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1761 &rss->key_len, 1762 "RSS hash key too large"); 1763 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1764 return rte_flow_error_set(error, ENOTSUP, 1765 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1766 &rss->queue_num, 1767 "number of queues too large"); 1768 if (rss->types & MLX5_RSS_HF_MASK) 1769 return rte_flow_error_set(error, ENOTSUP, 1770 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1771 &rss->types, 1772 "some RSS protocols are not" 1773 " supported"); 1774 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1775 !(rss->types & RTE_ETH_RSS_IP)) 1776 return rte_flow_error_set(error, EINVAL, 1777 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1778 "L3 partial RSS requested but L3 RSS" 1779 " type not specified"); 1780 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1781 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1782 return rte_flow_error_set(error, EINVAL, 1783 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1784 "L4 partial RSS requested but L4 RSS" 1785 " type not specified"); 1786 if (!priv->rxqs_n) 1787 return rte_flow_error_set(error, EINVAL, 1788 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1789 NULL, "No Rx queues configured"); 1790 if (!rss->queue_num) 1791 return rte_flow_error_set(error, EINVAL, 1792 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1793 NULL, "No queues configured"); 1794 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1795 &message, &queue_idx); 1796 if (ret != 0) { 1797 return rte_flow_error_set(error, -ret, 1798 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1799 &rss->queue[queue_idx], message); 1800 } 1801 return 0; 1802 } 1803 1804 /* 1805 * Validate the rss action. 1806 * 1807 * @param[in] action 1808 * Pointer to the queue action. 1809 * @param[in] action_flags 1810 * Bit-fields that holds the actions detected until now. 1811 * @param[in] dev 1812 * Pointer to the Ethernet device structure. 1813 * @param[in] attr 1814 * Attributes of flow that includes this action. 1815 * @param[in] item_flags 1816 * Items that were detected. 1817 * @param[out] error 1818 * Pointer to error structure. 1819 * 1820 * @return 1821 * 0 on success, a negative errno value otherwise and rte_errno is set. 1822 */ 1823 int 1824 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1825 uint64_t action_flags, 1826 struct rte_eth_dev *dev, 1827 const struct rte_flow_attr *attr, 1828 uint64_t item_flags, 1829 struct rte_flow_error *error) 1830 { 1831 const struct rte_flow_action_rss *rss = action->conf; 1832 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1833 int ret; 1834 1835 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1836 return rte_flow_error_set(error, EINVAL, 1837 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1838 "can't have 2 fate actions" 1839 " in same flow"); 1840 ret = mlx5_validate_action_rss(dev, action, error); 1841 if (ret) 1842 return ret; 1843 if (attr->egress) 1844 return rte_flow_error_set(error, ENOTSUP, 1845 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1846 "rss action not supported for " 1847 "egress"); 1848 if (rss->level > 1 && !tunnel) 1849 return rte_flow_error_set(error, EINVAL, 1850 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1851 "inner RSS is not supported for " 1852 "non-tunnel flows"); 1853 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1854 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1855 return rte_flow_error_set(error, EINVAL, 1856 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1857 "RSS on eCPRI is not supported now"); 1858 } 1859 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1860 !(item_flags & 1861 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1862 rss->level > 1) 1863 return rte_flow_error_set(error, EINVAL, 1864 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1865 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1866 return 0; 1867 } 1868 1869 /* 1870 * Validate the default miss action. 1871 * 1872 * @param[in] action_flags 1873 * Bit-fields that holds the actions detected until now. 1874 * @param[out] error 1875 * Pointer to error structure. 1876 * 1877 * @return 1878 * 0 on success, a negative errno value otherwise and rte_errno is set. 1879 */ 1880 int 1881 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1882 const struct rte_flow_attr *attr, 1883 struct rte_flow_error *error) 1884 { 1885 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1886 return rte_flow_error_set(error, EINVAL, 1887 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1888 "can't have 2 fate actions in" 1889 " same flow"); 1890 if (attr->egress) 1891 return rte_flow_error_set(error, ENOTSUP, 1892 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1893 "default miss action not supported " 1894 "for egress"); 1895 if (attr->group) 1896 return rte_flow_error_set(error, ENOTSUP, 1897 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1898 "only group 0 is supported"); 1899 if (attr->transfer) 1900 return rte_flow_error_set(error, ENOTSUP, 1901 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1902 NULL, "transfer is not supported"); 1903 return 0; 1904 } 1905 1906 /* 1907 * Validate the count action. 1908 * 1909 * @param[in] dev 1910 * Pointer to the Ethernet device structure. 1911 * @param[in] attr 1912 * Attributes of flow that includes this action. 1913 * @param[out] error 1914 * Pointer to error structure. 1915 * 1916 * @return 1917 * 0 on success, a negative errno value otherwise and rte_errno is set. 1918 */ 1919 int 1920 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1921 const struct rte_flow_attr *attr, 1922 struct rte_flow_error *error) 1923 { 1924 if (attr->egress) 1925 return rte_flow_error_set(error, ENOTSUP, 1926 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1927 "count action not supported for " 1928 "egress"); 1929 return 0; 1930 } 1931 1932 /* 1933 * Validate the ASO CT action. 1934 * 1935 * @param[in] dev 1936 * Pointer to the Ethernet device structure. 1937 * @param[in] conntrack 1938 * Pointer to the CT action profile. 1939 * @param[out] error 1940 * Pointer to error structure. 1941 * 1942 * @return 1943 * 0 on success, a negative errno value otherwise and rte_errno is set. 1944 */ 1945 int 1946 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1947 const struct rte_flow_action_conntrack *conntrack, 1948 struct rte_flow_error *error) 1949 { 1950 RTE_SET_USED(dev); 1951 1952 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1953 return rte_flow_error_set(error, EINVAL, 1954 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1955 "Invalid CT state"); 1956 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1957 return rte_flow_error_set(error, EINVAL, 1958 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1959 "Invalid last TCP packet flag"); 1960 return 0; 1961 } 1962 1963 /** 1964 * Verify the @p attributes will be correctly understood by the NIC and store 1965 * them in the @p flow if everything is correct. 1966 * 1967 * @param[in] dev 1968 * Pointer to the Ethernet device structure. 1969 * @param[in] attributes 1970 * Pointer to flow attributes 1971 * @param[out] error 1972 * Pointer to error structure. 1973 * 1974 * @return 1975 * 0 on success, a negative errno value otherwise and rte_errno is set. 1976 */ 1977 int 1978 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1979 const struct rte_flow_attr *attributes, 1980 struct rte_flow_error *error) 1981 { 1982 struct mlx5_priv *priv = dev->data->dev_private; 1983 uint32_t priority_max = priv->sh->flow_max_priority - 1; 1984 1985 if (attributes->group) 1986 return rte_flow_error_set(error, ENOTSUP, 1987 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1988 NULL, "groups is not supported"); 1989 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1990 attributes->priority >= priority_max) 1991 return rte_flow_error_set(error, ENOTSUP, 1992 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1993 NULL, "priority out of range"); 1994 if (attributes->egress) 1995 return rte_flow_error_set(error, ENOTSUP, 1996 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1997 "egress is not supported"); 1998 if (attributes->transfer && !priv->sh->config.dv_esw_en) 1999 return rte_flow_error_set(error, ENOTSUP, 2000 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2001 NULL, "transfer is not supported"); 2002 if (!attributes->ingress) 2003 return rte_flow_error_set(error, EINVAL, 2004 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2005 NULL, 2006 "ingress attribute is mandatory"); 2007 return 0; 2008 } 2009 2010 /** 2011 * Validate ICMP6 item. 2012 * 2013 * @param[in] item 2014 * Item specification. 2015 * @param[in] item_flags 2016 * Bit-fields that holds the items detected until now. 2017 * @param[in] ext_vlan_sup 2018 * Whether extended VLAN features are supported or not. 2019 * @param[out] error 2020 * Pointer to error structure. 2021 * 2022 * @return 2023 * 0 on success, a negative errno value otherwise and rte_errno is set. 2024 */ 2025 int 2026 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2027 uint64_t item_flags, 2028 uint8_t target_protocol, 2029 struct rte_flow_error *error) 2030 { 2031 const struct rte_flow_item_icmp6 *mask = item->mask; 2032 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2033 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2034 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2035 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2036 MLX5_FLOW_LAYER_OUTER_L4; 2037 int ret; 2038 2039 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2040 return rte_flow_error_set(error, EINVAL, 2041 RTE_FLOW_ERROR_TYPE_ITEM, item, 2042 "protocol filtering not compatible" 2043 " with ICMP6 layer"); 2044 if (!(item_flags & l3m)) 2045 return rte_flow_error_set(error, EINVAL, 2046 RTE_FLOW_ERROR_TYPE_ITEM, item, 2047 "IPv6 is mandatory to filter on" 2048 " ICMP6"); 2049 if (item_flags & l4m) 2050 return rte_flow_error_set(error, EINVAL, 2051 RTE_FLOW_ERROR_TYPE_ITEM, item, 2052 "multiple L4 layers not supported"); 2053 if (!mask) 2054 mask = &rte_flow_item_icmp6_mask; 2055 ret = mlx5_flow_item_acceptable 2056 (item, (const uint8_t *)mask, 2057 (const uint8_t *)&rte_flow_item_icmp6_mask, 2058 sizeof(struct rte_flow_item_icmp6), 2059 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2060 if (ret < 0) 2061 return ret; 2062 return 0; 2063 } 2064 2065 /** 2066 * Validate ICMP item. 2067 * 2068 * @param[in] item 2069 * Item specification. 2070 * @param[in] item_flags 2071 * Bit-fields that holds the items detected until now. 2072 * @param[out] error 2073 * Pointer to error structure. 2074 * 2075 * @return 2076 * 0 on success, a negative errno value otherwise and rte_errno is set. 2077 */ 2078 int 2079 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2080 uint64_t item_flags, 2081 uint8_t target_protocol, 2082 struct rte_flow_error *error) 2083 { 2084 const struct rte_flow_item_icmp *mask = item->mask; 2085 const struct rte_flow_item_icmp nic_mask = { 2086 .hdr.icmp_type = 0xff, 2087 .hdr.icmp_code = 0xff, 2088 .hdr.icmp_ident = RTE_BE16(0xffff), 2089 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2090 }; 2091 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2092 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2093 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2094 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2095 MLX5_FLOW_LAYER_OUTER_L4; 2096 int ret; 2097 2098 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2099 return rte_flow_error_set(error, EINVAL, 2100 RTE_FLOW_ERROR_TYPE_ITEM, item, 2101 "protocol filtering not compatible" 2102 " with ICMP layer"); 2103 if (!(item_flags & l3m)) 2104 return rte_flow_error_set(error, EINVAL, 2105 RTE_FLOW_ERROR_TYPE_ITEM, item, 2106 "IPv4 is mandatory to filter" 2107 " on ICMP"); 2108 if (item_flags & l4m) 2109 return rte_flow_error_set(error, EINVAL, 2110 RTE_FLOW_ERROR_TYPE_ITEM, item, 2111 "multiple L4 layers not supported"); 2112 if (!mask) 2113 mask = &nic_mask; 2114 ret = mlx5_flow_item_acceptable 2115 (item, (const uint8_t *)mask, 2116 (const uint8_t *)&nic_mask, 2117 sizeof(struct rte_flow_item_icmp), 2118 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2119 if (ret < 0) 2120 return ret; 2121 return 0; 2122 } 2123 2124 /** 2125 * Validate Ethernet item. 2126 * 2127 * @param[in] item 2128 * Item specification. 2129 * @param[in] item_flags 2130 * Bit-fields that holds the items detected until now. 2131 * @param[out] error 2132 * Pointer to error structure. 2133 * 2134 * @return 2135 * 0 on success, a negative errno value otherwise and rte_errno is set. 2136 */ 2137 int 2138 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2139 uint64_t item_flags, bool ext_vlan_sup, 2140 struct rte_flow_error *error) 2141 { 2142 const struct rte_flow_item_eth *mask = item->mask; 2143 const struct rte_flow_item_eth nic_mask = { 2144 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2145 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2146 .type = RTE_BE16(0xffff), 2147 .has_vlan = ext_vlan_sup ? 1 : 0, 2148 }; 2149 int ret; 2150 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2151 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2152 MLX5_FLOW_LAYER_OUTER_L2; 2153 2154 if (item_flags & ethm) 2155 return rte_flow_error_set(error, ENOTSUP, 2156 RTE_FLOW_ERROR_TYPE_ITEM, item, 2157 "multiple L2 layers not supported"); 2158 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2159 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2160 return rte_flow_error_set(error, EINVAL, 2161 RTE_FLOW_ERROR_TYPE_ITEM, item, 2162 "L2 layer should not follow " 2163 "L3 layers"); 2164 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2165 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2166 return rte_flow_error_set(error, EINVAL, 2167 RTE_FLOW_ERROR_TYPE_ITEM, item, 2168 "L2 layer should not follow VLAN"); 2169 if (item_flags & MLX5_FLOW_LAYER_GTP) 2170 return rte_flow_error_set(error, EINVAL, 2171 RTE_FLOW_ERROR_TYPE_ITEM, item, 2172 "L2 layer should not follow GTP"); 2173 if (!mask) 2174 mask = &rte_flow_item_eth_mask; 2175 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2176 (const uint8_t *)&nic_mask, 2177 sizeof(struct rte_flow_item_eth), 2178 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2179 return ret; 2180 } 2181 2182 /** 2183 * Validate VLAN item. 2184 * 2185 * @param[in] item 2186 * Item specification. 2187 * @param[in] item_flags 2188 * Bit-fields that holds the items detected until now. 2189 * @param[in] dev 2190 * Ethernet device flow is being created on. 2191 * @param[out] error 2192 * Pointer to error structure. 2193 * 2194 * @return 2195 * 0 on success, a negative errno value otherwise and rte_errno is set. 2196 */ 2197 int 2198 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2199 uint64_t item_flags, 2200 struct rte_eth_dev *dev, 2201 struct rte_flow_error *error) 2202 { 2203 const struct rte_flow_item_vlan *spec = item->spec; 2204 const struct rte_flow_item_vlan *mask = item->mask; 2205 const struct rte_flow_item_vlan nic_mask = { 2206 .tci = RTE_BE16(UINT16_MAX), 2207 .inner_type = RTE_BE16(UINT16_MAX), 2208 }; 2209 uint16_t vlan_tag = 0; 2210 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2211 int ret; 2212 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2213 MLX5_FLOW_LAYER_INNER_L4) : 2214 (MLX5_FLOW_LAYER_OUTER_L3 | 2215 MLX5_FLOW_LAYER_OUTER_L4); 2216 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2217 MLX5_FLOW_LAYER_OUTER_VLAN; 2218 2219 if (item_flags & vlanm) 2220 return rte_flow_error_set(error, EINVAL, 2221 RTE_FLOW_ERROR_TYPE_ITEM, item, 2222 "multiple VLAN layers not supported"); 2223 else if ((item_flags & l34m) != 0) 2224 return rte_flow_error_set(error, EINVAL, 2225 RTE_FLOW_ERROR_TYPE_ITEM, item, 2226 "VLAN cannot follow L3/L4 layer"); 2227 if (!mask) 2228 mask = &rte_flow_item_vlan_mask; 2229 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2230 (const uint8_t *)&nic_mask, 2231 sizeof(struct rte_flow_item_vlan), 2232 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2233 if (ret) 2234 return ret; 2235 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2236 struct mlx5_priv *priv = dev->data->dev_private; 2237 2238 if (priv->vmwa_context) { 2239 /* 2240 * Non-NULL context means we have a virtual machine 2241 * and SR-IOV enabled, we have to create VLAN interface 2242 * to make hypervisor to setup E-Switch vport 2243 * context correctly. We avoid creating the multiple 2244 * VLAN interfaces, so we cannot support VLAN tag mask. 2245 */ 2246 return rte_flow_error_set(error, EINVAL, 2247 RTE_FLOW_ERROR_TYPE_ITEM, 2248 item, 2249 "VLAN tag mask is not" 2250 " supported in virtual" 2251 " environment"); 2252 } 2253 } 2254 if (spec) { 2255 vlan_tag = spec->tci; 2256 vlan_tag &= mask->tci; 2257 } 2258 /* 2259 * From verbs perspective an empty VLAN is equivalent 2260 * to a packet without VLAN layer. 2261 */ 2262 if (!vlan_tag) 2263 return rte_flow_error_set(error, EINVAL, 2264 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2265 item->spec, 2266 "VLAN cannot be empty"); 2267 return 0; 2268 } 2269 2270 /** 2271 * Validate IPV4 item. 2272 * 2273 * @param[in] item 2274 * Item specification. 2275 * @param[in] item_flags 2276 * Bit-fields that holds the items detected until now. 2277 * @param[in] last_item 2278 * Previous validated item in the pattern items. 2279 * @param[in] ether_type 2280 * Type in the ethernet layer header (including dot1q). 2281 * @param[in] acc_mask 2282 * Acceptable mask, if NULL default internal default mask 2283 * will be used to check whether item fields are supported. 2284 * @param[in] range_accepted 2285 * True if range of values is accepted for specific fields, false otherwise. 2286 * @param[out] error 2287 * Pointer to error structure. 2288 * 2289 * @return 2290 * 0 on success, a negative errno value otherwise and rte_errno is set. 2291 */ 2292 int 2293 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2294 uint64_t item_flags, 2295 uint64_t last_item, 2296 uint16_t ether_type, 2297 const struct rte_flow_item_ipv4 *acc_mask, 2298 bool range_accepted, 2299 struct rte_flow_error *error) 2300 { 2301 const struct rte_flow_item_ipv4 *mask = item->mask; 2302 const struct rte_flow_item_ipv4 *spec = item->spec; 2303 const struct rte_flow_item_ipv4 nic_mask = { 2304 .hdr = { 2305 .src_addr = RTE_BE32(0xffffffff), 2306 .dst_addr = RTE_BE32(0xffffffff), 2307 .type_of_service = 0xff, 2308 .next_proto_id = 0xff, 2309 }, 2310 }; 2311 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2312 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2313 MLX5_FLOW_LAYER_OUTER_L3; 2314 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2315 MLX5_FLOW_LAYER_OUTER_L4; 2316 int ret; 2317 uint8_t next_proto = 0xFF; 2318 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2319 MLX5_FLOW_LAYER_OUTER_VLAN | 2320 MLX5_FLOW_LAYER_INNER_VLAN); 2321 2322 if ((last_item & l2_vlan) && ether_type && 2323 ether_type != RTE_ETHER_TYPE_IPV4) 2324 return rte_flow_error_set(error, EINVAL, 2325 RTE_FLOW_ERROR_TYPE_ITEM, item, 2326 "IPv4 cannot follow L2/VLAN layer " 2327 "which ether type is not IPv4"); 2328 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2329 if (mask && spec) 2330 next_proto = mask->hdr.next_proto_id & 2331 spec->hdr.next_proto_id; 2332 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2333 return rte_flow_error_set(error, EINVAL, 2334 RTE_FLOW_ERROR_TYPE_ITEM, 2335 item, 2336 "multiple tunnel " 2337 "not supported"); 2338 } 2339 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2340 return rte_flow_error_set(error, EINVAL, 2341 RTE_FLOW_ERROR_TYPE_ITEM, item, 2342 "wrong tunnel type - IPv6 specified " 2343 "but IPv4 item provided"); 2344 if (item_flags & l3m) 2345 return rte_flow_error_set(error, ENOTSUP, 2346 RTE_FLOW_ERROR_TYPE_ITEM, item, 2347 "multiple L3 layers not supported"); 2348 else if (item_flags & l4m) 2349 return rte_flow_error_set(error, EINVAL, 2350 RTE_FLOW_ERROR_TYPE_ITEM, item, 2351 "L3 cannot follow an L4 layer."); 2352 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2353 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2354 return rte_flow_error_set(error, EINVAL, 2355 RTE_FLOW_ERROR_TYPE_ITEM, item, 2356 "L3 cannot follow an NVGRE layer."); 2357 if (!mask) 2358 mask = &rte_flow_item_ipv4_mask; 2359 else if (mask->hdr.next_proto_id != 0 && 2360 mask->hdr.next_proto_id != 0xff) 2361 return rte_flow_error_set(error, EINVAL, 2362 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2363 "partial mask is not supported" 2364 " for protocol"); 2365 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2366 acc_mask ? (const uint8_t *)acc_mask 2367 : (const uint8_t *)&nic_mask, 2368 sizeof(struct rte_flow_item_ipv4), 2369 range_accepted, error); 2370 if (ret < 0) 2371 return ret; 2372 return 0; 2373 } 2374 2375 /** 2376 * Validate IPV6 item. 2377 * 2378 * @param[in] item 2379 * Item specification. 2380 * @param[in] item_flags 2381 * Bit-fields that holds the items detected until now. 2382 * @param[in] last_item 2383 * Previous validated item in the pattern items. 2384 * @param[in] ether_type 2385 * Type in the ethernet layer header (including dot1q). 2386 * @param[in] acc_mask 2387 * Acceptable mask, if NULL default internal default mask 2388 * will be used to check whether item fields are supported. 2389 * @param[out] error 2390 * Pointer to error structure. 2391 * 2392 * @return 2393 * 0 on success, a negative errno value otherwise and rte_errno is set. 2394 */ 2395 int 2396 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2397 uint64_t item_flags, 2398 uint64_t last_item, 2399 uint16_t ether_type, 2400 const struct rte_flow_item_ipv6 *acc_mask, 2401 struct rte_flow_error *error) 2402 { 2403 const struct rte_flow_item_ipv6 *mask = item->mask; 2404 const struct rte_flow_item_ipv6 *spec = item->spec; 2405 const struct rte_flow_item_ipv6 nic_mask = { 2406 .hdr = { 2407 .src_addr = 2408 "\xff\xff\xff\xff\xff\xff\xff\xff" 2409 "\xff\xff\xff\xff\xff\xff\xff\xff", 2410 .dst_addr = 2411 "\xff\xff\xff\xff\xff\xff\xff\xff" 2412 "\xff\xff\xff\xff\xff\xff\xff\xff", 2413 .vtc_flow = RTE_BE32(0xffffffff), 2414 .proto = 0xff, 2415 }, 2416 }; 2417 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2418 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2419 MLX5_FLOW_LAYER_OUTER_L3; 2420 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2421 MLX5_FLOW_LAYER_OUTER_L4; 2422 int ret; 2423 uint8_t next_proto = 0xFF; 2424 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2425 MLX5_FLOW_LAYER_OUTER_VLAN | 2426 MLX5_FLOW_LAYER_INNER_VLAN); 2427 2428 if ((last_item & l2_vlan) && ether_type && 2429 ether_type != RTE_ETHER_TYPE_IPV6) 2430 return rte_flow_error_set(error, EINVAL, 2431 RTE_FLOW_ERROR_TYPE_ITEM, item, 2432 "IPv6 cannot follow L2/VLAN layer " 2433 "which ether type is not IPv6"); 2434 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2435 next_proto = spec->hdr.proto; 2436 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2437 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2438 return rte_flow_error_set(error, EINVAL, 2439 RTE_FLOW_ERROR_TYPE_ITEM, 2440 item, 2441 "multiple tunnel " 2442 "not supported"); 2443 } 2444 if (next_proto == IPPROTO_HOPOPTS || 2445 next_proto == IPPROTO_ROUTING || 2446 next_proto == IPPROTO_FRAGMENT || 2447 next_proto == IPPROTO_ESP || 2448 next_proto == IPPROTO_AH || 2449 next_proto == IPPROTO_DSTOPTS) 2450 return rte_flow_error_set(error, EINVAL, 2451 RTE_FLOW_ERROR_TYPE_ITEM, item, 2452 "IPv6 proto (next header) should " 2453 "not be set as extension header"); 2454 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2455 return rte_flow_error_set(error, EINVAL, 2456 RTE_FLOW_ERROR_TYPE_ITEM, item, 2457 "wrong tunnel type - IPv4 specified " 2458 "but IPv6 item provided"); 2459 if (item_flags & l3m) 2460 return rte_flow_error_set(error, ENOTSUP, 2461 RTE_FLOW_ERROR_TYPE_ITEM, item, 2462 "multiple L3 layers not supported"); 2463 else if (item_flags & l4m) 2464 return rte_flow_error_set(error, EINVAL, 2465 RTE_FLOW_ERROR_TYPE_ITEM, item, 2466 "L3 cannot follow an L4 layer."); 2467 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2468 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2469 return rte_flow_error_set(error, EINVAL, 2470 RTE_FLOW_ERROR_TYPE_ITEM, item, 2471 "L3 cannot follow an NVGRE layer."); 2472 if (!mask) 2473 mask = &rte_flow_item_ipv6_mask; 2474 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2475 acc_mask ? (const uint8_t *)acc_mask 2476 : (const uint8_t *)&nic_mask, 2477 sizeof(struct rte_flow_item_ipv6), 2478 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2479 if (ret < 0) 2480 return ret; 2481 return 0; 2482 } 2483 2484 /** 2485 * Validate UDP item. 2486 * 2487 * @param[in] item 2488 * Item specification. 2489 * @param[in] item_flags 2490 * Bit-fields that holds the items detected until now. 2491 * @param[in] target_protocol 2492 * The next protocol in the previous item. 2493 * @param[in] flow_mask 2494 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2495 * @param[out] error 2496 * Pointer to error structure. 2497 * 2498 * @return 2499 * 0 on success, a negative errno value otherwise and rte_errno is set. 2500 */ 2501 int 2502 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2503 uint64_t item_flags, 2504 uint8_t target_protocol, 2505 struct rte_flow_error *error) 2506 { 2507 const struct rte_flow_item_udp *mask = item->mask; 2508 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2509 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2510 MLX5_FLOW_LAYER_OUTER_L3; 2511 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2512 MLX5_FLOW_LAYER_OUTER_L4; 2513 int ret; 2514 2515 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2516 return rte_flow_error_set(error, EINVAL, 2517 RTE_FLOW_ERROR_TYPE_ITEM, item, 2518 "protocol filtering not compatible" 2519 " with UDP layer"); 2520 if (!(item_flags & l3m)) 2521 return rte_flow_error_set(error, EINVAL, 2522 RTE_FLOW_ERROR_TYPE_ITEM, item, 2523 "L3 is mandatory to filter on L4"); 2524 if (item_flags & l4m) 2525 return rte_flow_error_set(error, EINVAL, 2526 RTE_FLOW_ERROR_TYPE_ITEM, item, 2527 "multiple L4 layers not supported"); 2528 if (!mask) 2529 mask = &rte_flow_item_udp_mask; 2530 ret = mlx5_flow_item_acceptable 2531 (item, (const uint8_t *)mask, 2532 (const uint8_t *)&rte_flow_item_udp_mask, 2533 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2534 error); 2535 if (ret < 0) 2536 return ret; 2537 return 0; 2538 } 2539 2540 /** 2541 * Validate TCP item. 2542 * 2543 * @param[in] item 2544 * Item specification. 2545 * @param[in] item_flags 2546 * Bit-fields that holds the items detected until now. 2547 * @param[in] target_protocol 2548 * The next protocol in the previous item. 2549 * @param[out] error 2550 * Pointer to error structure. 2551 * 2552 * @return 2553 * 0 on success, a negative errno value otherwise and rte_errno is set. 2554 */ 2555 int 2556 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2557 uint64_t item_flags, 2558 uint8_t target_protocol, 2559 const struct rte_flow_item_tcp *flow_mask, 2560 struct rte_flow_error *error) 2561 { 2562 const struct rte_flow_item_tcp *mask = item->mask; 2563 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2564 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2565 MLX5_FLOW_LAYER_OUTER_L3; 2566 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2567 MLX5_FLOW_LAYER_OUTER_L4; 2568 int ret; 2569 2570 MLX5_ASSERT(flow_mask); 2571 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2572 return rte_flow_error_set(error, EINVAL, 2573 RTE_FLOW_ERROR_TYPE_ITEM, item, 2574 "protocol filtering not compatible" 2575 " with TCP layer"); 2576 if (!(item_flags & l3m)) 2577 return rte_flow_error_set(error, EINVAL, 2578 RTE_FLOW_ERROR_TYPE_ITEM, item, 2579 "L3 is mandatory to filter on L4"); 2580 if (item_flags & l4m) 2581 return rte_flow_error_set(error, EINVAL, 2582 RTE_FLOW_ERROR_TYPE_ITEM, item, 2583 "multiple L4 layers not supported"); 2584 if (!mask) 2585 mask = &rte_flow_item_tcp_mask; 2586 ret = mlx5_flow_item_acceptable 2587 (item, (const uint8_t *)mask, 2588 (const uint8_t *)flow_mask, 2589 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2590 error); 2591 if (ret < 0) 2592 return ret; 2593 return 0; 2594 } 2595 2596 /** 2597 * Validate VXLAN item. 2598 * 2599 * @param[in] dev 2600 * Pointer to the Ethernet device structure. 2601 * @param[in] udp_dport 2602 * UDP destination port 2603 * @param[in] item 2604 * Item specification. 2605 * @param[in] item_flags 2606 * Bit-fields that holds the items detected until now. 2607 * @param[in] attr 2608 * Flow rule attributes. 2609 * @param[out] error 2610 * Pointer to error structure. 2611 * 2612 * @return 2613 * 0 on success, a negative errno value otherwise and rte_errno is set. 2614 */ 2615 int 2616 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2617 uint16_t udp_dport, 2618 const struct rte_flow_item *item, 2619 uint64_t item_flags, 2620 const struct rte_flow_attr *attr, 2621 struct rte_flow_error *error) 2622 { 2623 const struct rte_flow_item_vxlan *spec = item->spec; 2624 const struct rte_flow_item_vxlan *mask = item->mask; 2625 int ret; 2626 struct mlx5_priv *priv = dev->data->dev_private; 2627 union vni { 2628 uint32_t vlan_id; 2629 uint8_t vni[4]; 2630 } id = { .vlan_id = 0, }; 2631 const struct rte_flow_item_vxlan nic_mask = { 2632 .vni = "\xff\xff\xff", 2633 .rsvd1 = 0xff, 2634 }; 2635 const struct rte_flow_item_vxlan *valid_mask; 2636 2637 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2638 return rte_flow_error_set(error, ENOTSUP, 2639 RTE_FLOW_ERROR_TYPE_ITEM, item, 2640 "multiple tunnel layers not" 2641 " supported"); 2642 valid_mask = &rte_flow_item_vxlan_mask; 2643 /* 2644 * Verify only UDPv4 is present as defined in 2645 * https://tools.ietf.org/html/rfc7348 2646 */ 2647 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2648 return rte_flow_error_set(error, EINVAL, 2649 RTE_FLOW_ERROR_TYPE_ITEM, item, 2650 "no outer UDP layer found"); 2651 if (!mask) 2652 mask = &rte_flow_item_vxlan_mask; 2653 2654 if (priv->sh->steering_format_version != 2655 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2656 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2657 /* FDB domain & NIC domain non-zero group */ 2658 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2659 valid_mask = &nic_mask; 2660 /* Group zero in NIC domain */ 2661 if (!attr->group && !attr->transfer && 2662 priv->sh->tunnel_header_0_1) 2663 valid_mask = &nic_mask; 2664 } 2665 ret = mlx5_flow_item_acceptable 2666 (item, (const uint8_t *)mask, 2667 (const uint8_t *)valid_mask, 2668 sizeof(struct rte_flow_item_vxlan), 2669 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2670 if (ret < 0) 2671 return ret; 2672 if (spec) { 2673 memcpy(&id.vni[1], spec->vni, 3); 2674 memcpy(&id.vni[1], mask->vni, 3); 2675 } 2676 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2677 return rte_flow_error_set(error, ENOTSUP, 2678 RTE_FLOW_ERROR_TYPE_ITEM, item, 2679 "VXLAN tunnel must be fully defined"); 2680 return 0; 2681 } 2682 2683 /** 2684 * Validate VXLAN_GPE item. 2685 * 2686 * @param[in] item 2687 * Item specification. 2688 * @param[in] item_flags 2689 * Bit-fields that holds the items detected until now. 2690 * @param[in] priv 2691 * Pointer to the private data structure. 2692 * @param[in] target_protocol 2693 * The next protocol in the previous item. 2694 * @param[out] error 2695 * Pointer to error structure. 2696 * 2697 * @return 2698 * 0 on success, a negative errno value otherwise and rte_errno is set. 2699 */ 2700 int 2701 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2702 uint64_t item_flags, 2703 struct rte_eth_dev *dev, 2704 struct rte_flow_error *error) 2705 { 2706 struct mlx5_priv *priv = dev->data->dev_private; 2707 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2708 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2709 int ret; 2710 union vni { 2711 uint32_t vlan_id; 2712 uint8_t vni[4]; 2713 } id = { .vlan_id = 0, }; 2714 2715 if (!priv->sh->config.l3_vxlan_en) 2716 return rte_flow_error_set(error, ENOTSUP, 2717 RTE_FLOW_ERROR_TYPE_ITEM, item, 2718 "L3 VXLAN is not enabled by device" 2719 " parameter and/or not configured in" 2720 " firmware"); 2721 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2722 return rte_flow_error_set(error, ENOTSUP, 2723 RTE_FLOW_ERROR_TYPE_ITEM, item, 2724 "multiple tunnel layers not" 2725 " supported"); 2726 /* 2727 * Verify only UDPv4 is present as defined in 2728 * https://tools.ietf.org/html/rfc7348 2729 */ 2730 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2731 return rte_flow_error_set(error, EINVAL, 2732 RTE_FLOW_ERROR_TYPE_ITEM, item, 2733 "no outer UDP layer found"); 2734 if (!mask) 2735 mask = &rte_flow_item_vxlan_gpe_mask; 2736 ret = mlx5_flow_item_acceptable 2737 (item, (const uint8_t *)mask, 2738 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2739 sizeof(struct rte_flow_item_vxlan_gpe), 2740 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2741 if (ret < 0) 2742 return ret; 2743 if (spec) { 2744 if (spec->protocol) 2745 return rte_flow_error_set(error, ENOTSUP, 2746 RTE_FLOW_ERROR_TYPE_ITEM, 2747 item, 2748 "VxLAN-GPE protocol" 2749 " not supported"); 2750 memcpy(&id.vni[1], spec->vni, 3); 2751 memcpy(&id.vni[1], mask->vni, 3); 2752 } 2753 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2754 return rte_flow_error_set(error, ENOTSUP, 2755 RTE_FLOW_ERROR_TYPE_ITEM, item, 2756 "VXLAN-GPE tunnel must be fully" 2757 " defined"); 2758 return 0; 2759 } 2760 /** 2761 * Validate GRE Key item. 2762 * 2763 * @param[in] item 2764 * Item specification. 2765 * @param[in] item_flags 2766 * Bit flags to mark detected items. 2767 * @param[in] gre_item 2768 * Pointer to gre_item 2769 * @param[out] error 2770 * Pointer to error structure. 2771 * 2772 * @return 2773 * 0 on success, a negative errno value otherwise and rte_errno is set. 2774 */ 2775 int 2776 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2777 uint64_t item_flags, 2778 const struct rte_flow_item *gre_item, 2779 struct rte_flow_error *error) 2780 { 2781 const rte_be32_t *mask = item->mask; 2782 int ret = 0; 2783 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2784 const struct rte_flow_item_gre *gre_spec; 2785 const struct rte_flow_item_gre *gre_mask; 2786 2787 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2788 return rte_flow_error_set(error, ENOTSUP, 2789 RTE_FLOW_ERROR_TYPE_ITEM, item, 2790 "Multiple GRE key not support"); 2791 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2792 return rte_flow_error_set(error, ENOTSUP, 2793 RTE_FLOW_ERROR_TYPE_ITEM, item, 2794 "No preceding GRE header"); 2795 if (item_flags & MLX5_FLOW_LAYER_INNER) 2796 return rte_flow_error_set(error, ENOTSUP, 2797 RTE_FLOW_ERROR_TYPE_ITEM, item, 2798 "GRE key following a wrong item"); 2799 gre_mask = gre_item->mask; 2800 if (!gre_mask) 2801 gre_mask = &rte_flow_item_gre_mask; 2802 gre_spec = gre_item->spec; 2803 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2804 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2805 return rte_flow_error_set(error, EINVAL, 2806 RTE_FLOW_ERROR_TYPE_ITEM, item, 2807 "Key bit must be on"); 2808 2809 if (!mask) 2810 mask = &gre_key_default_mask; 2811 ret = mlx5_flow_item_acceptable 2812 (item, (const uint8_t *)mask, 2813 (const uint8_t *)&gre_key_default_mask, 2814 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2815 return ret; 2816 } 2817 2818 /** 2819 * Validate GRE item. 2820 * 2821 * @param[in] item 2822 * Item specification. 2823 * @param[in] item_flags 2824 * Bit flags to mark detected items. 2825 * @param[in] target_protocol 2826 * The next protocol in the previous item. 2827 * @param[out] error 2828 * Pointer to error structure. 2829 * 2830 * @return 2831 * 0 on success, a negative errno value otherwise and rte_errno is set. 2832 */ 2833 int 2834 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2835 uint64_t item_flags, 2836 uint8_t target_protocol, 2837 struct rte_flow_error *error) 2838 { 2839 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2840 const struct rte_flow_item_gre *mask = item->mask; 2841 int ret; 2842 const struct rte_flow_item_gre nic_mask = { 2843 .c_rsvd0_ver = RTE_BE16(0xB000), 2844 .protocol = RTE_BE16(UINT16_MAX), 2845 }; 2846 2847 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2848 return rte_flow_error_set(error, EINVAL, 2849 RTE_FLOW_ERROR_TYPE_ITEM, item, 2850 "protocol filtering not compatible" 2851 " with this GRE layer"); 2852 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2853 return rte_flow_error_set(error, ENOTSUP, 2854 RTE_FLOW_ERROR_TYPE_ITEM, item, 2855 "multiple tunnel layers not" 2856 " supported"); 2857 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2858 return rte_flow_error_set(error, ENOTSUP, 2859 RTE_FLOW_ERROR_TYPE_ITEM, item, 2860 "L3 Layer is missing"); 2861 if (!mask) 2862 mask = &rte_flow_item_gre_mask; 2863 ret = mlx5_flow_item_acceptable 2864 (item, (const uint8_t *)mask, 2865 (const uint8_t *)&nic_mask, 2866 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2867 error); 2868 if (ret < 0) 2869 return ret; 2870 #ifndef HAVE_MLX5DV_DR 2871 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2872 if (spec && (spec->protocol & mask->protocol)) 2873 return rte_flow_error_set(error, ENOTSUP, 2874 RTE_FLOW_ERROR_TYPE_ITEM, item, 2875 "without MPLS support the" 2876 " specification cannot be used for" 2877 " filtering"); 2878 #endif 2879 #endif 2880 return 0; 2881 } 2882 2883 /** 2884 * Validate Geneve item. 2885 * 2886 * @param[in] item 2887 * Item specification. 2888 * @param[in] itemFlags 2889 * Bit-fields that holds the items detected until now. 2890 * @param[in] enPriv 2891 * Pointer to the private data structure. 2892 * @param[out] error 2893 * Pointer to error structure. 2894 * 2895 * @return 2896 * 0 on success, a negative errno value otherwise and rte_errno is set. 2897 */ 2898 2899 int 2900 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2901 uint64_t item_flags, 2902 struct rte_eth_dev *dev, 2903 struct rte_flow_error *error) 2904 { 2905 struct mlx5_priv *priv = dev->data->dev_private; 2906 const struct rte_flow_item_geneve *spec = item->spec; 2907 const struct rte_flow_item_geneve *mask = item->mask; 2908 int ret; 2909 uint16_t gbhdr; 2910 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 2911 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2912 const struct rte_flow_item_geneve nic_mask = { 2913 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2914 .vni = "\xff\xff\xff", 2915 .protocol = RTE_BE16(UINT16_MAX), 2916 }; 2917 2918 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 2919 return rte_flow_error_set(error, ENOTSUP, 2920 RTE_FLOW_ERROR_TYPE_ITEM, item, 2921 "L3 Geneve is not enabled by device" 2922 " parameter and/or not configured in" 2923 " firmware"); 2924 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2925 return rte_flow_error_set(error, ENOTSUP, 2926 RTE_FLOW_ERROR_TYPE_ITEM, item, 2927 "multiple tunnel layers not" 2928 " supported"); 2929 /* 2930 * Verify only UDPv4 is present as defined in 2931 * https://tools.ietf.org/html/rfc7348 2932 */ 2933 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2934 return rte_flow_error_set(error, EINVAL, 2935 RTE_FLOW_ERROR_TYPE_ITEM, item, 2936 "no outer UDP layer found"); 2937 if (!mask) 2938 mask = &rte_flow_item_geneve_mask; 2939 ret = mlx5_flow_item_acceptable 2940 (item, (const uint8_t *)mask, 2941 (const uint8_t *)&nic_mask, 2942 sizeof(struct rte_flow_item_geneve), 2943 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2944 if (ret) 2945 return ret; 2946 if (spec) { 2947 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2948 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2949 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2950 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2951 return rte_flow_error_set(error, ENOTSUP, 2952 RTE_FLOW_ERROR_TYPE_ITEM, 2953 item, 2954 "Geneve protocol unsupported" 2955 " fields are being used"); 2956 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2957 return rte_flow_error_set 2958 (error, ENOTSUP, 2959 RTE_FLOW_ERROR_TYPE_ITEM, 2960 item, 2961 "Unsupported Geneve options length"); 2962 } 2963 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2964 return rte_flow_error_set 2965 (error, ENOTSUP, 2966 RTE_FLOW_ERROR_TYPE_ITEM, item, 2967 "Geneve tunnel must be fully defined"); 2968 return 0; 2969 } 2970 2971 /** 2972 * Validate Geneve TLV option item. 2973 * 2974 * @param[in] item 2975 * Item specification. 2976 * @param[in] last_item 2977 * Previous validated item in the pattern items. 2978 * @param[in] geneve_item 2979 * Previous GENEVE item specification. 2980 * @param[in] dev 2981 * Pointer to the rte_eth_dev structure. 2982 * @param[out] error 2983 * Pointer to error structure. 2984 * 2985 * @return 2986 * 0 on success, a negative errno value otherwise and rte_errno is set. 2987 */ 2988 int 2989 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2990 uint64_t last_item, 2991 const struct rte_flow_item *geneve_item, 2992 struct rte_eth_dev *dev, 2993 struct rte_flow_error *error) 2994 { 2995 struct mlx5_priv *priv = dev->data->dev_private; 2996 struct mlx5_dev_ctx_shared *sh = priv->sh; 2997 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2998 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 2999 uint8_t data_max_supported = 3000 hca_attr->max_geneve_tlv_option_data_len * 4; 3001 const struct rte_flow_item_geneve *geneve_spec; 3002 const struct rte_flow_item_geneve *geneve_mask; 3003 const struct rte_flow_item_geneve_opt *spec = item->spec; 3004 const struct rte_flow_item_geneve_opt *mask = item->mask; 3005 unsigned int i; 3006 unsigned int data_len; 3007 uint8_t tlv_option_len; 3008 uint16_t optlen_m, optlen_v; 3009 const struct rte_flow_item_geneve_opt full_mask = { 3010 .option_class = RTE_BE16(0xffff), 3011 .option_type = 0xff, 3012 .option_len = 0x1f, 3013 }; 3014 3015 if (!mask) 3016 mask = &rte_flow_item_geneve_opt_mask; 3017 if (!spec) 3018 return rte_flow_error_set 3019 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3020 "Geneve TLV opt class/type/length must be specified"); 3021 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3022 return rte_flow_error_set 3023 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3024 "Geneve TLV opt length exceeds the limit (31)"); 3025 /* Check if class type and length masks are full. */ 3026 if (full_mask.option_class != mask->option_class || 3027 full_mask.option_type != mask->option_type || 3028 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3029 return rte_flow_error_set 3030 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3031 "Geneve TLV opt class/type/length masks must be full"); 3032 /* Check if length is supported */ 3033 if ((uint32_t)spec->option_len > 3034 hca_attr->max_geneve_tlv_option_data_len) 3035 return rte_flow_error_set 3036 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3037 "Geneve TLV opt length not supported"); 3038 if (hca_attr->max_geneve_tlv_options > 1) 3039 DRV_LOG(DEBUG, 3040 "max_geneve_tlv_options supports more than 1 option"); 3041 /* Check GENEVE item preceding. */ 3042 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3043 return rte_flow_error_set 3044 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3045 "Geneve opt item must be preceded with Geneve item"); 3046 geneve_spec = geneve_item->spec; 3047 geneve_mask = geneve_item->mask ? geneve_item->mask : 3048 &rte_flow_item_geneve_mask; 3049 /* Check if GENEVE TLV option size doesn't exceed option length */ 3050 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3051 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3052 tlv_option_len = spec->option_len & mask->option_len; 3053 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3054 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3055 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3056 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3057 if ((optlen_v & optlen_m) <= tlv_option_len) 3058 return rte_flow_error_set 3059 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3060 "GENEVE TLV option length exceeds optlen"); 3061 } 3062 /* Check if length is 0 or data is 0. */ 3063 if (spec->data == NULL || spec->option_len == 0) 3064 return rte_flow_error_set 3065 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3066 "Geneve TLV opt with zero data/length not supported"); 3067 /* Check not all data & mask are 0. */ 3068 data_len = spec->option_len * 4; 3069 if (mask->data == NULL) { 3070 for (i = 0; i < data_len; i++) 3071 if (spec->data[i]) 3072 break; 3073 if (i == data_len) 3074 return rte_flow_error_set(error, ENOTSUP, 3075 RTE_FLOW_ERROR_TYPE_ITEM, item, 3076 "Can't match on Geneve option data 0"); 3077 } else { 3078 for (i = 0; i < data_len; i++) 3079 if (spec->data[i] & mask->data[i]) 3080 break; 3081 if (i == data_len) 3082 return rte_flow_error_set(error, ENOTSUP, 3083 RTE_FLOW_ERROR_TYPE_ITEM, item, 3084 "Can't match on Geneve option data and mask 0"); 3085 /* Check data mask supported. */ 3086 for (i = data_max_supported; i < data_len ; i++) 3087 if (mask->data[i]) 3088 return rte_flow_error_set(error, ENOTSUP, 3089 RTE_FLOW_ERROR_TYPE_ITEM, item, 3090 "Data mask is of unsupported size"); 3091 } 3092 /* Check GENEVE option is supported in NIC. */ 3093 if (!hca_attr->geneve_tlv_opt) 3094 return rte_flow_error_set 3095 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3096 "Geneve TLV opt not supported"); 3097 /* Check if we already have geneve option with different type/class. */ 3098 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3099 geneve_opt_resource = sh->geneve_tlv_option_resource; 3100 if (geneve_opt_resource != NULL) 3101 if (geneve_opt_resource->option_class != spec->option_class || 3102 geneve_opt_resource->option_type != spec->option_type || 3103 geneve_opt_resource->length != spec->option_len) { 3104 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3105 return rte_flow_error_set(error, ENOTSUP, 3106 RTE_FLOW_ERROR_TYPE_ITEM, item, 3107 "Only one Geneve TLV option supported"); 3108 } 3109 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3110 return 0; 3111 } 3112 3113 /** 3114 * Validate MPLS item. 3115 * 3116 * @param[in] dev 3117 * Pointer to the rte_eth_dev structure. 3118 * @param[in] item 3119 * Item specification. 3120 * @param[in] item_flags 3121 * Bit-fields that holds the items detected until now. 3122 * @param[in] prev_layer 3123 * The protocol layer indicated in previous item. 3124 * @param[out] error 3125 * Pointer to error structure. 3126 * 3127 * @return 3128 * 0 on success, a negative errno value otherwise and rte_errno is set. 3129 */ 3130 int 3131 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3132 const struct rte_flow_item *item __rte_unused, 3133 uint64_t item_flags __rte_unused, 3134 uint64_t prev_layer __rte_unused, 3135 struct rte_flow_error *error) 3136 { 3137 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3138 const struct rte_flow_item_mpls *mask = item->mask; 3139 struct mlx5_priv *priv = dev->data->dev_private; 3140 int ret; 3141 3142 if (!priv->sh->dev_cap.mpls_en) 3143 return rte_flow_error_set(error, ENOTSUP, 3144 RTE_FLOW_ERROR_TYPE_ITEM, item, 3145 "MPLS not supported or" 3146 " disabled in firmware" 3147 " configuration."); 3148 /* MPLS over UDP, GRE is allowed */ 3149 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3150 MLX5_FLOW_LAYER_GRE | 3151 MLX5_FLOW_LAYER_GRE_KEY))) 3152 return rte_flow_error_set(error, EINVAL, 3153 RTE_FLOW_ERROR_TYPE_ITEM, item, 3154 "protocol filtering not compatible" 3155 " with MPLS layer"); 3156 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3157 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3158 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3159 return rte_flow_error_set(error, ENOTSUP, 3160 RTE_FLOW_ERROR_TYPE_ITEM, item, 3161 "multiple tunnel layers not" 3162 " supported"); 3163 if (!mask) 3164 mask = &rte_flow_item_mpls_mask; 3165 ret = mlx5_flow_item_acceptable 3166 (item, (const uint8_t *)mask, 3167 (const uint8_t *)&rte_flow_item_mpls_mask, 3168 sizeof(struct rte_flow_item_mpls), 3169 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3170 if (ret < 0) 3171 return ret; 3172 return 0; 3173 #else 3174 return rte_flow_error_set(error, ENOTSUP, 3175 RTE_FLOW_ERROR_TYPE_ITEM, item, 3176 "MPLS is not supported by Verbs, please" 3177 " update."); 3178 #endif 3179 } 3180 3181 /** 3182 * Validate NVGRE item. 3183 * 3184 * @param[in] item 3185 * Item specification. 3186 * @param[in] item_flags 3187 * Bit flags to mark detected items. 3188 * @param[in] target_protocol 3189 * The next protocol in the previous item. 3190 * @param[out] error 3191 * Pointer to error structure. 3192 * 3193 * @return 3194 * 0 on success, a negative errno value otherwise and rte_errno is set. 3195 */ 3196 int 3197 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3198 uint64_t item_flags, 3199 uint8_t target_protocol, 3200 struct rte_flow_error *error) 3201 { 3202 const struct rte_flow_item_nvgre *mask = item->mask; 3203 int ret; 3204 3205 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3206 return rte_flow_error_set(error, EINVAL, 3207 RTE_FLOW_ERROR_TYPE_ITEM, item, 3208 "protocol filtering not compatible" 3209 " with this GRE layer"); 3210 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3211 return rte_flow_error_set(error, ENOTSUP, 3212 RTE_FLOW_ERROR_TYPE_ITEM, item, 3213 "multiple tunnel layers not" 3214 " supported"); 3215 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3216 return rte_flow_error_set(error, ENOTSUP, 3217 RTE_FLOW_ERROR_TYPE_ITEM, item, 3218 "L3 Layer is missing"); 3219 if (!mask) 3220 mask = &rte_flow_item_nvgre_mask; 3221 ret = mlx5_flow_item_acceptable 3222 (item, (const uint8_t *)mask, 3223 (const uint8_t *)&rte_flow_item_nvgre_mask, 3224 sizeof(struct rte_flow_item_nvgre), 3225 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3226 if (ret < 0) 3227 return ret; 3228 return 0; 3229 } 3230 3231 /** 3232 * Validate eCPRI item. 3233 * 3234 * @param[in] item 3235 * Item specification. 3236 * @param[in] item_flags 3237 * Bit-fields that holds the items detected until now. 3238 * @param[in] last_item 3239 * Previous validated item in the pattern items. 3240 * @param[in] ether_type 3241 * Type in the ethernet layer header (including dot1q). 3242 * @param[in] acc_mask 3243 * Acceptable mask, if NULL default internal default mask 3244 * will be used to check whether item fields are supported. 3245 * @param[out] error 3246 * Pointer to error structure. 3247 * 3248 * @return 3249 * 0 on success, a negative errno value otherwise and rte_errno is set. 3250 */ 3251 int 3252 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3253 uint64_t item_flags, 3254 uint64_t last_item, 3255 uint16_t ether_type, 3256 const struct rte_flow_item_ecpri *acc_mask, 3257 struct rte_flow_error *error) 3258 { 3259 const struct rte_flow_item_ecpri *mask = item->mask; 3260 const struct rte_flow_item_ecpri nic_mask = { 3261 .hdr = { 3262 .common = { 3263 .u32 = 3264 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3265 .type = 0xFF, 3266 }).u32), 3267 }, 3268 .dummy[0] = 0xFFFFFFFF, 3269 }, 3270 }; 3271 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3272 MLX5_FLOW_LAYER_OUTER_VLAN); 3273 struct rte_flow_item_ecpri mask_lo; 3274 3275 if (!(last_item & outer_l2_vlan) && 3276 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3277 return rte_flow_error_set(error, EINVAL, 3278 RTE_FLOW_ERROR_TYPE_ITEM, item, 3279 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3280 if ((last_item & outer_l2_vlan) && ether_type && 3281 ether_type != RTE_ETHER_TYPE_ECPRI) 3282 return rte_flow_error_set(error, EINVAL, 3283 RTE_FLOW_ERROR_TYPE_ITEM, item, 3284 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3285 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3286 return rte_flow_error_set(error, EINVAL, 3287 RTE_FLOW_ERROR_TYPE_ITEM, item, 3288 "eCPRI with tunnel is not supported right now"); 3289 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3290 return rte_flow_error_set(error, ENOTSUP, 3291 RTE_FLOW_ERROR_TYPE_ITEM, item, 3292 "multiple L3 layers not supported"); 3293 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3294 return rte_flow_error_set(error, EINVAL, 3295 RTE_FLOW_ERROR_TYPE_ITEM, item, 3296 "eCPRI cannot coexist with a TCP layer"); 3297 /* In specification, eCPRI could be over UDP layer. */ 3298 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3299 return rte_flow_error_set(error, EINVAL, 3300 RTE_FLOW_ERROR_TYPE_ITEM, item, 3301 "eCPRI over UDP layer is not yet supported right now"); 3302 /* Mask for type field in common header could be zero. */ 3303 if (!mask) 3304 mask = &rte_flow_item_ecpri_mask; 3305 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3306 /* Input mask is in big-endian format. */ 3307 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3308 return rte_flow_error_set(error, EINVAL, 3309 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3310 "partial mask is not supported for protocol"); 3311 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3312 return rte_flow_error_set(error, EINVAL, 3313 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3314 "message header mask must be after a type mask"); 3315 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3316 acc_mask ? (const uint8_t *)acc_mask 3317 : (const uint8_t *)&nic_mask, 3318 sizeof(struct rte_flow_item_ecpri), 3319 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3320 } 3321 3322 static int 3323 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3324 const struct rte_flow_attr *attr __rte_unused, 3325 const struct rte_flow_item items[] __rte_unused, 3326 const struct rte_flow_action actions[] __rte_unused, 3327 bool external __rte_unused, 3328 int hairpin __rte_unused, 3329 struct rte_flow_error *error) 3330 { 3331 return rte_flow_error_set(error, ENOTSUP, 3332 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3333 } 3334 3335 static struct mlx5_flow * 3336 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3337 const struct rte_flow_attr *attr __rte_unused, 3338 const struct rte_flow_item items[] __rte_unused, 3339 const struct rte_flow_action actions[] __rte_unused, 3340 struct rte_flow_error *error) 3341 { 3342 rte_flow_error_set(error, ENOTSUP, 3343 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3344 return NULL; 3345 } 3346 3347 static int 3348 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3349 struct mlx5_flow *dev_flow __rte_unused, 3350 const struct rte_flow_attr *attr __rte_unused, 3351 const struct rte_flow_item items[] __rte_unused, 3352 const struct rte_flow_action actions[] __rte_unused, 3353 struct rte_flow_error *error) 3354 { 3355 return rte_flow_error_set(error, ENOTSUP, 3356 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3357 } 3358 3359 static int 3360 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3361 struct rte_flow *flow __rte_unused, 3362 struct rte_flow_error *error) 3363 { 3364 return rte_flow_error_set(error, ENOTSUP, 3365 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3366 } 3367 3368 static void 3369 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3370 struct rte_flow *flow __rte_unused) 3371 { 3372 } 3373 3374 static void 3375 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3376 struct rte_flow *flow __rte_unused) 3377 { 3378 } 3379 3380 static int 3381 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3382 struct rte_flow *flow __rte_unused, 3383 const struct rte_flow_action *actions __rte_unused, 3384 void *data __rte_unused, 3385 struct rte_flow_error *error) 3386 { 3387 return rte_flow_error_set(error, ENOTSUP, 3388 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3389 } 3390 3391 static int 3392 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3393 uint32_t domains __rte_unused, 3394 uint32_t flags __rte_unused) 3395 { 3396 return 0; 3397 } 3398 3399 /* Void driver to protect from null pointer reference. */ 3400 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3401 .validate = flow_null_validate, 3402 .prepare = flow_null_prepare, 3403 .translate = flow_null_translate, 3404 .apply = flow_null_apply, 3405 .remove = flow_null_remove, 3406 .destroy = flow_null_destroy, 3407 .query = flow_null_query, 3408 .sync_domain = flow_null_sync_domain, 3409 }; 3410 3411 /** 3412 * Select flow driver type according to flow attributes and device 3413 * configuration. 3414 * 3415 * @param[in] dev 3416 * Pointer to the dev structure. 3417 * @param[in] attr 3418 * Pointer to the flow attributes. 3419 * 3420 * @return 3421 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3422 */ 3423 static enum mlx5_flow_drv_type 3424 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3425 { 3426 struct mlx5_priv *priv = dev->data->dev_private; 3427 /* The OS can determine first a specific flow type (DV, VERBS) */ 3428 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3429 3430 if (type != MLX5_FLOW_TYPE_MAX) 3431 return type; 3432 /* If no OS specific type - continue with DV/VERBS selection */ 3433 if (attr->transfer && priv->sh->config.dv_esw_en) 3434 type = MLX5_FLOW_TYPE_DV; 3435 if (!attr->transfer) 3436 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3437 MLX5_FLOW_TYPE_VERBS; 3438 return type; 3439 } 3440 3441 #define flow_get_drv_ops(type) flow_drv_ops[type] 3442 3443 /** 3444 * Flow driver validation API. This abstracts calling driver specific functions. 3445 * The type of flow driver is determined according to flow attributes. 3446 * 3447 * @param[in] dev 3448 * Pointer to the dev structure. 3449 * @param[in] attr 3450 * Pointer to the flow attributes. 3451 * @param[in] items 3452 * Pointer to the list of items. 3453 * @param[in] actions 3454 * Pointer to the list of actions. 3455 * @param[in] external 3456 * This flow rule is created by request external to PMD. 3457 * @param[in] hairpin 3458 * Number of hairpin TX actions, 0 means classic flow. 3459 * @param[out] error 3460 * Pointer to the error structure. 3461 * 3462 * @return 3463 * 0 on success, a negative errno value otherwise and rte_errno is set. 3464 */ 3465 static inline int 3466 flow_drv_validate(struct rte_eth_dev *dev, 3467 const struct rte_flow_attr *attr, 3468 const struct rte_flow_item items[], 3469 const struct rte_flow_action actions[], 3470 bool external, int hairpin, struct rte_flow_error *error) 3471 { 3472 const struct mlx5_flow_driver_ops *fops; 3473 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3474 3475 fops = flow_get_drv_ops(type); 3476 return fops->validate(dev, attr, items, actions, external, 3477 hairpin, error); 3478 } 3479 3480 /** 3481 * Flow driver preparation API. This abstracts calling driver specific 3482 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3483 * calculates the size of memory required for device flow, allocates the memory, 3484 * initializes the device flow and returns the pointer. 3485 * 3486 * @note 3487 * This function initializes device flow structure such as dv or verbs in 3488 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3489 * rest. For example, adding returning device flow to flow->dev_flow list and 3490 * setting backward reference to the flow should be done out of this function. 3491 * layers field is not filled either. 3492 * 3493 * @param[in] dev 3494 * Pointer to the dev structure. 3495 * @param[in] attr 3496 * Pointer to the flow attributes. 3497 * @param[in] items 3498 * Pointer to the list of items. 3499 * @param[in] actions 3500 * Pointer to the list of actions. 3501 * @param[in] flow_idx 3502 * This memory pool index to the flow. 3503 * @param[out] error 3504 * Pointer to the error structure. 3505 * 3506 * @return 3507 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3508 */ 3509 static inline struct mlx5_flow * 3510 flow_drv_prepare(struct rte_eth_dev *dev, 3511 const struct rte_flow *flow, 3512 const struct rte_flow_attr *attr, 3513 const struct rte_flow_item items[], 3514 const struct rte_flow_action actions[], 3515 uint32_t flow_idx, 3516 struct rte_flow_error *error) 3517 { 3518 const struct mlx5_flow_driver_ops *fops; 3519 enum mlx5_flow_drv_type type = flow->drv_type; 3520 struct mlx5_flow *mlx5_flow = NULL; 3521 3522 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3523 fops = flow_get_drv_ops(type); 3524 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3525 if (mlx5_flow) 3526 mlx5_flow->flow_idx = flow_idx; 3527 return mlx5_flow; 3528 } 3529 3530 /** 3531 * Flow driver translation API. This abstracts calling driver specific 3532 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3533 * translates a generic flow into a driver flow. flow_drv_prepare() must 3534 * precede. 3535 * 3536 * @note 3537 * dev_flow->layers could be filled as a result of parsing during translation 3538 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3539 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3540 * flow->actions could be overwritten even though all the expanded dev_flows 3541 * have the same actions. 3542 * 3543 * @param[in] dev 3544 * Pointer to the rte dev structure. 3545 * @param[in, out] dev_flow 3546 * Pointer to the mlx5 flow. 3547 * @param[in] attr 3548 * Pointer to the flow attributes. 3549 * @param[in] items 3550 * Pointer to the list of items. 3551 * @param[in] actions 3552 * Pointer to the list of actions. 3553 * @param[out] error 3554 * Pointer to the error structure. 3555 * 3556 * @return 3557 * 0 on success, a negative errno value otherwise and rte_errno is set. 3558 */ 3559 static inline int 3560 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3561 const struct rte_flow_attr *attr, 3562 const struct rte_flow_item items[], 3563 const struct rte_flow_action actions[], 3564 struct rte_flow_error *error) 3565 { 3566 const struct mlx5_flow_driver_ops *fops; 3567 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3568 3569 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3570 fops = flow_get_drv_ops(type); 3571 return fops->translate(dev, dev_flow, attr, items, actions, error); 3572 } 3573 3574 /** 3575 * Flow driver apply API. This abstracts calling driver specific functions. 3576 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3577 * translated driver flows on to device. flow_drv_translate() must precede. 3578 * 3579 * @param[in] dev 3580 * Pointer to Ethernet device structure. 3581 * @param[in, out] flow 3582 * Pointer to flow structure. 3583 * @param[out] error 3584 * Pointer to error structure. 3585 * 3586 * @return 3587 * 0 on success, a negative errno value otherwise and rte_errno is set. 3588 */ 3589 static inline int 3590 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3591 struct rte_flow_error *error) 3592 { 3593 const struct mlx5_flow_driver_ops *fops; 3594 enum mlx5_flow_drv_type type = flow->drv_type; 3595 3596 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3597 fops = flow_get_drv_ops(type); 3598 return fops->apply(dev, flow, error); 3599 } 3600 3601 /** 3602 * Flow driver destroy API. This abstracts calling driver specific functions. 3603 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3604 * on device and releases resources of the flow. 3605 * 3606 * @param[in] dev 3607 * Pointer to Ethernet device. 3608 * @param[in, out] flow 3609 * Pointer to flow structure. 3610 */ 3611 static inline void 3612 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3613 { 3614 const struct mlx5_flow_driver_ops *fops; 3615 enum mlx5_flow_drv_type type = flow->drv_type; 3616 3617 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3618 fops = flow_get_drv_ops(type); 3619 fops->destroy(dev, flow); 3620 } 3621 3622 /** 3623 * Flow driver find RSS policy tbl API. This abstracts calling driver 3624 * specific functions. Parent flow (rte_flow) should have driver 3625 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3626 * 3627 * @param[in] dev 3628 * Pointer to Ethernet device. 3629 * @param[in, out] flow 3630 * Pointer to flow structure. 3631 * @param[in] policy 3632 * Pointer to meter policy table. 3633 * @param[in] rss_desc 3634 * Pointer to rss_desc 3635 */ 3636 static struct mlx5_flow_meter_sub_policy * 3637 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3638 struct rte_flow *flow, 3639 struct mlx5_flow_meter_policy *policy, 3640 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3641 { 3642 const struct mlx5_flow_driver_ops *fops; 3643 enum mlx5_flow_drv_type type = flow->drv_type; 3644 3645 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3646 fops = flow_get_drv_ops(type); 3647 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3648 } 3649 3650 /** 3651 * Flow driver color tag rule API. This abstracts calling driver 3652 * specific functions. Parent flow (rte_flow) should have driver 3653 * type (drv_type). It will create the color tag rules in hierarchy meter. 3654 * 3655 * @param[in] dev 3656 * Pointer to Ethernet device. 3657 * @param[in, out] flow 3658 * Pointer to flow structure. 3659 * @param[in] fm 3660 * Pointer to flow meter structure. 3661 * @param[in] src_port 3662 * The src port this extra rule should use. 3663 * @param[in] item 3664 * The src port id match item. 3665 * @param[out] error 3666 * Pointer to error structure. 3667 */ 3668 static int 3669 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3670 struct rte_flow *flow, 3671 struct mlx5_flow_meter_info *fm, 3672 int32_t src_port, 3673 const struct rte_flow_item *item, 3674 struct rte_flow_error *error) 3675 { 3676 const struct mlx5_flow_driver_ops *fops; 3677 enum mlx5_flow_drv_type type = flow->drv_type; 3678 3679 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3680 fops = flow_get_drv_ops(type); 3681 return fops->meter_hierarchy_rule_create(dev, fm, 3682 src_port, item, error); 3683 } 3684 3685 /** 3686 * Get RSS action from the action list. 3687 * 3688 * @param[in] dev 3689 * Pointer to Ethernet device. 3690 * @param[in] actions 3691 * Pointer to the list of actions. 3692 * @param[in] flow 3693 * Parent flow structure pointer. 3694 * 3695 * @return 3696 * Pointer to the RSS action if exist, else return NULL. 3697 */ 3698 static const struct rte_flow_action_rss* 3699 flow_get_rss_action(struct rte_eth_dev *dev, 3700 const struct rte_flow_action actions[]) 3701 { 3702 struct mlx5_priv *priv = dev->data->dev_private; 3703 const struct rte_flow_action_rss *rss = NULL; 3704 struct mlx5_meter_policy_action_container *acg; 3705 struct mlx5_meter_policy_action_container *acy; 3706 3707 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3708 switch (actions->type) { 3709 case RTE_FLOW_ACTION_TYPE_RSS: 3710 rss = actions->conf; 3711 break; 3712 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3713 { 3714 const struct rte_flow_action_sample *sample = 3715 actions->conf; 3716 const struct rte_flow_action *act = sample->actions; 3717 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3718 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3719 rss = act->conf; 3720 break; 3721 } 3722 case RTE_FLOW_ACTION_TYPE_METER: 3723 { 3724 uint32_t mtr_idx; 3725 struct mlx5_flow_meter_info *fm; 3726 struct mlx5_flow_meter_policy *policy; 3727 const struct rte_flow_action_meter *mtr = actions->conf; 3728 3729 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3730 if (fm && !fm->def_policy) { 3731 policy = mlx5_flow_meter_policy_find(dev, 3732 fm->policy_id, NULL); 3733 MLX5_ASSERT(policy); 3734 if (policy->is_hierarchy) { 3735 policy = 3736 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3737 policy); 3738 if (!policy) 3739 return NULL; 3740 } 3741 if (policy->is_rss) { 3742 acg = 3743 &policy->act_cnt[RTE_COLOR_GREEN]; 3744 acy = 3745 &policy->act_cnt[RTE_COLOR_YELLOW]; 3746 if (acg->fate_action == 3747 MLX5_FLOW_FATE_SHARED_RSS) 3748 rss = acg->rss->conf; 3749 else if (acy->fate_action == 3750 MLX5_FLOW_FATE_SHARED_RSS) 3751 rss = acy->rss->conf; 3752 } 3753 } 3754 break; 3755 } 3756 default: 3757 break; 3758 } 3759 } 3760 return rss; 3761 } 3762 3763 /** 3764 * Get ASO age action by index. 3765 * 3766 * @param[in] dev 3767 * Pointer to the Ethernet device structure. 3768 * @param[in] age_idx 3769 * Index to the ASO age action. 3770 * 3771 * @return 3772 * The specified ASO age action. 3773 */ 3774 struct mlx5_aso_age_action* 3775 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3776 { 3777 uint16_t pool_idx = age_idx & UINT16_MAX; 3778 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3779 struct mlx5_priv *priv = dev->data->dev_private; 3780 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3781 struct mlx5_aso_age_pool *pool; 3782 3783 rte_rwlock_read_lock(&mng->resize_rwl); 3784 pool = mng->pools[pool_idx]; 3785 rte_rwlock_read_unlock(&mng->resize_rwl); 3786 return &pool->actions[offset - 1]; 3787 } 3788 3789 /* maps indirect action to translated direct in some actions array */ 3790 struct mlx5_translated_action_handle { 3791 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3792 int index; /**< Index in related array of rte_flow_action. */ 3793 }; 3794 3795 /** 3796 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3797 * direct action if translation possible. 3798 * This functionality used to run same execution path for both direct and 3799 * indirect actions on flow create. All necessary preparations for indirect 3800 * action handling should be performed on *handle* actions list returned 3801 * from this call. 3802 * 3803 * @param[in] dev 3804 * Pointer to Ethernet device. 3805 * @param[in] actions 3806 * List of actions to translate. 3807 * @param[out] handle 3808 * List to store translated indirect action object handles. 3809 * @param[in, out] indir_n 3810 * Size of *handle* array. On return should be updated with number of 3811 * indirect actions retrieved from the *actions* list. 3812 * @param[out] translated_actions 3813 * List of actions where all indirect actions were translated to direct 3814 * if possible. NULL if no translation took place. 3815 * @param[out] error 3816 * Pointer to the error structure. 3817 * 3818 * @return 3819 * 0 on success, a negative errno value otherwise and rte_errno is set. 3820 */ 3821 static int 3822 flow_action_handles_translate(struct rte_eth_dev *dev, 3823 const struct rte_flow_action actions[], 3824 struct mlx5_translated_action_handle *handle, 3825 int *indir_n, 3826 struct rte_flow_action **translated_actions, 3827 struct rte_flow_error *error) 3828 { 3829 struct mlx5_priv *priv = dev->data->dev_private; 3830 struct rte_flow_action *translated = NULL; 3831 size_t actions_size; 3832 int n; 3833 int copied_n = 0; 3834 struct mlx5_translated_action_handle *handle_end = NULL; 3835 3836 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3837 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3838 continue; 3839 if (copied_n == *indir_n) { 3840 return rte_flow_error_set 3841 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3842 NULL, "too many shared actions"); 3843 } 3844 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3845 sizeof(actions[n].conf)); 3846 handle[copied_n].index = n; 3847 copied_n++; 3848 } 3849 n++; 3850 *indir_n = copied_n; 3851 if (!copied_n) 3852 return 0; 3853 actions_size = sizeof(struct rte_flow_action) * n; 3854 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3855 if (!translated) { 3856 rte_errno = ENOMEM; 3857 return -ENOMEM; 3858 } 3859 memcpy(translated, actions, actions_size); 3860 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3861 struct mlx5_shared_action_rss *shared_rss; 3862 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3863 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3864 uint32_t idx = act_idx & 3865 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3866 3867 switch (type) { 3868 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3869 shared_rss = mlx5_ipool_get 3870 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3871 translated[handle->index].type = 3872 RTE_FLOW_ACTION_TYPE_RSS; 3873 translated[handle->index].conf = 3874 &shared_rss->origin; 3875 break; 3876 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3877 translated[handle->index].type = 3878 (enum rte_flow_action_type) 3879 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3880 translated[handle->index].conf = (void *)(uintptr_t)idx; 3881 break; 3882 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3883 if (priv->sh->flow_hit_aso_en) { 3884 translated[handle->index].type = 3885 (enum rte_flow_action_type) 3886 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3887 translated[handle->index].conf = 3888 (void *)(uintptr_t)idx; 3889 break; 3890 } 3891 /* Fall-through */ 3892 case MLX5_INDIRECT_ACTION_TYPE_CT: 3893 if (priv->sh->ct_aso_en) { 3894 translated[handle->index].type = 3895 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3896 translated[handle->index].conf = 3897 (void *)(uintptr_t)idx; 3898 break; 3899 } 3900 /* Fall-through */ 3901 default: 3902 mlx5_free(translated); 3903 return rte_flow_error_set 3904 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3905 NULL, "invalid indirect action type"); 3906 } 3907 } 3908 *translated_actions = translated; 3909 return 0; 3910 } 3911 3912 /** 3913 * Get Shared RSS action from the action list. 3914 * 3915 * @param[in] dev 3916 * Pointer to Ethernet device. 3917 * @param[in] shared 3918 * Pointer to the list of actions. 3919 * @param[in] shared_n 3920 * Actions list length. 3921 * 3922 * @return 3923 * The MLX5 RSS action ID if exists, otherwise return 0. 3924 */ 3925 static uint32_t 3926 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3927 struct mlx5_translated_action_handle *handle, 3928 int shared_n) 3929 { 3930 struct mlx5_translated_action_handle *handle_end; 3931 struct mlx5_priv *priv = dev->data->dev_private; 3932 struct mlx5_shared_action_rss *shared_rss; 3933 3934 3935 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3936 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3937 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3938 uint32_t idx = act_idx & 3939 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3940 switch (type) { 3941 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3942 shared_rss = mlx5_ipool_get 3943 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3944 idx); 3945 __atomic_add_fetch(&shared_rss->refcnt, 1, 3946 __ATOMIC_RELAXED); 3947 return idx; 3948 default: 3949 break; 3950 } 3951 } 3952 return 0; 3953 } 3954 3955 static unsigned int 3956 find_graph_root(uint32_t rss_level) 3957 { 3958 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3959 MLX5_EXPANSION_ROOT_OUTER; 3960 } 3961 3962 /** 3963 * Get layer flags from the prefix flow. 3964 * 3965 * Some flows may be split to several subflows, the prefix subflow gets the 3966 * match items and the suffix sub flow gets the actions. 3967 * Some actions need the user defined match item flags to get the detail for 3968 * the action. 3969 * This function helps the suffix flow to get the item layer flags from prefix 3970 * subflow. 3971 * 3972 * @param[in] dev_flow 3973 * Pointer the created prefix subflow. 3974 * 3975 * @return 3976 * The layers get from prefix subflow. 3977 */ 3978 static inline uint64_t 3979 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3980 { 3981 uint64_t layers = 0; 3982 3983 /* 3984 * Layers bits could be localization, but usually the compiler will 3985 * help to do the optimization work for source code. 3986 * If no decap actions, use the layers directly. 3987 */ 3988 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3989 return dev_flow->handle->layers; 3990 /* Convert L3 layers with decap action. */ 3991 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3992 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3993 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3994 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3995 /* Convert L4 layers with decap action. */ 3996 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3997 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3998 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3999 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4000 return layers; 4001 } 4002 4003 /** 4004 * Get metadata split action information. 4005 * 4006 * @param[in] actions 4007 * Pointer to the list of actions. 4008 * @param[out] qrss 4009 * Pointer to the return pointer. 4010 * @param[out] qrss_type 4011 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4012 * if no QUEUE/RSS is found. 4013 * @param[out] encap_idx 4014 * Pointer to the index of the encap action if exists, otherwise the last 4015 * action index. 4016 * 4017 * @return 4018 * Total number of actions. 4019 */ 4020 static int 4021 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4022 const struct rte_flow_action **qrss, 4023 int *encap_idx) 4024 { 4025 const struct rte_flow_action_raw_encap *raw_encap; 4026 int actions_n = 0; 4027 int raw_decap_idx = -1; 4028 4029 *encap_idx = -1; 4030 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4031 switch (actions->type) { 4032 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4033 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4034 *encap_idx = actions_n; 4035 break; 4036 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4037 raw_decap_idx = actions_n; 4038 break; 4039 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4040 raw_encap = actions->conf; 4041 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4042 *encap_idx = raw_decap_idx != -1 ? 4043 raw_decap_idx : actions_n; 4044 break; 4045 case RTE_FLOW_ACTION_TYPE_QUEUE: 4046 case RTE_FLOW_ACTION_TYPE_RSS: 4047 *qrss = actions; 4048 break; 4049 default: 4050 break; 4051 } 4052 actions_n++; 4053 } 4054 if (*encap_idx == -1) 4055 *encap_idx = actions_n; 4056 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4057 return actions_n + 1; 4058 } 4059 4060 /** 4061 * Check if the action will change packet. 4062 * 4063 * @param dev 4064 * Pointer to Ethernet device. 4065 * @param[in] type 4066 * action type. 4067 * 4068 * @return 4069 * true if action will change packet, false otherwise. 4070 */ 4071 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4072 enum rte_flow_action_type type) 4073 { 4074 struct mlx5_priv *priv = dev->data->dev_private; 4075 4076 switch (type) { 4077 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4078 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4079 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4080 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4081 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4082 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4083 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4084 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4085 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4086 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4087 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4088 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4089 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4090 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4091 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4092 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4093 case RTE_FLOW_ACTION_TYPE_SET_META: 4094 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4095 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4096 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4097 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4098 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4099 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4100 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4101 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4102 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4103 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4104 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4105 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4106 return true; 4107 case RTE_FLOW_ACTION_TYPE_FLAG: 4108 case RTE_FLOW_ACTION_TYPE_MARK: 4109 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4110 return true; 4111 else 4112 return false; 4113 default: 4114 return false; 4115 } 4116 } 4117 4118 /** 4119 * Check meter action from the action list. 4120 * 4121 * @param dev 4122 * Pointer to Ethernet device. 4123 * @param[in] actions 4124 * Pointer to the list of actions. 4125 * @param[out] has_mtr 4126 * Pointer to the meter exist flag. 4127 * @param[out] has_modify 4128 * Pointer to the flag showing there's packet change action. 4129 * @param[out] meter_id 4130 * Pointer to the meter id. 4131 * 4132 * @return 4133 * Total number of actions. 4134 */ 4135 static int 4136 flow_check_meter_action(struct rte_eth_dev *dev, 4137 const struct rte_flow_action actions[], 4138 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4139 { 4140 const struct rte_flow_action_meter *mtr = NULL; 4141 int actions_n = 0; 4142 4143 MLX5_ASSERT(has_mtr); 4144 *has_mtr = false; 4145 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4146 switch (actions->type) { 4147 case RTE_FLOW_ACTION_TYPE_METER: 4148 mtr = actions->conf; 4149 *meter_id = mtr->mtr_id; 4150 *has_mtr = true; 4151 break; 4152 default: 4153 break; 4154 } 4155 if (!*has_mtr) 4156 *has_modify |= flow_check_modify_action_type(dev, 4157 actions->type); 4158 actions_n++; 4159 } 4160 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4161 return actions_n + 1; 4162 } 4163 4164 /** 4165 * Check if the flow should be split due to hairpin. 4166 * The reason for the split is that in current HW we can't 4167 * support encap and push-vlan on Rx, so if a flow contains 4168 * these actions we move it to Tx. 4169 * 4170 * @param dev 4171 * Pointer to Ethernet device. 4172 * @param[in] attr 4173 * Flow rule attributes. 4174 * @param[in] actions 4175 * Associated actions (list terminated by the END action). 4176 * 4177 * @return 4178 * > 0 the number of actions and the flow should be split, 4179 * 0 when no split required. 4180 */ 4181 static int 4182 flow_check_hairpin_split(struct rte_eth_dev *dev, 4183 const struct rte_flow_attr *attr, 4184 const struct rte_flow_action actions[]) 4185 { 4186 int queue_action = 0; 4187 int action_n = 0; 4188 int split = 0; 4189 const struct rte_flow_action_queue *queue; 4190 const struct rte_flow_action_rss *rss; 4191 const struct rte_flow_action_raw_encap *raw_encap; 4192 const struct rte_eth_hairpin_conf *conf; 4193 4194 if (!attr->ingress) 4195 return 0; 4196 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4197 switch (actions->type) { 4198 case RTE_FLOW_ACTION_TYPE_QUEUE: 4199 queue = actions->conf; 4200 if (queue == NULL) 4201 return 0; 4202 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4203 if (conf == NULL || conf->tx_explicit != 0) 4204 return 0; 4205 queue_action = 1; 4206 action_n++; 4207 break; 4208 case RTE_FLOW_ACTION_TYPE_RSS: 4209 rss = actions->conf; 4210 if (rss == NULL || rss->queue_num == 0) 4211 return 0; 4212 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4213 if (conf == NULL || conf->tx_explicit != 0) 4214 return 0; 4215 queue_action = 1; 4216 action_n++; 4217 break; 4218 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4219 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4220 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4221 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4222 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4223 split++; 4224 action_n++; 4225 break; 4226 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4227 raw_encap = actions->conf; 4228 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4229 split++; 4230 action_n++; 4231 break; 4232 default: 4233 action_n++; 4234 break; 4235 } 4236 } 4237 if (split && queue_action) 4238 return action_n; 4239 return 0; 4240 } 4241 4242 /* Declare flow create/destroy prototype in advance. */ 4243 static uint32_t 4244 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4245 const struct rte_flow_attr *attr, 4246 const struct rte_flow_item items[], 4247 const struct rte_flow_action actions[], 4248 bool external, struct rte_flow_error *error); 4249 4250 static void 4251 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4252 uint32_t flow_idx); 4253 4254 int 4255 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4256 struct mlx5_list_entry *entry, void *cb_ctx) 4257 { 4258 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4259 struct mlx5_flow_mreg_copy_resource *mcp_res = 4260 container_of(entry, typeof(*mcp_res), hlist_ent); 4261 4262 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4263 } 4264 4265 struct mlx5_list_entry * 4266 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4267 { 4268 struct rte_eth_dev *dev = tool_ctx; 4269 struct mlx5_priv *priv = dev->data->dev_private; 4270 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4271 struct mlx5_flow_mreg_copy_resource *mcp_res; 4272 struct rte_flow_error *error = ctx->error; 4273 uint32_t idx = 0; 4274 int ret; 4275 uint32_t mark_id = *(uint32_t *)(ctx->data); 4276 struct rte_flow_attr attr = { 4277 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4278 .ingress = 1, 4279 }; 4280 struct mlx5_rte_flow_item_tag tag_spec = { 4281 .data = mark_id, 4282 }; 4283 struct rte_flow_item items[] = { 4284 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4285 }; 4286 struct rte_flow_action_mark ftag = { 4287 .id = mark_id, 4288 }; 4289 struct mlx5_flow_action_copy_mreg cp_mreg = { 4290 .dst = REG_B, 4291 .src = REG_NON, 4292 }; 4293 struct rte_flow_action_jump jump = { 4294 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4295 }; 4296 struct rte_flow_action actions[] = { 4297 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4298 }; 4299 4300 /* Fill the register fields in the flow. */ 4301 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4302 if (ret < 0) 4303 return NULL; 4304 tag_spec.id = ret; 4305 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4306 if (ret < 0) 4307 return NULL; 4308 cp_mreg.src = ret; 4309 /* Provide the full width of FLAG specific value. */ 4310 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4311 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4312 /* Build a new flow. */ 4313 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4314 items[0] = (struct rte_flow_item){ 4315 .type = (enum rte_flow_item_type) 4316 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4317 .spec = &tag_spec, 4318 }; 4319 items[1] = (struct rte_flow_item){ 4320 .type = RTE_FLOW_ITEM_TYPE_END, 4321 }; 4322 actions[0] = (struct rte_flow_action){ 4323 .type = (enum rte_flow_action_type) 4324 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4325 .conf = &ftag, 4326 }; 4327 actions[1] = (struct rte_flow_action){ 4328 .type = (enum rte_flow_action_type) 4329 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4330 .conf = &cp_mreg, 4331 }; 4332 actions[2] = (struct rte_flow_action){ 4333 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4334 .conf = &jump, 4335 }; 4336 actions[3] = (struct rte_flow_action){ 4337 .type = RTE_FLOW_ACTION_TYPE_END, 4338 }; 4339 } else { 4340 /* Default rule, wildcard match. */ 4341 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4342 items[0] = (struct rte_flow_item){ 4343 .type = RTE_FLOW_ITEM_TYPE_END, 4344 }; 4345 actions[0] = (struct rte_flow_action){ 4346 .type = (enum rte_flow_action_type) 4347 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4348 .conf = &cp_mreg, 4349 }; 4350 actions[1] = (struct rte_flow_action){ 4351 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4352 .conf = &jump, 4353 }; 4354 actions[2] = (struct rte_flow_action){ 4355 .type = RTE_FLOW_ACTION_TYPE_END, 4356 }; 4357 } 4358 /* Build a new entry. */ 4359 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4360 if (!mcp_res) { 4361 rte_errno = ENOMEM; 4362 return NULL; 4363 } 4364 mcp_res->idx = idx; 4365 mcp_res->mark_id = mark_id; 4366 /* 4367 * The copy Flows are not included in any list. There 4368 * ones are referenced from other Flows and can not 4369 * be applied, removed, deleted in arbitrary order 4370 * by list traversing. 4371 */ 4372 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4373 &attr, items, actions, false, error); 4374 if (!mcp_res->rix_flow) { 4375 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4376 return NULL; 4377 } 4378 return &mcp_res->hlist_ent; 4379 } 4380 4381 struct mlx5_list_entry * 4382 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4383 void *cb_ctx __rte_unused) 4384 { 4385 struct rte_eth_dev *dev = tool_ctx; 4386 struct mlx5_priv *priv = dev->data->dev_private; 4387 struct mlx5_flow_mreg_copy_resource *mcp_res; 4388 uint32_t idx = 0; 4389 4390 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4391 if (!mcp_res) { 4392 rte_errno = ENOMEM; 4393 return NULL; 4394 } 4395 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4396 mcp_res->idx = idx; 4397 return &mcp_res->hlist_ent; 4398 } 4399 4400 void 4401 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4402 { 4403 struct mlx5_flow_mreg_copy_resource *mcp_res = 4404 container_of(entry, typeof(*mcp_res), hlist_ent); 4405 struct rte_eth_dev *dev = tool_ctx; 4406 struct mlx5_priv *priv = dev->data->dev_private; 4407 4408 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4409 } 4410 4411 /** 4412 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4413 * 4414 * As mark_id is unique, if there's already a registered flow for the mark_id, 4415 * return by increasing the reference counter of the resource. Otherwise, create 4416 * the resource (mcp_res) and flow. 4417 * 4418 * Flow looks like, 4419 * - If ingress port is ANY and reg_c[1] is mark_id, 4420 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4421 * 4422 * For default flow (zero mark_id), flow is like, 4423 * - If ingress port is ANY, 4424 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4425 * 4426 * @param dev 4427 * Pointer to Ethernet device. 4428 * @param mark_id 4429 * ID of MARK action, zero means default flow for META. 4430 * @param[out] error 4431 * Perform verbose error reporting if not NULL. 4432 * 4433 * @return 4434 * Associated resource on success, NULL otherwise and rte_errno is set. 4435 */ 4436 static struct mlx5_flow_mreg_copy_resource * 4437 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4438 struct rte_flow_error *error) 4439 { 4440 struct mlx5_priv *priv = dev->data->dev_private; 4441 struct mlx5_list_entry *entry; 4442 struct mlx5_flow_cb_ctx ctx = { 4443 .dev = dev, 4444 .error = error, 4445 .data = &mark_id, 4446 }; 4447 4448 /* Check if already registered. */ 4449 MLX5_ASSERT(priv->mreg_cp_tbl); 4450 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4451 if (!entry) 4452 return NULL; 4453 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4454 hlist_ent); 4455 } 4456 4457 void 4458 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4459 { 4460 struct mlx5_flow_mreg_copy_resource *mcp_res = 4461 container_of(entry, typeof(*mcp_res), hlist_ent); 4462 struct rte_eth_dev *dev = tool_ctx; 4463 struct mlx5_priv *priv = dev->data->dev_private; 4464 4465 MLX5_ASSERT(mcp_res->rix_flow); 4466 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4467 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4468 } 4469 4470 /** 4471 * Release flow in RX_CP_TBL. 4472 * 4473 * @param dev 4474 * Pointer to Ethernet device. 4475 * @flow 4476 * Parent flow for wich copying is provided. 4477 */ 4478 static void 4479 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4480 struct rte_flow *flow) 4481 { 4482 struct mlx5_flow_mreg_copy_resource *mcp_res; 4483 struct mlx5_priv *priv = dev->data->dev_private; 4484 4485 if (!flow->rix_mreg_copy) 4486 return; 4487 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4488 flow->rix_mreg_copy); 4489 if (!mcp_res || !priv->mreg_cp_tbl) 4490 return; 4491 MLX5_ASSERT(mcp_res->rix_flow); 4492 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4493 flow->rix_mreg_copy = 0; 4494 } 4495 4496 /** 4497 * Remove the default copy action from RX_CP_TBL. 4498 * 4499 * This functions is called in the mlx5_dev_start(). No thread safe 4500 * is guaranteed. 4501 * 4502 * @param dev 4503 * Pointer to Ethernet device. 4504 */ 4505 static void 4506 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4507 { 4508 struct mlx5_list_entry *entry; 4509 struct mlx5_priv *priv = dev->data->dev_private; 4510 struct mlx5_flow_cb_ctx ctx; 4511 uint32_t mark_id; 4512 4513 /* Check if default flow is registered. */ 4514 if (!priv->mreg_cp_tbl) 4515 return; 4516 mark_id = MLX5_DEFAULT_COPY_ID; 4517 ctx.data = &mark_id; 4518 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4519 if (!entry) 4520 return; 4521 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4522 } 4523 4524 /** 4525 * Add the default copy action in in RX_CP_TBL. 4526 * 4527 * This functions is called in the mlx5_dev_start(). No thread safe 4528 * is guaranteed. 4529 * 4530 * @param dev 4531 * Pointer to Ethernet device. 4532 * @param[out] error 4533 * Perform verbose error reporting if not NULL. 4534 * 4535 * @return 4536 * 0 for success, negative value otherwise and rte_errno is set. 4537 */ 4538 static int 4539 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4540 struct rte_flow_error *error) 4541 { 4542 struct mlx5_priv *priv = dev->data->dev_private; 4543 struct mlx5_flow_mreg_copy_resource *mcp_res; 4544 struct mlx5_flow_cb_ctx ctx; 4545 uint32_t mark_id; 4546 4547 /* Check whether extensive metadata feature is engaged. */ 4548 if (!priv->sh->config.dv_flow_en || 4549 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4550 !mlx5_flow_ext_mreg_supported(dev) || 4551 !priv->sh->dv_regc0_mask) 4552 return 0; 4553 /* 4554 * Add default mreg copy flow may be called multiple time, but 4555 * only be called once in stop. Avoid register it twice. 4556 */ 4557 mark_id = MLX5_DEFAULT_COPY_ID; 4558 ctx.data = &mark_id; 4559 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4560 return 0; 4561 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4562 if (!mcp_res) 4563 return -rte_errno; 4564 return 0; 4565 } 4566 4567 /** 4568 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4569 * 4570 * All the flow having Q/RSS action should be split by 4571 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4572 * performs the following, 4573 * - CQE->flow_tag := reg_c[1] (MARK) 4574 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4575 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4576 * but there should be a flow per each MARK ID set by MARK action. 4577 * 4578 * For the aforementioned reason, if there's a MARK action in flow's action 4579 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4580 * the MARK ID to CQE's flow_tag like, 4581 * - If reg_c[1] is mark_id, 4582 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4583 * 4584 * For SET_META action which stores value in reg_c[0], as the destination is 4585 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4586 * MARK ID means the default flow. The default flow looks like, 4587 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4588 * 4589 * @param dev 4590 * Pointer to Ethernet device. 4591 * @param flow 4592 * Pointer to flow structure. 4593 * @param[in] actions 4594 * Pointer to the list of actions. 4595 * @param[out] error 4596 * Perform verbose error reporting if not NULL. 4597 * 4598 * @return 4599 * 0 on success, negative value otherwise and rte_errno is set. 4600 */ 4601 static int 4602 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4603 struct rte_flow *flow, 4604 const struct rte_flow_action *actions, 4605 struct rte_flow_error *error) 4606 { 4607 struct mlx5_priv *priv = dev->data->dev_private; 4608 struct mlx5_sh_config *config = &priv->sh->config; 4609 struct mlx5_flow_mreg_copy_resource *mcp_res; 4610 const struct rte_flow_action_mark *mark; 4611 4612 /* Check whether extensive metadata feature is engaged. */ 4613 if (!config->dv_flow_en || 4614 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4615 !mlx5_flow_ext_mreg_supported(dev) || 4616 !priv->sh->dv_regc0_mask) 4617 return 0; 4618 /* Find MARK action. */ 4619 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4620 switch (actions->type) { 4621 case RTE_FLOW_ACTION_TYPE_FLAG: 4622 mcp_res = flow_mreg_add_copy_action 4623 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4624 if (!mcp_res) 4625 return -rte_errno; 4626 flow->rix_mreg_copy = mcp_res->idx; 4627 return 0; 4628 case RTE_FLOW_ACTION_TYPE_MARK: 4629 mark = (const struct rte_flow_action_mark *) 4630 actions->conf; 4631 mcp_res = 4632 flow_mreg_add_copy_action(dev, mark->id, error); 4633 if (!mcp_res) 4634 return -rte_errno; 4635 flow->rix_mreg_copy = mcp_res->idx; 4636 return 0; 4637 default: 4638 break; 4639 } 4640 } 4641 return 0; 4642 } 4643 4644 #define MLX5_MAX_SPLIT_ACTIONS 24 4645 #define MLX5_MAX_SPLIT_ITEMS 24 4646 4647 /** 4648 * Split the hairpin flow. 4649 * Since HW can't support encap and push-vlan on Rx, we move these 4650 * actions to Tx. 4651 * If the count action is after the encap then we also 4652 * move the count action. in this case the count will also measure 4653 * the outer bytes. 4654 * 4655 * @param dev 4656 * Pointer to Ethernet device. 4657 * @param[in] actions 4658 * Associated actions (list terminated by the END action). 4659 * @param[out] actions_rx 4660 * Rx flow actions. 4661 * @param[out] actions_tx 4662 * Tx flow actions.. 4663 * @param[out] pattern_tx 4664 * The pattern items for the Tx flow. 4665 * @param[out] flow_id 4666 * The flow ID connected to this flow. 4667 * 4668 * @return 4669 * 0 on success. 4670 */ 4671 static int 4672 flow_hairpin_split(struct rte_eth_dev *dev, 4673 const struct rte_flow_action actions[], 4674 struct rte_flow_action actions_rx[], 4675 struct rte_flow_action actions_tx[], 4676 struct rte_flow_item pattern_tx[], 4677 uint32_t flow_id) 4678 { 4679 const struct rte_flow_action_raw_encap *raw_encap; 4680 const struct rte_flow_action_raw_decap *raw_decap; 4681 struct mlx5_rte_flow_action_set_tag *set_tag; 4682 struct rte_flow_action *tag_action; 4683 struct mlx5_rte_flow_item_tag *tag_item; 4684 struct rte_flow_item *item; 4685 char *addr; 4686 int encap = 0; 4687 4688 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4689 switch (actions->type) { 4690 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4691 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4692 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4693 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4694 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4695 rte_memcpy(actions_tx, actions, 4696 sizeof(struct rte_flow_action)); 4697 actions_tx++; 4698 break; 4699 case RTE_FLOW_ACTION_TYPE_COUNT: 4700 if (encap) { 4701 rte_memcpy(actions_tx, actions, 4702 sizeof(struct rte_flow_action)); 4703 actions_tx++; 4704 } else { 4705 rte_memcpy(actions_rx, actions, 4706 sizeof(struct rte_flow_action)); 4707 actions_rx++; 4708 } 4709 break; 4710 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4711 raw_encap = actions->conf; 4712 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4713 memcpy(actions_tx, actions, 4714 sizeof(struct rte_flow_action)); 4715 actions_tx++; 4716 encap = 1; 4717 } else { 4718 rte_memcpy(actions_rx, actions, 4719 sizeof(struct rte_flow_action)); 4720 actions_rx++; 4721 } 4722 break; 4723 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4724 raw_decap = actions->conf; 4725 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4726 memcpy(actions_tx, actions, 4727 sizeof(struct rte_flow_action)); 4728 actions_tx++; 4729 } else { 4730 rte_memcpy(actions_rx, actions, 4731 sizeof(struct rte_flow_action)); 4732 actions_rx++; 4733 } 4734 break; 4735 default: 4736 rte_memcpy(actions_rx, actions, 4737 sizeof(struct rte_flow_action)); 4738 actions_rx++; 4739 break; 4740 } 4741 } 4742 /* Add set meta action and end action for the Rx flow. */ 4743 tag_action = actions_rx; 4744 tag_action->type = (enum rte_flow_action_type) 4745 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4746 actions_rx++; 4747 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4748 actions_rx++; 4749 set_tag = (void *)actions_rx; 4750 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4751 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4752 .data = flow_id, 4753 }; 4754 MLX5_ASSERT(set_tag->id > REG_NON); 4755 tag_action->conf = set_tag; 4756 /* Create Tx item list. */ 4757 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4758 addr = (void *)&pattern_tx[2]; 4759 item = pattern_tx; 4760 item->type = (enum rte_flow_item_type) 4761 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4762 tag_item = (void *)addr; 4763 tag_item->data = flow_id; 4764 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4765 MLX5_ASSERT(set_tag->id > REG_NON); 4766 item->spec = tag_item; 4767 addr += sizeof(struct mlx5_rte_flow_item_tag); 4768 tag_item = (void *)addr; 4769 tag_item->data = UINT32_MAX; 4770 tag_item->id = UINT16_MAX; 4771 item->mask = tag_item; 4772 item->last = NULL; 4773 item++; 4774 item->type = RTE_FLOW_ITEM_TYPE_END; 4775 return 0; 4776 } 4777 4778 /** 4779 * The last stage of splitting chain, just creates the subflow 4780 * without any modification. 4781 * 4782 * @param[in] dev 4783 * Pointer to Ethernet device. 4784 * @param[in] flow 4785 * Parent flow structure pointer. 4786 * @param[in, out] sub_flow 4787 * Pointer to return the created subflow, may be NULL. 4788 * @param[in] attr 4789 * Flow rule attributes. 4790 * @param[in] items 4791 * Pattern specification (list terminated by the END pattern item). 4792 * @param[in] actions 4793 * Associated actions (list terminated by the END action). 4794 * @param[in] flow_split_info 4795 * Pointer to flow split info structure. 4796 * @param[out] error 4797 * Perform verbose error reporting if not NULL. 4798 * @return 4799 * 0 on success, negative value otherwise 4800 */ 4801 static int 4802 flow_create_split_inner(struct rte_eth_dev *dev, 4803 struct rte_flow *flow, 4804 struct mlx5_flow **sub_flow, 4805 const struct rte_flow_attr *attr, 4806 const struct rte_flow_item items[], 4807 const struct rte_flow_action actions[], 4808 struct mlx5_flow_split_info *flow_split_info, 4809 struct rte_flow_error *error) 4810 { 4811 struct mlx5_flow *dev_flow; 4812 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 4813 4814 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4815 flow_split_info->flow_idx, error); 4816 if (!dev_flow) 4817 return -rte_errno; 4818 dev_flow->flow = flow; 4819 dev_flow->external = flow_split_info->external; 4820 dev_flow->skip_scale = flow_split_info->skip_scale; 4821 /* Subflow object was created, we must include one in the list. */ 4822 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4823 dev_flow->handle, next); 4824 /* 4825 * If dev_flow is as one of the suffix flow, some actions in suffix 4826 * flow may need some user defined item layer flags, and pass the 4827 * Metadata rxq mark flag to suffix flow as well. 4828 */ 4829 if (flow_split_info->prefix_layers) 4830 dev_flow->handle->layers = flow_split_info->prefix_layers; 4831 if (flow_split_info->prefix_mark) { 4832 MLX5_ASSERT(wks); 4833 wks->mark = 1; 4834 } 4835 if (sub_flow) 4836 *sub_flow = dev_flow; 4837 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4838 dev_flow->dv.table_id = flow_split_info->table_id; 4839 #endif 4840 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4841 } 4842 4843 /** 4844 * Get the sub policy of a meter. 4845 * 4846 * @param[in] dev 4847 * Pointer to Ethernet device. 4848 * @param[in] flow 4849 * Parent flow structure pointer. 4850 * @param wks 4851 * Pointer to thread flow work space. 4852 * @param[in] attr 4853 * Flow rule attributes. 4854 * @param[in] items 4855 * Pattern specification (list terminated by the END pattern item). 4856 * @param[out] error 4857 * Perform verbose error reporting if not NULL. 4858 * 4859 * @return 4860 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4861 */ 4862 static struct mlx5_flow_meter_sub_policy * 4863 get_meter_sub_policy(struct rte_eth_dev *dev, 4864 struct rte_flow *flow, 4865 struct mlx5_flow_workspace *wks, 4866 const struct rte_flow_attr *attr, 4867 const struct rte_flow_item items[], 4868 struct rte_flow_error *error) 4869 { 4870 struct mlx5_flow_meter_policy *policy; 4871 struct mlx5_flow_meter_policy *final_policy; 4872 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4873 4874 policy = wks->policy; 4875 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4876 if (final_policy->is_rss || final_policy->is_queue) { 4877 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4878 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4879 uint32_t i; 4880 4881 /* 4882 * This is a tmp dev_flow, 4883 * no need to register any matcher for it in translate. 4884 */ 4885 wks->skip_matcher_reg = 1; 4886 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4887 struct mlx5_flow dev_flow = {0}; 4888 struct mlx5_flow_handle dev_handle = { {0} }; 4889 uint8_t fate = final_policy->act_cnt[i].fate_action; 4890 4891 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4892 const struct rte_flow_action_rss *rss_act = 4893 final_policy->act_cnt[i].rss->conf; 4894 struct rte_flow_action rss_actions[2] = { 4895 [0] = { 4896 .type = RTE_FLOW_ACTION_TYPE_RSS, 4897 .conf = rss_act, 4898 }, 4899 [1] = { 4900 .type = RTE_FLOW_ACTION_TYPE_END, 4901 .conf = NULL, 4902 } 4903 }; 4904 4905 dev_flow.handle = &dev_handle; 4906 dev_flow.ingress = attr->ingress; 4907 dev_flow.flow = flow; 4908 dev_flow.external = 0; 4909 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4910 dev_flow.dv.transfer = attr->transfer; 4911 #endif 4912 /** 4913 * Translate RSS action to get rss hash fields. 4914 */ 4915 if (flow_drv_translate(dev, &dev_flow, attr, 4916 items, rss_actions, error)) 4917 goto exit; 4918 rss_desc_v[i] = wks->rss_desc; 4919 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4920 rss_desc_v[i].hash_fields = 4921 dev_flow.hash_fields; 4922 rss_desc_v[i].queue_num = 4923 rss_desc_v[i].hash_fields ? 4924 rss_desc_v[i].queue_num : 1; 4925 rss_desc_v[i].tunnel = 4926 !!(dev_flow.handle->layers & 4927 MLX5_FLOW_LAYER_TUNNEL); 4928 /* Use the RSS queues in the containers. */ 4929 rss_desc_v[i].queue = 4930 (uint16_t *)(uintptr_t)rss_act->queue; 4931 rss_desc[i] = &rss_desc_v[i]; 4932 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 4933 /* This is queue action. */ 4934 rss_desc_v[i] = wks->rss_desc; 4935 rss_desc_v[i].key_len = 0; 4936 rss_desc_v[i].hash_fields = 0; 4937 rss_desc_v[i].queue = 4938 &final_policy->act_cnt[i].queue; 4939 rss_desc_v[i].queue_num = 1; 4940 rss_desc[i] = &rss_desc_v[i]; 4941 } else { 4942 rss_desc[i] = NULL; 4943 } 4944 } 4945 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4946 flow, policy, rss_desc); 4947 } else { 4948 enum mlx5_meter_domain mtr_domain = 4949 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4950 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4951 MLX5_MTR_DOMAIN_INGRESS); 4952 sub_policy = policy->sub_policys[mtr_domain][0]; 4953 } 4954 if (!sub_policy) 4955 rte_flow_error_set(error, EINVAL, 4956 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4957 "Failed to get meter sub-policy."); 4958 exit: 4959 return sub_policy; 4960 } 4961 4962 /** 4963 * Split the meter flow. 4964 * 4965 * As meter flow will split to three sub flow, other than meter 4966 * action, the other actions make sense to only meter accepts 4967 * the packet. If it need to be dropped, no other additional 4968 * actions should be take. 4969 * 4970 * One kind of special action which decapsulates the L3 tunnel 4971 * header will be in the prefix sub flow, as not to take the 4972 * L3 tunnel header into account. 4973 * 4974 * @param[in] dev 4975 * Pointer to Ethernet device. 4976 * @param[in] flow 4977 * Parent flow structure pointer. 4978 * @param wks 4979 * Pointer to thread flow work space. 4980 * @param[in] attr 4981 * Flow rule attributes. 4982 * @param[in] items 4983 * Pattern specification (list terminated by the END pattern item). 4984 * @param[out] sfx_items 4985 * Suffix flow match items (list terminated by the END pattern item). 4986 * @param[in] actions 4987 * Associated actions (list terminated by the END action). 4988 * @param[out] actions_sfx 4989 * Suffix flow actions. 4990 * @param[out] actions_pre 4991 * Prefix flow actions. 4992 * @param[out] mtr_flow_id 4993 * Pointer to meter flow id. 4994 * @param[out] error 4995 * Perform verbose error reporting if not NULL. 4996 * 4997 * @return 4998 * 0 on success, a negative errno value otherwise and rte_errno is set. 4999 */ 5000 static int 5001 flow_meter_split_prep(struct rte_eth_dev *dev, 5002 struct rte_flow *flow, 5003 struct mlx5_flow_workspace *wks, 5004 const struct rte_flow_attr *attr, 5005 const struct rte_flow_item items[], 5006 struct rte_flow_item sfx_items[], 5007 const struct rte_flow_action actions[], 5008 struct rte_flow_action actions_sfx[], 5009 struct rte_flow_action actions_pre[], 5010 uint32_t *mtr_flow_id, 5011 struct rte_flow_error *error) 5012 { 5013 struct mlx5_priv *priv = dev->data->dev_private; 5014 struct mlx5_flow_meter_info *fm = wks->fm; 5015 struct rte_flow_action *tag_action = NULL; 5016 struct rte_flow_item *tag_item; 5017 struct mlx5_rte_flow_action_set_tag *set_tag; 5018 const struct rte_flow_action_raw_encap *raw_encap; 5019 const struct rte_flow_action_raw_decap *raw_decap; 5020 struct mlx5_rte_flow_item_tag *tag_item_spec; 5021 struct mlx5_rte_flow_item_tag *tag_item_mask; 5022 uint32_t tag_id = 0; 5023 struct rte_flow_item *vlan_item_dst = NULL; 5024 const struct rte_flow_item *vlan_item_src = NULL; 5025 const struct rte_flow_item *orig_items = items; 5026 struct rte_flow_action *hw_mtr_action; 5027 struct rte_flow_action *action_pre_head = NULL; 5028 int32_t flow_src_port = priv->representor_id; 5029 bool mtr_first; 5030 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5031 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5032 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5033 uint32_t flow_id = 0; 5034 uint32_t flow_id_reversed = 0; 5035 uint8_t flow_id_bits = 0; 5036 int shift; 5037 5038 /* Prepare the suffix subflow items. */ 5039 tag_item = sfx_items++; 5040 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5041 struct mlx5_priv *port_priv; 5042 const struct rte_flow_item_port_id *pid_v; 5043 int item_type = items->type; 5044 5045 switch (item_type) { 5046 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5047 pid_v = items->spec; 5048 MLX5_ASSERT(pid_v); 5049 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5050 if (!port_priv) 5051 return rte_flow_error_set(error, 5052 rte_errno, 5053 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5054 pid_v, 5055 "Failed to get port info."); 5056 flow_src_port = port_priv->representor_id; 5057 if (!fm->def_policy && wks->policy->is_hierarchy && 5058 flow_src_port != priv->representor_id) { 5059 if (flow_drv_mtr_hierarchy_rule_create(dev, 5060 flow, fm, 5061 flow_src_port, 5062 items, 5063 error)) 5064 return -rte_errno; 5065 } 5066 memcpy(sfx_items, items, sizeof(*sfx_items)); 5067 sfx_items++; 5068 break; 5069 case RTE_FLOW_ITEM_TYPE_VLAN: 5070 /* Determine if copy vlan item below. */ 5071 vlan_item_src = items; 5072 vlan_item_dst = sfx_items++; 5073 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5074 break; 5075 default: 5076 break; 5077 } 5078 } 5079 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5080 sfx_items++; 5081 mtr_first = priv->sh->meter_aso_en && 5082 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5083 /* For ASO meter, meter must be before tag in TX direction. */ 5084 if (mtr_first) { 5085 action_pre_head = actions_pre++; 5086 /* Leave space for tag action. */ 5087 tag_action = actions_pre++; 5088 } 5089 /* Prepare the actions for prefix and suffix flow. */ 5090 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5091 struct rte_flow_action *action_cur = NULL; 5092 5093 switch (actions->type) { 5094 case RTE_FLOW_ACTION_TYPE_METER: 5095 if (mtr_first) { 5096 action_cur = action_pre_head; 5097 } else { 5098 /* Leave space for tag action. */ 5099 tag_action = actions_pre++; 5100 action_cur = actions_pre++; 5101 } 5102 break; 5103 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5104 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5105 action_cur = actions_pre++; 5106 break; 5107 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5108 raw_encap = actions->conf; 5109 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5110 action_cur = actions_pre++; 5111 break; 5112 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5113 raw_decap = actions->conf; 5114 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5115 action_cur = actions_pre++; 5116 break; 5117 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5118 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5119 if (vlan_item_dst && vlan_item_src) { 5120 memcpy(vlan_item_dst, vlan_item_src, 5121 sizeof(*vlan_item_dst)); 5122 /* 5123 * Convert to internal match item, it is used 5124 * for vlan push and set vid. 5125 */ 5126 vlan_item_dst->type = (enum rte_flow_item_type) 5127 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5128 } 5129 break; 5130 default: 5131 break; 5132 } 5133 if (!action_cur) 5134 action_cur = (fm->def_policy) ? 5135 actions_sfx++ : actions_pre++; 5136 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5137 } 5138 /* Add end action to the actions. */ 5139 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5140 if (priv->sh->meter_aso_en) { 5141 /** 5142 * For ASO meter, need to add an extra jump action explicitly, 5143 * to jump from meter to policer table. 5144 */ 5145 struct mlx5_flow_meter_sub_policy *sub_policy; 5146 struct mlx5_flow_tbl_data_entry *tbl_data; 5147 5148 if (!fm->def_policy) { 5149 sub_policy = get_meter_sub_policy(dev, flow, wks, 5150 attr, orig_items, 5151 error); 5152 if (!sub_policy) 5153 return -rte_errno; 5154 } else { 5155 enum mlx5_meter_domain mtr_domain = 5156 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5157 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5158 MLX5_MTR_DOMAIN_INGRESS); 5159 5160 sub_policy = 5161 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5162 } 5163 tbl_data = container_of(sub_policy->tbl_rsc, 5164 struct mlx5_flow_tbl_data_entry, tbl); 5165 hw_mtr_action = actions_pre++; 5166 hw_mtr_action->type = (enum rte_flow_action_type) 5167 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5168 hw_mtr_action->conf = tbl_data->jump.action; 5169 } 5170 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5171 actions_pre++; 5172 if (!tag_action) 5173 return rte_flow_error_set(error, ENOMEM, 5174 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5175 NULL, "No tag action space."); 5176 if (!mtr_flow_id) { 5177 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5178 goto exit; 5179 } 5180 /* Only default-policy Meter creates mtr flow id. */ 5181 if (fm->def_policy) { 5182 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5183 if (!tag_id) 5184 return rte_flow_error_set(error, ENOMEM, 5185 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5186 "Failed to allocate meter flow id."); 5187 flow_id = tag_id - 1; 5188 flow_id_bits = (!flow_id) ? 1 : 5189 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5190 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5191 mtr_reg_bits) { 5192 mlx5_ipool_free(fm->flow_ipool, tag_id); 5193 return rte_flow_error_set(error, EINVAL, 5194 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5195 "Meter flow id exceeds max limit."); 5196 } 5197 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5198 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5199 } 5200 /* Build tag actions and items for meter_id/meter flow_id. */ 5201 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5202 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5203 tag_item_mask = tag_item_spec + 1; 5204 /* Both flow_id and meter_id share the same register. */ 5205 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5206 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5207 0, error), 5208 .offset = mtr_id_offset, 5209 .length = mtr_reg_bits, 5210 .data = flow->meter, 5211 }; 5212 /* 5213 * The color Reg bits used by flow_id are growing from 5214 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5215 */ 5216 for (shift = 0; shift < flow_id_bits; shift++) 5217 flow_id_reversed = (flow_id_reversed << 1) | 5218 ((flow_id >> shift) & 0x1); 5219 set_tag->data |= 5220 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5221 tag_item_spec->id = set_tag->id; 5222 tag_item_spec->data = set_tag->data << mtr_id_offset; 5223 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5224 tag_action->type = (enum rte_flow_action_type) 5225 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5226 tag_action->conf = set_tag; 5227 tag_item->type = (enum rte_flow_item_type) 5228 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5229 tag_item->spec = tag_item_spec; 5230 tag_item->last = NULL; 5231 tag_item->mask = tag_item_mask; 5232 exit: 5233 if (mtr_flow_id) 5234 *mtr_flow_id = tag_id; 5235 return 0; 5236 } 5237 5238 /** 5239 * Split action list having QUEUE/RSS for metadata register copy. 5240 * 5241 * Once Q/RSS action is detected in user's action list, the flow action 5242 * should be split in order to copy metadata registers, which will happen in 5243 * RX_CP_TBL like, 5244 * - CQE->flow_tag := reg_c[1] (MARK) 5245 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5246 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5247 * This is because the last action of each flow must be a terminal action 5248 * (QUEUE, RSS or DROP). 5249 * 5250 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5251 * stored and kept in the mlx5_flow structure per each sub_flow. 5252 * 5253 * The Q/RSS action is replaced with, 5254 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5255 * And the following JUMP action is added at the end, 5256 * - JUMP, to RX_CP_TBL. 5257 * 5258 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5259 * flow_create_split_metadata() routine. The flow will look like, 5260 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5261 * 5262 * @param dev 5263 * Pointer to Ethernet device. 5264 * @param[out] split_actions 5265 * Pointer to store split actions to jump to CP_TBL. 5266 * @param[in] actions 5267 * Pointer to the list of original flow actions. 5268 * @param[in] qrss 5269 * Pointer to the Q/RSS action. 5270 * @param[in] actions_n 5271 * Number of original actions. 5272 * @param[in] mtr_sfx 5273 * Check if it is in meter suffix table. 5274 * @param[out] error 5275 * Perform verbose error reporting if not NULL. 5276 * 5277 * @return 5278 * non-zero unique flow_id on success, otherwise 0 and 5279 * error/rte_error are set. 5280 */ 5281 static uint32_t 5282 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5283 struct rte_flow_action *split_actions, 5284 const struct rte_flow_action *actions, 5285 const struct rte_flow_action *qrss, 5286 int actions_n, int mtr_sfx, 5287 struct rte_flow_error *error) 5288 { 5289 struct mlx5_priv *priv = dev->data->dev_private; 5290 struct mlx5_rte_flow_action_set_tag *set_tag; 5291 struct rte_flow_action_jump *jump; 5292 const int qrss_idx = qrss - actions; 5293 uint32_t flow_id = 0; 5294 int ret = 0; 5295 5296 /* 5297 * Given actions will be split 5298 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5299 * - Add jump to mreg CP_TBL. 5300 * As a result, there will be one more action. 5301 */ 5302 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5303 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5304 ++actions_n; 5305 set_tag = (void *)(split_actions + actions_n); 5306 /* 5307 * If we are not the meter suffix flow, add the tag action. 5308 * Since meter suffix flow already has the tag added. 5309 */ 5310 if (!mtr_sfx) { 5311 /* 5312 * Allocate the new subflow ID. This one is unique within 5313 * device and not shared with representors. Otherwise, 5314 * we would have to resolve multi-thread access synch 5315 * issue. Each flow on the shared device is appended 5316 * with source vport identifier, so the resulting 5317 * flows will be unique in the shared (by master and 5318 * representors) domain even if they have coinciding 5319 * IDs. 5320 */ 5321 mlx5_ipool_malloc(priv->sh->ipool 5322 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5323 if (!flow_id) 5324 return rte_flow_error_set(error, ENOMEM, 5325 RTE_FLOW_ERROR_TYPE_ACTION, 5326 NULL, "can't allocate id " 5327 "for split Q/RSS subflow"); 5328 /* Internal SET_TAG action to set flow ID. */ 5329 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5330 .data = flow_id, 5331 }; 5332 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5333 if (ret < 0) 5334 return ret; 5335 set_tag->id = ret; 5336 /* Construct new actions array. */ 5337 /* Replace QUEUE/RSS action. */ 5338 split_actions[qrss_idx] = (struct rte_flow_action){ 5339 .type = (enum rte_flow_action_type) 5340 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5341 .conf = set_tag, 5342 }; 5343 } else { 5344 /* 5345 * If we are the suffix flow of meter, tag already exist. 5346 * Set the QUEUE/RSS action to void. 5347 */ 5348 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5349 } 5350 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5351 jump = (void *)(set_tag + 1); 5352 *jump = (struct rte_flow_action_jump){ 5353 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5354 }; 5355 split_actions[actions_n - 2] = (struct rte_flow_action){ 5356 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5357 .conf = jump, 5358 }; 5359 split_actions[actions_n - 1] = (struct rte_flow_action){ 5360 .type = RTE_FLOW_ACTION_TYPE_END, 5361 }; 5362 return flow_id; 5363 } 5364 5365 /** 5366 * Extend the given action list for Tx metadata copy. 5367 * 5368 * Copy the given action list to the ext_actions and add flow metadata register 5369 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5370 * 5371 * @param[out] ext_actions 5372 * Pointer to the extended action list. 5373 * @param[in] actions 5374 * Pointer to the list of actions. 5375 * @param[in] actions_n 5376 * Number of actions in the list. 5377 * @param[out] error 5378 * Perform verbose error reporting if not NULL. 5379 * @param[in] encap_idx 5380 * The encap action index. 5381 * 5382 * @return 5383 * 0 on success, negative value otherwise 5384 */ 5385 static int 5386 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5387 struct rte_flow_action *ext_actions, 5388 const struct rte_flow_action *actions, 5389 int actions_n, struct rte_flow_error *error, 5390 int encap_idx) 5391 { 5392 struct mlx5_flow_action_copy_mreg *cp_mreg = 5393 (struct mlx5_flow_action_copy_mreg *) 5394 (ext_actions + actions_n + 1); 5395 int ret; 5396 5397 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5398 if (ret < 0) 5399 return ret; 5400 cp_mreg->dst = ret; 5401 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5402 if (ret < 0) 5403 return ret; 5404 cp_mreg->src = ret; 5405 if (encap_idx != 0) 5406 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5407 if (encap_idx == actions_n - 1) { 5408 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5409 .type = (enum rte_flow_action_type) 5410 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5411 .conf = cp_mreg, 5412 }; 5413 ext_actions[actions_n] = (struct rte_flow_action){ 5414 .type = RTE_FLOW_ACTION_TYPE_END, 5415 }; 5416 } else { 5417 ext_actions[encap_idx] = (struct rte_flow_action){ 5418 .type = (enum rte_flow_action_type) 5419 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5420 .conf = cp_mreg, 5421 }; 5422 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5423 sizeof(*ext_actions) * (actions_n - encap_idx)); 5424 } 5425 return 0; 5426 } 5427 5428 /** 5429 * Check the match action from the action list. 5430 * 5431 * @param[in] actions 5432 * Pointer to the list of actions. 5433 * @param[in] attr 5434 * Flow rule attributes. 5435 * @param[in] action 5436 * The action to be check if exist. 5437 * @param[out] match_action_pos 5438 * Pointer to the position of the matched action if exists, otherwise is -1. 5439 * @param[out] qrss_action_pos 5440 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5441 * @param[out] modify_after_mirror 5442 * Pointer to the flag of modify action after FDB mirroring. 5443 * 5444 * @return 5445 * > 0 the total number of actions. 5446 * 0 if not found match action in action list. 5447 */ 5448 static int 5449 flow_check_match_action(const struct rte_flow_action actions[], 5450 const struct rte_flow_attr *attr, 5451 enum rte_flow_action_type action, 5452 int *match_action_pos, int *qrss_action_pos, 5453 int *modify_after_mirror) 5454 { 5455 const struct rte_flow_action_sample *sample; 5456 const struct rte_flow_action_raw_decap *decap; 5457 int actions_n = 0; 5458 uint32_t ratio = 0; 5459 int sub_type = 0; 5460 int flag = 0; 5461 int fdb_mirror = 0; 5462 5463 *match_action_pos = -1; 5464 *qrss_action_pos = -1; 5465 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5466 if (actions->type == action) { 5467 flag = 1; 5468 *match_action_pos = actions_n; 5469 } 5470 switch (actions->type) { 5471 case RTE_FLOW_ACTION_TYPE_QUEUE: 5472 case RTE_FLOW_ACTION_TYPE_RSS: 5473 *qrss_action_pos = actions_n; 5474 break; 5475 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5476 sample = actions->conf; 5477 ratio = sample->ratio; 5478 sub_type = ((const struct rte_flow_action *) 5479 (sample->actions))->type; 5480 if (ratio == 1 && attr->transfer) 5481 fdb_mirror = 1; 5482 break; 5483 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5484 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5485 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5486 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5487 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5488 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5489 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5490 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5491 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5492 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5493 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5494 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5495 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5496 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5497 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5498 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5499 case RTE_FLOW_ACTION_TYPE_FLAG: 5500 case RTE_FLOW_ACTION_TYPE_MARK: 5501 case RTE_FLOW_ACTION_TYPE_SET_META: 5502 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5503 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5504 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5505 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5506 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5507 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5508 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5509 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5510 case RTE_FLOW_ACTION_TYPE_METER: 5511 if (fdb_mirror) 5512 *modify_after_mirror = 1; 5513 break; 5514 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5515 decap = actions->conf; 5516 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5517 ; 5518 actions_n++; 5519 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5520 const struct rte_flow_action_raw_encap *encap = 5521 actions->conf; 5522 if (decap->size <= 5523 MLX5_ENCAPSULATION_DECISION_SIZE && 5524 encap->size > 5525 MLX5_ENCAPSULATION_DECISION_SIZE) 5526 /* L3 encap. */ 5527 break; 5528 } 5529 if (fdb_mirror) 5530 *modify_after_mirror = 1; 5531 break; 5532 default: 5533 break; 5534 } 5535 actions_n++; 5536 } 5537 if (flag && fdb_mirror && !*modify_after_mirror) { 5538 /* FDB mirroring uses the destination array to implement 5539 * instead of FLOW_SAMPLER object. 5540 */ 5541 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5542 flag = 0; 5543 } 5544 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5545 return flag ? actions_n + 1 : 0; 5546 } 5547 5548 #define SAMPLE_SUFFIX_ITEM 2 5549 5550 /** 5551 * Split the sample flow. 5552 * 5553 * As sample flow will split to two sub flow, sample flow with 5554 * sample action, the other actions will move to new suffix flow. 5555 * 5556 * Also add unique tag id with tag action in the sample flow, 5557 * the same tag id will be as match in the suffix flow. 5558 * 5559 * @param dev 5560 * Pointer to Ethernet device. 5561 * @param[in] add_tag 5562 * Add extra tag action flag. 5563 * @param[out] sfx_items 5564 * Suffix flow match items (list terminated by the END pattern item). 5565 * @param[in] actions 5566 * Associated actions (list terminated by the END action). 5567 * @param[out] actions_sfx 5568 * Suffix flow actions. 5569 * @param[out] actions_pre 5570 * Prefix flow actions. 5571 * @param[in] actions_n 5572 * The total number of actions. 5573 * @param[in] sample_action_pos 5574 * The sample action position. 5575 * @param[in] qrss_action_pos 5576 * The Queue/RSS action position. 5577 * @param[in] jump_table 5578 * Add extra jump action flag. 5579 * @param[out] error 5580 * Perform verbose error reporting if not NULL. 5581 * 5582 * @return 5583 * 0 on success, or unique flow_id, a negative errno value 5584 * otherwise and rte_errno is set. 5585 */ 5586 static int 5587 flow_sample_split_prep(struct rte_eth_dev *dev, 5588 int add_tag, 5589 struct rte_flow_item sfx_items[], 5590 const struct rte_flow_action actions[], 5591 struct rte_flow_action actions_sfx[], 5592 struct rte_flow_action actions_pre[], 5593 int actions_n, 5594 int sample_action_pos, 5595 int qrss_action_pos, 5596 int jump_table, 5597 struct rte_flow_error *error) 5598 { 5599 struct mlx5_priv *priv = dev->data->dev_private; 5600 struct mlx5_rte_flow_action_set_tag *set_tag; 5601 struct mlx5_rte_flow_item_tag *tag_spec; 5602 struct mlx5_rte_flow_item_tag *tag_mask; 5603 struct rte_flow_action_jump *jump_action; 5604 uint32_t tag_id = 0; 5605 int index; 5606 int append_index = 0; 5607 int ret; 5608 5609 if (sample_action_pos < 0) 5610 return rte_flow_error_set(error, EINVAL, 5611 RTE_FLOW_ERROR_TYPE_ACTION, 5612 NULL, "invalid position of sample " 5613 "action in list"); 5614 /* Prepare the actions for prefix and suffix flow. */ 5615 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5616 index = qrss_action_pos; 5617 /* Put the preceding the Queue/RSS action into prefix flow. */ 5618 if (index != 0) 5619 memcpy(actions_pre, actions, 5620 sizeof(struct rte_flow_action) * index); 5621 /* Put others preceding the sample action into prefix flow. */ 5622 if (sample_action_pos > index + 1) 5623 memcpy(actions_pre + index, actions + index + 1, 5624 sizeof(struct rte_flow_action) * 5625 (sample_action_pos - index - 1)); 5626 index = sample_action_pos - 1; 5627 /* Put Queue/RSS action into Suffix flow. */ 5628 memcpy(actions_sfx, actions + qrss_action_pos, 5629 sizeof(struct rte_flow_action)); 5630 actions_sfx++; 5631 } else { 5632 index = sample_action_pos; 5633 if (index != 0) 5634 memcpy(actions_pre, actions, 5635 sizeof(struct rte_flow_action) * index); 5636 } 5637 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5638 * For CX6DX and above, metadata registers Cx preserve their value, 5639 * add an extra tag action for NIC-RX and E-Switch Domain. 5640 */ 5641 if (add_tag) { 5642 /* Prepare the prefix tag action. */ 5643 append_index++; 5644 set_tag = (void *)(actions_pre + actions_n + append_index); 5645 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5646 if (ret < 0) 5647 return ret; 5648 mlx5_ipool_malloc(priv->sh->ipool 5649 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5650 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5651 .id = ret, 5652 .data = tag_id, 5653 }; 5654 /* Prepare the suffix subflow items. */ 5655 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5656 tag_spec->data = tag_id; 5657 tag_spec->id = set_tag->id; 5658 tag_mask = tag_spec + 1; 5659 tag_mask->data = UINT32_MAX; 5660 sfx_items[0] = (struct rte_flow_item){ 5661 .type = (enum rte_flow_item_type) 5662 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5663 .spec = tag_spec, 5664 .last = NULL, 5665 .mask = tag_mask, 5666 }; 5667 sfx_items[1] = (struct rte_flow_item){ 5668 .type = (enum rte_flow_item_type) 5669 RTE_FLOW_ITEM_TYPE_END, 5670 }; 5671 /* Prepare the tag action in prefix subflow. */ 5672 actions_pre[index++] = 5673 (struct rte_flow_action){ 5674 .type = (enum rte_flow_action_type) 5675 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5676 .conf = set_tag, 5677 }; 5678 } 5679 memcpy(actions_pre + index, actions + sample_action_pos, 5680 sizeof(struct rte_flow_action)); 5681 index += 1; 5682 /* For the modify action after the sample action in E-Switch mirroring, 5683 * Add the extra jump action in prefix subflow and jump into the next 5684 * table, then do the modify action in the new table. 5685 */ 5686 if (jump_table) { 5687 /* Prepare the prefix jump action. */ 5688 append_index++; 5689 jump_action = (void *)(actions_pre + actions_n + append_index); 5690 jump_action->group = jump_table; 5691 actions_pre[index++] = 5692 (struct rte_flow_action){ 5693 .type = (enum rte_flow_action_type) 5694 RTE_FLOW_ACTION_TYPE_JUMP, 5695 .conf = jump_action, 5696 }; 5697 } 5698 actions_pre[index] = (struct rte_flow_action){ 5699 .type = (enum rte_flow_action_type) 5700 RTE_FLOW_ACTION_TYPE_END, 5701 }; 5702 /* Put the actions after sample into Suffix flow. */ 5703 memcpy(actions_sfx, actions + sample_action_pos + 1, 5704 sizeof(struct rte_flow_action) * 5705 (actions_n - sample_action_pos - 1)); 5706 return tag_id; 5707 } 5708 5709 /** 5710 * The splitting for metadata feature. 5711 * 5712 * - Q/RSS action on NIC Rx should be split in order to pass by 5713 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5714 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5715 * 5716 * - All the actions on NIC Tx should have a mreg copy action to 5717 * copy reg_a from WQE to reg_c[0]. 5718 * 5719 * @param dev 5720 * Pointer to Ethernet device. 5721 * @param[in] flow 5722 * Parent flow structure pointer. 5723 * @param[in] attr 5724 * Flow rule attributes. 5725 * @param[in] items 5726 * Pattern specification (list terminated by the END pattern item). 5727 * @param[in] actions 5728 * Associated actions (list terminated by the END action). 5729 * @param[in] flow_split_info 5730 * Pointer to flow split info structure. 5731 * @param[out] error 5732 * Perform verbose error reporting if not NULL. 5733 * @return 5734 * 0 on success, negative value otherwise 5735 */ 5736 static int 5737 flow_create_split_metadata(struct rte_eth_dev *dev, 5738 struct rte_flow *flow, 5739 const struct rte_flow_attr *attr, 5740 const struct rte_flow_item items[], 5741 const struct rte_flow_action actions[], 5742 struct mlx5_flow_split_info *flow_split_info, 5743 struct rte_flow_error *error) 5744 { 5745 struct mlx5_priv *priv = dev->data->dev_private; 5746 struct mlx5_sh_config *config = &priv->sh->config; 5747 const struct rte_flow_action *qrss = NULL; 5748 struct rte_flow_action *ext_actions = NULL; 5749 struct mlx5_flow *dev_flow = NULL; 5750 uint32_t qrss_id = 0; 5751 int mtr_sfx = 0; 5752 size_t act_size; 5753 int actions_n; 5754 int encap_idx; 5755 int ret; 5756 5757 /* Check whether extensive metadata feature is engaged. */ 5758 if (!config->dv_flow_en || 5759 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5760 !mlx5_flow_ext_mreg_supported(dev)) 5761 return flow_create_split_inner(dev, flow, NULL, attr, items, 5762 actions, flow_split_info, error); 5763 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5764 &encap_idx); 5765 if (qrss) { 5766 /* Exclude hairpin flows from splitting. */ 5767 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5768 const struct rte_flow_action_queue *queue; 5769 5770 queue = qrss->conf; 5771 if (mlx5_rxq_get_type(dev, queue->index) == 5772 MLX5_RXQ_TYPE_HAIRPIN) 5773 qrss = NULL; 5774 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5775 const struct rte_flow_action_rss *rss; 5776 5777 rss = qrss->conf; 5778 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5779 MLX5_RXQ_TYPE_HAIRPIN) 5780 qrss = NULL; 5781 } 5782 } 5783 if (qrss) { 5784 /* Check if it is in meter suffix table. */ 5785 mtr_sfx = attr->group == (attr->transfer ? 5786 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5787 MLX5_FLOW_TABLE_LEVEL_METER); 5788 /* 5789 * Q/RSS action on NIC Rx should be split in order to pass by 5790 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5791 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5792 */ 5793 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5794 sizeof(struct rte_flow_action_set_tag) + 5795 sizeof(struct rte_flow_action_jump); 5796 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5797 SOCKET_ID_ANY); 5798 if (!ext_actions) 5799 return rte_flow_error_set(error, ENOMEM, 5800 RTE_FLOW_ERROR_TYPE_ACTION, 5801 NULL, "no memory to split " 5802 "metadata flow"); 5803 /* 5804 * Create the new actions list with removed Q/RSS action 5805 * and appended set tag and jump to register copy table 5806 * (RX_CP_TBL). We should preallocate unique tag ID here 5807 * in advance, because it is needed for set tag action. 5808 */ 5809 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5810 qrss, actions_n, 5811 mtr_sfx, error); 5812 if (!mtr_sfx && !qrss_id) { 5813 ret = -rte_errno; 5814 goto exit; 5815 } 5816 } else if (attr->egress && !attr->transfer) { 5817 /* 5818 * All the actions on NIC Tx should have a metadata register 5819 * copy action to copy reg_a from WQE to reg_c[meta] 5820 */ 5821 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5822 sizeof(struct mlx5_flow_action_copy_mreg); 5823 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5824 SOCKET_ID_ANY); 5825 if (!ext_actions) 5826 return rte_flow_error_set(error, ENOMEM, 5827 RTE_FLOW_ERROR_TYPE_ACTION, 5828 NULL, "no memory to split " 5829 "metadata flow"); 5830 /* Create the action list appended with copy register. */ 5831 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5832 actions_n, error, encap_idx); 5833 if (ret < 0) 5834 goto exit; 5835 } 5836 /* Add the unmodified original or prefix subflow. */ 5837 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5838 items, ext_actions ? ext_actions : 5839 actions, flow_split_info, error); 5840 if (ret < 0) 5841 goto exit; 5842 MLX5_ASSERT(dev_flow); 5843 if (qrss) { 5844 const struct rte_flow_attr q_attr = { 5845 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5846 .ingress = 1, 5847 }; 5848 /* Internal PMD action to set register. */ 5849 struct mlx5_rte_flow_item_tag q_tag_spec = { 5850 .data = qrss_id, 5851 .id = REG_NON, 5852 }; 5853 struct rte_flow_item q_items[] = { 5854 { 5855 .type = (enum rte_flow_item_type) 5856 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5857 .spec = &q_tag_spec, 5858 .last = NULL, 5859 .mask = NULL, 5860 }, 5861 { 5862 .type = RTE_FLOW_ITEM_TYPE_END, 5863 }, 5864 }; 5865 struct rte_flow_action q_actions[] = { 5866 { 5867 .type = qrss->type, 5868 .conf = qrss->conf, 5869 }, 5870 { 5871 .type = RTE_FLOW_ACTION_TYPE_END, 5872 }, 5873 }; 5874 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5875 5876 /* 5877 * Configure the tag item only if there is no meter subflow. 5878 * Since tag is already marked in the meter suffix subflow 5879 * we can just use the meter suffix items as is. 5880 */ 5881 if (qrss_id) { 5882 /* Not meter subflow. */ 5883 MLX5_ASSERT(!mtr_sfx); 5884 /* 5885 * Put unique id in prefix flow due to it is destroyed 5886 * after suffix flow and id will be freed after there 5887 * is no actual flows with this id and identifier 5888 * reallocation becomes possible (for example, for 5889 * other flows in other threads). 5890 */ 5891 dev_flow->handle->split_flow_id = qrss_id; 5892 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5893 error); 5894 if (ret < 0) 5895 goto exit; 5896 q_tag_spec.id = ret; 5897 } 5898 dev_flow = NULL; 5899 /* Add suffix subflow to execute Q/RSS. */ 5900 flow_split_info->prefix_layers = layers; 5901 flow_split_info->prefix_mark = 0; 5902 flow_split_info->table_id = 0; 5903 ret = flow_create_split_inner(dev, flow, &dev_flow, 5904 &q_attr, mtr_sfx ? items : 5905 q_items, q_actions, 5906 flow_split_info, error); 5907 if (ret < 0) 5908 goto exit; 5909 /* qrss ID should be freed if failed. */ 5910 qrss_id = 0; 5911 MLX5_ASSERT(dev_flow); 5912 } 5913 5914 exit: 5915 /* 5916 * We do not destroy the partially created sub_flows in case of error. 5917 * These ones are included into parent flow list and will be destroyed 5918 * by flow_drv_destroy. 5919 */ 5920 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5921 qrss_id); 5922 mlx5_free(ext_actions); 5923 return ret; 5924 } 5925 5926 /** 5927 * Create meter internal drop flow with the original pattern. 5928 * 5929 * @param dev 5930 * Pointer to Ethernet device. 5931 * @param[in] flow 5932 * Parent flow structure pointer. 5933 * @param[in] attr 5934 * Flow rule attributes. 5935 * @param[in] items 5936 * Pattern specification (list terminated by the END pattern item). 5937 * @param[in] flow_split_info 5938 * Pointer to flow split info structure. 5939 * @param[in] fm 5940 * Pointer to flow meter structure. 5941 * @param[out] error 5942 * Perform verbose error reporting if not NULL. 5943 * @return 5944 * 0 on success, negative value otherwise 5945 */ 5946 static uint32_t 5947 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5948 struct rte_flow *flow, 5949 const struct rte_flow_attr *attr, 5950 const struct rte_flow_item items[], 5951 struct mlx5_flow_split_info *flow_split_info, 5952 struct mlx5_flow_meter_info *fm, 5953 struct rte_flow_error *error) 5954 { 5955 struct mlx5_flow *dev_flow = NULL; 5956 struct rte_flow_attr drop_attr = *attr; 5957 struct rte_flow_action drop_actions[3]; 5958 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5959 5960 MLX5_ASSERT(fm->drop_cnt); 5961 drop_actions[0].type = 5962 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5963 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5964 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5965 drop_actions[1].conf = NULL; 5966 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5967 drop_actions[2].conf = NULL; 5968 drop_split_info.external = false; 5969 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5970 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5971 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5972 return flow_create_split_inner(dev, flow, &dev_flow, 5973 &drop_attr, items, drop_actions, 5974 &drop_split_info, error); 5975 } 5976 5977 /** 5978 * The splitting for meter feature. 5979 * 5980 * - The meter flow will be split to two flows as prefix and 5981 * suffix flow. The packets make sense only it pass the prefix 5982 * meter action. 5983 * 5984 * - Reg_C_5 is used for the packet to match betweend prefix and 5985 * suffix flow. 5986 * 5987 * @param dev 5988 * Pointer to Ethernet device. 5989 * @param[in] flow 5990 * Parent flow structure pointer. 5991 * @param[in] attr 5992 * Flow rule attributes. 5993 * @param[in] items 5994 * Pattern specification (list terminated by the END pattern item). 5995 * @param[in] actions 5996 * Associated actions (list terminated by the END action). 5997 * @param[in] flow_split_info 5998 * Pointer to flow split info structure. 5999 * @param[out] error 6000 * Perform verbose error reporting if not NULL. 6001 * @return 6002 * 0 on success, negative value otherwise 6003 */ 6004 static int 6005 flow_create_split_meter(struct rte_eth_dev *dev, 6006 struct rte_flow *flow, 6007 const struct rte_flow_attr *attr, 6008 const struct rte_flow_item items[], 6009 const struct rte_flow_action actions[], 6010 struct mlx5_flow_split_info *flow_split_info, 6011 struct rte_flow_error *error) 6012 { 6013 struct mlx5_priv *priv = dev->data->dev_private; 6014 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6015 struct rte_flow_action *sfx_actions = NULL; 6016 struct rte_flow_action *pre_actions = NULL; 6017 struct rte_flow_item *sfx_items = NULL; 6018 struct mlx5_flow *dev_flow = NULL; 6019 struct rte_flow_attr sfx_attr = *attr; 6020 struct mlx5_flow_meter_info *fm = NULL; 6021 uint8_t skip_scale_restore; 6022 bool has_mtr = false; 6023 bool has_modify = false; 6024 bool set_mtr_reg = true; 6025 bool is_mtr_hierarchy = false; 6026 uint32_t meter_id = 0; 6027 uint32_t mtr_idx = 0; 6028 uint32_t mtr_flow_id = 0; 6029 size_t act_size; 6030 size_t item_size; 6031 int actions_n = 0; 6032 int ret = 0; 6033 6034 if (priv->mtr_en) 6035 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6036 &has_modify, &meter_id); 6037 if (has_mtr) { 6038 if (flow->meter) { 6039 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6040 if (!fm) 6041 return rte_flow_error_set(error, EINVAL, 6042 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6043 NULL, "Meter not found."); 6044 } else { 6045 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6046 if (!fm) 6047 return rte_flow_error_set(error, EINVAL, 6048 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6049 NULL, "Meter not found."); 6050 ret = mlx5_flow_meter_attach(priv, fm, 6051 &sfx_attr, error); 6052 if (ret) 6053 return -rte_errno; 6054 flow->meter = mtr_idx; 6055 } 6056 MLX5_ASSERT(wks); 6057 wks->fm = fm; 6058 if (!fm->def_policy) { 6059 wks->policy = mlx5_flow_meter_policy_find(dev, 6060 fm->policy_id, 6061 NULL); 6062 MLX5_ASSERT(wks->policy); 6063 if (wks->policy->is_hierarchy) { 6064 wks->final_policy = 6065 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6066 wks->policy); 6067 if (!wks->final_policy) 6068 return rte_flow_error_set(error, 6069 EINVAL, 6070 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6071 "Failed to find terminal policy of hierarchy."); 6072 is_mtr_hierarchy = true; 6073 } 6074 } 6075 /* 6076 * If it isn't default-policy Meter, and 6077 * 1. There's no action in flow to change 6078 * packet (modify/encap/decap etc.), OR 6079 * 2. No drop count needed for this meter. 6080 * 3. It's not meter hierarchy. 6081 * Then no need to use regC to save meter id anymore. 6082 */ 6083 if (!fm->def_policy && !is_mtr_hierarchy && 6084 (!has_modify || !fm->drop_cnt)) 6085 set_mtr_reg = false; 6086 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6087 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6088 sizeof(struct mlx5_rte_flow_action_set_tag); 6089 /* Suffix items: tag, vlan, port id, end. */ 6090 #define METER_SUFFIX_ITEM 4 6091 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6092 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6093 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6094 0, SOCKET_ID_ANY); 6095 if (!sfx_actions) 6096 return rte_flow_error_set(error, ENOMEM, 6097 RTE_FLOW_ERROR_TYPE_ACTION, 6098 NULL, "no memory to split " 6099 "meter flow"); 6100 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6101 act_size); 6102 /* There's no suffix flow for meter of non-default policy. */ 6103 if (!fm->def_policy) 6104 pre_actions = sfx_actions + 1; 6105 else 6106 pre_actions = sfx_actions + actions_n; 6107 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6108 items, sfx_items, actions, 6109 sfx_actions, pre_actions, 6110 (set_mtr_reg ? &mtr_flow_id : NULL), 6111 error); 6112 if (ret) { 6113 ret = -rte_errno; 6114 goto exit; 6115 } 6116 /* Add the prefix subflow. */ 6117 skip_scale_restore = flow_split_info->skip_scale; 6118 flow_split_info->skip_scale |= 6119 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6120 ret = flow_create_split_inner(dev, flow, &dev_flow, 6121 attr, items, pre_actions, 6122 flow_split_info, error); 6123 flow_split_info->skip_scale = skip_scale_restore; 6124 if (ret) { 6125 if (mtr_flow_id) 6126 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6127 ret = -rte_errno; 6128 goto exit; 6129 } 6130 if (mtr_flow_id) { 6131 dev_flow->handle->split_flow_id = mtr_flow_id; 6132 dev_flow->handle->is_meter_flow_id = 1; 6133 } 6134 if (!fm->def_policy) { 6135 if (!set_mtr_reg && fm->drop_cnt) 6136 ret = 6137 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6138 &sfx_attr, items, 6139 flow_split_info, 6140 fm, error); 6141 goto exit; 6142 } 6143 /* Setting the sfx group atrr. */ 6144 sfx_attr.group = sfx_attr.transfer ? 6145 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6146 MLX5_FLOW_TABLE_LEVEL_METER; 6147 flow_split_info->prefix_layers = 6148 flow_get_prefix_layer_flags(dev_flow); 6149 flow_split_info->prefix_mark |= wks->mark; 6150 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6151 } 6152 /* Add the prefix subflow. */ 6153 ret = flow_create_split_metadata(dev, flow, 6154 &sfx_attr, sfx_items ? 6155 sfx_items : items, 6156 sfx_actions ? sfx_actions : actions, 6157 flow_split_info, error); 6158 exit: 6159 if (sfx_actions) 6160 mlx5_free(sfx_actions); 6161 return ret; 6162 } 6163 6164 /** 6165 * The splitting for sample feature. 6166 * 6167 * Once Sample action is detected in the action list, the flow actions should 6168 * be split into prefix sub flow and suffix sub flow. 6169 * 6170 * The original items remain in the prefix sub flow, all actions preceding the 6171 * sample action and the sample action itself will be copied to the prefix 6172 * sub flow, the actions following the sample action will be copied to the 6173 * suffix sub flow, Queue action always be located in the suffix sub flow. 6174 * 6175 * In order to make the packet from prefix sub flow matches with suffix sub 6176 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6177 * flow uses tag item with the unique flow id. 6178 * 6179 * @param dev 6180 * Pointer to Ethernet device. 6181 * @param[in] flow 6182 * Parent flow structure pointer. 6183 * @param[in] attr 6184 * Flow rule attributes. 6185 * @param[in] items 6186 * Pattern specification (list terminated by the END pattern item). 6187 * @param[in] actions 6188 * Associated actions (list terminated by the END action). 6189 * @param[in] flow_split_info 6190 * Pointer to flow split info structure. 6191 * @param[out] error 6192 * Perform verbose error reporting if not NULL. 6193 * @return 6194 * 0 on success, negative value otherwise 6195 */ 6196 static int 6197 flow_create_split_sample(struct rte_eth_dev *dev, 6198 struct rte_flow *flow, 6199 const struct rte_flow_attr *attr, 6200 const struct rte_flow_item items[], 6201 const struct rte_flow_action actions[], 6202 struct mlx5_flow_split_info *flow_split_info, 6203 struct rte_flow_error *error) 6204 { 6205 struct mlx5_priv *priv = dev->data->dev_private; 6206 struct rte_flow_action *sfx_actions = NULL; 6207 struct rte_flow_action *pre_actions = NULL; 6208 struct rte_flow_item *sfx_items = NULL; 6209 struct mlx5_flow *dev_flow = NULL; 6210 struct rte_flow_attr sfx_attr = *attr; 6211 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6212 struct mlx5_flow_dv_sample_resource *sample_res; 6213 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6214 struct mlx5_flow_tbl_resource *sfx_tbl; 6215 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6216 #endif 6217 size_t act_size; 6218 size_t item_size; 6219 uint32_t fdb_tx = 0; 6220 int32_t tag_id = 0; 6221 int actions_n = 0; 6222 int sample_action_pos; 6223 int qrss_action_pos; 6224 int add_tag = 0; 6225 int modify_after_mirror = 0; 6226 uint16_t jump_table = 0; 6227 const uint32_t next_ft_step = 1; 6228 int ret = 0; 6229 6230 if (priv->sampler_en) 6231 actions_n = flow_check_match_action(actions, attr, 6232 RTE_FLOW_ACTION_TYPE_SAMPLE, 6233 &sample_action_pos, &qrss_action_pos, 6234 &modify_after_mirror); 6235 if (actions_n) { 6236 /* The prefix actions must includes sample, tag, end. */ 6237 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6238 + sizeof(struct mlx5_rte_flow_action_set_tag); 6239 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6240 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6241 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6242 item_size), 0, SOCKET_ID_ANY); 6243 if (!sfx_actions) 6244 return rte_flow_error_set(error, ENOMEM, 6245 RTE_FLOW_ERROR_TYPE_ACTION, 6246 NULL, "no memory to split " 6247 "sample flow"); 6248 /* The representor_id is UINT16_MAX for uplink. */ 6249 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6250 /* 6251 * When reg_c_preserve is set, metadata registers Cx preserve 6252 * their value even through packet duplication. 6253 */ 6254 add_tag = (!fdb_tx || 6255 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6256 if (add_tag) 6257 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6258 + act_size); 6259 if (modify_after_mirror) 6260 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6261 next_ft_step; 6262 pre_actions = sfx_actions + actions_n; 6263 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6264 actions, sfx_actions, 6265 pre_actions, actions_n, 6266 sample_action_pos, 6267 qrss_action_pos, jump_table, 6268 error); 6269 if (tag_id < 0 || (add_tag && !tag_id)) { 6270 ret = -rte_errno; 6271 goto exit; 6272 } 6273 if (modify_after_mirror) 6274 flow_split_info->skip_scale = 6275 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6276 /* Add the prefix subflow. */ 6277 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6278 items, pre_actions, 6279 flow_split_info, error); 6280 if (ret) { 6281 ret = -rte_errno; 6282 goto exit; 6283 } 6284 dev_flow->handle->split_flow_id = tag_id; 6285 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6286 if (!modify_after_mirror) { 6287 /* Set the sfx group attr. */ 6288 sample_res = (struct mlx5_flow_dv_sample_resource *) 6289 dev_flow->dv.sample_res; 6290 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6291 sample_res->normal_path_tbl; 6292 sfx_tbl_data = container_of(sfx_tbl, 6293 struct mlx5_flow_tbl_data_entry, 6294 tbl); 6295 sfx_attr.group = sfx_attr.transfer ? 6296 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6297 } else { 6298 MLX5_ASSERT(attr->transfer); 6299 sfx_attr.group = jump_table; 6300 } 6301 flow_split_info->prefix_layers = 6302 flow_get_prefix_layer_flags(dev_flow); 6303 MLX5_ASSERT(wks); 6304 flow_split_info->prefix_mark |= wks->mark; 6305 /* Suffix group level already be scaled with factor, set 6306 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6307 * again in translation. 6308 */ 6309 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6310 #endif 6311 } 6312 /* Add the suffix subflow. */ 6313 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6314 sfx_items ? sfx_items : items, 6315 sfx_actions ? sfx_actions : actions, 6316 flow_split_info, error); 6317 exit: 6318 if (sfx_actions) 6319 mlx5_free(sfx_actions); 6320 return ret; 6321 } 6322 6323 /** 6324 * Split the flow to subflow set. The splitters might be linked 6325 * in the chain, like this: 6326 * flow_create_split_outer() calls: 6327 * flow_create_split_meter() calls: 6328 * flow_create_split_metadata(meter_subflow_0) calls: 6329 * flow_create_split_inner(metadata_subflow_0) 6330 * flow_create_split_inner(metadata_subflow_1) 6331 * flow_create_split_inner(metadata_subflow_2) 6332 * flow_create_split_metadata(meter_subflow_1) calls: 6333 * flow_create_split_inner(metadata_subflow_0) 6334 * flow_create_split_inner(metadata_subflow_1) 6335 * flow_create_split_inner(metadata_subflow_2) 6336 * 6337 * This provide flexible way to add new levels of flow splitting. 6338 * The all of successfully created subflows are included to the 6339 * parent flow dev_flow list. 6340 * 6341 * @param dev 6342 * Pointer to Ethernet device. 6343 * @param[in] flow 6344 * Parent flow structure pointer. 6345 * @param[in] attr 6346 * Flow rule attributes. 6347 * @param[in] items 6348 * Pattern specification (list terminated by the END pattern item). 6349 * @param[in] actions 6350 * Associated actions (list terminated by the END action). 6351 * @param[in] flow_split_info 6352 * Pointer to flow split info structure. 6353 * @param[out] error 6354 * Perform verbose error reporting if not NULL. 6355 * @return 6356 * 0 on success, negative value otherwise 6357 */ 6358 static int 6359 flow_create_split_outer(struct rte_eth_dev *dev, 6360 struct rte_flow *flow, 6361 const struct rte_flow_attr *attr, 6362 const struct rte_flow_item items[], 6363 const struct rte_flow_action actions[], 6364 struct mlx5_flow_split_info *flow_split_info, 6365 struct rte_flow_error *error) 6366 { 6367 int ret; 6368 6369 ret = flow_create_split_sample(dev, flow, attr, items, 6370 actions, flow_split_info, error); 6371 MLX5_ASSERT(ret <= 0); 6372 return ret; 6373 } 6374 6375 static inline struct mlx5_flow_tunnel * 6376 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6377 { 6378 struct mlx5_flow_tunnel *tunnel; 6379 6380 #pragma GCC diagnostic push 6381 #pragma GCC diagnostic ignored "-Wcast-qual" 6382 tunnel = (typeof(tunnel))flow->tunnel; 6383 #pragma GCC diagnostic pop 6384 6385 return tunnel; 6386 } 6387 6388 /** 6389 * Adjust flow RSS workspace if needed. 6390 * 6391 * @param wks 6392 * Pointer to thread flow work space. 6393 * @param rss_desc 6394 * Pointer to RSS descriptor. 6395 * @param[in] nrssq_num 6396 * New RSS queue number. 6397 * 6398 * @return 6399 * 0 on success, -1 otherwise and rte_errno is set. 6400 */ 6401 static int 6402 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6403 struct mlx5_flow_rss_desc *rss_desc, 6404 uint32_t nrssq_num) 6405 { 6406 if (likely(nrssq_num <= wks->rssq_num)) 6407 return 0; 6408 rss_desc->queue = realloc(rss_desc->queue, 6409 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6410 if (!rss_desc->queue) { 6411 rte_errno = ENOMEM; 6412 return -1; 6413 } 6414 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6415 return 0; 6416 } 6417 6418 /** 6419 * Create a flow and add it to @p list. 6420 * 6421 * @param dev 6422 * Pointer to Ethernet device. 6423 * @param list 6424 * Pointer to a TAILQ flow list. If this parameter NULL, 6425 * no list insertion occurred, flow is just created, 6426 * this is caller's responsibility to track the 6427 * created flow. 6428 * @param[in] attr 6429 * Flow rule attributes. 6430 * @param[in] items 6431 * Pattern specification (list terminated by the END pattern item). 6432 * @param[in] actions 6433 * Associated actions (list terminated by the END action). 6434 * @param[in] external 6435 * This flow rule is created by request external to PMD. 6436 * @param[out] error 6437 * Perform verbose error reporting if not NULL. 6438 * 6439 * @return 6440 * A flow index on success, 0 otherwise and rte_errno is set. 6441 */ 6442 static uint32_t 6443 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6444 const struct rte_flow_attr *attr, 6445 const struct rte_flow_item items[], 6446 const struct rte_flow_action original_actions[], 6447 bool external, struct rte_flow_error *error) 6448 { 6449 struct mlx5_priv *priv = dev->data->dev_private; 6450 struct rte_flow *flow = NULL; 6451 struct mlx5_flow *dev_flow; 6452 const struct rte_flow_action_rss *rss = NULL; 6453 struct mlx5_translated_action_handle 6454 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6455 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6456 union { 6457 struct mlx5_flow_expand_rss buf; 6458 uint8_t buffer[4096]; 6459 } expand_buffer; 6460 union { 6461 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6462 uint8_t buffer[2048]; 6463 } actions_rx; 6464 union { 6465 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6466 uint8_t buffer[2048]; 6467 } actions_hairpin_tx; 6468 union { 6469 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6470 uint8_t buffer[2048]; 6471 } items_tx; 6472 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6473 struct mlx5_flow_rss_desc *rss_desc; 6474 const struct rte_flow_action *p_actions_rx; 6475 uint32_t i; 6476 uint32_t idx = 0; 6477 int hairpin_flow; 6478 struct rte_flow_attr attr_tx = { .priority = 0 }; 6479 const struct rte_flow_action *actions; 6480 struct rte_flow_action *translated_actions = NULL; 6481 struct mlx5_flow_tunnel *tunnel; 6482 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6483 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6484 struct mlx5_flow_split_info flow_split_info = { 6485 .external = !!external, 6486 .skip_scale = 0, 6487 .flow_idx = 0, 6488 .prefix_mark = 0, 6489 .prefix_layers = 0, 6490 .table_id = 0 6491 }; 6492 int ret; 6493 6494 MLX5_ASSERT(wks); 6495 rss_desc = &wks->rss_desc; 6496 ret = flow_action_handles_translate(dev, original_actions, 6497 indir_actions, 6498 &indir_actions_n, 6499 &translated_actions, error); 6500 if (ret < 0) { 6501 MLX5_ASSERT(translated_actions == NULL); 6502 return 0; 6503 } 6504 actions = translated_actions ? translated_actions : original_actions; 6505 p_actions_rx = actions; 6506 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6507 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6508 external, hairpin_flow, error); 6509 if (ret < 0) 6510 goto error_before_hairpin_split; 6511 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6512 if (!flow) { 6513 rte_errno = ENOMEM; 6514 goto error_before_hairpin_split; 6515 } 6516 if (hairpin_flow > 0) { 6517 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6518 rte_errno = EINVAL; 6519 goto error_before_hairpin_split; 6520 } 6521 flow_hairpin_split(dev, actions, actions_rx.actions, 6522 actions_hairpin_tx.actions, items_tx.items, 6523 idx); 6524 p_actions_rx = actions_rx.actions; 6525 } 6526 flow_split_info.flow_idx = idx; 6527 flow->drv_type = flow_get_drv_type(dev, attr); 6528 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6529 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6530 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6531 /* RSS Action only works on NIC RX domain */ 6532 if (attr->ingress && !attr->transfer) 6533 rss = flow_get_rss_action(dev, p_actions_rx); 6534 if (rss) { 6535 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6536 return 0; 6537 /* 6538 * The following information is required by 6539 * mlx5_flow_hashfields_adjust() in advance. 6540 */ 6541 rss_desc->level = rss->level; 6542 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6543 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6544 } 6545 flow->dev_handles = 0; 6546 if (rss && rss->types) { 6547 unsigned int graph_root; 6548 6549 graph_root = find_graph_root(rss->level); 6550 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6551 items, rss->types, 6552 mlx5_support_expansion, graph_root); 6553 MLX5_ASSERT(ret > 0 && 6554 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6555 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6556 for (i = 0; i < buf->entries; ++i) 6557 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6558 } 6559 } else { 6560 buf->entries = 1; 6561 buf->entry[0].pattern = (void *)(uintptr_t)items; 6562 } 6563 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6564 indir_actions_n); 6565 for (i = 0; i < buf->entries; ++i) { 6566 /* Initialize flow split data. */ 6567 flow_split_info.prefix_layers = 0; 6568 flow_split_info.prefix_mark = 0; 6569 flow_split_info.skip_scale = 0; 6570 /* 6571 * The splitter may create multiple dev_flows, 6572 * depending on configuration. In the simplest 6573 * case it just creates unmodified original flow. 6574 */ 6575 ret = flow_create_split_outer(dev, flow, attr, 6576 buf->entry[i].pattern, 6577 p_actions_rx, &flow_split_info, 6578 error); 6579 if (ret < 0) 6580 goto error; 6581 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6582 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6583 p_actions_rx, 6584 idx, 6585 wks->flows[0].tunnel, 6586 &default_miss_ctx, 6587 error); 6588 if (ret < 0) { 6589 mlx5_free(default_miss_ctx.queue); 6590 goto error; 6591 } 6592 } 6593 } 6594 /* Create the tx flow. */ 6595 if (hairpin_flow) { 6596 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6597 attr_tx.ingress = 0; 6598 attr_tx.egress = 1; 6599 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6600 actions_hairpin_tx.actions, 6601 idx, error); 6602 if (!dev_flow) 6603 goto error; 6604 dev_flow->flow = flow; 6605 dev_flow->external = 0; 6606 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6607 dev_flow->handle, next); 6608 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6609 items_tx.items, 6610 actions_hairpin_tx.actions, error); 6611 if (ret < 0) 6612 goto error; 6613 } 6614 /* 6615 * Update the metadata register copy table. If extensive 6616 * metadata feature is enabled and registers are supported 6617 * we might create the extra rte_flow for each unique 6618 * MARK/FLAG action ID. 6619 * 6620 * The table is updated for ingress Flows only, because 6621 * the egress Flows belong to the different device and 6622 * copy table should be updated in peer NIC Rx domain. 6623 */ 6624 if (attr->ingress && 6625 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6626 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6627 if (ret) 6628 goto error; 6629 } 6630 /* 6631 * If the flow is external (from application) OR device is started, 6632 * OR mreg discover, then apply immediately. 6633 */ 6634 if (external || dev->data->dev_started || 6635 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6636 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6637 ret = flow_drv_apply(dev, flow, error); 6638 if (ret < 0) 6639 goto error; 6640 } 6641 flow->type = type; 6642 flow_rxq_flags_set(dev, flow); 6643 rte_free(translated_actions); 6644 tunnel = flow_tunnel_from_rule(wks->flows); 6645 if (tunnel) { 6646 flow->tunnel = 1; 6647 flow->tunnel_id = tunnel->tunnel_id; 6648 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6649 mlx5_free(default_miss_ctx.queue); 6650 } 6651 mlx5_flow_pop_thread_workspace(); 6652 return idx; 6653 error: 6654 MLX5_ASSERT(flow); 6655 ret = rte_errno; /* Save rte_errno before cleanup. */ 6656 flow_mreg_del_copy_action(dev, flow); 6657 flow_drv_destroy(dev, flow); 6658 if (rss_desc->shared_rss) 6659 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6660 mlx5_ipool_get 6661 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6662 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6663 mlx5_ipool_free(priv->flows[type], idx); 6664 rte_errno = ret; /* Restore rte_errno. */ 6665 ret = rte_errno; 6666 rte_errno = ret; 6667 mlx5_flow_pop_thread_workspace(); 6668 error_before_hairpin_split: 6669 rte_free(translated_actions); 6670 return 0; 6671 } 6672 6673 /** 6674 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6675 * incoming packets to table 1. 6676 * 6677 * Other flow rules, requested for group n, will be created in 6678 * e-switch table n+1. 6679 * Jump action to e-switch group n will be created to group n+1. 6680 * 6681 * Used when working in switchdev mode, to utilise advantages of table 1 6682 * and above. 6683 * 6684 * @param dev 6685 * Pointer to Ethernet device. 6686 * 6687 * @return 6688 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6689 */ 6690 struct rte_flow * 6691 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6692 { 6693 const struct rte_flow_attr attr = { 6694 .group = 0, 6695 .priority = 0, 6696 .ingress = 1, 6697 .egress = 0, 6698 .transfer = 1, 6699 }; 6700 const struct rte_flow_item pattern = { 6701 .type = RTE_FLOW_ITEM_TYPE_END, 6702 }; 6703 struct rte_flow_action_jump jump = { 6704 .group = 1, 6705 }; 6706 const struct rte_flow_action actions[] = { 6707 { 6708 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6709 .conf = &jump, 6710 }, 6711 { 6712 .type = RTE_FLOW_ACTION_TYPE_END, 6713 }, 6714 }; 6715 struct rte_flow_error error; 6716 6717 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6718 &attr, &pattern, 6719 actions, false, &error); 6720 } 6721 6722 /** 6723 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6724 * and sq number, directs all packets to peer vport. 6725 * 6726 * @param dev 6727 * Pointer to Ethernet device. 6728 * @param txq 6729 * Txq index. 6730 * 6731 * @return 6732 * Flow ID on success, 0 otherwise and rte_errno is set. 6733 */ 6734 uint32_t 6735 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6736 { 6737 struct rte_flow_attr attr = { 6738 .group = 0, 6739 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6740 .ingress = 1, 6741 .egress = 0, 6742 .transfer = 1, 6743 }; 6744 struct rte_flow_item_port_id port_spec = { 6745 .id = MLX5_PORT_ESW_MGR, 6746 }; 6747 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6748 .queue = txq, 6749 }; 6750 struct rte_flow_item pattern[] = { 6751 { 6752 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6753 .spec = &port_spec, 6754 }, 6755 { 6756 .type = (enum rte_flow_item_type) 6757 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6758 .spec = &txq_spec, 6759 }, 6760 { 6761 .type = RTE_FLOW_ITEM_TYPE_END, 6762 }, 6763 }; 6764 struct rte_flow_action_jump jump = { 6765 .group = 1, 6766 }; 6767 struct rte_flow_action_port_id port = { 6768 .id = dev->data->port_id, 6769 }; 6770 struct rte_flow_action actions[] = { 6771 { 6772 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6773 .conf = &jump, 6774 }, 6775 { 6776 .type = RTE_FLOW_ACTION_TYPE_END, 6777 }, 6778 }; 6779 struct rte_flow_error error; 6780 6781 /* 6782 * Creates group 0, highest priority jump flow. 6783 * Matches txq to bypass kernel packets. 6784 */ 6785 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 6786 false, &error) == 0) 6787 return 0; 6788 /* Create group 1, lowest priority redirect flow for txq. */ 6789 attr.group = 1; 6790 actions[0].conf = &port; 6791 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 6792 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 6793 actions, false, &error); 6794 } 6795 6796 /** 6797 * Validate a flow supported by the NIC. 6798 * 6799 * @see rte_flow_validate() 6800 * @see rte_flow_ops 6801 */ 6802 int 6803 mlx5_flow_validate(struct rte_eth_dev *dev, 6804 const struct rte_flow_attr *attr, 6805 const struct rte_flow_item items[], 6806 const struct rte_flow_action original_actions[], 6807 struct rte_flow_error *error) 6808 { 6809 int hairpin_flow; 6810 struct mlx5_translated_action_handle 6811 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6812 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6813 const struct rte_flow_action *actions; 6814 struct rte_flow_action *translated_actions = NULL; 6815 int ret = flow_action_handles_translate(dev, original_actions, 6816 indir_actions, 6817 &indir_actions_n, 6818 &translated_actions, error); 6819 6820 if (ret) 6821 return ret; 6822 actions = translated_actions ? translated_actions : original_actions; 6823 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6824 ret = flow_drv_validate(dev, attr, items, actions, 6825 true, hairpin_flow, error); 6826 rte_free(translated_actions); 6827 return ret; 6828 } 6829 6830 /** 6831 * Create a flow. 6832 * 6833 * @see rte_flow_create() 6834 * @see rte_flow_ops 6835 */ 6836 struct rte_flow * 6837 mlx5_flow_create(struct rte_eth_dev *dev, 6838 const struct rte_flow_attr *attr, 6839 const struct rte_flow_item items[], 6840 const struct rte_flow_action actions[], 6841 struct rte_flow_error *error) 6842 { 6843 struct mlx5_priv *priv = dev->data->dev_private; 6844 6845 if (priv->sh->config.dv_flow_en == 2) { 6846 rte_flow_error_set(error, ENOTSUP, 6847 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6848 NULL, 6849 "Flow non-Q creation not supported"); 6850 return NULL; 6851 } 6852 /* 6853 * If the device is not started yet, it is not allowed to created a 6854 * flow from application. PMD default flows and traffic control flows 6855 * are not affected. 6856 */ 6857 if (unlikely(!dev->data->dev_started)) { 6858 DRV_LOG(DEBUG, "port %u is not started when " 6859 "inserting a flow", dev->data->port_id); 6860 rte_flow_error_set(error, ENODEV, 6861 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6862 NULL, 6863 "port not started"); 6864 return NULL; 6865 } 6866 6867 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6868 attr, items, actions, 6869 true, error); 6870 } 6871 6872 /** 6873 * Destroy a flow in a list. 6874 * 6875 * @param dev 6876 * Pointer to Ethernet device. 6877 * @param[in] flow_idx 6878 * Index of flow to destroy. 6879 */ 6880 static void 6881 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6882 uint32_t flow_idx) 6883 { 6884 struct mlx5_priv *priv = dev->data->dev_private; 6885 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6886 6887 if (!flow) 6888 return; 6889 MLX5_ASSERT(flow->type == type); 6890 /* 6891 * Update RX queue flags only if port is started, otherwise it is 6892 * already clean. 6893 */ 6894 if (dev->data->dev_started) 6895 flow_rxq_flags_trim(dev, flow); 6896 flow_drv_destroy(dev, flow); 6897 if (flow->tunnel) { 6898 struct mlx5_flow_tunnel *tunnel; 6899 6900 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6901 RTE_VERIFY(tunnel); 6902 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6903 mlx5_flow_tunnel_free(dev, tunnel); 6904 } 6905 flow_mreg_del_copy_action(dev, flow); 6906 mlx5_ipool_free(priv->flows[type], flow_idx); 6907 } 6908 6909 /** 6910 * Destroy all flows. 6911 * 6912 * @param dev 6913 * Pointer to Ethernet device. 6914 * @param type 6915 * Flow type to be flushed. 6916 * @param active 6917 * If flushing is called actively. 6918 */ 6919 void 6920 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6921 bool active) 6922 { 6923 struct mlx5_priv *priv = dev->data->dev_private; 6924 uint32_t num_flushed = 0, fidx = 1; 6925 struct rte_flow *flow; 6926 6927 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6928 flow_list_destroy(dev, type, fidx); 6929 num_flushed++; 6930 } 6931 if (active) { 6932 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6933 dev->data->port_id, num_flushed); 6934 } 6935 } 6936 6937 /** 6938 * Stop all default actions for flows. 6939 * 6940 * @param dev 6941 * Pointer to Ethernet device. 6942 */ 6943 void 6944 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6945 { 6946 flow_mreg_del_default_copy_action(dev); 6947 flow_rxq_flags_clear(dev); 6948 } 6949 6950 /** 6951 * Start all default actions for flows. 6952 * 6953 * @param dev 6954 * Pointer to Ethernet device. 6955 * @return 6956 * 0 on success, a negative errno value otherwise and rte_errno is set. 6957 */ 6958 int 6959 mlx5_flow_start_default(struct rte_eth_dev *dev) 6960 { 6961 struct rte_flow_error error; 6962 6963 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6964 return flow_mreg_add_default_copy_action(dev, &error); 6965 } 6966 6967 /** 6968 * Release key of thread specific flow workspace data. 6969 */ 6970 void 6971 flow_release_workspace(void *data) 6972 { 6973 struct mlx5_flow_workspace *wks = data; 6974 struct mlx5_flow_workspace *next; 6975 6976 while (wks) { 6977 next = wks->next; 6978 free(wks->rss_desc.queue); 6979 free(wks); 6980 wks = next; 6981 } 6982 } 6983 6984 /** 6985 * Get thread specific current flow workspace. 6986 * 6987 * @return pointer to thread specific flow workspace data, NULL on error. 6988 */ 6989 struct mlx5_flow_workspace* 6990 mlx5_flow_get_thread_workspace(void) 6991 { 6992 struct mlx5_flow_workspace *data; 6993 6994 data = mlx5_flow_os_get_specific_workspace(); 6995 MLX5_ASSERT(data && data->inuse); 6996 if (!data || !data->inuse) 6997 DRV_LOG(ERR, "flow workspace not initialized."); 6998 return data; 6999 } 7000 7001 /** 7002 * Allocate and init new flow workspace. 7003 * 7004 * @return pointer to flow workspace data, NULL on error. 7005 */ 7006 static struct mlx5_flow_workspace* 7007 flow_alloc_thread_workspace(void) 7008 { 7009 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7010 7011 if (!data) { 7012 DRV_LOG(ERR, "Failed to allocate flow workspace " 7013 "memory."); 7014 return NULL; 7015 } 7016 data->rss_desc.queue = calloc(1, 7017 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7018 if (!data->rss_desc.queue) 7019 goto err; 7020 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7021 return data; 7022 err: 7023 free(data->rss_desc.queue); 7024 free(data); 7025 return NULL; 7026 } 7027 7028 /** 7029 * Get new thread specific flow workspace. 7030 * 7031 * If current workspace inuse, create new one and set as current. 7032 * 7033 * @return pointer to thread specific flow workspace data, NULL on error. 7034 */ 7035 static struct mlx5_flow_workspace* 7036 mlx5_flow_push_thread_workspace(void) 7037 { 7038 struct mlx5_flow_workspace *curr; 7039 struct mlx5_flow_workspace *data; 7040 7041 curr = mlx5_flow_os_get_specific_workspace(); 7042 if (!curr) { 7043 data = flow_alloc_thread_workspace(); 7044 if (!data) 7045 return NULL; 7046 } else if (!curr->inuse) { 7047 data = curr; 7048 } else if (curr->next) { 7049 data = curr->next; 7050 } else { 7051 data = flow_alloc_thread_workspace(); 7052 if (!data) 7053 return NULL; 7054 curr->next = data; 7055 data->prev = curr; 7056 } 7057 data->inuse = 1; 7058 data->flow_idx = 0; 7059 /* Set as current workspace */ 7060 if (mlx5_flow_os_set_specific_workspace(data)) 7061 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7062 return data; 7063 } 7064 7065 /** 7066 * Close current thread specific flow workspace. 7067 * 7068 * If previous workspace available, set it as current. 7069 * 7070 * @return pointer to thread specific flow workspace data, NULL on error. 7071 */ 7072 static void 7073 mlx5_flow_pop_thread_workspace(void) 7074 { 7075 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7076 7077 if (!data) 7078 return; 7079 if (!data->inuse) { 7080 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7081 return; 7082 } 7083 data->inuse = 0; 7084 if (!data->prev) 7085 return; 7086 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7087 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7088 } 7089 7090 /** 7091 * Verify the flow list is empty 7092 * 7093 * @param dev 7094 * Pointer to Ethernet device. 7095 * 7096 * @return the number of flows not released. 7097 */ 7098 int 7099 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7100 { 7101 struct mlx5_priv *priv = dev->data->dev_private; 7102 struct rte_flow *flow; 7103 uint32_t idx = 0; 7104 int ret = 0, i; 7105 7106 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7107 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7108 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7109 dev->data->port_id, (void *)flow); 7110 ret++; 7111 } 7112 } 7113 return ret; 7114 } 7115 7116 /** 7117 * Enable default hairpin egress flow. 7118 * 7119 * @param dev 7120 * Pointer to Ethernet device. 7121 * @param queue 7122 * The queue index. 7123 * 7124 * @return 7125 * 0 on success, a negative errno value otherwise and rte_errno is set. 7126 */ 7127 int 7128 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7129 uint32_t queue) 7130 { 7131 const struct rte_flow_attr attr = { 7132 .egress = 1, 7133 .priority = 0, 7134 }; 7135 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7136 .queue = queue, 7137 }; 7138 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7139 .queue = UINT32_MAX, 7140 }; 7141 struct rte_flow_item items[] = { 7142 { 7143 .type = (enum rte_flow_item_type) 7144 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7145 .spec = &queue_spec, 7146 .last = NULL, 7147 .mask = &queue_mask, 7148 }, 7149 { 7150 .type = RTE_FLOW_ITEM_TYPE_END, 7151 }, 7152 }; 7153 struct rte_flow_action_jump jump = { 7154 .group = MLX5_HAIRPIN_TX_TABLE, 7155 }; 7156 struct rte_flow_action actions[2]; 7157 uint32_t flow_idx; 7158 struct rte_flow_error error; 7159 7160 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7161 actions[0].conf = &jump; 7162 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7163 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7164 &attr, items, actions, false, &error); 7165 if (!flow_idx) { 7166 DRV_LOG(DEBUG, 7167 "Failed to create ctrl flow: rte_errno(%d)," 7168 " type(%d), message(%s)", 7169 rte_errno, error.type, 7170 error.message ? error.message : " (no stated reason)"); 7171 return -rte_errno; 7172 } 7173 return 0; 7174 } 7175 7176 /** 7177 * Enable a control flow configured from the control plane. 7178 * 7179 * @param dev 7180 * Pointer to Ethernet device. 7181 * @param eth_spec 7182 * An Ethernet flow spec to apply. 7183 * @param eth_mask 7184 * An Ethernet flow mask to apply. 7185 * @param vlan_spec 7186 * A VLAN flow spec to apply. 7187 * @param vlan_mask 7188 * A VLAN flow mask to apply. 7189 * 7190 * @return 7191 * 0 on success, a negative errno value otherwise and rte_errno is set. 7192 */ 7193 int 7194 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7195 struct rte_flow_item_eth *eth_spec, 7196 struct rte_flow_item_eth *eth_mask, 7197 struct rte_flow_item_vlan *vlan_spec, 7198 struct rte_flow_item_vlan *vlan_mask) 7199 { 7200 struct mlx5_priv *priv = dev->data->dev_private; 7201 const struct rte_flow_attr attr = { 7202 .ingress = 1, 7203 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7204 }; 7205 struct rte_flow_item items[] = { 7206 { 7207 .type = RTE_FLOW_ITEM_TYPE_ETH, 7208 .spec = eth_spec, 7209 .last = NULL, 7210 .mask = eth_mask, 7211 }, 7212 { 7213 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7214 RTE_FLOW_ITEM_TYPE_END, 7215 .spec = vlan_spec, 7216 .last = NULL, 7217 .mask = vlan_mask, 7218 }, 7219 { 7220 .type = RTE_FLOW_ITEM_TYPE_END, 7221 }, 7222 }; 7223 uint16_t queue[priv->reta_idx_n]; 7224 struct rte_flow_action_rss action_rss = { 7225 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7226 .level = 0, 7227 .types = priv->rss_conf.rss_hf, 7228 .key_len = priv->rss_conf.rss_key_len, 7229 .queue_num = priv->reta_idx_n, 7230 .key = priv->rss_conf.rss_key, 7231 .queue = queue, 7232 }; 7233 struct rte_flow_action actions[] = { 7234 { 7235 .type = RTE_FLOW_ACTION_TYPE_RSS, 7236 .conf = &action_rss, 7237 }, 7238 { 7239 .type = RTE_FLOW_ACTION_TYPE_END, 7240 }, 7241 }; 7242 uint32_t flow_idx; 7243 struct rte_flow_error error; 7244 unsigned int i; 7245 7246 if (!priv->reta_idx_n || !priv->rxqs_n) { 7247 return 0; 7248 } 7249 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7250 action_rss.types = 0; 7251 for (i = 0; i != priv->reta_idx_n; ++i) 7252 queue[i] = (*priv->reta_idx)[i]; 7253 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7254 &attr, items, actions, false, &error); 7255 if (!flow_idx) 7256 return -rte_errno; 7257 return 0; 7258 } 7259 7260 /** 7261 * Enable a flow control configured from the control plane. 7262 * 7263 * @param dev 7264 * Pointer to Ethernet device. 7265 * @param eth_spec 7266 * An Ethernet flow spec to apply. 7267 * @param eth_mask 7268 * An Ethernet flow mask to apply. 7269 * 7270 * @return 7271 * 0 on success, a negative errno value otherwise and rte_errno is set. 7272 */ 7273 int 7274 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7275 struct rte_flow_item_eth *eth_spec, 7276 struct rte_flow_item_eth *eth_mask) 7277 { 7278 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7279 } 7280 7281 /** 7282 * Create default miss flow rule matching lacp traffic 7283 * 7284 * @param dev 7285 * Pointer to Ethernet device. 7286 * @param eth_spec 7287 * An Ethernet flow spec to apply. 7288 * 7289 * @return 7290 * 0 on success, a negative errno value otherwise and rte_errno is set. 7291 */ 7292 int 7293 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7294 { 7295 /* 7296 * The LACP matching is done by only using ether type since using 7297 * a multicast dst mac causes kernel to give low priority to this flow. 7298 */ 7299 static const struct rte_flow_item_eth lacp_spec = { 7300 .type = RTE_BE16(0x8809), 7301 }; 7302 static const struct rte_flow_item_eth lacp_mask = { 7303 .type = 0xffff, 7304 }; 7305 const struct rte_flow_attr attr = { 7306 .ingress = 1, 7307 }; 7308 struct rte_flow_item items[] = { 7309 { 7310 .type = RTE_FLOW_ITEM_TYPE_ETH, 7311 .spec = &lacp_spec, 7312 .mask = &lacp_mask, 7313 }, 7314 { 7315 .type = RTE_FLOW_ITEM_TYPE_END, 7316 }, 7317 }; 7318 struct rte_flow_action actions[] = { 7319 { 7320 .type = (enum rte_flow_action_type) 7321 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7322 }, 7323 { 7324 .type = RTE_FLOW_ACTION_TYPE_END, 7325 }, 7326 }; 7327 struct rte_flow_error error; 7328 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7329 &attr, items, actions, 7330 false, &error); 7331 7332 if (!flow_idx) 7333 return -rte_errno; 7334 return 0; 7335 } 7336 7337 /** 7338 * Destroy a flow. 7339 * 7340 * @see rte_flow_destroy() 7341 * @see rte_flow_ops 7342 */ 7343 int 7344 mlx5_flow_destroy(struct rte_eth_dev *dev, 7345 struct rte_flow *flow, 7346 struct rte_flow_error *error __rte_unused) 7347 { 7348 struct mlx5_priv *priv = dev->data->dev_private; 7349 7350 if (priv->sh->config.dv_flow_en == 2) 7351 return rte_flow_error_set(error, ENOTSUP, 7352 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7353 NULL, 7354 "Flow non-Q destruction not supported"); 7355 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7356 (uintptr_t)(void *)flow); 7357 return 0; 7358 } 7359 7360 /** 7361 * Destroy all flows. 7362 * 7363 * @see rte_flow_flush() 7364 * @see rte_flow_ops 7365 */ 7366 int 7367 mlx5_flow_flush(struct rte_eth_dev *dev, 7368 struct rte_flow_error *error __rte_unused) 7369 { 7370 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7371 return 0; 7372 } 7373 7374 /** 7375 * Isolated mode. 7376 * 7377 * @see rte_flow_isolate() 7378 * @see rte_flow_ops 7379 */ 7380 int 7381 mlx5_flow_isolate(struct rte_eth_dev *dev, 7382 int enable, 7383 struct rte_flow_error *error) 7384 { 7385 struct mlx5_priv *priv = dev->data->dev_private; 7386 7387 if (dev->data->dev_started) { 7388 rte_flow_error_set(error, EBUSY, 7389 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7390 NULL, 7391 "port must be stopped first"); 7392 return -rte_errno; 7393 } 7394 priv->isolated = !!enable; 7395 if (enable) 7396 dev->dev_ops = &mlx5_dev_ops_isolate; 7397 else 7398 dev->dev_ops = &mlx5_dev_ops; 7399 7400 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7401 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7402 7403 return 0; 7404 } 7405 7406 /** 7407 * Query a flow. 7408 * 7409 * @see rte_flow_query() 7410 * @see rte_flow_ops 7411 */ 7412 static int 7413 flow_drv_query(struct rte_eth_dev *dev, 7414 uint32_t flow_idx, 7415 const struct rte_flow_action *actions, 7416 void *data, 7417 struct rte_flow_error *error) 7418 { 7419 struct mlx5_priv *priv = dev->data->dev_private; 7420 const struct mlx5_flow_driver_ops *fops; 7421 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7422 flow_idx); 7423 enum mlx5_flow_drv_type ftype; 7424 7425 if (!flow) { 7426 return rte_flow_error_set(error, ENOENT, 7427 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7428 NULL, 7429 "invalid flow handle"); 7430 } 7431 ftype = flow->drv_type; 7432 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7433 fops = flow_get_drv_ops(ftype); 7434 7435 return fops->query(dev, flow, actions, data, error); 7436 } 7437 7438 /** 7439 * Query a flow. 7440 * 7441 * @see rte_flow_query() 7442 * @see rte_flow_ops 7443 */ 7444 int 7445 mlx5_flow_query(struct rte_eth_dev *dev, 7446 struct rte_flow *flow, 7447 const struct rte_flow_action *actions, 7448 void *data, 7449 struct rte_flow_error *error) 7450 { 7451 int ret; 7452 struct mlx5_priv *priv = dev->data->dev_private; 7453 7454 if (priv->sh->config.dv_flow_en == 2) 7455 return rte_flow_error_set(error, ENOTSUP, 7456 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7457 NULL, 7458 "Flow non-Q query not supported"); 7459 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7460 error); 7461 if (ret < 0) 7462 return ret; 7463 return 0; 7464 } 7465 7466 /** 7467 * Get rte_flow callbacks. 7468 * 7469 * @param dev 7470 * Pointer to Ethernet device structure. 7471 * @param ops 7472 * Pointer to operation-specific structure. 7473 * 7474 * @return 0 7475 */ 7476 int 7477 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7478 const struct rte_flow_ops **ops) 7479 { 7480 *ops = &mlx5_flow_ops; 7481 return 0; 7482 } 7483 7484 /** 7485 * Validate meter policy actions. 7486 * Dispatcher for action type specific validation. 7487 * 7488 * @param[in] dev 7489 * Pointer to the Ethernet device structure. 7490 * @param[in] action 7491 * The meter policy action object to validate. 7492 * @param[in] attr 7493 * Attributes of flow to determine steering domain. 7494 * @param[out] is_rss 7495 * Is RSS or not. 7496 * @param[out] domain_bitmap 7497 * Domain bitmap. 7498 * @param[out] is_def_policy 7499 * Is default policy or not. 7500 * @param[out] error 7501 * Perform verbose error reporting if not NULL. Initialized in case of 7502 * error only. 7503 * 7504 * @return 7505 * 0 on success, otherwise negative errno value. 7506 */ 7507 int 7508 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7509 const struct rte_flow_action *actions[RTE_COLORS], 7510 struct rte_flow_attr *attr, 7511 bool *is_rss, 7512 uint8_t *domain_bitmap, 7513 uint8_t *policy_mode, 7514 struct rte_mtr_error *error) 7515 { 7516 const struct mlx5_flow_driver_ops *fops; 7517 7518 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7519 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7520 domain_bitmap, policy_mode, error); 7521 } 7522 7523 /** 7524 * Destroy the meter table set. 7525 * 7526 * @param[in] dev 7527 * Pointer to Ethernet device. 7528 * @param[in] mtr_policy 7529 * Meter policy struct. 7530 */ 7531 void 7532 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7533 struct mlx5_flow_meter_policy *mtr_policy) 7534 { 7535 const struct mlx5_flow_driver_ops *fops; 7536 7537 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7538 fops->destroy_mtr_acts(dev, mtr_policy); 7539 } 7540 7541 /** 7542 * Create policy action, lock free, 7543 * (mutex should be acquired by caller). 7544 * Dispatcher for action type specific call. 7545 * 7546 * @param[in] dev 7547 * Pointer to the Ethernet device structure. 7548 * @param[in] mtr_policy 7549 * Meter policy struct. 7550 * @param[in] action 7551 * Action specification used to create meter actions. 7552 * @param[out] error 7553 * Perform verbose error reporting if not NULL. Initialized in case of 7554 * error only. 7555 * 7556 * @return 7557 * 0 on success, otherwise negative errno value. 7558 */ 7559 int 7560 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7561 struct mlx5_flow_meter_policy *mtr_policy, 7562 const struct rte_flow_action *actions[RTE_COLORS], 7563 struct rte_mtr_error *error) 7564 { 7565 const struct mlx5_flow_driver_ops *fops; 7566 7567 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7568 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7569 } 7570 7571 /** 7572 * Create policy rules, lock free, 7573 * (mutex should be acquired by caller). 7574 * Dispatcher for action type specific call. 7575 * 7576 * @param[in] dev 7577 * Pointer to the Ethernet device structure. 7578 * @param[in] mtr_policy 7579 * Meter policy struct. 7580 * 7581 * @return 7582 * 0 on success, -1 otherwise. 7583 */ 7584 int 7585 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7586 struct mlx5_flow_meter_policy *mtr_policy) 7587 { 7588 const struct mlx5_flow_driver_ops *fops; 7589 7590 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7591 return fops->create_policy_rules(dev, mtr_policy); 7592 } 7593 7594 /** 7595 * Destroy policy rules, lock free, 7596 * (mutex should be acquired by caller). 7597 * Dispatcher for action type specific call. 7598 * 7599 * @param[in] dev 7600 * Pointer to the Ethernet device structure. 7601 * @param[in] mtr_policy 7602 * Meter policy struct. 7603 */ 7604 void 7605 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7606 struct mlx5_flow_meter_policy *mtr_policy) 7607 { 7608 const struct mlx5_flow_driver_ops *fops; 7609 7610 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7611 fops->destroy_policy_rules(dev, mtr_policy); 7612 } 7613 7614 /** 7615 * Destroy the default policy table set. 7616 * 7617 * @param[in] dev 7618 * Pointer to Ethernet device. 7619 */ 7620 void 7621 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7622 { 7623 const struct mlx5_flow_driver_ops *fops; 7624 7625 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7626 fops->destroy_def_policy(dev); 7627 } 7628 7629 /** 7630 * Destroy the default policy table set. 7631 * 7632 * @param[in] dev 7633 * Pointer to Ethernet device. 7634 * 7635 * @return 7636 * 0 on success, -1 otherwise. 7637 */ 7638 int 7639 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7640 { 7641 const struct mlx5_flow_driver_ops *fops; 7642 7643 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7644 return fops->create_def_policy(dev); 7645 } 7646 7647 /** 7648 * Create the needed meter and suffix tables. 7649 * 7650 * @param[in] dev 7651 * Pointer to Ethernet device. 7652 * 7653 * @return 7654 * 0 on success, -1 otherwise. 7655 */ 7656 int 7657 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7658 struct mlx5_flow_meter_info *fm, 7659 uint32_t mtr_idx, 7660 uint8_t domain_bitmap) 7661 { 7662 const struct mlx5_flow_driver_ops *fops; 7663 7664 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7665 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7666 } 7667 7668 /** 7669 * Destroy the meter table set. 7670 * 7671 * @param[in] dev 7672 * Pointer to Ethernet device. 7673 * @param[in] tbl 7674 * Pointer to the meter table set. 7675 */ 7676 void 7677 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7678 struct mlx5_flow_meter_info *fm) 7679 { 7680 const struct mlx5_flow_driver_ops *fops; 7681 7682 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7683 fops->destroy_mtr_tbls(dev, fm); 7684 } 7685 7686 /** 7687 * Destroy the global meter drop table. 7688 * 7689 * @param[in] dev 7690 * Pointer to Ethernet device. 7691 */ 7692 void 7693 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7694 { 7695 const struct mlx5_flow_driver_ops *fops; 7696 7697 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7698 fops->destroy_mtr_drop_tbls(dev); 7699 } 7700 7701 /** 7702 * Destroy the sub policy table with RX queue. 7703 * 7704 * @param[in] dev 7705 * Pointer to Ethernet device. 7706 * @param[in] mtr_policy 7707 * Pointer to meter policy table. 7708 */ 7709 void 7710 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7711 struct mlx5_flow_meter_policy *mtr_policy) 7712 { 7713 const struct mlx5_flow_driver_ops *fops; 7714 7715 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7716 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7717 } 7718 7719 /** 7720 * Allocate the needed aso flow meter id. 7721 * 7722 * @param[in] dev 7723 * Pointer to Ethernet device. 7724 * 7725 * @return 7726 * Index to aso flow meter on success, NULL otherwise. 7727 */ 7728 uint32_t 7729 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7730 { 7731 const struct mlx5_flow_driver_ops *fops; 7732 7733 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7734 return fops->create_meter(dev); 7735 } 7736 7737 /** 7738 * Free the aso flow meter id. 7739 * 7740 * @param[in] dev 7741 * Pointer to Ethernet device. 7742 * @param[in] mtr_idx 7743 * Index to aso flow meter to be free. 7744 * 7745 * @return 7746 * 0 on success. 7747 */ 7748 void 7749 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7750 { 7751 const struct mlx5_flow_driver_ops *fops; 7752 7753 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7754 fops->free_meter(dev, mtr_idx); 7755 } 7756 7757 /** 7758 * Allocate a counter. 7759 * 7760 * @param[in] dev 7761 * Pointer to Ethernet device structure. 7762 * 7763 * @return 7764 * Index to allocated counter on success, 0 otherwise. 7765 */ 7766 uint32_t 7767 mlx5_counter_alloc(struct rte_eth_dev *dev) 7768 { 7769 const struct mlx5_flow_driver_ops *fops; 7770 struct rte_flow_attr attr = { .transfer = 0 }; 7771 7772 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7773 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7774 return fops->counter_alloc(dev); 7775 } 7776 DRV_LOG(ERR, 7777 "port %u counter allocate is not supported.", 7778 dev->data->port_id); 7779 return 0; 7780 } 7781 7782 /** 7783 * Free a counter. 7784 * 7785 * @param[in] dev 7786 * Pointer to Ethernet device structure. 7787 * @param[in] cnt 7788 * Index to counter to be free. 7789 */ 7790 void 7791 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7792 { 7793 const struct mlx5_flow_driver_ops *fops; 7794 struct rte_flow_attr attr = { .transfer = 0 }; 7795 7796 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7797 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7798 fops->counter_free(dev, cnt); 7799 return; 7800 } 7801 DRV_LOG(ERR, 7802 "port %u counter free is not supported.", 7803 dev->data->port_id); 7804 } 7805 7806 /** 7807 * Query counter statistics. 7808 * 7809 * @param[in] dev 7810 * Pointer to Ethernet device structure. 7811 * @param[in] cnt 7812 * Index to counter to query. 7813 * @param[in] clear 7814 * Set to clear counter statistics. 7815 * @param[out] pkts 7816 * The counter hits packets number to save. 7817 * @param[out] bytes 7818 * The counter hits bytes number to save. 7819 * 7820 * @return 7821 * 0 on success, a negative errno value otherwise. 7822 */ 7823 int 7824 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7825 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 7826 { 7827 const struct mlx5_flow_driver_ops *fops; 7828 struct rte_flow_attr attr = { .transfer = 0 }; 7829 7830 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7831 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7832 return fops->counter_query(dev, cnt, clear, pkts, 7833 bytes, action); 7834 } 7835 DRV_LOG(ERR, 7836 "port %u counter query is not supported.", 7837 dev->data->port_id); 7838 return -ENOTSUP; 7839 } 7840 7841 /** 7842 * Allocate a new memory for the counter values wrapped by all the needed 7843 * management. 7844 * 7845 * @param[in] sh 7846 * Pointer to mlx5_dev_ctx_shared object. 7847 * 7848 * @return 7849 * 0 on success, a negative errno value otherwise. 7850 */ 7851 static int 7852 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7853 { 7854 struct mlx5_counter_stats_mem_mng *mem_mng; 7855 volatile struct flow_counter_stats *raw_data; 7856 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7857 int size = (sizeof(struct flow_counter_stats) * 7858 MLX5_COUNTERS_PER_POOL + 7859 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7860 sizeof(struct mlx5_counter_stats_mem_mng); 7861 size_t pgsize = rte_mem_page_size(); 7862 uint8_t *mem; 7863 int ret; 7864 int i; 7865 7866 if (pgsize == (size_t)-1) { 7867 DRV_LOG(ERR, "Failed to get mem page size"); 7868 rte_errno = ENOMEM; 7869 return -ENOMEM; 7870 } 7871 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7872 if (!mem) { 7873 rte_errno = ENOMEM; 7874 return -ENOMEM; 7875 } 7876 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7877 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7878 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 7879 sh->cdev->pdn, mem, size, 7880 &mem_mng->wm); 7881 if (ret) { 7882 rte_errno = errno; 7883 mlx5_free(mem); 7884 return -rte_errno; 7885 } 7886 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7887 raw_data = (volatile struct flow_counter_stats *)mem; 7888 for (i = 0; i < raws_n; ++i) { 7889 mem_mng->raws[i].mem_mng = mem_mng; 7890 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7891 } 7892 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7893 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7894 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7895 next); 7896 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7897 sh->cmng.mem_mng = mem_mng; 7898 return 0; 7899 } 7900 7901 /** 7902 * Set the statistic memory to the new counter pool. 7903 * 7904 * @param[in] sh 7905 * Pointer to mlx5_dev_ctx_shared object. 7906 * @param[in] pool 7907 * Pointer to the pool to set the statistic memory. 7908 * 7909 * @return 7910 * 0 on success, a negative errno value otherwise. 7911 */ 7912 static int 7913 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7914 struct mlx5_flow_counter_pool *pool) 7915 { 7916 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7917 /* Resize statistic memory once used out. */ 7918 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7919 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7920 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7921 return -1; 7922 } 7923 rte_spinlock_lock(&pool->sl); 7924 pool->raw = cmng->mem_mng->raws + pool->index % 7925 MLX5_CNT_CONTAINER_RESIZE; 7926 rte_spinlock_unlock(&pool->sl); 7927 pool->raw_hw = NULL; 7928 return 0; 7929 } 7930 7931 #define MLX5_POOL_QUERY_FREQ_US 1000000 7932 7933 /** 7934 * Set the periodic procedure for triggering asynchronous batch queries for all 7935 * the counter pools. 7936 * 7937 * @param[in] sh 7938 * Pointer to mlx5_dev_ctx_shared object. 7939 */ 7940 void 7941 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7942 { 7943 uint32_t pools_n, us; 7944 7945 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7946 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7947 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7948 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7949 sh->cmng.query_thread_on = 0; 7950 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7951 } else { 7952 sh->cmng.query_thread_on = 1; 7953 } 7954 } 7955 7956 /** 7957 * The periodic procedure for triggering asynchronous batch queries for all the 7958 * counter pools. This function is probably called by the host thread. 7959 * 7960 * @param[in] arg 7961 * The parameter for the alarm process. 7962 */ 7963 void 7964 mlx5_flow_query_alarm(void *arg) 7965 { 7966 struct mlx5_dev_ctx_shared *sh = arg; 7967 int ret; 7968 uint16_t pool_index = sh->cmng.pool_index; 7969 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7970 struct mlx5_flow_counter_pool *pool; 7971 uint16_t n_valid; 7972 7973 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7974 goto set_alarm; 7975 rte_spinlock_lock(&cmng->pool_update_sl); 7976 pool = cmng->pools[pool_index]; 7977 n_valid = cmng->n_valid; 7978 rte_spinlock_unlock(&cmng->pool_update_sl); 7979 /* Set the statistic memory to the new created pool. */ 7980 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7981 goto set_alarm; 7982 if (pool->raw_hw) 7983 /* There is a pool query in progress. */ 7984 goto set_alarm; 7985 pool->raw_hw = 7986 LIST_FIRST(&sh->cmng.free_stat_raws); 7987 if (!pool->raw_hw) 7988 /* No free counter statistics raw memory. */ 7989 goto set_alarm; 7990 /* 7991 * Identify the counters released between query trigger and query 7992 * handle more efficiently. The counter released in this gap period 7993 * should wait for a new round of query as the new arrived packets 7994 * will not be taken into account. 7995 */ 7996 pool->query_gen++; 7997 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7998 MLX5_COUNTERS_PER_POOL, 7999 NULL, NULL, 8000 pool->raw_hw->mem_mng->wm.lkey, 8001 (void *)(uintptr_t) 8002 pool->raw_hw->data, 8003 sh->devx_comp, 8004 (uint64_t)(uintptr_t)pool); 8005 if (ret) { 8006 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 8007 " %d", pool->min_dcs->id); 8008 pool->raw_hw = NULL; 8009 goto set_alarm; 8010 } 8011 LIST_REMOVE(pool->raw_hw, next); 8012 sh->cmng.pending_queries++; 8013 pool_index++; 8014 if (pool_index >= n_valid) 8015 pool_index = 0; 8016 set_alarm: 8017 sh->cmng.pool_index = pool_index; 8018 mlx5_set_query_alarm(sh); 8019 } 8020 8021 /** 8022 * Check and callback event for new aged flow in the counter pool 8023 * 8024 * @param[in] sh 8025 * Pointer to mlx5_dev_ctx_shared object. 8026 * @param[in] pool 8027 * Pointer to Current counter pool. 8028 */ 8029 static void 8030 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8031 struct mlx5_flow_counter_pool *pool) 8032 { 8033 struct mlx5_priv *priv; 8034 struct mlx5_flow_counter *cnt; 8035 struct mlx5_age_info *age_info; 8036 struct mlx5_age_param *age_param; 8037 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8038 struct mlx5_counter_stats_raw *prev = pool->raw; 8039 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8040 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8041 uint16_t expected = AGE_CANDIDATE; 8042 uint32_t i; 8043 8044 pool->time_of_last_age_check = curr_time; 8045 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8046 cnt = MLX5_POOL_GET_CNT(pool, i); 8047 age_param = MLX5_CNT_TO_AGE(cnt); 8048 if (__atomic_load_n(&age_param->state, 8049 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8050 continue; 8051 if (cur->data[i].hits != prev->data[i].hits) { 8052 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8053 __ATOMIC_RELAXED); 8054 continue; 8055 } 8056 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8057 time_delta, 8058 __ATOMIC_RELAXED) <= age_param->timeout) 8059 continue; 8060 /** 8061 * Hold the lock first, or if between the 8062 * state AGE_TMOUT and tailq operation the 8063 * release happened, the release procedure 8064 * may delete a non-existent tailq node. 8065 */ 8066 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8067 age_info = GET_PORT_AGE_INFO(priv); 8068 rte_spinlock_lock(&age_info->aged_sl); 8069 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8070 AGE_TMOUT, false, 8071 __ATOMIC_RELAXED, 8072 __ATOMIC_RELAXED)) { 8073 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8074 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8075 } 8076 rte_spinlock_unlock(&age_info->aged_sl); 8077 } 8078 mlx5_age_event_prepare(sh); 8079 } 8080 8081 /** 8082 * Handler for the HW respond about ready values from an asynchronous batch 8083 * query. This function is probably called by the host thread. 8084 * 8085 * @param[in] sh 8086 * The pointer to the shared device context. 8087 * @param[in] async_id 8088 * The Devx async ID. 8089 * @param[in] status 8090 * The status of the completion. 8091 */ 8092 void 8093 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8094 uint64_t async_id, int status) 8095 { 8096 struct mlx5_flow_counter_pool *pool = 8097 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8098 struct mlx5_counter_stats_raw *raw_to_free; 8099 uint8_t query_gen = pool->query_gen ^ 1; 8100 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8101 enum mlx5_counter_type cnt_type = 8102 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8103 MLX5_COUNTER_TYPE_ORIGIN; 8104 8105 if (unlikely(status)) { 8106 raw_to_free = pool->raw_hw; 8107 } else { 8108 raw_to_free = pool->raw; 8109 if (pool->is_aged) 8110 mlx5_flow_aging_check(sh, pool); 8111 rte_spinlock_lock(&pool->sl); 8112 pool->raw = pool->raw_hw; 8113 rte_spinlock_unlock(&pool->sl); 8114 /* Be sure the new raw counters data is updated in memory. */ 8115 rte_io_wmb(); 8116 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8117 rte_spinlock_lock(&cmng->csl[cnt_type]); 8118 TAILQ_CONCAT(&cmng->counters[cnt_type], 8119 &pool->counters[query_gen], next); 8120 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8121 } 8122 } 8123 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8124 pool->raw_hw = NULL; 8125 sh->cmng.pending_queries--; 8126 } 8127 8128 static int 8129 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8130 const struct flow_grp_info *grp_info, 8131 struct rte_flow_error *error) 8132 { 8133 if (grp_info->transfer && grp_info->external && 8134 grp_info->fdb_def_rule) { 8135 if (group == UINT32_MAX) 8136 return rte_flow_error_set 8137 (error, EINVAL, 8138 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8139 NULL, 8140 "group index not supported"); 8141 *table = group + 1; 8142 } else { 8143 *table = group; 8144 } 8145 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8146 return 0; 8147 } 8148 8149 /** 8150 * Translate the rte_flow group index to HW table value. 8151 * 8152 * If tunnel offload is disabled, all group ids converted to flow table 8153 * id using the standard method. 8154 * If tunnel offload is enabled, group id can be converted using the 8155 * standard or tunnel conversion method. Group conversion method 8156 * selection depends on flags in `grp_info` parameter: 8157 * - Internal (grp_info.external == 0) groups conversion uses the 8158 * standard method. 8159 * - Group ids in JUMP action converted with the tunnel conversion. 8160 * - Group id in rule attribute conversion depends on a rule type and 8161 * group id value: 8162 * ** non zero group attributes converted with the tunnel method 8163 * ** zero group attribute in non-tunnel rule is converted using the 8164 * standard method - there's only one root table 8165 * ** zero group attribute in steer tunnel rule is converted with the 8166 * standard method - single root table 8167 * ** zero group attribute in match tunnel rule is a special OvS 8168 * case: that value is used for portability reasons. That group 8169 * id is converted with the tunnel conversion method. 8170 * 8171 * @param[in] dev 8172 * Port device 8173 * @param[in] tunnel 8174 * PMD tunnel offload object 8175 * @param[in] group 8176 * rte_flow group index value. 8177 * @param[out] table 8178 * HW table value. 8179 * @param[in] grp_info 8180 * flags used for conversion 8181 * @param[out] error 8182 * Pointer to error structure. 8183 * 8184 * @return 8185 * 0 on success, a negative errno value otherwise and rte_errno is set. 8186 */ 8187 int 8188 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8189 const struct mlx5_flow_tunnel *tunnel, 8190 uint32_t group, uint32_t *table, 8191 const struct flow_grp_info *grp_info, 8192 struct rte_flow_error *error) 8193 { 8194 int ret; 8195 bool standard_translation; 8196 8197 if (!grp_info->skip_scale && grp_info->external && 8198 group < MLX5_MAX_TABLES_EXTERNAL) 8199 group *= MLX5_FLOW_TABLE_FACTOR; 8200 if (is_tunnel_offload_active(dev)) { 8201 standard_translation = !grp_info->external || 8202 grp_info->std_tbl_fix; 8203 } else { 8204 standard_translation = true; 8205 } 8206 DRV_LOG(DEBUG, 8207 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8208 dev->data->port_id, group, grp_info->transfer, 8209 grp_info->external, grp_info->fdb_def_rule, 8210 standard_translation ? "STANDARD" : "TUNNEL"); 8211 if (standard_translation) 8212 ret = flow_group_to_table(dev->data->port_id, group, table, 8213 grp_info, error); 8214 else 8215 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8216 table, error); 8217 8218 return ret; 8219 } 8220 8221 /** 8222 * Discover availability of metadata reg_c's. 8223 * 8224 * Iteratively use test flows to check availability. 8225 * 8226 * @param[in] dev 8227 * Pointer to the Ethernet device structure. 8228 * 8229 * @return 8230 * 0 on success, a negative errno value otherwise and rte_errno is set. 8231 */ 8232 int 8233 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8234 { 8235 struct mlx5_priv *priv = dev->data->dev_private; 8236 enum modify_reg idx; 8237 int n = 0; 8238 8239 /* reg_c[0] and reg_c[1] are reserved. */ 8240 priv->sh->flow_mreg_c[n++] = REG_C_0; 8241 priv->sh->flow_mreg_c[n++] = REG_C_1; 8242 /* Discover availability of other reg_c's. */ 8243 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 8244 struct rte_flow_attr attr = { 8245 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 8246 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8247 .ingress = 1, 8248 }; 8249 struct rte_flow_item items[] = { 8250 [0] = { 8251 .type = RTE_FLOW_ITEM_TYPE_END, 8252 }, 8253 }; 8254 struct rte_flow_action actions[] = { 8255 [0] = { 8256 .type = (enum rte_flow_action_type) 8257 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8258 .conf = &(struct mlx5_flow_action_copy_mreg){ 8259 .src = REG_C_1, 8260 .dst = idx, 8261 }, 8262 }, 8263 [1] = { 8264 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8265 .conf = &(struct rte_flow_action_jump){ 8266 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8267 }, 8268 }, 8269 [2] = { 8270 .type = RTE_FLOW_ACTION_TYPE_END, 8271 }, 8272 }; 8273 uint32_t flow_idx; 8274 struct rte_flow *flow; 8275 struct rte_flow_error error; 8276 8277 if (!priv->sh->config.dv_flow_en) 8278 break; 8279 /* Create internal flow, validation skips copy action. */ 8280 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8281 items, actions, false, &error); 8282 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8283 flow_idx); 8284 if (!flow) 8285 continue; 8286 priv->sh->flow_mreg_c[n++] = idx; 8287 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8288 } 8289 for (; n < MLX5_MREG_C_NUM; ++n) 8290 priv->sh->flow_mreg_c[n] = REG_NON; 8291 priv->sh->metadata_regc_check_flag = 1; 8292 return 0; 8293 } 8294 8295 int 8296 save_dump_file(const uint8_t *data, uint32_t size, 8297 uint32_t type, uint64_t id, void *arg, FILE *file) 8298 { 8299 char line[BUF_SIZE]; 8300 uint32_t out = 0; 8301 uint32_t k; 8302 uint32_t actions_num; 8303 struct rte_flow_query_count *count; 8304 8305 memset(line, 0, BUF_SIZE); 8306 switch (type) { 8307 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8308 actions_num = *(uint32_t *)(arg); 8309 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 8310 type, id, actions_num); 8311 break; 8312 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8313 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 8314 type, id); 8315 break; 8316 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8317 count = (struct rte_flow_query_count *)arg; 8318 fprintf(file, 8319 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 8320 type, id, count->hits, count->bytes); 8321 return 0; 8322 default: 8323 return -1; 8324 } 8325 8326 for (k = 0; k < size; k++) { 8327 /* Make sure we do not overrun the line buffer length. */ 8328 if (out >= BUF_SIZE - 4) { 8329 line[out] = '\0'; 8330 break; 8331 } 8332 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8333 (data[k]) & 0xff); 8334 } 8335 fprintf(file, "%s\n", line); 8336 return 0; 8337 } 8338 8339 int 8340 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8341 struct rte_flow_query_count *count, struct rte_flow_error *error) 8342 { 8343 struct rte_flow_action action[2]; 8344 enum mlx5_flow_drv_type ftype; 8345 const struct mlx5_flow_driver_ops *fops; 8346 8347 if (!flow) { 8348 return rte_flow_error_set(error, ENOENT, 8349 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8350 NULL, 8351 "invalid flow handle"); 8352 } 8353 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8354 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8355 if (flow->counter) { 8356 memset(count, 0, sizeof(struct rte_flow_query_count)); 8357 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8358 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8359 ftype < MLX5_FLOW_TYPE_MAX); 8360 fops = flow_get_drv_ops(ftype); 8361 return fops->query(dev, flow, action, count, error); 8362 } 8363 return -1; 8364 } 8365 8366 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8367 /** 8368 * Dump flow ipool data to file 8369 * 8370 * @param[in] dev 8371 * The pointer to Ethernet device. 8372 * @param[in] file 8373 * A pointer to a file for output. 8374 * @param[out] error 8375 * Perform verbose error reporting if not NULL. PMDs initialize this 8376 * structure in case of error only. 8377 * @return 8378 * 0 on success, a negative value otherwise. 8379 */ 8380 int 8381 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8382 struct rte_flow *flow, FILE *file, 8383 struct rte_flow_error *error) 8384 { 8385 struct mlx5_priv *priv = dev->data->dev_private; 8386 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8387 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8388 uint32_t handle_idx; 8389 struct mlx5_flow_handle *dh; 8390 struct rte_flow_query_count count; 8391 uint32_t actions_num; 8392 const uint8_t *data; 8393 size_t size; 8394 uint64_t id; 8395 uint32_t type; 8396 void *action = NULL; 8397 8398 if (!flow) { 8399 return rte_flow_error_set(error, ENOENT, 8400 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8401 NULL, 8402 "invalid flow handle"); 8403 } 8404 handle_idx = flow->dev_handles; 8405 /* query counter */ 8406 if (flow->counter && 8407 (!mlx5_counter_query(dev, flow->counter, false, 8408 &count.hits, &count.bytes, &action)) && action) { 8409 id = (uint64_t)(uintptr_t)action; 8410 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8411 save_dump_file(NULL, 0, type, 8412 id, (void *)&count, file); 8413 } 8414 8415 while (handle_idx) { 8416 dh = mlx5_ipool_get(priv->sh->ipool 8417 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8418 if (!dh) 8419 continue; 8420 handle_idx = dh->next.next; 8421 8422 /* Get modify_hdr and encap_decap buf from ipools. */ 8423 encap_decap = NULL; 8424 modify_hdr = dh->dvh.modify_hdr; 8425 8426 if (dh->dvh.rix_encap_decap) { 8427 encap_decap = mlx5_ipool_get(priv->sh->ipool 8428 [MLX5_IPOOL_DECAP_ENCAP], 8429 dh->dvh.rix_encap_decap); 8430 } 8431 if (modify_hdr) { 8432 data = (const uint8_t *)modify_hdr->actions; 8433 size = (size_t)(modify_hdr->actions_num) * 8; 8434 id = (uint64_t)(uintptr_t)modify_hdr->action; 8435 actions_num = modify_hdr->actions_num; 8436 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8437 save_dump_file(data, size, type, id, 8438 (void *)(&actions_num), file); 8439 } 8440 if (encap_decap) { 8441 data = encap_decap->buf; 8442 size = encap_decap->size; 8443 id = (uint64_t)(uintptr_t)encap_decap->action; 8444 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8445 save_dump_file(data, size, type, 8446 id, NULL, file); 8447 } 8448 } 8449 return 0; 8450 } 8451 8452 /** 8453 * Dump all flow's encap_decap/modify_hdr/counter data to file 8454 * 8455 * @param[in] dev 8456 * The pointer to Ethernet device. 8457 * @param[in] file 8458 * A pointer to a file for output. 8459 * @param[out] error 8460 * Perform verbose error reporting if not NULL. PMDs initialize this 8461 * structure in case of error only. 8462 * @return 8463 * 0 on success, a negative value otherwise. 8464 */ 8465 static int 8466 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 8467 FILE *file, struct rte_flow_error *error __rte_unused) 8468 { 8469 struct mlx5_priv *priv = dev->data->dev_private; 8470 struct mlx5_dev_ctx_shared *sh = priv->sh; 8471 struct mlx5_hlist *h; 8472 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8473 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8474 struct rte_flow_query_count count; 8475 uint32_t actions_num; 8476 const uint8_t *data; 8477 size_t size; 8478 uint64_t id; 8479 uint32_t type; 8480 uint32_t i; 8481 uint32_t j; 8482 struct mlx5_list_inconst *l_inconst; 8483 struct mlx5_list_entry *e; 8484 int lcore_index; 8485 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 8486 uint32_t max; 8487 void *action; 8488 8489 /* encap_decap hlist is lcore_share, get global core cache. */ 8490 i = MLX5_LIST_GLOBAL; 8491 h = sh->encaps_decaps; 8492 if (h) { 8493 for (j = 0; j <= h->mask; j++) { 8494 l_inconst = &h->buckets[j].l; 8495 if (!l_inconst || !l_inconst->cache[i]) 8496 continue; 8497 8498 e = LIST_FIRST(&l_inconst->cache[i]->h); 8499 while (e) { 8500 encap_decap = 8501 (struct mlx5_flow_dv_encap_decap_resource *)e; 8502 data = encap_decap->buf; 8503 size = encap_decap->size; 8504 id = (uint64_t)(uintptr_t)encap_decap->action; 8505 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8506 save_dump_file(data, size, type, 8507 id, NULL, file); 8508 e = LIST_NEXT(e, next); 8509 } 8510 } 8511 } 8512 8513 /* get modify_hdr */ 8514 h = sh->modify_cmds; 8515 if (h) { 8516 lcore_index = rte_lcore_index(rte_lcore_id()); 8517 if (unlikely(lcore_index == -1)) { 8518 lcore_index = MLX5_LIST_NLCORE; 8519 rte_spinlock_lock(&h->l_const.lcore_lock); 8520 } 8521 i = lcore_index; 8522 8523 for (j = 0; j <= h->mask; j++) { 8524 l_inconst = &h->buckets[j].l; 8525 if (!l_inconst || !l_inconst->cache[i]) 8526 continue; 8527 8528 e = LIST_FIRST(&l_inconst->cache[i]->h); 8529 while (e) { 8530 modify_hdr = 8531 (struct mlx5_flow_dv_modify_hdr_resource *)e; 8532 data = (const uint8_t *)modify_hdr->actions; 8533 size = (size_t)(modify_hdr->actions_num) * 8; 8534 actions_num = modify_hdr->actions_num; 8535 id = (uint64_t)(uintptr_t)modify_hdr->action; 8536 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8537 save_dump_file(data, size, type, id, 8538 (void *)(&actions_num), file); 8539 e = LIST_NEXT(e, next); 8540 } 8541 } 8542 8543 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 8544 rte_spinlock_unlock(&h->l_const.lcore_lock); 8545 } 8546 8547 /* get counter */ 8548 MLX5_ASSERT(cmng->n_valid <= cmng->n); 8549 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 8550 for (j = 1; j <= max; j++) { 8551 action = NULL; 8552 if ((!mlx5_counter_query(dev, j, false, &count.hits, 8553 &count.bytes, &action)) && action) { 8554 id = (uint64_t)(uintptr_t)action; 8555 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8556 save_dump_file(NULL, 0, type, 8557 id, (void *)&count, file); 8558 } 8559 } 8560 return 0; 8561 } 8562 #endif 8563 8564 /** 8565 * Dump flow raw hw data to file 8566 * 8567 * @param[in] dev 8568 * The pointer to Ethernet device. 8569 * @param[in] file 8570 * A pointer to a file for output. 8571 * @param[out] error 8572 * Perform verbose error reporting if not NULL. PMDs initialize this 8573 * structure in case of error only. 8574 * @return 8575 * 0 on success, a negative value otherwise. 8576 */ 8577 int 8578 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8579 FILE *file, 8580 struct rte_flow_error *error __rte_unused) 8581 { 8582 struct mlx5_priv *priv = dev->data->dev_private; 8583 struct mlx5_dev_ctx_shared *sh = priv->sh; 8584 uint32_t handle_idx; 8585 int ret; 8586 struct mlx5_flow_handle *dh; 8587 struct rte_flow *flow; 8588 8589 if (!sh->config.dv_flow_en) { 8590 if (fputs("device dv flow disabled\n", file) <= 0) 8591 return -errno; 8592 return -ENOTSUP; 8593 } 8594 8595 /* dump all */ 8596 if (!flow_idx) { 8597 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8598 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 8599 return -EINVAL; 8600 #endif 8601 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8602 sh->rx_domain, 8603 sh->tx_domain, file); 8604 } 8605 /* dump one */ 8606 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8607 (uintptr_t)(void *)flow_idx); 8608 if (!flow) 8609 return -EINVAL; 8610 8611 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8612 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8613 #endif 8614 handle_idx = flow->dev_handles; 8615 while (handle_idx) { 8616 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8617 handle_idx); 8618 if (!dh) 8619 return -ENOENT; 8620 if (dh->drv_flow) { 8621 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8622 file); 8623 if (ret) 8624 return -ENOENT; 8625 } 8626 handle_idx = dh->next.next; 8627 } 8628 return 0; 8629 } 8630 8631 /** 8632 * Get aged-out flows. 8633 * 8634 * @param[in] dev 8635 * Pointer to the Ethernet device structure. 8636 * @param[in] context 8637 * The address of an array of pointers to the aged-out flows contexts. 8638 * @param[in] nb_countexts 8639 * The length of context array pointers. 8640 * @param[out] error 8641 * Perform verbose error reporting if not NULL. Initialized in case of 8642 * error only. 8643 * 8644 * @return 8645 * how many contexts get in success, otherwise negative errno value. 8646 * if nb_contexts is 0, return the amount of all aged contexts. 8647 * if nb_contexts is not 0 , return the amount of aged flows reported 8648 * in the context array. 8649 */ 8650 int 8651 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8652 uint32_t nb_contexts, struct rte_flow_error *error) 8653 { 8654 const struct mlx5_flow_driver_ops *fops; 8655 struct rte_flow_attr attr = { .transfer = 0 }; 8656 8657 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8658 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8659 return fops->get_aged_flows(dev, contexts, nb_contexts, 8660 error); 8661 } 8662 DRV_LOG(ERR, 8663 "port %u get aged flows is not supported.", 8664 dev->data->port_id); 8665 return -ENOTSUP; 8666 } 8667 8668 /* Wrapper for driver action_validate op callback */ 8669 static int 8670 flow_drv_action_validate(struct rte_eth_dev *dev, 8671 const struct rte_flow_indir_action_conf *conf, 8672 const struct rte_flow_action *action, 8673 const struct mlx5_flow_driver_ops *fops, 8674 struct rte_flow_error *error) 8675 { 8676 static const char err_msg[] = "indirect action validation unsupported"; 8677 8678 if (!fops->action_validate) { 8679 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8680 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8681 NULL, err_msg); 8682 return -rte_errno; 8683 } 8684 return fops->action_validate(dev, conf, action, error); 8685 } 8686 8687 /** 8688 * Destroys the shared action by handle. 8689 * 8690 * @param dev 8691 * Pointer to Ethernet device structure. 8692 * @param[in] handle 8693 * Handle for the indirect action object to be destroyed. 8694 * @param[out] error 8695 * Perform verbose error reporting if not NULL. PMDs initialize this 8696 * structure in case of error only. 8697 * 8698 * @return 8699 * 0 on success, a negative errno value otherwise and rte_errno is set. 8700 * 8701 * @note: wrapper for driver action_create op callback. 8702 */ 8703 static int 8704 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8705 struct rte_flow_action_handle *handle, 8706 struct rte_flow_error *error) 8707 { 8708 static const char err_msg[] = "indirect action destruction unsupported"; 8709 struct rte_flow_attr attr = { .transfer = 0 }; 8710 const struct mlx5_flow_driver_ops *fops = 8711 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8712 8713 if (!fops->action_destroy) { 8714 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8715 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8716 NULL, err_msg); 8717 return -rte_errno; 8718 } 8719 return fops->action_destroy(dev, handle, error); 8720 } 8721 8722 /* Wrapper for driver action_destroy op callback */ 8723 static int 8724 flow_drv_action_update(struct rte_eth_dev *dev, 8725 struct rte_flow_action_handle *handle, 8726 const void *update, 8727 const struct mlx5_flow_driver_ops *fops, 8728 struct rte_flow_error *error) 8729 { 8730 static const char err_msg[] = "indirect action update unsupported"; 8731 8732 if (!fops->action_update) { 8733 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8734 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8735 NULL, err_msg); 8736 return -rte_errno; 8737 } 8738 return fops->action_update(dev, handle, update, error); 8739 } 8740 8741 /* Wrapper for driver action_destroy op callback */ 8742 static int 8743 flow_drv_action_query(struct rte_eth_dev *dev, 8744 const struct rte_flow_action_handle *handle, 8745 void *data, 8746 const struct mlx5_flow_driver_ops *fops, 8747 struct rte_flow_error *error) 8748 { 8749 static const char err_msg[] = "indirect action query unsupported"; 8750 8751 if (!fops->action_query) { 8752 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8753 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8754 NULL, err_msg); 8755 return -rte_errno; 8756 } 8757 return fops->action_query(dev, handle, data, error); 8758 } 8759 8760 /** 8761 * Create indirect action for reuse in multiple flow rules. 8762 * 8763 * @param dev 8764 * Pointer to Ethernet device structure. 8765 * @param conf 8766 * Pointer to indirect action object configuration. 8767 * @param[in] action 8768 * Action configuration for indirect action object creation. 8769 * @param[out] error 8770 * Perform verbose error reporting if not NULL. PMDs initialize this 8771 * structure in case of error only. 8772 * @return 8773 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8774 */ 8775 static struct rte_flow_action_handle * 8776 mlx5_action_handle_create(struct rte_eth_dev *dev, 8777 const struct rte_flow_indir_action_conf *conf, 8778 const struct rte_flow_action *action, 8779 struct rte_flow_error *error) 8780 { 8781 static const char err_msg[] = "indirect action creation unsupported"; 8782 struct rte_flow_attr attr = { .transfer = 0 }; 8783 const struct mlx5_flow_driver_ops *fops = 8784 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8785 8786 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8787 return NULL; 8788 if (!fops->action_create) { 8789 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8790 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8791 NULL, err_msg); 8792 return NULL; 8793 } 8794 return fops->action_create(dev, conf, action, error); 8795 } 8796 8797 /** 8798 * Updates inplace the indirect action configuration pointed by *handle* 8799 * with the configuration provided as *update* argument. 8800 * The update of the indirect action configuration effects all flow rules 8801 * reusing the action via handle. 8802 * 8803 * @param dev 8804 * Pointer to Ethernet device structure. 8805 * @param[in] handle 8806 * Handle for the indirect action to be updated. 8807 * @param[in] update 8808 * Action specification used to modify the action pointed by handle. 8809 * *update* could be of same type with the action pointed by the *handle* 8810 * handle argument, or some other structures like a wrapper, depending on 8811 * the indirect action type. 8812 * @param[out] error 8813 * Perform verbose error reporting if not NULL. PMDs initialize this 8814 * structure in case of error only. 8815 * 8816 * @return 8817 * 0 on success, a negative errno value otherwise and rte_errno is set. 8818 */ 8819 static int 8820 mlx5_action_handle_update(struct rte_eth_dev *dev, 8821 struct rte_flow_action_handle *handle, 8822 const void *update, 8823 struct rte_flow_error *error) 8824 { 8825 struct rte_flow_attr attr = { .transfer = 0 }; 8826 const struct mlx5_flow_driver_ops *fops = 8827 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8828 int ret; 8829 8830 ret = flow_drv_action_validate(dev, NULL, 8831 (const struct rte_flow_action *)update, fops, error); 8832 if (ret) 8833 return ret; 8834 return flow_drv_action_update(dev, handle, update, fops, 8835 error); 8836 } 8837 8838 /** 8839 * Query the indirect action by handle. 8840 * 8841 * This function allows retrieving action-specific data such as counters. 8842 * Data is gathered by special action which may be present/referenced in 8843 * more than one flow rule definition. 8844 * 8845 * see @RTE_FLOW_ACTION_TYPE_COUNT 8846 * 8847 * @param dev 8848 * Pointer to Ethernet device structure. 8849 * @param[in] handle 8850 * Handle for the indirect action to query. 8851 * @param[in, out] data 8852 * Pointer to storage for the associated query data type. 8853 * @param[out] error 8854 * Perform verbose error reporting if not NULL. PMDs initialize this 8855 * structure in case of error only. 8856 * 8857 * @return 8858 * 0 on success, a negative errno value otherwise and rte_errno is set. 8859 */ 8860 static int 8861 mlx5_action_handle_query(struct rte_eth_dev *dev, 8862 const struct rte_flow_action_handle *handle, 8863 void *data, 8864 struct rte_flow_error *error) 8865 { 8866 struct rte_flow_attr attr = { .transfer = 0 }; 8867 const struct mlx5_flow_driver_ops *fops = 8868 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8869 8870 return flow_drv_action_query(dev, handle, data, fops, error); 8871 } 8872 8873 /** 8874 * Destroy all indirect actions (shared RSS). 8875 * 8876 * @param dev 8877 * Pointer to Ethernet device. 8878 * 8879 * @return 8880 * 0 on success, a negative errno value otherwise and rte_errno is set. 8881 */ 8882 int 8883 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8884 { 8885 struct rte_flow_error error; 8886 struct mlx5_priv *priv = dev->data->dev_private; 8887 struct mlx5_shared_action_rss *shared_rss; 8888 int ret = 0; 8889 uint32_t idx; 8890 8891 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8892 priv->rss_shared_actions, idx, shared_rss, next) { 8893 ret |= mlx5_action_handle_destroy(dev, 8894 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8895 } 8896 return ret; 8897 } 8898 8899 /** 8900 * Validate existing indirect actions against current device configuration 8901 * and attach them to device resources. 8902 * 8903 * @param dev 8904 * Pointer to Ethernet device. 8905 * 8906 * @return 8907 * 0 on success, a negative errno value otherwise and rte_errno is set. 8908 */ 8909 int 8910 mlx5_action_handle_attach(struct rte_eth_dev *dev) 8911 { 8912 struct mlx5_priv *priv = dev->data->dev_private; 8913 struct mlx5_indexed_pool *ipool = 8914 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 8915 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 8916 int ret = 0; 8917 uint32_t idx; 8918 8919 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8920 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8921 const char *message; 8922 uint32_t queue_idx; 8923 8924 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 8925 ind_tbl->queues_n, 8926 &message, &queue_idx); 8927 if (ret != 0) { 8928 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 8929 dev->data->port_id, ind_tbl->queues[queue_idx], 8930 message); 8931 break; 8932 } 8933 } 8934 if (ret != 0) 8935 return ret; 8936 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8937 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8938 8939 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 8940 if (ret != 0) { 8941 DRV_LOG(ERR, "Port %u could not attach " 8942 "indirection table obj %p", 8943 dev->data->port_id, (void *)ind_tbl); 8944 goto error; 8945 } 8946 } 8947 return 0; 8948 error: 8949 shared_rss_last = shared_rss; 8950 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8951 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8952 8953 if (shared_rss == shared_rss_last) 8954 break; 8955 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 8956 DRV_LOG(CRIT, "Port %u could not detach " 8957 "indirection table obj %p on rollback", 8958 dev->data->port_id, (void *)ind_tbl); 8959 } 8960 return ret; 8961 } 8962 8963 /** 8964 * Detach indirect actions of the device from its resources. 8965 * 8966 * @param dev 8967 * Pointer to Ethernet device. 8968 * 8969 * @return 8970 * 0 on success, a negative errno value otherwise and rte_errno is set. 8971 */ 8972 int 8973 mlx5_action_handle_detach(struct rte_eth_dev *dev) 8974 { 8975 struct mlx5_priv *priv = dev->data->dev_private; 8976 struct mlx5_indexed_pool *ipool = 8977 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 8978 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 8979 int ret = 0; 8980 uint32_t idx; 8981 8982 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8983 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8984 8985 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 8986 if (ret != 0) { 8987 DRV_LOG(ERR, "Port %u could not detach " 8988 "indirection table obj %p", 8989 dev->data->port_id, (void *)ind_tbl); 8990 goto error; 8991 } 8992 } 8993 return 0; 8994 error: 8995 shared_rss_last = shared_rss; 8996 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8997 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8998 8999 if (shared_rss == shared_rss_last) 9000 break; 9001 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 9002 DRV_LOG(CRIT, "Port %u could not attach " 9003 "indirection table obj %p on rollback", 9004 dev->data->port_id, (void *)ind_tbl); 9005 } 9006 return ret; 9007 } 9008 9009 #ifndef HAVE_MLX5DV_DR 9010 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 9011 #else 9012 #define MLX5_DOMAIN_SYNC_FLOW \ 9013 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 9014 #endif 9015 9016 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 9017 { 9018 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 9019 const struct mlx5_flow_driver_ops *fops; 9020 int ret; 9021 struct rte_flow_attr attr = { .transfer = 0 }; 9022 9023 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9024 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9025 if (ret > 0) 9026 ret = -ret; 9027 return ret; 9028 } 9029 9030 const struct mlx5_flow_tunnel * 9031 mlx5_get_tof(const struct rte_flow_item *item, 9032 const struct rte_flow_action *action, 9033 enum mlx5_tof_rule_type *rule_type) 9034 { 9035 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9036 if (item->type == (typeof(item->type)) 9037 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9038 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9039 return flow_items_to_tunnel(item); 9040 } 9041 } 9042 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9043 if (action->type == (typeof(action->type)) 9044 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9045 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9046 return flow_actions_to_tunnel(action); 9047 } 9048 } 9049 return NULL; 9050 } 9051 9052 /** 9053 * tunnel offload functionality is defined for DV environment only 9054 */ 9055 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9056 __extension__ 9057 union tunnel_offload_mark { 9058 uint32_t val; 9059 struct { 9060 uint32_t app_reserve:8; 9061 uint32_t table_id:15; 9062 uint32_t transfer:1; 9063 uint32_t _unused_:8; 9064 }; 9065 }; 9066 9067 static bool 9068 mlx5_access_tunnel_offload_db 9069 (struct rte_eth_dev *dev, 9070 bool (*match)(struct rte_eth_dev *, 9071 struct mlx5_flow_tunnel *, const void *), 9072 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9073 void (*miss)(struct rte_eth_dev *, void *), 9074 void *ctx, bool lock_op); 9075 9076 static int 9077 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9078 struct rte_flow *flow, 9079 const struct rte_flow_attr *attr, 9080 const struct rte_flow_action *app_actions, 9081 uint32_t flow_idx, 9082 const struct mlx5_flow_tunnel *tunnel, 9083 struct tunnel_default_miss_ctx *ctx, 9084 struct rte_flow_error *error) 9085 { 9086 struct mlx5_priv *priv = dev->data->dev_private; 9087 struct mlx5_flow *dev_flow; 9088 struct rte_flow_attr miss_attr = *attr; 9089 const struct rte_flow_item miss_items[2] = { 9090 { 9091 .type = RTE_FLOW_ITEM_TYPE_ETH, 9092 .spec = NULL, 9093 .last = NULL, 9094 .mask = NULL 9095 }, 9096 { 9097 .type = RTE_FLOW_ITEM_TYPE_END, 9098 .spec = NULL, 9099 .last = NULL, 9100 .mask = NULL 9101 } 9102 }; 9103 union tunnel_offload_mark mark_id; 9104 struct rte_flow_action_mark miss_mark; 9105 struct rte_flow_action miss_actions[3] = { 9106 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9107 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9108 }; 9109 const struct rte_flow_action_jump *jump_data; 9110 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9111 struct flow_grp_info grp_info = { 9112 .external = 1, 9113 .transfer = attr->transfer, 9114 .fdb_def_rule = !!priv->fdb_def_rule, 9115 .std_tbl_fix = 0, 9116 }; 9117 int ret; 9118 9119 if (!attr->transfer) { 9120 uint32_t q_size; 9121 9122 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9123 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9124 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9125 0, SOCKET_ID_ANY); 9126 if (!ctx->queue) 9127 return rte_flow_error_set 9128 (error, ENOMEM, 9129 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9130 NULL, "invalid default miss RSS"); 9131 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9132 ctx->action_rss.level = 0, 9133 ctx->action_rss.types = priv->rss_conf.rss_hf, 9134 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9135 ctx->action_rss.queue_num = priv->reta_idx_n, 9136 ctx->action_rss.key = priv->rss_conf.rss_key, 9137 ctx->action_rss.queue = ctx->queue; 9138 if (!priv->reta_idx_n || !priv->rxqs_n) 9139 return rte_flow_error_set 9140 (error, EINVAL, 9141 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9142 NULL, "invalid port configuration"); 9143 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9144 ctx->action_rss.types = 0; 9145 for (i = 0; i != priv->reta_idx_n; ++i) 9146 ctx->queue[i] = (*priv->reta_idx)[i]; 9147 } else { 9148 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9149 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9150 } 9151 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9152 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9153 jump_data = app_actions->conf; 9154 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9155 miss_attr.group = jump_data->group; 9156 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9157 &flow_table, &grp_info, error); 9158 if (ret) 9159 return rte_flow_error_set(error, EINVAL, 9160 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9161 NULL, "invalid tunnel id"); 9162 mark_id.app_reserve = 0; 9163 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9164 mark_id.transfer = !!attr->transfer; 9165 mark_id._unused_ = 0; 9166 miss_mark.id = mark_id.val; 9167 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9168 miss_items, miss_actions, flow_idx, error); 9169 if (!dev_flow) 9170 return -rte_errno; 9171 dev_flow->flow = flow; 9172 dev_flow->external = true; 9173 dev_flow->tunnel = tunnel; 9174 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9175 /* Subflow object was created, we must include one in the list. */ 9176 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9177 dev_flow->handle, next); 9178 DRV_LOG(DEBUG, 9179 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9180 dev->data->port_id, tunnel->app_tunnel.type, 9181 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9182 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9183 miss_actions, error); 9184 if (!ret) 9185 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9186 error); 9187 9188 return ret; 9189 } 9190 9191 static const struct mlx5_flow_tbl_data_entry * 9192 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9193 { 9194 struct mlx5_priv *priv = dev->data->dev_private; 9195 struct mlx5_dev_ctx_shared *sh = priv->sh; 9196 struct mlx5_list_entry *he; 9197 union tunnel_offload_mark mbits = { .val = mark }; 9198 union mlx5_flow_tbl_key table_key = { 9199 { 9200 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9201 .id = 0, 9202 .reserved = 0, 9203 .dummy = 0, 9204 .is_fdb = !!mbits.transfer, 9205 .is_egress = 0, 9206 } 9207 }; 9208 struct mlx5_flow_cb_ctx ctx = { 9209 .data = &table_key.v64, 9210 }; 9211 9212 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9213 return he ? 9214 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9215 } 9216 9217 static void 9218 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9219 struct mlx5_list_entry *entry) 9220 { 9221 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9222 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9223 9224 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9225 tunnel_flow_tbl_to_id(tte->flow_table)); 9226 mlx5_free(tte); 9227 } 9228 9229 static int 9230 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9231 struct mlx5_list_entry *entry, void *cb_ctx) 9232 { 9233 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9234 union tunnel_tbl_key tbl = { 9235 .val = *(uint64_t *)(ctx->data), 9236 }; 9237 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9238 9239 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9240 } 9241 9242 static struct mlx5_list_entry * 9243 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 9244 { 9245 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9246 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9247 struct tunnel_tbl_entry *tte; 9248 union tunnel_tbl_key tbl = { 9249 .val = *(uint64_t *)(ctx->data), 9250 }; 9251 9252 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 9253 sizeof(*tte), 0, 9254 SOCKET_ID_ANY); 9255 if (!tte) 9256 goto err; 9257 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9258 &tte->flow_table); 9259 if (tte->flow_table >= MLX5_MAX_TABLES) { 9260 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 9261 tte->flow_table); 9262 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9263 tte->flow_table); 9264 goto err; 9265 } else if (!tte->flow_table) { 9266 goto err; 9267 } 9268 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 9269 tte->tunnel_id = tbl.tunnel_id; 9270 tte->group = tbl.group; 9271 return &tte->hash; 9272 err: 9273 if (tte) 9274 mlx5_free(tte); 9275 return NULL; 9276 } 9277 9278 static struct mlx5_list_entry * 9279 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 9280 struct mlx5_list_entry *oentry, 9281 void *cb_ctx __rte_unused) 9282 { 9283 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 9284 0, SOCKET_ID_ANY); 9285 9286 if (!tte) 9287 return NULL; 9288 memcpy(tte, oentry, sizeof(*tte)); 9289 return &tte->hash; 9290 } 9291 9292 static void 9293 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 9294 struct mlx5_list_entry *entry) 9295 { 9296 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9297 9298 mlx5_free(tte); 9299 } 9300 9301 static uint32_t 9302 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 9303 const struct mlx5_flow_tunnel *tunnel, 9304 uint32_t group, uint32_t *table, 9305 struct rte_flow_error *error) 9306 { 9307 struct mlx5_list_entry *he; 9308 struct tunnel_tbl_entry *tte; 9309 union tunnel_tbl_key key = { 9310 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 9311 .group = group 9312 }; 9313 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9314 struct mlx5_hlist *group_hash; 9315 struct mlx5_flow_cb_ctx ctx = { 9316 .data = &key.val, 9317 }; 9318 9319 group_hash = tunnel ? tunnel->groups : thub->groups; 9320 he = mlx5_hlist_register(group_hash, key.val, &ctx); 9321 if (!he) 9322 return rte_flow_error_set(error, EINVAL, 9323 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9324 NULL, 9325 "tunnel group index not supported"); 9326 tte = container_of(he, typeof(*tte), hash); 9327 *table = tte->flow_table; 9328 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 9329 dev->data->port_id, key.tunnel_id, group, *table); 9330 return 0; 9331 } 9332 9333 static void 9334 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 9335 struct mlx5_flow_tunnel *tunnel) 9336 { 9337 struct mlx5_priv *priv = dev->data->dev_private; 9338 struct mlx5_indexed_pool *ipool; 9339 9340 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 9341 dev->data->port_id, tunnel->tunnel_id); 9342 LIST_REMOVE(tunnel, chain); 9343 mlx5_hlist_destroy(tunnel->groups); 9344 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9345 mlx5_ipool_free(ipool, tunnel->tunnel_id); 9346 } 9347 9348 static bool 9349 mlx5_access_tunnel_offload_db 9350 (struct rte_eth_dev *dev, 9351 bool (*match)(struct rte_eth_dev *, 9352 struct mlx5_flow_tunnel *, const void *), 9353 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9354 void (*miss)(struct rte_eth_dev *, void *), 9355 void *ctx, bool lock_op) 9356 { 9357 bool verdict = false; 9358 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9359 struct mlx5_flow_tunnel *tunnel; 9360 9361 rte_spinlock_lock(&thub->sl); 9362 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 9363 verdict = match(dev, tunnel, (const void *)ctx); 9364 if (verdict) 9365 break; 9366 } 9367 if (!lock_op) 9368 rte_spinlock_unlock(&thub->sl); 9369 if (verdict && hit) 9370 hit(dev, tunnel, ctx); 9371 if (!verdict && miss) 9372 miss(dev, ctx); 9373 if (lock_op) 9374 rte_spinlock_unlock(&thub->sl); 9375 9376 return verdict; 9377 } 9378 9379 struct tunnel_db_find_tunnel_id_ctx { 9380 uint32_t tunnel_id; 9381 struct mlx5_flow_tunnel *tunnel; 9382 }; 9383 9384 static bool 9385 find_tunnel_id_match(struct rte_eth_dev *dev, 9386 struct mlx5_flow_tunnel *tunnel, const void *x) 9387 { 9388 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9389 9390 RTE_SET_USED(dev); 9391 return tunnel->tunnel_id == ctx->tunnel_id; 9392 } 9393 9394 static void 9395 find_tunnel_id_hit(struct rte_eth_dev *dev, 9396 struct mlx5_flow_tunnel *tunnel, void *x) 9397 { 9398 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9399 RTE_SET_USED(dev); 9400 ctx->tunnel = tunnel; 9401 } 9402 9403 static struct mlx5_flow_tunnel * 9404 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 9405 { 9406 struct tunnel_db_find_tunnel_id_ctx ctx = { 9407 .tunnel_id = id, 9408 }; 9409 9410 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 9411 find_tunnel_id_hit, NULL, &ctx, true); 9412 9413 return ctx.tunnel; 9414 } 9415 9416 static struct mlx5_flow_tunnel * 9417 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 9418 const struct rte_flow_tunnel *app_tunnel) 9419 { 9420 struct mlx5_priv *priv = dev->data->dev_private; 9421 struct mlx5_indexed_pool *ipool; 9422 struct mlx5_flow_tunnel *tunnel; 9423 uint32_t id; 9424 9425 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9426 tunnel = mlx5_ipool_zmalloc(ipool, &id); 9427 if (!tunnel) 9428 return NULL; 9429 if (id >= MLX5_MAX_TUNNELS) { 9430 mlx5_ipool_free(ipool, id); 9431 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 9432 return NULL; 9433 } 9434 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 9435 priv->sh, 9436 mlx5_flow_tunnel_grp2tbl_create_cb, 9437 mlx5_flow_tunnel_grp2tbl_match_cb, 9438 mlx5_flow_tunnel_grp2tbl_remove_cb, 9439 mlx5_flow_tunnel_grp2tbl_clone_cb, 9440 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9441 if (!tunnel->groups) { 9442 mlx5_ipool_free(ipool, id); 9443 return NULL; 9444 } 9445 /* initiate new PMD tunnel */ 9446 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 9447 tunnel->tunnel_id = id; 9448 tunnel->action.type = (typeof(tunnel->action.type)) 9449 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 9450 tunnel->action.conf = tunnel; 9451 tunnel->item.type = (typeof(tunnel->item.type)) 9452 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 9453 tunnel->item.spec = tunnel; 9454 tunnel->item.last = NULL; 9455 tunnel->item.mask = NULL; 9456 9457 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 9458 dev->data->port_id, tunnel->tunnel_id); 9459 9460 return tunnel; 9461 } 9462 9463 struct tunnel_db_get_tunnel_ctx { 9464 const struct rte_flow_tunnel *app_tunnel; 9465 struct mlx5_flow_tunnel *tunnel; 9466 }; 9467 9468 static bool get_tunnel_match(struct rte_eth_dev *dev, 9469 struct mlx5_flow_tunnel *tunnel, const void *x) 9470 { 9471 const struct tunnel_db_get_tunnel_ctx *ctx = x; 9472 9473 RTE_SET_USED(dev); 9474 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 9475 sizeof(*ctx->app_tunnel)); 9476 } 9477 9478 static void get_tunnel_hit(struct rte_eth_dev *dev, 9479 struct mlx5_flow_tunnel *tunnel, void *x) 9480 { 9481 /* called under tunnel spinlock protection */ 9482 struct tunnel_db_get_tunnel_ctx *ctx = x; 9483 9484 RTE_SET_USED(dev); 9485 tunnel->refctn++; 9486 ctx->tunnel = tunnel; 9487 } 9488 9489 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 9490 { 9491 /* called under tunnel spinlock protection */ 9492 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9493 struct tunnel_db_get_tunnel_ctx *ctx = x; 9494 9495 rte_spinlock_unlock(&thub->sl); 9496 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 9497 rte_spinlock_lock(&thub->sl); 9498 if (ctx->tunnel) { 9499 ctx->tunnel->refctn = 1; 9500 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 9501 } 9502 } 9503 9504 9505 static int 9506 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 9507 const struct rte_flow_tunnel *app_tunnel, 9508 struct mlx5_flow_tunnel **tunnel) 9509 { 9510 struct tunnel_db_get_tunnel_ctx ctx = { 9511 .app_tunnel = app_tunnel, 9512 }; 9513 9514 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 9515 get_tunnel_miss, &ctx, true); 9516 *tunnel = ctx.tunnel; 9517 return ctx.tunnel ? 0 : -ENOMEM; 9518 } 9519 9520 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 9521 { 9522 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 9523 9524 if (!thub) 9525 return; 9526 if (!LIST_EMPTY(&thub->tunnels)) 9527 DRV_LOG(WARNING, "port %u tunnels present", port_id); 9528 mlx5_hlist_destroy(thub->groups); 9529 mlx5_free(thub); 9530 } 9531 9532 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 9533 { 9534 int err; 9535 struct mlx5_flow_tunnel_hub *thub; 9536 9537 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 9538 0, SOCKET_ID_ANY); 9539 if (!thub) 9540 return -ENOMEM; 9541 LIST_INIT(&thub->tunnels); 9542 rte_spinlock_init(&thub->sl); 9543 thub->groups = mlx5_hlist_create("flow groups", 64, 9544 false, true, sh, 9545 mlx5_flow_tunnel_grp2tbl_create_cb, 9546 mlx5_flow_tunnel_grp2tbl_match_cb, 9547 mlx5_flow_tunnel_grp2tbl_remove_cb, 9548 mlx5_flow_tunnel_grp2tbl_clone_cb, 9549 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9550 if (!thub->groups) { 9551 err = -rte_errno; 9552 goto err; 9553 } 9554 sh->tunnel_hub = thub; 9555 9556 return 0; 9557 9558 err: 9559 if (thub->groups) 9560 mlx5_hlist_destroy(thub->groups); 9561 if (thub) 9562 mlx5_free(thub); 9563 return err; 9564 } 9565 9566 static inline int 9567 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 9568 struct rte_flow_tunnel *tunnel, 9569 struct rte_flow_error *error) 9570 { 9571 struct mlx5_priv *priv = dev->data->dev_private; 9572 9573 if (!priv->sh->config.dv_flow_en) 9574 return rte_flow_error_set(error, ENOTSUP, 9575 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9576 "flow DV interface is off"); 9577 if (!is_tunnel_offload_active(dev)) 9578 return rte_flow_error_set(error, ENOTSUP, 9579 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9580 "tunnel offload was not activated"); 9581 if (!tunnel) 9582 return rte_flow_error_set(error, EINVAL, 9583 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9584 "no application tunnel"); 9585 switch (tunnel->type) { 9586 default: 9587 return rte_flow_error_set(error, EINVAL, 9588 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9589 "unsupported tunnel type"); 9590 case RTE_FLOW_ITEM_TYPE_VXLAN: 9591 case RTE_FLOW_ITEM_TYPE_GRE: 9592 case RTE_FLOW_ITEM_TYPE_NVGRE: 9593 case RTE_FLOW_ITEM_TYPE_GENEVE: 9594 break; 9595 } 9596 return 0; 9597 } 9598 9599 static int 9600 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 9601 struct rte_flow_tunnel *app_tunnel, 9602 struct rte_flow_action **actions, 9603 uint32_t *num_of_actions, 9604 struct rte_flow_error *error) 9605 { 9606 struct mlx5_flow_tunnel *tunnel; 9607 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9608 9609 if (ret) 9610 return ret; 9611 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9612 if (ret < 0) { 9613 return rte_flow_error_set(error, ret, 9614 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9615 "failed to initialize pmd tunnel"); 9616 } 9617 *actions = &tunnel->action; 9618 *num_of_actions = 1; 9619 return 0; 9620 } 9621 9622 static int 9623 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9624 struct rte_flow_tunnel *app_tunnel, 9625 struct rte_flow_item **items, 9626 uint32_t *num_of_items, 9627 struct rte_flow_error *error) 9628 { 9629 struct mlx5_flow_tunnel *tunnel; 9630 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9631 9632 if (ret) 9633 return ret; 9634 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9635 if (ret < 0) { 9636 return rte_flow_error_set(error, ret, 9637 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9638 "failed to initialize pmd tunnel"); 9639 } 9640 *items = &tunnel->item; 9641 *num_of_items = 1; 9642 return 0; 9643 } 9644 9645 struct tunnel_db_element_release_ctx { 9646 struct rte_flow_item *items; 9647 struct rte_flow_action *actions; 9648 uint32_t num_elements; 9649 struct rte_flow_error *error; 9650 int ret; 9651 }; 9652 9653 static bool 9654 tunnel_element_release_match(struct rte_eth_dev *dev, 9655 struct mlx5_flow_tunnel *tunnel, const void *x) 9656 { 9657 const struct tunnel_db_element_release_ctx *ctx = x; 9658 9659 RTE_SET_USED(dev); 9660 if (ctx->num_elements != 1) 9661 return false; 9662 else if (ctx->items) 9663 return ctx->items == &tunnel->item; 9664 else if (ctx->actions) 9665 return ctx->actions == &tunnel->action; 9666 9667 return false; 9668 } 9669 9670 static void 9671 tunnel_element_release_hit(struct rte_eth_dev *dev, 9672 struct mlx5_flow_tunnel *tunnel, void *x) 9673 { 9674 struct tunnel_db_element_release_ctx *ctx = x; 9675 ctx->ret = 0; 9676 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9677 mlx5_flow_tunnel_free(dev, tunnel); 9678 } 9679 9680 static void 9681 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9682 { 9683 struct tunnel_db_element_release_ctx *ctx = x; 9684 RTE_SET_USED(dev); 9685 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9686 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9687 "invalid argument"); 9688 } 9689 9690 static int 9691 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9692 struct rte_flow_item *pmd_items, 9693 uint32_t num_items, struct rte_flow_error *err) 9694 { 9695 struct tunnel_db_element_release_ctx ctx = { 9696 .items = pmd_items, 9697 .actions = NULL, 9698 .num_elements = num_items, 9699 .error = err, 9700 }; 9701 9702 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9703 tunnel_element_release_hit, 9704 tunnel_element_release_miss, &ctx, false); 9705 9706 return ctx.ret; 9707 } 9708 9709 static int 9710 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9711 struct rte_flow_action *pmd_actions, 9712 uint32_t num_actions, struct rte_flow_error *err) 9713 { 9714 struct tunnel_db_element_release_ctx ctx = { 9715 .items = NULL, 9716 .actions = pmd_actions, 9717 .num_elements = num_actions, 9718 .error = err, 9719 }; 9720 9721 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9722 tunnel_element_release_hit, 9723 tunnel_element_release_miss, &ctx, false); 9724 9725 return ctx.ret; 9726 } 9727 9728 static int 9729 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9730 struct rte_mbuf *m, 9731 struct rte_flow_restore_info *info, 9732 struct rte_flow_error *err) 9733 { 9734 uint64_t ol_flags = m->ol_flags; 9735 const struct mlx5_flow_tbl_data_entry *tble; 9736 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 9737 9738 if (!is_tunnel_offload_active(dev)) { 9739 info->flags = 0; 9740 return 0; 9741 } 9742 9743 if ((ol_flags & mask) != mask) 9744 goto err; 9745 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9746 if (!tble) { 9747 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9748 dev->data->port_id, m->hash.fdir.hi); 9749 goto err; 9750 } 9751 MLX5_ASSERT(tble->tunnel); 9752 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9753 info->group_id = tble->group_id; 9754 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9755 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9756 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9757 9758 return 0; 9759 9760 err: 9761 return rte_flow_error_set(err, EINVAL, 9762 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9763 "failed to get restore info"); 9764 } 9765 9766 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9767 static int 9768 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9769 __rte_unused struct rte_flow_tunnel *app_tunnel, 9770 __rte_unused struct rte_flow_action **actions, 9771 __rte_unused uint32_t *num_of_actions, 9772 __rte_unused struct rte_flow_error *error) 9773 { 9774 return -ENOTSUP; 9775 } 9776 9777 static int 9778 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9779 __rte_unused struct rte_flow_tunnel *app_tunnel, 9780 __rte_unused struct rte_flow_item **items, 9781 __rte_unused uint32_t *num_of_items, 9782 __rte_unused struct rte_flow_error *error) 9783 { 9784 return -ENOTSUP; 9785 } 9786 9787 static int 9788 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9789 __rte_unused struct rte_flow_item *pmd_items, 9790 __rte_unused uint32_t num_items, 9791 __rte_unused struct rte_flow_error *err) 9792 { 9793 return -ENOTSUP; 9794 } 9795 9796 static int 9797 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9798 __rte_unused struct rte_flow_action *pmd_action, 9799 __rte_unused uint32_t num_actions, 9800 __rte_unused struct rte_flow_error *err) 9801 { 9802 return -ENOTSUP; 9803 } 9804 9805 static int 9806 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9807 __rte_unused struct rte_mbuf *m, 9808 __rte_unused struct rte_flow_restore_info *i, 9809 __rte_unused struct rte_flow_error *err) 9810 { 9811 return -ENOTSUP; 9812 } 9813 9814 static int 9815 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9816 __rte_unused struct rte_flow *flow, 9817 __rte_unused const struct rte_flow_attr *attr, 9818 __rte_unused const struct rte_flow_action *actions, 9819 __rte_unused uint32_t flow_idx, 9820 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9821 __rte_unused struct tunnel_default_miss_ctx *ctx, 9822 __rte_unused struct rte_flow_error *error) 9823 { 9824 return -ENOTSUP; 9825 } 9826 9827 static struct mlx5_flow_tunnel * 9828 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9829 __rte_unused uint32_t id) 9830 { 9831 return NULL; 9832 } 9833 9834 static void 9835 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9836 __rte_unused struct mlx5_flow_tunnel *tunnel) 9837 { 9838 } 9839 9840 static uint32_t 9841 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9842 __rte_unused const struct mlx5_flow_tunnel *t, 9843 __rte_unused uint32_t group, 9844 __rte_unused uint32_t *table, 9845 struct rte_flow_error *error) 9846 { 9847 return rte_flow_error_set(error, ENOTSUP, 9848 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9849 "tunnel offload requires DV support"); 9850 } 9851 9852 void 9853 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9854 __rte_unused uint16_t port_id) 9855 { 9856 } 9857 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9858 9859 /* Flex flow item API */ 9860 static struct rte_flow_item_flex_handle * 9861 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 9862 const struct rte_flow_item_flex_conf *conf, 9863 struct rte_flow_error *error) 9864 { 9865 static const char err_msg[] = "flex item creation unsupported"; 9866 struct rte_flow_attr attr = { .transfer = 0 }; 9867 const struct mlx5_flow_driver_ops *fops = 9868 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9869 9870 if (!fops->item_create) { 9871 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9872 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9873 NULL, err_msg); 9874 return NULL; 9875 } 9876 return fops->item_create(dev, conf, error); 9877 } 9878 9879 static int 9880 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 9881 const struct rte_flow_item_flex_handle *handle, 9882 struct rte_flow_error *error) 9883 { 9884 static const char err_msg[] = "flex item release unsupported"; 9885 struct rte_flow_attr attr = { .transfer = 0 }; 9886 const struct mlx5_flow_driver_ops *fops = 9887 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9888 9889 if (!fops->item_release) { 9890 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9891 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9892 NULL, err_msg); 9893 return -rte_errno; 9894 } 9895 return fops->item_release(dev, handle, error); 9896 } 9897 9898 static void 9899 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9900 { 9901 int ret; 9902 struct rte_flow_error error; 9903 9904 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9905 char *item_name; 9906 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9907 sizeof(item_name), 9908 (void *)(uintptr_t)item->type, &error); 9909 if (ret > 0) 9910 printf("%s ", item_name); 9911 else 9912 printf("%d\n", (int)item->type); 9913 } 9914 printf("END\n"); 9915 } 9916 9917 static int 9918 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 9919 { 9920 const struct rte_flow_item_udp *spec = udp_item->spec; 9921 const struct rte_flow_item_udp *mask = udp_item->mask; 9922 uint16_t udp_dport = 0; 9923 9924 if (spec != NULL) { 9925 if (!mask) 9926 mask = &rte_flow_item_udp_mask; 9927 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 9928 mask->hdr.dst_port); 9929 } 9930 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 9931 } 9932 9933 static const struct mlx5_flow_expand_node * 9934 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 9935 unsigned int item_idx, 9936 const struct mlx5_flow_expand_node graph[], 9937 const struct mlx5_flow_expand_node *node) 9938 { 9939 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 9940 9941 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 9942 node != NULL && 9943 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 9944 /* 9945 * The expansion node is VXLAN and it is also the last 9946 * expandable item in the pattern, so need to continue 9947 * expansion of the inner tunnel. 9948 */ 9949 MLX5_ASSERT(item_idx > 0); 9950 prev_item = pattern + item_idx - 1; 9951 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 9952 if (mlx5_flow_is_std_vxlan_port(prev_item)) 9953 return &graph[MLX5_EXPANSION_STD_VXLAN]; 9954 return &graph[MLX5_EXPANSION_L3_VXLAN]; 9955 } 9956 return node; 9957 } 9958 9959 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 9960 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 9961 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 9962 }; 9963 9964 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 9965 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 9966 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 9967 { 9, 10, 11 }, { 12, 13, 14 }, 9968 }; 9969 9970 /** 9971 * Discover the number of available flow priorities. 9972 * 9973 * @param dev 9974 * Ethernet device. 9975 * 9976 * @return 9977 * On success, number of available flow priorities. 9978 * On failure, a negative errno-style code and rte_errno is set. 9979 */ 9980 int 9981 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 9982 { 9983 static const uint16_t vprio[] = {8, 16}; 9984 const struct mlx5_priv *priv = dev->data->dev_private; 9985 const struct mlx5_flow_driver_ops *fops; 9986 enum mlx5_flow_drv_type type; 9987 int ret; 9988 9989 type = mlx5_flow_os_get_type(); 9990 if (type == MLX5_FLOW_TYPE_MAX) { 9991 type = MLX5_FLOW_TYPE_VERBS; 9992 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 9993 type = MLX5_FLOW_TYPE_DV; 9994 } 9995 fops = flow_get_drv_ops(type); 9996 if (fops->discover_priorities == NULL) { 9997 DRV_LOG(ERR, "Priority discovery not supported"); 9998 rte_errno = ENOTSUP; 9999 return -rte_errno; 10000 } 10001 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 10002 if (ret < 0) 10003 return ret; 10004 switch (ret) { 10005 case 8: 10006 ret = RTE_DIM(priority_map_3); 10007 break; 10008 case 16: 10009 ret = RTE_DIM(priority_map_5); 10010 break; 10011 default: 10012 rte_errno = ENOTSUP; 10013 DRV_LOG(ERR, 10014 "port %u maximum priority: %d expected 8/16", 10015 dev->data->port_id, ret); 10016 return -rte_errno; 10017 } 10018 DRV_LOG(INFO, "port %u supported flow priorities:" 10019 " 0-%d for ingress or egress root table," 10020 " 0-%d for non-root table or transfer root table.", 10021 dev->data->port_id, ret - 2, 10022 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 10023 return ret; 10024 } 10025 10026 /** 10027 * Adjust flow priority based on the highest layer and the request priority. 10028 * 10029 * @param[in] dev 10030 * Pointer to the Ethernet device structure. 10031 * @param[in] priority 10032 * The rule base priority. 10033 * @param[in] subpriority 10034 * The priority based on the items. 10035 * 10036 * @return 10037 * The new priority. 10038 */ 10039 uint32_t 10040 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10041 uint32_t subpriority) 10042 { 10043 uint32_t res = 0; 10044 struct mlx5_priv *priv = dev->data->dev_private; 10045 10046 switch (priv->sh->flow_max_priority) { 10047 case RTE_DIM(priority_map_3): 10048 res = priority_map_3[priority][subpriority]; 10049 break; 10050 case RTE_DIM(priority_map_5): 10051 res = priority_map_5[priority][subpriority]; 10052 break; 10053 } 10054 return res; 10055 } 10056