1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 80 #endif 81 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 82 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 83 }; 84 85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 87 (const int []){ \ 88 __VA_ARGS__, 0, \ 89 } 90 91 /** Node object of input graph for mlx5_flow_expand_rss(). */ 92 struct mlx5_flow_expand_node { 93 const int *const next; 94 /**< 95 * List of next node indexes. Index 0 is interpreted as a terminator. 96 */ 97 const enum rte_flow_item_type type; 98 /**< Pattern item type of current node. */ 99 uint64_t rss_types; 100 /**< 101 * RSS types bit-field associated with this node 102 * (see RTE_ETH_RSS_* definitions). 103 */ 104 uint64_t node_flags; 105 /**< 106 * Bit-fields that define how the node is used in the expansion. 107 * (see MLX5_EXPANSION_NODE_* definitions). 108 */ 109 }; 110 111 /* Optional expand field. The expansion alg will not go deeper. */ 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 113 114 /* The node is not added implicitly as expansion to the flow pattern. 115 * If the node type does not match the flow pattern item type, the 116 * expansion alg will go deeper to its next items. 117 * In the current implementation, the list of next nodes indexes can 118 * have up to one node with this flag set and it has to be the last 119 * node index (before the list terminator). 120 */ 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 122 123 /** Object returned by mlx5_flow_expand_rss(). */ 124 struct mlx5_flow_expand_rss { 125 uint32_t entries; 126 /**< Number of entries @p patterns and @p priorities. */ 127 struct { 128 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 129 uint32_t priority; /**< Priority offset for each expansion. */ 130 } entry[]; 131 }; 132 133 static void 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 135 136 static const struct mlx5_flow_expand_node * 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 138 unsigned int item_idx, 139 const struct mlx5_flow_expand_node graph[], 140 const struct mlx5_flow_expand_node *node); 141 142 static bool 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 144 { 145 switch (item->type) { 146 case RTE_FLOW_ITEM_TYPE_ETH: 147 case RTE_FLOW_ITEM_TYPE_VLAN: 148 case RTE_FLOW_ITEM_TYPE_IPV4: 149 case RTE_FLOW_ITEM_TYPE_IPV6: 150 case RTE_FLOW_ITEM_TYPE_UDP: 151 case RTE_FLOW_ITEM_TYPE_TCP: 152 case RTE_FLOW_ITEM_TYPE_VXLAN: 153 case RTE_FLOW_ITEM_TYPE_NVGRE: 154 case RTE_FLOW_ITEM_TYPE_GRE: 155 case RTE_FLOW_ITEM_TYPE_GENEVE: 156 case RTE_FLOW_ITEM_TYPE_MPLS: 157 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 158 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 159 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 160 case RTE_FLOW_ITEM_TYPE_GTP: 161 return true; 162 default: 163 break; 164 } 165 return false; 166 } 167 168 /** 169 * Network Service Header (NSH) and its next protocol values 170 * are described in RFC-8393. 171 */ 172 static enum rte_flow_item_type 173 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 174 { 175 enum rte_flow_item_type type; 176 177 switch (proto_mask & proto_spec) { 178 case RTE_VXLAN_GPE_TYPE_IPV4: 179 type = RTE_FLOW_ITEM_TYPE_IPV4; 180 break; 181 case RTE_VXLAN_GPE_TYPE_IPV6: 182 type = RTE_VXLAN_GPE_TYPE_IPV6; 183 break; 184 case RTE_VXLAN_GPE_TYPE_ETH: 185 type = RTE_FLOW_ITEM_TYPE_ETH; 186 break; 187 default: 188 type = RTE_FLOW_ITEM_TYPE_END; 189 } 190 return type; 191 } 192 193 static enum rte_flow_item_type 194 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 195 { 196 enum rte_flow_item_type type; 197 198 switch (proto_mask & proto_spec) { 199 case IPPROTO_UDP: 200 type = RTE_FLOW_ITEM_TYPE_UDP; 201 break; 202 case IPPROTO_TCP: 203 type = RTE_FLOW_ITEM_TYPE_TCP; 204 break; 205 case IPPROTO_IP: 206 type = RTE_FLOW_ITEM_TYPE_IPV4; 207 break; 208 case IPPROTO_IPV6: 209 type = RTE_FLOW_ITEM_TYPE_IPV6; 210 break; 211 default: 212 type = RTE_FLOW_ITEM_TYPE_END; 213 } 214 return type; 215 } 216 217 static enum rte_flow_item_type 218 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 219 rte_be16_t type_mask, bool is_tunnel) 220 { 221 enum rte_flow_item_type type; 222 223 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 224 case RTE_ETHER_TYPE_TEB: 225 type = is_tunnel ? 226 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 227 break; 228 case RTE_ETHER_TYPE_VLAN: 229 type = !is_tunnel ? 230 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 231 break; 232 case RTE_ETHER_TYPE_IPV4: 233 type = RTE_FLOW_ITEM_TYPE_IPV4; 234 break; 235 case RTE_ETHER_TYPE_IPV6: 236 type = RTE_FLOW_ITEM_TYPE_IPV6; 237 break; 238 default: 239 type = RTE_FLOW_ITEM_TYPE_END; 240 } 241 return type; 242 } 243 244 static enum rte_flow_item_type 245 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 246 { 247 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 248 do { \ 249 const void *m = item->mask; \ 250 const void *s = item->spec; \ 251 mask = m ? \ 252 ((const struct rte_flow_item_##type *)m)->fld : \ 253 rte_flow_item_##type##_mask.fld; \ 254 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 255 } while (0) 256 257 enum rte_flow_item_type ret; 258 uint16_t spec, mask; 259 260 if (item == NULL || item->spec == NULL) 261 return RTE_FLOW_ITEM_TYPE_VOID; 262 switch (item->type) { 263 case RTE_FLOW_ITEM_TYPE_ETH: 264 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 265 if (!mask) 266 return RTE_FLOW_ITEM_TYPE_VOID; 267 ret = mlx5_ethertype_to_item_type(spec, mask, false); 268 break; 269 case RTE_FLOW_ITEM_TYPE_VLAN: 270 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 271 if (!mask) 272 return RTE_FLOW_ITEM_TYPE_VOID; 273 ret = mlx5_ethertype_to_item_type(spec, mask, false); 274 break; 275 case RTE_FLOW_ITEM_TYPE_IPV4: 276 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 277 if (!mask) 278 return RTE_FLOW_ITEM_TYPE_VOID; 279 ret = mlx5_inet_proto_to_item_type(spec, mask); 280 break; 281 case RTE_FLOW_ITEM_TYPE_IPV6: 282 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 283 if (!mask) 284 return RTE_FLOW_ITEM_TYPE_VOID; 285 ret = mlx5_inet_proto_to_item_type(spec, mask); 286 break; 287 case RTE_FLOW_ITEM_TYPE_GENEVE: 288 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 289 ret = mlx5_ethertype_to_item_type(spec, mask, true); 290 break; 291 case RTE_FLOW_ITEM_TYPE_GRE: 292 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 293 ret = mlx5_ethertype_to_item_type(spec, mask, true); 294 break; 295 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 296 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 297 ret = mlx5_nsh_proto_to_item_type(spec, mask); 298 break; 299 default: 300 ret = RTE_FLOW_ITEM_TYPE_VOID; 301 break; 302 } 303 return ret; 304 #undef MLX5_XSET_ITEM_MASK_SPEC 305 } 306 307 static const int * 308 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 309 const int *next_node) 310 { 311 const struct mlx5_flow_expand_node *node = NULL; 312 const int *next = next_node; 313 314 while (next && *next) { 315 /* 316 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 317 * flag set, because they were not found in the flow pattern. 318 */ 319 node = &graph[*next]; 320 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 321 break; 322 next = node->next; 323 } 324 return next; 325 } 326 327 #define MLX5_RSS_EXP_ELT_N 16 328 329 /** 330 * Expand RSS flows into several possible flows according to the RSS hash 331 * fields requested and the driver capabilities. 332 * 333 * @param[out] buf 334 * Buffer to store the result expansion. 335 * @param[in] size 336 * Buffer size in bytes. If 0, @p buf can be NULL. 337 * @param[in] pattern 338 * User flow pattern. 339 * @param[in] types 340 * RSS types to expand (see RTE_ETH_RSS_* definitions). 341 * @param[in] graph 342 * Input graph to expand @p pattern according to @p types. 343 * @param[in] graph_root_index 344 * Index of root node in @p graph, typically 0. 345 * 346 * @return 347 * A positive value representing the size of @p buf in bytes regardless of 348 * @p size on success, a negative errno value otherwise and rte_errno is 349 * set, the following errors are defined: 350 * 351 * -E2BIG: graph-depth @p graph is too deep. 352 * -EINVAL: @p size has not enough space for expanded pattern. 353 */ 354 static int 355 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 356 const struct rte_flow_item *pattern, uint64_t types, 357 const struct mlx5_flow_expand_node graph[], 358 int graph_root_index) 359 { 360 const struct rte_flow_item *item; 361 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 362 const int *next_node; 363 const int *stack[MLX5_RSS_EXP_ELT_N]; 364 int stack_pos = 0; 365 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 366 unsigned int i, item_idx, last_expand_item_idx = 0; 367 size_t lsize; 368 size_t user_pattern_size = 0; 369 void *addr = NULL; 370 const struct mlx5_flow_expand_node *next = NULL; 371 struct rte_flow_item missed_item; 372 int missed = 0; 373 int elt = 0; 374 const struct rte_flow_item *last_expand_item = NULL; 375 376 memset(&missed_item, 0, sizeof(missed_item)); 377 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 378 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 379 if (lsize > size) 380 return -EINVAL; 381 buf->entry[0].priority = 0; 382 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 383 buf->entries = 0; 384 addr = buf->entry[0].pattern; 385 for (item = pattern, item_idx = 0; 386 item->type != RTE_FLOW_ITEM_TYPE_END; 387 item++, item_idx++) { 388 if (!mlx5_flow_is_rss_expandable_item(item)) { 389 user_pattern_size += sizeof(*item); 390 continue; 391 } 392 last_expand_item = item; 393 last_expand_item_idx = item_idx; 394 i = 0; 395 while (node->next && node->next[i]) { 396 next = &graph[node->next[i]]; 397 if (next->type == item->type) 398 break; 399 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 400 node = next; 401 i = 0; 402 } else { 403 ++i; 404 } 405 } 406 if (next) 407 node = next; 408 user_pattern_size += sizeof(*item); 409 } 410 user_pattern_size += sizeof(*item); /* Handle END item. */ 411 lsize += user_pattern_size; 412 if (lsize > size) 413 return -EINVAL; 414 /* Copy the user pattern in the first entry of the buffer. */ 415 rte_memcpy(addr, pattern, user_pattern_size); 416 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 417 buf->entries = 1; 418 /* Start expanding. */ 419 memset(flow_items, 0, sizeof(flow_items)); 420 user_pattern_size -= sizeof(*item); 421 /* 422 * Check if the last valid item has spec set, need complete pattern, 423 * and the pattern can be used for expansion. 424 */ 425 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 426 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 427 /* Item type END indicates expansion is not required. */ 428 return lsize; 429 } 430 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 431 next = NULL; 432 missed = 1; 433 i = 0; 434 while (node->next && node->next[i]) { 435 next = &graph[node->next[i]]; 436 if (next->type == missed_item.type) { 437 flow_items[0].type = missed_item.type; 438 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 439 break; 440 } 441 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 442 node = next; 443 i = 0; 444 } else { 445 ++i; 446 } 447 next = NULL; 448 } 449 } 450 if (next && missed) { 451 elt = 2; /* missed item + item end. */ 452 node = next; 453 lsize += elt * sizeof(*item) + user_pattern_size; 454 if (lsize > size) 455 return -EINVAL; 456 if (node->rss_types & types) { 457 buf->entry[buf->entries].priority = 1; 458 buf->entry[buf->entries].pattern = addr; 459 buf->entries++; 460 rte_memcpy(addr, buf->entry[0].pattern, 461 user_pattern_size); 462 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 463 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 464 addr = (void *)(((uintptr_t)addr) + 465 elt * sizeof(*item)); 466 } 467 } else if (last_expand_item != NULL) { 468 node = mlx5_flow_expand_rss_adjust_node(pattern, 469 last_expand_item_idx, graph, node); 470 } 471 memset(flow_items, 0, sizeof(flow_items)); 472 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 473 node->next); 474 stack[stack_pos] = next_node; 475 node = next_node ? &graph[*next_node] : NULL; 476 while (node) { 477 flow_items[stack_pos].type = node->type; 478 if (node->rss_types & types) { 479 size_t n; 480 /* 481 * compute the number of items to copy from the 482 * expansion and copy it. 483 * When the stack_pos is 0, there are 1 element in it, 484 * plus the addition END item. 485 */ 486 elt = stack_pos + 2; 487 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 488 lsize += elt * sizeof(*item) + user_pattern_size; 489 if (lsize > size) 490 return -EINVAL; 491 n = elt * sizeof(*item); 492 buf->entry[buf->entries].priority = 493 stack_pos + 1 + missed; 494 buf->entry[buf->entries].pattern = addr; 495 buf->entries++; 496 rte_memcpy(addr, buf->entry[0].pattern, 497 user_pattern_size); 498 addr = (void *)(((uintptr_t)addr) + 499 user_pattern_size); 500 rte_memcpy(addr, &missed_item, 501 missed * sizeof(*item)); 502 addr = (void *)(((uintptr_t)addr) + 503 missed * sizeof(*item)); 504 rte_memcpy(addr, flow_items, n); 505 addr = (void *)(((uintptr_t)addr) + n); 506 } 507 /* Go deeper. */ 508 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 509 node->next) { 510 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 511 node->next); 512 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 513 rte_errno = E2BIG; 514 return -rte_errno; 515 } 516 stack[stack_pos] = next_node; 517 } else if (*(next_node + 1)) { 518 /* Follow up with the next possibility. */ 519 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 520 ++next_node); 521 } else if (!stack_pos) { 522 /* 523 * Completing the traverse over the different paths. 524 * The next_node is advanced to the terminator. 525 */ 526 ++next_node; 527 } else { 528 /* Move to the next path. */ 529 while (stack_pos) { 530 next_node = stack[--stack_pos]; 531 next_node++; 532 if (*next_node) 533 break; 534 } 535 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 536 next_node); 537 stack[stack_pos] = next_node; 538 } 539 node = next_node && *next_node ? &graph[*next_node] : NULL; 540 }; 541 return lsize; 542 } 543 544 enum mlx5_expansion { 545 MLX5_EXPANSION_ROOT, 546 MLX5_EXPANSION_ROOT_OUTER, 547 MLX5_EXPANSION_OUTER_ETH, 548 MLX5_EXPANSION_OUTER_VLAN, 549 MLX5_EXPANSION_OUTER_IPV4, 550 MLX5_EXPANSION_OUTER_IPV4_UDP, 551 MLX5_EXPANSION_OUTER_IPV4_TCP, 552 MLX5_EXPANSION_OUTER_IPV6, 553 MLX5_EXPANSION_OUTER_IPV6_UDP, 554 MLX5_EXPANSION_OUTER_IPV6_TCP, 555 MLX5_EXPANSION_VXLAN, 556 MLX5_EXPANSION_STD_VXLAN, 557 MLX5_EXPANSION_L3_VXLAN, 558 MLX5_EXPANSION_VXLAN_GPE, 559 MLX5_EXPANSION_GRE, 560 MLX5_EXPANSION_NVGRE, 561 MLX5_EXPANSION_GRE_KEY, 562 MLX5_EXPANSION_MPLS, 563 MLX5_EXPANSION_ETH, 564 MLX5_EXPANSION_VLAN, 565 MLX5_EXPANSION_IPV4, 566 MLX5_EXPANSION_IPV4_UDP, 567 MLX5_EXPANSION_IPV4_TCP, 568 MLX5_EXPANSION_IPV6, 569 MLX5_EXPANSION_IPV6_UDP, 570 MLX5_EXPANSION_IPV6_TCP, 571 MLX5_EXPANSION_IPV6_FRAG_EXT, 572 MLX5_EXPANSION_GTP, 573 MLX5_EXPANSION_GENEVE, 574 }; 575 576 /** Supported expansion of items. */ 577 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 578 [MLX5_EXPANSION_ROOT] = { 579 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 580 MLX5_EXPANSION_IPV4, 581 MLX5_EXPANSION_IPV6), 582 .type = RTE_FLOW_ITEM_TYPE_END, 583 }, 584 [MLX5_EXPANSION_ROOT_OUTER] = { 585 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 586 MLX5_EXPANSION_OUTER_IPV4, 587 MLX5_EXPANSION_OUTER_IPV6), 588 .type = RTE_FLOW_ITEM_TYPE_END, 589 }, 590 [MLX5_EXPANSION_OUTER_ETH] = { 591 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 592 .type = RTE_FLOW_ITEM_TYPE_ETH, 593 .rss_types = 0, 594 }, 595 [MLX5_EXPANSION_OUTER_VLAN] = { 596 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 597 MLX5_EXPANSION_OUTER_IPV6), 598 .type = RTE_FLOW_ITEM_TYPE_VLAN, 599 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 600 }, 601 [MLX5_EXPANSION_OUTER_IPV4] = { 602 .next = MLX5_FLOW_EXPAND_RSS_NEXT 603 (MLX5_EXPANSION_OUTER_IPV4_UDP, 604 MLX5_EXPANSION_OUTER_IPV4_TCP, 605 MLX5_EXPANSION_GRE, 606 MLX5_EXPANSION_NVGRE, 607 MLX5_EXPANSION_IPV4, 608 MLX5_EXPANSION_IPV6), 609 .type = RTE_FLOW_ITEM_TYPE_IPV4, 610 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 611 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 612 }, 613 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 614 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 615 MLX5_EXPANSION_VXLAN_GPE, 616 MLX5_EXPANSION_MPLS, 617 MLX5_EXPANSION_GENEVE, 618 MLX5_EXPANSION_GTP), 619 .type = RTE_FLOW_ITEM_TYPE_UDP, 620 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 621 }, 622 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 623 .type = RTE_FLOW_ITEM_TYPE_TCP, 624 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 625 }, 626 [MLX5_EXPANSION_OUTER_IPV6] = { 627 .next = MLX5_FLOW_EXPAND_RSS_NEXT 628 (MLX5_EXPANSION_OUTER_IPV6_UDP, 629 MLX5_EXPANSION_OUTER_IPV6_TCP, 630 MLX5_EXPANSION_IPV4, 631 MLX5_EXPANSION_IPV6, 632 MLX5_EXPANSION_GRE, 633 MLX5_EXPANSION_NVGRE), 634 .type = RTE_FLOW_ITEM_TYPE_IPV6, 635 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 636 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 637 }, 638 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 639 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 640 MLX5_EXPANSION_VXLAN_GPE, 641 MLX5_EXPANSION_MPLS, 642 MLX5_EXPANSION_GENEVE, 643 MLX5_EXPANSION_GTP), 644 .type = RTE_FLOW_ITEM_TYPE_UDP, 645 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 646 }, 647 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 648 .type = RTE_FLOW_ITEM_TYPE_TCP, 649 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 650 }, 651 [MLX5_EXPANSION_VXLAN] = { 652 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 653 MLX5_EXPANSION_IPV4, 654 MLX5_EXPANSION_IPV6), 655 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 656 }, 657 [MLX5_EXPANSION_STD_VXLAN] = { 658 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 659 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 660 }, 661 [MLX5_EXPANSION_L3_VXLAN] = { 662 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 663 MLX5_EXPANSION_IPV6), 664 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 665 }, 666 [MLX5_EXPANSION_VXLAN_GPE] = { 667 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 668 MLX5_EXPANSION_IPV4, 669 MLX5_EXPANSION_IPV6), 670 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 671 }, 672 [MLX5_EXPANSION_GRE] = { 673 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 674 MLX5_EXPANSION_IPV4, 675 MLX5_EXPANSION_IPV6, 676 MLX5_EXPANSION_GRE_KEY, 677 MLX5_EXPANSION_MPLS), 678 .type = RTE_FLOW_ITEM_TYPE_GRE, 679 }, 680 [MLX5_EXPANSION_GRE_KEY] = { 681 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 682 MLX5_EXPANSION_IPV6, 683 MLX5_EXPANSION_MPLS), 684 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 685 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 686 }, 687 [MLX5_EXPANSION_NVGRE] = { 688 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 689 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 690 }, 691 [MLX5_EXPANSION_MPLS] = { 692 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 693 MLX5_EXPANSION_IPV6, 694 MLX5_EXPANSION_ETH), 695 .type = RTE_FLOW_ITEM_TYPE_MPLS, 696 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 697 }, 698 [MLX5_EXPANSION_ETH] = { 699 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 700 .type = RTE_FLOW_ITEM_TYPE_ETH, 701 }, 702 [MLX5_EXPANSION_VLAN] = { 703 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 704 MLX5_EXPANSION_IPV6), 705 .type = RTE_FLOW_ITEM_TYPE_VLAN, 706 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 707 }, 708 [MLX5_EXPANSION_IPV4] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 710 MLX5_EXPANSION_IPV4_TCP), 711 .type = RTE_FLOW_ITEM_TYPE_IPV4, 712 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 713 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 714 }, 715 [MLX5_EXPANSION_IPV4_UDP] = { 716 .type = RTE_FLOW_ITEM_TYPE_UDP, 717 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 718 }, 719 [MLX5_EXPANSION_IPV4_TCP] = { 720 .type = RTE_FLOW_ITEM_TYPE_TCP, 721 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 722 }, 723 [MLX5_EXPANSION_IPV6] = { 724 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 725 MLX5_EXPANSION_IPV6_TCP, 726 MLX5_EXPANSION_IPV6_FRAG_EXT), 727 .type = RTE_FLOW_ITEM_TYPE_IPV6, 728 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 729 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 730 }, 731 [MLX5_EXPANSION_IPV6_UDP] = { 732 .type = RTE_FLOW_ITEM_TYPE_UDP, 733 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 734 }, 735 [MLX5_EXPANSION_IPV6_TCP] = { 736 .type = RTE_FLOW_ITEM_TYPE_TCP, 737 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 738 }, 739 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 740 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 741 }, 742 [MLX5_EXPANSION_GTP] = { 743 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_GTP, 746 }, 747 [MLX5_EXPANSION_GENEVE] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 749 MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 752 }, 753 }; 754 755 static struct rte_flow_action_handle * 756 mlx5_action_handle_create(struct rte_eth_dev *dev, 757 const struct rte_flow_indir_action_conf *conf, 758 const struct rte_flow_action *action, 759 struct rte_flow_error *error); 760 static int mlx5_action_handle_destroy 761 (struct rte_eth_dev *dev, 762 struct rte_flow_action_handle *handle, 763 struct rte_flow_error *error); 764 static int mlx5_action_handle_update 765 (struct rte_eth_dev *dev, 766 struct rte_flow_action_handle *handle, 767 const void *update, 768 struct rte_flow_error *error); 769 static int mlx5_action_handle_query 770 (struct rte_eth_dev *dev, 771 const struct rte_flow_action_handle *handle, 772 void *data, 773 struct rte_flow_error *error); 774 static int 775 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 776 struct rte_flow_tunnel *app_tunnel, 777 struct rte_flow_action **actions, 778 uint32_t *num_of_actions, 779 struct rte_flow_error *error); 780 static int 781 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 782 struct rte_flow_tunnel *app_tunnel, 783 struct rte_flow_item **items, 784 uint32_t *num_of_items, 785 struct rte_flow_error *error); 786 static int 787 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 788 struct rte_flow_item *pmd_items, 789 uint32_t num_items, struct rte_flow_error *err); 790 static int 791 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 792 struct rte_flow_action *pmd_actions, 793 uint32_t num_actions, 794 struct rte_flow_error *err); 795 static int 796 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 797 struct rte_mbuf *m, 798 struct rte_flow_restore_info *info, 799 struct rte_flow_error *err); 800 static struct rte_flow_item_flex_handle * 801 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 802 const struct rte_flow_item_flex_conf *conf, 803 struct rte_flow_error *error); 804 static int 805 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 806 const struct rte_flow_item_flex_handle *handle, 807 struct rte_flow_error *error); 808 static int 809 mlx5_flow_info_get(struct rte_eth_dev *dev, 810 struct rte_flow_port_info *port_info, 811 struct rte_flow_queue_info *queue_info, 812 struct rte_flow_error *error); 813 static int 814 mlx5_flow_port_configure(struct rte_eth_dev *dev, 815 const struct rte_flow_port_attr *port_attr, 816 uint16_t nb_queue, 817 const struct rte_flow_queue_attr *queue_attr[], 818 struct rte_flow_error *err); 819 820 static struct rte_flow_pattern_template * 821 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 822 const struct rte_flow_pattern_template_attr *attr, 823 const struct rte_flow_item items[], 824 struct rte_flow_error *error); 825 826 static int 827 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 828 struct rte_flow_pattern_template *template, 829 struct rte_flow_error *error); 830 static struct rte_flow_actions_template * 831 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 832 const struct rte_flow_actions_template_attr *attr, 833 const struct rte_flow_action actions[], 834 const struct rte_flow_action masks[], 835 struct rte_flow_error *error); 836 static int 837 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 838 struct rte_flow_actions_template *template, 839 struct rte_flow_error *error); 840 841 static struct rte_flow_template_table * 842 mlx5_flow_table_create(struct rte_eth_dev *dev, 843 const struct rte_flow_template_table_attr *attr, 844 struct rte_flow_pattern_template *item_templates[], 845 uint8_t nb_item_templates, 846 struct rte_flow_actions_template *action_templates[], 847 uint8_t nb_action_templates, 848 struct rte_flow_error *error); 849 static int 850 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 851 struct rte_flow_template_table *table, 852 struct rte_flow_error *error); 853 static struct rte_flow * 854 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 855 uint32_t queue, 856 const struct rte_flow_op_attr *attr, 857 struct rte_flow_template_table *table, 858 const struct rte_flow_item items[], 859 uint8_t pattern_template_index, 860 const struct rte_flow_action actions[], 861 uint8_t action_template_index, 862 void *user_data, 863 struct rte_flow_error *error); 864 static int 865 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 866 uint32_t queue, 867 const struct rte_flow_op_attr *attr, 868 struct rte_flow *flow, 869 void *user_data, 870 struct rte_flow_error *error); 871 static int 872 mlx5_flow_pull(struct rte_eth_dev *dev, 873 uint32_t queue, 874 struct rte_flow_op_result res[], 875 uint16_t n_res, 876 struct rte_flow_error *error); 877 static int 878 mlx5_flow_push(struct rte_eth_dev *dev, 879 uint32_t queue, 880 struct rte_flow_error *error); 881 882 static struct rte_flow_action_handle * 883 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 884 const struct rte_flow_op_attr *attr, 885 const struct rte_flow_indir_action_conf *conf, 886 const struct rte_flow_action *action, 887 void *user_data, 888 struct rte_flow_error *error); 889 890 static int 891 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 892 const struct rte_flow_op_attr *attr, 893 struct rte_flow_action_handle *handle, 894 const void *update, 895 void *user_data, 896 struct rte_flow_error *error); 897 898 static int 899 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 900 const struct rte_flow_op_attr *attr, 901 struct rte_flow_action_handle *handle, 902 void *user_data, 903 struct rte_flow_error *error); 904 905 static const struct rte_flow_ops mlx5_flow_ops = { 906 .validate = mlx5_flow_validate, 907 .create = mlx5_flow_create, 908 .destroy = mlx5_flow_destroy, 909 .flush = mlx5_flow_flush, 910 .isolate = mlx5_flow_isolate, 911 .query = mlx5_flow_query, 912 .dev_dump = mlx5_flow_dev_dump, 913 .get_aged_flows = mlx5_flow_get_aged_flows, 914 .action_handle_create = mlx5_action_handle_create, 915 .action_handle_destroy = mlx5_action_handle_destroy, 916 .action_handle_update = mlx5_action_handle_update, 917 .action_handle_query = mlx5_action_handle_query, 918 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 919 .tunnel_match = mlx5_flow_tunnel_match, 920 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 921 .tunnel_item_release = mlx5_flow_tunnel_item_release, 922 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 923 .flex_item_create = mlx5_flow_flex_item_create, 924 .flex_item_release = mlx5_flow_flex_item_release, 925 .info_get = mlx5_flow_info_get, 926 .configure = mlx5_flow_port_configure, 927 .pattern_template_create = mlx5_flow_pattern_template_create, 928 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 929 .actions_template_create = mlx5_flow_actions_template_create, 930 .actions_template_destroy = mlx5_flow_actions_template_destroy, 931 .template_table_create = mlx5_flow_table_create, 932 .template_table_destroy = mlx5_flow_table_destroy, 933 .async_create = mlx5_flow_async_flow_create, 934 .async_destroy = mlx5_flow_async_flow_destroy, 935 .pull = mlx5_flow_pull, 936 .push = mlx5_flow_push, 937 .async_action_handle_create = mlx5_flow_async_action_handle_create, 938 .async_action_handle_update = mlx5_flow_async_action_handle_update, 939 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 940 }; 941 942 /* Tunnel information. */ 943 struct mlx5_flow_tunnel_info { 944 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 945 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 946 }; 947 948 static struct mlx5_flow_tunnel_info tunnels_info[] = { 949 { 950 .tunnel = MLX5_FLOW_LAYER_VXLAN, 951 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 952 }, 953 { 954 .tunnel = MLX5_FLOW_LAYER_GENEVE, 955 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 956 }, 957 { 958 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 959 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 960 }, 961 { 962 .tunnel = MLX5_FLOW_LAYER_GRE, 963 .ptype = RTE_PTYPE_TUNNEL_GRE, 964 }, 965 { 966 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 967 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 968 }, 969 { 970 .tunnel = MLX5_FLOW_LAYER_MPLS, 971 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 972 }, 973 { 974 .tunnel = MLX5_FLOW_LAYER_NVGRE, 975 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 976 }, 977 { 978 .tunnel = MLX5_FLOW_LAYER_IPIP, 979 .ptype = RTE_PTYPE_TUNNEL_IP, 980 }, 981 { 982 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 983 .ptype = RTE_PTYPE_TUNNEL_IP, 984 }, 985 { 986 .tunnel = MLX5_FLOW_LAYER_GTP, 987 .ptype = RTE_PTYPE_TUNNEL_GTPU, 988 }, 989 }; 990 991 992 993 /** 994 * Translate tag ID to register. 995 * 996 * @param[in] dev 997 * Pointer to the Ethernet device structure. 998 * @param[in] feature 999 * The feature that request the register. 1000 * @param[in] id 1001 * The request register ID. 1002 * @param[out] error 1003 * Error description in case of any. 1004 * 1005 * @return 1006 * The request register on success, a negative errno 1007 * value otherwise and rte_errno is set. 1008 */ 1009 int 1010 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1011 enum mlx5_feature_name feature, 1012 uint32_t id, 1013 struct rte_flow_error *error) 1014 { 1015 struct mlx5_priv *priv = dev->data->dev_private; 1016 struct mlx5_sh_config *config = &priv->sh->config; 1017 enum modify_reg start_reg; 1018 bool skip_mtr_reg = false; 1019 1020 switch (feature) { 1021 case MLX5_HAIRPIN_RX: 1022 return REG_B; 1023 case MLX5_HAIRPIN_TX: 1024 return REG_A; 1025 case MLX5_METADATA_RX: 1026 switch (config->dv_xmeta_en) { 1027 case MLX5_XMETA_MODE_LEGACY: 1028 return REG_B; 1029 case MLX5_XMETA_MODE_META16: 1030 return REG_C_0; 1031 case MLX5_XMETA_MODE_META32: 1032 return REG_C_1; 1033 } 1034 break; 1035 case MLX5_METADATA_TX: 1036 return REG_A; 1037 case MLX5_METADATA_FDB: 1038 switch (config->dv_xmeta_en) { 1039 case MLX5_XMETA_MODE_LEGACY: 1040 return REG_NON; 1041 case MLX5_XMETA_MODE_META16: 1042 return REG_C_0; 1043 case MLX5_XMETA_MODE_META32: 1044 return REG_C_1; 1045 } 1046 break; 1047 case MLX5_FLOW_MARK: 1048 switch (config->dv_xmeta_en) { 1049 case MLX5_XMETA_MODE_LEGACY: 1050 return REG_NON; 1051 case MLX5_XMETA_MODE_META16: 1052 return REG_C_1; 1053 case MLX5_XMETA_MODE_META32: 1054 return REG_C_0; 1055 } 1056 break; 1057 case MLX5_MTR_ID: 1058 /* 1059 * If meter color and meter id share one register, flow match 1060 * should use the meter color register for match. 1061 */ 1062 if (priv->mtr_reg_share) 1063 return priv->mtr_color_reg; 1064 else 1065 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1066 REG_C_3; 1067 case MLX5_MTR_COLOR: 1068 case MLX5_ASO_FLOW_HIT: 1069 case MLX5_ASO_CONNTRACK: 1070 case MLX5_SAMPLE_ID: 1071 /* All features use the same REG_C. */ 1072 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1073 return priv->mtr_color_reg; 1074 case MLX5_COPY_MARK: 1075 /* 1076 * Metadata COPY_MARK register using is in meter suffix sub 1077 * flow while with meter. It's safe to share the same register. 1078 */ 1079 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1080 case MLX5_APP_TAG: 1081 /* 1082 * If meter is enable, it will engage the register for color 1083 * match and flow match. If meter color match is not using the 1084 * REG_C_2, need to skip the REG_C_x be used by meter color 1085 * match. 1086 * If meter is disable, free to use all available registers. 1087 */ 1088 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1089 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1090 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1091 if (id > (uint32_t)(REG_C_7 - start_reg)) 1092 return rte_flow_error_set(error, EINVAL, 1093 RTE_FLOW_ERROR_TYPE_ITEM, 1094 NULL, "invalid tag id"); 1095 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1096 return rte_flow_error_set(error, ENOTSUP, 1097 RTE_FLOW_ERROR_TYPE_ITEM, 1098 NULL, "unsupported tag id"); 1099 /* 1100 * This case means meter is using the REG_C_x great than 2. 1101 * Take care not to conflict with meter color REG_C_x. 1102 * If the available index REG_C_y >= REG_C_x, skip the 1103 * color register. 1104 */ 1105 if (skip_mtr_reg && priv->sh->flow_mreg_c 1106 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1107 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1108 return rte_flow_error_set(error, EINVAL, 1109 RTE_FLOW_ERROR_TYPE_ITEM, 1110 NULL, "invalid tag id"); 1111 if (priv->sh->flow_mreg_c 1112 [id + 1 + start_reg - REG_C_0] != REG_NON) 1113 return priv->sh->flow_mreg_c 1114 [id + 1 + start_reg - REG_C_0]; 1115 return rte_flow_error_set(error, ENOTSUP, 1116 RTE_FLOW_ERROR_TYPE_ITEM, 1117 NULL, "unsupported tag id"); 1118 } 1119 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1120 } 1121 MLX5_ASSERT(false); 1122 return rte_flow_error_set(error, EINVAL, 1123 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1124 NULL, "invalid feature name"); 1125 } 1126 1127 /** 1128 * Check extensive flow metadata register support. 1129 * 1130 * @param dev 1131 * Pointer to rte_eth_dev structure. 1132 * 1133 * @return 1134 * True if device supports extensive flow metadata register, otherwise false. 1135 */ 1136 bool 1137 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1138 { 1139 struct mlx5_priv *priv = dev->data->dev_private; 1140 1141 /* 1142 * Having available reg_c can be regarded inclusively as supporting 1143 * extensive flow metadata register, which could mean, 1144 * - metadata register copy action by modify header. 1145 * - 16 modify header actions is supported. 1146 * - reg_c's are preserved across different domain (FDB and NIC) on 1147 * packet loopback by flow lookup miss. 1148 */ 1149 return priv->sh->flow_mreg_c[2] != REG_NON; 1150 } 1151 1152 /** 1153 * Get the lowest priority. 1154 * 1155 * @param[in] dev 1156 * Pointer to the Ethernet device structure. 1157 * @param[in] attributes 1158 * Pointer to device flow rule attributes. 1159 * 1160 * @return 1161 * The value of lowest priority of flow. 1162 */ 1163 uint32_t 1164 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1165 const struct rte_flow_attr *attr) 1166 { 1167 struct mlx5_priv *priv = dev->data->dev_private; 1168 1169 if (!attr->group && !attr->transfer) 1170 return priv->sh->flow_max_priority - 2; 1171 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1172 } 1173 1174 /** 1175 * Calculate matcher priority of the flow. 1176 * 1177 * @param[in] dev 1178 * Pointer to the Ethernet device structure. 1179 * @param[in] attr 1180 * Pointer to device flow rule attributes. 1181 * @param[in] subpriority 1182 * The priority based on the items. 1183 * @param[in] external 1184 * Flow is user flow. 1185 * @return 1186 * The matcher priority of the flow. 1187 */ 1188 uint16_t 1189 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1190 const struct rte_flow_attr *attr, 1191 uint32_t subpriority, bool external) 1192 { 1193 uint16_t priority = (uint16_t)attr->priority; 1194 struct mlx5_priv *priv = dev->data->dev_private; 1195 1196 if (!attr->group && !attr->transfer) { 1197 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1198 priority = priv->sh->flow_max_priority - 1; 1199 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1200 } else if (!external && attr->transfer && attr->group == 0 && 1201 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1202 return (priv->sh->flow_max_priority - 1) * 3; 1203 } 1204 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1205 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1206 return priority * 3 + subpriority; 1207 } 1208 1209 /** 1210 * Verify the @p item specifications (spec, last, mask) are compatible with the 1211 * NIC capabilities. 1212 * 1213 * @param[in] item 1214 * Item specification. 1215 * @param[in] mask 1216 * @p item->mask or flow default bit-masks. 1217 * @param[in] nic_mask 1218 * Bit-masks covering supported fields by the NIC to compare with user mask. 1219 * @param[in] size 1220 * Bit-masks size in bytes. 1221 * @param[in] range_accepted 1222 * True if range of values is accepted for specific fields, false otherwise. 1223 * @param[out] error 1224 * Pointer to error structure. 1225 * 1226 * @return 1227 * 0 on success, a negative errno value otherwise and rte_errno is set. 1228 */ 1229 int 1230 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1231 const uint8_t *mask, 1232 const uint8_t *nic_mask, 1233 unsigned int size, 1234 bool range_accepted, 1235 struct rte_flow_error *error) 1236 { 1237 unsigned int i; 1238 1239 MLX5_ASSERT(nic_mask); 1240 for (i = 0; i < size; ++i) 1241 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1242 return rte_flow_error_set(error, ENOTSUP, 1243 RTE_FLOW_ERROR_TYPE_ITEM, 1244 item, 1245 "mask enables non supported" 1246 " bits"); 1247 if (!item->spec && (item->mask || item->last)) 1248 return rte_flow_error_set(error, EINVAL, 1249 RTE_FLOW_ERROR_TYPE_ITEM, item, 1250 "mask/last without a spec is not" 1251 " supported"); 1252 if (item->spec && item->last && !range_accepted) { 1253 uint8_t spec[size]; 1254 uint8_t last[size]; 1255 unsigned int i; 1256 int ret; 1257 1258 for (i = 0; i < size; ++i) { 1259 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1260 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1261 } 1262 ret = memcmp(spec, last, size); 1263 if (ret != 0) 1264 return rte_flow_error_set(error, EINVAL, 1265 RTE_FLOW_ERROR_TYPE_ITEM, 1266 item, 1267 "range is not valid"); 1268 } 1269 return 0; 1270 } 1271 1272 /** 1273 * Adjust the hash fields according to the @p flow information. 1274 * 1275 * @param[in] dev_flow. 1276 * Pointer to the mlx5_flow. 1277 * @param[in] tunnel 1278 * 1 when the hash field is for a tunnel item. 1279 * @param[in] layer_types 1280 * RTE_ETH_RSS_* types. 1281 * @param[in] hash_fields 1282 * Item hash fields. 1283 * 1284 * @return 1285 * The hash fields that should be used. 1286 */ 1287 uint64_t 1288 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1289 int tunnel __rte_unused, uint64_t layer_types, 1290 uint64_t hash_fields) 1291 { 1292 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1293 int rss_request_inner = rss_desc->level >= 2; 1294 1295 /* Check RSS hash level for tunnel. */ 1296 if (tunnel && rss_request_inner) 1297 hash_fields |= IBV_RX_HASH_INNER; 1298 else if (tunnel || rss_request_inner) 1299 return 0; 1300 #endif 1301 /* Check if requested layer matches RSS hash fields. */ 1302 if (!(rss_desc->types & layer_types)) 1303 return 0; 1304 return hash_fields; 1305 } 1306 1307 /** 1308 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1309 * if several tunnel rules are used on this queue, the tunnel ptype will be 1310 * cleared. 1311 * 1312 * @param rxq_ctrl 1313 * Rx queue to update. 1314 */ 1315 static void 1316 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1317 { 1318 unsigned int i; 1319 uint32_t tunnel_ptype = 0; 1320 1321 /* Look up for the ptype to use. */ 1322 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1323 if (!rxq_ctrl->flow_tunnels_n[i]) 1324 continue; 1325 if (!tunnel_ptype) { 1326 tunnel_ptype = tunnels_info[i].ptype; 1327 } else { 1328 tunnel_ptype = 0; 1329 break; 1330 } 1331 } 1332 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1333 } 1334 1335 /** 1336 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1337 * flow. 1338 * 1339 * @param[in] dev 1340 * Pointer to the Ethernet device structure. 1341 * @param[in] dev_handle 1342 * Pointer to device flow handle structure. 1343 */ 1344 void 1345 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1346 struct mlx5_flow_handle *dev_handle) 1347 { 1348 struct mlx5_priv *priv = dev->data->dev_private; 1349 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1350 struct mlx5_ind_table_obj *ind_tbl = NULL; 1351 unsigned int i; 1352 1353 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1354 struct mlx5_hrxq *hrxq; 1355 1356 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1357 dev_handle->rix_hrxq); 1358 if (hrxq) 1359 ind_tbl = hrxq->ind_table; 1360 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1361 struct mlx5_shared_action_rss *shared_rss; 1362 1363 shared_rss = mlx5_ipool_get 1364 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1365 dev_handle->rix_srss); 1366 if (shared_rss) 1367 ind_tbl = shared_rss->ind_tbl; 1368 } 1369 if (!ind_tbl) 1370 return; 1371 for (i = 0; i != ind_tbl->queues_n; ++i) { 1372 int idx = ind_tbl->queues[i]; 1373 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1374 1375 MLX5_ASSERT(rxq_ctrl != NULL); 1376 if (rxq_ctrl == NULL) 1377 continue; 1378 /* 1379 * To support metadata register copy on Tx loopback, 1380 * this must be always enabled (metadata may arive 1381 * from other port - not from local flows only. 1382 */ 1383 if (tunnel) { 1384 unsigned int j; 1385 1386 /* Increase the counter matching the flow. */ 1387 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1388 if ((tunnels_info[j].tunnel & 1389 dev_handle->layers) == 1390 tunnels_info[j].tunnel) { 1391 rxq_ctrl->flow_tunnels_n[j]++; 1392 break; 1393 } 1394 } 1395 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1396 } 1397 } 1398 } 1399 1400 static void 1401 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1402 { 1403 struct mlx5_priv *priv = dev->data->dev_private; 1404 struct mlx5_rxq_ctrl *rxq_ctrl; 1405 1406 if (priv->mark_enabled) 1407 return; 1408 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1409 rxq_ctrl->rxq.mark = 1; 1410 } 1411 priv->mark_enabled = 1; 1412 } 1413 1414 /** 1415 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1416 * 1417 * @param[in] dev 1418 * Pointer to the Ethernet device structure. 1419 * @param[in] flow 1420 * Pointer to flow structure. 1421 */ 1422 static void 1423 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1424 { 1425 struct mlx5_priv *priv = dev->data->dev_private; 1426 uint32_t handle_idx; 1427 struct mlx5_flow_handle *dev_handle; 1428 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1429 1430 MLX5_ASSERT(wks); 1431 if (wks->mark) 1432 flow_rxq_mark_flag_set(dev); 1433 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1434 handle_idx, dev_handle, next) 1435 flow_drv_rxq_flags_set(dev, dev_handle); 1436 } 1437 1438 /** 1439 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1440 * device flow if no other flow uses it with the same kind of request. 1441 * 1442 * @param dev 1443 * Pointer to Ethernet device. 1444 * @param[in] dev_handle 1445 * Pointer to the device flow handle structure. 1446 */ 1447 static void 1448 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1449 struct mlx5_flow_handle *dev_handle) 1450 { 1451 struct mlx5_priv *priv = dev->data->dev_private; 1452 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1453 struct mlx5_ind_table_obj *ind_tbl = NULL; 1454 unsigned int i; 1455 1456 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1457 struct mlx5_hrxq *hrxq; 1458 1459 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1460 dev_handle->rix_hrxq); 1461 if (hrxq) 1462 ind_tbl = hrxq->ind_table; 1463 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1464 struct mlx5_shared_action_rss *shared_rss; 1465 1466 shared_rss = mlx5_ipool_get 1467 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1468 dev_handle->rix_srss); 1469 if (shared_rss) 1470 ind_tbl = shared_rss->ind_tbl; 1471 } 1472 if (!ind_tbl) 1473 return; 1474 MLX5_ASSERT(dev->data->dev_started); 1475 for (i = 0; i != ind_tbl->queues_n; ++i) { 1476 int idx = ind_tbl->queues[i]; 1477 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1478 1479 MLX5_ASSERT(rxq_ctrl != NULL); 1480 if (rxq_ctrl == NULL) 1481 continue; 1482 if (tunnel) { 1483 unsigned int j; 1484 1485 /* Decrease the counter matching the flow. */ 1486 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1487 if ((tunnels_info[j].tunnel & 1488 dev_handle->layers) == 1489 tunnels_info[j].tunnel) { 1490 rxq_ctrl->flow_tunnels_n[j]--; 1491 break; 1492 } 1493 } 1494 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1495 } 1496 } 1497 } 1498 1499 /** 1500 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1501 * @p flow if no other flow uses it with the same kind of request. 1502 * 1503 * @param dev 1504 * Pointer to Ethernet device. 1505 * @param[in] flow 1506 * Pointer to the flow. 1507 */ 1508 static void 1509 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1510 { 1511 struct mlx5_priv *priv = dev->data->dev_private; 1512 uint32_t handle_idx; 1513 struct mlx5_flow_handle *dev_handle; 1514 1515 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1516 handle_idx, dev_handle, next) 1517 flow_drv_rxq_flags_trim(dev, dev_handle); 1518 } 1519 1520 /** 1521 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1522 * 1523 * @param dev 1524 * Pointer to Ethernet device. 1525 */ 1526 static void 1527 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1528 { 1529 struct mlx5_priv *priv = dev->data->dev_private; 1530 unsigned int i; 1531 1532 for (i = 0; i != priv->rxqs_n; ++i) { 1533 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1534 unsigned int j; 1535 1536 if (rxq == NULL || rxq->ctrl == NULL) 1537 continue; 1538 rxq->ctrl->rxq.mark = 0; 1539 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1540 rxq->ctrl->flow_tunnels_n[j] = 0; 1541 rxq->ctrl->rxq.tunnel = 0; 1542 } 1543 priv->mark_enabled = 0; 1544 } 1545 1546 /** 1547 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1548 * 1549 * @param[in] dev 1550 * Pointer to the Ethernet device structure. 1551 */ 1552 void 1553 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1554 { 1555 struct mlx5_priv *priv = dev->data->dev_private; 1556 unsigned int i; 1557 1558 for (i = 0; i != priv->rxqs_n; ++i) { 1559 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1560 struct mlx5_rxq_data *data; 1561 1562 if (rxq == NULL || rxq->ctrl == NULL) 1563 continue; 1564 data = &rxq->ctrl->rxq; 1565 if (!rte_flow_dynf_metadata_avail()) { 1566 data->dynf_meta = 0; 1567 data->flow_meta_mask = 0; 1568 data->flow_meta_offset = -1; 1569 data->flow_meta_port_mask = 0; 1570 } else { 1571 data->dynf_meta = 1; 1572 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1573 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1574 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1575 } 1576 } 1577 } 1578 1579 /* 1580 * return a pointer to the desired action in the list of actions. 1581 * 1582 * @param[in] actions 1583 * The list of actions to search the action in. 1584 * @param[in] action 1585 * The action to find. 1586 * 1587 * @return 1588 * Pointer to the action in the list, if found. NULL otherwise. 1589 */ 1590 const struct rte_flow_action * 1591 mlx5_flow_find_action(const struct rte_flow_action *actions, 1592 enum rte_flow_action_type action) 1593 { 1594 if (actions == NULL) 1595 return NULL; 1596 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1597 if (actions->type == action) 1598 return actions; 1599 return NULL; 1600 } 1601 1602 /* 1603 * Validate the flag action. 1604 * 1605 * @param[in] action_flags 1606 * Bit-fields that holds the actions detected until now. 1607 * @param[in] attr 1608 * Attributes of flow that includes this action. 1609 * @param[out] error 1610 * Pointer to error structure. 1611 * 1612 * @return 1613 * 0 on success, a negative errno value otherwise and rte_errno is set. 1614 */ 1615 int 1616 mlx5_flow_validate_action_flag(uint64_t action_flags, 1617 const struct rte_flow_attr *attr, 1618 struct rte_flow_error *error) 1619 { 1620 if (action_flags & MLX5_FLOW_ACTION_MARK) 1621 return rte_flow_error_set(error, EINVAL, 1622 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1623 "can't mark and flag in same flow"); 1624 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1625 return rte_flow_error_set(error, EINVAL, 1626 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1627 "can't have 2 flag" 1628 " actions in same flow"); 1629 if (attr->egress) 1630 return rte_flow_error_set(error, ENOTSUP, 1631 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1632 "flag action not supported for " 1633 "egress"); 1634 return 0; 1635 } 1636 1637 /* 1638 * Validate the mark action. 1639 * 1640 * @param[in] action 1641 * Pointer to the queue action. 1642 * @param[in] action_flags 1643 * Bit-fields that holds the actions detected until now. 1644 * @param[in] attr 1645 * Attributes of flow that includes this action. 1646 * @param[out] error 1647 * Pointer to error structure. 1648 * 1649 * @return 1650 * 0 on success, a negative errno value otherwise and rte_errno is set. 1651 */ 1652 int 1653 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1654 uint64_t action_flags, 1655 const struct rte_flow_attr *attr, 1656 struct rte_flow_error *error) 1657 { 1658 const struct rte_flow_action_mark *mark = action->conf; 1659 1660 if (!mark) 1661 return rte_flow_error_set(error, EINVAL, 1662 RTE_FLOW_ERROR_TYPE_ACTION, 1663 action, 1664 "configuration cannot be null"); 1665 if (mark->id >= MLX5_FLOW_MARK_MAX) 1666 return rte_flow_error_set(error, EINVAL, 1667 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1668 &mark->id, 1669 "mark id must in 0 <= id < " 1670 RTE_STR(MLX5_FLOW_MARK_MAX)); 1671 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1672 return rte_flow_error_set(error, EINVAL, 1673 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1674 "can't flag and mark in same flow"); 1675 if (action_flags & MLX5_FLOW_ACTION_MARK) 1676 return rte_flow_error_set(error, EINVAL, 1677 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1678 "can't have 2 mark actions in same" 1679 " flow"); 1680 if (attr->egress) 1681 return rte_flow_error_set(error, ENOTSUP, 1682 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1683 "mark action not supported for " 1684 "egress"); 1685 return 0; 1686 } 1687 1688 /* 1689 * Validate the drop action. 1690 * 1691 * @param[in] action_flags 1692 * Bit-fields that holds the actions detected until now. 1693 * @param[in] attr 1694 * Attributes of flow that includes this action. 1695 * @param[out] error 1696 * Pointer to error structure. 1697 * 1698 * @return 1699 * 0 on success, a negative errno value otherwise and rte_errno is set. 1700 */ 1701 int 1702 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1703 const struct rte_flow_attr *attr, 1704 struct rte_flow_error *error) 1705 { 1706 if (attr->egress) 1707 return rte_flow_error_set(error, ENOTSUP, 1708 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1709 "drop action not supported for " 1710 "egress"); 1711 return 0; 1712 } 1713 1714 /* 1715 * Validate the queue action. 1716 * 1717 * @param[in] action 1718 * Pointer to the queue action. 1719 * @param[in] action_flags 1720 * Bit-fields that holds the actions detected until now. 1721 * @param[in] dev 1722 * Pointer to the Ethernet device structure. 1723 * @param[in] attr 1724 * Attributes of flow that includes this action. 1725 * @param[out] error 1726 * Pointer to error structure. 1727 * 1728 * @return 1729 * 0 on success, a negative errno value otherwise and rte_errno is set. 1730 */ 1731 int 1732 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1733 uint64_t action_flags, 1734 struct rte_eth_dev *dev, 1735 const struct rte_flow_attr *attr, 1736 struct rte_flow_error *error) 1737 { 1738 struct mlx5_priv *priv = dev->data->dev_private; 1739 const struct rte_flow_action_queue *queue = action->conf; 1740 1741 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1742 return rte_flow_error_set(error, EINVAL, 1743 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1744 "can't have 2 fate actions in" 1745 " same flow"); 1746 if (!priv->rxqs_n) 1747 return rte_flow_error_set(error, EINVAL, 1748 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1749 NULL, "No Rx queues configured"); 1750 if (queue->index >= priv->rxqs_n) 1751 return rte_flow_error_set(error, EINVAL, 1752 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1753 &queue->index, 1754 "queue index out of range"); 1755 if (mlx5_rxq_get(dev, queue->index) == NULL) 1756 return rte_flow_error_set(error, EINVAL, 1757 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1758 &queue->index, 1759 "queue is not configured"); 1760 if (attr->egress) 1761 return rte_flow_error_set(error, ENOTSUP, 1762 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1763 "queue action not supported for " 1764 "egress"); 1765 return 0; 1766 } 1767 1768 /** 1769 * Validate queue numbers for device RSS. 1770 * 1771 * @param[in] dev 1772 * Configured device. 1773 * @param[in] queues 1774 * Array of queue numbers. 1775 * @param[in] queues_n 1776 * Size of the @p queues array. 1777 * @param[out] error 1778 * On error, filled with a textual error description. 1779 * @param[out] queue 1780 * On error, filled with an offending queue index in @p queues array. 1781 * 1782 * @return 1783 * 0 on success, a negative errno code on error. 1784 */ 1785 static int 1786 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1787 const uint16_t *queues, uint32_t queues_n, 1788 const char **error, uint32_t *queue_idx) 1789 { 1790 const struct mlx5_priv *priv = dev->data->dev_private; 1791 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1792 uint32_t i; 1793 1794 for (i = 0; i != queues_n; ++i) { 1795 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, 1796 queues[i]); 1797 1798 if (queues[i] >= priv->rxqs_n) { 1799 *error = "queue index out of range"; 1800 *queue_idx = i; 1801 return -EINVAL; 1802 } 1803 if (rxq_ctrl == NULL) { 1804 *error = "queue is not configured"; 1805 *queue_idx = i; 1806 return -EINVAL; 1807 } 1808 if (i == 0) 1809 rxq_type = rxq_ctrl->type; 1810 if (rxq_type != rxq_ctrl->type) { 1811 *error = "combining hairpin and regular RSS queues is not supported"; 1812 *queue_idx = i; 1813 return -ENOTSUP; 1814 } 1815 } 1816 return 0; 1817 } 1818 1819 /* 1820 * Validate the rss action. 1821 * 1822 * @param[in] dev 1823 * Pointer to the Ethernet device structure. 1824 * @param[in] action 1825 * Pointer to the queue action. 1826 * @param[out] error 1827 * Pointer to error structure. 1828 * 1829 * @return 1830 * 0 on success, a negative errno value otherwise and rte_errno is set. 1831 */ 1832 int 1833 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1834 const struct rte_flow_action *action, 1835 struct rte_flow_error *error) 1836 { 1837 struct mlx5_priv *priv = dev->data->dev_private; 1838 const struct rte_flow_action_rss *rss = action->conf; 1839 int ret; 1840 const char *message; 1841 uint32_t queue_idx; 1842 1843 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1844 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1845 return rte_flow_error_set(error, ENOTSUP, 1846 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1847 &rss->func, 1848 "RSS hash function not supported"); 1849 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1850 if (rss->level > 2) 1851 #else 1852 if (rss->level > 1) 1853 #endif 1854 return rte_flow_error_set(error, ENOTSUP, 1855 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1856 &rss->level, 1857 "tunnel RSS is not supported"); 1858 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1859 if (rss->key_len == 0 && rss->key != NULL) 1860 return rte_flow_error_set(error, ENOTSUP, 1861 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1862 &rss->key_len, 1863 "RSS hash key length 0"); 1864 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1865 return rte_flow_error_set(error, ENOTSUP, 1866 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1867 &rss->key_len, 1868 "RSS hash key too small"); 1869 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1870 return rte_flow_error_set(error, ENOTSUP, 1871 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1872 &rss->key_len, 1873 "RSS hash key too large"); 1874 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1875 return rte_flow_error_set(error, ENOTSUP, 1876 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1877 &rss->queue_num, 1878 "number of queues too large"); 1879 if (rss->types & MLX5_RSS_HF_MASK) 1880 return rte_flow_error_set(error, ENOTSUP, 1881 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1882 &rss->types, 1883 "some RSS protocols are not" 1884 " supported"); 1885 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1886 !(rss->types & RTE_ETH_RSS_IP)) 1887 return rte_flow_error_set(error, EINVAL, 1888 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1889 "L3 partial RSS requested but L3 RSS" 1890 " type not specified"); 1891 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1892 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1893 return rte_flow_error_set(error, EINVAL, 1894 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1895 "L4 partial RSS requested but L4 RSS" 1896 " type not specified"); 1897 if (!priv->rxqs_n) 1898 return rte_flow_error_set(error, EINVAL, 1899 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1900 NULL, "No Rx queues configured"); 1901 if (!rss->queue_num) 1902 return rte_flow_error_set(error, EINVAL, 1903 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1904 NULL, "No queues configured"); 1905 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1906 &message, &queue_idx); 1907 if (ret != 0) { 1908 return rte_flow_error_set(error, -ret, 1909 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1910 &rss->queue[queue_idx], message); 1911 } 1912 return 0; 1913 } 1914 1915 /* 1916 * Validate the rss action. 1917 * 1918 * @param[in] action 1919 * Pointer to the queue action. 1920 * @param[in] action_flags 1921 * Bit-fields that holds the actions detected until now. 1922 * @param[in] dev 1923 * Pointer to the Ethernet device structure. 1924 * @param[in] attr 1925 * Attributes of flow that includes this action. 1926 * @param[in] item_flags 1927 * Items that were detected. 1928 * @param[out] error 1929 * Pointer to error structure. 1930 * 1931 * @return 1932 * 0 on success, a negative errno value otherwise and rte_errno is set. 1933 */ 1934 int 1935 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1936 uint64_t action_flags, 1937 struct rte_eth_dev *dev, 1938 const struct rte_flow_attr *attr, 1939 uint64_t item_flags, 1940 struct rte_flow_error *error) 1941 { 1942 const struct rte_flow_action_rss *rss = action->conf; 1943 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1944 int ret; 1945 1946 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1947 return rte_flow_error_set(error, EINVAL, 1948 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1949 "can't have 2 fate actions" 1950 " in same flow"); 1951 ret = mlx5_validate_action_rss(dev, action, error); 1952 if (ret) 1953 return ret; 1954 if (attr->egress) 1955 return rte_flow_error_set(error, ENOTSUP, 1956 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1957 "rss action not supported for " 1958 "egress"); 1959 if (rss->level > 1 && !tunnel) 1960 return rte_flow_error_set(error, EINVAL, 1961 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1962 "inner RSS is not supported for " 1963 "non-tunnel flows"); 1964 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1965 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1966 return rte_flow_error_set(error, EINVAL, 1967 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1968 "RSS on eCPRI is not supported now"); 1969 } 1970 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1971 !(item_flags & 1972 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1973 rss->level > 1) 1974 return rte_flow_error_set(error, EINVAL, 1975 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1976 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1977 return 0; 1978 } 1979 1980 /* 1981 * Validate the default miss action. 1982 * 1983 * @param[in] action_flags 1984 * Bit-fields that holds the actions detected until now. 1985 * @param[out] error 1986 * Pointer to error structure. 1987 * 1988 * @return 1989 * 0 on success, a negative errno value otherwise and rte_errno is set. 1990 */ 1991 int 1992 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1993 const struct rte_flow_attr *attr, 1994 struct rte_flow_error *error) 1995 { 1996 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1997 return rte_flow_error_set(error, EINVAL, 1998 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1999 "can't have 2 fate actions in" 2000 " same flow"); 2001 if (attr->egress) 2002 return rte_flow_error_set(error, ENOTSUP, 2003 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2004 "default miss action not supported " 2005 "for egress"); 2006 if (attr->group) 2007 return rte_flow_error_set(error, ENOTSUP, 2008 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2009 "only group 0 is supported"); 2010 if (attr->transfer) 2011 return rte_flow_error_set(error, ENOTSUP, 2012 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2013 NULL, "transfer is not supported"); 2014 return 0; 2015 } 2016 2017 /* 2018 * Validate the count action. 2019 * 2020 * @param[in] dev 2021 * Pointer to the Ethernet device structure. 2022 * @param[in] attr 2023 * Attributes of flow that includes this action. 2024 * @param[out] error 2025 * Pointer to error structure. 2026 * 2027 * @return 2028 * 0 on success, a negative errno value otherwise and rte_errno is set. 2029 */ 2030 int 2031 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2032 const struct rte_flow_attr *attr, 2033 struct rte_flow_error *error) 2034 { 2035 if (attr->egress) 2036 return rte_flow_error_set(error, ENOTSUP, 2037 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2038 "count action not supported for " 2039 "egress"); 2040 return 0; 2041 } 2042 2043 /* 2044 * Validate the ASO CT action. 2045 * 2046 * @param[in] dev 2047 * Pointer to the Ethernet device structure. 2048 * @param[in] conntrack 2049 * Pointer to the CT action profile. 2050 * @param[out] error 2051 * Pointer to error structure. 2052 * 2053 * @return 2054 * 0 on success, a negative errno value otherwise and rte_errno is set. 2055 */ 2056 int 2057 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2058 const struct rte_flow_action_conntrack *conntrack, 2059 struct rte_flow_error *error) 2060 { 2061 RTE_SET_USED(dev); 2062 2063 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2064 return rte_flow_error_set(error, EINVAL, 2065 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2066 "Invalid CT state"); 2067 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2068 return rte_flow_error_set(error, EINVAL, 2069 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2070 "Invalid last TCP packet flag"); 2071 return 0; 2072 } 2073 2074 /** 2075 * Verify the @p attributes will be correctly understood by the NIC and store 2076 * them in the @p flow if everything is correct. 2077 * 2078 * @param[in] dev 2079 * Pointer to the Ethernet device structure. 2080 * @param[in] attributes 2081 * Pointer to flow attributes 2082 * @param[out] error 2083 * Pointer to error structure. 2084 * 2085 * @return 2086 * 0 on success, a negative errno value otherwise and rte_errno is set. 2087 */ 2088 int 2089 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2090 const struct rte_flow_attr *attributes, 2091 struct rte_flow_error *error) 2092 { 2093 struct mlx5_priv *priv = dev->data->dev_private; 2094 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2095 2096 if (attributes->group) 2097 return rte_flow_error_set(error, ENOTSUP, 2098 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2099 NULL, "groups is not supported"); 2100 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2101 attributes->priority >= priority_max) 2102 return rte_flow_error_set(error, ENOTSUP, 2103 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2104 NULL, "priority out of range"); 2105 if (attributes->egress) 2106 return rte_flow_error_set(error, ENOTSUP, 2107 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2108 "egress is not supported"); 2109 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2110 return rte_flow_error_set(error, ENOTSUP, 2111 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2112 NULL, "transfer is not supported"); 2113 if (!attributes->ingress) 2114 return rte_flow_error_set(error, EINVAL, 2115 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2116 NULL, 2117 "ingress attribute is mandatory"); 2118 return 0; 2119 } 2120 2121 /** 2122 * Validate ICMP6 item. 2123 * 2124 * @param[in] item 2125 * Item specification. 2126 * @param[in] item_flags 2127 * Bit-fields that holds the items detected until now. 2128 * @param[in] ext_vlan_sup 2129 * Whether extended VLAN features are supported or not. 2130 * @param[out] error 2131 * Pointer to error structure. 2132 * 2133 * @return 2134 * 0 on success, a negative errno value otherwise and rte_errno is set. 2135 */ 2136 int 2137 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2138 uint64_t item_flags, 2139 uint8_t target_protocol, 2140 struct rte_flow_error *error) 2141 { 2142 const struct rte_flow_item_icmp6 *mask = item->mask; 2143 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2144 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2145 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2146 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2147 MLX5_FLOW_LAYER_OUTER_L4; 2148 int ret; 2149 2150 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2151 return rte_flow_error_set(error, EINVAL, 2152 RTE_FLOW_ERROR_TYPE_ITEM, item, 2153 "protocol filtering not compatible" 2154 " with ICMP6 layer"); 2155 if (!(item_flags & l3m)) 2156 return rte_flow_error_set(error, EINVAL, 2157 RTE_FLOW_ERROR_TYPE_ITEM, item, 2158 "IPv6 is mandatory to filter on" 2159 " ICMP6"); 2160 if (item_flags & l4m) 2161 return rte_flow_error_set(error, EINVAL, 2162 RTE_FLOW_ERROR_TYPE_ITEM, item, 2163 "multiple L4 layers not supported"); 2164 if (!mask) 2165 mask = &rte_flow_item_icmp6_mask; 2166 ret = mlx5_flow_item_acceptable 2167 (item, (const uint8_t *)mask, 2168 (const uint8_t *)&rte_flow_item_icmp6_mask, 2169 sizeof(struct rte_flow_item_icmp6), 2170 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2171 if (ret < 0) 2172 return ret; 2173 return 0; 2174 } 2175 2176 /** 2177 * Validate ICMP item. 2178 * 2179 * @param[in] item 2180 * Item specification. 2181 * @param[in] item_flags 2182 * Bit-fields that holds the items detected until now. 2183 * @param[out] error 2184 * Pointer to error structure. 2185 * 2186 * @return 2187 * 0 on success, a negative errno value otherwise and rte_errno is set. 2188 */ 2189 int 2190 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2191 uint64_t item_flags, 2192 uint8_t target_protocol, 2193 struct rte_flow_error *error) 2194 { 2195 const struct rte_flow_item_icmp *mask = item->mask; 2196 const struct rte_flow_item_icmp nic_mask = { 2197 .hdr.icmp_type = 0xff, 2198 .hdr.icmp_code = 0xff, 2199 .hdr.icmp_ident = RTE_BE16(0xffff), 2200 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2201 }; 2202 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2203 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2204 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2205 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2206 MLX5_FLOW_LAYER_OUTER_L4; 2207 int ret; 2208 2209 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2210 return rte_flow_error_set(error, EINVAL, 2211 RTE_FLOW_ERROR_TYPE_ITEM, item, 2212 "protocol filtering not compatible" 2213 " with ICMP layer"); 2214 if (!(item_flags & l3m)) 2215 return rte_flow_error_set(error, EINVAL, 2216 RTE_FLOW_ERROR_TYPE_ITEM, item, 2217 "IPv4 is mandatory to filter" 2218 " on ICMP"); 2219 if (item_flags & l4m) 2220 return rte_flow_error_set(error, EINVAL, 2221 RTE_FLOW_ERROR_TYPE_ITEM, item, 2222 "multiple L4 layers not supported"); 2223 if (!mask) 2224 mask = &nic_mask; 2225 ret = mlx5_flow_item_acceptable 2226 (item, (const uint8_t *)mask, 2227 (const uint8_t *)&nic_mask, 2228 sizeof(struct rte_flow_item_icmp), 2229 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2230 if (ret < 0) 2231 return ret; 2232 return 0; 2233 } 2234 2235 /** 2236 * Validate Ethernet item. 2237 * 2238 * @param[in] item 2239 * Item specification. 2240 * @param[in] item_flags 2241 * Bit-fields that holds the items detected until now. 2242 * @param[out] error 2243 * Pointer to error structure. 2244 * 2245 * @return 2246 * 0 on success, a negative errno value otherwise and rte_errno is set. 2247 */ 2248 int 2249 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2250 uint64_t item_flags, bool ext_vlan_sup, 2251 struct rte_flow_error *error) 2252 { 2253 const struct rte_flow_item_eth *mask = item->mask; 2254 const struct rte_flow_item_eth nic_mask = { 2255 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2256 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2257 .type = RTE_BE16(0xffff), 2258 .has_vlan = ext_vlan_sup ? 1 : 0, 2259 }; 2260 int ret; 2261 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2262 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2263 MLX5_FLOW_LAYER_OUTER_L2; 2264 2265 if (item_flags & ethm) 2266 return rte_flow_error_set(error, ENOTSUP, 2267 RTE_FLOW_ERROR_TYPE_ITEM, item, 2268 "multiple L2 layers not supported"); 2269 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2270 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2271 return rte_flow_error_set(error, EINVAL, 2272 RTE_FLOW_ERROR_TYPE_ITEM, item, 2273 "L2 layer should not follow " 2274 "L3 layers"); 2275 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2276 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2277 return rte_flow_error_set(error, EINVAL, 2278 RTE_FLOW_ERROR_TYPE_ITEM, item, 2279 "L2 layer should not follow VLAN"); 2280 if (item_flags & MLX5_FLOW_LAYER_GTP) 2281 return rte_flow_error_set(error, EINVAL, 2282 RTE_FLOW_ERROR_TYPE_ITEM, item, 2283 "L2 layer should not follow GTP"); 2284 if (!mask) 2285 mask = &rte_flow_item_eth_mask; 2286 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2287 (const uint8_t *)&nic_mask, 2288 sizeof(struct rte_flow_item_eth), 2289 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2290 return ret; 2291 } 2292 2293 /** 2294 * Validate VLAN item. 2295 * 2296 * @param[in] item 2297 * Item specification. 2298 * @param[in] item_flags 2299 * Bit-fields that holds the items detected until now. 2300 * @param[in] dev 2301 * Ethernet device flow is being created on. 2302 * @param[out] error 2303 * Pointer to error structure. 2304 * 2305 * @return 2306 * 0 on success, a negative errno value otherwise and rte_errno is set. 2307 */ 2308 int 2309 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2310 uint64_t item_flags, 2311 struct rte_eth_dev *dev, 2312 struct rte_flow_error *error) 2313 { 2314 const struct rte_flow_item_vlan *spec = item->spec; 2315 const struct rte_flow_item_vlan *mask = item->mask; 2316 const struct rte_flow_item_vlan nic_mask = { 2317 .tci = RTE_BE16(UINT16_MAX), 2318 .inner_type = RTE_BE16(UINT16_MAX), 2319 }; 2320 uint16_t vlan_tag = 0; 2321 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2322 int ret; 2323 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2324 MLX5_FLOW_LAYER_INNER_L4) : 2325 (MLX5_FLOW_LAYER_OUTER_L3 | 2326 MLX5_FLOW_LAYER_OUTER_L4); 2327 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2328 MLX5_FLOW_LAYER_OUTER_VLAN; 2329 2330 if (item_flags & vlanm) 2331 return rte_flow_error_set(error, EINVAL, 2332 RTE_FLOW_ERROR_TYPE_ITEM, item, 2333 "multiple VLAN layers not supported"); 2334 else if ((item_flags & l34m) != 0) 2335 return rte_flow_error_set(error, EINVAL, 2336 RTE_FLOW_ERROR_TYPE_ITEM, item, 2337 "VLAN cannot follow L3/L4 layer"); 2338 if (!mask) 2339 mask = &rte_flow_item_vlan_mask; 2340 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2341 (const uint8_t *)&nic_mask, 2342 sizeof(struct rte_flow_item_vlan), 2343 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2344 if (ret) 2345 return ret; 2346 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2347 struct mlx5_priv *priv = dev->data->dev_private; 2348 2349 if (priv->vmwa_context) { 2350 /* 2351 * Non-NULL context means we have a virtual machine 2352 * and SR-IOV enabled, we have to create VLAN interface 2353 * to make hypervisor to setup E-Switch vport 2354 * context correctly. We avoid creating the multiple 2355 * VLAN interfaces, so we cannot support VLAN tag mask. 2356 */ 2357 return rte_flow_error_set(error, EINVAL, 2358 RTE_FLOW_ERROR_TYPE_ITEM, 2359 item, 2360 "VLAN tag mask is not" 2361 " supported in virtual" 2362 " environment"); 2363 } 2364 } 2365 if (spec) { 2366 vlan_tag = spec->tci; 2367 vlan_tag &= mask->tci; 2368 } 2369 /* 2370 * From verbs perspective an empty VLAN is equivalent 2371 * to a packet without VLAN layer. 2372 */ 2373 if (!vlan_tag) 2374 return rte_flow_error_set(error, EINVAL, 2375 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2376 item->spec, 2377 "VLAN cannot be empty"); 2378 return 0; 2379 } 2380 2381 /** 2382 * Validate IPV4 item. 2383 * 2384 * @param[in] item 2385 * Item specification. 2386 * @param[in] item_flags 2387 * Bit-fields that holds the items detected until now. 2388 * @param[in] last_item 2389 * Previous validated item in the pattern items. 2390 * @param[in] ether_type 2391 * Type in the ethernet layer header (including dot1q). 2392 * @param[in] acc_mask 2393 * Acceptable mask, if NULL default internal default mask 2394 * will be used to check whether item fields are supported. 2395 * @param[in] range_accepted 2396 * True if range of values is accepted for specific fields, false otherwise. 2397 * @param[out] error 2398 * Pointer to error structure. 2399 * 2400 * @return 2401 * 0 on success, a negative errno value otherwise and rte_errno is set. 2402 */ 2403 int 2404 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2405 uint64_t item_flags, 2406 uint64_t last_item, 2407 uint16_t ether_type, 2408 const struct rte_flow_item_ipv4 *acc_mask, 2409 bool range_accepted, 2410 struct rte_flow_error *error) 2411 { 2412 const struct rte_flow_item_ipv4 *mask = item->mask; 2413 const struct rte_flow_item_ipv4 *spec = item->spec; 2414 const struct rte_flow_item_ipv4 nic_mask = { 2415 .hdr = { 2416 .src_addr = RTE_BE32(0xffffffff), 2417 .dst_addr = RTE_BE32(0xffffffff), 2418 .type_of_service = 0xff, 2419 .next_proto_id = 0xff, 2420 }, 2421 }; 2422 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2423 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2424 MLX5_FLOW_LAYER_OUTER_L3; 2425 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2426 MLX5_FLOW_LAYER_OUTER_L4; 2427 int ret; 2428 uint8_t next_proto = 0xFF; 2429 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2430 MLX5_FLOW_LAYER_OUTER_VLAN | 2431 MLX5_FLOW_LAYER_INNER_VLAN); 2432 2433 if ((last_item & l2_vlan) && ether_type && 2434 ether_type != RTE_ETHER_TYPE_IPV4) 2435 return rte_flow_error_set(error, EINVAL, 2436 RTE_FLOW_ERROR_TYPE_ITEM, item, 2437 "IPv4 cannot follow L2/VLAN layer " 2438 "which ether type is not IPv4"); 2439 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2440 if (mask && spec) 2441 next_proto = mask->hdr.next_proto_id & 2442 spec->hdr.next_proto_id; 2443 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2444 return rte_flow_error_set(error, EINVAL, 2445 RTE_FLOW_ERROR_TYPE_ITEM, 2446 item, 2447 "multiple tunnel " 2448 "not supported"); 2449 } 2450 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2451 return rte_flow_error_set(error, EINVAL, 2452 RTE_FLOW_ERROR_TYPE_ITEM, item, 2453 "wrong tunnel type - IPv6 specified " 2454 "but IPv4 item provided"); 2455 if (item_flags & l3m) 2456 return rte_flow_error_set(error, ENOTSUP, 2457 RTE_FLOW_ERROR_TYPE_ITEM, item, 2458 "multiple L3 layers not supported"); 2459 else if (item_flags & l4m) 2460 return rte_flow_error_set(error, EINVAL, 2461 RTE_FLOW_ERROR_TYPE_ITEM, item, 2462 "L3 cannot follow an L4 layer."); 2463 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2464 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2465 return rte_flow_error_set(error, EINVAL, 2466 RTE_FLOW_ERROR_TYPE_ITEM, item, 2467 "L3 cannot follow an NVGRE layer."); 2468 if (!mask) 2469 mask = &rte_flow_item_ipv4_mask; 2470 else if (mask->hdr.next_proto_id != 0 && 2471 mask->hdr.next_proto_id != 0xff) 2472 return rte_flow_error_set(error, EINVAL, 2473 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2474 "partial mask is not supported" 2475 " for protocol"); 2476 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2477 acc_mask ? (const uint8_t *)acc_mask 2478 : (const uint8_t *)&nic_mask, 2479 sizeof(struct rte_flow_item_ipv4), 2480 range_accepted, error); 2481 if (ret < 0) 2482 return ret; 2483 return 0; 2484 } 2485 2486 /** 2487 * Validate IPV6 item. 2488 * 2489 * @param[in] item 2490 * Item specification. 2491 * @param[in] item_flags 2492 * Bit-fields that holds the items detected until now. 2493 * @param[in] last_item 2494 * Previous validated item in the pattern items. 2495 * @param[in] ether_type 2496 * Type in the ethernet layer header (including dot1q). 2497 * @param[in] acc_mask 2498 * Acceptable mask, if NULL default internal default mask 2499 * will be used to check whether item fields are supported. 2500 * @param[out] error 2501 * Pointer to error structure. 2502 * 2503 * @return 2504 * 0 on success, a negative errno value otherwise and rte_errno is set. 2505 */ 2506 int 2507 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2508 uint64_t item_flags, 2509 uint64_t last_item, 2510 uint16_t ether_type, 2511 const struct rte_flow_item_ipv6 *acc_mask, 2512 struct rte_flow_error *error) 2513 { 2514 const struct rte_flow_item_ipv6 *mask = item->mask; 2515 const struct rte_flow_item_ipv6 *spec = item->spec; 2516 const struct rte_flow_item_ipv6 nic_mask = { 2517 .hdr = { 2518 .src_addr = 2519 "\xff\xff\xff\xff\xff\xff\xff\xff" 2520 "\xff\xff\xff\xff\xff\xff\xff\xff", 2521 .dst_addr = 2522 "\xff\xff\xff\xff\xff\xff\xff\xff" 2523 "\xff\xff\xff\xff\xff\xff\xff\xff", 2524 .vtc_flow = RTE_BE32(0xffffffff), 2525 .proto = 0xff, 2526 }, 2527 }; 2528 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2529 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2530 MLX5_FLOW_LAYER_OUTER_L3; 2531 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2532 MLX5_FLOW_LAYER_OUTER_L4; 2533 int ret; 2534 uint8_t next_proto = 0xFF; 2535 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2536 MLX5_FLOW_LAYER_OUTER_VLAN | 2537 MLX5_FLOW_LAYER_INNER_VLAN); 2538 2539 if ((last_item & l2_vlan) && ether_type && 2540 ether_type != RTE_ETHER_TYPE_IPV6) 2541 return rte_flow_error_set(error, EINVAL, 2542 RTE_FLOW_ERROR_TYPE_ITEM, item, 2543 "IPv6 cannot follow L2/VLAN layer " 2544 "which ether type is not IPv6"); 2545 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2546 next_proto = spec->hdr.proto; 2547 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2548 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2549 return rte_flow_error_set(error, EINVAL, 2550 RTE_FLOW_ERROR_TYPE_ITEM, 2551 item, 2552 "multiple tunnel " 2553 "not supported"); 2554 } 2555 if (next_proto == IPPROTO_HOPOPTS || 2556 next_proto == IPPROTO_ROUTING || 2557 next_proto == IPPROTO_FRAGMENT || 2558 next_proto == IPPROTO_ESP || 2559 next_proto == IPPROTO_AH || 2560 next_proto == IPPROTO_DSTOPTS) 2561 return rte_flow_error_set(error, EINVAL, 2562 RTE_FLOW_ERROR_TYPE_ITEM, item, 2563 "IPv6 proto (next header) should " 2564 "not be set as extension header"); 2565 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2566 return rte_flow_error_set(error, EINVAL, 2567 RTE_FLOW_ERROR_TYPE_ITEM, item, 2568 "wrong tunnel type - IPv4 specified " 2569 "but IPv6 item provided"); 2570 if (item_flags & l3m) 2571 return rte_flow_error_set(error, ENOTSUP, 2572 RTE_FLOW_ERROR_TYPE_ITEM, item, 2573 "multiple L3 layers not supported"); 2574 else if (item_flags & l4m) 2575 return rte_flow_error_set(error, EINVAL, 2576 RTE_FLOW_ERROR_TYPE_ITEM, item, 2577 "L3 cannot follow an L4 layer."); 2578 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2579 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2580 return rte_flow_error_set(error, EINVAL, 2581 RTE_FLOW_ERROR_TYPE_ITEM, item, 2582 "L3 cannot follow an NVGRE layer."); 2583 if (!mask) 2584 mask = &rte_flow_item_ipv6_mask; 2585 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2586 acc_mask ? (const uint8_t *)acc_mask 2587 : (const uint8_t *)&nic_mask, 2588 sizeof(struct rte_flow_item_ipv6), 2589 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2590 if (ret < 0) 2591 return ret; 2592 return 0; 2593 } 2594 2595 /** 2596 * Validate UDP item. 2597 * 2598 * @param[in] item 2599 * Item specification. 2600 * @param[in] item_flags 2601 * Bit-fields that holds the items detected until now. 2602 * @param[in] target_protocol 2603 * The next protocol in the previous item. 2604 * @param[in] flow_mask 2605 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2606 * @param[out] error 2607 * Pointer to error structure. 2608 * 2609 * @return 2610 * 0 on success, a negative errno value otherwise and rte_errno is set. 2611 */ 2612 int 2613 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2614 uint64_t item_flags, 2615 uint8_t target_protocol, 2616 struct rte_flow_error *error) 2617 { 2618 const struct rte_flow_item_udp *mask = item->mask; 2619 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2620 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2621 MLX5_FLOW_LAYER_OUTER_L3; 2622 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2623 MLX5_FLOW_LAYER_OUTER_L4; 2624 int ret; 2625 2626 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2627 return rte_flow_error_set(error, EINVAL, 2628 RTE_FLOW_ERROR_TYPE_ITEM, item, 2629 "protocol filtering not compatible" 2630 " with UDP layer"); 2631 if (!(item_flags & l3m)) 2632 return rte_flow_error_set(error, EINVAL, 2633 RTE_FLOW_ERROR_TYPE_ITEM, item, 2634 "L3 is mandatory to filter on L4"); 2635 if (item_flags & l4m) 2636 return rte_flow_error_set(error, EINVAL, 2637 RTE_FLOW_ERROR_TYPE_ITEM, item, 2638 "multiple L4 layers not supported"); 2639 if (!mask) 2640 mask = &rte_flow_item_udp_mask; 2641 ret = mlx5_flow_item_acceptable 2642 (item, (const uint8_t *)mask, 2643 (const uint8_t *)&rte_flow_item_udp_mask, 2644 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2645 error); 2646 if (ret < 0) 2647 return ret; 2648 return 0; 2649 } 2650 2651 /** 2652 * Validate TCP item. 2653 * 2654 * @param[in] item 2655 * Item specification. 2656 * @param[in] item_flags 2657 * Bit-fields that holds the items detected until now. 2658 * @param[in] target_protocol 2659 * The next protocol in the previous item. 2660 * @param[out] error 2661 * Pointer to error structure. 2662 * 2663 * @return 2664 * 0 on success, a negative errno value otherwise and rte_errno is set. 2665 */ 2666 int 2667 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2668 uint64_t item_flags, 2669 uint8_t target_protocol, 2670 const struct rte_flow_item_tcp *flow_mask, 2671 struct rte_flow_error *error) 2672 { 2673 const struct rte_flow_item_tcp *mask = item->mask; 2674 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2675 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2676 MLX5_FLOW_LAYER_OUTER_L3; 2677 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2678 MLX5_FLOW_LAYER_OUTER_L4; 2679 int ret; 2680 2681 MLX5_ASSERT(flow_mask); 2682 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2683 return rte_flow_error_set(error, EINVAL, 2684 RTE_FLOW_ERROR_TYPE_ITEM, item, 2685 "protocol filtering not compatible" 2686 " with TCP layer"); 2687 if (!(item_flags & l3m)) 2688 return rte_flow_error_set(error, EINVAL, 2689 RTE_FLOW_ERROR_TYPE_ITEM, item, 2690 "L3 is mandatory to filter on L4"); 2691 if (item_flags & l4m) 2692 return rte_flow_error_set(error, EINVAL, 2693 RTE_FLOW_ERROR_TYPE_ITEM, item, 2694 "multiple L4 layers not supported"); 2695 if (!mask) 2696 mask = &rte_flow_item_tcp_mask; 2697 ret = mlx5_flow_item_acceptable 2698 (item, (const uint8_t *)mask, 2699 (const uint8_t *)flow_mask, 2700 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2701 error); 2702 if (ret < 0) 2703 return ret; 2704 return 0; 2705 } 2706 2707 /** 2708 * Validate VXLAN item. 2709 * 2710 * @param[in] dev 2711 * Pointer to the Ethernet device structure. 2712 * @param[in] udp_dport 2713 * UDP destination port 2714 * @param[in] item 2715 * Item specification. 2716 * @param[in] item_flags 2717 * Bit-fields that holds the items detected until now. 2718 * @param[in] attr 2719 * Flow rule attributes. 2720 * @param[out] error 2721 * Pointer to error structure. 2722 * 2723 * @return 2724 * 0 on success, a negative errno value otherwise and rte_errno is set. 2725 */ 2726 int 2727 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2728 uint16_t udp_dport, 2729 const struct rte_flow_item *item, 2730 uint64_t item_flags, 2731 const struct rte_flow_attr *attr, 2732 struct rte_flow_error *error) 2733 { 2734 const struct rte_flow_item_vxlan *spec = item->spec; 2735 const struct rte_flow_item_vxlan *mask = item->mask; 2736 int ret; 2737 struct mlx5_priv *priv = dev->data->dev_private; 2738 union vni { 2739 uint32_t vlan_id; 2740 uint8_t vni[4]; 2741 } id = { .vlan_id = 0, }; 2742 const struct rte_flow_item_vxlan nic_mask = { 2743 .vni = "\xff\xff\xff", 2744 .rsvd1 = 0xff, 2745 }; 2746 const struct rte_flow_item_vxlan *valid_mask; 2747 2748 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2749 return rte_flow_error_set(error, ENOTSUP, 2750 RTE_FLOW_ERROR_TYPE_ITEM, item, 2751 "multiple tunnel layers not" 2752 " supported"); 2753 valid_mask = &rte_flow_item_vxlan_mask; 2754 /* 2755 * Verify only UDPv4 is present as defined in 2756 * https://tools.ietf.org/html/rfc7348 2757 */ 2758 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2759 return rte_flow_error_set(error, EINVAL, 2760 RTE_FLOW_ERROR_TYPE_ITEM, item, 2761 "no outer UDP layer found"); 2762 if (!mask) 2763 mask = &rte_flow_item_vxlan_mask; 2764 2765 if (priv->sh->steering_format_version != 2766 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2767 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2768 /* FDB domain & NIC domain non-zero group */ 2769 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2770 valid_mask = &nic_mask; 2771 /* Group zero in NIC domain */ 2772 if (!attr->group && !attr->transfer && 2773 priv->sh->tunnel_header_0_1) 2774 valid_mask = &nic_mask; 2775 } 2776 ret = mlx5_flow_item_acceptable 2777 (item, (const uint8_t *)mask, 2778 (const uint8_t *)valid_mask, 2779 sizeof(struct rte_flow_item_vxlan), 2780 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2781 if (ret < 0) 2782 return ret; 2783 if (spec) { 2784 memcpy(&id.vni[1], spec->vni, 3); 2785 memcpy(&id.vni[1], mask->vni, 3); 2786 } 2787 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2788 return rte_flow_error_set(error, ENOTSUP, 2789 RTE_FLOW_ERROR_TYPE_ITEM, item, 2790 "VXLAN tunnel must be fully defined"); 2791 return 0; 2792 } 2793 2794 /** 2795 * Validate VXLAN_GPE item. 2796 * 2797 * @param[in] item 2798 * Item specification. 2799 * @param[in] item_flags 2800 * Bit-fields that holds the items detected until now. 2801 * @param[in] priv 2802 * Pointer to the private data structure. 2803 * @param[in] target_protocol 2804 * The next protocol in the previous item. 2805 * @param[out] error 2806 * Pointer to error structure. 2807 * 2808 * @return 2809 * 0 on success, a negative errno value otherwise and rte_errno is set. 2810 */ 2811 int 2812 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2813 uint64_t item_flags, 2814 struct rte_eth_dev *dev, 2815 struct rte_flow_error *error) 2816 { 2817 struct mlx5_priv *priv = dev->data->dev_private; 2818 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2819 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2820 int ret; 2821 union vni { 2822 uint32_t vlan_id; 2823 uint8_t vni[4]; 2824 } id = { .vlan_id = 0, }; 2825 2826 if (!priv->sh->config.l3_vxlan_en) 2827 return rte_flow_error_set(error, ENOTSUP, 2828 RTE_FLOW_ERROR_TYPE_ITEM, item, 2829 "L3 VXLAN is not enabled by device" 2830 " parameter and/or not configured in" 2831 " firmware"); 2832 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2833 return rte_flow_error_set(error, ENOTSUP, 2834 RTE_FLOW_ERROR_TYPE_ITEM, item, 2835 "multiple tunnel layers not" 2836 " supported"); 2837 /* 2838 * Verify only UDPv4 is present as defined in 2839 * https://tools.ietf.org/html/rfc7348 2840 */ 2841 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2842 return rte_flow_error_set(error, EINVAL, 2843 RTE_FLOW_ERROR_TYPE_ITEM, item, 2844 "no outer UDP layer found"); 2845 if (!mask) 2846 mask = &rte_flow_item_vxlan_gpe_mask; 2847 ret = mlx5_flow_item_acceptable 2848 (item, (const uint8_t *)mask, 2849 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2850 sizeof(struct rte_flow_item_vxlan_gpe), 2851 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2852 if (ret < 0) 2853 return ret; 2854 if (spec) { 2855 if (spec->protocol) 2856 return rte_flow_error_set(error, ENOTSUP, 2857 RTE_FLOW_ERROR_TYPE_ITEM, 2858 item, 2859 "VxLAN-GPE protocol" 2860 " not supported"); 2861 memcpy(&id.vni[1], spec->vni, 3); 2862 memcpy(&id.vni[1], mask->vni, 3); 2863 } 2864 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2865 return rte_flow_error_set(error, ENOTSUP, 2866 RTE_FLOW_ERROR_TYPE_ITEM, item, 2867 "VXLAN-GPE tunnel must be fully" 2868 " defined"); 2869 return 0; 2870 } 2871 /** 2872 * Validate GRE Key item. 2873 * 2874 * @param[in] item 2875 * Item specification. 2876 * @param[in] item_flags 2877 * Bit flags to mark detected items. 2878 * @param[in] gre_item 2879 * Pointer to gre_item 2880 * @param[out] error 2881 * Pointer to error structure. 2882 * 2883 * @return 2884 * 0 on success, a negative errno value otherwise and rte_errno is set. 2885 */ 2886 int 2887 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2888 uint64_t item_flags, 2889 const struct rte_flow_item *gre_item, 2890 struct rte_flow_error *error) 2891 { 2892 const rte_be32_t *mask = item->mask; 2893 int ret = 0; 2894 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2895 const struct rte_flow_item_gre *gre_spec; 2896 const struct rte_flow_item_gre *gre_mask; 2897 2898 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2899 return rte_flow_error_set(error, ENOTSUP, 2900 RTE_FLOW_ERROR_TYPE_ITEM, item, 2901 "Multiple GRE key not support"); 2902 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2903 return rte_flow_error_set(error, ENOTSUP, 2904 RTE_FLOW_ERROR_TYPE_ITEM, item, 2905 "No preceding GRE header"); 2906 if (item_flags & MLX5_FLOW_LAYER_INNER) 2907 return rte_flow_error_set(error, ENOTSUP, 2908 RTE_FLOW_ERROR_TYPE_ITEM, item, 2909 "GRE key following a wrong item"); 2910 gre_mask = gre_item->mask; 2911 if (!gre_mask) 2912 gre_mask = &rte_flow_item_gre_mask; 2913 gre_spec = gre_item->spec; 2914 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2915 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2916 return rte_flow_error_set(error, EINVAL, 2917 RTE_FLOW_ERROR_TYPE_ITEM, item, 2918 "Key bit must be on"); 2919 2920 if (!mask) 2921 mask = &gre_key_default_mask; 2922 ret = mlx5_flow_item_acceptable 2923 (item, (const uint8_t *)mask, 2924 (const uint8_t *)&gre_key_default_mask, 2925 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2926 return ret; 2927 } 2928 2929 /** 2930 * Validate GRE item. 2931 * 2932 * @param[in] item 2933 * Item specification. 2934 * @param[in] item_flags 2935 * Bit flags to mark detected items. 2936 * @param[in] target_protocol 2937 * The next protocol in the previous item. 2938 * @param[out] error 2939 * Pointer to error structure. 2940 * 2941 * @return 2942 * 0 on success, a negative errno value otherwise and rte_errno is set. 2943 */ 2944 int 2945 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2946 uint64_t item_flags, 2947 uint8_t target_protocol, 2948 struct rte_flow_error *error) 2949 { 2950 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2951 const struct rte_flow_item_gre *mask = item->mask; 2952 int ret; 2953 const struct rte_flow_item_gre nic_mask = { 2954 .c_rsvd0_ver = RTE_BE16(0xB000), 2955 .protocol = RTE_BE16(UINT16_MAX), 2956 }; 2957 2958 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2959 return rte_flow_error_set(error, EINVAL, 2960 RTE_FLOW_ERROR_TYPE_ITEM, item, 2961 "protocol filtering not compatible" 2962 " with this GRE layer"); 2963 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2964 return rte_flow_error_set(error, ENOTSUP, 2965 RTE_FLOW_ERROR_TYPE_ITEM, item, 2966 "multiple tunnel layers not" 2967 " supported"); 2968 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2969 return rte_flow_error_set(error, ENOTSUP, 2970 RTE_FLOW_ERROR_TYPE_ITEM, item, 2971 "L3 Layer is missing"); 2972 if (!mask) 2973 mask = &rte_flow_item_gre_mask; 2974 ret = mlx5_flow_item_acceptable 2975 (item, (const uint8_t *)mask, 2976 (const uint8_t *)&nic_mask, 2977 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2978 error); 2979 if (ret < 0) 2980 return ret; 2981 #ifndef HAVE_MLX5DV_DR 2982 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2983 if (spec && (spec->protocol & mask->protocol)) 2984 return rte_flow_error_set(error, ENOTSUP, 2985 RTE_FLOW_ERROR_TYPE_ITEM, item, 2986 "without MPLS support the" 2987 " specification cannot be used for" 2988 " filtering"); 2989 #endif 2990 #endif 2991 return 0; 2992 } 2993 2994 /** 2995 * Validate Geneve item. 2996 * 2997 * @param[in] item 2998 * Item specification. 2999 * @param[in] itemFlags 3000 * Bit-fields that holds the items detected until now. 3001 * @param[in] enPriv 3002 * Pointer to the private data structure. 3003 * @param[out] error 3004 * Pointer to error structure. 3005 * 3006 * @return 3007 * 0 on success, a negative errno value otherwise and rte_errno is set. 3008 */ 3009 3010 int 3011 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3012 uint64_t item_flags, 3013 struct rte_eth_dev *dev, 3014 struct rte_flow_error *error) 3015 { 3016 struct mlx5_priv *priv = dev->data->dev_private; 3017 const struct rte_flow_item_geneve *spec = item->spec; 3018 const struct rte_flow_item_geneve *mask = item->mask; 3019 int ret; 3020 uint16_t gbhdr; 3021 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3022 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3023 const struct rte_flow_item_geneve nic_mask = { 3024 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3025 .vni = "\xff\xff\xff", 3026 .protocol = RTE_BE16(UINT16_MAX), 3027 }; 3028 3029 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3030 return rte_flow_error_set(error, ENOTSUP, 3031 RTE_FLOW_ERROR_TYPE_ITEM, item, 3032 "L3 Geneve is not enabled by device" 3033 " parameter and/or not configured in" 3034 " firmware"); 3035 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3036 return rte_flow_error_set(error, ENOTSUP, 3037 RTE_FLOW_ERROR_TYPE_ITEM, item, 3038 "multiple tunnel layers not" 3039 " supported"); 3040 /* 3041 * Verify only UDPv4 is present as defined in 3042 * https://tools.ietf.org/html/rfc7348 3043 */ 3044 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3045 return rte_flow_error_set(error, EINVAL, 3046 RTE_FLOW_ERROR_TYPE_ITEM, item, 3047 "no outer UDP layer found"); 3048 if (!mask) 3049 mask = &rte_flow_item_geneve_mask; 3050 ret = mlx5_flow_item_acceptable 3051 (item, (const uint8_t *)mask, 3052 (const uint8_t *)&nic_mask, 3053 sizeof(struct rte_flow_item_geneve), 3054 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3055 if (ret) 3056 return ret; 3057 if (spec) { 3058 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3059 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3060 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3061 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3062 return rte_flow_error_set(error, ENOTSUP, 3063 RTE_FLOW_ERROR_TYPE_ITEM, 3064 item, 3065 "Geneve protocol unsupported" 3066 " fields are being used"); 3067 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3068 return rte_flow_error_set 3069 (error, ENOTSUP, 3070 RTE_FLOW_ERROR_TYPE_ITEM, 3071 item, 3072 "Unsupported Geneve options length"); 3073 } 3074 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3075 return rte_flow_error_set 3076 (error, ENOTSUP, 3077 RTE_FLOW_ERROR_TYPE_ITEM, item, 3078 "Geneve tunnel must be fully defined"); 3079 return 0; 3080 } 3081 3082 /** 3083 * Validate Geneve TLV option item. 3084 * 3085 * @param[in] item 3086 * Item specification. 3087 * @param[in] last_item 3088 * Previous validated item in the pattern items. 3089 * @param[in] geneve_item 3090 * Previous GENEVE item specification. 3091 * @param[in] dev 3092 * Pointer to the rte_eth_dev structure. 3093 * @param[out] error 3094 * Pointer to error structure. 3095 * 3096 * @return 3097 * 0 on success, a negative errno value otherwise and rte_errno is set. 3098 */ 3099 int 3100 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3101 uint64_t last_item, 3102 const struct rte_flow_item *geneve_item, 3103 struct rte_eth_dev *dev, 3104 struct rte_flow_error *error) 3105 { 3106 struct mlx5_priv *priv = dev->data->dev_private; 3107 struct mlx5_dev_ctx_shared *sh = priv->sh; 3108 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3109 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3110 uint8_t data_max_supported = 3111 hca_attr->max_geneve_tlv_option_data_len * 4; 3112 const struct rte_flow_item_geneve *geneve_spec; 3113 const struct rte_flow_item_geneve *geneve_mask; 3114 const struct rte_flow_item_geneve_opt *spec = item->spec; 3115 const struct rte_flow_item_geneve_opt *mask = item->mask; 3116 unsigned int i; 3117 unsigned int data_len; 3118 uint8_t tlv_option_len; 3119 uint16_t optlen_m, optlen_v; 3120 const struct rte_flow_item_geneve_opt full_mask = { 3121 .option_class = RTE_BE16(0xffff), 3122 .option_type = 0xff, 3123 .option_len = 0x1f, 3124 }; 3125 3126 if (!mask) 3127 mask = &rte_flow_item_geneve_opt_mask; 3128 if (!spec) 3129 return rte_flow_error_set 3130 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3131 "Geneve TLV opt class/type/length must be specified"); 3132 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3133 return rte_flow_error_set 3134 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3135 "Geneve TLV opt length exceeds the limit (31)"); 3136 /* Check if class type and length masks are full. */ 3137 if (full_mask.option_class != mask->option_class || 3138 full_mask.option_type != mask->option_type || 3139 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3140 return rte_flow_error_set 3141 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3142 "Geneve TLV opt class/type/length masks must be full"); 3143 /* Check if length is supported */ 3144 if ((uint32_t)spec->option_len > 3145 hca_attr->max_geneve_tlv_option_data_len) 3146 return rte_flow_error_set 3147 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3148 "Geneve TLV opt length not supported"); 3149 if (hca_attr->max_geneve_tlv_options > 1) 3150 DRV_LOG(DEBUG, 3151 "max_geneve_tlv_options supports more than 1 option"); 3152 /* Check GENEVE item preceding. */ 3153 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3154 return rte_flow_error_set 3155 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3156 "Geneve opt item must be preceded with Geneve item"); 3157 geneve_spec = geneve_item->spec; 3158 geneve_mask = geneve_item->mask ? geneve_item->mask : 3159 &rte_flow_item_geneve_mask; 3160 /* Check if GENEVE TLV option size doesn't exceed option length */ 3161 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3162 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3163 tlv_option_len = spec->option_len & mask->option_len; 3164 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3165 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3166 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3167 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3168 if ((optlen_v & optlen_m) <= tlv_option_len) 3169 return rte_flow_error_set 3170 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3171 "GENEVE TLV option length exceeds optlen"); 3172 } 3173 /* Check if length is 0 or data is 0. */ 3174 if (spec->data == NULL || spec->option_len == 0) 3175 return rte_flow_error_set 3176 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3177 "Geneve TLV opt with zero data/length not supported"); 3178 /* Check not all data & mask are 0. */ 3179 data_len = spec->option_len * 4; 3180 if (mask->data == NULL) { 3181 for (i = 0; i < data_len; i++) 3182 if (spec->data[i]) 3183 break; 3184 if (i == data_len) 3185 return rte_flow_error_set(error, ENOTSUP, 3186 RTE_FLOW_ERROR_TYPE_ITEM, item, 3187 "Can't match on Geneve option data 0"); 3188 } else { 3189 for (i = 0; i < data_len; i++) 3190 if (spec->data[i] & mask->data[i]) 3191 break; 3192 if (i == data_len) 3193 return rte_flow_error_set(error, ENOTSUP, 3194 RTE_FLOW_ERROR_TYPE_ITEM, item, 3195 "Can't match on Geneve option data and mask 0"); 3196 /* Check data mask supported. */ 3197 for (i = data_max_supported; i < data_len ; i++) 3198 if (mask->data[i]) 3199 return rte_flow_error_set(error, ENOTSUP, 3200 RTE_FLOW_ERROR_TYPE_ITEM, item, 3201 "Data mask is of unsupported size"); 3202 } 3203 /* Check GENEVE option is supported in NIC. */ 3204 if (!hca_attr->geneve_tlv_opt) 3205 return rte_flow_error_set 3206 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3207 "Geneve TLV opt not supported"); 3208 /* Check if we already have geneve option with different type/class. */ 3209 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3210 geneve_opt_resource = sh->geneve_tlv_option_resource; 3211 if (geneve_opt_resource != NULL) 3212 if (geneve_opt_resource->option_class != spec->option_class || 3213 geneve_opt_resource->option_type != spec->option_type || 3214 geneve_opt_resource->length != spec->option_len) { 3215 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3216 return rte_flow_error_set(error, ENOTSUP, 3217 RTE_FLOW_ERROR_TYPE_ITEM, item, 3218 "Only one Geneve TLV option supported"); 3219 } 3220 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3221 return 0; 3222 } 3223 3224 /** 3225 * Validate MPLS item. 3226 * 3227 * @param[in] dev 3228 * Pointer to the rte_eth_dev structure. 3229 * @param[in] item 3230 * Item specification. 3231 * @param[in] item_flags 3232 * Bit-fields that holds the items detected until now. 3233 * @param[in] prev_layer 3234 * The protocol layer indicated in previous item. 3235 * @param[out] error 3236 * Pointer to error structure. 3237 * 3238 * @return 3239 * 0 on success, a negative errno value otherwise and rte_errno is set. 3240 */ 3241 int 3242 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3243 const struct rte_flow_item *item __rte_unused, 3244 uint64_t item_flags __rte_unused, 3245 uint64_t prev_layer __rte_unused, 3246 struct rte_flow_error *error) 3247 { 3248 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3249 const struct rte_flow_item_mpls *mask = item->mask; 3250 struct mlx5_priv *priv = dev->data->dev_private; 3251 int ret; 3252 3253 if (!priv->sh->dev_cap.mpls_en) 3254 return rte_flow_error_set(error, ENOTSUP, 3255 RTE_FLOW_ERROR_TYPE_ITEM, item, 3256 "MPLS not supported or" 3257 " disabled in firmware" 3258 " configuration."); 3259 /* MPLS over UDP, GRE is allowed */ 3260 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3261 MLX5_FLOW_LAYER_GRE | 3262 MLX5_FLOW_LAYER_GRE_KEY))) 3263 return rte_flow_error_set(error, EINVAL, 3264 RTE_FLOW_ERROR_TYPE_ITEM, item, 3265 "protocol filtering not compatible" 3266 " with MPLS layer"); 3267 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3268 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3269 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3270 return rte_flow_error_set(error, ENOTSUP, 3271 RTE_FLOW_ERROR_TYPE_ITEM, item, 3272 "multiple tunnel layers not" 3273 " supported"); 3274 if (!mask) 3275 mask = &rte_flow_item_mpls_mask; 3276 ret = mlx5_flow_item_acceptable 3277 (item, (const uint8_t *)mask, 3278 (const uint8_t *)&rte_flow_item_mpls_mask, 3279 sizeof(struct rte_flow_item_mpls), 3280 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3281 if (ret < 0) 3282 return ret; 3283 return 0; 3284 #else 3285 return rte_flow_error_set(error, ENOTSUP, 3286 RTE_FLOW_ERROR_TYPE_ITEM, item, 3287 "MPLS is not supported by Verbs, please" 3288 " update."); 3289 #endif 3290 } 3291 3292 /** 3293 * Validate NVGRE item. 3294 * 3295 * @param[in] item 3296 * Item specification. 3297 * @param[in] item_flags 3298 * Bit flags to mark detected items. 3299 * @param[in] target_protocol 3300 * The next protocol in the previous item. 3301 * @param[out] error 3302 * Pointer to error structure. 3303 * 3304 * @return 3305 * 0 on success, a negative errno value otherwise and rte_errno is set. 3306 */ 3307 int 3308 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3309 uint64_t item_flags, 3310 uint8_t target_protocol, 3311 struct rte_flow_error *error) 3312 { 3313 const struct rte_flow_item_nvgre *mask = item->mask; 3314 int ret; 3315 3316 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3317 return rte_flow_error_set(error, EINVAL, 3318 RTE_FLOW_ERROR_TYPE_ITEM, item, 3319 "protocol filtering not compatible" 3320 " with this GRE layer"); 3321 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3322 return rte_flow_error_set(error, ENOTSUP, 3323 RTE_FLOW_ERROR_TYPE_ITEM, item, 3324 "multiple tunnel layers not" 3325 " supported"); 3326 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3327 return rte_flow_error_set(error, ENOTSUP, 3328 RTE_FLOW_ERROR_TYPE_ITEM, item, 3329 "L3 Layer is missing"); 3330 if (!mask) 3331 mask = &rte_flow_item_nvgre_mask; 3332 ret = mlx5_flow_item_acceptable 3333 (item, (const uint8_t *)mask, 3334 (const uint8_t *)&rte_flow_item_nvgre_mask, 3335 sizeof(struct rte_flow_item_nvgre), 3336 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3337 if (ret < 0) 3338 return ret; 3339 return 0; 3340 } 3341 3342 /** 3343 * Validate eCPRI item. 3344 * 3345 * @param[in] item 3346 * Item specification. 3347 * @param[in] item_flags 3348 * Bit-fields that holds the items detected until now. 3349 * @param[in] last_item 3350 * Previous validated item in the pattern items. 3351 * @param[in] ether_type 3352 * Type in the ethernet layer header (including dot1q). 3353 * @param[in] acc_mask 3354 * Acceptable mask, if NULL default internal default mask 3355 * will be used to check whether item fields are supported. 3356 * @param[out] error 3357 * Pointer to error structure. 3358 * 3359 * @return 3360 * 0 on success, a negative errno value otherwise and rte_errno is set. 3361 */ 3362 int 3363 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3364 uint64_t item_flags, 3365 uint64_t last_item, 3366 uint16_t ether_type, 3367 const struct rte_flow_item_ecpri *acc_mask, 3368 struct rte_flow_error *error) 3369 { 3370 const struct rte_flow_item_ecpri *mask = item->mask; 3371 const struct rte_flow_item_ecpri nic_mask = { 3372 .hdr = { 3373 .common = { 3374 .u32 = 3375 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3376 .type = 0xFF, 3377 }).u32), 3378 }, 3379 .dummy[0] = 0xFFFFFFFF, 3380 }, 3381 }; 3382 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3383 MLX5_FLOW_LAYER_OUTER_VLAN); 3384 struct rte_flow_item_ecpri mask_lo; 3385 3386 if (!(last_item & outer_l2_vlan) && 3387 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3388 return rte_flow_error_set(error, EINVAL, 3389 RTE_FLOW_ERROR_TYPE_ITEM, item, 3390 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3391 if ((last_item & outer_l2_vlan) && ether_type && 3392 ether_type != RTE_ETHER_TYPE_ECPRI) 3393 return rte_flow_error_set(error, EINVAL, 3394 RTE_FLOW_ERROR_TYPE_ITEM, item, 3395 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3396 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3397 return rte_flow_error_set(error, EINVAL, 3398 RTE_FLOW_ERROR_TYPE_ITEM, item, 3399 "eCPRI with tunnel is not supported right now"); 3400 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3401 return rte_flow_error_set(error, ENOTSUP, 3402 RTE_FLOW_ERROR_TYPE_ITEM, item, 3403 "multiple L3 layers not supported"); 3404 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3405 return rte_flow_error_set(error, EINVAL, 3406 RTE_FLOW_ERROR_TYPE_ITEM, item, 3407 "eCPRI cannot coexist with a TCP layer"); 3408 /* In specification, eCPRI could be over UDP layer. */ 3409 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3410 return rte_flow_error_set(error, EINVAL, 3411 RTE_FLOW_ERROR_TYPE_ITEM, item, 3412 "eCPRI over UDP layer is not yet supported right now"); 3413 /* Mask for type field in common header could be zero. */ 3414 if (!mask) 3415 mask = &rte_flow_item_ecpri_mask; 3416 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3417 /* Input mask is in big-endian format. */ 3418 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3419 return rte_flow_error_set(error, EINVAL, 3420 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3421 "partial mask is not supported for protocol"); 3422 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3423 return rte_flow_error_set(error, EINVAL, 3424 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3425 "message header mask must be after a type mask"); 3426 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3427 acc_mask ? (const uint8_t *)acc_mask 3428 : (const uint8_t *)&nic_mask, 3429 sizeof(struct rte_flow_item_ecpri), 3430 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3431 } 3432 3433 static int 3434 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3435 const struct rte_flow_attr *attr __rte_unused, 3436 const struct rte_flow_item items[] __rte_unused, 3437 const struct rte_flow_action actions[] __rte_unused, 3438 bool external __rte_unused, 3439 int hairpin __rte_unused, 3440 struct rte_flow_error *error) 3441 { 3442 return rte_flow_error_set(error, ENOTSUP, 3443 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3444 } 3445 3446 static struct mlx5_flow * 3447 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3448 const struct rte_flow_attr *attr __rte_unused, 3449 const struct rte_flow_item items[] __rte_unused, 3450 const struct rte_flow_action actions[] __rte_unused, 3451 struct rte_flow_error *error) 3452 { 3453 rte_flow_error_set(error, ENOTSUP, 3454 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3455 return NULL; 3456 } 3457 3458 static int 3459 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3460 struct mlx5_flow *dev_flow __rte_unused, 3461 const struct rte_flow_attr *attr __rte_unused, 3462 const struct rte_flow_item items[] __rte_unused, 3463 const struct rte_flow_action actions[] __rte_unused, 3464 struct rte_flow_error *error) 3465 { 3466 return rte_flow_error_set(error, ENOTSUP, 3467 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3468 } 3469 3470 static int 3471 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3472 struct rte_flow *flow __rte_unused, 3473 struct rte_flow_error *error) 3474 { 3475 return rte_flow_error_set(error, ENOTSUP, 3476 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3477 } 3478 3479 static void 3480 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3481 struct rte_flow *flow __rte_unused) 3482 { 3483 } 3484 3485 static void 3486 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3487 struct rte_flow *flow __rte_unused) 3488 { 3489 } 3490 3491 static int 3492 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3493 struct rte_flow *flow __rte_unused, 3494 const struct rte_flow_action *actions __rte_unused, 3495 void *data __rte_unused, 3496 struct rte_flow_error *error) 3497 { 3498 return rte_flow_error_set(error, ENOTSUP, 3499 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3500 } 3501 3502 static int 3503 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3504 uint32_t domains __rte_unused, 3505 uint32_t flags __rte_unused) 3506 { 3507 return 0; 3508 } 3509 3510 /* Void driver to protect from null pointer reference. */ 3511 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3512 .validate = flow_null_validate, 3513 .prepare = flow_null_prepare, 3514 .translate = flow_null_translate, 3515 .apply = flow_null_apply, 3516 .remove = flow_null_remove, 3517 .destroy = flow_null_destroy, 3518 .query = flow_null_query, 3519 .sync_domain = flow_null_sync_domain, 3520 }; 3521 3522 /** 3523 * Select flow driver type according to flow attributes and device 3524 * configuration. 3525 * 3526 * @param[in] dev 3527 * Pointer to the dev structure. 3528 * @param[in] attr 3529 * Pointer to the flow attributes. 3530 * 3531 * @return 3532 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3533 */ 3534 static enum mlx5_flow_drv_type 3535 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3536 { 3537 struct mlx5_priv *priv = dev->data->dev_private; 3538 /* The OS can determine first a specific flow type (DV, VERBS) */ 3539 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3540 3541 if (type != MLX5_FLOW_TYPE_MAX) 3542 return type; 3543 /* 3544 * Currently when dv_flow_en == 2, only HW steering engine is 3545 * supported. New engines can also be chosen here if ready. 3546 */ 3547 if (priv->sh->config.dv_flow_en == 2) 3548 return MLX5_FLOW_TYPE_HW; 3549 /* If no OS specific type - continue with DV/VERBS selection */ 3550 if (attr->transfer && priv->sh->config.dv_esw_en) 3551 type = MLX5_FLOW_TYPE_DV; 3552 if (!attr->transfer) 3553 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3554 MLX5_FLOW_TYPE_VERBS; 3555 return type; 3556 } 3557 3558 #define flow_get_drv_ops(type) flow_drv_ops[type] 3559 3560 /** 3561 * Flow driver validation API. This abstracts calling driver specific functions. 3562 * The type of flow driver is determined according to flow attributes. 3563 * 3564 * @param[in] dev 3565 * Pointer to the dev structure. 3566 * @param[in] attr 3567 * Pointer to the flow attributes. 3568 * @param[in] items 3569 * Pointer to the list of items. 3570 * @param[in] actions 3571 * Pointer to the list of actions. 3572 * @param[in] external 3573 * This flow rule is created by request external to PMD. 3574 * @param[in] hairpin 3575 * Number of hairpin TX actions, 0 means classic flow. 3576 * @param[out] error 3577 * Pointer to the error structure. 3578 * 3579 * @return 3580 * 0 on success, a negative errno value otherwise and rte_errno is set. 3581 */ 3582 static inline int 3583 flow_drv_validate(struct rte_eth_dev *dev, 3584 const struct rte_flow_attr *attr, 3585 const struct rte_flow_item items[], 3586 const struct rte_flow_action actions[], 3587 bool external, int hairpin, struct rte_flow_error *error) 3588 { 3589 const struct mlx5_flow_driver_ops *fops; 3590 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3591 3592 fops = flow_get_drv_ops(type); 3593 return fops->validate(dev, attr, items, actions, external, 3594 hairpin, error); 3595 } 3596 3597 /** 3598 * Flow driver preparation API. This abstracts calling driver specific 3599 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3600 * calculates the size of memory required for device flow, allocates the memory, 3601 * initializes the device flow and returns the pointer. 3602 * 3603 * @note 3604 * This function initializes device flow structure such as dv or verbs in 3605 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3606 * rest. For example, adding returning device flow to flow->dev_flow list and 3607 * setting backward reference to the flow should be done out of this function. 3608 * layers field is not filled either. 3609 * 3610 * @param[in] dev 3611 * Pointer to the dev structure. 3612 * @param[in] attr 3613 * Pointer to the flow attributes. 3614 * @param[in] items 3615 * Pointer to the list of items. 3616 * @param[in] actions 3617 * Pointer to the list of actions. 3618 * @param[in] flow_idx 3619 * This memory pool index to the flow. 3620 * @param[out] error 3621 * Pointer to the error structure. 3622 * 3623 * @return 3624 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3625 */ 3626 static inline struct mlx5_flow * 3627 flow_drv_prepare(struct rte_eth_dev *dev, 3628 const struct rte_flow *flow, 3629 const struct rte_flow_attr *attr, 3630 const struct rte_flow_item items[], 3631 const struct rte_flow_action actions[], 3632 uint32_t flow_idx, 3633 struct rte_flow_error *error) 3634 { 3635 const struct mlx5_flow_driver_ops *fops; 3636 enum mlx5_flow_drv_type type = flow->drv_type; 3637 struct mlx5_flow *mlx5_flow = NULL; 3638 3639 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3640 fops = flow_get_drv_ops(type); 3641 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3642 if (mlx5_flow) 3643 mlx5_flow->flow_idx = flow_idx; 3644 return mlx5_flow; 3645 } 3646 3647 /** 3648 * Flow driver translation API. This abstracts calling driver specific 3649 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3650 * translates a generic flow into a driver flow. flow_drv_prepare() must 3651 * precede. 3652 * 3653 * @note 3654 * dev_flow->layers could be filled as a result of parsing during translation 3655 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3656 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3657 * flow->actions could be overwritten even though all the expanded dev_flows 3658 * have the same actions. 3659 * 3660 * @param[in] dev 3661 * Pointer to the rte dev structure. 3662 * @param[in, out] dev_flow 3663 * Pointer to the mlx5 flow. 3664 * @param[in] attr 3665 * Pointer to the flow attributes. 3666 * @param[in] items 3667 * Pointer to the list of items. 3668 * @param[in] actions 3669 * Pointer to the list of actions. 3670 * @param[out] error 3671 * Pointer to the error structure. 3672 * 3673 * @return 3674 * 0 on success, a negative errno value otherwise and rte_errno is set. 3675 */ 3676 static inline int 3677 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3678 const struct rte_flow_attr *attr, 3679 const struct rte_flow_item items[], 3680 const struct rte_flow_action actions[], 3681 struct rte_flow_error *error) 3682 { 3683 const struct mlx5_flow_driver_ops *fops; 3684 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3685 3686 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3687 fops = flow_get_drv_ops(type); 3688 return fops->translate(dev, dev_flow, attr, items, actions, error); 3689 } 3690 3691 /** 3692 * Flow driver apply API. This abstracts calling driver specific functions. 3693 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3694 * translated driver flows on to device. flow_drv_translate() must precede. 3695 * 3696 * @param[in] dev 3697 * Pointer to Ethernet device structure. 3698 * @param[in, out] flow 3699 * Pointer to flow structure. 3700 * @param[out] error 3701 * Pointer to error structure. 3702 * 3703 * @return 3704 * 0 on success, a negative errno value otherwise and rte_errno is set. 3705 */ 3706 static inline int 3707 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3708 struct rte_flow_error *error) 3709 { 3710 const struct mlx5_flow_driver_ops *fops; 3711 enum mlx5_flow_drv_type type = flow->drv_type; 3712 3713 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3714 fops = flow_get_drv_ops(type); 3715 return fops->apply(dev, flow, error); 3716 } 3717 3718 /** 3719 * Flow driver destroy API. This abstracts calling driver specific functions. 3720 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3721 * on device and releases resources of the flow. 3722 * 3723 * @param[in] dev 3724 * Pointer to Ethernet device. 3725 * @param[in, out] flow 3726 * Pointer to flow structure. 3727 */ 3728 static inline void 3729 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3730 { 3731 const struct mlx5_flow_driver_ops *fops; 3732 enum mlx5_flow_drv_type type = flow->drv_type; 3733 3734 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3735 fops = flow_get_drv_ops(type); 3736 fops->destroy(dev, flow); 3737 } 3738 3739 /** 3740 * Flow driver find RSS policy tbl API. This abstracts calling driver 3741 * specific functions. Parent flow (rte_flow) should have driver 3742 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3743 * 3744 * @param[in] dev 3745 * Pointer to Ethernet device. 3746 * @param[in, out] flow 3747 * Pointer to flow structure. 3748 * @param[in] policy 3749 * Pointer to meter policy table. 3750 * @param[in] rss_desc 3751 * Pointer to rss_desc 3752 */ 3753 static struct mlx5_flow_meter_sub_policy * 3754 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3755 struct rte_flow *flow, 3756 struct mlx5_flow_meter_policy *policy, 3757 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3758 { 3759 const struct mlx5_flow_driver_ops *fops; 3760 enum mlx5_flow_drv_type type = flow->drv_type; 3761 3762 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3763 fops = flow_get_drv_ops(type); 3764 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3765 } 3766 3767 /** 3768 * Flow driver color tag rule API. This abstracts calling driver 3769 * specific functions. Parent flow (rte_flow) should have driver 3770 * type (drv_type). It will create the color tag rules in hierarchy meter. 3771 * 3772 * @param[in] dev 3773 * Pointer to Ethernet device. 3774 * @param[in, out] flow 3775 * Pointer to flow structure. 3776 * @param[in] fm 3777 * Pointer to flow meter structure. 3778 * @param[in] src_port 3779 * The src port this extra rule should use. 3780 * @param[in] item 3781 * The src port id match item. 3782 * @param[out] error 3783 * Pointer to error structure. 3784 */ 3785 static int 3786 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3787 struct rte_flow *flow, 3788 struct mlx5_flow_meter_info *fm, 3789 int32_t src_port, 3790 const struct rte_flow_item *item, 3791 struct rte_flow_error *error) 3792 { 3793 const struct mlx5_flow_driver_ops *fops; 3794 enum mlx5_flow_drv_type type = flow->drv_type; 3795 3796 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3797 fops = flow_get_drv_ops(type); 3798 return fops->meter_hierarchy_rule_create(dev, fm, 3799 src_port, item, error); 3800 } 3801 3802 /** 3803 * Get RSS action from the action list. 3804 * 3805 * @param[in] dev 3806 * Pointer to Ethernet device. 3807 * @param[in] actions 3808 * Pointer to the list of actions. 3809 * @param[in] flow 3810 * Parent flow structure pointer. 3811 * 3812 * @return 3813 * Pointer to the RSS action if exist, else return NULL. 3814 */ 3815 static const struct rte_flow_action_rss* 3816 flow_get_rss_action(struct rte_eth_dev *dev, 3817 const struct rte_flow_action actions[]) 3818 { 3819 struct mlx5_priv *priv = dev->data->dev_private; 3820 const struct rte_flow_action_rss *rss = NULL; 3821 struct mlx5_meter_policy_action_container *acg; 3822 struct mlx5_meter_policy_action_container *acy; 3823 3824 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3825 switch (actions->type) { 3826 case RTE_FLOW_ACTION_TYPE_RSS: 3827 rss = actions->conf; 3828 break; 3829 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3830 { 3831 const struct rte_flow_action_sample *sample = 3832 actions->conf; 3833 const struct rte_flow_action *act = sample->actions; 3834 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3835 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3836 rss = act->conf; 3837 break; 3838 } 3839 case RTE_FLOW_ACTION_TYPE_METER: 3840 { 3841 uint32_t mtr_idx; 3842 struct mlx5_flow_meter_info *fm; 3843 struct mlx5_flow_meter_policy *policy; 3844 const struct rte_flow_action_meter *mtr = actions->conf; 3845 3846 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3847 if (fm && !fm->def_policy) { 3848 policy = mlx5_flow_meter_policy_find(dev, 3849 fm->policy_id, NULL); 3850 MLX5_ASSERT(policy); 3851 if (policy->is_hierarchy) { 3852 policy = 3853 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3854 policy); 3855 if (!policy) 3856 return NULL; 3857 } 3858 if (policy->is_rss) { 3859 acg = 3860 &policy->act_cnt[RTE_COLOR_GREEN]; 3861 acy = 3862 &policy->act_cnt[RTE_COLOR_YELLOW]; 3863 if (acg->fate_action == 3864 MLX5_FLOW_FATE_SHARED_RSS) 3865 rss = acg->rss->conf; 3866 else if (acy->fate_action == 3867 MLX5_FLOW_FATE_SHARED_RSS) 3868 rss = acy->rss->conf; 3869 } 3870 } 3871 break; 3872 } 3873 default: 3874 break; 3875 } 3876 } 3877 return rss; 3878 } 3879 3880 /** 3881 * Get ASO age action by index. 3882 * 3883 * @param[in] dev 3884 * Pointer to the Ethernet device structure. 3885 * @param[in] age_idx 3886 * Index to the ASO age action. 3887 * 3888 * @return 3889 * The specified ASO age action. 3890 */ 3891 struct mlx5_aso_age_action* 3892 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3893 { 3894 uint16_t pool_idx = age_idx & UINT16_MAX; 3895 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3896 struct mlx5_priv *priv = dev->data->dev_private; 3897 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3898 struct mlx5_aso_age_pool *pool; 3899 3900 rte_rwlock_read_lock(&mng->resize_rwl); 3901 pool = mng->pools[pool_idx]; 3902 rte_rwlock_read_unlock(&mng->resize_rwl); 3903 return &pool->actions[offset - 1]; 3904 } 3905 3906 /* maps indirect action to translated direct in some actions array */ 3907 struct mlx5_translated_action_handle { 3908 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3909 int index; /**< Index in related array of rte_flow_action. */ 3910 }; 3911 3912 /** 3913 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3914 * direct action if translation possible. 3915 * This functionality used to run same execution path for both direct and 3916 * indirect actions on flow create. All necessary preparations for indirect 3917 * action handling should be performed on *handle* actions list returned 3918 * from this call. 3919 * 3920 * @param[in] dev 3921 * Pointer to Ethernet device. 3922 * @param[in] actions 3923 * List of actions to translate. 3924 * @param[out] handle 3925 * List to store translated indirect action object handles. 3926 * @param[in, out] indir_n 3927 * Size of *handle* array. On return should be updated with number of 3928 * indirect actions retrieved from the *actions* list. 3929 * @param[out] translated_actions 3930 * List of actions where all indirect actions were translated to direct 3931 * if possible. NULL if no translation took place. 3932 * @param[out] error 3933 * Pointer to the error structure. 3934 * 3935 * @return 3936 * 0 on success, a negative errno value otherwise and rte_errno is set. 3937 */ 3938 static int 3939 flow_action_handles_translate(struct rte_eth_dev *dev, 3940 const struct rte_flow_action actions[], 3941 struct mlx5_translated_action_handle *handle, 3942 int *indir_n, 3943 struct rte_flow_action **translated_actions, 3944 struct rte_flow_error *error) 3945 { 3946 struct mlx5_priv *priv = dev->data->dev_private; 3947 struct rte_flow_action *translated = NULL; 3948 size_t actions_size; 3949 int n; 3950 int copied_n = 0; 3951 struct mlx5_translated_action_handle *handle_end = NULL; 3952 3953 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3954 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3955 continue; 3956 if (copied_n == *indir_n) { 3957 return rte_flow_error_set 3958 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3959 NULL, "too many shared actions"); 3960 } 3961 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3962 sizeof(actions[n].conf)); 3963 handle[copied_n].index = n; 3964 copied_n++; 3965 } 3966 n++; 3967 *indir_n = copied_n; 3968 if (!copied_n) 3969 return 0; 3970 actions_size = sizeof(struct rte_flow_action) * n; 3971 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3972 if (!translated) { 3973 rte_errno = ENOMEM; 3974 return -ENOMEM; 3975 } 3976 memcpy(translated, actions, actions_size); 3977 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3978 struct mlx5_shared_action_rss *shared_rss; 3979 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3980 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3981 uint32_t idx = act_idx & 3982 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3983 3984 switch (type) { 3985 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3986 shared_rss = mlx5_ipool_get 3987 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3988 translated[handle->index].type = 3989 RTE_FLOW_ACTION_TYPE_RSS; 3990 translated[handle->index].conf = 3991 &shared_rss->origin; 3992 break; 3993 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3994 translated[handle->index].type = 3995 (enum rte_flow_action_type) 3996 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3997 translated[handle->index].conf = (void *)(uintptr_t)idx; 3998 break; 3999 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4000 if (priv->sh->flow_hit_aso_en) { 4001 translated[handle->index].type = 4002 (enum rte_flow_action_type) 4003 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4004 translated[handle->index].conf = 4005 (void *)(uintptr_t)idx; 4006 break; 4007 } 4008 /* Fall-through */ 4009 case MLX5_INDIRECT_ACTION_TYPE_CT: 4010 if (priv->sh->ct_aso_en) { 4011 translated[handle->index].type = 4012 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4013 translated[handle->index].conf = 4014 (void *)(uintptr_t)idx; 4015 break; 4016 } 4017 /* Fall-through */ 4018 default: 4019 mlx5_free(translated); 4020 return rte_flow_error_set 4021 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4022 NULL, "invalid indirect action type"); 4023 } 4024 } 4025 *translated_actions = translated; 4026 return 0; 4027 } 4028 4029 /** 4030 * Get Shared RSS action from the action list. 4031 * 4032 * @param[in] dev 4033 * Pointer to Ethernet device. 4034 * @param[in] shared 4035 * Pointer to the list of actions. 4036 * @param[in] shared_n 4037 * Actions list length. 4038 * 4039 * @return 4040 * The MLX5 RSS action ID if exists, otherwise return 0. 4041 */ 4042 static uint32_t 4043 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4044 struct mlx5_translated_action_handle *handle, 4045 int shared_n) 4046 { 4047 struct mlx5_translated_action_handle *handle_end; 4048 struct mlx5_priv *priv = dev->data->dev_private; 4049 struct mlx5_shared_action_rss *shared_rss; 4050 4051 4052 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4053 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4054 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4055 uint32_t idx = act_idx & 4056 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4057 switch (type) { 4058 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4059 shared_rss = mlx5_ipool_get 4060 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4061 idx); 4062 __atomic_add_fetch(&shared_rss->refcnt, 1, 4063 __ATOMIC_RELAXED); 4064 return idx; 4065 default: 4066 break; 4067 } 4068 } 4069 return 0; 4070 } 4071 4072 static unsigned int 4073 find_graph_root(uint32_t rss_level) 4074 { 4075 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4076 MLX5_EXPANSION_ROOT_OUTER; 4077 } 4078 4079 /** 4080 * Get layer flags from the prefix flow. 4081 * 4082 * Some flows may be split to several subflows, the prefix subflow gets the 4083 * match items and the suffix sub flow gets the actions. 4084 * Some actions need the user defined match item flags to get the detail for 4085 * the action. 4086 * This function helps the suffix flow to get the item layer flags from prefix 4087 * subflow. 4088 * 4089 * @param[in] dev_flow 4090 * Pointer the created prefix subflow. 4091 * 4092 * @return 4093 * The layers get from prefix subflow. 4094 */ 4095 static inline uint64_t 4096 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4097 { 4098 uint64_t layers = 0; 4099 4100 /* 4101 * Layers bits could be localization, but usually the compiler will 4102 * help to do the optimization work for source code. 4103 * If no decap actions, use the layers directly. 4104 */ 4105 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4106 return dev_flow->handle->layers; 4107 /* Convert L3 layers with decap action. */ 4108 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4109 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4110 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4111 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4112 /* Convert L4 layers with decap action. */ 4113 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4114 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4115 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4116 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4117 return layers; 4118 } 4119 4120 /** 4121 * Get metadata split action information. 4122 * 4123 * @param[in] actions 4124 * Pointer to the list of actions. 4125 * @param[out] qrss 4126 * Pointer to the return pointer. 4127 * @param[out] qrss_type 4128 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4129 * if no QUEUE/RSS is found. 4130 * @param[out] encap_idx 4131 * Pointer to the index of the encap action if exists, otherwise the last 4132 * action index. 4133 * 4134 * @return 4135 * Total number of actions. 4136 */ 4137 static int 4138 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4139 const struct rte_flow_action **qrss, 4140 int *encap_idx) 4141 { 4142 const struct rte_flow_action_raw_encap *raw_encap; 4143 int actions_n = 0; 4144 int raw_decap_idx = -1; 4145 4146 *encap_idx = -1; 4147 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4148 switch (actions->type) { 4149 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4150 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4151 *encap_idx = actions_n; 4152 break; 4153 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4154 raw_decap_idx = actions_n; 4155 break; 4156 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4157 raw_encap = actions->conf; 4158 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4159 *encap_idx = raw_decap_idx != -1 ? 4160 raw_decap_idx : actions_n; 4161 break; 4162 case RTE_FLOW_ACTION_TYPE_QUEUE: 4163 case RTE_FLOW_ACTION_TYPE_RSS: 4164 *qrss = actions; 4165 break; 4166 default: 4167 break; 4168 } 4169 actions_n++; 4170 } 4171 if (*encap_idx == -1) 4172 *encap_idx = actions_n; 4173 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4174 return actions_n + 1; 4175 } 4176 4177 /** 4178 * Check if the action will change packet. 4179 * 4180 * @param dev 4181 * Pointer to Ethernet device. 4182 * @param[in] type 4183 * action type. 4184 * 4185 * @return 4186 * true if action will change packet, false otherwise. 4187 */ 4188 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4189 enum rte_flow_action_type type) 4190 { 4191 struct mlx5_priv *priv = dev->data->dev_private; 4192 4193 switch (type) { 4194 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4195 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4196 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4197 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4198 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4199 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4200 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4201 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4202 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4203 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4204 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4205 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4206 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4207 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4208 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4209 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4210 case RTE_FLOW_ACTION_TYPE_SET_META: 4211 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4212 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4213 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4214 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4215 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4216 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4217 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4218 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4219 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4220 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4221 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4222 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4223 return true; 4224 case RTE_FLOW_ACTION_TYPE_FLAG: 4225 case RTE_FLOW_ACTION_TYPE_MARK: 4226 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4227 return true; 4228 else 4229 return false; 4230 default: 4231 return false; 4232 } 4233 } 4234 4235 /** 4236 * Check meter action from the action list. 4237 * 4238 * @param dev 4239 * Pointer to Ethernet device. 4240 * @param[in] actions 4241 * Pointer to the list of actions. 4242 * @param[out] has_mtr 4243 * Pointer to the meter exist flag. 4244 * @param[out] has_modify 4245 * Pointer to the flag showing there's packet change action. 4246 * @param[out] meter_id 4247 * Pointer to the meter id. 4248 * 4249 * @return 4250 * Total number of actions. 4251 */ 4252 static int 4253 flow_check_meter_action(struct rte_eth_dev *dev, 4254 const struct rte_flow_action actions[], 4255 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4256 { 4257 const struct rte_flow_action_meter *mtr = NULL; 4258 int actions_n = 0; 4259 4260 MLX5_ASSERT(has_mtr); 4261 *has_mtr = false; 4262 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4263 switch (actions->type) { 4264 case RTE_FLOW_ACTION_TYPE_METER: 4265 mtr = actions->conf; 4266 *meter_id = mtr->mtr_id; 4267 *has_mtr = true; 4268 break; 4269 default: 4270 break; 4271 } 4272 if (!*has_mtr) 4273 *has_modify |= flow_check_modify_action_type(dev, 4274 actions->type); 4275 actions_n++; 4276 } 4277 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4278 return actions_n + 1; 4279 } 4280 4281 /** 4282 * Check if the flow should be split due to hairpin. 4283 * The reason for the split is that in current HW we can't 4284 * support encap and push-vlan on Rx, so if a flow contains 4285 * these actions we move it to Tx. 4286 * 4287 * @param dev 4288 * Pointer to Ethernet device. 4289 * @param[in] attr 4290 * Flow rule attributes. 4291 * @param[in] actions 4292 * Associated actions (list terminated by the END action). 4293 * 4294 * @return 4295 * > 0 the number of actions and the flow should be split, 4296 * 0 when no split required. 4297 */ 4298 static int 4299 flow_check_hairpin_split(struct rte_eth_dev *dev, 4300 const struct rte_flow_attr *attr, 4301 const struct rte_flow_action actions[]) 4302 { 4303 int queue_action = 0; 4304 int action_n = 0; 4305 int split = 0; 4306 const struct rte_flow_action_queue *queue; 4307 const struct rte_flow_action_rss *rss; 4308 const struct rte_flow_action_raw_encap *raw_encap; 4309 const struct rte_eth_hairpin_conf *conf; 4310 4311 if (!attr->ingress) 4312 return 0; 4313 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4314 switch (actions->type) { 4315 case RTE_FLOW_ACTION_TYPE_QUEUE: 4316 queue = actions->conf; 4317 if (queue == NULL) 4318 return 0; 4319 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4320 if (conf == NULL || conf->tx_explicit != 0) 4321 return 0; 4322 queue_action = 1; 4323 action_n++; 4324 break; 4325 case RTE_FLOW_ACTION_TYPE_RSS: 4326 rss = actions->conf; 4327 if (rss == NULL || rss->queue_num == 0) 4328 return 0; 4329 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4330 if (conf == NULL || conf->tx_explicit != 0) 4331 return 0; 4332 queue_action = 1; 4333 action_n++; 4334 break; 4335 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4336 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4337 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4338 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4339 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4340 split++; 4341 action_n++; 4342 break; 4343 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4344 raw_encap = actions->conf; 4345 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4346 split++; 4347 action_n++; 4348 break; 4349 default: 4350 action_n++; 4351 break; 4352 } 4353 } 4354 if (split && queue_action) 4355 return action_n; 4356 return 0; 4357 } 4358 4359 /* Declare flow create/destroy prototype in advance. */ 4360 static uint32_t 4361 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4362 const struct rte_flow_attr *attr, 4363 const struct rte_flow_item items[], 4364 const struct rte_flow_action actions[], 4365 bool external, struct rte_flow_error *error); 4366 4367 static void 4368 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4369 uint32_t flow_idx); 4370 4371 int 4372 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4373 struct mlx5_list_entry *entry, void *cb_ctx) 4374 { 4375 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4376 struct mlx5_flow_mreg_copy_resource *mcp_res = 4377 container_of(entry, typeof(*mcp_res), hlist_ent); 4378 4379 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4380 } 4381 4382 struct mlx5_list_entry * 4383 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4384 { 4385 struct rte_eth_dev *dev = tool_ctx; 4386 struct mlx5_priv *priv = dev->data->dev_private; 4387 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4388 struct mlx5_flow_mreg_copy_resource *mcp_res; 4389 struct rte_flow_error *error = ctx->error; 4390 uint32_t idx = 0; 4391 int ret; 4392 uint32_t mark_id = *(uint32_t *)(ctx->data); 4393 struct rte_flow_attr attr = { 4394 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4395 .ingress = 1, 4396 }; 4397 struct mlx5_rte_flow_item_tag tag_spec = { 4398 .data = mark_id, 4399 }; 4400 struct rte_flow_item items[] = { 4401 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4402 }; 4403 struct rte_flow_action_mark ftag = { 4404 .id = mark_id, 4405 }; 4406 struct mlx5_flow_action_copy_mreg cp_mreg = { 4407 .dst = REG_B, 4408 .src = REG_NON, 4409 }; 4410 struct rte_flow_action_jump jump = { 4411 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4412 }; 4413 struct rte_flow_action actions[] = { 4414 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4415 }; 4416 4417 /* Fill the register fields in the flow. */ 4418 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4419 if (ret < 0) 4420 return NULL; 4421 tag_spec.id = ret; 4422 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4423 if (ret < 0) 4424 return NULL; 4425 cp_mreg.src = ret; 4426 /* Provide the full width of FLAG specific value. */ 4427 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4428 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4429 /* Build a new flow. */ 4430 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4431 items[0] = (struct rte_flow_item){ 4432 .type = (enum rte_flow_item_type) 4433 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4434 .spec = &tag_spec, 4435 }; 4436 items[1] = (struct rte_flow_item){ 4437 .type = RTE_FLOW_ITEM_TYPE_END, 4438 }; 4439 actions[0] = (struct rte_flow_action){ 4440 .type = (enum rte_flow_action_type) 4441 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4442 .conf = &ftag, 4443 }; 4444 actions[1] = (struct rte_flow_action){ 4445 .type = (enum rte_flow_action_type) 4446 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4447 .conf = &cp_mreg, 4448 }; 4449 actions[2] = (struct rte_flow_action){ 4450 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4451 .conf = &jump, 4452 }; 4453 actions[3] = (struct rte_flow_action){ 4454 .type = RTE_FLOW_ACTION_TYPE_END, 4455 }; 4456 } else { 4457 /* Default rule, wildcard match. */ 4458 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4459 items[0] = (struct rte_flow_item){ 4460 .type = RTE_FLOW_ITEM_TYPE_END, 4461 }; 4462 actions[0] = (struct rte_flow_action){ 4463 .type = (enum rte_flow_action_type) 4464 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4465 .conf = &cp_mreg, 4466 }; 4467 actions[1] = (struct rte_flow_action){ 4468 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4469 .conf = &jump, 4470 }; 4471 actions[2] = (struct rte_flow_action){ 4472 .type = RTE_FLOW_ACTION_TYPE_END, 4473 }; 4474 } 4475 /* Build a new entry. */ 4476 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4477 if (!mcp_res) { 4478 rte_errno = ENOMEM; 4479 return NULL; 4480 } 4481 mcp_res->idx = idx; 4482 mcp_res->mark_id = mark_id; 4483 /* 4484 * The copy Flows are not included in any list. There 4485 * ones are referenced from other Flows and can not 4486 * be applied, removed, deleted in arbitrary order 4487 * by list traversing. 4488 */ 4489 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4490 &attr, items, actions, false, error); 4491 if (!mcp_res->rix_flow) { 4492 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4493 return NULL; 4494 } 4495 return &mcp_res->hlist_ent; 4496 } 4497 4498 struct mlx5_list_entry * 4499 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4500 void *cb_ctx __rte_unused) 4501 { 4502 struct rte_eth_dev *dev = tool_ctx; 4503 struct mlx5_priv *priv = dev->data->dev_private; 4504 struct mlx5_flow_mreg_copy_resource *mcp_res; 4505 uint32_t idx = 0; 4506 4507 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4508 if (!mcp_res) { 4509 rte_errno = ENOMEM; 4510 return NULL; 4511 } 4512 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4513 mcp_res->idx = idx; 4514 return &mcp_res->hlist_ent; 4515 } 4516 4517 void 4518 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4519 { 4520 struct mlx5_flow_mreg_copy_resource *mcp_res = 4521 container_of(entry, typeof(*mcp_res), hlist_ent); 4522 struct rte_eth_dev *dev = tool_ctx; 4523 struct mlx5_priv *priv = dev->data->dev_private; 4524 4525 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4526 } 4527 4528 /** 4529 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4530 * 4531 * As mark_id is unique, if there's already a registered flow for the mark_id, 4532 * return by increasing the reference counter of the resource. Otherwise, create 4533 * the resource (mcp_res) and flow. 4534 * 4535 * Flow looks like, 4536 * - If ingress port is ANY and reg_c[1] is mark_id, 4537 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4538 * 4539 * For default flow (zero mark_id), flow is like, 4540 * - If ingress port is ANY, 4541 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4542 * 4543 * @param dev 4544 * Pointer to Ethernet device. 4545 * @param mark_id 4546 * ID of MARK action, zero means default flow for META. 4547 * @param[out] error 4548 * Perform verbose error reporting if not NULL. 4549 * 4550 * @return 4551 * Associated resource on success, NULL otherwise and rte_errno is set. 4552 */ 4553 static struct mlx5_flow_mreg_copy_resource * 4554 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4555 struct rte_flow_error *error) 4556 { 4557 struct mlx5_priv *priv = dev->data->dev_private; 4558 struct mlx5_list_entry *entry; 4559 struct mlx5_flow_cb_ctx ctx = { 4560 .dev = dev, 4561 .error = error, 4562 .data = &mark_id, 4563 }; 4564 4565 /* Check if already registered. */ 4566 MLX5_ASSERT(priv->mreg_cp_tbl); 4567 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4568 if (!entry) 4569 return NULL; 4570 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4571 hlist_ent); 4572 } 4573 4574 void 4575 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4576 { 4577 struct mlx5_flow_mreg_copy_resource *mcp_res = 4578 container_of(entry, typeof(*mcp_res), hlist_ent); 4579 struct rte_eth_dev *dev = tool_ctx; 4580 struct mlx5_priv *priv = dev->data->dev_private; 4581 4582 MLX5_ASSERT(mcp_res->rix_flow); 4583 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4584 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4585 } 4586 4587 /** 4588 * Release flow in RX_CP_TBL. 4589 * 4590 * @param dev 4591 * Pointer to Ethernet device. 4592 * @flow 4593 * Parent flow for wich copying is provided. 4594 */ 4595 static void 4596 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4597 struct rte_flow *flow) 4598 { 4599 struct mlx5_flow_mreg_copy_resource *mcp_res; 4600 struct mlx5_priv *priv = dev->data->dev_private; 4601 4602 if (!flow->rix_mreg_copy) 4603 return; 4604 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4605 flow->rix_mreg_copy); 4606 if (!mcp_res || !priv->mreg_cp_tbl) 4607 return; 4608 MLX5_ASSERT(mcp_res->rix_flow); 4609 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4610 flow->rix_mreg_copy = 0; 4611 } 4612 4613 /** 4614 * Remove the default copy action from RX_CP_TBL. 4615 * 4616 * This functions is called in the mlx5_dev_start(). No thread safe 4617 * is guaranteed. 4618 * 4619 * @param dev 4620 * Pointer to Ethernet device. 4621 */ 4622 static void 4623 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4624 { 4625 struct mlx5_list_entry *entry; 4626 struct mlx5_priv *priv = dev->data->dev_private; 4627 struct mlx5_flow_cb_ctx ctx; 4628 uint32_t mark_id; 4629 4630 /* Check if default flow is registered. */ 4631 if (!priv->mreg_cp_tbl) 4632 return; 4633 mark_id = MLX5_DEFAULT_COPY_ID; 4634 ctx.data = &mark_id; 4635 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4636 if (!entry) 4637 return; 4638 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4639 } 4640 4641 /** 4642 * Add the default copy action in in RX_CP_TBL. 4643 * 4644 * This functions is called in the mlx5_dev_start(). No thread safe 4645 * is guaranteed. 4646 * 4647 * @param dev 4648 * Pointer to Ethernet device. 4649 * @param[out] error 4650 * Perform verbose error reporting if not NULL. 4651 * 4652 * @return 4653 * 0 for success, negative value otherwise and rte_errno is set. 4654 */ 4655 static int 4656 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4657 struct rte_flow_error *error) 4658 { 4659 struct mlx5_priv *priv = dev->data->dev_private; 4660 struct mlx5_flow_mreg_copy_resource *mcp_res; 4661 struct mlx5_flow_cb_ctx ctx; 4662 uint32_t mark_id; 4663 4664 /* Check whether extensive metadata feature is engaged. */ 4665 if (!priv->sh->config.dv_flow_en || 4666 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4667 !mlx5_flow_ext_mreg_supported(dev) || 4668 !priv->sh->dv_regc0_mask) 4669 return 0; 4670 /* 4671 * Add default mreg copy flow may be called multiple time, but 4672 * only be called once in stop. Avoid register it twice. 4673 */ 4674 mark_id = MLX5_DEFAULT_COPY_ID; 4675 ctx.data = &mark_id; 4676 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4677 return 0; 4678 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4679 if (!mcp_res) 4680 return -rte_errno; 4681 return 0; 4682 } 4683 4684 /** 4685 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4686 * 4687 * All the flow having Q/RSS action should be split by 4688 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4689 * performs the following, 4690 * - CQE->flow_tag := reg_c[1] (MARK) 4691 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4692 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4693 * but there should be a flow per each MARK ID set by MARK action. 4694 * 4695 * For the aforementioned reason, if there's a MARK action in flow's action 4696 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4697 * the MARK ID to CQE's flow_tag like, 4698 * - If reg_c[1] is mark_id, 4699 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4700 * 4701 * For SET_META action which stores value in reg_c[0], as the destination is 4702 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4703 * MARK ID means the default flow. The default flow looks like, 4704 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4705 * 4706 * @param dev 4707 * Pointer to Ethernet device. 4708 * @param flow 4709 * Pointer to flow structure. 4710 * @param[in] actions 4711 * Pointer to the list of actions. 4712 * @param[out] error 4713 * Perform verbose error reporting if not NULL. 4714 * 4715 * @return 4716 * 0 on success, negative value otherwise and rte_errno is set. 4717 */ 4718 static int 4719 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4720 struct rte_flow *flow, 4721 const struct rte_flow_action *actions, 4722 struct rte_flow_error *error) 4723 { 4724 struct mlx5_priv *priv = dev->data->dev_private; 4725 struct mlx5_sh_config *config = &priv->sh->config; 4726 struct mlx5_flow_mreg_copy_resource *mcp_res; 4727 const struct rte_flow_action_mark *mark; 4728 4729 /* Check whether extensive metadata feature is engaged. */ 4730 if (!config->dv_flow_en || 4731 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4732 !mlx5_flow_ext_mreg_supported(dev) || 4733 !priv->sh->dv_regc0_mask) 4734 return 0; 4735 /* Find MARK action. */ 4736 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4737 switch (actions->type) { 4738 case RTE_FLOW_ACTION_TYPE_FLAG: 4739 mcp_res = flow_mreg_add_copy_action 4740 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4741 if (!mcp_res) 4742 return -rte_errno; 4743 flow->rix_mreg_copy = mcp_res->idx; 4744 return 0; 4745 case RTE_FLOW_ACTION_TYPE_MARK: 4746 mark = (const struct rte_flow_action_mark *) 4747 actions->conf; 4748 mcp_res = 4749 flow_mreg_add_copy_action(dev, mark->id, error); 4750 if (!mcp_res) 4751 return -rte_errno; 4752 flow->rix_mreg_copy = mcp_res->idx; 4753 return 0; 4754 default: 4755 break; 4756 } 4757 } 4758 return 0; 4759 } 4760 4761 #define MLX5_MAX_SPLIT_ACTIONS 24 4762 #define MLX5_MAX_SPLIT_ITEMS 24 4763 4764 /** 4765 * Split the hairpin flow. 4766 * Since HW can't support encap and push-vlan on Rx, we move these 4767 * actions to Tx. 4768 * If the count action is after the encap then we also 4769 * move the count action. in this case the count will also measure 4770 * the outer bytes. 4771 * 4772 * @param dev 4773 * Pointer to Ethernet device. 4774 * @param[in] actions 4775 * Associated actions (list terminated by the END action). 4776 * @param[out] actions_rx 4777 * Rx flow actions. 4778 * @param[out] actions_tx 4779 * Tx flow actions.. 4780 * @param[out] pattern_tx 4781 * The pattern items for the Tx flow. 4782 * @param[out] flow_id 4783 * The flow ID connected to this flow. 4784 * 4785 * @return 4786 * 0 on success. 4787 */ 4788 static int 4789 flow_hairpin_split(struct rte_eth_dev *dev, 4790 const struct rte_flow_action actions[], 4791 struct rte_flow_action actions_rx[], 4792 struct rte_flow_action actions_tx[], 4793 struct rte_flow_item pattern_tx[], 4794 uint32_t flow_id) 4795 { 4796 const struct rte_flow_action_raw_encap *raw_encap; 4797 const struct rte_flow_action_raw_decap *raw_decap; 4798 struct mlx5_rte_flow_action_set_tag *set_tag; 4799 struct rte_flow_action *tag_action; 4800 struct mlx5_rte_flow_item_tag *tag_item; 4801 struct rte_flow_item *item; 4802 char *addr; 4803 int encap = 0; 4804 4805 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4806 switch (actions->type) { 4807 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4808 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4809 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4810 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4811 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4812 rte_memcpy(actions_tx, actions, 4813 sizeof(struct rte_flow_action)); 4814 actions_tx++; 4815 break; 4816 case RTE_FLOW_ACTION_TYPE_COUNT: 4817 if (encap) { 4818 rte_memcpy(actions_tx, actions, 4819 sizeof(struct rte_flow_action)); 4820 actions_tx++; 4821 } else { 4822 rte_memcpy(actions_rx, actions, 4823 sizeof(struct rte_flow_action)); 4824 actions_rx++; 4825 } 4826 break; 4827 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4828 raw_encap = actions->conf; 4829 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4830 memcpy(actions_tx, actions, 4831 sizeof(struct rte_flow_action)); 4832 actions_tx++; 4833 encap = 1; 4834 } else { 4835 rte_memcpy(actions_rx, actions, 4836 sizeof(struct rte_flow_action)); 4837 actions_rx++; 4838 } 4839 break; 4840 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4841 raw_decap = actions->conf; 4842 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4843 memcpy(actions_tx, actions, 4844 sizeof(struct rte_flow_action)); 4845 actions_tx++; 4846 } else { 4847 rte_memcpy(actions_rx, actions, 4848 sizeof(struct rte_flow_action)); 4849 actions_rx++; 4850 } 4851 break; 4852 default: 4853 rte_memcpy(actions_rx, actions, 4854 sizeof(struct rte_flow_action)); 4855 actions_rx++; 4856 break; 4857 } 4858 } 4859 /* Add set meta action and end action for the Rx flow. */ 4860 tag_action = actions_rx; 4861 tag_action->type = (enum rte_flow_action_type) 4862 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4863 actions_rx++; 4864 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4865 actions_rx++; 4866 set_tag = (void *)actions_rx; 4867 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4868 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4869 .data = flow_id, 4870 }; 4871 MLX5_ASSERT(set_tag->id > REG_NON); 4872 tag_action->conf = set_tag; 4873 /* Create Tx item list. */ 4874 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4875 addr = (void *)&pattern_tx[2]; 4876 item = pattern_tx; 4877 item->type = (enum rte_flow_item_type) 4878 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4879 tag_item = (void *)addr; 4880 tag_item->data = flow_id; 4881 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4882 MLX5_ASSERT(set_tag->id > REG_NON); 4883 item->spec = tag_item; 4884 addr += sizeof(struct mlx5_rte_flow_item_tag); 4885 tag_item = (void *)addr; 4886 tag_item->data = UINT32_MAX; 4887 tag_item->id = UINT16_MAX; 4888 item->mask = tag_item; 4889 item->last = NULL; 4890 item++; 4891 item->type = RTE_FLOW_ITEM_TYPE_END; 4892 return 0; 4893 } 4894 4895 /** 4896 * The last stage of splitting chain, just creates the subflow 4897 * without any modification. 4898 * 4899 * @param[in] dev 4900 * Pointer to Ethernet device. 4901 * @param[in] flow 4902 * Parent flow structure pointer. 4903 * @param[in, out] sub_flow 4904 * Pointer to return the created subflow, may be NULL. 4905 * @param[in] attr 4906 * Flow rule attributes. 4907 * @param[in] items 4908 * Pattern specification (list terminated by the END pattern item). 4909 * @param[in] actions 4910 * Associated actions (list terminated by the END action). 4911 * @param[in] flow_split_info 4912 * Pointer to flow split info structure. 4913 * @param[out] error 4914 * Perform verbose error reporting if not NULL. 4915 * @return 4916 * 0 on success, negative value otherwise 4917 */ 4918 static int 4919 flow_create_split_inner(struct rte_eth_dev *dev, 4920 struct rte_flow *flow, 4921 struct mlx5_flow **sub_flow, 4922 const struct rte_flow_attr *attr, 4923 const struct rte_flow_item items[], 4924 const struct rte_flow_action actions[], 4925 struct mlx5_flow_split_info *flow_split_info, 4926 struct rte_flow_error *error) 4927 { 4928 struct mlx5_flow *dev_flow; 4929 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 4930 4931 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4932 flow_split_info->flow_idx, error); 4933 if (!dev_flow) 4934 return -rte_errno; 4935 dev_flow->flow = flow; 4936 dev_flow->external = flow_split_info->external; 4937 dev_flow->skip_scale = flow_split_info->skip_scale; 4938 /* Subflow object was created, we must include one in the list. */ 4939 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4940 dev_flow->handle, next); 4941 /* 4942 * If dev_flow is as one of the suffix flow, some actions in suffix 4943 * flow may need some user defined item layer flags, and pass the 4944 * Metadata rxq mark flag to suffix flow as well. 4945 */ 4946 if (flow_split_info->prefix_layers) 4947 dev_flow->handle->layers = flow_split_info->prefix_layers; 4948 if (flow_split_info->prefix_mark) { 4949 MLX5_ASSERT(wks); 4950 wks->mark = 1; 4951 } 4952 if (sub_flow) 4953 *sub_flow = dev_flow; 4954 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4955 dev_flow->dv.table_id = flow_split_info->table_id; 4956 #endif 4957 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4958 } 4959 4960 /** 4961 * Get the sub policy of a meter. 4962 * 4963 * @param[in] dev 4964 * Pointer to Ethernet device. 4965 * @param[in] flow 4966 * Parent flow structure pointer. 4967 * @param wks 4968 * Pointer to thread flow work space. 4969 * @param[in] attr 4970 * Flow rule attributes. 4971 * @param[in] items 4972 * Pattern specification (list terminated by the END pattern item). 4973 * @param[out] error 4974 * Perform verbose error reporting if not NULL. 4975 * 4976 * @return 4977 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4978 */ 4979 static struct mlx5_flow_meter_sub_policy * 4980 get_meter_sub_policy(struct rte_eth_dev *dev, 4981 struct rte_flow *flow, 4982 struct mlx5_flow_workspace *wks, 4983 const struct rte_flow_attr *attr, 4984 const struct rte_flow_item items[], 4985 struct rte_flow_error *error) 4986 { 4987 struct mlx5_flow_meter_policy *policy; 4988 struct mlx5_flow_meter_policy *final_policy; 4989 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4990 4991 policy = wks->policy; 4992 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4993 if (final_policy->is_rss || final_policy->is_queue) { 4994 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4995 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4996 uint32_t i; 4997 4998 /* 4999 * This is a tmp dev_flow, 5000 * no need to register any matcher for it in translate. 5001 */ 5002 wks->skip_matcher_reg = 1; 5003 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5004 struct mlx5_flow dev_flow = {0}; 5005 struct mlx5_flow_handle dev_handle = { {0} }; 5006 uint8_t fate = final_policy->act_cnt[i].fate_action; 5007 5008 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5009 const struct rte_flow_action_rss *rss_act = 5010 final_policy->act_cnt[i].rss->conf; 5011 struct rte_flow_action rss_actions[2] = { 5012 [0] = { 5013 .type = RTE_FLOW_ACTION_TYPE_RSS, 5014 .conf = rss_act, 5015 }, 5016 [1] = { 5017 .type = RTE_FLOW_ACTION_TYPE_END, 5018 .conf = NULL, 5019 } 5020 }; 5021 5022 dev_flow.handle = &dev_handle; 5023 dev_flow.ingress = attr->ingress; 5024 dev_flow.flow = flow; 5025 dev_flow.external = 0; 5026 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5027 dev_flow.dv.transfer = attr->transfer; 5028 #endif 5029 /** 5030 * Translate RSS action to get rss hash fields. 5031 */ 5032 if (flow_drv_translate(dev, &dev_flow, attr, 5033 items, rss_actions, error)) 5034 goto exit; 5035 rss_desc_v[i] = wks->rss_desc; 5036 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5037 rss_desc_v[i].hash_fields = 5038 dev_flow.hash_fields; 5039 rss_desc_v[i].queue_num = 5040 rss_desc_v[i].hash_fields ? 5041 rss_desc_v[i].queue_num : 1; 5042 rss_desc_v[i].tunnel = 5043 !!(dev_flow.handle->layers & 5044 MLX5_FLOW_LAYER_TUNNEL); 5045 /* Use the RSS queues in the containers. */ 5046 rss_desc_v[i].queue = 5047 (uint16_t *)(uintptr_t)rss_act->queue; 5048 rss_desc[i] = &rss_desc_v[i]; 5049 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5050 /* This is queue action. */ 5051 rss_desc_v[i] = wks->rss_desc; 5052 rss_desc_v[i].key_len = 0; 5053 rss_desc_v[i].hash_fields = 0; 5054 rss_desc_v[i].queue = 5055 &final_policy->act_cnt[i].queue; 5056 rss_desc_v[i].queue_num = 1; 5057 rss_desc[i] = &rss_desc_v[i]; 5058 } else { 5059 rss_desc[i] = NULL; 5060 } 5061 } 5062 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5063 flow, policy, rss_desc); 5064 } else { 5065 enum mlx5_meter_domain mtr_domain = 5066 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5067 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5068 MLX5_MTR_DOMAIN_INGRESS); 5069 sub_policy = policy->sub_policys[mtr_domain][0]; 5070 } 5071 if (!sub_policy) 5072 rte_flow_error_set(error, EINVAL, 5073 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5074 "Failed to get meter sub-policy."); 5075 exit: 5076 return sub_policy; 5077 } 5078 5079 /** 5080 * Split the meter flow. 5081 * 5082 * As meter flow will split to three sub flow, other than meter 5083 * action, the other actions make sense to only meter accepts 5084 * the packet. If it need to be dropped, no other additional 5085 * actions should be take. 5086 * 5087 * One kind of special action which decapsulates the L3 tunnel 5088 * header will be in the prefix sub flow, as not to take the 5089 * L3 tunnel header into account. 5090 * 5091 * @param[in] dev 5092 * Pointer to Ethernet device. 5093 * @param[in] flow 5094 * Parent flow structure pointer. 5095 * @param wks 5096 * Pointer to thread flow work space. 5097 * @param[in] attr 5098 * Flow rule attributes. 5099 * @param[in] items 5100 * Pattern specification (list terminated by the END pattern item). 5101 * @param[out] sfx_items 5102 * Suffix flow match items (list terminated by the END pattern item). 5103 * @param[in] actions 5104 * Associated actions (list terminated by the END action). 5105 * @param[out] actions_sfx 5106 * Suffix flow actions. 5107 * @param[out] actions_pre 5108 * Prefix flow actions. 5109 * @param[out] mtr_flow_id 5110 * Pointer to meter flow id. 5111 * @param[out] error 5112 * Perform verbose error reporting if not NULL. 5113 * 5114 * @return 5115 * 0 on success, a negative errno value otherwise and rte_errno is set. 5116 */ 5117 static int 5118 flow_meter_split_prep(struct rte_eth_dev *dev, 5119 struct rte_flow *flow, 5120 struct mlx5_flow_workspace *wks, 5121 const struct rte_flow_attr *attr, 5122 const struct rte_flow_item items[], 5123 struct rte_flow_item sfx_items[], 5124 const struct rte_flow_action actions[], 5125 struct rte_flow_action actions_sfx[], 5126 struct rte_flow_action actions_pre[], 5127 uint32_t *mtr_flow_id, 5128 struct rte_flow_error *error) 5129 { 5130 struct mlx5_priv *priv = dev->data->dev_private; 5131 struct mlx5_flow_meter_info *fm = wks->fm; 5132 struct rte_flow_action *tag_action = NULL; 5133 struct rte_flow_item *tag_item; 5134 struct mlx5_rte_flow_action_set_tag *set_tag; 5135 const struct rte_flow_action_raw_encap *raw_encap; 5136 const struct rte_flow_action_raw_decap *raw_decap; 5137 struct mlx5_rte_flow_item_tag *tag_item_spec; 5138 struct mlx5_rte_flow_item_tag *tag_item_mask; 5139 uint32_t tag_id = 0; 5140 struct rte_flow_item *vlan_item_dst = NULL; 5141 const struct rte_flow_item *vlan_item_src = NULL; 5142 const struct rte_flow_item *orig_items = items; 5143 struct rte_flow_action *hw_mtr_action; 5144 struct rte_flow_action *action_pre_head = NULL; 5145 int32_t flow_src_port = priv->representor_id; 5146 bool mtr_first; 5147 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5148 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5149 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5150 uint32_t flow_id = 0; 5151 uint32_t flow_id_reversed = 0; 5152 uint8_t flow_id_bits = 0; 5153 int shift; 5154 5155 /* Prepare the suffix subflow items. */ 5156 tag_item = sfx_items++; 5157 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5158 struct mlx5_priv *port_priv; 5159 const struct rte_flow_item_port_id *pid_v; 5160 int item_type = items->type; 5161 5162 switch (item_type) { 5163 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5164 pid_v = items->spec; 5165 MLX5_ASSERT(pid_v); 5166 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5167 if (!port_priv) 5168 return rte_flow_error_set(error, 5169 rte_errno, 5170 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5171 pid_v, 5172 "Failed to get port info."); 5173 flow_src_port = port_priv->representor_id; 5174 if (!fm->def_policy && wks->policy->is_hierarchy && 5175 flow_src_port != priv->representor_id) { 5176 if (flow_drv_mtr_hierarchy_rule_create(dev, 5177 flow, fm, 5178 flow_src_port, 5179 items, 5180 error)) 5181 return -rte_errno; 5182 } 5183 memcpy(sfx_items, items, sizeof(*sfx_items)); 5184 sfx_items++; 5185 break; 5186 case RTE_FLOW_ITEM_TYPE_VLAN: 5187 /* Determine if copy vlan item below. */ 5188 vlan_item_src = items; 5189 vlan_item_dst = sfx_items++; 5190 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5191 break; 5192 default: 5193 break; 5194 } 5195 } 5196 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5197 sfx_items++; 5198 mtr_first = priv->sh->meter_aso_en && 5199 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5200 /* For ASO meter, meter must be before tag in TX direction. */ 5201 if (mtr_first) { 5202 action_pre_head = actions_pre++; 5203 /* Leave space for tag action. */ 5204 tag_action = actions_pre++; 5205 } 5206 /* Prepare the actions for prefix and suffix flow. */ 5207 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5208 struct rte_flow_action *action_cur = NULL; 5209 5210 switch (actions->type) { 5211 case RTE_FLOW_ACTION_TYPE_METER: 5212 if (mtr_first) { 5213 action_cur = action_pre_head; 5214 } else { 5215 /* Leave space for tag action. */ 5216 tag_action = actions_pre++; 5217 action_cur = actions_pre++; 5218 } 5219 break; 5220 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5221 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5222 action_cur = actions_pre++; 5223 break; 5224 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5225 raw_encap = actions->conf; 5226 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5227 action_cur = actions_pre++; 5228 break; 5229 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5230 raw_decap = actions->conf; 5231 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5232 action_cur = actions_pre++; 5233 break; 5234 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5235 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5236 if (vlan_item_dst && vlan_item_src) { 5237 memcpy(vlan_item_dst, vlan_item_src, 5238 sizeof(*vlan_item_dst)); 5239 /* 5240 * Convert to internal match item, it is used 5241 * for vlan push and set vid. 5242 */ 5243 vlan_item_dst->type = (enum rte_flow_item_type) 5244 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5245 } 5246 break; 5247 default: 5248 break; 5249 } 5250 if (!action_cur) 5251 action_cur = (fm->def_policy) ? 5252 actions_sfx++ : actions_pre++; 5253 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5254 } 5255 /* Add end action to the actions. */ 5256 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5257 if (priv->sh->meter_aso_en) { 5258 /** 5259 * For ASO meter, need to add an extra jump action explicitly, 5260 * to jump from meter to policer table. 5261 */ 5262 struct mlx5_flow_meter_sub_policy *sub_policy; 5263 struct mlx5_flow_tbl_data_entry *tbl_data; 5264 5265 if (!fm->def_policy) { 5266 sub_policy = get_meter_sub_policy(dev, flow, wks, 5267 attr, orig_items, 5268 error); 5269 if (!sub_policy) 5270 return -rte_errno; 5271 } else { 5272 enum mlx5_meter_domain mtr_domain = 5273 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5274 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5275 MLX5_MTR_DOMAIN_INGRESS); 5276 5277 sub_policy = 5278 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5279 } 5280 tbl_data = container_of(sub_policy->tbl_rsc, 5281 struct mlx5_flow_tbl_data_entry, tbl); 5282 hw_mtr_action = actions_pre++; 5283 hw_mtr_action->type = (enum rte_flow_action_type) 5284 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5285 hw_mtr_action->conf = tbl_data->jump.action; 5286 } 5287 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5288 actions_pre++; 5289 if (!tag_action) 5290 return rte_flow_error_set(error, ENOMEM, 5291 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5292 NULL, "No tag action space."); 5293 if (!mtr_flow_id) { 5294 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5295 goto exit; 5296 } 5297 /* Only default-policy Meter creates mtr flow id. */ 5298 if (fm->def_policy) { 5299 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5300 if (!tag_id) 5301 return rte_flow_error_set(error, ENOMEM, 5302 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5303 "Failed to allocate meter flow id."); 5304 flow_id = tag_id - 1; 5305 flow_id_bits = (!flow_id) ? 1 : 5306 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5307 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5308 mtr_reg_bits) { 5309 mlx5_ipool_free(fm->flow_ipool, tag_id); 5310 return rte_flow_error_set(error, EINVAL, 5311 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5312 "Meter flow id exceeds max limit."); 5313 } 5314 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5315 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5316 } 5317 /* Build tag actions and items for meter_id/meter flow_id. */ 5318 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5319 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5320 tag_item_mask = tag_item_spec + 1; 5321 /* Both flow_id and meter_id share the same register. */ 5322 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5323 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5324 0, error), 5325 .offset = mtr_id_offset, 5326 .length = mtr_reg_bits, 5327 .data = flow->meter, 5328 }; 5329 /* 5330 * The color Reg bits used by flow_id are growing from 5331 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5332 */ 5333 for (shift = 0; shift < flow_id_bits; shift++) 5334 flow_id_reversed = (flow_id_reversed << 1) | 5335 ((flow_id >> shift) & 0x1); 5336 set_tag->data |= 5337 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5338 tag_item_spec->id = set_tag->id; 5339 tag_item_spec->data = set_tag->data << mtr_id_offset; 5340 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5341 tag_action->type = (enum rte_flow_action_type) 5342 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5343 tag_action->conf = set_tag; 5344 tag_item->type = (enum rte_flow_item_type) 5345 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5346 tag_item->spec = tag_item_spec; 5347 tag_item->last = NULL; 5348 tag_item->mask = tag_item_mask; 5349 exit: 5350 if (mtr_flow_id) 5351 *mtr_flow_id = tag_id; 5352 return 0; 5353 } 5354 5355 /** 5356 * Split action list having QUEUE/RSS for metadata register copy. 5357 * 5358 * Once Q/RSS action is detected in user's action list, the flow action 5359 * should be split in order to copy metadata registers, which will happen in 5360 * RX_CP_TBL like, 5361 * - CQE->flow_tag := reg_c[1] (MARK) 5362 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5363 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5364 * This is because the last action of each flow must be a terminal action 5365 * (QUEUE, RSS or DROP). 5366 * 5367 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5368 * stored and kept in the mlx5_flow structure per each sub_flow. 5369 * 5370 * The Q/RSS action is replaced with, 5371 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5372 * And the following JUMP action is added at the end, 5373 * - JUMP, to RX_CP_TBL. 5374 * 5375 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5376 * flow_create_split_metadata() routine. The flow will look like, 5377 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5378 * 5379 * @param dev 5380 * Pointer to Ethernet device. 5381 * @param[out] split_actions 5382 * Pointer to store split actions to jump to CP_TBL. 5383 * @param[in] actions 5384 * Pointer to the list of original flow actions. 5385 * @param[in] qrss 5386 * Pointer to the Q/RSS action. 5387 * @param[in] actions_n 5388 * Number of original actions. 5389 * @param[in] mtr_sfx 5390 * Check if it is in meter suffix table. 5391 * @param[out] error 5392 * Perform verbose error reporting if not NULL. 5393 * 5394 * @return 5395 * non-zero unique flow_id on success, otherwise 0 and 5396 * error/rte_error are set. 5397 */ 5398 static uint32_t 5399 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5400 struct rte_flow_action *split_actions, 5401 const struct rte_flow_action *actions, 5402 const struct rte_flow_action *qrss, 5403 int actions_n, int mtr_sfx, 5404 struct rte_flow_error *error) 5405 { 5406 struct mlx5_priv *priv = dev->data->dev_private; 5407 struct mlx5_rte_flow_action_set_tag *set_tag; 5408 struct rte_flow_action_jump *jump; 5409 const int qrss_idx = qrss - actions; 5410 uint32_t flow_id = 0; 5411 int ret = 0; 5412 5413 /* 5414 * Given actions will be split 5415 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5416 * - Add jump to mreg CP_TBL. 5417 * As a result, there will be one more action. 5418 */ 5419 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5420 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5421 ++actions_n; 5422 set_tag = (void *)(split_actions + actions_n); 5423 /* 5424 * If we are not the meter suffix flow, add the tag action. 5425 * Since meter suffix flow already has the tag added. 5426 */ 5427 if (!mtr_sfx) { 5428 /* 5429 * Allocate the new subflow ID. This one is unique within 5430 * device and not shared with representors. Otherwise, 5431 * we would have to resolve multi-thread access synch 5432 * issue. Each flow on the shared device is appended 5433 * with source vport identifier, so the resulting 5434 * flows will be unique in the shared (by master and 5435 * representors) domain even if they have coinciding 5436 * IDs. 5437 */ 5438 mlx5_ipool_malloc(priv->sh->ipool 5439 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5440 if (!flow_id) 5441 return rte_flow_error_set(error, ENOMEM, 5442 RTE_FLOW_ERROR_TYPE_ACTION, 5443 NULL, "can't allocate id " 5444 "for split Q/RSS subflow"); 5445 /* Internal SET_TAG action to set flow ID. */ 5446 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5447 .data = flow_id, 5448 }; 5449 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5450 if (ret < 0) 5451 return ret; 5452 set_tag->id = ret; 5453 /* Construct new actions array. */ 5454 /* Replace QUEUE/RSS action. */ 5455 split_actions[qrss_idx] = (struct rte_flow_action){ 5456 .type = (enum rte_flow_action_type) 5457 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5458 .conf = set_tag, 5459 }; 5460 } else { 5461 /* 5462 * If we are the suffix flow of meter, tag already exist. 5463 * Set the QUEUE/RSS action to void. 5464 */ 5465 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5466 } 5467 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5468 jump = (void *)(set_tag + 1); 5469 *jump = (struct rte_flow_action_jump){ 5470 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5471 }; 5472 split_actions[actions_n - 2] = (struct rte_flow_action){ 5473 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5474 .conf = jump, 5475 }; 5476 split_actions[actions_n - 1] = (struct rte_flow_action){ 5477 .type = RTE_FLOW_ACTION_TYPE_END, 5478 }; 5479 return flow_id; 5480 } 5481 5482 /** 5483 * Extend the given action list for Tx metadata copy. 5484 * 5485 * Copy the given action list to the ext_actions and add flow metadata register 5486 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5487 * 5488 * @param[out] ext_actions 5489 * Pointer to the extended action list. 5490 * @param[in] actions 5491 * Pointer to the list of actions. 5492 * @param[in] actions_n 5493 * Number of actions in the list. 5494 * @param[out] error 5495 * Perform verbose error reporting if not NULL. 5496 * @param[in] encap_idx 5497 * The encap action index. 5498 * 5499 * @return 5500 * 0 on success, negative value otherwise 5501 */ 5502 static int 5503 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5504 struct rte_flow_action *ext_actions, 5505 const struct rte_flow_action *actions, 5506 int actions_n, struct rte_flow_error *error, 5507 int encap_idx) 5508 { 5509 struct mlx5_flow_action_copy_mreg *cp_mreg = 5510 (struct mlx5_flow_action_copy_mreg *) 5511 (ext_actions + actions_n + 1); 5512 int ret; 5513 5514 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5515 if (ret < 0) 5516 return ret; 5517 cp_mreg->dst = ret; 5518 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5519 if (ret < 0) 5520 return ret; 5521 cp_mreg->src = ret; 5522 if (encap_idx != 0) 5523 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5524 if (encap_idx == actions_n - 1) { 5525 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5526 .type = (enum rte_flow_action_type) 5527 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5528 .conf = cp_mreg, 5529 }; 5530 ext_actions[actions_n] = (struct rte_flow_action){ 5531 .type = RTE_FLOW_ACTION_TYPE_END, 5532 }; 5533 } else { 5534 ext_actions[encap_idx] = (struct rte_flow_action){ 5535 .type = (enum rte_flow_action_type) 5536 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5537 .conf = cp_mreg, 5538 }; 5539 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5540 sizeof(*ext_actions) * (actions_n - encap_idx)); 5541 } 5542 return 0; 5543 } 5544 5545 /** 5546 * Check the match action from the action list. 5547 * 5548 * @param[in] actions 5549 * Pointer to the list of actions. 5550 * @param[in] attr 5551 * Flow rule attributes. 5552 * @param[in] action 5553 * The action to be check if exist. 5554 * @param[out] match_action_pos 5555 * Pointer to the position of the matched action if exists, otherwise is -1. 5556 * @param[out] qrss_action_pos 5557 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5558 * @param[out] modify_after_mirror 5559 * Pointer to the flag of modify action after FDB mirroring. 5560 * 5561 * @return 5562 * > 0 the total number of actions. 5563 * 0 if not found match action in action list. 5564 */ 5565 static int 5566 flow_check_match_action(const struct rte_flow_action actions[], 5567 const struct rte_flow_attr *attr, 5568 enum rte_flow_action_type action, 5569 int *match_action_pos, int *qrss_action_pos, 5570 int *modify_after_mirror) 5571 { 5572 const struct rte_flow_action_sample *sample; 5573 const struct rte_flow_action_raw_decap *decap; 5574 int actions_n = 0; 5575 uint32_t ratio = 0; 5576 int sub_type = 0; 5577 int flag = 0; 5578 int fdb_mirror = 0; 5579 5580 *match_action_pos = -1; 5581 *qrss_action_pos = -1; 5582 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5583 if (actions->type == action) { 5584 flag = 1; 5585 *match_action_pos = actions_n; 5586 } 5587 switch (actions->type) { 5588 case RTE_FLOW_ACTION_TYPE_QUEUE: 5589 case RTE_FLOW_ACTION_TYPE_RSS: 5590 *qrss_action_pos = actions_n; 5591 break; 5592 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5593 sample = actions->conf; 5594 ratio = sample->ratio; 5595 sub_type = ((const struct rte_flow_action *) 5596 (sample->actions))->type; 5597 if (ratio == 1 && attr->transfer) 5598 fdb_mirror = 1; 5599 break; 5600 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5601 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5602 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5603 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5604 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5605 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5606 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5607 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5608 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5609 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5610 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5611 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5612 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5613 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5614 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5615 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5616 case RTE_FLOW_ACTION_TYPE_FLAG: 5617 case RTE_FLOW_ACTION_TYPE_MARK: 5618 case RTE_FLOW_ACTION_TYPE_SET_META: 5619 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5620 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5621 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5622 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5623 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5624 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5625 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5626 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5627 case RTE_FLOW_ACTION_TYPE_METER: 5628 if (fdb_mirror) 5629 *modify_after_mirror = 1; 5630 break; 5631 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5632 decap = actions->conf; 5633 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5634 ; 5635 actions_n++; 5636 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5637 const struct rte_flow_action_raw_encap *encap = 5638 actions->conf; 5639 if (decap->size <= 5640 MLX5_ENCAPSULATION_DECISION_SIZE && 5641 encap->size > 5642 MLX5_ENCAPSULATION_DECISION_SIZE) 5643 /* L3 encap. */ 5644 break; 5645 } 5646 if (fdb_mirror) 5647 *modify_after_mirror = 1; 5648 break; 5649 default: 5650 break; 5651 } 5652 actions_n++; 5653 } 5654 if (flag && fdb_mirror && !*modify_after_mirror) { 5655 /* FDB mirroring uses the destination array to implement 5656 * instead of FLOW_SAMPLER object. 5657 */ 5658 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5659 flag = 0; 5660 } 5661 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5662 return flag ? actions_n + 1 : 0; 5663 } 5664 5665 #define SAMPLE_SUFFIX_ITEM 2 5666 5667 /** 5668 * Split the sample flow. 5669 * 5670 * As sample flow will split to two sub flow, sample flow with 5671 * sample action, the other actions will move to new suffix flow. 5672 * 5673 * Also add unique tag id with tag action in the sample flow, 5674 * the same tag id will be as match in the suffix flow. 5675 * 5676 * @param dev 5677 * Pointer to Ethernet device. 5678 * @param[in] add_tag 5679 * Add extra tag action flag. 5680 * @param[out] sfx_items 5681 * Suffix flow match items (list terminated by the END pattern item). 5682 * @param[in] actions 5683 * Associated actions (list terminated by the END action). 5684 * @param[out] actions_sfx 5685 * Suffix flow actions. 5686 * @param[out] actions_pre 5687 * Prefix flow actions. 5688 * @param[in] actions_n 5689 * The total number of actions. 5690 * @param[in] sample_action_pos 5691 * The sample action position. 5692 * @param[in] qrss_action_pos 5693 * The Queue/RSS action position. 5694 * @param[in] jump_table 5695 * Add extra jump action flag. 5696 * @param[out] error 5697 * Perform verbose error reporting if not NULL. 5698 * 5699 * @return 5700 * 0 on success, or unique flow_id, a negative errno value 5701 * otherwise and rte_errno is set. 5702 */ 5703 static int 5704 flow_sample_split_prep(struct rte_eth_dev *dev, 5705 int add_tag, 5706 struct rte_flow_item sfx_items[], 5707 const struct rte_flow_action actions[], 5708 struct rte_flow_action actions_sfx[], 5709 struct rte_flow_action actions_pre[], 5710 int actions_n, 5711 int sample_action_pos, 5712 int qrss_action_pos, 5713 int jump_table, 5714 struct rte_flow_error *error) 5715 { 5716 struct mlx5_priv *priv = dev->data->dev_private; 5717 struct mlx5_rte_flow_action_set_tag *set_tag; 5718 struct mlx5_rte_flow_item_tag *tag_spec; 5719 struct mlx5_rte_flow_item_tag *tag_mask; 5720 struct rte_flow_action_jump *jump_action; 5721 uint32_t tag_id = 0; 5722 int index; 5723 int append_index = 0; 5724 int ret; 5725 5726 if (sample_action_pos < 0) 5727 return rte_flow_error_set(error, EINVAL, 5728 RTE_FLOW_ERROR_TYPE_ACTION, 5729 NULL, "invalid position of sample " 5730 "action in list"); 5731 /* Prepare the actions for prefix and suffix flow. */ 5732 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5733 index = qrss_action_pos; 5734 /* Put the preceding the Queue/RSS action into prefix flow. */ 5735 if (index != 0) 5736 memcpy(actions_pre, actions, 5737 sizeof(struct rte_flow_action) * index); 5738 /* Put others preceding the sample action into prefix flow. */ 5739 if (sample_action_pos > index + 1) 5740 memcpy(actions_pre + index, actions + index + 1, 5741 sizeof(struct rte_flow_action) * 5742 (sample_action_pos - index - 1)); 5743 index = sample_action_pos - 1; 5744 /* Put Queue/RSS action into Suffix flow. */ 5745 memcpy(actions_sfx, actions + qrss_action_pos, 5746 sizeof(struct rte_flow_action)); 5747 actions_sfx++; 5748 } else { 5749 index = sample_action_pos; 5750 if (index != 0) 5751 memcpy(actions_pre, actions, 5752 sizeof(struct rte_flow_action) * index); 5753 } 5754 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5755 * For CX6DX and above, metadata registers Cx preserve their value, 5756 * add an extra tag action for NIC-RX and E-Switch Domain. 5757 */ 5758 if (add_tag) { 5759 /* Prepare the prefix tag action. */ 5760 append_index++; 5761 set_tag = (void *)(actions_pre + actions_n + append_index); 5762 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5763 if (ret < 0) 5764 return ret; 5765 mlx5_ipool_malloc(priv->sh->ipool 5766 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5767 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5768 .id = ret, 5769 .data = tag_id, 5770 }; 5771 /* Prepare the suffix subflow items. */ 5772 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5773 tag_spec->data = tag_id; 5774 tag_spec->id = set_tag->id; 5775 tag_mask = tag_spec + 1; 5776 tag_mask->data = UINT32_MAX; 5777 sfx_items[0] = (struct rte_flow_item){ 5778 .type = (enum rte_flow_item_type) 5779 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5780 .spec = tag_spec, 5781 .last = NULL, 5782 .mask = tag_mask, 5783 }; 5784 sfx_items[1] = (struct rte_flow_item){ 5785 .type = (enum rte_flow_item_type) 5786 RTE_FLOW_ITEM_TYPE_END, 5787 }; 5788 /* Prepare the tag action in prefix subflow. */ 5789 actions_pre[index++] = 5790 (struct rte_flow_action){ 5791 .type = (enum rte_flow_action_type) 5792 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5793 .conf = set_tag, 5794 }; 5795 } 5796 memcpy(actions_pre + index, actions + sample_action_pos, 5797 sizeof(struct rte_flow_action)); 5798 index += 1; 5799 /* For the modify action after the sample action in E-Switch mirroring, 5800 * Add the extra jump action in prefix subflow and jump into the next 5801 * table, then do the modify action in the new table. 5802 */ 5803 if (jump_table) { 5804 /* Prepare the prefix jump action. */ 5805 append_index++; 5806 jump_action = (void *)(actions_pre + actions_n + append_index); 5807 jump_action->group = jump_table; 5808 actions_pre[index++] = 5809 (struct rte_flow_action){ 5810 .type = (enum rte_flow_action_type) 5811 RTE_FLOW_ACTION_TYPE_JUMP, 5812 .conf = jump_action, 5813 }; 5814 } 5815 actions_pre[index] = (struct rte_flow_action){ 5816 .type = (enum rte_flow_action_type) 5817 RTE_FLOW_ACTION_TYPE_END, 5818 }; 5819 /* Put the actions after sample into Suffix flow. */ 5820 memcpy(actions_sfx, actions + sample_action_pos + 1, 5821 sizeof(struct rte_flow_action) * 5822 (actions_n - sample_action_pos - 1)); 5823 return tag_id; 5824 } 5825 5826 /** 5827 * The splitting for metadata feature. 5828 * 5829 * - Q/RSS action on NIC Rx should be split in order to pass by 5830 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5831 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5832 * 5833 * - All the actions on NIC Tx should have a mreg copy action to 5834 * copy reg_a from WQE to reg_c[0]. 5835 * 5836 * @param dev 5837 * Pointer to Ethernet device. 5838 * @param[in] flow 5839 * Parent flow structure pointer. 5840 * @param[in] attr 5841 * Flow rule attributes. 5842 * @param[in] items 5843 * Pattern specification (list terminated by the END pattern item). 5844 * @param[in] actions 5845 * Associated actions (list terminated by the END action). 5846 * @param[in] flow_split_info 5847 * Pointer to flow split info structure. 5848 * @param[out] error 5849 * Perform verbose error reporting if not NULL. 5850 * @return 5851 * 0 on success, negative value otherwise 5852 */ 5853 static int 5854 flow_create_split_metadata(struct rte_eth_dev *dev, 5855 struct rte_flow *flow, 5856 const struct rte_flow_attr *attr, 5857 const struct rte_flow_item items[], 5858 const struct rte_flow_action actions[], 5859 struct mlx5_flow_split_info *flow_split_info, 5860 struct rte_flow_error *error) 5861 { 5862 struct mlx5_priv *priv = dev->data->dev_private; 5863 struct mlx5_sh_config *config = &priv->sh->config; 5864 const struct rte_flow_action *qrss = NULL; 5865 struct rte_flow_action *ext_actions = NULL; 5866 struct mlx5_flow *dev_flow = NULL; 5867 uint32_t qrss_id = 0; 5868 int mtr_sfx = 0; 5869 size_t act_size; 5870 int actions_n; 5871 int encap_idx; 5872 int ret; 5873 5874 /* Check whether extensive metadata feature is engaged. */ 5875 if (!config->dv_flow_en || 5876 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5877 !mlx5_flow_ext_mreg_supported(dev)) 5878 return flow_create_split_inner(dev, flow, NULL, attr, items, 5879 actions, flow_split_info, error); 5880 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5881 &encap_idx); 5882 if (qrss) { 5883 /* Exclude hairpin flows from splitting. */ 5884 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5885 const struct rte_flow_action_queue *queue; 5886 5887 queue = qrss->conf; 5888 if (mlx5_rxq_get_type(dev, queue->index) == 5889 MLX5_RXQ_TYPE_HAIRPIN) 5890 qrss = NULL; 5891 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5892 const struct rte_flow_action_rss *rss; 5893 5894 rss = qrss->conf; 5895 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5896 MLX5_RXQ_TYPE_HAIRPIN) 5897 qrss = NULL; 5898 } 5899 } 5900 if (qrss) { 5901 /* Check if it is in meter suffix table. */ 5902 mtr_sfx = attr->group == (attr->transfer ? 5903 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5904 MLX5_FLOW_TABLE_LEVEL_METER); 5905 /* 5906 * Q/RSS action on NIC Rx should be split in order to pass by 5907 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5908 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5909 */ 5910 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5911 sizeof(struct rte_flow_action_set_tag) + 5912 sizeof(struct rte_flow_action_jump); 5913 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5914 SOCKET_ID_ANY); 5915 if (!ext_actions) 5916 return rte_flow_error_set(error, ENOMEM, 5917 RTE_FLOW_ERROR_TYPE_ACTION, 5918 NULL, "no memory to split " 5919 "metadata flow"); 5920 /* 5921 * Create the new actions list with removed Q/RSS action 5922 * and appended set tag and jump to register copy table 5923 * (RX_CP_TBL). We should preallocate unique tag ID here 5924 * in advance, because it is needed for set tag action. 5925 */ 5926 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5927 qrss, actions_n, 5928 mtr_sfx, error); 5929 if (!mtr_sfx && !qrss_id) { 5930 ret = -rte_errno; 5931 goto exit; 5932 } 5933 } else if (attr->egress && !attr->transfer) { 5934 /* 5935 * All the actions on NIC Tx should have a metadata register 5936 * copy action to copy reg_a from WQE to reg_c[meta] 5937 */ 5938 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5939 sizeof(struct mlx5_flow_action_copy_mreg); 5940 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5941 SOCKET_ID_ANY); 5942 if (!ext_actions) 5943 return rte_flow_error_set(error, ENOMEM, 5944 RTE_FLOW_ERROR_TYPE_ACTION, 5945 NULL, "no memory to split " 5946 "metadata flow"); 5947 /* Create the action list appended with copy register. */ 5948 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5949 actions_n, error, encap_idx); 5950 if (ret < 0) 5951 goto exit; 5952 } 5953 /* Add the unmodified original or prefix subflow. */ 5954 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5955 items, ext_actions ? ext_actions : 5956 actions, flow_split_info, error); 5957 if (ret < 0) 5958 goto exit; 5959 MLX5_ASSERT(dev_flow); 5960 if (qrss) { 5961 const struct rte_flow_attr q_attr = { 5962 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5963 .ingress = 1, 5964 }; 5965 /* Internal PMD action to set register. */ 5966 struct mlx5_rte_flow_item_tag q_tag_spec = { 5967 .data = qrss_id, 5968 .id = REG_NON, 5969 }; 5970 struct rte_flow_item q_items[] = { 5971 { 5972 .type = (enum rte_flow_item_type) 5973 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5974 .spec = &q_tag_spec, 5975 .last = NULL, 5976 .mask = NULL, 5977 }, 5978 { 5979 .type = RTE_FLOW_ITEM_TYPE_END, 5980 }, 5981 }; 5982 struct rte_flow_action q_actions[] = { 5983 { 5984 .type = qrss->type, 5985 .conf = qrss->conf, 5986 }, 5987 { 5988 .type = RTE_FLOW_ACTION_TYPE_END, 5989 }, 5990 }; 5991 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5992 5993 /* 5994 * Configure the tag item only if there is no meter subflow. 5995 * Since tag is already marked in the meter suffix subflow 5996 * we can just use the meter suffix items as is. 5997 */ 5998 if (qrss_id) { 5999 /* Not meter subflow. */ 6000 MLX5_ASSERT(!mtr_sfx); 6001 /* 6002 * Put unique id in prefix flow due to it is destroyed 6003 * after suffix flow and id will be freed after there 6004 * is no actual flows with this id and identifier 6005 * reallocation becomes possible (for example, for 6006 * other flows in other threads). 6007 */ 6008 dev_flow->handle->split_flow_id = qrss_id; 6009 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6010 error); 6011 if (ret < 0) 6012 goto exit; 6013 q_tag_spec.id = ret; 6014 } 6015 dev_flow = NULL; 6016 /* Add suffix subflow to execute Q/RSS. */ 6017 flow_split_info->prefix_layers = layers; 6018 flow_split_info->prefix_mark = 0; 6019 flow_split_info->table_id = 0; 6020 ret = flow_create_split_inner(dev, flow, &dev_flow, 6021 &q_attr, mtr_sfx ? items : 6022 q_items, q_actions, 6023 flow_split_info, error); 6024 if (ret < 0) 6025 goto exit; 6026 /* qrss ID should be freed if failed. */ 6027 qrss_id = 0; 6028 MLX5_ASSERT(dev_flow); 6029 } 6030 6031 exit: 6032 /* 6033 * We do not destroy the partially created sub_flows in case of error. 6034 * These ones are included into parent flow list and will be destroyed 6035 * by flow_drv_destroy. 6036 */ 6037 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6038 qrss_id); 6039 mlx5_free(ext_actions); 6040 return ret; 6041 } 6042 6043 /** 6044 * Create meter internal drop flow with the original pattern. 6045 * 6046 * @param dev 6047 * Pointer to Ethernet device. 6048 * @param[in] flow 6049 * Parent flow structure pointer. 6050 * @param[in] attr 6051 * Flow rule attributes. 6052 * @param[in] items 6053 * Pattern specification (list terminated by the END pattern item). 6054 * @param[in] flow_split_info 6055 * Pointer to flow split info structure. 6056 * @param[in] fm 6057 * Pointer to flow meter structure. 6058 * @param[out] error 6059 * Perform verbose error reporting if not NULL. 6060 * @return 6061 * 0 on success, negative value otherwise 6062 */ 6063 static uint32_t 6064 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6065 struct rte_flow *flow, 6066 const struct rte_flow_attr *attr, 6067 const struct rte_flow_item items[], 6068 struct mlx5_flow_split_info *flow_split_info, 6069 struct mlx5_flow_meter_info *fm, 6070 struct rte_flow_error *error) 6071 { 6072 struct mlx5_flow *dev_flow = NULL; 6073 struct rte_flow_attr drop_attr = *attr; 6074 struct rte_flow_action drop_actions[3]; 6075 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6076 6077 MLX5_ASSERT(fm->drop_cnt); 6078 drop_actions[0].type = 6079 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6080 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6081 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6082 drop_actions[1].conf = NULL; 6083 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6084 drop_actions[2].conf = NULL; 6085 drop_split_info.external = false; 6086 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6087 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6088 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6089 return flow_create_split_inner(dev, flow, &dev_flow, 6090 &drop_attr, items, drop_actions, 6091 &drop_split_info, error); 6092 } 6093 6094 /** 6095 * The splitting for meter feature. 6096 * 6097 * - The meter flow will be split to two flows as prefix and 6098 * suffix flow. The packets make sense only it pass the prefix 6099 * meter action. 6100 * 6101 * - Reg_C_5 is used for the packet to match betweend prefix and 6102 * suffix flow. 6103 * 6104 * @param dev 6105 * Pointer to Ethernet device. 6106 * @param[in] flow 6107 * Parent flow structure pointer. 6108 * @param[in] attr 6109 * Flow rule attributes. 6110 * @param[in] items 6111 * Pattern specification (list terminated by the END pattern item). 6112 * @param[in] actions 6113 * Associated actions (list terminated by the END action). 6114 * @param[in] flow_split_info 6115 * Pointer to flow split info structure. 6116 * @param[out] error 6117 * Perform verbose error reporting if not NULL. 6118 * @return 6119 * 0 on success, negative value otherwise 6120 */ 6121 static int 6122 flow_create_split_meter(struct rte_eth_dev *dev, 6123 struct rte_flow *flow, 6124 const struct rte_flow_attr *attr, 6125 const struct rte_flow_item items[], 6126 const struct rte_flow_action actions[], 6127 struct mlx5_flow_split_info *flow_split_info, 6128 struct rte_flow_error *error) 6129 { 6130 struct mlx5_priv *priv = dev->data->dev_private; 6131 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6132 struct rte_flow_action *sfx_actions = NULL; 6133 struct rte_flow_action *pre_actions = NULL; 6134 struct rte_flow_item *sfx_items = NULL; 6135 struct mlx5_flow *dev_flow = NULL; 6136 struct rte_flow_attr sfx_attr = *attr; 6137 struct mlx5_flow_meter_info *fm = NULL; 6138 uint8_t skip_scale_restore; 6139 bool has_mtr = false; 6140 bool has_modify = false; 6141 bool set_mtr_reg = true; 6142 bool is_mtr_hierarchy = false; 6143 uint32_t meter_id = 0; 6144 uint32_t mtr_idx = 0; 6145 uint32_t mtr_flow_id = 0; 6146 size_t act_size; 6147 size_t item_size; 6148 int actions_n = 0; 6149 int ret = 0; 6150 6151 if (priv->mtr_en) 6152 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6153 &has_modify, &meter_id); 6154 if (has_mtr) { 6155 if (flow->meter) { 6156 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6157 if (!fm) 6158 return rte_flow_error_set(error, EINVAL, 6159 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6160 NULL, "Meter not found."); 6161 } else { 6162 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6163 if (!fm) 6164 return rte_flow_error_set(error, EINVAL, 6165 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6166 NULL, "Meter not found."); 6167 ret = mlx5_flow_meter_attach(priv, fm, 6168 &sfx_attr, error); 6169 if (ret) 6170 return -rte_errno; 6171 flow->meter = mtr_idx; 6172 } 6173 MLX5_ASSERT(wks); 6174 wks->fm = fm; 6175 if (!fm->def_policy) { 6176 wks->policy = mlx5_flow_meter_policy_find(dev, 6177 fm->policy_id, 6178 NULL); 6179 MLX5_ASSERT(wks->policy); 6180 if (wks->policy->is_hierarchy) { 6181 wks->final_policy = 6182 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6183 wks->policy); 6184 if (!wks->final_policy) 6185 return rte_flow_error_set(error, 6186 EINVAL, 6187 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6188 "Failed to find terminal policy of hierarchy."); 6189 is_mtr_hierarchy = true; 6190 } 6191 } 6192 /* 6193 * If it isn't default-policy Meter, and 6194 * 1. There's no action in flow to change 6195 * packet (modify/encap/decap etc.), OR 6196 * 2. No drop count needed for this meter. 6197 * 3. It's not meter hierarchy. 6198 * Then no need to use regC to save meter id anymore. 6199 */ 6200 if (!fm->def_policy && !is_mtr_hierarchy && 6201 (!has_modify || !fm->drop_cnt)) 6202 set_mtr_reg = false; 6203 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6204 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6205 sizeof(struct mlx5_rte_flow_action_set_tag); 6206 /* Suffix items: tag, vlan, port id, end. */ 6207 #define METER_SUFFIX_ITEM 4 6208 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6209 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6210 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6211 0, SOCKET_ID_ANY); 6212 if (!sfx_actions) 6213 return rte_flow_error_set(error, ENOMEM, 6214 RTE_FLOW_ERROR_TYPE_ACTION, 6215 NULL, "no memory to split " 6216 "meter flow"); 6217 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6218 act_size); 6219 /* There's no suffix flow for meter of non-default policy. */ 6220 if (!fm->def_policy) 6221 pre_actions = sfx_actions + 1; 6222 else 6223 pre_actions = sfx_actions + actions_n; 6224 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6225 items, sfx_items, actions, 6226 sfx_actions, pre_actions, 6227 (set_mtr_reg ? &mtr_flow_id : NULL), 6228 error); 6229 if (ret) { 6230 ret = -rte_errno; 6231 goto exit; 6232 } 6233 /* Add the prefix subflow. */ 6234 skip_scale_restore = flow_split_info->skip_scale; 6235 flow_split_info->skip_scale |= 6236 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6237 ret = flow_create_split_inner(dev, flow, &dev_flow, 6238 attr, items, pre_actions, 6239 flow_split_info, error); 6240 flow_split_info->skip_scale = skip_scale_restore; 6241 if (ret) { 6242 if (mtr_flow_id) 6243 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6244 ret = -rte_errno; 6245 goto exit; 6246 } 6247 if (mtr_flow_id) { 6248 dev_flow->handle->split_flow_id = mtr_flow_id; 6249 dev_flow->handle->is_meter_flow_id = 1; 6250 } 6251 if (!fm->def_policy) { 6252 if (!set_mtr_reg && fm->drop_cnt) 6253 ret = 6254 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6255 &sfx_attr, items, 6256 flow_split_info, 6257 fm, error); 6258 goto exit; 6259 } 6260 /* Setting the sfx group atrr. */ 6261 sfx_attr.group = sfx_attr.transfer ? 6262 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6263 MLX5_FLOW_TABLE_LEVEL_METER; 6264 flow_split_info->prefix_layers = 6265 flow_get_prefix_layer_flags(dev_flow); 6266 flow_split_info->prefix_mark |= wks->mark; 6267 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6268 } 6269 /* Add the prefix subflow. */ 6270 ret = flow_create_split_metadata(dev, flow, 6271 &sfx_attr, sfx_items ? 6272 sfx_items : items, 6273 sfx_actions ? sfx_actions : actions, 6274 flow_split_info, error); 6275 exit: 6276 if (sfx_actions) 6277 mlx5_free(sfx_actions); 6278 return ret; 6279 } 6280 6281 /** 6282 * The splitting for sample feature. 6283 * 6284 * Once Sample action is detected in the action list, the flow actions should 6285 * be split into prefix sub flow and suffix sub flow. 6286 * 6287 * The original items remain in the prefix sub flow, all actions preceding the 6288 * sample action and the sample action itself will be copied to the prefix 6289 * sub flow, the actions following the sample action will be copied to the 6290 * suffix sub flow, Queue action always be located in the suffix sub flow. 6291 * 6292 * In order to make the packet from prefix sub flow matches with suffix sub 6293 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6294 * flow uses tag item with the unique flow id. 6295 * 6296 * @param dev 6297 * Pointer to Ethernet device. 6298 * @param[in] flow 6299 * Parent flow structure pointer. 6300 * @param[in] attr 6301 * Flow rule attributes. 6302 * @param[in] items 6303 * Pattern specification (list terminated by the END pattern item). 6304 * @param[in] actions 6305 * Associated actions (list terminated by the END action). 6306 * @param[in] flow_split_info 6307 * Pointer to flow split info structure. 6308 * @param[out] error 6309 * Perform verbose error reporting if not NULL. 6310 * @return 6311 * 0 on success, negative value otherwise 6312 */ 6313 static int 6314 flow_create_split_sample(struct rte_eth_dev *dev, 6315 struct rte_flow *flow, 6316 const struct rte_flow_attr *attr, 6317 const struct rte_flow_item items[], 6318 const struct rte_flow_action actions[], 6319 struct mlx5_flow_split_info *flow_split_info, 6320 struct rte_flow_error *error) 6321 { 6322 struct mlx5_priv *priv = dev->data->dev_private; 6323 struct rte_flow_action *sfx_actions = NULL; 6324 struct rte_flow_action *pre_actions = NULL; 6325 struct rte_flow_item *sfx_items = NULL; 6326 struct mlx5_flow *dev_flow = NULL; 6327 struct rte_flow_attr sfx_attr = *attr; 6328 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6329 struct mlx5_flow_dv_sample_resource *sample_res; 6330 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6331 struct mlx5_flow_tbl_resource *sfx_tbl; 6332 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6333 #endif 6334 size_t act_size; 6335 size_t item_size; 6336 uint32_t fdb_tx = 0; 6337 int32_t tag_id = 0; 6338 int actions_n = 0; 6339 int sample_action_pos; 6340 int qrss_action_pos; 6341 int add_tag = 0; 6342 int modify_after_mirror = 0; 6343 uint16_t jump_table = 0; 6344 const uint32_t next_ft_step = 1; 6345 int ret = 0; 6346 6347 if (priv->sampler_en) 6348 actions_n = flow_check_match_action(actions, attr, 6349 RTE_FLOW_ACTION_TYPE_SAMPLE, 6350 &sample_action_pos, &qrss_action_pos, 6351 &modify_after_mirror); 6352 if (actions_n) { 6353 /* The prefix actions must includes sample, tag, end. */ 6354 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6355 + sizeof(struct mlx5_rte_flow_action_set_tag); 6356 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6357 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6358 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6359 item_size), 0, SOCKET_ID_ANY); 6360 if (!sfx_actions) 6361 return rte_flow_error_set(error, ENOMEM, 6362 RTE_FLOW_ERROR_TYPE_ACTION, 6363 NULL, "no memory to split " 6364 "sample flow"); 6365 /* The representor_id is UINT16_MAX for uplink. */ 6366 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6367 /* 6368 * When reg_c_preserve is set, metadata registers Cx preserve 6369 * their value even through packet duplication. 6370 */ 6371 add_tag = (!fdb_tx || 6372 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6373 if (add_tag) 6374 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6375 + act_size); 6376 if (modify_after_mirror) 6377 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6378 next_ft_step; 6379 pre_actions = sfx_actions + actions_n; 6380 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6381 actions, sfx_actions, 6382 pre_actions, actions_n, 6383 sample_action_pos, 6384 qrss_action_pos, jump_table, 6385 error); 6386 if (tag_id < 0 || (add_tag && !tag_id)) { 6387 ret = -rte_errno; 6388 goto exit; 6389 } 6390 if (modify_after_mirror) 6391 flow_split_info->skip_scale = 6392 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6393 /* Add the prefix subflow. */ 6394 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6395 items, pre_actions, 6396 flow_split_info, error); 6397 if (ret) { 6398 ret = -rte_errno; 6399 goto exit; 6400 } 6401 dev_flow->handle->split_flow_id = tag_id; 6402 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6403 if (!modify_after_mirror) { 6404 /* Set the sfx group attr. */ 6405 sample_res = (struct mlx5_flow_dv_sample_resource *) 6406 dev_flow->dv.sample_res; 6407 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6408 sample_res->normal_path_tbl; 6409 sfx_tbl_data = container_of(sfx_tbl, 6410 struct mlx5_flow_tbl_data_entry, 6411 tbl); 6412 sfx_attr.group = sfx_attr.transfer ? 6413 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6414 } else { 6415 MLX5_ASSERT(attr->transfer); 6416 sfx_attr.group = jump_table; 6417 } 6418 flow_split_info->prefix_layers = 6419 flow_get_prefix_layer_flags(dev_flow); 6420 MLX5_ASSERT(wks); 6421 flow_split_info->prefix_mark |= wks->mark; 6422 /* Suffix group level already be scaled with factor, set 6423 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6424 * again in translation. 6425 */ 6426 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6427 #endif 6428 } 6429 /* Add the suffix subflow. */ 6430 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6431 sfx_items ? sfx_items : items, 6432 sfx_actions ? sfx_actions : actions, 6433 flow_split_info, error); 6434 exit: 6435 if (sfx_actions) 6436 mlx5_free(sfx_actions); 6437 return ret; 6438 } 6439 6440 /** 6441 * Split the flow to subflow set. The splitters might be linked 6442 * in the chain, like this: 6443 * flow_create_split_outer() calls: 6444 * flow_create_split_meter() calls: 6445 * flow_create_split_metadata(meter_subflow_0) calls: 6446 * flow_create_split_inner(metadata_subflow_0) 6447 * flow_create_split_inner(metadata_subflow_1) 6448 * flow_create_split_inner(metadata_subflow_2) 6449 * flow_create_split_metadata(meter_subflow_1) calls: 6450 * flow_create_split_inner(metadata_subflow_0) 6451 * flow_create_split_inner(metadata_subflow_1) 6452 * flow_create_split_inner(metadata_subflow_2) 6453 * 6454 * This provide flexible way to add new levels of flow splitting. 6455 * The all of successfully created subflows are included to the 6456 * parent flow dev_flow list. 6457 * 6458 * @param dev 6459 * Pointer to Ethernet device. 6460 * @param[in] flow 6461 * Parent flow structure pointer. 6462 * @param[in] attr 6463 * Flow rule attributes. 6464 * @param[in] items 6465 * Pattern specification (list terminated by the END pattern item). 6466 * @param[in] actions 6467 * Associated actions (list terminated by the END action). 6468 * @param[in] flow_split_info 6469 * Pointer to flow split info structure. 6470 * @param[out] error 6471 * Perform verbose error reporting if not NULL. 6472 * @return 6473 * 0 on success, negative value otherwise 6474 */ 6475 static int 6476 flow_create_split_outer(struct rte_eth_dev *dev, 6477 struct rte_flow *flow, 6478 const struct rte_flow_attr *attr, 6479 const struct rte_flow_item items[], 6480 const struct rte_flow_action actions[], 6481 struct mlx5_flow_split_info *flow_split_info, 6482 struct rte_flow_error *error) 6483 { 6484 int ret; 6485 6486 ret = flow_create_split_sample(dev, flow, attr, items, 6487 actions, flow_split_info, error); 6488 MLX5_ASSERT(ret <= 0); 6489 return ret; 6490 } 6491 6492 static inline struct mlx5_flow_tunnel * 6493 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6494 { 6495 struct mlx5_flow_tunnel *tunnel; 6496 6497 #pragma GCC diagnostic push 6498 #pragma GCC diagnostic ignored "-Wcast-qual" 6499 tunnel = (typeof(tunnel))flow->tunnel; 6500 #pragma GCC diagnostic pop 6501 6502 return tunnel; 6503 } 6504 6505 /** 6506 * Adjust flow RSS workspace if needed. 6507 * 6508 * @param wks 6509 * Pointer to thread flow work space. 6510 * @param rss_desc 6511 * Pointer to RSS descriptor. 6512 * @param[in] nrssq_num 6513 * New RSS queue number. 6514 * 6515 * @return 6516 * 0 on success, -1 otherwise and rte_errno is set. 6517 */ 6518 static int 6519 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6520 struct mlx5_flow_rss_desc *rss_desc, 6521 uint32_t nrssq_num) 6522 { 6523 if (likely(nrssq_num <= wks->rssq_num)) 6524 return 0; 6525 rss_desc->queue = realloc(rss_desc->queue, 6526 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6527 if (!rss_desc->queue) { 6528 rte_errno = ENOMEM; 6529 return -1; 6530 } 6531 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6532 return 0; 6533 } 6534 6535 /** 6536 * Create a flow and add it to @p list. 6537 * 6538 * @param dev 6539 * Pointer to Ethernet device. 6540 * @param list 6541 * Pointer to a TAILQ flow list. If this parameter NULL, 6542 * no list insertion occurred, flow is just created, 6543 * this is caller's responsibility to track the 6544 * created flow. 6545 * @param[in] attr 6546 * Flow rule attributes. 6547 * @param[in] items 6548 * Pattern specification (list terminated by the END pattern item). 6549 * @param[in] actions 6550 * Associated actions (list terminated by the END action). 6551 * @param[in] external 6552 * This flow rule is created by request external to PMD. 6553 * @param[out] error 6554 * Perform verbose error reporting if not NULL. 6555 * 6556 * @return 6557 * A flow index on success, 0 otherwise and rte_errno is set. 6558 */ 6559 static uint32_t 6560 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6561 const struct rte_flow_attr *attr, 6562 const struct rte_flow_item items[], 6563 const struct rte_flow_action original_actions[], 6564 bool external, struct rte_flow_error *error) 6565 { 6566 struct mlx5_priv *priv = dev->data->dev_private; 6567 struct rte_flow *flow = NULL; 6568 struct mlx5_flow *dev_flow; 6569 const struct rte_flow_action_rss *rss = NULL; 6570 struct mlx5_translated_action_handle 6571 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6572 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6573 union { 6574 struct mlx5_flow_expand_rss buf; 6575 uint8_t buffer[4096]; 6576 } expand_buffer; 6577 union { 6578 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6579 uint8_t buffer[2048]; 6580 } actions_rx; 6581 union { 6582 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6583 uint8_t buffer[2048]; 6584 } actions_hairpin_tx; 6585 union { 6586 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6587 uint8_t buffer[2048]; 6588 } items_tx; 6589 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6590 struct mlx5_flow_rss_desc *rss_desc; 6591 const struct rte_flow_action *p_actions_rx; 6592 uint32_t i; 6593 uint32_t idx = 0; 6594 int hairpin_flow; 6595 struct rte_flow_attr attr_tx = { .priority = 0 }; 6596 const struct rte_flow_action *actions; 6597 struct rte_flow_action *translated_actions = NULL; 6598 struct mlx5_flow_tunnel *tunnel; 6599 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6600 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6601 struct mlx5_flow_split_info flow_split_info = { 6602 .external = !!external, 6603 .skip_scale = 0, 6604 .flow_idx = 0, 6605 .prefix_mark = 0, 6606 .prefix_layers = 0, 6607 .table_id = 0 6608 }; 6609 int ret; 6610 6611 MLX5_ASSERT(wks); 6612 rss_desc = &wks->rss_desc; 6613 ret = flow_action_handles_translate(dev, original_actions, 6614 indir_actions, 6615 &indir_actions_n, 6616 &translated_actions, error); 6617 if (ret < 0) { 6618 MLX5_ASSERT(translated_actions == NULL); 6619 return 0; 6620 } 6621 actions = translated_actions ? translated_actions : original_actions; 6622 p_actions_rx = actions; 6623 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6624 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6625 external, hairpin_flow, error); 6626 if (ret < 0) 6627 goto error_before_hairpin_split; 6628 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6629 if (!flow) { 6630 rte_errno = ENOMEM; 6631 goto error_before_hairpin_split; 6632 } 6633 if (hairpin_flow > 0) { 6634 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6635 rte_errno = EINVAL; 6636 goto error_before_hairpin_split; 6637 } 6638 flow_hairpin_split(dev, actions, actions_rx.actions, 6639 actions_hairpin_tx.actions, items_tx.items, 6640 idx); 6641 p_actions_rx = actions_rx.actions; 6642 } 6643 flow_split_info.flow_idx = idx; 6644 flow->drv_type = flow_get_drv_type(dev, attr); 6645 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6646 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6647 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6648 /* RSS Action only works on NIC RX domain */ 6649 if (attr->ingress && !attr->transfer) 6650 rss = flow_get_rss_action(dev, p_actions_rx); 6651 if (rss) { 6652 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6653 return 0; 6654 /* 6655 * The following information is required by 6656 * mlx5_flow_hashfields_adjust() in advance. 6657 */ 6658 rss_desc->level = rss->level; 6659 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6660 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6661 } 6662 flow->dev_handles = 0; 6663 if (rss && rss->types) { 6664 unsigned int graph_root; 6665 6666 graph_root = find_graph_root(rss->level); 6667 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6668 items, rss->types, 6669 mlx5_support_expansion, graph_root); 6670 MLX5_ASSERT(ret > 0 && 6671 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6672 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6673 for (i = 0; i < buf->entries; ++i) 6674 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6675 } 6676 } else { 6677 buf->entries = 1; 6678 buf->entry[0].pattern = (void *)(uintptr_t)items; 6679 } 6680 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6681 indir_actions_n); 6682 for (i = 0; i < buf->entries; ++i) { 6683 /* Initialize flow split data. */ 6684 flow_split_info.prefix_layers = 0; 6685 flow_split_info.prefix_mark = 0; 6686 flow_split_info.skip_scale = 0; 6687 /* 6688 * The splitter may create multiple dev_flows, 6689 * depending on configuration. In the simplest 6690 * case it just creates unmodified original flow. 6691 */ 6692 ret = flow_create_split_outer(dev, flow, attr, 6693 buf->entry[i].pattern, 6694 p_actions_rx, &flow_split_info, 6695 error); 6696 if (ret < 0) 6697 goto error; 6698 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6699 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6700 p_actions_rx, 6701 idx, 6702 wks->flows[0].tunnel, 6703 &default_miss_ctx, 6704 error); 6705 if (ret < 0) { 6706 mlx5_free(default_miss_ctx.queue); 6707 goto error; 6708 } 6709 } 6710 } 6711 /* Create the tx flow. */ 6712 if (hairpin_flow) { 6713 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6714 attr_tx.ingress = 0; 6715 attr_tx.egress = 1; 6716 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6717 actions_hairpin_tx.actions, 6718 idx, error); 6719 if (!dev_flow) 6720 goto error; 6721 dev_flow->flow = flow; 6722 dev_flow->external = 0; 6723 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6724 dev_flow->handle, next); 6725 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6726 items_tx.items, 6727 actions_hairpin_tx.actions, error); 6728 if (ret < 0) 6729 goto error; 6730 } 6731 /* 6732 * Update the metadata register copy table. If extensive 6733 * metadata feature is enabled and registers are supported 6734 * we might create the extra rte_flow for each unique 6735 * MARK/FLAG action ID. 6736 * 6737 * The table is updated for ingress Flows only, because 6738 * the egress Flows belong to the different device and 6739 * copy table should be updated in peer NIC Rx domain. 6740 */ 6741 if (attr->ingress && 6742 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6743 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6744 if (ret) 6745 goto error; 6746 } 6747 /* 6748 * If the flow is external (from application) OR device is started, 6749 * OR mreg discover, then apply immediately. 6750 */ 6751 if (external || dev->data->dev_started || 6752 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6753 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6754 ret = flow_drv_apply(dev, flow, error); 6755 if (ret < 0) 6756 goto error; 6757 } 6758 flow->type = type; 6759 flow_rxq_flags_set(dev, flow); 6760 rte_free(translated_actions); 6761 tunnel = flow_tunnel_from_rule(wks->flows); 6762 if (tunnel) { 6763 flow->tunnel = 1; 6764 flow->tunnel_id = tunnel->tunnel_id; 6765 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6766 mlx5_free(default_miss_ctx.queue); 6767 } 6768 mlx5_flow_pop_thread_workspace(); 6769 return idx; 6770 error: 6771 MLX5_ASSERT(flow); 6772 ret = rte_errno; /* Save rte_errno before cleanup. */ 6773 flow_mreg_del_copy_action(dev, flow); 6774 flow_drv_destroy(dev, flow); 6775 if (rss_desc->shared_rss) 6776 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6777 mlx5_ipool_get 6778 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6779 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6780 mlx5_ipool_free(priv->flows[type], idx); 6781 rte_errno = ret; /* Restore rte_errno. */ 6782 ret = rte_errno; 6783 rte_errno = ret; 6784 mlx5_flow_pop_thread_workspace(); 6785 error_before_hairpin_split: 6786 rte_free(translated_actions); 6787 return 0; 6788 } 6789 6790 /** 6791 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6792 * incoming packets to table 1. 6793 * 6794 * Other flow rules, requested for group n, will be created in 6795 * e-switch table n+1. 6796 * Jump action to e-switch group n will be created to group n+1. 6797 * 6798 * Used when working in switchdev mode, to utilise advantages of table 1 6799 * and above. 6800 * 6801 * @param dev 6802 * Pointer to Ethernet device. 6803 * 6804 * @return 6805 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6806 */ 6807 struct rte_flow * 6808 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6809 { 6810 const struct rte_flow_attr attr = { 6811 .group = 0, 6812 .priority = 0, 6813 .ingress = 1, 6814 .egress = 0, 6815 .transfer = 1, 6816 }; 6817 const struct rte_flow_item pattern = { 6818 .type = RTE_FLOW_ITEM_TYPE_END, 6819 }; 6820 struct rte_flow_action_jump jump = { 6821 .group = 1, 6822 }; 6823 const struct rte_flow_action actions[] = { 6824 { 6825 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6826 .conf = &jump, 6827 }, 6828 { 6829 .type = RTE_FLOW_ACTION_TYPE_END, 6830 }, 6831 }; 6832 struct rte_flow_error error; 6833 6834 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6835 &attr, &pattern, 6836 actions, false, &error); 6837 } 6838 6839 /** 6840 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6841 * and sq number, directs all packets to peer vport. 6842 * 6843 * @param dev 6844 * Pointer to Ethernet device. 6845 * @param txq 6846 * Txq index. 6847 * 6848 * @return 6849 * Flow ID on success, 0 otherwise and rte_errno is set. 6850 */ 6851 uint32_t 6852 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6853 { 6854 struct rte_flow_attr attr = { 6855 .group = 0, 6856 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6857 .ingress = 1, 6858 .egress = 0, 6859 .transfer = 1, 6860 }; 6861 struct rte_flow_item_port_id port_spec = { 6862 .id = MLX5_PORT_ESW_MGR, 6863 }; 6864 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6865 .queue = txq, 6866 }; 6867 struct rte_flow_item pattern[] = { 6868 { 6869 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6870 .spec = &port_spec, 6871 }, 6872 { 6873 .type = (enum rte_flow_item_type) 6874 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6875 .spec = &txq_spec, 6876 }, 6877 { 6878 .type = RTE_FLOW_ITEM_TYPE_END, 6879 }, 6880 }; 6881 struct rte_flow_action_jump jump = { 6882 .group = 1, 6883 }; 6884 struct rte_flow_action_port_id port = { 6885 .id = dev->data->port_id, 6886 }; 6887 struct rte_flow_action actions[] = { 6888 { 6889 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6890 .conf = &jump, 6891 }, 6892 { 6893 .type = RTE_FLOW_ACTION_TYPE_END, 6894 }, 6895 }; 6896 struct rte_flow_error error; 6897 6898 /* 6899 * Creates group 0, highest priority jump flow. 6900 * Matches txq to bypass kernel packets. 6901 */ 6902 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 6903 false, &error) == 0) 6904 return 0; 6905 /* Create group 1, lowest priority redirect flow for txq. */ 6906 attr.group = 1; 6907 actions[0].conf = &port; 6908 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 6909 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 6910 actions, false, &error); 6911 } 6912 6913 /** 6914 * Validate a flow supported by the NIC. 6915 * 6916 * @see rte_flow_validate() 6917 * @see rte_flow_ops 6918 */ 6919 int 6920 mlx5_flow_validate(struct rte_eth_dev *dev, 6921 const struct rte_flow_attr *attr, 6922 const struct rte_flow_item items[], 6923 const struct rte_flow_action original_actions[], 6924 struct rte_flow_error *error) 6925 { 6926 int hairpin_flow; 6927 struct mlx5_translated_action_handle 6928 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6929 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6930 const struct rte_flow_action *actions; 6931 struct rte_flow_action *translated_actions = NULL; 6932 int ret = flow_action_handles_translate(dev, original_actions, 6933 indir_actions, 6934 &indir_actions_n, 6935 &translated_actions, error); 6936 6937 if (ret) 6938 return ret; 6939 actions = translated_actions ? translated_actions : original_actions; 6940 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6941 ret = flow_drv_validate(dev, attr, items, actions, 6942 true, hairpin_flow, error); 6943 rte_free(translated_actions); 6944 return ret; 6945 } 6946 6947 /** 6948 * Create a flow. 6949 * 6950 * @see rte_flow_create() 6951 * @see rte_flow_ops 6952 */ 6953 struct rte_flow * 6954 mlx5_flow_create(struct rte_eth_dev *dev, 6955 const struct rte_flow_attr *attr, 6956 const struct rte_flow_item items[], 6957 const struct rte_flow_action actions[], 6958 struct rte_flow_error *error) 6959 { 6960 struct mlx5_priv *priv = dev->data->dev_private; 6961 6962 if (priv->sh->config.dv_flow_en == 2) { 6963 rte_flow_error_set(error, ENOTSUP, 6964 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6965 NULL, 6966 "Flow non-Q creation not supported"); 6967 return NULL; 6968 } 6969 /* 6970 * If the device is not started yet, it is not allowed to created a 6971 * flow from application. PMD default flows and traffic control flows 6972 * are not affected. 6973 */ 6974 if (unlikely(!dev->data->dev_started)) { 6975 DRV_LOG(DEBUG, "port %u is not started when " 6976 "inserting a flow", dev->data->port_id); 6977 rte_flow_error_set(error, ENODEV, 6978 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6979 NULL, 6980 "port not started"); 6981 return NULL; 6982 } 6983 6984 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6985 attr, items, actions, 6986 true, error); 6987 } 6988 6989 /** 6990 * Destroy a flow in a list. 6991 * 6992 * @param dev 6993 * Pointer to Ethernet device. 6994 * @param[in] flow_idx 6995 * Index of flow to destroy. 6996 */ 6997 static void 6998 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6999 uint32_t flow_idx) 7000 { 7001 struct mlx5_priv *priv = dev->data->dev_private; 7002 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7003 7004 if (!flow) 7005 return; 7006 MLX5_ASSERT(flow->type == type); 7007 /* 7008 * Update RX queue flags only if port is started, otherwise it is 7009 * already clean. 7010 */ 7011 if (dev->data->dev_started) 7012 flow_rxq_flags_trim(dev, flow); 7013 flow_drv_destroy(dev, flow); 7014 if (flow->tunnel) { 7015 struct mlx5_flow_tunnel *tunnel; 7016 7017 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7018 RTE_VERIFY(tunnel); 7019 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7020 mlx5_flow_tunnel_free(dev, tunnel); 7021 } 7022 flow_mreg_del_copy_action(dev, flow); 7023 mlx5_ipool_free(priv->flows[type], flow_idx); 7024 } 7025 7026 /** 7027 * Destroy all flows. 7028 * 7029 * @param dev 7030 * Pointer to Ethernet device. 7031 * @param type 7032 * Flow type to be flushed. 7033 * @param active 7034 * If flushing is called actively. 7035 */ 7036 void 7037 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7038 bool active) 7039 { 7040 struct mlx5_priv *priv = dev->data->dev_private; 7041 uint32_t num_flushed = 0, fidx = 1; 7042 struct rte_flow *flow; 7043 7044 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7045 if (priv->sh->config.dv_flow_en == 2 && 7046 type == MLX5_FLOW_TYPE_GEN) { 7047 flow_hw_q_flow_flush(dev, NULL); 7048 return; 7049 } 7050 #endif 7051 7052 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7053 flow_list_destroy(dev, type, fidx); 7054 num_flushed++; 7055 } 7056 if (active) { 7057 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7058 dev->data->port_id, num_flushed); 7059 } 7060 } 7061 7062 /** 7063 * Stop all default actions for flows. 7064 * 7065 * @param dev 7066 * Pointer to Ethernet device. 7067 */ 7068 void 7069 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7070 { 7071 flow_mreg_del_default_copy_action(dev); 7072 flow_rxq_flags_clear(dev); 7073 } 7074 7075 /** 7076 * Start all default actions for flows. 7077 * 7078 * @param dev 7079 * Pointer to Ethernet device. 7080 * @return 7081 * 0 on success, a negative errno value otherwise and rte_errno is set. 7082 */ 7083 int 7084 mlx5_flow_start_default(struct rte_eth_dev *dev) 7085 { 7086 struct rte_flow_error error; 7087 7088 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7089 return flow_mreg_add_default_copy_action(dev, &error); 7090 } 7091 7092 /** 7093 * Release key of thread specific flow workspace data. 7094 */ 7095 void 7096 flow_release_workspace(void *data) 7097 { 7098 struct mlx5_flow_workspace *wks = data; 7099 struct mlx5_flow_workspace *next; 7100 7101 while (wks) { 7102 next = wks->next; 7103 free(wks->rss_desc.queue); 7104 free(wks); 7105 wks = next; 7106 } 7107 } 7108 7109 /** 7110 * Get thread specific current flow workspace. 7111 * 7112 * @return pointer to thread specific flow workspace data, NULL on error. 7113 */ 7114 struct mlx5_flow_workspace* 7115 mlx5_flow_get_thread_workspace(void) 7116 { 7117 struct mlx5_flow_workspace *data; 7118 7119 data = mlx5_flow_os_get_specific_workspace(); 7120 MLX5_ASSERT(data && data->inuse); 7121 if (!data || !data->inuse) 7122 DRV_LOG(ERR, "flow workspace not initialized."); 7123 return data; 7124 } 7125 7126 /** 7127 * Allocate and init new flow workspace. 7128 * 7129 * @return pointer to flow workspace data, NULL on error. 7130 */ 7131 static struct mlx5_flow_workspace* 7132 flow_alloc_thread_workspace(void) 7133 { 7134 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7135 7136 if (!data) { 7137 DRV_LOG(ERR, "Failed to allocate flow workspace " 7138 "memory."); 7139 return NULL; 7140 } 7141 data->rss_desc.queue = calloc(1, 7142 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7143 if (!data->rss_desc.queue) 7144 goto err; 7145 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7146 return data; 7147 err: 7148 free(data->rss_desc.queue); 7149 free(data); 7150 return NULL; 7151 } 7152 7153 /** 7154 * Get new thread specific flow workspace. 7155 * 7156 * If current workspace inuse, create new one and set as current. 7157 * 7158 * @return pointer to thread specific flow workspace data, NULL on error. 7159 */ 7160 static struct mlx5_flow_workspace* 7161 mlx5_flow_push_thread_workspace(void) 7162 { 7163 struct mlx5_flow_workspace *curr; 7164 struct mlx5_flow_workspace *data; 7165 7166 curr = mlx5_flow_os_get_specific_workspace(); 7167 if (!curr) { 7168 data = flow_alloc_thread_workspace(); 7169 if (!data) 7170 return NULL; 7171 } else if (!curr->inuse) { 7172 data = curr; 7173 } else if (curr->next) { 7174 data = curr->next; 7175 } else { 7176 data = flow_alloc_thread_workspace(); 7177 if (!data) 7178 return NULL; 7179 curr->next = data; 7180 data->prev = curr; 7181 } 7182 data->inuse = 1; 7183 data->flow_idx = 0; 7184 /* Set as current workspace */ 7185 if (mlx5_flow_os_set_specific_workspace(data)) 7186 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7187 return data; 7188 } 7189 7190 /** 7191 * Close current thread specific flow workspace. 7192 * 7193 * If previous workspace available, set it as current. 7194 * 7195 * @return pointer to thread specific flow workspace data, NULL on error. 7196 */ 7197 static void 7198 mlx5_flow_pop_thread_workspace(void) 7199 { 7200 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7201 7202 if (!data) 7203 return; 7204 if (!data->inuse) { 7205 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7206 return; 7207 } 7208 data->inuse = 0; 7209 if (!data->prev) 7210 return; 7211 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7212 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7213 } 7214 7215 /** 7216 * Verify the flow list is empty 7217 * 7218 * @param dev 7219 * Pointer to Ethernet device. 7220 * 7221 * @return the number of flows not released. 7222 */ 7223 int 7224 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7225 { 7226 struct mlx5_priv *priv = dev->data->dev_private; 7227 struct rte_flow *flow; 7228 uint32_t idx = 0; 7229 int ret = 0, i; 7230 7231 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7232 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7233 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7234 dev->data->port_id, (void *)flow); 7235 ret++; 7236 } 7237 } 7238 return ret; 7239 } 7240 7241 /** 7242 * Enable default hairpin egress flow. 7243 * 7244 * @param dev 7245 * Pointer to Ethernet device. 7246 * @param queue 7247 * The queue index. 7248 * 7249 * @return 7250 * 0 on success, a negative errno value otherwise and rte_errno is set. 7251 */ 7252 int 7253 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7254 uint32_t queue) 7255 { 7256 const struct rte_flow_attr attr = { 7257 .egress = 1, 7258 .priority = 0, 7259 }; 7260 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7261 .queue = queue, 7262 }; 7263 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7264 .queue = UINT32_MAX, 7265 }; 7266 struct rte_flow_item items[] = { 7267 { 7268 .type = (enum rte_flow_item_type) 7269 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7270 .spec = &queue_spec, 7271 .last = NULL, 7272 .mask = &queue_mask, 7273 }, 7274 { 7275 .type = RTE_FLOW_ITEM_TYPE_END, 7276 }, 7277 }; 7278 struct rte_flow_action_jump jump = { 7279 .group = MLX5_HAIRPIN_TX_TABLE, 7280 }; 7281 struct rte_flow_action actions[2]; 7282 uint32_t flow_idx; 7283 struct rte_flow_error error; 7284 7285 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7286 actions[0].conf = &jump; 7287 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7288 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7289 &attr, items, actions, false, &error); 7290 if (!flow_idx) { 7291 DRV_LOG(DEBUG, 7292 "Failed to create ctrl flow: rte_errno(%d)," 7293 " type(%d), message(%s)", 7294 rte_errno, error.type, 7295 error.message ? error.message : " (no stated reason)"); 7296 return -rte_errno; 7297 } 7298 return 0; 7299 } 7300 7301 /** 7302 * Enable a control flow configured from the control plane. 7303 * 7304 * @param dev 7305 * Pointer to Ethernet device. 7306 * @param eth_spec 7307 * An Ethernet flow spec to apply. 7308 * @param eth_mask 7309 * An Ethernet flow mask to apply. 7310 * @param vlan_spec 7311 * A VLAN flow spec to apply. 7312 * @param vlan_mask 7313 * A VLAN flow mask to apply. 7314 * 7315 * @return 7316 * 0 on success, a negative errno value otherwise and rte_errno is set. 7317 */ 7318 int 7319 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7320 struct rte_flow_item_eth *eth_spec, 7321 struct rte_flow_item_eth *eth_mask, 7322 struct rte_flow_item_vlan *vlan_spec, 7323 struct rte_flow_item_vlan *vlan_mask) 7324 { 7325 struct mlx5_priv *priv = dev->data->dev_private; 7326 const struct rte_flow_attr attr = { 7327 .ingress = 1, 7328 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7329 }; 7330 struct rte_flow_item items[] = { 7331 { 7332 .type = RTE_FLOW_ITEM_TYPE_ETH, 7333 .spec = eth_spec, 7334 .last = NULL, 7335 .mask = eth_mask, 7336 }, 7337 { 7338 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7339 RTE_FLOW_ITEM_TYPE_END, 7340 .spec = vlan_spec, 7341 .last = NULL, 7342 .mask = vlan_mask, 7343 }, 7344 { 7345 .type = RTE_FLOW_ITEM_TYPE_END, 7346 }, 7347 }; 7348 uint16_t queue[priv->reta_idx_n]; 7349 struct rte_flow_action_rss action_rss = { 7350 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7351 .level = 0, 7352 .types = priv->rss_conf.rss_hf, 7353 .key_len = priv->rss_conf.rss_key_len, 7354 .queue_num = priv->reta_idx_n, 7355 .key = priv->rss_conf.rss_key, 7356 .queue = queue, 7357 }; 7358 struct rte_flow_action actions[] = { 7359 { 7360 .type = RTE_FLOW_ACTION_TYPE_RSS, 7361 .conf = &action_rss, 7362 }, 7363 { 7364 .type = RTE_FLOW_ACTION_TYPE_END, 7365 }, 7366 }; 7367 uint32_t flow_idx; 7368 struct rte_flow_error error; 7369 unsigned int i; 7370 7371 if (!priv->reta_idx_n || !priv->rxqs_n) { 7372 return 0; 7373 } 7374 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7375 action_rss.types = 0; 7376 for (i = 0; i != priv->reta_idx_n; ++i) 7377 queue[i] = (*priv->reta_idx)[i]; 7378 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7379 &attr, items, actions, false, &error); 7380 if (!flow_idx) 7381 return -rte_errno; 7382 return 0; 7383 } 7384 7385 /** 7386 * Enable a flow control configured from the control plane. 7387 * 7388 * @param dev 7389 * Pointer to Ethernet device. 7390 * @param eth_spec 7391 * An Ethernet flow spec to apply. 7392 * @param eth_mask 7393 * An Ethernet flow mask to apply. 7394 * 7395 * @return 7396 * 0 on success, a negative errno value otherwise and rte_errno is set. 7397 */ 7398 int 7399 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7400 struct rte_flow_item_eth *eth_spec, 7401 struct rte_flow_item_eth *eth_mask) 7402 { 7403 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7404 } 7405 7406 /** 7407 * Create default miss flow rule matching lacp traffic 7408 * 7409 * @param dev 7410 * Pointer to Ethernet device. 7411 * @param eth_spec 7412 * An Ethernet flow spec to apply. 7413 * 7414 * @return 7415 * 0 on success, a negative errno value otherwise and rte_errno is set. 7416 */ 7417 int 7418 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7419 { 7420 /* 7421 * The LACP matching is done by only using ether type since using 7422 * a multicast dst mac causes kernel to give low priority to this flow. 7423 */ 7424 static const struct rte_flow_item_eth lacp_spec = { 7425 .type = RTE_BE16(0x8809), 7426 }; 7427 static const struct rte_flow_item_eth lacp_mask = { 7428 .type = 0xffff, 7429 }; 7430 const struct rte_flow_attr attr = { 7431 .ingress = 1, 7432 }; 7433 struct rte_flow_item items[] = { 7434 { 7435 .type = RTE_FLOW_ITEM_TYPE_ETH, 7436 .spec = &lacp_spec, 7437 .mask = &lacp_mask, 7438 }, 7439 { 7440 .type = RTE_FLOW_ITEM_TYPE_END, 7441 }, 7442 }; 7443 struct rte_flow_action actions[] = { 7444 { 7445 .type = (enum rte_flow_action_type) 7446 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7447 }, 7448 { 7449 .type = RTE_FLOW_ACTION_TYPE_END, 7450 }, 7451 }; 7452 struct rte_flow_error error; 7453 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7454 &attr, items, actions, 7455 false, &error); 7456 7457 if (!flow_idx) 7458 return -rte_errno; 7459 return 0; 7460 } 7461 7462 /** 7463 * Destroy a flow. 7464 * 7465 * @see rte_flow_destroy() 7466 * @see rte_flow_ops 7467 */ 7468 int 7469 mlx5_flow_destroy(struct rte_eth_dev *dev, 7470 struct rte_flow *flow, 7471 struct rte_flow_error *error __rte_unused) 7472 { 7473 struct mlx5_priv *priv = dev->data->dev_private; 7474 7475 if (priv->sh->config.dv_flow_en == 2) 7476 return rte_flow_error_set(error, ENOTSUP, 7477 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7478 NULL, 7479 "Flow non-Q destruction not supported"); 7480 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7481 (uintptr_t)(void *)flow); 7482 return 0; 7483 } 7484 7485 /** 7486 * Destroy all flows. 7487 * 7488 * @see rte_flow_flush() 7489 * @see rte_flow_ops 7490 */ 7491 int 7492 mlx5_flow_flush(struct rte_eth_dev *dev, 7493 struct rte_flow_error *error __rte_unused) 7494 { 7495 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7496 return 0; 7497 } 7498 7499 /** 7500 * Isolated mode. 7501 * 7502 * @see rte_flow_isolate() 7503 * @see rte_flow_ops 7504 */ 7505 int 7506 mlx5_flow_isolate(struct rte_eth_dev *dev, 7507 int enable, 7508 struct rte_flow_error *error) 7509 { 7510 struct mlx5_priv *priv = dev->data->dev_private; 7511 7512 if (dev->data->dev_started) { 7513 rte_flow_error_set(error, EBUSY, 7514 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7515 NULL, 7516 "port must be stopped first"); 7517 return -rte_errno; 7518 } 7519 priv->isolated = !!enable; 7520 if (enable) 7521 dev->dev_ops = &mlx5_dev_ops_isolate; 7522 else 7523 dev->dev_ops = &mlx5_dev_ops; 7524 7525 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7526 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7527 7528 return 0; 7529 } 7530 7531 /** 7532 * Query a flow. 7533 * 7534 * @see rte_flow_query() 7535 * @see rte_flow_ops 7536 */ 7537 static int 7538 flow_drv_query(struct rte_eth_dev *dev, 7539 uint32_t flow_idx, 7540 const struct rte_flow_action *actions, 7541 void *data, 7542 struct rte_flow_error *error) 7543 { 7544 struct mlx5_priv *priv = dev->data->dev_private; 7545 const struct mlx5_flow_driver_ops *fops; 7546 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7547 flow_idx); 7548 enum mlx5_flow_drv_type ftype; 7549 7550 if (!flow) { 7551 return rte_flow_error_set(error, ENOENT, 7552 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7553 NULL, 7554 "invalid flow handle"); 7555 } 7556 ftype = flow->drv_type; 7557 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7558 fops = flow_get_drv_ops(ftype); 7559 7560 return fops->query(dev, flow, actions, data, error); 7561 } 7562 7563 /** 7564 * Query a flow. 7565 * 7566 * @see rte_flow_query() 7567 * @see rte_flow_ops 7568 */ 7569 int 7570 mlx5_flow_query(struct rte_eth_dev *dev, 7571 struct rte_flow *flow, 7572 const struct rte_flow_action *actions, 7573 void *data, 7574 struct rte_flow_error *error) 7575 { 7576 int ret; 7577 struct mlx5_priv *priv = dev->data->dev_private; 7578 7579 if (priv->sh->config.dv_flow_en == 2) 7580 return rte_flow_error_set(error, ENOTSUP, 7581 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7582 NULL, 7583 "Flow non-Q query not supported"); 7584 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7585 error); 7586 if (ret < 0) 7587 return ret; 7588 return 0; 7589 } 7590 7591 /** 7592 * Get rte_flow callbacks. 7593 * 7594 * @param dev 7595 * Pointer to Ethernet device structure. 7596 * @param ops 7597 * Pointer to operation-specific structure. 7598 * 7599 * @return 0 7600 */ 7601 int 7602 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7603 const struct rte_flow_ops **ops) 7604 { 7605 *ops = &mlx5_flow_ops; 7606 return 0; 7607 } 7608 7609 /** 7610 * Validate meter policy actions. 7611 * Dispatcher for action type specific validation. 7612 * 7613 * @param[in] dev 7614 * Pointer to the Ethernet device structure. 7615 * @param[in] action 7616 * The meter policy action object to validate. 7617 * @param[in] attr 7618 * Attributes of flow to determine steering domain. 7619 * @param[out] is_rss 7620 * Is RSS or not. 7621 * @param[out] domain_bitmap 7622 * Domain bitmap. 7623 * @param[out] is_def_policy 7624 * Is default policy or not. 7625 * @param[out] error 7626 * Perform verbose error reporting if not NULL. Initialized in case of 7627 * error only. 7628 * 7629 * @return 7630 * 0 on success, otherwise negative errno value. 7631 */ 7632 int 7633 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7634 const struct rte_flow_action *actions[RTE_COLORS], 7635 struct rte_flow_attr *attr, 7636 bool *is_rss, 7637 uint8_t *domain_bitmap, 7638 uint8_t *policy_mode, 7639 struct rte_mtr_error *error) 7640 { 7641 const struct mlx5_flow_driver_ops *fops; 7642 7643 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7644 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7645 domain_bitmap, policy_mode, error); 7646 } 7647 7648 /** 7649 * Destroy the meter table set. 7650 * 7651 * @param[in] dev 7652 * Pointer to Ethernet device. 7653 * @param[in] mtr_policy 7654 * Meter policy struct. 7655 */ 7656 void 7657 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7658 struct mlx5_flow_meter_policy *mtr_policy) 7659 { 7660 const struct mlx5_flow_driver_ops *fops; 7661 7662 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7663 fops->destroy_mtr_acts(dev, mtr_policy); 7664 } 7665 7666 /** 7667 * Create policy action, lock free, 7668 * (mutex should be acquired by caller). 7669 * Dispatcher for action type specific call. 7670 * 7671 * @param[in] dev 7672 * Pointer to the Ethernet device structure. 7673 * @param[in] mtr_policy 7674 * Meter policy struct. 7675 * @param[in] action 7676 * Action specification used to create meter actions. 7677 * @param[out] error 7678 * Perform verbose error reporting if not NULL. Initialized in case of 7679 * error only. 7680 * 7681 * @return 7682 * 0 on success, otherwise negative errno value. 7683 */ 7684 int 7685 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7686 struct mlx5_flow_meter_policy *mtr_policy, 7687 const struct rte_flow_action *actions[RTE_COLORS], 7688 struct rte_mtr_error *error) 7689 { 7690 const struct mlx5_flow_driver_ops *fops; 7691 7692 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7693 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7694 } 7695 7696 /** 7697 * Create policy rules, lock free, 7698 * (mutex should be acquired by caller). 7699 * Dispatcher for action type specific call. 7700 * 7701 * @param[in] dev 7702 * Pointer to the Ethernet device structure. 7703 * @param[in] mtr_policy 7704 * Meter policy struct. 7705 * 7706 * @return 7707 * 0 on success, -1 otherwise. 7708 */ 7709 int 7710 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7711 struct mlx5_flow_meter_policy *mtr_policy) 7712 { 7713 const struct mlx5_flow_driver_ops *fops; 7714 7715 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7716 return fops->create_policy_rules(dev, mtr_policy); 7717 } 7718 7719 /** 7720 * Destroy policy rules, lock free, 7721 * (mutex should be acquired by caller). 7722 * Dispatcher for action type specific call. 7723 * 7724 * @param[in] dev 7725 * Pointer to the Ethernet device structure. 7726 * @param[in] mtr_policy 7727 * Meter policy struct. 7728 */ 7729 void 7730 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7731 struct mlx5_flow_meter_policy *mtr_policy) 7732 { 7733 const struct mlx5_flow_driver_ops *fops; 7734 7735 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7736 fops->destroy_policy_rules(dev, mtr_policy); 7737 } 7738 7739 /** 7740 * Destroy the default policy table set. 7741 * 7742 * @param[in] dev 7743 * Pointer to Ethernet device. 7744 */ 7745 void 7746 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7747 { 7748 const struct mlx5_flow_driver_ops *fops; 7749 7750 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7751 fops->destroy_def_policy(dev); 7752 } 7753 7754 /** 7755 * Destroy the default policy table set. 7756 * 7757 * @param[in] dev 7758 * Pointer to Ethernet device. 7759 * 7760 * @return 7761 * 0 on success, -1 otherwise. 7762 */ 7763 int 7764 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7765 { 7766 const struct mlx5_flow_driver_ops *fops; 7767 7768 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7769 return fops->create_def_policy(dev); 7770 } 7771 7772 /** 7773 * Create the needed meter and suffix tables. 7774 * 7775 * @param[in] dev 7776 * Pointer to Ethernet device. 7777 * 7778 * @return 7779 * 0 on success, -1 otherwise. 7780 */ 7781 int 7782 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7783 struct mlx5_flow_meter_info *fm, 7784 uint32_t mtr_idx, 7785 uint8_t domain_bitmap) 7786 { 7787 const struct mlx5_flow_driver_ops *fops; 7788 7789 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7790 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7791 } 7792 7793 /** 7794 * Destroy the meter table set. 7795 * 7796 * @param[in] dev 7797 * Pointer to Ethernet device. 7798 * @param[in] tbl 7799 * Pointer to the meter table set. 7800 */ 7801 void 7802 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7803 struct mlx5_flow_meter_info *fm) 7804 { 7805 const struct mlx5_flow_driver_ops *fops; 7806 7807 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7808 fops->destroy_mtr_tbls(dev, fm); 7809 } 7810 7811 /** 7812 * Destroy the global meter drop table. 7813 * 7814 * @param[in] dev 7815 * Pointer to Ethernet device. 7816 */ 7817 void 7818 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7819 { 7820 const struct mlx5_flow_driver_ops *fops; 7821 7822 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7823 fops->destroy_mtr_drop_tbls(dev); 7824 } 7825 7826 /** 7827 * Destroy the sub policy table with RX queue. 7828 * 7829 * @param[in] dev 7830 * Pointer to Ethernet device. 7831 * @param[in] mtr_policy 7832 * Pointer to meter policy table. 7833 */ 7834 void 7835 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7836 struct mlx5_flow_meter_policy *mtr_policy) 7837 { 7838 const struct mlx5_flow_driver_ops *fops; 7839 7840 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7841 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7842 } 7843 7844 /** 7845 * Allocate the needed aso flow meter id. 7846 * 7847 * @param[in] dev 7848 * Pointer to Ethernet device. 7849 * 7850 * @return 7851 * Index to aso flow meter on success, NULL otherwise. 7852 */ 7853 uint32_t 7854 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7855 { 7856 const struct mlx5_flow_driver_ops *fops; 7857 7858 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7859 return fops->create_meter(dev); 7860 } 7861 7862 /** 7863 * Free the aso flow meter id. 7864 * 7865 * @param[in] dev 7866 * Pointer to Ethernet device. 7867 * @param[in] mtr_idx 7868 * Index to aso flow meter to be free. 7869 * 7870 * @return 7871 * 0 on success. 7872 */ 7873 void 7874 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7875 { 7876 const struct mlx5_flow_driver_ops *fops; 7877 7878 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7879 fops->free_meter(dev, mtr_idx); 7880 } 7881 7882 /** 7883 * Allocate a counter. 7884 * 7885 * @param[in] dev 7886 * Pointer to Ethernet device structure. 7887 * 7888 * @return 7889 * Index to allocated counter on success, 0 otherwise. 7890 */ 7891 uint32_t 7892 mlx5_counter_alloc(struct rte_eth_dev *dev) 7893 { 7894 const struct mlx5_flow_driver_ops *fops; 7895 struct rte_flow_attr attr = { .transfer = 0 }; 7896 7897 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7898 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7899 return fops->counter_alloc(dev); 7900 } 7901 DRV_LOG(ERR, 7902 "port %u counter allocate is not supported.", 7903 dev->data->port_id); 7904 return 0; 7905 } 7906 7907 /** 7908 * Free a counter. 7909 * 7910 * @param[in] dev 7911 * Pointer to Ethernet device structure. 7912 * @param[in] cnt 7913 * Index to counter to be free. 7914 */ 7915 void 7916 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7917 { 7918 const struct mlx5_flow_driver_ops *fops; 7919 struct rte_flow_attr attr = { .transfer = 0 }; 7920 7921 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7922 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7923 fops->counter_free(dev, cnt); 7924 return; 7925 } 7926 DRV_LOG(ERR, 7927 "port %u counter free is not supported.", 7928 dev->data->port_id); 7929 } 7930 7931 /** 7932 * Query counter statistics. 7933 * 7934 * @param[in] dev 7935 * Pointer to Ethernet device structure. 7936 * @param[in] cnt 7937 * Index to counter to query. 7938 * @param[in] clear 7939 * Set to clear counter statistics. 7940 * @param[out] pkts 7941 * The counter hits packets number to save. 7942 * @param[out] bytes 7943 * The counter hits bytes number to save. 7944 * 7945 * @return 7946 * 0 on success, a negative errno value otherwise. 7947 */ 7948 int 7949 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7950 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 7951 { 7952 const struct mlx5_flow_driver_ops *fops; 7953 struct rte_flow_attr attr = { .transfer = 0 }; 7954 7955 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7956 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7957 return fops->counter_query(dev, cnt, clear, pkts, 7958 bytes, action); 7959 } 7960 DRV_LOG(ERR, 7961 "port %u counter query is not supported.", 7962 dev->data->port_id); 7963 return -ENOTSUP; 7964 } 7965 7966 /** 7967 * Get information about HWS pre-configurable resources. 7968 * 7969 * @param[in] dev 7970 * Pointer to the rte_eth_dev structure. 7971 * @param[out] port_info 7972 * Pointer to port information. 7973 * @param[out] queue_info 7974 * Pointer to queue information. 7975 * @param[out] error 7976 * Pointer to error structure. 7977 * 7978 * @return 7979 * 0 on success, a negative errno value otherwise and rte_errno is set. 7980 */ 7981 static int 7982 mlx5_flow_info_get(struct rte_eth_dev *dev, 7983 struct rte_flow_port_info *port_info, 7984 struct rte_flow_queue_info *queue_info, 7985 struct rte_flow_error *error) 7986 { 7987 const struct mlx5_flow_driver_ops *fops; 7988 7989 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 7990 return rte_flow_error_set(error, ENOTSUP, 7991 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7992 NULL, 7993 "info get with incorrect steering mode"); 7994 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 7995 return fops->info_get(dev, port_info, queue_info, error); 7996 } 7997 7998 /** 7999 * Configure port HWS resources. 8000 * 8001 * @param[in] dev 8002 * Pointer to the rte_eth_dev structure. 8003 * @param[in] port_attr 8004 * Port configuration attributes. 8005 * @param[in] nb_queue 8006 * Number of queue. 8007 * @param[in] queue_attr 8008 * Array that holds attributes for each flow queue. 8009 * @param[out] error 8010 * Pointer to error structure. 8011 * 8012 * @return 8013 * 0 on success, a negative errno value otherwise and rte_errno is set. 8014 */ 8015 static int 8016 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8017 const struct rte_flow_port_attr *port_attr, 8018 uint16_t nb_queue, 8019 const struct rte_flow_queue_attr *queue_attr[], 8020 struct rte_flow_error *error) 8021 { 8022 const struct mlx5_flow_driver_ops *fops; 8023 8024 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8025 return rte_flow_error_set(error, ENOTSUP, 8026 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8027 NULL, 8028 "port configure with incorrect steering mode"); 8029 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8030 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8031 } 8032 8033 /** 8034 * Create flow item template. 8035 * 8036 * @param[in] dev 8037 * Pointer to the rte_eth_dev structure. 8038 * @param[in] attr 8039 * Pointer to the item template attributes. 8040 * @param[in] items 8041 * The template item pattern. 8042 * @param[out] error 8043 * Pointer to error structure. 8044 * 8045 * @return 8046 * 0 on success, a negative errno value otherwise and rte_errno is set. 8047 */ 8048 static struct rte_flow_pattern_template * 8049 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8050 const struct rte_flow_pattern_template_attr *attr, 8051 const struct rte_flow_item items[], 8052 struct rte_flow_error *error) 8053 { 8054 const struct mlx5_flow_driver_ops *fops; 8055 8056 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8057 rte_flow_error_set(error, ENOTSUP, 8058 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8059 NULL, 8060 "pattern create with incorrect steering mode"); 8061 return NULL; 8062 } 8063 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8064 return fops->pattern_template_create(dev, attr, items, error); 8065 } 8066 8067 /** 8068 * Destroy flow item template. 8069 * 8070 * @param[in] dev 8071 * Pointer to the rte_eth_dev structure. 8072 * @param[in] template 8073 * Pointer to the item template to be destroyed. 8074 * @param[out] error 8075 * Pointer to error structure. 8076 * 8077 * @return 8078 * 0 on success, a negative errno value otherwise and rte_errno is set. 8079 */ 8080 static int 8081 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8082 struct rte_flow_pattern_template *template, 8083 struct rte_flow_error *error) 8084 { 8085 const struct mlx5_flow_driver_ops *fops; 8086 8087 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8088 return rte_flow_error_set(error, ENOTSUP, 8089 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8090 NULL, 8091 "pattern destroy with incorrect steering mode"); 8092 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8093 return fops->pattern_template_destroy(dev, template, error); 8094 } 8095 8096 /** 8097 * Create flow item template. 8098 * 8099 * @param[in] dev 8100 * Pointer to the rte_eth_dev structure. 8101 * @param[in] attr 8102 * Pointer to the action template attributes. 8103 * @param[in] actions 8104 * Associated actions (list terminated by the END action). 8105 * @param[in] masks 8106 * List of actions that marks which of the action's member is constant. 8107 * @param[out] error 8108 * Pointer to error structure. 8109 * 8110 * @return 8111 * 0 on success, a negative errno value otherwise and rte_errno is set. 8112 */ 8113 static struct rte_flow_actions_template * 8114 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8115 const struct rte_flow_actions_template_attr *attr, 8116 const struct rte_flow_action actions[], 8117 const struct rte_flow_action masks[], 8118 struct rte_flow_error *error) 8119 { 8120 const struct mlx5_flow_driver_ops *fops; 8121 8122 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8123 rte_flow_error_set(error, ENOTSUP, 8124 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8125 NULL, 8126 "action create with incorrect steering mode"); 8127 return NULL; 8128 } 8129 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8130 return fops->actions_template_create(dev, attr, actions, masks, error); 8131 } 8132 8133 /** 8134 * Destroy flow action template. 8135 * 8136 * @param[in] dev 8137 * Pointer to the rte_eth_dev structure. 8138 * @param[in] template 8139 * Pointer to the action template to be destroyed. 8140 * @param[out] error 8141 * Pointer to error structure. 8142 * 8143 * @return 8144 * 0 on success, a negative errno value otherwise and rte_errno is set. 8145 */ 8146 static int 8147 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8148 struct rte_flow_actions_template *template, 8149 struct rte_flow_error *error) 8150 { 8151 const struct mlx5_flow_driver_ops *fops; 8152 8153 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8154 return rte_flow_error_set(error, ENOTSUP, 8155 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8156 NULL, 8157 "action destroy with incorrect steering mode"); 8158 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8159 return fops->actions_template_destroy(dev, template, error); 8160 } 8161 8162 /** 8163 * Create flow table. 8164 * 8165 * @param[in] dev 8166 * Pointer to the rte_eth_dev structure. 8167 * @param[in] attr 8168 * Pointer to the table attributes. 8169 * @param[in] item_templates 8170 * Item template array to be binded to the table. 8171 * @param[in] nb_item_templates 8172 * Number of item template. 8173 * @param[in] action_templates 8174 * Action template array to be binded to the table. 8175 * @param[in] nb_action_templates 8176 * Number of action template. 8177 * @param[out] error 8178 * Pointer to error structure. 8179 * 8180 * @return 8181 * Table on success, NULL otherwise and rte_errno is set. 8182 */ 8183 static struct rte_flow_template_table * 8184 mlx5_flow_table_create(struct rte_eth_dev *dev, 8185 const struct rte_flow_template_table_attr *attr, 8186 struct rte_flow_pattern_template *item_templates[], 8187 uint8_t nb_item_templates, 8188 struct rte_flow_actions_template *action_templates[], 8189 uint8_t nb_action_templates, 8190 struct rte_flow_error *error) 8191 { 8192 const struct mlx5_flow_driver_ops *fops; 8193 8194 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8195 rte_flow_error_set(error, ENOTSUP, 8196 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8197 NULL, 8198 "table create with incorrect steering mode"); 8199 return NULL; 8200 } 8201 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8202 return fops->template_table_create(dev, 8203 attr, 8204 item_templates, 8205 nb_item_templates, 8206 action_templates, 8207 nb_action_templates, 8208 error); 8209 } 8210 8211 /** 8212 * PMD destroy flow table. 8213 * 8214 * @param[in] dev 8215 * Pointer to the rte_eth_dev structure. 8216 * @param[in] table 8217 * Pointer to the table to be destroyed. 8218 * @param[out] error 8219 * Pointer to error structure. 8220 * 8221 * @return 8222 * 0 on success, a negative errno value otherwise and rte_errno is set. 8223 */ 8224 static int 8225 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8226 struct rte_flow_template_table *table, 8227 struct rte_flow_error *error) 8228 { 8229 const struct mlx5_flow_driver_ops *fops; 8230 8231 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8232 return rte_flow_error_set(error, ENOTSUP, 8233 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8234 NULL, 8235 "table destroy with incorrect steering mode"); 8236 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8237 return fops->template_table_destroy(dev, table, error); 8238 } 8239 8240 /** 8241 * Enqueue flow creation. 8242 * 8243 * @param[in] dev 8244 * Pointer to the rte_eth_dev structure. 8245 * @param[in] queue_id 8246 * The queue to create the flow. 8247 * @param[in] attr 8248 * Pointer to the flow operation attributes. 8249 * @param[in] items 8250 * Items with flow spec value. 8251 * @param[in] pattern_template_index 8252 * The item pattern flow follows from the table. 8253 * @param[in] actions 8254 * Action with flow spec value. 8255 * @param[in] action_template_index 8256 * The action pattern flow follows from the table. 8257 * @param[in] user_data 8258 * Pointer to the user_data. 8259 * @param[out] error 8260 * Pointer to error structure. 8261 * 8262 * @return 8263 * Flow pointer on success, NULL otherwise and rte_errno is set. 8264 */ 8265 static struct rte_flow * 8266 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8267 uint32_t queue_id, 8268 const struct rte_flow_op_attr *attr, 8269 struct rte_flow_template_table *table, 8270 const struct rte_flow_item items[], 8271 uint8_t pattern_template_index, 8272 const struct rte_flow_action actions[], 8273 uint8_t action_template_index, 8274 void *user_data, 8275 struct rte_flow_error *error) 8276 { 8277 const struct mlx5_flow_driver_ops *fops; 8278 8279 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8280 rte_flow_error_set(error, ENOTSUP, 8281 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8282 NULL, 8283 "flow_q create with incorrect steering mode"); 8284 return NULL; 8285 } 8286 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8287 return fops->async_flow_create(dev, queue_id, attr, table, 8288 items, pattern_template_index, 8289 actions, action_template_index, 8290 user_data, error); 8291 } 8292 8293 /** 8294 * Enqueue flow destruction. 8295 * 8296 * @param[in] dev 8297 * Pointer to the rte_eth_dev structure. 8298 * @param[in] queue 8299 * The queue to destroy the flow. 8300 * @param[in] attr 8301 * Pointer to the flow operation attributes. 8302 * @param[in] flow 8303 * Pointer to the flow to be destroyed. 8304 * @param[in] user_data 8305 * Pointer to the user_data. 8306 * @param[out] error 8307 * Pointer to error structure. 8308 * 8309 * @return 8310 * 0 on success, negative value otherwise and rte_errno is set. 8311 */ 8312 static int 8313 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8314 uint32_t queue, 8315 const struct rte_flow_op_attr *attr, 8316 struct rte_flow *flow, 8317 void *user_data, 8318 struct rte_flow_error *error) 8319 { 8320 const struct mlx5_flow_driver_ops *fops; 8321 8322 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8323 return rte_flow_error_set(error, ENOTSUP, 8324 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8325 NULL, 8326 "flow_q destroy with incorrect steering mode"); 8327 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8328 return fops->async_flow_destroy(dev, queue, attr, flow, 8329 user_data, error); 8330 } 8331 8332 /** 8333 * Pull the enqueued flows. 8334 * 8335 * @param[in] dev 8336 * Pointer to the rte_eth_dev structure. 8337 * @param[in] queue 8338 * The queue to pull the result. 8339 * @param[in/out] res 8340 * Array to save the results. 8341 * @param[in] n_res 8342 * Available result with the array. 8343 * @param[out] error 8344 * Pointer to error structure. 8345 * 8346 * @return 8347 * Result number on success, negative value otherwise and rte_errno is set. 8348 */ 8349 static int 8350 mlx5_flow_pull(struct rte_eth_dev *dev, 8351 uint32_t queue, 8352 struct rte_flow_op_result res[], 8353 uint16_t n_res, 8354 struct rte_flow_error *error) 8355 { 8356 const struct mlx5_flow_driver_ops *fops; 8357 8358 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8359 return rte_flow_error_set(error, ENOTSUP, 8360 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8361 NULL, 8362 "flow_q pull with incorrect steering mode"); 8363 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8364 return fops->pull(dev, queue, res, n_res, error); 8365 } 8366 8367 /** 8368 * Push the enqueued flows. 8369 * 8370 * @param[in] dev 8371 * Pointer to the rte_eth_dev structure. 8372 * @param[in] queue 8373 * The queue to push the flows. 8374 * @param[out] error 8375 * Pointer to error structure. 8376 * 8377 * @return 8378 * 0 on success, negative value otherwise and rte_errno is set. 8379 */ 8380 static int 8381 mlx5_flow_push(struct rte_eth_dev *dev, 8382 uint32_t queue, 8383 struct rte_flow_error *error) 8384 { 8385 const struct mlx5_flow_driver_ops *fops; 8386 8387 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8388 return rte_flow_error_set(error, ENOTSUP, 8389 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8390 NULL, 8391 "flow_q push with incorrect steering mode"); 8392 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8393 return fops->push(dev, queue, error); 8394 } 8395 8396 /** 8397 * Create shared action. 8398 * 8399 * @param[in] dev 8400 * Pointer to the rte_eth_dev structure. 8401 * @param[in] queue 8402 * Which queue to be used.. 8403 * @param[in] attr 8404 * Operation attribute. 8405 * @param[in] conf 8406 * Indirect action configuration. 8407 * @param[in] action 8408 * rte_flow action detail. 8409 * @param[in] user_data 8410 * Pointer to the user_data. 8411 * @param[out] error 8412 * Pointer to error structure. 8413 * 8414 * @return 8415 * Action handle on success, NULL otherwise and rte_errno is set. 8416 */ 8417 static struct rte_flow_action_handle * 8418 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8419 const struct rte_flow_op_attr *attr, 8420 const struct rte_flow_indir_action_conf *conf, 8421 const struct rte_flow_action *action, 8422 void *user_data, 8423 struct rte_flow_error *error) 8424 { 8425 const struct mlx5_flow_driver_ops *fops = 8426 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8427 8428 return fops->async_action_create(dev, queue, attr, conf, action, 8429 user_data, error); 8430 } 8431 8432 /** 8433 * Update shared action. 8434 * 8435 * @param[in] dev 8436 * Pointer to the rte_eth_dev structure. 8437 * @param[in] queue 8438 * Which queue to be used.. 8439 * @param[in] attr 8440 * Operation attribute. 8441 * @param[in] handle 8442 * Action handle to be updated. 8443 * @param[in] update 8444 * Update value. 8445 * @param[in] user_data 8446 * Pointer to the user_data. 8447 * @param[out] error 8448 * Pointer to error structure. 8449 * 8450 * @return 8451 * 0 on success, negative value otherwise and rte_errno is set. 8452 */ 8453 static int 8454 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 8455 const struct rte_flow_op_attr *attr, 8456 struct rte_flow_action_handle *handle, 8457 const void *update, 8458 void *user_data, 8459 struct rte_flow_error *error) 8460 { 8461 const struct mlx5_flow_driver_ops *fops = 8462 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8463 8464 return fops->async_action_update(dev, queue, attr, handle, 8465 update, user_data, error); 8466 } 8467 8468 /** 8469 * Destroy shared action. 8470 * 8471 * @param[in] dev 8472 * Pointer to the rte_eth_dev structure. 8473 * @param[in] queue 8474 * Which queue to be used.. 8475 * @param[in] attr 8476 * Operation attribute. 8477 * @param[in] handle 8478 * Action handle to be destroyed. 8479 * @param[in] user_data 8480 * Pointer to the user_data. 8481 * @param[out] error 8482 * Pointer to error structure. 8483 * 8484 * @return 8485 * 0 on success, negative value otherwise and rte_errno is set. 8486 */ 8487 static int 8488 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 8489 const struct rte_flow_op_attr *attr, 8490 struct rte_flow_action_handle *handle, 8491 void *user_data, 8492 struct rte_flow_error *error) 8493 { 8494 const struct mlx5_flow_driver_ops *fops = 8495 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8496 8497 return fops->async_action_destroy(dev, queue, attr, handle, 8498 user_data, error); 8499 } 8500 8501 /** 8502 * Allocate a new memory for the counter values wrapped by all the needed 8503 * management. 8504 * 8505 * @param[in] sh 8506 * Pointer to mlx5_dev_ctx_shared object. 8507 * 8508 * @return 8509 * 0 on success, a negative errno value otherwise. 8510 */ 8511 static int 8512 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 8513 { 8514 struct mlx5_counter_stats_mem_mng *mem_mng; 8515 volatile struct flow_counter_stats *raw_data; 8516 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 8517 int size = (sizeof(struct flow_counter_stats) * 8518 MLX5_COUNTERS_PER_POOL + 8519 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 8520 sizeof(struct mlx5_counter_stats_mem_mng); 8521 size_t pgsize = rte_mem_page_size(); 8522 uint8_t *mem; 8523 int ret; 8524 int i; 8525 8526 if (pgsize == (size_t)-1) { 8527 DRV_LOG(ERR, "Failed to get mem page size"); 8528 rte_errno = ENOMEM; 8529 return -ENOMEM; 8530 } 8531 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 8532 if (!mem) { 8533 rte_errno = ENOMEM; 8534 return -ENOMEM; 8535 } 8536 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 8537 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 8538 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 8539 sh->cdev->pdn, mem, size, 8540 &mem_mng->wm); 8541 if (ret) { 8542 rte_errno = errno; 8543 mlx5_free(mem); 8544 return -rte_errno; 8545 } 8546 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 8547 raw_data = (volatile struct flow_counter_stats *)mem; 8548 for (i = 0; i < raws_n; ++i) { 8549 mem_mng->raws[i].mem_mng = mem_mng; 8550 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 8551 } 8552 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 8553 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 8554 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 8555 next); 8556 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 8557 sh->cmng.mem_mng = mem_mng; 8558 return 0; 8559 } 8560 8561 /** 8562 * Set the statistic memory to the new counter pool. 8563 * 8564 * @param[in] sh 8565 * Pointer to mlx5_dev_ctx_shared object. 8566 * @param[in] pool 8567 * Pointer to the pool to set the statistic memory. 8568 * 8569 * @return 8570 * 0 on success, a negative errno value otherwise. 8571 */ 8572 static int 8573 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 8574 struct mlx5_flow_counter_pool *pool) 8575 { 8576 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8577 /* Resize statistic memory once used out. */ 8578 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 8579 mlx5_flow_create_counter_stat_mem_mng(sh)) { 8580 DRV_LOG(ERR, "Cannot resize counter stat mem."); 8581 return -1; 8582 } 8583 rte_spinlock_lock(&pool->sl); 8584 pool->raw = cmng->mem_mng->raws + pool->index % 8585 MLX5_CNT_CONTAINER_RESIZE; 8586 rte_spinlock_unlock(&pool->sl); 8587 pool->raw_hw = NULL; 8588 return 0; 8589 } 8590 8591 #define MLX5_POOL_QUERY_FREQ_US 1000000 8592 8593 /** 8594 * Set the periodic procedure for triggering asynchronous batch queries for all 8595 * the counter pools. 8596 * 8597 * @param[in] sh 8598 * Pointer to mlx5_dev_ctx_shared object. 8599 */ 8600 void 8601 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 8602 { 8603 uint32_t pools_n, us; 8604 8605 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 8606 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 8607 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 8608 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 8609 sh->cmng.query_thread_on = 0; 8610 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 8611 } else { 8612 sh->cmng.query_thread_on = 1; 8613 } 8614 } 8615 8616 /** 8617 * The periodic procedure for triggering asynchronous batch queries for all the 8618 * counter pools. This function is probably called by the host thread. 8619 * 8620 * @param[in] arg 8621 * The parameter for the alarm process. 8622 */ 8623 void 8624 mlx5_flow_query_alarm(void *arg) 8625 { 8626 struct mlx5_dev_ctx_shared *sh = arg; 8627 int ret; 8628 uint16_t pool_index = sh->cmng.pool_index; 8629 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8630 struct mlx5_flow_counter_pool *pool; 8631 uint16_t n_valid; 8632 8633 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 8634 goto set_alarm; 8635 rte_spinlock_lock(&cmng->pool_update_sl); 8636 pool = cmng->pools[pool_index]; 8637 n_valid = cmng->n_valid; 8638 rte_spinlock_unlock(&cmng->pool_update_sl); 8639 /* Set the statistic memory to the new created pool. */ 8640 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 8641 goto set_alarm; 8642 if (pool->raw_hw) 8643 /* There is a pool query in progress. */ 8644 goto set_alarm; 8645 pool->raw_hw = 8646 LIST_FIRST(&sh->cmng.free_stat_raws); 8647 if (!pool->raw_hw) 8648 /* No free counter statistics raw memory. */ 8649 goto set_alarm; 8650 /* 8651 * Identify the counters released between query trigger and query 8652 * handle more efficiently. The counter released in this gap period 8653 * should wait for a new round of query as the new arrived packets 8654 * will not be taken into account. 8655 */ 8656 pool->query_gen++; 8657 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 8658 MLX5_COUNTERS_PER_POOL, 8659 NULL, NULL, 8660 pool->raw_hw->mem_mng->wm.lkey, 8661 (void *)(uintptr_t) 8662 pool->raw_hw->data, 8663 sh->devx_comp, 8664 (uint64_t)(uintptr_t)pool); 8665 if (ret) { 8666 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 8667 " %d", pool->min_dcs->id); 8668 pool->raw_hw = NULL; 8669 goto set_alarm; 8670 } 8671 LIST_REMOVE(pool->raw_hw, next); 8672 sh->cmng.pending_queries++; 8673 pool_index++; 8674 if (pool_index >= n_valid) 8675 pool_index = 0; 8676 set_alarm: 8677 sh->cmng.pool_index = pool_index; 8678 mlx5_set_query_alarm(sh); 8679 } 8680 8681 /** 8682 * Check and callback event for new aged flow in the counter pool 8683 * 8684 * @param[in] sh 8685 * Pointer to mlx5_dev_ctx_shared object. 8686 * @param[in] pool 8687 * Pointer to Current counter pool. 8688 */ 8689 static void 8690 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8691 struct mlx5_flow_counter_pool *pool) 8692 { 8693 struct mlx5_priv *priv; 8694 struct mlx5_flow_counter *cnt; 8695 struct mlx5_age_info *age_info; 8696 struct mlx5_age_param *age_param; 8697 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8698 struct mlx5_counter_stats_raw *prev = pool->raw; 8699 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8700 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8701 uint16_t expected = AGE_CANDIDATE; 8702 uint32_t i; 8703 8704 pool->time_of_last_age_check = curr_time; 8705 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8706 cnt = MLX5_POOL_GET_CNT(pool, i); 8707 age_param = MLX5_CNT_TO_AGE(cnt); 8708 if (__atomic_load_n(&age_param->state, 8709 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8710 continue; 8711 if (cur->data[i].hits != prev->data[i].hits) { 8712 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8713 __ATOMIC_RELAXED); 8714 continue; 8715 } 8716 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8717 time_delta, 8718 __ATOMIC_RELAXED) <= age_param->timeout) 8719 continue; 8720 /** 8721 * Hold the lock first, or if between the 8722 * state AGE_TMOUT and tailq operation the 8723 * release happened, the release procedure 8724 * may delete a non-existent tailq node. 8725 */ 8726 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8727 age_info = GET_PORT_AGE_INFO(priv); 8728 rte_spinlock_lock(&age_info->aged_sl); 8729 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8730 AGE_TMOUT, false, 8731 __ATOMIC_RELAXED, 8732 __ATOMIC_RELAXED)) { 8733 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8734 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8735 } 8736 rte_spinlock_unlock(&age_info->aged_sl); 8737 } 8738 mlx5_age_event_prepare(sh); 8739 } 8740 8741 /** 8742 * Handler for the HW respond about ready values from an asynchronous batch 8743 * query. This function is probably called by the host thread. 8744 * 8745 * @param[in] sh 8746 * The pointer to the shared device context. 8747 * @param[in] async_id 8748 * The Devx async ID. 8749 * @param[in] status 8750 * The status of the completion. 8751 */ 8752 void 8753 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8754 uint64_t async_id, int status) 8755 { 8756 struct mlx5_flow_counter_pool *pool = 8757 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8758 struct mlx5_counter_stats_raw *raw_to_free; 8759 uint8_t query_gen = pool->query_gen ^ 1; 8760 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8761 enum mlx5_counter_type cnt_type = 8762 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8763 MLX5_COUNTER_TYPE_ORIGIN; 8764 8765 if (unlikely(status)) { 8766 raw_to_free = pool->raw_hw; 8767 } else { 8768 raw_to_free = pool->raw; 8769 if (pool->is_aged) 8770 mlx5_flow_aging_check(sh, pool); 8771 rte_spinlock_lock(&pool->sl); 8772 pool->raw = pool->raw_hw; 8773 rte_spinlock_unlock(&pool->sl); 8774 /* Be sure the new raw counters data is updated in memory. */ 8775 rte_io_wmb(); 8776 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8777 rte_spinlock_lock(&cmng->csl[cnt_type]); 8778 TAILQ_CONCAT(&cmng->counters[cnt_type], 8779 &pool->counters[query_gen], next); 8780 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8781 } 8782 } 8783 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8784 pool->raw_hw = NULL; 8785 sh->cmng.pending_queries--; 8786 } 8787 8788 static int 8789 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8790 const struct flow_grp_info *grp_info, 8791 struct rte_flow_error *error) 8792 { 8793 if (grp_info->transfer && grp_info->external && 8794 grp_info->fdb_def_rule) { 8795 if (group == UINT32_MAX) 8796 return rte_flow_error_set 8797 (error, EINVAL, 8798 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8799 NULL, 8800 "group index not supported"); 8801 *table = group + 1; 8802 } else { 8803 *table = group; 8804 } 8805 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8806 return 0; 8807 } 8808 8809 /** 8810 * Translate the rte_flow group index to HW table value. 8811 * 8812 * If tunnel offload is disabled, all group ids converted to flow table 8813 * id using the standard method. 8814 * If tunnel offload is enabled, group id can be converted using the 8815 * standard or tunnel conversion method. Group conversion method 8816 * selection depends on flags in `grp_info` parameter: 8817 * - Internal (grp_info.external == 0) groups conversion uses the 8818 * standard method. 8819 * - Group ids in JUMP action converted with the tunnel conversion. 8820 * - Group id in rule attribute conversion depends on a rule type and 8821 * group id value: 8822 * ** non zero group attributes converted with the tunnel method 8823 * ** zero group attribute in non-tunnel rule is converted using the 8824 * standard method - there's only one root table 8825 * ** zero group attribute in steer tunnel rule is converted with the 8826 * standard method - single root table 8827 * ** zero group attribute in match tunnel rule is a special OvS 8828 * case: that value is used for portability reasons. That group 8829 * id is converted with the tunnel conversion method. 8830 * 8831 * @param[in] dev 8832 * Port device 8833 * @param[in] tunnel 8834 * PMD tunnel offload object 8835 * @param[in] group 8836 * rte_flow group index value. 8837 * @param[out] table 8838 * HW table value. 8839 * @param[in] grp_info 8840 * flags used for conversion 8841 * @param[out] error 8842 * Pointer to error structure. 8843 * 8844 * @return 8845 * 0 on success, a negative errno value otherwise and rte_errno is set. 8846 */ 8847 int 8848 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8849 const struct mlx5_flow_tunnel *tunnel, 8850 uint32_t group, uint32_t *table, 8851 const struct flow_grp_info *grp_info, 8852 struct rte_flow_error *error) 8853 { 8854 int ret; 8855 bool standard_translation; 8856 8857 if (!grp_info->skip_scale && grp_info->external && 8858 group < MLX5_MAX_TABLES_EXTERNAL) 8859 group *= MLX5_FLOW_TABLE_FACTOR; 8860 if (is_tunnel_offload_active(dev)) { 8861 standard_translation = !grp_info->external || 8862 grp_info->std_tbl_fix; 8863 } else { 8864 standard_translation = true; 8865 } 8866 DRV_LOG(DEBUG, 8867 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8868 dev->data->port_id, group, grp_info->transfer, 8869 grp_info->external, grp_info->fdb_def_rule, 8870 standard_translation ? "STANDARD" : "TUNNEL"); 8871 if (standard_translation) 8872 ret = flow_group_to_table(dev->data->port_id, group, table, 8873 grp_info, error); 8874 else 8875 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8876 table, error); 8877 8878 return ret; 8879 } 8880 8881 /** 8882 * Discover availability of metadata reg_c's. 8883 * 8884 * Iteratively use test flows to check availability. 8885 * 8886 * @param[in] dev 8887 * Pointer to the Ethernet device structure. 8888 * 8889 * @return 8890 * 0 on success, a negative errno value otherwise and rte_errno is set. 8891 */ 8892 int 8893 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8894 { 8895 struct mlx5_priv *priv = dev->data->dev_private; 8896 enum modify_reg idx; 8897 int n = 0; 8898 8899 /* reg_c[0] and reg_c[1] are reserved. */ 8900 priv->sh->flow_mreg_c[n++] = REG_C_0; 8901 priv->sh->flow_mreg_c[n++] = REG_C_1; 8902 /* Discover availability of other reg_c's. */ 8903 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 8904 struct rte_flow_attr attr = { 8905 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 8906 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8907 .ingress = 1, 8908 }; 8909 struct rte_flow_item items[] = { 8910 [0] = { 8911 .type = RTE_FLOW_ITEM_TYPE_END, 8912 }, 8913 }; 8914 struct rte_flow_action actions[] = { 8915 [0] = { 8916 .type = (enum rte_flow_action_type) 8917 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8918 .conf = &(struct mlx5_flow_action_copy_mreg){ 8919 .src = REG_C_1, 8920 .dst = idx, 8921 }, 8922 }, 8923 [1] = { 8924 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8925 .conf = &(struct rte_flow_action_jump){ 8926 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8927 }, 8928 }, 8929 [2] = { 8930 .type = RTE_FLOW_ACTION_TYPE_END, 8931 }, 8932 }; 8933 uint32_t flow_idx; 8934 struct rte_flow *flow; 8935 struct rte_flow_error error; 8936 8937 if (!priv->sh->config.dv_flow_en) 8938 break; 8939 /* Create internal flow, validation skips copy action. */ 8940 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8941 items, actions, false, &error); 8942 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8943 flow_idx); 8944 if (!flow) 8945 continue; 8946 priv->sh->flow_mreg_c[n++] = idx; 8947 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8948 } 8949 for (; n < MLX5_MREG_C_NUM; ++n) 8950 priv->sh->flow_mreg_c[n] = REG_NON; 8951 priv->sh->metadata_regc_check_flag = 1; 8952 return 0; 8953 } 8954 8955 int 8956 save_dump_file(const uint8_t *data, uint32_t size, 8957 uint32_t type, uint64_t id, void *arg, FILE *file) 8958 { 8959 char line[BUF_SIZE]; 8960 uint32_t out = 0; 8961 uint32_t k; 8962 uint32_t actions_num; 8963 struct rte_flow_query_count *count; 8964 8965 memset(line, 0, BUF_SIZE); 8966 switch (type) { 8967 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8968 actions_num = *(uint32_t *)(arg); 8969 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 8970 type, id, actions_num); 8971 break; 8972 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8973 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 8974 type, id); 8975 break; 8976 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8977 count = (struct rte_flow_query_count *)arg; 8978 fprintf(file, 8979 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 8980 type, id, count->hits, count->bytes); 8981 return 0; 8982 default: 8983 return -1; 8984 } 8985 8986 for (k = 0; k < size; k++) { 8987 /* Make sure we do not overrun the line buffer length. */ 8988 if (out >= BUF_SIZE - 4) { 8989 line[out] = '\0'; 8990 break; 8991 } 8992 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8993 (data[k]) & 0xff); 8994 } 8995 fprintf(file, "%s\n", line); 8996 return 0; 8997 } 8998 8999 int 9000 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9001 struct rte_flow_query_count *count, struct rte_flow_error *error) 9002 { 9003 struct rte_flow_action action[2]; 9004 enum mlx5_flow_drv_type ftype; 9005 const struct mlx5_flow_driver_ops *fops; 9006 9007 if (!flow) { 9008 return rte_flow_error_set(error, ENOENT, 9009 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9010 NULL, 9011 "invalid flow handle"); 9012 } 9013 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9014 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9015 if (flow->counter) { 9016 memset(count, 0, sizeof(struct rte_flow_query_count)); 9017 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9018 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9019 ftype < MLX5_FLOW_TYPE_MAX); 9020 fops = flow_get_drv_ops(ftype); 9021 return fops->query(dev, flow, action, count, error); 9022 } 9023 return -1; 9024 } 9025 9026 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9027 /** 9028 * Dump flow ipool data to file 9029 * 9030 * @param[in] dev 9031 * The pointer to Ethernet device. 9032 * @param[in] file 9033 * A pointer to a file for output. 9034 * @param[out] error 9035 * Perform verbose error reporting if not NULL. PMDs initialize this 9036 * structure in case of error only. 9037 * @return 9038 * 0 on success, a negative value otherwise. 9039 */ 9040 int 9041 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9042 struct rte_flow *flow, FILE *file, 9043 struct rte_flow_error *error) 9044 { 9045 struct mlx5_priv *priv = dev->data->dev_private; 9046 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9047 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9048 uint32_t handle_idx; 9049 struct mlx5_flow_handle *dh; 9050 struct rte_flow_query_count count; 9051 uint32_t actions_num; 9052 const uint8_t *data; 9053 size_t size; 9054 uint64_t id; 9055 uint32_t type; 9056 void *action = NULL; 9057 9058 if (!flow) { 9059 return rte_flow_error_set(error, ENOENT, 9060 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9061 NULL, 9062 "invalid flow handle"); 9063 } 9064 handle_idx = flow->dev_handles; 9065 /* query counter */ 9066 if (flow->counter && 9067 (!mlx5_counter_query(dev, flow->counter, false, 9068 &count.hits, &count.bytes, &action)) && action) { 9069 id = (uint64_t)(uintptr_t)action; 9070 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9071 save_dump_file(NULL, 0, type, 9072 id, (void *)&count, file); 9073 } 9074 9075 while (handle_idx) { 9076 dh = mlx5_ipool_get(priv->sh->ipool 9077 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9078 if (!dh) 9079 continue; 9080 handle_idx = dh->next.next; 9081 9082 /* Get modify_hdr and encap_decap buf from ipools. */ 9083 encap_decap = NULL; 9084 modify_hdr = dh->dvh.modify_hdr; 9085 9086 if (dh->dvh.rix_encap_decap) { 9087 encap_decap = mlx5_ipool_get(priv->sh->ipool 9088 [MLX5_IPOOL_DECAP_ENCAP], 9089 dh->dvh.rix_encap_decap); 9090 } 9091 if (modify_hdr) { 9092 data = (const uint8_t *)modify_hdr->actions; 9093 size = (size_t)(modify_hdr->actions_num) * 8; 9094 id = (uint64_t)(uintptr_t)modify_hdr->action; 9095 actions_num = modify_hdr->actions_num; 9096 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9097 save_dump_file(data, size, type, id, 9098 (void *)(&actions_num), file); 9099 } 9100 if (encap_decap) { 9101 data = encap_decap->buf; 9102 size = encap_decap->size; 9103 id = (uint64_t)(uintptr_t)encap_decap->action; 9104 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9105 save_dump_file(data, size, type, 9106 id, NULL, file); 9107 } 9108 } 9109 return 0; 9110 } 9111 9112 /** 9113 * Dump all flow's encap_decap/modify_hdr/counter data to file 9114 * 9115 * @param[in] dev 9116 * The pointer to Ethernet device. 9117 * @param[in] file 9118 * A pointer to a file for output. 9119 * @param[out] error 9120 * Perform verbose error reporting if not NULL. PMDs initialize this 9121 * structure in case of error only. 9122 * @return 9123 * 0 on success, a negative value otherwise. 9124 */ 9125 static int 9126 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9127 FILE *file, struct rte_flow_error *error __rte_unused) 9128 { 9129 struct mlx5_priv *priv = dev->data->dev_private; 9130 struct mlx5_dev_ctx_shared *sh = priv->sh; 9131 struct mlx5_hlist *h; 9132 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9133 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9134 struct rte_flow_query_count count; 9135 uint32_t actions_num; 9136 const uint8_t *data; 9137 size_t size; 9138 uint64_t id; 9139 uint32_t type; 9140 uint32_t i; 9141 uint32_t j; 9142 struct mlx5_list_inconst *l_inconst; 9143 struct mlx5_list_entry *e; 9144 int lcore_index; 9145 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 9146 uint32_t max; 9147 void *action; 9148 9149 /* encap_decap hlist is lcore_share, get global core cache. */ 9150 i = MLX5_LIST_GLOBAL; 9151 h = sh->encaps_decaps; 9152 if (h) { 9153 for (j = 0; j <= h->mask; j++) { 9154 l_inconst = &h->buckets[j].l; 9155 if (!l_inconst || !l_inconst->cache[i]) 9156 continue; 9157 9158 e = LIST_FIRST(&l_inconst->cache[i]->h); 9159 while (e) { 9160 encap_decap = 9161 (struct mlx5_flow_dv_encap_decap_resource *)e; 9162 data = encap_decap->buf; 9163 size = encap_decap->size; 9164 id = (uint64_t)(uintptr_t)encap_decap->action; 9165 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9166 save_dump_file(data, size, type, 9167 id, NULL, file); 9168 e = LIST_NEXT(e, next); 9169 } 9170 } 9171 } 9172 9173 /* get modify_hdr */ 9174 h = sh->modify_cmds; 9175 if (h) { 9176 lcore_index = rte_lcore_index(rte_lcore_id()); 9177 if (unlikely(lcore_index == -1)) { 9178 lcore_index = MLX5_LIST_NLCORE; 9179 rte_spinlock_lock(&h->l_const.lcore_lock); 9180 } 9181 i = lcore_index; 9182 9183 for (j = 0; j <= h->mask; j++) { 9184 l_inconst = &h->buckets[j].l; 9185 if (!l_inconst || !l_inconst->cache[i]) 9186 continue; 9187 9188 e = LIST_FIRST(&l_inconst->cache[i]->h); 9189 while (e) { 9190 modify_hdr = 9191 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9192 data = (const uint8_t *)modify_hdr->actions; 9193 size = (size_t)(modify_hdr->actions_num) * 8; 9194 actions_num = modify_hdr->actions_num; 9195 id = (uint64_t)(uintptr_t)modify_hdr->action; 9196 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9197 save_dump_file(data, size, type, id, 9198 (void *)(&actions_num), file); 9199 e = LIST_NEXT(e, next); 9200 } 9201 } 9202 9203 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9204 rte_spinlock_unlock(&h->l_const.lcore_lock); 9205 } 9206 9207 /* get counter */ 9208 MLX5_ASSERT(cmng->n_valid <= cmng->n); 9209 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9210 for (j = 1; j <= max; j++) { 9211 action = NULL; 9212 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9213 &count.bytes, &action)) && action) { 9214 id = (uint64_t)(uintptr_t)action; 9215 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9216 save_dump_file(NULL, 0, type, 9217 id, (void *)&count, file); 9218 } 9219 } 9220 return 0; 9221 } 9222 #endif 9223 9224 /** 9225 * Dump flow raw hw data to file 9226 * 9227 * @param[in] dev 9228 * The pointer to Ethernet device. 9229 * @param[in] file 9230 * A pointer to a file for output. 9231 * @param[out] error 9232 * Perform verbose error reporting if not NULL. PMDs initialize this 9233 * structure in case of error only. 9234 * @return 9235 * 0 on success, a negative value otherwise. 9236 */ 9237 int 9238 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9239 FILE *file, 9240 struct rte_flow_error *error __rte_unused) 9241 { 9242 struct mlx5_priv *priv = dev->data->dev_private; 9243 struct mlx5_dev_ctx_shared *sh = priv->sh; 9244 uint32_t handle_idx; 9245 int ret; 9246 struct mlx5_flow_handle *dh; 9247 struct rte_flow *flow; 9248 9249 if (!sh->config.dv_flow_en) { 9250 if (fputs("device dv flow disabled\n", file) <= 0) 9251 return -errno; 9252 return -ENOTSUP; 9253 } 9254 9255 /* dump all */ 9256 if (!flow_idx) { 9257 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9258 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9259 return -EINVAL; 9260 #endif 9261 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9262 sh->rx_domain, 9263 sh->tx_domain, file); 9264 } 9265 /* dump one */ 9266 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9267 (uintptr_t)(void *)flow_idx); 9268 if (!flow) 9269 return -EINVAL; 9270 9271 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9272 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9273 #endif 9274 handle_idx = flow->dev_handles; 9275 while (handle_idx) { 9276 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9277 handle_idx); 9278 if (!dh) 9279 return -ENOENT; 9280 if (dh->drv_flow) { 9281 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9282 file); 9283 if (ret) 9284 return -ENOENT; 9285 } 9286 handle_idx = dh->next.next; 9287 } 9288 return 0; 9289 } 9290 9291 /** 9292 * Get aged-out flows. 9293 * 9294 * @param[in] dev 9295 * Pointer to the Ethernet device structure. 9296 * @param[in] context 9297 * The address of an array of pointers to the aged-out flows contexts. 9298 * @param[in] nb_countexts 9299 * The length of context array pointers. 9300 * @param[out] error 9301 * Perform verbose error reporting if not NULL. Initialized in case of 9302 * error only. 9303 * 9304 * @return 9305 * how many contexts get in success, otherwise negative errno value. 9306 * if nb_contexts is 0, return the amount of all aged contexts. 9307 * if nb_contexts is not 0 , return the amount of aged flows reported 9308 * in the context array. 9309 */ 9310 int 9311 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9312 uint32_t nb_contexts, struct rte_flow_error *error) 9313 { 9314 const struct mlx5_flow_driver_ops *fops; 9315 struct rte_flow_attr attr = { .transfer = 0 }; 9316 9317 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 9318 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 9319 return fops->get_aged_flows(dev, contexts, nb_contexts, 9320 error); 9321 } 9322 DRV_LOG(ERR, 9323 "port %u get aged flows is not supported.", 9324 dev->data->port_id); 9325 return -ENOTSUP; 9326 } 9327 9328 /* Wrapper for driver action_validate op callback */ 9329 static int 9330 flow_drv_action_validate(struct rte_eth_dev *dev, 9331 const struct rte_flow_indir_action_conf *conf, 9332 const struct rte_flow_action *action, 9333 const struct mlx5_flow_driver_ops *fops, 9334 struct rte_flow_error *error) 9335 { 9336 static const char err_msg[] = "indirect action validation unsupported"; 9337 9338 if (!fops->action_validate) { 9339 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9340 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9341 NULL, err_msg); 9342 return -rte_errno; 9343 } 9344 return fops->action_validate(dev, conf, action, error); 9345 } 9346 9347 /** 9348 * Destroys the shared action by handle. 9349 * 9350 * @param dev 9351 * Pointer to Ethernet device structure. 9352 * @param[in] handle 9353 * Handle for the indirect action object to be destroyed. 9354 * @param[out] error 9355 * Perform verbose error reporting if not NULL. PMDs initialize this 9356 * structure in case of error only. 9357 * 9358 * @return 9359 * 0 on success, a negative errno value otherwise and rte_errno is set. 9360 * 9361 * @note: wrapper for driver action_create op callback. 9362 */ 9363 static int 9364 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9365 struct rte_flow_action_handle *handle, 9366 struct rte_flow_error *error) 9367 { 9368 static const char err_msg[] = "indirect action destruction unsupported"; 9369 struct rte_flow_attr attr = { .transfer = 0 }; 9370 const struct mlx5_flow_driver_ops *fops = 9371 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9372 9373 if (!fops->action_destroy) { 9374 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9375 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9376 NULL, err_msg); 9377 return -rte_errno; 9378 } 9379 return fops->action_destroy(dev, handle, error); 9380 } 9381 9382 /* Wrapper for driver action_destroy op callback */ 9383 static int 9384 flow_drv_action_update(struct rte_eth_dev *dev, 9385 struct rte_flow_action_handle *handle, 9386 const void *update, 9387 const struct mlx5_flow_driver_ops *fops, 9388 struct rte_flow_error *error) 9389 { 9390 static const char err_msg[] = "indirect action update unsupported"; 9391 9392 if (!fops->action_update) { 9393 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9394 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9395 NULL, err_msg); 9396 return -rte_errno; 9397 } 9398 return fops->action_update(dev, handle, update, error); 9399 } 9400 9401 /* Wrapper for driver action_destroy op callback */ 9402 static int 9403 flow_drv_action_query(struct rte_eth_dev *dev, 9404 const struct rte_flow_action_handle *handle, 9405 void *data, 9406 const struct mlx5_flow_driver_ops *fops, 9407 struct rte_flow_error *error) 9408 { 9409 static const char err_msg[] = "indirect action query unsupported"; 9410 9411 if (!fops->action_query) { 9412 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9413 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9414 NULL, err_msg); 9415 return -rte_errno; 9416 } 9417 return fops->action_query(dev, handle, data, error); 9418 } 9419 9420 /** 9421 * Create indirect action for reuse in multiple flow rules. 9422 * 9423 * @param dev 9424 * Pointer to Ethernet device structure. 9425 * @param conf 9426 * Pointer to indirect action object configuration. 9427 * @param[in] action 9428 * Action configuration for indirect action object creation. 9429 * @param[out] error 9430 * Perform verbose error reporting if not NULL. PMDs initialize this 9431 * structure in case of error only. 9432 * @return 9433 * A valid handle in case of success, NULL otherwise and rte_errno is set. 9434 */ 9435 static struct rte_flow_action_handle * 9436 mlx5_action_handle_create(struct rte_eth_dev *dev, 9437 const struct rte_flow_indir_action_conf *conf, 9438 const struct rte_flow_action *action, 9439 struct rte_flow_error *error) 9440 { 9441 static const char err_msg[] = "indirect action creation unsupported"; 9442 struct rte_flow_attr attr = { .transfer = 0 }; 9443 const struct mlx5_flow_driver_ops *fops = 9444 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9445 9446 if (flow_drv_action_validate(dev, conf, action, fops, error)) 9447 return NULL; 9448 if (!fops->action_create) { 9449 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9450 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9451 NULL, err_msg); 9452 return NULL; 9453 } 9454 return fops->action_create(dev, conf, action, error); 9455 } 9456 9457 /** 9458 * Updates inplace the indirect action configuration pointed by *handle* 9459 * with the configuration provided as *update* argument. 9460 * The update of the indirect action configuration effects all flow rules 9461 * reusing the action via handle. 9462 * 9463 * @param dev 9464 * Pointer to Ethernet device structure. 9465 * @param[in] handle 9466 * Handle for the indirect action to be updated. 9467 * @param[in] update 9468 * Action specification used to modify the action pointed by handle. 9469 * *update* could be of same type with the action pointed by the *handle* 9470 * handle argument, or some other structures like a wrapper, depending on 9471 * the indirect action type. 9472 * @param[out] error 9473 * Perform verbose error reporting if not NULL. PMDs initialize this 9474 * structure in case of error only. 9475 * 9476 * @return 9477 * 0 on success, a negative errno value otherwise and rte_errno is set. 9478 */ 9479 static int 9480 mlx5_action_handle_update(struct rte_eth_dev *dev, 9481 struct rte_flow_action_handle *handle, 9482 const void *update, 9483 struct rte_flow_error *error) 9484 { 9485 struct rte_flow_attr attr = { .transfer = 0 }; 9486 const struct mlx5_flow_driver_ops *fops = 9487 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9488 int ret; 9489 9490 ret = flow_drv_action_validate(dev, NULL, 9491 (const struct rte_flow_action *)update, fops, error); 9492 if (ret) 9493 return ret; 9494 return flow_drv_action_update(dev, handle, update, fops, 9495 error); 9496 } 9497 9498 /** 9499 * Query the indirect action by handle. 9500 * 9501 * This function allows retrieving action-specific data such as counters. 9502 * Data is gathered by special action which may be present/referenced in 9503 * more than one flow rule definition. 9504 * 9505 * see @RTE_FLOW_ACTION_TYPE_COUNT 9506 * 9507 * @param dev 9508 * Pointer to Ethernet device structure. 9509 * @param[in] handle 9510 * Handle for the indirect action to query. 9511 * @param[in, out] data 9512 * Pointer to storage for the associated query data type. 9513 * @param[out] error 9514 * Perform verbose error reporting if not NULL. PMDs initialize this 9515 * structure in case of error only. 9516 * 9517 * @return 9518 * 0 on success, a negative errno value otherwise and rte_errno is set. 9519 */ 9520 static int 9521 mlx5_action_handle_query(struct rte_eth_dev *dev, 9522 const struct rte_flow_action_handle *handle, 9523 void *data, 9524 struct rte_flow_error *error) 9525 { 9526 struct rte_flow_attr attr = { .transfer = 0 }; 9527 const struct mlx5_flow_driver_ops *fops = 9528 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9529 9530 return flow_drv_action_query(dev, handle, data, fops, error); 9531 } 9532 9533 /** 9534 * Destroy all indirect actions (shared RSS). 9535 * 9536 * @param dev 9537 * Pointer to Ethernet device. 9538 * 9539 * @return 9540 * 0 on success, a negative errno value otherwise and rte_errno is set. 9541 */ 9542 int 9543 mlx5_action_handle_flush(struct rte_eth_dev *dev) 9544 { 9545 struct rte_flow_error error; 9546 struct mlx5_priv *priv = dev->data->dev_private; 9547 struct mlx5_shared_action_rss *shared_rss; 9548 int ret = 0; 9549 uint32_t idx; 9550 9551 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 9552 priv->rss_shared_actions, idx, shared_rss, next) { 9553 ret |= mlx5_action_handle_destroy(dev, 9554 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 9555 } 9556 return ret; 9557 } 9558 9559 /** 9560 * Validate existing indirect actions against current device configuration 9561 * and attach them to device resources. 9562 * 9563 * @param dev 9564 * Pointer to Ethernet device. 9565 * 9566 * @return 9567 * 0 on success, a negative errno value otherwise and rte_errno is set. 9568 */ 9569 int 9570 mlx5_action_handle_attach(struct rte_eth_dev *dev) 9571 { 9572 struct mlx5_priv *priv = dev->data->dev_private; 9573 int ret = 0; 9574 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9575 9576 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9577 const char *message; 9578 uint32_t queue_idx; 9579 9580 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 9581 ind_tbl->queues_n, 9582 &message, &queue_idx); 9583 if (ret != 0) { 9584 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 9585 dev->data->port_id, ind_tbl->queues[queue_idx], 9586 message); 9587 break; 9588 } 9589 } 9590 if (ret != 0) 9591 return ret; 9592 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9593 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 9594 if (ret != 0) { 9595 DRV_LOG(ERR, "Port %u could not attach " 9596 "indirection table obj %p", 9597 dev->data->port_id, (void *)ind_tbl); 9598 goto error; 9599 } 9600 } 9601 9602 return 0; 9603 error: 9604 ind_tbl_last = ind_tbl; 9605 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9606 if (ind_tbl == ind_tbl_last) 9607 break; 9608 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 9609 DRV_LOG(CRIT, "Port %u could not detach " 9610 "indirection table obj %p on rollback", 9611 dev->data->port_id, (void *)ind_tbl); 9612 } 9613 return ret; 9614 } 9615 9616 /** 9617 * Detach indirect actions of the device from its resources. 9618 * 9619 * @param dev 9620 * Pointer to Ethernet device. 9621 * 9622 * @return 9623 * 0 on success, a negative errno value otherwise and rte_errno is set. 9624 */ 9625 int 9626 mlx5_action_handle_detach(struct rte_eth_dev *dev) 9627 { 9628 struct mlx5_priv *priv = dev->data->dev_private; 9629 int ret = 0; 9630 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9631 9632 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9633 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 9634 if (ret != 0) { 9635 DRV_LOG(ERR, "Port %u could not detach " 9636 "indirection table obj %p", 9637 dev->data->port_id, (void *)ind_tbl); 9638 goto error; 9639 } 9640 } 9641 return 0; 9642 error: 9643 ind_tbl_last = ind_tbl; 9644 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9645 if (ind_tbl == ind_tbl_last) 9646 break; 9647 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 9648 DRV_LOG(CRIT, "Port %u could not attach " 9649 "indirection table obj %p on rollback", 9650 dev->data->port_id, (void *)ind_tbl); 9651 } 9652 return ret; 9653 } 9654 9655 #ifndef HAVE_MLX5DV_DR 9656 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 9657 #else 9658 #define MLX5_DOMAIN_SYNC_FLOW \ 9659 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 9660 #endif 9661 9662 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 9663 { 9664 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 9665 const struct mlx5_flow_driver_ops *fops; 9666 int ret; 9667 struct rte_flow_attr attr = { .transfer = 0 }; 9668 9669 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9670 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9671 if (ret > 0) 9672 ret = -ret; 9673 return ret; 9674 } 9675 9676 const struct mlx5_flow_tunnel * 9677 mlx5_get_tof(const struct rte_flow_item *item, 9678 const struct rte_flow_action *action, 9679 enum mlx5_tof_rule_type *rule_type) 9680 { 9681 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9682 if (item->type == (typeof(item->type)) 9683 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9684 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9685 return flow_items_to_tunnel(item); 9686 } 9687 } 9688 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9689 if (action->type == (typeof(action->type)) 9690 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9691 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9692 return flow_actions_to_tunnel(action); 9693 } 9694 } 9695 return NULL; 9696 } 9697 9698 /** 9699 * tunnel offload functionality is defined for DV environment only 9700 */ 9701 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9702 __extension__ 9703 union tunnel_offload_mark { 9704 uint32_t val; 9705 struct { 9706 uint32_t app_reserve:8; 9707 uint32_t table_id:15; 9708 uint32_t transfer:1; 9709 uint32_t _unused_:8; 9710 }; 9711 }; 9712 9713 static bool 9714 mlx5_access_tunnel_offload_db 9715 (struct rte_eth_dev *dev, 9716 bool (*match)(struct rte_eth_dev *, 9717 struct mlx5_flow_tunnel *, const void *), 9718 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9719 void (*miss)(struct rte_eth_dev *, void *), 9720 void *ctx, bool lock_op); 9721 9722 static int 9723 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9724 struct rte_flow *flow, 9725 const struct rte_flow_attr *attr, 9726 const struct rte_flow_action *app_actions, 9727 uint32_t flow_idx, 9728 const struct mlx5_flow_tunnel *tunnel, 9729 struct tunnel_default_miss_ctx *ctx, 9730 struct rte_flow_error *error) 9731 { 9732 struct mlx5_priv *priv = dev->data->dev_private; 9733 struct mlx5_flow *dev_flow; 9734 struct rte_flow_attr miss_attr = *attr; 9735 const struct rte_flow_item miss_items[2] = { 9736 { 9737 .type = RTE_FLOW_ITEM_TYPE_ETH, 9738 .spec = NULL, 9739 .last = NULL, 9740 .mask = NULL 9741 }, 9742 { 9743 .type = RTE_FLOW_ITEM_TYPE_END, 9744 .spec = NULL, 9745 .last = NULL, 9746 .mask = NULL 9747 } 9748 }; 9749 union tunnel_offload_mark mark_id; 9750 struct rte_flow_action_mark miss_mark; 9751 struct rte_flow_action miss_actions[3] = { 9752 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9753 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9754 }; 9755 const struct rte_flow_action_jump *jump_data; 9756 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9757 struct flow_grp_info grp_info = { 9758 .external = 1, 9759 .transfer = attr->transfer, 9760 .fdb_def_rule = !!priv->fdb_def_rule, 9761 .std_tbl_fix = 0, 9762 }; 9763 int ret; 9764 9765 if (!attr->transfer) { 9766 uint32_t q_size; 9767 9768 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9769 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9770 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9771 0, SOCKET_ID_ANY); 9772 if (!ctx->queue) 9773 return rte_flow_error_set 9774 (error, ENOMEM, 9775 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9776 NULL, "invalid default miss RSS"); 9777 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9778 ctx->action_rss.level = 0, 9779 ctx->action_rss.types = priv->rss_conf.rss_hf, 9780 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9781 ctx->action_rss.queue_num = priv->reta_idx_n, 9782 ctx->action_rss.key = priv->rss_conf.rss_key, 9783 ctx->action_rss.queue = ctx->queue; 9784 if (!priv->reta_idx_n || !priv->rxqs_n) 9785 return rte_flow_error_set 9786 (error, EINVAL, 9787 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9788 NULL, "invalid port configuration"); 9789 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9790 ctx->action_rss.types = 0; 9791 for (i = 0; i != priv->reta_idx_n; ++i) 9792 ctx->queue[i] = (*priv->reta_idx)[i]; 9793 } else { 9794 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9795 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9796 } 9797 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9798 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9799 jump_data = app_actions->conf; 9800 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9801 miss_attr.group = jump_data->group; 9802 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9803 &flow_table, &grp_info, error); 9804 if (ret) 9805 return rte_flow_error_set(error, EINVAL, 9806 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9807 NULL, "invalid tunnel id"); 9808 mark_id.app_reserve = 0; 9809 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9810 mark_id.transfer = !!attr->transfer; 9811 mark_id._unused_ = 0; 9812 miss_mark.id = mark_id.val; 9813 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9814 miss_items, miss_actions, flow_idx, error); 9815 if (!dev_flow) 9816 return -rte_errno; 9817 dev_flow->flow = flow; 9818 dev_flow->external = true; 9819 dev_flow->tunnel = tunnel; 9820 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9821 /* Subflow object was created, we must include one in the list. */ 9822 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9823 dev_flow->handle, next); 9824 DRV_LOG(DEBUG, 9825 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9826 dev->data->port_id, tunnel->app_tunnel.type, 9827 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9828 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9829 miss_actions, error); 9830 if (!ret) 9831 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9832 error); 9833 9834 return ret; 9835 } 9836 9837 static const struct mlx5_flow_tbl_data_entry * 9838 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9839 { 9840 struct mlx5_priv *priv = dev->data->dev_private; 9841 struct mlx5_dev_ctx_shared *sh = priv->sh; 9842 struct mlx5_list_entry *he; 9843 union tunnel_offload_mark mbits = { .val = mark }; 9844 union mlx5_flow_tbl_key table_key = { 9845 { 9846 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9847 .id = 0, 9848 .reserved = 0, 9849 .dummy = 0, 9850 .is_fdb = !!mbits.transfer, 9851 .is_egress = 0, 9852 } 9853 }; 9854 struct mlx5_flow_cb_ctx ctx = { 9855 .data = &table_key.v64, 9856 }; 9857 9858 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9859 return he ? 9860 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9861 } 9862 9863 static void 9864 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9865 struct mlx5_list_entry *entry) 9866 { 9867 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9868 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9869 9870 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9871 tunnel_flow_tbl_to_id(tte->flow_table)); 9872 mlx5_free(tte); 9873 } 9874 9875 static int 9876 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9877 struct mlx5_list_entry *entry, void *cb_ctx) 9878 { 9879 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9880 union tunnel_tbl_key tbl = { 9881 .val = *(uint64_t *)(ctx->data), 9882 }; 9883 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9884 9885 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9886 } 9887 9888 static struct mlx5_list_entry * 9889 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 9890 { 9891 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9892 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9893 struct tunnel_tbl_entry *tte; 9894 union tunnel_tbl_key tbl = { 9895 .val = *(uint64_t *)(ctx->data), 9896 }; 9897 9898 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 9899 sizeof(*tte), 0, 9900 SOCKET_ID_ANY); 9901 if (!tte) 9902 goto err; 9903 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9904 &tte->flow_table); 9905 if (tte->flow_table >= MLX5_MAX_TABLES) { 9906 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 9907 tte->flow_table); 9908 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9909 tte->flow_table); 9910 goto err; 9911 } else if (!tte->flow_table) { 9912 goto err; 9913 } 9914 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 9915 tte->tunnel_id = tbl.tunnel_id; 9916 tte->group = tbl.group; 9917 return &tte->hash; 9918 err: 9919 if (tte) 9920 mlx5_free(tte); 9921 return NULL; 9922 } 9923 9924 static struct mlx5_list_entry * 9925 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 9926 struct mlx5_list_entry *oentry, 9927 void *cb_ctx __rte_unused) 9928 { 9929 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 9930 0, SOCKET_ID_ANY); 9931 9932 if (!tte) 9933 return NULL; 9934 memcpy(tte, oentry, sizeof(*tte)); 9935 return &tte->hash; 9936 } 9937 9938 static void 9939 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 9940 struct mlx5_list_entry *entry) 9941 { 9942 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9943 9944 mlx5_free(tte); 9945 } 9946 9947 static uint32_t 9948 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 9949 const struct mlx5_flow_tunnel *tunnel, 9950 uint32_t group, uint32_t *table, 9951 struct rte_flow_error *error) 9952 { 9953 struct mlx5_list_entry *he; 9954 struct tunnel_tbl_entry *tte; 9955 union tunnel_tbl_key key = { 9956 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 9957 .group = group 9958 }; 9959 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9960 struct mlx5_hlist *group_hash; 9961 struct mlx5_flow_cb_ctx ctx = { 9962 .data = &key.val, 9963 }; 9964 9965 group_hash = tunnel ? tunnel->groups : thub->groups; 9966 he = mlx5_hlist_register(group_hash, key.val, &ctx); 9967 if (!he) 9968 return rte_flow_error_set(error, EINVAL, 9969 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9970 NULL, 9971 "tunnel group index not supported"); 9972 tte = container_of(he, typeof(*tte), hash); 9973 *table = tte->flow_table; 9974 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 9975 dev->data->port_id, key.tunnel_id, group, *table); 9976 return 0; 9977 } 9978 9979 static void 9980 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 9981 struct mlx5_flow_tunnel *tunnel) 9982 { 9983 struct mlx5_priv *priv = dev->data->dev_private; 9984 struct mlx5_indexed_pool *ipool; 9985 9986 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 9987 dev->data->port_id, tunnel->tunnel_id); 9988 LIST_REMOVE(tunnel, chain); 9989 mlx5_hlist_destroy(tunnel->groups); 9990 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9991 mlx5_ipool_free(ipool, tunnel->tunnel_id); 9992 } 9993 9994 static bool 9995 mlx5_access_tunnel_offload_db 9996 (struct rte_eth_dev *dev, 9997 bool (*match)(struct rte_eth_dev *, 9998 struct mlx5_flow_tunnel *, const void *), 9999 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10000 void (*miss)(struct rte_eth_dev *, void *), 10001 void *ctx, bool lock_op) 10002 { 10003 bool verdict = false; 10004 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10005 struct mlx5_flow_tunnel *tunnel; 10006 10007 rte_spinlock_lock(&thub->sl); 10008 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10009 verdict = match(dev, tunnel, (const void *)ctx); 10010 if (verdict) 10011 break; 10012 } 10013 if (!lock_op) 10014 rte_spinlock_unlock(&thub->sl); 10015 if (verdict && hit) 10016 hit(dev, tunnel, ctx); 10017 if (!verdict && miss) 10018 miss(dev, ctx); 10019 if (lock_op) 10020 rte_spinlock_unlock(&thub->sl); 10021 10022 return verdict; 10023 } 10024 10025 struct tunnel_db_find_tunnel_id_ctx { 10026 uint32_t tunnel_id; 10027 struct mlx5_flow_tunnel *tunnel; 10028 }; 10029 10030 static bool 10031 find_tunnel_id_match(struct rte_eth_dev *dev, 10032 struct mlx5_flow_tunnel *tunnel, const void *x) 10033 { 10034 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10035 10036 RTE_SET_USED(dev); 10037 return tunnel->tunnel_id == ctx->tunnel_id; 10038 } 10039 10040 static void 10041 find_tunnel_id_hit(struct rte_eth_dev *dev, 10042 struct mlx5_flow_tunnel *tunnel, void *x) 10043 { 10044 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10045 RTE_SET_USED(dev); 10046 ctx->tunnel = tunnel; 10047 } 10048 10049 static struct mlx5_flow_tunnel * 10050 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10051 { 10052 struct tunnel_db_find_tunnel_id_ctx ctx = { 10053 .tunnel_id = id, 10054 }; 10055 10056 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10057 find_tunnel_id_hit, NULL, &ctx, true); 10058 10059 return ctx.tunnel; 10060 } 10061 10062 static struct mlx5_flow_tunnel * 10063 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10064 const struct rte_flow_tunnel *app_tunnel) 10065 { 10066 struct mlx5_priv *priv = dev->data->dev_private; 10067 struct mlx5_indexed_pool *ipool; 10068 struct mlx5_flow_tunnel *tunnel; 10069 uint32_t id; 10070 10071 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10072 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10073 if (!tunnel) 10074 return NULL; 10075 if (id >= MLX5_MAX_TUNNELS) { 10076 mlx5_ipool_free(ipool, id); 10077 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10078 return NULL; 10079 } 10080 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10081 priv->sh, 10082 mlx5_flow_tunnel_grp2tbl_create_cb, 10083 mlx5_flow_tunnel_grp2tbl_match_cb, 10084 mlx5_flow_tunnel_grp2tbl_remove_cb, 10085 mlx5_flow_tunnel_grp2tbl_clone_cb, 10086 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10087 if (!tunnel->groups) { 10088 mlx5_ipool_free(ipool, id); 10089 return NULL; 10090 } 10091 /* initiate new PMD tunnel */ 10092 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10093 tunnel->tunnel_id = id; 10094 tunnel->action.type = (typeof(tunnel->action.type)) 10095 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10096 tunnel->action.conf = tunnel; 10097 tunnel->item.type = (typeof(tunnel->item.type)) 10098 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10099 tunnel->item.spec = tunnel; 10100 tunnel->item.last = NULL; 10101 tunnel->item.mask = NULL; 10102 10103 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10104 dev->data->port_id, tunnel->tunnel_id); 10105 10106 return tunnel; 10107 } 10108 10109 struct tunnel_db_get_tunnel_ctx { 10110 const struct rte_flow_tunnel *app_tunnel; 10111 struct mlx5_flow_tunnel *tunnel; 10112 }; 10113 10114 static bool get_tunnel_match(struct rte_eth_dev *dev, 10115 struct mlx5_flow_tunnel *tunnel, const void *x) 10116 { 10117 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10118 10119 RTE_SET_USED(dev); 10120 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10121 sizeof(*ctx->app_tunnel)); 10122 } 10123 10124 static void get_tunnel_hit(struct rte_eth_dev *dev, 10125 struct mlx5_flow_tunnel *tunnel, void *x) 10126 { 10127 /* called under tunnel spinlock protection */ 10128 struct tunnel_db_get_tunnel_ctx *ctx = x; 10129 10130 RTE_SET_USED(dev); 10131 tunnel->refctn++; 10132 ctx->tunnel = tunnel; 10133 } 10134 10135 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10136 { 10137 /* called under tunnel spinlock protection */ 10138 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10139 struct tunnel_db_get_tunnel_ctx *ctx = x; 10140 10141 rte_spinlock_unlock(&thub->sl); 10142 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10143 rte_spinlock_lock(&thub->sl); 10144 if (ctx->tunnel) { 10145 ctx->tunnel->refctn = 1; 10146 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10147 } 10148 } 10149 10150 10151 static int 10152 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10153 const struct rte_flow_tunnel *app_tunnel, 10154 struct mlx5_flow_tunnel **tunnel) 10155 { 10156 struct tunnel_db_get_tunnel_ctx ctx = { 10157 .app_tunnel = app_tunnel, 10158 }; 10159 10160 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10161 get_tunnel_miss, &ctx, true); 10162 *tunnel = ctx.tunnel; 10163 return ctx.tunnel ? 0 : -ENOMEM; 10164 } 10165 10166 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10167 { 10168 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10169 10170 if (!thub) 10171 return; 10172 if (!LIST_EMPTY(&thub->tunnels)) 10173 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10174 mlx5_hlist_destroy(thub->groups); 10175 mlx5_free(thub); 10176 } 10177 10178 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10179 { 10180 int err; 10181 struct mlx5_flow_tunnel_hub *thub; 10182 10183 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10184 0, SOCKET_ID_ANY); 10185 if (!thub) 10186 return -ENOMEM; 10187 LIST_INIT(&thub->tunnels); 10188 rte_spinlock_init(&thub->sl); 10189 thub->groups = mlx5_hlist_create("flow groups", 64, 10190 false, true, sh, 10191 mlx5_flow_tunnel_grp2tbl_create_cb, 10192 mlx5_flow_tunnel_grp2tbl_match_cb, 10193 mlx5_flow_tunnel_grp2tbl_remove_cb, 10194 mlx5_flow_tunnel_grp2tbl_clone_cb, 10195 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10196 if (!thub->groups) { 10197 err = -rte_errno; 10198 goto err; 10199 } 10200 sh->tunnel_hub = thub; 10201 10202 return 0; 10203 10204 err: 10205 if (thub->groups) 10206 mlx5_hlist_destroy(thub->groups); 10207 if (thub) 10208 mlx5_free(thub); 10209 return err; 10210 } 10211 10212 static inline int 10213 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10214 struct rte_flow_tunnel *tunnel, 10215 struct rte_flow_error *error) 10216 { 10217 struct mlx5_priv *priv = dev->data->dev_private; 10218 10219 if (!priv->sh->config.dv_flow_en) 10220 return rte_flow_error_set(error, ENOTSUP, 10221 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10222 "flow DV interface is off"); 10223 if (!is_tunnel_offload_active(dev)) 10224 return rte_flow_error_set(error, ENOTSUP, 10225 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10226 "tunnel offload was not activated"); 10227 if (!tunnel) 10228 return rte_flow_error_set(error, EINVAL, 10229 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10230 "no application tunnel"); 10231 switch (tunnel->type) { 10232 default: 10233 return rte_flow_error_set(error, EINVAL, 10234 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10235 "unsupported tunnel type"); 10236 case RTE_FLOW_ITEM_TYPE_VXLAN: 10237 case RTE_FLOW_ITEM_TYPE_GRE: 10238 case RTE_FLOW_ITEM_TYPE_NVGRE: 10239 case RTE_FLOW_ITEM_TYPE_GENEVE: 10240 break; 10241 } 10242 return 0; 10243 } 10244 10245 static int 10246 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10247 struct rte_flow_tunnel *app_tunnel, 10248 struct rte_flow_action **actions, 10249 uint32_t *num_of_actions, 10250 struct rte_flow_error *error) 10251 { 10252 struct mlx5_flow_tunnel *tunnel; 10253 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10254 10255 if (ret) 10256 return ret; 10257 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10258 if (ret < 0) { 10259 return rte_flow_error_set(error, ret, 10260 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10261 "failed to initialize pmd tunnel"); 10262 } 10263 *actions = &tunnel->action; 10264 *num_of_actions = 1; 10265 return 0; 10266 } 10267 10268 static int 10269 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10270 struct rte_flow_tunnel *app_tunnel, 10271 struct rte_flow_item **items, 10272 uint32_t *num_of_items, 10273 struct rte_flow_error *error) 10274 { 10275 struct mlx5_flow_tunnel *tunnel; 10276 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10277 10278 if (ret) 10279 return ret; 10280 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10281 if (ret < 0) { 10282 return rte_flow_error_set(error, ret, 10283 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10284 "failed to initialize pmd tunnel"); 10285 } 10286 *items = &tunnel->item; 10287 *num_of_items = 1; 10288 return 0; 10289 } 10290 10291 struct tunnel_db_element_release_ctx { 10292 struct rte_flow_item *items; 10293 struct rte_flow_action *actions; 10294 uint32_t num_elements; 10295 struct rte_flow_error *error; 10296 int ret; 10297 }; 10298 10299 static bool 10300 tunnel_element_release_match(struct rte_eth_dev *dev, 10301 struct mlx5_flow_tunnel *tunnel, const void *x) 10302 { 10303 const struct tunnel_db_element_release_ctx *ctx = x; 10304 10305 RTE_SET_USED(dev); 10306 if (ctx->num_elements != 1) 10307 return false; 10308 else if (ctx->items) 10309 return ctx->items == &tunnel->item; 10310 else if (ctx->actions) 10311 return ctx->actions == &tunnel->action; 10312 10313 return false; 10314 } 10315 10316 static void 10317 tunnel_element_release_hit(struct rte_eth_dev *dev, 10318 struct mlx5_flow_tunnel *tunnel, void *x) 10319 { 10320 struct tunnel_db_element_release_ctx *ctx = x; 10321 ctx->ret = 0; 10322 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10323 mlx5_flow_tunnel_free(dev, tunnel); 10324 } 10325 10326 static void 10327 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10328 { 10329 struct tunnel_db_element_release_ctx *ctx = x; 10330 RTE_SET_USED(dev); 10331 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10332 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10333 "invalid argument"); 10334 } 10335 10336 static int 10337 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10338 struct rte_flow_item *pmd_items, 10339 uint32_t num_items, struct rte_flow_error *err) 10340 { 10341 struct tunnel_db_element_release_ctx ctx = { 10342 .items = pmd_items, 10343 .actions = NULL, 10344 .num_elements = num_items, 10345 .error = err, 10346 }; 10347 10348 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10349 tunnel_element_release_hit, 10350 tunnel_element_release_miss, &ctx, false); 10351 10352 return ctx.ret; 10353 } 10354 10355 static int 10356 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10357 struct rte_flow_action *pmd_actions, 10358 uint32_t num_actions, struct rte_flow_error *err) 10359 { 10360 struct tunnel_db_element_release_ctx ctx = { 10361 .items = NULL, 10362 .actions = pmd_actions, 10363 .num_elements = num_actions, 10364 .error = err, 10365 }; 10366 10367 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10368 tunnel_element_release_hit, 10369 tunnel_element_release_miss, &ctx, false); 10370 10371 return ctx.ret; 10372 } 10373 10374 static int 10375 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10376 struct rte_mbuf *m, 10377 struct rte_flow_restore_info *info, 10378 struct rte_flow_error *err) 10379 { 10380 uint64_t ol_flags = m->ol_flags; 10381 const struct mlx5_flow_tbl_data_entry *tble; 10382 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10383 10384 if (!is_tunnel_offload_active(dev)) { 10385 info->flags = 0; 10386 return 0; 10387 } 10388 10389 if ((ol_flags & mask) != mask) 10390 goto err; 10391 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10392 if (!tble) { 10393 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10394 dev->data->port_id, m->hash.fdir.hi); 10395 goto err; 10396 } 10397 MLX5_ASSERT(tble->tunnel); 10398 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10399 info->group_id = tble->group_id; 10400 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10401 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10402 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10403 10404 return 0; 10405 10406 err: 10407 return rte_flow_error_set(err, EINVAL, 10408 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10409 "failed to get restore info"); 10410 } 10411 10412 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 10413 static int 10414 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 10415 __rte_unused struct rte_flow_tunnel *app_tunnel, 10416 __rte_unused struct rte_flow_action **actions, 10417 __rte_unused uint32_t *num_of_actions, 10418 __rte_unused struct rte_flow_error *error) 10419 { 10420 return -ENOTSUP; 10421 } 10422 10423 static int 10424 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 10425 __rte_unused struct rte_flow_tunnel *app_tunnel, 10426 __rte_unused struct rte_flow_item **items, 10427 __rte_unused uint32_t *num_of_items, 10428 __rte_unused struct rte_flow_error *error) 10429 { 10430 return -ENOTSUP; 10431 } 10432 10433 static int 10434 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 10435 __rte_unused struct rte_flow_item *pmd_items, 10436 __rte_unused uint32_t num_items, 10437 __rte_unused struct rte_flow_error *err) 10438 { 10439 return -ENOTSUP; 10440 } 10441 10442 static int 10443 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 10444 __rte_unused struct rte_flow_action *pmd_action, 10445 __rte_unused uint32_t num_actions, 10446 __rte_unused struct rte_flow_error *err) 10447 { 10448 return -ENOTSUP; 10449 } 10450 10451 static int 10452 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 10453 __rte_unused struct rte_mbuf *m, 10454 __rte_unused struct rte_flow_restore_info *i, 10455 __rte_unused struct rte_flow_error *err) 10456 { 10457 return -ENOTSUP; 10458 } 10459 10460 static int 10461 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 10462 __rte_unused struct rte_flow *flow, 10463 __rte_unused const struct rte_flow_attr *attr, 10464 __rte_unused const struct rte_flow_action *actions, 10465 __rte_unused uint32_t flow_idx, 10466 __rte_unused const struct mlx5_flow_tunnel *tunnel, 10467 __rte_unused struct tunnel_default_miss_ctx *ctx, 10468 __rte_unused struct rte_flow_error *error) 10469 { 10470 return -ENOTSUP; 10471 } 10472 10473 static struct mlx5_flow_tunnel * 10474 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 10475 __rte_unused uint32_t id) 10476 { 10477 return NULL; 10478 } 10479 10480 static void 10481 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 10482 __rte_unused struct mlx5_flow_tunnel *tunnel) 10483 { 10484 } 10485 10486 static uint32_t 10487 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 10488 __rte_unused const struct mlx5_flow_tunnel *t, 10489 __rte_unused uint32_t group, 10490 __rte_unused uint32_t *table, 10491 struct rte_flow_error *error) 10492 { 10493 return rte_flow_error_set(error, ENOTSUP, 10494 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10495 "tunnel offload requires DV support"); 10496 } 10497 10498 void 10499 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 10500 __rte_unused uint16_t port_id) 10501 { 10502 } 10503 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10504 10505 /* Flex flow item API */ 10506 static struct rte_flow_item_flex_handle * 10507 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 10508 const struct rte_flow_item_flex_conf *conf, 10509 struct rte_flow_error *error) 10510 { 10511 static const char err_msg[] = "flex item creation unsupported"; 10512 struct rte_flow_attr attr = { .transfer = 0 }; 10513 const struct mlx5_flow_driver_ops *fops = 10514 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10515 10516 if (!fops->item_create) { 10517 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10518 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10519 NULL, err_msg); 10520 return NULL; 10521 } 10522 return fops->item_create(dev, conf, error); 10523 } 10524 10525 static int 10526 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 10527 const struct rte_flow_item_flex_handle *handle, 10528 struct rte_flow_error *error) 10529 { 10530 static const char err_msg[] = "flex item release unsupported"; 10531 struct rte_flow_attr attr = { .transfer = 0 }; 10532 const struct mlx5_flow_driver_ops *fops = 10533 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10534 10535 if (!fops->item_release) { 10536 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10537 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10538 NULL, err_msg); 10539 return -rte_errno; 10540 } 10541 return fops->item_release(dev, handle, error); 10542 } 10543 10544 static void 10545 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 10546 { 10547 int ret; 10548 struct rte_flow_error error; 10549 10550 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10551 char *item_name; 10552 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 10553 sizeof(item_name), 10554 (void *)(uintptr_t)item->type, &error); 10555 if (ret > 0) 10556 printf("%s ", item_name); 10557 else 10558 printf("%d\n", (int)item->type); 10559 } 10560 printf("END\n"); 10561 } 10562 10563 static int 10564 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 10565 { 10566 const struct rte_flow_item_udp *spec = udp_item->spec; 10567 const struct rte_flow_item_udp *mask = udp_item->mask; 10568 uint16_t udp_dport = 0; 10569 10570 if (spec != NULL) { 10571 if (!mask) 10572 mask = &rte_flow_item_udp_mask; 10573 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 10574 mask->hdr.dst_port); 10575 } 10576 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 10577 } 10578 10579 static const struct mlx5_flow_expand_node * 10580 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 10581 unsigned int item_idx, 10582 const struct mlx5_flow_expand_node graph[], 10583 const struct mlx5_flow_expand_node *node) 10584 { 10585 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 10586 10587 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 10588 node != NULL && 10589 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 10590 /* 10591 * The expansion node is VXLAN and it is also the last 10592 * expandable item in the pattern, so need to continue 10593 * expansion of the inner tunnel. 10594 */ 10595 MLX5_ASSERT(item_idx > 0); 10596 prev_item = pattern + item_idx - 1; 10597 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 10598 if (mlx5_flow_is_std_vxlan_port(prev_item)) 10599 return &graph[MLX5_EXPANSION_STD_VXLAN]; 10600 return &graph[MLX5_EXPANSION_L3_VXLAN]; 10601 } 10602 return node; 10603 } 10604 10605 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 10606 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 10607 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 10608 }; 10609 10610 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 10611 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 10612 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 10613 { 9, 10, 11 }, { 12, 13, 14 }, 10614 }; 10615 10616 /** 10617 * Discover the number of available flow priorities. 10618 * 10619 * @param dev 10620 * Ethernet device. 10621 * 10622 * @return 10623 * On success, number of available flow priorities. 10624 * On failure, a negative errno-style code and rte_errno is set. 10625 */ 10626 int 10627 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 10628 { 10629 static const uint16_t vprio[] = {8, 16}; 10630 const struct mlx5_priv *priv = dev->data->dev_private; 10631 const struct mlx5_flow_driver_ops *fops; 10632 enum mlx5_flow_drv_type type; 10633 int ret; 10634 10635 type = mlx5_flow_os_get_type(); 10636 if (type == MLX5_FLOW_TYPE_MAX) { 10637 type = MLX5_FLOW_TYPE_VERBS; 10638 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 10639 type = MLX5_FLOW_TYPE_DV; 10640 } 10641 fops = flow_get_drv_ops(type); 10642 if (fops->discover_priorities == NULL) { 10643 DRV_LOG(ERR, "Priority discovery not supported"); 10644 rte_errno = ENOTSUP; 10645 return -rte_errno; 10646 } 10647 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 10648 if (ret < 0) 10649 return ret; 10650 switch (ret) { 10651 case 8: 10652 ret = RTE_DIM(priority_map_3); 10653 break; 10654 case 16: 10655 ret = RTE_DIM(priority_map_5); 10656 break; 10657 default: 10658 rte_errno = ENOTSUP; 10659 DRV_LOG(ERR, 10660 "port %u maximum priority: %d expected 8/16", 10661 dev->data->port_id, ret); 10662 return -rte_errno; 10663 } 10664 DRV_LOG(INFO, "port %u supported flow priorities:" 10665 " 0-%d for ingress or egress root table," 10666 " 0-%d for non-root table or transfer root table.", 10667 dev->data->port_id, ret - 2, 10668 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 10669 return ret; 10670 } 10671 10672 /** 10673 * Adjust flow priority based on the highest layer and the request priority. 10674 * 10675 * @param[in] dev 10676 * Pointer to the Ethernet device structure. 10677 * @param[in] priority 10678 * The rule base priority. 10679 * @param[in] subpriority 10680 * The priority based on the items. 10681 * 10682 * @return 10683 * The new priority. 10684 */ 10685 uint32_t 10686 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10687 uint32_t subpriority) 10688 { 10689 uint32_t res = 0; 10690 struct mlx5_priv *priv = dev->data->dev_private; 10691 10692 switch (priv->sh->flow_max_priority) { 10693 case RTE_DIM(priority_map_3): 10694 res = priority_map_3[priority][subpriority]; 10695 break; 10696 case RTE_DIM(priority_map_5): 10697 res = priority_map_5[priority][subpriority]; 10698 break; 10699 } 10700 return res; 10701 } 10702