1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see RTE_ETH_RSS_* definitions). 102 */ 103 uint64_t node_flags; 104 /**< 105 * Bit-fields that define how the node is used in the expansion. 106 * (see MLX5_EXPANSION_NODE_* definitions). 107 */ 108 }; 109 110 /* Optional expand field. The expansion alg will not go deeper. */ 111 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 112 113 /* The node is not added implicitly as expansion to the flow pattern. 114 * If the node type does not match the flow pattern item type, the 115 * expansion alg will go deeper to its next items. 116 * In the current implementation, the list of next nodes indexes can 117 * have up to one node with this flag set and it has to be the last 118 * node index (before the list terminator). 119 */ 120 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 121 122 /** Object returned by mlx5_flow_expand_rss(). */ 123 struct mlx5_flow_expand_rss { 124 uint32_t entries; 125 /**< Number of entries @p patterns and @p priorities. */ 126 struct { 127 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 128 uint32_t priority; /**< Priority offset for each expansion. */ 129 } entry[]; 130 }; 131 132 static void 133 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 134 135 static const struct mlx5_flow_expand_node * 136 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 137 unsigned int item_idx, 138 const struct mlx5_flow_expand_node graph[], 139 const struct mlx5_flow_expand_node *node); 140 141 static bool 142 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 143 { 144 switch (item->type) { 145 case RTE_FLOW_ITEM_TYPE_ETH: 146 case RTE_FLOW_ITEM_TYPE_VLAN: 147 case RTE_FLOW_ITEM_TYPE_IPV4: 148 case RTE_FLOW_ITEM_TYPE_IPV6: 149 case RTE_FLOW_ITEM_TYPE_UDP: 150 case RTE_FLOW_ITEM_TYPE_TCP: 151 case RTE_FLOW_ITEM_TYPE_VXLAN: 152 case RTE_FLOW_ITEM_TYPE_NVGRE: 153 case RTE_FLOW_ITEM_TYPE_GRE: 154 case RTE_FLOW_ITEM_TYPE_GENEVE: 155 case RTE_FLOW_ITEM_TYPE_MPLS: 156 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 157 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 158 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 159 case RTE_FLOW_ITEM_TYPE_GTP: 160 return true; 161 default: 162 break; 163 } 164 return false; 165 } 166 167 /** 168 * Network Service Header (NSH) and its next protocol values 169 * are described in RFC-8393. 170 */ 171 static enum rte_flow_item_type 172 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 173 { 174 enum rte_flow_item_type type; 175 176 switch (proto_mask & proto_spec) { 177 case RTE_VXLAN_GPE_TYPE_IPV4: 178 type = RTE_FLOW_ITEM_TYPE_IPV4; 179 break; 180 case RTE_VXLAN_GPE_TYPE_IPV6: 181 type = RTE_VXLAN_GPE_TYPE_IPV6; 182 break; 183 case RTE_VXLAN_GPE_TYPE_ETH: 184 type = RTE_FLOW_ITEM_TYPE_ETH; 185 break; 186 default: 187 type = RTE_FLOW_ITEM_TYPE_END; 188 } 189 return type; 190 } 191 192 static enum rte_flow_item_type 193 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 194 { 195 enum rte_flow_item_type type; 196 197 switch (proto_mask & proto_spec) { 198 case IPPROTO_UDP: 199 type = RTE_FLOW_ITEM_TYPE_UDP; 200 break; 201 case IPPROTO_TCP: 202 type = RTE_FLOW_ITEM_TYPE_TCP; 203 break; 204 case IPPROTO_IP: 205 type = RTE_FLOW_ITEM_TYPE_IPV4; 206 break; 207 case IPPROTO_IPV6: 208 type = RTE_FLOW_ITEM_TYPE_IPV6; 209 break; 210 default: 211 type = RTE_FLOW_ITEM_TYPE_END; 212 } 213 return type; 214 } 215 216 static enum rte_flow_item_type 217 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 218 rte_be16_t type_mask, bool is_tunnel) 219 { 220 enum rte_flow_item_type type; 221 222 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 223 case RTE_ETHER_TYPE_TEB: 224 type = is_tunnel ? 225 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 226 break; 227 case RTE_ETHER_TYPE_VLAN: 228 type = !is_tunnel ? 229 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 230 break; 231 case RTE_ETHER_TYPE_IPV4: 232 type = RTE_FLOW_ITEM_TYPE_IPV4; 233 break; 234 case RTE_ETHER_TYPE_IPV6: 235 type = RTE_FLOW_ITEM_TYPE_IPV6; 236 break; 237 default: 238 type = RTE_FLOW_ITEM_TYPE_END; 239 } 240 return type; 241 } 242 243 static enum rte_flow_item_type 244 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 245 { 246 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 247 do { \ 248 const void *m = item->mask; \ 249 const void *s = item->spec; \ 250 mask = m ? \ 251 ((const struct rte_flow_item_##type *)m)->fld : \ 252 rte_flow_item_##type##_mask.fld; \ 253 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 254 } while (0) 255 256 enum rte_flow_item_type ret; 257 uint16_t spec, mask; 258 259 if (item == NULL || item->spec == NULL) 260 return RTE_FLOW_ITEM_TYPE_VOID; 261 switch (item->type) { 262 case RTE_FLOW_ITEM_TYPE_ETH: 263 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 264 if (!mask) 265 return RTE_FLOW_ITEM_TYPE_VOID; 266 ret = mlx5_ethertype_to_item_type(spec, mask, false); 267 break; 268 case RTE_FLOW_ITEM_TYPE_VLAN: 269 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 270 if (!mask) 271 return RTE_FLOW_ITEM_TYPE_VOID; 272 ret = mlx5_ethertype_to_item_type(spec, mask, false); 273 break; 274 case RTE_FLOW_ITEM_TYPE_IPV4: 275 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 276 if (!mask) 277 return RTE_FLOW_ITEM_TYPE_VOID; 278 ret = mlx5_inet_proto_to_item_type(spec, mask); 279 break; 280 case RTE_FLOW_ITEM_TYPE_IPV6: 281 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 282 if (!mask) 283 return RTE_FLOW_ITEM_TYPE_VOID; 284 ret = mlx5_inet_proto_to_item_type(spec, mask); 285 break; 286 case RTE_FLOW_ITEM_TYPE_GENEVE: 287 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 288 ret = mlx5_ethertype_to_item_type(spec, mask, true); 289 break; 290 case RTE_FLOW_ITEM_TYPE_GRE: 291 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 292 ret = mlx5_ethertype_to_item_type(spec, mask, true); 293 break; 294 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 295 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 296 ret = mlx5_nsh_proto_to_item_type(spec, mask); 297 break; 298 default: 299 ret = RTE_FLOW_ITEM_TYPE_VOID; 300 break; 301 } 302 return ret; 303 #undef MLX5_XSET_ITEM_MASK_SPEC 304 } 305 306 static const int * 307 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 308 const int *next_node) 309 { 310 const struct mlx5_flow_expand_node *node = NULL; 311 const int *next = next_node; 312 313 while (next && *next) { 314 /* 315 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 316 * flag set, because they were not found in the flow pattern. 317 */ 318 node = &graph[*next]; 319 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 320 break; 321 next = node->next; 322 } 323 return next; 324 } 325 326 #define MLX5_RSS_EXP_ELT_N 16 327 328 /** 329 * Expand RSS flows into several possible flows according to the RSS hash 330 * fields requested and the driver capabilities. 331 * 332 * @param[out] buf 333 * Buffer to store the result expansion. 334 * @param[in] size 335 * Buffer size in bytes. If 0, @p buf can be NULL. 336 * @param[in] pattern 337 * User flow pattern. 338 * @param[in] types 339 * RSS types to expand (see RTE_ETH_RSS_* definitions). 340 * @param[in] graph 341 * Input graph to expand @p pattern according to @p types. 342 * @param[in] graph_root_index 343 * Index of root node in @p graph, typically 0. 344 * 345 * @return 346 * A positive value representing the size of @p buf in bytes regardless of 347 * @p size on success, a negative errno value otherwise and rte_errno is 348 * set, the following errors are defined: 349 * 350 * -E2BIG: graph-depth @p graph is too deep. 351 * -EINVAL: @p size has not enough space for expanded pattern. 352 */ 353 static int 354 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 355 const struct rte_flow_item *pattern, uint64_t types, 356 const struct mlx5_flow_expand_node graph[], 357 int graph_root_index) 358 { 359 const struct rte_flow_item *item; 360 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 361 const int *next_node; 362 const int *stack[MLX5_RSS_EXP_ELT_N]; 363 int stack_pos = 0; 364 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 365 unsigned int i, item_idx, last_expand_item_idx = 0; 366 size_t lsize; 367 size_t user_pattern_size = 0; 368 void *addr = NULL; 369 const struct mlx5_flow_expand_node *next = NULL; 370 struct rte_flow_item missed_item; 371 int missed = 0; 372 int elt = 0; 373 const struct rte_flow_item *last_expand_item = NULL; 374 375 memset(&missed_item, 0, sizeof(missed_item)); 376 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 377 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 378 if (lsize > size) 379 return -EINVAL; 380 buf->entry[0].priority = 0; 381 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 382 buf->entries = 0; 383 addr = buf->entry[0].pattern; 384 for (item = pattern, item_idx = 0; 385 item->type != RTE_FLOW_ITEM_TYPE_END; 386 item++, item_idx++) { 387 if (!mlx5_flow_is_rss_expandable_item(item)) { 388 user_pattern_size += sizeof(*item); 389 continue; 390 } 391 last_expand_item = item; 392 last_expand_item_idx = item_idx; 393 i = 0; 394 while (node->next && node->next[i]) { 395 next = &graph[node->next[i]]; 396 if (next->type == item->type) 397 break; 398 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 399 node = next; 400 i = 0; 401 } else { 402 ++i; 403 } 404 } 405 if (next) 406 node = next; 407 user_pattern_size += sizeof(*item); 408 } 409 user_pattern_size += sizeof(*item); /* Handle END item. */ 410 lsize += user_pattern_size; 411 if (lsize > size) 412 return -EINVAL; 413 /* Copy the user pattern in the first entry of the buffer. */ 414 rte_memcpy(addr, pattern, user_pattern_size); 415 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 416 buf->entries = 1; 417 /* Start expanding. */ 418 memset(flow_items, 0, sizeof(flow_items)); 419 user_pattern_size -= sizeof(*item); 420 /* 421 * Check if the last valid item has spec set, need complete pattern, 422 * and the pattern can be used for expansion. 423 */ 424 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 425 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 426 /* Item type END indicates expansion is not required. */ 427 return lsize; 428 } 429 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 430 next = NULL; 431 missed = 1; 432 i = 0; 433 while (node->next && node->next[i]) { 434 next = &graph[node->next[i]]; 435 if (next->type == missed_item.type) { 436 flow_items[0].type = missed_item.type; 437 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 438 break; 439 } 440 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 441 node = next; 442 i = 0; 443 } else { 444 ++i; 445 } 446 next = NULL; 447 } 448 } 449 if (next && missed) { 450 elt = 2; /* missed item + item end. */ 451 node = next; 452 lsize += elt * sizeof(*item) + user_pattern_size; 453 if (lsize > size) 454 return -EINVAL; 455 if (node->rss_types & types) { 456 buf->entry[buf->entries].priority = 1; 457 buf->entry[buf->entries].pattern = addr; 458 buf->entries++; 459 rte_memcpy(addr, buf->entry[0].pattern, 460 user_pattern_size); 461 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 462 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 463 addr = (void *)(((uintptr_t)addr) + 464 elt * sizeof(*item)); 465 } 466 } else if (last_expand_item != NULL) { 467 node = mlx5_flow_expand_rss_adjust_node(pattern, 468 last_expand_item_idx, graph, node); 469 } 470 memset(flow_items, 0, sizeof(flow_items)); 471 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 472 node->next); 473 stack[stack_pos] = next_node; 474 node = next_node ? &graph[*next_node] : NULL; 475 while (node) { 476 flow_items[stack_pos].type = node->type; 477 if (node->rss_types & types) { 478 size_t n; 479 /* 480 * compute the number of items to copy from the 481 * expansion and copy it. 482 * When the stack_pos is 0, there are 1 element in it, 483 * plus the addition END item. 484 */ 485 elt = stack_pos + 2; 486 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 487 lsize += elt * sizeof(*item) + user_pattern_size; 488 if (lsize > size) 489 return -EINVAL; 490 n = elt * sizeof(*item); 491 buf->entry[buf->entries].priority = 492 stack_pos + 1 + missed; 493 buf->entry[buf->entries].pattern = addr; 494 buf->entries++; 495 rte_memcpy(addr, buf->entry[0].pattern, 496 user_pattern_size); 497 addr = (void *)(((uintptr_t)addr) + 498 user_pattern_size); 499 rte_memcpy(addr, &missed_item, 500 missed * sizeof(*item)); 501 addr = (void *)(((uintptr_t)addr) + 502 missed * sizeof(*item)); 503 rte_memcpy(addr, flow_items, n); 504 addr = (void *)(((uintptr_t)addr) + n); 505 } 506 /* Go deeper. */ 507 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 508 node->next) { 509 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 510 node->next); 511 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 512 rte_errno = E2BIG; 513 return -rte_errno; 514 } 515 stack[stack_pos] = next_node; 516 } else if (*(next_node + 1)) { 517 /* Follow up with the next possibility. */ 518 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 519 ++next_node); 520 } else if (!stack_pos) { 521 /* 522 * Completing the traverse over the different paths. 523 * The next_node is advanced to the terminator. 524 */ 525 ++next_node; 526 } else { 527 /* Move to the next path. */ 528 while (stack_pos) { 529 next_node = stack[--stack_pos]; 530 next_node++; 531 if (*next_node) 532 break; 533 } 534 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 535 next_node); 536 stack[stack_pos] = next_node; 537 } 538 node = next_node && *next_node ? &graph[*next_node] : NULL; 539 }; 540 return lsize; 541 } 542 543 enum mlx5_expansion { 544 MLX5_EXPANSION_ROOT, 545 MLX5_EXPANSION_ROOT_OUTER, 546 MLX5_EXPANSION_OUTER_ETH, 547 MLX5_EXPANSION_OUTER_VLAN, 548 MLX5_EXPANSION_OUTER_IPV4, 549 MLX5_EXPANSION_OUTER_IPV4_UDP, 550 MLX5_EXPANSION_OUTER_IPV4_TCP, 551 MLX5_EXPANSION_OUTER_IPV6, 552 MLX5_EXPANSION_OUTER_IPV6_UDP, 553 MLX5_EXPANSION_OUTER_IPV6_TCP, 554 MLX5_EXPANSION_VXLAN, 555 MLX5_EXPANSION_STD_VXLAN, 556 MLX5_EXPANSION_L3_VXLAN, 557 MLX5_EXPANSION_VXLAN_GPE, 558 MLX5_EXPANSION_GRE, 559 MLX5_EXPANSION_NVGRE, 560 MLX5_EXPANSION_GRE_KEY, 561 MLX5_EXPANSION_MPLS, 562 MLX5_EXPANSION_ETH, 563 MLX5_EXPANSION_VLAN, 564 MLX5_EXPANSION_IPV4, 565 MLX5_EXPANSION_IPV4_UDP, 566 MLX5_EXPANSION_IPV4_TCP, 567 MLX5_EXPANSION_IPV6, 568 MLX5_EXPANSION_IPV6_UDP, 569 MLX5_EXPANSION_IPV6_TCP, 570 MLX5_EXPANSION_IPV6_FRAG_EXT, 571 MLX5_EXPANSION_GTP, 572 MLX5_EXPANSION_GENEVE, 573 }; 574 575 /** Supported expansion of items. */ 576 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 577 [MLX5_EXPANSION_ROOT] = { 578 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 579 MLX5_EXPANSION_IPV4, 580 MLX5_EXPANSION_IPV6), 581 .type = RTE_FLOW_ITEM_TYPE_END, 582 }, 583 [MLX5_EXPANSION_ROOT_OUTER] = { 584 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 585 MLX5_EXPANSION_OUTER_IPV4, 586 MLX5_EXPANSION_OUTER_IPV6), 587 .type = RTE_FLOW_ITEM_TYPE_END, 588 }, 589 [MLX5_EXPANSION_OUTER_ETH] = { 590 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 591 .type = RTE_FLOW_ITEM_TYPE_ETH, 592 .rss_types = 0, 593 }, 594 [MLX5_EXPANSION_OUTER_VLAN] = { 595 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 596 MLX5_EXPANSION_OUTER_IPV6), 597 .type = RTE_FLOW_ITEM_TYPE_VLAN, 598 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 599 }, 600 [MLX5_EXPANSION_OUTER_IPV4] = { 601 .next = MLX5_FLOW_EXPAND_RSS_NEXT 602 (MLX5_EXPANSION_OUTER_IPV4_UDP, 603 MLX5_EXPANSION_OUTER_IPV4_TCP, 604 MLX5_EXPANSION_GRE, 605 MLX5_EXPANSION_NVGRE, 606 MLX5_EXPANSION_IPV4, 607 MLX5_EXPANSION_IPV6), 608 .type = RTE_FLOW_ITEM_TYPE_IPV4, 609 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 610 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 611 }, 612 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 613 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 614 MLX5_EXPANSION_VXLAN_GPE, 615 MLX5_EXPANSION_MPLS, 616 MLX5_EXPANSION_GENEVE, 617 MLX5_EXPANSION_GTP), 618 .type = RTE_FLOW_ITEM_TYPE_UDP, 619 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 620 }, 621 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 622 .type = RTE_FLOW_ITEM_TYPE_TCP, 623 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 624 }, 625 [MLX5_EXPANSION_OUTER_IPV6] = { 626 .next = MLX5_FLOW_EXPAND_RSS_NEXT 627 (MLX5_EXPANSION_OUTER_IPV6_UDP, 628 MLX5_EXPANSION_OUTER_IPV6_TCP, 629 MLX5_EXPANSION_IPV4, 630 MLX5_EXPANSION_IPV6, 631 MLX5_EXPANSION_GRE, 632 MLX5_EXPANSION_NVGRE), 633 .type = RTE_FLOW_ITEM_TYPE_IPV6, 634 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 635 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 636 }, 637 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 638 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 639 MLX5_EXPANSION_VXLAN_GPE, 640 MLX5_EXPANSION_MPLS, 641 MLX5_EXPANSION_GENEVE, 642 MLX5_EXPANSION_GTP), 643 .type = RTE_FLOW_ITEM_TYPE_UDP, 644 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 645 }, 646 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 647 .type = RTE_FLOW_ITEM_TYPE_TCP, 648 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 649 }, 650 [MLX5_EXPANSION_VXLAN] = { 651 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 652 MLX5_EXPANSION_IPV4, 653 MLX5_EXPANSION_IPV6), 654 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 655 }, 656 [MLX5_EXPANSION_STD_VXLAN] = { 657 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 658 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 659 }, 660 [MLX5_EXPANSION_L3_VXLAN] = { 661 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 662 MLX5_EXPANSION_IPV6), 663 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 664 }, 665 [MLX5_EXPANSION_VXLAN_GPE] = { 666 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 667 MLX5_EXPANSION_IPV4, 668 MLX5_EXPANSION_IPV6), 669 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 670 }, 671 [MLX5_EXPANSION_GRE] = { 672 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 673 MLX5_EXPANSION_IPV4, 674 MLX5_EXPANSION_IPV6, 675 MLX5_EXPANSION_GRE_KEY, 676 MLX5_EXPANSION_MPLS), 677 .type = RTE_FLOW_ITEM_TYPE_GRE, 678 }, 679 [MLX5_EXPANSION_GRE_KEY] = { 680 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 681 MLX5_EXPANSION_IPV6, 682 MLX5_EXPANSION_MPLS), 683 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 684 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 685 }, 686 [MLX5_EXPANSION_NVGRE] = { 687 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 688 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 689 }, 690 [MLX5_EXPANSION_MPLS] = { 691 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 692 MLX5_EXPANSION_IPV6, 693 MLX5_EXPANSION_ETH), 694 .type = RTE_FLOW_ITEM_TYPE_MPLS, 695 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 696 }, 697 [MLX5_EXPANSION_ETH] = { 698 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 699 .type = RTE_FLOW_ITEM_TYPE_ETH, 700 }, 701 [MLX5_EXPANSION_VLAN] = { 702 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 703 MLX5_EXPANSION_IPV6), 704 .type = RTE_FLOW_ITEM_TYPE_VLAN, 705 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 706 }, 707 [MLX5_EXPANSION_IPV4] = { 708 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 709 MLX5_EXPANSION_IPV4_TCP), 710 .type = RTE_FLOW_ITEM_TYPE_IPV4, 711 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 712 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 713 }, 714 [MLX5_EXPANSION_IPV4_UDP] = { 715 .type = RTE_FLOW_ITEM_TYPE_UDP, 716 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 717 }, 718 [MLX5_EXPANSION_IPV4_TCP] = { 719 .type = RTE_FLOW_ITEM_TYPE_TCP, 720 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 721 }, 722 [MLX5_EXPANSION_IPV6] = { 723 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 724 MLX5_EXPANSION_IPV6_TCP, 725 MLX5_EXPANSION_IPV6_FRAG_EXT), 726 .type = RTE_FLOW_ITEM_TYPE_IPV6, 727 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 728 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 729 }, 730 [MLX5_EXPANSION_IPV6_UDP] = { 731 .type = RTE_FLOW_ITEM_TYPE_UDP, 732 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 733 }, 734 [MLX5_EXPANSION_IPV6_TCP] = { 735 .type = RTE_FLOW_ITEM_TYPE_TCP, 736 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 737 }, 738 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 739 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 740 }, 741 [MLX5_EXPANSION_GTP] = { 742 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 743 MLX5_EXPANSION_IPV6), 744 .type = RTE_FLOW_ITEM_TYPE_GTP, 745 }, 746 [MLX5_EXPANSION_GENEVE] = { 747 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 748 MLX5_EXPANSION_IPV4, 749 MLX5_EXPANSION_IPV6), 750 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 751 }, 752 }; 753 754 static struct rte_flow_action_handle * 755 mlx5_action_handle_create(struct rte_eth_dev *dev, 756 const struct rte_flow_indir_action_conf *conf, 757 const struct rte_flow_action *action, 758 struct rte_flow_error *error); 759 static int mlx5_action_handle_destroy 760 (struct rte_eth_dev *dev, 761 struct rte_flow_action_handle *handle, 762 struct rte_flow_error *error); 763 static int mlx5_action_handle_update 764 (struct rte_eth_dev *dev, 765 struct rte_flow_action_handle *handle, 766 const void *update, 767 struct rte_flow_error *error); 768 static int mlx5_action_handle_query 769 (struct rte_eth_dev *dev, 770 const struct rte_flow_action_handle *handle, 771 void *data, 772 struct rte_flow_error *error); 773 static int 774 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 775 struct rte_flow_tunnel *app_tunnel, 776 struct rte_flow_action **actions, 777 uint32_t *num_of_actions, 778 struct rte_flow_error *error); 779 static int 780 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 781 struct rte_flow_tunnel *app_tunnel, 782 struct rte_flow_item **items, 783 uint32_t *num_of_items, 784 struct rte_flow_error *error); 785 static int 786 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 787 struct rte_flow_item *pmd_items, 788 uint32_t num_items, struct rte_flow_error *err); 789 static int 790 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 791 struct rte_flow_action *pmd_actions, 792 uint32_t num_actions, 793 struct rte_flow_error *err); 794 static int 795 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 796 struct rte_mbuf *m, 797 struct rte_flow_restore_info *info, 798 struct rte_flow_error *err); 799 static struct rte_flow_item_flex_handle * 800 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 801 const struct rte_flow_item_flex_conf *conf, 802 struct rte_flow_error *error); 803 static int 804 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 805 const struct rte_flow_item_flex_handle *handle, 806 struct rte_flow_error *error); 807 808 static const struct rte_flow_ops mlx5_flow_ops = { 809 .validate = mlx5_flow_validate, 810 .create = mlx5_flow_create, 811 .destroy = mlx5_flow_destroy, 812 .flush = mlx5_flow_flush, 813 .isolate = mlx5_flow_isolate, 814 .query = mlx5_flow_query, 815 .dev_dump = mlx5_flow_dev_dump, 816 .get_aged_flows = mlx5_flow_get_aged_flows, 817 .action_handle_create = mlx5_action_handle_create, 818 .action_handle_destroy = mlx5_action_handle_destroy, 819 .action_handle_update = mlx5_action_handle_update, 820 .action_handle_query = mlx5_action_handle_query, 821 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 822 .tunnel_match = mlx5_flow_tunnel_match, 823 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 824 .tunnel_item_release = mlx5_flow_tunnel_item_release, 825 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 826 .flex_item_create = mlx5_flow_flex_item_create, 827 .flex_item_release = mlx5_flow_flex_item_release, 828 }; 829 830 /* Tunnel information. */ 831 struct mlx5_flow_tunnel_info { 832 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 833 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 834 }; 835 836 static struct mlx5_flow_tunnel_info tunnels_info[] = { 837 { 838 .tunnel = MLX5_FLOW_LAYER_VXLAN, 839 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 840 }, 841 { 842 .tunnel = MLX5_FLOW_LAYER_GENEVE, 843 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 844 }, 845 { 846 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 847 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 848 }, 849 { 850 .tunnel = MLX5_FLOW_LAYER_GRE, 851 .ptype = RTE_PTYPE_TUNNEL_GRE, 852 }, 853 { 854 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 855 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 856 }, 857 { 858 .tunnel = MLX5_FLOW_LAYER_MPLS, 859 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 860 }, 861 { 862 .tunnel = MLX5_FLOW_LAYER_NVGRE, 863 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 864 }, 865 { 866 .tunnel = MLX5_FLOW_LAYER_IPIP, 867 .ptype = RTE_PTYPE_TUNNEL_IP, 868 }, 869 { 870 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 871 .ptype = RTE_PTYPE_TUNNEL_IP, 872 }, 873 { 874 .tunnel = MLX5_FLOW_LAYER_GTP, 875 .ptype = RTE_PTYPE_TUNNEL_GTPU, 876 }, 877 }; 878 879 880 881 /** 882 * Translate tag ID to register. 883 * 884 * @param[in] dev 885 * Pointer to the Ethernet device structure. 886 * @param[in] feature 887 * The feature that request the register. 888 * @param[in] id 889 * The request register ID. 890 * @param[out] error 891 * Error description in case of any. 892 * 893 * @return 894 * The request register on success, a negative errno 895 * value otherwise and rte_errno is set. 896 */ 897 int 898 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 899 enum mlx5_feature_name feature, 900 uint32_t id, 901 struct rte_flow_error *error) 902 { 903 struct mlx5_priv *priv = dev->data->dev_private; 904 struct mlx5_sh_config *config = &priv->sh->config; 905 enum modify_reg start_reg; 906 bool skip_mtr_reg = false; 907 908 switch (feature) { 909 case MLX5_HAIRPIN_RX: 910 return REG_B; 911 case MLX5_HAIRPIN_TX: 912 return REG_A; 913 case MLX5_METADATA_RX: 914 switch (config->dv_xmeta_en) { 915 case MLX5_XMETA_MODE_LEGACY: 916 return REG_B; 917 case MLX5_XMETA_MODE_META16: 918 return REG_C_0; 919 case MLX5_XMETA_MODE_META32: 920 return REG_C_1; 921 } 922 break; 923 case MLX5_METADATA_TX: 924 return REG_A; 925 case MLX5_METADATA_FDB: 926 switch (config->dv_xmeta_en) { 927 case MLX5_XMETA_MODE_LEGACY: 928 return REG_NON; 929 case MLX5_XMETA_MODE_META16: 930 return REG_C_0; 931 case MLX5_XMETA_MODE_META32: 932 return REG_C_1; 933 } 934 break; 935 case MLX5_FLOW_MARK: 936 switch (config->dv_xmeta_en) { 937 case MLX5_XMETA_MODE_LEGACY: 938 return REG_NON; 939 case MLX5_XMETA_MODE_META16: 940 return REG_C_1; 941 case MLX5_XMETA_MODE_META32: 942 return REG_C_0; 943 } 944 break; 945 case MLX5_MTR_ID: 946 /* 947 * If meter color and meter id share one register, flow match 948 * should use the meter color register for match. 949 */ 950 if (priv->mtr_reg_share) 951 return priv->mtr_color_reg; 952 else 953 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 954 REG_C_3; 955 case MLX5_MTR_COLOR: 956 case MLX5_ASO_FLOW_HIT: 957 case MLX5_ASO_CONNTRACK: 958 case MLX5_SAMPLE_ID: 959 /* All features use the same REG_C. */ 960 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 961 return priv->mtr_color_reg; 962 case MLX5_COPY_MARK: 963 /* 964 * Metadata COPY_MARK register using is in meter suffix sub 965 * flow while with meter. It's safe to share the same register. 966 */ 967 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 968 case MLX5_APP_TAG: 969 /* 970 * If meter is enable, it will engage the register for color 971 * match and flow match. If meter color match is not using the 972 * REG_C_2, need to skip the REG_C_x be used by meter color 973 * match. 974 * If meter is disable, free to use all available registers. 975 */ 976 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 977 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 978 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 979 if (id > (uint32_t)(REG_C_7 - start_reg)) 980 return rte_flow_error_set(error, EINVAL, 981 RTE_FLOW_ERROR_TYPE_ITEM, 982 NULL, "invalid tag id"); 983 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 984 return rte_flow_error_set(error, ENOTSUP, 985 RTE_FLOW_ERROR_TYPE_ITEM, 986 NULL, "unsupported tag id"); 987 /* 988 * This case means meter is using the REG_C_x great than 2. 989 * Take care not to conflict with meter color REG_C_x. 990 * If the available index REG_C_y >= REG_C_x, skip the 991 * color register. 992 */ 993 if (skip_mtr_reg && priv->sh->flow_mreg_c 994 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 995 if (id >= (uint32_t)(REG_C_7 - start_reg)) 996 return rte_flow_error_set(error, EINVAL, 997 RTE_FLOW_ERROR_TYPE_ITEM, 998 NULL, "invalid tag id"); 999 if (priv->sh->flow_mreg_c 1000 [id + 1 + start_reg - REG_C_0] != REG_NON) 1001 return priv->sh->flow_mreg_c 1002 [id + 1 + start_reg - REG_C_0]; 1003 return rte_flow_error_set(error, ENOTSUP, 1004 RTE_FLOW_ERROR_TYPE_ITEM, 1005 NULL, "unsupported tag id"); 1006 } 1007 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1008 } 1009 MLX5_ASSERT(false); 1010 return rte_flow_error_set(error, EINVAL, 1011 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1012 NULL, "invalid feature name"); 1013 } 1014 1015 /** 1016 * Check extensive flow metadata register support. 1017 * 1018 * @param dev 1019 * Pointer to rte_eth_dev structure. 1020 * 1021 * @return 1022 * True if device supports extensive flow metadata register, otherwise false. 1023 */ 1024 bool 1025 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1026 { 1027 struct mlx5_priv *priv = dev->data->dev_private; 1028 1029 /* 1030 * Having available reg_c can be regarded inclusively as supporting 1031 * extensive flow metadata register, which could mean, 1032 * - metadata register copy action by modify header. 1033 * - 16 modify header actions is supported. 1034 * - reg_c's are preserved across different domain (FDB and NIC) on 1035 * packet loopback by flow lookup miss. 1036 */ 1037 return priv->sh->flow_mreg_c[2] != REG_NON; 1038 } 1039 1040 /** 1041 * Get the lowest priority. 1042 * 1043 * @param[in] dev 1044 * Pointer to the Ethernet device structure. 1045 * @param[in] attributes 1046 * Pointer to device flow rule attributes. 1047 * 1048 * @return 1049 * The value of lowest priority of flow. 1050 */ 1051 uint32_t 1052 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1053 const struct rte_flow_attr *attr) 1054 { 1055 struct mlx5_priv *priv = dev->data->dev_private; 1056 1057 if (!attr->group && !attr->transfer) 1058 return priv->sh->flow_max_priority - 2; 1059 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1060 } 1061 1062 /** 1063 * Calculate matcher priority of the flow. 1064 * 1065 * @param[in] dev 1066 * Pointer to the Ethernet device structure. 1067 * @param[in] attr 1068 * Pointer to device flow rule attributes. 1069 * @param[in] subpriority 1070 * The priority based on the items. 1071 * @param[in] external 1072 * Flow is user flow. 1073 * @return 1074 * The matcher priority of the flow. 1075 */ 1076 uint16_t 1077 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1078 const struct rte_flow_attr *attr, 1079 uint32_t subpriority, bool external) 1080 { 1081 uint16_t priority = (uint16_t)attr->priority; 1082 struct mlx5_priv *priv = dev->data->dev_private; 1083 1084 if (!attr->group && !attr->transfer) { 1085 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1086 priority = priv->sh->flow_max_priority - 1; 1087 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1088 } else if (!external && attr->transfer && attr->group == 0 && 1089 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1090 return (priv->sh->flow_max_priority - 1) * 3; 1091 } 1092 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1093 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1094 return priority * 3 + subpriority; 1095 } 1096 1097 /** 1098 * Verify the @p item specifications (spec, last, mask) are compatible with the 1099 * NIC capabilities. 1100 * 1101 * @param[in] item 1102 * Item specification. 1103 * @param[in] mask 1104 * @p item->mask or flow default bit-masks. 1105 * @param[in] nic_mask 1106 * Bit-masks covering supported fields by the NIC to compare with user mask. 1107 * @param[in] size 1108 * Bit-masks size in bytes. 1109 * @param[in] range_accepted 1110 * True if range of values is accepted for specific fields, false otherwise. 1111 * @param[out] error 1112 * Pointer to error structure. 1113 * 1114 * @return 1115 * 0 on success, a negative errno value otherwise and rte_errno is set. 1116 */ 1117 int 1118 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1119 const uint8_t *mask, 1120 const uint8_t *nic_mask, 1121 unsigned int size, 1122 bool range_accepted, 1123 struct rte_flow_error *error) 1124 { 1125 unsigned int i; 1126 1127 MLX5_ASSERT(nic_mask); 1128 for (i = 0; i < size; ++i) 1129 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1130 return rte_flow_error_set(error, ENOTSUP, 1131 RTE_FLOW_ERROR_TYPE_ITEM, 1132 item, 1133 "mask enables non supported" 1134 " bits"); 1135 if (!item->spec && (item->mask || item->last)) 1136 return rte_flow_error_set(error, EINVAL, 1137 RTE_FLOW_ERROR_TYPE_ITEM, item, 1138 "mask/last without a spec is not" 1139 " supported"); 1140 if (item->spec && item->last && !range_accepted) { 1141 uint8_t spec[size]; 1142 uint8_t last[size]; 1143 unsigned int i; 1144 int ret; 1145 1146 for (i = 0; i < size; ++i) { 1147 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1148 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1149 } 1150 ret = memcmp(spec, last, size); 1151 if (ret != 0) 1152 return rte_flow_error_set(error, EINVAL, 1153 RTE_FLOW_ERROR_TYPE_ITEM, 1154 item, 1155 "range is not valid"); 1156 } 1157 return 0; 1158 } 1159 1160 /** 1161 * Adjust the hash fields according to the @p flow information. 1162 * 1163 * @param[in] dev_flow. 1164 * Pointer to the mlx5_flow. 1165 * @param[in] tunnel 1166 * 1 when the hash field is for a tunnel item. 1167 * @param[in] layer_types 1168 * RTE_ETH_RSS_* types. 1169 * @param[in] hash_fields 1170 * Item hash fields. 1171 * 1172 * @return 1173 * The hash fields that should be used. 1174 */ 1175 uint64_t 1176 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1177 int tunnel __rte_unused, uint64_t layer_types, 1178 uint64_t hash_fields) 1179 { 1180 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1181 int rss_request_inner = rss_desc->level >= 2; 1182 1183 /* Check RSS hash level for tunnel. */ 1184 if (tunnel && rss_request_inner) 1185 hash_fields |= IBV_RX_HASH_INNER; 1186 else if (tunnel || rss_request_inner) 1187 return 0; 1188 #endif 1189 /* Check if requested layer matches RSS hash fields. */ 1190 if (!(rss_desc->types & layer_types)) 1191 return 0; 1192 return hash_fields; 1193 } 1194 1195 /** 1196 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1197 * if several tunnel rules are used on this queue, the tunnel ptype will be 1198 * cleared. 1199 * 1200 * @param rxq_ctrl 1201 * Rx queue to update. 1202 */ 1203 static void 1204 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1205 { 1206 unsigned int i; 1207 uint32_t tunnel_ptype = 0; 1208 1209 /* Look up for the ptype to use. */ 1210 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1211 if (!rxq_ctrl->flow_tunnels_n[i]) 1212 continue; 1213 if (!tunnel_ptype) { 1214 tunnel_ptype = tunnels_info[i].ptype; 1215 } else { 1216 tunnel_ptype = 0; 1217 break; 1218 } 1219 } 1220 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1221 } 1222 1223 /** 1224 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1225 * flow. 1226 * 1227 * @param[in] dev 1228 * Pointer to the Ethernet device structure. 1229 * @param[in] dev_handle 1230 * Pointer to device flow handle structure. 1231 */ 1232 void 1233 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1234 struct mlx5_flow_handle *dev_handle) 1235 { 1236 struct mlx5_priv *priv = dev->data->dev_private; 1237 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1238 struct mlx5_ind_table_obj *ind_tbl = NULL; 1239 unsigned int i; 1240 1241 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1242 struct mlx5_hrxq *hrxq; 1243 1244 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1245 dev_handle->rix_hrxq); 1246 if (hrxq) 1247 ind_tbl = hrxq->ind_table; 1248 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1249 struct mlx5_shared_action_rss *shared_rss; 1250 1251 shared_rss = mlx5_ipool_get 1252 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1253 dev_handle->rix_srss); 1254 if (shared_rss) 1255 ind_tbl = shared_rss->ind_tbl; 1256 } 1257 if (!ind_tbl) 1258 return; 1259 for (i = 0; i != ind_tbl->queues_n; ++i) { 1260 int idx = ind_tbl->queues[i]; 1261 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1262 1263 MLX5_ASSERT(rxq_ctrl != NULL); 1264 if (rxq_ctrl == NULL) 1265 continue; 1266 /* 1267 * To support metadata register copy on Tx loopback, 1268 * this must be always enabled (metadata may arive 1269 * from other port - not from local flows only. 1270 */ 1271 if (tunnel) { 1272 unsigned int j; 1273 1274 /* Increase the counter matching the flow. */ 1275 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1276 if ((tunnels_info[j].tunnel & 1277 dev_handle->layers) == 1278 tunnels_info[j].tunnel) { 1279 rxq_ctrl->flow_tunnels_n[j]++; 1280 break; 1281 } 1282 } 1283 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1284 } 1285 } 1286 } 1287 1288 static void 1289 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1290 { 1291 struct mlx5_priv *priv = dev->data->dev_private; 1292 struct mlx5_rxq_ctrl *rxq_ctrl; 1293 1294 if (priv->mark_enabled) 1295 return; 1296 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1297 rxq_ctrl->rxq.mark = 1; 1298 } 1299 priv->mark_enabled = 1; 1300 } 1301 1302 /** 1303 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1304 * 1305 * @param[in] dev 1306 * Pointer to the Ethernet device structure. 1307 * @param[in] flow 1308 * Pointer to flow structure. 1309 */ 1310 static void 1311 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1312 { 1313 struct mlx5_priv *priv = dev->data->dev_private; 1314 uint32_t handle_idx; 1315 struct mlx5_flow_handle *dev_handle; 1316 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1317 1318 MLX5_ASSERT(wks); 1319 if (wks->mark) 1320 flow_rxq_mark_flag_set(dev); 1321 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1322 handle_idx, dev_handle, next) 1323 flow_drv_rxq_flags_set(dev, dev_handle); 1324 } 1325 1326 /** 1327 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1328 * device flow if no other flow uses it with the same kind of request. 1329 * 1330 * @param dev 1331 * Pointer to Ethernet device. 1332 * @param[in] dev_handle 1333 * Pointer to the device flow handle structure. 1334 */ 1335 static void 1336 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1337 struct mlx5_flow_handle *dev_handle) 1338 { 1339 struct mlx5_priv *priv = dev->data->dev_private; 1340 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1341 struct mlx5_ind_table_obj *ind_tbl = NULL; 1342 unsigned int i; 1343 1344 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1345 struct mlx5_hrxq *hrxq; 1346 1347 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1348 dev_handle->rix_hrxq); 1349 if (hrxq) 1350 ind_tbl = hrxq->ind_table; 1351 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1352 struct mlx5_shared_action_rss *shared_rss; 1353 1354 shared_rss = mlx5_ipool_get 1355 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1356 dev_handle->rix_srss); 1357 if (shared_rss) 1358 ind_tbl = shared_rss->ind_tbl; 1359 } 1360 if (!ind_tbl) 1361 return; 1362 MLX5_ASSERT(dev->data->dev_started); 1363 for (i = 0; i != ind_tbl->queues_n; ++i) { 1364 int idx = ind_tbl->queues[i]; 1365 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1366 1367 MLX5_ASSERT(rxq_ctrl != NULL); 1368 if (rxq_ctrl == NULL) 1369 continue; 1370 if (tunnel) { 1371 unsigned int j; 1372 1373 /* Decrease the counter matching the flow. */ 1374 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1375 if ((tunnels_info[j].tunnel & 1376 dev_handle->layers) == 1377 tunnels_info[j].tunnel) { 1378 rxq_ctrl->flow_tunnels_n[j]--; 1379 break; 1380 } 1381 } 1382 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1383 } 1384 } 1385 } 1386 1387 /** 1388 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1389 * @p flow if no other flow uses it with the same kind of request. 1390 * 1391 * @param dev 1392 * Pointer to Ethernet device. 1393 * @param[in] flow 1394 * Pointer to the flow. 1395 */ 1396 static void 1397 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1398 { 1399 struct mlx5_priv *priv = dev->data->dev_private; 1400 uint32_t handle_idx; 1401 struct mlx5_flow_handle *dev_handle; 1402 1403 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1404 handle_idx, dev_handle, next) 1405 flow_drv_rxq_flags_trim(dev, dev_handle); 1406 } 1407 1408 /** 1409 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1410 * 1411 * @param dev 1412 * Pointer to Ethernet device. 1413 */ 1414 static void 1415 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1416 { 1417 struct mlx5_priv *priv = dev->data->dev_private; 1418 unsigned int i; 1419 1420 for (i = 0; i != priv->rxqs_n; ++i) { 1421 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1422 unsigned int j; 1423 1424 if (rxq == NULL || rxq->ctrl == NULL) 1425 continue; 1426 rxq->ctrl->rxq.mark = 0; 1427 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1428 rxq->ctrl->flow_tunnels_n[j] = 0; 1429 rxq->ctrl->rxq.tunnel = 0; 1430 } 1431 priv->mark_enabled = 0; 1432 } 1433 1434 /** 1435 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1436 * 1437 * @param[in] dev 1438 * Pointer to the Ethernet device structure. 1439 */ 1440 void 1441 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1442 { 1443 struct mlx5_priv *priv = dev->data->dev_private; 1444 unsigned int i; 1445 1446 for (i = 0; i != priv->rxqs_n; ++i) { 1447 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1448 struct mlx5_rxq_data *data; 1449 1450 if (rxq == NULL || rxq->ctrl == NULL) 1451 continue; 1452 data = &rxq->ctrl->rxq; 1453 if (!rte_flow_dynf_metadata_avail()) { 1454 data->dynf_meta = 0; 1455 data->flow_meta_mask = 0; 1456 data->flow_meta_offset = -1; 1457 data->flow_meta_port_mask = 0; 1458 } else { 1459 data->dynf_meta = 1; 1460 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1461 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1462 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1463 } 1464 } 1465 } 1466 1467 /* 1468 * return a pointer to the desired action in the list of actions. 1469 * 1470 * @param[in] actions 1471 * The list of actions to search the action in. 1472 * @param[in] action 1473 * The action to find. 1474 * 1475 * @return 1476 * Pointer to the action in the list, if found. NULL otherwise. 1477 */ 1478 const struct rte_flow_action * 1479 mlx5_flow_find_action(const struct rte_flow_action *actions, 1480 enum rte_flow_action_type action) 1481 { 1482 if (actions == NULL) 1483 return NULL; 1484 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1485 if (actions->type == action) 1486 return actions; 1487 return NULL; 1488 } 1489 1490 /* 1491 * Validate the flag action. 1492 * 1493 * @param[in] action_flags 1494 * Bit-fields that holds the actions detected until now. 1495 * @param[in] attr 1496 * Attributes of flow that includes this action. 1497 * @param[out] error 1498 * Pointer to error structure. 1499 * 1500 * @return 1501 * 0 on success, a negative errno value otherwise and rte_errno is set. 1502 */ 1503 int 1504 mlx5_flow_validate_action_flag(uint64_t action_flags, 1505 const struct rte_flow_attr *attr, 1506 struct rte_flow_error *error) 1507 { 1508 if (action_flags & MLX5_FLOW_ACTION_MARK) 1509 return rte_flow_error_set(error, EINVAL, 1510 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1511 "can't mark and flag in same flow"); 1512 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1513 return rte_flow_error_set(error, EINVAL, 1514 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1515 "can't have 2 flag" 1516 " actions in same flow"); 1517 if (attr->egress) 1518 return rte_flow_error_set(error, ENOTSUP, 1519 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1520 "flag action not supported for " 1521 "egress"); 1522 return 0; 1523 } 1524 1525 /* 1526 * Validate the mark action. 1527 * 1528 * @param[in] action 1529 * Pointer to the queue action. 1530 * @param[in] action_flags 1531 * Bit-fields that holds the actions detected until now. 1532 * @param[in] attr 1533 * Attributes of flow that includes this action. 1534 * @param[out] error 1535 * Pointer to error structure. 1536 * 1537 * @return 1538 * 0 on success, a negative errno value otherwise and rte_errno is set. 1539 */ 1540 int 1541 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1542 uint64_t action_flags, 1543 const struct rte_flow_attr *attr, 1544 struct rte_flow_error *error) 1545 { 1546 const struct rte_flow_action_mark *mark = action->conf; 1547 1548 if (!mark) 1549 return rte_flow_error_set(error, EINVAL, 1550 RTE_FLOW_ERROR_TYPE_ACTION, 1551 action, 1552 "configuration cannot be null"); 1553 if (mark->id >= MLX5_FLOW_MARK_MAX) 1554 return rte_flow_error_set(error, EINVAL, 1555 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1556 &mark->id, 1557 "mark id must in 0 <= id < " 1558 RTE_STR(MLX5_FLOW_MARK_MAX)); 1559 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1560 return rte_flow_error_set(error, EINVAL, 1561 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1562 "can't flag and mark in same flow"); 1563 if (action_flags & MLX5_FLOW_ACTION_MARK) 1564 return rte_flow_error_set(error, EINVAL, 1565 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1566 "can't have 2 mark actions in same" 1567 " flow"); 1568 if (attr->egress) 1569 return rte_flow_error_set(error, ENOTSUP, 1570 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1571 "mark action not supported for " 1572 "egress"); 1573 return 0; 1574 } 1575 1576 /* 1577 * Validate the drop action. 1578 * 1579 * @param[in] action_flags 1580 * Bit-fields that holds the actions detected until now. 1581 * @param[in] attr 1582 * Attributes of flow that includes this action. 1583 * @param[out] error 1584 * Pointer to error structure. 1585 * 1586 * @return 1587 * 0 on success, a negative errno value otherwise and rte_errno is set. 1588 */ 1589 int 1590 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1591 const struct rte_flow_attr *attr, 1592 struct rte_flow_error *error) 1593 { 1594 if (attr->egress) 1595 return rte_flow_error_set(error, ENOTSUP, 1596 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1597 "drop action not supported for " 1598 "egress"); 1599 return 0; 1600 } 1601 1602 /* 1603 * Validate the queue action. 1604 * 1605 * @param[in] action 1606 * Pointer to the queue action. 1607 * @param[in] action_flags 1608 * Bit-fields that holds the actions detected until now. 1609 * @param[in] dev 1610 * Pointer to the Ethernet device structure. 1611 * @param[in] attr 1612 * Attributes of flow that includes this action. 1613 * @param[out] error 1614 * Pointer to error structure. 1615 * 1616 * @return 1617 * 0 on success, a negative errno value otherwise and rte_errno is set. 1618 */ 1619 int 1620 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1621 uint64_t action_flags, 1622 struct rte_eth_dev *dev, 1623 const struct rte_flow_attr *attr, 1624 struct rte_flow_error *error) 1625 { 1626 struct mlx5_priv *priv = dev->data->dev_private; 1627 const struct rte_flow_action_queue *queue = action->conf; 1628 1629 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1630 return rte_flow_error_set(error, EINVAL, 1631 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1632 "can't have 2 fate actions in" 1633 " same flow"); 1634 if (!priv->rxqs_n) 1635 return rte_flow_error_set(error, EINVAL, 1636 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1637 NULL, "No Rx queues configured"); 1638 if (queue->index >= priv->rxqs_n) 1639 return rte_flow_error_set(error, EINVAL, 1640 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1641 &queue->index, 1642 "queue index out of range"); 1643 if (mlx5_rxq_get(dev, queue->index) == NULL) 1644 return rte_flow_error_set(error, EINVAL, 1645 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1646 &queue->index, 1647 "queue is not configured"); 1648 if (attr->egress) 1649 return rte_flow_error_set(error, ENOTSUP, 1650 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1651 "queue action not supported for " 1652 "egress"); 1653 return 0; 1654 } 1655 1656 /** 1657 * Validate queue numbers for device RSS. 1658 * 1659 * @param[in] dev 1660 * Configured device. 1661 * @param[in] queues 1662 * Array of queue numbers. 1663 * @param[in] queues_n 1664 * Size of the @p queues array. 1665 * @param[out] error 1666 * On error, filled with a textual error description. 1667 * @param[out] queue 1668 * On error, filled with an offending queue index in @p queues array. 1669 * 1670 * @return 1671 * 0 on success, a negative errno code on error. 1672 */ 1673 static int 1674 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1675 const uint16_t *queues, uint32_t queues_n, 1676 const char **error, uint32_t *queue_idx) 1677 { 1678 const struct mlx5_priv *priv = dev->data->dev_private; 1679 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1680 uint32_t i; 1681 1682 for (i = 0; i != queues_n; ++i) { 1683 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, 1684 queues[i]); 1685 1686 if (queues[i] >= priv->rxqs_n) { 1687 *error = "queue index out of range"; 1688 *queue_idx = i; 1689 return -EINVAL; 1690 } 1691 if (rxq_ctrl == NULL) { 1692 *error = "queue is not configured"; 1693 *queue_idx = i; 1694 return -EINVAL; 1695 } 1696 if (i == 0) 1697 rxq_type = rxq_ctrl->type; 1698 if (rxq_type != rxq_ctrl->type) { 1699 *error = "combining hairpin and regular RSS queues is not supported"; 1700 *queue_idx = i; 1701 return -ENOTSUP; 1702 } 1703 } 1704 return 0; 1705 } 1706 1707 /* 1708 * Validate the rss action. 1709 * 1710 * @param[in] dev 1711 * Pointer to the Ethernet device structure. 1712 * @param[in] action 1713 * Pointer to the queue action. 1714 * @param[out] error 1715 * Pointer to error structure. 1716 * 1717 * @return 1718 * 0 on success, a negative errno value otherwise and rte_errno is set. 1719 */ 1720 int 1721 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1722 const struct rte_flow_action *action, 1723 struct rte_flow_error *error) 1724 { 1725 struct mlx5_priv *priv = dev->data->dev_private; 1726 const struct rte_flow_action_rss *rss = action->conf; 1727 int ret; 1728 const char *message; 1729 uint32_t queue_idx; 1730 1731 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1732 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1733 return rte_flow_error_set(error, ENOTSUP, 1734 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1735 &rss->func, 1736 "RSS hash function not supported"); 1737 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1738 if (rss->level > 2) 1739 #else 1740 if (rss->level > 1) 1741 #endif 1742 return rte_flow_error_set(error, ENOTSUP, 1743 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1744 &rss->level, 1745 "tunnel RSS is not supported"); 1746 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1747 if (rss->key_len == 0 && rss->key != NULL) 1748 return rte_flow_error_set(error, ENOTSUP, 1749 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1750 &rss->key_len, 1751 "RSS hash key length 0"); 1752 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1753 return rte_flow_error_set(error, ENOTSUP, 1754 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1755 &rss->key_len, 1756 "RSS hash key too small"); 1757 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1758 return rte_flow_error_set(error, ENOTSUP, 1759 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1760 &rss->key_len, 1761 "RSS hash key too large"); 1762 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1763 return rte_flow_error_set(error, ENOTSUP, 1764 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1765 &rss->queue_num, 1766 "number of queues too large"); 1767 if (rss->types & MLX5_RSS_HF_MASK) 1768 return rte_flow_error_set(error, ENOTSUP, 1769 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1770 &rss->types, 1771 "some RSS protocols are not" 1772 " supported"); 1773 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1774 !(rss->types & RTE_ETH_RSS_IP)) 1775 return rte_flow_error_set(error, EINVAL, 1776 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1777 "L3 partial RSS requested but L3 RSS" 1778 " type not specified"); 1779 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1780 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1781 return rte_flow_error_set(error, EINVAL, 1782 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1783 "L4 partial RSS requested but L4 RSS" 1784 " type not specified"); 1785 if (!priv->rxqs_n) 1786 return rte_flow_error_set(error, EINVAL, 1787 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1788 NULL, "No Rx queues configured"); 1789 if (!rss->queue_num) 1790 return rte_flow_error_set(error, EINVAL, 1791 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1792 NULL, "No queues configured"); 1793 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1794 &message, &queue_idx); 1795 if (ret != 0) { 1796 return rte_flow_error_set(error, -ret, 1797 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1798 &rss->queue[queue_idx], message); 1799 } 1800 return 0; 1801 } 1802 1803 /* 1804 * Validate the rss action. 1805 * 1806 * @param[in] action 1807 * Pointer to the queue action. 1808 * @param[in] action_flags 1809 * Bit-fields that holds the actions detected until now. 1810 * @param[in] dev 1811 * Pointer to the Ethernet device structure. 1812 * @param[in] attr 1813 * Attributes of flow that includes this action. 1814 * @param[in] item_flags 1815 * Items that were detected. 1816 * @param[out] error 1817 * Pointer to error structure. 1818 * 1819 * @return 1820 * 0 on success, a negative errno value otherwise and rte_errno is set. 1821 */ 1822 int 1823 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1824 uint64_t action_flags, 1825 struct rte_eth_dev *dev, 1826 const struct rte_flow_attr *attr, 1827 uint64_t item_flags, 1828 struct rte_flow_error *error) 1829 { 1830 const struct rte_flow_action_rss *rss = action->conf; 1831 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1832 int ret; 1833 1834 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1835 return rte_flow_error_set(error, EINVAL, 1836 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1837 "can't have 2 fate actions" 1838 " in same flow"); 1839 ret = mlx5_validate_action_rss(dev, action, error); 1840 if (ret) 1841 return ret; 1842 if (attr->egress) 1843 return rte_flow_error_set(error, ENOTSUP, 1844 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1845 "rss action not supported for " 1846 "egress"); 1847 if (rss->level > 1 && !tunnel) 1848 return rte_flow_error_set(error, EINVAL, 1849 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1850 "inner RSS is not supported for " 1851 "non-tunnel flows"); 1852 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1853 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1854 return rte_flow_error_set(error, EINVAL, 1855 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1856 "RSS on eCPRI is not supported now"); 1857 } 1858 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1859 !(item_flags & 1860 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1861 rss->level > 1) 1862 return rte_flow_error_set(error, EINVAL, 1863 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1864 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1865 return 0; 1866 } 1867 1868 /* 1869 * Validate the default miss action. 1870 * 1871 * @param[in] action_flags 1872 * Bit-fields that holds the actions detected until now. 1873 * @param[out] error 1874 * Pointer to error structure. 1875 * 1876 * @return 1877 * 0 on success, a negative errno value otherwise and rte_errno is set. 1878 */ 1879 int 1880 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1881 const struct rte_flow_attr *attr, 1882 struct rte_flow_error *error) 1883 { 1884 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1885 return rte_flow_error_set(error, EINVAL, 1886 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1887 "can't have 2 fate actions in" 1888 " same flow"); 1889 if (attr->egress) 1890 return rte_flow_error_set(error, ENOTSUP, 1891 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1892 "default miss action not supported " 1893 "for egress"); 1894 if (attr->group) 1895 return rte_flow_error_set(error, ENOTSUP, 1896 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1897 "only group 0 is supported"); 1898 if (attr->transfer) 1899 return rte_flow_error_set(error, ENOTSUP, 1900 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1901 NULL, "transfer is not supported"); 1902 return 0; 1903 } 1904 1905 /* 1906 * Validate the count action. 1907 * 1908 * @param[in] dev 1909 * Pointer to the Ethernet device structure. 1910 * @param[in] attr 1911 * Attributes of flow that includes this action. 1912 * @param[out] error 1913 * Pointer to error structure. 1914 * 1915 * @return 1916 * 0 on success, a negative errno value otherwise and rte_errno is set. 1917 */ 1918 int 1919 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1920 const struct rte_flow_attr *attr, 1921 struct rte_flow_error *error) 1922 { 1923 if (attr->egress) 1924 return rte_flow_error_set(error, ENOTSUP, 1925 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1926 "count action not supported for " 1927 "egress"); 1928 return 0; 1929 } 1930 1931 /* 1932 * Validate the ASO CT action. 1933 * 1934 * @param[in] dev 1935 * Pointer to the Ethernet device structure. 1936 * @param[in] conntrack 1937 * Pointer to the CT action profile. 1938 * @param[out] error 1939 * Pointer to error structure. 1940 * 1941 * @return 1942 * 0 on success, a negative errno value otherwise and rte_errno is set. 1943 */ 1944 int 1945 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1946 const struct rte_flow_action_conntrack *conntrack, 1947 struct rte_flow_error *error) 1948 { 1949 RTE_SET_USED(dev); 1950 1951 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1952 return rte_flow_error_set(error, EINVAL, 1953 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1954 "Invalid CT state"); 1955 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1956 return rte_flow_error_set(error, EINVAL, 1957 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1958 "Invalid last TCP packet flag"); 1959 return 0; 1960 } 1961 1962 /** 1963 * Verify the @p attributes will be correctly understood by the NIC and store 1964 * them in the @p flow if everything is correct. 1965 * 1966 * @param[in] dev 1967 * Pointer to the Ethernet device structure. 1968 * @param[in] attributes 1969 * Pointer to flow attributes 1970 * @param[out] error 1971 * Pointer to error structure. 1972 * 1973 * @return 1974 * 0 on success, a negative errno value otherwise and rte_errno is set. 1975 */ 1976 int 1977 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1978 const struct rte_flow_attr *attributes, 1979 struct rte_flow_error *error) 1980 { 1981 struct mlx5_priv *priv = dev->data->dev_private; 1982 uint32_t priority_max = priv->sh->flow_max_priority - 1; 1983 1984 if (attributes->group) 1985 return rte_flow_error_set(error, ENOTSUP, 1986 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1987 NULL, "groups is not supported"); 1988 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1989 attributes->priority >= priority_max) 1990 return rte_flow_error_set(error, ENOTSUP, 1991 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1992 NULL, "priority out of range"); 1993 if (attributes->egress) 1994 return rte_flow_error_set(error, ENOTSUP, 1995 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1996 "egress is not supported"); 1997 if (attributes->transfer && !priv->sh->config.dv_esw_en) 1998 return rte_flow_error_set(error, ENOTSUP, 1999 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2000 NULL, "transfer is not supported"); 2001 if (!attributes->ingress) 2002 return rte_flow_error_set(error, EINVAL, 2003 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2004 NULL, 2005 "ingress attribute is mandatory"); 2006 return 0; 2007 } 2008 2009 /** 2010 * Validate ICMP6 item. 2011 * 2012 * @param[in] item 2013 * Item specification. 2014 * @param[in] item_flags 2015 * Bit-fields that holds the items detected until now. 2016 * @param[in] ext_vlan_sup 2017 * Whether extended VLAN features are supported or not. 2018 * @param[out] error 2019 * Pointer to error structure. 2020 * 2021 * @return 2022 * 0 on success, a negative errno value otherwise and rte_errno is set. 2023 */ 2024 int 2025 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2026 uint64_t item_flags, 2027 uint8_t target_protocol, 2028 struct rte_flow_error *error) 2029 { 2030 const struct rte_flow_item_icmp6 *mask = item->mask; 2031 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2032 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2033 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2034 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2035 MLX5_FLOW_LAYER_OUTER_L4; 2036 int ret; 2037 2038 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2039 return rte_flow_error_set(error, EINVAL, 2040 RTE_FLOW_ERROR_TYPE_ITEM, item, 2041 "protocol filtering not compatible" 2042 " with ICMP6 layer"); 2043 if (!(item_flags & l3m)) 2044 return rte_flow_error_set(error, EINVAL, 2045 RTE_FLOW_ERROR_TYPE_ITEM, item, 2046 "IPv6 is mandatory to filter on" 2047 " ICMP6"); 2048 if (item_flags & l4m) 2049 return rte_flow_error_set(error, EINVAL, 2050 RTE_FLOW_ERROR_TYPE_ITEM, item, 2051 "multiple L4 layers not supported"); 2052 if (!mask) 2053 mask = &rte_flow_item_icmp6_mask; 2054 ret = mlx5_flow_item_acceptable 2055 (item, (const uint8_t *)mask, 2056 (const uint8_t *)&rte_flow_item_icmp6_mask, 2057 sizeof(struct rte_flow_item_icmp6), 2058 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2059 if (ret < 0) 2060 return ret; 2061 return 0; 2062 } 2063 2064 /** 2065 * Validate ICMP item. 2066 * 2067 * @param[in] item 2068 * Item specification. 2069 * @param[in] item_flags 2070 * Bit-fields that holds the items detected until now. 2071 * @param[out] error 2072 * Pointer to error structure. 2073 * 2074 * @return 2075 * 0 on success, a negative errno value otherwise and rte_errno is set. 2076 */ 2077 int 2078 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2079 uint64_t item_flags, 2080 uint8_t target_protocol, 2081 struct rte_flow_error *error) 2082 { 2083 const struct rte_flow_item_icmp *mask = item->mask; 2084 const struct rte_flow_item_icmp nic_mask = { 2085 .hdr.icmp_type = 0xff, 2086 .hdr.icmp_code = 0xff, 2087 .hdr.icmp_ident = RTE_BE16(0xffff), 2088 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2089 }; 2090 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2091 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2092 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2093 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2094 MLX5_FLOW_LAYER_OUTER_L4; 2095 int ret; 2096 2097 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2098 return rte_flow_error_set(error, EINVAL, 2099 RTE_FLOW_ERROR_TYPE_ITEM, item, 2100 "protocol filtering not compatible" 2101 " with ICMP layer"); 2102 if (!(item_flags & l3m)) 2103 return rte_flow_error_set(error, EINVAL, 2104 RTE_FLOW_ERROR_TYPE_ITEM, item, 2105 "IPv4 is mandatory to filter" 2106 " on ICMP"); 2107 if (item_flags & l4m) 2108 return rte_flow_error_set(error, EINVAL, 2109 RTE_FLOW_ERROR_TYPE_ITEM, item, 2110 "multiple L4 layers not supported"); 2111 if (!mask) 2112 mask = &nic_mask; 2113 ret = mlx5_flow_item_acceptable 2114 (item, (const uint8_t *)mask, 2115 (const uint8_t *)&nic_mask, 2116 sizeof(struct rte_flow_item_icmp), 2117 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2118 if (ret < 0) 2119 return ret; 2120 return 0; 2121 } 2122 2123 /** 2124 * Validate Ethernet item. 2125 * 2126 * @param[in] item 2127 * Item specification. 2128 * @param[in] item_flags 2129 * Bit-fields that holds the items detected until now. 2130 * @param[out] error 2131 * Pointer to error structure. 2132 * 2133 * @return 2134 * 0 on success, a negative errno value otherwise and rte_errno is set. 2135 */ 2136 int 2137 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2138 uint64_t item_flags, bool ext_vlan_sup, 2139 struct rte_flow_error *error) 2140 { 2141 const struct rte_flow_item_eth *mask = item->mask; 2142 const struct rte_flow_item_eth nic_mask = { 2143 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2144 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2145 .type = RTE_BE16(0xffff), 2146 .has_vlan = ext_vlan_sup ? 1 : 0, 2147 }; 2148 int ret; 2149 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2150 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2151 MLX5_FLOW_LAYER_OUTER_L2; 2152 2153 if (item_flags & ethm) 2154 return rte_flow_error_set(error, ENOTSUP, 2155 RTE_FLOW_ERROR_TYPE_ITEM, item, 2156 "multiple L2 layers not supported"); 2157 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2158 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2159 return rte_flow_error_set(error, EINVAL, 2160 RTE_FLOW_ERROR_TYPE_ITEM, item, 2161 "L2 layer should not follow " 2162 "L3 layers"); 2163 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2164 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2165 return rte_flow_error_set(error, EINVAL, 2166 RTE_FLOW_ERROR_TYPE_ITEM, item, 2167 "L2 layer should not follow VLAN"); 2168 if (item_flags & MLX5_FLOW_LAYER_GTP) 2169 return rte_flow_error_set(error, EINVAL, 2170 RTE_FLOW_ERROR_TYPE_ITEM, item, 2171 "L2 layer should not follow GTP"); 2172 if (!mask) 2173 mask = &rte_flow_item_eth_mask; 2174 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2175 (const uint8_t *)&nic_mask, 2176 sizeof(struct rte_flow_item_eth), 2177 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2178 return ret; 2179 } 2180 2181 /** 2182 * Validate VLAN item. 2183 * 2184 * @param[in] item 2185 * Item specification. 2186 * @param[in] item_flags 2187 * Bit-fields that holds the items detected until now. 2188 * @param[in] dev 2189 * Ethernet device flow is being created on. 2190 * @param[out] error 2191 * Pointer to error structure. 2192 * 2193 * @return 2194 * 0 on success, a negative errno value otherwise and rte_errno is set. 2195 */ 2196 int 2197 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2198 uint64_t item_flags, 2199 struct rte_eth_dev *dev, 2200 struct rte_flow_error *error) 2201 { 2202 const struct rte_flow_item_vlan *spec = item->spec; 2203 const struct rte_flow_item_vlan *mask = item->mask; 2204 const struct rte_flow_item_vlan nic_mask = { 2205 .tci = RTE_BE16(UINT16_MAX), 2206 .inner_type = RTE_BE16(UINT16_MAX), 2207 }; 2208 uint16_t vlan_tag = 0; 2209 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2210 int ret; 2211 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2212 MLX5_FLOW_LAYER_INNER_L4) : 2213 (MLX5_FLOW_LAYER_OUTER_L3 | 2214 MLX5_FLOW_LAYER_OUTER_L4); 2215 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2216 MLX5_FLOW_LAYER_OUTER_VLAN; 2217 2218 if (item_flags & vlanm) 2219 return rte_flow_error_set(error, EINVAL, 2220 RTE_FLOW_ERROR_TYPE_ITEM, item, 2221 "multiple VLAN layers not supported"); 2222 else if ((item_flags & l34m) != 0) 2223 return rte_flow_error_set(error, EINVAL, 2224 RTE_FLOW_ERROR_TYPE_ITEM, item, 2225 "VLAN cannot follow L3/L4 layer"); 2226 if (!mask) 2227 mask = &rte_flow_item_vlan_mask; 2228 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2229 (const uint8_t *)&nic_mask, 2230 sizeof(struct rte_flow_item_vlan), 2231 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2232 if (ret) 2233 return ret; 2234 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2235 struct mlx5_priv *priv = dev->data->dev_private; 2236 2237 if (priv->vmwa_context) { 2238 /* 2239 * Non-NULL context means we have a virtual machine 2240 * and SR-IOV enabled, we have to create VLAN interface 2241 * to make hypervisor to setup E-Switch vport 2242 * context correctly. We avoid creating the multiple 2243 * VLAN interfaces, so we cannot support VLAN tag mask. 2244 */ 2245 return rte_flow_error_set(error, EINVAL, 2246 RTE_FLOW_ERROR_TYPE_ITEM, 2247 item, 2248 "VLAN tag mask is not" 2249 " supported in virtual" 2250 " environment"); 2251 } 2252 } 2253 if (spec) { 2254 vlan_tag = spec->tci; 2255 vlan_tag &= mask->tci; 2256 } 2257 /* 2258 * From verbs perspective an empty VLAN is equivalent 2259 * to a packet without VLAN layer. 2260 */ 2261 if (!vlan_tag) 2262 return rte_flow_error_set(error, EINVAL, 2263 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2264 item->spec, 2265 "VLAN cannot be empty"); 2266 return 0; 2267 } 2268 2269 /** 2270 * Validate IPV4 item. 2271 * 2272 * @param[in] item 2273 * Item specification. 2274 * @param[in] item_flags 2275 * Bit-fields that holds the items detected until now. 2276 * @param[in] last_item 2277 * Previous validated item in the pattern items. 2278 * @param[in] ether_type 2279 * Type in the ethernet layer header (including dot1q). 2280 * @param[in] acc_mask 2281 * Acceptable mask, if NULL default internal default mask 2282 * will be used to check whether item fields are supported. 2283 * @param[in] range_accepted 2284 * True if range of values is accepted for specific fields, false otherwise. 2285 * @param[out] error 2286 * Pointer to error structure. 2287 * 2288 * @return 2289 * 0 on success, a negative errno value otherwise and rte_errno is set. 2290 */ 2291 int 2292 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2293 uint64_t item_flags, 2294 uint64_t last_item, 2295 uint16_t ether_type, 2296 const struct rte_flow_item_ipv4 *acc_mask, 2297 bool range_accepted, 2298 struct rte_flow_error *error) 2299 { 2300 const struct rte_flow_item_ipv4 *mask = item->mask; 2301 const struct rte_flow_item_ipv4 *spec = item->spec; 2302 const struct rte_flow_item_ipv4 nic_mask = { 2303 .hdr = { 2304 .src_addr = RTE_BE32(0xffffffff), 2305 .dst_addr = RTE_BE32(0xffffffff), 2306 .type_of_service = 0xff, 2307 .next_proto_id = 0xff, 2308 }, 2309 }; 2310 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2311 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2312 MLX5_FLOW_LAYER_OUTER_L3; 2313 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2314 MLX5_FLOW_LAYER_OUTER_L4; 2315 int ret; 2316 uint8_t next_proto = 0xFF; 2317 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2318 MLX5_FLOW_LAYER_OUTER_VLAN | 2319 MLX5_FLOW_LAYER_INNER_VLAN); 2320 2321 if ((last_item & l2_vlan) && ether_type && 2322 ether_type != RTE_ETHER_TYPE_IPV4) 2323 return rte_flow_error_set(error, EINVAL, 2324 RTE_FLOW_ERROR_TYPE_ITEM, item, 2325 "IPv4 cannot follow L2/VLAN layer " 2326 "which ether type is not IPv4"); 2327 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2328 if (mask && spec) 2329 next_proto = mask->hdr.next_proto_id & 2330 spec->hdr.next_proto_id; 2331 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2332 return rte_flow_error_set(error, EINVAL, 2333 RTE_FLOW_ERROR_TYPE_ITEM, 2334 item, 2335 "multiple tunnel " 2336 "not supported"); 2337 } 2338 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2339 return rte_flow_error_set(error, EINVAL, 2340 RTE_FLOW_ERROR_TYPE_ITEM, item, 2341 "wrong tunnel type - IPv6 specified " 2342 "but IPv4 item provided"); 2343 if (item_flags & l3m) 2344 return rte_flow_error_set(error, ENOTSUP, 2345 RTE_FLOW_ERROR_TYPE_ITEM, item, 2346 "multiple L3 layers not supported"); 2347 else if (item_flags & l4m) 2348 return rte_flow_error_set(error, EINVAL, 2349 RTE_FLOW_ERROR_TYPE_ITEM, item, 2350 "L3 cannot follow an L4 layer."); 2351 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2352 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2353 return rte_flow_error_set(error, EINVAL, 2354 RTE_FLOW_ERROR_TYPE_ITEM, item, 2355 "L3 cannot follow an NVGRE layer."); 2356 if (!mask) 2357 mask = &rte_flow_item_ipv4_mask; 2358 else if (mask->hdr.next_proto_id != 0 && 2359 mask->hdr.next_proto_id != 0xff) 2360 return rte_flow_error_set(error, EINVAL, 2361 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2362 "partial mask is not supported" 2363 " for protocol"); 2364 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2365 acc_mask ? (const uint8_t *)acc_mask 2366 : (const uint8_t *)&nic_mask, 2367 sizeof(struct rte_flow_item_ipv4), 2368 range_accepted, error); 2369 if (ret < 0) 2370 return ret; 2371 return 0; 2372 } 2373 2374 /** 2375 * Validate IPV6 item. 2376 * 2377 * @param[in] item 2378 * Item specification. 2379 * @param[in] item_flags 2380 * Bit-fields that holds the items detected until now. 2381 * @param[in] last_item 2382 * Previous validated item in the pattern items. 2383 * @param[in] ether_type 2384 * Type in the ethernet layer header (including dot1q). 2385 * @param[in] acc_mask 2386 * Acceptable mask, if NULL default internal default mask 2387 * will be used to check whether item fields are supported. 2388 * @param[out] error 2389 * Pointer to error structure. 2390 * 2391 * @return 2392 * 0 on success, a negative errno value otherwise and rte_errno is set. 2393 */ 2394 int 2395 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2396 uint64_t item_flags, 2397 uint64_t last_item, 2398 uint16_t ether_type, 2399 const struct rte_flow_item_ipv6 *acc_mask, 2400 struct rte_flow_error *error) 2401 { 2402 const struct rte_flow_item_ipv6 *mask = item->mask; 2403 const struct rte_flow_item_ipv6 *spec = item->spec; 2404 const struct rte_flow_item_ipv6 nic_mask = { 2405 .hdr = { 2406 .src_addr = 2407 "\xff\xff\xff\xff\xff\xff\xff\xff" 2408 "\xff\xff\xff\xff\xff\xff\xff\xff", 2409 .dst_addr = 2410 "\xff\xff\xff\xff\xff\xff\xff\xff" 2411 "\xff\xff\xff\xff\xff\xff\xff\xff", 2412 .vtc_flow = RTE_BE32(0xffffffff), 2413 .proto = 0xff, 2414 }, 2415 }; 2416 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2417 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2418 MLX5_FLOW_LAYER_OUTER_L3; 2419 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2420 MLX5_FLOW_LAYER_OUTER_L4; 2421 int ret; 2422 uint8_t next_proto = 0xFF; 2423 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2424 MLX5_FLOW_LAYER_OUTER_VLAN | 2425 MLX5_FLOW_LAYER_INNER_VLAN); 2426 2427 if ((last_item & l2_vlan) && ether_type && 2428 ether_type != RTE_ETHER_TYPE_IPV6) 2429 return rte_flow_error_set(error, EINVAL, 2430 RTE_FLOW_ERROR_TYPE_ITEM, item, 2431 "IPv6 cannot follow L2/VLAN layer " 2432 "which ether type is not IPv6"); 2433 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2434 next_proto = spec->hdr.proto; 2435 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2436 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2437 return rte_flow_error_set(error, EINVAL, 2438 RTE_FLOW_ERROR_TYPE_ITEM, 2439 item, 2440 "multiple tunnel " 2441 "not supported"); 2442 } 2443 if (next_proto == IPPROTO_HOPOPTS || 2444 next_proto == IPPROTO_ROUTING || 2445 next_proto == IPPROTO_FRAGMENT || 2446 next_proto == IPPROTO_ESP || 2447 next_proto == IPPROTO_AH || 2448 next_proto == IPPROTO_DSTOPTS) 2449 return rte_flow_error_set(error, EINVAL, 2450 RTE_FLOW_ERROR_TYPE_ITEM, item, 2451 "IPv6 proto (next header) should " 2452 "not be set as extension header"); 2453 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2454 return rte_flow_error_set(error, EINVAL, 2455 RTE_FLOW_ERROR_TYPE_ITEM, item, 2456 "wrong tunnel type - IPv4 specified " 2457 "but IPv6 item provided"); 2458 if (item_flags & l3m) 2459 return rte_flow_error_set(error, ENOTSUP, 2460 RTE_FLOW_ERROR_TYPE_ITEM, item, 2461 "multiple L3 layers not supported"); 2462 else if (item_flags & l4m) 2463 return rte_flow_error_set(error, EINVAL, 2464 RTE_FLOW_ERROR_TYPE_ITEM, item, 2465 "L3 cannot follow an L4 layer."); 2466 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2467 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2468 return rte_flow_error_set(error, EINVAL, 2469 RTE_FLOW_ERROR_TYPE_ITEM, item, 2470 "L3 cannot follow an NVGRE layer."); 2471 if (!mask) 2472 mask = &rte_flow_item_ipv6_mask; 2473 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2474 acc_mask ? (const uint8_t *)acc_mask 2475 : (const uint8_t *)&nic_mask, 2476 sizeof(struct rte_flow_item_ipv6), 2477 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2478 if (ret < 0) 2479 return ret; 2480 return 0; 2481 } 2482 2483 /** 2484 * Validate UDP item. 2485 * 2486 * @param[in] item 2487 * Item specification. 2488 * @param[in] item_flags 2489 * Bit-fields that holds the items detected until now. 2490 * @param[in] target_protocol 2491 * The next protocol in the previous item. 2492 * @param[in] flow_mask 2493 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2494 * @param[out] error 2495 * Pointer to error structure. 2496 * 2497 * @return 2498 * 0 on success, a negative errno value otherwise and rte_errno is set. 2499 */ 2500 int 2501 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2502 uint64_t item_flags, 2503 uint8_t target_protocol, 2504 struct rte_flow_error *error) 2505 { 2506 const struct rte_flow_item_udp *mask = item->mask; 2507 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2508 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2509 MLX5_FLOW_LAYER_OUTER_L3; 2510 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2511 MLX5_FLOW_LAYER_OUTER_L4; 2512 int ret; 2513 2514 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2515 return rte_flow_error_set(error, EINVAL, 2516 RTE_FLOW_ERROR_TYPE_ITEM, item, 2517 "protocol filtering not compatible" 2518 " with UDP layer"); 2519 if (!(item_flags & l3m)) 2520 return rte_flow_error_set(error, EINVAL, 2521 RTE_FLOW_ERROR_TYPE_ITEM, item, 2522 "L3 is mandatory to filter on L4"); 2523 if (item_flags & l4m) 2524 return rte_flow_error_set(error, EINVAL, 2525 RTE_FLOW_ERROR_TYPE_ITEM, item, 2526 "multiple L4 layers not supported"); 2527 if (!mask) 2528 mask = &rte_flow_item_udp_mask; 2529 ret = mlx5_flow_item_acceptable 2530 (item, (const uint8_t *)mask, 2531 (const uint8_t *)&rte_flow_item_udp_mask, 2532 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2533 error); 2534 if (ret < 0) 2535 return ret; 2536 return 0; 2537 } 2538 2539 /** 2540 * Validate TCP item. 2541 * 2542 * @param[in] item 2543 * Item specification. 2544 * @param[in] item_flags 2545 * Bit-fields that holds the items detected until now. 2546 * @param[in] target_protocol 2547 * The next protocol in the previous item. 2548 * @param[out] error 2549 * Pointer to error structure. 2550 * 2551 * @return 2552 * 0 on success, a negative errno value otherwise and rte_errno is set. 2553 */ 2554 int 2555 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2556 uint64_t item_flags, 2557 uint8_t target_protocol, 2558 const struct rte_flow_item_tcp *flow_mask, 2559 struct rte_flow_error *error) 2560 { 2561 const struct rte_flow_item_tcp *mask = item->mask; 2562 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2563 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2564 MLX5_FLOW_LAYER_OUTER_L3; 2565 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2566 MLX5_FLOW_LAYER_OUTER_L4; 2567 int ret; 2568 2569 MLX5_ASSERT(flow_mask); 2570 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2571 return rte_flow_error_set(error, EINVAL, 2572 RTE_FLOW_ERROR_TYPE_ITEM, item, 2573 "protocol filtering not compatible" 2574 " with TCP layer"); 2575 if (!(item_flags & l3m)) 2576 return rte_flow_error_set(error, EINVAL, 2577 RTE_FLOW_ERROR_TYPE_ITEM, item, 2578 "L3 is mandatory to filter on L4"); 2579 if (item_flags & l4m) 2580 return rte_flow_error_set(error, EINVAL, 2581 RTE_FLOW_ERROR_TYPE_ITEM, item, 2582 "multiple L4 layers not supported"); 2583 if (!mask) 2584 mask = &rte_flow_item_tcp_mask; 2585 ret = mlx5_flow_item_acceptable 2586 (item, (const uint8_t *)mask, 2587 (const uint8_t *)flow_mask, 2588 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2589 error); 2590 if (ret < 0) 2591 return ret; 2592 return 0; 2593 } 2594 2595 /** 2596 * Validate VXLAN item. 2597 * 2598 * @param[in] dev 2599 * Pointer to the Ethernet device structure. 2600 * @param[in] udp_dport 2601 * UDP destination port 2602 * @param[in] item 2603 * Item specification. 2604 * @param[in] item_flags 2605 * Bit-fields that holds the items detected until now. 2606 * @param[in] attr 2607 * Flow rule attributes. 2608 * @param[out] error 2609 * Pointer to error structure. 2610 * 2611 * @return 2612 * 0 on success, a negative errno value otherwise and rte_errno is set. 2613 */ 2614 int 2615 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2616 uint16_t udp_dport, 2617 const struct rte_flow_item *item, 2618 uint64_t item_flags, 2619 const struct rte_flow_attr *attr, 2620 struct rte_flow_error *error) 2621 { 2622 const struct rte_flow_item_vxlan *spec = item->spec; 2623 const struct rte_flow_item_vxlan *mask = item->mask; 2624 int ret; 2625 struct mlx5_priv *priv = dev->data->dev_private; 2626 union vni { 2627 uint32_t vlan_id; 2628 uint8_t vni[4]; 2629 } id = { .vlan_id = 0, }; 2630 const struct rte_flow_item_vxlan nic_mask = { 2631 .vni = "\xff\xff\xff", 2632 .rsvd1 = 0xff, 2633 }; 2634 const struct rte_flow_item_vxlan *valid_mask; 2635 2636 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2637 return rte_flow_error_set(error, ENOTSUP, 2638 RTE_FLOW_ERROR_TYPE_ITEM, item, 2639 "multiple tunnel layers not" 2640 " supported"); 2641 valid_mask = &rte_flow_item_vxlan_mask; 2642 /* 2643 * Verify only UDPv4 is present as defined in 2644 * https://tools.ietf.org/html/rfc7348 2645 */ 2646 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2647 return rte_flow_error_set(error, EINVAL, 2648 RTE_FLOW_ERROR_TYPE_ITEM, item, 2649 "no outer UDP layer found"); 2650 if (!mask) 2651 mask = &rte_flow_item_vxlan_mask; 2652 2653 if (priv->sh->steering_format_version != 2654 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2655 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2656 /* FDB domain & NIC domain non-zero group */ 2657 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2658 valid_mask = &nic_mask; 2659 /* Group zero in NIC domain */ 2660 if (!attr->group && !attr->transfer && 2661 priv->sh->tunnel_header_0_1) 2662 valid_mask = &nic_mask; 2663 } 2664 ret = mlx5_flow_item_acceptable 2665 (item, (const uint8_t *)mask, 2666 (const uint8_t *)valid_mask, 2667 sizeof(struct rte_flow_item_vxlan), 2668 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2669 if (ret < 0) 2670 return ret; 2671 if (spec) { 2672 memcpy(&id.vni[1], spec->vni, 3); 2673 memcpy(&id.vni[1], mask->vni, 3); 2674 } 2675 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2676 return rte_flow_error_set(error, ENOTSUP, 2677 RTE_FLOW_ERROR_TYPE_ITEM, item, 2678 "VXLAN tunnel must be fully defined"); 2679 return 0; 2680 } 2681 2682 /** 2683 * Validate VXLAN_GPE item. 2684 * 2685 * @param[in] item 2686 * Item specification. 2687 * @param[in] item_flags 2688 * Bit-fields that holds the items detected until now. 2689 * @param[in] priv 2690 * Pointer to the private data structure. 2691 * @param[in] target_protocol 2692 * The next protocol in the previous item. 2693 * @param[out] error 2694 * Pointer to error structure. 2695 * 2696 * @return 2697 * 0 on success, a negative errno value otherwise and rte_errno is set. 2698 */ 2699 int 2700 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2701 uint64_t item_flags, 2702 struct rte_eth_dev *dev, 2703 struct rte_flow_error *error) 2704 { 2705 struct mlx5_priv *priv = dev->data->dev_private; 2706 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2707 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2708 int ret; 2709 union vni { 2710 uint32_t vlan_id; 2711 uint8_t vni[4]; 2712 } id = { .vlan_id = 0, }; 2713 2714 if (!priv->sh->config.l3_vxlan_en) 2715 return rte_flow_error_set(error, ENOTSUP, 2716 RTE_FLOW_ERROR_TYPE_ITEM, item, 2717 "L3 VXLAN is not enabled by device" 2718 " parameter and/or not configured in" 2719 " firmware"); 2720 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2721 return rte_flow_error_set(error, ENOTSUP, 2722 RTE_FLOW_ERROR_TYPE_ITEM, item, 2723 "multiple tunnel layers not" 2724 " supported"); 2725 /* 2726 * Verify only UDPv4 is present as defined in 2727 * https://tools.ietf.org/html/rfc7348 2728 */ 2729 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2730 return rte_flow_error_set(error, EINVAL, 2731 RTE_FLOW_ERROR_TYPE_ITEM, item, 2732 "no outer UDP layer found"); 2733 if (!mask) 2734 mask = &rte_flow_item_vxlan_gpe_mask; 2735 ret = mlx5_flow_item_acceptable 2736 (item, (const uint8_t *)mask, 2737 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2738 sizeof(struct rte_flow_item_vxlan_gpe), 2739 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2740 if (ret < 0) 2741 return ret; 2742 if (spec) { 2743 if (spec->protocol) 2744 return rte_flow_error_set(error, ENOTSUP, 2745 RTE_FLOW_ERROR_TYPE_ITEM, 2746 item, 2747 "VxLAN-GPE protocol" 2748 " not supported"); 2749 memcpy(&id.vni[1], spec->vni, 3); 2750 memcpy(&id.vni[1], mask->vni, 3); 2751 } 2752 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2753 return rte_flow_error_set(error, ENOTSUP, 2754 RTE_FLOW_ERROR_TYPE_ITEM, item, 2755 "VXLAN-GPE tunnel must be fully" 2756 " defined"); 2757 return 0; 2758 } 2759 /** 2760 * Validate GRE Key item. 2761 * 2762 * @param[in] item 2763 * Item specification. 2764 * @param[in] item_flags 2765 * Bit flags to mark detected items. 2766 * @param[in] gre_item 2767 * Pointer to gre_item 2768 * @param[out] error 2769 * Pointer to error structure. 2770 * 2771 * @return 2772 * 0 on success, a negative errno value otherwise and rte_errno is set. 2773 */ 2774 int 2775 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2776 uint64_t item_flags, 2777 const struct rte_flow_item *gre_item, 2778 struct rte_flow_error *error) 2779 { 2780 const rte_be32_t *mask = item->mask; 2781 int ret = 0; 2782 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2783 const struct rte_flow_item_gre *gre_spec; 2784 const struct rte_flow_item_gre *gre_mask; 2785 2786 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2787 return rte_flow_error_set(error, ENOTSUP, 2788 RTE_FLOW_ERROR_TYPE_ITEM, item, 2789 "Multiple GRE key not support"); 2790 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2791 return rte_flow_error_set(error, ENOTSUP, 2792 RTE_FLOW_ERROR_TYPE_ITEM, item, 2793 "No preceding GRE header"); 2794 if (item_flags & MLX5_FLOW_LAYER_INNER) 2795 return rte_flow_error_set(error, ENOTSUP, 2796 RTE_FLOW_ERROR_TYPE_ITEM, item, 2797 "GRE key following a wrong item"); 2798 gre_mask = gre_item->mask; 2799 if (!gre_mask) 2800 gre_mask = &rte_flow_item_gre_mask; 2801 gre_spec = gre_item->spec; 2802 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2803 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2804 return rte_flow_error_set(error, EINVAL, 2805 RTE_FLOW_ERROR_TYPE_ITEM, item, 2806 "Key bit must be on"); 2807 2808 if (!mask) 2809 mask = &gre_key_default_mask; 2810 ret = mlx5_flow_item_acceptable 2811 (item, (const uint8_t *)mask, 2812 (const uint8_t *)&gre_key_default_mask, 2813 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2814 return ret; 2815 } 2816 2817 /** 2818 * Validate GRE item. 2819 * 2820 * @param[in] item 2821 * Item specification. 2822 * @param[in] item_flags 2823 * Bit flags to mark detected items. 2824 * @param[in] target_protocol 2825 * The next protocol in the previous item. 2826 * @param[out] error 2827 * Pointer to error structure. 2828 * 2829 * @return 2830 * 0 on success, a negative errno value otherwise and rte_errno is set. 2831 */ 2832 int 2833 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2834 uint64_t item_flags, 2835 uint8_t target_protocol, 2836 struct rte_flow_error *error) 2837 { 2838 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2839 const struct rte_flow_item_gre *mask = item->mask; 2840 int ret; 2841 const struct rte_flow_item_gre nic_mask = { 2842 .c_rsvd0_ver = RTE_BE16(0xB000), 2843 .protocol = RTE_BE16(UINT16_MAX), 2844 }; 2845 2846 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2847 return rte_flow_error_set(error, EINVAL, 2848 RTE_FLOW_ERROR_TYPE_ITEM, item, 2849 "protocol filtering not compatible" 2850 " with this GRE layer"); 2851 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2852 return rte_flow_error_set(error, ENOTSUP, 2853 RTE_FLOW_ERROR_TYPE_ITEM, item, 2854 "multiple tunnel layers not" 2855 " supported"); 2856 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2857 return rte_flow_error_set(error, ENOTSUP, 2858 RTE_FLOW_ERROR_TYPE_ITEM, item, 2859 "L3 Layer is missing"); 2860 if (!mask) 2861 mask = &rte_flow_item_gre_mask; 2862 ret = mlx5_flow_item_acceptable 2863 (item, (const uint8_t *)mask, 2864 (const uint8_t *)&nic_mask, 2865 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2866 error); 2867 if (ret < 0) 2868 return ret; 2869 #ifndef HAVE_MLX5DV_DR 2870 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2871 if (spec && (spec->protocol & mask->protocol)) 2872 return rte_flow_error_set(error, ENOTSUP, 2873 RTE_FLOW_ERROR_TYPE_ITEM, item, 2874 "without MPLS support the" 2875 " specification cannot be used for" 2876 " filtering"); 2877 #endif 2878 #endif 2879 return 0; 2880 } 2881 2882 /** 2883 * Validate Geneve item. 2884 * 2885 * @param[in] item 2886 * Item specification. 2887 * @param[in] itemFlags 2888 * Bit-fields that holds the items detected until now. 2889 * @param[in] enPriv 2890 * Pointer to the private data structure. 2891 * @param[out] error 2892 * Pointer to error structure. 2893 * 2894 * @return 2895 * 0 on success, a negative errno value otherwise and rte_errno is set. 2896 */ 2897 2898 int 2899 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2900 uint64_t item_flags, 2901 struct rte_eth_dev *dev, 2902 struct rte_flow_error *error) 2903 { 2904 struct mlx5_priv *priv = dev->data->dev_private; 2905 const struct rte_flow_item_geneve *spec = item->spec; 2906 const struct rte_flow_item_geneve *mask = item->mask; 2907 int ret; 2908 uint16_t gbhdr; 2909 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 2910 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2911 const struct rte_flow_item_geneve nic_mask = { 2912 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2913 .vni = "\xff\xff\xff", 2914 .protocol = RTE_BE16(UINT16_MAX), 2915 }; 2916 2917 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 2918 return rte_flow_error_set(error, ENOTSUP, 2919 RTE_FLOW_ERROR_TYPE_ITEM, item, 2920 "L3 Geneve is not enabled by device" 2921 " parameter and/or not configured in" 2922 " firmware"); 2923 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2924 return rte_flow_error_set(error, ENOTSUP, 2925 RTE_FLOW_ERROR_TYPE_ITEM, item, 2926 "multiple tunnel layers not" 2927 " supported"); 2928 /* 2929 * Verify only UDPv4 is present as defined in 2930 * https://tools.ietf.org/html/rfc7348 2931 */ 2932 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2933 return rte_flow_error_set(error, EINVAL, 2934 RTE_FLOW_ERROR_TYPE_ITEM, item, 2935 "no outer UDP layer found"); 2936 if (!mask) 2937 mask = &rte_flow_item_geneve_mask; 2938 ret = mlx5_flow_item_acceptable 2939 (item, (const uint8_t *)mask, 2940 (const uint8_t *)&nic_mask, 2941 sizeof(struct rte_flow_item_geneve), 2942 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2943 if (ret) 2944 return ret; 2945 if (spec) { 2946 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2947 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2948 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2949 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2950 return rte_flow_error_set(error, ENOTSUP, 2951 RTE_FLOW_ERROR_TYPE_ITEM, 2952 item, 2953 "Geneve protocol unsupported" 2954 " fields are being used"); 2955 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2956 return rte_flow_error_set 2957 (error, ENOTSUP, 2958 RTE_FLOW_ERROR_TYPE_ITEM, 2959 item, 2960 "Unsupported Geneve options length"); 2961 } 2962 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2963 return rte_flow_error_set 2964 (error, ENOTSUP, 2965 RTE_FLOW_ERROR_TYPE_ITEM, item, 2966 "Geneve tunnel must be fully defined"); 2967 return 0; 2968 } 2969 2970 /** 2971 * Validate Geneve TLV option item. 2972 * 2973 * @param[in] item 2974 * Item specification. 2975 * @param[in] last_item 2976 * Previous validated item in the pattern items. 2977 * @param[in] geneve_item 2978 * Previous GENEVE item specification. 2979 * @param[in] dev 2980 * Pointer to the rte_eth_dev structure. 2981 * @param[out] error 2982 * Pointer to error structure. 2983 * 2984 * @return 2985 * 0 on success, a negative errno value otherwise and rte_errno is set. 2986 */ 2987 int 2988 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2989 uint64_t last_item, 2990 const struct rte_flow_item *geneve_item, 2991 struct rte_eth_dev *dev, 2992 struct rte_flow_error *error) 2993 { 2994 struct mlx5_priv *priv = dev->data->dev_private; 2995 struct mlx5_dev_ctx_shared *sh = priv->sh; 2996 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2997 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 2998 uint8_t data_max_supported = 2999 hca_attr->max_geneve_tlv_option_data_len * 4; 3000 const struct rte_flow_item_geneve *geneve_spec; 3001 const struct rte_flow_item_geneve *geneve_mask; 3002 const struct rte_flow_item_geneve_opt *spec = item->spec; 3003 const struct rte_flow_item_geneve_opt *mask = item->mask; 3004 unsigned int i; 3005 unsigned int data_len; 3006 uint8_t tlv_option_len; 3007 uint16_t optlen_m, optlen_v; 3008 const struct rte_flow_item_geneve_opt full_mask = { 3009 .option_class = RTE_BE16(0xffff), 3010 .option_type = 0xff, 3011 .option_len = 0x1f, 3012 }; 3013 3014 if (!mask) 3015 mask = &rte_flow_item_geneve_opt_mask; 3016 if (!spec) 3017 return rte_flow_error_set 3018 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3019 "Geneve TLV opt class/type/length must be specified"); 3020 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3021 return rte_flow_error_set 3022 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3023 "Geneve TLV opt length exceeds the limit (31)"); 3024 /* Check if class type and length masks are full. */ 3025 if (full_mask.option_class != mask->option_class || 3026 full_mask.option_type != mask->option_type || 3027 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3028 return rte_flow_error_set 3029 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3030 "Geneve TLV opt class/type/length masks must be full"); 3031 /* Check if length is supported */ 3032 if ((uint32_t)spec->option_len > 3033 hca_attr->max_geneve_tlv_option_data_len) 3034 return rte_flow_error_set 3035 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3036 "Geneve TLV opt length not supported"); 3037 if (hca_attr->max_geneve_tlv_options > 1) 3038 DRV_LOG(DEBUG, 3039 "max_geneve_tlv_options supports more than 1 option"); 3040 /* Check GENEVE item preceding. */ 3041 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3042 return rte_flow_error_set 3043 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3044 "Geneve opt item must be preceded with Geneve item"); 3045 geneve_spec = geneve_item->spec; 3046 geneve_mask = geneve_item->mask ? geneve_item->mask : 3047 &rte_flow_item_geneve_mask; 3048 /* Check if GENEVE TLV option size doesn't exceed option length */ 3049 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3050 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3051 tlv_option_len = spec->option_len & mask->option_len; 3052 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3053 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3054 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3055 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3056 if ((optlen_v & optlen_m) <= tlv_option_len) 3057 return rte_flow_error_set 3058 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3059 "GENEVE TLV option length exceeds optlen"); 3060 } 3061 /* Check if length is 0 or data is 0. */ 3062 if (spec->data == NULL || spec->option_len == 0) 3063 return rte_flow_error_set 3064 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3065 "Geneve TLV opt with zero data/length not supported"); 3066 /* Check not all data & mask are 0. */ 3067 data_len = spec->option_len * 4; 3068 if (mask->data == NULL) { 3069 for (i = 0; i < data_len; i++) 3070 if (spec->data[i]) 3071 break; 3072 if (i == data_len) 3073 return rte_flow_error_set(error, ENOTSUP, 3074 RTE_FLOW_ERROR_TYPE_ITEM, item, 3075 "Can't match on Geneve option data 0"); 3076 } else { 3077 for (i = 0; i < data_len; i++) 3078 if (spec->data[i] & mask->data[i]) 3079 break; 3080 if (i == data_len) 3081 return rte_flow_error_set(error, ENOTSUP, 3082 RTE_FLOW_ERROR_TYPE_ITEM, item, 3083 "Can't match on Geneve option data and mask 0"); 3084 /* Check data mask supported. */ 3085 for (i = data_max_supported; i < data_len ; i++) 3086 if (mask->data[i]) 3087 return rte_flow_error_set(error, ENOTSUP, 3088 RTE_FLOW_ERROR_TYPE_ITEM, item, 3089 "Data mask is of unsupported size"); 3090 } 3091 /* Check GENEVE option is supported in NIC. */ 3092 if (!hca_attr->geneve_tlv_opt) 3093 return rte_flow_error_set 3094 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3095 "Geneve TLV opt not supported"); 3096 /* Check if we already have geneve option with different type/class. */ 3097 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3098 geneve_opt_resource = sh->geneve_tlv_option_resource; 3099 if (geneve_opt_resource != NULL) 3100 if (geneve_opt_resource->option_class != spec->option_class || 3101 geneve_opt_resource->option_type != spec->option_type || 3102 geneve_opt_resource->length != spec->option_len) { 3103 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3104 return rte_flow_error_set(error, ENOTSUP, 3105 RTE_FLOW_ERROR_TYPE_ITEM, item, 3106 "Only one Geneve TLV option supported"); 3107 } 3108 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3109 return 0; 3110 } 3111 3112 /** 3113 * Validate MPLS item. 3114 * 3115 * @param[in] dev 3116 * Pointer to the rte_eth_dev structure. 3117 * @param[in] item 3118 * Item specification. 3119 * @param[in] item_flags 3120 * Bit-fields that holds the items detected until now. 3121 * @param[in] prev_layer 3122 * The protocol layer indicated in previous item. 3123 * @param[out] error 3124 * Pointer to error structure. 3125 * 3126 * @return 3127 * 0 on success, a negative errno value otherwise and rte_errno is set. 3128 */ 3129 int 3130 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3131 const struct rte_flow_item *item __rte_unused, 3132 uint64_t item_flags __rte_unused, 3133 uint64_t prev_layer __rte_unused, 3134 struct rte_flow_error *error) 3135 { 3136 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3137 const struct rte_flow_item_mpls *mask = item->mask; 3138 struct mlx5_priv *priv = dev->data->dev_private; 3139 int ret; 3140 3141 if (!priv->sh->dev_cap.mpls_en) 3142 return rte_flow_error_set(error, ENOTSUP, 3143 RTE_FLOW_ERROR_TYPE_ITEM, item, 3144 "MPLS not supported or" 3145 " disabled in firmware" 3146 " configuration."); 3147 /* MPLS over UDP, GRE is allowed */ 3148 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3149 MLX5_FLOW_LAYER_GRE | 3150 MLX5_FLOW_LAYER_GRE_KEY))) 3151 return rte_flow_error_set(error, EINVAL, 3152 RTE_FLOW_ERROR_TYPE_ITEM, item, 3153 "protocol filtering not compatible" 3154 " with MPLS layer"); 3155 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3156 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3157 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3158 return rte_flow_error_set(error, ENOTSUP, 3159 RTE_FLOW_ERROR_TYPE_ITEM, item, 3160 "multiple tunnel layers not" 3161 " supported"); 3162 if (!mask) 3163 mask = &rte_flow_item_mpls_mask; 3164 ret = mlx5_flow_item_acceptable 3165 (item, (const uint8_t *)mask, 3166 (const uint8_t *)&rte_flow_item_mpls_mask, 3167 sizeof(struct rte_flow_item_mpls), 3168 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3169 if (ret < 0) 3170 return ret; 3171 return 0; 3172 #else 3173 return rte_flow_error_set(error, ENOTSUP, 3174 RTE_FLOW_ERROR_TYPE_ITEM, item, 3175 "MPLS is not supported by Verbs, please" 3176 " update."); 3177 #endif 3178 } 3179 3180 /** 3181 * Validate NVGRE item. 3182 * 3183 * @param[in] item 3184 * Item specification. 3185 * @param[in] item_flags 3186 * Bit flags to mark detected items. 3187 * @param[in] target_protocol 3188 * The next protocol in the previous item. 3189 * @param[out] error 3190 * Pointer to error structure. 3191 * 3192 * @return 3193 * 0 on success, a negative errno value otherwise and rte_errno is set. 3194 */ 3195 int 3196 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3197 uint64_t item_flags, 3198 uint8_t target_protocol, 3199 struct rte_flow_error *error) 3200 { 3201 const struct rte_flow_item_nvgre *mask = item->mask; 3202 int ret; 3203 3204 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3205 return rte_flow_error_set(error, EINVAL, 3206 RTE_FLOW_ERROR_TYPE_ITEM, item, 3207 "protocol filtering not compatible" 3208 " with this GRE layer"); 3209 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3210 return rte_flow_error_set(error, ENOTSUP, 3211 RTE_FLOW_ERROR_TYPE_ITEM, item, 3212 "multiple tunnel layers not" 3213 " supported"); 3214 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3215 return rte_flow_error_set(error, ENOTSUP, 3216 RTE_FLOW_ERROR_TYPE_ITEM, item, 3217 "L3 Layer is missing"); 3218 if (!mask) 3219 mask = &rte_flow_item_nvgre_mask; 3220 ret = mlx5_flow_item_acceptable 3221 (item, (const uint8_t *)mask, 3222 (const uint8_t *)&rte_flow_item_nvgre_mask, 3223 sizeof(struct rte_flow_item_nvgre), 3224 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3225 if (ret < 0) 3226 return ret; 3227 return 0; 3228 } 3229 3230 /** 3231 * Validate eCPRI item. 3232 * 3233 * @param[in] item 3234 * Item specification. 3235 * @param[in] item_flags 3236 * Bit-fields that holds the items detected until now. 3237 * @param[in] last_item 3238 * Previous validated item in the pattern items. 3239 * @param[in] ether_type 3240 * Type in the ethernet layer header (including dot1q). 3241 * @param[in] acc_mask 3242 * Acceptable mask, if NULL default internal default mask 3243 * will be used to check whether item fields are supported. 3244 * @param[out] error 3245 * Pointer to error structure. 3246 * 3247 * @return 3248 * 0 on success, a negative errno value otherwise and rte_errno is set. 3249 */ 3250 int 3251 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3252 uint64_t item_flags, 3253 uint64_t last_item, 3254 uint16_t ether_type, 3255 const struct rte_flow_item_ecpri *acc_mask, 3256 struct rte_flow_error *error) 3257 { 3258 const struct rte_flow_item_ecpri *mask = item->mask; 3259 const struct rte_flow_item_ecpri nic_mask = { 3260 .hdr = { 3261 .common = { 3262 .u32 = 3263 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3264 .type = 0xFF, 3265 }).u32), 3266 }, 3267 .dummy[0] = 0xFFFFFFFF, 3268 }, 3269 }; 3270 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3271 MLX5_FLOW_LAYER_OUTER_VLAN); 3272 struct rte_flow_item_ecpri mask_lo; 3273 3274 if (!(last_item & outer_l2_vlan) && 3275 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3276 return rte_flow_error_set(error, EINVAL, 3277 RTE_FLOW_ERROR_TYPE_ITEM, item, 3278 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3279 if ((last_item & outer_l2_vlan) && ether_type && 3280 ether_type != RTE_ETHER_TYPE_ECPRI) 3281 return rte_flow_error_set(error, EINVAL, 3282 RTE_FLOW_ERROR_TYPE_ITEM, item, 3283 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3284 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3285 return rte_flow_error_set(error, EINVAL, 3286 RTE_FLOW_ERROR_TYPE_ITEM, item, 3287 "eCPRI with tunnel is not supported right now"); 3288 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3289 return rte_flow_error_set(error, ENOTSUP, 3290 RTE_FLOW_ERROR_TYPE_ITEM, item, 3291 "multiple L3 layers not supported"); 3292 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3293 return rte_flow_error_set(error, EINVAL, 3294 RTE_FLOW_ERROR_TYPE_ITEM, item, 3295 "eCPRI cannot coexist with a TCP layer"); 3296 /* In specification, eCPRI could be over UDP layer. */ 3297 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3298 return rte_flow_error_set(error, EINVAL, 3299 RTE_FLOW_ERROR_TYPE_ITEM, item, 3300 "eCPRI over UDP layer is not yet supported right now"); 3301 /* Mask for type field in common header could be zero. */ 3302 if (!mask) 3303 mask = &rte_flow_item_ecpri_mask; 3304 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3305 /* Input mask is in big-endian format. */ 3306 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3307 return rte_flow_error_set(error, EINVAL, 3308 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3309 "partial mask is not supported for protocol"); 3310 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3311 return rte_flow_error_set(error, EINVAL, 3312 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3313 "message header mask must be after a type mask"); 3314 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3315 acc_mask ? (const uint8_t *)acc_mask 3316 : (const uint8_t *)&nic_mask, 3317 sizeof(struct rte_flow_item_ecpri), 3318 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3319 } 3320 3321 static int 3322 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3323 const struct rte_flow_attr *attr __rte_unused, 3324 const struct rte_flow_item items[] __rte_unused, 3325 const struct rte_flow_action actions[] __rte_unused, 3326 bool external __rte_unused, 3327 int hairpin __rte_unused, 3328 struct rte_flow_error *error) 3329 { 3330 return rte_flow_error_set(error, ENOTSUP, 3331 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3332 } 3333 3334 static struct mlx5_flow * 3335 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3336 const struct rte_flow_attr *attr __rte_unused, 3337 const struct rte_flow_item items[] __rte_unused, 3338 const struct rte_flow_action actions[] __rte_unused, 3339 struct rte_flow_error *error) 3340 { 3341 rte_flow_error_set(error, ENOTSUP, 3342 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3343 return NULL; 3344 } 3345 3346 static int 3347 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3348 struct mlx5_flow *dev_flow __rte_unused, 3349 const struct rte_flow_attr *attr __rte_unused, 3350 const struct rte_flow_item items[] __rte_unused, 3351 const struct rte_flow_action actions[] __rte_unused, 3352 struct rte_flow_error *error) 3353 { 3354 return rte_flow_error_set(error, ENOTSUP, 3355 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3356 } 3357 3358 static int 3359 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3360 struct rte_flow *flow __rte_unused, 3361 struct rte_flow_error *error) 3362 { 3363 return rte_flow_error_set(error, ENOTSUP, 3364 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3365 } 3366 3367 static void 3368 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3369 struct rte_flow *flow __rte_unused) 3370 { 3371 } 3372 3373 static void 3374 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3375 struct rte_flow *flow __rte_unused) 3376 { 3377 } 3378 3379 static int 3380 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3381 struct rte_flow *flow __rte_unused, 3382 const struct rte_flow_action *actions __rte_unused, 3383 void *data __rte_unused, 3384 struct rte_flow_error *error) 3385 { 3386 return rte_flow_error_set(error, ENOTSUP, 3387 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3388 } 3389 3390 static int 3391 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3392 uint32_t domains __rte_unused, 3393 uint32_t flags __rte_unused) 3394 { 3395 return 0; 3396 } 3397 3398 /* Void driver to protect from null pointer reference. */ 3399 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3400 .validate = flow_null_validate, 3401 .prepare = flow_null_prepare, 3402 .translate = flow_null_translate, 3403 .apply = flow_null_apply, 3404 .remove = flow_null_remove, 3405 .destroy = flow_null_destroy, 3406 .query = flow_null_query, 3407 .sync_domain = flow_null_sync_domain, 3408 }; 3409 3410 /** 3411 * Select flow driver type according to flow attributes and device 3412 * configuration. 3413 * 3414 * @param[in] dev 3415 * Pointer to the dev structure. 3416 * @param[in] attr 3417 * Pointer to the flow attributes. 3418 * 3419 * @return 3420 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3421 */ 3422 static enum mlx5_flow_drv_type 3423 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3424 { 3425 struct mlx5_priv *priv = dev->data->dev_private; 3426 /* The OS can determine first a specific flow type (DV, VERBS) */ 3427 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3428 3429 if (type != MLX5_FLOW_TYPE_MAX) 3430 return type; 3431 /* If no OS specific type - continue with DV/VERBS selection */ 3432 if (attr->transfer && priv->sh->config.dv_esw_en) 3433 type = MLX5_FLOW_TYPE_DV; 3434 if (!attr->transfer) 3435 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3436 MLX5_FLOW_TYPE_VERBS; 3437 return type; 3438 } 3439 3440 #define flow_get_drv_ops(type) flow_drv_ops[type] 3441 3442 /** 3443 * Flow driver validation API. This abstracts calling driver specific functions. 3444 * The type of flow driver is determined according to flow attributes. 3445 * 3446 * @param[in] dev 3447 * Pointer to the dev structure. 3448 * @param[in] attr 3449 * Pointer to the flow attributes. 3450 * @param[in] items 3451 * Pointer to the list of items. 3452 * @param[in] actions 3453 * Pointer to the list of actions. 3454 * @param[in] external 3455 * This flow rule is created by request external to PMD. 3456 * @param[in] hairpin 3457 * Number of hairpin TX actions, 0 means classic flow. 3458 * @param[out] error 3459 * Pointer to the error structure. 3460 * 3461 * @return 3462 * 0 on success, a negative errno value otherwise and rte_errno is set. 3463 */ 3464 static inline int 3465 flow_drv_validate(struct rte_eth_dev *dev, 3466 const struct rte_flow_attr *attr, 3467 const struct rte_flow_item items[], 3468 const struct rte_flow_action actions[], 3469 bool external, int hairpin, struct rte_flow_error *error) 3470 { 3471 const struct mlx5_flow_driver_ops *fops; 3472 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3473 3474 fops = flow_get_drv_ops(type); 3475 return fops->validate(dev, attr, items, actions, external, 3476 hairpin, error); 3477 } 3478 3479 /** 3480 * Flow driver preparation API. This abstracts calling driver specific 3481 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3482 * calculates the size of memory required for device flow, allocates the memory, 3483 * initializes the device flow and returns the pointer. 3484 * 3485 * @note 3486 * This function initializes device flow structure such as dv or verbs in 3487 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3488 * rest. For example, adding returning device flow to flow->dev_flow list and 3489 * setting backward reference to the flow should be done out of this function. 3490 * layers field is not filled either. 3491 * 3492 * @param[in] dev 3493 * Pointer to the dev structure. 3494 * @param[in] attr 3495 * Pointer to the flow attributes. 3496 * @param[in] items 3497 * Pointer to the list of items. 3498 * @param[in] actions 3499 * Pointer to the list of actions. 3500 * @param[in] flow_idx 3501 * This memory pool index to the flow. 3502 * @param[out] error 3503 * Pointer to the error structure. 3504 * 3505 * @return 3506 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3507 */ 3508 static inline struct mlx5_flow * 3509 flow_drv_prepare(struct rte_eth_dev *dev, 3510 const struct rte_flow *flow, 3511 const struct rte_flow_attr *attr, 3512 const struct rte_flow_item items[], 3513 const struct rte_flow_action actions[], 3514 uint32_t flow_idx, 3515 struct rte_flow_error *error) 3516 { 3517 const struct mlx5_flow_driver_ops *fops; 3518 enum mlx5_flow_drv_type type = flow->drv_type; 3519 struct mlx5_flow *mlx5_flow = NULL; 3520 3521 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3522 fops = flow_get_drv_ops(type); 3523 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3524 if (mlx5_flow) 3525 mlx5_flow->flow_idx = flow_idx; 3526 return mlx5_flow; 3527 } 3528 3529 /** 3530 * Flow driver translation API. This abstracts calling driver specific 3531 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3532 * translates a generic flow into a driver flow. flow_drv_prepare() must 3533 * precede. 3534 * 3535 * @note 3536 * dev_flow->layers could be filled as a result of parsing during translation 3537 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3538 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3539 * flow->actions could be overwritten even though all the expanded dev_flows 3540 * have the same actions. 3541 * 3542 * @param[in] dev 3543 * Pointer to the rte dev structure. 3544 * @param[in, out] dev_flow 3545 * Pointer to the mlx5 flow. 3546 * @param[in] attr 3547 * Pointer to the flow attributes. 3548 * @param[in] items 3549 * Pointer to the list of items. 3550 * @param[in] actions 3551 * Pointer to the list of actions. 3552 * @param[out] error 3553 * Pointer to the error structure. 3554 * 3555 * @return 3556 * 0 on success, a negative errno value otherwise and rte_errno is set. 3557 */ 3558 static inline int 3559 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3560 const struct rte_flow_attr *attr, 3561 const struct rte_flow_item items[], 3562 const struct rte_flow_action actions[], 3563 struct rte_flow_error *error) 3564 { 3565 const struct mlx5_flow_driver_ops *fops; 3566 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3567 3568 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3569 fops = flow_get_drv_ops(type); 3570 return fops->translate(dev, dev_flow, attr, items, actions, error); 3571 } 3572 3573 /** 3574 * Flow driver apply API. This abstracts calling driver specific functions. 3575 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3576 * translated driver flows on to device. flow_drv_translate() must precede. 3577 * 3578 * @param[in] dev 3579 * Pointer to Ethernet device structure. 3580 * @param[in, out] flow 3581 * Pointer to flow structure. 3582 * @param[out] error 3583 * Pointer to error structure. 3584 * 3585 * @return 3586 * 0 on success, a negative errno value otherwise and rte_errno is set. 3587 */ 3588 static inline int 3589 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3590 struct rte_flow_error *error) 3591 { 3592 const struct mlx5_flow_driver_ops *fops; 3593 enum mlx5_flow_drv_type type = flow->drv_type; 3594 3595 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3596 fops = flow_get_drv_ops(type); 3597 return fops->apply(dev, flow, error); 3598 } 3599 3600 /** 3601 * Flow driver destroy API. This abstracts calling driver specific functions. 3602 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3603 * on device and releases resources of the flow. 3604 * 3605 * @param[in] dev 3606 * Pointer to Ethernet device. 3607 * @param[in, out] flow 3608 * Pointer to flow structure. 3609 */ 3610 static inline void 3611 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3612 { 3613 const struct mlx5_flow_driver_ops *fops; 3614 enum mlx5_flow_drv_type type = flow->drv_type; 3615 3616 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3617 fops = flow_get_drv_ops(type); 3618 fops->destroy(dev, flow); 3619 } 3620 3621 /** 3622 * Flow driver find RSS policy tbl API. This abstracts calling driver 3623 * specific functions. Parent flow (rte_flow) should have driver 3624 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3625 * 3626 * @param[in] dev 3627 * Pointer to Ethernet device. 3628 * @param[in, out] flow 3629 * Pointer to flow structure. 3630 * @param[in] policy 3631 * Pointer to meter policy table. 3632 * @param[in] rss_desc 3633 * Pointer to rss_desc 3634 */ 3635 static struct mlx5_flow_meter_sub_policy * 3636 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3637 struct rte_flow *flow, 3638 struct mlx5_flow_meter_policy *policy, 3639 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3640 { 3641 const struct mlx5_flow_driver_ops *fops; 3642 enum mlx5_flow_drv_type type = flow->drv_type; 3643 3644 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3645 fops = flow_get_drv_ops(type); 3646 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3647 } 3648 3649 /** 3650 * Flow driver color tag rule API. This abstracts calling driver 3651 * specific functions. Parent flow (rte_flow) should have driver 3652 * type (drv_type). It will create the color tag rules in hierarchy meter. 3653 * 3654 * @param[in] dev 3655 * Pointer to Ethernet device. 3656 * @param[in, out] flow 3657 * Pointer to flow structure. 3658 * @param[in] fm 3659 * Pointer to flow meter structure. 3660 * @param[in] src_port 3661 * The src port this extra rule should use. 3662 * @param[in] item 3663 * The src port id match item. 3664 * @param[out] error 3665 * Pointer to error structure. 3666 */ 3667 static int 3668 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3669 struct rte_flow *flow, 3670 struct mlx5_flow_meter_info *fm, 3671 int32_t src_port, 3672 const struct rte_flow_item *item, 3673 struct rte_flow_error *error) 3674 { 3675 const struct mlx5_flow_driver_ops *fops; 3676 enum mlx5_flow_drv_type type = flow->drv_type; 3677 3678 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3679 fops = flow_get_drv_ops(type); 3680 return fops->meter_hierarchy_rule_create(dev, fm, 3681 src_port, item, error); 3682 } 3683 3684 /** 3685 * Get RSS action from the action list. 3686 * 3687 * @param[in] dev 3688 * Pointer to Ethernet device. 3689 * @param[in] actions 3690 * Pointer to the list of actions. 3691 * @param[in] flow 3692 * Parent flow structure pointer. 3693 * 3694 * @return 3695 * Pointer to the RSS action if exist, else return NULL. 3696 */ 3697 static const struct rte_flow_action_rss* 3698 flow_get_rss_action(struct rte_eth_dev *dev, 3699 const struct rte_flow_action actions[]) 3700 { 3701 struct mlx5_priv *priv = dev->data->dev_private; 3702 const struct rte_flow_action_rss *rss = NULL; 3703 struct mlx5_meter_policy_action_container *acg; 3704 struct mlx5_meter_policy_action_container *acy; 3705 3706 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3707 switch (actions->type) { 3708 case RTE_FLOW_ACTION_TYPE_RSS: 3709 rss = actions->conf; 3710 break; 3711 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3712 { 3713 const struct rte_flow_action_sample *sample = 3714 actions->conf; 3715 const struct rte_flow_action *act = sample->actions; 3716 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3717 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3718 rss = act->conf; 3719 break; 3720 } 3721 case RTE_FLOW_ACTION_TYPE_METER: 3722 { 3723 uint32_t mtr_idx; 3724 struct mlx5_flow_meter_info *fm; 3725 struct mlx5_flow_meter_policy *policy; 3726 const struct rte_flow_action_meter *mtr = actions->conf; 3727 3728 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3729 if (fm && !fm->def_policy) { 3730 policy = mlx5_flow_meter_policy_find(dev, 3731 fm->policy_id, NULL); 3732 MLX5_ASSERT(policy); 3733 if (policy->is_hierarchy) { 3734 policy = 3735 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3736 policy); 3737 if (!policy) 3738 return NULL; 3739 } 3740 if (policy->is_rss) { 3741 acg = 3742 &policy->act_cnt[RTE_COLOR_GREEN]; 3743 acy = 3744 &policy->act_cnt[RTE_COLOR_YELLOW]; 3745 if (acg->fate_action == 3746 MLX5_FLOW_FATE_SHARED_RSS) 3747 rss = acg->rss->conf; 3748 else if (acy->fate_action == 3749 MLX5_FLOW_FATE_SHARED_RSS) 3750 rss = acy->rss->conf; 3751 } 3752 } 3753 break; 3754 } 3755 default: 3756 break; 3757 } 3758 } 3759 return rss; 3760 } 3761 3762 /** 3763 * Get ASO age action by index. 3764 * 3765 * @param[in] dev 3766 * Pointer to the Ethernet device structure. 3767 * @param[in] age_idx 3768 * Index to the ASO age action. 3769 * 3770 * @return 3771 * The specified ASO age action. 3772 */ 3773 struct mlx5_aso_age_action* 3774 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3775 { 3776 uint16_t pool_idx = age_idx & UINT16_MAX; 3777 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3778 struct mlx5_priv *priv = dev->data->dev_private; 3779 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3780 struct mlx5_aso_age_pool *pool; 3781 3782 rte_rwlock_read_lock(&mng->resize_rwl); 3783 pool = mng->pools[pool_idx]; 3784 rte_rwlock_read_unlock(&mng->resize_rwl); 3785 return &pool->actions[offset - 1]; 3786 } 3787 3788 /* maps indirect action to translated direct in some actions array */ 3789 struct mlx5_translated_action_handle { 3790 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3791 int index; /**< Index in related array of rte_flow_action. */ 3792 }; 3793 3794 /** 3795 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3796 * direct action if translation possible. 3797 * This functionality used to run same execution path for both direct and 3798 * indirect actions on flow create. All necessary preparations for indirect 3799 * action handling should be performed on *handle* actions list returned 3800 * from this call. 3801 * 3802 * @param[in] dev 3803 * Pointer to Ethernet device. 3804 * @param[in] actions 3805 * List of actions to translate. 3806 * @param[out] handle 3807 * List to store translated indirect action object handles. 3808 * @param[in, out] indir_n 3809 * Size of *handle* array. On return should be updated with number of 3810 * indirect actions retrieved from the *actions* list. 3811 * @param[out] translated_actions 3812 * List of actions where all indirect actions were translated to direct 3813 * if possible. NULL if no translation took place. 3814 * @param[out] error 3815 * Pointer to the error structure. 3816 * 3817 * @return 3818 * 0 on success, a negative errno value otherwise and rte_errno is set. 3819 */ 3820 static int 3821 flow_action_handles_translate(struct rte_eth_dev *dev, 3822 const struct rte_flow_action actions[], 3823 struct mlx5_translated_action_handle *handle, 3824 int *indir_n, 3825 struct rte_flow_action **translated_actions, 3826 struct rte_flow_error *error) 3827 { 3828 struct mlx5_priv *priv = dev->data->dev_private; 3829 struct rte_flow_action *translated = NULL; 3830 size_t actions_size; 3831 int n; 3832 int copied_n = 0; 3833 struct mlx5_translated_action_handle *handle_end = NULL; 3834 3835 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3836 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3837 continue; 3838 if (copied_n == *indir_n) { 3839 return rte_flow_error_set 3840 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3841 NULL, "too many shared actions"); 3842 } 3843 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3844 sizeof(actions[n].conf)); 3845 handle[copied_n].index = n; 3846 copied_n++; 3847 } 3848 n++; 3849 *indir_n = copied_n; 3850 if (!copied_n) 3851 return 0; 3852 actions_size = sizeof(struct rte_flow_action) * n; 3853 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3854 if (!translated) { 3855 rte_errno = ENOMEM; 3856 return -ENOMEM; 3857 } 3858 memcpy(translated, actions, actions_size); 3859 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3860 struct mlx5_shared_action_rss *shared_rss; 3861 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3862 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3863 uint32_t idx = act_idx & 3864 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3865 3866 switch (type) { 3867 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3868 shared_rss = mlx5_ipool_get 3869 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3870 translated[handle->index].type = 3871 RTE_FLOW_ACTION_TYPE_RSS; 3872 translated[handle->index].conf = 3873 &shared_rss->origin; 3874 break; 3875 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3876 translated[handle->index].type = 3877 (enum rte_flow_action_type) 3878 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3879 translated[handle->index].conf = (void *)(uintptr_t)idx; 3880 break; 3881 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3882 if (priv->sh->flow_hit_aso_en) { 3883 translated[handle->index].type = 3884 (enum rte_flow_action_type) 3885 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3886 translated[handle->index].conf = 3887 (void *)(uintptr_t)idx; 3888 break; 3889 } 3890 /* Fall-through */ 3891 case MLX5_INDIRECT_ACTION_TYPE_CT: 3892 if (priv->sh->ct_aso_en) { 3893 translated[handle->index].type = 3894 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3895 translated[handle->index].conf = 3896 (void *)(uintptr_t)idx; 3897 break; 3898 } 3899 /* Fall-through */ 3900 default: 3901 mlx5_free(translated); 3902 return rte_flow_error_set 3903 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3904 NULL, "invalid indirect action type"); 3905 } 3906 } 3907 *translated_actions = translated; 3908 return 0; 3909 } 3910 3911 /** 3912 * Get Shared RSS action from the action list. 3913 * 3914 * @param[in] dev 3915 * Pointer to Ethernet device. 3916 * @param[in] shared 3917 * Pointer to the list of actions. 3918 * @param[in] shared_n 3919 * Actions list length. 3920 * 3921 * @return 3922 * The MLX5 RSS action ID if exists, otherwise return 0. 3923 */ 3924 static uint32_t 3925 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3926 struct mlx5_translated_action_handle *handle, 3927 int shared_n) 3928 { 3929 struct mlx5_translated_action_handle *handle_end; 3930 struct mlx5_priv *priv = dev->data->dev_private; 3931 struct mlx5_shared_action_rss *shared_rss; 3932 3933 3934 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3935 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3936 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3937 uint32_t idx = act_idx & 3938 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3939 switch (type) { 3940 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3941 shared_rss = mlx5_ipool_get 3942 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3943 idx); 3944 __atomic_add_fetch(&shared_rss->refcnt, 1, 3945 __ATOMIC_RELAXED); 3946 return idx; 3947 default: 3948 break; 3949 } 3950 } 3951 return 0; 3952 } 3953 3954 static unsigned int 3955 find_graph_root(uint32_t rss_level) 3956 { 3957 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3958 MLX5_EXPANSION_ROOT_OUTER; 3959 } 3960 3961 /** 3962 * Get layer flags from the prefix flow. 3963 * 3964 * Some flows may be split to several subflows, the prefix subflow gets the 3965 * match items and the suffix sub flow gets the actions. 3966 * Some actions need the user defined match item flags to get the detail for 3967 * the action. 3968 * This function helps the suffix flow to get the item layer flags from prefix 3969 * subflow. 3970 * 3971 * @param[in] dev_flow 3972 * Pointer the created prefix subflow. 3973 * 3974 * @return 3975 * The layers get from prefix subflow. 3976 */ 3977 static inline uint64_t 3978 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3979 { 3980 uint64_t layers = 0; 3981 3982 /* 3983 * Layers bits could be localization, but usually the compiler will 3984 * help to do the optimization work for source code. 3985 * If no decap actions, use the layers directly. 3986 */ 3987 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3988 return dev_flow->handle->layers; 3989 /* Convert L3 layers with decap action. */ 3990 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3991 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3992 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3993 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3994 /* Convert L4 layers with decap action. */ 3995 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3996 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3997 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3998 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3999 return layers; 4000 } 4001 4002 /** 4003 * Get metadata split action information. 4004 * 4005 * @param[in] actions 4006 * Pointer to the list of actions. 4007 * @param[out] qrss 4008 * Pointer to the return pointer. 4009 * @param[out] qrss_type 4010 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4011 * if no QUEUE/RSS is found. 4012 * @param[out] encap_idx 4013 * Pointer to the index of the encap action if exists, otherwise the last 4014 * action index. 4015 * 4016 * @return 4017 * Total number of actions. 4018 */ 4019 static int 4020 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4021 const struct rte_flow_action **qrss, 4022 int *encap_idx) 4023 { 4024 const struct rte_flow_action_raw_encap *raw_encap; 4025 int actions_n = 0; 4026 int raw_decap_idx = -1; 4027 4028 *encap_idx = -1; 4029 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4030 switch (actions->type) { 4031 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4032 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4033 *encap_idx = actions_n; 4034 break; 4035 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4036 raw_decap_idx = actions_n; 4037 break; 4038 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4039 raw_encap = actions->conf; 4040 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4041 *encap_idx = raw_decap_idx != -1 ? 4042 raw_decap_idx : actions_n; 4043 break; 4044 case RTE_FLOW_ACTION_TYPE_QUEUE: 4045 case RTE_FLOW_ACTION_TYPE_RSS: 4046 *qrss = actions; 4047 break; 4048 default: 4049 break; 4050 } 4051 actions_n++; 4052 } 4053 if (*encap_idx == -1) 4054 *encap_idx = actions_n; 4055 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4056 return actions_n + 1; 4057 } 4058 4059 /** 4060 * Check if the action will change packet. 4061 * 4062 * @param dev 4063 * Pointer to Ethernet device. 4064 * @param[in] type 4065 * action type. 4066 * 4067 * @return 4068 * true if action will change packet, false otherwise. 4069 */ 4070 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4071 enum rte_flow_action_type type) 4072 { 4073 struct mlx5_priv *priv = dev->data->dev_private; 4074 4075 switch (type) { 4076 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4077 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4078 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4079 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4080 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4081 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4082 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4083 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4084 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4085 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4086 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4087 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4088 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4089 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4090 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4091 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4092 case RTE_FLOW_ACTION_TYPE_SET_META: 4093 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4094 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4095 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4096 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4097 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4098 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4099 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4100 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4101 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4102 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4103 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4104 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4105 return true; 4106 case RTE_FLOW_ACTION_TYPE_FLAG: 4107 case RTE_FLOW_ACTION_TYPE_MARK: 4108 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4109 return true; 4110 else 4111 return false; 4112 default: 4113 return false; 4114 } 4115 } 4116 4117 /** 4118 * Check meter action from the action list. 4119 * 4120 * @param dev 4121 * Pointer to Ethernet device. 4122 * @param[in] actions 4123 * Pointer to the list of actions. 4124 * @param[out] has_mtr 4125 * Pointer to the meter exist flag. 4126 * @param[out] has_modify 4127 * Pointer to the flag showing there's packet change action. 4128 * @param[out] meter_id 4129 * Pointer to the meter id. 4130 * 4131 * @return 4132 * Total number of actions. 4133 */ 4134 static int 4135 flow_check_meter_action(struct rte_eth_dev *dev, 4136 const struct rte_flow_action actions[], 4137 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4138 { 4139 const struct rte_flow_action_meter *mtr = NULL; 4140 int actions_n = 0; 4141 4142 MLX5_ASSERT(has_mtr); 4143 *has_mtr = false; 4144 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4145 switch (actions->type) { 4146 case RTE_FLOW_ACTION_TYPE_METER: 4147 mtr = actions->conf; 4148 *meter_id = mtr->mtr_id; 4149 *has_mtr = true; 4150 break; 4151 default: 4152 break; 4153 } 4154 if (!*has_mtr) 4155 *has_modify |= flow_check_modify_action_type(dev, 4156 actions->type); 4157 actions_n++; 4158 } 4159 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4160 return actions_n + 1; 4161 } 4162 4163 /** 4164 * Check if the flow should be split due to hairpin. 4165 * The reason for the split is that in current HW we can't 4166 * support encap and push-vlan on Rx, so if a flow contains 4167 * these actions we move it to Tx. 4168 * 4169 * @param dev 4170 * Pointer to Ethernet device. 4171 * @param[in] attr 4172 * Flow rule attributes. 4173 * @param[in] actions 4174 * Associated actions (list terminated by the END action). 4175 * 4176 * @return 4177 * > 0 the number of actions and the flow should be split, 4178 * 0 when no split required. 4179 */ 4180 static int 4181 flow_check_hairpin_split(struct rte_eth_dev *dev, 4182 const struct rte_flow_attr *attr, 4183 const struct rte_flow_action actions[]) 4184 { 4185 int queue_action = 0; 4186 int action_n = 0; 4187 int split = 0; 4188 const struct rte_flow_action_queue *queue; 4189 const struct rte_flow_action_rss *rss; 4190 const struct rte_flow_action_raw_encap *raw_encap; 4191 const struct rte_eth_hairpin_conf *conf; 4192 4193 if (!attr->ingress) 4194 return 0; 4195 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4196 switch (actions->type) { 4197 case RTE_FLOW_ACTION_TYPE_QUEUE: 4198 queue = actions->conf; 4199 if (queue == NULL) 4200 return 0; 4201 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4202 if (conf == NULL || conf->tx_explicit != 0) 4203 return 0; 4204 queue_action = 1; 4205 action_n++; 4206 break; 4207 case RTE_FLOW_ACTION_TYPE_RSS: 4208 rss = actions->conf; 4209 if (rss == NULL || rss->queue_num == 0) 4210 return 0; 4211 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4212 if (conf == NULL || conf->tx_explicit != 0) 4213 return 0; 4214 queue_action = 1; 4215 action_n++; 4216 break; 4217 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4218 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4219 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4220 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4221 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4222 split++; 4223 action_n++; 4224 break; 4225 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4226 raw_encap = actions->conf; 4227 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4228 split++; 4229 action_n++; 4230 break; 4231 default: 4232 action_n++; 4233 break; 4234 } 4235 } 4236 if (split && queue_action) 4237 return action_n; 4238 return 0; 4239 } 4240 4241 /* Declare flow create/destroy prototype in advance. */ 4242 static uint32_t 4243 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4244 const struct rte_flow_attr *attr, 4245 const struct rte_flow_item items[], 4246 const struct rte_flow_action actions[], 4247 bool external, struct rte_flow_error *error); 4248 4249 static void 4250 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4251 uint32_t flow_idx); 4252 4253 int 4254 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4255 struct mlx5_list_entry *entry, void *cb_ctx) 4256 { 4257 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4258 struct mlx5_flow_mreg_copy_resource *mcp_res = 4259 container_of(entry, typeof(*mcp_res), hlist_ent); 4260 4261 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4262 } 4263 4264 struct mlx5_list_entry * 4265 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4266 { 4267 struct rte_eth_dev *dev = tool_ctx; 4268 struct mlx5_priv *priv = dev->data->dev_private; 4269 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4270 struct mlx5_flow_mreg_copy_resource *mcp_res; 4271 struct rte_flow_error *error = ctx->error; 4272 uint32_t idx = 0; 4273 int ret; 4274 uint32_t mark_id = *(uint32_t *)(ctx->data); 4275 struct rte_flow_attr attr = { 4276 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4277 .ingress = 1, 4278 }; 4279 struct mlx5_rte_flow_item_tag tag_spec = { 4280 .data = mark_id, 4281 }; 4282 struct rte_flow_item items[] = { 4283 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4284 }; 4285 struct rte_flow_action_mark ftag = { 4286 .id = mark_id, 4287 }; 4288 struct mlx5_flow_action_copy_mreg cp_mreg = { 4289 .dst = REG_B, 4290 .src = REG_NON, 4291 }; 4292 struct rte_flow_action_jump jump = { 4293 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4294 }; 4295 struct rte_flow_action actions[] = { 4296 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4297 }; 4298 4299 /* Fill the register fields in the flow. */ 4300 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4301 if (ret < 0) 4302 return NULL; 4303 tag_spec.id = ret; 4304 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4305 if (ret < 0) 4306 return NULL; 4307 cp_mreg.src = ret; 4308 /* Provide the full width of FLAG specific value. */ 4309 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4310 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4311 /* Build a new flow. */ 4312 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4313 items[0] = (struct rte_flow_item){ 4314 .type = (enum rte_flow_item_type) 4315 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4316 .spec = &tag_spec, 4317 }; 4318 items[1] = (struct rte_flow_item){ 4319 .type = RTE_FLOW_ITEM_TYPE_END, 4320 }; 4321 actions[0] = (struct rte_flow_action){ 4322 .type = (enum rte_flow_action_type) 4323 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4324 .conf = &ftag, 4325 }; 4326 actions[1] = (struct rte_flow_action){ 4327 .type = (enum rte_flow_action_type) 4328 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4329 .conf = &cp_mreg, 4330 }; 4331 actions[2] = (struct rte_flow_action){ 4332 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4333 .conf = &jump, 4334 }; 4335 actions[3] = (struct rte_flow_action){ 4336 .type = RTE_FLOW_ACTION_TYPE_END, 4337 }; 4338 } else { 4339 /* Default rule, wildcard match. */ 4340 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4341 items[0] = (struct rte_flow_item){ 4342 .type = RTE_FLOW_ITEM_TYPE_END, 4343 }; 4344 actions[0] = (struct rte_flow_action){ 4345 .type = (enum rte_flow_action_type) 4346 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4347 .conf = &cp_mreg, 4348 }; 4349 actions[1] = (struct rte_flow_action){ 4350 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4351 .conf = &jump, 4352 }; 4353 actions[2] = (struct rte_flow_action){ 4354 .type = RTE_FLOW_ACTION_TYPE_END, 4355 }; 4356 } 4357 /* Build a new entry. */ 4358 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4359 if (!mcp_res) { 4360 rte_errno = ENOMEM; 4361 return NULL; 4362 } 4363 mcp_res->idx = idx; 4364 mcp_res->mark_id = mark_id; 4365 /* 4366 * The copy Flows are not included in any list. There 4367 * ones are referenced from other Flows and can not 4368 * be applied, removed, deleted in arbitrary order 4369 * by list traversing. 4370 */ 4371 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4372 &attr, items, actions, false, error); 4373 if (!mcp_res->rix_flow) { 4374 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4375 return NULL; 4376 } 4377 return &mcp_res->hlist_ent; 4378 } 4379 4380 struct mlx5_list_entry * 4381 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4382 void *cb_ctx __rte_unused) 4383 { 4384 struct rte_eth_dev *dev = tool_ctx; 4385 struct mlx5_priv *priv = dev->data->dev_private; 4386 struct mlx5_flow_mreg_copy_resource *mcp_res; 4387 uint32_t idx = 0; 4388 4389 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4390 if (!mcp_res) { 4391 rte_errno = ENOMEM; 4392 return NULL; 4393 } 4394 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4395 mcp_res->idx = idx; 4396 return &mcp_res->hlist_ent; 4397 } 4398 4399 void 4400 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4401 { 4402 struct mlx5_flow_mreg_copy_resource *mcp_res = 4403 container_of(entry, typeof(*mcp_res), hlist_ent); 4404 struct rte_eth_dev *dev = tool_ctx; 4405 struct mlx5_priv *priv = dev->data->dev_private; 4406 4407 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4408 } 4409 4410 /** 4411 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4412 * 4413 * As mark_id is unique, if there's already a registered flow for the mark_id, 4414 * return by increasing the reference counter of the resource. Otherwise, create 4415 * the resource (mcp_res) and flow. 4416 * 4417 * Flow looks like, 4418 * - If ingress port is ANY and reg_c[1] is mark_id, 4419 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4420 * 4421 * For default flow (zero mark_id), flow is like, 4422 * - If ingress port is ANY, 4423 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4424 * 4425 * @param dev 4426 * Pointer to Ethernet device. 4427 * @param mark_id 4428 * ID of MARK action, zero means default flow for META. 4429 * @param[out] error 4430 * Perform verbose error reporting if not NULL. 4431 * 4432 * @return 4433 * Associated resource on success, NULL otherwise and rte_errno is set. 4434 */ 4435 static struct mlx5_flow_mreg_copy_resource * 4436 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4437 struct rte_flow_error *error) 4438 { 4439 struct mlx5_priv *priv = dev->data->dev_private; 4440 struct mlx5_list_entry *entry; 4441 struct mlx5_flow_cb_ctx ctx = { 4442 .dev = dev, 4443 .error = error, 4444 .data = &mark_id, 4445 }; 4446 4447 /* Check if already registered. */ 4448 MLX5_ASSERT(priv->mreg_cp_tbl); 4449 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4450 if (!entry) 4451 return NULL; 4452 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4453 hlist_ent); 4454 } 4455 4456 void 4457 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4458 { 4459 struct mlx5_flow_mreg_copy_resource *mcp_res = 4460 container_of(entry, typeof(*mcp_res), hlist_ent); 4461 struct rte_eth_dev *dev = tool_ctx; 4462 struct mlx5_priv *priv = dev->data->dev_private; 4463 4464 MLX5_ASSERT(mcp_res->rix_flow); 4465 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4466 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4467 } 4468 4469 /** 4470 * Release flow in RX_CP_TBL. 4471 * 4472 * @param dev 4473 * Pointer to Ethernet device. 4474 * @flow 4475 * Parent flow for wich copying is provided. 4476 */ 4477 static void 4478 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4479 struct rte_flow *flow) 4480 { 4481 struct mlx5_flow_mreg_copy_resource *mcp_res; 4482 struct mlx5_priv *priv = dev->data->dev_private; 4483 4484 if (!flow->rix_mreg_copy) 4485 return; 4486 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4487 flow->rix_mreg_copy); 4488 if (!mcp_res || !priv->mreg_cp_tbl) 4489 return; 4490 MLX5_ASSERT(mcp_res->rix_flow); 4491 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4492 flow->rix_mreg_copy = 0; 4493 } 4494 4495 /** 4496 * Remove the default copy action from RX_CP_TBL. 4497 * 4498 * This functions is called in the mlx5_dev_start(). No thread safe 4499 * is guaranteed. 4500 * 4501 * @param dev 4502 * Pointer to Ethernet device. 4503 */ 4504 static void 4505 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4506 { 4507 struct mlx5_list_entry *entry; 4508 struct mlx5_priv *priv = dev->data->dev_private; 4509 struct mlx5_flow_cb_ctx ctx; 4510 uint32_t mark_id; 4511 4512 /* Check if default flow is registered. */ 4513 if (!priv->mreg_cp_tbl) 4514 return; 4515 mark_id = MLX5_DEFAULT_COPY_ID; 4516 ctx.data = &mark_id; 4517 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4518 if (!entry) 4519 return; 4520 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4521 } 4522 4523 /** 4524 * Add the default copy action in in RX_CP_TBL. 4525 * 4526 * This functions is called in the mlx5_dev_start(). No thread safe 4527 * is guaranteed. 4528 * 4529 * @param dev 4530 * Pointer to Ethernet device. 4531 * @param[out] error 4532 * Perform verbose error reporting if not NULL. 4533 * 4534 * @return 4535 * 0 for success, negative value otherwise and rte_errno is set. 4536 */ 4537 static int 4538 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4539 struct rte_flow_error *error) 4540 { 4541 struct mlx5_priv *priv = dev->data->dev_private; 4542 struct mlx5_flow_mreg_copy_resource *mcp_res; 4543 struct mlx5_flow_cb_ctx ctx; 4544 uint32_t mark_id; 4545 4546 /* Check whether extensive metadata feature is engaged. */ 4547 if (!priv->sh->config.dv_flow_en || 4548 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4549 !mlx5_flow_ext_mreg_supported(dev) || 4550 !priv->sh->dv_regc0_mask) 4551 return 0; 4552 /* 4553 * Add default mreg copy flow may be called multiple time, but 4554 * only be called once in stop. Avoid register it twice. 4555 */ 4556 mark_id = MLX5_DEFAULT_COPY_ID; 4557 ctx.data = &mark_id; 4558 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4559 return 0; 4560 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4561 if (!mcp_res) 4562 return -rte_errno; 4563 return 0; 4564 } 4565 4566 /** 4567 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4568 * 4569 * All the flow having Q/RSS action should be split by 4570 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4571 * performs the following, 4572 * - CQE->flow_tag := reg_c[1] (MARK) 4573 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4574 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4575 * but there should be a flow per each MARK ID set by MARK action. 4576 * 4577 * For the aforementioned reason, if there's a MARK action in flow's action 4578 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4579 * the MARK ID to CQE's flow_tag like, 4580 * - If reg_c[1] is mark_id, 4581 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4582 * 4583 * For SET_META action which stores value in reg_c[0], as the destination is 4584 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4585 * MARK ID means the default flow. The default flow looks like, 4586 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4587 * 4588 * @param dev 4589 * Pointer to Ethernet device. 4590 * @param flow 4591 * Pointer to flow structure. 4592 * @param[in] actions 4593 * Pointer to the list of actions. 4594 * @param[out] error 4595 * Perform verbose error reporting if not NULL. 4596 * 4597 * @return 4598 * 0 on success, negative value otherwise and rte_errno is set. 4599 */ 4600 static int 4601 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4602 struct rte_flow *flow, 4603 const struct rte_flow_action *actions, 4604 struct rte_flow_error *error) 4605 { 4606 struct mlx5_priv *priv = dev->data->dev_private; 4607 struct mlx5_sh_config *config = &priv->sh->config; 4608 struct mlx5_flow_mreg_copy_resource *mcp_res; 4609 const struct rte_flow_action_mark *mark; 4610 4611 /* Check whether extensive metadata feature is engaged. */ 4612 if (!config->dv_flow_en || 4613 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4614 !mlx5_flow_ext_mreg_supported(dev) || 4615 !priv->sh->dv_regc0_mask) 4616 return 0; 4617 /* Find MARK action. */ 4618 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4619 switch (actions->type) { 4620 case RTE_FLOW_ACTION_TYPE_FLAG: 4621 mcp_res = flow_mreg_add_copy_action 4622 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4623 if (!mcp_res) 4624 return -rte_errno; 4625 flow->rix_mreg_copy = mcp_res->idx; 4626 return 0; 4627 case RTE_FLOW_ACTION_TYPE_MARK: 4628 mark = (const struct rte_flow_action_mark *) 4629 actions->conf; 4630 mcp_res = 4631 flow_mreg_add_copy_action(dev, mark->id, error); 4632 if (!mcp_res) 4633 return -rte_errno; 4634 flow->rix_mreg_copy = mcp_res->idx; 4635 return 0; 4636 default: 4637 break; 4638 } 4639 } 4640 return 0; 4641 } 4642 4643 #define MLX5_MAX_SPLIT_ACTIONS 24 4644 #define MLX5_MAX_SPLIT_ITEMS 24 4645 4646 /** 4647 * Split the hairpin flow. 4648 * Since HW can't support encap and push-vlan on Rx, we move these 4649 * actions to Tx. 4650 * If the count action is after the encap then we also 4651 * move the count action. in this case the count will also measure 4652 * the outer bytes. 4653 * 4654 * @param dev 4655 * Pointer to Ethernet device. 4656 * @param[in] actions 4657 * Associated actions (list terminated by the END action). 4658 * @param[out] actions_rx 4659 * Rx flow actions. 4660 * @param[out] actions_tx 4661 * Tx flow actions.. 4662 * @param[out] pattern_tx 4663 * The pattern items for the Tx flow. 4664 * @param[out] flow_id 4665 * The flow ID connected to this flow. 4666 * 4667 * @return 4668 * 0 on success. 4669 */ 4670 static int 4671 flow_hairpin_split(struct rte_eth_dev *dev, 4672 const struct rte_flow_action actions[], 4673 struct rte_flow_action actions_rx[], 4674 struct rte_flow_action actions_tx[], 4675 struct rte_flow_item pattern_tx[], 4676 uint32_t flow_id) 4677 { 4678 const struct rte_flow_action_raw_encap *raw_encap; 4679 const struct rte_flow_action_raw_decap *raw_decap; 4680 struct mlx5_rte_flow_action_set_tag *set_tag; 4681 struct rte_flow_action *tag_action; 4682 struct mlx5_rte_flow_item_tag *tag_item; 4683 struct rte_flow_item *item; 4684 char *addr; 4685 int encap = 0; 4686 4687 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4688 switch (actions->type) { 4689 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4690 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4691 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4692 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4693 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4694 rte_memcpy(actions_tx, actions, 4695 sizeof(struct rte_flow_action)); 4696 actions_tx++; 4697 break; 4698 case RTE_FLOW_ACTION_TYPE_COUNT: 4699 if (encap) { 4700 rte_memcpy(actions_tx, actions, 4701 sizeof(struct rte_flow_action)); 4702 actions_tx++; 4703 } else { 4704 rte_memcpy(actions_rx, actions, 4705 sizeof(struct rte_flow_action)); 4706 actions_rx++; 4707 } 4708 break; 4709 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4710 raw_encap = actions->conf; 4711 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4712 memcpy(actions_tx, actions, 4713 sizeof(struct rte_flow_action)); 4714 actions_tx++; 4715 encap = 1; 4716 } else { 4717 rte_memcpy(actions_rx, actions, 4718 sizeof(struct rte_flow_action)); 4719 actions_rx++; 4720 } 4721 break; 4722 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4723 raw_decap = actions->conf; 4724 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4725 memcpy(actions_tx, actions, 4726 sizeof(struct rte_flow_action)); 4727 actions_tx++; 4728 } else { 4729 rte_memcpy(actions_rx, actions, 4730 sizeof(struct rte_flow_action)); 4731 actions_rx++; 4732 } 4733 break; 4734 default: 4735 rte_memcpy(actions_rx, actions, 4736 sizeof(struct rte_flow_action)); 4737 actions_rx++; 4738 break; 4739 } 4740 } 4741 /* Add set meta action and end action for the Rx flow. */ 4742 tag_action = actions_rx; 4743 tag_action->type = (enum rte_flow_action_type) 4744 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4745 actions_rx++; 4746 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4747 actions_rx++; 4748 set_tag = (void *)actions_rx; 4749 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4750 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4751 .data = flow_id, 4752 }; 4753 MLX5_ASSERT(set_tag->id > REG_NON); 4754 tag_action->conf = set_tag; 4755 /* Create Tx item list. */ 4756 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4757 addr = (void *)&pattern_tx[2]; 4758 item = pattern_tx; 4759 item->type = (enum rte_flow_item_type) 4760 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4761 tag_item = (void *)addr; 4762 tag_item->data = flow_id; 4763 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4764 MLX5_ASSERT(set_tag->id > REG_NON); 4765 item->spec = tag_item; 4766 addr += sizeof(struct mlx5_rte_flow_item_tag); 4767 tag_item = (void *)addr; 4768 tag_item->data = UINT32_MAX; 4769 tag_item->id = UINT16_MAX; 4770 item->mask = tag_item; 4771 item->last = NULL; 4772 item++; 4773 item->type = RTE_FLOW_ITEM_TYPE_END; 4774 return 0; 4775 } 4776 4777 /** 4778 * The last stage of splitting chain, just creates the subflow 4779 * without any modification. 4780 * 4781 * @param[in] dev 4782 * Pointer to Ethernet device. 4783 * @param[in] flow 4784 * Parent flow structure pointer. 4785 * @param[in, out] sub_flow 4786 * Pointer to return the created subflow, may be NULL. 4787 * @param[in] attr 4788 * Flow rule attributes. 4789 * @param[in] items 4790 * Pattern specification (list terminated by the END pattern item). 4791 * @param[in] actions 4792 * Associated actions (list terminated by the END action). 4793 * @param[in] flow_split_info 4794 * Pointer to flow split info structure. 4795 * @param[out] error 4796 * Perform verbose error reporting if not NULL. 4797 * @return 4798 * 0 on success, negative value otherwise 4799 */ 4800 static int 4801 flow_create_split_inner(struct rte_eth_dev *dev, 4802 struct rte_flow *flow, 4803 struct mlx5_flow **sub_flow, 4804 const struct rte_flow_attr *attr, 4805 const struct rte_flow_item items[], 4806 const struct rte_flow_action actions[], 4807 struct mlx5_flow_split_info *flow_split_info, 4808 struct rte_flow_error *error) 4809 { 4810 struct mlx5_flow *dev_flow; 4811 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 4812 4813 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4814 flow_split_info->flow_idx, error); 4815 if (!dev_flow) 4816 return -rte_errno; 4817 dev_flow->flow = flow; 4818 dev_flow->external = flow_split_info->external; 4819 dev_flow->skip_scale = flow_split_info->skip_scale; 4820 /* Subflow object was created, we must include one in the list. */ 4821 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4822 dev_flow->handle, next); 4823 /* 4824 * If dev_flow is as one of the suffix flow, some actions in suffix 4825 * flow may need some user defined item layer flags, and pass the 4826 * Metadata rxq mark flag to suffix flow as well. 4827 */ 4828 if (flow_split_info->prefix_layers) 4829 dev_flow->handle->layers = flow_split_info->prefix_layers; 4830 if (flow_split_info->prefix_mark) { 4831 MLX5_ASSERT(wks); 4832 wks->mark = 1; 4833 } 4834 if (sub_flow) 4835 *sub_flow = dev_flow; 4836 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4837 dev_flow->dv.table_id = flow_split_info->table_id; 4838 #endif 4839 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4840 } 4841 4842 /** 4843 * Get the sub policy of a meter. 4844 * 4845 * @param[in] dev 4846 * Pointer to Ethernet device. 4847 * @param[in] flow 4848 * Parent flow structure pointer. 4849 * @param wks 4850 * Pointer to thread flow work space. 4851 * @param[in] attr 4852 * Flow rule attributes. 4853 * @param[in] items 4854 * Pattern specification (list terminated by the END pattern item). 4855 * @param[out] error 4856 * Perform verbose error reporting if not NULL. 4857 * 4858 * @return 4859 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4860 */ 4861 static struct mlx5_flow_meter_sub_policy * 4862 get_meter_sub_policy(struct rte_eth_dev *dev, 4863 struct rte_flow *flow, 4864 struct mlx5_flow_workspace *wks, 4865 const struct rte_flow_attr *attr, 4866 const struct rte_flow_item items[], 4867 struct rte_flow_error *error) 4868 { 4869 struct mlx5_flow_meter_policy *policy; 4870 struct mlx5_flow_meter_policy *final_policy; 4871 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4872 4873 policy = wks->policy; 4874 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4875 if (final_policy->is_rss || final_policy->is_queue) { 4876 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4877 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4878 uint32_t i; 4879 4880 /* 4881 * This is a tmp dev_flow, 4882 * no need to register any matcher for it in translate. 4883 */ 4884 wks->skip_matcher_reg = 1; 4885 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4886 struct mlx5_flow dev_flow = {0}; 4887 struct mlx5_flow_handle dev_handle = { {0} }; 4888 uint8_t fate = final_policy->act_cnt[i].fate_action; 4889 4890 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4891 const struct rte_flow_action_rss *rss_act = 4892 final_policy->act_cnt[i].rss->conf; 4893 struct rte_flow_action rss_actions[2] = { 4894 [0] = { 4895 .type = RTE_FLOW_ACTION_TYPE_RSS, 4896 .conf = rss_act, 4897 }, 4898 [1] = { 4899 .type = RTE_FLOW_ACTION_TYPE_END, 4900 .conf = NULL, 4901 } 4902 }; 4903 4904 dev_flow.handle = &dev_handle; 4905 dev_flow.ingress = attr->ingress; 4906 dev_flow.flow = flow; 4907 dev_flow.external = 0; 4908 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4909 dev_flow.dv.transfer = attr->transfer; 4910 #endif 4911 /** 4912 * Translate RSS action to get rss hash fields. 4913 */ 4914 if (flow_drv_translate(dev, &dev_flow, attr, 4915 items, rss_actions, error)) 4916 goto exit; 4917 rss_desc_v[i] = wks->rss_desc; 4918 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4919 rss_desc_v[i].hash_fields = 4920 dev_flow.hash_fields; 4921 rss_desc_v[i].queue_num = 4922 rss_desc_v[i].hash_fields ? 4923 rss_desc_v[i].queue_num : 1; 4924 rss_desc_v[i].tunnel = 4925 !!(dev_flow.handle->layers & 4926 MLX5_FLOW_LAYER_TUNNEL); 4927 /* Use the RSS queues in the containers. */ 4928 rss_desc_v[i].queue = 4929 (uint16_t *)(uintptr_t)rss_act->queue; 4930 rss_desc[i] = &rss_desc_v[i]; 4931 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 4932 /* This is queue action. */ 4933 rss_desc_v[i] = wks->rss_desc; 4934 rss_desc_v[i].key_len = 0; 4935 rss_desc_v[i].hash_fields = 0; 4936 rss_desc_v[i].queue = 4937 &final_policy->act_cnt[i].queue; 4938 rss_desc_v[i].queue_num = 1; 4939 rss_desc[i] = &rss_desc_v[i]; 4940 } else { 4941 rss_desc[i] = NULL; 4942 } 4943 } 4944 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4945 flow, policy, rss_desc); 4946 } else { 4947 enum mlx5_meter_domain mtr_domain = 4948 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4949 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4950 MLX5_MTR_DOMAIN_INGRESS); 4951 sub_policy = policy->sub_policys[mtr_domain][0]; 4952 } 4953 if (!sub_policy) 4954 rte_flow_error_set(error, EINVAL, 4955 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4956 "Failed to get meter sub-policy."); 4957 exit: 4958 return sub_policy; 4959 } 4960 4961 /** 4962 * Split the meter flow. 4963 * 4964 * As meter flow will split to three sub flow, other than meter 4965 * action, the other actions make sense to only meter accepts 4966 * the packet. If it need to be dropped, no other additional 4967 * actions should be take. 4968 * 4969 * One kind of special action which decapsulates the L3 tunnel 4970 * header will be in the prefix sub flow, as not to take the 4971 * L3 tunnel header into account. 4972 * 4973 * @param[in] dev 4974 * Pointer to Ethernet device. 4975 * @param[in] flow 4976 * Parent flow structure pointer. 4977 * @param wks 4978 * Pointer to thread flow work space. 4979 * @param[in] attr 4980 * Flow rule attributes. 4981 * @param[in] items 4982 * Pattern specification (list terminated by the END pattern item). 4983 * @param[out] sfx_items 4984 * Suffix flow match items (list terminated by the END pattern item). 4985 * @param[in] actions 4986 * Associated actions (list terminated by the END action). 4987 * @param[out] actions_sfx 4988 * Suffix flow actions. 4989 * @param[out] actions_pre 4990 * Prefix flow actions. 4991 * @param[out] mtr_flow_id 4992 * Pointer to meter flow id. 4993 * @param[out] error 4994 * Perform verbose error reporting if not NULL. 4995 * 4996 * @return 4997 * 0 on success, a negative errno value otherwise and rte_errno is set. 4998 */ 4999 static int 5000 flow_meter_split_prep(struct rte_eth_dev *dev, 5001 struct rte_flow *flow, 5002 struct mlx5_flow_workspace *wks, 5003 const struct rte_flow_attr *attr, 5004 const struct rte_flow_item items[], 5005 struct rte_flow_item sfx_items[], 5006 const struct rte_flow_action actions[], 5007 struct rte_flow_action actions_sfx[], 5008 struct rte_flow_action actions_pre[], 5009 uint32_t *mtr_flow_id, 5010 struct rte_flow_error *error) 5011 { 5012 struct mlx5_priv *priv = dev->data->dev_private; 5013 struct mlx5_flow_meter_info *fm = wks->fm; 5014 struct rte_flow_action *tag_action = NULL; 5015 struct rte_flow_item *tag_item; 5016 struct mlx5_rte_flow_action_set_tag *set_tag; 5017 const struct rte_flow_action_raw_encap *raw_encap; 5018 const struct rte_flow_action_raw_decap *raw_decap; 5019 struct mlx5_rte_flow_item_tag *tag_item_spec; 5020 struct mlx5_rte_flow_item_tag *tag_item_mask; 5021 uint32_t tag_id = 0; 5022 struct rte_flow_item *vlan_item_dst = NULL; 5023 const struct rte_flow_item *vlan_item_src = NULL; 5024 const struct rte_flow_item *orig_items = items; 5025 struct rte_flow_action *hw_mtr_action; 5026 struct rte_flow_action *action_pre_head = NULL; 5027 int32_t flow_src_port = priv->representor_id; 5028 bool mtr_first; 5029 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5030 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5031 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5032 uint32_t flow_id = 0; 5033 uint32_t flow_id_reversed = 0; 5034 uint8_t flow_id_bits = 0; 5035 int shift; 5036 5037 /* Prepare the suffix subflow items. */ 5038 tag_item = sfx_items++; 5039 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5040 struct mlx5_priv *port_priv; 5041 const struct rte_flow_item_port_id *pid_v; 5042 int item_type = items->type; 5043 5044 switch (item_type) { 5045 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5046 pid_v = items->spec; 5047 MLX5_ASSERT(pid_v); 5048 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5049 if (!port_priv) 5050 return rte_flow_error_set(error, 5051 rte_errno, 5052 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5053 pid_v, 5054 "Failed to get port info."); 5055 flow_src_port = port_priv->representor_id; 5056 if (!fm->def_policy && wks->policy->is_hierarchy && 5057 flow_src_port != priv->representor_id) { 5058 if (flow_drv_mtr_hierarchy_rule_create(dev, 5059 flow, fm, 5060 flow_src_port, 5061 items, 5062 error)) 5063 return -rte_errno; 5064 } 5065 memcpy(sfx_items, items, sizeof(*sfx_items)); 5066 sfx_items++; 5067 break; 5068 case RTE_FLOW_ITEM_TYPE_VLAN: 5069 /* Determine if copy vlan item below. */ 5070 vlan_item_src = items; 5071 vlan_item_dst = sfx_items++; 5072 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5073 break; 5074 default: 5075 break; 5076 } 5077 } 5078 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5079 sfx_items++; 5080 mtr_first = priv->sh->meter_aso_en && 5081 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5082 /* For ASO meter, meter must be before tag in TX direction. */ 5083 if (mtr_first) { 5084 action_pre_head = actions_pre++; 5085 /* Leave space for tag action. */ 5086 tag_action = actions_pre++; 5087 } 5088 /* Prepare the actions for prefix and suffix flow. */ 5089 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5090 struct rte_flow_action *action_cur = NULL; 5091 5092 switch (actions->type) { 5093 case RTE_FLOW_ACTION_TYPE_METER: 5094 if (mtr_first) { 5095 action_cur = action_pre_head; 5096 } else { 5097 /* Leave space for tag action. */ 5098 tag_action = actions_pre++; 5099 action_cur = actions_pre++; 5100 } 5101 break; 5102 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5103 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5104 action_cur = actions_pre++; 5105 break; 5106 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5107 raw_encap = actions->conf; 5108 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5109 action_cur = actions_pre++; 5110 break; 5111 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5112 raw_decap = actions->conf; 5113 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5114 action_cur = actions_pre++; 5115 break; 5116 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5117 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5118 if (vlan_item_dst && vlan_item_src) { 5119 memcpy(vlan_item_dst, vlan_item_src, 5120 sizeof(*vlan_item_dst)); 5121 /* 5122 * Convert to internal match item, it is used 5123 * for vlan push and set vid. 5124 */ 5125 vlan_item_dst->type = (enum rte_flow_item_type) 5126 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5127 } 5128 break; 5129 default: 5130 break; 5131 } 5132 if (!action_cur) 5133 action_cur = (fm->def_policy) ? 5134 actions_sfx++ : actions_pre++; 5135 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5136 } 5137 /* Add end action to the actions. */ 5138 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5139 if (priv->sh->meter_aso_en) { 5140 /** 5141 * For ASO meter, need to add an extra jump action explicitly, 5142 * to jump from meter to policer table. 5143 */ 5144 struct mlx5_flow_meter_sub_policy *sub_policy; 5145 struct mlx5_flow_tbl_data_entry *tbl_data; 5146 5147 if (!fm->def_policy) { 5148 sub_policy = get_meter_sub_policy(dev, flow, wks, 5149 attr, orig_items, 5150 error); 5151 if (!sub_policy) 5152 return -rte_errno; 5153 } else { 5154 enum mlx5_meter_domain mtr_domain = 5155 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5156 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5157 MLX5_MTR_DOMAIN_INGRESS); 5158 5159 sub_policy = 5160 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5161 } 5162 tbl_data = container_of(sub_policy->tbl_rsc, 5163 struct mlx5_flow_tbl_data_entry, tbl); 5164 hw_mtr_action = actions_pre++; 5165 hw_mtr_action->type = (enum rte_flow_action_type) 5166 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5167 hw_mtr_action->conf = tbl_data->jump.action; 5168 } 5169 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5170 actions_pre++; 5171 if (!tag_action) 5172 return rte_flow_error_set(error, ENOMEM, 5173 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5174 NULL, "No tag action space."); 5175 if (!mtr_flow_id) { 5176 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5177 goto exit; 5178 } 5179 /* Only default-policy Meter creates mtr flow id. */ 5180 if (fm->def_policy) { 5181 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5182 if (!tag_id) 5183 return rte_flow_error_set(error, ENOMEM, 5184 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5185 "Failed to allocate meter flow id."); 5186 flow_id = tag_id - 1; 5187 flow_id_bits = (!flow_id) ? 1 : 5188 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5189 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5190 mtr_reg_bits) { 5191 mlx5_ipool_free(fm->flow_ipool, tag_id); 5192 return rte_flow_error_set(error, EINVAL, 5193 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5194 "Meter flow id exceeds max limit."); 5195 } 5196 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5197 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5198 } 5199 /* Build tag actions and items for meter_id/meter flow_id. */ 5200 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5201 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5202 tag_item_mask = tag_item_spec + 1; 5203 /* Both flow_id and meter_id share the same register. */ 5204 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5205 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5206 0, error), 5207 .offset = mtr_id_offset, 5208 .length = mtr_reg_bits, 5209 .data = flow->meter, 5210 }; 5211 /* 5212 * The color Reg bits used by flow_id are growing from 5213 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5214 */ 5215 for (shift = 0; shift < flow_id_bits; shift++) 5216 flow_id_reversed = (flow_id_reversed << 1) | 5217 ((flow_id >> shift) & 0x1); 5218 set_tag->data |= 5219 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5220 tag_item_spec->id = set_tag->id; 5221 tag_item_spec->data = set_tag->data << mtr_id_offset; 5222 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5223 tag_action->type = (enum rte_flow_action_type) 5224 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5225 tag_action->conf = set_tag; 5226 tag_item->type = (enum rte_flow_item_type) 5227 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5228 tag_item->spec = tag_item_spec; 5229 tag_item->last = NULL; 5230 tag_item->mask = tag_item_mask; 5231 exit: 5232 if (mtr_flow_id) 5233 *mtr_flow_id = tag_id; 5234 return 0; 5235 } 5236 5237 /** 5238 * Split action list having QUEUE/RSS for metadata register copy. 5239 * 5240 * Once Q/RSS action is detected in user's action list, the flow action 5241 * should be split in order to copy metadata registers, which will happen in 5242 * RX_CP_TBL like, 5243 * - CQE->flow_tag := reg_c[1] (MARK) 5244 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5245 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5246 * This is because the last action of each flow must be a terminal action 5247 * (QUEUE, RSS or DROP). 5248 * 5249 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5250 * stored and kept in the mlx5_flow structure per each sub_flow. 5251 * 5252 * The Q/RSS action is replaced with, 5253 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5254 * And the following JUMP action is added at the end, 5255 * - JUMP, to RX_CP_TBL. 5256 * 5257 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5258 * flow_create_split_metadata() routine. The flow will look like, 5259 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5260 * 5261 * @param dev 5262 * Pointer to Ethernet device. 5263 * @param[out] split_actions 5264 * Pointer to store split actions to jump to CP_TBL. 5265 * @param[in] actions 5266 * Pointer to the list of original flow actions. 5267 * @param[in] qrss 5268 * Pointer to the Q/RSS action. 5269 * @param[in] actions_n 5270 * Number of original actions. 5271 * @param[in] mtr_sfx 5272 * Check if it is in meter suffix table. 5273 * @param[out] error 5274 * Perform verbose error reporting if not NULL. 5275 * 5276 * @return 5277 * non-zero unique flow_id on success, otherwise 0 and 5278 * error/rte_error are set. 5279 */ 5280 static uint32_t 5281 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5282 struct rte_flow_action *split_actions, 5283 const struct rte_flow_action *actions, 5284 const struct rte_flow_action *qrss, 5285 int actions_n, int mtr_sfx, 5286 struct rte_flow_error *error) 5287 { 5288 struct mlx5_priv *priv = dev->data->dev_private; 5289 struct mlx5_rte_flow_action_set_tag *set_tag; 5290 struct rte_flow_action_jump *jump; 5291 const int qrss_idx = qrss - actions; 5292 uint32_t flow_id = 0; 5293 int ret = 0; 5294 5295 /* 5296 * Given actions will be split 5297 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5298 * - Add jump to mreg CP_TBL. 5299 * As a result, there will be one more action. 5300 */ 5301 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5302 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5303 ++actions_n; 5304 set_tag = (void *)(split_actions + actions_n); 5305 /* 5306 * If we are not the meter suffix flow, add the tag action. 5307 * Since meter suffix flow already has the tag added. 5308 */ 5309 if (!mtr_sfx) { 5310 /* 5311 * Allocate the new subflow ID. This one is unique within 5312 * device and not shared with representors. Otherwise, 5313 * we would have to resolve multi-thread access synch 5314 * issue. Each flow on the shared device is appended 5315 * with source vport identifier, so the resulting 5316 * flows will be unique in the shared (by master and 5317 * representors) domain even if they have coinciding 5318 * IDs. 5319 */ 5320 mlx5_ipool_malloc(priv->sh->ipool 5321 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5322 if (!flow_id) 5323 return rte_flow_error_set(error, ENOMEM, 5324 RTE_FLOW_ERROR_TYPE_ACTION, 5325 NULL, "can't allocate id " 5326 "for split Q/RSS subflow"); 5327 /* Internal SET_TAG action to set flow ID. */ 5328 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5329 .data = flow_id, 5330 }; 5331 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5332 if (ret < 0) 5333 return ret; 5334 set_tag->id = ret; 5335 /* Construct new actions array. */ 5336 /* Replace QUEUE/RSS action. */ 5337 split_actions[qrss_idx] = (struct rte_flow_action){ 5338 .type = (enum rte_flow_action_type) 5339 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5340 .conf = set_tag, 5341 }; 5342 } else { 5343 /* 5344 * If we are the suffix flow of meter, tag already exist. 5345 * Set the QUEUE/RSS action to void. 5346 */ 5347 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5348 } 5349 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5350 jump = (void *)(set_tag + 1); 5351 *jump = (struct rte_flow_action_jump){ 5352 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5353 }; 5354 split_actions[actions_n - 2] = (struct rte_flow_action){ 5355 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5356 .conf = jump, 5357 }; 5358 split_actions[actions_n - 1] = (struct rte_flow_action){ 5359 .type = RTE_FLOW_ACTION_TYPE_END, 5360 }; 5361 return flow_id; 5362 } 5363 5364 /** 5365 * Extend the given action list for Tx metadata copy. 5366 * 5367 * Copy the given action list to the ext_actions and add flow metadata register 5368 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5369 * 5370 * @param[out] ext_actions 5371 * Pointer to the extended action list. 5372 * @param[in] actions 5373 * Pointer to the list of actions. 5374 * @param[in] actions_n 5375 * Number of actions in the list. 5376 * @param[out] error 5377 * Perform verbose error reporting if not NULL. 5378 * @param[in] encap_idx 5379 * The encap action index. 5380 * 5381 * @return 5382 * 0 on success, negative value otherwise 5383 */ 5384 static int 5385 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5386 struct rte_flow_action *ext_actions, 5387 const struct rte_flow_action *actions, 5388 int actions_n, struct rte_flow_error *error, 5389 int encap_idx) 5390 { 5391 struct mlx5_flow_action_copy_mreg *cp_mreg = 5392 (struct mlx5_flow_action_copy_mreg *) 5393 (ext_actions + actions_n + 1); 5394 int ret; 5395 5396 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5397 if (ret < 0) 5398 return ret; 5399 cp_mreg->dst = ret; 5400 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5401 if (ret < 0) 5402 return ret; 5403 cp_mreg->src = ret; 5404 if (encap_idx != 0) 5405 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5406 if (encap_idx == actions_n - 1) { 5407 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5408 .type = (enum rte_flow_action_type) 5409 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5410 .conf = cp_mreg, 5411 }; 5412 ext_actions[actions_n] = (struct rte_flow_action){ 5413 .type = RTE_FLOW_ACTION_TYPE_END, 5414 }; 5415 } else { 5416 ext_actions[encap_idx] = (struct rte_flow_action){ 5417 .type = (enum rte_flow_action_type) 5418 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5419 .conf = cp_mreg, 5420 }; 5421 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5422 sizeof(*ext_actions) * (actions_n - encap_idx)); 5423 } 5424 return 0; 5425 } 5426 5427 /** 5428 * Check the match action from the action list. 5429 * 5430 * @param[in] actions 5431 * Pointer to the list of actions. 5432 * @param[in] attr 5433 * Flow rule attributes. 5434 * @param[in] action 5435 * The action to be check if exist. 5436 * @param[out] match_action_pos 5437 * Pointer to the position of the matched action if exists, otherwise is -1. 5438 * @param[out] qrss_action_pos 5439 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5440 * @param[out] modify_after_mirror 5441 * Pointer to the flag of modify action after FDB mirroring. 5442 * 5443 * @return 5444 * > 0 the total number of actions. 5445 * 0 if not found match action in action list. 5446 */ 5447 static int 5448 flow_check_match_action(const struct rte_flow_action actions[], 5449 const struct rte_flow_attr *attr, 5450 enum rte_flow_action_type action, 5451 int *match_action_pos, int *qrss_action_pos, 5452 int *modify_after_mirror) 5453 { 5454 const struct rte_flow_action_sample *sample; 5455 const struct rte_flow_action_raw_decap *decap; 5456 int actions_n = 0; 5457 uint32_t ratio = 0; 5458 int sub_type = 0; 5459 int flag = 0; 5460 int fdb_mirror = 0; 5461 5462 *match_action_pos = -1; 5463 *qrss_action_pos = -1; 5464 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5465 if (actions->type == action) { 5466 flag = 1; 5467 *match_action_pos = actions_n; 5468 } 5469 switch (actions->type) { 5470 case RTE_FLOW_ACTION_TYPE_QUEUE: 5471 case RTE_FLOW_ACTION_TYPE_RSS: 5472 *qrss_action_pos = actions_n; 5473 break; 5474 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5475 sample = actions->conf; 5476 ratio = sample->ratio; 5477 sub_type = ((const struct rte_flow_action *) 5478 (sample->actions))->type; 5479 if (ratio == 1 && attr->transfer) 5480 fdb_mirror = 1; 5481 break; 5482 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5483 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5484 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5485 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5486 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5487 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5488 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5489 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5490 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5491 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5492 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5493 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5494 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5495 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5496 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5497 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5498 case RTE_FLOW_ACTION_TYPE_FLAG: 5499 case RTE_FLOW_ACTION_TYPE_MARK: 5500 case RTE_FLOW_ACTION_TYPE_SET_META: 5501 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5502 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5503 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5504 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5505 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5506 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5507 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5508 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5509 case RTE_FLOW_ACTION_TYPE_METER: 5510 if (fdb_mirror) 5511 *modify_after_mirror = 1; 5512 break; 5513 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5514 decap = actions->conf; 5515 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5516 ; 5517 actions_n++; 5518 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5519 const struct rte_flow_action_raw_encap *encap = 5520 actions->conf; 5521 if (decap->size <= 5522 MLX5_ENCAPSULATION_DECISION_SIZE && 5523 encap->size > 5524 MLX5_ENCAPSULATION_DECISION_SIZE) 5525 /* L3 encap. */ 5526 break; 5527 } 5528 if (fdb_mirror) 5529 *modify_after_mirror = 1; 5530 break; 5531 default: 5532 break; 5533 } 5534 actions_n++; 5535 } 5536 if (flag && fdb_mirror && !*modify_after_mirror) { 5537 /* FDB mirroring uses the destination array to implement 5538 * instead of FLOW_SAMPLER object. 5539 */ 5540 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5541 flag = 0; 5542 } 5543 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5544 return flag ? actions_n + 1 : 0; 5545 } 5546 5547 #define SAMPLE_SUFFIX_ITEM 2 5548 5549 /** 5550 * Split the sample flow. 5551 * 5552 * As sample flow will split to two sub flow, sample flow with 5553 * sample action, the other actions will move to new suffix flow. 5554 * 5555 * Also add unique tag id with tag action in the sample flow, 5556 * the same tag id will be as match in the suffix flow. 5557 * 5558 * @param dev 5559 * Pointer to Ethernet device. 5560 * @param[in] add_tag 5561 * Add extra tag action flag. 5562 * @param[out] sfx_items 5563 * Suffix flow match items (list terminated by the END pattern item). 5564 * @param[in] actions 5565 * Associated actions (list terminated by the END action). 5566 * @param[out] actions_sfx 5567 * Suffix flow actions. 5568 * @param[out] actions_pre 5569 * Prefix flow actions. 5570 * @param[in] actions_n 5571 * The total number of actions. 5572 * @param[in] sample_action_pos 5573 * The sample action position. 5574 * @param[in] qrss_action_pos 5575 * The Queue/RSS action position. 5576 * @param[in] jump_table 5577 * Add extra jump action flag. 5578 * @param[out] error 5579 * Perform verbose error reporting if not NULL. 5580 * 5581 * @return 5582 * 0 on success, or unique flow_id, a negative errno value 5583 * otherwise and rte_errno is set. 5584 */ 5585 static int 5586 flow_sample_split_prep(struct rte_eth_dev *dev, 5587 int add_tag, 5588 struct rte_flow_item sfx_items[], 5589 const struct rte_flow_action actions[], 5590 struct rte_flow_action actions_sfx[], 5591 struct rte_flow_action actions_pre[], 5592 int actions_n, 5593 int sample_action_pos, 5594 int qrss_action_pos, 5595 int jump_table, 5596 struct rte_flow_error *error) 5597 { 5598 struct mlx5_priv *priv = dev->data->dev_private; 5599 struct mlx5_rte_flow_action_set_tag *set_tag; 5600 struct mlx5_rte_flow_item_tag *tag_spec; 5601 struct mlx5_rte_flow_item_tag *tag_mask; 5602 struct rte_flow_action_jump *jump_action; 5603 uint32_t tag_id = 0; 5604 int index; 5605 int append_index = 0; 5606 int ret; 5607 5608 if (sample_action_pos < 0) 5609 return rte_flow_error_set(error, EINVAL, 5610 RTE_FLOW_ERROR_TYPE_ACTION, 5611 NULL, "invalid position of sample " 5612 "action in list"); 5613 /* Prepare the actions for prefix and suffix flow. */ 5614 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5615 index = qrss_action_pos; 5616 /* Put the preceding the Queue/RSS action into prefix flow. */ 5617 if (index != 0) 5618 memcpy(actions_pre, actions, 5619 sizeof(struct rte_flow_action) * index); 5620 /* Put others preceding the sample action into prefix flow. */ 5621 if (sample_action_pos > index + 1) 5622 memcpy(actions_pre + index, actions + index + 1, 5623 sizeof(struct rte_flow_action) * 5624 (sample_action_pos - index - 1)); 5625 index = sample_action_pos - 1; 5626 /* Put Queue/RSS action into Suffix flow. */ 5627 memcpy(actions_sfx, actions + qrss_action_pos, 5628 sizeof(struct rte_flow_action)); 5629 actions_sfx++; 5630 } else { 5631 index = sample_action_pos; 5632 if (index != 0) 5633 memcpy(actions_pre, actions, 5634 sizeof(struct rte_flow_action) * index); 5635 } 5636 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5637 * For CX6DX and above, metadata registers Cx preserve their value, 5638 * add an extra tag action for NIC-RX and E-Switch Domain. 5639 */ 5640 if (add_tag) { 5641 /* Prepare the prefix tag action. */ 5642 append_index++; 5643 set_tag = (void *)(actions_pre + actions_n + append_index); 5644 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5645 if (ret < 0) 5646 return ret; 5647 mlx5_ipool_malloc(priv->sh->ipool 5648 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5649 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5650 .id = ret, 5651 .data = tag_id, 5652 }; 5653 /* Prepare the suffix subflow items. */ 5654 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5655 tag_spec->data = tag_id; 5656 tag_spec->id = set_tag->id; 5657 tag_mask = tag_spec + 1; 5658 tag_mask->data = UINT32_MAX; 5659 sfx_items[0] = (struct rte_flow_item){ 5660 .type = (enum rte_flow_item_type) 5661 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5662 .spec = tag_spec, 5663 .last = NULL, 5664 .mask = tag_mask, 5665 }; 5666 sfx_items[1] = (struct rte_flow_item){ 5667 .type = (enum rte_flow_item_type) 5668 RTE_FLOW_ITEM_TYPE_END, 5669 }; 5670 /* Prepare the tag action in prefix subflow. */ 5671 actions_pre[index++] = 5672 (struct rte_flow_action){ 5673 .type = (enum rte_flow_action_type) 5674 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5675 .conf = set_tag, 5676 }; 5677 } 5678 memcpy(actions_pre + index, actions + sample_action_pos, 5679 sizeof(struct rte_flow_action)); 5680 index += 1; 5681 /* For the modify action after the sample action in E-Switch mirroring, 5682 * Add the extra jump action in prefix subflow and jump into the next 5683 * table, then do the modify action in the new table. 5684 */ 5685 if (jump_table) { 5686 /* Prepare the prefix jump action. */ 5687 append_index++; 5688 jump_action = (void *)(actions_pre + actions_n + append_index); 5689 jump_action->group = jump_table; 5690 actions_pre[index++] = 5691 (struct rte_flow_action){ 5692 .type = (enum rte_flow_action_type) 5693 RTE_FLOW_ACTION_TYPE_JUMP, 5694 .conf = jump_action, 5695 }; 5696 } 5697 actions_pre[index] = (struct rte_flow_action){ 5698 .type = (enum rte_flow_action_type) 5699 RTE_FLOW_ACTION_TYPE_END, 5700 }; 5701 /* Put the actions after sample into Suffix flow. */ 5702 memcpy(actions_sfx, actions + sample_action_pos + 1, 5703 sizeof(struct rte_flow_action) * 5704 (actions_n - sample_action_pos - 1)); 5705 return tag_id; 5706 } 5707 5708 /** 5709 * The splitting for metadata feature. 5710 * 5711 * - Q/RSS action on NIC Rx should be split in order to pass by 5712 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5713 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5714 * 5715 * - All the actions on NIC Tx should have a mreg copy action to 5716 * copy reg_a from WQE to reg_c[0]. 5717 * 5718 * @param dev 5719 * Pointer to Ethernet device. 5720 * @param[in] flow 5721 * Parent flow structure pointer. 5722 * @param[in] attr 5723 * Flow rule attributes. 5724 * @param[in] items 5725 * Pattern specification (list terminated by the END pattern item). 5726 * @param[in] actions 5727 * Associated actions (list terminated by the END action). 5728 * @param[in] flow_split_info 5729 * Pointer to flow split info structure. 5730 * @param[out] error 5731 * Perform verbose error reporting if not NULL. 5732 * @return 5733 * 0 on success, negative value otherwise 5734 */ 5735 static int 5736 flow_create_split_metadata(struct rte_eth_dev *dev, 5737 struct rte_flow *flow, 5738 const struct rte_flow_attr *attr, 5739 const struct rte_flow_item items[], 5740 const struct rte_flow_action actions[], 5741 struct mlx5_flow_split_info *flow_split_info, 5742 struct rte_flow_error *error) 5743 { 5744 struct mlx5_priv *priv = dev->data->dev_private; 5745 struct mlx5_sh_config *config = &priv->sh->config; 5746 const struct rte_flow_action *qrss = NULL; 5747 struct rte_flow_action *ext_actions = NULL; 5748 struct mlx5_flow *dev_flow = NULL; 5749 uint32_t qrss_id = 0; 5750 int mtr_sfx = 0; 5751 size_t act_size; 5752 int actions_n; 5753 int encap_idx; 5754 int ret; 5755 5756 /* Check whether extensive metadata feature is engaged. */ 5757 if (!config->dv_flow_en || 5758 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5759 !mlx5_flow_ext_mreg_supported(dev)) 5760 return flow_create_split_inner(dev, flow, NULL, attr, items, 5761 actions, flow_split_info, error); 5762 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5763 &encap_idx); 5764 if (qrss) { 5765 /* Exclude hairpin flows from splitting. */ 5766 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5767 const struct rte_flow_action_queue *queue; 5768 5769 queue = qrss->conf; 5770 if (mlx5_rxq_get_type(dev, queue->index) == 5771 MLX5_RXQ_TYPE_HAIRPIN) 5772 qrss = NULL; 5773 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5774 const struct rte_flow_action_rss *rss; 5775 5776 rss = qrss->conf; 5777 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5778 MLX5_RXQ_TYPE_HAIRPIN) 5779 qrss = NULL; 5780 } 5781 } 5782 if (qrss) { 5783 /* Check if it is in meter suffix table. */ 5784 mtr_sfx = attr->group == (attr->transfer ? 5785 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5786 MLX5_FLOW_TABLE_LEVEL_METER); 5787 /* 5788 * Q/RSS action on NIC Rx should be split in order to pass by 5789 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5790 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5791 */ 5792 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5793 sizeof(struct rte_flow_action_set_tag) + 5794 sizeof(struct rte_flow_action_jump); 5795 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5796 SOCKET_ID_ANY); 5797 if (!ext_actions) 5798 return rte_flow_error_set(error, ENOMEM, 5799 RTE_FLOW_ERROR_TYPE_ACTION, 5800 NULL, "no memory to split " 5801 "metadata flow"); 5802 /* 5803 * Create the new actions list with removed Q/RSS action 5804 * and appended set tag and jump to register copy table 5805 * (RX_CP_TBL). We should preallocate unique tag ID here 5806 * in advance, because it is needed for set tag action. 5807 */ 5808 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5809 qrss, actions_n, 5810 mtr_sfx, error); 5811 if (!mtr_sfx && !qrss_id) { 5812 ret = -rte_errno; 5813 goto exit; 5814 } 5815 } else if (attr->egress && !attr->transfer) { 5816 /* 5817 * All the actions on NIC Tx should have a metadata register 5818 * copy action to copy reg_a from WQE to reg_c[meta] 5819 */ 5820 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5821 sizeof(struct mlx5_flow_action_copy_mreg); 5822 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5823 SOCKET_ID_ANY); 5824 if (!ext_actions) 5825 return rte_flow_error_set(error, ENOMEM, 5826 RTE_FLOW_ERROR_TYPE_ACTION, 5827 NULL, "no memory to split " 5828 "metadata flow"); 5829 /* Create the action list appended with copy register. */ 5830 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5831 actions_n, error, encap_idx); 5832 if (ret < 0) 5833 goto exit; 5834 } 5835 /* Add the unmodified original or prefix subflow. */ 5836 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5837 items, ext_actions ? ext_actions : 5838 actions, flow_split_info, error); 5839 if (ret < 0) 5840 goto exit; 5841 MLX5_ASSERT(dev_flow); 5842 if (qrss) { 5843 const struct rte_flow_attr q_attr = { 5844 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5845 .ingress = 1, 5846 }; 5847 /* Internal PMD action to set register. */ 5848 struct mlx5_rte_flow_item_tag q_tag_spec = { 5849 .data = qrss_id, 5850 .id = REG_NON, 5851 }; 5852 struct rte_flow_item q_items[] = { 5853 { 5854 .type = (enum rte_flow_item_type) 5855 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5856 .spec = &q_tag_spec, 5857 .last = NULL, 5858 .mask = NULL, 5859 }, 5860 { 5861 .type = RTE_FLOW_ITEM_TYPE_END, 5862 }, 5863 }; 5864 struct rte_flow_action q_actions[] = { 5865 { 5866 .type = qrss->type, 5867 .conf = qrss->conf, 5868 }, 5869 { 5870 .type = RTE_FLOW_ACTION_TYPE_END, 5871 }, 5872 }; 5873 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5874 5875 /* 5876 * Configure the tag item only if there is no meter subflow. 5877 * Since tag is already marked in the meter suffix subflow 5878 * we can just use the meter suffix items as is. 5879 */ 5880 if (qrss_id) { 5881 /* Not meter subflow. */ 5882 MLX5_ASSERT(!mtr_sfx); 5883 /* 5884 * Put unique id in prefix flow due to it is destroyed 5885 * after suffix flow and id will be freed after there 5886 * is no actual flows with this id and identifier 5887 * reallocation becomes possible (for example, for 5888 * other flows in other threads). 5889 */ 5890 dev_flow->handle->split_flow_id = qrss_id; 5891 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5892 error); 5893 if (ret < 0) 5894 goto exit; 5895 q_tag_spec.id = ret; 5896 } 5897 dev_flow = NULL; 5898 /* Add suffix subflow to execute Q/RSS. */ 5899 flow_split_info->prefix_layers = layers; 5900 flow_split_info->prefix_mark = 0; 5901 flow_split_info->table_id = 0; 5902 ret = flow_create_split_inner(dev, flow, &dev_flow, 5903 &q_attr, mtr_sfx ? items : 5904 q_items, q_actions, 5905 flow_split_info, error); 5906 if (ret < 0) 5907 goto exit; 5908 /* qrss ID should be freed if failed. */ 5909 qrss_id = 0; 5910 MLX5_ASSERT(dev_flow); 5911 } 5912 5913 exit: 5914 /* 5915 * We do not destroy the partially created sub_flows in case of error. 5916 * These ones are included into parent flow list and will be destroyed 5917 * by flow_drv_destroy. 5918 */ 5919 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5920 qrss_id); 5921 mlx5_free(ext_actions); 5922 return ret; 5923 } 5924 5925 /** 5926 * Create meter internal drop flow with the original pattern. 5927 * 5928 * @param dev 5929 * Pointer to Ethernet device. 5930 * @param[in] flow 5931 * Parent flow structure pointer. 5932 * @param[in] attr 5933 * Flow rule attributes. 5934 * @param[in] items 5935 * Pattern specification (list terminated by the END pattern item). 5936 * @param[in] flow_split_info 5937 * Pointer to flow split info structure. 5938 * @param[in] fm 5939 * Pointer to flow meter structure. 5940 * @param[out] error 5941 * Perform verbose error reporting if not NULL. 5942 * @return 5943 * 0 on success, negative value otherwise 5944 */ 5945 static uint32_t 5946 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5947 struct rte_flow *flow, 5948 const struct rte_flow_attr *attr, 5949 const struct rte_flow_item items[], 5950 struct mlx5_flow_split_info *flow_split_info, 5951 struct mlx5_flow_meter_info *fm, 5952 struct rte_flow_error *error) 5953 { 5954 struct mlx5_flow *dev_flow = NULL; 5955 struct rte_flow_attr drop_attr = *attr; 5956 struct rte_flow_action drop_actions[3]; 5957 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5958 5959 MLX5_ASSERT(fm->drop_cnt); 5960 drop_actions[0].type = 5961 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5962 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5963 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5964 drop_actions[1].conf = NULL; 5965 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5966 drop_actions[2].conf = NULL; 5967 drop_split_info.external = false; 5968 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5969 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5970 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5971 return flow_create_split_inner(dev, flow, &dev_flow, 5972 &drop_attr, items, drop_actions, 5973 &drop_split_info, error); 5974 } 5975 5976 /** 5977 * The splitting for meter feature. 5978 * 5979 * - The meter flow will be split to two flows as prefix and 5980 * suffix flow. The packets make sense only it pass the prefix 5981 * meter action. 5982 * 5983 * - Reg_C_5 is used for the packet to match betweend prefix and 5984 * suffix flow. 5985 * 5986 * @param dev 5987 * Pointer to Ethernet device. 5988 * @param[in] flow 5989 * Parent flow structure pointer. 5990 * @param[in] attr 5991 * Flow rule attributes. 5992 * @param[in] items 5993 * Pattern specification (list terminated by the END pattern item). 5994 * @param[in] actions 5995 * Associated actions (list terminated by the END action). 5996 * @param[in] flow_split_info 5997 * Pointer to flow split info structure. 5998 * @param[out] error 5999 * Perform verbose error reporting if not NULL. 6000 * @return 6001 * 0 on success, negative value otherwise 6002 */ 6003 static int 6004 flow_create_split_meter(struct rte_eth_dev *dev, 6005 struct rte_flow *flow, 6006 const struct rte_flow_attr *attr, 6007 const struct rte_flow_item items[], 6008 const struct rte_flow_action actions[], 6009 struct mlx5_flow_split_info *flow_split_info, 6010 struct rte_flow_error *error) 6011 { 6012 struct mlx5_priv *priv = dev->data->dev_private; 6013 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6014 struct rte_flow_action *sfx_actions = NULL; 6015 struct rte_flow_action *pre_actions = NULL; 6016 struct rte_flow_item *sfx_items = NULL; 6017 struct mlx5_flow *dev_flow = NULL; 6018 struct rte_flow_attr sfx_attr = *attr; 6019 struct mlx5_flow_meter_info *fm = NULL; 6020 uint8_t skip_scale_restore; 6021 bool has_mtr = false; 6022 bool has_modify = false; 6023 bool set_mtr_reg = true; 6024 bool is_mtr_hierarchy = false; 6025 uint32_t meter_id = 0; 6026 uint32_t mtr_idx = 0; 6027 uint32_t mtr_flow_id = 0; 6028 size_t act_size; 6029 size_t item_size; 6030 int actions_n = 0; 6031 int ret = 0; 6032 6033 if (priv->mtr_en) 6034 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6035 &has_modify, &meter_id); 6036 if (has_mtr) { 6037 if (flow->meter) { 6038 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6039 if (!fm) 6040 return rte_flow_error_set(error, EINVAL, 6041 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6042 NULL, "Meter not found."); 6043 } else { 6044 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6045 if (!fm) 6046 return rte_flow_error_set(error, EINVAL, 6047 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6048 NULL, "Meter not found."); 6049 ret = mlx5_flow_meter_attach(priv, fm, 6050 &sfx_attr, error); 6051 if (ret) 6052 return -rte_errno; 6053 flow->meter = mtr_idx; 6054 } 6055 MLX5_ASSERT(wks); 6056 wks->fm = fm; 6057 if (!fm->def_policy) { 6058 wks->policy = mlx5_flow_meter_policy_find(dev, 6059 fm->policy_id, 6060 NULL); 6061 MLX5_ASSERT(wks->policy); 6062 if (wks->policy->is_hierarchy) { 6063 wks->final_policy = 6064 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6065 wks->policy); 6066 if (!wks->final_policy) 6067 return rte_flow_error_set(error, 6068 EINVAL, 6069 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6070 "Failed to find terminal policy of hierarchy."); 6071 is_mtr_hierarchy = true; 6072 } 6073 } 6074 /* 6075 * If it isn't default-policy Meter, and 6076 * 1. There's no action in flow to change 6077 * packet (modify/encap/decap etc.), OR 6078 * 2. No drop count needed for this meter. 6079 * 3. It's not meter hierarchy. 6080 * Then no need to use regC to save meter id anymore. 6081 */ 6082 if (!fm->def_policy && !is_mtr_hierarchy && 6083 (!has_modify || !fm->drop_cnt)) 6084 set_mtr_reg = false; 6085 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6086 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6087 sizeof(struct mlx5_rte_flow_action_set_tag); 6088 /* Suffix items: tag, vlan, port id, end. */ 6089 #define METER_SUFFIX_ITEM 4 6090 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6091 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6092 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6093 0, SOCKET_ID_ANY); 6094 if (!sfx_actions) 6095 return rte_flow_error_set(error, ENOMEM, 6096 RTE_FLOW_ERROR_TYPE_ACTION, 6097 NULL, "no memory to split " 6098 "meter flow"); 6099 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6100 act_size); 6101 /* There's no suffix flow for meter of non-default policy. */ 6102 if (!fm->def_policy) 6103 pre_actions = sfx_actions + 1; 6104 else 6105 pre_actions = sfx_actions + actions_n; 6106 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6107 items, sfx_items, actions, 6108 sfx_actions, pre_actions, 6109 (set_mtr_reg ? &mtr_flow_id : NULL), 6110 error); 6111 if (ret) { 6112 ret = -rte_errno; 6113 goto exit; 6114 } 6115 /* Add the prefix subflow. */ 6116 skip_scale_restore = flow_split_info->skip_scale; 6117 flow_split_info->skip_scale |= 6118 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6119 ret = flow_create_split_inner(dev, flow, &dev_flow, 6120 attr, items, pre_actions, 6121 flow_split_info, error); 6122 flow_split_info->skip_scale = skip_scale_restore; 6123 if (ret) { 6124 if (mtr_flow_id) 6125 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6126 ret = -rte_errno; 6127 goto exit; 6128 } 6129 if (mtr_flow_id) { 6130 dev_flow->handle->split_flow_id = mtr_flow_id; 6131 dev_flow->handle->is_meter_flow_id = 1; 6132 } 6133 if (!fm->def_policy) { 6134 if (!set_mtr_reg && fm->drop_cnt) 6135 ret = 6136 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6137 &sfx_attr, items, 6138 flow_split_info, 6139 fm, error); 6140 goto exit; 6141 } 6142 /* Setting the sfx group atrr. */ 6143 sfx_attr.group = sfx_attr.transfer ? 6144 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6145 MLX5_FLOW_TABLE_LEVEL_METER; 6146 flow_split_info->prefix_layers = 6147 flow_get_prefix_layer_flags(dev_flow); 6148 flow_split_info->prefix_mark |= wks->mark; 6149 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6150 } 6151 /* Add the prefix subflow. */ 6152 ret = flow_create_split_metadata(dev, flow, 6153 &sfx_attr, sfx_items ? 6154 sfx_items : items, 6155 sfx_actions ? sfx_actions : actions, 6156 flow_split_info, error); 6157 exit: 6158 if (sfx_actions) 6159 mlx5_free(sfx_actions); 6160 return ret; 6161 } 6162 6163 /** 6164 * The splitting for sample feature. 6165 * 6166 * Once Sample action is detected in the action list, the flow actions should 6167 * be split into prefix sub flow and suffix sub flow. 6168 * 6169 * The original items remain in the prefix sub flow, all actions preceding the 6170 * sample action and the sample action itself will be copied to the prefix 6171 * sub flow, the actions following the sample action will be copied to the 6172 * suffix sub flow, Queue action always be located in the suffix sub flow. 6173 * 6174 * In order to make the packet from prefix sub flow matches with suffix sub 6175 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6176 * flow uses tag item with the unique flow id. 6177 * 6178 * @param dev 6179 * Pointer to Ethernet device. 6180 * @param[in] flow 6181 * Parent flow structure pointer. 6182 * @param[in] attr 6183 * Flow rule attributes. 6184 * @param[in] items 6185 * Pattern specification (list terminated by the END pattern item). 6186 * @param[in] actions 6187 * Associated actions (list terminated by the END action). 6188 * @param[in] flow_split_info 6189 * Pointer to flow split info structure. 6190 * @param[out] error 6191 * Perform verbose error reporting if not NULL. 6192 * @return 6193 * 0 on success, negative value otherwise 6194 */ 6195 static int 6196 flow_create_split_sample(struct rte_eth_dev *dev, 6197 struct rte_flow *flow, 6198 const struct rte_flow_attr *attr, 6199 const struct rte_flow_item items[], 6200 const struct rte_flow_action actions[], 6201 struct mlx5_flow_split_info *flow_split_info, 6202 struct rte_flow_error *error) 6203 { 6204 struct mlx5_priv *priv = dev->data->dev_private; 6205 struct rte_flow_action *sfx_actions = NULL; 6206 struct rte_flow_action *pre_actions = NULL; 6207 struct rte_flow_item *sfx_items = NULL; 6208 struct mlx5_flow *dev_flow = NULL; 6209 struct rte_flow_attr sfx_attr = *attr; 6210 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6211 struct mlx5_flow_dv_sample_resource *sample_res; 6212 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6213 struct mlx5_flow_tbl_resource *sfx_tbl; 6214 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6215 #endif 6216 size_t act_size; 6217 size_t item_size; 6218 uint32_t fdb_tx = 0; 6219 int32_t tag_id = 0; 6220 int actions_n = 0; 6221 int sample_action_pos; 6222 int qrss_action_pos; 6223 int add_tag = 0; 6224 int modify_after_mirror = 0; 6225 uint16_t jump_table = 0; 6226 const uint32_t next_ft_step = 1; 6227 int ret = 0; 6228 6229 if (priv->sampler_en) 6230 actions_n = flow_check_match_action(actions, attr, 6231 RTE_FLOW_ACTION_TYPE_SAMPLE, 6232 &sample_action_pos, &qrss_action_pos, 6233 &modify_after_mirror); 6234 if (actions_n) { 6235 /* The prefix actions must includes sample, tag, end. */ 6236 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6237 + sizeof(struct mlx5_rte_flow_action_set_tag); 6238 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6239 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6240 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6241 item_size), 0, SOCKET_ID_ANY); 6242 if (!sfx_actions) 6243 return rte_flow_error_set(error, ENOMEM, 6244 RTE_FLOW_ERROR_TYPE_ACTION, 6245 NULL, "no memory to split " 6246 "sample flow"); 6247 /* The representor_id is UINT16_MAX for uplink. */ 6248 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6249 /* 6250 * When reg_c_preserve is set, metadata registers Cx preserve 6251 * their value even through packet duplication. 6252 */ 6253 add_tag = (!fdb_tx || 6254 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6255 if (add_tag) 6256 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6257 + act_size); 6258 if (modify_after_mirror) 6259 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6260 next_ft_step; 6261 pre_actions = sfx_actions + actions_n; 6262 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6263 actions, sfx_actions, 6264 pre_actions, actions_n, 6265 sample_action_pos, 6266 qrss_action_pos, jump_table, 6267 error); 6268 if (tag_id < 0 || (add_tag && !tag_id)) { 6269 ret = -rte_errno; 6270 goto exit; 6271 } 6272 if (modify_after_mirror) 6273 flow_split_info->skip_scale = 6274 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6275 /* Add the prefix subflow. */ 6276 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6277 items, pre_actions, 6278 flow_split_info, error); 6279 if (ret) { 6280 ret = -rte_errno; 6281 goto exit; 6282 } 6283 dev_flow->handle->split_flow_id = tag_id; 6284 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6285 if (!modify_after_mirror) { 6286 /* Set the sfx group attr. */ 6287 sample_res = (struct mlx5_flow_dv_sample_resource *) 6288 dev_flow->dv.sample_res; 6289 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6290 sample_res->normal_path_tbl; 6291 sfx_tbl_data = container_of(sfx_tbl, 6292 struct mlx5_flow_tbl_data_entry, 6293 tbl); 6294 sfx_attr.group = sfx_attr.transfer ? 6295 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6296 } else { 6297 MLX5_ASSERT(attr->transfer); 6298 sfx_attr.group = jump_table; 6299 } 6300 flow_split_info->prefix_layers = 6301 flow_get_prefix_layer_flags(dev_flow); 6302 MLX5_ASSERT(wks); 6303 flow_split_info->prefix_mark |= wks->mark; 6304 /* Suffix group level already be scaled with factor, set 6305 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6306 * again in translation. 6307 */ 6308 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6309 #endif 6310 } 6311 /* Add the suffix subflow. */ 6312 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6313 sfx_items ? sfx_items : items, 6314 sfx_actions ? sfx_actions : actions, 6315 flow_split_info, error); 6316 exit: 6317 if (sfx_actions) 6318 mlx5_free(sfx_actions); 6319 return ret; 6320 } 6321 6322 /** 6323 * Split the flow to subflow set. The splitters might be linked 6324 * in the chain, like this: 6325 * flow_create_split_outer() calls: 6326 * flow_create_split_meter() calls: 6327 * flow_create_split_metadata(meter_subflow_0) calls: 6328 * flow_create_split_inner(metadata_subflow_0) 6329 * flow_create_split_inner(metadata_subflow_1) 6330 * flow_create_split_inner(metadata_subflow_2) 6331 * flow_create_split_metadata(meter_subflow_1) calls: 6332 * flow_create_split_inner(metadata_subflow_0) 6333 * flow_create_split_inner(metadata_subflow_1) 6334 * flow_create_split_inner(metadata_subflow_2) 6335 * 6336 * This provide flexible way to add new levels of flow splitting. 6337 * The all of successfully created subflows are included to the 6338 * parent flow dev_flow list. 6339 * 6340 * @param dev 6341 * Pointer to Ethernet device. 6342 * @param[in] flow 6343 * Parent flow structure pointer. 6344 * @param[in] attr 6345 * Flow rule attributes. 6346 * @param[in] items 6347 * Pattern specification (list terminated by the END pattern item). 6348 * @param[in] actions 6349 * Associated actions (list terminated by the END action). 6350 * @param[in] flow_split_info 6351 * Pointer to flow split info structure. 6352 * @param[out] error 6353 * Perform verbose error reporting if not NULL. 6354 * @return 6355 * 0 on success, negative value otherwise 6356 */ 6357 static int 6358 flow_create_split_outer(struct rte_eth_dev *dev, 6359 struct rte_flow *flow, 6360 const struct rte_flow_attr *attr, 6361 const struct rte_flow_item items[], 6362 const struct rte_flow_action actions[], 6363 struct mlx5_flow_split_info *flow_split_info, 6364 struct rte_flow_error *error) 6365 { 6366 int ret; 6367 6368 ret = flow_create_split_sample(dev, flow, attr, items, 6369 actions, flow_split_info, error); 6370 MLX5_ASSERT(ret <= 0); 6371 return ret; 6372 } 6373 6374 static inline struct mlx5_flow_tunnel * 6375 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6376 { 6377 struct mlx5_flow_tunnel *tunnel; 6378 6379 #pragma GCC diagnostic push 6380 #pragma GCC diagnostic ignored "-Wcast-qual" 6381 tunnel = (typeof(tunnel))flow->tunnel; 6382 #pragma GCC diagnostic pop 6383 6384 return tunnel; 6385 } 6386 6387 /** 6388 * Adjust flow RSS workspace if needed. 6389 * 6390 * @param wks 6391 * Pointer to thread flow work space. 6392 * @param rss_desc 6393 * Pointer to RSS descriptor. 6394 * @param[in] nrssq_num 6395 * New RSS queue number. 6396 * 6397 * @return 6398 * 0 on success, -1 otherwise and rte_errno is set. 6399 */ 6400 static int 6401 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6402 struct mlx5_flow_rss_desc *rss_desc, 6403 uint32_t nrssq_num) 6404 { 6405 if (likely(nrssq_num <= wks->rssq_num)) 6406 return 0; 6407 rss_desc->queue = realloc(rss_desc->queue, 6408 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6409 if (!rss_desc->queue) { 6410 rte_errno = ENOMEM; 6411 return -1; 6412 } 6413 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6414 return 0; 6415 } 6416 6417 /** 6418 * Create a flow and add it to @p list. 6419 * 6420 * @param dev 6421 * Pointer to Ethernet device. 6422 * @param list 6423 * Pointer to a TAILQ flow list. If this parameter NULL, 6424 * no list insertion occurred, flow is just created, 6425 * this is caller's responsibility to track the 6426 * created flow. 6427 * @param[in] attr 6428 * Flow rule attributes. 6429 * @param[in] items 6430 * Pattern specification (list terminated by the END pattern item). 6431 * @param[in] actions 6432 * Associated actions (list terminated by the END action). 6433 * @param[in] external 6434 * This flow rule is created by request external to PMD. 6435 * @param[out] error 6436 * Perform verbose error reporting if not NULL. 6437 * 6438 * @return 6439 * A flow index on success, 0 otherwise and rte_errno is set. 6440 */ 6441 static uint32_t 6442 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6443 const struct rte_flow_attr *attr, 6444 const struct rte_flow_item items[], 6445 const struct rte_flow_action original_actions[], 6446 bool external, struct rte_flow_error *error) 6447 { 6448 struct mlx5_priv *priv = dev->data->dev_private; 6449 struct rte_flow *flow = NULL; 6450 struct mlx5_flow *dev_flow; 6451 const struct rte_flow_action_rss *rss = NULL; 6452 struct mlx5_translated_action_handle 6453 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6454 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6455 union { 6456 struct mlx5_flow_expand_rss buf; 6457 uint8_t buffer[4096]; 6458 } expand_buffer; 6459 union { 6460 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6461 uint8_t buffer[2048]; 6462 } actions_rx; 6463 union { 6464 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6465 uint8_t buffer[2048]; 6466 } actions_hairpin_tx; 6467 union { 6468 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6469 uint8_t buffer[2048]; 6470 } items_tx; 6471 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6472 struct mlx5_flow_rss_desc *rss_desc; 6473 const struct rte_flow_action *p_actions_rx; 6474 uint32_t i; 6475 uint32_t idx = 0; 6476 int hairpin_flow; 6477 struct rte_flow_attr attr_tx = { .priority = 0 }; 6478 const struct rte_flow_action *actions; 6479 struct rte_flow_action *translated_actions = NULL; 6480 struct mlx5_flow_tunnel *tunnel; 6481 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6482 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6483 struct mlx5_flow_split_info flow_split_info = { 6484 .external = !!external, 6485 .skip_scale = 0, 6486 .flow_idx = 0, 6487 .prefix_mark = 0, 6488 .prefix_layers = 0, 6489 .table_id = 0 6490 }; 6491 int ret; 6492 6493 MLX5_ASSERT(wks); 6494 rss_desc = &wks->rss_desc; 6495 ret = flow_action_handles_translate(dev, original_actions, 6496 indir_actions, 6497 &indir_actions_n, 6498 &translated_actions, error); 6499 if (ret < 0) { 6500 MLX5_ASSERT(translated_actions == NULL); 6501 return 0; 6502 } 6503 actions = translated_actions ? translated_actions : original_actions; 6504 p_actions_rx = actions; 6505 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6506 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6507 external, hairpin_flow, error); 6508 if (ret < 0) 6509 goto error_before_hairpin_split; 6510 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6511 if (!flow) { 6512 rte_errno = ENOMEM; 6513 goto error_before_hairpin_split; 6514 } 6515 if (hairpin_flow > 0) { 6516 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6517 rte_errno = EINVAL; 6518 goto error_before_hairpin_split; 6519 } 6520 flow_hairpin_split(dev, actions, actions_rx.actions, 6521 actions_hairpin_tx.actions, items_tx.items, 6522 idx); 6523 p_actions_rx = actions_rx.actions; 6524 } 6525 flow_split_info.flow_idx = idx; 6526 flow->drv_type = flow_get_drv_type(dev, attr); 6527 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6528 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6529 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6530 /* RSS Action only works on NIC RX domain */ 6531 if (attr->ingress && !attr->transfer) 6532 rss = flow_get_rss_action(dev, p_actions_rx); 6533 if (rss) { 6534 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6535 return 0; 6536 /* 6537 * The following information is required by 6538 * mlx5_flow_hashfields_adjust() in advance. 6539 */ 6540 rss_desc->level = rss->level; 6541 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6542 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6543 } 6544 flow->dev_handles = 0; 6545 if (rss && rss->types) { 6546 unsigned int graph_root; 6547 6548 graph_root = find_graph_root(rss->level); 6549 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6550 items, rss->types, 6551 mlx5_support_expansion, graph_root); 6552 MLX5_ASSERT(ret > 0 && 6553 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6554 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6555 for (i = 0; i < buf->entries; ++i) 6556 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6557 } 6558 } else { 6559 buf->entries = 1; 6560 buf->entry[0].pattern = (void *)(uintptr_t)items; 6561 } 6562 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6563 indir_actions_n); 6564 for (i = 0; i < buf->entries; ++i) { 6565 /* Initialize flow split data. */ 6566 flow_split_info.prefix_layers = 0; 6567 flow_split_info.prefix_mark = 0; 6568 flow_split_info.skip_scale = 0; 6569 /* 6570 * The splitter may create multiple dev_flows, 6571 * depending on configuration. In the simplest 6572 * case it just creates unmodified original flow. 6573 */ 6574 ret = flow_create_split_outer(dev, flow, attr, 6575 buf->entry[i].pattern, 6576 p_actions_rx, &flow_split_info, 6577 error); 6578 if (ret < 0) 6579 goto error; 6580 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6581 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6582 p_actions_rx, 6583 idx, 6584 wks->flows[0].tunnel, 6585 &default_miss_ctx, 6586 error); 6587 if (ret < 0) { 6588 mlx5_free(default_miss_ctx.queue); 6589 goto error; 6590 } 6591 } 6592 } 6593 /* Create the tx flow. */ 6594 if (hairpin_flow) { 6595 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6596 attr_tx.ingress = 0; 6597 attr_tx.egress = 1; 6598 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6599 actions_hairpin_tx.actions, 6600 idx, error); 6601 if (!dev_flow) 6602 goto error; 6603 dev_flow->flow = flow; 6604 dev_flow->external = 0; 6605 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6606 dev_flow->handle, next); 6607 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6608 items_tx.items, 6609 actions_hairpin_tx.actions, error); 6610 if (ret < 0) 6611 goto error; 6612 } 6613 /* 6614 * Update the metadata register copy table. If extensive 6615 * metadata feature is enabled and registers are supported 6616 * we might create the extra rte_flow for each unique 6617 * MARK/FLAG action ID. 6618 * 6619 * The table is updated for ingress Flows only, because 6620 * the egress Flows belong to the different device and 6621 * copy table should be updated in peer NIC Rx domain. 6622 */ 6623 if (attr->ingress && 6624 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6625 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6626 if (ret) 6627 goto error; 6628 } 6629 /* 6630 * If the flow is external (from application) OR device is started, 6631 * OR mreg discover, then apply immediately. 6632 */ 6633 if (external || dev->data->dev_started || 6634 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6635 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6636 ret = flow_drv_apply(dev, flow, error); 6637 if (ret < 0) 6638 goto error; 6639 } 6640 flow->type = type; 6641 flow_rxq_flags_set(dev, flow); 6642 rte_free(translated_actions); 6643 tunnel = flow_tunnel_from_rule(wks->flows); 6644 if (tunnel) { 6645 flow->tunnel = 1; 6646 flow->tunnel_id = tunnel->tunnel_id; 6647 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6648 mlx5_free(default_miss_ctx.queue); 6649 } 6650 mlx5_flow_pop_thread_workspace(); 6651 return idx; 6652 error: 6653 MLX5_ASSERT(flow); 6654 ret = rte_errno; /* Save rte_errno before cleanup. */ 6655 flow_mreg_del_copy_action(dev, flow); 6656 flow_drv_destroy(dev, flow); 6657 if (rss_desc->shared_rss) 6658 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6659 mlx5_ipool_get 6660 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6661 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6662 mlx5_ipool_free(priv->flows[type], idx); 6663 rte_errno = ret; /* Restore rte_errno. */ 6664 ret = rte_errno; 6665 rte_errno = ret; 6666 mlx5_flow_pop_thread_workspace(); 6667 error_before_hairpin_split: 6668 rte_free(translated_actions); 6669 return 0; 6670 } 6671 6672 /** 6673 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6674 * incoming packets to table 1. 6675 * 6676 * Other flow rules, requested for group n, will be created in 6677 * e-switch table n+1. 6678 * Jump action to e-switch group n will be created to group n+1. 6679 * 6680 * Used when working in switchdev mode, to utilise advantages of table 1 6681 * and above. 6682 * 6683 * @param dev 6684 * Pointer to Ethernet device. 6685 * 6686 * @return 6687 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6688 */ 6689 struct rte_flow * 6690 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6691 { 6692 const struct rte_flow_attr attr = { 6693 .group = 0, 6694 .priority = 0, 6695 .ingress = 1, 6696 .egress = 0, 6697 .transfer = 1, 6698 }; 6699 const struct rte_flow_item pattern = { 6700 .type = RTE_FLOW_ITEM_TYPE_END, 6701 }; 6702 struct rte_flow_action_jump jump = { 6703 .group = 1, 6704 }; 6705 const struct rte_flow_action actions[] = { 6706 { 6707 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6708 .conf = &jump, 6709 }, 6710 { 6711 .type = RTE_FLOW_ACTION_TYPE_END, 6712 }, 6713 }; 6714 struct rte_flow_error error; 6715 6716 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6717 &attr, &pattern, 6718 actions, false, &error); 6719 } 6720 6721 /** 6722 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6723 * and sq number, directs all packets to peer vport. 6724 * 6725 * @param dev 6726 * Pointer to Ethernet device. 6727 * @param txq 6728 * Txq index. 6729 * 6730 * @return 6731 * Flow ID on success, 0 otherwise and rte_errno is set. 6732 */ 6733 uint32_t 6734 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6735 { 6736 struct rte_flow_attr attr = { 6737 .group = 0, 6738 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6739 .ingress = 1, 6740 .egress = 0, 6741 .transfer = 1, 6742 }; 6743 struct rte_flow_item_port_id port_spec = { 6744 .id = MLX5_PORT_ESW_MGR, 6745 }; 6746 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6747 .queue = txq, 6748 }; 6749 struct rte_flow_item pattern[] = { 6750 { 6751 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6752 .spec = &port_spec, 6753 }, 6754 { 6755 .type = (enum rte_flow_item_type) 6756 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6757 .spec = &txq_spec, 6758 }, 6759 { 6760 .type = RTE_FLOW_ITEM_TYPE_END, 6761 }, 6762 }; 6763 struct rte_flow_action_jump jump = { 6764 .group = 1, 6765 }; 6766 struct rte_flow_action_port_id port = { 6767 .id = dev->data->port_id, 6768 }; 6769 struct rte_flow_action actions[] = { 6770 { 6771 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6772 .conf = &jump, 6773 }, 6774 { 6775 .type = RTE_FLOW_ACTION_TYPE_END, 6776 }, 6777 }; 6778 struct rte_flow_error error; 6779 6780 /* 6781 * Creates group 0, highest priority jump flow. 6782 * Matches txq to bypass kernel packets. 6783 */ 6784 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 6785 false, &error) == 0) 6786 return 0; 6787 /* Create group 1, lowest priority redirect flow for txq. */ 6788 attr.group = 1; 6789 actions[0].conf = &port; 6790 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 6791 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 6792 actions, false, &error); 6793 } 6794 6795 /** 6796 * Validate a flow supported by the NIC. 6797 * 6798 * @see rte_flow_validate() 6799 * @see rte_flow_ops 6800 */ 6801 int 6802 mlx5_flow_validate(struct rte_eth_dev *dev, 6803 const struct rte_flow_attr *attr, 6804 const struct rte_flow_item items[], 6805 const struct rte_flow_action original_actions[], 6806 struct rte_flow_error *error) 6807 { 6808 int hairpin_flow; 6809 struct mlx5_translated_action_handle 6810 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6811 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6812 const struct rte_flow_action *actions; 6813 struct rte_flow_action *translated_actions = NULL; 6814 int ret = flow_action_handles_translate(dev, original_actions, 6815 indir_actions, 6816 &indir_actions_n, 6817 &translated_actions, error); 6818 6819 if (ret) 6820 return ret; 6821 actions = translated_actions ? translated_actions : original_actions; 6822 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6823 ret = flow_drv_validate(dev, attr, items, actions, 6824 true, hairpin_flow, error); 6825 rte_free(translated_actions); 6826 return ret; 6827 } 6828 6829 /** 6830 * Create a flow. 6831 * 6832 * @see rte_flow_create() 6833 * @see rte_flow_ops 6834 */ 6835 struct rte_flow * 6836 mlx5_flow_create(struct rte_eth_dev *dev, 6837 const struct rte_flow_attr *attr, 6838 const struct rte_flow_item items[], 6839 const struct rte_flow_action actions[], 6840 struct rte_flow_error *error) 6841 { 6842 /* 6843 * If the device is not started yet, it is not allowed to created a 6844 * flow from application. PMD default flows and traffic control flows 6845 * are not affected. 6846 */ 6847 if (unlikely(!dev->data->dev_started)) { 6848 DRV_LOG(DEBUG, "port %u is not started when " 6849 "inserting a flow", dev->data->port_id); 6850 rte_flow_error_set(error, ENODEV, 6851 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6852 NULL, 6853 "port not started"); 6854 return NULL; 6855 } 6856 6857 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6858 attr, items, actions, 6859 true, error); 6860 } 6861 6862 /** 6863 * Destroy a flow in a list. 6864 * 6865 * @param dev 6866 * Pointer to Ethernet device. 6867 * @param[in] flow_idx 6868 * Index of flow to destroy. 6869 */ 6870 static void 6871 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6872 uint32_t flow_idx) 6873 { 6874 struct mlx5_priv *priv = dev->data->dev_private; 6875 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6876 6877 if (!flow) 6878 return; 6879 MLX5_ASSERT(flow->type == type); 6880 /* 6881 * Update RX queue flags only if port is started, otherwise it is 6882 * already clean. 6883 */ 6884 if (dev->data->dev_started) 6885 flow_rxq_flags_trim(dev, flow); 6886 flow_drv_destroy(dev, flow); 6887 if (flow->tunnel) { 6888 struct mlx5_flow_tunnel *tunnel; 6889 6890 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6891 RTE_VERIFY(tunnel); 6892 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6893 mlx5_flow_tunnel_free(dev, tunnel); 6894 } 6895 flow_mreg_del_copy_action(dev, flow); 6896 mlx5_ipool_free(priv->flows[type], flow_idx); 6897 } 6898 6899 /** 6900 * Destroy all flows. 6901 * 6902 * @param dev 6903 * Pointer to Ethernet device. 6904 * @param type 6905 * Flow type to be flushed. 6906 * @param active 6907 * If flushing is called actively. 6908 */ 6909 void 6910 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6911 bool active) 6912 { 6913 struct mlx5_priv *priv = dev->data->dev_private; 6914 uint32_t num_flushed = 0, fidx = 1; 6915 struct rte_flow *flow; 6916 6917 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6918 flow_list_destroy(dev, type, fidx); 6919 num_flushed++; 6920 } 6921 if (active) { 6922 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6923 dev->data->port_id, num_flushed); 6924 } 6925 } 6926 6927 /** 6928 * Stop all default actions for flows. 6929 * 6930 * @param dev 6931 * Pointer to Ethernet device. 6932 */ 6933 void 6934 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6935 { 6936 flow_mreg_del_default_copy_action(dev); 6937 flow_rxq_flags_clear(dev); 6938 } 6939 6940 /** 6941 * Start all default actions for flows. 6942 * 6943 * @param dev 6944 * Pointer to Ethernet device. 6945 * @return 6946 * 0 on success, a negative errno value otherwise and rte_errno is set. 6947 */ 6948 int 6949 mlx5_flow_start_default(struct rte_eth_dev *dev) 6950 { 6951 struct rte_flow_error error; 6952 6953 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6954 return flow_mreg_add_default_copy_action(dev, &error); 6955 } 6956 6957 /** 6958 * Release key of thread specific flow workspace data. 6959 */ 6960 void 6961 flow_release_workspace(void *data) 6962 { 6963 struct mlx5_flow_workspace *wks = data; 6964 struct mlx5_flow_workspace *next; 6965 6966 while (wks) { 6967 next = wks->next; 6968 free(wks->rss_desc.queue); 6969 free(wks); 6970 wks = next; 6971 } 6972 } 6973 6974 /** 6975 * Get thread specific current flow workspace. 6976 * 6977 * @return pointer to thread specific flow workspace data, NULL on error. 6978 */ 6979 struct mlx5_flow_workspace* 6980 mlx5_flow_get_thread_workspace(void) 6981 { 6982 struct mlx5_flow_workspace *data; 6983 6984 data = mlx5_flow_os_get_specific_workspace(); 6985 MLX5_ASSERT(data && data->inuse); 6986 if (!data || !data->inuse) 6987 DRV_LOG(ERR, "flow workspace not initialized."); 6988 return data; 6989 } 6990 6991 /** 6992 * Allocate and init new flow workspace. 6993 * 6994 * @return pointer to flow workspace data, NULL on error. 6995 */ 6996 static struct mlx5_flow_workspace* 6997 flow_alloc_thread_workspace(void) 6998 { 6999 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7000 7001 if (!data) { 7002 DRV_LOG(ERR, "Failed to allocate flow workspace " 7003 "memory."); 7004 return NULL; 7005 } 7006 data->rss_desc.queue = calloc(1, 7007 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7008 if (!data->rss_desc.queue) 7009 goto err; 7010 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7011 return data; 7012 err: 7013 free(data->rss_desc.queue); 7014 free(data); 7015 return NULL; 7016 } 7017 7018 /** 7019 * Get new thread specific flow workspace. 7020 * 7021 * If current workspace inuse, create new one and set as current. 7022 * 7023 * @return pointer to thread specific flow workspace data, NULL on error. 7024 */ 7025 static struct mlx5_flow_workspace* 7026 mlx5_flow_push_thread_workspace(void) 7027 { 7028 struct mlx5_flow_workspace *curr; 7029 struct mlx5_flow_workspace *data; 7030 7031 curr = mlx5_flow_os_get_specific_workspace(); 7032 if (!curr) { 7033 data = flow_alloc_thread_workspace(); 7034 if (!data) 7035 return NULL; 7036 } else if (!curr->inuse) { 7037 data = curr; 7038 } else if (curr->next) { 7039 data = curr->next; 7040 } else { 7041 data = flow_alloc_thread_workspace(); 7042 if (!data) 7043 return NULL; 7044 curr->next = data; 7045 data->prev = curr; 7046 } 7047 data->inuse = 1; 7048 data->flow_idx = 0; 7049 /* Set as current workspace */ 7050 if (mlx5_flow_os_set_specific_workspace(data)) 7051 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7052 return data; 7053 } 7054 7055 /** 7056 * Close current thread specific flow workspace. 7057 * 7058 * If previous workspace available, set it as current. 7059 * 7060 * @return pointer to thread specific flow workspace data, NULL on error. 7061 */ 7062 static void 7063 mlx5_flow_pop_thread_workspace(void) 7064 { 7065 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7066 7067 if (!data) 7068 return; 7069 if (!data->inuse) { 7070 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7071 return; 7072 } 7073 data->inuse = 0; 7074 if (!data->prev) 7075 return; 7076 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7077 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7078 } 7079 7080 /** 7081 * Verify the flow list is empty 7082 * 7083 * @param dev 7084 * Pointer to Ethernet device. 7085 * 7086 * @return the number of flows not released. 7087 */ 7088 int 7089 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7090 { 7091 struct mlx5_priv *priv = dev->data->dev_private; 7092 struct rte_flow *flow; 7093 uint32_t idx = 0; 7094 int ret = 0, i; 7095 7096 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7097 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7098 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7099 dev->data->port_id, (void *)flow); 7100 ret++; 7101 } 7102 } 7103 return ret; 7104 } 7105 7106 /** 7107 * Enable default hairpin egress flow. 7108 * 7109 * @param dev 7110 * Pointer to Ethernet device. 7111 * @param queue 7112 * The queue index. 7113 * 7114 * @return 7115 * 0 on success, a negative errno value otherwise and rte_errno is set. 7116 */ 7117 int 7118 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7119 uint32_t queue) 7120 { 7121 const struct rte_flow_attr attr = { 7122 .egress = 1, 7123 .priority = 0, 7124 }; 7125 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7126 .queue = queue, 7127 }; 7128 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7129 .queue = UINT32_MAX, 7130 }; 7131 struct rte_flow_item items[] = { 7132 { 7133 .type = (enum rte_flow_item_type) 7134 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7135 .spec = &queue_spec, 7136 .last = NULL, 7137 .mask = &queue_mask, 7138 }, 7139 { 7140 .type = RTE_FLOW_ITEM_TYPE_END, 7141 }, 7142 }; 7143 struct rte_flow_action_jump jump = { 7144 .group = MLX5_HAIRPIN_TX_TABLE, 7145 }; 7146 struct rte_flow_action actions[2]; 7147 uint32_t flow_idx; 7148 struct rte_flow_error error; 7149 7150 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7151 actions[0].conf = &jump; 7152 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7153 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7154 &attr, items, actions, false, &error); 7155 if (!flow_idx) { 7156 DRV_LOG(DEBUG, 7157 "Failed to create ctrl flow: rte_errno(%d)," 7158 " type(%d), message(%s)", 7159 rte_errno, error.type, 7160 error.message ? error.message : " (no stated reason)"); 7161 return -rte_errno; 7162 } 7163 return 0; 7164 } 7165 7166 /** 7167 * Enable a control flow configured from the control plane. 7168 * 7169 * @param dev 7170 * Pointer to Ethernet device. 7171 * @param eth_spec 7172 * An Ethernet flow spec to apply. 7173 * @param eth_mask 7174 * An Ethernet flow mask to apply. 7175 * @param vlan_spec 7176 * A VLAN flow spec to apply. 7177 * @param vlan_mask 7178 * A VLAN flow mask to apply. 7179 * 7180 * @return 7181 * 0 on success, a negative errno value otherwise and rte_errno is set. 7182 */ 7183 int 7184 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7185 struct rte_flow_item_eth *eth_spec, 7186 struct rte_flow_item_eth *eth_mask, 7187 struct rte_flow_item_vlan *vlan_spec, 7188 struct rte_flow_item_vlan *vlan_mask) 7189 { 7190 struct mlx5_priv *priv = dev->data->dev_private; 7191 const struct rte_flow_attr attr = { 7192 .ingress = 1, 7193 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7194 }; 7195 struct rte_flow_item items[] = { 7196 { 7197 .type = RTE_FLOW_ITEM_TYPE_ETH, 7198 .spec = eth_spec, 7199 .last = NULL, 7200 .mask = eth_mask, 7201 }, 7202 { 7203 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7204 RTE_FLOW_ITEM_TYPE_END, 7205 .spec = vlan_spec, 7206 .last = NULL, 7207 .mask = vlan_mask, 7208 }, 7209 { 7210 .type = RTE_FLOW_ITEM_TYPE_END, 7211 }, 7212 }; 7213 uint16_t queue[priv->reta_idx_n]; 7214 struct rte_flow_action_rss action_rss = { 7215 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7216 .level = 0, 7217 .types = priv->rss_conf.rss_hf, 7218 .key_len = priv->rss_conf.rss_key_len, 7219 .queue_num = priv->reta_idx_n, 7220 .key = priv->rss_conf.rss_key, 7221 .queue = queue, 7222 }; 7223 struct rte_flow_action actions[] = { 7224 { 7225 .type = RTE_FLOW_ACTION_TYPE_RSS, 7226 .conf = &action_rss, 7227 }, 7228 { 7229 .type = RTE_FLOW_ACTION_TYPE_END, 7230 }, 7231 }; 7232 uint32_t flow_idx; 7233 struct rte_flow_error error; 7234 unsigned int i; 7235 7236 if (!priv->reta_idx_n || !priv->rxqs_n) { 7237 return 0; 7238 } 7239 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7240 action_rss.types = 0; 7241 for (i = 0; i != priv->reta_idx_n; ++i) 7242 queue[i] = (*priv->reta_idx)[i]; 7243 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7244 &attr, items, actions, false, &error); 7245 if (!flow_idx) 7246 return -rte_errno; 7247 return 0; 7248 } 7249 7250 /** 7251 * Enable a flow control configured from the control plane. 7252 * 7253 * @param dev 7254 * Pointer to Ethernet device. 7255 * @param eth_spec 7256 * An Ethernet flow spec to apply. 7257 * @param eth_mask 7258 * An Ethernet flow mask to apply. 7259 * 7260 * @return 7261 * 0 on success, a negative errno value otherwise and rte_errno is set. 7262 */ 7263 int 7264 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7265 struct rte_flow_item_eth *eth_spec, 7266 struct rte_flow_item_eth *eth_mask) 7267 { 7268 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7269 } 7270 7271 /** 7272 * Create default miss flow rule matching lacp traffic 7273 * 7274 * @param dev 7275 * Pointer to Ethernet device. 7276 * @param eth_spec 7277 * An Ethernet flow spec to apply. 7278 * 7279 * @return 7280 * 0 on success, a negative errno value otherwise and rte_errno is set. 7281 */ 7282 int 7283 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7284 { 7285 /* 7286 * The LACP matching is done by only using ether type since using 7287 * a multicast dst mac causes kernel to give low priority to this flow. 7288 */ 7289 static const struct rte_flow_item_eth lacp_spec = { 7290 .type = RTE_BE16(0x8809), 7291 }; 7292 static const struct rte_flow_item_eth lacp_mask = { 7293 .type = 0xffff, 7294 }; 7295 const struct rte_flow_attr attr = { 7296 .ingress = 1, 7297 }; 7298 struct rte_flow_item items[] = { 7299 { 7300 .type = RTE_FLOW_ITEM_TYPE_ETH, 7301 .spec = &lacp_spec, 7302 .mask = &lacp_mask, 7303 }, 7304 { 7305 .type = RTE_FLOW_ITEM_TYPE_END, 7306 }, 7307 }; 7308 struct rte_flow_action actions[] = { 7309 { 7310 .type = (enum rte_flow_action_type) 7311 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7312 }, 7313 { 7314 .type = RTE_FLOW_ACTION_TYPE_END, 7315 }, 7316 }; 7317 struct rte_flow_error error; 7318 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7319 &attr, items, actions, 7320 false, &error); 7321 7322 if (!flow_idx) 7323 return -rte_errno; 7324 return 0; 7325 } 7326 7327 /** 7328 * Destroy a flow. 7329 * 7330 * @see rte_flow_destroy() 7331 * @see rte_flow_ops 7332 */ 7333 int 7334 mlx5_flow_destroy(struct rte_eth_dev *dev, 7335 struct rte_flow *flow, 7336 struct rte_flow_error *error __rte_unused) 7337 { 7338 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7339 (uintptr_t)(void *)flow); 7340 return 0; 7341 } 7342 7343 /** 7344 * Destroy all flows. 7345 * 7346 * @see rte_flow_flush() 7347 * @see rte_flow_ops 7348 */ 7349 int 7350 mlx5_flow_flush(struct rte_eth_dev *dev, 7351 struct rte_flow_error *error __rte_unused) 7352 { 7353 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7354 return 0; 7355 } 7356 7357 /** 7358 * Isolated mode. 7359 * 7360 * @see rte_flow_isolate() 7361 * @see rte_flow_ops 7362 */ 7363 int 7364 mlx5_flow_isolate(struct rte_eth_dev *dev, 7365 int enable, 7366 struct rte_flow_error *error) 7367 { 7368 struct mlx5_priv *priv = dev->data->dev_private; 7369 7370 if (dev->data->dev_started) { 7371 rte_flow_error_set(error, EBUSY, 7372 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7373 NULL, 7374 "port must be stopped first"); 7375 return -rte_errno; 7376 } 7377 priv->isolated = !!enable; 7378 if (enable) 7379 dev->dev_ops = &mlx5_dev_ops_isolate; 7380 else 7381 dev->dev_ops = &mlx5_dev_ops; 7382 7383 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7384 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7385 7386 return 0; 7387 } 7388 7389 /** 7390 * Query a flow. 7391 * 7392 * @see rte_flow_query() 7393 * @see rte_flow_ops 7394 */ 7395 static int 7396 flow_drv_query(struct rte_eth_dev *dev, 7397 uint32_t flow_idx, 7398 const struct rte_flow_action *actions, 7399 void *data, 7400 struct rte_flow_error *error) 7401 { 7402 struct mlx5_priv *priv = dev->data->dev_private; 7403 const struct mlx5_flow_driver_ops *fops; 7404 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7405 flow_idx); 7406 enum mlx5_flow_drv_type ftype; 7407 7408 if (!flow) { 7409 return rte_flow_error_set(error, ENOENT, 7410 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7411 NULL, 7412 "invalid flow handle"); 7413 } 7414 ftype = flow->drv_type; 7415 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7416 fops = flow_get_drv_ops(ftype); 7417 7418 return fops->query(dev, flow, actions, data, error); 7419 } 7420 7421 /** 7422 * Query a flow. 7423 * 7424 * @see rte_flow_query() 7425 * @see rte_flow_ops 7426 */ 7427 int 7428 mlx5_flow_query(struct rte_eth_dev *dev, 7429 struct rte_flow *flow, 7430 const struct rte_flow_action *actions, 7431 void *data, 7432 struct rte_flow_error *error) 7433 { 7434 int ret; 7435 7436 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7437 error); 7438 if (ret < 0) 7439 return ret; 7440 return 0; 7441 } 7442 7443 /** 7444 * Get rte_flow callbacks. 7445 * 7446 * @param dev 7447 * Pointer to Ethernet device structure. 7448 * @param ops 7449 * Pointer to operation-specific structure. 7450 * 7451 * @return 0 7452 */ 7453 int 7454 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7455 const struct rte_flow_ops **ops) 7456 { 7457 *ops = &mlx5_flow_ops; 7458 return 0; 7459 } 7460 7461 /** 7462 * Validate meter policy actions. 7463 * Dispatcher for action type specific validation. 7464 * 7465 * @param[in] dev 7466 * Pointer to the Ethernet device structure. 7467 * @param[in] action 7468 * The meter policy action object to validate. 7469 * @param[in] attr 7470 * Attributes of flow to determine steering domain. 7471 * @param[out] is_rss 7472 * Is RSS or not. 7473 * @param[out] domain_bitmap 7474 * Domain bitmap. 7475 * @param[out] is_def_policy 7476 * Is default policy or not. 7477 * @param[out] error 7478 * Perform verbose error reporting if not NULL. Initialized in case of 7479 * error only. 7480 * 7481 * @return 7482 * 0 on success, otherwise negative errno value. 7483 */ 7484 int 7485 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7486 const struct rte_flow_action *actions[RTE_COLORS], 7487 struct rte_flow_attr *attr, 7488 bool *is_rss, 7489 uint8_t *domain_bitmap, 7490 uint8_t *policy_mode, 7491 struct rte_mtr_error *error) 7492 { 7493 const struct mlx5_flow_driver_ops *fops; 7494 7495 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7496 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7497 domain_bitmap, policy_mode, error); 7498 } 7499 7500 /** 7501 * Destroy the meter table set. 7502 * 7503 * @param[in] dev 7504 * Pointer to Ethernet device. 7505 * @param[in] mtr_policy 7506 * Meter policy struct. 7507 */ 7508 void 7509 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7510 struct mlx5_flow_meter_policy *mtr_policy) 7511 { 7512 const struct mlx5_flow_driver_ops *fops; 7513 7514 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7515 fops->destroy_mtr_acts(dev, mtr_policy); 7516 } 7517 7518 /** 7519 * Create policy action, lock free, 7520 * (mutex should be acquired by caller). 7521 * Dispatcher for action type specific call. 7522 * 7523 * @param[in] dev 7524 * Pointer to the Ethernet device structure. 7525 * @param[in] mtr_policy 7526 * Meter policy struct. 7527 * @param[in] action 7528 * Action specification used to create meter actions. 7529 * @param[out] error 7530 * Perform verbose error reporting if not NULL. Initialized in case of 7531 * error only. 7532 * 7533 * @return 7534 * 0 on success, otherwise negative errno value. 7535 */ 7536 int 7537 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7538 struct mlx5_flow_meter_policy *mtr_policy, 7539 const struct rte_flow_action *actions[RTE_COLORS], 7540 struct rte_mtr_error *error) 7541 { 7542 const struct mlx5_flow_driver_ops *fops; 7543 7544 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7545 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7546 } 7547 7548 /** 7549 * Create policy rules, lock free, 7550 * (mutex should be acquired by caller). 7551 * Dispatcher for action type specific call. 7552 * 7553 * @param[in] dev 7554 * Pointer to the Ethernet device structure. 7555 * @param[in] mtr_policy 7556 * Meter policy struct. 7557 * 7558 * @return 7559 * 0 on success, -1 otherwise. 7560 */ 7561 int 7562 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7563 struct mlx5_flow_meter_policy *mtr_policy) 7564 { 7565 const struct mlx5_flow_driver_ops *fops; 7566 7567 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7568 return fops->create_policy_rules(dev, mtr_policy); 7569 } 7570 7571 /** 7572 * Destroy policy rules, lock free, 7573 * (mutex should be acquired by caller). 7574 * Dispatcher for action type specific call. 7575 * 7576 * @param[in] dev 7577 * Pointer to the Ethernet device structure. 7578 * @param[in] mtr_policy 7579 * Meter policy struct. 7580 */ 7581 void 7582 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7583 struct mlx5_flow_meter_policy *mtr_policy) 7584 { 7585 const struct mlx5_flow_driver_ops *fops; 7586 7587 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7588 fops->destroy_policy_rules(dev, mtr_policy); 7589 } 7590 7591 /** 7592 * Destroy the default policy table set. 7593 * 7594 * @param[in] dev 7595 * Pointer to Ethernet device. 7596 */ 7597 void 7598 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7599 { 7600 const struct mlx5_flow_driver_ops *fops; 7601 7602 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7603 fops->destroy_def_policy(dev); 7604 } 7605 7606 /** 7607 * Destroy the default policy table set. 7608 * 7609 * @param[in] dev 7610 * Pointer to Ethernet device. 7611 * 7612 * @return 7613 * 0 on success, -1 otherwise. 7614 */ 7615 int 7616 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7617 { 7618 const struct mlx5_flow_driver_ops *fops; 7619 7620 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7621 return fops->create_def_policy(dev); 7622 } 7623 7624 /** 7625 * Create the needed meter and suffix tables. 7626 * 7627 * @param[in] dev 7628 * Pointer to Ethernet device. 7629 * 7630 * @return 7631 * 0 on success, -1 otherwise. 7632 */ 7633 int 7634 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7635 struct mlx5_flow_meter_info *fm, 7636 uint32_t mtr_idx, 7637 uint8_t domain_bitmap) 7638 { 7639 const struct mlx5_flow_driver_ops *fops; 7640 7641 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7642 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7643 } 7644 7645 /** 7646 * Destroy the meter table set. 7647 * 7648 * @param[in] dev 7649 * Pointer to Ethernet device. 7650 * @param[in] tbl 7651 * Pointer to the meter table set. 7652 */ 7653 void 7654 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7655 struct mlx5_flow_meter_info *fm) 7656 { 7657 const struct mlx5_flow_driver_ops *fops; 7658 7659 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7660 fops->destroy_mtr_tbls(dev, fm); 7661 } 7662 7663 /** 7664 * Destroy the global meter drop table. 7665 * 7666 * @param[in] dev 7667 * Pointer to Ethernet device. 7668 */ 7669 void 7670 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7671 { 7672 const struct mlx5_flow_driver_ops *fops; 7673 7674 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7675 fops->destroy_mtr_drop_tbls(dev); 7676 } 7677 7678 /** 7679 * Destroy the sub policy table with RX queue. 7680 * 7681 * @param[in] dev 7682 * Pointer to Ethernet device. 7683 * @param[in] mtr_policy 7684 * Pointer to meter policy table. 7685 */ 7686 void 7687 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7688 struct mlx5_flow_meter_policy *mtr_policy) 7689 { 7690 const struct mlx5_flow_driver_ops *fops; 7691 7692 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7693 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7694 } 7695 7696 /** 7697 * Allocate the needed aso flow meter id. 7698 * 7699 * @param[in] dev 7700 * Pointer to Ethernet device. 7701 * 7702 * @return 7703 * Index to aso flow meter on success, NULL otherwise. 7704 */ 7705 uint32_t 7706 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7707 { 7708 const struct mlx5_flow_driver_ops *fops; 7709 7710 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7711 return fops->create_meter(dev); 7712 } 7713 7714 /** 7715 * Free the aso flow meter id. 7716 * 7717 * @param[in] dev 7718 * Pointer to Ethernet device. 7719 * @param[in] mtr_idx 7720 * Index to aso flow meter to be free. 7721 * 7722 * @return 7723 * 0 on success. 7724 */ 7725 void 7726 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7727 { 7728 const struct mlx5_flow_driver_ops *fops; 7729 7730 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7731 fops->free_meter(dev, mtr_idx); 7732 } 7733 7734 /** 7735 * Allocate a counter. 7736 * 7737 * @param[in] dev 7738 * Pointer to Ethernet device structure. 7739 * 7740 * @return 7741 * Index to allocated counter on success, 0 otherwise. 7742 */ 7743 uint32_t 7744 mlx5_counter_alloc(struct rte_eth_dev *dev) 7745 { 7746 const struct mlx5_flow_driver_ops *fops; 7747 struct rte_flow_attr attr = { .transfer = 0 }; 7748 7749 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7750 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7751 return fops->counter_alloc(dev); 7752 } 7753 DRV_LOG(ERR, 7754 "port %u counter allocate is not supported.", 7755 dev->data->port_id); 7756 return 0; 7757 } 7758 7759 /** 7760 * Free a counter. 7761 * 7762 * @param[in] dev 7763 * Pointer to Ethernet device structure. 7764 * @param[in] cnt 7765 * Index to counter to be free. 7766 */ 7767 void 7768 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7769 { 7770 const struct mlx5_flow_driver_ops *fops; 7771 struct rte_flow_attr attr = { .transfer = 0 }; 7772 7773 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7774 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7775 fops->counter_free(dev, cnt); 7776 return; 7777 } 7778 DRV_LOG(ERR, 7779 "port %u counter free is not supported.", 7780 dev->data->port_id); 7781 } 7782 7783 /** 7784 * Query counter statistics. 7785 * 7786 * @param[in] dev 7787 * Pointer to Ethernet device structure. 7788 * @param[in] cnt 7789 * Index to counter to query. 7790 * @param[in] clear 7791 * Set to clear counter statistics. 7792 * @param[out] pkts 7793 * The counter hits packets number to save. 7794 * @param[out] bytes 7795 * The counter hits bytes number to save. 7796 * 7797 * @return 7798 * 0 on success, a negative errno value otherwise. 7799 */ 7800 int 7801 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7802 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 7803 { 7804 const struct mlx5_flow_driver_ops *fops; 7805 struct rte_flow_attr attr = { .transfer = 0 }; 7806 7807 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7808 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7809 return fops->counter_query(dev, cnt, clear, pkts, 7810 bytes, action); 7811 } 7812 DRV_LOG(ERR, 7813 "port %u counter query is not supported.", 7814 dev->data->port_id); 7815 return -ENOTSUP; 7816 } 7817 7818 /** 7819 * Allocate a new memory for the counter values wrapped by all the needed 7820 * management. 7821 * 7822 * @param[in] sh 7823 * Pointer to mlx5_dev_ctx_shared object. 7824 * 7825 * @return 7826 * 0 on success, a negative errno value otherwise. 7827 */ 7828 static int 7829 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7830 { 7831 struct mlx5_counter_stats_mem_mng *mem_mng; 7832 volatile struct flow_counter_stats *raw_data; 7833 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7834 int size = (sizeof(struct flow_counter_stats) * 7835 MLX5_COUNTERS_PER_POOL + 7836 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7837 sizeof(struct mlx5_counter_stats_mem_mng); 7838 size_t pgsize = rte_mem_page_size(); 7839 uint8_t *mem; 7840 int ret; 7841 int i; 7842 7843 if (pgsize == (size_t)-1) { 7844 DRV_LOG(ERR, "Failed to get mem page size"); 7845 rte_errno = ENOMEM; 7846 return -ENOMEM; 7847 } 7848 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7849 if (!mem) { 7850 rte_errno = ENOMEM; 7851 return -ENOMEM; 7852 } 7853 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7854 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7855 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 7856 sh->cdev->pdn, mem, size, 7857 &mem_mng->wm); 7858 if (ret) { 7859 rte_errno = errno; 7860 mlx5_free(mem); 7861 return -rte_errno; 7862 } 7863 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7864 raw_data = (volatile struct flow_counter_stats *)mem; 7865 for (i = 0; i < raws_n; ++i) { 7866 mem_mng->raws[i].mem_mng = mem_mng; 7867 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7868 } 7869 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7870 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7871 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7872 next); 7873 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7874 sh->cmng.mem_mng = mem_mng; 7875 return 0; 7876 } 7877 7878 /** 7879 * Set the statistic memory to the new counter pool. 7880 * 7881 * @param[in] sh 7882 * Pointer to mlx5_dev_ctx_shared object. 7883 * @param[in] pool 7884 * Pointer to the pool to set the statistic memory. 7885 * 7886 * @return 7887 * 0 on success, a negative errno value otherwise. 7888 */ 7889 static int 7890 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7891 struct mlx5_flow_counter_pool *pool) 7892 { 7893 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7894 /* Resize statistic memory once used out. */ 7895 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7896 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7897 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7898 return -1; 7899 } 7900 rte_spinlock_lock(&pool->sl); 7901 pool->raw = cmng->mem_mng->raws + pool->index % 7902 MLX5_CNT_CONTAINER_RESIZE; 7903 rte_spinlock_unlock(&pool->sl); 7904 pool->raw_hw = NULL; 7905 return 0; 7906 } 7907 7908 #define MLX5_POOL_QUERY_FREQ_US 1000000 7909 7910 /** 7911 * Set the periodic procedure for triggering asynchronous batch queries for all 7912 * the counter pools. 7913 * 7914 * @param[in] sh 7915 * Pointer to mlx5_dev_ctx_shared object. 7916 */ 7917 void 7918 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7919 { 7920 uint32_t pools_n, us; 7921 7922 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7923 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7924 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7925 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7926 sh->cmng.query_thread_on = 0; 7927 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7928 } else { 7929 sh->cmng.query_thread_on = 1; 7930 } 7931 } 7932 7933 /** 7934 * The periodic procedure for triggering asynchronous batch queries for all the 7935 * counter pools. This function is probably called by the host thread. 7936 * 7937 * @param[in] arg 7938 * The parameter for the alarm process. 7939 */ 7940 void 7941 mlx5_flow_query_alarm(void *arg) 7942 { 7943 struct mlx5_dev_ctx_shared *sh = arg; 7944 int ret; 7945 uint16_t pool_index = sh->cmng.pool_index; 7946 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7947 struct mlx5_flow_counter_pool *pool; 7948 uint16_t n_valid; 7949 7950 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7951 goto set_alarm; 7952 rte_spinlock_lock(&cmng->pool_update_sl); 7953 pool = cmng->pools[pool_index]; 7954 n_valid = cmng->n_valid; 7955 rte_spinlock_unlock(&cmng->pool_update_sl); 7956 /* Set the statistic memory to the new created pool. */ 7957 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7958 goto set_alarm; 7959 if (pool->raw_hw) 7960 /* There is a pool query in progress. */ 7961 goto set_alarm; 7962 pool->raw_hw = 7963 LIST_FIRST(&sh->cmng.free_stat_raws); 7964 if (!pool->raw_hw) 7965 /* No free counter statistics raw memory. */ 7966 goto set_alarm; 7967 /* 7968 * Identify the counters released between query trigger and query 7969 * handle more efficiently. The counter released in this gap period 7970 * should wait for a new round of query as the new arrived packets 7971 * will not be taken into account. 7972 */ 7973 pool->query_gen++; 7974 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7975 MLX5_COUNTERS_PER_POOL, 7976 NULL, NULL, 7977 pool->raw_hw->mem_mng->wm.lkey, 7978 (void *)(uintptr_t) 7979 pool->raw_hw->data, 7980 sh->devx_comp, 7981 (uint64_t)(uintptr_t)pool); 7982 if (ret) { 7983 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7984 " %d", pool->min_dcs->id); 7985 pool->raw_hw = NULL; 7986 goto set_alarm; 7987 } 7988 LIST_REMOVE(pool->raw_hw, next); 7989 sh->cmng.pending_queries++; 7990 pool_index++; 7991 if (pool_index >= n_valid) 7992 pool_index = 0; 7993 set_alarm: 7994 sh->cmng.pool_index = pool_index; 7995 mlx5_set_query_alarm(sh); 7996 } 7997 7998 /** 7999 * Check and callback event for new aged flow in the counter pool 8000 * 8001 * @param[in] sh 8002 * Pointer to mlx5_dev_ctx_shared object. 8003 * @param[in] pool 8004 * Pointer to Current counter pool. 8005 */ 8006 static void 8007 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8008 struct mlx5_flow_counter_pool *pool) 8009 { 8010 struct mlx5_priv *priv; 8011 struct mlx5_flow_counter *cnt; 8012 struct mlx5_age_info *age_info; 8013 struct mlx5_age_param *age_param; 8014 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8015 struct mlx5_counter_stats_raw *prev = pool->raw; 8016 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8017 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8018 uint16_t expected = AGE_CANDIDATE; 8019 uint32_t i; 8020 8021 pool->time_of_last_age_check = curr_time; 8022 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8023 cnt = MLX5_POOL_GET_CNT(pool, i); 8024 age_param = MLX5_CNT_TO_AGE(cnt); 8025 if (__atomic_load_n(&age_param->state, 8026 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8027 continue; 8028 if (cur->data[i].hits != prev->data[i].hits) { 8029 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8030 __ATOMIC_RELAXED); 8031 continue; 8032 } 8033 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8034 time_delta, 8035 __ATOMIC_RELAXED) <= age_param->timeout) 8036 continue; 8037 /** 8038 * Hold the lock first, or if between the 8039 * state AGE_TMOUT and tailq operation the 8040 * release happened, the release procedure 8041 * may delete a non-existent tailq node. 8042 */ 8043 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8044 age_info = GET_PORT_AGE_INFO(priv); 8045 rte_spinlock_lock(&age_info->aged_sl); 8046 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8047 AGE_TMOUT, false, 8048 __ATOMIC_RELAXED, 8049 __ATOMIC_RELAXED)) { 8050 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8051 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8052 } 8053 rte_spinlock_unlock(&age_info->aged_sl); 8054 } 8055 mlx5_age_event_prepare(sh); 8056 } 8057 8058 /** 8059 * Handler for the HW respond about ready values from an asynchronous batch 8060 * query. This function is probably called by the host thread. 8061 * 8062 * @param[in] sh 8063 * The pointer to the shared device context. 8064 * @param[in] async_id 8065 * The Devx async ID. 8066 * @param[in] status 8067 * The status of the completion. 8068 */ 8069 void 8070 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8071 uint64_t async_id, int status) 8072 { 8073 struct mlx5_flow_counter_pool *pool = 8074 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8075 struct mlx5_counter_stats_raw *raw_to_free; 8076 uint8_t query_gen = pool->query_gen ^ 1; 8077 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8078 enum mlx5_counter_type cnt_type = 8079 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8080 MLX5_COUNTER_TYPE_ORIGIN; 8081 8082 if (unlikely(status)) { 8083 raw_to_free = pool->raw_hw; 8084 } else { 8085 raw_to_free = pool->raw; 8086 if (pool->is_aged) 8087 mlx5_flow_aging_check(sh, pool); 8088 rte_spinlock_lock(&pool->sl); 8089 pool->raw = pool->raw_hw; 8090 rte_spinlock_unlock(&pool->sl); 8091 /* Be sure the new raw counters data is updated in memory. */ 8092 rte_io_wmb(); 8093 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8094 rte_spinlock_lock(&cmng->csl[cnt_type]); 8095 TAILQ_CONCAT(&cmng->counters[cnt_type], 8096 &pool->counters[query_gen], next); 8097 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8098 } 8099 } 8100 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8101 pool->raw_hw = NULL; 8102 sh->cmng.pending_queries--; 8103 } 8104 8105 static int 8106 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8107 const struct flow_grp_info *grp_info, 8108 struct rte_flow_error *error) 8109 { 8110 if (grp_info->transfer && grp_info->external && 8111 grp_info->fdb_def_rule) { 8112 if (group == UINT32_MAX) 8113 return rte_flow_error_set 8114 (error, EINVAL, 8115 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8116 NULL, 8117 "group index not supported"); 8118 *table = group + 1; 8119 } else { 8120 *table = group; 8121 } 8122 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8123 return 0; 8124 } 8125 8126 /** 8127 * Translate the rte_flow group index to HW table value. 8128 * 8129 * If tunnel offload is disabled, all group ids converted to flow table 8130 * id using the standard method. 8131 * If tunnel offload is enabled, group id can be converted using the 8132 * standard or tunnel conversion method. Group conversion method 8133 * selection depends on flags in `grp_info` parameter: 8134 * - Internal (grp_info.external == 0) groups conversion uses the 8135 * standard method. 8136 * - Group ids in JUMP action converted with the tunnel conversion. 8137 * - Group id in rule attribute conversion depends on a rule type and 8138 * group id value: 8139 * ** non zero group attributes converted with the tunnel method 8140 * ** zero group attribute in non-tunnel rule is converted using the 8141 * standard method - there's only one root table 8142 * ** zero group attribute in steer tunnel rule is converted with the 8143 * standard method - single root table 8144 * ** zero group attribute in match tunnel rule is a special OvS 8145 * case: that value is used for portability reasons. That group 8146 * id is converted with the tunnel conversion method. 8147 * 8148 * @param[in] dev 8149 * Port device 8150 * @param[in] tunnel 8151 * PMD tunnel offload object 8152 * @param[in] group 8153 * rte_flow group index value. 8154 * @param[out] table 8155 * HW table value. 8156 * @param[in] grp_info 8157 * flags used for conversion 8158 * @param[out] error 8159 * Pointer to error structure. 8160 * 8161 * @return 8162 * 0 on success, a negative errno value otherwise and rte_errno is set. 8163 */ 8164 int 8165 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8166 const struct mlx5_flow_tunnel *tunnel, 8167 uint32_t group, uint32_t *table, 8168 const struct flow_grp_info *grp_info, 8169 struct rte_flow_error *error) 8170 { 8171 int ret; 8172 bool standard_translation; 8173 8174 if (!grp_info->skip_scale && grp_info->external && 8175 group < MLX5_MAX_TABLES_EXTERNAL) 8176 group *= MLX5_FLOW_TABLE_FACTOR; 8177 if (is_tunnel_offload_active(dev)) { 8178 standard_translation = !grp_info->external || 8179 grp_info->std_tbl_fix; 8180 } else { 8181 standard_translation = true; 8182 } 8183 DRV_LOG(DEBUG, 8184 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8185 dev->data->port_id, group, grp_info->transfer, 8186 grp_info->external, grp_info->fdb_def_rule, 8187 standard_translation ? "STANDARD" : "TUNNEL"); 8188 if (standard_translation) 8189 ret = flow_group_to_table(dev->data->port_id, group, table, 8190 grp_info, error); 8191 else 8192 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8193 table, error); 8194 8195 return ret; 8196 } 8197 8198 /** 8199 * Discover availability of metadata reg_c's. 8200 * 8201 * Iteratively use test flows to check availability. 8202 * 8203 * @param[in] dev 8204 * Pointer to the Ethernet device structure. 8205 * 8206 * @return 8207 * 0 on success, a negative errno value otherwise and rte_errno is set. 8208 */ 8209 int 8210 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8211 { 8212 struct mlx5_priv *priv = dev->data->dev_private; 8213 enum modify_reg idx; 8214 int n = 0; 8215 8216 /* reg_c[0] and reg_c[1] are reserved. */ 8217 priv->sh->flow_mreg_c[n++] = REG_C_0; 8218 priv->sh->flow_mreg_c[n++] = REG_C_1; 8219 /* Discover availability of other reg_c's. */ 8220 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 8221 struct rte_flow_attr attr = { 8222 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 8223 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8224 .ingress = 1, 8225 }; 8226 struct rte_flow_item items[] = { 8227 [0] = { 8228 .type = RTE_FLOW_ITEM_TYPE_END, 8229 }, 8230 }; 8231 struct rte_flow_action actions[] = { 8232 [0] = { 8233 .type = (enum rte_flow_action_type) 8234 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8235 .conf = &(struct mlx5_flow_action_copy_mreg){ 8236 .src = REG_C_1, 8237 .dst = idx, 8238 }, 8239 }, 8240 [1] = { 8241 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8242 .conf = &(struct rte_flow_action_jump){ 8243 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8244 }, 8245 }, 8246 [2] = { 8247 .type = RTE_FLOW_ACTION_TYPE_END, 8248 }, 8249 }; 8250 uint32_t flow_idx; 8251 struct rte_flow *flow; 8252 struct rte_flow_error error; 8253 8254 if (!priv->sh->config.dv_flow_en) 8255 break; 8256 /* Create internal flow, validation skips copy action. */ 8257 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8258 items, actions, false, &error); 8259 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8260 flow_idx); 8261 if (!flow) 8262 continue; 8263 priv->sh->flow_mreg_c[n++] = idx; 8264 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8265 } 8266 for (; n < MLX5_MREG_C_NUM; ++n) 8267 priv->sh->flow_mreg_c[n] = REG_NON; 8268 priv->sh->metadata_regc_check_flag = 1; 8269 return 0; 8270 } 8271 8272 int 8273 save_dump_file(const uint8_t *data, uint32_t size, 8274 uint32_t type, uint64_t id, void *arg, FILE *file) 8275 { 8276 char line[BUF_SIZE]; 8277 uint32_t out = 0; 8278 uint32_t k; 8279 uint32_t actions_num; 8280 struct rte_flow_query_count *count; 8281 8282 memset(line, 0, BUF_SIZE); 8283 switch (type) { 8284 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8285 actions_num = *(uint32_t *)(arg); 8286 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 8287 type, id, actions_num); 8288 break; 8289 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8290 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 8291 type, id); 8292 break; 8293 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8294 count = (struct rte_flow_query_count *)arg; 8295 fprintf(file, 8296 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 8297 type, id, count->hits, count->bytes); 8298 return 0; 8299 default: 8300 return -1; 8301 } 8302 8303 for (k = 0; k < size; k++) { 8304 /* Make sure we do not overrun the line buffer length. */ 8305 if (out >= BUF_SIZE - 4) { 8306 line[out] = '\0'; 8307 break; 8308 } 8309 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8310 (data[k]) & 0xff); 8311 } 8312 fprintf(file, "%s\n", line); 8313 return 0; 8314 } 8315 8316 int 8317 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8318 struct rte_flow_query_count *count, struct rte_flow_error *error) 8319 { 8320 struct rte_flow_action action[2]; 8321 enum mlx5_flow_drv_type ftype; 8322 const struct mlx5_flow_driver_ops *fops; 8323 8324 if (!flow) { 8325 return rte_flow_error_set(error, ENOENT, 8326 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8327 NULL, 8328 "invalid flow handle"); 8329 } 8330 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8331 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8332 if (flow->counter) { 8333 memset(count, 0, sizeof(struct rte_flow_query_count)); 8334 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8335 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8336 ftype < MLX5_FLOW_TYPE_MAX); 8337 fops = flow_get_drv_ops(ftype); 8338 return fops->query(dev, flow, action, count, error); 8339 } 8340 return -1; 8341 } 8342 8343 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8344 /** 8345 * Dump flow ipool data to file 8346 * 8347 * @param[in] dev 8348 * The pointer to Ethernet device. 8349 * @param[in] file 8350 * A pointer to a file for output. 8351 * @param[out] error 8352 * Perform verbose error reporting if not NULL. PMDs initialize this 8353 * structure in case of error only. 8354 * @return 8355 * 0 on success, a negative value otherwise. 8356 */ 8357 int 8358 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8359 struct rte_flow *flow, FILE *file, 8360 struct rte_flow_error *error) 8361 { 8362 struct mlx5_priv *priv = dev->data->dev_private; 8363 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8364 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8365 uint32_t handle_idx; 8366 struct mlx5_flow_handle *dh; 8367 struct rte_flow_query_count count; 8368 uint32_t actions_num; 8369 const uint8_t *data; 8370 size_t size; 8371 uint64_t id; 8372 uint32_t type; 8373 void *action = NULL; 8374 8375 if (!flow) { 8376 return rte_flow_error_set(error, ENOENT, 8377 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8378 NULL, 8379 "invalid flow handle"); 8380 } 8381 handle_idx = flow->dev_handles; 8382 /* query counter */ 8383 if (flow->counter && 8384 (!mlx5_counter_query(dev, flow->counter, false, 8385 &count.hits, &count.bytes, &action)) && action) { 8386 id = (uint64_t)(uintptr_t)action; 8387 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8388 save_dump_file(NULL, 0, type, 8389 id, (void *)&count, file); 8390 } 8391 8392 while (handle_idx) { 8393 dh = mlx5_ipool_get(priv->sh->ipool 8394 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8395 if (!dh) 8396 continue; 8397 handle_idx = dh->next.next; 8398 8399 /* Get modify_hdr and encap_decap buf from ipools. */ 8400 encap_decap = NULL; 8401 modify_hdr = dh->dvh.modify_hdr; 8402 8403 if (dh->dvh.rix_encap_decap) { 8404 encap_decap = mlx5_ipool_get(priv->sh->ipool 8405 [MLX5_IPOOL_DECAP_ENCAP], 8406 dh->dvh.rix_encap_decap); 8407 } 8408 if (modify_hdr) { 8409 data = (const uint8_t *)modify_hdr->actions; 8410 size = (size_t)(modify_hdr->actions_num) * 8; 8411 id = (uint64_t)(uintptr_t)modify_hdr->action; 8412 actions_num = modify_hdr->actions_num; 8413 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8414 save_dump_file(data, size, type, id, 8415 (void *)(&actions_num), file); 8416 } 8417 if (encap_decap) { 8418 data = encap_decap->buf; 8419 size = encap_decap->size; 8420 id = (uint64_t)(uintptr_t)encap_decap->action; 8421 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8422 save_dump_file(data, size, type, 8423 id, NULL, file); 8424 } 8425 } 8426 return 0; 8427 } 8428 8429 /** 8430 * Dump all flow's encap_decap/modify_hdr/counter data to file 8431 * 8432 * @param[in] dev 8433 * The pointer to Ethernet device. 8434 * @param[in] file 8435 * A pointer to a file for output. 8436 * @param[out] error 8437 * Perform verbose error reporting if not NULL. PMDs initialize this 8438 * structure in case of error only. 8439 * @return 8440 * 0 on success, a negative value otherwise. 8441 */ 8442 static int 8443 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 8444 FILE *file, struct rte_flow_error *error __rte_unused) 8445 { 8446 struct mlx5_priv *priv = dev->data->dev_private; 8447 struct mlx5_dev_ctx_shared *sh = priv->sh; 8448 struct mlx5_hlist *h; 8449 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8450 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8451 struct rte_flow_query_count count; 8452 uint32_t actions_num; 8453 const uint8_t *data; 8454 size_t size; 8455 uint64_t id; 8456 uint32_t type; 8457 uint32_t i; 8458 uint32_t j; 8459 struct mlx5_list_inconst *l_inconst; 8460 struct mlx5_list_entry *e; 8461 int lcore_index; 8462 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 8463 uint32_t max; 8464 void *action; 8465 8466 /* encap_decap hlist is lcore_share, get global core cache. */ 8467 i = MLX5_LIST_GLOBAL; 8468 h = sh->encaps_decaps; 8469 if (h) { 8470 for (j = 0; j <= h->mask; j++) { 8471 l_inconst = &h->buckets[j].l; 8472 if (!l_inconst || !l_inconst->cache[i]) 8473 continue; 8474 8475 e = LIST_FIRST(&l_inconst->cache[i]->h); 8476 while (e) { 8477 encap_decap = 8478 (struct mlx5_flow_dv_encap_decap_resource *)e; 8479 data = encap_decap->buf; 8480 size = encap_decap->size; 8481 id = (uint64_t)(uintptr_t)encap_decap->action; 8482 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8483 save_dump_file(data, size, type, 8484 id, NULL, file); 8485 e = LIST_NEXT(e, next); 8486 } 8487 } 8488 } 8489 8490 /* get modify_hdr */ 8491 h = sh->modify_cmds; 8492 if (h) { 8493 lcore_index = rte_lcore_index(rte_lcore_id()); 8494 if (unlikely(lcore_index == -1)) { 8495 lcore_index = MLX5_LIST_NLCORE; 8496 rte_spinlock_lock(&h->l_const.lcore_lock); 8497 } 8498 i = lcore_index; 8499 8500 for (j = 0; j <= h->mask; j++) { 8501 l_inconst = &h->buckets[j].l; 8502 if (!l_inconst || !l_inconst->cache[i]) 8503 continue; 8504 8505 e = LIST_FIRST(&l_inconst->cache[i]->h); 8506 while (e) { 8507 modify_hdr = 8508 (struct mlx5_flow_dv_modify_hdr_resource *)e; 8509 data = (const uint8_t *)modify_hdr->actions; 8510 size = (size_t)(modify_hdr->actions_num) * 8; 8511 actions_num = modify_hdr->actions_num; 8512 id = (uint64_t)(uintptr_t)modify_hdr->action; 8513 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8514 save_dump_file(data, size, type, id, 8515 (void *)(&actions_num), file); 8516 e = LIST_NEXT(e, next); 8517 } 8518 } 8519 8520 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 8521 rte_spinlock_unlock(&h->l_const.lcore_lock); 8522 } 8523 8524 /* get counter */ 8525 MLX5_ASSERT(cmng->n_valid <= cmng->n); 8526 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 8527 for (j = 1; j <= max; j++) { 8528 action = NULL; 8529 if ((!mlx5_counter_query(dev, j, false, &count.hits, 8530 &count.bytes, &action)) && action) { 8531 id = (uint64_t)(uintptr_t)action; 8532 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8533 save_dump_file(NULL, 0, type, 8534 id, (void *)&count, file); 8535 } 8536 } 8537 return 0; 8538 } 8539 #endif 8540 8541 /** 8542 * Dump flow raw hw data to file 8543 * 8544 * @param[in] dev 8545 * The pointer to Ethernet device. 8546 * @param[in] file 8547 * A pointer to a file for output. 8548 * @param[out] error 8549 * Perform verbose error reporting if not NULL. PMDs initialize this 8550 * structure in case of error only. 8551 * @return 8552 * 0 on success, a negative value otherwise. 8553 */ 8554 int 8555 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8556 FILE *file, 8557 struct rte_flow_error *error __rte_unused) 8558 { 8559 struct mlx5_priv *priv = dev->data->dev_private; 8560 struct mlx5_dev_ctx_shared *sh = priv->sh; 8561 uint32_t handle_idx; 8562 int ret; 8563 struct mlx5_flow_handle *dh; 8564 struct rte_flow *flow; 8565 8566 if (!sh->config.dv_flow_en) { 8567 if (fputs("device dv flow disabled\n", file) <= 0) 8568 return -errno; 8569 return -ENOTSUP; 8570 } 8571 8572 /* dump all */ 8573 if (!flow_idx) { 8574 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8575 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 8576 return -EINVAL; 8577 #endif 8578 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8579 sh->rx_domain, 8580 sh->tx_domain, file); 8581 } 8582 /* dump one */ 8583 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8584 (uintptr_t)(void *)flow_idx); 8585 if (!flow) 8586 return -EINVAL; 8587 8588 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8589 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8590 #endif 8591 handle_idx = flow->dev_handles; 8592 while (handle_idx) { 8593 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8594 handle_idx); 8595 if (!dh) 8596 return -ENOENT; 8597 if (dh->drv_flow) { 8598 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8599 file); 8600 if (ret) 8601 return -ENOENT; 8602 } 8603 handle_idx = dh->next.next; 8604 } 8605 return 0; 8606 } 8607 8608 /** 8609 * Get aged-out flows. 8610 * 8611 * @param[in] dev 8612 * Pointer to the Ethernet device structure. 8613 * @param[in] context 8614 * The address of an array of pointers to the aged-out flows contexts. 8615 * @param[in] nb_countexts 8616 * The length of context array pointers. 8617 * @param[out] error 8618 * Perform verbose error reporting if not NULL. Initialized in case of 8619 * error only. 8620 * 8621 * @return 8622 * how many contexts get in success, otherwise negative errno value. 8623 * if nb_contexts is 0, return the amount of all aged contexts. 8624 * if nb_contexts is not 0 , return the amount of aged flows reported 8625 * in the context array. 8626 */ 8627 int 8628 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8629 uint32_t nb_contexts, struct rte_flow_error *error) 8630 { 8631 const struct mlx5_flow_driver_ops *fops; 8632 struct rte_flow_attr attr = { .transfer = 0 }; 8633 8634 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8635 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8636 return fops->get_aged_flows(dev, contexts, nb_contexts, 8637 error); 8638 } 8639 DRV_LOG(ERR, 8640 "port %u get aged flows is not supported.", 8641 dev->data->port_id); 8642 return -ENOTSUP; 8643 } 8644 8645 /* Wrapper for driver action_validate op callback */ 8646 static int 8647 flow_drv_action_validate(struct rte_eth_dev *dev, 8648 const struct rte_flow_indir_action_conf *conf, 8649 const struct rte_flow_action *action, 8650 const struct mlx5_flow_driver_ops *fops, 8651 struct rte_flow_error *error) 8652 { 8653 static const char err_msg[] = "indirect action validation unsupported"; 8654 8655 if (!fops->action_validate) { 8656 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8657 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8658 NULL, err_msg); 8659 return -rte_errno; 8660 } 8661 return fops->action_validate(dev, conf, action, error); 8662 } 8663 8664 /** 8665 * Destroys the shared action by handle. 8666 * 8667 * @param dev 8668 * Pointer to Ethernet device structure. 8669 * @param[in] handle 8670 * Handle for the indirect action object to be destroyed. 8671 * @param[out] error 8672 * Perform verbose error reporting if not NULL. PMDs initialize this 8673 * structure in case of error only. 8674 * 8675 * @return 8676 * 0 on success, a negative errno value otherwise and rte_errno is set. 8677 * 8678 * @note: wrapper for driver action_create op callback. 8679 */ 8680 static int 8681 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8682 struct rte_flow_action_handle *handle, 8683 struct rte_flow_error *error) 8684 { 8685 static const char err_msg[] = "indirect action destruction unsupported"; 8686 struct rte_flow_attr attr = { .transfer = 0 }; 8687 const struct mlx5_flow_driver_ops *fops = 8688 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8689 8690 if (!fops->action_destroy) { 8691 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8692 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8693 NULL, err_msg); 8694 return -rte_errno; 8695 } 8696 return fops->action_destroy(dev, handle, error); 8697 } 8698 8699 /* Wrapper for driver action_destroy op callback */ 8700 static int 8701 flow_drv_action_update(struct rte_eth_dev *dev, 8702 struct rte_flow_action_handle *handle, 8703 const void *update, 8704 const struct mlx5_flow_driver_ops *fops, 8705 struct rte_flow_error *error) 8706 { 8707 static const char err_msg[] = "indirect action update unsupported"; 8708 8709 if (!fops->action_update) { 8710 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8711 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8712 NULL, err_msg); 8713 return -rte_errno; 8714 } 8715 return fops->action_update(dev, handle, update, error); 8716 } 8717 8718 /* Wrapper for driver action_destroy op callback */ 8719 static int 8720 flow_drv_action_query(struct rte_eth_dev *dev, 8721 const struct rte_flow_action_handle *handle, 8722 void *data, 8723 const struct mlx5_flow_driver_ops *fops, 8724 struct rte_flow_error *error) 8725 { 8726 static const char err_msg[] = "indirect action query unsupported"; 8727 8728 if (!fops->action_query) { 8729 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8730 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8731 NULL, err_msg); 8732 return -rte_errno; 8733 } 8734 return fops->action_query(dev, handle, data, error); 8735 } 8736 8737 /** 8738 * Create indirect action for reuse in multiple flow rules. 8739 * 8740 * @param dev 8741 * Pointer to Ethernet device structure. 8742 * @param conf 8743 * Pointer to indirect action object configuration. 8744 * @param[in] action 8745 * Action configuration for indirect action object creation. 8746 * @param[out] error 8747 * Perform verbose error reporting if not NULL. PMDs initialize this 8748 * structure in case of error only. 8749 * @return 8750 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8751 */ 8752 static struct rte_flow_action_handle * 8753 mlx5_action_handle_create(struct rte_eth_dev *dev, 8754 const struct rte_flow_indir_action_conf *conf, 8755 const struct rte_flow_action *action, 8756 struct rte_flow_error *error) 8757 { 8758 static const char err_msg[] = "indirect action creation unsupported"; 8759 struct rte_flow_attr attr = { .transfer = 0 }; 8760 const struct mlx5_flow_driver_ops *fops = 8761 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8762 8763 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8764 return NULL; 8765 if (!fops->action_create) { 8766 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8767 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8768 NULL, err_msg); 8769 return NULL; 8770 } 8771 return fops->action_create(dev, conf, action, error); 8772 } 8773 8774 /** 8775 * Updates inplace the indirect action configuration pointed by *handle* 8776 * with the configuration provided as *update* argument. 8777 * The update of the indirect action configuration effects all flow rules 8778 * reusing the action via handle. 8779 * 8780 * @param dev 8781 * Pointer to Ethernet device structure. 8782 * @param[in] handle 8783 * Handle for the indirect action to be updated. 8784 * @param[in] update 8785 * Action specification used to modify the action pointed by handle. 8786 * *update* could be of same type with the action pointed by the *handle* 8787 * handle argument, or some other structures like a wrapper, depending on 8788 * the indirect action type. 8789 * @param[out] error 8790 * Perform verbose error reporting if not NULL. PMDs initialize this 8791 * structure in case of error only. 8792 * 8793 * @return 8794 * 0 on success, a negative errno value otherwise and rte_errno is set. 8795 */ 8796 static int 8797 mlx5_action_handle_update(struct rte_eth_dev *dev, 8798 struct rte_flow_action_handle *handle, 8799 const void *update, 8800 struct rte_flow_error *error) 8801 { 8802 struct rte_flow_attr attr = { .transfer = 0 }; 8803 const struct mlx5_flow_driver_ops *fops = 8804 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8805 int ret; 8806 8807 ret = flow_drv_action_validate(dev, NULL, 8808 (const struct rte_flow_action *)update, fops, error); 8809 if (ret) 8810 return ret; 8811 return flow_drv_action_update(dev, handle, update, fops, 8812 error); 8813 } 8814 8815 /** 8816 * Query the indirect action by handle. 8817 * 8818 * This function allows retrieving action-specific data such as counters. 8819 * Data is gathered by special action which may be present/referenced in 8820 * more than one flow rule definition. 8821 * 8822 * see @RTE_FLOW_ACTION_TYPE_COUNT 8823 * 8824 * @param dev 8825 * Pointer to Ethernet device structure. 8826 * @param[in] handle 8827 * Handle for the indirect action to query. 8828 * @param[in, out] data 8829 * Pointer to storage for the associated query data type. 8830 * @param[out] error 8831 * Perform verbose error reporting if not NULL. PMDs initialize this 8832 * structure in case of error only. 8833 * 8834 * @return 8835 * 0 on success, a negative errno value otherwise and rte_errno is set. 8836 */ 8837 static int 8838 mlx5_action_handle_query(struct rte_eth_dev *dev, 8839 const struct rte_flow_action_handle *handle, 8840 void *data, 8841 struct rte_flow_error *error) 8842 { 8843 struct rte_flow_attr attr = { .transfer = 0 }; 8844 const struct mlx5_flow_driver_ops *fops = 8845 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8846 8847 return flow_drv_action_query(dev, handle, data, fops, error); 8848 } 8849 8850 /** 8851 * Destroy all indirect actions (shared RSS). 8852 * 8853 * @param dev 8854 * Pointer to Ethernet device. 8855 * 8856 * @return 8857 * 0 on success, a negative errno value otherwise and rte_errno is set. 8858 */ 8859 int 8860 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8861 { 8862 struct rte_flow_error error; 8863 struct mlx5_priv *priv = dev->data->dev_private; 8864 struct mlx5_shared_action_rss *shared_rss; 8865 int ret = 0; 8866 uint32_t idx; 8867 8868 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8869 priv->rss_shared_actions, idx, shared_rss, next) { 8870 ret |= mlx5_action_handle_destroy(dev, 8871 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8872 } 8873 return ret; 8874 } 8875 8876 /** 8877 * Validate existing indirect actions against current device configuration 8878 * and attach them to device resources. 8879 * 8880 * @param dev 8881 * Pointer to Ethernet device. 8882 * 8883 * @return 8884 * 0 on success, a negative errno value otherwise and rte_errno is set. 8885 */ 8886 int 8887 mlx5_action_handle_attach(struct rte_eth_dev *dev) 8888 { 8889 struct mlx5_priv *priv = dev->data->dev_private; 8890 struct mlx5_indexed_pool *ipool = 8891 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 8892 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 8893 int ret = 0; 8894 uint32_t idx; 8895 8896 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8897 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8898 const char *message; 8899 uint32_t queue_idx; 8900 8901 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 8902 ind_tbl->queues_n, 8903 &message, &queue_idx); 8904 if (ret != 0) { 8905 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 8906 dev->data->port_id, ind_tbl->queues[queue_idx], 8907 message); 8908 break; 8909 } 8910 } 8911 if (ret != 0) 8912 return ret; 8913 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8914 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8915 8916 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 8917 if (ret != 0) { 8918 DRV_LOG(ERR, "Port %u could not attach " 8919 "indirection table obj %p", 8920 dev->data->port_id, (void *)ind_tbl); 8921 goto error; 8922 } 8923 } 8924 return 0; 8925 error: 8926 shared_rss_last = shared_rss; 8927 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8928 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8929 8930 if (shared_rss == shared_rss_last) 8931 break; 8932 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 8933 DRV_LOG(CRIT, "Port %u could not detach " 8934 "indirection table obj %p on rollback", 8935 dev->data->port_id, (void *)ind_tbl); 8936 } 8937 return ret; 8938 } 8939 8940 /** 8941 * Detach indirect actions of the device from its resources. 8942 * 8943 * @param dev 8944 * Pointer to Ethernet device. 8945 * 8946 * @return 8947 * 0 on success, a negative errno value otherwise and rte_errno is set. 8948 */ 8949 int 8950 mlx5_action_handle_detach(struct rte_eth_dev *dev) 8951 { 8952 struct mlx5_priv *priv = dev->data->dev_private; 8953 struct mlx5_indexed_pool *ipool = 8954 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 8955 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 8956 int ret = 0; 8957 uint32_t idx; 8958 8959 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8960 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8961 8962 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 8963 if (ret != 0) { 8964 DRV_LOG(ERR, "Port %u could not detach " 8965 "indirection table obj %p", 8966 dev->data->port_id, (void *)ind_tbl); 8967 goto error; 8968 } 8969 } 8970 return 0; 8971 error: 8972 shared_rss_last = shared_rss; 8973 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8974 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8975 8976 if (shared_rss == shared_rss_last) 8977 break; 8978 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 8979 DRV_LOG(CRIT, "Port %u could not attach " 8980 "indirection table obj %p on rollback", 8981 dev->data->port_id, (void *)ind_tbl); 8982 } 8983 return ret; 8984 } 8985 8986 #ifndef HAVE_MLX5DV_DR 8987 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8988 #else 8989 #define MLX5_DOMAIN_SYNC_FLOW \ 8990 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8991 #endif 8992 8993 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8994 { 8995 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8996 const struct mlx5_flow_driver_ops *fops; 8997 int ret; 8998 struct rte_flow_attr attr = { .transfer = 0 }; 8999 9000 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9001 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9002 if (ret > 0) 9003 ret = -ret; 9004 return ret; 9005 } 9006 9007 const struct mlx5_flow_tunnel * 9008 mlx5_get_tof(const struct rte_flow_item *item, 9009 const struct rte_flow_action *action, 9010 enum mlx5_tof_rule_type *rule_type) 9011 { 9012 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9013 if (item->type == (typeof(item->type)) 9014 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9015 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9016 return flow_items_to_tunnel(item); 9017 } 9018 } 9019 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9020 if (action->type == (typeof(action->type)) 9021 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9022 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9023 return flow_actions_to_tunnel(action); 9024 } 9025 } 9026 return NULL; 9027 } 9028 9029 /** 9030 * tunnel offload functionality is defined for DV environment only 9031 */ 9032 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9033 __extension__ 9034 union tunnel_offload_mark { 9035 uint32_t val; 9036 struct { 9037 uint32_t app_reserve:8; 9038 uint32_t table_id:15; 9039 uint32_t transfer:1; 9040 uint32_t _unused_:8; 9041 }; 9042 }; 9043 9044 static bool 9045 mlx5_access_tunnel_offload_db 9046 (struct rte_eth_dev *dev, 9047 bool (*match)(struct rte_eth_dev *, 9048 struct mlx5_flow_tunnel *, const void *), 9049 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9050 void (*miss)(struct rte_eth_dev *, void *), 9051 void *ctx, bool lock_op); 9052 9053 static int 9054 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9055 struct rte_flow *flow, 9056 const struct rte_flow_attr *attr, 9057 const struct rte_flow_action *app_actions, 9058 uint32_t flow_idx, 9059 const struct mlx5_flow_tunnel *tunnel, 9060 struct tunnel_default_miss_ctx *ctx, 9061 struct rte_flow_error *error) 9062 { 9063 struct mlx5_priv *priv = dev->data->dev_private; 9064 struct mlx5_flow *dev_flow; 9065 struct rte_flow_attr miss_attr = *attr; 9066 const struct rte_flow_item miss_items[2] = { 9067 { 9068 .type = RTE_FLOW_ITEM_TYPE_ETH, 9069 .spec = NULL, 9070 .last = NULL, 9071 .mask = NULL 9072 }, 9073 { 9074 .type = RTE_FLOW_ITEM_TYPE_END, 9075 .spec = NULL, 9076 .last = NULL, 9077 .mask = NULL 9078 } 9079 }; 9080 union tunnel_offload_mark mark_id; 9081 struct rte_flow_action_mark miss_mark; 9082 struct rte_flow_action miss_actions[3] = { 9083 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9084 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9085 }; 9086 const struct rte_flow_action_jump *jump_data; 9087 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9088 struct flow_grp_info grp_info = { 9089 .external = 1, 9090 .transfer = attr->transfer, 9091 .fdb_def_rule = !!priv->fdb_def_rule, 9092 .std_tbl_fix = 0, 9093 }; 9094 int ret; 9095 9096 if (!attr->transfer) { 9097 uint32_t q_size; 9098 9099 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9100 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9101 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9102 0, SOCKET_ID_ANY); 9103 if (!ctx->queue) 9104 return rte_flow_error_set 9105 (error, ENOMEM, 9106 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9107 NULL, "invalid default miss RSS"); 9108 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9109 ctx->action_rss.level = 0, 9110 ctx->action_rss.types = priv->rss_conf.rss_hf, 9111 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9112 ctx->action_rss.queue_num = priv->reta_idx_n, 9113 ctx->action_rss.key = priv->rss_conf.rss_key, 9114 ctx->action_rss.queue = ctx->queue; 9115 if (!priv->reta_idx_n || !priv->rxqs_n) 9116 return rte_flow_error_set 9117 (error, EINVAL, 9118 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9119 NULL, "invalid port configuration"); 9120 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9121 ctx->action_rss.types = 0; 9122 for (i = 0; i != priv->reta_idx_n; ++i) 9123 ctx->queue[i] = (*priv->reta_idx)[i]; 9124 } else { 9125 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9126 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9127 } 9128 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9129 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9130 jump_data = app_actions->conf; 9131 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9132 miss_attr.group = jump_data->group; 9133 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9134 &flow_table, &grp_info, error); 9135 if (ret) 9136 return rte_flow_error_set(error, EINVAL, 9137 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9138 NULL, "invalid tunnel id"); 9139 mark_id.app_reserve = 0; 9140 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9141 mark_id.transfer = !!attr->transfer; 9142 mark_id._unused_ = 0; 9143 miss_mark.id = mark_id.val; 9144 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9145 miss_items, miss_actions, flow_idx, error); 9146 if (!dev_flow) 9147 return -rte_errno; 9148 dev_flow->flow = flow; 9149 dev_flow->external = true; 9150 dev_flow->tunnel = tunnel; 9151 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9152 /* Subflow object was created, we must include one in the list. */ 9153 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9154 dev_flow->handle, next); 9155 DRV_LOG(DEBUG, 9156 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9157 dev->data->port_id, tunnel->app_tunnel.type, 9158 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9159 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9160 miss_actions, error); 9161 if (!ret) 9162 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9163 error); 9164 9165 return ret; 9166 } 9167 9168 static const struct mlx5_flow_tbl_data_entry * 9169 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9170 { 9171 struct mlx5_priv *priv = dev->data->dev_private; 9172 struct mlx5_dev_ctx_shared *sh = priv->sh; 9173 struct mlx5_list_entry *he; 9174 union tunnel_offload_mark mbits = { .val = mark }; 9175 union mlx5_flow_tbl_key table_key = { 9176 { 9177 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9178 .id = 0, 9179 .reserved = 0, 9180 .dummy = 0, 9181 .is_fdb = !!mbits.transfer, 9182 .is_egress = 0, 9183 } 9184 }; 9185 struct mlx5_flow_cb_ctx ctx = { 9186 .data = &table_key.v64, 9187 }; 9188 9189 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9190 return he ? 9191 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9192 } 9193 9194 static void 9195 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9196 struct mlx5_list_entry *entry) 9197 { 9198 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9199 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9200 9201 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9202 tunnel_flow_tbl_to_id(tte->flow_table)); 9203 mlx5_free(tte); 9204 } 9205 9206 static int 9207 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9208 struct mlx5_list_entry *entry, void *cb_ctx) 9209 { 9210 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9211 union tunnel_tbl_key tbl = { 9212 .val = *(uint64_t *)(ctx->data), 9213 }; 9214 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9215 9216 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9217 } 9218 9219 static struct mlx5_list_entry * 9220 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 9221 { 9222 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9223 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9224 struct tunnel_tbl_entry *tte; 9225 union tunnel_tbl_key tbl = { 9226 .val = *(uint64_t *)(ctx->data), 9227 }; 9228 9229 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 9230 sizeof(*tte), 0, 9231 SOCKET_ID_ANY); 9232 if (!tte) 9233 goto err; 9234 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9235 &tte->flow_table); 9236 if (tte->flow_table >= MLX5_MAX_TABLES) { 9237 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 9238 tte->flow_table); 9239 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9240 tte->flow_table); 9241 goto err; 9242 } else if (!tte->flow_table) { 9243 goto err; 9244 } 9245 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 9246 tte->tunnel_id = tbl.tunnel_id; 9247 tte->group = tbl.group; 9248 return &tte->hash; 9249 err: 9250 if (tte) 9251 mlx5_free(tte); 9252 return NULL; 9253 } 9254 9255 static struct mlx5_list_entry * 9256 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 9257 struct mlx5_list_entry *oentry, 9258 void *cb_ctx __rte_unused) 9259 { 9260 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 9261 0, SOCKET_ID_ANY); 9262 9263 if (!tte) 9264 return NULL; 9265 memcpy(tte, oentry, sizeof(*tte)); 9266 return &tte->hash; 9267 } 9268 9269 static void 9270 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 9271 struct mlx5_list_entry *entry) 9272 { 9273 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9274 9275 mlx5_free(tte); 9276 } 9277 9278 static uint32_t 9279 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 9280 const struct mlx5_flow_tunnel *tunnel, 9281 uint32_t group, uint32_t *table, 9282 struct rte_flow_error *error) 9283 { 9284 struct mlx5_list_entry *he; 9285 struct tunnel_tbl_entry *tte; 9286 union tunnel_tbl_key key = { 9287 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 9288 .group = group 9289 }; 9290 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9291 struct mlx5_hlist *group_hash; 9292 struct mlx5_flow_cb_ctx ctx = { 9293 .data = &key.val, 9294 }; 9295 9296 group_hash = tunnel ? tunnel->groups : thub->groups; 9297 he = mlx5_hlist_register(group_hash, key.val, &ctx); 9298 if (!he) 9299 return rte_flow_error_set(error, EINVAL, 9300 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9301 NULL, 9302 "tunnel group index not supported"); 9303 tte = container_of(he, typeof(*tte), hash); 9304 *table = tte->flow_table; 9305 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 9306 dev->data->port_id, key.tunnel_id, group, *table); 9307 return 0; 9308 } 9309 9310 static void 9311 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 9312 struct mlx5_flow_tunnel *tunnel) 9313 { 9314 struct mlx5_priv *priv = dev->data->dev_private; 9315 struct mlx5_indexed_pool *ipool; 9316 9317 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 9318 dev->data->port_id, tunnel->tunnel_id); 9319 LIST_REMOVE(tunnel, chain); 9320 mlx5_hlist_destroy(tunnel->groups); 9321 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9322 mlx5_ipool_free(ipool, tunnel->tunnel_id); 9323 } 9324 9325 static bool 9326 mlx5_access_tunnel_offload_db 9327 (struct rte_eth_dev *dev, 9328 bool (*match)(struct rte_eth_dev *, 9329 struct mlx5_flow_tunnel *, const void *), 9330 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9331 void (*miss)(struct rte_eth_dev *, void *), 9332 void *ctx, bool lock_op) 9333 { 9334 bool verdict = false; 9335 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9336 struct mlx5_flow_tunnel *tunnel; 9337 9338 rte_spinlock_lock(&thub->sl); 9339 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 9340 verdict = match(dev, tunnel, (const void *)ctx); 9341 if (verdict) 9342 break; 9343 } 9344 if (!lock_op) 9345 rte_spinlock_unlock(&thub->sl); 9346 if (verdict && hit) 9347 hit(dev, tunnel, ctx); 9348 if (!verdict && miss) 9349 miss(dev, ctx); 9350 if (lock_op) 9351 rte_spinlock_unlock(&thub->sl); 9352 9353 return verdict; 9354 } 9355 9356 struct tunnel_db_find_tunnel_id_ctx { 9357 uint32_t tunnel_id; 9358 struct mlx5_flow_tunnel *tunnel; 9359 }; 9360 9361 static bool 9362 find_tunnel_id_match(struct rte_eth_dev *dev, 9363 struct mlx5_flow_tunnel *tunnel, const void *x) 9364 { 9365 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9366 9367 RTE_SET_USED(dev); 9368 return tunnel->tunnel_id == ctx->tunnel_id; 9369 } 9370 9371 static void 9372 find_tunnel_id_hit(struct rte_eth_dev *dev, 9373 struct mlx5_flow_tunnel *tunnel, void *x) 9374 { 9375 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9376 RTE_SET_USED(dev); 9377 ctx->tunnel = tunnel; 9378 } 9379 9380 static struct mlx5_flow_tunnel * 9381 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 9382 { 9383 struct tunnel_db_find_tunnel_id_ctx ctx = { 9384 .tunnel_id = id, 9385 }; 9386 9387 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 9388 find_tunnel_id_hit, NULL, &ctx, true); 9389 9390 return ctx.tunnel; 9391 } 9392 9393 static struct mlx5_flow_tunnel * 9394 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 9395 const struct rte_flow_tunnel *app_tunnel) 9396 { 9397 struct mlx5_priv *priv = dev->data->dev_private; 9398 struct mlx5_indexed_pool *ipool; 9399 struct mlx5_flow_tunnel *tunnel; 9400 uint32_t id; 9401 9402 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9403 tunnel = mlx5_ipool_zmalloc(ipool, &id); 9404 if (!tunnel) 9405 return NULL; 9406 if (id >= MLX5_MAX_TUNNELS) { 9407 mlx5_ipool_free(ipool, id); 9408 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 9409 return NULL; 9410 } 9411 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 9412 priv->sh, 9413 mlx5_flow_tunnel_grp2tbl_create_cb, 9414 mlx5_flow_tunnel_grp2tbl_match_cb, 9415 mlx5_flow_tunnel_grp2tbl_remove_cb, 9416 mlx5_flow_tunnel_grp2tbl_clone_cb, 9417 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9418 if (!tunnel->groups) { 9419 mlx5_ipool_free(ipool, id); 9420 return NULL; 9421 } 9422 /* initiate new PMD tunnel */ 9423 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 9424 tunnel->tunnel_id = id; 9425 tunnel->action.type = (typeof(tunnel->action.type)) 9426 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 9427 tunnel->action.conf = tunnel; 9428 tunnel->item.type = (typeof(tunnel->item.type)) 9429 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 9430 tunnel->item.spec = tunnel; 9431 tunnel->item.last = NULL; 9432 tunnel->item.mask = NULL; 9433 9434 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 9435 dev->data->port_id, tunnel->tunnel_id); 9436 9437 return tunnel; 9438 } 9439 9440 struct tunnel_db_get_tunnel_ctx { 9441 const struct rte_flow_tunnel *app_tunnel; 9442 struct mlx5_flow_tunnel *tunnel; 9443 }; 9444 9445 static bool get_tunnel_match(struct rte_eth_dev *dev, 9446 struct mlx5_flow_tunnel *tunnel, const void *x) 9447 { 9448 const struct tunnel_db_get_tunnel_ctx *ctx = x; 9449 9450 RTE_SET_USED(dev); 9451 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 9452 sizeof(*ctx->app_tunnel)); 9453 } 9454 9455 static void get_tunnel_hit(struct rte_eth_dev *dev, 9456 struct mlx5_flow_tunnel *tunnel, void *x) 9457 { 9458 /* called under tunnel spinlock protection */ 9459 struct tunnel_db_get_tunnel_ctx *ctx = x; 9460 9461 RTE_SET_USED(dev); 9462 tunnel->refctn++; 9463 ctx->tunnel = tunnel; 9464 } 9465 9466 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 9467 { 9468 /* called under tunnel spinlock protection */ 9469 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9470 struct tunnel_db_get_tunnel_ctx *ctx = x; 9471 9472 rte_spinlock_unlock(&thub->sl); 9473 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 9474 rte_spinlock_lock(&thub->sl); 9475 if (ctx->tunnel) { 9476 ctx->tunnel->refctn = 1; 9477 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 9478 } 9479 } 9480 9481 9482 static int 9483 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 9484 const struct rte_flow_tunnel *app_tunnel, 9485 struct mlx5_flow_tunnel **tunnel) 9486 { 9487 struct tunnel_db_get_tunnel_ctx ctx = { 9488 .app_tunnel = app_tunnel, 9489 }; 9490 9491 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 9492 get_tunnel_miss, &ctx, true); 9493 *tunnel = ctx.tunnel; 9494 return ctx.tunnel ? 0 : -ENOMEM; 9495 } 9496 9497 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 9498 { 9499 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 9500 9501 if (!thub) 9502 return; 9503 if (!LIST_EMPTY(&thub->tunnels)) 9504 DRV_LOG(WARNING, "port %u tunnels present", port_id); 9505 mlx5_hlist_destroy(thub->groups); 9506 mlx5_free(thub); 9507 } 9508 9509 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 9510 { 9511 int err; 9512 struct mlx5_flow_tunnel_hub *thub; 9513 9514 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 9515 0, SOCKET_ID_ANY); 9516 if (!thub) 9517 return -ENOMEM; 9518 LIST_INIT(&thub->tunnels); 9519 rte_spinlock_init(&thub->sl); 9520 thub->groups = mlx5_hlist_create("flow groups", 64, 9521 false, true, sh, 9522 mlx5_flow_tunnel_grp2tbl_create_cb, 9523 mlx5_flow_tunnel_grp2tbl_match_cb, 9524 mlx5_flow_tunnel_grp2tbl_remove_cb, 9525 mlx5_flow_tunnel_grp2tbl_clone_cb, 9526 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9527 if (!thub->groups) { 9528 err = -rte_errno; 9529 goto err; 9530 } 9531 sh->tunnel_hub = thub; 9532 9533 return 0; 9534 9535 err: 9536 if (thub->groups) 9537 mlx5_hlist_destroy(thub->groups); 9538 if (thub) 9539 mlx5_free(thub); 9540 return err; 9541 } 9542 9543 static inline int 9544 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 9545 struct rte_flow_tunnel *tunnel, 9546 struct rte_flow_error *error) 9547 { 9548 struct mlx5_priv *priv = dev->data->dev_private; 9549 9550 if (!priv->sh->config.dv_flow_en) 9551 return rte_flow_error_set(error, ENOTSUP, 9552 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9553 "flow DV interface is off"); 9554 if (!is_tunnel_offload_active(dev)) 9555 return rte_flow_error_set(error, ENOTSUP, 9556 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9557 "tunnel offload was not activated"); 9558 if (!tunnel) 9559 return rte_flow_error_set(error, EINVAL, 9560 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9561 "no application tunnel"); 9562 switch (tunnel->type) { 9563 default: 9564 return rte_flow_error_set(error, EINVAL, 9565 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9566 "unsupported tunnel type"); 9567 case RTE_FLOW_ITEM_TYPE_VXLAN: 9568 case RTE_FLOW_ITEM_TYPE_GRE: 9569 case RTE_FLOW_ITEM_TYPE_NVGRE: 9570 case RTE_FLOW_ITEM_TYPE_GENEVE: 9571 break; 9572 } 9573 return 0; 9574 } 9575 9576 static int 9577 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 9578 struct rte_flow_tunnel *app_tunnel, 9579 struct rte_flow_action **actions, 9580 uint32_t *num_of_actions, 9581 struct rte_flow_error *error) 9582 { 9583 struct mlx5_flow_tunnel *tunnel; 9584 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9585 9586 if (ret) 9587 return ret; 9588 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9589 if (ret < 0) { 9590 return rte_flow_error_set(error, ret, 9591 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9592 "failed to initialize pmd tunnel"); 9593 } 9594 *actions = &tunnel->action; 9595 *num_of_actions = 1; 9596 return 0; 9597 } 9598 9599 static int 9600 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9601 struct rte_flow_tunnel *app_tunnel, 9602 struct rte_flow_item **items, 9603 uint32_t *num_of_items, 9604 struct rte_flow_error *error) 9605 { 9606 struct mlx5_flow_tunnel *tunnel; 9607 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9608 9609 if (ret) 9610 return ret; 9611 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9612 if (ret < 0) { 9613 return rte_flow_error_set(error, ret, 9614 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9615 "failed to initialize pmd tunnel"); 9616 } 9617 *items = &tunnel->item; 9618 *num_of_items = 1; 9619 return 0; 9620 } 9621 9622 struct tunnel_db_element_release_ctx { 9623 struct rte_flow_item *items; 9624 struct rte_flow_action *actions; 9625 uint32_t num_elements; 9626 struct rte_flow_error *error; 9627 int ret; 9628 }; 9629 9630 static bool 9631 tunnel_element_release_match(struct rte_eth_dev *dev, 9632 struct mlx5_flow_tunnel *tunnel, const void *x) 9633 { 9634 const struct tunnel_db_element_release_ctx *ctx = x; 9635 9636 RTE_SET_USED(dev); 9637 if (ctx->num_elements != 1) 9638 return false; 9639 else if (ctx->items) 9640 return ctx->items == &tunnel->item; 9641 else if (ctx->actions) 9642 return ctx->actions == &tunnel->action; 9643 9644 return false; 9645 } 9646 9647 static void 9648 tunnel_element_release_hit(struct rte_eth_dev *dev, 9649 struct mlx5_flow_tunnel *tunnel, void *x) 9650 { 9651 struct tunnel_db_element_release_ctx *ctx = x; 9652 ctx->ret = 0; 9653 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9654 mlx5_flow_tunnel_free(dev, tunnel); 9655 } 9656 9657 static void 9658 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9659 { 9660 struct tunnel_db_element_release_ctx *ctx = x; 9661 RTE_SET_USED(dev); 9662 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9663 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9664 "invalid argument"); 9665 } 9666 9667 static int 9668 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9669 struct rte_flow_item *pmd_items, 9670 uint32_t num_items, struct rte_flow_error *err) 9671 { 9672 struct tunnel_db_element_release_ctx ctx = { 9673 .items = pmd_items, 9674 .actions = NULL, 9675 .num_elements = num_items, 9676 .error = err, 9677 }; 9678 9679 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9680 tunnel_element_release_hit, 9681 tunnel_element_release_miss, &ctx, false); 9682 9683 return ctx.ret; 9684 } 9685 9686 static int 9687 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9688 struct rte_flow_action *pmd_actions, 9689 uint32_t num_actions, struct rte_flow_error *err) 9690 { 9691 struct tunnel_db_element_release_ctx ctx = { 9692 .items = NULL, 9693 .actions = pmd_actions, 9694 .num_elements = num_actions, 9695 .error = err, 9696 }; 9697 9698 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9699 tunnel_element_release_hit, 9700 tunnel_element_release_miss, &ctx, false); 9701 9702 return ctx.ret; 9703 } 9704 9705 static int 9706 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9707 struct rte_mbuf *m, 9708 struct rte_flow_restore_info *info, 9709 struct rte_flow_error *err) 9710 { 9711 uint64_t ol_flags = m->ol_flags; 9712 const struct mlx5_flow_tbl_data_entry *tble; 9713 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 9714 9715 if (!is_tunnel_offload_active(dev)) { 9716 info->flags = 0; 9717 return 0; 9718 } 9719 9720 if ((ol_flags & mask) != mask) 9721 goto err; 9722 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9723 if (!tble) { 9724 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9725 dev->data->port_id, m->hash.fdir.hi); 9726 goto err; 9727 } 9728 MLX5_ASSERT(tble->tunnel); 9729 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9730 info->group_id = tble->group_id; 9731 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9732 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9733 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9734 9735 return 0; 9736 9737 err: 9738 return rte_flow_error_set(err, EINVAL, 9739 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9740 "failed to get restore info"); 9741 } 9742 9743 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9744 static int 9745 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9746 __rte_unused struct rte_flow_tunnel *app_tunnel, 9747 __rte_unused struct rte_flow_action **actions, 9748 __rte_unused uint32_t *num_of_actions, 9749 __rte_unused struct rte_flow_error *error) 9750 { 9751 return -ENOTSUP; 9752 } 9753 9754 static int 9755 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9756 __rte_unused struct rte_flow_tunnel *app_tunnel, 9757 __rte_unused struct rte_flow_item **items, 9758 __rte_unused uint32_t *num_of_items, 9759 __rte_unused struct rte_flow_error *error) 9760 { 9761 return -ENOTSUP; 9762 } 9763 9764 static int 9765 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9766 __rte_unused struct rte_flow_item *pmd_items, 9767 __rte_unused uint32_t num_items, 9768 __rte_unused struct rte_flow_error *err) 9769 { 9770 return -ENOTSUP; 9771 } 9772 9773 static int 9774 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9775 __rte_unused struct rte_flow_action *pmd_action, 9776 __rte_unused uint32_t num_actions, 9777 __rte_unused struct rte_flow_error *err) 9778 { 9779 return -ENOTSUP; 9780 } 9781 9782 static int 9783 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9784 __rte_unused struct rte_mbuf *m, 9785 __rte_unused struct rte_flow_restore_info *i, 9786 __rte_unused struct rte_flow_error *err) 9787 { 9788 return -ENOTSUP; 9789 } 9790 9791 static int 9792 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9793 __rte_unused struct rte_flow *flow, 9794 __rte_unused const struct rte_flow_attr *attr, 9795 __rte_unused const struct rte_flow_action *actions, 9796 __rte_unused uint32_t flow_idx, 9797 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9798 __rte_unused struct tunnel_default_miss_ctx *ctx, 9799 __rte_unused struct rte_flow_error *error) 9800 { 9801 return -ENOTSUP; 9802 } 9803 9804 static struct mlx5_flow_tunnel * 9805 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9806 __rte_unused uint32_t id) 9807 { 9808 return NULL; 9809 } 9810 9811 static void 9812 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9813 __rte_unused struct mlx5_flow_tunnel *tunnel) 9814 { 9815 } 9816 9817 static uint32_t 9818 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9819 __rte_unused const struct mlx5_flow_tunnel *t, 9820 __rte_unused uint32_t group, 9821 __rte_unused uint32_t *table, 9822 struct rte_flow_error *error) 9823 { 9824 return rte_flow_error_set(error, ENOTSUP, 9825 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9826 "tunnel offload requires DV support"); 9827 } 9828 9829 void 9830 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9831 __rte_unused uint16_t port_id) 9832 { 9833 } 9834 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9835 9836 /* Flex flow item API */ 9837 static struct rte_flow_item_flex_handle * 9838 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 9839 const struct rte_flow_item_flex_conf *conf, 9840 struct rte_flow_error *error) 9841 { 9842 static const char err_msg[] = "flex item creation unsupported"; 9843 struct rte_flow_attr attr = { .transfer = 0 }; 9844 const struct mlx5_flow_driver_ops *fops = 9845 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9846 9847 if (!fops->item_create) { 9848 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9849 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9850 NULL, err_msg); 9851 return NULL; 9852 } 9853 return fops->item_create(dev, conf, error); 9854 } 9855 9856 static int 9857 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 9858 const struct rte_flow_item_flex_handle *handle, 9859 struct rte_flow_error *error) 9860 { 9861 static const char err_msg[] = "flex item release unsupported"; 9862 struct rte_flow_attr attr = { .transfer = 0 }; 9863 const struct mlx5_flow_driver_ops *fops = 9864 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9865 9866 if (!fops->item_release) { 9867 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9868 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9869 NULL, err_msg); 9870 return -rte_errno; 9871 } 9872 return fops->item_release(dev, handle, error); 9873 } 9874 9875 static void 9876 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9877 { 9878 int ret; 9879 struct rte_flow_error error; 9880 9881 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9882 char *item_name; 9883 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9884 sizeof(item_name), 9885 (void *)(uintptr_t)item->type, &error); 9886 if (ret > 0) 9887 printf("%s ", item_name); 9888 else 9889 printf("%d\n", (int)item->type); 9890 } 9891 printf("END\n"); 9892 } 9893 9894 static int 9895 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 9896 { 9897 const struct rte_flow_item_udp *spec = udp_item->spec; 9898 const struct rte_flow_item_udp *mask = udp_item->mask; 9899 uint16_t udp_dport = 0; 9900 9901 if (spec != NULL) { 9902 if (!mask) 9903 mask = &rte_flow_item_udp_mask; 9904 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 9905 mask->hdr.dst_port); 9906 } 9907 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 9908 } 9909 9910 static const struct mlx5_flow_expand_node * 9911 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 9912 unsigned int item_idx, 9913 const struct mlx5_flow_expand_node graph[], 9914 const struct mlx5_flow_expand_node *node) 9915 { 9916 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 9917 9918 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 9919 node != NULL && 9920 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 9921 /* 9922 * The expansion node is VXLAN and it is also the last 9923 * expandable item in the pattern, so need to continue 9924 * expansion of the inner tunnel. 9925 */ 9926 MLX5_ASSERT(item_idx > 0); 9927 prev_item = pattern + item_idx - 1; 9928 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 9929 if (mlx5_flow_is_std_vxlan_port(prev_item)) 9930 return &graph[MLX5_EXPANSION_STD_VXLAN]; 9931 return &graph[MLX5_EXPANSION_L3_VXLAN]; 9932 } 9933 return node; 9934 } 9935 9936 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 9937 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 9938 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 9939 }; 9940 9941 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 9942 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 9943 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 9944 { 9, 10, 11 }, { 12, 13, 14 }, 9945 }; 9946 9947 /** 9948 * Discover the number of available flow priorities. 9949 * 9950 * @param dev 9951 * Ethernet device. 9952 * 9953 * @return 9954 * On success, number of available flow priorities. 9955 * On failure, a negative errno-style code and rte_errno is set. 9956 */ 9957 int 9958 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 9959 { 9960 static const uint16_t vprio[] = {8, 16}; 9961 const struct mlx5_priv *priv = dev->data->dev_private; 9962 const struct mlx5_flow_driver_ops *fops; 9963 enum mlx5_flow_drv_type type; 9964 int ret; 9965 9966 type = mlx5_flow_os_get_type(); 9967 if (type == MLX5_FLOW_TYPE_MAX) { 9968 type = MLX5_FLOW_TYPE_VERBS; 9969 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 9970 type = MLX5_FLOW_TYPE_DV; 9971 } 9972 fops = flow_get_drv_ops(type); 9973 if (fops->discover_priorities == NULL) { 9974 DRV_LOG(ERR, "Priority discovery not supported"); 9975 rte_errno = ENOTSUP; 9976 return -rte_errno; 9977 } 9978 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 9979 if (ret < 0) 9980 return ret; 9981 switch (ret) { 9982 case 8: 9983 ret = RTE_DIM(priority_map_3); 9984 break; 9985 case 16: 9986 ret = RTE_DIM(priority_map_5); 9987 break; 9988 default: 9989 rte_errno = ENOTSUP; 9990 DRV_LOG(ERR, 9991 "port %u maximum priority: %d expected 8/16", 9992 dev->data->port_id, ret); 9993 return -rte_errno; 9994 } 9995 DRV_LOG(INFO, "port %u supported flow priorities:" 9996 " 0-%d for ingress or egress root table," 9997 " 0-%d for non-root table or transfer root table.", 9998 dev->data->port_id, ret - 2, 9999 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 10000 return ret; 10001 } 10002 10003 /** 10004 * Adjust flow priority based on the highest layer and the request priority. 10005 * 10006 * @param[in] dev 10007 * Pointer to the Ethernet device structure. 10008 * @param[in] priority 10009 * The rule base priority. 10010 * @param[in] subpriority 10011 * The priority based on the items. 10012 * 10013 * @return 10014 * The new priority. 10015 */ 10016 uint32_t 10017 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10018 uint32_t subpriority) 10019 { 10020 uint32_t res = 0; 10021 struct mlx5_priv *priv = dev->data->dev_private; 10022 10023 switch (priv->sh->flow_max_priority) { 10024 case RTE_DIM(priority_map_3): 10025 res = priority_map_3[priority][subpriority]; 10026 break; 10027 case RTE_DIM(priority_map_5): 10028 res = priority_map_5[priority][subpriority]; 10029 break; 10030 } 10031 return res; 10032 } 10033