1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see RTE_ETH_RSS_* definitions). 102 */ 103 uint64_t node_flags; 104 /**< 105 * Bit-fields that define how the node is used in the expansion. 106 * (see MLX5_EXPANSION_NODE_* definitions). 107 */ 108 }; 109 110 /* Optional expand field. The expansion alg will not go deeper. */ 111 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 112 113 /* The node is not added implicitly as expansion to the flow pattern. 114 * If the node type does not match the flow pattern item type, the 115 * expansion alg will go deeper to its next items. 116 * In the current implementation, the list of next nodes indexes can 117 * have up to one node with this flag set and it has to be the last 118 * node index (before the list terminator). 119 */ 120 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 121 122 /** Object returned by mlx5_flow_expand_rss(). */ 123 struct mlx5_flow_expand_rss { 124 uint32_t entries; 125 /**< Number of entries @p patterns and @p priorities. */ 126 struct { 127 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 128 uint32_t priority; /**< Priority offset for each expansion. */ 129 } entry[]; 130 }; 131 132 static void 133 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 134 135 static const struct mlx5_flow_expand_node * 136 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 137 unsigned int item_idx, 138 const struct mlx5_flow_expand_node graph[], 139 const struct mlx5_flow_expand_node *node); 140 141 static bool 142 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 143 { 144 switch (item->type) { 145 case RTE_FLOW_ITEM_TYPE_ETH: 146 case RTE_FLOW_ITEM_TYPE_VLAN: 147 case RTE_FLOW_ITEM_TYPE_IPV4: 148 case RTE_FLOW_ITEM_TYPE_IPV6: 149 case RTE_FLOW_ITEM_TYPE_UDP: 150 case RTE_FLOW_ITEM_TYPE_TCP: 151 case RTE_FLOW_ITEM_TYPE_VXLAN: 152 case RTE_FLOW_ITEM_TYPE_NVGRE: 153 case RTE_FLOW_ITEM_TYPE_GRE: 154 case RTE_FLOW_ITEM_TYPE_GENEVE: 155 case RTE_FLOW_ITEM_TYPE_MPLS: 156 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 157 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 158 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 159 case RTE_FLOW_ITEM_TYPE_GTP: 160 return true; 161 default: 162 break; 163 } 164 return false; 165 } 166 167 /** 168 * Network Service Header (NSH) and its next protocol values 169 * are described in RFC-8393. 170 */ 171 static enum rte_flow_item_type 172 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 173 { 174 enum rte_flow_item_type type; 175 176 switch (proto_mask & proto_spec) { 177 case RTE_VXLAN_GPE_TYPE_IPV4: 178 type = RTE_FLOW_ITEM_TYPE_IPV4; 179 break; 180 case RTE_VXLAN_GPE_TYPE_IPV6: 181 type = RTE_VXLAN_GPE_TYPE_IPV6; 182 break; 183 case RTE_VXLAN_GPE_TYPE_ETH: 184 type = RTE_FLOW_ITEM_TYPE_ETH; 185 break; 186 default: 187 type = RTE_FLOW_ITEM_TYPE_END; 188 } 189 return type; 190 } 191 192 static enum rte_flow_item_type 193 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 194 { 195 enum rte_flow_item_type type; 196 197 switch (proto_mask & proto_spec) { 198 case IPPROTO_UDP: 199 type = RTE_FLOW_ITEM_TYPE_UDP; 200 break; 201 case IPPROTO_TCP: 202 type = RTE_FLOW_ITEM_TYPE_TCP; 203 break; 204 case IPPROTO_IP: 205 type = RTE_FLOW_ITEM_TYPE_IPV4; 206 break; 207 case IPPROTO_IPV6: 208 type = RTE_FLOW_ITEM_TYPE_IPV6; 209 break; 210 default: 211 type = RTE_FLOW_ITEM_TYPE_END; 212 } 213 return type; 214 } 215 216 static enum rte_flow_item_type 217 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 218 rte_be16_t type_mask, bool is_tunnel) 219 { 220 enum rte_flow_item_type type; 221 222 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 223 case RTE_ETHER_TYPE_TEB: 224 type = is_tunnel ? 225 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 226 break; 227 case RTE_ETHER_TYPE_VLAN: 228 type = !is_tunnel ? 229 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 230 break; 231 case RTE_ETHER_TYPE_IPV4: 232 type = RTE_FLOW_ITEM_TYPE_IPV4; 233 break; 234 case RTE_ETHER_TYPE_IPV6: 235 type = RTE_FLOW_ITEM_TYPE_IPV6; 236 break; 237 default: 238 type = RTE_FLOW_ITEM_TYPE_END; 239 } 240 return type; 241 } 242 243 static enum rte_flow_item_type 244 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 245 { 246 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 247 do { \ 248 const void *m = item->mask; \ 249 const void *s = item->spec; \ 250 mask = m ? \ 251 ((const struct rte_flow_item_##type *)m)->fld : \ 252 rte_flow_item_##type##_mask.fld; \ 253 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 254 } while (0) 255 256 enum rte_flow_item_type ret; 257 uint16_t spec, mask; 258 259 if (item == NULL || item->spec == NULL) 260 return RTE_FLOW_ITEM_TYPE_VOID; 261 switch (item->type) { 262 case RTE_FLOW_ITEM_TYPE_ETH: 263 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 264 if (!mask) 265 return RTE_FLOW_ITEM_TYPE_VOID; 266 ret = mlx5_ethertype_to_item_type(spec, mask, false); 267 break; 268 case RTE_FLOW_ITEM_TYPE_VLAN: 269 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 270 if (!mask) 271 return RTE_FLOW_ITEM_TYPE_VOID; 272 ret = mlx5_ethertype_to_item_type(spec, mask, false); 273 break; 274 case RTE_FLOW_ITEM_TYPE_IPV4: 275 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 276 if (!mask) 277 return RTE_FLOW_ITEM_TYPE_VOID; 278 ret = mlx5_inet_proto_to_item_type(spec, mask); 279 break; 280 case RTE_FLOW_ITEM_TYPE_IPV6: 281 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 282 if (!mask) 283 return RTE_FLOW_ITEM_TYPE_VOID; 284 ret = mlx5_inet_proto_to_item_type(spec, mask); 285 break; 286 case RTE_FLOW_ITEM_TYPE_GENEVE: 287 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 288 ret = mlx5_ethertype_to_item_type(spec, mask, true); 289 break; 290 case RTE_FLOW_ITEM_TYPE_GRE: 291 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 292 ret = mlx5_ethertype_to_item_type(spec, mask, true); 293 break; 294 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 295 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 296 ret = mlx5_nsh_proto_to_item_type(spec, mask); 297 break; 298 default: 299 ret = RTE_FLOW_ITEM_TYPE_VOID; 300 break; 301 } 302 return ret; 303 #undef MLX5_XSET_ITEM_MASK_SPEC 304 } 305 306 static const int * 307 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 308 const int *next_node) 309 { 310 const struct mlx5_flow_expand_node *node = NULL; 311 const int *next = next_node; 312 313 while (next && *next) { 314 /* 315 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 316 * flag set, because they were not found in the flow pattern. 317 */ 318 node = &graph[*next]; 319 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 320 break; 321 next = node->next; 322 } 323 return next; 324 } 325 326 #define MLX5_RSS_EXP_ELT_N 16 327 328 /** 329 * Expand RSS flows into several possible flows according to the RSS hash 330 * fields requested and the driver capabilities. 331 * 332 * @param[out] buf 333 * Buffer to store the result expansion. 334 * @param[in] size 335 * Buffer size in bytes. If 0, @p buf can be NULL. 336 * @param[in] pattern 337 * User flow pattern. 338 * @param[in] types 339 * RSS types to expand (see RTE_ETH_RSS_* definitions). 340 * @param[in] graph 341 * Input graph to expand @p pattern according to @p types. 342 * @param[in] graph_root_index 343 * Index of root node in @p graph, typically 0. 344 * 345 * @return 346 * A positive value representing the size of @p buf in bytes regardless of 347 * @p size on success, a negative errno value otherwise and rte_errno is 348 * set, the following errors are defined: 349 * 350 * -E2BIG: graph-depth @p graph is too deep. 351 * -EINVAL: @p size has not enough space for expanded pattern. 352 */ 353 static int 354 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 355 const struct rte_flow_item *pattern, uint64_t types, 356 const struct mlx5_flow_expand_node graph[], 357 int graph_root_index) 358 { 359 const struct rte_flow_item *item; 360 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 361 const int *next_node; 362 const int *stack[MLX5_RSS_EXP_ELT_N]; 363 int stack_pos = 0; 364 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 365 unsigned int i, item_idx, last_expand_item_idx = 0; 366 size_t lsize; 367 size_t user_pattern_size = 0; 368 void *addr = NULL; 369 const struct mlx5_flow_expand_node *next = NULL; 370 struct rte_flow_item missed_item; 371 int missed = 0; 372 int elt = 0; 373 const struct rte_flow_item *last_expand_item = NULL; 374 375 memset(&missed_item, 0, sizeof(missed_item)); 376 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 377 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 378 if (lsize > size) 379 return -EINVAL; 380 buf->entry[0].priority = 0; 381 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 382 buf->entries = 0; 383 addr = buf->entry[0].pattern; 384 for (item = pattern, item_idx = 0; 385 item->type != RTE_FLOW_ITEM_TYPE_END; 386 item++, item_idx++) { 387 if (!mlx5_flow_is_rss_expandable_item(item)) { 388 user_pattern_size += sizeof(*item); 389 continue; 390 } 391 last_expand_item = item; 392 last_expand_item_idx = item_idx; 393 i = 0; 394 while (node->next && node->next[i]) { 395 next = &graph[node->next[i]]; 396 if (next->type == item->type) 397 break; 398 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 399 node = next; 400 i = 0; 401 } else { 402 ++i; 403 } 404 } 405 if (next) 406 node = next; 407 user_pattern_size += sizeof(*item); 408 } 409 user_pattern_size += sizeof(*item); /* Handle END item. */ 410 lsize += user_pattern_size; 411 if (lsize > size) 412 return -EINVAL; 413 /* Copy the user pattern in the first entry of the buffer. */ 414 rte_memcpy(addr, pattern, user_pattern_size); 415 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 416 buf->entries = 1; 417 /* Start expanding. */ 418 memset(flow_items, 0, sizeof(flow_items)); 419 user_pattern_size -= sizeof(*item); 420 /* 421 * Check if the last valid item has spec set, need complete pattern, 422 * and the pattern can be used for expansion. 423 */ 424 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 425 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 426 /* Item type END indicates expansion is not required. */ 427 return lsize; 428 } 429 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 430 next = NULL; 431 missed = 1; 432 i = 0; 433 while (node->next && node->next[i]) { 434 next = &graph[node->next[i]]; 435 if (next->type == missed_item.type) { 436 flow_items[0].type = missed_item.type; 437 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 438 break; 439 } 440 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 441 node = next; 442 i = 0; 443 } else { 444 ++i; 445 } 446 next = NULL; 447 } 448 } 449 if (next && missed) { 450 elt = 2; /* missed item + item end. */ 451 node = next; 452 lsize += elt * sizeof(*item) + user_pattern_size; 453 if (lsize > size) 454 return -EINVAL; 455 if (node->rss_types & types) { 456 buf->entry[buf->entries].priority = 1; 457 buf->entry[buf->entries].pattern = addr; 458 buf->entries++; 459 rte_memcpy(addr, buf->entry[0].pattern, 460 user_pattern_size); 461 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 462 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 463 addr = (void *)(((uintptr_t)addr) + 464 elt * sizeof(*item)); 465 } 466 } else if (last_expand_item != NULL) { 467 node = mlx5_flow_expand_rss_adjust_node(pattern, 468 last_expand_item_idx, graph, node); 469 } 470 memset(flow_items, 0, sizeof(flow_items)); 471 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 472 node->next); 473 stack[stack_pos] = next_node; 474 node = next_node ? &graph[*next_node] : NULL; 475 while (node) { 476 flow_items[stack_pos].type = node->type; 477 if (node->rss_types & types) { 478 size_t n; 479 /* 480 * compute the number of items to copy from the 481 * expansion and copy it. 482 * When the stack_pos is 0, there are 1 element in it, 483 * plus the addition END item. 484 */ 485 elt = stack_pos + 2; 486 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 487 lsize += elt * sizeof(*item) + user_pattern_size; 488 if (lsize > size) 489 return -EINVAL; 490 n = elt * sizeof(*item); 491 buf->entry[buf->entries].priority = 492 stack_pos + 1 + missed; 493 buf->entry[buf->entries].pattern = addr; 494 buf->entries++; 495 rte_memcpy(addr, buf->entry[0].pattern, 496 user_pattern_size); 497 addr = (void *)(((uintptr_t)addr) + 498 user_pattern_size); 499 rte_memcpy(addr, &missed_item, 500 missed * sizeof(*item)); 501 addr = (void *)(((uintptr_t)addr) + 502 missed * sizeof(*item)); 503 rte_memcpy(addr, flow_items, n); 504 addr = (void *)(((uintptr_t)addr) + n); 505 } 506 /* Go deeper. */ 507 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 508 node->next) { 509 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 510 node->next); 511 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 512 rte_errno = E2BIG; 513 return -rte_errno; 514 } 515 stack[stack_pos] = next_node; 516 } else if (*(next_node + 1)) { 517 /* Follow up with the next possibility. */ 518 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 519 ++next_node); 520 } else if (!stack_pos) { 521 /* 522 * Completing the traverse over the different paths. 523 * The next_node is advanced to the terminator. 524 */ 525 ++next_node; 526 } else { 527 /* Move to the next path. */ 528 while (stack_pos) { 529 next_node = stack[--stack_pos]; 530 next_node++; 531 if (*next_node) 532 break; 533 } 534 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 535 next_node); 536 stack[stack_pos] = next_node; 537 } 538 node = next_node && *next_node ? &graph[*next_node] : NULL; 539 }; 540 return lsize; 541 } 542 543 enum mlx5_expansion { 544 MLX5_EXPANSION_ROOT, 545 MLX5_EXPANSION_ROOT_OUTER, 546 MLX5_EXPANSION_OUTER_ETH, 547 MLX5_EXPANSION_OUTER_VLAN, 548 MLX5_EXPANSION_OUTER_IPV4, 549 MLX5_EXPANSION_OUTER_IPV4_UDP, 550 MLX5_EXPANSION_OUTER_IPV4_TCP, 551 MLX5_EXPANSION_OUTER_IPV6, 552 MLX5_EXPANSION_OUTER_IPV6_UDP, 553 MLX5_EXPANSION_OUTER_IPV6_TCP, 554 MLX5_EXPANSION_VXLAN, 555 MLX5_EXPANSION_STD_VXLAN, 556 MLX5_EXPANSION_L3_VXLAN, 557 MLX5_EXPANSION_VXLAN_GPE, 558 MLX5_EXPANSION_GRE, 559 MLX5_EXPANSION_NVGRE, 560 MLX5_EXPANSION_GRE_KEY, 561 MLX5_EXPANSION_MPLS, 562 MLX5_EXPANSION_ETH, 563 MLX5_EXPANSION_VLAN, 564 MLX5_EXPANSION_IPV4, 565 MLX5_EXPANSION_IPV4_UDP, 566 MLX5_EXPANSION_IPV4_TCP, 567 MLX5_EXPANSION_IPV6, 568 MLX5_EXPANSION_IPV6_UDP, 569 MLX5_EXPANSION_IPV6_TCP, 570 MLX5_EXPANSION_IPV6_FRAG_EXT, 571 MLX5_EXPANSION_GTP, 572 MLX5_EXPANSION_GENEVE, 573 }; 574 575 /** Supported expansion of items. */ 576 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 577 [MLX5_EXPANSION_ROOT] = { 578 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 579 MLX5_EXPANSION_IPV4, 580 MLX5_EXPANSION_IPV6), 581 .type = RTE_FLOW_ITEM_TYPE_END, 582 }, 583 [MLX5_EXPANSION_ROOT_OUTER] = { 584 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 585 MLX5_EXPANSION_OUTER_IPV4, 586 MLX5_EXPANSION_OUTER_IPV6), 587 .type = RTE_FLOW_ITEM_TYPE_END, 588 }, 589 [MLX5_EXPANSION_OUTER_ETH] = { 590 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 591 .type = RTE_FLOW_ITEM_TYPE_ETH, 592 .rss_types = 0, 593 }, 594 [MLX5_EXPANSION_OUTER_VLAN] = { 595 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 596 MLX5_EXPANSION_OUTER_IPV6), 597 .type = RTE_FLOW_ITEM_TYPE_VLAN, 598 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 599 }, 600 [MLX5_EXPANSION_OUTER_IPV4] = { 601 .next = MLX5_FLOW_EXPAND_RSS_NEXT 602 (MLX5_EXPANSION_OUTER_IPV4_UDP, 603 MLX5_EXPANSION_OUTER_IPV4_TCP, 604 MLX5_EXPANSION_GRE, 605 MLX5_EXPANSION_NVGRE, 606 MLX5_EXPANSION_IPV4, 607 MLX5_EXPANSION_IPV6), 608 .type = RTE_FLOW_ITEM_TYPE_IPV4, 609 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 610 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 611 }, 612 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 613 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 614 MLX5_EXPANSION_VXLAN_GPE, 615 MLX5_EXPANSION_MPLS, 616 MLX5_EXPANSION_GENEVE, 617 MLX5_EXPANSION_GTP), 618 .type = RTE_FLOW_ITEM_TYPE_UDP, 619 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 620 }, 621 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 622 .type = RTE_FLOW_ITEM_TYPE_TCP, 623 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 624 }, 625 [MLX5_EXPANSION_OUTER_IPV6] = { 626 .next = MLX5_FLOW_EXPAND_RSS_NEXT 627 (MLX5_EXPANSION_OUTER_IPV6_UDP, 628 MLX5_EXPANSION_OUTER_IPV6_TCP, 629 MLX5_EXPANSION_IPV4, 630 MLX5_EXPANSION_IPV6, 631 MLX5_EXPANSION_GRE, 632 MLX5_EXPANSION_NVGRE), 633 .type = RTE_FLOW_ITEM_TYPE_IPV6, 634 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 635 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 636 }, 637 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 638 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 639 MLX5_EXPANSION_VXLAN_GPE, 640 MLX5_EXPANSION_MPLS, 641 MLX5_EXPANSION_GENEVE, 642 MLX5_EXPANSION_GTP), 643 .type = RTE_FLOW_ITEM_TYPE_UDP, 644 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 645 }, 646 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 647 .type = RTE_FLOW_ITEM_TYPE_TCP, 648 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 649 }, 650 [MLX5_EXPANSION_VXLAN] = { 651 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 652 MLX5_EXPANSION_IPV4, 653 MLX5_EXPANSION_IPV6), 654 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 655 }, 656 [MLX5_EXPANSION_STD_VXLAN] = { 657 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 658 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 659 }, 660 [MLX5_EXPANSION_L3_VXLAN] = { 661 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 662 MLX5_EXPANSION_IPV6), 663 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 664 }, 665 [MLX5_EXPANSION_VXLAN_GPE] = { 666 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 667 MLX5_EXPANSION_IPV4, 668 MLX5_EXPANSION_IPV6), 669 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 670 }, 671 [MLX5_EXPANSION_GRE] = { 672 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 673 MLX5_EXPANSION_IPV4, 674 MLX5_EXPANSION_IPV6, 675 MLX5_EXPANSION_GRE_KEY, 676 MLX5_EXPANSION_MPLS), 677 .type = RTE_FLOW_ITEM_TYPE_GRE, 678 }, 679 [MLX5_EXPANSION_GRE_KEY] = { 680 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 681 MLX5_EXPANSION_IPV6, 682 MLX5_EXPANSION_MPLS), 683 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 684 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 685 }, 686 [MLX5_EXPANSION_NVGRE] = { 687 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 688 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 689 }, 690 [MLX5_EXPANSION_MPLS] = { 691 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 692 MLX5_EXPANSION_IPV6, 693 MLX5_EXPANSION_ETH), 694 .type = RTE_FLOW_ITEM_TYPE_MPLS, 695 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 696 }, 697 [MLX5_EXPANSION_ETH] = { 698 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 699 .type = RTE_FLOW_ITEM_TYPE_ETH, 700 }, 701 [MLX5_EXPANSION_VLAN] = { 702 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 703 MLX5_EXPANSION_IPV6), 704 .type = RTE_FLOW_ITEM_TYPE_VLAN, 705 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 706 }, 707 [MLX5_EXPANSION_IPV4] = { 708 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 709 MLX5_EXPANSION_IPV4_TCP), 710 .type = RTE_FLOW_ITEM_TYPE_IPV4, 711 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 712 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 713 }, 714 [MLX5_EXPANSION_IPV4_UDP] = { 715 .type = RTE_FLOW_ITEM_TYPE_UDP, 716 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 717 }, 718 [MLX5_EXPANSION_IPV4_TCP] = { 719 .type = RTE_FLOW_ITEM_TYPE_TCP, 720 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 721 }, 722 [MLX5_EXPANSION_IPV6] = { 723 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 724 MLX5_EXPANSION_IPV6_TCP, 725 MLX5_EXPANSION_IPV6_FRAG_EXT), 726 .type = RTE_FLOW_ITEM_TYPE_IPV6, 727 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 728 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 729 }, 730 [MLX5_EXPANSION_IPV6_UDP] = { 731 .type = RTE_FLOW_ITEM_TYPE_UDP, 732 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 733 }, 734 [MLX5_EXPANSION_IPV6_TCP] = { 735 .type = RTE_FLOW_ITEM_TYPE_TCP, 736 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 737 }, 738 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 739 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 740 }, 741 [MLX5_EXPANSION_GTP] = { 742 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 743 MLX5_EXPANSION_IPV6), 744 .type = RTE_FLOW_ITEM_TYPE_GTP, 745 }, 746 [MLX5_EXPANSION_GENEVE] = { 747 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 748 MLX5_EXPANSION_IPV4, 749 MLX5_EXPANSION_IPV6), 750 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 751 }, 752 }; 753 754 static struct rte_flow_action_handle * 755 mlx5_action_handle_create(struct rte_eth_dev *dev, 756 const struct rte_flow_indir_action_conf *conf, 757 const struct rte_flow_action *action, 758 struct rte_flow_error *error); 759 static int mlx5_action_handle_destroy 760 (struct rte_eth_dev *dev, 761 struct rte_flow_action_handle *handle, 762 struct rte_flow_error *error); 763 static int mlx5_action_handle_update 764 (struct rte_eth_dev *dev, 765 struct rte_flow_action_handle *handle, 766 const void *update, 767 struct rte_flow_error *error); 768 static int mlx5_action_handle_query 769 (struct rte_eth_dev *dev, 770 const struct rte_flow_action_handle *handle, 771 void *data, 772 struct rte_flow_error *error); 773 static int 774 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 775 struct rte_flow_tunnel *app_tunnel, 776 struct rte_flow_action **actions, 777 uint32_t *num_of_actions, 778 struct rte_flow_error *error); 779 static int 780 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 781 struct rte_flow_tunnel *app_tunnel, 782 struct rte_flow_item **items, 783 uint32_t *num_of_items, 784 struct rte_flow_error *error); 785 static int 786 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 787 struct rte_flow_item *pmd_items, 788 uint32_t num_items, struct rte_flow_error *err); 789 static int 790 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 791 struct rte_flow_action *pmd_actions, 792 uint32_t num_actions, 793 struct rte_flow_error *err); 794 static int 795 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 796 struct rte_mbuf *m, 797 struct rte_flow_restore_info *info, 798 struct rte_flow_error *err); 799 static struct rte_flow_item_flex_handle * 800 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 801 const struct rte_flow_item_flex_conf *conf, 802 struct rte_flow_error *error); 803 static int 804 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 805 const struct rte_flow_item_flex_handle *handle, 806 struct rte_flow_error *error); 807 808 static const struct rte_flow_ops mlx5_flow_ops = { 809 .validate = mlx5_flow_validate, 810 .create = mlx5_flow_create, 811 .destroy = mlx5_flow_destroy, 812 .flush = mlx5_flow_flush, 813 .isolate = mlx5_flow_isolate, 814 .query = mlx5_flow_query, 815 .dev_dump = mlx5_flow_dev_dump, 816 .get_aged_flows = mlx5_flow_get_aged_flows, 817 .action_handle_create = mlx5_action_handle_create, 818 .action_handle_destroy = mlx5_action_handle_destroy, 819 .action_handle_update = mlx5_action_handle_update, 820 .action_handle_query = mlx5_action_handle_query, 821 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 822 .tunnel_match = mlx5_flow_tunnel_match, 823 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 824 .tunnel_item_release = mlx5_flow_tunnel_item_release, 825 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 826 .flex_item_create = mlx5_flow_flex_item_create, 827 .flex_item_release = mlx5_flow_flex_item_release, 828 }; 829 830 /* Tunnel information. */ 831 struct mlx5_flow_tunnel_info { 832 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 833 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 834 }; 835 836 static struct mlx5_flow_tunnel_info tunnels_info[] = { 837 { 838 .tunnel = MLX5_FLOW_LAYER_VXLAN, 839 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 840 }, 841 { 842 .tunnel = MLX5_FLOW_LAYER_GENEVE, 843 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 844 }, 845 { 846 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 847 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 848 }, 849 { 850 .tunnel = MLX5_FLOW_LAYER_GRE, 851 .ptype = RTE_PTYPE_TUNNEL_GRE, 852 }, 853 { 854 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 855 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 856 }, 857 { 858 .tunnel = MLX5_FLOW_LAYER_MPLS, 859 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 860 }, 861 { 862 .tunnel = MLX5_FLOW_LAYER_NVGRE, 863 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 864 }, 865 { 866 .tunnel = MLX5_FLOW_LAYER_IPIP, 867 .ptype = RTE_PTYPE_TUNNEL_IP, 868 }, 869 { 870 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 871 .ptype = RTE_PTYPE_TUNNEL_IP, 872 }, 873 { 874 .tunnel = MLX5_FLOW_LAYER_GTP, 875 .ptype = RTE_PTYPE_TUNNEL_GTPU, 876 }, 877 }; 878 879 880 881 /** 882 * Translate tag ID to register. 883 * 884 * @param[in] dev 885 * Pointer to the Ethernet device structure. 886 * @param[in] feature 887 * The feature that request the register. 888 * @param[in] id 889 * The request register ID. 890 * @param[out] error 891 * Error description in case of any. 892 * 893 * @return 894 * The request register on success, a negative errno 895 * value otherwise and rte_errno is set. 896 */ 897 int 898 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 899 enum mlx5_feature_name feature, 900 uint32_t id, 901 struct rte_flow_error *error) 902 { 903 struct mlx5_priv *priv = dev->data->dev_private; 904 struct mlx5_dev_config *config = &priv->config; 905 enum modify_reg start_reg; 906 bool skip_mtr_reg = false; 907 908 switch (feature) { 909 case MLX5_HAIRPIN_RX: 910 return REG_B; 911 case MLX5_HAIRPIN_TX: 912 return REG_A; 913 case MLX5_METADATA_RX: 914 switch (config->dv_xmeta_en) { 915 case MLX5_XMETA_MODE_LEGACY: 916 return REG_B; 917 case MLX5_XMETA_MODE_META16: 918 return REG_C_0; 919 case MLX5_XMETA_MODE_META32: 920 return REG_C_1; 921 } 922 break; 923 case MLX5_METADATA_TX: 924 return REG_A; 925 case MLX5_METADATA_FDB: 926 switch (config->dv_xmeta_en) { 927 case MLX5_XMETA_MODE_LEGACY: 928 return REG_NON; 929 case MLX5_XMETA_MODE_META16: 930 return REG_C_0; 931 case MLX5_XMETA_MODE_META32: 932 return REG_C_1; 933 } 934 break; 935 case MLX5_FLOW_MARK: 936 switch (config->dv_xmeta_en) { 937 case MLX5_XMETA_MODE_LEGACY: 938 return REG_NON; 939 case MLX5_XMETA_MODE_META16: 940 return REG_C_1; 941 case MLX5_XMETA_MODE_META32: 942 return REG_C_0; 943 } 944 break; 945 case MLX5_MTR_ID: 946 /* 947 * If meter color and meter id share one register, flow match 948 * should use the meter color register for match. 949 */ 950 if (priv->mtr_reg_share) 951 return priv->mtr_color_reg; 952 else 953 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 954 REG_C_3; 955 case MLX5_MTR_COLOR: 956 case MLX5_ASO_FLOW_HIT: 957 case MLX5_ASO_CONNTRACK: 958 case MLX5_SAMPLE_ID: 959 /* All features use the same REG_C. */ 960 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 961 return priv->mtr_color_reg; 962 case MLX5_COPY_MARK: 963 /* 964 * Metadata COPY_MARK register using is in meter suffix sub 965 * flow while with meter. It's safe to share the same register. 966 */ 967 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 968 case MLX5_APP_TAG: 969 /* 970 * If meter is enable, it will engage the register for color 971 * match and flow match. If meter color match is not using the 972 * REG_C_2, need to skip the REG_C_x be used by meter color 973 * match. 974 * If meter is disable, free to use all available registers. 975 */ 976 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 977 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 978 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 979 if (id > (uint32_t)(REG_C_7 - start_reg)) 980 return rte_flow_error_set(error, EINVAL, 981 RTE_FLOW_ERROR_TYPE_ITEM, 982 NULL, "invalid tag id"); 983 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 984 return rte_flow_error_set(error, ENOTSUP, 985 RTE_FLOW_ERROR_TYPE_ITEM, 986 NULL, "unsupported tag id"); 987 /* 988 * This case means meter is using the REG_C_x great than 2. 989 * Take care not to conflict with meter color REG_C_x. 990 * If the available index REG_C_y >= REG_C_x, skip the 991 * color register. 992 */ 993 if (skip_mtr_reg && priv->sh->flow_mreg_c 994 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 995 if (id >= (uint32_t)(REG_C_7 - start_reg)) 996 return rte_flow_error_set(error, EINVAL, 997 RTE_FLOW_ERROR_TYPE_ITEM, 998 NULL, "invalid tag id"); 999 if (priv->sh->flow_mreg_c 1000 [id + 1 + start_reg - REG_C_0] != REG_NON) 1001 return priv->sh->flow_mreg_c 1002 [id + 1 + start_reg - REG_C_0]; 1003 return rte_flow_error_set(error, ENOTSUP, 1004 RTE_FLOW_ERROR_TYPE_ITEM, 1005 NULL, "unsupported tag id"); 1006 } 1007 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1008 } 1009 MLX5_ASSERT(false); 1010 return rte_flow_error_set(error, EINVAL, 1011 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1012 NULL, "invalid feature name"); 1013 } 1014 1015 /** 1016 * Check extensive flow metadata register support. 1017 * 1018 * @param dev 1019 * Pointer to rte_eth_dev structure. 1020 * 1021 * @return 1022 * True if device supports extensive flow metadata register, otherwise false. 1023 */ 1024 bool 1025 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1026 { 1027 struct mlx5_priv *priv = dev->data->dev_private; 1028 1029 /* 1030 * Having available reg_c can be regarded inclusively as supporting 1031 * extensive flow metadata register, which could mean, 1032 * - metadata register copy action by modify header. 1033 * - 16 modify header actions is supported. 1034 * - reg_c's are preserved across different domain (FDB and NIC) on 1035 * packet loopback by flow lookup miss. 1036 */ 1037 return priv->sh->flow_mreg_c[2] != REG_NON; 1038 } 1039 1040 /** 1041 * Get the lowest priority. 1042 * 1043 * @param[in] dev 1044 * Pointer to the Ethernet device structure. 1045 * @param[in] attributes 1046 * Pointer to device flow rule attributes. 1047 * 1048 * @return 1049 * The value of lowest priority of flow. 1050 */ 1051 uint32_t 1052 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1053 const struct rte_flow_attr *attr) 1054 { 1055 struct mlx5_priv *priv = dev->data->dev_private; 1056 1057 if (!attr->group && !attr->transfer) 1058 return priv->sh->flow_max_priority - 2; 1059 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1060 } 1061 1062 /** 1063 * Calculate matcher priority of the flow. 1064 * 1065 * @param[in] dev 1066 * Pointer to the Ethernet device structure. 1067 * @param[in] attr 1068 * Pointer to device flow rule attributes. 1069 * @param[in] subpriority 1070 * The priority based on the items. 1071 * @param[in] external 1072 * Flow is user flow. 1073 * @return 1074 * The matcher priority of the flow. 1075 */ 1076 uint16_t 1077 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1078 const struct rte_flow_attr *attr, 1079 uint32_t subpriority, bool external) 1080 { 1081 uint16_t priority = (uint16_t)attr->priority; 1082 struct mlx5_priv *priv = dev->data->dev_private; 1083 1084 if (!attr->group && !attr->transfer) { 1085 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1086 priority = priv->sh->flow_max_priority - 1; 1087 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1088 } else if (!external && attr->transfer && attr->group == 0 && 1089 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1090 return (priv->sh->flow_max_priority - 1) * 3; 1091 } 1092 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1093 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1094 return priority * 3 + subpriority; 1095 } 1096 1097 /** 1098 * Verify the @p item specifications (spec, last, mask) are compatible with the 1099 * NIC capabilities. 1100 * 1101 * @param[in] item 1102 * Item specification. 1103 * @param[in] mask 1104 * @p item->mask or flow default bit-masks. 1105 * @param[in] nic_mask 1106 * Bit-masks covering supported fields by the NIC to compare with user mask. 1107 * @param[in] size 1108 * Bit-masks size in bytes. 1109 * @param[in] range_accepted 1110 * True if range of values is accepted for specific fields, false otherwise. 1111 * @param[out] error 1112 * Pointer to error structure. 1113 * 1114 * @return 1115 * 0 on success, a negative errno value otherwise and rte_errno is set. 1116 */ 1117 int 1118 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1119 const uint8_t *mask, 1120 const uint8_t *nic_mask, 1121 unsigned int size, 1122 bool range_accepted, 1123 struct rte_flow_error *error) 1124 { 1125 unsigned int i; 1126 1127 MLX5_ASSERT(nic_mask); 1128 for (i = 0; i < size; ++i) 1129 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1130 return rte_flow_error_set(error, ENOTSUP, 1131 RTE_FLOW_ERROR_TYPE_ITEM, 1132 item, 1133 "mask enables non supported" 1134 " bits"); 1135 if (!item->spec && (item->mask || item->last)) 1136 return rte_flow_error_set(error, EINVAL, 1137 RTE_FLOW_ERROR_TYPE_ITEM, item, 1138 "mask/last without a spec is not" 1139 " supported"); 1140 if (item->spec && item->last && !range_accepted) { 1141 uint8_t spec[size]; 1142 uint8_t last[size]; 1143 unsigned int i; 1144 int ret; 1145 1146 for (i = 0; i < size; ++i) { 1147 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1148 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1149 } 1150 ret = memcmp(spec, last, size); 1151 if (ret != 0) 1152 return rte_flow_error_set(error, EINVAL, 1153 RTE_FLOW_ERROR_TYPE_ITEM, 1154 item, 1155 "range is not valid"); 1156 } 1157 return 0; 1158 } 1159 1160 /** 1161 * Adjust the hash fields according to the @p flow information. 1162 * 1163 * @param[in] dev_flow. 1164 * Pointer to the mlx5_flow. 1165 * @param[in] tunnel 1166 * 1 when the hash field is for a tunnel item. 1167 * @param[in] layer_types 1168 * RTE_ETH_RSS_* types. 1169 * @param[in] hash_fields 1170 * Item hash fields. 1171 * 1172 * @return 1173 * The hash fields that should be used. 1174 */ 1175 uint64_t 1176 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1177 int tunnel __rte_unused, uint64_t layer_types, 1178 uint64_t hash_fields) 1179 { 1180 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1181 int rss_request_inner = rss_desc->level >= 2; 1182 1183 /* Check RSS hash level for tunnel. */ 1184 if (tunnel && rss_request_inner) 1185 hash_fields |= IBV_RX_HASH_INNER; 1186 else if (tunnel || rss_request_inner) 1187 return 0; 1188 #endif 1189 /* Check if requested layer matches RSS hash fields. */ 1190 if (!(rss_desc->types & layer_types)) 1191 return 0; 1192 return hash_fields; 1193 } 1194 1195 /** 1196 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1197 * if several tunnel rules are used on this queue, the tunnel ptype will be 1198 * cleared. 1199 * 1200 * @param rxq_ctrl 1201 * Rx queue to update. 1202 */ 1203 static void 1204 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1205 { 1206 unsigned int i; 1207 uint32_t tunnel_ptype = 0; 1208 1209 /* Look up for the ptype to use. */ 1210 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1211 if (!rxq_ctrl->flow_tunnels_n[i]) 1212 continue; 1213 if (!tunnel_ptype) { 1214 tunnel_ptype = tunnels_info[i].ptype; 1215 } else { 1216 tunnel_ptype = 0; 1217 break; 1218 } 1219 } 1220 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1221 } 1222 1223 /** 1224 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1225 * flow. 1226 * 1227 * @param[in] dev 1228 * Pointer to the Ethernet device structure. 1229 * @param[in] dev_handle 1230 * Pointer to device flow handle structure. 1231 */ 1232 void 1233 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1234 struct mlx5_flow_handle *dev_handle) 1235 { 1236 struct mlx5_priv *priv = dev->data->dev_private; 1237 const int mark = dev_handle->mark; 1238 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1239 struct mlx5_ind_table_obj *ind_tbl = NULL; 1240 unsigned int i; 1241 1242 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1243 struct mlx5_hrxq *hrxq; 1244 1245 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1246 dev_handle->rix_hrxq); 1247 if (hrxq) 1248 ind_tbl = hrxq->ind_table; 1249 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1250 struct mlx5_shared_action_rss *shared_rss; 1251 1252 shared_rss = mlx5_ipool_get 1253 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1254 dev_handle->rix_srss); 1255 if (shared_rss) 1256 ind_tbl = shared_rss->ind_tbl; 1257 } 1258 if (!ind_tbl) 1259 return; 1260 for (i = 0; i != ind_tbl->queues_n; ++i) { 1261 int idx = ind_tbl->queues[i]; 1262 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1263 1264 MLX5_ASSERT(rxq_ctrl != NULL); 1265 if (rxq_ctrl == NULL) 1266 continue; 1267 /* 1268 * To support metadata register copy on Tx loopback, 1269 * this must be always enabled (metadata may arive 1270 * from other port - not from local flows only. 1271 */ 1272 if (priv->config.dv_flow_en && 1273 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1274 mlx5_flow_ext_mreg_supported(dev)) { 1275 rxq_ctrl->rxq.mark = 1; 1276 rxq_ctrl->flow_mark_n = 1; 1277 } else if (mark) { 1278 rxq_ctrl->rxq.mark = 1; 1279 rxq_ctrl->flow_mark_n++; 1280 } 1281 if (tunnel) { 1282 unsigned int j; 1283 1284 /* Increase the counter matching the flow. */ 1285 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1286 if ((tunnels_info[j].tunnel & 1287 dev_handle->layers) == 1288 tunnels_info[j].tunnel) { 1289 rxq_ctrl->flow_tunnels_n[j]++; 1290 break; 1291 } 1292 } 1293 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1294 } 1295 } 1296 } 1297 1298 /** 1299 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1300 * 1301 * @param[in] dev 1302 * Pointer to the Ethernet device structure. 1303 * @param[in] flow 1304 * Pointer to flow structure. 1305 */ 1306 static void 1307 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1308 { 1309 struct mlx5_priv *priv = dev->data->dev_private; 1310 uint32_t handle_idx; 1311 struct mlx5_flow_handle *dev_handle; 1312 1313 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1314 handle_idx, dev_handle, next) 1315 flow_drv_rxq_flags_set(dev, dev_handle); 1316 } 1317 1318 /** 1319 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1320 * device flow if no other flow uses it with the same kind of request. 1321 * 1322 * @param dev 1323 * Pointer to Ethernet device. 1324 * @param[in] dev_handle 1325 * Pointer to the device flow handle structure. 1326 */ 1327 static void 1328 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1329 struct mlx5_flow_handle *dev_handle) 1330 { 1331 struct mlx5_priv *priv = dev->data->dev_private; 1332 const int mark = dev_handle->mark; 1333 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1334 struct mlx5_ind_table_obj *ind_tbl = NULL; 1335 unsigned int i; 1336 1337 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1338 struct mlx5_hrxq *hrxq; 1339 1340 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1341 dev_handle->rix_hrxq); 1342 if (hrxq) 1343 ind_tbl = hrxq->ind_table; 1344 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1345 struct mlx5_shared_action_rss *shared_rss; 1346 1347 shared_rss = mlx5_ipool_get 1348 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1349 dev_handle->rix_srss); 1350 if (shared_rss) 1351 ind_tbl = shared_rss->ind_tbl; 1352 } 1353 if (!ind_tbl) 1354 return; 1355 MLX5_ASSERT(dev->data->dev_started); 1356 for (i = 0; i != ind_tbl->queues_n; ++i) { 1357 int idx = ind_tbl->queues[i]; 1358 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1359 1360 MLX5_ASSERT(rxq_ctrl != NULL); 1361 if (rxq_ctrl == NULL) 1362 continue; 1363 if (priv->config.dv_flow_en && 1364 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1365 mlx5_flow_ext_mreg_supported(dev)) { 1366 rxq_ctrl->rxq.mark = 1; 1367 rxq_ctrl->flow_mark_n = 1; 1368 } else if (mark) { 1369 rxq_ctrl->flow_mark_n--; 1370 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1371 } 1372 if (tunnel) { 1373 unsigned int j; 1374 1375 /* Decrease the counter matching the flow. */ 1376 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1377 if ((tunnels_info[j].tunnel & 1378 dev_handle->layers) == 1379 tunnels_info[j].tunnel) { 1380 rxq_ctrl->flow_tunnels_n[j]--; 1381 break; 1382 } 1383 } 1384 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1385 } 1386 } 1387 } 1388 1389 /** 1390 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1391 * @p flow if no other flow uses it with the same kind of request. 1392 * 1393 * @param dev 1394 * Pointer to Ethernet device. 1395 * @param[in] flow 1396 * Pointer to the flow. 1397 */ 1398 static void 1399 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1400 { 1401 struct mlx5_priv *priv = dev->data->dev_private; 1402 uint32_t handle_idx; 1403 struct mlx5_flow_handle *dev_handle; 1404 1405 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1406 handle_idx, dev_handle, next) 1407 flow_drv_rxq_flags_trim(dev, dev_handle); 1408 } 1409 1410 /** 1411 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1412 * 1413 * @param dev 1414 * Pointer to Ethernet device. 1415 */ 1416 static void 1417 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1418 { 1419 struct mlx5_priv *priv = dev->data->dev_private; 1420 unsigned int i; 1421 1422 for (i = 0; i != priv->rxqs_n; ++i) { 1423 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1424 unsigned int j; 1425 1426 if (rxq == NULL || rxq->ctrl == NULL) 1427 continue; 1428 rxq->ctrl->flow_mark_n = 0; 1429 rxq->ctrl->rxq.mark = 0; 1430 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1431 rxq->ctrl->flow_tunnels_n[j] = 0; 1432 rxq->ctrl->rxq.tunnel = 0; 1433 } 1434 } 1435 1436 /** 1437 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1438 * 1439 * @param[in] dev 1440 * Pointer to the Ethernet device structure. 1441 */ 1442 void 1443 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1444 { 1445 struct mlx5_priv *priv = dev->data->dev_private; 1446 unsigned int i; 1447 1448 for (i = 0; i != priv->rxqs_n; ++i) { 1449 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1450 struct mlx5_rxq_data *data; 1451 1452 if (rxq == NULL || rxq->ctrl == NULL) 1453 continue; 1454 data = &rxq->ctrl->rxq; 1455 if (!rte_flow_dynf_metadata_avail()) { 1456 data->dynf_meta = 0; 1457 data->flow_meta_mask = 0; 1458 data->flow_meta_offset = -1; 1459 data->flow_meta_port_mask = 0; 1460 } else { 1461 data->dynf_meta = 1; 1462 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1463 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1464 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1465 } 1466 } 1467 } 1468 1469 /* 1470 * return a pointer to the desired action in the list of actions. 1471 * 1472 * @param[in] actions 1473 * The list of actions to search the action in. 1474 * @param[in] action 1475 * The action to find. 1476 * 1477 * @return 1478 * Pointer to the action in the list, if found. NULL otherwise. 1479 */ 1480 const struct rte_flow_action * 1481 mlx5_flow_find_action(const struct rte_flow_action *actions, 1482 enum rte_flow_action_type action) 1483 { 1484 if (actions == NULL) 1485 return NULL; 1486 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1487 if (actions->type == action) 1488 return actions; 1489 return NULL; 1490 } 1491 1492 /* 1493 * Validate the flag action. 1494 * 1495 * @param[in] action_flags 1496 * Bit-fields that holds the actions detected until now. 1497 * @param[in] attr 1498 * Attributes of flow that includes this action. 1499 * @param[out] error 1500 * Pointer to error structure. 1501 * 1502 * @return 1503 * 0 on success, a negative errno value otherwise and rte_errno is set. 1504 */ 1505 int 1506 mlx5_flow_validate_action_flag(uint64_t action_flags, 1507 const struct rte_flow_attr *attr, 1508 struct rte_flow_error *error) 1509 { 1510 if (action_flags & MLX5_FLOW_ACTION_MARK) 1511 return rte_flow_error_set(error, EINVAL, 1512 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1513 "can't mark and flag in same flow"); 1514 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1515 return rte_flow_error_set(error, EINVAL, 1516 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1517 "can't have 2 flag" 1518 " actions in same flow"); 1519 if (attr->egress) 1520 return rte_flow_error_set(error, ENOTSUP, 1521 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1522 "flag action not supported for " 1523 "egress"); 1524 return 0; 1525 } 1526 1527 /* 1528 * Validate the mark action. 1529 * 1530 * @param[in] action 1531 * Pointer to the queue action. 1532 * @param[in] action_flags 1533 * Bit-fields that holds the actions detected until now. 1534 * @param[in] attr 1535 * Attributes of flow that includes this action. 1536 * @param[out] error 1537 * Pointer to error structure. 1538 * 1539 * @return 1540 * 0 on success, a negative errno value otherwise and rte_errno is set. 1541 */ 1542 int 1543 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1544 uint64_t action_flags, 1545 const struct rte_flow_attr *attr, 1546 struct rte_flow_error *error) 1547 { 1548 const struct rte_flow_action_mark *mark = action->conf; 1549 1550 if (!mark) 1551 return rte_flow_error_set(error, EINVAL, 1552 RTE_FLOW_ERROR_TYPE_ACTION, 1553 action, 1554 "configuration cannot be null"); 1555 if (mark->id >= MLX5_FLOW_MARK_MAX) 1556 return rte_flow_error_set(error, EINVAL, 1557 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1558 &mark->id, 1559 "mark id must in 0 <= id < " 1560 RTE_STR(MLX5_FLOW_MARK_MAX)); 1561 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1562 return rte_flow_error_set(error, EINVAL, 1563 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1564 "can't flag and mark in same flow"); 1565 if (action_flags & MLX5_FLOW_ACTION_MARK) 1566 return rte_flow_error_set(error, EINVAL, 1567 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1568 "can't have 2 mark actions in same" 1569 " flow"); 1570 if (attr->egress) 1571 return rte_flow_error_set(error, ENOTSUP, 1572 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1573 "mark action not supported for " 1574 "egress"); 1575 return 0; 1576 } 1577 1578 /* 1579 * Validate the drop action. 1580 * 1581 * @param[in] action_flags 1582 * Bit-fields that holds the actions detected until now. 1583 * @param[in] attr 1584 * Attributes of flow that includes this action. 1585 * @param[out] error 1586 * Pointer to error structure. 1587 * 1588 * @return 1589 * 0 on success, a negative errno value otherwise and rte_errno is set. 1590 */ 1591 int 1592 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1593 const struct rte_flow_attr *attr, 1594 struct rte_flow_error *error) 1595 { 1596 if (attr->egress) 1597 return rte_flow_error_set(error, ENOTSUP, 1598 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1599 "drop action not supported for " 1600 "egress"); 1601 return 0; 1602 } 1603 1604 /* 1605 * Validate the queue action. 1606 * 1607 * @param[in] action 1608 * Pointer to the queue action. 1609 * @param[in] action_flags 1610 * Bit-fields that holds the actions detected until now. 1611 * @param[in] dev 1612 * Pointer to the Ethernet device structure. 1613 * @param[in] attr 1614 * Attributes of flow that includes this action. 1615 * @param[out] error 1616 * Pointer to error structure. 1617 * 1618 * @return 1619 * 0 on success, a negative errno value otherwise and rte_errno is set. 1620 */ 1621 int 1622 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1623 uint64_t action_flags, 1624 struct rte_eth_dev *dev, 1625 const struct rte_flow_attr *attr, 1626 struct rte_flow_error *error) 1627 { 1628 struct mlx5_priv *priv = dev->data->dev_private; 1629 const struct rte_flow_action_queue *queue = action->conf; 1630 1631 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1632 return rte_flow_error_set(error, EINVAL, 1633 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1634 "can't have 2 fate actions in" 1635 " same flow"); 1636 if (!priv->rxqs_n) 1637 return rte_flow_error_set(error, EINVAL, 1638 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1639 NULL, "No Rx queues configured"); 1640 if (queue->index >= priv->rxqs_n) 1641 return rte_flow_error_set(error, EINVAL, 1642 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1643 &queue->index, 1644 "queue index out of range"); 1645 if (mlx5_rxq_get(dev, queue->index) == NULL) 1646 return rte_flow_error_set(error, EINVAL, 1647 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1648 &queue->index, 1649 "queue is not configured"); 1650 if (attr->egress) 1651 return rte_flow_error_set(error, ENOTSUP, 1652 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1653 "queue action not supported for " 1654 "egress"); 1655 return 0; 1656 } 1657 1658 /** 1659 * Validate queue numbers for device RSS. 1660 * 1661 * @param[in] dev 1662 * Configured device. 1663 * @param[in] queues 1664 * Array of queue numbers. 1665 * @param[in] queues_n 1666 * Size of the @p queues array. 1667 * @param[out] error 1668 * On error, filled with a textual error description. 1669 * @param[out] queue 1670 * On error, filled with an offending queue index in @p queues array. 1671 * 1672 * @return 1673 * 0 on success, a negative errno code on error. 1674 */ 1675 static int 1676 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1677 const uint16_t *queues, uint32_t queues_n, 1678 const char **error, uint32_t *queue_idx) 1679 { 1680 const struct mlx5_priv *priv = dev->data->dev_private; 1681 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1682 uint32_t i; 1683 1684 for (i = 0; i != queues_n; ++i) { 1685 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, 1686 queues[i]); 1687 1688 if (queues[i] >= priv->rxqs_n) { 1689 *error = "queue index out of range"; 1690 *queue_idx = i; 1691 return -EINVAL; 1692 } 1693 if (rxq_ctrl == NULL) { 1694 *error = "queue is not configured"; 1695 *queue_idx = i; 1696 return -EINVAL; 1697 } 1698 if (i == 0) 1699 rxq_type = rxq_ctrl->type; 1700 if (rxq_type != rxq_ctrl->type) { 1701 *error = "combining hairpin and regular RSS queues is not supported"; 1702 *queue_idx = i; 1703 return -ENOTSUP; 1704 } 1705 } 1706 return 0; 1707 } 1708 1709 /* 1710 * Validate the rss action. 1711 * 1712 * @param[in] dev 1713 * Pointer to the Ethernet device structure. 1714 * @param[in] action 1715 * Pointer to the queue action. 1716 * @param[out] error 1717 * Pointer to error structure. 1718 * 1719 * @return 1720 * 0 on success, a negative errno value otherwise and rte_errno is set. 1721 */ 1722 int 1723 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1724 const struct rte_flow_action *action, 1725 struct rte_flow_error *error) 1726 { 1727 struct mlx5_priv *priv = dev->data->dev_private; 1728 const struct rte_flow_action_rss *rss = action->conf; 1729 int ret; 1730 const char *message; 1731 uint32_t queue_idx; 1732 1733 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1734 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1735 return rte_flow_error_set(error, ENOTSUP, 1736 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1737 &rss->func, 1738 "RSS hash function not supported"); 1739 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1740 if (rss->level > 2) 1741 #else 1742 if (rss->level > 1) 1743 #endif 1744 return rte_flow_error_set(error, ENOTSUP, 1745 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1746 &rss->level, 1747 "tunnel RSS is not supported"); 1748 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1749 if (rss->key_len == 0 && rss->key != NULL) 1750 return rte_flow_error_set(error, ENOTSUP, 1751 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1752 &rss->key_len, 1753 "RSS hash key length 0"); 1754 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1755 return rte_flow_error_set(error, ENOTSUP, 1756 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1757 &rss->key_len, 1758 "RSS hash key too small"); 1759 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1760 return rte_flow_error_set(error, ENOTSUP, 1761 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1762 &rss->key_len, 1763 "RSS hash key too large"); 1764 if (rss->queue_num > priv->config.ind_table_max_size) 1765 return rte_flow_error_set(error, ENOTSUP, 1766 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1767 &rss->queue_num, 1768 "number of queues too large"); 1769 if (rss->types & MLX5_RSS_HF_MASK) 1770 return rte_flow_error_set(error, ENOTSUP, 1771 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1772 &rss->types, 1773 "some RSS protocols are not" 1774 " supported"); 1775 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1776 !(rss->types & RTE_ETH_RSS_IP)) 1777 return rte_flow_error_set(error, EINVAL, 1778 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1779 "L3 partial RSS requested but L3 RSS" 1780 " type not specified"); 1781 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1782 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1783 return rte_flow_error_set(error, EINVAL, 1784 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1785 "L4 partial RSS requested but L4 RSS" 1786 " type not specified"); 1787 if (!priv->rxqs_n) 1788 return rte_flow_error_set(error, EINVAL, 1789 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1790 NULL, "No Rx queues configured"); 1791 if (!rss->queue_num) 1792 return rte_flow_error_set(error, EINVAL, 1793 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1794 NULL, "No queues configured"); 1795 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1796 &message, &queue_idx); 1797 if (ret != 0) { 1798 return rte_flow_error_set(error, -ret, 1799 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1800 &rss->queue[queue_idx], message); 1801 } 1802 return 0; 1803 } 1804 1805 /* 1806 * Validate the rss action. 1807 * 1808 * @param[in] action 1809 * Pointer to the queue action. 1810 * @param[in] action_flags 1811 * Bit-fields that holds the actions detected until now. 1812 * @param[in] dev 1813 * Pointer to the Ethernet device structure. 1814 * @param[in] attr 1815 * Attributes of flow that includes this action. 1816 * @param[in] item_flags 1817 * Items that were detected. 1818 * @param[out] error 1819 * Pointer to error structure. 1820 * 1821 * @return 1822 * 0 on success, a negative errno value otherwise and rte_errno is set. 1823 */ 1824 int 1825 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1826 uint64_t action_flags, 1827 struct rte_eth_dev *dev, 1828 const struct rte_flow_attr *attr, 1829 uint64_t item_flags, 1830 struct rte_flow_error *error) 1831 { 1832 const struct rte_flow_action_rss *rss = action->conf; 1833 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1834 int ret; 1835 1836 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1837 return rte_flow_error_set(error, EINVAL, 1838 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1839 "can't have 2 fate actions" 1840 " in same flow"); 1841 ret = mlx5_validate_action_rss(dev, action, error); 1842 if (ret) 1843 return ret; 1844 if (attr->egress) 1845 return rte_flow_error_set(error, ENOTSUP, 1846 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1847 "rss action not supported for " 1848 "egress"); 1849 if (rss->level > 1 && !tunnel) 1850 return rte_flow_error_set(error, EINVAL, 1851 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1852 "inner RSS is not supported for " 1853 "non-tunnel flows"); 1854 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1855 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1856 return rte_flow_error_set(error, EINVAL, 1857 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1858 "RSS on eCPRI is not supported now"); 1859 } 1860 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1861 !(item_flags & 1862 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1863 rss->level > 1) 1864 return rte_flow_error_set(error, EINVAL, 1865 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1866 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1867 return 0; 1868 } 1869 1870 /* 1871 * Validate the default miss action. 1872 * 1873 * @param[in] action_flags 1874 * Bit-fields that holds the actions detected until now. 1875 * @param[out] error 1876 * Pointer to error structure. 1877 * 1878 * @return 1879 * 0 on success, a negative errno value otherwise and rte_errno is set. 1880 */ 1881 int 1882 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1883 const struct rte_flow_attr *attr, 1884 struct rte_flow_error *error) 1885 { 1886 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1887 return rte_flow_error_set(error, EINVAL, 1888 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1889 "can't have 2 fate actions in" 1890 " same flow"); 1891 if (attr->egress) 1892 return rte_flow_error_set(error, ENOTSUP, 1893 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1894 "default miss action not supported " 1895 "for egress"); 1896 if (attr->group) 1897 return rte_flow_error_set(error, ENOTSUP, 1898 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1899 "only group 0 is supported"); 1900 if (attr->transfer) 1901 return rte_flow_error_set(error, ENOTSUP, 1902 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1903 NULL, "transfer is not supported"); 1904 return 0; 1905 } 1906 1907 /* 1908 * Validate the count action. 1909 * 1910 * @param[in] dev 1911 * Pointer to the Ethernet device structure. 1912 * @param[in] attr 1913 * Attributes of flow that includes this action. 1914 * @param[out] error 1915 * Pointer to error structure. 1916 * 1917 * @return 1918 * 0 on success, a negative errno value otherwise and rte_errno is set. 1919 */ 1920 int 1921 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1922 const struct rte_flow_attr *attr, 1923 struct rte_flow_error *error) 1924 { 1925 if (attr->egress) 1926 return rte_flow_error_set(error, ENOTSUP, 1927 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1928 "count action not supported for " 1929 "egress"); 1930 return 0; 1931 } 1932 1933 /* 1934 * Validate the ASO CT action. 1935 * 1936 * @param[in] dev 1937 * Pointer to the Ethernet device structure. 1938 * @param[in] conntrack 1939 * Pointer to the CT action profile. 1940 * @param[out] error 1941 * Pointer to error structure. 1942 * 1943 * @return 1944 * 0 on success, a negative errno value otherwise and rte_errno is set. 1945 */ 1946 int 1947 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1948 const struct rte_flow_action_conntrack *conntrack, 1949 struct rte_flow_error *error) 1950 { 1951 RTE_SET_USED(dev); 1952 1953 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1954 return rte_flow_error_set(error, EINVAL, 1955 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1956 "Invalid CT state"); 1957 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1958 return rte_flow_error_set(error, EINVAL, 1959 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1960 "Invalid last TCP packet flag"); 1961 return 0; 1962 } 1963 1964 /** 1965 * Verify the @p attributes will be correctly understood by the NIC and store 1966 * them in the @p flow if everything is correct. 1967 * 1968 * @param[in] dev 1969 * Pointer to the Ethernet device structure. 1970 * @param[in] attributes 1971 * Pointer to flow attributes 1972 * @param[out] error 1973 * Pointer to error structure. 1974 * 1975 * @return 1976 * 0 on success, a negative errno value otherwise and rte_errno is set. 1977 */ 1978 int 1979 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1980 const struct rte_flow_attr *attributes, 1981 struct rte_flow_error *error) 1982 { 1983 struct mlx5_priv *priv = dev->data->dev_private; 1984 uint32_t priority_max = priv->sh->flow_max_priority - 1; 1985 1986 if (attributes->group) 1987 return rte_flow_error_set(error, ENOTSUP, 1988 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1989 NULL, "groups is not supported"); 1990 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1991 attributes->priority >= priority_max) 1992 return rte_flow_error_set(error, ENOTSUP, 1993 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1994 NULL, "priority out of range"); 1995 if (attributes->egress) 1996 return rte_flow_error_set(error, ENOTSUP, 1997 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1998 "egress is not supported"); 1999 if (attributes->transfer && !priv->config.dv_esw_en) 2000 return rte_flow_error_set(error, ENOTSUP, 2001 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2002 NULL, "transfer is not supported"); 2003 if (!attributes->ingress) 2004 return rte_flow_error_set(error, EINVAL, 2005 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2006 NULL, 2007 "ingress attribute is mandatory"); 2008 return 0; 2009 } 2010 2011 /** 2012 * Validate ICMP6 item. 2013 * 2014 * @param[in] item 2015 * Item specification. 2016 * @param[in] item_flags 2017 * Bit-fields that holds the items detected until now. 2018 * @param[in] ext_vlan_sup 2019 * Whether extended VLAN features are supported or not. 2020 * @param[out] error 2021 * Pointer to error structure. 2022 * 2023 * @return 2024 * 0 on success, a negative errno value otherwise and rte_errno is set. 2025 */ 2026 int 2027 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2028 uint64_t item_flags, 2029 uint8_t target_protocol, 2030 struct rte_flow_error *error) 2031 { 2032 const struct rte_flow_item_icmp6 *mask = item->mask; 2033 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2034 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2035 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2036 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2037 MLX5_FLOW_LAYER_OUTER_L4; 2038 int ret; 2039 2040 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2041 return rte_flow_error_set(error, EINVAL, 2042 RTE_FLOW_ERROR_TYPE_ITEM, item, 2043 "protocol filtering not compatible" 2044 " with ICMP6 layer"); 2045 if (!(item_flags & l3m)) 2046 return rte_flow_error_set(error, EINVAL, 2047 RTE_FLOW_ERROR_TYPE_ITEM, item, 2048 "IPv6 is mandatory to filter on" 2049 " ICMP6"); 2050 if (item_flags & l4m) 2051 return rte_flow_error_set(error, EINVAL, 2052 RTE_FLOW_ERROR_TYPE_ITEM, item, 2053 "multiple L4 layers not supported"); 2054 if (!mask) 2055 mask = &rte_flow_item_icmp6_mask; 2056 ret = mlx5_flow_item_acceptable 2057 (item, (const uint8_t *)mask, 2058 (const uint8_t *)&rte_flow_item_icmp6_mask, 2059 sizeof(struct rte_flow_item_icmp6), 2060 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2061 if (ret < 0) 2062 return ret; 2063 return 0; 2064 } 2065 2066 /** 2067 * Validate ICMP item. 2068 * 2069 * @param[in] item 2070 * Item specification. 2071 * @param[in] item_flags 2072 * Bit-fields that holds the items detected until now. 2073 * @param[out] error 2074 * Pointer to error structure. 2075 * 2076 * @return 2077 * 0 on success, a negative errno value otherwise and rte_errno is set. 2078 */ 2079 int 2080 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2081 uint64_t item_flags, 2082 uint8_t target_protocol, 2083 struct rte_flow_error *error) 2084 { 2085 const struct rte_flow_item_icmp *mask = item->mask; 2086 const struct rte_flow_item_icmp nic_mask = { 2087 .hdr.icmp_type = 0xff, 2088 .hdr.icmp_code = 0xff, 2089 .hdr.icmp_ident = RTE_BE16(0xffff), 2090 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2091 }; 2092 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2093 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2094 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2095 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2096 MLX5_FLOW_LAYER_OUTER_L4; 2097 int ret; 2098 2099 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2100 return rte_flow_error_set(error, EINVAL, 2101 RTE_FLOW_ERROR_TYPE_ITEM, item, 2102 "protocol filtering not compatible" 2103 " with ICMP layer"); 2104 if (!(item_flags & l3m)) 2105 return rte_flow_error_set(error, EINVAL, 2106 RTE_FLOW_ERROR_TYPE_ITEM, item, 2107 "IPv4 is mandatory to filter" 2108 " on ICMP"); 2109 if (item_flags & l4m) 2110 return rte_flow_error_set(error, EINVAL, 2111 RTE_FLOW_ERROR_TYPE_ITEM, item, 2112 "multiple L4 layers not supported"); 2113 if (!mask) 2114 mask = &nic_mask; 2115 ret = mlx5_flow_item_acceptable 2116 (item, (const uint8_t *)mask, 2117 (const uint8_t *)&nic_mask, 2118 sizeof(struct rte_flow_item_icmp), 2119 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2120 if (ret < 0) 2121 return ret; 2122 return 0; 2123 } 2124 2125 /** 2126 * Validate Ethernet item. 2127 * 2128 * @param[in] item 2129 * Item specification. 2130 * @param[in] item_flags 2131 * Bit-fields that holds the items detected until now. 2132 * @param[out] error 2133 * Pointer to error structure. 2134 * 2135 * @return 2136 * 0 on success, a negative errno value otherwise and rte_errno is set. 2137 */ 2138 int 2139 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2140 uint64_t item_flags, bool ext_vlan_sup, 2141 struct rte_flow_error *error) 2142 { 2143 const struct rte_flow_item_eth *mask = item->mask; 2144 const struct rte_flow_item_eth nic_mask = { 2145 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2146 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2147 .type = RTE_BE16(0xffff), 2148 .has_vlan = ext_vlan_sup ? 1 : 0, 2149 }; 2150 int ret; 2151 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2152 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2153 MLX5_FLOW_LAYER_OUTER_L2; 2154 2155 if (item_flags & ethm) 2156 return rte_flow_error_set(error, ENOTSUP, 2157 RTE_FLOW_ERROR_TYPE_ITEM, item, 2158 "multiple L2 layers not supported"); 2159 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2160 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2161 return rte_flow_error_set(error, EINVAL, 2162 RTE_FLOW_ERROR_TYPE_ITEM, item, 2163 "L2 layer should not follow " 2164 "L3 layers"); 2165 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2166 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2167 return rte_flow_error_set(error, EINVAL, 2168 RTE_FLOW_ERROR_TYPE_ITEM, item, 2169 "L2 layer should not follow VLAN"); 2170 if (item_flags & MLX5_FLOW_LAYER_GTP) 2171 return rte_flow_error_set(error, EINVAL, 2172 RTE_FLOW_ERROR_TYPE_ITEM, item, 2173 "L2 layer should not follow GTP"); 2174 if (!mask) 2175 mask = &rte_flow_item_eth_mask; 2176 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2177 (const uint8_t *)&nic_mask, 2178 sizeof(struct rte_flow_item_eth), 2179 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2180 return ret; 2181 } 2182 2183 /** 2184 * Validate VLAN item. 2185 * 2186 * @param[in] item 2187 * Item specification. 2188 * @param[in] item_flags 2189 * Bit-fields that holds the items detected until now. 2190 * @param[in] dev 2191 * Ethernet device flow is being created on. 2192 * @param[out] error 2193 * Pointer to error structure. 2194 * 2195 * @return 2196 * 0 on success, a negative errno value otherwise and rte_errno is set. 2197 */ 2198 int 2199 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2200 uint64_t item_flags, 2201 struct rte_eth_dev *dev, 2202 struct rte_flow_error *error) 2203 { 2204 const struct rte_flow_item_vlan *spec = item->spec; 2205 const struct rte_flow_item_vlan *mask = item->mask; 2206 const struct rte_flow_item_vlan nic_mask = { 2207 .tci = RTE_BE16(UINT16_MAX), 2208 .inner_type = RTE_BE16(UINT16_MAX), 2209 }; 2210 uint16_t vlan_tag = 0; 2211 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2212 int ret; 2213 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2214 MLX5_FLOW_LAYER_INNER_L4) : 2215 (MLX5_FLOW_LAYER_OUTER_L3 | 2216 MLX5_FLOW_LAYER_OUTER_L4); 2217 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2218 MLX5_FLOW_LAYER_OUTER_VLAN; 2219 2220 if (item_flags & vlanm) 2221 return rte_flow_error_set(error, EINVAL, 2222 RTE_FLOW_ERROR_TYPE_ITEM, item, 2223 "multiple VLAN layers not supported"); 2224 else if ((item_flags & l34m) != 0) 2225 return rte_flow_error_set(error, EINVAL, 2226 RTE_FLOW_ERROR_TYPE_ITEM, item, 2227 "VLAN cannot follow L3/L4 layer"); 2228 if (!mask) 2229 mask = &rte_flow_item_vlan_mask; 2230 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2231 (const uint8_t *)&nic_mask, 2232 sizeof(struct rte_flow_item_vlan), 2233 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2234 if (ret) 2235 return ret; 2236 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2237 struct mlx5_priv *priv = dev->data->dev_private; 2238 2239 if (priv->vmwa_context) { 2240 /* 2241 * Non-NULL context means we have a virtual machine 2242 * and SR-IOV enabled, we have to create VLAN interface 2243 * to make hypervisor to setup E-Switch vport 2244 * context correctly. We avoid creating the multiple 2245 * VLAN interfaces, so we cannot support VLAN tag mask. 2246 */ 2247 return rte_flow_error_set(error, EINVAL, 2248 RTE_FLOW_ERROR_TYPE_ITEM, 2249 item, 2250 "VLAN tag mask is not" 2251 " supported in virtual" 2252 " environment"); 2253 } 2254 } 2255 if (spec) { 2256 vlan_tag = spec->tci; 2257 vlan_tag &= mask->tci; 2258 } 2259 /* 2260 * From verbs perspective an empty VLAN is equivalent 2261 * to a packet without VLAN layer. 2262 */ 2263 if (!vlan_tag) 2264 return rte_flow_error_set(error, EINVAL, 2265 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2266 item->spec, 2267 "VLAN cannot be empty"); 2268 return 0; 2269 } 2270 2271 /** 2272 * Validate IPV4 item. 2273 * 2274 * @param[in] item 2275 * Item specification. 2276 * @param[in] item_flags 2277 * Bit-fields that holds the items detected until now. 2278 * @param[in] last_item 2279 * Previous validated item in the pattern items. 2280 * @param[in] ether_type 2281 * Type in the ethernet layer header (including dot1q). 2282 * @param[in] acc_mask 2283 * Acceptable mask, if NULL default internal default mask 2284 * will be used to check whether item fields are supported. 2285 * @param[in] range_accepted 2286 * True if range of values is accepted for specific fields, false otherwise. 2287 * @param[out] error 2288 * Pointer to error structure. 2289 * 2290 * @return 2291 * 0 on success, a negative errno value otherwise and rte_errno is set. 2292 */ 2293 int 2294 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2295 uint64_t item_flags, 2296 uint64_t last_item, 2297 uint16_t ether_type, 2298 const struct rte_flow_item_ipv4 *acc_mask, 2299 bool range_accepted, 2300 struct rte_flow_error *error) 2301 { 2302 const struct rte_flow_item_ipv4 *mask = item->mask; 2303 const struct rte_flow_item_ipv4 *spec = item->spec; 2304 const struct rte_flow_item_ipv4 nic_mask = { 2305 .hdr = { 2306 .src_addr = RTE_BE32(0xffffffff), 2307 .dst_addr = RTE_BE32(0xffffffff), 2308 .type_of_service = 0xff, 2309 .next_proto_id = 0xff, 2310 }, 2311 }; 2312 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2313 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2314 MLX5_FLOW_LAYER_OUTER_L3; 2315 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2316 MLX5_FLOW_LAYER_OUTER_L4; 2317 int ret; 2318 uint8_t next_proto = 0xFF; 2319 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2320 MLX5_FLOW_LAYER_OUTER_VLAN | 2321 MLX5_FLOW_LAYER_INNER_VLAN); 2322 2323 if ((last_item & l2_vlan) && ether_type && 2324 ether_type != RTE_ETHER_TYPE_IPV4) 2325 return rte_flow_error_set(error, EINVAL, 2326 RTE_FLOW_ERROR_TYPE_ITEM, item, 2327 "IPv4 cannot follow L2/VLAN layer " 2328 "which ether type is not IPv4"); 2329 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2330 if (mask && spec) 2331 next_proto = mask->hdr.next_proto_id & 2332 spec->hdr.next_proto_id; 2333 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2334 return rte_flow_error_set(error, EINVAL, 2335 RTE_FLOW_ERROR_TYPE_ITEM, 2336 item, 2337 "multiple tunnel " 2338 "not supported"); 2339 } 2340 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2341 return rte_flow_error_set(error, EINVAL, 2342 RTE_FLOW_ERROR_TYPE_ITEM, item, 2343 "wrong tunnel type - IPv6 specified " 2344 "but IPv4 item provided"); 2345 if (item_flags & l3m) 2346 return rte_flow_error_set(error, ENOTSUP, 2347 RTE_FLOW_ERROR_TYPE_ITEM, item, 2348 "multiple L3 layers not supported"); 2349 else if (item_flags & l4m) 2350 return rte_flow_error_set(error, EINVAL, 2351 RTE_FLOW_ERROR_TYPE_ITEM, item, 2352 "L3 cannot follow an L4 layer."); 2353 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2354 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2355 return rte_flow_error_set(error, EINVAL, 2356 RTE_FLOW_ERROR_TYPE_ITEM, item, 2357 "L3 cannot follow an NVGRE layer."); 2358 if (!mask) 2359 mask = &rte_flow_item_ipv4_mask; 2360 else if (mask->hdr.next_proto_id != 0 && 2361 mask->hdr.next_proto_id != 0xff) 2362 return rte_flow_error_set(error, EINVAL, 2363 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2364 "partial mask is not supported" 2365 " for protocol"); 2366 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2367 acc_mask ? (const uint8_t *)acc_mask 2368 : (const uint8_t *)&nic_mask, 2369 sizeof(struct rte_flow_item_ipv4), 2370 range_accepted, error); 2371 if (ret < 0) 2372 return ret; 2373 return 0; 2374 } 2375 2376 /** 2377 * Validate IPV6 item. 2378 * 2379 * @param[in] item 2380 * Item specification. 2381 * @param[in] item_flags 2382 * Bit-fields that holds the items detected until now. 2383 * @param[in] last_item 2384 * Previous validated item in the pattern items. 2385 * @param[in] ether_type 2386 * Type in the ethernet layer header (including dot1q). 2387 * @param[in] acc_mask 2388 * Acceptable mask, if NULL default internal default mask 2389 * will be used to check whether item fields are supported. 2390 * @param[out] error 2391 * Pointer to error structure. 2392 * 2393 * @return 2394 * 0 on success, a negative errno value otherwise and rte_errno is set. 2395 */ 2396 int 2397 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2398 uint64_t item_flags, 2399 uint64_t last_item, 2400 uint16_t ether_type, 2401 const struct rte_flow_item_ipv6 *acc_mask, 2402 struct rte_flow_error *error) 2403 { 2404 const struct rte_flow_item_ipv6 *mask = item->mask; 2405 const struct rte_flow_item_ipv6 *spec = item->spec; 2406 const struct rte_flow_item_ipv6 nic_mask = { 2407 .hdr = { 2408 .src_addr = 2409 "\xff\xff\xff\xff\xff\xff\xff\xff" 2410 "\xff\xff\xff\xff\xff\xff\xff\xff", 2411 .dst_addr = 2412 "\xff\xff\xff\xff\xff\xff\xff\xff" 2413 "\xff\xff\xff\xff\xff\xff\xff\xff", 2414 .vtc_flow = RTE_BE32(0xffffffff), 2415 .proto = 0xff, 2416 }, 2417 }; 2418 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2419 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2420 MLX5_FLOW_LAYER_OUTER_L3; 2421 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2422 MLX5_FLOW_LAYER_OUTER_L4; 2423 int ret; 2424 uint8_t next_proto = 0xFF; 2425 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2426 MLX5_FLOW_LAYER_OUTER_VLAN | 2427 MLX5_FLOW_LAYER_INNER_VLAN); 2428 2429 if ((last_item & l2_vlan) && ether_type && 2430 ether_type != RTE_ETHER_TYPE_IPV6) 2431 return rte_flow_error_set(error, EINVAL, 2432 RTE_FLOW_ERROR_TYPE_ITEM, item, 2433 "IPv6 cannot follow L2/VLAN layer " 2434 "which ether type is not IPv6"); 2435 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2436 next_proto = spec->hdr.proto; 2437 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2438 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2439 return rte_flow_error_set(error, EINVAL, 2440 RTE_FLOW_ERROR_TYPE_ITEM, 2441 item, 2442 "multiple tunnel " 2443 "not supported"); 2444 } 2445 if (next_proto == IPPROTO_HOPOPTS || 2446 next_proto == IPPROTO_ROUTING || 2447 next_proto == IPPROTO_FRAGMENT || 2448 next_proto == IPPROTO_ESP || 2449 next_proto == IPPROTO_AH || 2450 next_proto == IPPROTO_DSTOPTS) 2451 return rte_flow_error_set(error, EINVAL, 2452 RTE_FLOW_ERROR_TYPE_ITEM, item, 2453 "IPv6 proto (next header) should " 2454 "not be set as extension header"); 2455 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2456 return rte_flow_error_set(error, EINVAL, 2457 RTE_FLOW_ERROR_TYPE_ITEM, item, 2458 "wrong tunnel type - IPv4 specified " 2459 "but IPv6 item provided"); 2460 if (item_flags & l3m) 2461 return rte_flow_error_set(error, ENOTSUP, 2462 RTE_FLOW_ERROR_TYPE_ITEM, item, 2463 "multiple L3 layers not supported"); 2464 else if (item_flags & l4m) 2465 return rte_flow_error_set(error, EINVAL, 2466 RTE_FLOW_ERROR_TYPE_ITEM, item, 2467 "L3 cannot follow an L4 layer."); 2468 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2469 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2470 return rte_flow_error_set(error, EINVAL, 2471 RTE_FLOW_ERROR_TYPE_ITEM, item, 2472 "L3 cannot follow an NVGRE layer."); 2473 if (!mask) 2474 mask = &rte_flow_item_ipv6_mask; 2475 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2476 acc_mask ? (const uint8_t *)acc_mask 2477 : (const uint8_t *)&nic_mask, 2478 sizeof(struct rte_flow_item_ipv6), 2479 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2480 if (ret < 0) 2481 return ret; 2482 return 0; 2483 } 2484 2485 /** 2486 * Validate UDP item. 2487 * 2488 * @param[in] item 2489 * Item specification. 2490 * @param[in] item_flags 2491 * Bit-fields that holds the items detected until now. 2492 * @param[in] target_protocol 2493 * The next protocol in the previous item. 2494 * @param[in] flow_mask 2495 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2496 * @param[out] error 2497 * Pointer to error structure. 2498 * 2499 * @return 2500 * 0 on success, a negative errno value otherwise and rte_errno is set. 2501 */ 2502 int 2503 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2504 uint64_t item_flags, 2505 uint8_t target_protocol, 2506 struct rte_flow_error *error) 2507 { 2508 const struct rte_flow_item_udp *mask = item->mask; 2509 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2510 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2511 MLX5_FLOW_LAYER_OUTER_L3; 2512 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2513 MLX5_FLOW_LAYER_OUTER_L4; 2514 int ret; 2515 2516 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2517 return rte_flow_error_set(error, EINVAL, 2518 RTE_FLOW_ERROR_TYPE_ITEM, item, 2519 "protocol filtering not compatible" 2520 " with UDP layer"); 2521 if (!(item_flags & l3m)) 2522 return rte_flow_error_set(error, EINVAL, 2523 RTE_FLOW_ERROR_TYPE_ITEM, item, 2524 "L3 is mandatory to filter on L4"); 2525 if (item_flags & l4m) 2526 return rte_flow_error_set(error, EINVAL, 2527 RTE_FLOW_ERROR_TYPE_ITEM, item, 2528 "multiple L4 layers not supported"); 2529 if (!mask) 2530 mask = &rte_flow_item_udp_mask; 2531 ret = mlx5_flow_item_acceptable 2532 (item, (const uint8_t *)mask, 2533 (const uint8_t *)&rte_flow_item_udp_mask, 2534 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2535 error); 2536 if (ret < 0) 2537 return ret; 2538 return 0; 2539 } 2540 2541 /** 2542 * Validate TCP item. 2543 * 2544 * @param[in] item 2545 * Item specification. 2546 * @param[in] item_flags 2547 * Bit-fields that holds the items detected until now. 2548 * @param[in] target_protocol 2549 * The next protocol in the previous item. 2550 * @param[out] error 2551 * Pointer to error structure. 2552 * 2553 * @return 2554 * 0 on success, a negative errno value otherwise and rte_errno is set. 2555 */ 2556 int 2557 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2558 uint64_t item_flags, 2559 uint8_t target_protocol, 2560 const struct rte_flow_item_tcp *flow_mask, 2561 struct rte_flow_error *error) 2562 { 2563 const struct rte_flow_item_tcp *mask = item->mask; 2564 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2565 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2566 MLX5_FLOW_LAYER_OUTER_L3; 2567 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2568 MLX5_FLOW_LAYER_OUTER_L4; 2569 int ret; 2570 2571 MLX5_ASSERT(flow_mask); 2572 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2573 return rte_flow_error_set(error, EINVAL, 2574 RTE_FLOW_ERROR_TYPE_ITEM, item, 2575 "protocol filtering not compatible" 2576 " with TCP layer"); 2577 if (!(item_flags & l3m)) 2578 return rte_flow_error_set(error, EINVAL, 2579 RTE_FLOW_ERROR_TYPE_ITEM, item, 2580 "L3 is mandatory to filter on L4"); 2581 if (item_flags & l4m) 2582 return rte_flow_error_set(error, EINVAL, 2583 RTE_FLOW_ERROR_TYPE_ITEM, item, 2584 "multiple L4 layers not supported"); 2585 if (!mask) 2586 mask = &rte_flow_item_tcp_mask; 2587 ret = mlx5_flow_item_acceptable 2588 (item, (const uint8_t *)mask, 2589 (const uint8_t *)flow_mask, 2590 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2591 error); 2592 if (ret < 0) 2593 return ret; 2594 return 0; 2595 } 2596 2597 /** 2598 * Validate VXLAN item. 2599 * 2600 * @param[in] dev 2601 * Pointer to the Ethernet device structure. 2602 * @param[in] udp_dport 2603 * UDP destination port 2604 * @param[in] item 2605 * Item specification. 2606 * @param[in] item_flags 2607 * Bit-fields that holds the items detected until now. 2608 * @param[in] attr 2609 * Flow rule attributes. 2610 * @param[out] error 2611 * Pointer to error structure. 2612 * 2613 * @return 2614 * 0 on success, a negative errno value otherwise and rte_errno is set. 2615 */ 2616 int 2617 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2618 uint16_t udp_dport, 2619 const struct rte_flow_item *item, 2620 uint64_t item_flags, 2621 const struct rte_flow_attr *attr, 2622 struct rte_flow_error *error) 2623 { 2624 const struct rte_flow_item_vxlan *spec = item->spec; 2625 const struct rte_flow_item_vxlan *mask = item->mask; 2626 int ret; 2627 struct mlx5_priv *priv = dev->data->dev_private; 2628 union vni { 2629 uint32_t vlan_id; 2630 uint8_t vni[4]; 2631 } id = { .vlan_id = 0, }; 2632 const struct rte_flow_item_vxlan nic_mask = { 2633 .vni = "\xff\xff\xff", 2634 .rsvd1 = 0xff, 2635 }; 2636 const struct rte_flow_item_vxlan *valid_mask; 2637 2638 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2639 return rte_flow_error_set(error, ENOTSUP, 2640 RTE_FLOW_ERROR_TYPE_ITEM, item, 2641 "multiple tunnel layers not" 2642 " supported"); 2643 valid_mask = &rte_flow_item_vxlan_mask; 2644 /* 2645 * Verify only UDPv4 is present as defined in 2646 * https://tools.ietf.org/html/rfc7348 2647 */ 2648 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2649 return rte_flow_error_set(error, EINVAL, 2650 RTE_FLOW_ERROR_TYPE_ITEM, item, 2651 "no outer UDP layer found"); 2652 if (!mask) 2653 mask = &rte_flow_item_vxlan_mask; 2654 2655 if (priv->sh->steering_format_version != 2656 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2657 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2658 /* FDB domain & NIC domain non-zero group */ 2659 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2660 valid_mask = &nic_mask; 2661 /* Group zero in NIC domain */ 2662 if (!attr->group && !attr->transfer && 2663 priv->sh->tunnel_header_0_1) 2664 valid_mask = &nic_mask; 2665 } 2666 ret = mlx5_flow_item_acceptable 2667 (item, (const uint8_t *)mask, 2668 (const uint8_t *)valid_mask, 2669 sizeof(struct rte_flow_item_vxlan), 2670 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2671 if (ret < 0) 2672 return ret; 2673 if (spec) { 2674 memcpy(&id.vni[1], spec->vni, 3); 2675 memcpy(&id.vni[1], mask->vni, 3); 2676 } 2677 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2678 return rte_flow_error_set(error, ENOTSUP, 2679 RTE_FLOW_ERROR_TYPE_ITEM, item, 2680 "VXLAN tunnel must be fully defined"); 2681 return 0; 2682 } 2683 2684 /** 2685 * Validate VXLAN_GPE item. 2686 * 2687 * @param[in] item 2688 * Item specification. 2689 * @param[in] item_flags 2690 * Bit-fields that holds the items detected until now. 2691 * @param[in] priv 2692 * Pointer to the private data structure. 2693 * @param[in] target_protocol 2694 * The next protocol in the previous item. 2695 * @param[out] error 2696 * Pointer to error structure. 2697 * 2698 * @return 2699 * 0 on success, a negative errno value otherwise and rte_errno is set. 2700 */ 2701 int 2702 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2703 uint64_t item_flags, 2704 struct rte_eth_dev *dev, 2705 struct rte_flow_error *error) 2706 { 2707 struct mlx5_priv *priv = dev->data->dev_private; 2708 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2709 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2710 int ret; 2711 union vni { 2712 uint32_t vlan_id; 2713 uint8_t vni[4]; 2714 } id = { .vlan_id = 0, }; 2715 2716 if (!priv->config.l3_vxlan_en) 2717 return rte_flow_error_set(error, ENOTSUP, 2718 RTE_FLOW_ERROR_TYPE_ITEM, item, 2719 "L3 VXLAN is not enabled by device" 2720 " parameter and/or not configured in" 2721 " firmware"); 2722 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2723 return rte_flow_error_set(error, ENOTSUP, 2724 RTE_FLOW_ERROR_TYPE_ITEM, item, 2725 "multiple tunnel layers not" 2726 " supported"); 2727 /* 2728 * Verify only UDPv4 is present as defined in 2729 * https://tools.ietf.org/html/rfc7348 2730 */ 2731 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2732 return rte_flow_error_set(error, EINVAL, 2733 RTE_FLOW_ERROR_TYPE_ITEM, item, 2734 "no outer UDP layer found"); 2735 if (!mask) 2736 mask = &rte_flow_item_vxlan_gpe_mask; 2737 ret = mlx5_flow_item_acceptable 2738 (item, (const uint8_t *)mask, 2739 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2740 sizeof(struct rte_flow_item_vxlan_gpe), 2741 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2742 if (ret < 0) 2743 return ret; 2744 if (spec) { 2745 if (spec->protocol) 2746 return rte_flow_error_set(error, ENOTSUP, 2747 RTE_FLOW_ERROR_TYPE_ITEM, 2748 item, 2749 "VxLAN-GPE protocol" 2750 " not supported"); 2751 memcpy(&id.vni[1], spec->vni, 3); 2752 memcpy(&id.vni[1], mask->vni, 3); 2753 } 2754 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2755 return rte_flow_error_set(error, ENOTSUP, 2756 RTE_FLOW_ERROR_TYPE_ITEM, item, 2757 "VXLAN-GPE tunnel must be fully" 2758 " defined"); 2759 return 0; 2760 } 2761 /** 2762 * Validate GRE Key item. 2763 * 2764 * @param[in] item 2765 * Item specification. 2766 * @param[in] item_flags 2767 * Bit flags to mark detected items. 2768 * @param[in] gre_item 2769 * Pointer to gre_item 2770 * @param[out] error 2771 * Pointer to error structure. 2772 * 2773 * @return 2774 * 0 on success, a negative errno value otherwise and rte_errno is set. 2775 */ 2776 int 2777 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2778 uint64_t item_flags, 2779 const struct rte_flow_item *gre_item, 2780 struct rte_flow_error *error) 2781 { 2782 const rte_be32_t *mask = item->mask; 2783 int ret = 0; 2784 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2785 const struct rte_flow_item_gre *gre_spec; 2786 const struct rte_flow_item_gre *gre_mask; 2787 2788 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2789 return rte_flow_error_set(error, ENOTSUP, 2790 RTE_FLOW_ERROR_TYPE_ITEM, item, 2791 "Multiple GRE key not support"); 2792 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2793 return rte_flow_error_set(error, ENOTSUP, 2794 RTE_FLOW_ERROR_TYPE_ITEM, item, 2795 "No preceding GRE header"); 2796 if (item_flags & MLX5_FLOW_LAYER_INNER) 2797 return rte_flow_error_set(error, ENOTSUP, 2798 RTE_FLOW_ERROR_TYPE_ITEM, item, 2799 "GRE key following a wrong item"); 2800 gre_mask = gre_item->mask; 2801 if (!gre_mask) 2802 gre_mask = &rte_flow_item_gre_mask; 2803 gre_spec = gre_item->spec; 2804 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2805 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2806 return rte_flow_error_set(error, EINVAL, 2807 RTE_FLOW_ERROR_TYPE_ITEM, item, 2808 "Key bit must be on"); 2809 2810 if (!mask) 2811 mask = &gre_key_default_mask; 2812 ret = mlx5_flow_item_acceptable 2813 (item, (const uint8_t *)mask, 2814 (const uint8_t *)&gre_key_default_mask, 2815 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2816 return ret; 2817 } 2818 2819 /** 2820 * Validate GRE item. 2821 * 2822 * @param[in] item 2823 * Item specification. 2824 * @param[in] item_flags 2825 * Bit flags to mark detected items. 2826 * @param[in] target_protocol 2827 * The next protocol in the previous item. 2828 * @param[out] error 2829 * Pointer to error structure. 2830 * 2831 * @return 2832 * 0 on success, a negative errno value otherwise and rte_errno is set. 2833 */ 2834 int 2835 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2836 uint64_t item_flags, 2837 uint8_t target_protocol, 2838 struct rte_flow_error *error) 2839 { 2840 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2841 const struct rte_flow_item_gre *mask = item->mask; 2842 int ret; 2843 const struct rte_flow_item_gre nic_mask = { 2844 .c_rsvd0_ver = RTE_BE16(0xB000), 2845 .protocol = RTE_BE16(UINT16_MAX), 2846 }; 2847 2848 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2849 return rte_flow_error_set(error, EINVAL, 2850 RTE_FLOW_ERROR_TYPE_ITEM, item, 2851 "protocol filtering not compatible" 2852 " with this GRE layer"); 2853 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2854 return rte_flow_error_set(error, ENOTSUP, 2855 RTE_FLOW_ERROR_TYPE_ITEM, item, 2856 "multiple tunnel layers not" 2857 " supported"); 2858 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2859 return rte_flow_error_set(error, ENOTSUP, 2860 RTE_FLOW_ERROR_TYPE_ITEM, item, 2861 "L3 Layer is missing"); 2862 if (!mask) 2863 mask = &rte_flow_item_gre_mask; 2864 ret = mlx5_flow_item_acceptable 2865 (item, (const uint8_t *)mask, 2866 (const uint8_t *)&nic_mask, 2867 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2868 error); 2869 if (ret < 0) 2870 return ret; 2871 #ifndef HAVE_MLX5DV_DR 2872 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2873 if (spec && (spec->protocol & mask->protocol)) 2874 return rte_flow_error_set(error, ENOTSUP, 2875 RTE_FLOW_ERROR_TYPE_ITEM, item, 2876 "without MPLS support the" 2877 " specification cannot be used for" 2878 " filtering"); 2879 #endif 2880 #endif 2881 return 0; 2882 } 2883 2884 /** 2885 * Validate Geneve item. 2886 * 2887 * @param[in] item 2888 * Item specification. 2889 * @param[in] itemFlags 2890 * Bit-fields that holds the items detected until now. 2891 * @param[in] enPriv 2892 * Pointer to the private data structure. 2893 * @param[out] error 2894 * Pointer to error structure. 2895 * 2896 * @return 2897 * 0 on success, a negative errno value otherwise and rte_errno is set. 2898 */ 2899 2900 int 2901 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2902 uint64_t item_flags, 2903 struct rte_eth_dev *dev, 2904 struct rte_flow_error *error) 2905 { 2906 struct mlx5_priv *priv = dev->data->dev_private; 2907 const struct rte_flow_item_geneve *spec = item->spec; 2908 const struct rte_flow_item_geneve *mask = item->mask; 2909 int ret; 2910 uint16_t gbhdr; 2911 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2912 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2913 const struct rte_flow_item_geneve nic_mask = { 2914 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2915 .vni = "\xff\xff\xff", 2916 .protocol = RTE_BE16(UINT16_MAX), 2917 }; 2918 2919 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2920 return rte_flow_error_set(error, ENOTSUP, 2921 RTE_FLOW_ERROR_TYPE_ITEM, item, 2922 "L3 Geneve is not enabled by device" 2923 " parameter and/or not configured in" 2924 " firmware"); 2925 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2926 return rte_flow_error_set(error, ENOTSUP, 2927 RTE_FLOW_ERROR_TYPE_ITEM, item, 2928 "multiple tunnel layers not" 2929 " supported"); 2930 /* 2931 * Verify only UDPv4 is present as defined in 2932 * https://tools.ietf.org/html/rfc7348 2933 */ 2934 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2935 return rte_flow_error_set(error, EINVAL, 2936 RTE_FLOW_ERROR_TYPE_ITEM, item, 2937 "no outer UDP layer found"); 2938 if (!mask) 2939 mask = &rte_flow_item_geneve_mask; 2940 ret = mlx5_flow_item_acceptable 2941 (item, (const uint8_t *)mask, 2942 (const uint8_t *)&nic_mask, 2943 sizeof(struct rte_flow_item_geneve), 2944 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2945 if (ret) 2946 return ret; 2947 if (spec) { 2948 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2949 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2950 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2951 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2952 return rte_flow_error_set(error, ENOTSUP, 2953 RTE_FLOW_ERROR_TYPE_ITEM, 2954 item, 2955 "Geneve protocol unsupported" 2956 " fields are being used"); 2957 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2958 return rte_flow_error_set 2959 (error, ENOTSUP, 2960 RTE_FLOW_ERROR_TYPE_ITEM, 2961 item, 2962 "Unsupported Geneve options length"); 2963 } 2964 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2965 return rte_flow_error_set 2966 (error, ENOTSUP, 2967 RTE_FLOW_ERROR_TYPE_ITEM, item, 2968 "Geneve tunnel must be fully defined"); 2969 return 0; 2970 } 2971 2972 /** 2973 * Validate Geneve TLV option item. 2974 * 2975 * @param[in] item 2976 * Item specification. 2977 * @param[in] last_item 2978 * Previous validated item in the pattern items. 2979 * @param[in] geneve_item 2980 * Previous GENEVE item specification. 2981 * @param[in] dev 2982 * Pointer to the rte_eth_dev structure. 2983 * @param[out] error 2984 * Pointer to error structure. 2985 * 2986 * @return 2987 * 0 on success, a negative errno value otherwise and rte_errno is set. 2988 */ 2989 int 2990 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2991 uint64_t last_item, 2992 const struct rte_flow_item *geneve_item, 2993 struct rte_eth_dev *dev, 2994 struct rte_flow_error *error) 2995 { 2996 struct mlx5_priv *priv = dev->data->dev_private; 2997 struct mlx5_dev_ctx_shared *sh = priv->sh; 2998 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2999 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 3000 uint8_t data_max_supported = 3001 hca_attr->max_geneve_tlv_option_data_len * 4; 3002 struct mlx5_dev_config *config = &priv->config; 3003 const struct rte_flow_item_geneve *geneve_spec; 3004 const struct rte_flow_item_geneve *geneve_mask; 3005 const struct rte_flow_item_geneve_opt *spec = item->spec; 3006 const struct rte_flow_item_geneve_opt *mask = item->mask; 3007 unsigned int i; 3008 unsigned int data_len; 3009 uint8_t tlv_option_len; 3010 uint16_t optlen_m, optlen_v; 3011 const struct rte_flow_item_geneve_opt full_mask = { 3012 .option_class = RTE_BE16(0xffff), 3013 .option_type = 0xff, 3014 .option_len = 0x1f, 3015 }; 3016 3017 if (!mask) 3018 mask = &rte_flow_item_geneve_opt_mask; 3019 if (!spec) 3020 return rte_flow_error_set 3021 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3022 "Geneve TLV opt class/type/length must be specified"); 3023 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3024 return rte_flow_error_set 3025 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3026 "Geneve TLV opt length exceeds the limit (31)"); 3027 /* Check if class type and length masks are full. */ 3028 if (full_mask.option_class != mask->option_class || 3029 full_mask.option_type != mask->option_type || 3030 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3031 return rte_flow_error_set 3032 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3033 "Geneve TLV opt class/type/length masks must be full"); 3034 /* Check if length is supported */ 3035 if ((uint32_t)spec->option_len > 3036 config->hca_attr.max_geneve_tlv_option_data_len) 3037 return rte_flow_error_set 3038 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3039 "Geneve TLV opt length not supported"); 3040 if (config->hca_attr.max_geneve_tlv_options > 1) 3041 DRV_LOG(DEBUG, 3042 "max_geneve_tlv_options supports more than 1 option"); 3043 /* Check GENEVE item preceding. */ 3044 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3045 return rte_flow_error_set 3046 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3047 "Geneve opt item must be preceded with Geneve item"); 3048 geneve_spec = geneve_item->spec; 3049 geneve_mask = geneve_item->mask ? geneve_item->mask : 3050 &rte_flow_item_geneve_mask; 3051 /* Check if GENEVE TLV option size doesn't exceed option length */ 3052 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3053 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3054 tlv_option_len = spec->option_len & mask->option_len; 3055 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3056 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3057 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3058 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3059 if ((optlen_v & optlen_m) <= tlv_option_len) 3060 return rte_flow_error_set 3061 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3062 "GENEVE TLV option length exceeds optlen"); 3063 } 3064 /* Check if length is 0 or data is 0. */ 3065 if (spec->data == NULL || spec->option_len == 0) 3066 return rte_flow_error_set 3067 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3068 "Geneve TLV opt with zero data/length not supported"); 3069 /* Check not all data & mask are 0. */ 3070 data_len = spec->option_len * 4; 3071 if (mask->data == NULL) { 3072 for (i = 0; i < data_len; i++) 3073 if (spec->data[i]) 3074 break; 3075 if (i == data_len) 3076 return rte_flow_error_set(error, ENOTSUP, 3077 RTE_FLOW_ERROR_TYPE_ITEM, item, 3078 "Can't match on Geneve option data 0"); 3079 } else { 3080 for (i = 0; i < data_len; i++) 3081 if (spec->data[i] & mask->data[i]) 3082 break; 3083 if (i == data_len) 3084 return rte_flow_error_set(error, ENOTSUP, 3085 RTE_FLOW_ERROR_TYPE_ITEM, item, 3086 "Can't match on Geneve option data and mask 0"); 3087 /* Check data mask supported. */ 3088 for (i = data_max_supported; i < data_len ; i++) 3089 if (mask->data[i]) 3090 return rte_flow_error_set(error, ENOTSUP, 3091 RTE_FLOW_ERROR_TYPE_ITEM, item, 3092 "Data mask is of unsupported size"); 3093 } 3094 /* Check GENEVE option is supported in NIC. */ 3095 if (!config->hca_attr.geneve_tlv_opt) 3096 return rte_flow_error_set 3097 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3098 "Geneve TLV opt not supported"); 3099 /* Check if we already have geneve option with different type/class. */ 3100 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3101 geneve_opt_resource = sh->geneve_tlv_option_resource; 3102 if (geneve_opt_resource != NULL) 3103 if (geneve_opt_resource->option_class != spec->option_class || 3104 geneve_opt_resource->option_type != spec->option_type || 3105 geneve_opt_resource->length != spec->option_len) { 3106 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3107 return rte_flow_error_set(error, ENOTSUP, 3108 RTE_FLOW_ERROR_TYPE_ITEM, item, 3109 "Only one Geneve TLV option supported"); 3110 } 3111 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3112 return 0; 3113 } 3114 3115 /** 3116 * Validate MPLS item. 3117 * 3118 * @param[in] dev 3119 * Pointer to the rte_eth_dev structure. 3120 * @param[in] item 3121 * Item specification. 3122 * @param[in] item_flags 3123 * Bit-fields that holds the items detected until now. 3124 * @param[in] prev_layer 3125 * The protocol layer indicated in previous item. 3126 * @param[out] error 3127 * Pointer to error structure. 3128 * 3129 * @return 3130 * 0 on success, a negative errno value otherwise and rte_errno is set. 3131 */ 3132 int 3133 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3134 const struct rte_flow_item *item __rte_unused, 3135 uint64_t item_flags __rte_unused, 3136 uint64_t prev_layer __rte_unused, 3137 struct rte_flow_error *error) 3138 { 3139 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3140 const struct rte_flow_item_mpls *mask = item->mask; 3141 struct mlx5_priv *priv = dev->data->dev_private; 3142 int ret; 3143 3144 if (!priv->config.mpls_en) 3145 return rte_flow_error_set(error, ENOTSUP, 3146 RTE_FLOW_ERROR_TYPE_ITEM, item, 3147 "MPLS not supported or" 3148 " disabled in firmware" 3149 " configuration."); 3150 /* MPLS over UDP, GRE is allowed */ 3151 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3152 MLX5_FLOW_LAYER_GRE | 3153 MLX5_FLOW_LAYER_GRE_KEY))) 3154 return rte_flow_error_set(error, EINVAL, 3155 RTE_FLOW_ERROR_TYPE_ITEM, item, 3156 "protocol filtering not compatible" 3157 " with MPLS layer"); 3158 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3159 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3160 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3161 return rte_flow_error_set(error, ENOTSUP, 3162 RTE_FLOW_ERROR_TYPE_ITEM, item, 3163 "multiple tunnel layers not" 3164 " supported"); 3165 if (!mask) 3166 mask = &rte_flow_item_mpls_mask; 3167 ret = mlx5_flow_item_acceptable 3168 (item, (const uint8_t *)mask, 3169 (const uint8_t *)&rte_flow_item_mpls_mask, 3170 sizeof(struct rte_flow_item_mpls), 3171 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3172 if (ret < 0) 3173 return ret; 3174 return 0; 3175 #else 3176 return rte_flow_error_set(error, ENOTSUP, 3177 RTE_FLOW_ERROR_TYPE_ITEM, item, 3178 "MPLS is not supported by Verbs, please" 3179 " update."); 3180 #endif 3181 } 3182 3183 /** 3184 * Validate NVGRE item. 3185 * 3186 * @param[in] item 3187 * Item specification. 3188 * @param[in] item_flags 3189 * Bit flags to mark detected items. 3190 * @param[in] target_protocol 3191 * The next protocol in the previous item. 3192 * @param[out] error 3193 * Pointer to error structure. 3194 * 3195 * @return 3196 * 0 on success, a negative errno value otherwise and rte_errno is set. 3197 */ 3198 int 3199 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3200 uint64_t item_flags, 3201 uint8_t target_protocol, 3202 struct rte_flow_error *error) 3203 { 3204 const struct rte_flow_item_nvgre *mask = item->mask; 3205 int ret; 3206 3207 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3208 return rte_flow_error_set(error, EINVAL, 3209 RTE_FLOW_ERROR_TYPE_ITEM, item, 3210 "protocol filtering not compatible" 3211 " with this GRE layer"); 3212 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3213 return rte_flow_error_set(error, ENOTSUP, 3214 RTE_FLOW_ERROR_TYPE_ITEM, item, 3215 "multiple tunnel layers not" 3216 " supported"); 3217 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3218 return rte_flow_error_set(error, ENOTSUP, 3219 RTE_FLOW_ERROR_TYPE_ITEM, item, 3220 "L3 Layer is missing"); 3221 if (!mask) 3222 mask = &rte_flow_item_nvgre_mask; 3223 ret = mlx5_flow_item_acceptable 3224 (item, (const uint8_t *)mask, 3225 (const uint8_t *)&rte_flow_item_nvgre_mask, 3226 sizeof(struct rte_flow_item_nvgre), 3227 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3228 if (ret < 0) 3229 return ret; 3230 return 0; 3231 } 3232 3233 /** 3234 * Validate eCPRI item. 3235 * 3236 * @param[in] item 3237 * Item specification. 3238 * @param[in] item_flags 3239 * Bit-fields that holds the items detected until now. 3240 * @param[in] last_item 3241 * Previous validated item in the pattern items. 3242 * @param[in] ether_type 3243 * Type in the ethernet layer header (including dot1q). 3244 * @param[in] acc_mask 3245 * Acceptable mask, if NULL default internal default mask 3246 * will be used to check whether item fields are supported. 3247 * @param[out] error 3248 * Pointer to error structure. 3249 * 3250 * @return 3251 * 0 on success, a negative errno value otherwise and rte_errno is set. 3252 */ 3253 int 3254 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3255 uint64_t item_flags, 3256 uint64_t last_item, 3257 uint16_t ether_type, 3258 const struct rte_flow_item_ecpri *acc_mask, 3259 struct rte_flow_error *error) 3260 { 3261 const struct rte_flow_item_ecpri *mask = item->mask; 3262 const struct rte_flow_item_ecpri nic_mask = { 3263 .hdr = { 3264 .common = { 3265 .u32 = 3266 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3267 .type = 0xFF, 3268 }).u32), 3269 }, 3270 .dummy[0] = 0xFFFFFFFF, 3271 }, 3272 }; 3273 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3274 MLX5_FLOW_LAYER_OUTER_VLAN); 3275 struct rte_flow_item_ecpri mask_lo; 3276 3277 if (!(last_item & outer_l2_vlan) && 3278 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3279 return rte_flow_error_set(error, EINVAL, 3280 RTE_FLOW_ERROR_TYPE_ITEM, item, 3281 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3282 if ((last_item & outer_l2_vlan) && ether_type && 3283 ether_type != RTE_ETHER_TYPE_ECPRI) 3284 return rte_flow_error_set(error, EINVAL, 3285 RTE_FLOW_ERROR_TYPE_ITEM, item, 3286 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3287 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3288 return rte_flow_error_set(error, EINVAL, 3289 RTE_FLOW_ERROR_TYPE_ITEM, item, 3290 "eCPRI with tunnel is not supported right now"); 3291 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3292 return rte_flow_error_set(error, ENOTSUP, 3293 RTE_FLOW_ERROR_TYPE_ITEM, item, 3294 "multiple L3 layers not supported"); 3295 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3296 return rte_flow_error_set(error, EINVAL, 3297 RTE_FLOW_ERROR_TYPE_ITEM, item, 3298 "eCPRI cannot coexist with a TCP layer"); 3299 /* In specification, eCPRI could be over UDP layer. */ 3300 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3301 return rte_flow_error_set(error, EINVAL, 3302 RTE_FLOW_ERROR_TYPE_ITEM, item, 3303 "eCPRI over UDP layer is not yet supported right now"); 3304 /* Mask for type field in common header could be zero. */ 3305 if (!mask) 3306 mask = &rte_flow_item_ecpri_mask; 3307 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3308 /* Input mask is in big-endian format. */ 3309 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3310 return rte_flow_error_set(error, EINVAL, 3311 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3312 "partial mask is not supported for protocol"); 3313 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3314 return rte_flow_error_set(error, EINVAL, 3315 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3316 "message header mask must be after a type mask"); 3317 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3318 acc_mask ? (const uint8_t *)acc_mask 3319 : (const uint8_t *)&nic_mask, 3320 sizeof(struct rte_flow_item_ecpri), 3321 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3322 } 3323 3324 static int 3325 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3326 const struct rte_flow_attr *attr __rte_unused, 3327 const struct rte_flow_item items[] __rte_unused, 3328 const struct rte_flow_action actions[] __rte_unused, 3329 bool external __rte_unused, 3330 int hairpin __rte_unused, 3331 struct rte_flow_error *error) 3332 { 3333 return rte_flow_error_set(error, ENOTSUP, 3334 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3335 } 3336 3337 static struct mlx5_flow * 3338 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3339 const struct rte_flow_attr *attr __rte_unused, 3340 const struct rte_flow_item items[] __rte_unused, 3341 const struct rte_flow_action actions[] __rte_unused, 3342 struct rte_flow_error *error) 3343 { 3344 rte_flow_error_set(error, ENOTSUP, 3345 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3346 return NULL; 3347 } 3348 3349 static int 3350 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3351 struct mlx5_flow *dev_flow __rte_unused, 3352 const struct rte_flow_attr *attr __rte_unused, 3353 const struct rte_flow_item items[] __rte_unused, 3354 const struct rte_flow_action actions[] __rte_unused, 3355 struct rte_flow_error *error) 3356 { 3357 return rte_flow_error_set(error, ENOTSUP, 3358 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3359 } 3360 3361 static int 3362 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3363 struct rte_flow *flow __rte_unused, 3364 struct rte_flow_error *error) 3365 { 3366 return rte_flow_error_set(error, ENOTSUP, 3367 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3368 } 3369 3370 static void 3371 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3372 struct rte_flow *flow __rte_unused) 3373 { 3374 } 3375 3376 static void 3377 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3378 struct rte_flow *flow __rte_unused) 3379 { 3380 } 3381 3382 static int 3383 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3384 struct rte_flow *flow __rte_unused, 3385 const struct rte_flow_action *actions __rte_unused, 3386 void *data __rte_unused, 3387 struct rte_flow_error *error) 3388 { 3389 return rte_flow_error_set(error, ENOTSUP, 3390 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3391 } 3392 3393 static int 3394 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3395 uint32_t domains __rte_unused, 3396 uint32_t flags __rte_unused) 3397 { 3398 return 0; 3399 } 3400 3401 /* Void driver to protect from null pointer reference. */ 3402 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3403 .validate = flow_null_validate, 3404 .prepare = flow_null_prepare, 3405 .translate = flow_null_translate, 3406 .apply = flow_null_apply, 3407 .remove = flow_null_remove, 3408 .destroy = flow_null_destroy, 3409 .query = flow_null_query, 3410 .sync_domain = flow_null_sync_domain, 3411 }; 3412 3413 /** 3414 * Select flow driver type according to flow attributes and device 3415 * configuration. 3416 * 3417 * @param[in] dev 3418 * Pointer to the dev structure. 3419 * @param[in] attr 3420 * Pointer to the flow attributes. 3421 * 3422 * @return 3423 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3424 */ 3425 static enum mlx5_flow_drv_type 3426 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3427 { 3428 struct mlx5_priv *priv = dev->data->dev_private; 3429 /* The OS can determine first a specific flow type (DV, VERBS) */ 3430 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3431 3432 if (type != MLX5_FLOW_TYPE_MAX) 3433 return type; 3434 /* If no OS specific type - continue with DV/VERBS selection */ 3435 if (attr->transfer && priv->config.dv_esw_en) 3436 type = MLX5_FLOW_TYPE_DV; 3437 if (!attr->transfer) 3438 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3439 MLX5_FLOW_TYPE_VERBS; 3440 return type; 3441 } 3442 3443 #define flow_get_drv_ops(type) flow_drv_ops[type] 3444 3445 /** 3446 * Flow driver validation API. This abstracts calling driver specific functions. 3447 * The type of flow driver is determined according to flow attributes. 3448 * 3449 * @param[in] dev 3450 * Pointer to the dev structure. 3451 * @param[in] attr 3452 * Pointer to the flow attributes. 3453 * @param[in] items 3454 * Pointer to the list of items. 3455 * @param[in] actions 3456 * Pointer to the list of actions. 3457 * @param[in] external 3458 * This flow rule is created by request external to PMD. 3459 * @param[in] hairpin 3460 * Number of hairpin TX actions, 0 means classic flow. 3461 * @param[out] error 3462 * Pointer to the error structure. 3463 * 3464 * @return 3465 * 0 on success, a negative errno value otherwise and rte_errno is set. 3466 */ 3467 static inline int 3468 flow_drv_validate(struct rte_eth_dev *dev, 3469 const struct rte_flow_attr *attr, 3470 const struct rte_flow_item items[], 3471 const struct rte_flow_action actions[], 3472 bool external, int hairpin, struct rte_flow_error *error) 3473 { 3474 const struct mlx5_flow_driver_ops *fops; 3475 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3476 3477 fops = flow_get_drv_ops(type); 3478 return fops->validate(dev, attr, items, actions, external, 3479 hairpin, error); 3480 } 3481 3482 /** 3483 * Flow driver preparation API. This abstracts calling driver specific 3484 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3485 * calculates the size of memory required for device flow, allocates the memory, 3486 * initializes the device flow and returns the pointer. 3487 * 3488 * @note 3489 * This function initializes device flow structure such as dv or verbs in 3490 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3491 * rest. For example, adding returning device flow to flow->dev_flow list and 3492 * setting backward reference to the flow should be done out of this function. 3493 * layers field is not filled either. 3494 * 3495 * @param[in] dev 3496 * Pointer to the dev structure. 3497 * @param[in] attr 3498 * Pointer to the flow attributes. 3499 * @param[in] items 3500 * Pointer to the list of items. 3501 * @param[in] actions 3502 * Pointer to the list of actions. 3503 * @param[in] flow_idx 3504 * This memory pool index to the flow. 3505 * @param[out] error 3506 * Pointer to the error structure. 3507 * 3508 * @return 3509 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3510 */ 3511 static inline struct mlx5_flow * 3512 flow_drv_prepare(struct rte_eth_dev *dev, 3513 const struct rte_flow *flow, 3514 const struct rte_flow_attr *attr, 3515 const struct rte_flow_item items[], 3516 const struct rte_flow_action actions[], 3517 uint32_t flow_idx, 3518 struct rte_flow_error *error) 3519 { 3520 const struct mlx5_flow_driver_ops *fops; 3521 enum mlx5_flow_drv_type type = flow->drv_type; 3522 struct mlx5_flow *mlx5_flow = NULL; 3523 3524 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3525 fops = flow_get_drv_ops(type); 3526 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3527 if (mlx5_flow) 3528 mlx5_flow->flow_idx = flow_idx; 3529 return mlx5_flow; 3530 } 3531 3532 /** 3533 * Flow driver translation API. This abstracts calling driver specific 3534 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3535 * translates a generic flow into a driver flow. flow_drv_prepare() must 3536 * precede. 3537 * 3538 * @note 3539 * dev_flow->layers could be filled as a result of parsing during translation 3540 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3541 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3542 * flow->actions could be overwritten even though all the expanded dev_flows 3543 * have the same actions. 3544 * 3545 * @param[in] dev 3546 * Pointer to the rte dev structure. 3547 * @param[in, out] dev_flow 3548 * Pointer to the mlx5 flow. 3549 * @param[in] attr 3550 * Pointer to the flow attributes. 3551 * @param[in] items 3552 * Pointer to the list of items. 3553 * @param[in] actions 3554 * Pointer to the list of actions. 3555 * @param[out] error 3556 * Pointer to the error structure. 3557 * 3558 * @return 3559 * 0 on success, a negative errno value otherwise and rte_errno is set. 3560 */ 3561 static inline int 3562 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3563 const struct rte_flow_attr *attr, 3564 const struct rte_flow_item items[], 3565 const struct rte_flow_action actions[], 3566 struct rte_flow_error *error) 3567 { 3568 const struct mlx5_flow_driver_ops *fops; 3569 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3570 3571 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3572 fops = flow_get_drv_ops(type); 3573 return fops->translate(dev, dev_flow, attr, items, actions, error); 3574 } 3575 3576 /** 3577 * Flow driver apply API. This abstracts calling driver specific functions. 3578 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3579 * translated driver flows on to device. flow_drv_translate() must precede. 3580 * 3581 * @param[in] dev 3582 * Pointer to Ethernet device structure. 3583 * @param[in, out] flow 3584 * Pointer to flow structure. 3585 * @param[out] error 3586 * Pointer to error structure. 3587 * 3588 * @return 3589 * 0 on success, a negative errno value otherwise and rte_errno is set. 3590 */ 3591 static inline int 3592 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3593 struct rte_flow_error *error) 3594 { 3595 const struct mlx5_flow_driver_ops *fops; 3596 enum mlx5_flow_drv_type type = flow->drv_type; 3597 3598 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3599 fops = flow_get_drv_ops(type); 3600 return fops->apply(dev, flow, error); 3601 } 3602 3603 /** 3604 * Flow driver destroy API. This abstracts calling driver specific functions. 3605 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3606 * on device and releases resources of the flow. 3607 * 3608 * @param[in] dev 3609 * Pointer to Ethernet device. 3610 * @param[in, out] flow 3611 * Pointer to flow structure. 3612 */ 3613 static inline void 3614 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3615 { 3616 const struct mlx5_flow_driver_ops *fops; 3617 enum mlx5_flow_drv_type type = flow->drv_type; 3618 3619 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3620 fops = flow_get_drv_ops(type); 3621 fops->destroy(dev, flow); 3622 } 3623 3624 /** 3625 * Flow driver find RSS policy tbl API. This abstracts calling driver 3626 * specific functions. Parent flow (rte_flow) should have driver 3627 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3628 * 3629 * @param[in] dev 3630 * Pointer to Ethernet device. 3631 * @param[in, out] flow 3632 * Pointer to flow structure. 3633 * @param[in] policy 3634 * Pointer to meter policy table. 3635 * @param[in] rss_desc 3636 * Pointer to rss_desc 3637 */ 3638 static struct mlx5_flow_meter_sub_policy * 3639 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3640 struct rte_flow *flow, 3641 struct mlx5_flow_meter_policy *policy, 3642 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3643 { 3644 const struct mlx5_flow_driver_ops *fops; 3645 enum mlx5_flow_drv_type type = flow->drv_type; 3646 3647 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3648 fops = flow_get_drv_ops(type); 3649 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3650 } 3651 3652 /** 3653 * Flow driver color tag rule API. This abstracts calling driver 3654 * specific functions. Parent flow (rte_flow) should have driver 3655 * type (drv_type). It will create the color tag rules in hierarchy meter. 3656 * 3657 * @param[in] dev 3658 * Pointer to Ethernet device. 3659 * @param[in, out] flow 3660 * Pointer to flow structure. 3661 * @param[in] fm 3662 * Pointer to flow meter structure. 3663 * @param[in] src_port 3664 * The src port this extra rule should use. 3665 * @param[in] item 3666 * The src port id match item. 3667 * @param[out] error 3668 * Pointer to error structure. 3669 */ 3670 static int 3671 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3672 struct rte_flow *flow, 3673 struct mlx5_flow_meter_info *fm, 3674 int32_t src_port, 3675 const struct rte_flow_item *item, 3676 struct rte_flow_error *error) 3677 { 3678 const struct mlx5_flow_driver_ops *fops; 3679 enum mlx5_flow_drv_type type = flow->drv_type; 3680 3681 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3682 fops = flow_get_drv_ops(type); 3683 return fops->meter_hierarchy_rule_create(dev, fm, 3684 src_port, item, error); 3685 } 3686 3687 /** 3688 * Get RSS action from the action list. 3689 * 3690 * @param[in] dev 3691 * Pointer to Ethernet device. 3692 * @param[in] actions 3693 * Pointer to the list of actions. 3694 * @param[in] flow 3695 * Parent flow structure pointer. 3696 * 3697 * @return 3698 * Pointer to the RSS action if exist, else return NULL. 3699 */ 3700 static const struct rte_flow_action_rss* 3701 flow_get_rss_action(struct rte_eth_dev *dev, 3702 const struct rte_flow_action actions[]) 3703 { 3704 struct mlx5_priv *priv = dev->data->dev_private; 3705 const struct rte_flow_action_rss *rss = NULL; 3706 struct mlx5_meter_policy_action_container *acg; 3707 struct mlx5_meter_policy_action_container *acy; 3708 3709 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3710 switch (actions->type) { 3711 case RTE_FLOW_ACTION_TYPE_RSS: 3712 rss = actions->conf; 3713 break; 3714 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3715 { 3716 const struct rte_flow_action_sample *sample = 3717 actions->conf; 3718 const struct rte_flow_action *act = sample->actions; 3719 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3720 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3721 rss = act->conf; 3722 break; 3723 } 3724 case RTE_FLOW_ACTION_TYPE_METER: 3725 { 3726 uint32_t mtr_idx; 3727 struct mlx5_flow_meter_info *fm; 3728 struct mlx5_flow_meter_policy *policy; 3729 const struct rte_flow_action_meter *mtr = actions->conf; 3730 3731 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3732 if (fm && !fm->def_policy) { 3733 policy = mlx5_flow_meter_policy_find(dev, 3734 fm->policy_id, NULL); 3735 MLX5_ASSERT(policy); 3736 if (policy->is_hierarchy) { 3737 policy = 3738 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3739 policy); 3740 if (!policy) 3741 return NULL; 3742 } 3743 if (policy->is_rss) { 3744 acg = 3745 &policy->act_cnt[RTE_COLOR_GREEN]; 3746 acy = 3747 &policy->act_cnt[RTE_COLOR_YELLOW]; 3748 if (acg->fate_action == 3749 MLX5_FLOW_FATE_SHARED_RSS) 3750 rss = acg->rss->conf; 3751 else if (acy->fate_action == 3752 MLX5_FLOW_FATE_SHARED_RSS) 3753 rss = acy->rss->conf; 3754 } 3755 } 3756 break; 3757 } 3758 default: 3759 break; 3760 } 3761 } 3762 return rss; 3763 } 3764 3765 /** 3766 * Get ASO age action by index. 3767 * 3768 * @param[in] dev 3769 * Pointer to the Ethernet device structure. 3770 * @param[in] age_idx 3771 * Index to the ASO age action. 3772 * 3773 * @return 3774 * The specified ASO age action. 3775 */ 3776 struct mlx5_aso_age_action* 3777 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3778 { 3779 uint16_t pool_idx = age_idx & UINT16_MAX; 3780 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3781 struct mlx5_priv *priv = dev->data->dev_private; 3782 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3783 struct mlx5_aso_age_pool *pool; 3784 3785 rte_rwlock_read_lock(&mng->resize_rwl); 3786 pool = mng->pools[pool_idx]; 3787 rte_rwlock_read_unlock(&mng->resize_rwl); 3788 return &pool->actions[offset - 1]; 3789 } 3790 3791 /* maps indirect action to translated direct in some actions array */ 3792 struct mlx5_translated_action_handle { 3793 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3794 int index; /**< Index in related array of rte_flow_action. */ 3795 }; 3796 3797 /** 3798 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3799 * direct action if translation possible. 3800 * This functionality used to run same execution path for both direct and 3801 * indirect actions on flow create. All necessary preparations for indirect 3802 * action handling should be performed on *handle* actions list returned 3803 * from this call. 3804 * 3805 * @param[in] dev 3806 * Pointer to Ethernet device. 3807 * @param[in] actions 3808 * List of actions to translate. 3809 * @param[out] handle 3810 * List to store translated indirect action object handles. 3811 * @param[in, out] indir_n 3812 * Size of *handle* array. On return should be updated with number of 3813 * indirect actions retrieved from the *actions* list. 3814 * @param[out] translated_actions 3815 * List of actions where all indirect actions were translated to direct 3816 * if possible. NULL if no translation took place. 3817 * @param[out] error 3818 * Pointer to the error structure. 3819 * 3820 * @return 3821 * 0 on success, a negative errno value otherwise and rte_errno is set. 3822 */ 3823 static int 3824 flow_action_handles_translate(struct rte_eth_dev *dev, 3825 const struct rte_flow_action actions[], 3826 struct mlx5_translated_action_handle *handle, 3827 int *indir_n, 3828 struct rte_flow_action **translated_actions, 3829 struct rte_flow_error *error) 3830 { 3831 struct mlx5_priv *priv = dev->data->dev_private; 3832 struct rte_flow_action *translated = NULL; 3833 size_t actions_size; 3834 int n; 3835 int copied_n = 0; 3836 struct mlx5_translated_action_handle *handle_end = NULL; 3837 3838 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3839 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3840 continue; 3841 if (copied_n == *indir_n) { 3842 return rte_flow_error_set 3843 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3844 NULL, "too many shared actions"); 3845 } 3846 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3847 sizeof(actions[n].conf)); 3848 handle[copied_n].index = n; 3849 copied_n++; 3850 } 3851 n++; 3852 *indir_n = copied_n; 3853 if (!copied_n) 3854 return 0; 3855 actions_size = sizeof(struct rte_flow_action) * n; 3856 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3857 if (!translated) { 3858 rte_errno = ENOMEM; 3859 return -ENOMEM; 3860 } 3861 memcpy(translated, actions, actions_size); 3862 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3863 struct mlx5_shared_action_rss *shared_rss; 3864 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3865 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3866 uint32_t idx = act_idx & 3867 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3868 3869 switch (type) { 3870 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3871 shared_rss = mlx5_ipool_get 3872 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3873 translated[handle->index].type = 3874 RTE_FLOW_ACTION_TYPE_RSS; 3875 translated[handle->index].conf = 3876 &shared_rss->origin; 3877 break; 3878 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3879 translated[handle->index].type = 3880 (enum rte_flow_action_type) 3881 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3882 translated[handle->index].conf = (void *)(uintptr_t)idx; 3883 break; 3884 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3885 if (priv->sh->flow_hit_aso_en) { 3886 translated[handle->index].type = 3887 (enum rte_flow_action_type) 3888 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3889 translated[handle->index].conf = 3890 (void *)(uintptr_t)idx; 3891 break; 3892 } 3893 /* Fall-through */ 3894 case MLX5_INDIRECT_ACTION_TYPE_CT: 3895 if (priv->sh->ct_aso_en) { 3896 translated[handle->index].type = 3897 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3898 translated[handle->index].conf = 3899 (void *)(uintptr_t)idx; 3900 break; 3901 } 3902 /* Fall-through */ 3903 default: 3904 mlx5_free(translated); 3905 return rte_flow_error_set 3906 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3907 NULL, "invalid indirect action type"); 3908 } 3909 } 3910 *translated_actions = translated; 3911 return 0; 3912 } 3913 3914 /** 3915 * Get Shared RSS action from the action list. 3916 * 3917 * @param[in] dev 3918 * Pointer to Ethernet device. 3919 * @param[in] shared 3920 * Pointer to the list of actions. 3921 * @param[in] shared_n 3922 * Actions list length. 3923 * 3924 * @return 3925 * The MLX5 RSS action ID if exists, otherwise return 0. 3926 */ 3927 static uint32_t 3928 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3929 struct mlx5_translated_action_handle *handle, 3930 int shared_n) 3931 { 3932 struct mlx5_translated_action_handle *handle_end; 3933 struct mlx5_priv *priv = dev->data->dev_private; 3934 struct mlx5_shared_action_rss *shared_rss; 3935 3936 3937 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3938 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3939 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3940 uint32_t idx = act_idx & 3941 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3942 switch (type) { 3943 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3944 shared_rss = mlx5_ipool_get 3945 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3946 idx); 3947 __atomic_add_fetch(&shared_rss->refcnt, 1, 3948 __ATOMIC_RELAXED); 3949 return idx; 3950 default: 3951 break; 3952 } 3953 } 3954 return 0; 3955 } 3956 3957 static unsigned int 3958 find_graph_root(uint32_t rss_level) 3959 { 3960 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3961 MLX5_EXPANSION_ROOT_OUTER; 3962 } 3963 3964 /** 3965 * Get layer flags from the prefix flow. 3966 * 3967 * Some flows may be split to several subflows, the prefix subflow gets the 3968 * match items and the suffix sub flow gets the actions. 3969 * Some actions need the user defined match item flags to get the detail for 3970 * the action. 3971 * This function helps the suffix flow to get the item layer flags from prefix 3972 * subflow. 3973 * 3974 * @param[in] dev_flow 3975 * Pointer the created prefix subflow. 3976 * 3977 * @return 3978 * The layers get from prefix subflow. 3979 */ 3980 static inline uint64_t 3981 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3982 { 3983 uint64_t layers = 0; 3984 3985 /* 3986 * Layers bits could be localization, but usually the compiler will 3987 * help to do the optimization work for source code. 3988 * If no decap actions, use the layers directly. 3989 */ 3990 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3991 return dev_flow->handle->layers; 3992 /* Convert L3 layers with decap action. */ 3993 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3994 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3995 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3996 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3997 /* Convert L4 layers with decap action. */ 3998 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3999 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4000 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4001 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4002 return layers; 4003 } 4004 4005 /** 4006 * Get metadata split action information. 4007 * 4008 * @param[in] actions 4009 * Pointer to the list of actions. 4010 * @param[out] qrss 4011 * Pointer to the return pointer. 4012 * @param[out] qrss_type 4013 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4014 * if no QUEUE/RSS is found. 4015 * @param[out] encap_idx 4016 * Pointer to the index of the encap action if exists, otherwise the last 4017 * action index. 4018 * 4019 * @return 4020 * Total number of actions. 4021 */ 4022 static int 4023 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4024 const struct rte_flow_action **qrss, 4025 int *encap_idx) 4026 { 4027 const struct rte_flow_action_raw_encap *raw_encap; 4028 int actions_n = 0; 4029 int raw_decap_idx = -1; 4030 4031 *encap_idx = -1; 4032 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4033 switch (actions->type) { 4034 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4035 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4036 *encap_idx = actions_n; 4037 break; 4038 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4039 raw_decap_idx = actions_n; 4040 break; 4041 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4042 raw_encap = actions->conf; 4043 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4044 *encap_idx = raw_decap_idx != -1 ? 4045 raw_decap_idx : actions_n; 4046 break; 4047 case RTE_FLOW_ACTION_TYPE_QUEUE: 4048 case RTE_FLOW_ACTION_TYPE_RSS: 4049 *qrss = actions; 4050 break; 4051 default: 4052 break; 4053 } 4054 actions_n++; 4055 } 4056 if (*encap_idx == -1) 4057 *encap_idx = actions_n; 4058 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4059 return actions_n + 1; 4060 } 4061 4062 /** 4063 * Check if the action will change packet. 4064 * 4065 * @param dev 4066 * Pointer to Ethernet device. 4067 * @param[in] type 4068 * action type. 4069 * 4070 * @return 4071 * true if action will change packet, false otherwise. 4072 */ 4073 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4074 enum rte_flow_action_type type) 4075 { 4076 struct mlx5_priv *priv = dev->data->dev_private; 4077 4078 switch (type) { 4079 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4080 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4081 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4082 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4083 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4084 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4085 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4086 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4087 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4088 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4089 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4090 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4091 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4092 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4093 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4094 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4095 case RTE_FLOW_ACTION_TYPE_SET_META: 4096 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4097 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4098 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4099 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4100 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4101 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4102 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4103 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4104 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4105 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4106 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4107 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4108 return true; 4109 case RTE_FLOW_ACTION_TYPE_FLAG: 4110 case RTE_FLOW_ACTION_TYPE_MARK: 4111 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4112 return true; 4113 else 4114 return false; 4115 default: 4116 return false; 4117 } 4118 } 4119 4120 /** 4121 * Check meter action from the action list. 4122 * 4123 * @param dev 4124 * Pointer to Ethernet device. 4125 * @param[in] actions 4126 * Pointer to the list of actions. 4127 * @param[out] has_mtr 4128 * Pointer to the meter exist flag. 4129 * @param[out] has_modify 4130 * Pointer to the flag showing there's packet change action. 4131 * @param[out] meter_id 4132 * Pointer to the meter id. 4133 * 4134 * @return 4135 * Total number of actions. 4136 */ 4137 static int 4138 flow_check_meter_action(struct rte_eth_dev *dev, 4139 const struct rte_flow_action actions[], 4140 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4141 { 4142 const struct rte_flow_action_meter *mtr = NULL; 4143 int actions_n = 0; 4144 4145 MLX5_ASSERT(has_mtr); 4146 *has_mtr = false; 4147 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4148 switch (actions->type) { 4149 case RTE_FLOW_ACTION_TYPE_METER: 4150 mtr = actions->conf; 4151 *meter_id = mtr->mtr_id; 4152 *has_mtr = true; 4153 break; 4154 default: 4155 break; 4156 } 4157 if (!*has_mtr) 4158 *has_modify |= flow_check_modify_action_type(dev, 4159 actions->type); 4160 actions_n++; 4161 } 4162 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4163 return actions_n + 1; 4164 } 4165 4166 /** 4167 * Check if the flow should be split due to hairpin. 4168 * The reason for the split is that in current HW we can't 4169 * support encap and push-vlan on Rx, so if a flow contains 4170 * these actions we move it to Tx. 4171 * 4172 * @param dev 4173 * Pointer to Ethernet device. 4174 * @param[in] attr 4175 * Flow rule attributes. 4176 * @param[in] actions 4177 * Associated actions (list terminated by the END action). 4178 * 4179 * @return 4180 * > 0 the number of actions and the flow should be split, 4181 * 0 when no split required. 4182 */ 4183 static int 4184 flow_check_hairpin_split(struct rte_eth_dev *dev, 4185 const struct rte_flow_attr *attr, 4186 const struct rte_flow_action actions[]) 4187 { 4188 int queue_action = 0; 4189 int action_n = 0; 4190 int split = 0; 4191 const struct rte_flow_action_queue *queue; 4192 const struct rte_flow_action_rss *rss; 4193 const struct rte_flow_action_raw_encap *raw_encap; 4194 const struct rte_eth_hairpin_conf *conf; 4195 4196 if (!attr->ingress) 4197 return 0; 4198 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4199 switch (actions->type) { 4200 case RTE_FLOW_ACTION_TYPE_QUEUE: 4201 queue = actions->conf; 4202 if (queue == NULL) 4203 return 0; 4204 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4205 if (conf == NULL || conf->tx_explicit != 0) 4206 return 0; 4207 queue_action = 1; 4208 action_n++; 4209 break; 4210 case RTE_FLOW_ACTION_TYPE_RSS: 4211 rss = actions->conf; 4212 if (rss == NULL || rss->queue_num == 0) 4213 return 0; 4214 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4215 if (conf == NULL || conf->tx_explicit != 0) 4216 return 0; 4217 queue_action = 1; 4218 action_n++; 4219 break; 4220 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4221 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4222 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4223 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4224 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4225 split++; 4226 action_n++; 4227 break; 4228 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4229 raw_encap = actions->conf; 4230 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4231 split++; 4232 action_n++; 4233 break; 4234 default: 4235 action_n++; 4236 break; 4237 } 4238 } 4239 if (split && queue_action) 4240 return action_n; 4241 return 0; 4242 } 4243 4244 /* Declare flow create/destroy prototype in advance. */ 4245 static uint32_t 4246 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4247 const struct rte_flow_attr *attr, 4248 const struct rte_flow_item items[], 4249 const struct rte_flow_action actions[], 4250 bool external, struct rte_flow_error *error); 4251 4252 static void 4253 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4254 uint32_t flow_idx); 4255 4256 int 4257 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4258 struct mlx5_list_entry *entry, void *cb_ctx) 4259 { 4260 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4261 struct mlx5_flow_mreg_copy_resource *mcp_res = 4262 container_of(entry, typeof(*mcp_res), hlist_ent); 4263 4264 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4265 } 4266 4267 struct mlx5_list_entry * 4268 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4269 { 4270 struct rte_eth_dev *dev = tool_ctx; 4271 struct mlx5_priv *priv = dev->data->dev_private; 4272 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4273 struct mlx5_flow_mreg_copy_resource *mcp_res; 4274 struct rte_flow_error *error = ctx->error; 4275 uint32_t idx = 0; 4276 int ret; 4277 uint32_t mark_id = *(uint32_t *)(ctx->data); 4278 struct rte_flow_attr attr = { 4279 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4280 .ingress = 1, 4281 }; 4282 struct mlx5_rte_flow_item_tag tag_spec = { 4283 .data = mark_id, 4284 }; 4285 struct rte_flow_item items[] = { 4286 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4287 }; 4288 struct rte_flow_action_mark ftag = { 4289 .id = mark_id, 4290 }; 4291 struct mlx5_flow_action_copy_mreg cp_mreg = { 4292 .dst = REG_B, 4293 .src = REG_NON, 4294 }; 4295 struct rte_flow_action_jump jump = { 4296 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4297 }; 4298 struct rte_flow_action actions[] = { 4299 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4300 }; 4301 4302 /* Fill the register fields in the flow. */ 4303 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4304 if (ret < 0) 4305 return NULL; 4306 tag_spec.id = ret; 4307 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4308 if (ret < 0) 4309 return NULL; 4310 cp_mreg.src = ret; 4311 /* Provide the full width of FLAG specific value. */ 4312 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4313 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4314 /* Build a new flow. */ 4315 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4316 items[0] = (struct rte_flow_item){ 4317 .type = (enum rte_flow_item_type) 4318 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4319 .spec = &tag_spec, 4320 }; 4321 items[1] = (struct rte_flow_item){ 4322 .type = RTE_FLOW_ITEM_TYPE_END, 4323 }; 4324 actions[0] = (struct rte_flow_action){ 4325 .type = (enum rte_flow_action_type) 4326 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4327 .conf = &ftag, 4328 }; 4329 actions[1] = (struct rte_flow_action){ 4330 .type = (enum rte_flow_action_type) 4331 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4332 .conf = &cp_mreg, 4333 }; 4334 actions[2] = (struct rte_flow_action){ 4335 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4336 .conf = &jump, 4337 }; 4338 actions[3] = (struct rte_flow_action){ 4339 .type = RTE_FLOW_ACTION_TYPE_END, 4340 }; 4341 } else { 4342 /* Default rule, wildcard match. */ 4343 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4344 items[0] = (struct rte_flow_item){ 4345 .type = RTE_FLOW_ITEM_TYPE_END, 4346 }; 4347 actions[0] = (struct rte_flow_action){ 4348 .type = (enum rte_flow_action_type) 4349 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4350 .conf = &cp_mreg, 4351 }; 4352 actions[1] = (struct rte_flow_action){ 4353 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4354 .conf = &jump, 4355 }; 4356 actions[2] = (struct rte_flow_action){ 4357 .type = RTE_FLOW_ACTION_TYPE_END, 4358 }; 4359 } 4360 /* Build a new entry. */ 4361 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4362 if (!mcp_res) { 4363 rte_errno = ENOMEM; 4364 return NULL; 4365 } 4366 mcp_res->idx = idx; 4367 mcp_res->mark_id = mark_id; 4368 /* 4369 * The copy Flows are not included in any list. There 4370 * ones are referenced from other Flows and can not 4371 * be applied, removed, deleted in arbitrary order 4372 * by list traversing. 4373 */ 4374 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4375 &attr, items, actions, false, error); 4376 if (!mcp_res->rix_flow) { 4377 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4378 return NULL; 4379 } 4380 return &mcp_res->hlist_ent; 4381 } 4382 4383 struct mlx5_list_entry * 4384 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4385 void *cb_ctx __rte_unused) 4386 { 4387 struct rte_eth_dev *dev = tool_ctx; 4388 struct mlx5_priv *priv = dev->data->dev_private; 4389 struct mlx5_flow_mreg_copy_resource *mcp_res; 4390 uint32_t idx = 0; 4391 4392 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4393 if (!mcp_res) { 4394 rte_errno = ENOMEM; 4395 return NULL; 4396 } 4397 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4398 mcp_res->idx = idx; 4399 return &mcp_res->hlist_ent; 4400 } 4401 4402 void 4403 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4404 { 4405 struct mlx5_flow_mreg_copy_resource *mcp_res = 4406 container_of(entry, typeof(*mcp_res), hlist_ent); 4407 struct rte_eth_dev *dev = tool_ctx; 4408 struct mlx5_priv *priv = dev->data->dev_private; 4409 4410 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4411 } 4412 4413 /** 4414 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4415 * 4416 * As mark_id is unique, if there's already a registered flow for the mark_id, 4417 * return by increasing the reference counter of the resource. Otherwise, create 4418 * the resource (mcp_res) and flow. 4419 * 4420 * Flow looks like, 4421 * - If ingress port is ANY and reg_c[1] is mark_id, 4422 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4423 * 4424 * For default flow (zero mark_id), flow is like, 4425 * - If ingress port is ANY, 4426 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4427 * 4428 * @param dev 4429 * Pointer to Ethernet device. 4430 * @param mark_id 4431 * ID of MARK action, zero means default flow for META. 4432 * @param[out] error 4433 * Perform verbose error reporting if not NULL. 4434 * 4435 * @return 4436 * Associated resource on success, NULL otherwise and rte_errno is set. 4437 */ 4438 static struct mlx5_flow_mreg_copy_resource * 4439 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4440 struct rte_flow_error *error) 4441 { 4442 struct mlx5_priv *priv = dev->data->dev_private; 4443 struct mlx5_list_entry *entry; 4444 struct mlx5_flow_cb_ctx ctx = { 4445 .dev = dev, 4446 .error = error, 4447 .data = &mark_id, 4448 }; 4449 4450 /* Check if already registered. */ 4451 MLX5_ASSERT(priv->mreg_cp_tbl); 4452 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4453 if (!entry) 4454 return NULL; 4455 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4456 hlist_ent); 4457 } 4458 4459 void 4460 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4461 { 4462 struct mlx5_flow_mreg_copy_resource *mcp_res = 4463 container_of(entry, typeof(*mcp_res), hlist_ent); 4464 struct rte_eth_dev *dev = tool_ctx; 4465 struct mlx5_priv *priv = dev->data->dev_private; 4466 4467 MLX5_ASSERT(mcp_res->rix_flow); 4468 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4469 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4470 } 4471 4472 /** 4473 * Release flow in RX_CP_TBL. 4474 * 4475 * @param dev 4476 * Pointer to Ethernet device. 4477 * @flow 4478 * Parent flow for wich copying is provided. 4479 */ 4480 static void 4481 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4482 struct rte_flow *flow) 4483 { 4484 struct mlx5_flow_mreg_copy_resource *mcp_res; 4485 struct mlx5_priv *priv = dev->data->dev_private; 4486 4487 if (!flow->rix_mreg_copy) 4488 return; 4489 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4490 flow->rix_mreg_copy); 4491 if (!mcp_res || !priv->mreg_cp_tbl) 4492 return; 4493 MLX5_ASSERT(mcp_res->rix_flow); 4494 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4495 flow->rix_mreg_copy = 0; 4496 } 4497 4498 /** 4499 * Remove the default copy action from RX_CP_TBL. 4500 * 4501 * This functions is called in the mlx5_dev_start(). No thread safe 4502 * is guaranteed. 4503 * 4504 * @param dev 4505 * Pointer to Ethernet device. 4506 */ 4507 static void 4508 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4509 { 4510 struct mlx5_list_entry *entry; 4511 struct mlx5_priv *priv = dev->data->dev_private; 4512 struct mlx5_flow_cb_ctx ctx; 4513 uint32_t mark_id; 4514 4515 /* Check if default flow is registered. */ 4516 if (!priv->mreg_cp_tbl) 4517 return; 4518 mark_id = MLX5_DEFAULT_COPY_ID; 4519 ctx.data = &mark_id; 4520 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4521 if (!entry) 4522 return; 4523 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4524 } 4525 4526 /** 4527 * Add the default copy action in in RX_CP_TBL. 4528 * 4529 * This functions is called in the mlx5_dev_start(). No thread safe 4530 * is guaranteed. 4531 * 4532 * @param dev 4533 * Pointer to Ethernet device. 4534 * @param[out] error 4535 * Perform verbose error reporting if not NULL. 4536 * 4537 * @return 4538 * 0 for success, negative value otherwise and rte_errno is set. 4539 */ 4540 static int 4541 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4542 struct rte_flow_error *error) 4543 { 4544 struct mlx5_priv *priv = dev->data->dev_private; 4545 struct mlx5_flow_mreg_copy_resource *mcp_res; 4546 struct mlx5_flow_cb_ctx ctx; 4547 uint32_t mark_id; 4548 4549 /* Check whether extensive metadata feature is engaged. */ 4550 if (!priv->config.dv_flow_en || 4551 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4552 !mlx5_flow_ext_mreg_supported(dev) || 4553 !priv->sh->dv_regc0_mask) 4554 return 0; 4555 /* 4556 * Add default mreg copy flow may be called multiple time, but 4557 * only be called once in stop. Avoid register it twice. 4558 */ 4559 mark_id = MLX5_DEFAULT_COPY_ID; 4560 ctx.data = &mark_id; 4561 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4562 return 0; 4563 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4564 if (!mcp_res) 4565 return -rte_errno; 4566 return 0; 4567 } 4568 4569 /** 4570 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4571 * 4572 * All the flow having Q/RSS action should be split by 4573 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4574 * performs the following, 4575 * - CQE->flow_tag := reg_c[1] (MARK) 4576 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4577 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4578 * but there should be a flow per each MARK ID set by MARK action. 4579 * 4580 * For the aforementioned reason, if there's a MARK action in flow's action 4581 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4582 * the MARK ID to CQE's flow_tag like, 4583 * - If reg_c[1] is mark_id, 4584 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4585 * 4586 * For SET_META action which stores value in reg_c[0], as the destination is 4587 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4588 * MARK ID means the default flow. The default flow looks like, 4589 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4590 * 4591 * @param dev 4592 * Pointer to Ethernet device. 4593 * @param flow 4594 * Pointer to flow structure. 4595 * @param[in] actions 4596 * Pointer to the list of actions. 4597 * @param[out] error 4598 * Perform verbose error reporting if not NULL. 4599 * 4600 * @return 4601 * 0 on success, negative value otherwise and rte_errno is set. 4602 */ 4603 static int 4604 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4605 struct rte_flow *flow, 4606 const struct rte_flow_action *actions, 4607 struct rte_flow_error *error) 4608 { 4609 struct mlx5_priv *priv = dev->data->dev_private; 4610 struct mlx5_dev_config *config = &priv->config; 4611 struct mlx5_flow_mreg_copy_resource *mcp_res; 4612 const struct rte_flow_action_mark *mark; 4613 4614 /* Check whether extensive metadata feature is engaged. */ 4615 if (!config->dv_flow_en || 4616 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4617 !mlx5_flow_ext_mreg_supported(dev) || 4618 !priv->sh->dv_regc0_mask) 4619 return 0; 4620 /* Find MARK action. */ 4621 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4622 switch (actions->type) { 4623 case RTE_FLOW_ACTION_TYPE_FLAG: 4624 mcp_res = flow_mreg_add_copy_action 4625 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4626 if (!mcp_res) 4627 return -rte_errno; 4628 flow->rix_mreg_copy = mcp_res->idx; 4629 return 0; 4630 case RTE_FLOW_ACTION_TYPE_MARK: 4631 mark = (const struct rte_flow_action_mark *) 4632 actions->conf; 4633 mcp_res = 4634 flow_mreg_add_copy_action(dev, mark->id, error); 4635 if (!mcp_res) 4636 return -rte_errno; 4637 flow->rix_mreg_copy = mcp_res->idx; 4638 return 0; 4639 default: 4640 break; 4641 } 4642 } 4643 return 0; 4644 } 4645 4646 #define MLX5_MAX_SPLIT_ACTIONS 24 4647 #define MLX5_MAX_SPLIT_ITEMS 24 4648 4649 /** 4650 * Split the hairpin flow. 4651 * Since HW can't support encap and push-vlan on Rx, we move these 4652 * actions to Tx. 4653 * If the count action is after the encap then we also 4654 * move the count action. in this case the count will also measure 4655 * the outer bytes. 4656 * 4657 * @param dev 4658 * Pointer to Ethernet device. 4659 * @param[in] actions 4660 * Associated actions (list terminated by the END action). 4661 * @param[out] actions_rx 4662 * Rx flow actions. 4663 * @param[out] actions_tx 4664 * Tx flow actions.. 4665 * @param[out] pattern_tx 4666 * The pattern items for the Tx flow. 4667 * @param[out] flow_id 4668 * The flow ID connected to this flow. 4669 * 4670 * @return 4671 * 0 on success. 4672 */ 4673 static int 4674 flow_hairpin_split(struct rte_eth_dev *dev, 4675 const struct rte_flow_action actions[], 4676 struct rte_flow_action actions_rx[], 4677 struct rte_flow_action actions_tx[], 4678 struct rte_flow_item pattern_tx[], 4679 uint32_t flow_id) 4680 { 4681 const struct rte_flow_action_raw_encap *raw_encap; 4682 const struct rte_flow_action_raw_decap *raw_decap; 4683 struct mlx5_rte_flow_action_set_tag *set_tag; 4684 struct rte_flow_action *tag_action; 4685 struct mlx5_rte_flow_item_tag *tag_item; 4686 struct rte_flow_item *item; 4687 char *addr; 4688 int encap = 0; 4689 4690 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4691 switch (actions->type) { 4692 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4693 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4694 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4695 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4696 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4697 rte_memcpy(actions_tx, actions, 4698 sizeof(struct rte_flow_action)); 4699 actions_tx++; 4700 break; 4701 case RTE_FLOW_ACTION_TYPE_COUNT: 4702 if (encap) { 4703 rte_memcpy(actions_tx, actions, 4704 sizeof(struct rte_flow_action)); 4705 actions_tx++; 4706 } else { 4707 rte_memcpy(actions_rx, actions, 4708 sizeof(struct rte_flow_action)); 4709 actions_rx++; 4710 } 4711 break; 4712 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4713 raw_encap = actions->conf; 4714 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4715 memcpy(actions_tx, actions, 4716 sizeof(struct rte_flow_action)); 4717 actions_tx++; 4718 encap = 1; 4719 } else { 4720 rte_memcpy(actions_rx, actions, 4721 sizeof(struct rte_flow_action)); 4722 actions_rx++; 4723 } 4724 break; 4725 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4726 raw_decap = actions->conf; 4727 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4728 memcpy(actions_tx, actions, 4729 sizeof(struct rte_flow_action)); 4730 actions_tx++; 4731 } else { 4732 rte_memcpy(actions_rx, actions, 4733 sizeof(struct rte_flow_action)); 4734 actions_rx++; 4735 } 4736 break; 4737 default: 4738 rte_memcpy(actions_rx, actions, 4739 sizeof(struct rte_flow_action)); 4740 actions_rx++; 4741 break; 4742 } 4743 } 4744 /* Add set meta action and end action for the Rx flow. */ 4745 tag_action = actions_rx; 4746 tag_action->type = (enum rte_flow_action_type) 4747 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4748 actions_rx++; 4749 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4750 actions_rx++; 4751 set_tag = (void *)actions_rx; 4752 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4753 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4754 .data = flow_id, 4755 }; 4756 MLX5_ASSERT(set_tag->id > REG_NON); 4757 tag_action->conf = set_tag; 4758 /* Create Tx item list. */ 4759 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4760 addr = (void *)&pattern_tx[2]; 4761 item = pattern_tx; 4762 item->type = (enum rte_flow_item_type) 4763 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4764 tag_item = (void *)addr; 4765 tag_item->data = flow_id; 4766 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4767 MLX5_ASSERT(set_tag->id > REG_NON); 4768 item->spec = tag_item; 4769 addr += sizeof(struct mlx5_rte_flow_item_tag); 4770 tag_item = (void *)addr; 4771 tag_item->data = UINT32_MAX; 4772 tag_item->id = UINT16_MAX; 4773 item->mask = tag_item; 4774 item->last = NULL; 4775 item++; 4776 item->type = RTE_FLOW_ITEM_TYPE_END; 4777 return 0; 4778 } 4779 4780 /** 4781 * The last stage of splitting chain, just creates the subflow 4782 * without any modification. 4783 * 4784 * @param[in] dev 4785 * Pointer to Ethernet device. 4786 * @param[in] flow 4787 * Parent flow structure pointer. 4788 * @param[in, out] sub_flow 4789 * Pointer to return the created subflow, may be NULL. 4790 * @param[in] attr 4791 * Flow rule attributes. 4792 * @param[in] items 4793 * Pattern specification (list terminated by the END pattern item). 4794 * @param[in] actions 4795 * Associated actions (list terminated by the END action). 4796 * @param[in] flow_split_info 4797 * Pointer to flow split info structure. 4798 * @param[out] error 4799 * Perform verbose error reporting if not NULL. 4800 * @return 4801 * 0 on success, negative value otherwise 4802 */ 4803 static int 4804 flow_create_split_inner(struct rte_eth_dev *dev, 4805 struct rte_flow *flow, 4806 struct mlx5_flow **sub_flow, 4807 const struct rte_flow_attr *attr, 4808 const struct rte_flow_item items[], 4809 const struct rte_flow_action actions[], 4810 struct mlx5_flow_split_info *flow_split_info, 4811 struct rte_flow_error *error) 4812 { 4813 struct mlx5_flow *dev_flow; 4814 4815 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4816 flow_split_info->flow_idx, error); 4817 if (!dev_flow) 4818 return -rte_errno; 4819 dev_flow->flow = flow; 4820 dev_flow->external = flow_split_info->external; 4821 dev_flow->skip_scale = flow_split_info->skip_scale; 4822 /* Subflow object was created, we must include one in the list. */ 4823 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4824 dev_flow->handle, next); 4825 /* 4826 * If dev_flow is as one of the suffix flow, some actions in suffix 4827 * flow may need some user defined item layer flags, and pass the 4828 * Metadata rxq mark flag to suffix flow as well. 4829 */ 4830 if (flow_split_info->prefix_layers) 4831 dev_flow->handle->layers = flow_split_info->prefix_layers; 4832 if (flow_split_info->prefix_mark) 4833 dev_flow->handle->mark = 1; 4834 if (sub_flow) 4835 *sub_flow = dev_flow; 4836 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4837 dev_flow->dv.table_id = flow_split_info->table_id; 4838 #endif 4839 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4840 } 4841 4842 /** 4843 * Get the sub policy of a meter. 4844 * 4845 * @param[in] dev 4846 * Pointer to Ethernet device. 4847 * @param[in] flow 4848 * Parent flow structure pointer. 4849 * @param wks 4850 * Pointer to thread flow work space. 4851 * @param[in] attr 4852 * Flow rule attributes. 4853 * @param[in] items 4854 * Pattern specification (list terminated by the END pattern item). 4855 * @param[out] error 4856 * Perform verbose error reporting if not NULL. 4857 * 4858 * @return 4859 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4860 */ 4861 static struct mlx5_flow_meter_sub_policy * 4862 get_meter_sub_policy(struct rte_eth_dev *dev, 4863 struct rte_flow *flow, 4864 struct mlx5_flow_workspace *wks, 4865 const struct rte_flow_attr *attr, 4866 const struct rte_flow_item items[], 4867 struct rte_flow_error *error) 4868 { 4869 struct mlx5_flow_meter_policy *policy; 4870 struct mlx5_flow_meter_policy *final_policy; 4871 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4872 4873 policy = wks->policy; 4874 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4875 if (final_policy->is_rss || final_policy->is_queue) { 4876 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4877 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4878 uint32_t i; 4879 4880 /* 4881 * This is a tmp dev_flow, 4882 * no need to register any matcher for it in translate. 4883 */ 4884 wks->skip_matcher_reg = 1; 4885 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4886 struct mlx5_flow dev_flow = {0}; 4887 struct mlx5_flow_handle dev_handle = { {0} }; 4888 uint8_t fate = final_policy->act_cnt[i].fate_action; 4889 4890 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4891 const struct rte_flow_action_rss *rss_act = 4892 final_policy->act_cnt[i].rss->conf; 4893 struct rte_flow_action rss_actions[2] = { 4894 [0] = { 4895 .type = RTE_FLOW_ACTION_TYPE_RSS, 4896 .conf = rss_act, 4897 }, 4898 [1] = { 4899 .type = RTE_FLOW_ACTION_TYPE_END, 4900 .conf = NULL, 4901 } 4902 }; 4903 4904 dev_flow.handle = &dev_handle; 4905 dev_flow.ingress = attr->ingress; 4906 dev_flow.flow = flow; 4907 dev_flow.external = 0; 4908 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4909 dev_flow.dv.transfer = attr->transfer; 4910 #endif 4911 /** 4912 * Translate RSS action to get rss hash fields. 4913 */ 4914 if (flow_drv_translate(dev, &dev_flow, attr, 4915 items, rss_actions, error)) 4916 goto exit; 4917 rss_desc_v[i] = wks->rss_desc; 4918 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4919 rss_desc_v[i].hash_fields = 4920 dev_flow.hash_fields; 4921 rss_desc_v[i].queue_num = 4922 rss_desc_v[i].hash_fields ? 4923 rss_desc_v[i].queue_num : 1; 4924 rss_desc_v[i].tunnel = 4925 !!(dev_flow.handle->layers & 4926 MLX5_FLOW_LAYER_TUNNEL); 4927 /* Use the RSS queues in the containers. */ 4928 rss_desc_v[i].queue = 4929 (uint16_t *)(uintptr_t)rss_act->queue; 4930 rss_desc[i] = &rss_desc_v[i]; 4931 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 4932 /* This is queue action. */ 4933 rss_desc_v[i] = wks->rss_desc; 4934 rss_desc_v[i].key_len = 0; 4935 rss_desc_v[i].hash_fields = 0; 4936 rss_desc_v[i].queue = 4937 &final_policy->act_cnt[i].queue; 4938 rss_desc_v[i].queue_num = 1; 4939 rss_desc[i] = &rss_desc_v[i]; 4940 } else { 4941 rss_desc[i] = NULL; 4942 } 4943 } 4944 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4945 flow, policy, rss_desc); 4946 } else { 4947 enum mlx5_meter_domain mtr_domain = 4948 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4949 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4950 MLX5_MTR_DOMAIN_INGRESS); 4951 sub_policy = policy->sub_policys[mtr_domain][0]; 4952 } 4953 if (!sub_policy) 4954 rte_flow_error_set(error, EINVAL, 4955 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4956 "Failed to get meter sub-policy."); 4957 exit: 4958 return sub_policy; 4959 } 4960 4961 /** 4962 * Split the meter flow. 4963 * 4964 * As meter flow will split to three sub flow, other than meter 4965 * action, the other actions make sense to only meter accepts 4966 * the packet. If it need to be dropped, no other additional 4967 * actions should be take. 4968 * 4969 * One kind of special action which decapsulates the L3 tunnel 4970 * header will be in the prefix sub flow, as not to take the 4971 * L3 tunnel header into account. 4972 * 4973 * @param[in] dev 4974 * Pointer to Ethernet device. 4975 * @param[in] flow 4976 * Parent flow structure pointer. 4977 * @param wks 4978 * Pointer to thread flow work space. 4979 * @param[in] attr 4980 * Flow rule attributes. 4981 * @param[in] items 4982 * Pattern specification (list terminated by the END pattern item). 4983 * @param[out] sfx_items 4984 * Suffix flow match items (list terminated by the END pattern item). 4985 * @param[in] actions 4986 * Associated actions (list terminated by the END action). 4987 * @param[out] actions_sfx 4988 * Suffix flow actions. 4989 * @param[out] actions_pre 4990 * Prefix flow actions. 4991 * @param[out] mtr_flow_id 4992 * Pointer to meter flow id. 4993 * @param[out] error 4994 * Perform verbose error reporting if not NULL. 4995 * 4996 * @return 4997 * 0 on success, a negative errno value otherwise and rte_errno is set. 4998 */ 4999 static int 5000 flow_meter_split_prep(struct rte_eth_dev *dev, 5001 struct rte_flow *flow, 5002 struct mlx5_flow_workspace *wks, 5003 const struct rte_flow_attr *attr, 5004 const struct rte_flow_item items[], 5005 struct rte_flow_item sfx_items[], 5006 const struct rte_flow_action actions[], 5007 struct rte_flow_action actions_sfx[], 5008 struct rte_flow_action actions_pre[], 5009 uint32_t *mtr_flow_id, 5010 struct rte_flow_error *error) 5011 { 5012 struct mlx5_priv *priv = dev->data->dev_private; 5013 struct mlx5_flow_meter_info *fm = wks->fm; 5014 struct rte_flow_action *tag_action = NULL; 5015 struct rte_flow_item *tag_item; 5016 struct mlx5_rte_flow_action_set_tag *set_tag; 5017 const struct rte_flow_action_raw_encap *raw_encap; 5018 const struct rte_flow_action_raw_decap *raw_decap; 5019 struct mlx5_rte_flow_item_tag *tag_item_spec; 5020 struct mlx5_rte_flow_item_tag *tag_item_mask; 5021 uint32_t tag_id = 0; 5022 struct rte_flow_item *vlan_item_dst = NULL; 5023 const struct rte_flow_item *vlan_item_src = NULL; 5024 struct rte_flow_action *hw_mtr_action; 5025 struct rte_flow_action *action_pre_head = NULL; 5026 int32_t flow_src_port = priv->representor_id; 5027 bool mtr_first; 5028 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5029 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5030 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5031 uint32_t flow_id = 0; 5032 uint32_t flow_id_reversed = 0; 5033 uint8_t flow_id_bits = 0; 5034 int shift; 5035 5036 /* Prepare the suffix subflow items. */ 5037 tag_item = sfx_items++; 5038 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5039 struct mlx5_priv *port_priv; 5040 const struct rte_flow_item_port_id *pid_v; 5041 int item_type = items->type; 5042 5043 switch (item_type) { 5044 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5045 pid_v = items->spec; 5046 MLX5_ASSERT(pid_v); 5047 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5048 if (!port_priv) 5049 return rte_flow_error_set(error, 5050 rte_errno, 5051 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5052 pid_v, 5053 "Failed to get port info."); 5054 flow_src_port = port_priv->representor_id; 5055 if (!fm->def_policy && wks->policy->is_hierarchy && 5056 flow_src_port != priv->representor_id) { 5057 if (flow_drv_mtr_hierarchy_rule_create(dev, 5058 flow, fm, 5059 flow_src_port, 5060 items, 5061 error)) 5062 return -rte_errno; 5063 } 5064 memcpy(sfx_items, items, sizeof(*sfx_items)); 5065 sfx_items++; 5066 break; 5067 case RTE_FLOW_ITEM_TYPE_VLAN: 5068 /* Determine if copy vlan item below. */ 5069 vlan_item_src = items; 5070 vlan_item_dst = sfx_items++; 5071 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5072 break; 5073 default: 5074 break; 5075 } 5076 } 5077 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5078 sfx_items++; 5079 mtr_first = priv->sh->meter_aso_en && 5080 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5081 /* For ASO meter, meter must be before tag in TX direction. */ 5082 if (mtr_first) { 5083 action_pre_head = actions_pre++; 5084 /* Leave space for tag action. */ 5085 tag_action = actions_pre++; 5086 } 5087 /* Prepare the actions for prefix and suffix flow. */ 5088 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5089 struct rte_flow_action *action_cur = NULL; 5090 5091 switch (actions->type) { 5092 case RTE_FLOW_ACTION_TYPE_METER: 5093 if (mtr_first) { 5094 action_cur = action_pre_head; 5095 } else { 5096 /* Leave space for tag action. */ 5097 tag_action = actions_pre++; 5098 action_cur = actions_pre++; 5099 } 5100 break; 5101 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5102 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5103 action_cur = actions_pre++; 5104 break; 5105 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5106 raw_encap = actions->conf; 5107 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5108 action_cur = actions_pre++; 5109 break; 5110 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5111 raw_decap = actions->conf; 5112 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5113 action_cur = actions_pre++; 5114 break; 5115 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5116 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5117 if (vlan_item_dst && vlan_item_src) { 5118 memcpy(vlan_item_dst, vlan_item_src, 5119 sizeof(*vlan_item_dst)); 5120 /* 5121 * Convert to internal match item, it is used 5122 * for vlan push and set vid. 5123 */ 5124 vlan_item_dst->type = (enum rte_flow_item_type) 5125 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5126 } 5127 break; 5128 default: 5129 break; 5130 } 5131 if (!action_cur) 5132 action_cur = (fm->def_policy) ? 5133 actions_sfx++ : actions_pre++; 5134 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5135 } 5136 /* Add end action to the actions. */ 5137 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5138 if (priv->sh->meter_aso_en) { 5139 /** 5140 * For ASO meter, need to add an extra jump action explicitly, 5141 * to jump from meter to policer table. 5142 */ 5143 struct mlx5_flow_meter_sub_policy *sub_policy; 5144 struct mlx5_flow_tbl_data_entry *tbl_data; 5145 5146 if (!fm->def_policy) { 5147 sub_policy = get_meter_sub_policy(dev, flow, wks, 5148 attr, items, error); 5149 if (!sub_policy) 5150 return -rte_errno; 5151 } else { 5152 enum mlx5_meter_domain mtr_domain = 5153 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5154 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5155 MLX5_MTR_DOMAIN_INGRESS); 5156 5157 sub_policy = 5158 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5159 } 5160 tbl_data = container_of(sub_policy->tbl_rsc, 5161 struct mlx5_flow_tbl_data_entry, tbl); 5162 hw_mtr_action = actions_pre++; 5163 hw_mtr_action->type = (enum rte_flow_action_type) 5164 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5165 hw_mtr_action->conf = tbl_data->jump.action; 5166 } 5167 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5168 actions_pre++; 5169 if (!tag_action) 5170 return rte_flow_error_set(error, ENOMEM, 5171 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5172 NULL, "No tag action space."); 5173 if (!mtr_flow_id) { 5174 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5175 goto exit; 5176 } 5177 /* Only default-policy Meter creates mtr flow id. */ 5178 if (fm->def_policy) { 5179 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5180 if (!tag_id) 5181 return rte_flow_error_set(error, ENOMEM, 5182 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5183 "Failed to allocate meter flow id."); 5184 flow_id = tag_id - 1; 5185 flow_id_bits = (!flow_id) ? 1 : 5186 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5187 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5188 mtr_reg_bits) { 5189 mlx5_ipool_free(fm->flow_ipool, tag_id); 5190 return rte_flow_error_set(error, EINVAL, 5191 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5192 "Meter flow id exceeds max limit."); 5193 } 5194 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5195 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5196 } 5197 /* Build tag actions and items for meter_id/meter flow_id. */ 5198 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5199 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5200 tag_item_mask = tag_item_spec + 1; 5201 /* Both flow_id and meter_id share the same register. */ 5202 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5203 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5204 0, error), 5205 .offset = mtr_id_offset, 5206 .length = mtr_reg_bits, 5207 .data = flow->meter, 5208 }; 5209 /* 5210 * The color Reg bits used by flow_id are growing from 5211 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5212 */ 5213 for (shift = 0; shift < flow_id_bits; shift++) 5214 flow_id_reversed = (flow_id_reversed << 1) | 5215 ((flow_id >> shift) & 0x1); 5216 set_tag->data |= 5217 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5218 tag_item_spec->id = set_tag->id; 5219 tag_item_spec->data = set_tag->data << mtr_id_offset; 5220 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5221 tag_action->type = (enum rte_flow_action_type) 5222 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5223 tag_action->conf = set_tag; 5224 tag_item->type = (enum rte_flow_item_type) 5225 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5226 tag_item->spec = tag_item_spec; 5227 tag_item->last = NULL; 5228 tag_item->mask = tag_item_mask; 5229 exit: 5230 if (mtr_flow_id) 5231 *mtr_flow_id = tag_id; 5232 return 0; 5233 } 5234 5235 /** 5236 * Split action list having QUEUE/RSS for metadata register copy. 5237 * 5238 * Once Q/RSS action is detected in user's action list, the flow action 5239 * should be split in order to copy metadata registers, which will happen in 5240 * RX_CP_TBL like, 5241 * - CQE->flow_tag := reg_c[1] (MARK) 5242 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5243 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5244 * This is because the last action of each flow must be a terminal action 5245 * (QUEUE, RSS or DROP). 5246 * 5247 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5248 * stored and kept in the mlx5_flow structure per each sub_flow. 5249 * 5250 * The Q/RSS action is replaced with, 5251 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5252 * And the following JUMP action is added at the end, 5253 * - JUMP, to RX_CP_TBL. 5254 * 5255 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5256 * flow_create_split_metadata() routine. The flow will look like, 5257 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5258 * 5259 * @param dev 5260 * Pointer to Ethernet device. 5261 * @param[out] split_actions 5262 * Pointer to store split actions to jump to CP_TBL. 5263 * @param[in] actions 5264 * Pointer to the list of original flow actions. 5265 * @param[in] qrss 5266 * Pointer to the Q/RSS action. 5267 * @param[in] actions_n 5268 * Number of original actions. 5269 * @param[in] mtr_sfx 5270 * Check if it is in meter suffix table. 5271 * @param[out] error 5272 * Perform verbose error reporting if not NULL. 5273 * 5274 * @return 5275 * non-zero unique flow_id on success, otherwise 0 and 5276 * error/rte_error are set. 5277 */ 5278 static uint32_t 5279 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5280 struct rte_flow_action *split_actions, 5281 const struct rte_flow_action *actions, 5282 const struct rte_flow_action *qrss, 5283 int actions_n, int mtr_sfx, 5284 struct rte_flow_error *error) 5285 { 5286 struct mlx5_priv *priv = dev->data->dev_private; 5287 struct mlx5_rte_flow_action_set_tag *set_tag; 5288 struct rte_flow_action_jump *jump; 5289 const int qrss_idx = qrss - actions; 5290 uint32_t flow_id = 0; 5291 int ret = 0; 5292 5293 /* 5294 * Given actions will be split 5295 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5296 * - Add jump to mreg CP_TBL. 5297 * As a result, there will be one more action. 5298 */ 5299 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5300 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5301 ++actions_n; 5302 set_tag = (void *)(split_actions + actions_n); 5303 /* 5304 * If we are not the meter suffix flow, add the tag action. 5305 * Since meter suffix flow already has the tag added. 5306 */ 5307 if (!mtr_sfx) { 5308 /* 5309 * Allocate the new subflow ID. This one is unique within 5310 * device and not shared with representors. Otherwise, 5311 * we would have to resolve multi-thread access synch 5312 * issue. Each flow on the shared device is appended 5313 * with source vport identifier, so the resulting 5314 * flows will be unique in the shared (by master and 5315 * representors) domain even if they have coinciding 5316 * IDs. 5317 */ 5318 mlx5_ipool_malloc(priv->sh->ipool 5319 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5320 if (!flow_id) 5321 return rte_flow_error_set(error, ENOMEM, 5322 RTE_FLOW_ERROR_TYPE_ACTION, 5323 NULL, "can't allocate id " 5324 "for split Q/RSS subflow"); 5325 /* Internal SET_TAG action to set flow ID. */ 5326 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5327 .data = flow_id, 5328 }; 5329 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5330 if (ret < 0) 5331 return ret; 5332 set_tag->id = ret; 5333 /* Construct new actions array. */ 5334 /* Replace QUEUE/RSS action. */ 5335 split_actions[qrss_idx] = (struct rte_flow_action){ 5336 .type = (enum rte_flow_action_type) 5337 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5338 .conf = set_tag, 5339 }; 5340 } else { 5341 /* 5342 * If we are the suffix flow of meter, tag already exist. 5343 * Set the QUEUE/RSS action to void. 5344 */ 5345 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5346 } 5347 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5348 jump = (void *)(set_tag + 1); 5349 *jump = (struct rte_flow_action_jump){ 5350 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5351 }; 5352 split_actions[actions_n - 2] = (struct rte_flow_action){ 5353 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5354 .conf = jump, 5355 }; 5356 split_actions[actions_n - 1] = (struct rte_flow_action){ 5357 .type = RTE_FLOW_ACTION_TYPE_END, 5358 }; 5359 return flow_id; 5360 } 5361 5362 /** 5363 * Extend the given action list for Tx metadata copy. 5364 * 5365 * Copy the given action list to the ext_actions and add flow metadata register 5366 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5367 * 5368 * @param[out] ext_actions 5369 * Pointer to the extended action list. 5370 * @param[in] actions 5371 * Pointer to the list of actions. 5372 * @param[in] actions_n 5373 * Number of actions in the list. 5374 * @param[out] error 5375 * Perform verbose error reporting if not NULL. 5376 * @param[in] encap_idx 5377 * The encap action index. 5378 * 5379 * @return 5380 * 0 on success, negative value otherwise 5381 */ 5382 static int 5383 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5384 struct rte_flow_action *ext_actions, 5385 const struct rte_flow_action *actions, 5386 int actions_n, struct rte_flow_error *error, 5387 int encap_idx) 5388 { 5389 struct mlx5_flow_action_copy_mreg *cp_mreg = 5390 (struct mlx5_flow_action_copy_mreg *) 5391 (ext_actions + actions_n + 1); 5392 int ret; 5393 5394 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5395 if (ret < 0) 5396 return ret; 5397 cp_mreg->dst = ret; 5398 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5399 if (ret < 0) 5400 return ret; 5401 cp_mreg->src = ret; 5402 if (encap_idx != 0) 5403 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5404 if (encap_idx == actions_n - 1) { 5405 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5406 .type = (enum rte_flow_action_type) 5407 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5408 .conf = cp_mreg, 5409 }; 5410 ext_actions[actions_n] = (struct rte_flow_action){ 5411 .type = RTE_FLOW_ACTION_TYPE_END, 5412 }; 5413 } else { 5414 ext_actions[encap_idx] = (struct rte_flow_action){ 5415 .type = (enum rte_flow_action_type) 5416 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5417 .conf = cp_mreg, 5418 }; 5419 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5420 sizeof(*ext_actions) * (actions_n - encap_idx)); 5421 } 5422 return 0; 5423 } 5424 5425 /** 5426 * Check the match action from the action list. 5427 * 5428 * @param[in] actions 5429 * Pointer to the list of actions. 5430 * @param[in] attr 5431 * Flow rule attributes. 5432 * @param[in] action 5433 * The action to be check if exist. 5434 * @param[out] match_action_pos 5435 * Pointer to the position of the matched action if exists, otherwise is -1. 5436 * @param[out] qrss_action_pos 5437 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5438 * @param[out] modify_after_mirror 5439 * Pointer to the flag of modify action after FDB mirroring. 5440 * 5441 * @return 5442 * > 0 the total number of actions. 5443 * 0 if not found match action in action list. 5444 */ 5445 static int 5446 flow_check_match_action(const struct rte_flow_action actions[], 5447 const struct rte_flow_attr *attr, 5448 enum rte_flow_action_type action, 5449 int *match_action_pos, int *qrss_action_pos, 5450 int *modify_after_mirror) 5451 { 5452 const struct rte_flow_action_sample *sample; 5453 const struct rte_flow_action_raw_decap *decap; 5454 int actions_n = 0; 5455 uint32_t ratio = 0; 5456 int sub_type = 0; 5457 int flag = 0; 5458 int fdb_mirror = 0; 5459 5460 *match_action_pos = -1; 5461 *qrss_action_pos = -1; 5462 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5463 if (actions->type == action) { 5464 flag = 1; 5465 *match_action_pos = actions_n; 5466 } 5467 switch (actions->type) { 5468 case RTE_FLOW_ACTION_TYPE_QUEUE: 5469 case RTE_FLOW_ACTION_TYPE_RSS: 5470 *qrss_action_pos = actions_n; 5471 break; 5472 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5473 sample = actions->conf; 5474 ratio = sample->ratio; 5475 sub_type = ((const struct rte_flow_action *) 5476 (sample->actions))->type; 5477 if (ratio == 1 && attr->transfer) 5478 fdb_mirror = 1; 5479 break; 5480 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5481 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5482 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5483 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5484 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5485 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5486 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5487 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5488 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5489 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5490 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5491 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5492 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5493 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5494 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5495 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5496 case RTE_FLOW_ACTION_TYPE_FLAG: 5497 case RTE_FLOW_ACTION_TYPE_MARK: 5498 case RTE_FLOW_ACTION_TYPE_SET_META: 5499 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5500 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5501 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5502 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5503 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5504 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5505 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5506 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5507 case RTE_FLOW_ACTION_TYPE_METER: 5508 if (fdb_mirror) 5509 *modify_after_mirror = 1; 5510 break; 5511 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5512 decap = actions->conf; 5513 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5514 ; 5515 actions_n++; 5516 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5517 const struct rte_flow_action_raw_encap *encap = 5518 actions->conf; 5519 if (decap->size <= 5520 MLX5_ENCAPSULATION_DECISION_SIZE && 5521 encap->size > 5522 MLX5_ENCAPSULATION_DECISION_SIZE) 5523 /* L3 encap. */ 5524 break; 5525 } 5526 if (fdb_mirror) 5527 *modify_after_mirror = 1; 5528 break; 5529 default: 5530 break; 5531 } 5532 actions_n++; 5533 } 5534 if (flag && fdb_mirror && !*modify_after_mirror) { 5535 /* FDB mirroring uses the destination array to implement 5536 * instead of FLOW_SAMPLER object. 5537 */ 5538 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5539 flag = 0; 5540 } 5541 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5542 return flag ? actions_n + 1 : 0; 5543 } 5544 5545 #define SAMPLE_SUFFIX_ITEM 2 5546 5547 /** 5548 * Split the sample flow. 5549 * 5550 * As sample flow will split to two sub flow, sample flow with 5551 * sample action, the other actions will move to new suffix flow. 5552 * 5553 * Also add unique tag id with tag action in the sample flow, 5554 * the same tag id will be as match in the suffix flow. 5555 * 5556 * @param dev 5557 * Pointer to Ethernet device. 5558 * @param[in] add_tag 5559 * Add extra tag action flag. 5560 * @param[out] sfx_items 5561 * Suffix flow match items (list terminated by the END pattern item). 5562 * @param[in] actions 5563 * Associated actions (list terminated by the END action). 5564 * @param[out] actions_sfx 5565 * Suffix flow actions. 5566 * @param[out] actions_pre 5567 * Prefix flow actions. 5568 * @param[in] actions_n 5569 * The total number of actions. 5570 * @param[in] sample_action_pos 5571 * The sample action position. 5572 * @param[in] qrss_action_pos 5573 * The Queue/RSS action position. 5574 * @param[in] jump_table 5575 * Add extra jump action flag. 5576 * @param[out] error 5577 * Perform verbose error reporting if not NULL. 5578 * 5579 * @return 5580 * 0 on success, or unique flow_id, a negative errno value 5581 * otherwise and rte_errno is set. 5582 */ 5583 static int 5584 flow_sample_split_prep(struct rte_eth_dev *dev, 5585 int add_tag, 5586 struct rte_flow_item sfx_items[], 5587 const struct rte_flow_action actions[], 5588 struct rte_flow_action actions_sfx[], 5589 struct rte_flow_action actions_pre[], 5590 int actions_n, 5591 int sample_action_pos, 5592 int qrss_action_pos, 5593 int jump_table, 5594 struct rte_flow_error *error) 5595 { 5596 struct mlx5_priv *priv = dev->data->dev_private; 5597 struct mlx5_rte_flow_action_set_tag *set_tag; 5598 struct mlx5_rte_flow_item_tag *tag_spec; 5599 struct mlx5_rte_flow_item_tag *tag_mask; 5600 struct rte_flow_action_jump *jump_action; 5601 uint32_t tag_id = 0; 5602 int index; 5603 int append_index = 0; 5604 int ret; 5605 5606 if (sample_action_pos < 0) 5607 return rte_flow_error_set(error, EINVAL, 5608 RTE_FLOW_ERROR_TYPE_ACTION, 5609 NULL, "invalid position of sample " 5610 "action in list"); 5611 /* Prepare the actions for prefix and suffix flow. */ 5612 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5613 index = qrss_action_pos; 5614 /* Put the preceding the Queue/RSS action into prefix flow. */ 5615 if (index != 0) 5616 memcpy(actions_pre, actions, 5617 sizeof(struct rte_flow_action) * index); 5618 /* Put others preceding the sample action into prefix flow. */ 5619 if (sample_action_pos > index + 1) 5620 memcpy(actions_pre + index, actions + index + 1, 5621 sizeof(struct rte_flow_action) * 5622 (sample_action_pos - index - 1)); 5623 index = sample_action_pos - 1; 5624 /* Put Queue/RSS action into Suffix flow. */ 5625 memcpy(actions_sfx, actions + qrss_action_pos, 5626 sizeof(struct rte_flow_action)); 5627 actions_sfx++; 5628 } else { 5629 index = sample_action_pos; 5630 if (index != 0) 5631 memcpy(actions_pre, actions, 5632 sizeof(struct rte_flow_action) * index); 5633 } 5634 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5635 * For CX6DX and above, metadata registers Cx preserve their value, 5636 * add an extra tag action for NIC-RX and E-Switch Domain. 5637 */ 5638 if (add_tag) { 5639 /* Prepare the prefix tag action. */ 5640 append_index++; 5641 set_tag = (void *)(actions_pre + actions_n + append_index); 5642 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5643 if (ret < 0) 5644 return ret; 5645 mlx5_ipool_malloc(priv->sh->ipool 5646 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5647 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5648 .id = ret, 5649 .data = tag_id, 5650 }; 5651 /* Prepare the suffix subflow items. */ 5652 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5653 tag_spec->data = tag_id; 5654 tag_spec->id = set_tag->id; 5655 tag_mask = tag_spec + 1; 5656 tag_mask->data = UINT32_MAX; 5657 sfx_items[0] = (struct rte_flow_item){ 5658 .type = (enum rte_flow_item_type) 5659 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5660 .spec = tag_spec, 5661 .last = NULL, 5662 .mask = tag_mask, 5663 }; 5664 sfx_items[1] = (struct rte_flow_item){ 5665 .type = (enum rte_flow_item_type) 5666 RTE_FLOW_ITEM_TYPE_END, 5667 }; 5668 /* Prepare the tag action in prefix subflow. */ 5669 actions_pre[index++] = 5670 (struct rte_flow_action){ 5671 .type = (enum rte_flow_action_type) 5672 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5673 .conf = set_tag, 5674 }; 5675 } 5676 memcpy(actions_pre + index, actions + sample_action_pos, 5677 sizeof(struct rte_flow_action)); 5678 index += 1; 5679 /* For the modify action after the sample action in E-Switch mirroring, 5680 * Add the extra jump action in prefix subflow and jump into the next 5681 * table, then do the modify action in the new table. 5682 */ 5683 if (jump_table) { 5684 /* Prepare the prefix jump action. */ 5685 append_index++; 5686 jump_action = (void *)(actions_pre + actions_n + append_index); 5687 jump_action->group = jump_table; 5688 actions_pre[index++] = 5689 (struct rte_flow_action){ 5690 .type = (enum rte_flow_action_type) 5691 RTE_FLOW_ACTION_TYPE_JUMP, 5692 .conf = jump_action, 5693 }; 5694 } 5695 actions_pre[index] = (struct rte_flow_action){ 5696 .type = (enum rte_flow_action_type) 5697 RTE_FLOW_ACTION_TYPE_END, 5698 }; 5699 /* Put the actions after sample into Suffix flow. */ 5700 memcpy(actions_sfx, actions + sample_action_pos + 1, 5701 sizeof(struct rte_flow_action) * 5702 (actions_n - sample_action_pos - 1)); 5703 return tag_id; 5704 } 5705 5706 /** 5707 * The splitting for metadata feature. 5708 * 5709 * - Q/RSS action on NIC Rx should be split in order to pass by 5710 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5711 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5712 * 5713 * - All the actions on NIC Tx should have a mreg copy action to 5714 * copy reg_a from WQE to reg_c[0]. 5715 * 5716 * @param dev 5717 * Pointer to Ethernet device. 5718 * @param[in] flow 5719 * Parent flow structure pointer. 5720 * @param[in] attr 5721 * Flow rule attributes. 5722 * @param[in] items 5723 * Pattern specification (list terminated by the END pattern item). 5724 * @param[in] actions 5725 * Associated actions (list terminated by the END action). 5726 * @param[in] flow_split_info 5727 * Pointer to flow split info structure. 5728 * @param[out] error 5729 * Perform verbose error reporting if not NULL. 5730 * @return 5731 * 0 on success, negative value otherwise 5732 */ 5733 static int 5734 flow_create_split_metadata(struct rte_eth_dev *dev, 5735 struct rte_flow *flow, 5736 const struct rte_flow_attr *attr, 5737 const struct rte_flow_item items[], 5738 const struct rte_flow_action actions[], 5739 struct mlx5_flow_split_info *flow_split_info, 5740 struct rte_flow_error *error) 5741 { 5742 struct mlx5_priv *priv = dev->data->dev_private; 5743 struct mlx5_dev_config *config = &priv->config; 5744 const struct rte_flow_action *qrss = NULL; 5745 struct rte_flow_action *ext_actions = NULL; 5746 struct mlx5_flow *dev_flow = NULL; 5747 uint32_t qrss_id = 0; 5748 int mtr_sfx = 0; 5749 size_t act_size; 5750 int actions_n; 5751 int encap_idx; 5752 int ret; 5753 5754 /* Check whether extensive metadata feature is engaged. */ 5755 if (!config->dv_flow_en || 5756 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5757 !mlx5_flow_ext_mreg_supported(dev)) 5758 return flow_create_split_inner(dev, flow, NULL, attr, items, 5759 actions, flow_split_info, error); 5760 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5761 &encap_idx); 5762 if (qrss) { 5763 /* Exclude hairpin flows from splitting. */ 5764 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5765 const struct rte_flow_action_queue *queue; 5766 5767 queue = qrss->conf; 5768 if (mlx5_rxq_get_type(dev, queue->index) == 5769 MLX5_RXQ_TYPE_HAIRPIN) 5770 qrss = NULL; 5771 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5772 const struct rte_flow_action_rss *rss; 5773 5774 rss = qrss->conf; 5775 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5776 MLX5_RXQ_TYPE_HAIRPIN) 5777 qrss = NULL; 5778 } 5779 } 5780 if (qrss) { 5781 /* Check if it is in meter suffix table. */ 5782 mtr_sfx = attr->group == (attr->transfer ? 5783 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5784 MLX5_FLOW_TABLE_LEVEL_METER); 5785 /* 5786 * Q/RSS action on NIC Rx should be split in order to pass by 5787 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5788 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5789 */ 5790 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5791 sizeof(struct rte_flow_action_set_tag) + 5792 sizeof(struct rte_flow_action_jump); 5793 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5794 SOCKET_ID_ANY); 5795 if (!ext_actions) 5796 return rte_flow_error_set(error, ENOMEM, 5797 RTE_FLOW_ERROR_TYPE_ACTION, 5798 NULL, "no memory to split " 5799 "metadata flow"); 5800 /* 5801 * Create the new actions list with removed Q/RSS action 5802 * and appended set tag and jump to register copy table 5803 * (RX_CP_TBL). We should preallocate unique tag ID here 5804 * in advance, because it is needed for set tag action. 5805 */ 5806 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5807 qrss, actions_n, 5808 mtr_sfx, error); 5809 if (!mtr_sfx && !qrss_id) { 5810 ret = -rte_errno; 5811 goto exit; 5812 } 5813 } else if (attr->egress && !attr->transfer) { 5814 /* 5815 * All the actions on NIC Tx should have a metadata register 5816 * copy action to copy reg_a from WQE to reg_c[meta] 5817 */ 5818 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5819 sizeof(struct mlx5_flow_action_copy_mreg); 5820 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5821 SOCKET_ID_ANY); 5822 if (!ext_actions) 5823 return rte_flow_error_set(error, ENOMEM, 5824 RTE_FLOW_ERROR_TYPE_ACTION, 5825 NULL, "no memory to split " 5826 "metadata flow"); 5827 /* Create the action list appended with copy register. */ 5828 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5829 actions_n, error, encap_idx); 5830 if (ret < 0) 5831 goto exit; 5832 } 5833 /* Add the unmodified original or prefix subflow. */ 5834 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5835 items, ext_actions ? ext_actions : 5836 actions, flow_split_info, error); 5837 if (ret < 0) 5838 goto exit; 5839 MLX5_ASSERT(dev_flow); 5840 if (qrss) { 5841 const struct rte_flow_attr q_attr = { 5842 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5843 .ingress = 1, 5844 }; 5845 /* Internal PMD action to set register. */ 5846 struct mlx5_rte_flow_item_tag q_tag_spec = { 5847 .data = qrss_id, 5848 .id = REG_NON, 5849 }; 5850 struct rte_flow_item q_items[] = { 5851 { 5852 .type = (enum rte_flow_item_type) 5853 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5854 .spec = &q_tag_spec, 5855 .last = NULL, 5856 .mask = NULL, 5857 }, 5858 { 5859 .type = RTE_FLOW_ITEM_TYPE_END, 5860 }, 5861 }; 5862 struct rte_flow_action q_actions[] = { 5863 { 5864 .type = qrss->type, 5865 .conf = qrss->conf, 5866 }, 5867 { 5868 .type = RTE_FLOW_ACTION_TYPE_END, 5869 }, 5870 }; 5871 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5872 5873 /* 5874 * Configure the tag item only if there is no meter subflow. 5875 * Since tag is already marked in the meter suffix subflow 5876 * we can just use the meter suffix items as is. 5877 */ 5878 if (qrss_id) { 5879 /* Not meter subflow. */ 5880 MLX5_ASSERT(!mtr_sfx); 5881 /* 5882 * Put unique id in prefix flow due to it is destroyed 5883 * after suffix flow and id will be freed after there 5884 * is no actual flows with this id and identifier 5885 * reallocation becomes possible (for example, for 5886 * other flows in other threads). 5887 */ 5888 dev_flow->handle->split_flow_id = qrss_id; 5889 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5890 error); 5891 if (ret < 0) 5892 goto exit; 5893 q_tag_spec.id = ret; 5894 } 5895 dev_flow = NULL; 5896 /* Add suffix subflow to execute Q/RSS. */ 5897 flow_split_info->prefix_layers = layers; 5898 flow_split_info->prefix_mark = 0; 5899 flow_split_info->table_id = 0; 5900 ret = flow_create_split_inner(dev, flow, &dev_flow, 5901 &q_attr, mtr_sfx ? items : 5902 q_items, q_actions, 5903 flow_split_info, error); 5904 if (ret < 0) 5905 goto exit; 5906 /* qrss ID should be freed if failed. */ 5907 qrss_id = 0; 5908 MLX5_ASSERT(dev_flow); 5909 } 5910 5911 exit: 5912 /* 5913 * We do not destroy the partially created sub_flows in case of error. 5914 * These ones are included into parent flow list and will be destroyed 5915 * by flow_drv_destroy. 5916 */ 5917 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5918 qrss_id); 5919 mlx5_free(ext_actions); 5920 return ret; 5921 } 5922 5923 /** 5924 * Create meter internal drop flow with the original pattern. 5925 * 5926 * @param dev 5927 * Pointer to Ethernet device. 5928 * @param[in] flow 5929 * Parent flow structure pointer. 5930 * @param[in] attr 5931 * Flow rule attributes. 5932 * @param[in] items 5933 * Pattern specification (list terminated by the END pattern item). 5934 * @param[in] flow_split_info 5935 * Pointer to flow split info structure. 5936 * @param[in] fm 5937 * Pointer to flow meter structure. 5938 * @param[out] error 5939 * Perform verbose error reporting if not NULL. 5940 * @return 5941 * 0 on success, negative value otherwise 5942 */ 5943 static uint32_t 5944 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5945 struct rte_flow *flow, 5946 const struct rte_flow_attr *attr, 5947 const struct rte_flow_item items[], 5948 struct mlx5_flow_split_info *flow_split_info, 5949 struct mlx5_flow_meter_info *fm, 5950 struct rte_flow_error *error) 5951 { 5952 struct mlx5_flow *dev_flow = NULL; 5953 struct rte_flow_attr drop_attr = *attr; 5954 struct rte_flow_action drop_actions[3]; 5955 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5956 5957 MLX5_ASSERT(fm->drop_cnt); 5958 drop_actions[0].type = 5959 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5960 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5961 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5962 drop_actions[1].conf = NULL; 5963 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5964 drop_actions[2].conf = NULL; 5965 drop_split_info.external = false; 5966 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5967 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5968 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5969 return flow_create_split_inner(dev, flow, &dev_flow, 5970 &drop_attr, items, drop_actions, 5971 &drop_split_info, error); 5972 } 5973 5974 /** 5975 * The splitting for meter feature. 5976 * 5977 * - The meter flow will be split to two flows as prefix and 5978 * suffix flow. The packets make sense only it pass the prefix 5979 * meter action. 5980 * 5981 * - Reg_C_5 is used for the packet to match betweend prefix and 5982 * suffix flow. 5983 * 5984 * @param dev 5985 * Pointer to Ethernet device. 5986 * @param[in] flow 5987 * Parent flow structure pointer. 5988 * @param[in] attr 5989 * Flow rule attributes. 5990 * @param[in] items 5991 * Pattern specification (list terminated by the END pattern item). 5992 * @param[in] actions 5993 * Associated actions (list terminated by the END action). 5994 * @param[in] flow_split_info 5995 * Pointer to flow split info structure. 5996 * @param[out] error 5997 * Perform verbose error reporting if not NULL. 5998 * @return 5999 * 0 on success, negative value otherwise 6000 */ 6001 static int 6002 flow_create_split_meter(struct rte_eth_dev *dev, 6003 struct rte_flow *flow, 6004 const struct rte_flow_attr *attr, 6005 const struct rte_flow_item items[], 6006 const struct rte_flow_action actions[], 6007 struct mlx5_flow_split_info *flow_split_info, 6008 struct rte_flow_error *error) 6009 { 6010 struct mlx5_priv *priv = dev->data->dev_private; 6011 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6012 struct rte_flow_action *sfx_actions = NULL; 6013 struct rte_flow_action *pre_actions = NULL; 6014 struct rte_flow_item *sfx_items = NULL; 6015 struct mlx5_flow *dev_flow = NULL; 6016 struct rte_flow_attr sfx_attr = *attr; 6017 struct mlx5_flow_meter_info *fm = NULL; 6018 uint8_t skip_scale_restore; 6019 bool has_mtr = false; 6020 bool has_modify = false; 6021 bool set_mtr_reg = true; 6022 bool is_mtr_hierarchy = false; 6023 uint32_t meter_id = 0; 6024 uint32_t mtr_idx = 0; 6025 uint32_t mtr_flow_id = 0; 6026 size_t act_size; 6027 size_t item_size; 6028 int actions_n = 0; 6029 int ret = 0; 6030 6031 if (priv->mtr_en) 6032 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6033 &has_modify, &meter_id); 6034 if (has_mtr) { 6035 if (flow->meter) { 6036 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6037 if (!fm) 6038 return rte_flow_error_set(error, EINVAL, 6039 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6040 NULL, "Meter not found."); 6041 } else { 6042 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6043 if (!fm) 6044 return rte_flow_error_set(error, EINVAL, 6045 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6046 NULL, "Meter not found."); 6047 ret = mlx5_flow_meter_attach(priv, fm, 6048 &sfx_attr, error); 6049 if (ret) 6050 return -rte_errno; 6051 flow->meter = mtr_idx; 6052 } 6053 MLX5_ASSERT(wks); 6054 wks->fm = fm; 6055 if (!fm->def_policy) { 6056 wks->policy = mlx5_flow_meter_policy_find(dev, 6057 fm->policy_id, 6058 NULL); 6059 MLX5_ASSERT(wks->policy); 6060 if (wks->policy->is_hierarchy) { 6061 wks->final_policy = 6062 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6063 wks->policy); 6064 if (!wks->final_policy) 6065 return rte_flow_error_set(error, 6066 EINVAL, 6067 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6068 "Failed to find terminal policy of hierarchy."); 6069 is_mtr_hierarchy = true; 6070 } 6071 } 6072 /* 6073 * If it isn't default-policy Meter, and 6074 * 1. There's no action in flow to change 6075 * packet (modify/encap/decap etc.), OR 6076 * 2. No drop count needed for this meter. 6077 * 3. It's not meter hierarchy. 6078 * Then no need to use regC to save meter id anymore. 6079 */ 6080 if (!fm->def_policy && !is_mtr_hierarchy && 6081 (!has_modify || !fm->drop_cnt)) 6082 set_mtr_reg = false; 6083 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6084 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6085 sizeof(struct mlx5_rte_flow_action_set_tag); 6086 /* Suffix items: tag, vlan, port id, end. */ 6087 #define METER_SUFFIX_ITEM 4 6088 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6089 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6090 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6091 0, SOCKET_ID_ANY); 6092 if (!sfx_actions) 6093 return rte_flow_error_set(error, ENOMEM, 6094 RTE_FLOW_ERROR_TYPE_ACTION, 6095 NULL, "no memory to split " 6096 "meter flow"); 6097 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6098 act_size); 6099 /* There's no suffix flow for meter of non-default policy. */ 6100 if (!fm->def_policy) 6101 pre_actions = sfx_actions + 1; 6102 else 6103 pre_actions = sfx_actions + actions_n; 6104 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6105 items, sfx_items, actions, 6106 sfx_actions, pre_actions, 6107 (set_mtr_reg ? &mtr_flow_id : NULL), 6108 error); 6109 if (ret) { 6110 ret = -rte_errno; 6111 goto exit; 6112 } 6113 /* Add the prefix subflow. */ 6114 skip_scale_restore = flow_split_info->skip_scale; 6115 flow_split_info->skip_scale |= 6116 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6117 ret = flow_create_split_inner(dev, flow, &dev_flow, 6118 attr, items, pre_actions, 6119 flow_split_info, error); 6120 flow_split_info->skip_scale = skip_scale_restore; 6121 if (ret) { 6122 if (mtr_flow_id) 6123 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6124 ret = -rte_errno; 6125 goto exit; 6126 } 6127 if (mtr_flow_id) { 6128 dev_flow->handle->split_flow_id = mtr_flow_id; 6129 dev_flow->handle->is_meter_flow_id = 1; 6130 } 6131 if (!fm->def_policy) { 6132 if (!set_mtr_reg && fm->drop_cnt) 6133 ret = 6134 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6135 &sfx_attr, items, 6136 flow_split_info, 6137 fm, error); 6138 goto exit; 6139 } 6140 /* Setting the sfx group atrr. */ 6141 sfx_attr.group = sfx_attr.transfer ? 6142 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6143 MLX5_FLOW_TABLE_LEVEL_METER; 6144 flow_split_info->prefix_layers = 6145 flow_get_prefix_layer_flags(dev_flow); 6146 flow_split_info->prefix_mark |= dev_flow->handle->mark; 6147 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6148 } 6149 /* Add the prefix subflow. */ 6150 ret = flow_create_split_metadata(dev, flow, 6151 &sfx_attr, sfx_items ? 6152 sfx_items : items, 6153 sfx_actions ? sfx_actions : actions, 6154 flow_split_info, error); 6155 exit: 6156 if (sfx_actions) 6157 mlx5_free(sfx_actions); 6158 return ret; 6159 } 6160 6161 /** 6162 * The splitting for sample feature. 6163 * 6164 * Once Sample action is detected in the action list, the flow actions should 6165 * be split into prefix sub flow and suffix sub flow. 6166 * 6167 * The original items remain in the prefix sub flow, all actions preceding the 6168 * sample action and the sample action itself will be copied to the prefix 6169 * sub flow, the actions following the sample action will be copied to the 6170 * suffix sub flow, Queue action always be located in the suffix sub flow. 6171 * 6172 * In order to make the packet from prefix sub flow matches with suffix sub 6173 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6174 * flow uses tag item with the unique flow id. 6175 * 6176 * @param dev 6177 * Pointer to Ethernet device. 6178 * @param[in] flow 6179 * Parent flow structure pointer. 6180 * @param[in] attr 6181 * Flow rule attributes. 6182 * @param[in] items 6183 * Pattern specification (list terminated by the END pattern item). 6184 * @param[in] actions 6185 * Associated actions (list terminated by the END action). 6186 * @param[in] flow_split_info 6187 * Pointer to flow split info structure. 6188 * @param[out] error 6189 * Perform verbose error reporting if not NULL. 6190 * @return 6191 * 0 on success, negative value otherwise 6192 */ 6193 static int 6194 flow_create_split_sample(struct rte_eth_dev *dev, 6195 struct rte_flow *flow, 6196 const struct rte_flow_attr *attr, 6197 const struct rte_flow_item items[], 6198 const struct rte_flow_action actions[], 6199 struct mlx5_flow_split_info *flow_split_info, 6200 struct rte_flow_error *error) 6201 { 6202 struct mlx5_priv *priv = dev->data->dev_private; 6203 struct rte_flow_action *sfx_actions = NULL; 6204 struct rte_flow_action *pre_actions = NULL; 6205 struct rte_flow_item *sfx_items = NULL; 6206 struct mlx5_flow *dev_flow = NULL; 6207 struct rte_flow_attr sfx_attr = *attr; 6208 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6209 struct mlx5_flow_dv_sample_resource *sample_res; 6210 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6211 struct mlx5_flow_tbl_resource *sfx_tbl; 6212 #endif 6213 size_t act_size; 6214 size_t item_size; 6215 uint32_t fdb_tx = 0; 6216 int32_t tag_id = 0; 6217 int actions_n = 0; 6218 int sample_action_pos; 6219 int qrss_action_pos; 6220 int add_tag = 0; 6221 int modify_after_mirror = 0; 6222 uint16_t jump_table = 0; 6223 const uint32_t next_ft_step = 1; 6224 int ret = 0; 6225 6226 if (priv->sampler_en) 6227 actions_n = flow_check_match_action(actions, attr, 6228 RTE_FLOW_ACTION_TYPE_SAMPLE, 6229 &sample_action_pos, &qrss_action_pos, 6230 &modify_after_mirror); 6231 if (actions_n) { 6232 /* The prefix actions must includes sample, tag, end. */ 6233 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6234 + sizeof(struct mlx5_rte_flow_action_set_tag); 6235 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6236 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6237 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6238 item_size), 0, SOCKET_ID_ANY); 6239 if (!sfx_actions) 6240 return rte_flow_error_set(error, ENOMEM, 6241 RTE_FLOW_ERROR_TYPE_ACTION, 6242 NULL, "no memory to split " 6243 "sample flow"); 6244 /* The representor_id is UINT16_MAX for uplink. */ 6245 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6246 /* 6247 * When reg_c_preserve is set, metadata registers Cx preserve 6248 * their value even through packet duplication. 6249 */ 6250 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 6251 if (add_tag) 6252 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6253 + act_size); 6254 if (modify_after_mirror) 6255 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6256 next_ft_step; 6257 pre_actions = sfx_actions + actions_n; 6258 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6259 actions, sfx_actions, 6260 pre_actions, actions_n, 6261 sample_action_pos, 6262 qrss_action_pos, jump_table, 6263 error); 6264 if (tag_id < 0 || (add_tag && !tag_id)) { 6265 ret = -rte_errno; 6266 goto exit; 6267 } 6268 if (modify_after_mirror) 6269 flow_split_info->skip_scale = 6270 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6271 /* Add the prefix subflow. */ 6272 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6273 items, pre_actions, 6274 flow_split_info, error); 6275 if (ret) { 6276 ret = -rte_errno; 6277 goto exit; 6278 } 6279 dev_flow->handle->split_flow_id = tag_id; 6280 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6281 if (!modify_after_mirror) { 6282 /* Set the sfx group attr. */ 6283 sample_res = (struct mlx5_flow_dv_sample_resource *) 6284 dev_flow->dv.sample_res; 6285 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6286 sample_res->normal_path_tbl; 6287 sfx_tbl_data = container_of(sfx_tbl, 6288 struct mlx5_flow_tbl_data_entry, 6289 tbl); 6290 sfx_attr.group = sfx_attr.transfer ? 6291 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6292 } else { 6293 MLX5_ASSERT(attr->transfer); 6294 sfx_attr.group = jump_table; 6295 } 6296 flow_split_info->prefix_layers = 6297 flow_get_prefix_layer_flags(dev_flow); 6298 flow_split_info->prefix_mark |= dev_flow->handle->mark; 6299 /* Suffix group level already be scaled with factor, set 6300 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6301 * again in translation. 6302 */ 6303 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6304 #endif 6305 } 6306 /* Add the suffix subflow. */ 6307 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6308 sfx_items ? sfx_items : items, 6309 sfx_actions ? sfx_actions : actions, 6310 flow_split_info, error); 6311 exit: 6312 if (sfx_actions) 6313 mlx5_free(sfx_actions); 6314 return ret; 6315 } 6316 6317 /** 6318 * Split the flow to subflow set. The splitters might be linked 6319 * in the chain, like this: 6320 * flow_create_split_outer() calls: 6321 * flow_create_split_meter() calls: 6322 * flow_create_split_metadata(meter_subflow_0) calls: 6323 * flow_create_split_inner(metadata_subflow_0) 6324 * flow_create_split_inner(metadata_subflow_1) 6325 * flow_create_split_inner(metadata_subflow_2) 6326 * flow_create_split_metadata(meter_subflow_1) calls: 6327 * flow_create_split_inner(metadata_subflow_0) 6328 * flow_create_split_inner(metadata_subflow_1) 6329 * flow_create_split_inner(metadata_subflow_2) 6330 * 6331 * This provide flexible way to add new levels of flow splitting. 6332 * The all of successfully created subflows are included to the 6333 * parent flow dev_flow list. 6334 * 6335 * @param dev 6336 * Pointer to Ethernet device. 6337 * @param[in] flow 6338 * Parent flow structure pointer. 6339 * @param[in] attr 6340 * Flow rule attributes. 6341 * @param[in] items 6342 * Pattern specification (list terminated by the END pattern item). 6343 * @param[in] actions 6344 * Associated actions (list terminated by the END action). 6345 * @param[in] flow_split_info 6346 * Pointer to flow split info structure. 6347 * @param[out] error 6348 * Perform verbose error reporting if not NULL. 6349 * @return 6350 * 0 on success, negative value otherwise 6351 */ 6352 static int 6353 flow_create_split_outer(struct rte_eth_dev *dev, 6354 struct rte_flow *flow, 6355 const struct rte_flow_attr *attr, 6356 const struct rte_flow_item items[], 6357 const struct rte_flow_action actions[], 6358 struct mlx5_flow_split_info *flow_split_info, 6359 struct rte_flow_error *error) 6360 { 6361 int ret; 6362 6363 ret = flow_create_split_sample(dev, flow, attr, items, 6364 actions, flow_split_info, error); 6365 MLX5_ASSERT(ret <= 0); 6366 return ret; 6367 } 6368 6369 static inline struct mlx5_flow_tunnel * 6370 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6371 { 6372 struct mlx5_flow_tunnel *tunnel; 6373 6374 #pragma GCC diagnostic push 6375 #pragma GCC diagnostic ignored "-Wcast-qual" 6376 tunnel = (typeof(tunnel))flow->tunnel; 6377 #pragma GCC diagnostic pop 6378 6379 return tunnel; 6380 } 6381 6382 /** 6383 * Adjust flow RSS workspace if needed. 6384 * 6385 * @param wks 6386 * Pointer to thread flow work space. 6387 * @param rss_desc 6388 * Pointer to RSS descriptor. 6389 * @param[in] nrssq_num 6390 * New RSS queue number. 6391 * 6392 * @return 6393 * 0 on success, -1 otherwise and rte_errno is set. 6394 */ 6395 static int 6396 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6397 struct mlx5_flow_rss_desc *rss_desc, 6398 uint32_t nrssq_num) 6399 { 6400 if (likely(nrssq_num <= wks->rssq_num)) 6401 return 0; 6402 rss_desc->queue = realloc(rss_desc->queue, 6403 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6404 if (!rss_desc->queue) { 6405 rte_errno = ENOMEM; 6406 return -1; 6407 } 6408 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6409 return 0; 6410 } 6411 6412 /** 6413 * Create a flow and add it to @p list. 6414 * 6415 * @param dev 6416 * Pointer to Ethernet device. 6417 * @param list 6418 * Pointer to a TAILQ flow list. If this parameter NULL, 6419 * no list insertion occurred, flow is just created, 6420 * this is caller's responsibility to track the 6421 * created flow. 6422 * @param[in] attr 6423 * Flow rule attributes. 6424 * @param[in] items 6425 * Pattern specification (list terminated by the END pattern item). 6426 * @param[in] actions 6427 * Associated actions (list terminated by the END action). 6428 * @param[in] external 6429 * This flow rule is created by request external to PMD. 6430 * @param[out] error 6431 * Perform verbose error reporting if not NULL. 6432 * 6433 * @return 6434 * A flow index on success, 0 otherwise and rte_errno is set. 6435 */ 6436 static uint32_t 6437 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6438 const struct rte_flow_attr *attr, 6439 const struct rte_flow_item items[], 6440 const struct rte_flow_action original_actions[], 6441 bool external, struct rte_flow_error *error) 6442 { 6443 struct mlx5_priv *priv = dev->data->dev_private; 6444 struct rte_flow *flow = NULL; 6445 struct mlx5_flow *dev_flow; 6446 const struct rte_flow_action_rss *rss = NULL; 6447 struct mlx5_translated_action_handle 6448 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6449 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6450 union { 6451 struct mlx5_flow_expand_rss buf; 6452 uint8_t buffer[4096]; 6453 } expand_buffer; 6454 union { 6455 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6456 uint8_t buffer[2048]; 6457 } actions_rx; 6458 union { 6459 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6460 uint8_t buffer[2048]; 6461 } actions_hairpin_tx; 6462 union { 6463 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6464 uint8_t buffer[2048]; 6465 } items_tx; 6466 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6467 struct mlx5_flow_rss_desc *rss_desc; 6468 const struct rte_flow_action *p_actions_rx; 6469 uint32_t i; 6470 uint32_t idx = 0; 6471 int hairpin_flow; 6472 struct rte_flow_attr attr_tx = { .priority = 0 }; 6473 const struct rte_flow_action *actions; 6474 struct rte_flow_action *translated_actions = NULL; 6475 struct mlx5_flow_tunnel *tunnel; 6476 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6477 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6478 struct mlx5_flow_split_info flow_split_info = { 6479 .external = !!external, 6480 .skip_scale = 0, 6481 .flow_idx = 0, 6482 .prefix_mark = 0, 6483 .prefix_layers = 0, 6484 .table_id = 0 6485 }; 6486 int ret; 6487 6488 MLX5_ASSERT(wks); 6489 rss_desc = &wks->rss_desc; 6490 ret = flow_action_handles_translate(dev, original_actions, 6491 indir_actions, 6492 &indir_actions_n, 6493 &translated_actions, error); 6494 if (ret < 0) { 6495 MLX5_ASSERT(translated_actions == NULL); 6496 return 0; 6497 } 6498 actions = translated_actions ? translated_actions : original_actions; 6499 p_actions_rx = actions; 6500 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6501 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6502 external, hairpin_flow, error); 6503 if (ret < 0) 6504 goto error_before_hairpin_split; 6505 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6506 if (!flow) { 6507 rte_errno = ENOMEM; 6508 goto error_before_hairpin_split; 6509 } 6510 if (hairpin_flow > 0) { 6511 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6512 rte_errno = EINVAL; 6513 goto error_before_hairpin_split; 6514 } 6515 flow_hairpin_split(dev, actions, actions_rx.actions, 6516 actions_hairpin_tx.actions, items_tx.items, 6517 idx); 6518 p_actions_rx = actions_rx.actions; 6519 } 6520 flow_split_info.flow_idx = idx; 6521 flow->drv_type = flow_get_drv_type(dev, attr); 6522 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6523 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6524 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6525 /* RSS Action only works on NIC RX domain */ 6526 if (attr->ingress && !attr->transfer) 6527 rss = flow_get_rss_action(dev, p_actions_rx); 6528 if (rss) { 6529 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6530 return 0; 6531 /* 6532 * The following information is required by 6533 * mlx5_flow_hashfields_adjust() in advance. 6534 */ 6535 rss_desc->level = rss->level; 6536 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6537 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6538 } 6539 flow->dev_handles = 0; 6540 if (rss && rss->types) { 6541 unsigned int graph_root; 6542 6543 graph_root = find_graph_root(rss->level); 6544 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6545 items, rss->types, 6546 mlx5_support_expansion, graph_root); 6547 MLX5_ASSERT(ret > 0 && 6548 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6549 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6550 for (i = 0; i < buf->entries; ++i) 6551 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6552 } 6553 } else { 6554 buf->entries = 1; 6555 buf->entry[0].pattern = (void *)(uintptr_t)items; 6556 } 6557 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6558 indir_actions_n); 6559 for (i = 0; i < buf->entries; ++i) { 6560 /* Initialize flow split data. */ 6561 flow_split_info.prefix_layers = 0; 6562 flow_split_info.prefix_mark = 0; 6563 flow_split_info.skip_scale = 0; 6564 /* 6565 * The splitter may create multiple dev_flows, 6566 * depending on configuration. In the simplest 6567 * case it just creates unmodified original flow. 6568 */ 6569 ret = flow_create_split_outer(dev, flow, attr, 6570 buf->entry[i].pattern, 6571 p_actions_rx, &flow_split_info, 6572 error); 6573 if (ret < 0) 6574 goto error; 6575 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6576 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6577 p_actions_rx, 6578 idx, 6579 wks->flows[0].tunnel, 6580 &default_miss_ctx, 6581 error); 6582 if (ret < 0) { 6583 mlx5_free(default_miss_ctx.queue); 6584 goto error; 6585 } 6586 } 6587 } 6588 /* Create the tx flow. */ 6589 if (hairpin_flow) { 6590 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6591 attr_tx.ingress = 0; 6592 attr_tx.egress = 1; 6593 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6594 actions_hairpin_tx.actions, 6595 idx, error); 6596 if (!dev_flow) 6597 goto error; 6598 dev_flow->flow = flow; 6599 dev_flow->external = 0; 6600 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6601 dev_flow->handle, next); 6602 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6603 items_tx.items, 6604 actions_hairpin_tx.actions, error); 6605 if (ret < 0) 6606 goto error; 6607 } 6608 /* 6609 * Update the metadata register copy table. If extensive 6610 * metadata feature is enabled and registers are supported 6611 * we might create the extra rte_flow for each unique 6612 * MARK/FLAG action ID. 6613 * 6614 * The table is updated for ingress Flows only, because 6615 * the egress Flows belong to the different device and 6616 * copy table should be updated in peer NIC Rx domain. 6617 */ 6618 if (attr->ingress && 6619 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6620 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6621 if (ret) 6622 goto error; 6623 } 6624 /* 6625 * If the flow is external (from application) OR device is started, 6626 * OR mreg discover, then apply immediately. 6627 */ 6628 if (external || dev->data->dev_started || 6629 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6630 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6631 ret = flow_drv_apply(dev, flow, error); 6632 if (ret < 0) 6633 goto error; 6634 } 6635 flow->type = type; 6636 flow_rxq_flags_set(dev, flow); 6637 rte_free(translated_actions); 6638 tunnel = flow_tunnel_from_rule(wks->flows); 6639 if (tunnel) { 6640 flow->tunnel = 1; 6641 flow->tunnel_id = tunnel->tunnel_id; 6642 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6643 mlx5_free(default_miss_ctx.queue); 6644 } 6645 mlx5_flow_pop_thread_workspace(); 6646 return idx; 6647 error: 6648 MLX5_ASSERT(flow); 6649 ret = rte_errno; /* Save rte_errno before cleanup. */ 6650 flow_mreg_del_copy_action(dev, flow); 6651 flow_drv_destroy(dev, flow); 6652 if (rss_desc->shared_rss) 6653 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6654 mlx5_ipool_get 6655 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6656 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6657 mlx5_ipool_free(priv->flows[type], idx); 6658 rte_errno = ret; /* Restore rte_errno. */ 6659 ret = rte_errno; 6660 rte_errno = ret; 6661 mlx5_flow_pop_thread_workspace(); 6662 error_before_hairpin_split: 6663 rte_free(translated_actions); 6664 return 0; 6665 } 6666 6667 /** 6668 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6669 * incoming packets to table 1. 6670 * 6671 * Other flow rules, requested for group n, will be created in 6672 * e-switch table n+1. 6673 * Jump action to e-switch group n will be created to group n+1. 6674 * 6675 * Used when working in switchdev mode, to utilise advantages of table 1 6676 * and above. 6677 * 6678 * @param dev 6679 * Pointer to Ethernet device. 6680 * 6681 * @return 6682 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6683 */ 6684 struct rte_flow * 6685 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6686 { 6687 const struct rte_flow_attr attr = { 6688 .group = 0, 6689 .priority = 0, 6690 .ingress = 1, 6691 .egress = 0, 6692 .transfer = 1, 6693 }; 6694 const struct rte_flow_item pattern = { 6695 .type = RTE_FLOW_ITEM_TYPE_END, 6696 }; 6697 struct rte_flow_action_jump jump = { 6698 .group = 1, 6699 }; 6700 const struct rte_flow_action actions[] = { 6701 { 6702 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6703 .conf = &jump, 6704 }, 6705 { 6706 .type = RTE_FLOW_ACTION_TYPE_END, 6707 }, 6708 }; 6709 struct rte_flow_error error; 6710 6711 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6712 &attr, &pattern, 6713 actions, false, &error); 6714 } 6715 6716 /** 6717 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6718 * and sq number, directs all packets to peer vport. 6719 * 6720 * @param dev 6721 * Pointer to Ethernet device. 6722 * @param txq 6723 * Txq index. 6724 * 6725 * @return 6726 * Flow ID on success, 0 otherwise and rte_errno is set. 6727 */ 6728 uint32_t 6729 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6730 { 6731 struct rte_flow_attr attr = { 6732 .group = 0, 6733 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6734 .ingress = 1, 6735 .egress = 0, 6736 .transfer = 1, 6737 }; 6738 struct rte_flow_item_port_id port_spec = { 6739 .id = MLX5_PORT_ESW_MGR, 6740 }; 6741 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6742 .queue = txq, 6743 }; 6744 struct rte_flow_item pattern[] = { 6745 { 6746 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6747 .spec = &port_spec, 6748 }, 6749 { 6750 .type = (enum rte_flow_item_type) 6751 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6752 .spec = &txq_spec, 6753 }, 6754 { 6755 .type = RTE_FLOW_ITEM_TYPE_END, 6756 }, 6757 }; 6758 struct rte_flow_action_jump jump = { 6759 .group = 1, 6760 }; 6761 struct rte_flow_action_port_id port = { 6762 .id = dev->data->port_id, 6763 }; 6764 struct rte_flow_action actions[] = { 6765 { 6766 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6767 .conf = &jump, 6768 }, 6769 { 6770 .type = RTE_FLOW_ACTION_TYPE_END, 6771 }, 6772 }; 6773 struct rte_flow_error error; 6774 6775 /* 6776 * Creates group 0, highest priority jump flow. 6777 * Matches txq to bypass kernel packets. 6778 */ 6779 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 6780 false, &error) == 0) 6781 return 0; 6782 /* Create group 1, lowest priority redirect flow for txq. */ 6783 attr.group = 1; 6784 actions[0].conf = &port; 6785 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 6786 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 6787 actions, false, &error); 6788 } 6789 6790 /** 6791 * Validate a flow supported by the NIC. 6792 * 6793 * @see rte_flow_validate() 6794 * @see rte_flow_ops 6795 */ 6796 int 6797 mlx5_flow_validate(struct rte_eth_dev *dev, 6798 const struct rte_flow_attr *attr, 6799 const struct rte_flow_item items[], 6800 const struct rte_flow_action original_actions[], 6801 struct rte_flow_error *error) 6802 { 6803 int hairpin_flow; 6804 struct mlx5_translated_action_handle 6805 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6806 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6807 const struct rte_flow_action *actions; 6808 struct rte_flow_action *translated_actions = NULL; 6809 int ret = flow_action_handles_translate(dev, original_actions, 6810 indir_actions, 6811 &indir_actions_n, 6812 &translated_actions, error); 6813 6814 if (ret) 6815 return ret; 6816 actions = translated_actions ? translated_actions : original_actions; 6817 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6818 ret = flow_drv_validate(dev, attr, items, actions, 6819 true, hairpin_flow, error); 6820 rte_free(translated_actions); 6821 return ret; 6822 } 6823 6824 /** 6825 * Create a flow. 6826 * 6827 * @see rte_flow_create() 6828 * @see rte_flow_ops 6829 */ 6830 struct rte_flow * 6831 mlx5_flow_create(struct rte_eth_dev *dev, 6832 const struct rte_flow_attr *attr, 6833 const struct rte_flow_item items[], 6834 const struct rte_flow_action actions[], 6835 struct rte_flow_error *error) 6836 { 6837 /* 6838 * If the device is not started yet, it is not allowed to created a 6839 * flow from application. PMD default flows and traffic control flows 6840 * are not affected. 6841 */ 6842 if (unlikely(!dev->data->dev_started)) { 6843 DRV_LOG(DEBUG, "port %u is not started when " 6844 "inserting a flow", dev->data->port_id); 6845 rte_flow_error_set(error, ENODEV, 6846 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6847 NULL, 6848 "port not started"); 6849 return NULL; 6850 } 6851 6852 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6853 attr, items, actions, 6854 true, error); 6855 } 6856 6857 /** 6858 * Destroy a flow in a list. 6859 * 6860 * @param dev 6861 * Pointer to Ethernet device. 6862 * @param[in] flow_idx 6863 * Index of flow to destroy. 6864 */ 6865 static void 6866 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6867 uint32_t flow_idx) 6868 { 6869 struct mlx5_priv *priv = dev->data->dev_private; 6870 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6871 6872 if (!flow) 6873 return; 6874 MLX5_ASSERT(flow->type == type); 6875 /* 6876 * Update RX queue flags only if port is started, otherwise it is 6877 * already clean. 6878 */ 6879 if (dev->data->dev_started) 6880 flow_rxq_flags_trim(dev, flow); 6881 flow_drv_destroy(dev, flow); 6882 if (flow->tunnel) { 6883 struct mlx5_flow_tunnel *tunnel; 6884 6885 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6886 RTE_VERIFY(tunnel); 6887 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6888 mlx5_flow_tunnel_free(dev, tunnel); 6889 } 6890 flow_mreg_del_copy_action(dev, flow); 6891 mlx5_ipool_free(priv->flows[type], flow_idx); 6892 } 6893 6894 /** 6895 * Destroy all flows. 6896 * 6897 * @param dev 6898 * Pointer to Ethernet device. 6899 * @param type 6900 * Flow type to be flushed. 6901 * @param active 6902 * If flushing is called actively. 6903 */ 6904 void 6905 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6906 bool active) 6907 { 6908 struct mlx5_priv *priv = dev->data->dev_private; 6909 uint32_t num_flushed = 0, fidx = 1; 6910 struct rte_flow *flow; 6911 6912 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6913 flow_list_destroy(dev, type, fidx); 6914 num_flushed++; 6915 } 6916 if (active) { 6917 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6918 dev->data->port_id, num_flushed); 6919 } 6920 } 6921 6922 /** 6923 * Stop all default actions for flows. 6924 * 6925 * @param dev 6926 * Pointer to Ethernet device. 6927 */ 6928 void 6929 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6930 { 6931 flow_mreg_del_default_copy_action(dev); 6932 flow_rxq_flags_clear(dev); 6933 } 6934 6935 /** 6936 * Start all default actions for flows. 6937 * 6938 * @param dev 6939 * Pointer to Ethernet device. 6940 * @return 6941 * 0 on success, a negative errno value otherwise and rte_errno is set. 6942 */ 6943 int 6944 mlx5_flow_start_default(struct rte_eth_dev *dev) 6945 { 6946 struct rte_flow_error error; 6947 6948 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6949 return flow_mreg_add_default_copy_action(dev, &error); 6950 } 6951 6952 /** 6953 * Release key of thread specific flow workspace data. 6954 */ 6955 void 6956 flow_release_workspace(void *data) 6957 { 6958 struct mlx5_flow_workspace *wks = data; 6959 struct mlx5_flow_workspace *next; 6960 6961 while (wks) { 6962 next = wks->next; 6963 free(wks->rss_desc.queue); 6964 free(wks); 6965 wks = next; 6966 } 6967 } 6968 6969 /** 6970 * Get thread specific current flow workspace. 6971 * 6972 * @return pointer to thread specific flow workspace data, NULL on error. 6973 */ 6974 struct mlx5_flow_workspace* 6975 mlx5_flow_get_thread_workspace(void) 6976 { 6977 struct mlx5_flow_workspace *data; 6978 6979 data = mlx5_flow_os_get_specific_workspace(); 6980 MLX5_ASSERT(data && data->inuse); 6981 if (!data || !data->inuse) 6982 DRV_LOG(ERR, "flow workspace not initialized."); 6983 return data; 6984 } 6985 6986 /** 6987 * Allocate and init new flow workspace. 6988 * 6989 * @return pointer to flow workspace data, NULL on error. 6990 */ 6991 static struct mlx5_flow_workspace* 6992 flow_alloc_thread_workspace(void) 6993 { 6994 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6995 6996 if (!data) { 6997 DRV_LOG(ERR, "Failed to allocate flow workspace " 6998 "memory."); 6999 return NULL; 7000 } 7001 data->rss_desc.queue = calloc(1, 7002 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7003 if (!data->rss_desc.queue) 7004 goto err; 7005 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7006 return data; 7007 err: 7008 if (data->rss_desc.queue) 7009 free(data->rss_desc.queue); 7010 free(data); 7011 return NULL; 7012 } 7013 7014 /** 7015 * Get new thread specific flow workspace. 7016 * 7017 * If current workspace inuse, create new one and set as current. 7018 * 7019 * @return pointer to thread specific flow workspace data, NULL on error. 7020 */ 7021 static struct mlx5_flow_workspace* 7022 mlx5_flow_push_thread_workspace(void) 7023 { 7024 struct mlx5_flow_workspace *curr; 7025 struct mlx5_flow_workspace *data; 7026 7027 curr = mlx5_flow_os_get_specific_workspace(); 7028 if (!curr) { 7029 data = flow_alloc_thread_workspace(); 7030 if (!data) 7031 return NULL; 7032 } else if (!curr->inuse) { 7033 data = curr; 7034 } else if (curr->next) { 7035 data = curr->next; 7036 } else { 7037 data = flow_alloc_thread_workspace(); 7038 if (!data) 7039 return NULL; 7040 curr->next = data; 7041 data->prev = curr; 7042 } 7043 data->inuse = 1; 7044 data->flow_idx = 0; 7045 /* Set as current workspace */ 7046 if (mlx5_flow_os_set_specific_workspace(data)) 7047 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7048 return data; 7049 } 7050 7051 /** 7052 * Close current thread specific flow workspace. 7053 * 7054 * If previous workspace available, set it as current. 7055 * 7056 * @return pointer to thread specific flow workspace data, NULL on error. 7057 */ 7058 static void 7059 mlx5_flow_pop_thread_workspace(void) 7060 { 7061 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7062 7063 if (!data) 7064 return; 7065 if (!data->inuse) { 7066 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7067 return; 7068 } 7069 data->inuse = 0; 7070 if (!data->prev) 7071 return; 7072 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7073 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7074 } 7075 7076 /** 7077 * Verify the flow list is empty 7078 * 7079 * @param dev 7080 * Pointer to Ethernet device. 7081 * 7082 * @return the number of flows not released. 7083 */ 7084 int 7085 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7086 { 7087 struct mlx5_priv *priv = dev->data->dev_private; 7088 struct rte_flow *flow; 7089 uint32_t idx = 0; 7090 int ret = 0, i; 7091 7092 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7093 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7094 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7095 dev->data->port_id, (void *)flow); 7096 ret++; 7097 } 7098 } 7099 return ret; 7100 } 7101 7102 /** 7103 * Enable default hairpin egress flow. 7104 * 7105 * @param dev 7106 * Pointer to Ethernet device. 7107 * @param queue 7108 * The queue index. 7109 * 7110 * @return 7111 * 0 on success, a negative errno value otherwise and rte_errno is set. 7112 */ 7113 int 7114 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7115 uint32_t queue) 7116 { 7117 const struct rte_flow_attr attr = { 7118 .egress = 1, 7119 .priority = 0, 7120 }; 7121 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7122 .queue = queue, 7123 }; 7124 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7125 .queue = UINT32_MAX, 7126 }; 7127 struct rte_flow_item items[] = { 7128 { 7129 .type = (enum rte_flow_item_type) 7130 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7131 .spec = &queue_spec, 7132 .last = NULL, 7133 .mask = &queue_mask, 7134 }, 7135 { 7136 .type = RTE_FLOW_ITEM_TYPE_END, 7137 }, 7138 }; 7139 struct rte_flow_action_jump jump = { 7140 .group = MLX5_HAIRPIN_TX_TABLE, 7141 }; 7142 struct rte_flow_action actions[2]; 7143 uint32_t flow_idx; 7144 struct rte_flow_error error; 7145 7146 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7147 actions[0].conf = &jump; 7148 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7149 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7150 &attr, items, actions, false, &error); 7151 if (!flow_idx) { 7152 DRV_LOG(DEBUG, 7153 "Failed to create ctrl flow: rte_errno(%d)," 7154 " type(%d), message(%s)", 7155 rte_errno, error.type, 7156 error.message ? error.message : " (no stated reason)"); 7157 return -rte_errno; 7158 } 7159 return 0; 7160 } 7161 7162 /** 7163 * Enable a control flow configured from the control plane. 7164 * 7165 * @param dev 7166 * Pointer to Ethernet device. 7167 * @param eth_spec 7168 * An Ethernet flow spec to apply. 7169 * @param eth_mask 7170 * An Ethernet flow mask to apply. 7171 * @param vlan_spec 7172 * A VLAN flow spec to apply. 7173 * @param vlan_mask 7174 * A VLAN flow mask to apply. 7175 * 7176 * @return 7177 * 0 on success, a negative errno value otherwise and rte_errno is set. 7178 */ 7179 int 7180 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7181 struct rte_flow_item_eth *eth_spec, 7182 struct rte_flow_item_eth *eth_mask, 7183 struct rte_flow_item_vlan *vlan_spec, 7184 struct rte_flow_item_vlan *vlan_mask) 7185 { 7186 struct mlx5_priv *priv = dev->data->dev_private; 7187 const struct rte_flow_attr attr = { 7188 .ingress = 1, 7189 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7190 }; 7191 struct rte_flow_item items[] = { 7192 { 7193 .type = RTE_FLOW_ITEM_TYPE_ETH, 7194 .spec = eth_spec, 7195 .last = NULL, 7196 .mask = eth_mask, 7197 }, 7198 { 7199 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7200 RTE_FLOW_ITEM_TYPE_END, 7201 .spec = vlan_spec, 7202 .last = NULL, 7203 .mask = vlan_mask, 7204 }, 7205 { 7206 .type = RTE_FLOW_ITEM_TYPE_END, 7207 }, 7208 }; 7209 uint16_t queue[priv->reta_idx_n]; 7210 struct rte_flow_action_rss action_rss = { 7211 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7212 .level = 0, 7213 .types = priv->rss_conf.rss_hf, 7214 .key_len = priv->rss_conf.rss_key_len, 7215 .queue_num = priv->reta_idx_n, 7216 .key = priv->rss_conf.rss_key, 7217 .queue = queue, 7218 }; 7219 struct rte_flow_action actions[] = { 7220 { 7221 .type = RTE_FLOW_ACTION_TYPE_RSS, 7222 .conf = &action_rss, 7223 }, 7224 { 7225 .type = RTE_FLOW_ACTION_TYPE_END, 7226 }, 7227 }; 7228 uint32_t flow_idx; 7229 struct rte_flow_error error; 7230 unsigned int i; 7231 7232 if (!priv->reta_idx_n || !priv->rxqs_n) { 7233 return 0; 7234 } 7235 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7236 action_rss.types = 0; 7237 for (i = 0; i != priv->reta_idx_n; ++i) 7238 queue[i] = (*priv->reta_idx)[i]; 7239 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7240 &attr, items, actions, false, &error); 7241 if (!flow_idx) 7242 return -rte_errno; 7243 return 0; 7244 } 7245 7246 /** 7247 * Enable a flow control configured from the control plane. 7248 * 7249 * @param dev 7250 * Pointer to Ethernet device. 7251 * @param eth_spec 7252 * An Ethernet flow spec to apply. 7253 * @param eth_mask 7254 * An Ethernet flow mask to apply. 7255 * 7256 * @return 7257 * 0 on success, a negative errno value otherwise and rte_errno is set. 7258 */ 7259 int 7260 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7261 struct rte_flow_item_eth *eth_spec, 7262 struct rte_flow_item_eth *eth_mask) 7263 { 7264 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7265 } 7266 7267 /** 7268 * Create default miss flow rule matching lacp traffic 7269 * 7270 * @param dev 7271 * Pointer to Ethernet device. 7272 * @param eth_spec 7273 * An Ethernet flow spec to apply. 7274 * 7275 * @return 7276 * 0 on success, a negative errno value otherwise and rte_errno is set. 7277 */ 7278 int 7279 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7280 { 7281 /* 7282 * The LACP matching is done by only using ether type since using 7283 * a multicast dst mac causes kernel to give low priority to this flow. 7284 */ 7285 static const struct rte_flow_item_eth lacp_spec = { 7286 .type = RTE_BE16(0x8809), 7287 }; 7288 static const struct rte_flow_item_eth lacp_mask = { 7289 .type = 0xffff, 7290 }; 7291 const struct rte_flow_attr attr = { 7292 .ingress = 1, 7293 }; 7294 struct rte_flow_item items[] = { 7295 { 7296 .type = RTE_FLOW_ITEM_TYPE_ETH, 7297 .spec = &lacp_spec, 7298 .mask = &lacp_mask, 7299 }, 7300 { 7301 .type = RTE_FLOW_ITEM_TYPE_END, 7302 }, 7303 }; 7304 struct rte_flow_action actions[] = { 7305 { 7306 .type = (enum rte_flow_action_type) 7307 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7308 }, 7309 { 7310 .type = RTE_FLOW_ACTION_TYPE_END, 7311 }, 7312 }; 7313 struct rte_flow_error error; 7314 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7315 &attr, items, actions, 7316 false, &error); 7317 7318 if (!flow_idx) 7319 return -rte_errno; 7320 return 0; 7321 } 7322 7323 /** 7324 * Destroy a flow. 7325 * 7326 * @see rte_flow_destroy() 7327 * @see rte_flow_ops 7328 */ 7329 int 7330 mlx5_flow_destroy(struct rte_eth_dev *dev, 7331 struct rte_flow *flow, 7332 struct rte_flow_error *error __rte_unused) 7333 { 7334 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7335 (uintptr_t)(void *)flow); 7336 return 0; 7337 } 7338 7339 /** 7340 * Destroy all flows. 7341 * 7342 * @see rte_flow_flush() 7343 * @see rte_flow_ops 7344 */ 7345 int 7346 mlx5_flow_flush(struct rte_eth_dev *dev, 7347 struct rte_flow_error *error __rte_unused) 7348 { 7349 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7350 return 0; 7351 } 7352 7353 /** 7354 * Isolated mode. 7355 * 7356 * @see rte_flow_isolate() 7357 * @see rte_flow_ops 7358 */ 7359 int 7360 mlx5_flow_isolate(struct rte_eth_dev *dev, 7361 int enable, 7362 struct rte_flow_error *error) 7363 { 7364 struct mlx5_priv *priv = dev->data->dev_private; 7365 7366 if (dev->data->dev_started) { 7367 rte_flow_error_set(error, EBUSY, 7368 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7369 NULL, 7370 "port must be stopped first"); 7371 return -rte_errno; 7372 } 7373 priv->isolated = !!enable; 7374 if (enable) 7375 dev->dev_ops = &mlx5_dev_ops_isolate; 7376 else 7377 dev->dev_ops = &mlx5_dev_ops; 7378 7379 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7380 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7381 7382 return 0; 7383 } 7384 7385 /** 7386 * Query a flow. 7387 * 7388 * @see rte_flow_query() 7389 * @see rte_flow_ops 7390 */ 7391 static int 7392 flow_drv_query(struct rte_eth_dev *dev, 7393 uint32_t flow_idx, 7394 const struct rte_flow_action *actions, 7395 void *data, 7396 struct rte_flow_error *error) 7397 { 7398 struct mlx5_priv *priv = dev->data->dev_private; 7399 const struct mlx5_flow_driver_ops *fops; 7400 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7401 flow_idx); 7402 enum mlx5_flow_drv_type ftype; 7403 7404 if (!flow) { 7405 return rte_flow_error_set(error, ENOENT, 7406 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7407 NULL, 7408 "invalid flow handle"); 7409 } 7410 ftype = flow->drv_type; 7411 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7412 fops = flow_get_drv_ops(ftype); 7413 7414 return fops->query(dev, flow, actions, data, error); 7415 } 7416 7417 /** 7418 * Query a flow. 7419 * 7420 * @see rte_flow_query() 7421 * @see rte_flow_ops 7422 */ 7423 int 7424 mlx5_flow_query(struct rte_eth_dev *dev, 7425 struct rte_flow *flow, 7426 const struct rte_flow_action *actions, 7427 void *data, 7428 struct rte_flow_error *error) 7429 { 7430 int ret; 7431 7432 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7433 error); 7434 if (ret < 0) 7435 return ret; 7436 return 0; 7437 } 7438 7439 /** 7440 * Get rte_flow callbacks. 7441 * 7442 * @param dev 7443 * Pointer to Ethernet device structure. 7444 * @param ops 7445 * Pointer to operation-specific structure. 7446 * 7447 * @return 0 7448 */ 7449 int 7450 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7451 const struct rte_flow_ops **ops) 7452 { 7453 *ops = &mlx5_flow_ops; 7454 return 0; 7455 } 7456 7457 /** 7458 * Validate meter policy actions. 7459 * Dispatcher for action type specific validation. 7460 * 7461 * @param[in] dev 7462 * Pointer to the Ethernet device structure. 7463 * @param[in] action 7464 * The meter policy action object to validate. 7465 * @param[in] attr 7466 * Attributes of flow to determine steering domain. 7467 * @param[out] is_rss 7468 * Is RSS or not. 7469 * @param[out] domain_bitmap 7470 * Domain bitmap. 7471 * @param[out] is_def_policy 7472 * Is default policy or not. 7473 * @param[out] error 7474 * Perform verbose error reporting if not NULL. Initialized in case of 7475 * error only. 7476 * 7477 * @return 7478 * 0 on success, otherwise negative errno value. 7479 */ 7480 int 7481 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7482 const struct rte_flow_action *actions[RTE_COLORS], 7483 struct rte_flow_attr *attr, 7484 bool *is_rss, 7485 uint8_t *domain_bitmap, 7486 uint8_t *policy_mode, 7487 struct rte_mtr_error *error) 7488 { 7489 const struct mlx5_flow_driver_ops *fops; 7490 7491 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7492 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7493 domain_bitmap, policy_mode, error); 7494 } 7495 7496 /** 7497 * Destroy the meter table set. 7498 * 7499 * @param[in] dev 7500 * Pointer to Ethernet device. 7501 * @param[in] mtr_policy 7502 * Meter policy struct. 7503 */ 7504 void 7505 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7506 struct mlx5_flow_meter_policy *mtr_policy) 7507 { 7508 const struct mlx5_flow_driver_ops *fops; 7509 7510 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7511 fops->destroy_mtr_acts(dev, mtr_policy); 7512 } 7513 7514 /** 7515 * Create policy action, lock free, 7516 * (mutex should be acquired by caller). 7517 * Dispatcher for action type specific call. 7518 * 7519 * @param[in] dev 7520 * Pointer to the Ethernet device structure. 7521 * @param[in] mtr_policy 7522 * Meter policy struct. 7523 * @param[in] action 7524 * Action specification used to create meter actions. 7525 * @param[out] error 7526 * Perform verbose error reporting if not NULL. Initialized in case of 7527 * error only. 7528 * 7529 * @return 7530 * 0 on success, otherwise negative errno value. 7531 */ 7532 int 7533 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7534 struct mlx5_flow_meter_policy *mtr_policy, 7535 const struct rte_flow_action *actions[RTE_COLORS], 7536 struct rte_mtr_error *error) 7537 { 7538 const struct mlx5_flow_driver_ops *fops; 7539 7540 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7541 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7542 } 7543 7544 /** 7545 * Create policy rules, lock free, 7546 * (mutex should be acquired by caller). 7547 * Dispatcher for action type specific call. 7548 * 7549 * @param[in] dev 7550 * Pointer to the Ethernet device structure. 7551 * @param[in] mtr_policy 7552 * Meter policy struct. 7553 * 7554 * @return 7555 * 0 on success, -1 otherwise. 7556 */ 7557 int 7558 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7559 struct mlx5_flow_meter_policy *mtr_policy) 7560 { 7561 const struct mlx5_flow_driver_ops *fops; 7562 7563 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7564 return fops->create_policy_rules(dev, mtr_policy); 7565 } 7566 7567 /** 7568 * Destroy policy rules, lock free, 7569 * (mutex should be acquired by caller). 7570 * Dispatcher for action type specific call. 7571 * 7572 * @param[in] dev 7573 * Pointer to the Ethernet device structure. 7574 * @param[in] mtr_policy 7575 * Meter policy struct. 7576 */ 7577 void 7578 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7579 struct mlx5_flow_meter_policy *mtr_policy) 7580 { 7581 const struct mlx5_flow_driver_ops *fops; 7582 7583 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7584 fops->destroy_policy_rules(dev, mtr_policy); 7585 } 7586 7587 /** 7588 * Destroy the default policy table set. 7589 * 7590 * @param[in] dev 7591 * Pointer to Ethernet device. 7592 */ 7593 void 7594 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7595 { 7596 const struct mlx5_flow_driver_ops *fops; 7597 7598 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7599 fops->destroy_def_policy(dev); 7600 } 7601 7602 /** 7603 * Destroy the default policy table set. 7604 * 7605 * @param[in] dev 7606 * Pointer to Ethernet device. 7607 * 7608 * @return 7609 * 0 on success, -1 otherwise. 7610 */ 7611 int 7612 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7613 { 7614 const struct mlx5_flow_driver_ops *fops; 7615 7616 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7617 return fops->create_def_policy(dev); 7618 } 7619 7620 /** 7621 * Create the needed meter and suffix tables. 7622 * 7623 * @param[in] dev 7624 * Pointer to Ethernet device. 7625 * 7626 * @return 7627 * 0 on success, -1 otherwise. 7628 */ 7629 int 7630 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7631 struct mlx5_flow_meter_info *fm, 7632 uint32_t mtr_idx, 7633 uint8_t domain_bitmap) 7634 { 7635 const struct mlx5_flow_driver_ops *fops; 7636 7637 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7638 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7639 } 7640 7641 /** 7642 * Destroy the meter table set. 7643 * 7644 * @param[in] dev 7645 * Pointer to Ethernet device. 7646 * @param[in] tbl 7647 * Pointer to the meter table set. 7648 */ 7649 void 7650 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7651 struct mlx5_flow_meter_info *fm) 7652 { 7653 const struct mlx5_flow_driver_ops *fops; 7654 7655 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7656 fops->destroy_mtr_tbls(dev, fm); 7657 } 7658 7659 /** 7660 * Destroy the global meter drop table. 7661 * 7662 * @param[in] dev 7663 * Pointer to Ethernet device. 7664 */ 7665 void 7666 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7667 { 7668 const struct mlx5_flow_driver_ops *fops; 7669 7670 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7671 fops->destroy_mtr_drop_tbls(dev); 7672 } 7673 7674 /** 7675 * Destroy the sub policy table with RX queue. 7676 * 7677 * @param[in] dev 7678 * Pointer to Ethernet device. 7679 * @param[in] mtr_policy 7680 * Pointer to meter policy table. 7681 */ 7682 void 7683 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7684 struct mlx5_flow_meter_policy *mtr_policy) 7685 { 7686 const struct mlx5_flow_driver_ops *fops; 7687 7688 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7689 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7690 } 7691 7692 /** 7693 * Allocate the needed aso flow meter id. 7694 * 7695 * @param[in] dev 7696 * Pointer to Ethernet device. 7697 * 7698 * @return 7699 * Index to aso flow meter on success, NULL otherwise. 7700 */ 7701 uint32_t 7702 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7703 { 7704 const struct mlx5_flow_driver_ops *fops; 7705 7706 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7707 return fops->create_meter(dev); 7708 } 7709 7710 /** 7711 * Free the aso flow meter id. 7712 * 7713 * @param[in] dev 7714 * Pointer to Ethernet device. 7715 * @param[in] mtr_idx 7716 * Index to aso flow meter to be free. 7717 * 7718 * @return 7719 * 0 on success. 7720 */ 7721 void 7722 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7723 { 7724 const struct mlx5_flow_driver_ops *fops; 7725 7726 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7727 fops->free_meter(dev, mtr_idx); 7728 } 7729 7730 /** 7731 * Allocate a counter. 7732 * 7733 * @param[in] dev 7734 * Pointer to Ethernet device structure. 7735 * 7736 * @return 7737 * Index to allocated counter on success, 0 otherwise. 7738 */ 7739 uint32_t 7740 mlx5_counter_alloc(struct rte_eth_dev *dev) 7741 { 7742 const struct mlx5_flow_driver_ops *fops; 7743 struct rte_flow_attr attr = { .transfer = 0 }; 7744 7745 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7746 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7747 return fops->counter_alloc(dev); 7748 } 7749 DRV_LOG(ERR, 7750 "port %u counter allocate is not supported.", 7751 dev->data->port_id); 7752 return 0; 7753 } 7754 7755 /** 7756 * Free a counter. 7757 * 7758 * @param[in] dev 7759 * Pointer to Ethernet device structure. 7760 * @param[in] cnt 7761 * Index to counter to be free. 7762 */ 7763 void 7764 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7765 { 7766 const struct mlx5_flow_driver_ops *fops; 7767 struct rte_flow_attr attr = { .transfer = 0 }; 7768 7769 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7770 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7771 fops->counter_free(dev, cnt); 7772 return; 7773 } 7774 DRV_LOG(ERR, 7775 "port %u counter free is not supported.", 7776 dev->data->port_id); 7777 } 7778 7779 /** 7780 * Query counter statistics. 7781 * 7782 * @param[in] dev 7783 * Pointer to Ethernet device structure. 7784 * @param[in] cnt 7785 * Index to counter to query. 7786 * @param[in] clear 7787 * Set to clear counter statistics. 7788 * @param[out] pkts 7789 * The counter hits packets number to save. 7790 * @param[out] bytes 7791 * The counter hits bytes number to save. 7792 * 7793 * @return 7794 * 0 on success, a negative errno value otherwise. 7795 */ 7796 int 7797 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7798 bool clear, uint64_t *pkts, uint64_t *bytes) 7799 { 7800 const struct mlx5_flow_driver_ops *fops; 7801 struct rte_flow_attr attr = { .transfer = 0 }; 7802 7803 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7804 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7805 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7806 } 7807 DRV_LOG(ERR, 7808 "port %u counter query is not supported.", 7809 dev->data->port_id); 7810 return -ENOTSUP; 7811 } 7812 7813 /** 7814 * Allocate a new memory for the counter values wrapped by all the needed 7815 * management. 7816 * 7817 * @param[in] sh 7818 * Pointer to mlx5_dev_ctx_shared object. 7819 * 7820 * @return 7821 * 0 on success, a negative errno value otherwise. 7822 */ 7823 static int 7824 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7825 { 7826 struct mlx5_counter_stats_mem_mng *mem_mng; 7827 volatile struct flow_counter_stats *raw_data; 7828 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7829 int size = (sizeof(struct flow_counter_stats) * 7830 MLX5_COUNTERS_PER_POOL + 7831 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7832 sizeof(struct mlx5_counter_stats_mem_mng); 7833 size_t pgsize = rte_mem_page_size(); 7834 uint8_t *mem; 7835 int ret; 7836 int i; 7837 7838 if (pgsize == (size_t)-1) { 7839 DRV_LOG(ERR, "Failed to get mem page size"); 7840 rte_errno = ENOMEM; 7841 return -ENOMEM; 7842 } 7843 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7844 if (!mem) { 7845 rte_errno = ENOMEM; 7846 return -ENOMEM; 7847 } 7848 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7849 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7850 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 7851 sh->cdev->pdn, mem, size, 7852 &mem_mng->wm); 7853 if (ret) { 7854 rte_errno = errno; 7855 mlx5_free(mem); 7856 return -rte_errno; 7857 } 7858 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7859 raw_data = (volatile struct flow_counter_stats *)mem; 7860 for (i = 0; i < raws_n; ++i) { 7861 mem_mng->raws[i].mem_mng = mem_mng; 7862 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7863 } 7864 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7865 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7866 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7867 next); 7868 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7869 sh->cmng.mem_mng = mem_mng; 7870 return 0; 7871 } 7872 7873 /** 7874 * Set the statistic memory to the new counter pool. 7875 * 7876 * @param[in] sh 7877 * Pointer to mlx5_dev_ctx_shared object. 7878 * @param[in] pool 7879 * Pointer to the pool to set the statistic memory. 7880 * 7881 * @return 7882 * 0 on success, a negative errno value otherwise. 7883 */ 7884 static int 7885 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7886 struct mlx5_flow_counter_pool *pool) 7887 { 7888 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7889 /* Resize statistic memory once used out. */ 7890 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7891 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7892 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7893 return -1; 7894 } 7895 rte_spinlock_lock(&pool->sl); 7896 pool->raw = cmng->mem_mng->raws + pool->index % 7897 MLX5_CNT_CONTAINER_RESIZE; 7898 rte_spinlock_unlock(&pool->sl); 7899 pool->raw_hw = NULL; 7900 return 0; 7901 } 7902 7903 #define MLX5_POOL_QUERY_FREQ_US 1000000 7904 7905 /** 7906 * Set the periodic procedure for triggering asynchronous batch queries for all 7907 * the counter pools. 7908 * 7909 * @param[in] sh 7910 * Pointer to mlx5_dev_ctx_shared object. 7911 */ 7912 void 7913 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7914 { 7915 uint32_t pools_n, us; 7916 7917 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7918 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7919 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7920 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7921 sh->cmng.query_thread_on = 0; 7922 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7923 } else { 7924 sh->cmng.query_thread_on = 1; 7925 } 7926 } 7927 7928 /** 7929 * The periodic procedure for triggering asynchronous batch queries for all the 7930 * counter pools. This function is probably called by the host thread. 7931 * 7932 * @param[in] arg 7933 * The parameter for the alarm process. 7934 */ 7935 void 7936 mlx5_flow_query_alarm(void *arg) 7937 { 7938 struct mlx5_dev_ctx_shared *sh = arg; 7939 int ret; 7940 uint16_t pool_index = sh->cmng.pool_index; 7941 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7942 struct mlx5_flow_counter_pool *pool; 7943 uint16_t n_valid; 7944 7945 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7946 goto set_alarm; 7947 rte_spinlock_lock(&cmng->pool_update_sl); 7948 pool = cmng->pools[pool_index]; 7949 n_valid = cmng->n_valid; 7950 rte_spinlock_unlock(&cmng->pool_update_sl); 7951 /* Set the statistic memory to the new created pool. */ 7952 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7953 goto set_alarm; 7954 if (pool->raw_hw) 7955 /* There is a pool query in progress. */ 7956 goto set_alarm; 7957 pool->raw_hw = 7958 LIST_FIRST(&sh->cmng.free_stat_raws); 7959 if (!pool->raw_hw) 7960 /* No free counter statistics raw memory. */ 7961 goto set_alarm; 7962 /* 7963 * Identify the counters released between query trigger and query 7964 * handle more efficiently. The counter released in this gap period 7965 * should wait for a new round of query as the new arrived packets 7966 * will not be taken into account. 7967 */ 7968 pool->query_gen++; 7969 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7970 MLX5_COUNTERS_PER_POOL, 7971 NULL, NULL, 7972 pool->raw_hw->mem_mng->wm.lkey, 7973 (void *)(uintptr_t) 7974 pool->raw_hw->data, 7975 sh->devx_comp, 7976 (uint64_t)(uintptr_t)pool); 7977 if (ret) { 7978 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7979 " %d", pool->min_dcs->id); 7980 pool->raw_hw = NULL; 7981 goto set_alarm; 7982 } 7983 LIST_REMOVE(pool->raw_hw, next); 7984 sh->cmng.pending_queries++; 7985 pool_index++; 7986 if (pool_index >= n_valid) 7987 pool_index = 0; 7988 set_alarm: 7989 sh->cmng.pool_index = pool_index; 7990 mlx5_set_query_alarm(sh); 7991 } 7992 7993 /** 7994 * Check and callback event for new aged flow in the counter pool 7995 * 7996 * @param[in] sh 7997 * Pointer to mlx5_dev_ctx_shared object. 7998 * @param[in] pool 7999 * Pointer to Current counter pool. 8000 */ 8001 static void 8002 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8003 struct mlx5_flow_counter_pool *pool) 8004 { 8005 struct mlx5_priv *priv; 8006 struct mlx5_flow_counter *cnt; 8007 struct mlx5_age_info *age_info; 8008 struct mlx5_age_param *age_param; 8009 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8010 struct mlx5_counter_stats_raw *prev = pool->raw; 8011 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8012 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8013 uint16_t expected = AGE_CANDIDATE; 8014 uint32_t i; 8015 8016 pool->time_of_last_age_check = curr_time; 8017 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8018 cnt = MLX5_POOL_GET_CNT(pool, i); 8019 age_param = MLX5_CNT_TO_AGE(cnt); 8020 if (__atomic_load_n(&age_param->state, 8021 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8022 continue; 8023 if (cur->data[i].hits != prev->data[i].hits) { 8024 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8025 __ATOMIC_RELAXED); 8026 continue; 8027 } 8028 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8029 time_delta, 8030 __ATOMIC_RELAXED) <= age_param->timeout) 8031 continue; 8032 /** 8033 * Hold the lock first, or if between the 8034 * state AGE_TMOUT and tailq operation the 8035 * release happened, the release procedure 8036 * may delete a non-existent tailq node. 8037 */ 8038 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8039 age_info = GET_PORT_AGE_INFO(priv); 8040 rte_spinlock_lock(&age_info->aged_sl); 8041 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8042 AGE_TMOUT, false, 8043 __ATOMIC_RELAXED, 8044 __ATOMIC_RELAXED)) { 8045 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8046 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8047 } 8048 rte_spinlock_unlock(&age_info->aged_sl); 8049 } 8050 mlx5_age_event_prepare(sh); 8051 } 8052 8053 /** 8054 * Handler for the HW respond about ready values from an asynchronous batch 8055 * query. This function is probably called by the host thread. 8056 * 8057 * @param[in] sh 8058 * The pointer to the shared device context. 8059 * @param[in] async_id 8060 * The Devx async ID. 8061 * @param[in] status 8062 * The status of the completion. 8063 */ 8064 void 8065 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8066 uint64_t async_id, int status) 8067 { 8068 struct mlx5_flow_counter_pool *pool = 8069 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8070 struct mlx5_counter_stats_raw *raw_to_free; 8071 uint8_t query_gen = pool->query_gen ^ 1; 8072 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8073 enum mlx5_counter_type cnt_type = 8074 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8075 MLX5_COUNTER_TYPE_ORIGIN; 8076 8077 if (unlikely(status)) { 8078 raw_to_free = pool->raw_hw; 8079 } else { 8080 raw_to_free = pool->raw; 8081 if (pool->is_aged) 8082 mlx5_flow_aging_check(sh, pool); 8083 rte_spinlock_lock(&pool->sl); 8084 pool->raw = pool->raw_hw; 8085 rte_spinlock_unlock(&pool->sl); 8086 /* Be sure the new raw counters data is updated in memory. */ 8087 rte_io_wmb(); 8088 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8089 rte_spinlock_lock(&cmng->csl[cnt_type]); 8090 TAILQ_CONCAT(&cmng->counters[cnt_type], 8091 &pool->counters[query_gen], next); 8092 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8093 } 8094 } 8095 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8096 pool->raw_hw = NULL; 8097 sh->cmng.pending_queries--; 8098 } 8099 8100 static int 8101 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8102 const struct flow_grp_info *grp_info, 8103 struct rte_flow_error *error) 8104 { 8105 if (grp_info->transfer && grp_info->external && 8106 grp_info->fdb_def_rule) { 8107 if (group == UINT32_MAX) 8108 return rte_flow_error_set 8109 (error, EINVAL, 8110 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8111 NULL, 8112 "group index not supported"); 8113 *table = group + 1; 8114 } else { 8115 *table = group; 8116 } 8117 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8118 return 0; 8119 } 8120 8121 /** 8122 * Translate the rte_flow group index to HW table value. 8123 * 8124 * If tunnel offload is disabled, all group ids converted to flow table 8125 * id using the standard method. 8126 * If tunnel offload is enabled, group id can be converted using the 8127 * standard or tunnel conversion method. Group conversion method 8128 * selection depends on flags in `grp_info` parameter: 8129 * - Internal (grp_info.external == 0) groups conversion uses the 8130 * standard method. 8131 * - Group ids in JUMP action converted with the tunnel conversion. 8132 * - Group id in rule attribute conversion depends on a rule type and 8133 * group id value: 8134 * ** non zero group attributes converted with the tunnel method 8135 * ** zero group attribute in non-tunnel rule is converted using the 8136 * standard method - there's only one root table 8137 * ** zero group attribute in steer tunnel rule is converted with the 8138 * standard method - single root table 8139 * ** zero group attribute in match tunnel rule is a special OvS 8140 * case: that value is used for portability reasons. That group 8141 * id is converted with the tunnel conversion method. 8142 * 8143 * @param[in] dev 8144 * Port device 8145 * @param[in] tunnel 8146 * PMD tunnel offload object 8147 * @param[in] group 8148 * rte_flow group index value. 8149 * @param[out] table 8150 * HW table value. 8151 * @param[in] grp_info 8152 * flags used for conversion 8153 * @param[out] error 8154 * Pointer to error structure. 8155 * 8156 * @return 8157 * 0 on success, a negative errno value otherwise and rte_errno is set. 8158 */ 8159 int 8160 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8161 const struct mlx5_flow_tunnel *tunnel, 8162 uint32_t group, uint32_t *table, 8163 const struct flow_grp_info *grp_info, 8164 struct rte_flow_error *error) 8165 { 8166 int ret; 8167 bool standard_translation; 8168 8169 if (!grp_info->skip_scale && grp_info->external && 8170 group < MLX5_MAX_TABLES_EXTERNAL) 8171 group *= MLX5_FLOW_TABLE_FACTOR; 8172 if (is_tunnel_offload_active(dev)) { 8173 standard_translation = !grp_info->external || 8174 grp_info->std_tbl_fix; 8175 } else { 8176 standard_translation = true; 8177 } 8178 DRV_LOG(DEBUG, 8179 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8180 dev->data->port_id, group, grp_info->transfer, 8181 grp_info->external, grp_info->fdb_def_rule, 8182 standard_translation ? "STANDARD" : "TUNNEL"); 8183 if (standard_translation) 8184 ret = flow_group_to_table(dev->data->port_id, group, table, 8185 grp_info, error); 8186 else 8187 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8188 table, error); 8189 8190 return ret; 8191 } 8192 8193 /** 8194 * Discover availability of metadata reg_c's. 8195 * 8196 * Iteratively use test flows to check availability. 8197 * 8198 * @param[in] dev 8199 * Pointer to the Ethernet device structure. 8200 * 8201 * @return 8202 * 0 on success, a negative errno value otherwise and rte_errno is set. 8203 */ 8204 int 8205 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8206 { 8207 struct mlx5_priv *priv = dev->data->dev_private; 8208 enum modify_reg idx; 8209 int n = 0; 8210 8211 /* reg_c[0] and reg_c[1] are reserved. */ 8212 priv->sh->flow_mreg_c[n++] = REG_C_0; 8213 priv->sh->flow_mreg_c[n++] = REG_C_1; 8214 /* Discover availability of other reg_c's. */ 8215 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 8216 struct rte_flow_attr attr = { 8217 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 8218 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8219 .ingress = 1, 8220 }; 8221 struct rte_flow_item items[] = { 8222 [0] = { 8223 .type = RTE_FLOW_ITEM_TYPE_END, 8224 }, 8225 }; 8226 struct rte_flow_action actions[] = { 8227 [0] = { 8228 .type = (enum rte_flow_action_type) 8229 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8230 .conf = &(struct mlx5_flow_action_copy_mreg){ 8231 .src = REG_C_1, 8232 .dst = idx, 8233 }, 8234 }, 8235 [1] = { 8236 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8237 .conf = &(struct rte_flow_action_jump){ 8238 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8239 }, 8240 }, 8241 [2] = { 8242 .type = RTE_FLOW_ACTION_TYPE_END, 8243 }, 8244 }; 8245 uint32_t flow_idx; 8246 struct rte_flow *flow; 8247 struct rte_flow_error error; 8248 8249 if (!priv->config.dv_flow_en) 8250 break; 8251 /* Create internal flow, validation skips copy action. */ 8252 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8253 items, actions, false, &error); 8254 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8255 flow_idx); 8256 if (!flow) 8257 continue; 8258 priv->sh->flow_mreg_c[n++] = idx; 8259 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8260 } 8261 for (; n < MLX5_MREG_C_NUM; ++n) 8262 priv->sh->flow_mreg_c[n] = REG_NON; 8263 priv->sh->metadata_regc_check_flag = 1; 8264 return 0; 8265 } 8266 8267 int 8268 save_dump_file(const uint8_t *data, uint32_t size, 8269 uint32_t type, uint64_t id, void *arg, FILE *file) 8270 { 8271 char line[BUF_SIZE]; 8272 uint32_t out = 0; 8273 uint32_t k; 8274 uint32_t actions_num; 8275 struct rte_flow_query_count *count; 8276 8277 memset(line, 0, BUF_SIZE); 8278 switch (type) { 8279 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8280 actions_num = *(uint32_t *)(arg); 8281 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 8282 type, id, actions_num); 8283 break; 8284 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8285 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 8286 type, id); 8287 break; 8288 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8289 count = (struct rte_flow_query_count *)arg; 8290 fprintf(file, 8291 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 8292 type, id, count->hits, count->bytes); 8293 return 0; 8294 default: 8295 return -1; 8296 } 8297 8298 for (k = 0; k < size; k++) { 8299 /* Make sure we do not overrun the line buffer length. */ 8300 if (out >= BUF_SIZE - 4) { 8301 line[out] = '\0'; 8302 break; 8303 } 8304 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8305 (data[k]) & 0xff); 8306 } 8307 fprintf(file, "%s\n", line); 8308 return 0; 8309 } 8310 8311 int 8312 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8313 struct rte_flow_query_count *count, struct rte_flow_error *error) 8314 { 8315 struct rte_flow_action action[2]; 8316 enum mlx5_flow_drv_type ftype; 8317 const struct mlx5_flow_driver_ops *fops; 8318 8319 if (!flow) { 8320 return rte_flow_error_set(error, ENOENT, 8321 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8322 NULL, 8323 "invalid flow handle"); 8324 } 8325 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8326 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8327 if (flow->counter) { 8328 memset(count, 0, sizeof(struct rte_flow_query_count)); 8329 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8330 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8331 ftype < MLX5_FLOW_TYPE_MAX); 8332 fops = flow_get_drv_ops(ftype); 8333 return fops->query(dev, flow, action, count, error); 8334 } 8335 return -1; 8336 } 8337 8338 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8339 /** 8340 * Dump flow ipool data to file 8341 * 8342 * @param[in] dev 8343 * The pointer to Ethernet device. 8344 * @param[in] file 8345 * A pointer to a file for output. 8346 * @param[out] error 8347 * Perform verbose error reporting if not NULL. PMDs initialize this 8348 * structure in case of error only. 8349 * @return 8350 * 0 on success, a negative value otherwise. 8351 */ 8352 int 8353 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8354 struct rte_flow *flow, FILE *file, 8355 struct rte_flow_error *error) 8356 { 8357 struct mlx5_priv *priv = dev->data->dev_private; 8358 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8359 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8360 uint32_t handle_idx; 8361 struct mlx5_flow_handle *dh; 8362 struct rte_flow_query_count count; 8363 uint32_t actions_num; 8364 const uint8_t *data; 8365 size_t size; 8366 uint64_t id; 8367 uint32_t type; 8368 void *action = NULL; 8369 8370 if (!flow) { 8371 return rte_flow_error_set(error, ENOENT, 8372 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8373 NULL, 8374 "invalid flow handle"); 8375 } 8376 handle_idx = flow->dev_handles; 8377 while (handle_idx) { 8378 dh = mlx5_ipool_get(priv->sh->ipool 8379 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8380 if (!dh) 8381 continue; 8382 handle_idx = dh->next.next; 8383 8384 /* query counter */ 8385 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8386 flow_dv_query_count_ptr(dev, flow->counter, 8387 &action, error); 8388 if (action) { 8389 id = (uint64_t)(uintptr_t)action; 8390 if (!mlx5_flow_query_counter(dev, flow, &count, error)) 8391 save_dump_file(NULL, 0, type, 8392 id, (void *)&count, file); 8393 } 8394 /* Get modify_hdr and encap_decap buf from ipools. */ 8395 encap_decap = NULL; 8396 modify_hdr = dh->dvh.modify_hdr; 8397 8398 if (dh->dvh.rix_encap_decap) { 8399 encap_decap = mlx5_ipool_get(priv->sh->ipool 8400 [MLX5_IPOOL_DECAP_ENCAP], 8401 dh->dvh.rix_encap_decap); 8402 } 8403 if (modify_hdr) { 8404 data = (const uint8_t *)modify_hdr->actions; 8405 size = (size_t)(modify_hdr->actions_num) * 8; 8406 id = (uint64_t)(uintptr_t)modify_hdr->action; 8407 actions_num = modify_hdr->actions_num; 8408 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8409 save_dump_file(data, size, type, id, 8410 (void *)(&actions_num), file); 8411 } 8412 if (encap_decap) { 8413 data = encap_decap->buf; 8414 size = encap_decap->size; 8415 id = (uint64_t)(uintptr_t)encap_decap->action; 8416 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8417 save_dump_file(data, size, type, 8418 id, NULL, file); 8419 } 8420 } 8421 return 0; 8422 } 8423 8424 /** 8425 * Dump all flow's encap_decap/modify_hdr/counter data to file 8426 * 8427 * @param[in] dev 8428 * The pointer to Ethernet device. 8429 * @param[in] file 8430 * A pointer to a file for output. 8431 * @param[out] error 8432 * Perform verbose error reporting if not NULL. PMDs initialize this 8433 * structure in case of error only. 8434 * @return 8435 * 0 on success, a negative value otherwise. 8436 */ 8437 static int 8438 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 8439 FILE *file, struct rte_flow_error *error) 8440 { 8441 struct mlx5_priv *priv = dev->data->dev_private; 8442 struct mlx5_dev_ctx_shared *sh = priv->sh; 8443 struct mlx5_hlist *h; 8444 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8445 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8446 struct rte_flow_query_count count; 8447 uint32_t actions_num; 8448 const uint8_t *data; 8449 size_t size; 8450 uint64_t id; 8451 uint32_t type; 8452 uint32_t i; 8453 uint32_t j; 8454 struct mlx5_list_inconst *l_inconst; 8455 struct mlx5_list_entry *e; 8456 int lcore_index; 8457 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 8458 uint32_t max; 8459 void *action; 8460 8461 /* encap_decap hlist is lcore_share, get global core cache. */ 8462 i = MLX5_LIST_GLOBAL; 8463 h = sh->encaps_decaps; 8464 if (h) { 8465 for (j = 0; j <= h->mask; j++) { 8466 l_inconst = &h->buckets[j].l; 8467 if (!l_inconst || !l_inconst->cache[i]) 8468 continue; 8469 8470 e = LIST_FIRST(&l_inconst->cache[i]->h); 8471 while (e) { 8472 encap_decap = 8473 (struct mlx5_flow_dv_encap_decap_resource *)e; 8474 data = encap_decap->buf; 8475 size = encap_decap->size; 8476 id = (uint64_t)(uintptr_t)encap_decap->action; 8477 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8478 save_dump_file(data, size, type, 8479 id, NULL, file); 8480 e = LIST_NEXT(e, next); 8481 } 8482 } 8483 } 8484 8485 /* get modify_hdr */ 8486 h = sh->modify_cmds; 8487 if (h) { 8488 lcore_index = rte_lcore_index(rte_lcore_id()); 8489 if (unlikely(lcore_index == -1)) { 8490 lcore_index = MLX5_LIST_NLCORE; 8491 rte_spinlock_lock(&h->l_const.lcore_lock); 8492 } 8493 i = lcore_index; 8494 8495 for (j = 0; j <= h->mask; j++) { 8496 l_inconst = &h->buckets[j].l; 8497 if (!l_inconst || !l_inconst->cache[i]) 8498 continue; 8499 8500 e = LIST_FIRST(&l_inconst->cache[i]->h); 8501 while (e) { 8502 modify_hdr = 8503 (struct mlx5_flow_dv_modify_hdr_resource *)e; 8504 data = (const uint8_t *)modify_hdr->actions; 8505 size = (size_t)(modify_hdr->actions_num) * 8; 8506 actions_num = modify_hdr->actions_num; 8507 id = (uint64_t)(uintptr_t)modify_hdr->action; 8508 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8509 save_dump_file(data, size, type, id, 8510 (void *)(&actions_num), file); 8511 e = LIST_NEXT(e, next); 8512 } 8513 } 8514 8515 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 8516 rte_spinlock_unlock(&h->l_const.lcore_lock); 8517 } 8518 8519 /* get counter */ 8520 MLX5_ASSERT(cmng->n_valid <= cmng->n); 8521 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 8522 for (j = 1; j <= max; j++) { 8523 action = NULL; 8524 flow_dv_query_count_ptr(dev, j, &action, error); 8525 if (action) { 8526 if (!flow_dv_query_count(dev, j, &count, error)) { 8527 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8528 id = (uint64_t)(uintptr_t)action; 8529 save_dump_file(NULL, 0, type, 8530 id, (void *)&count, file); 8531 } 8532 } 8533 } 8534 return 0; 8535 } 8536 #endif 8537 8538 /** 8539 * Dump flow raw hw data to file 8540 * 8541 * @param[in] dev 8542 * The pointer to Ethernet device. 8543 * @param[in] file 8544 * A pointer to a file for output. 8545 * @param[out] error 8546 * Perform verbose error reporting if not NULL. PMDs initialize this 8547 * structure in case of error only. 8548 * @return 8549 * 0 on success, a negative value otherwise. 8550 */ 8551 int 8552 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8553 FILE *file, 8554 struct rte_flow_error *error __rte_unused) 8555 { 8556 struct mlx5_priv *priv = dev->data->dev_private; 8557 struct mlx5_dev_ctx_shared *sh = priv->sh; 8558 uint32_t handle_idx; 8559 int ret; 8560 struct mlx5_flow_handle *dh; 8561 struct rte_flow *flow; 8562 8563 if (!priv->config.dv_flow_en) { 8564 if (fputs("device dv flow disabled\n", file) <= 0) 8565 return -errno; 8566 return -ENOTSUP; 8567 } 8568 8569 /* dump all */ 8570 if (!flow_idx) { 8571 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8572 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 8573 return -EINVAL; 8574 #endif 8575 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8576 sh->rx_domain, 8577 sh->tx_domain, file); 8578 } 8579 /* dump one */ 8580 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8581 (uintptr_t)(void *)flow_idx); 8582 if (!flow) 8583 return -EINVAL; 8584 8585 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8586 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8587 #endif 8588 handle_idx = flow->dev_handles; 8589 while (handle_idx) { 8590 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8591 handle_idx); 8592 if (!dh) 8593 return -ENOENT; 8594 if (dh->drv_flow) { 8595 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8596 file); 8597 if (ret) 8598 return -ENOENT; 8599 } 8600 handle_idx = dh->next.next; 8601 } 8602 return 0; 8603 } 8604 8605 /** 8606 * Get aged-out flows. 8607 * 8608 * @param[in] dev 8609 * Pointer to the Ethernet device structure. 8610 * @param[in] context 8611 * The address of an array of pointers to the aged-out flows contexts. 8612 * @param[in] nb_countexts 8613 * The length of context array pointers. 8614 * @param[out] error 8615 * Perform verbose error reporting if not NULL. Initialized in case of 8616 * error only. 8617 * 8618 * @return 8619 * how many contexts get in success, otherwise negative errno value. 8620 * if nb_contexts is 0, return the amount of all aged contexts. 8621 * if nb_contexts is not 0 , return the amount of aged flows reported 8622 * in the context array. 8623 */ 8624 int 8625 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8626 uint32_t nb_contexts, struct rte_flow_error *error) 8627 { 8628 const struct mlx5_flow_driver_ops *fops; 8629 struct rte_flow_attr attr = { .transfer = 0 }; 8630 8631 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8632 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8633 return fops->get_aged_flows(dev, contexts, nb_contexts, 8634 error); 8635 } 8636 DRV_LOG(ERR, 8637 "port %u get aged flows is not supported.", 8638 dev->data->port_id); 8639 return -ENOTSUP; 8640 } 8641 8642 /* Wrapper for driver action_validate op callback */ 8643 static int 8644 flow_drv_action_validate(struct rte_eth_dev *dev, 8645 const struct rte_flow_indir_action_conf *conf, 8646 const struct rte_flow_action *action, 8647 const struct mlx5_flow_driver_ops *fops, 8648 struct rte_flow_error *error) 8649 { 8650 static const char err_msg[] = "indirect action validation unsupported"; 8651 8652 if (!fops->action_validate) { 8653 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8654 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8655 NULL, err_msg); 8656 return -rte_errno; 8657 } 8658 return fops->action_validate(dev, conf, action, error); 8659 } 8660 8661 /** 8662 * Destroys the shared action by handle. 8663 * 8664 * @param dev 8665 * Pointer to Ethernet device structure. 8666 * @param[in] handle 8667 * Handle for the indirect action object to be destroyed. 8668 * @param[out] error 8669 * Perform verbose error reporting if not NULL. PMDs initialize this 8670 * structure in case of error only. 8671 * 8672 * @return 8673 * 0 on success, a negative errno value otherwise and rte_errno is set. 8674 * 8675 * @note: wrapper for driver action_create op callback. 8676 */ 8677 static int 8678 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8679 struct rte_flow_action_handle *handle, 8680 struct rte_flow_error *error) 8681 { 8682 static const char err_msg[] = "indirect action destruction unsupported"; 8683 struct rte_flow_attr attr = { .transfer = 0 }; 8684 const struct mlx5_flow_driver_ops *fops = 8685 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8686 8687 if (!fops->action_destroy) { 8688 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8689 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8690 NULL, err_msg); 8691 return -rte_errno; 8692 } 8693 return fops->action_destroy(dev, handle, error); 8694 } 8695 8696 /* Wrapper for driver action_destroy op callback */ 8697 static int 8698 flow_drv_action_update(struct rte_eth_dev *dev, 8699 struct rte_flow_action_handle *handle, 8700 const void *update, 8701 const struct mlx5_flow_driver_ops *fops, 8702 struct rte_flow_error *error) 8703 { 8704 static const char err_msg[] = "indirect action update unsupported"; 8705 8706 if (!fops->action_update) { 8707 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8708 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8709 NULL, err_msg); 8710 return -rte_errno; 8711 } 8712 return fops->action_update(dev, handle, update, error); 8713 } 8714 8715 /* Wrapper for driver action_destroy op callback */ 8716 static int 8717 flow_drv_action_query(struct rte_eth_dev *dev, 8718 const struct rte_flow_action_handle *handle, 8719 void *data, 8720 const struct mlx5_flow_driver_ops *fops, 8721 struct rte_flow_error *error) 8722 { 8723 static const char err_msg[] = "indirect action query unsupported"; 8724 8725 if (!fops->action_query) { 8726 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8727 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8728 NULL, err_msg); 8729 return -rte_errno; 8730 } 8731 return fops->action_query(dev, handle, data, error); 8732 } 8733 8734 /** 8735 * Create indirect action for reuse in multiple flow rules. 8736 * 8737 * @param dev 8738 * Pointer to Ethernet device structure. 8739 * @param conf 8740 * Pointer to indirect action object configuration. 8741 * @param[in] action 8742 * Action configuration for indirect action object creation. 8743 * @param[out] error 8744 * Perform verbose error reporting if not NULL. PMDs initialize this 8745 * structure in case of error only. 8746 * @return 8747 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8748 */ 8749 static struct rte_flow_action_handle * 8750 mlx5_action_handle_create(struct rte_eth_dev *dev, 8751 const struct rte_flow_indir_action_conf *conf, 8752 const struct rte_flow_action *action, 8753 struct rte_flow_error *error) 8754 { 8755 static const char err_msg[] = "indirect action creation unsupported"; 8756 struct rte_flow_attr attr = { .transfer = 0 }; 8757 const struct mlx5_flow_driver_ops *fops = 8758 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8759 8760 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8761 return NULL; 8762 if (!fops->action_create) { 8763 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8764 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8765 NULL, err_msg); 8766 return NULL; 8767 } 8768 return fops->action_create(dev, conf, action, error); 8769 } 8770 8771 /** 8772 * Updates inplace the indirect action configuration pointed by *handle* 8773 * with the configuration provided as *update* argument. 8774 * The update of the indirect action configuration effects all flow rules 8775 * reusing the action via handle. 8776 * 8777 * @param dev 8778 * Pointer to Ethernet device structure. 8779 * @param[in] handle 8780 * Handle for the indirect action to be updated. 8781 * @param[in] update 8782 * Action specification used to modify the action pointed by handle. 8783 * *update* could be of same type with the action pointed by the *handle* 8784 * handle argument, or some other structures like a wrapper, depending on 8785 * the indirect action type. 8786 * @param[out] error 8787 * Perform verbose error reporting if not NULL. PMDs initialize this 8788 * structure in case of error only. 8789 * 8790 * @return 8791 * 0 on success, a negative errno value otherwise and rte_errno is set. 8792 */ 8793 static int 8794 mlx5_action_handle_update(struct rte_eth_dev *dev, 8795 struct rte_flow_action_handle *handle, 8796 const void *update, 8797 struct rte_flow_error *error) 8798 { 8799 struct rte_flow_attr attr = { .transfer = 0 }; 8800 const struct mlx5_flow_driver_ops *fops = 8801 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8802 int ret; 8803 8804 ret = flow_drv_action_validate(dev, NULL, 8805 (const struct rte_flow_action *)update, fops, error); 8806 if (ret) 8807 return ret; 8808 return flow_drv_action_update(dev, handle, update, fops, 8809 error); 8810 } 8811 8812 /** 8813 * Query the indirect action by handle. 8814 * 8815 * This function allows retrieving action-specific data such as counters. 8816 * Data is gathered by special action which may be present/referenced in 8817 * more than one flow rule definition. 8818 * 8819 * see @RTE_FLOW_ACTION_TYPE_COUNT 8820 * 8821 * @param dev 8822 * Pointer to Ethernet device structure. 8823 * @param[in] handle 8824 * Handle for the indirect action to query. 8825 * @param[in, out] data 8826 * Pointer to storage for the associated query data type. 8827 * @param[out] error 8828 * Perform verbose error reporting if not NULL. PMDs initialize this 8829 * structure in case of error only. 8830 * 8831 * @return 8832 * 0 on success, a negative errno value otherwise and rte_errno is set. 8833 */ 8834 static int 8835 mlx5_action_handle_query(struct rte_eth_dev *dev, 8836 const struct rte_flow_action_handle *handle, 8837 void *data, 8838 struct rte_flow_error *error) 8839 { 8840 struct rte_flow_attr attr = { .transfer = 0 }; 8841 const struct mlx5_flow_driver_ops *fops = 8842 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8843 8844 return flow_drv_action_query(dev, handle, data, fops, error); 8845 } 8846 8847 /** 8848 * Destroy all indirect actions (shared RSS). 8849 * 8850 * @param dev 8851 * Pointer to Ethernet device. 8852 * 8853 * @return 8854 * 0 on success, a negative errno value otherwise and rte_errno is set. 8855 */ 8856 int 8857 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8858 { 8859 struct rte_flow_error error; 8860 struct mlx5_priv *priv = dev->data->dev_private; 8861 struct mlx5_shared_action_rss *shared_rss; 8862 int ret = 0; 8863 uint32_t idx; 8864 8865 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8866 priv->rss_shared_actions, idx, shared_rss, next) { 8867 ret |= mlx5_action_handle_destroy(dev, 8868 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8869 } 8870 return ret; 8871 } 8872 8873 /** 8874 * Validate existing indirect actions against current device configuration 8875 * and attach them to device resources. 8876 * 8877 * @param dev 8878 * Pointer to Ethernet device. 8879 * 8880 * @return 8881 * 0 on success, a negative errno value otherwise and rte_errno is set. 8882 */ 8883 int 8884 mlx5_action_handle_attach(struct rte_eth_dev *dev) 8885 { 8886 struct mlx5_priv *priv = dev->data->dev_private; 8887 struct mlx5_indexed_pool *ipool = 8888 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 8889 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 8890 int ret = 0; 8891 uint32_t idx; 8892 8893 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8894 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8895 const char *message; 8896 uint32_t queue_idx; 8897 8898 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 8899 ind_tbl->queues_n, 8900 &message, &queue_idx); 8901 if (ret != 0) { 8902 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 8903 dev->data->port_id, ind_tbl->queues[queue_idx], 8904 message); 8905 break; 8906 } 8907 } 8908 if (ret != 0) 8909 return ret; 8910 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8911 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8912 8913 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 8914 if (ret != 0) { 8915 DRV_LOG(ERR, "Port %u could not attach " 8916 "indirection table obj %p", 8917 dev->data->port_id, (void *)ind_tbl); 8918 goto error; 8919 } 8920 } 8921 return 0; 8922 error: 8923 shared_rss_last = shared_rss; 8924 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8925 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8926 8927 if (shared_rss == shared_rss_last) 8928 break; 8929 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 8930 DRV_LOG(CRIT, "Port %u could not detach " 8931 "indirection table obj %p on rollback", 8932 dev->data->port_id, (void *)ind_tbl); 8933 } 8934 return ret; 8935 } 8936 8937 /** 8938 * Detach indirect actions of the device from its resources. 8939 * 8940 * @param dev 8941 * Pointer to Ethernet device. 8942 * 8943 * @return 8944 * 0 on success, a negative errno value otherwise and rte_errno is set. 8945 */ 8946 int 8947 mlx5_action_handle_detach(struct rte_eth_dev *dev) 8948 { 8949 struct mlx5_priv *priv = dev->data->dev_private; 8950 struct mlx5_indexed_pool *ipool = 8951 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 8952 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 8953 int ret = 0; 8954 uint32_t idx; 8955 8956 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8957 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8958 8959 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 8960 if (ret != 0) { 8961 DRV_LOG(ERR, "Port %u could not detach " 8962 "indirection table obj %p", 8963 dev->data->port_id, (void *)ind_tbl); 8964 goto error; 8965 } 8966 } 8967 return 0; 8968 error: 8969 shared_rss_last = shared_rss; 8970 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 8971 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 8972 8973 if (shared_rss == shared_rss_last) 8974 break; 8975 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 8976 DRV_LOG(CRIT, "Port %u could not attach " 8977 "indirection table obj %p on rollback", 8978 dev->data->port_id, (void *)ind_tbl); 8979 } 8980 return ret; 8981 } 8982 8983 #ifndef HAVE_MLX5DV_DR 8984 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8985 #else 8986 #define MLX5_DOMAIN_SYNC_FLOW \ 8987 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8988 #endif 8989 8990 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8991 { 8992 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8993 const struct mlx5_flow_driver_ops *fops; 8994 int ret; 8995 struct rte_flow_attr attr = { .transfer = 0 }; 8996 8997 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8998 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8999 if (ret > 0) 9000 ret = -ret; 9001 return ret; 9002 } 9003 9004 const struct mlx5_flow_tunnel * 9005 mlx5_get_tof(const struct rte_flow_item *item, 9006 const struct rte_flow_action *action, 9007 enum mlx5_tof_rule_type *rule_type) 9008 { 9009 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9010 if (item->type == (typeof(item->type)) 9011 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9012 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9013 return flow_items_to_tunnel(item); 9014 } 9015 } 9016 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9017 if (action->type == (typeof(action->type)) 9018 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9019 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9020 return flow_actions_to_tunnel(action); 9021 } 9022 } 9023 return NULL; 9024 } 9025 9026 /** 9027 * tunnel offload functionality is defined for DV environment only 9028 */ 9029 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9030 __extension__ 9031 union tunnel_offload_mark { 9032 uint32_t val; 9033 struct { 9034 uint32_t app_reserve:8; 9035 uint32_t table_id:15; 9036 uint32_t transfer:1; 9037 uint32_t _unused_:8; 9038 }; 9039 }; 9040 9041 static bool 9042 mlx5_access_tunnel_offload_db 9043 (struct rte_eth_dev *dev, 9044 bool (*match)(struct rte_eth_dev *, 9045 struct mlx5_flow_tunnel *, const void *), 9046 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9047 void (*miss)(struct rte_eth_dev *, void *), 9048 void *ctx, bool lock_op); 9049 9050 static int 9051 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9052 struct rte_flow *flow, 9053 const struct rte_flow_attr *attr, 9054 const struct rte_flow_action *app_actions, 9055 uint32_t flow_idx, 9056 const struct mlx5_flow_tunnel *tunnel, 9057 struct tunnel_default_miss_ctx *ctx, 9058 struct rte_flow_error *error) 9059 { 9060 struct mlx5_priv *priv = dev->data->dev_private; 9061 struct mlx5_flow *dev_flow; 9062 struct rte_flow_attr miss_attr = *attr; 9063 const struct rte_flow_item miss_items[2] = { 9064 { 9065 .type = RTE_FLOW_ITEM_TYPE_ETH, 9066 .spec = NULL, 9067 .last = NULL, 9068 .mask = NULL 9069 }, 9070 { 9071 .type = RTE_FLOW_ITEM_TYPE_END, 9072 .spec = NULL, 9073 .last = NULL, 9074 .mask = NULL 9075 } 9076 }; 9077 union tunnel_offload_mark mark_id; 9078 struct rte_flow_action_mark miss_mark; 9079 struct rte_flow_action miss_actions[3] = { 9080 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9081 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9082 }; 9083 const struct rte_flow_action_jump *jump_data; 9084 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9085 struct flow_grp_info grp_info = { 9086 .external = 1, 9087 .transfer = attr->transfer, 9088 .fdb_def_rule = !!priv->fdb_def_rule, 9089 .std_tbl_fix = 0, 9090 }; 9091 int ret; 9092 9093 if (!attr->transfer) { 9094 uint32_t q_size; 9095 9096 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9097 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9098 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9099 0, SOCKET_ID_ANY); 9100 if (!ctx->queue) 9101 return rte_flow_error_set 9102 (error, ENOMEM, 9103 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9104 NULL, "invalid default miss RSS"); 9105 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9106 ctx->action_rss.level = 0, 9107 ctx->action_rss.types = priv->rss_conf.rss_hf, 9108 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9109 ctx->action_rss.queue_num = priv->reta_idx_n, 9110 ctx->action_rss.key = priv->rss_conf.rss_key, 9111 ctx->action_rss.queue = ctx->queue; 9112 if (!priv->reta_idx_n || !priv->rxqs_n) 9113 return rte_flow_error_set 9114 (error, EINVAL, 9115 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9116 NULL, "invalid port configuration"); 9117 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9118 ctx->action_rss.types = 0; 9119 for (i = 0; i != priv->reta_idx_n; ++i) 9120 ctx->queue[i] = (*priv->reta_idx)[i]; 9121 } else { 9122 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9123 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9124 } 9125 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9126 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9127 jump_data = app_actions->conf; 9128 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9129 miss_attr.group = jump_data->group; 9130 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9131 &flow_table, &grp_info, error); 9132 if (ret) 9133 return rte_flow_error_set(error, EINVAL, 9134 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9135 NULL, "invalid tunnel id"); 9136 mark_id.app_reserve = 0; 9137 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9138 mark_id.transfer = !!attr->transfer; 9139 mark_id._unused_ = 0; 9140 miss_mark.id = mark_id.val; 9141 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9142 miss_items, miss_actions, flow_idx, error); 9143 if (!dev_flow) 9144 return -rte_errno; 9145 dev_flow->flow = flow; 9146 dev_flow->external = true; 9147 dev_flow->tunnel = tunnel; 9148 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9149 /* Subflow object was created, we must include one in the list. */ 9150 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9151 dev_flow->handle, next); 9152 DRV_LOG(DEBUG, 9153 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9154 dev->data->port_id, tunnel->app_tunnel.type, 9155 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9156 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9157 miss_actions, error); 9158 if (!ret) 9159 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9160 error); 9161 9162 return ret; 9163 } 9164 9165 static const struct mlx5_flow_tbl_data_entry * 9166 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9167 { 9168 struct mlx5_priv *priv = dev->data->dev_private; 9169 struct mlx5_dev_ctx_shared *sh = priv->sh; 9170 struct mlx5_list_entry *he; 9171 union tunnel_offload_mark mbits = { .val = mark }; 9172 union mlx5_flow_tbl_key table_key = { 9173 { 9174 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9175 .id = 0, 9176 .reserved = 0, 9177 .dummy = 0, 9178 .is_fdb = !!mbits.transfer, 9179 .is_egress = 0, 9180 } 9181 }; 9182 struct mlx5_flow_cb_ctx ctx = { 9183 .data = &table_key.v64, 9184 }; 9185 9186 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9187 return he ? 9188 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9189 } 9190 9191 static void 9192 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9193 struct mlx5_list_entry *entry) 9194 { 9195 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9196 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9197 9198 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9199 tunnel_flow_tbl_to_id(tte->flow_table)); 9200 mlx5_free(tte); 9201 } 9202 9203 static int 9204 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9205 struct mlx5_list_entry *entry, void *cb_ctx) 9206 { 9207 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9208 union tunnel_tbl_key tbl = { 9209 .val = *(uint64_t *)(ctx->data), 9210 }; 9211 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9212 9213 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9214 } 9215 9216 static struct mlx5_list_entry * 9217 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 9218 { 9219 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9220 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9221 struct tunnel_tbl_entry *tte; 9222 union tunnel_tbl_key tbl = { 9223 .val = *(uint64_t *)(ctx->data), 9224 }; 9225 9226 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 9227 sizeof(*tte), 0, 9228 SOCKET_ID_ANY); 9229 if (!tte) 9230 goto err; 9231 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9232 &tte->flow_table); 9233 if (tte->flow_table >= MLX5_MAX_TABLES) { 9234 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 9235 tte->flow_table); 9236 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9237 tte->flow_table); 9238 goto err; 9239 } else if (!tte->flow_table) { 9240 goto err; 9241 } 9242 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 9243 tte->tunnel_id = tbl.tunnel_id; 9244 tte->group = tbl.group; 9245 return &tte->hash; 9246 err: 9247 if (tte) 9248 mlx5_free(tte); 9249 return NULL; 9250 } 9251 9252 static struct mlx5_list_entry * 9253 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 9254 struct mlx5_list_entry *oentry, 9255 void *cb_ctx __rte_unused) 9256 { 9257 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 9258 0, SOCKET_ID_ANY); 9259 9260 if (!tte) 9261 return NULL; 9262 memcpy(tte, oentry, sizeof(*tte)); 9263 return &tte->hash; 9264 } 9265 9266 static void 9267 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 9268 struct mlx5_list_entry *entry) 9269 { 9270 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9271 9272 mlx5_free(tte); 9273 } 9274 9275 static uint32_t 9276 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 9277 const struct mlx5_flow_tunnel *tunnel, 9278 uint32_t group, uint32_t *table, 9279 struct rte_flow_error *error) 9280 { 9281 struct mlx5_list_entry *he; 9282 struct tunnel_tbl_entry *tte; 9283 union tunnel_tbl_key key = { 9284 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 9285 .group = group 9286 }; 9287 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9288 struct mlx5_hlist *group_hash; 9289 struct mlx5_flow_cb_ctx ctx = { 9290 .data = &key.val, 9291 }; 9292 9293 group_hash = tunnel ? tunnel->groups : thub->groups; 9294 he = mlx5_hlist_register(group_hash, key.val, &ctx); 9295 if (!he) 9296 return rte_flow_error_set(error, EINVAL, 9297 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9298 NULL, 9299 "tunnel group index not supported"); 9300 tte = container_of(he, typeof(*tte), hash); 9301 *table = tte->flow_table; 9302 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 9303 dev->data->port_id, key.tunnel_id, group, *table); 9304 return 0; 9305 } 9306 9307 static void 9308 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 9309 struct mlx5_flow_tunnel *tunnel) 9310 { 9311 struct mlx5_priv *priv = dev->data->dev_private; 9312 struct mlx5_indexed_pool *ipool; 9313 9314 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 9315 dev->data->port_id, tunnel->tunnel_id); 9316 LIST_REMOVE(tunnel, chain); 9317 mlx5_hlist_destroy(tunnel->groups); 9318 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9319 mlx5_ipool_free(ipool, tunnel->tunnel_id); 9320 } 9321 9322 static bool 9323 mlx5_access_tunnel_offload_db 9324 (struct rte_eth_dev *dev, 9325 bool (*match)(struct rte_eth_dev *, 9326 struct mlx5_flow_tunnel *, const void *), 9327 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9328 void (*miss)(struct rte_eth_dev *, void *), 9329 void *ctx, bool lock_op) 9330 { 9331 bool verdict = false; 9332 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9333 struct mlx5_flow_tunnel *tunnel; 9334 9335 rte_spinlock_lock(&thub->sl); 9336 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 9337 verdict = match(dev, tunnel, (const void *)ctx); 9338 if (verdict) 9339 break; 9340 } 9341 if (!lock_op) 9342 rte_spinlock_unlock(&thub->sl); 9343 if (verdict && hit) 9344 hit(dev, tunnel, ctx); 9345 if (!verdict && miss) 9346 miss(dev, ctx); 9347 if (lock_op) 9348 rte_spinlock_unlock(&thub->sl); 9349 9350 return verdict; 9351 } 9352 9353 struct tunnel_db_find_tunnel_id_ctx { 9354 uint32_t tunnel_id; 9355 struct mlx5_flow_tunnel *tunnel; 9356 }; 9357 9358 static bool 9359 find_tunnel_id_match(struct rte_eth_dev *dev, 9360 struct mlx5_flow_tunnel *tunnel, const void *x) 9361 { 9362 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9363 9364 RTE_SET_USED(dev); 9365 return tunnel->tunnel_id == ctx->tunnel_id; 9366 } 9367 9368 static void 9369 find_tunnel_id_hit(struct rte_eth_dev *dev, 9370 struct mlx5_flow_tunnel *tunnel, void *x) 9371 { 9372 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9373 RTE_SET_USED(dev); 9374 ctx->tunnel = tunnel; 9375 } 9376 9377 static struct mlx5_flow_tunnel * 9378 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 9379 { 9380 struct tunnel_db_find_tunnel_id_ctx ctx = { 9381 .tunnel_id = id, 9382 }; 9383 9384 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 9385 find_tunnel_id_hit, NULL, &ctx, true); 9386 9387 return ctx.tunnel; 9388 } 9389 9390 static struct mlx5_flow_tunnel * 9391 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 9392 const struct rte_flow_tunnel *app_tunnel) 9393 { 9394 struct mlx5_priv *priv = dev->data->dev_private; 9395 struct mlx5_indexed_pool *ipool; 9396 struct mlx5_flow_tunnel *tunnel; 9397 uint32_t id; 9398 9399 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9400 tunnel = mlx5_ipool_zmalloc(ipool, &id); 9401 if (!tunnel) 9402 return NULL; 9403 if (id >= MLX5_MAX_TUNNELS) { 9404 mlx5_ipool_free(ipool, id); 9405 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 9406 return NULL; 9407 } 9408 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 9409 priv->sh, 9410 mlx5_flow_tunnel_grp2tbl_create_cb, 9411 mlx5_flow_tunnel_grp2tbl_match_cb, 9412 mlx5_flow_tunnel_grp2tbl_remove_cb, 9413 mlx5_flow_tunnel_grp2tbl_clone_cb, 9414 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9415 if (!tunnel->groups) { 9416 mlx5_ipool_free(ipool, id); 9417 return NULL; 9418 } 9419 /* initiate new PMD tunnel */ 9420 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 9421 tunnel->tunnel_id = id; 9422 tunnel->action.type = (typeof(tunnel->action.type)) 9423 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 9424 tunnel->action.conf = tunnel; 9425 tunnel->item.type = (typeof(tunnel->item.type)) 9426 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 9427 tunnel->item.spec = tunnel; 9428 tunnel->item.last = NULL; 9429 tunnel->item.mask = NULL; 9430 9431 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 9432 dev->data->port_id, tunnel->tunnel_id); 9433 9434 return tunnel; 9435 } 9436 9437 struct tunnel_db_get_tunnel_ctx { 9438 const struct rte_flow_tunnel *app_tunnel; 9439 struct mlx5_flow_tunnel *tunnel; 9440 }; 9441 9442 static bool get_tunnel_match(struct rte_eth_dev *dev, 9443 struct mlx5_flow_tunnel *tunnel, const void *x) 9444 { 9445 const struct tunnel_db_get_tunnel_ctx *ctx = x; 9446 9447 RTE_SET_USED(dev); 9448 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 9449 sizeof(*ctx->app_tunnel)); 9450 } 9451 9452 static void get_tunnel_hit(struct rte_eth_dev *dev, 9453 struct mlx5_flow_tunnel *tunnel, void *x) 9454 { 9455 /* called under tunnel spinlock protection */ 9456 struct tunnel_db_get_tunnel_ctx *ctx = x; 9457 9458 RTE_SET_USED(dev); 9459 tunnel->refctn++; 9460 ctx->tunnel = tunnel; 9461 } 9462 9463 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 9464 { 9465 /* called under tunnel spinlock protection */ 9466 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9467 struct tunnel_db_get_tunnel_ctx *ctx = x; 9468 9469 rte_spinlock_unlock(&thub->sl); 9470 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 9471 rte_spinlock_lock(&thub->sl); 9472 if (ctx->tunnel) { 9473 ctx->tunnel->refctn = 1; 9474 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 9475 } 9476 } 9477 9478 9479 static int 9480 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 9481 const struct rte_flow_tunnel *app_tunnel, 9482 struct mlx5_flow_tunnel **tunnel) 9483 { 9484 struct tunnel_db_get_tunnel_ctx ctx = { 9485 .app_tunnel = app_tunnel, 9486 }; 9487 9488 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 9489 get_tunnel_miss, &ctx, true); 9490 *tunnel = ctx.tunnel; 9491 return ctx.tunnel ? 0 : -ENOMEM; 9492 } 9493 9494 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 9495 { 9496 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 9497 9498 if (!thub) 9499 return; 9500 if (!LIST_EMPTY(&thub->tunnels)) 9501 DRV_LOG(WARNING, "port %u tunnels present", port_id); 9502 mlx5_hlist_destroy(thub->groups); 9503 mlx5_free(thub); 9504 } 9505 9506 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 9507 { 9508 int err; 9509 struct mlx5_flow_tunnel_hub *thub; 9510 9511 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 9512 0, SOCKET_ID_ANY); 9513 if (!thub) 9514 return -ENOMEM; 9515 LIST_INIT(&thub->tunnels); 9516 rte_spinlock_init(&thub->sl); 9517 thub->groups = mlx5_hlist_create("flow groups", 64, 9518 false, true, sh, 9519 mlx5_flow_tunnel_grp2tbl_create_cb, 9520 mlx5_flow_tunnel_grp2tbl_match_cb, 9521 mlx5_flow_tunnel_grp2tbl_remove_cb, 9522 mlx5_flow_tunnel_grp2tbl_clone_cb, 9523 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9524 if (!thub->groups) { 9525 err = -rte_errno; 9526 goto err; 9527 } 9528 sh->tunnel_hub = thub; 9529 9530 return 0; 9531 9532 err: 9533 if (thub->groups) 9534 mlx5_hlist_destroy(thub->groups); 9535 if (thub) 9536 mlx5_free(thub); 9537 return err; 9538 } 9539 9540 static inline int 9541 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 9542 struct rte_flow_tunnel *tunnel, 9543 struct rte_flow_error *error) 9544 { 9545 struct mlx5_priv *priv = dev->data->dev_private; 9546 9547 if (!priv->config.dv_flow_en) 9548 return rte_flow_error_set(error, ENOTSUP, 9549 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9550 "flow DV interface is off"); 9551 if (!is_tunnel_offload_active(dev)) 9552 return rte_flow_error_set(error, ENOTSUP, 9553 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9554 "tunnel offload was not activated"); 9555 if (!tunnel) 9556 return rte_flow_error_set(error, EINVAL, 9557 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9558 "no application tunnel"); 9559 switch (tunnel->type) { 9560 default: 9561 return rte_flow_error_set(error, EINVAL, 9562 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9563 "unsupported tunnel type"); 9564 case RTE_FLOW_ITEM_TYPE_VXLAN: 9565 case RTE_FLOW_ITEM_TYPE_GRE: 9566 case RTE_FLOW_ITEM_TYPE_NVGRE: 9567 case RTE_FLOW_ITEM_TYPE_GENEVE: 9568 break; 9569 } 9570 return 0; 9571 } 9572 9573 static int 9574 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 9575 struct rte_flow_tunnel *app_tunnel, 9576 struct rte_flow_action **actions, 9577 uint32_t *num_of_actions, 9578 struct rte_flow_error *error) 9579 { 9580 struct mlx5_flow_tunnel *tunnel; 9581 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9582 9583 if (ret) 9584 return ret; 9585 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9586 if (ret < 0) { 9587 return rte_flow_error_set(error, ret, 9588 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9589 "failed to initialize pmd tunnel"); 9590 } 9591 *actions = &tunnel->action; 9592 *num_of_actions = 1; 9593 return 0; 9594 } 9595 9596 static int 9597 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9598 struct rte_flow_tunnel *app_tunnel, 9599 struct rte_flow_item **items, 9600 uint32_t *num_of_items, 9601 struct rte_flow_error *error) 9602 { 9603 struct mlx5_flow_tunnel *tunnel; 9604 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9605 9606 if (ret) 9607 return ret; 9608 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9609 if (ret < 0) { 9610 return rte_flow_error_set(error, ret, 9611 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9612 "failed to initialize pmd tunnel"); 9613 } 9614 *items = &tunnel->item; 9615 *num_of_items = 1; 9616 return 0; 9617 } 9618 9619 struct tunnel_db_element_release_ctx { 9620 struct rte_flow_item *items; 9621 struct rte_flow_action *actions; 9622 uint32_t num_elements; 9623 struct rte_flow_error *error; 9624 int ret; 9625 }; 9626 9627 static bool 9628 tunnel_element_release_match(struct rte_eth_dev *dev, 9629 struct mlx5_flow_tunnel *tunnel, const void *x) 9630 { 9631 const struct tunnel_db_element_release_ctx *ctx = x; 9632 9633 RTE_SET_USED(dev); 9634 if (ctx->num_elements != 1) 9635 return false; 9636 else if (ctx->items) 9637 return ctx->items == &tunnel->item; 9638 else if (ctx->actions) 9639 return ctx->actions == &tunnel->action; 9640 9641 return false; 9642 } 9643 9644 static void 9645 tunnel_element_release_hit(struct rte_eth_dev *dev, 9646 struct mlx5_flow_tunnel *tunnel, void *x) 9647 { 9648 struct tunnel_db_element_release_ctx *ctx = x; 9649 ctx->ret = 0; 9650 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9651 mlx5_flow_tunnel_free(dev, tunnel); 9652 } 9653 9654 static void 9655 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9656 { 9657 struct tunnel_db_element_release_ctx *ctx = x; 9658 RTE_SET_USED(dev); 9659 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9660 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9661 "invalid argument"); 9662 } 9663 9664 static int 9665 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9666 struct rte_flow_item *pmd_items, 9667 uint32_t num_items, struct rte_flow_error *err) 9668 { 9669 struct tunnel_db_element_release_ctx ctx = { 9670 .items = pmd_items, 9671 .actions = NULL, 9672 .num_elements = num_items, 9673 .error = err, 9674 }; 9675 9676 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9677 tunnel_element_release_hit, 9678 tunnel_element_release_miss, &ctx, false); 9679 9680 return ctx.ret; 9681 } 9682 9683 static int 9684 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9685 struct rte_flow_action *pmd_actions, 9686 uint32_t num_actions, struct rte_flow_error *err) 9687 { 9688 struct tunnel_db_element_release_ctx ctx = { 9689 .items = NULL, 9690 .actions = pmd_actions, 9691 .num_elements = num_actions, 9692 .error = err, 9693 }; 9694 9695 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9696 tunnel_element_release_hit, 9697 tunnel_element_release_miss, &ctx, false); 9698 9699 return ctx.ret; 9700 } 9701 9702 static int 9703 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9704 struct rte_mbuf *m, 9705 struct rte_flow_restore_info *info, 9706 struct rte_flow_error *err) 9707 { 9708 uint64_t ol_flags = m->ol_flags; 9709 const struct mlx5_flow_tbl_data_entry *tble; 9710 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 9711 9712 if (!is_tunnel_offload_active(dev)) { 9713 info->flags = 0; 9714 return 0; 9715 } 9716 9717 if ((ol_flags & mask) != mask) 9718 goto err; 9719 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9720 if (!tble) { 9721 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9722 dev->data->port_id, m->hash.fdir.hi); 9723 goto err; 9724 } 9725 MLX5_ASSERT(tble->tunnel); 9726 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9727 info->group_id = tble->group_id; 9728 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9729 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9730 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9731 9732 return 0; 9733 9734 err: 9735 return rte_flow_error_set(err, EINVAL, 9736 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9737 "failed to get restore info"); 9738 } 9739 9740 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9741 static int 9742 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9743 __rte_unused struct rte_flow_tunnel *app_tunnel, 9744 __rte_unused struct rte_flow_action **actions, 9745 __rte_unused uint32_t *num_of_actions, 9746 __rte_unused struct rte_flow_error *error) 9747 { 9748 return -ENOTSUP; 9749 } 9750 9751 static int 9752 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9753 __rte_unused struct rte_flow_tunnel *app_tunnel, 9754 __rte_unused struct rte_flow_item **items, 9755 __rte_unused uint32_t *num_of_items, 9756 __rte_unused struct rte_flow_error *error) 9757 { 9758 return -ENOTSUP; 9759 } 9760 9761 static int 9762 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9763 __rte_unused struct rte_flow_item *pmd_items, 9764 __rte_unused uint32_t num_items, 9765 __rte_unused struct rte_flow_error *err) 9766 { 9767 return -ENOTSUP; 9768 } 9769 9770 static int 9771 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9772 __rte_unused struct rte_flow_action *pmd_action, 9773 __rte_unused uint32_t num_actions, 9774 __rte_unused struct rte_flow_error *err) 9775 { 9776 return -ENOTSUP; 9777 } 9778 9779 static int 9780 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9781 __rte_unused struct rte_mbuf *m, 9782 __rte_unused struct rte_flow_restore_info *i, 9783 __rte_unused struct rte_flow_error *err) 9784 { 9785 return -ENOTSUP; 9786 } 9787 9788 static int 9789 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9790 __rte_unused struct rte_flow *flow, 9791 __rte_unused const struct rte_flow_attr *attr, 9792 __rte_unused const struct rte_flow_action *actions, 9793 __rte_unused uint32_t flow_idx, 9794 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9795 __rte_unused struct tunnel_default_miss_ctx *ctx, 9796 __rte_unused struct rte_flow_error *error) 9797 { 9798 return -ENOTSUP; 9799 } 9800 9801 static struct mlx5_flow_tunnel * 9802 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9803 __rte_unused uint32_t id) 9804 { 9805 return NULL; 9806 } 9807 9808 static void 9809 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9810 __rte_unused struct mlx5_flow_tunnel *tunnel) 9811 { 9812 } 9813 9814 static uint32_t 9815 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9816 __rte_unused const struct mlx5_flow_tunnel *t, 9817 __rte_unused uint32_t group, 9818 __rte_unused uint32_t *table, 9819 struct rte_flow_error *error) 9820 { 9821 return rte_flow_error_set(error, ENOTSUP, 9822 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9823 "tunnel offload requires DV support"); 9824 } 9825 9826 void 9827 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9828 __rte_unused uint16_t port_id) 9829 { 9830 } 9831 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9832 9833 /* Flex flow item API */ 9834 static struct rte_flow_item_flex_handle * 9835 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 9836 const struct rte_flow_item_flex_conf *conf, 9837 struct rte_flow_error *error) 9838 { 9839 static const char err_msg[] = "flex item creation unsupported"; 9840 struct rte_flow_attr attr = { .transfer = 0 }; 9841 const struct mlx5_flow_driver_ops *fops = 9842 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9843 9844 if (!fops->item_create) { 9845 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9846 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9847 NULL, err_msg); 9848 return NULL; 9849 } 9850 return fops->item_create(dev, conf, error); 9851 } 9852 9853 static int 9854 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 9855 const struct rte_flow_item_flex_handle *handle, 9856 struct rte_flow_error *error) 9857 { 9858 static const char err_msg[] = "flex item release unsupported"; 9859 struct rte_flow_attr attr = { .transfer = 0 }; 9860 const struct mlx5_flow_driver_ops *fops = 9861 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9862 9863 if (!fops->item_release) { 9864 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9865 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9866 NULL, err_msg); 9867 return -rte_errno; 9868 } 9869 return fops->item_release(dev, handle, error); 9870 } 9871 9872 static void 9873 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9874 { 9875 int ret; 9876 struct rte_flow_error error; 9877 9878 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9879 char *item_name; 9880 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9881 sizeof(item_name), 9882 (void *)(uintptr_t)item->type, &error); 9883 if (ret > 0) 9884 printf("%s ", item_name); 9885 else 9886 printf("%d\n", (int)item->type); 9887 } 9888 printf("END\n"); 9889 } 9890 9891 static int 9892 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 9893 { 9894 const struct rte_flow_item_udp *spec = udp_item->spec; 9895 const struct rte_flow_item_udp *mask = udp_item->mask; 9896 uint16_t udp_dport = 0; 9897 9898 if (spec != NULL) { 9899 if (!mask) 9900 mask = &rte_flow_item_udp_mask; 9901 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 9902 mask->hdr.dst_port); 9903 } 9904 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 9905 } 9906 9907 static const struct mlx5_flow_expand_node * 9908 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 9909 unsigned int item_idx, 9910 const struct mlx5_flow_expand_node graph[], 9911 const struct mlx5_flow_expand_node *node) 9912 { 9913 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 9914 9915 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 9916 node != NULL && 9917 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 9918 /* 9919 * The expansion node is VXLAN and it is also the last 9920 * expandable item in the pattern, so need to continue 9921 * expansion of the inner tunnel. 9922 */ 9923 MLX5_ASSERT(item_idx > 0); 9924 prev_item = pattern + item_idx - 1; 9925 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 9926 if (mlx5_flow_is_std_vxlan_port(prev_item)) 9927 return &graph[MLX5_EXPANSION_STD_VXLAN]; 9928 return &graph[MLX5_EXPANSION_L3_VXLAN]; 9929 } 9930 return node; 9931 } 9932 9933 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 9934 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 9935 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 9936 }; 9937 9938 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 9939 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 9940 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 9941 { 9, 10, 11 }, { 12, 13, 14 }, 9942 }; 9943 9944 /** 9945 * Discover the number of available flow priorities. 9946 * 9947 * @param dev 9948 * Ethernet device. 9949 * 9950 * @return 9951 * On success, number of available flow priorities. 9952 * On failure, a negative errno-style code and rte_errno is set. 9953 */ 9954 int 9955 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 9956 { 9957 static const uint16_t vprio[] = {8, 16}; 9958 const struct mlx5_priv *priv = dev->data->dev_private; 9959 const struct mlx5_flow_driver_ops *fops; 9960 enum mlx5_flow_drv_type type; 9961 int ret; 9962 9963 type = mlx5_flow_os_get_type(); 9964 if (type == MLX5_FLOW_TYPE_MAX) { 9965 type = MLX5_FLOW_TYPE_VERBS; 9966 if (priv->sh->devx && priv->config.dv_flow_en) 9967 type = MLX5_FLOW_TYPE_DV; 9968 } 9969 fops = flow_get_drv_ops(type); 9970 if (fops->discover_priorities == NULL) { 9971 DRV_LOG(ERR, "Priority discovery not supported"); 9972 rte_errno = ENOTSUP; 9973 return -rte_errno; 9974 } 9975 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 9976 if (ret < 0) 9977 return ret; 9978 switch (ret) { 9979 case 8: 9980 ret = RTE_DIM(priority_map_3); 9981 break; 9982 case 16: 9983 ret = RTE_DIM(priority_map_5); 9984 break; 9985 default: 9986 rte_errno = ENOTSUP; 9987 DRV_LOG(ERR, 9988 "port %u maximum priority: %d expected 8/16", 9989 dev->data->port_id, ret); 9990 return -rte_errno; 9991 } 9992 DRV_LOG(INFO, "port %u supported flow priorities:" 9993 " 0-%d for ingress or egress root table," 9994 " 0-%d for non-root table or transfer root table.", 9995 dev->data->port_id, ret - 2, 9996 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 9997 return ret; 9998 } 9999 10000 /** 10001 * Adjust flow priority based on the highest layer and the request priority. 10002 * 10003 * @param[in] dev 10004 * Pointer to the Ethernet device structure. 10005 * @param[in] priority 10006 * The rule base priority. 10007 * @param[in] subpriority 10008 * The priority based on the items. 10009 * 10010 * @return 10011 * The new priority. 10012 */ 10013 uint32_t 10014 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10015 uint32_t subpriority) 10016 { 10017 uint32_t res = 0; 10018 struct mlx5_priv *priv = dev->data->dev_private; 10019 10020 switch (priv->sh->flow_max_priority) { 10021 case RTE_DIM(priority_map_3): 10022 res = priority_map_3[priority][subpriority]; 10023 break; 10024 case RTE_DIM(priority_map_5): 10025 res = priority_map_5[priority][subpriority]; 10026 break; 10027 } 10028 return res; 10029 } 10030