1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 80 #endif 81 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 82 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 83 }; 84 85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 87 (const int []){ \ 88 __VA_ARGS__, 0, \ 89 } 90 91 /** Node object of input graph for mlx5_flow_expand_rss(). */ 92 struct mlx5_flow_expand_node { 93 const int *const next; 94 /**< 95 * List of next node indexes. Index 0 is interpreted as a terminator. 96 */ 97 const enum rte_flow_item_type type; 98 /**< Pattern item type of current node. */ 99 uint64_t rss_types; 100 /**< 101 * RSS types bit-field associated with this node 102 * (see RTE_ETH_RSS_* definitions). 103 */ 104 uint64_t node_flags; 105 /**< 106 * Bit-fields that define how the node is used in the expansion. 107 * (see MLX5_EXPANSION_NODE_* definitions). 108 */ 109 }; 110 111 /* Optional expand field. The expansion alg will not go deeper. */ 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 113 114 /* The node is not added implicitly as expansion to the flow pattern. 115 * If the node type does not match the flow pattern item type, the 116 * expansion alg will go deeper to its next items. 117 * In the current implementation, the list of next nodes indexes can 118 * have up to one node with this flag set and it has to be the last 119 * node index (before the list terminator). 120 */ 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 122 123 /** Object returned by mlx5_flow_expand_rss(). */ 124 struct mlx5_flow_expand_rss { 125 uint32_t entries; 126 /**< Number of entries @p patterns and @p priorities. */ 127 struct { 128 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 129 uint32_t priority; /**< Priority offset for each expansion. */ 130 } entry[]; 131 }; 132 133 static void 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 135 136 static const struct mlx5_flow_expand_node * 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 138 unsigned int item_idx, 139 const struct mlx5_flow_expand_node graph[], 140 const struct mlx5_flow_expand_node *node); 141 142 static bool 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 144 { 145 switch (item->type) { 146 case RTE_FLOW_ITEM_TYPE_ETH: 147 case RTE_FLOW_ITEM_TYPE_VLAN: 148 case RTE_FLOW_ITEM_TYPE_IPV4: 149 case RTE_FLOW_ITEM_TYPE_IPV6: 150 case RTE_FLOW_ITEM_TYPE_UDP: 151 case RTE_FLOW_ITEM_TYPE_TCP: 152 case RTE_FLOW_ITEM_TYPE_VXLAN: 153 case RTE_FLOW_ITEM_TYPE_NVGRE: 154 case RTE_FLOW_ITEM_TYPE_GRE: 155 case RTE_FLOW_ITEM_TYPE_GENEVE: 156 case RTE_FLOW_ITEM_TYPE_MPLS: 157 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 158 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 159 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 160 case RTE_FLOW_ITEM_TYPE_GTP: 161 return true; 162 default: 163 break; 164 } 165 return false; 166 } 167 168 /** 169 * Network Service Header (NSH) and its next protocol values 170 * are described in RFC-8393. 171 */ 172 static enum rte_flow_item_type 173 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 174 { 175 enum rte_flow_item_type type; 176 177 switch (proto_mask & proto_spec) { 178 case 0: 179 type = RTE_FLOW_ITEM_TYPE_VOID; 180 break; 181 case RTE_VXLAN_GPE_TYPE_IPV4: 182 type = RTE_FLOW_ITEM_TYPE_IPV4; 183 break; 184 case RTE_VXLAN_GPE_TYPE_IPV6: 185 type = RTE_VXLAN_GPE_TYPE_IPV6; 186 break; 187 case RTE_VXLAN_GPE_TYPE_ETH: 188 type = RTE_FLOW_ITEM_TYPE_ETH; 189 break; 190 default: 191 type = RTE_FLOW_ITEM_TYPE_END; 192 } 193 return type; 194 } 195 196 static enum rte_flow_item_type 197 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 198 { 199 enum rte_flow_item_type type; 200 201 switch (proto_mask & proto_spec) { 202 case 0: 203 type = RTE_FLOW_ITEM_TYPE_VOID; 204 break; 205 case IPPROTO_UDP: 206 type = RTE_FLOW_ITEM_TYPE_UDP; 207 break; 208 case IPPROTO_TCP: 209 type = RTE_FLOW_ITEM_TYPE_TCP; 210 break; 211 case IPPROTO_IPIP: 212 type = RTE_FLOW_ITEM_TYPE_IPV4; 213 break; 214 case IPPROTO_IPV6: 215 type = RTE_FLOW_ITEM_TYPE_IPV6; 216 break; 217 default: 218 type = RTE_FLOW_ITEM_TYPE_END; 219 } 220 return type; 221 } 222 223 static enum rte_flow_item_type 224 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 225 rte_be16_t type_mask, bool is_tunnel) 226 { 227 enum rte_flow_item_type type; 228 229 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 230 case 0: 231 type = RTE_FLOW_ITEM_TYPE_VOID; 232 break; 233 case RTE_ETHER_TYPE_TEB: 234 type = is_tunnel ? 235 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 236 break; 237 case RTE_ETHER_TYPE_VLAN: 238 type = !is_tunnel ? 239 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 240 break; 241 case RTE_ETHER_TYPE_IPV4: 242 type = RTE_FLOW_ITEM_TYPE_IPV4; 243 break; 244 case RTE_ETHER_TYPE_IPV6: 245 type = RTE_FLOW_ITEM_TYPE_IPV6; 246 break; 247 default: 248 type = RTE_FLOW_ITEM_TYPE_END; 249 } 250 return type; 251 } 252 253 static enum rte_flow_item_type 254 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 255 { 256 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 257 do { \ 258 const void *m = item->mask; \ 259 const void *s = item->spec; \ 260 mask = m ? \ 261 ((const struct rte_flow_item_##type *)m)->fld : \ 262 rte_flow_item_##type##_mask.fld; \ 263 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 264 } while (0) 265 266 enum rte_flow_item_type ret; 267 uint16_t spec, mask; 268 269 if (item == NULL || item->spec == NULL) 270 return RTE_FLOW_ITEM_TYPE_VOID; 271 switch (item->type) { 272 case RTE_FLOW_ITEM_TYPE_ETH: 273 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 274 if (!mask) 275 return RTE_FLOW_ITEM_TYPE_VOID; 276 ret = mlx5_ethertype_to_item_type(spec, mask, false); 277 break; 278 case RTE_FLOW_ITEM_TYPE_VLAN: 279 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 280 if (!mask) 281 return RTE_FLOW_ITEM_TYPE_VOID; 282 ret = mlx5_ethertype_to_item_type(spec, mask, false); 283 break; 284 case RTE_FLOW_ITEM_TYPE_IPV4: 285 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 286 if (!mask) 287 return RTE_FLOW_ITEM_TYPE_VOID; 288 ret = mlx5_inet_proto_to_item_type(spec, mask); 289 break; 290 case RTE_FLOW_ITEM_TYPE_IPV6: 291 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 292 if (!mask) 293 return RTE_FLOW_ITEM_TYPE_VOID; 294 ret = mlx5_inet_proto_to_item_type(spec, mask); 295 break; 296 case RTE_FLOW_ITEM_TYPE_GENEVE: 297 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 298 ret = mlx5_ethertype_to_item_type(spec, mask, true); 299 break; 300 case RTE_FLOW_ITEM_TYPE_GRE: 301 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 302 ret = mlx5_ethertype_to_item_type(spec, mask, true); 303 break; 304 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 305 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 306 ret = mlx5_nsh_proto_to_item_type(spec, mask); 307 break; 308 default: 309 ret = RTE_FLOW_ITEM_TYPE_VOID; 310 break; 311 } 312 return ret; 313 #undef MLX5_XSET_ITEM_MASK_SPEC 314 } 315 316 static const int * 317 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 318 const int *next_node) 319 { 320 const struct mlx5_flow_expand_node *node = NULL; 321 const int *next = next_node; 322 323 while (next && *next) { 324 /* 325 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 326 * flag set, because they were not found in the flow pattern. 327 */ 328 node = &graph[*next]; 329 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 330 break; 331 next = node->next; 332 } 333 return next; 334 } 335 336 #define MLX5_RSS_EXP_ELT_N 16 337 338 /** 339 * Expand RSS flows into several possible flows according to the RSS hash 340 * fields requested and the driver capabilities. 341 * 342 * @param[out] buf 343 * Buffer to store the result expansion. 344 * @param[in] size 345 * Buffer size in bytes. If 0, @p buf can be NULL. 346 * @param[in] pattern 347 * User flow pattern. 348 * @param[in] types 349 * RSS types to expand (see RTE_ETH_RSS_* definitions). 350 * @param[in] graph 351 * Input graph to expand @p pattern according to @p types. 352 * @param[in] graph_root_index 353 * Index of root node in @p graph, typically 0. 354 * 355 * @return 356 * A positive value representing the size of @p buf in bytes regardless of 357 * @p size on success, a negative errno value otherwise and rte_errno is 358 * set, the following errors are defined: 359 * 360 * -E2BIG: graph-depth @p graph is too deep. 361 * -EINVAL: @p size has not enough space for expanded pattern. 362 */ 363 static int 364 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 365 const struct rte_flow_item *pattern, uint64_t types, 366 const struct mlx5_flow_expand_node graph[], 367 int graph_root_index) 368 { 369 const struct rte_flow_item *item; 370 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 371 const int *next_node; 372 const int *stack[MLX5_RSS_EXP_ELT_N]; 373 int stack_pos = 0; 374 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 375 unsigned int i, item_idx, last_expand_item_idx = 0; 376 size_t lsize; 377 size_t user_pattern_size = 0; 378 void *addr = NULL; 379 const struct mlx5_flow_expand_node *next = NULL; 380 struct rte_flow_item missed_item; 381 int missed = 0; 382 int elt = 0; 383 const struct rte_flow_item *last_expand_item = NULL; 384 385 memset(&missed_item, 0, sizeof(missed_item)); 386 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 387 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 388 if (lsize > size) 389 return -EINVAL; 390 buf->entry[0].priority = 0; 391 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 392 buf->entries = 0; 393 addr = buf->entry[0].pattern; 394 for (item = pattern, item_idx = 0; 395 item->type != RTE_FLOW_ITEM_TYPE_END; 396 item++, item_idx++) { 397 if (!mlx5_flow_is_rss_expandable_item(item)) { 398 user_pattern_size += sizeof(*item); 399 continue; 400 } 401 last_expand_item = item; 402 last_expand_item_idx = item_idx; 403 i = 0; 404 while (node->next && node->next[i]) { 405 next = &graph[node->next[i]]; 406 if (next->type == item->type) 407 break; 408 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 409 node = next; 410 i = 0; 411 } else { 412 ++i; 413 } 414 } 415 if (next) 416 node = next; 417 user_pattern_size += sizeof(*item); 418 } 419 user_pattern_size += sizeof(*item); /* Handle END item. */ 420 lsize += user_pattern_size; 421 if (lsize > size) 422 return -EINVAL; 423 /* Copy the user pattern in the first entry of the buffer. */ 424 rte_memcpy(addr, pattern, user_pattern_size); 425 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 426 buf->entries = 1; 427 /* Start expanding. */ 428 memset(flow_items, 0, sizeof(flow_items)); 429 user_pattern_size -= sizeof(*item); 430 /* 431 * Check if the last valid item has spec set, need complete pattern, 432 * and the pattern can be used for expansion. 433 */ 434 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 435 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 436 /* Item type END indicates expansion is not required. */ 437 return lsize; 438 } 439 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 440 next = NULL; 441 missed = 1; 442 i = 0; 443 while (node->next && node->next[i]) { 444 next = &graph[node->next[i]]; 445 if (next->type == missed_item.type) { 446 flow_items[0].type = missed_item.type; 447 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 448 break; 449 } 450 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 451 node = next; 452 i = 0; 453 } else { 454 ++i; 455 } 456 next = NULL; 457 } 458 } 459 if (next && missed) { 460 elt = 2; /* missed item + item end. */ 461 node = next; 462 lsize += elt * sizeof(*item) + user_pattern_size; 463 if (lsize > size) 464 return -EINVAL; 465 if (node->rss_types & types) { 466 buf->entry[buf->entries].priority = 1; 467 buf->entry[buf->entries].pattern = addr; 468 buf->entries++; 469 rte_memcpy(addr, buf->entry[0].pattern, 470 user_pattern_size); 471 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 472 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 473 addr = (void *)(((uintptr_t)addr) + 474 elt * sizeof(*item)); 475 } 476 } else if (last_expand_item != NULL) { 477 node = mlx5_flow_expand_rss_adjust_node(pattern, 478 last_expand_item_idx, graph, node); 479 } 480 memset(flow_items, 0, sizeof(flow_items)); 481 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 482 node->next); 483 stack[stack_pos] = next_node; 484 node = next_node ? &graph[*next_node] : NULL; 485 while (node) { 486 flow_items[stack_pos].type = node->type; 487 if (node->rss_types & types) { 488 size_t n; 489 /* 490 * compute the number of items to copy from the 491 * expansion and copy it. 492 * When the stack_pos is 0, there are 1 element in it, 493 * plus the addition END item. 494 */ 495 elt = stack_pos + 2; 496 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 497 lsize += elt * sizeof(*item) + user_pattern_size; 498 if (lsize > size) 499 return -EINVAL; 500 n = elt * sizeof(*item); 501 buf->entry[buf->entries].priority = 502 stack_pos + 1 + missed; 503 buf->entry[buf->entries].pattern = addr; 504 buf->entries++; 505 rte_memcpy(addr, buf->entry[0].pattern, 506 user_pattern_size); 507 addr = (void *)(((uintptr_t)addr) + 508 user_pattern_size); 509 rte_memcpy(addr, &missed_item, 510 missed * sizeof(*item)); 511 addr = (void *)(((uintptr_t)addr) + 512 missed * sizeof(*item)); 513 rte_memcpy(addr, flow_items, n); 514 addr = (void *)(((uintptr_t)addr) + n); 515 } 516 /* Go deeper. */ 517 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 518 node->next) { 519 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 520 node->next); 521 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 522 rte_errno = E2BIG; 523 return -rte_errno; 524 } 525 stack[stack_pos] = next_node; 526 } else if (*(next_node + 1)) { 527 /* Follow up with the next possibility. */ 528 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 529 ++next_node); 530 } else if (!stack_pos) { 531 /* 532 * Completing the traverse over the different paths. 533 * The next_node is advanced to the terminator. 534 */ 535 ++next_node; 536 } else { 537 /* Move to the next path. */ 538 while (stack_pos) { 539 next_node = stack[--stack_pos]; 540 next_node++; 541 if (*next_node) 542 break; 543 } 544 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 545 next_node); 546 stack[stack_pos] = next_node; 547 } 548 node = next_node && *next_node ? &graph[*next_node] : NULL; 549 }; 550 return lsize; 551 } 552 553 enum mlx5_expansion { 554 MLX5_EXPANSION_ROOT, 555 MLX5_EXPANSION_ROOT_OUTER, 556 MLX5_EXPANSION_OUTER_ETH, 557 MLX5_EXPANSION_OUTER_VLAN, 558 MLX5_EXPANSION_OUTER_IPV4, 559 MLX5_EXPANSION_OUTER_IPV4_UDP, 560 MLX5_EXPANSION_OUTER_IPV4_TCP, 561 MLX5_EXPANSION_OUTER_IPV6, 562 MLX5_EXPANSION_OUTER_IPV6_UDP, 563 MLX5_EXPANSION_OUTER_IPV6_TCP, 564 MLX5_EXPANSION_VXLAN, 565 MLX5_EXPANSION_STD_VXLAN, 566 MLX5_EXPANSION_L3_VXLAN, 567 MLX5_EXPANSION_VXLAN_GPE, 568 MLX5_EXPANSION_GRE, 569 MLX5_EXPANSION_NVGRE, 570 MLX5_EXPANSION_GRE_KEY, 571 MLX5_EXPANSION_MPLS, 572 MLX5_EXPANSION_ETH, 573 MLX5_EXPANSION_VLAN, 574 MLX5_EXPANSION_IPV4, 575 MLX5_EXPANSION_IPV4_UDP, 576 MLX5_EXPANSION_IPV4_TCP, 577 MLX5_EXPANSION_IPV6, 578 MLX5_EXPANSION_IPV6_UDP, 579 MLX5_EXPANSION_IPV6_TCP, 580 MLX5_EXPANSION_IPV6_FRAG_EXT, 581 MLX5_EXPANSION_GTP, 582 MLX5_EXPANSION_GENEVE, 583 }; 584 585 /** Supported expansion of items. */ 586 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 587 [MLX5_EXPANSION_ROOT] = { 588 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 589 MLX5_EXPANSION_IPV4, 590 MLX5_EXPANSION_IPV6), 591 .type = RTE_FLOW_ITEM_TYPE_END, 592 }, 593 [MLX5_EXPANSION_ROOT_OUTER] = { 594 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 595 MLX5_EXPANSION_OUTER_IPV4, 596 MLX5_EXPANSION_OUTER_IPV6), 597 .type = RTE_FLOW_ITEM_TYPE_END, 598 }, 599 [MLX5_EXPANSION_OUTER_ETH] = { 600 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 601 .type = RTE_FLOW_ITEM_TYPE_ETH, 602 .rss_types = 0, 603 }, 604 [MLX5_EXPANSION_OUTER_VLAN] = { 605 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 606 MLX5_EXPANSION_OUTER_IPV6), 607 .type = RTE_FLOW_ITEM_TYPE_VLAN, 608 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 609 }, 610 [MLX5_EXPANSION_OUTER_IPV4] = { 611 .next = MLX5_FLOW_EXPAND_RSS_NEXT 612 (MLX5_EXPANSION_OUTER_IPV4_UDP, 613 MLX5_EXPANSION_OUTER_IPV4_TCP, 614 MLX5_EXPANSION_GRE, 615 MLX5_EXPANSION_NVGRE, 616 MLX5_EXPANSION_IPV4, 617 MLX5_EXPANSION_IPV6), 618 .type = RTE_FLOW_ITEM_TYPE_IPV4, 619 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 620 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 621 }, 622 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 623 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 624 MLX5_EXPANSION_VXLAN_GPE, 625 MLX5_EXPANSION_MPLS, 626 MLX5_EXPANSION_GENEVE, 627 MLX5_EXPANSION_GTP), 628 .type = RTE_FLOW_ITEM_TYPE_UDP, 629 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 630 }, 631 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 632 .type = RTE_FLOW_ITEM_TYPE_TCP, 633 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 634 }, 635 [MLX5_EXPANSION_OUTER_IPV6] = { 636 .next = MLX5_FLOW_EXPAND_RSS_NEXT 637 (MLX5_EXPANSION_OUTER_IPV6_UDP, 638 MLX5_EXPANSION_OUTER_IPV6_TCP, 639 MLX5_EXPANSION_IPV4, 640 MLX5_EXPANSION_IPV6, 641 MLX5_EXPANSION_GRE, 642 MLX5_EXPANSION_NVGRE), 643 .type = RTE_FLOW_ITEM_TYPE_IPV6, 644 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 645 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 646 }, 647 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 648 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 649 MLX5_EXPANSION_VXLAN_GPE, 650 MLX5_EXPANSION_MPLS, 651 MLX5_EXPANSION_GENEVE, 652 MLX5_EXPANSION_GTP), 653 .type = RTE_FLOW_ITEM_TYPE_UDP, 654 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 655 }, 656 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 657 .type = RTE_FLOW_ITEM_TYPE_TCP, 658 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 659 }, 660 [MLX5_EXPANSION_VXLAN] = { 661 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 662 MLX5_EXPANSION_IPV4, 663 MLX5_EXPANSION_IPV6), 664 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 665 }, 666 [MLX5_EXPANSION_STD_VXLAN] = { 667 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 668 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 669 }, 670 [MLX5_EXPANSION_L3_VXLAN] = { 671 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 672 MLX5_EXPANSION_IPV6), 673 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 674 }, 675 [MLX5_EXPANSION_VXLAN_GPE] = { 676 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 677 MLX5_EXPANSION_IPV4, 678 MLX5_EXPANSION_IPV6), 679 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 680 }, 681 [MLX5_EXPANSION_GRE] = { 682 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 683 MLX5_EXPANSION_IPV4, 684 MLX5_EXPANSION_IPV6, 685 MLX5_EXPANSION_GRE_KEY, 686 MLX5_EXPANSION_MPLS), 687 .type = RTE_FLOW_ITEM_TYPE_GRE, 688 }, 689 [MLX5_EXPANSION_GRE_KEY] = { 690 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 691 MLX5_EXPANSION_IPV6, 692 MLX5_EXPANSION_MPLS), 693 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 694 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 695 }, 696 [MLX5_EXPANSION_NVGRE] = { 697 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 698 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 699 }, 700 [MLX5_EXPANSION_MPLS] = { 701 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 702 MLX5_EXPANSION_IPV6, 703 MLX5_EXPANSION_ETH), 704 .type = RTE_FLOW_ITEM_TYPE_MPLS, 705 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 706 }, 707 [MLX5_EXPANSION_ETH] = { 708 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 709 .type = RTE_FLOW_ITEM_TYPE_ETH, 710 }, 711 [MLX5_EXPANSION_VLAN] = { 712 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 713 MLX5_EXPANSION_IPV6), 714 .type = RTE_FLOW_ITEM_TYPE_VLAN, 715 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 716 }, 717 [MLX5_EXPANSION_IPV4] = { 718 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 719 MLX5_EXPANSION_IPV4_TCP), 720 .type = RTE_FLOW_ITEM_TYPE_IPV4, 721 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 722 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 723 }, 724 [MLX5_EXPANSION_IPV4_UDP] = { 725 .type = RTE_FLOW_ITEM_TYPE_UDP, 726 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 727 }, 728 [MLX5_EXPANSION_IPV4_TCP] = { 729 .type = RTE_FLOW_ITEM_TYPE_TCP, 730 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 731 }, 732 [MLX5_EXPANSION_IPV6] = { 733 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 734 MLX5_EXPANSION_IPV6_TCP, 735 MLX5_EXPANSION_IPV6_FRAG_EXT), 736 .type = RTE_FLOW_ITEM_TYPE_IPV6, 737 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 738 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 739 }, 740 [MLX5_EXPANSION_IPV6_UDP] = { 741 .type = RTE_FLOW_ITEM_TYPE_UDP, 742 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 743 }, 744 [MLX5_EXPANSION_IPV6_TCP] = { 745 .type = RTE_FLOW_ITEM_TYPE_TCP, 746 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 747 }, 748 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 749 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 750 }, 751 [MLX5_EXPANSION_GTP] = { 752 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 753 MLX5_EXPANSION_IPV6), 754 .type = RTE_FLOW_ITEM_TYPE_GTP, 755 }, 756 [MLX5_EXPANSION_GENEVE] = { 757 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 758 MLX5_EXPANSION_IPV4, 759 MLX5_EXPANSION_IPV6), 760 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 761 }, 762 }; 763 764 static struct rte_flow_action_handle * 765 mlx5_action_handle_create(struct rte_eth_dev *dev, 766 const struct rte_flow_indir_action_conf *conf, 767 const struct rte_flow_action *action, 768 struct rte_flow_error *error); 769 static int mlx5_action_handle_destroy 770 (struct rte_eth_dev *dev, 771 struct rte_flow_action_handle *handle, 772 struct rte_flow_error *error); 773 static int mlx5_action_handle_update 774 (struct rte_eth_dev *dev, 775 struct rte_flow_action_handle *handle, 776 const void *update, 777 struct rte_flow_error *error); 778 static int mlx5_action_handle_query 779 (struct rte_eth_dev *dev, 780 const struct rte_flow_action_handle *handle, 781 void *data, 782 struct rte_flow_error *error); 783 static int 784 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 785 struct rte_flow_tunnel *app_tunnel, 786 struct rte_flow_action **actions, 787 uint32_t *num_of_actions, 788 struct rte_flow_error *error); 789 static int 790 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 791 struct rte_flow_tunnel *app_tunnel, 792 struct rte_flow_item **items, 793 uint32_t *num_of_items, 794 struct rte_flow_error *error); 795 static int 796 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 797 struct rte_flow_item *pmd_items, 798 uint32_t num_items, struct rte_flow_error *err); 799 static int 800 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 801 struct rte_flow_action *pmd_actions, 802 uint32_t num_actions, 803 struct rte_flow_error *err); 804 static int 805 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 806 struct rte_mbuf *m, 807 struct rte_flow_restore_info *info, 808 struct rte_flow_error *err); 809 static struct rte_flow_item_flex_handle * 810 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 811 const struct rte_flow_item_flex_conf *conf, 812 struct rte_flow_error *error); 813 static int 814 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 815 const struct rte_flow_item_flex_handle *handle, 816 struct rte_flow_error *error); 817 static int 818 mlx5_flow_info_get(struct rte_eth_dev *dev, 819 struct rte_flow_port_info *port_info, 820 struct rte_flow_queue_info *queue_info, 821 struct rte_flow_error *error); 822 static int 823 mlx5_flow_port_configure(struct rte_eth_dev *dev, 824 const struct rte_flow_port_attr *port_attr, 825 uint16_t nb_queue, 826 const struct rte_flow_queue_attr *queue_attr[], 827 struct rte_flow_error *err); 828 829 static struct rte_flow_pattern_template * 830 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 831 const struct rte_flow_pattern_template_attr *attr, 832 const struct rte_flow_item items[], 833 struct rte_flow_error *error); 834 835 static int 836 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 837 struct rte_flow_pattern_template *template, 838 struct rte_flow_error *error); 839 static struct rte_flow_actions_template * 840 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 841 const struct rte_flow_actions_template_attr *attr, 842 const struct rte_flow_action actions[], 843 const struct rte_flow_action masks[], 844 struct rte_flow_error *error); 845 static int 846 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 847 struct rte_flow_actions_template *template, 848 struct rte_flow_error *error); 849 850 static struct rte_flow_template_table * 851 mlx5_flow_table_create(struct rte_eth_dev *dev, 852 const struct rte_flow_template_table_attr *attr, 853 struct rte_flow_pattern_template *item_templates[], 854 uint8_t nb_item_templates, 855 struct rte_flow_actions_template *action_templates[], 856 uint8_t nb_action_templates, 857 struct rte_flow_error *error); 858 static int 859 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 860 struct rte_flow_template_table *table, 861 struct rte_flow_error *error); 862 static struct rte_flow * 863 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 864 uint32_t queue, 865 const struct rte_flow_op_attr *attr, 866 struct rte_flow_template_table *table, 867 const struct rte_flow_item items[], 868 uint8_t pattern_template_index, 869 const struct rte_flow_action actions[], 870 uint8_t action_template_index, 871 void *user_data, 872 struct rte_flow_error *error); 873 static int 874 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 875 uint32_t queue, 876 const struct rte_flow_op_attr *attr, 877 struct rte_flow *flow, 878 void *user_data, 879 struct rte_flow_error *error); 880 static int 881 mlx5_flow_pull(struct rte_eth_dev *dev, 882 uint32_t queue, 883 struct rte_flow_op_result res[], 884 uint16_t n_res, 885 struct rte_flow_error *error); 886 static int 887 mlx5_flow_push(struct rte_eth_dev *dev, 888 uint32_t queue, 889 struct rte_flow_error *error); 890 891 static struct rte_flow_action_handle * 892 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 893 const struct rte_flow_op_attr *attr, 894 const struct rte_flow_indir_action_conf *conf, 895 const struct rte_flow_action *action, 896 void *user_data, 897 struct rte_flow_error *error); 898 899 static int 900 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 901 const struct rte_flow_op_attr *attr, 902 struct rte_flow_action_handle *handle, 903 const void *update, 904 void *user_data, 905 struct rte_flow_error *error); 906 907 static int 908 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 909 const struct rte_flow_op_attr *attr, 910 struct rte_flow_action_handle *handle, 911 void *user_data, 912 struct rte_flow_error *error); 913 914 static const struct rte_flow_ops mlx5_flow_ops = { 915 .validate = mlx5_flow_validate, 916 .create = mlx5_flow_create, 917 .destroy = mlx5_flow_destroy, 918 .flush = mlx5_flow_flush, 919 .isolate = mlx5_flow_isolate, 920 .query = mlx5_flow_query, 921 .dev_dump = mlx5_flow_dev_dump, 922 .get_aged_flows = mlx5_flow_get_aged_flows, 923 .action_handle_create = mlx5_action_handle_create, 924 .action_handle_destroy = mlx5_action_handle_destroy, 925 .action_handle_update = mlx5_action_handle_update, 926 .action_handle_query = mlx5_action_handle_query, 927 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 928 .tunnel_match = mlx5_flow_tunnel_match, 929 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 930 .tunnel_item_release = mlx5_flow_tunnel_item_release, 931 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 932 .flex_item_create = mlx5_flow_flex_item_create, 933 .flex_item_release = mlx5_flow_flex_item_release, 934 .info_get = mlx5_flow_info_get, 935 .configure = mlx5_flow_port_configure, 936 .pattern_template_create = mlx5_flow_pattern_template_create, 937 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 938 .actions_template_create = mlx5_flow_actions_template_create, 939 .actions_template_destroy = mlx5_flow_actions_template_destroy, 940 .template_table_create = mlx5_flow_table_create, 941 .template_table_destroy = mlx5_flow_table_destroy, 942 .async_create = mlx5_flow_async_flow_create, 943 .async_destroy = mlx5_flow_async_flow_destroy, 944 .pull = mlx5_flow_pull, 945 .push = mlx5_flow_push, 946 .async_action_handle_create = mlx5_flow_async_action_handle_create, 947 .async_action_handle_update = mlx5_flow_async_action_handle_update, 948 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 949 }; 950 951 /* Tunnel information. */ 952 struct mlx5_flow_tunnel_info { 953 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 954 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 955 }; 956 957 static struct mlx5_flow_tunnel_info tunnels_info[] = { 958 { 959 .tunnel = MLX5_FLOW_LAYER_VXLAN, 960 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 961 }, 962 { 963 .tunnel = MLX5_FLOW_LAYER_GENEVE, 964 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 965 }, 966 { 967 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 968 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 969 }, 970 { 971 .tunnel = MLX5_FLOW_LAYER_GRE, 972 .ptype = RTE_PTYPE_TUNNEL_GRE, 973 }, 974 { 975 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 976 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 977 }, 978 { 979 .tunnel = MLX5_FLOW_LAYER_MPLS, 980 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 981 }, 982 { 983 .tunnel = MLX5_FLOW_LAYER_NVGRE, 984 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 985 }, 986 { 987 .tunnel = MLX5_FLOW_LAYER_IPIP, 988 .ptype = RTE_PTYPE_TUNNEL_IP, 989 }, 990 { 991 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 992 .ptype = RTE_PTYPE_TUNNEL_IP, 993 }, 994 { 995 .tunnel = MLX5_FLOW_LAYER_GTP, 996 .ptype = RTE_PTYPE_TUNNEL_GTPU, 997 }, 998 }; 999 1000 1001 1002 /** 1003 * Translate tag ID to register. 1004 * 1005 * @param[in] dev 1006 * Pointer to the Ethernet device structure. 1007 * @param[in] feature 1008 * The feature that request the register. 1009 * @param[in] id 1010 * The request register ID. 1011 * @param[out] error 1012 * Error description in case of any. 1013 * 1014 * @return 1015 * The request register on success, a negative errno 1016 * value otherwise and rte_errno is set. 1017 */ 1018 int 1019 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1020 enum mlx5_feature_name feature, 1021 uint32_t id, 1022 struct rte_flow_error *error) 1023 { 1024 struct mlx5_priv *priv = dev->data->dev_private; 1025 struct mlx5_sh_config *config = &priv->sh->config; 1026 enum modify_reg start_reg; 1027 bool skip_mtr_reg = false; 1028 1029 switch (feature) { 1030 case MLX5_HAIRPIN_RX: 1031 return REG_B; 1032 case MLX5_HAIRPIN_TX: 1033 return REG_A; 1034 case MLX5_METADATA_RX: 1035 switch (config->dv_xmeta_en) { 1036 case MLX5_XMETA_MODE_LEGACY: 1037 return REG_B; 1038 case MLX5_XMETA_MODE_META16: 1039 return REG_C_0; 1040 case MLX5_XMETA_MODE_META32: 1041 return REG_C_1; 1042 } 1043 break; 1044 case MLX5_METADATA_TX: 1045 return REG_A; 1046 case MLX5_METADATA_FDB: 1047 switch (config->dv_xmeta_en) { 1048 case MLX5_XMETA_MODE_LEGACY: 1049 return REG_NON; 1050 case MLX5_XMETA_MODE_META16: 1051 return REG_C_0; 1052 case MLX5_XMETA_MODE_META32: 1053 return REG_C_1; 1054 } 1055 break; 1056 case MLX5_FLOW_MARK: 1057 switch (config->dv_xmeta_en) { 1058 case MLX5_XMETA_MODE_LEGACY: 1059 return REG_NON; 1060 case MLX5_XMETA_MODE_META16: 1061 return REG_C_1; 1062 case MLX5_XMETA_MODE_META32: 1063 return REG_C_0; 1064 } 1065 break; 1066 case MLX5_MTR_ID: 1067 /* 1068 * If meter color and meter id share one register, flow match 1069 * should use the meter color register for match. 1070 */ 1071 if (priv->mtr_reg_share) 1072 return priv->mtr_color_reg; 1073 else 1074 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1075 REG_C_3; 1076 case MLX5_MTR_COLOR: 1077 case MLX5_ASO_FLOW_HIT: 1078 case MLX5_ASO_CONNTRACK: 1079 case MLX5_SAMPLE_ID: 1080 /* All features use the same REG_C. */ 1081 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1082 return priv->mtr_color_reg; 1083 case MLX5_COPY_MARK: 1084 /* 1085 * Metadata COPY_MARK register using is in meter suffix sub 1086 * flow while with meter. It's safe to share the same register. 1087 */ 1088 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1089 case MLX5_APP_TAG: 1090 /* 1091 * If meter is enable, it will engage the register for color 1092 * match and flow match. If meter color match is not using the 1093 * REG_C_2, need to skip the REG_C_x be used by meter color 1094 * match. 1095 * If meter is disable, free to use all available registers. 1096 */ 1097 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1098 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1099 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1100 if (id > (uint32_t)(REG_C_7 - start_reg)) 1101 return rte_flow_error_set(error, EINVAL, 1102 RTE_FLOW_ERROR_TYPE_ITEM, 1103 NULL, "invalid tag id"); 1104 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1105 return rte_flow_error_set(error, ENOTSUP, 1106 RTE_FLOW_ERROR_TYPE_ITEM, 1107 NULL, "unsupported tag id"); 1108 /* 1109 * This case means meter is using the REG_C_x great than 2. 1110 * Take care not to conflict with meter color REG_C_x. 1111 * If the available index REG_C_y >= REG_C_x, skip the 1112 * color register. 1113 */ 1114 if (skip_mtr_reg && priv->sh->flow_mreg_c 1115 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1116 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1117 return rte_flow_error_set(error, EINVAL, 1118 RTE_FLOW_ERROR_TYPE_ITEM, 1119 NULL, "invalid tag id"); 1120 if (priv->sh->flow_mreg_c 1121 [id + 1 + start_reg - REG_C_0] != REG_NON) 1122 return priv->sh->flow_mreg_c 1123 [id + 1 + start_reg - REG_C_0]; 1124 return rte_flow_error_set(error, ENOTSUP, 1125 RTE_FLOW_ERROR_TYPE_ITEM, 1126 NULL, "unsupported tag id"); 1127 } 1128 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1129 } 1130 MLX5_ASSERT(false); 1131 return rte_flow_error_set(error, EINVAL, 1132 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1133 NULL, "invalid feature name"); 1134 } 1135 1136 /** 1137 * Check extensive flow metadata register support. 1138 * 1139 * @param dev 1140 * Pointer to rte_eth_dev structure. 1141 * 1142 * @return 1143 * True if device supports extensive flow metadata register, otherwise false. 1144 */ 1145 bool 1146 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1147 { 1148 struct mlx5_priv *priv = dev->data->dev_private; 1149 1150 /* 1151 * Having available reg_c can be regarded inclusively as supporting 1152 * extensive flow metadata register, which could mean, 1153 * - metadata register copy action by modify header. 1154 * - 16 modify header actions is supported. 1155 * - reg_c's are preserved across different domain (FDB and NIC) on 1156 * packet loopback by flow lookup miss. 1157 */ 1158 return priv->sh->flow_mreg_c[2] != REG_NON; 1159 } 1160 1161 /** 1162 * Get the lowest priority. 1163 * 1164 * @param[in] dev 1165 * Pointer to the Ethernet device structure. 1166 * @param[in] attributes 1167 * Pointer to device flow rule attributes. 1168 * 1169 * @return 1170 * The value of lowest priority of flow. 1171 */ 1172 uint32_t 1173 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1174 const struct rte_flow_attr *attr) 1175 { 1176 struct mlx5_priv *priv = dev->data->dev_private; 1177 1178 if (!attr->group && !attr->transfer) 1179 return priv->sh->flow_max_priority - 2; 1180 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1181 } 1182 1183 /** 1184 * Calculate matcher priority of the flow. 1185 * 1186 * @param[in] dev 1187 * Pointer to the Ethernet device structure. 1188 * @param[in] attr 1189 * Pointer to device flow rule attributes. 1190 * @param[in] subpriority 1191 * The priority based on the items. 1192 * @param[in] external 1193 * Flow is user flow. 1194 * @return 1195 * The matcher priority of the flow. 1196 */ 1197 uint16_t 1198 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1199 const struct rte_flow_attr *attr, 1200 uint32_t subpriority, bool external) 1201 { 1202 uint16_t priority = (uint16_t)attr->priority; 1203 struct mlx5_priv *priv = dev->data->dev_private; 1204 1205 if (!attr->group && !attr->transfer) { 1206 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1207 priority = priv->sh->flow_max_priority - 1; 1208 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1209 } else if (!external && attr->transfer && attr->group == 0 && 1210 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1211 return (priv->sh->flow_max_priority - 1) * 3; 1212 } 1213 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1214 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1215 return priority * 3 + subpriority; 1216 } 1217 1218 /** 1219 * Verify the @p item specifications (spec, last, mask) are compatible with the 1220 * NIC capabilities. 1221 * 1222 * @param[in] item 1223 * Item specification. 1224 * @param[in] mask 1225 * @p item->mask or flow default bit-masks. 1226 * @param[in] nic_mask 1227 * Bit-masks covering supported fields by the NIC to compare with user mask. 1228 * @param[in] size 1229 * Bit-masks size in bytes. 1230 * @param[in] range_accepted 1231 * True if range of values is accepted for specific fields, false otherwise. 1232 * @param[out] error 1233 * Pointer to error structure. 1234 * 1235 * @return 1236 * 0 on success, a negative errno value otherwise and rte_errno is set. 1237 */ 1238 int 1239 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1240 const uint8_t *mask, 1241 const uint8_t *nic_mask, 1242 unsigned int size, 1243 bool range_accepted, 1244 struct rte_flow_error *error) 1245 { 1246 unsigned int i; 1247 1248 MLX5_ASSERT(nic_mask); 1249 for (i = 0; i < size; ++i) 1250 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1251 return rte_flow_error_set(error, ENOTSUP, 1252 RTE_FLOW_ERROR_TYPE_ITEM, 1253 item, 1254 "mask enables non supported" 1255 " bits"); 1256 if (!item->spec && (item->mask || item->last)) 1257 return rte_flow_error_set(error, EINVAL, 1258 RTE_FLOW_ERROR_TYPE_ITEM, item, 1259 "mask/last without a spec is not" 1260 " supported"); 1261 if (item->spec && item->last && !range_accepted) { 1262 uint8_t spec[size]; 1263 uint8_t last[size]; 1264 unsigned int i; 1265 int ret; 1266 1267 for (i = 0; i < size; ++i) { 1268 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1269 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1270 } 1271 ret = memcmp(spec, last, size); 1272 if (ret != 0) 1273 return rte_flow_error_set(error, EINVAL, 1274 RTE_FLOW_ERROR_TYPE_ITEM, 1275 item, 1276 "range is not valid"); 1277 } 1278 return 0; 1279 } 1280 1281 /** 1282 * Adjust the hash fields according to the @p flow information. 1283 * 1284 * @param[in] dev_flow. 1285 * Pointer to the mlx5_flow. 1286 * @param[in] tunnel 1287 * 1 when the hash field is for a tunnel item. 1288 * @param[in] layer_types 1289 * RTE_ETH_RSS_* types. 1290 * @param[in] hash_fields 1291 * Item hash fields. 1292 * 1293 * @return 1294 * The hash fields that should be used. 1295 */ 1296 uint64_t 1297 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1298 int tunnel __rte_unused, uint64_t layer_types, 1299 uint64_t hash_fields) 1300 { 1301 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1302 int rss_request_inner = rss_desc->level >= 2; 1303 1304 /* Check RSS hash level for tunnel. */ 1305 if (tunnel && rss_request_inner) 1306 hash_fields |= IBV_RX_HASH_INNER; 1307 else if (tunnel || rss_request_inner) 1308 return 0; 1309 #endif 1310 /* Check if requested layer matches RSS hash fields. */ 1311 if (!(rss_desc->types & layer_types)) 1312 return 0; 1313 return hash_fields; 1314 } 1315 1316 /** 1317 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1318 * if several tunnel rules are used on this queue, the tunnel ptype will be 1319 * cleared. 1320 * 1321 * @param rxq_ctrl 1322 * Rx queue to update. 1323 */ 1324 static void 1325 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1326 { 1327 unsigned int i; 1328 uint32_t tunnel_ptype = 0; 1329 1330 /* Look up for the ptype to use. */ 1331 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1332 if (!rxq_ctrl->flow_tunnels_n[i]) 1333 continue; 1334 if (!tunnel_ptype) { 1335 tunnel_ptype = tunnels_info[i].ptype; 1336 } else { 1337 tunnel_ptype = 0; 1338 break; 1339 } 1340 } 1341 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1342 } 1343 1344 /** 1345 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1346 * flow. 1347 * 1348 * @param[in] dev 1349 * Pointer to the Ethernet device structure. 1350 * @param[in] dev_handle 1351 * Pointer to device flow handle structure. 1352 */ 1353 void 1354 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1355 struct mlx5_flow_handle *dev_handle) 1356 { 1357 struct mlx5_priv *priv = dev->data->dev_private; 1358 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1359 struct mlx5_ind_table_obj *ind_tbl = NULL; 1360 unsigned int i; 1361 1362 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1363 struct mlx5_hrxq *hrxq; 1364 1365 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1366 dev_handle->rix_hrxq); 1367 if (hrxq) 1368 ind_tbl = hrxq->ind_table; 1369 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1370 struct mlx5_shared_action_rss *shared_rss; 1371 1372 shared_rss = mlx5_ipool_get 1373 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1374 dev_handle->rix_srss); 1375 if (shared_rss) 1376 ind_tbl = shared_rss->ind_tbl; 1377 } 1378 if (!ind_tbl) 1379 return; 1380 for (i = 0; i != ind_tbl->queues_n; ++i) { 1381 int idx = ind_tbl->queues[i]; 1382 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1383 1384 MLX5_ASSERT(rxq_ctrl != NULL); 1385 if (rxq_ctrl == NULL) 1386 continue; 1387 /* 1388 * To support metadata register copy on Tx loopback, 1389 * this must be always enabled (metadata may arive 1390 * from other port - not from local flows only. 1391 */ 1392 if (tunnel) { 1393 unsigned int j; 1394 1395 /* Increase the counter matching the flow. */ 1396 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1397 if ((tunnels_info[j].tunnel & 1398 dev_handle->layers) == 1399 tunnels_info[j].tunnel) { 1400 rxq_ctrl->flow_tunnels_n[j]++; 1401 break; 1402 } 1403 } 1404 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1405 } 1406 } 1407 } 1408 1409 static void 1410 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1411 { 1412 struct mlx5_priv *priv = dev->data->dev_private; 1413 struct mlx5_rxq_ctrl *rxq_ctrl; 1414 1415 if (priv->mark_enabled) 1416 return; 1417 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1418 rxq_ctrl->rxq.mark = 1; 1419 } 1420 priv->mark_enabled = 1; 1421 } 1422 1423 /** 1424 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1425 * 1426 * @param[in] dev 1427 * Pointer to the Ethernet device structure. 1428 * @param[in] flow 1429 * Pointer to flow structure. 1430 */ 1431 static void 1432 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1433 { 1434 struct mlx5_priv *priv = dev->data->dev_private; 1435 uint32_t handle_idx; 1436 struct mlx5_flow_handle *dev_handle; 1437 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1438 1439 MLX5_ASSERT(wks); 1440 if (wks->mark) 1441 flow_rxq_mark_flag_set(dev); 1442 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1443 handle_idx, dev_handle, next) 1444 flow_drv_rxq_flags_set(dev, dev_handle); 1445 } 1446 1447 /** 1448 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1449 * device flow if no other flow uses it with the same kind of request. 1450 * 1451 * @param dev 1452 * Pointer to Ethernet device. 1453 * @param[in] dev_handle 1454 * Pointer to the device flow handle structure. 1455 */ 1456 static void 1457 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1458 struct mlx5_flow_handle *dev_handle) 1459 { 1460 struct mlx5_priv *priv = dev->data->dev_private; 1461 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1462 struct mlx5_ind_table_obj *ind_tbl = NULL; 1463 unsigned int i; 1464 1465 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1466 struct mlx5_hrxq *hrxq; 1467 1468 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1469 dev_handle->rix_hrxq); 1470 if (hrxq) 1471 ind_tbl = hrxq->ind_table; 1472 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1473 struct mlx5_shared_action_rss *shared_rss; 1474 1475 shared_rss = mlx5_ipool_get 1476 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1477 dev_handle->rix_srss); 1478 if (shared_rss) 1479 ind_tbl = shared_rss->ind_tbl; 1480 } 1481 if (!ind_tbl) 1482 return; 1483 MLX5_ASSERT(dev->data->dev_started); 1484 for (i = 0; i != ind_tbl->queues_n; ++i) { 1485 int idx = ind_tbl->queues[i]; 1486 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1487 1488 MLX5_ASSERT(rxq_ctrl != NULL); 1489 if (rxq_ctrl == NULL) 1490 continue; 1491 if (tunnel) { 1492 unsigned int j; 1493 1494 /* Decrease the counter matching the flow. */ 1495 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1496 if ((tunnels_info[j].tunnel & 1497 dev_handle->layers) == 1498 tunnels_info[j].tunnel) { 1499 rxq_ctrl->flow_tunnels_n[j]--; 1500 break; 1501 } 1502 } 1503 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1504 } 1505 } 1506 } 1507 1508 /** 1509 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1510 * @p flow if no other flow uses it with the same kind of request. 1511 * 1512 * @param dev 1513 * Pointer to Ethernet device. 1514 * @param[in] flow 1515 * Pointer to the flow. 1516 */ 1517 static void 1518 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1519 { 1520 struct mlx5_priv *priv = dev->data->dev_private; 1521 uint32_t handle_idx; 1522 struct mlx5_flow_handle *dev_handle; 1523 1524 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1525 handle_idx, dev_handle, next) 1526 flow_drv_rxq_flags_trim(dev, dev_handle); 1527 } 1528 1529 /** 1530 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1531 * 1532 * @param dev 1533 * Pointer to Ethernet device. 1534 */ 1535 static void 1536 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1537 { 1538 struct mlx5_priv *priv = dev->data->dev_private; 1539 unsigned int i; 1540 1541 for (i = 0; i != priv->rxqs_n; ++i) { 1542 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1543 unsigned int j; 1544 1545 if (rxq == NULL || rxq->ctrl == NULL) 1546 continue; 1547 rxq->ctrl->rxq.mark = 0; 1548 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1549 rxq->ctrl->flow_tunnels_n[j] = 0; 1550 rxq->ctrl->rxq.tunnel = 0; 1551 } 1552 priv->mark_enabled = 0; 1553 } 1554 1555 /** 1556 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1557 * 1558 * @param[in] dev 1559 * Pointer to the Ethernet device structure. 1560 */ 1561 void 1562 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1563 { 1564 struct mlx5_priv *priv = dev->data->dev_private; 1565 unsigned int i; 1566 1567 for (i = 0; i != priv->rxqs_n; ++i) { 1568 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1569 struct mlx5_rxq_data *data; 1570 1571 if (rxq == NULL || rxq->ctrl == NULL) 1572 continue; 1573 data = &rxq->ctrl->rxq; 1574 if (!rte_flow_dynf_metadata_avail()) { 1575 data->dynf_meta = 0; 1576 data->flow_meta_mask = 0; 1577 data->flow_meta_offset = -1; 1578 data->flow_meta_port_mask = 0; 1579 } else { 1580 data->dynf_meta = 1; 1581 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1582 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1583 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1584 } 1585 } 1586 } 1587 1588 /* 1589 * return a pointer to the desired action in the list of actions. 1590 * 1591 * @param[in] actions 1592 * The list of actions to search the action in. 1593 * @param[in] action 1594 * The action to find. 1595 * 1596 * @return 1597 * Pointer to the action in the list, if found. NULL otherwise. 1598 */ 1599 const struct rte_flow_action * 1600 mlx5_flow_find_action(const struct rte_flow_action *actions, 1601 enum rte_flow_action_type action) 1602 { 1603 if (actions == NULL) 1604 return NULL; 1605 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1606 if (actions->type == action) 1607 return actions; 1608 return NULL; 1609 } 1610 1611 /* 1612 * Validate the flag action. 1613 * 1614 * @param[in] action_flags 1615 * Bit-fields that holds the actions detected until now. 1616 * @param[in] attr 1617 * Attributes of flow that includes this action. 1618 * @param[out] error 1619 * Pointer to error structure. 1620 * 1621 * @return 1622 * 0 on success, a negative errno value otherwise and rte_errno is set. 1623 */ 1624 int 1625 mlx5_flow_validate_action_flag(uint64_t action_flags, 1626 const struct rte_flow_attr *attr, 1627 struct rte_flow_error *error) 1628 { 1629 if (action_flags & MLX5_FLOW_ACTION_MARK) 1630 return rte_flow_error_set(error, EINVAL, 1631 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1632 "can't mark and flag in same flow"); 1633 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1634 return rte_flow_error_set(error, EINVAL, 1635 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1636 "can't have 2 flag" 1637 " actions in same flow"); 1638 if (attr->egress) 1639 return rte_flow_error_set(error, ENOTSUP, 1640 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1641 "flag action not supported for " 1642 "egress"); 1643 return 0; 1644 } 1645 1646 /* 1647 * Validate the mark action. 1648 * 1649 * @param[in] action 1650 * Pointer to the queue action. 1651 * @param[in] action_flags 1652 * Bit-fields that holds the actions detected until now. 1653 * @param[in] attr 1654 * Attributes of flow that includes this action. 1655 * @param[out] error 1656 * Pointer to error structure. 1657 * 1658 * @return 1659 * 0 on success, a negative errno value otherwise and rte_errno is set. 1660 */ 1661 int 1662 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1663 uint64_t action_flags, 1664 const struct rte_flow_attr *attr, 1665 struct rte_flow_error *error) 1666 { 1667 const struct rte_flow_action_mark *mark = action->conf; 1668 1669 if (!mark) 1670 return rte_flow_error_set(error, EINVAL, 1671 RTE_FLOW_ERROR_TYPE_ACTION, 1672 action, 1673 "configuration cannot be null"); 1674 if (mark->id >= MLX5_FLOW_MARK_MAX) 1675 return rte_flow_error_set(error, EINVAL, 1676 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1677 &mark->id, 1678 "mark id must in 0 <= id < " 1679 RTE_STR(MLX5_FLOW_MARK_MAX)); 1680 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1681 return rte_flow_error_set(error, EINVAL, 1682 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1683 "can't flag and mark in same flow"); 1684 if (action_flags & MLX5_FLOW_ACTION_MARK) 1685 return rte_flow_error_set(error, EINVAL, 1686 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1687 "can't have 2 mark actions in same" 1688 " flow"); 1689 if (attr->egress) 1690 return rte_flow_error_set(error, ENOTSUP, 1691 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1692 "mark action not supported for " 1693 "egress"); 1694 return 0; 1695 } 1696 1697 /* 1698 * Validate the drop action. 1699 * 1700 * @param[in] action_flags 1701 * Bit-fields that holds the actions detected until now. 1702 * @param[in] attr 1703 * Attributes of flow that includes this action. 1704 * @param[out] error 1705 * Pointer to error structure. 1706 * 1707 * @return 1708 * 0 on success, a negative errno value otherwise and rte_errno is set. 1709 */ 1710 int 1711 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1712 const struct rte_flow_attr *attr, 1713 struct rte_flow_error *error) 1714 { 1715 if (attr->egress) 1716 return rte_flow_error_set(error, ENOTSUP, 1717 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1718 "drop action not supported for " 1719 "egress"); 1720 return 0; 1721 } 1722 1723 /* 1724 * Validate the queue action. 1725 * 1726 * @param[in] action 1727 * Pointer to the queue action. 1728 * @param[in] action_flags 1729 * Bit-fields that holds the actions detected until now. 1730 * @param[in] dev 1731 * Pointer to the Ethernet device structure. 1732 * @param[in] attr 1733 * Attributes of flow that includes this action. 1734 * @param[out] error 1735 * Pointer to error structure. 1736 * 1737 * @return 1738 * 0 on success, a negative errno value otherwise and rte_errno is set. 1739 */ 1740 int 1741 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1742 uint64_t action_flags, 1743 struct rte_eth_dev *dev, 1744 const struct rte_flow_attr *attr, 1745 struct rte_flow_error *error) 1746 { 1747 struct mlx5_priv *priv = dev->data->dev_private; 1748 const struct rte_flow_action_queue *queue = action->conf; 1749 1750 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1751 return rte_flow_error_set(error, EINVAL, 1752 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1753 "can't have 2 fate actions in" 1754 " same flow"); 1755 if (attr->egress) 1756 return rte_flow_error_set(error, ENOTSUP, 1757 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1758 "queue action not supported for egress."); 1759 if (mlx5_is_external_rxq(dev, queue->index)) 1760 return 0; 1761 if (!priv->rxqs_n) 1762 return rte_flow_error_set(error, EINVAL, 1763 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1764 NULL, "No Rx queues configured"); 1765 if (queue->index >= priv->rxqs_n) 1766 return rte_flow_error_set(error, EINVAL, 1767 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1768 &queue->index, 1769 "queue index out of range"); 1770 if (mlx5_rxq_get(dev, queue->index) == NULL) 1771 return rte_flow_error_set(error, EINVAL, 1772 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1773 &queue->index, 1774 "queue is not configured"); 1775 return 0; 1776 } 1777 1778 /** 1779 * Validate queue numbers for device RSS. 1780 * 1781 * @param[in] dev 1782 * Configured device. 1783 * @param[in] queues 1784 * Array of queue numbers. 1785 * @param[in] queues_n 1786 * Size of the @p queues array. 1787 * @param[out] error 1788 * On error, filled with a textual error description. 1789 * @param[out] queue_idx 1790 * On error, filled with an offending queue index in @p queues array. 1791 * 1792 * @return 1793 * 0 on success, a negative errno code on error. 1794 */ 1795 static int 1796 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1797 const uint16_t *queues, uint32_t queues_n, 1798 const char **error, uint32_t *queue_idx) 1799 { 1800 const struct mlx5_priv *priv = dev->data->dev_private; 1801 bool is_hairpin = false; 1802 bool is_ext_rss = false; 1803 uint32_t i; 1804 1805 for (i = 0; i != queues_n; ++i) { 1806 struct mlx5_rxq_ctrl *rxq_ctrl; 1807 1808 if (mlx5_is_external_rxq(dev, queues[0])) { 1809 is_ext_rss = true; 1810 continue; 1811 } 1812 if (is_ext_rss) { 1813 *error = "Combining external and regular RSS queues is not supported"; 1814 *queue_idx = i; 1815 return -ENOTSUP; 1816 } 1817 if (queues[i] >= priv->rxqs_n) { 1818 *error = "queue index out of range"; 1819 *queue_idx = i; 1820 return -EINVAL; 1821 } 1822 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 1823 if (rxq_ctrl == NULL) { 1824 *error = "queue is not configured"; 1825 *queue_idx = i; 1826 return -EINVAL; 1827 } 1828 if (i == 0 && rxq_ctrl->is_hairpin) 1829 is_hairpin = true; 1830 if (is_hairpin != rxq_ctrl->is_hairpin) { 1831 *error = "combining hairpin and regular RSS queues is not supported"; 1832 *queue_idx = i; 1833 return -ENOTSUP; 1834 } 1835 } 1836 return 0; 1837 } 1838 1839 /* 1840 * Validate the rss action. 1841 * 1842 * @param[in] dev 1843 * Pointer to the Ethernet device structure. 1844 * @param[in] action 1845 * Pointer to the queue action. 1846 * @param[out] error 1847 * Pointer to error structure. 1848 * 1849 * @return 1850 * 0 on success, a negative errno value otherwise and rte_errno is set. 1851 */ 1852 int 1853 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1854 const struct rte_flow_action *action, 1855 struct rte_flow_error *error) 1856 { 1857 struct mlx5_priv *priv = dev->data->dev_private; 1858 const struct rte_flow_action_rss *rss = action->conf; 1859 int ret; 1860 const char *message; 1861 uint32_t queue_idx; 1862 1863 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1864 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1865 return rte_flow_error_set(error, ENOTSUP, 1866 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1867 &rss->func, 1868 "RSS hash function not supported"); 1869 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1870 if (rss->level > 2) 1871 #else 1872 if (rss->level > 1) 1873 #endif 1874 return rte_flow_error_set(error, ENOTSUP, 1875 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1876 &rss->level, 1877 "tunnel RSS is not supported"); 1878 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1879 if (rss->key_len == 0 && rss->key != NULL) 1880 return rte_flow_error_set(error, ENOTSUP, 1881 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1882 &rss->key_len, 1883 "RSS hash key length 0"); 1884 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1885 return rte_flow_error_set(error, ENOTSUP, 1886 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1887 &rss->key_len, 1888 "RSS hash key too small"); 1889 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1890 return rte_flow_error_set(error, ENOTSUP, 1891 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1892 &rss->key_len, 1893 "RSS hash key too large"); 1894 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1895 return rte_flow_error_set(error, ENOTSUP, 1896 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1897 &rss->queue_num, 1898 "number of queues too large"); 1899 if (rss->types & MLX5_RSS_HF_MASK) 1900 return rte_flow_error_set(error, ENOTSUP, 1901 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1902 &rss->types, 1903 "some RSS protocols are not" 1904 " supported"); 1905 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1906 !(rss->types & RTE_ETH_RSS_IP)) 1907 return rte_flow_error_set(error, EINVAL, 1908 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1909 "L3 partial RSS requested but L3 RSS" 1910 " type not specified"); 1911 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1912 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1913 return rte_flow_error_set(error, EINVAL, 1914 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1915 "L4 partial RSS requested but L4 RSS" 1916 " type not specified"); 1917 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 1918 return rte_flow_error_set(error, EINVAL, 1919 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1920 NULL, "No Rx queues configured"); 1921 if (!rss->queue_num) 1922 return rte_flow_error_set(error, EINVAL, 1923 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1924 NULL, "No queues configured"); 1925 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1926 &message, &queue_idx); 1927 if (ret != 0) { 1928 return rte_flow_error_set(error, -ret, 1929 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1930 &rss->queue[queue_idx], message); 1931 } 1932 return 0; 1933 } 1934 1935 /* 1936 * Validate the rss action. 1937 * 1938 * @param[in] action 1939 * Pointer to the queue action. 1940 * @param[in] action_flags 1941 * Bit-fields that holds the actions detected until now. 1942 * @param[in] dev 1943 * Pointer to the Ethernet device structure. 1944 * @param[in] attr 1945 * Attributes of flow that includes this action. 1946 * @param[in] item_flags 1947 * Items that were detected. 1948 * @param[out] error 1949 * Pointer to error structure. 1950 * 1951 * @return 1952 * 0 on success, a negative errno value otherwise and rte_errno is set. 1953 */ 1954 int 1955 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1956 uint64_t action_flags, 1957 struct rte_eth_dev *dev, 1958 const struct rte_flow_attr *attr, 1959 uint64_t item_flags, 1960 struct rte_flow_error *error) 1961 { 1962 const struct rte_flow_action_rss *rss = action->conf; 1963 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1964 int ret; 1965 1966 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1967 return rte_flow_error_set(error, EINVAL, 1968 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1969 "can't have 2 fate actions" 1970 " in same flow"); 1971 ret = mlx5_validate_action_rss(dev, action, error); 1972 if (ret) 1973 return ret; 1974 if (attr->egress) 1975 return rte_flow_error_set(error, ENOTSUP, 1976 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1977 "rss action not supported for " 1978 "egress"); 1979 if (rss->level > 1 && !tunnel) 1980 return rte_flow_error_set(error, EINVAL, 1981 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1982 "inner RSS is not supported for " 1983 "non-tunnel flows"); 1984 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1985 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1986 return rte_flow_error_set(error, EINVAL, 1987 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1988 "RSS on eCPRI is not supported now"); 1989 } 1990 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1991 !(item_flags & 1992 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1993 rss->level > 1) 1994 return rte_flow_error_set(error, EINVAL, 1995 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1996 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1997 return 0; 1998 } 1999 2000 /* 2001 * Validate the default miss action. 2002 * 2003 * @param[in] action_flags 2004 * Bit-fields that holds the actions detected until now. 2005 * @param[out] error 2006 * Pointer to error structure. 2007 * 2008 * @return 2009 * 0 on success, a negative errno value otherwise and rte_errno is set. 2010 */ 2011 int 2012 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2013 const struct rte_flow_attr *attr, 2014 struct rte_flow_error *error) 2015 { 2016 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2017 return rte_flow_error_set(error, EINVAL, 2018 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2019 "can't have 2 fate actions in" 2020 " same flow"); 2021 if (attr->egress) 2022 return rte_flow_error_set(error, ENOTSUP, 2023 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2024 "default miss action not supported " 2025 "for egress"); 2026 if (attr->group) 2027 return rte_flow_error_set(error, ENOTSUP, 2028 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2029 "only group 0 is supported"); 2030 if (attr->transfer) 2031 return rte_flow_error_set(error, ENOTSUP, 2032 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2033 NULL, "transfer is not supported"); 2034 return 0; 2035 } 2036 2037 /* 2038 * Validate the count action. 2039 * 2040 * @param[in] dev 2041 * Pointer to the Ethernet device structure. 2042 * @param[in] attr 2043 * Attributes of flow that includes this action. 2044 * @param[out] error 2045 * Pointer to error structure. 2046 * 2047 * @return 2048 * 0 on success, a negative errno value otherwise and rte_errno is set. 2049 */ 2050 int 2051 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2052 const struct rte_flow_attr *attr, 2053 struct rte_flow_error *error) 2054 { 2055 if (attr->egress) 2056 return rte_flow_error_set(error, ENOTSUP, 2057 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2058 "count action not supported for " 2059 "egress"); 2060 return 0; 2061 } 2062 2063 /* 2064 * Validate the ASO CT action. 2065 * 2066 * @param[in] dev 2067 * Pointer to the Ethernet device structure. 2068 * @param[in] conntrack 2069 * Pointer to the CT action profile. 2070 * @param[out] error 2071 * Pointer to error structure. 2072 * 2073 * @return 2074 * 0 on success, a negative errno value otherwise and rte_errno is set. 2075 */ 2076 int 2077 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2078 const struct rte_flow_action_conntrack *conntrack, 2079 struct rte_flow_error *error) 2080 { 2081 RTE_SET_USED(dev); 2082 2083 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2084 return rte_flow_error_set(error, EINVAL, 2085 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2086 "Invalid CT state"); 2087 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2088 return rte_flow_error_set(error, EINVAL, 2089 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2090 "Invalid last TCP packet flag"); 2091 return 0; 2092 } 2093 2094 /** 2095 * Verify the @p attributes will be correctly understood by the NIC and store 2096 * them in the @p flow if everything is correct. 2097 * 2098 * @param[in] dev 2099 * Pointer to the Ethernet device structure. 2100 * @param[in] attributes 2101 * Pointer to flow attributes 2102 * @param[out] error 2103 * Pointer to error structure. 2104 * 2105 * @return 2106 * 0 on success, a negative errno value otherwise and rte_errno is set. 2107 */ 2108 int 2109 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2110 const struct rte_flow_attr *attributes, 2111 struct rte_flow_error *error) 2112 { 2113 struct mlx5_priv *priv = dev->data->dev_private; 2114 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2115 2116 if (attributes->group) 2117 return rte_flow_error_set(error, ENOTSUP, 2118 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2119 NULL, "groups is not supported"); 2120 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2121 attributes->priority >= priority_max) 2122 return rte_flow_error_set(error, ENOTSUP, 2123 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2124 NULL, "priority out of range"); 2125 if (attributes->egress) 2126 return rte_flow_error_set(error, ENOTSUP, 2127 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2128 "egress is not supported"); 2129 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2130 return rte_flow_error_set(error, ENOTSUP, 2131 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2132 NULL, "transfer is not supported"); 2133 if (!attributes->ingress) 2134 return rte_flow_error_set(error, EINVAL, 2135 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2136 NULL, 2137 "ingress attribute is mandatory"); 2138 return 0; 2139 } 2140 2141 /** 2142 * Validate ICMP6 item. 2143 * 2144 * @param[in] item 2145 * Item specification. 2146 * @param[in] item_flags 2147 * Bit-fields that holds the items detected until now. 2148 * @param[in] ext_vlan_sup 2149 * Whether extended VLAN features are supported or not. 2150 * @param[out] error 2151 * Pointer to error structure. 2152 * 2153 * @return 2154 * 0 on success, a negative errno value otherwise and rte_errno is set. 2155 */ 2156 int 2157 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2158 uint64_t item_flags, 2159 uint8_t target_protocol, 2160 struct rte_flow_error *error) 2161 { 2162 const struct rte_flow_item_icmp6 *mask = item->mask; 2163 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2164 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2165 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2166 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2167 MLX5_FLOW_LAYER_OUTER_L4; 2168 int ret; 2169 2170 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2171 return rte_flow_error_set(error, EINVAL, 2172 RTE_FLOW_ERROR_TYPE_ITEM, item, 2173 "protocol filtering not compatible" 2174 " with ICMP6 layer"); 2175 if (!(item_flags & l3m)) 2176 return rte_flow_error_set(error, EINVAL, 2177 RTE_FLOW_ERROR_TYPE_ITEM, item, 2178 "IPv6 is mandatory to filter on" 2179 " ICMP6"); 2180 if (item_flags & l4m) 2181 return rte_flow_error_set(error, EINVAL, 2182 RTE_FLOW_ERROR_TYPE_ITEM, item, 2183 "multiple L4 layers not supported"); 2184 if (!mask) 2185 mask = &rte_flow_item_icmp6_mask; 2186 ret = mlx5_flow_item_acceptable 2187 (item, (const uint8_t *)mask, 2188 (const uint8_t *)&rte_flow_item_icmp6_mask, 2189 sizeof(struct rte_flow_item_icmp6), 2190 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2191 if (ret < 0) 2192 return ret; 2193 return 0; 2194 } 2195 2196 /** 2197 * Validate ICMP item. 2198 * 2199 * @param[in] item 2200 * Item specification. 2201 * @param[in] item_flags 2202 * Bit-fields that holds the items detected until now. 2203 * @param[out] error 2204 * Pointer to error structure. 2205 * 2206 * @return 2207 * 0 on success, a negative errno value otherwise and rte_errno is set. 2208 */ 2209 int 2210 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2211 uint64_t item_flags, 2212 uint8_t target_protocol, 2213 struct rte_flow_error *error) 2214 { 2215 const struct rte_flow_item_icmp *mask = item->mask; 2216 const struct rte_flow_item_icmp nic_mask = { 2217 .hdr.icmp_type = 0xff, 2218 .hdr.icmp_code = 0xff, 2219 .hdr.icmp_ident = RTE_BE16(0xffff), 2220 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2221 }; 2222 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2223 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2224 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2225 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2226 MLX5_FLOW_LAYER_OUTER_L4; 2227 int ret; 2228 2229 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2230 return rte_flow_error_set(error, EINVAL, 2231 RTE_FLOW_ERROR_TYPE_ITEM, item, 2232 "protocol filtering not compatible" 2233 " with ICMP layer"); 2234 if (!(item_flags & l3m)) 2235 return rte_flow_error_set(error, EINVAL, 2236 RTE_FLOW_ERROR_TYPE_ITEM, item, 2237 "IPv4 is mandatory to filter" 2238 " on ICMP"); 2239 if (item_flags & l4m) 2240 return rte_flow_error_set(error, EINVAL, 2241 RTE_FLOW_ERROR_TYPE_ITEM, item, 2242 "multiple L4 layers not supported"); 2243 if (!mask) 2244 mask = &nic_mask; 2245 ret = mlx5_flow_item_acceptable 2246 (item, (const uint8_t *)mask, 2247 (const uint8_t *)&nic_mask, 2248 sizeof(struct rte_flow_item_icmp), 2249 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2250 if (ret < 0) 2251 return ret; 2252 return 0; 2253 } 2254 2255 /** 2256 * Validate Ethernet item. 2257 * 2258 * @param[in] item 2259 * Item specification. 2260 * @param[in] item_flags 2261 * Bit-fields that holds the items detected until now. 2262 * @param[out] error 2263 * Pointer to error structure. 2264 * 2265 * @return 2266 * 0 on success, a negative errno value otherwise and rte_errno is set. 2267 */ 2268 int 2269 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2270 uint64_t item_flags, bool ext_vlan_sup, 2271 struct rte_flow_error *error) 2272 { 2273 const struct rte_flow_item_eth *mask = item->mask; 2274 const struct rte_flow_item_eth nic_mask = { 2275 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2276 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2277 .type = RTE_BE16(0xffff), 2278 .has_vlan = ext_vlan_sup ? 1 : 0, 2279 }; 2280 int ret; 2281 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2282 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2283 MLX5_FLOW_LAYER_OUTER_L2; 2284 2285 if (item_flags & ethm) 2286 return rte_flow_error_set(error, ENOTSUP, 2287 RTE_FLOW_ERROR_TYPE_ITEM, item, 2288 "multiple L2 layers not supported"); 2289 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2290 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2291 return rte_flow_error_set(error, EINVAL, 2292 RTE_FLOW_ERROR_TYPE_ITEM, item, 2293 "L2 layer should not follow " 2294 "L3 layers"); 2295 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2296 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2297 return rte_flow_error_set(error, EINVAL, 2298 RTE_FLOW_ERROR_TYPE_ITEM, item, 2299 "L2 layer should not follow VLAN"); 2300 if (item_flags & MLX5_FLOW_LAYER_GTP) 2301 return rte_flow_error_set(error, EINVAL, 2302 RTE_FLOW_ERROR_TYPE_ITEM, item, 2303 "L2 layer should not follow GTP"); 2304 if (!mask) 2305 mask = &rte_flow_item_eth_mask; 2306 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2307 (const uint8_t *)&nic_mask, 2308 sizeof(struct rte_flow_item_eth), 2309 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2310 return ret; 2311 } 2312 2313 /** 2314 * Validate VLAN item. 2315 * 2316 * @param[in] item 2317 * Item specification. 2318 * @param[in] item_flags 2319 * Bit-fields that holds the items detected until now. 2320 * @param[in] dev 2321 * Ethernet device flow is being created on. 2322 * @param[out] error 2323 * Pointer to error structure. 2324 * 2325 * @return 2326 * 0 on success, a negative errno value otherwise and rte_errno is set. 2327 */ 2328 int 2329 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2330 uint64_t item_flags, 2331 struct rte_eth_dev *dev, 2332 struct rte_flow_error *error) 2333 { 2334 const struct rte_flow_item_vlan *spec = item->spec; 2335 const struct rte_flow_item_vlan *mask = item->mask; 2336 const struct rte_flow_item_vlan nic_mask = { 2337 .tci = RTE_BE16(UINT16_MAX), 2338 .inner_type = RTE_BE16(UINT16_MAX), 2339 }; 2340 uint16_t vlan_tag = 0; 2341 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2342 int ret; 2343 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2344 MLX5_FLOW_LAYER_INNER_L4) : 2345 (MLX5_FLOW_LAYER_OUTER_L3 | 2346 MLX5_FLOW_LAYER_OUTER_L4); 2347 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2348 MLX5_FLOW_LAYER_OUTER_VLAN; 2349 2350 if (item_flags & vlanm) 2351 return rte_flow_error_set(error, EINVAL, 2352 RTE_FLOW_ERROR_TYPE_ITEM, item, 2353 "multiple VLAN layers not supported"); 2354 else if ((item_flags & l34m) != 0) 2355 return rte_flow_error_set(error, EINVAL, 2356 RTE_FLOW_ERROR_TYPE_ITEM, item, 2357 "VLAN cannot follow L3/L4 layer"); 2358 if (!mask) 2359 mask = &rte_flow_item_vlan_mask; 2360 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2361 (const uint8_t *)&nic_mask, 2362 sizeof(struct rte_flow_item_vlan), 2363 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2364 if (ret) 2365 return ret; 2366 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2367 struct mlx5_priv *priv = dev->data->dev_private; 2368 2369 if (priv->vmwa_context) { 2370 /* 2371 * Non-NULL context means we have a virtual machine 2372 * and SR-IOV enabled, we have to create VLAN interface 2373 * to make hypervisor to setup E-Switch vport 2374 * context correctly. We avoid creating the multiple 2375 * VLAN interfaces, so we cannot support VLAN tag mask. 2376 */ 2377 return rte_flow_error_set(error, EINVAL, 2378 RTE_FLOW_ERROR_TYPE_ITEM, 2379 item, 2380 "VLAN tag mask is not" 2381 " supported in virtual" 2382 " environment"); 2383 } 2384 } 2385 if (spec) { 2386 vlan_tag = spec->tci; 2387 vlan_tag &= mask->tci; 2388 } 2389 /* 2390 * From verbs perspective an empty VLAN is equivalent 2391 * to a packet without VLAN layer. 2392 */ 2393 if (!vlan_tag) 2394 return rte_flow_error_set(error, EINVAL, 2395 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2396 item->spec, 2397 "VLAN cannot be empty"); 2398 return 0; 2399 } 2400 2401 /** 2402 * Validate IPV4 item. 2403 * 2404 * @param[in] item 2405 * Item specification. 2406 * @param[in] item_flags 2407 * Bit-fields that holds the items detected until now. 2408 * @param[in] last_item 2409 * Previous validated item in the pattern items. 2410 * @param[in] ether_type 2411 * Type in the ethernet layer header (including dot1q). 2412 * @param[in] acc_mask 2413 * Acceptable mask, if NULL default internal default mask 2414 * will be used to check whether item fields are supported. 2415 * @param[in] range_accepted 2416 * True if range of values is accepted for specific fields, false otherwise. 2417 * @param[out] error 2418 * Pointer to error structure. 2419 * 2420 * @return 2421 * 0 on success, a negative errno value otherwise and rte_errno is set. 2422 */ 2423 int 2424 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2425 uint64_t item_flags, 2426 uint64_t last_item, 2427 uint16_t ether_type, 2428 const struct rte_flow_item_ipv4 *acc_mask, 2429 bool range_accepted, 2430 struct rte_flow_error *error) 2431 { 2432 const struct rte_flow_item_ipv4 *mask = item->mask; 2433 const struct rte_flow_item_ipv4 *spec = item->spec; 2434 const struct rte_flow_item_ipv4 nic_mask = { 2435 .hdr = { 2436 .src_addr = RTE_BE32(0xffffffff), 2437 .dst_addr = RTE_BE32(0xffffffff), 2438 .type_of_service = 0xff, 2439 .next_proto_id = 0xff, 2440 }, 2441 }; 2442 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2443 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2444 MLX5_FLOW_LAYER_OUTER_L3; 2445 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2446 MLX5_FLOW_LAYER_OUTER_L4; 2447 int ret; 2448 uint8_t next_proto = 0xFF; 2449 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2450 MLX5_FLOW_LAYER_OUTER_VLAN | 2451 MLX5_FLOW_LAYER_INNER_VLAN); 2452 2453 if ((last_item & l2_vlan) && ether_type && 2454 ether_type != RTE_ETHER_TYPE_IPV4) 2455 return rte_flow_error_set(error, EINVAL, 2456 RTE_FLOW_ERROR_TYPE_ITEM, item, 2457 "IPv4 cannot follow L2/VLAN layer " 2458 "which ether type is not IPv4"); 2459 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2460 if (mask && spec) 2461 next_proto = mask->hdr.next_proto_id & 2462 spec->hdr.next_proto_id; 2463 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2464 return rte_flow_error_set(error, EINVAL, 2465 RTE_FLOW_ERROR_TYPE_ITEM, 2466 item, 2467 "multiple tunnel " 2468 "not supported"); 2469 } 2470 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2471 return rte_flow_error_set(error, EINVAL, 2472 RTE_FLOW_ERROR_TYPE_ITEM, item, 2473 "wrong tunnel type - IPv6 specified " 2474 "but IPv4 item provided"); 2475 if (item_flags & l3m) 2476 return rte_flow_error_set(error, ENOTSUP, 2477 RTE_FLOW_ERROR_TYPE_ITEM, item, 2478 "multiple L3 layers not supported"); 2479 else if (item_flags & l4m) 2480 return rte_flow_error_set(error, EINVAL, 2481 RTE_FLOW_ERROR_TYPE_ITEM, item, 2482 "L3 cannot follow an L4 layer."); 2483 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2484 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2485 return rte_flow_error_set(error, EINVAL, 2486 RTE_FLOW_ERROR_TYPE_ITEM, item, 2487 "L3 cannot follow an NVGRE layer."); 2488 if (!mask) 2489 mask = &rte_flow_item_ipv4_mask; 2490 else if (mask->hdr.next_proto_id != 0 && 2491 mask->hdr.next_proto_id != 0xff) 2492 return rte_flow_error_set(error, EINVAL, 2493 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2494 "partial mask is not supported" 2495 " for protocol"); 2496 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2497 acc_mask ? (const uint8_t *)acc_mask 2498 : (const uint8_t *)&nic_mask, 2499 sizeof(struct rte_flow_item_ipv4), 2500 range_accepted, error); 2501 if (ret < 0) 2502 return ret; 2503 return 0; 2504 } 2505 2506 /** 2507 * Validate IPV6 item. 2508 * 2509 * @param[in] item 2510 * Item specification. 2511 * @param[in] item_flags 2512 * Bit-fields that holds the items detected until now. 2513 * @param[in] last_item 2514 * Previous validated item in the pattern items. 2515 * @param[in] ether_type 2516 * Type in the ethernet layer header (including dot1q). 2517 * @param[in] acc_mask 2518 * Acceptable mask, if NULL default internal default mask 2519 * will be used to check whether item fields are supported. 2520 * @param[out] error 2521 * Pointer to error structure. 2522 * 2523 * @return 2524 * 0 on success, a negative errno value otherwise and rte_errno is set. 2525 */ 2526 int 2527 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2528 uint64_t item_flags, 2529 uint64_t last_item, 2530 uint16_t ether_type, 2531 const struct rte_flow_item_ipv6 *acc_mask, 2532 struct rte_flow_error *error) 2533 { 2534 const struct rte_flow_item_ipv6 *mask = item->mask; 2535 const struct rte_flow_item_ipv6 *spec = item->spec; 2536 const struct rte_flow_item_ipv6 nic_mask = { 2537 .hdr = { 2538 .src_addr = 2539 "\xff\xff\xff\xff\xff\xff\xff\xff" 2540 "\xff\xff\xff\xff\xff\xff\xff\xff", 2541 .dst_addr = 2542 "\xff\xff\xff\xff\xff\xff\xff\xff" 2543 "\xff\xff\xff\xff\xff\xff\xff\xff", 2544 .vtc_flow = RTE_BE32(0xffffffff), 2545 .proto = 0xff, 2546 }, 2547 }; 2548 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2549 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2550 MLX5_FLOW_LAYER_OUTER_L3; 2551 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2552 MLX5_FLOW_LAYER_OUTER_L4; 2553 int ret; 2554 uint8_t next_proto = 0xFF; 2555 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2556 MLX5_FLOW_LAYER_OUTER_VLAN | 2557 MLX5_FLOW_LAYER_INNER_VLAN); 2558 2559 if ((last_item & l2_vlan) && ether_type && 2560 ether_type != RTE_ETHER_TYPE_IPV6) 2561 return rte_flow_error_set(error, EINVAL, 2562 RTE_FLOW_ERROR_TYPE_ITEM, item, 2563 "IPv6 cannot follow L2/VLAN layer " 2564 "which ether type is not IPv6"); 2565 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2566 next_proto = spec->hdr.proto; 2567 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2568 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2569 return rte_flow_error_set(error, EINVAL, 2570 RTE_FLOW_ERROR_TYPE_ITEM, 2571 item, 2572 "multiple tunnel " 2573 "not supported"); 2574 } 2575 if (next_proto == IPPROTO_HOPOPTS || 2576 next_proto == IPPROTO_ROUTING || 2577 next_proto == IPPROTO_FRAGMENT || 2578 next_proto == IPPROTO_ESP || 2579 next_proto == IPPROTO_AH || 2580 next_proto == IPPROTO_DSTOPTS) 2581 return rte_flow_error_set(error, EINVAL, 2582 RTE_FLOW_ERROR_TYPE_ITEM, item, 2583 "IPv6 proto (next header) should " 2584 "not be set as extension header"); 2585 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2586 return rte_flow_error_set(error, EINVAL, 2587 RTE_FLOW_ERROR_TYPE_ITEM, item, 2588 "wrong tunnel type - IPv4 specified " 2589 "but IPv6 item provided"); 2590 if (item_flags & l3m) 2591 return rte_flow_error_set(error, ENOTSUP, 2592 RTE_FLOW_ERROR_TYPE_ITEM, item, 2593 "multiple L3 layers not supported"); 2594 else if (item_flags & l4m) 2595 return rte_flow_error_set(error, EINVAL, 2596 RTE_FLOW_ERROR_TYPE_ITEM, item, 2597 "L3 cannot follow an L4 layer."); 2598 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2599 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2600 return rte_flow_error_set(error, EINVAL, 2601 RTE_FLOW_ERROR_TYPE_ITEM, item, 2602 "L3 cannot follow an NVGRE layer."); 2603 if (!mask) 2604 mask = &rte_flow_item_ipv6_mask; 2605 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2606 acc_mask ? (const uint8_t *)acc_mask 2607 : (const uint8_t *)&nic_mask, 2608 sizeof(struct rte_flow_item_ipv6), 2609 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2610 if (ret < 0) 2611 return ret; 2612 return 0; 2613 } 2614 2615 /** 2616 * Validate UDP item. 2617 * 2618 * @param[in] item 2619 * Item specification. 2620 * @param[in] item_flags 2621 * Bit-fields that holds the items detected until now. 2622 * @param[in] target_protocol 2623 * The next protocol in the previous item. 2624 * @param[in] flow_mask 2625 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2626 * @param[out] error 2627 * Pointer to error structure. 2628 * 2629 * @return 2630 * 0 on success, a negative errno value otherwise and rte_errno is set. 2631 */ 2632 int 2633 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2634 uint64_t item_flags, 2635 uint8_t target_protocol, 2636 struct rte_flow_error *error) 2637 { 2638 const struct rte_flow_item_udp *mask = item->mask; 2639 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2640 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2641 MLX5_FLOW_LAYER_OUTER_L3; 2642 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2643 MLX5_FLOW_LAYER_OUTER_L4; 2644 int ret; 2645 2646 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2647 return rte_flow_error_set(error, EINVAL, 2648 RTE_FLOW_ERROR_TYPE_ITEM, item, 2649 "protocol filtering not compatible" 2650 " with UDP layer"); 2651 if (!(item_flags & l3m)) 2652 return rte_flow_error_set(error, EINVAL, 2653 RTE_FLOW_ERROR_TYPE_ITEM, item, 2654 "L3 is mandatory to filter on L4"); 2655 if (item_flags & l4m) 2656 return rte_flow_error_set(error, EINVAL, 2657 RTE_FLOW_ERROR_TYPE_ITEM, item, 2658 "multiple L4 layers not supported"); 2659 if (!mask) 2660 mask = &rte_flow_item_udp_mask; 2661 ret = mlx5_flow_item_acceptable 2662 (item, (const uint8_t *)mask, 2663 (const uint8_t *)&rte_flow_item_udp_mask, 2664 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2665 error); 2666 if (ret < 0) 2667 return ret; 2668 return 0; 2669 } 2670 2671 /** 2672 * Validate TCP item. 2673 * 2674 * @param[in] item 2675 * Item specification. 2676 * @param[in] item_flags 2677 * Bit-fields that holds the items detected until now. 2678 * @param[in] target_protocol 2679 * The next protocol in the previous item. 2680 * @param[out] error 2681 * Pointer to error structure. 2682 * 2683 * @return 2684 * 0 on success, a negative errno value otherwise and rte_errno is set. 2685 */ 2686 int 2687 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2688 uint64_t item_flags, 2689 uint8_t target_protocol, 2690 const struct rte_flow_item_tcp *flow_mask, 2691 struct rte_flow_error *error) 2692 { 2693 const struct rte_flow_item_tcp *mask = item->mask; 2694 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2695 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2696 MLX5_FLOW_LAYER_OUTER_L3; 2697 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2698 MLX5_FLOW_LAYER_OUTER_L4; 2699 int ret; 2700 2701 MLX5_ASSERT(flow_mask); 2702 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2703 return rte_flow_error_set(error, EINVAL, 2704 RTE_FLOW_ERROR_TYPE_ITEM, item, 2705 "protocol filtering not compatible" 2706 " with TCP layer"); 2707 if (!(item_flags & l3m)) 2708 return rte_flow_error_set(error, EINVAL, 2709 RTE_FLOW_ERROR_TYPE_ITEM, item, 2710 "L3 is mandatory to filter on L4"); 2711 if (item_flags & l4m) 2712 return rte_flow_error_set(error, EINVAL, 2713 RTE_FLOW_ERROR_TYPE_ITEM, item, 2714 "multiple L4 layers not supported"); 2715 if (!mask) 2716 mask = &rte_flow_item_tcp_mask; 2717 ret = mlx5_flow_item_acceptable 2718 (item, (const uint8_t *)mask, 2719 (const uint8_t *)flow_mask, 2720 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2721 error); 2722 if (ret < 0) 2723 return ret; 2724 return 0; 2725 } 2726 2727 /** 2728 * Validate VXLAN item. 2729 * 2730 * @param[in] dev 2731 * Pointer to the Ethernet device structure. 2732 * @param[in] udp_dport 2733 * UDP destination port 2734 * @param[in] item 2735 * Item specification. 2736 * @param[in] item_flags 2737 * Bit-fields that holds the items detected until now. 2738 * @param[in] attr 2739 * Flow rule attributes. 2740 * @param[out] error 2741 * Pointer to error structure. 2742 * 2743 * @return 2744 * 0 on success, a negative errno value otherwise and rte_errno is set. 2745 */ 2746 int 2747 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2748 uint16_t udp_dport, 2749 const struct rte_flow_item *item, 2750 uint64_t item_flags, 2751 const struct rte_flow_attr *attr, 2752 struct rte_flow_error *error) 2753 { 2754 const struct rte_flow_item_vxlan *spec = item->spec; 2755 const struct rte_flow_item_vxlan *mask = item->mask; 2756 int ret; 2757 struct mlx5_priv *priv = dev->data->dev_private; 2758 union vni { 2759 uint32_t vlan_id; 2760 uint8_t vni[4]; 2761 } id = { .vlan_id = 0, }; 2762 const struct rte_flow_item_vxlan nic_mask = { 2763 .vni = "\xff\xff\xff", 2764 .rsvd1 = 0xff, 2765 }; 2766 const struct rte_flow_item_vxlan *valid_mask; 2767 2768 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2769 return rte_flow_error_set(error, ENOTSUP, 2770 RTE_FLOW_ERROR_TYPE_ITEM, item, 2771 "multiple tunnel layers not" 2772 " supported"); 2773 valid_mask = &rte_flow_item_vxlan_mask; 2774 /* 2775 * Verify only UDPv4 is present as defined in 2776 * https://tools.ietf.org/html/rfc7348 2777 */ 2778 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2779 return rte_flow_error_set(error, EINVAL, 2780 RTE_FLOW_ERROR_TYPE_ITEM, item, 2781 "no outer UDP layer found"); 2782 if (!mask) 2783 mask = &rte_flow_item_vxlan_mask; 2784 2785 if (priv->sh->steering_format_version != 2786 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2787 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2788 /* FDB domain & NIC domain non-zero group */ 2789 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2790 valid_mask = &nic_mask; 2791 /* Group zero in NIC domain */ 2792 if (!attr->group && !attr->transfer && 2793 priv->sh->tunnel_header_0_1) 2794 valid_mask = &nic_mask; 2795 } 2796 ret = mlx5_flow_item_acceptable 2797 (item, (const uint8_t *)mask, 2798 (const uint8_t *)valid_mask, 2799 sizeof(struct rte_flow_item_vxlan), 2800 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2801 if (ret < 0) 2802 return ret; 2803 if (spec) { 2804 memcpy(&id.vni[1], spec->vni, 3); 2805 memcpy(&id.vni[1], mask->vni, 3); 2806 } 2807 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2808 return rte_flow_error_set(error, ENOTSUP, 2809 RTE_FLOW_ERROR_TYPE_ITEM, item, 2810 "VXLAN tunnel must be fully defined"); 2811 return 0; 2812 } 2813 2814 /** 2815 * Validate VXLAN_GPE item. 2816 * 2817 * @param[in] item 2818 * Item specification. 2819 * @param[in] item_flags 2820 * Bit-fields that holds the items detected until now. 2821 * @param[in] priv 2822 * Pointer to the private data structure. 2823 * @param[in] target_protocol 2824 * The next protocol in the previous item. 2825 * @param[out] error 2826 * Pointer to error structure. 2827 * 2828 * @return 2829 * 0 on success, a negative errno value otherwise and rte_errno is set. 2830 */ 2831 int 2832 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2833 uint64_t item_flags, 2834 struct rte_eth_dev *dev, 2835 struct rte_flow_error *error) 2836 { 2837 struct mlx5_priv *priv = dev->data->dev_private; 2838 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2839 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2840 int ret; 2841 union vni { 2842 uint32_t vlan_id; 2843 uint8_t vni[4]; 2844 } id = { .vlan_id = 0, }; 2845 2846 if (!priv->sh->config.l3_vxlan_en) 2847 return rte_flow_error_set(error, ENOTSUP, 2848 RTE_FLOW_ERROR_TYPE_ITEM, item, 2849 "L3 VXLAN is not enabled by device" 2850 " parameter and/or not configured in" 2851 " firmware"); 2852 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2853 return rte_flow_error_set(error, ENOTSUP, 2854 RTE_FLOW_ERROR_TYPE_ITEM, item, 2855 "multiple tunnel layers not" 2856 " supported"); 2857 /* 2858 * Verify only UDPv4 is present as defined in 2859 * https://tools.ietf.org/html/rfc7348 2860 */ 2861 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2862 return rte_flow_error_set(error, EINVAL, 2863 RTE_FLOW_ERROR_TYPE_ITEM, item, 2864 "no outer UDP layer found"); 2865 if (!mask) 2866 mask = &rte_flow_item_vxlan_gpe_mask; 2867 ret = mlx5_flow_item_acceptable 2868 (item, (const uint8_t *)mask, 2869 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2870 sizeof(struct rte_flow_item_vxlan_gpe), 2871 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2872 if (ret < 0) 2873 return ret; 2874 if (spec) { 2875 if (spec->protocol) 2876 return rte_flow_error_set(error, ENOTSUP, 2877 RTE_FLOW_ERROR_TYPE_ITEM, 2878 item, 2879 "VxLAN-GPE protocol" 2880 " not supported"); 2881 memcpy(&id.vni[1], spec->vni, 3); 2882 memcpy(&id.vni[1], mask->vni, 3); 2883 } 2884 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2885 return rte_flow_error_set(error, ENOTSUP, 2886 RTE_FLOW_ERROR_TYPE_ITEM, item, 2887 "VXLAN-GPE tunnel must be fully" 2888 " defined"); 2889 return 0; 2890 } 2891 /** 2892 * Validate GRE Key item. 2893 * 2894 * @param[in] item 2895 * Item specification. 2896 * @param[in] item_flags 2897 * Bit flags to mark detected items. 2898 * @param[in] gre_item 2899 * Pointer to gre_item 2900 * @param[out] error 2901 * Pointer to error structure. 2902 * 2903 * @return 2904 * 0 on success, a negative errno value otherwise and rte_errno is set. 2905 */ 2906 int 2907 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2908 uint64_t item_flags, 2909 const struct rte_flow_item *gre_item, 2910 struct rte_flow_error *error) 2911 { 2912 const rte_be32_t *mask = item->mask; 2913 int ret = 0; 2914 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2915 const struct rte_flow_item_gre *gre_spec; 2916 const struct rte_flow_item_gre *gre_mask; 2917 2918 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2919 return rte_flow_error_set(error, ENOTSUP, 2920 RTE_FLOW_ERROR_TYPE_ITEM, item, 2921 "Multiple GRE key not support"); 2922 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2923 return rte_flow_error_set(error, ENOTSUP, 2924 RTE_FLOW_ERROR_TYPE_ITEM, item, 2925 "No preceding GRE header"); 2926 if (item_flags & MLX5_FLOW_LAYER_INNER) 2927 return rte_flow_error_set(error, ENOTSUP, 2928 RTE_FLOW_ERROR_TYPE_ITEM, item, 2929 "GRE key following a wrong item"); 2930 gre_mask = gre_item->mask; 2931 if (!gre_mask) 2932 gre_mask = &rte_flow_item_gre_mask; 2933 gre_spec = gre_item->spec; 2934 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2935 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2936 return rte_flow_error_set(error, EINVAL, 2937 RTE_FLOW_ERROR_TYPE_ITEM, item, 2938 "Key bit must be on"); 2939 2940 if (!mask) 2941 mask = &gre_key_default_mask; 2942 ret = mlx5_flow_item_acceptable 2943 (item, (const uint8_t *)mask, 2944 (const uint8_t *)&gre_key_default_mask, 2945 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2946 return ret; 2947 } 2948 2949 /** 2950 * Validate GRE optional item. 2951 * 2952 * @param[in] dev 2953 * Pointer to the Ethernet device structure. 2954 * @param[in] item 2955 * Item specification. 2956 * @param[in] item_flags 2957 * Bit flags to mark detected items. 2958 * @param[in] attr 2959 * Flow rule attributes. 2960 * @param[in] gre_item 2961 * Pointer to gre_item 2962 * @param[out] error 2963 * Pointer to error structure. 2964 * 2965 * @return 2966 * 0 on success, a negative errno value otherwise and rte_errno is set. 2967 */ 2968 int 2969 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 2970 const struct rte_flow_item *item, 2971 uint64_t item_flags, 2972 const struct rte_flow_attr *attr, 2973 const struct rte_flow_item *gre_item, 2974 struct rte_flow_error *error) 2975 { 2976 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 2977 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 2978 const struct rte_flow_item_gre_opt *spec = item->spec; 2979 const struct rte_flow_item_gre_opt *mask = item->mask; 2980 struct mlx5_priv *priv = dev->data->dev_private; 2981 int ret = 0; 2982 struct rte_flow_item_gre_opt nic_mask = { 2983 .checksum_rsvd = { 2984 .checksum = RTE_BE16(UINT16_MAX), 2985 .reserved1 = 0x0, 2986 }, 2987 .key = { 2988 .key = RTE_BE32(UINT32_MAX), 2989 }, 2990 .sequence = { 2991 .sequence = RTE_BE32(UINT32_MAX), 2992 }, 2993 }; 2994 2995 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2996 return rte_flow_error_set(error, ENOTSUP, 2997 RTE_FLOW_ERROR_TYPE_ITEM, item, 2998 "No preceding GRE header"); 2999 if (item_flags & MLX5_FLOW_LAYER_INNER) 3000 return rte_flow_error_set(error, ENOTSUP, 3001 RTE_FLOW_ERROR_TYPE_ITEM, item, 3002 "GRE option following a wrong item"); 3003 if (!spec || !mask) 3004 return rte_flow_error_set(error, EINVAL, 3005 RTE_FLOW_ERROR_TYPE_ITEM, item, 3006 "At least one field gre_option(checksum/key/sequence) must be specified"); 3007 if (!gre_mask) 3008 gre_mask = &rte_flow_item_gre_mask; 3009 if (mask->checksum_rsvd.checksum) 3010 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3011 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3012 return rte_flow_error_set(error, EINVAL, 3013 RTE_FLOW_ERROR_TYPE_ITEM, 3014 item, 3015 "Checksum bit must be on"); 3016 if (mask->key.key) 3017 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3018 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3019 return rte_flow_error_set(error, EINVAL, 3020 RTE_FLOW_ERROR_TYPE_ITEM, 3021 item, "Key bit must be on"); 3022 if (mask->sequence.sequence) 3023 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3024 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3025 return rte_flow_error_set(error, EINVAL, 3026 RTE_FLOW_ERROR_TYPE_ITEM, 3027 item, 3028 "Sequence bit must be on"); 3029 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3030 if (priv->sh->steering_format_version == 3031 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3032 ((attr->group || attr->transfer) && 3033 !priv->sh->misc5_cap) || 3034 (!(priv->sh->tunnel_header_0_1 && 3035 priv->sh->tunnel_header_2_3) && 3036 !attr->group && !attr->transfer)) 3037 return rte_flow_error_set(error, EINVAL, 3038 RTE_FLOW_ERROR_TYPE_ITEM, 3039 item, 3040 "Checksum/Sequence not supported"); 3041 } 3042 ret = mlx5_flow_item_acceptable 3043 (item, (const uint8_t *)mask, 3044 (const uint8_t *)&nic_mask, 3045 sizeof(struct rte_flow_item_gre_opt), 3046 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3047 return ret; 3048 } 3049 3050 /** 3051 * Validate GRE item. 3052 * 3053 * @param[in] item 3054 * Item specification. 3055 * @param[in] item_flags 3056 * Bit flags to mark detected items. 3057 * @param[in] target_protocol 3058 * The next protocol in the previous item. 3059 * @param[out] error 3060 * Pointer to error structure. 3061 * 3062 * @return 3063 * 0 on success, a negative errno value otherwise and rte_errno is set. 3064 */ 3065 int 3066 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3067 uint64_t item_flags, 3068 uint8_t target_protocol, 3069 struct rte_flow_error *error) 3070 { 3071 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3072 const struct rte_flow_item_gre *mask = item->mask; 3073 int ret; 3074 const struct rte_flow_item_gre nic_mask = { 3075 .c_rsvd0_ver = RTE_BE16(0xB000), 3076 .protocol = RTE_BE16(UINT16_MAX), 3077 }; 3078 3079 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3080 return rte_flow_error_set(error, EINVAL, 3081 RTE_FLOW_ERROR_TYPE_ITEM, item, 3082 "protocol filtering not compatible" 3083 " with this GRE layer"); 3084 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3085 return rte_flow_error_set(error, ENOTSUP, 3086 RTE_FLOW_ERROR_TYPE_ITEM, item, 3087 "multiple tunnel layers not" 3088 " supported"); 3089 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3090 return rte_flow_error_set(error, ENOTSUP, 3091 RTE_FLOW_ERROR_TYPE_ITEM, item, 3092 "L3 Layer is missing"); 3093 if (!mask) 3094 mask = &rte_flow_item_gre_mask; 3095 ret = mlx5_flow_item_acceptable 3096 (item, (const uint8_t *)mask, 3097 (const uint8_t *)&nic_mask, 3098 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3099 error); 3100 if (ret < 0) 3101 return ret; 3102 #ifndef HAVE_MLX5DV_DR 3103 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3104 if (spec && (spec->protocol & mask->protocol)) 3105 return rte_flow_error_set(error, ENOTSUP, 3106 RTE_FLOW_ERROR_TYPE_ITEM, item, 3107 "without MPLS support the" 3108 " specification cannot be used for" 3109 " filtering"); 3110 #endif 3111 #endif 3112 return 0; 3113 } 3114 3115 /** 3116 * Validate Geneve item. 3117 * 3118 * @param[in] item 3119 * Item specification. 3120 * @param[in] itemFlags 3121 * Bit-fields that holds the items detected until now. 3122 * @param[in] enPriv 3123 * Pointer to the private data structure. 3124 * @param[out] error 3125 * Pointer to error structure. 3126 * 3127 * @return 3128 * 0 on success, a negative errno value otherwise and rte_errno is set. 3129 */ 3130 3131 int 3132 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3133 uint64_t item_flags, 3134 struct rte_eth_dev *dev, 3135 struct rte_flow_error *error) 3136 { 3137 struct mlx5_priv *priv = dev->data->dev_private; 3138 const struct rte_flow_item_geneve *spec = item->spec; 3139 const struct rte_flow_item_geneve *mask = item->mask; 3140 int ret; 3141 uint16_t gbhdr; 3142 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3143 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3144 const struct rte_flow_item_geneve nic_mask = { 3145 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3146 .vni = "\xff\xff\xff", 3147 .protocol = RTE_BE16(UINT16_MAX), 3148 }; 3149 3150 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3151 return rte_flow_error_set(error, ENOTSUP, 3152 RTE_FLOW_ERROR_TYPE_ITEM, item, 3153 "L3 Geneve is not enabled by device" 3154 " parameter and/or not configured in" 3155 " firmware"); 3156 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3157 return rte_flow_error_set(error, ENOTSUP, 3158 RTE_FLOW_ERROR_TYPE_ITEM, item, 3159 "multiple tunnel layers not" 3160 " supported"); 3161 /* 3162 * Verify only UDPv4 is present as defined in 3163 * https://tools.ietf.org/html/rfc7348 3164 */ 3165 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3166 return rte_flow_error_set(error, EINVAL, 3167 RTE_FLOW_ERROR_TYPE_ITEM, item, 3168 "no outer UDP layer found"); 3169 if (!mask) 3170 mask = &rte_flow_item_geneve_mask; 3171 ret = mlx5_flow_item_acceptable 3172 (item, (const uint8_t *)mask, 3173 (const uint8_t *)&nic_mask, 3174 sizeof(struct rte_flow_item_geneve), 3175 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3176 if (ret) 3177 return ret; 3178 if (spec) { 3179 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3180 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3181 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3182 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3183 return rte_flow_error_set(error, ENOTSUP, 3184 RTE_FLOW_ERROR_TYPE_ITEM, 3185 item, 3186 "Geneve protocol unsupported" 3187 " fields are being used"); 3188 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3189 return rte_flow_error_set 3190 (error, ENOTSUP, 3191 RTE_FLOW_ERROR_TYPE_ITEM, 3192 item, 3193 "Unsupported Geneve options length"); 3194 } 3195 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3196 return rte_flow_error_set 3197 (error, ENOTSUP, 3198 RTE_FLOW_ERROR_TYPE_ITEM, item, 3199 "Geneve tunnel must be fully defined"); 3200 return 0; 3201 } 3202 3203 /** 3204 * Validate Geneve TLV option item. 3205 * 3206 * @param[in] item 3207 * Item specification. 3208 * @param[in] last_item 3209 * Previous validated item in the pattern items. 3210 * @param[in] geneve_item 3211 * Previous GENEVE item specification. 3212 * @param[in] dev 3213 * Pointer to the rte_eth_dev structure. 3214 * @param[out] error 3215 * Pointer to error structure. 3216 * 3217 * @return 3218 * 0 on success, a negative errno value otherwise and rte_errno is set. 3219 */ 3220 int 3221 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3222 uint64_t last_item, 3223 const struct rte_flow_item *geneve_item, 3224 struct rte_eth_dev *dev, 3225 struct rte_flow_error *error) 3226 { 3227 struct mlx5_priv *priv = dev->data->dev_private; 3228 struct mlx5_dev_ctx_shared *sh = priv->sh; 3229 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3230 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3231 uint8_t data_max_supported = 3232 hca_attr->max_geneve_tlv_option_data_len * 4; 3233 const struct rte_flow_item_geneve *geneve_spec; 3234 const struct rte_flow_item_geneve *geneve_mask; 3235 const struct rte_flow_item_geneve_opt *spec = item->spec; 3236 const struct rte_flow_item_geneve_opt *mask = item->mask; 3237 unsigned int i; 3238 unsigned int data_len; 3239 uint8_t tlv_option_len; 3240 uint16_t optlen_m, optlen_v; 3241 const struct rte_flow_item_geneve_opt full_mask = { 3242 .option_class = RTE_BE16(0xffff), 3243 .option_type = 0xff, 3244 .option_len = 0x1f, 3245 }; 3246 3247 if (!mask) 3248 mask = &rte_flow_item_geneve_opt_mask; 3249 if (!spec) 3250 return rte_flow_error_set 3251 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3252 "Geneve TLV opt class/type/length must be specified"); 3253 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3254 return rte_flow_error_set 3255 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3256 "Geneve TLV opt length exceeds the limit (31)"); 3257 /* Check if class type and length masks are full. */ 3258 if (full_mask.option_class != mask->option_class || 3259 full_mask.option_type != mask->option_type || 3260 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3261 return rte_flow_error_set 3262 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3263 "Geneve TLV opt class/type/length masks must be full"); 3264 /* Check if length is supported */ 3265 if ((uint32_t)spec->option_len > 3266 hca_attr->max_geneve_tlv_option_data_len) 3267 return rte_flow_error_set 3268 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3269 "Geneve TLV opt length not supported"); 3270 if (hca_attr->max_geneve_tlv_options > 1) 3271 DRV_LOG(DEBUG, 3272 "max_geneve_tlv_options supports more than 1 option"); 3273 /* Check GENEVE item preceding. */ 3274 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3275 return rte_flow_error_set 3276 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3277 "Geneve opt item must be preceded with Geneve item"); 3278 geneve_spec = geneve_item->spec; 3279 geneve_mask = geneve_item->mask ? geneve_item->mask : 3280 &rte_flow_item_geneve_mask; 3281 /* Check if GENEVE TLV option size doesn't exceed option length */ 3282 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3283 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3284 tlv_option_len = spec->option_len & mask->option_len; 3285 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3286 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3287 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3288 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3289 if ((optlen_v & optlen_m) <= tlv_option_len) 3290 return rte_flow_error_set 3291 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3292 "GENEVE TLV option length exceeds optlen"); 3293 } 3294 /* Check if length is 0 or data is 0. */ 3295 if (spec->data == NULL || spec->option_len == 0) 3296 return rte_flow_error_set 3297 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3298 "Geneve TLV opt with zero data/length not supported"); 3299 /* Check not all data & mask are 0. */ 3300 data_len = spec->option_len * 4; 3301 if (mask->data == NULL) { 3302 for (i = 0; i < data_len; i++) 3303 if (spec->data[i]) 3304 break; 3305 if (i == data_len) 3306 return rte_flow_error_set(error, ENOTSUP, 3307 RTE_FLOW_ERROR_TYPE_ITEM, item, 3308 "Can't match on Geneve option data 0"); 3309 } else { 3310 for (i = 0; i < data_len; i++) 3311 if (spec->data[i] & mask->data[i]) 3312 break; 3313 if (i == data_len) 3314 return rte_flow_error_set(error, ENOTSUP, 3315 RTE_FLOW_ERROR_TYPE_ITEM, item, 3316 "Can't match on Geneve option data and mask 0"); 3317 /* Check data mask supported. */ 3318 for (i = data_max_supported; i < data_len ; i++) 3319 if (mask->data[i]) 3320 return rte_flow_error_set(error, ENOTSUP, 3321 RTE_FLOW_ERROR_TYPE_ITEM, item, 3322 "Data mask is of unsupported size"); 3323 } 3324 /* Check GENEVE option is supported in NIC. */ 3325 if (!hca_attr->geneve_tlv_opt) 3326 return rte_flow_error_set 3327 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3328 "Geneve TLV opt not supported"); 3329 /* Check if we already have geneve option with different type/class. */ 3330 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3331 geneve_opt_resource = sh->geneve_tlv_option_resource; 3332 if (geneve_opt_resource != NULL) 3333 if (geneve_opt_resource->option_class != spec->option_class || 3334 geneve_opt_resource->option_type != spec->option_type || 3335 geneve_opt_resource->length != spec->option_len) { 3336 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3337 return rte_flow_error_set(error, ENOTSUP, 3338 RTE_FLOW_ERROR_TYPE_ITEM, item, 3339 "Only one Geneve TLV option supported"); 3340 } 3341 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3342 return 0; 3343 } 3344 3345 /** 3346 * Validate MPLS item. 3347 * 3348 * @param[in] dev 3349 * Pointer to the rte_eth_dev structure. 3350 * @param[in] item 3351 * Item specification. 3352 * @param[in] item_flags 3353 * Bit-fields that holds the items detected until now. 3354 * @param[in] prev_layer 3355 * The protocol layer indicated in previous item. 3356 * @param[out] error 3357 * Pointer to error structure. 3358 * 3359 * @return 3360 * 0 on success, a negative errno value otherwise and rte_errno is set. 3361 */ 3362 int 3363 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3364 const struct rte_flow_item *item __rte_unused, 3365 uint64_t item_flags __rte_unused, 3366 uint64_t prev_layer __rte_unused, 3367 struct rte_flow_error *error) 3368 { 3369 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3370 const struct rte_flow_item_mpls *mask = item->mask; 3371 struct mlx5_priv *priv = dev->data->dev_private; 3372 int ret; 3373 3374 if (!priv->sh->dev_cap.mpls_en) 3375 return rte_flow_error_set(error, ENOTSUP, 3376 RTE_FLOW_ERROR_TYPE_ITEM, item, 3377 "MPLS not supported or" 3378 " disabled in firmware" 3379 " configuration."); 3380 /* MPLS over UDP, GRE is allowed */ 3381 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3382 MLX5_FLOW_LAYER_GRE | 3383 MLX5_FLOW_LAYER_GRE_KEY))) 3384 return rte_flow_error_set(error, EINVAL, 3385 RTE_FLOW_ERROR_TYPE_ITEM, item, 3386 "protocol filtering not compatible" 3387 " with MPLS layer"); 3388 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3389 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3390 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3391 return rte_flow_error_set(error, ENOTSUP, 3392 RTE_FLOW_ERROR_TYPE_ITEM, item, 3393 "multiple tunnel layers not" 3394 " supported"); 3395 if (!mask) 3396 mask = &rte_flow_item_mpls_mask; 3397 ret = mlx5_flow_item_acceptable 3398 (item, (const uint8_t *)mask, 3399 (const uint8_t *)&rte_flow_item_mpls_mask, 3400 sizeof(struct rte_flow_item_mpls), 3401 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3402 if (ret < 0) 3403 return ret; 3404 return 0; 3405 #else 3406 return rte_flow_error_set(error, ENOTSUP, 3407 RTE_FLOW_ERROR_TYPE_ITEM, item, 3408 "MPLS is not supported by Verbs, please" 3409 " update."); 3410 #endif 3411 } 3412 3413 /** 3414 * Validate NVGRE item. 3415 * 3416 * @param[in] item 3417 * Item specification. 3418 * @param[in] item_flags 3419 * Bit flags to mark detected items. 3420 * @param[in] target_protocol 3421 * The next protocol in the previous item. 3422 * @param[out] error 3423 * Pointer to error structure. 3424 * 3425 * @return 3426 * 0 on success, a negative errno value otherwise and rte_errno is set. 3427 */ 3428 int 3429 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3430 uint64_t item_flags, 3431 uint8_t target_protocol, 3432 struct rte_flow_error *error) 3433 { 3434 const struct rte_flow_item_nvgre *mask = item->mask; 3435 int ret; 3436 3437 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3438 return rte_flow_error_set(error, EINVAL, 3439 RTE_FLOW_ERROR_TYPE_ITEM, item, 3440 "protocol filtering not compatible" 3441 " with this GRE layer"); 3442 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3443 return rte_flow_error_set(error, ENOTSUP, 3444 RTE_FLOW_ERROR_TYPE_ITEM, item, 3445 "multiple tunnel layers not" 3446 " supported"); 3447 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3448 return rte_flow_error_set(error, ENOTSUP, 3449 RTE_FLOW_ERROR_TYPE_ITEM, item, 3450 "L3 Layer is missing"); 3451 if (!mask) 3452 mask = &rte_flow_item_nvgre_mask; 3453 ret = mlx5_flow_item_acceptable 3454 (item, (const uint8_t *)mask, 3455 (const uint8_t *)&rte_flow_item_nvgre_mask, 3456 sizeof(struct rte_flow_item_nvgre), 3457 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3458 if (ret < 0) 3459 return ret; 3460 return 0; 3461 } 3462 3463 /** 3464 * Validate eCPRI item. 3465 * 3466 * @param[in] item 3467 * Item specification. 3468 * @param[in] item_flags 3469 * Bit-fields that holds the items detected until now. 3470 * @param[in] last_item 3471 * Previous validated item in the pattern items. 3472 * @param[in] ether_type 3473 * Type in the ethernet layer header (including dot1q). 3474 * @param[in] acc_mask 3475 * Acceptable mask, if NULL default internal default mask 3476 * will be used to check whether item fields are supported. 3477 * @param[out] error 3478 * Pointer to error structure. 3479 * 3480 * @return 3481 * 0 on success, a negative errno value otherwise and rte_errno is set. 3482 */ 3483 int 3484 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3485 uint64_t item_flags, 3486 uint64_t last_item, 3487 uint16_t ether_type, 3488 const struct rte_flow_item_ecpri *acc_mask, 3489 struct rte_flow_error *error) 3490 { 3491 const struct rte_flow_item_ecpri *mask = item->mask; 3492 const struct rte_flow_item_ecpri nic_mask = { 3493 .hdr = { 3494 .common = { 3495 .u32 = 3496 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3497 .type = 0xFF, 3498 }).u32), 3499 }, 3500 .dummy[0] = 0xFFFFFFFF, 3501 }, 3502 }; 3503 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3504 MLX5_FLOW_LAYER_OUTER_VLAN); 3505 struct rte_flow_item_ecpri mask_lo; 3506 3507 if (!(last_item & outer_l2_vlan) && 3508 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3509 return rte_flow_error_set(error, EINVAL, 3510 RTE_FLOW_ERROR_TYPE_ITEM, item, 3511 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3512 if ((last_item & outer_l2_vlan) && ether_type && 3513 ether_type != RTE_ETHER_TYPE_ECPRI) 3514 return rte_flow_error_set(error, EINVAL, 3515 RTE_FLOW_ERROR_TYPE_ITEM, item, 3516 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3517 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3518 return rte_flow_error_set(error, EINVAL, 3519 RTE_FLOW_ERROR_TYPE_ITEM, item, 3520 "eCPRI with tunnel is not supported right now"); 3521 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3522 return rte_flow_error_set(error, ENOTSUP, 3523 RTE_FLOW_ERROR_TYPE_ITEM, item, 3524 "multiple L3 layers not supported"); 3525 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3526 return rte_flow_error_set(error, EINVAL, 3527 RTE_FLOW_ERROR_TYPE_ITEM, item, 3528 "eCPRI cannot coexist with a TCP layer"); 3529 /* In specification, eCPRI could be over UDP layer. */ 3530 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3531 return rte_flow_error_set(error, EINVAL, 3532 RTE_FLOW_ERROR_TYPE_ITEM, item, 3533 "eCPRI over UDP layer is not yet supported right now"); 3534 /* Mask for type field in common header could be zero. */ 3535 if (!mask) 3536 mask = &rte_flow_item_ecpri_mask; 3537 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3538 /* Input mask is in big-endian format. */ 3539 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3540 return rte_flow_error_set(error, EINVAL, 3541 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3542 "partial mask is not supported for protocol"); 3543 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3544 return rte_flow_error_set(error, EINVAL, 3545 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3546 "message header mask must be after a type mask"); 3547 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3548 acc_mask ? (const uint8_t *)acc_mask 3549 : (const uint8_t *)&nic_mask, 3550 sizeof(struct rte_flow_item_ecpri), 3551 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3552 } 3553 3554 static int 3555 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3556 const struct rte_flow_attr *attr __rte_unused, 3557 const struct rte_flow_item items[] __rte_unused, 3558 const struct rte_flow_action actions[] __rte_unused, 3559 bool external __rte_unused, 3560 int hairpin __rte_unused, 3561 struct rte_flow_error *error) 3562 { 3563 return rte_flow_error_set(error, ENOTSUP, 3564 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3565 } 3566 3567 static struct mlx5_flow * 3568 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3569 const struct rte_flow_attr *attr __rte_unused, 3570 const struct rte_flow_item items[] __rte_unused, 3571 const struct rte_flow_action actions[] __rte_unused, 3572 struct rte_flow_error *error) 3573 { 3574 rte_flow_error_set(error, ENOTSUP, 3575 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3576 return NULL; 3577 } 3578 3579 static int 3580 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3581 struct mlx5_flow *dev_flow __rte_unused, 3582 const struct rte_flow_attr *attr __rte_unused, 3583 const struct rte_flow_item items[] __rte_unused, 3584 const struct rte_flow_action actions[] __rte_unused, 3585 struct rte_flow_error *error) 3586 { 3587 return rte_flow_error_set(error, ENOTSUP, 3588 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3589 } 3590 3591 static int 3592 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3593 struct rte_flow *flow __rte_unused, 3594 struct rte_flow_error *error) 3595 { 3596 return rte_flow_error_set(error, ENOTSUP, 3597 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3598 } 3599 3600 static void 3601 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3602 struct rte_flow *flow __rte_unused) 3603 { 3604 } 3605 3606 static void 3607 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3608 struct rte_flow *flow __rte_unused) 3609 { 3610 } 3611 3612 static int 3613 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3614 struct rte_flow *flow __rte_unused, 3615 const struct rte_flow_action *actions __rte_unused, 3616 void *data __rte_unused, 3617 struct rte_flow_error *error) 3618 { 3619 return rte_flow_error_set(error, ENOTSUP, 3620 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3621 } 3622 3623 static int 3624 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3625 uint32_t domains __rte_unused, 3626 uint32_t flags __rte_unused) 3627 { 3628 return 0; 3629 } 3630 3631 /* Void driver to protect from null pointer reference. */ 3632 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3633 .validate = flow_null_validate, 3634 .prepare = flow_null_prepare, 3635 .translate = flow_null_translate, 3636 .apply = flow_null_apply, 3637 .remove = flow_null_remove, 3638 .destroy = flow_null_destroy, 3639 .query = flow_null_query, 3640 .sync_domain = flow_null_sync_domain, 3641 }; 3642 3643 /** 3644 * Select flow driver type according to flow attributes and device 3645 * configuration. 3646 * 3647 * @param[in] dev 3648 * Pointer to the dev structure. 3649 * @param[in] attr 3650 * Pointer to the flow attributes. 3651 * 3652 * @return 3653 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3654 */ 3655 static enum mlx5_flow_drv_type 3656 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3657 { 3658 struct mlx5_priv *priv = dev->data->dev_private; 3659 /* The OS can determine first a specific flow type (DV, VERBS) */ 3660 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3661 3662 if (type != MLX5_FLOW_TYPE_MAX) 3663 return type; 3664 /* 3665 * Currently when dv_flow_en == 2, only HW steering engine is 3666 * supported. New engines can also be chosen here if ready. 3667 */ 3668 if (priv->sh->config.dv_flow_en == 2) 3669 return MLX5_FLOW_TYPE_HW; 3670 /* If no OS specific type - continue with DV/VERBS selection */ 3671 if (attr->transfer && priv->sh->config.dv_esw_en) 3672 type = MLX5_FLOW_TYPE_DV; 3673 if (!attr->transfer) 3674 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3675 MLX5_FLOW_TYPE_VERBS; 3676 return type; 3677 } 3678 3679 #define flow_get_drv_ops(type) flow_drv_ops[type] 3680 3681 /** 3682 * Flow driver validation API. This abstracts calling driver specific functions. 3683 * The type of flow driver is determined according to flow attributes. 3684 * 3685 * @param[in] dev 3686 * Pointer to the dev structure. 3687 * @param[in] attr 3688 * Pointer to the flow attributes. 3689 * @param[in] items 3690 * Pointer to the list of items. 3691 * @param[in] actions 3692 * Pointer to the list of actions. 3693 * @param[in] external 3694 * This flow rule is created by request external to PMD. 3695 * @param[in] hairpin 3696 * Number of hairpin TX actions, 0 means classic flow. 3697 * @param[out] error 3698 * Pointer to the error structure. 3699 * 3700 * @return 3701 * 0 on success, a negative errno value otherwise and rte_errno is set. 3702 */ 3703 static inline int 3704 flow_drv_validate(struct rte_eth_dev *dev, 3705 const struct rte_flow_attr *attr, 3706 const struct rte_flow_item items[], 3707 const struct rte_flow_action actions[], 3708 bool external, int hairpin, struct rte_flow_error *error) 3709 { 3710 const struct mlx5_flow_driver_ops *fops; 3711 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3712 3713 fops = flow_get_drv_ops(type); 3714 return fops->validate(dev, attr, items, actions, external, 3715 hairpin, error); 3716 } 3717 3718 /** 3719 * Flow driver preparation API. This abstracts calling driver specific 3720 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3721 * calculates the size of memory required for device flow, allocates the memory, 3722 * initializes the device flow and returns the pointer. 3723 * 3724 * @note 3725 * This function initializes device flow structure such as dv or verbs in 3726 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3727 * rest. For example, adding returning device flow to flow->dev_flow list and 3728 * setting backward reference to the flow should be done out of this function. 3729 * layers field is not filled either. 3730 * 3731 * @param[in] dev 3732 * Pointer to the dev structure. 3733 * @param[in] attr 3734 * Pointer to the flow attributes. 3735 * @param[in] items 3736 * Pointer to the list of items. 3737 * @param[in] actions 3738 * Pointer to the list of actions. 3739 * @param[in] flow_idx 3740 * This memory pool index to the flow. 3741 * @param[out] error 3742 * Pointer to the error structure. 3743 * 3744 * @return 3745 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3746 */ 3747 static inline struct mlx5_flow * 3748 flow_drv_prepare(struct rte_eth_dev *dev, 3749 const struct rte_flow *flow, 3750 const struct rte_flow_attr *attr, 3751 const struct rte_flow_item items[], 3752 const struct rte_flow_action actions[], 3753 uint32_t flow_idx, 3754 struct rte_flow_error *error) 3755 { 3756 const struct mlx5_flow_driver_ops *fops; 3757 enum mlx5_flow_drv_type type = flow->drv_type; 3758 struct mlx5_flow *mlx5_flow = NULL; 3759 3760 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3761 fops = flow_get_drv_ops(type); 3762 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3763 if (mlx5_flow) 3764 mlx5_flow->flow_idx = flow_idx; 3765 return mlx5_flow; 3766 } 3767 3768 /** 3769 * Flow driver translation API. This abstracts calling driver specific 3770 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3771 * translates a generic flow into a driver flow. flow_drv_prepare() must 3772 * precede. 3773 * 3774 * @note 3775 * dev_flow->layers could be filled as a result of parsing during translation 3776 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3777 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3778 * flow->actions could be overwritten even though all the expanded dev_flows 3779 * have the same actions. 3780 * 3781 * @param[in] dev 3782 * Pointer to the rte dev structure. 3783 * @param[in, out] dev_flow 3784 * Pointer to the mlx5 flow. 3785 * @param[in] attr 3786 * Pointer to the flow attributes. 3787 * @param[in] items 3788 * Pointer to the list of items. 3789 * @param[in] actions 3790 * Pointer to the list of actions. 3791 * @param[out] error 3792 * Pointer to the error structure. 3793 * 3794 * @return 3795 * 0 on success, a negative errno value otherwise and rte_errno is set. 3796 */ 3797 static inline int 3798 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3799 const struct rte_flow_attr *attr, 3800 const struct rte_flow_item items[], 3801 const struct rte_flow_action actions[], 3802 struct rte_flow_error *error) 3803 { 3804 const struct mlx5_flow_driver_ops *fops; 3805 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3806 3807 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3808 fops = flow_get_drv_ops(type); 3809 return fops->translate(dev, dev_flow, attr, items, actions, error); 3810 } 3811 3812 /** 3813 * Flow driver apply API. This abstracts calling driver specific functions. 3814 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3815 * translated driver flows on to device. flow_drv_translate() must precede. 3816 * 3817 * @param[in] dev 3818 * Pointer to Ethernet device structure. 3819 * @param[in, out] flow 3820 * Pointer to flow structure. 3821 * @param[out] error 3822 * Pointer to error structure. 3823 * 3824 * @return 3825 * 0 on success, a negative errno value otherwise and rte_errno is set. 3826 */ 3827 static inline int 3828 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3829 struct rte_flow_error *error) 3830 { 3831 const struct mlx5_flow_driver_ops *fops; 3832 enum mlx5_flow_drv_type type = flow->drv_type; 3833 3834 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3835 fops = flow_get_drv_ops(type); 3836 return fops->apply(dev, flow, error); 3837 } 3838 3839 /** 3840 * Flow driver destroy API. This abstracts calling driver specific functions. 3841 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3842 * on device and releases resources of the flow. 3843 * 3844 * @param[in] dev 3845 * Pointer to Ethernet device. 3846 * @param[in, out] flow 3847 * Pointer to flow structure. 3848 */ 3849 static inline void 3850 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3851 { 3852 const struct mlx5_flow_driver_ops *fops; 3853 enum mlx5_flow_drv_type type = flow->drv_type; 3854 3855 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3856 fops = flow_get_drv_ops(type); 3857 fops->destroy(dev, flow); 3858 } 3859 3860 /** 3861 * Flow driver find RSS policy tbl API. This abstracts calling driver 3862 * specific functions. Parent flow (rte_flow) should have driver 3863 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3864 * 3865 * @param[in] dev 3866 * Pointer to Ethernet device. 3867 * @param[in, out] flow 3868 * Pointer to flow structure. 3869 * @param[in] policy 3870 * Pointer to meter policy table. 3871 * @param[in] rss_desc 3872 * Pointer to rss_desc 3873 */ 3874 static struct mlx5_flow_meter_sub_policy * 3875 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3876 struct rte_flow *flow, 3877 struct mlx5_flow_meter_policy *policy, 3878 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3879 { 3880 const struct mlx5_flow_driver_ops *fops; 3881 enum mlx5_flow_drv_type type = flow->drv_type; 3882 3883 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3884 fops = flow_get_drv_ops(type); 3885 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3886 } 3887 3888 /** 3889 * Flow driver color tag rule API. This abstracts calling driver 3890 * specific functions. Parent flow (rte_flow) should have driver 3891 * type (drv_type). It will create the color tag rules in hierarchy meter. 3892 * 3893 * @param[in] dev 3894 * Pointer to Ethernet device. 3895 * @param[in, out] flow 3896 * Pointer to flow structure. 3897 * @param[in] fm 3898 * Pointer to flow meter structure. 3899 * @param[in] src_port 3900 * The src port this extra rule should use. 3901 * @param[in] item 3902 * The src port id match item. 3903 * @param[out] error 3904 * Pointer to error structure. 3905 */ 3906 static int 3907 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3908 struct rte_flow *flow, 3909 struct mlx5_flow_meter_info *fm, 3910 int32_t src_port, 3911 const struct rte_flow_item *item, 3912 struct rte_flow_error *error) 3913 { 3914 const struct mlx5_flow_driver_ops *fops; 3915 enum mlx5_flow_drv_type type = flow->drv_type; 3916 3917 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3918 fops = flow_get_drv_ops(type); 3919 return fops->meter_hierarchy_rule_create(dev, fm, 3920 src_port, item, error); 3921 } 3922 3923 /** 3924 * Get RSS action from the action list. 3925 * 3926 * @param[in] dev 3927 * Pointer to Ethernet device. 3928 * @param[in] actions 3929 * Pointer to the list of actions. 3930 * @param[in] flow 3931 * Parent flow structure pointer. 3932 * 3933 * @return 3934 * Pointer to the RSS action if exist, else return NULL. 3935 */ 3936 static const struct rte_flow_action_rss* 3937 flow_get_rss_action(struct rte_eth_dev *dev, 3938 const struct rte_flow_action actions[]) 3939 { 3940 struct mlx5_priv *priv = dev->data->dev_private; 3941 const struct rte_flow_action_rss *rss = NULL; 3942 struct mlx5_meter_policy_action_container *acg; 3943 struct mlx5_meter_policy_action_container *acy; 3944 3945 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3946 switch (actions->type) { 3947 case RTE_FLOW_ACTION_TYPE_RSS: 3948 rss = actions->conf; 3949 break; 3950 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3951 { 3952 const struct rte_flow_action_sample *sample = 3953 actions->conf; 3954 const struct rte_flow_action *act = sample->actions; 3955 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3956 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3957 rss = act->conf; 3958 break; 3959 } 3960 case RTE_FLOW_ACTION_TYPE_METER: 3961 { 3962 uint32_t mtr_idx; 3963 struct mlx5_flow_meter_info *fm; 3964 struct mlx5_flow_meter_policy *policy; 3965 const struct rte_flow_action_meter *mtr = actions->conf; 3966 3967 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3968 if (fm && !fm->def_policy) { 3969 policy = mlx5_flow_meter_policy_find(dev, 3970 fm->policy_id, NULL); 3971 MLX5_ASSERT(policy); 3972 if (policy->is_hierarchy) { 3973 policy = 3974 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3975 policy); 3976 if (!policy) 3977 return NULL; 3978 } 3979 if (policy->is_rss) { 3980 acg = 3981 &policy->act_cnt[RTE_COLOR_GREEN]; 3982 acy = 3983 &policy->act_cnt[RTE_COLOR_YELLOW]; 3984 if (acg->fate_action == 3985 MLX5_FLOW_FATE_SHARED_RSS) 3986 rss = acg->rss->conf; 3987 else if (acy->fate_action == 3988 MLX5_FLOW_FATE_SHARED_RSS) 3989 rss = acy->rss->conf; 3990 } 3991 } 3992 break; 3993 } 3994 default: 3995 break; 3996 } 3997 } 3998 return rss; 3999 } 4000 4001 /** 4002 * Get ASO age action by index. 4003 * 4004 * @param[in] dev 4005 * Pointer to the Ethernet device structure. 4006 * @param[in] age_idx 4007 * Index to the ASO age action. 4008 * 4009 * @return 4010 * The specified ASO age action. 4011 */ 4012 struct mlx5_aso_age_action* 4013 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4014 { 4015 uint16_t pool_idx = age_idx & UINT16_MAX; 4016 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4017 struct mlx5_priv *priv = dev->data->dev_private; 4018 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4019 struct mlx5_aso_age_pool *pool; 4020 4021 rte_rwlock_read_lock(&mng->resize_rwl); 4022 pool = mng->pools[pool_idx]; 4023 rte_rwlock_read_unlock(&mng->resize_rwl); 4024 return &pool->actions[offset - 1]; 4025 } 4026 4027 /* maps indirect action to translated direct in some actions array */ 4028 struct mlx5_translated_action_handle { 4029 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4030 int index; /**< Index in related array of rte_flow_action. */ 4031 }; 4032 4033 /** 4034 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4035 * direct action if translation possible. 4036 * This functionality used to run same execution path for both direct and 4037 * indirect actions on flow create. All necessary preparations for indirect 4038 * action handling should be performed on *handle* actions list returned 4039 * from this call. 4040 * 4041 * @param[in] dev 4042 * Pointer to Ethernet device. 4043 * @param[in] actions 4044 * List of actions to translate. 4045 * @param[out] handle 4046 * List to store translated indirect action object handles. 4047 * @param[in, out] indir_n 4048 * Size of *handle* array. On return should be updated with number of 4049 * indirect actions retrieved from the *actions* list. 4050 * @param[out] translated_actions 4051 * List of actions where all indirect actions were translated to direct 4052 * if possible. NULL if no translation took place. 4053 * @param[out] error 4054 * Pointer to the error structure. 4055 * 4056 * @return 4057 * 0 on success, a negative errno value otherwise and rte_errno is set. 4058 */ 4059 static int 4060 flow_action_handles_translate(struct rte_eth_dev *dev, 4061 const struct rte_flow_action actions[], 4062 struct mlx5_translated_action_handle *handle, 4063 int *indir_n, 4064 struct rte_flow_action **translated_actions, 4065 struct rte_flow_error *error) 4066 { 4067 struct mlx5_priv *priv = dev->data->dev_private; 4068 struct rte_flow_action *translated = NULL; 4069 size_t actions_size; 4070 int n; 4071 int copied_n = 0; 4072 struct mlx5_translated_action_handle *handle_end = NULL; 4073 4074 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4075 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4076 continue; 4077 if (copied_n == *indir_n) { 4078 return rte_flow_error_set 4079 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4080 NULL, "too many shared actions"); 4081 } 4082 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4083 sizeof(actions[n].conf)); 4084 handle[copied_n].index = n; 4085 copied_n++; 4086 } 4087 n++; 4088 *indir_n = copied_n; 4089 if (!copied_n) 4090 return 0; 4091 actions_size = sizeof(struct rte_flow_action) * n; 4092 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4093 if (!translated) { 4094 rte_errno = ENOMEM; 4095 return -ENOMEM; 4096 } 4097 memcpy(translated, actions, actions_size); 4098 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4099 struct mlx5_shared_action_rss *shared_rss; 4100 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4101 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4102 uint32_t idx = act_idx & 4103 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4104 4105 switch (type) { 4106 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4107 shared_rss = mlx5_ipool_get 4108 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4109 translated[handle->index].type = 4110 RTE_FLOW_ACTION_TYPE_RSS; 4111 translated[handle->index].conf = 4112 &shared_rss->origin; 4113 break; 4114 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4115 translated[handle->index].type = 4116 (enum rte_flow_action_type) 4117 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4118 translated[handle->index].conf = (void *)(uintptr_t)idx; 4119 break; 4120 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4121 if (priv->sh->flow_hit_aso_en) { 4122 translated[handle->index].type = 4123 (enum rte_flow_action_type) 4124 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4125 translated[handle->index].conf = 4126 (void *)(uintptr_t)idx; 4127 break; 4128 } 4129 /* Fall-through */ 4130 case MLX5_INDIRECT_ACTION_TYPE_CT: 4131 if (priv->sh->ct_aso_en) { 4132 translated[handle->index].type = 4133 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4134 translated[handle->index].conf = 4135 (void *)(uintptr_t)idx; 4136 break; 4137 } 4138 /* Fall-through */ 4139 default: 4140 mlx5_free(translated); 4141 return rte_flow_error_set 4142 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4143 NULL, "invalid indirect action type"); 4144 } 4145 } 4146 *translated_actions = translated; 4147 return 0; 4148 } 4149 4150 /** 4151 * Get Shared RSS action from the action list. 4152 * 4153 * @param[in] dev 4154 * Pointer to Ethernet device. 4155 * @param[in] shared 4156 * Pointer to the list of actions. 4157 * @param[in] shared_n 4158 * Actions list length. 4159 * 4160 * @return 4161 * The MLX5 RSS action ID if exists, otherwise return 0. 4162 */ 4163 static uint32_t 4164 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4165 struct mlx5_translated_action_handle *handle, 4166 int shared_n) 4167 { 4168 struct mlx5_translated_action_handle *handle_end; 4169 struct mlx5_priv *priv = dev->data->dev_private; 4170 struct mlx5_shared_action_rss *shared_rss; 4171 4172 4173 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4174 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4175 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4176 uint32_t idx = act_idx & 4177 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4178 switch (type) { 4179 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4180 shared_rss = mlx5_ipool_get 4181 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4182 idx); 4183 __atomic_add_fetch(&shared_rss->refcnt, 1, 4184 __ATOMIC_RELAXED); 4185 return idx; 4186 default: 4187 break; 4188 } 4189 } 4190 return 0; 4191 } 4192 4193 static unsigned int 4194 find_graph_root(uint32_t rss_level) 4195 { 4196 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4197 MLX5_EXPANSION_ROOT_OUTER; 4198 } 4199 4200 /** 4201 * Get layer flags from the prefix flow. 4202 * 4203 * Some flows may be split to several subflows, the prefix subflow gets the 4204 * match items and the suffix sub flow gets the actions. 4205 * Some actions need the user defined match item flags to get the detail for 4206 * the action. 4207 * This function helps the suffix flow to get the item layer flags from prefix 4208 * subflow. 4209 * 4210 * @param[in] dev_flow 4211 * Pointer the created prefix subflow. 4212 * 4213 * @return 4214 * The layers get from prefix subflow. 4215 */ 4216 static inline uint64_t 4217 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4218 { 4219 uint64_t layers = 0; 4220 4221 /* 4222 * Layers bits could be localization, but usually the compiler will 4223 * help to do the optimization work for source code. 4224 * If no decap actions, use the layers directly. 4225 */ 4226 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4227 return dev_flow->handle->layers; 4228 /* Convert L3 layers with decap action. */ 4229 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4230 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4231 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4232 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4233 /* Convert L4 layers with decap action. */ 4234 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4235 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4236 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4237 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4238 return layers; 4239 } 4240 4241 /** 4242 * Get metadata split action information. 4243 * 4244 * @param[in] actions 4245 * Pointer to the list of actions. 4246 * @param[out] qrss 4247 * Pointer to the return pointer. 4248 * @param[out] qrss_type 4249 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4250 * if no QUEUE/RSS is found. 4251 * @param[out] encap_idx 4252 * Pointer to the index of the encap action if exists, otherwise the last 4253 * action index. 4254 * 4255 * @return 4256 * Total number of actions. 4257 */ 4258 static int 4259 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4260 const struct rte_flow_action **qrss, 4261 int *encap_idx) 4262 { 4263 const struct rte_flow_action_raw_encap *raw_encap; 4264 int actions_n = 0; 4265 int raw_decap_idx = -1; 4266 4267 *encap_idx = -1; 4268 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4269 switch (actions->type) { 4270 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4271 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4272 *encap_idx = actions_n; 4273 break; 4274 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4275 raw_decap_idx = actions_n; 4276 break; 4277 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4278 raw_encap = actions->conf; 4279 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4280 *encap_idx = raw_decap_idx != -1 ? 4281 raw_decap_idx : actions_n; 4282 break; 4283 case RTE_FLOW_ACTION_TYPE_QUEUE: 4284 case RTE_FLOW_ACTION_TYPE_RSS: 4285 *qrss = actions; 4286 break; 4287 default: 4288 break; 4289 } 4290 actions_n++; 4291 } 4292 if (*encap_idx == -1) 4293 *encap_idx = actions_n; 4294 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4295 return actions_n + 1; 4296 } 4297 4298 /** 4299 * Check if the action will change packet. 4300 * 4301 * @param dev 4302 * Pointer to Ethernet device. 4303 * @param[in] type 4304 * action type. 4305 * 4306 * @return 4307 * true if action will change packet, false otherwise. 4308 */ 4309 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4310 enum rte_flow_action_type type) 4311 { 4312 struct mlx5_priv *priv = dev->data->dev_private; 4313 4314 switch (type) { 4315 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4316 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4317 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4318 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4319 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4320 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4321 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4322 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4323 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4324 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4325 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4326 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4327 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4328 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4329 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4330 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4331 case RTE_FLOW_ACTION_TYPE_SET_META: 4332 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4333 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4334 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4335 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4336 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4337 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4338 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4339 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4340 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4341 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4342 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4343 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4344 return true; 4345 case RTE_FLOW_ACTION_TYPE_FLAG: 4346 case RTE_FLOW_ACTION_TYPE_MARK: 4347 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4348 return true; 4349 else 4350 return false; 4351 default: 4352 return false; 4353 } 4354 } 4355 4356 /** 4357 * Check meter action from the action list. 4358 * 4359 * @param dev 4360 * Pointer to Ethernet device. 4361 * @param[in] actions 4362 * Pointer to the list of actions. 4363 * @param[out] has_mtr 4364 * Pointer to the meter exist flag. 4365 * @param[out] has_modify 4366 * Pointer to the flag showing there's packet change action. 4367 * @param[out] meter_id 4368 * Pointer to the meter id. 4369 * 4370 * @return 4371 * Total number of actions. 4372 */ 4373 static int 4374 flow_check_meter_action(struct rte_eth_dev *dev, 4375 const struct rte_flow_action actions[], 4376 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4377 { 4378 const struct rte_flow_action_meter *mtr = NULL; 4379 int actions_n = 0; 4380 4381 MLX5_ASSERT(has_mtr); 4382 *has_mtr = false; 4383 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4384 switch (actions->type) { 4385 case RTE_FLOW_ACTION_TYPE_METER: 4386 mtr = actions->conf; 4387 *meter_id = mtr->mtr_id; 4388 *has_mtr = true; 4389 break; 4390 default: 4391 break; 4392 } 4393 if (!*has_mtr) 4394 *has_modify |= flow_check_modify_action_type(dev, 4395 actions->type); 4396 actions_n++; 4397 } 4398 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4399 return actions_n + 1; 4400 } 4401 4402 /** 4403 * Check if the flow should be split due to hairpin. 4404 * The reason for the split is that in current HW we can't 4405 * support encap and push-vlan on Rx, so if a flow contains 4406 * these actions we move it to Tx. 4407 * 4408 * @param dev 4409 * Pointer to Ethernet device. 4410 * @param[in] attr 4411 * Flow rule attributes. 4412 * @param[in] actions 4413 * Associated actions (list terminated by the END action). 4414 * 4415 * @return 4416 * > 0 the number of actions and the flow should be split, 4417 * 0 when no split required. 4418 */ 4419 static int 4420 flow_check_hairpin_split(struct rte_eth_dev *dev, 4421 const struct rte_flow_attr *attr, 4422 const struct rte_flow_action actions[]) 4423 { 4424 int queue_action = 0; 4425 int action_n = 0; 4426 int split = 0; 4427 const struct rte_flow_action_queue *queue; 4428 const struct rte_flow_action_rss *rss; 4429 const struct rte_flow_action_raw_encap *raw_encap; 4430 const struct rte_eth_hairpin_conf *conf; 4431 4432 if (!attr->ingress) 4433 return 0; 4434 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4435 switch (actions->type) { 4436 case RTE_FLOW_ACTION_TYPE_QUEUE: 4437 queue = actions->conf; 4438 if (queue == NULL) 4439 return 0; 4440 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4441 if (conf == NULL || conf->tx_explicit != 0) 4442 return 0; 4443 queue_action = 1; 4444 action_n++; 4445 break; 4446 case RTE_FLOW_ACTION_TYPE_RSS: 4447 rss = actions->conf; 4448 if (rss == NULL || rss->queue_num == 0) 4449 return 0; 4450 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4451 if (conf == NULL || conf->tx_explicit != 0) 4452 return 0; 4453 queue_action = 1; 4454 action_n++; 4455 break; 4456 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4457 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4458 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4459 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4460 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4461 split++; 4462 action_n++; 4463 break; 4464 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4465 raw_encap = actions->conf; 4466 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4467 split++; 4468 action_n++; 4469 break; 4470 default: 4471 action_n++; 4472 break; 4473 } 4474 } 4475 if (split && queue_action) 4476 return action_n; 4477 return 0; 4478 } 4479 4480 /* Declare flow create/destroy prototype in advance. */ 4481 static uint32_t 4482 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4483 const struct rte_flow_attr *attr, 4484 const struct rte_flow_item items[], 4485 const struct rte_flow_action actions[], 4486 bool external, struct rte_flow_error *error); 4487 4488 static void 4489 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4490 uint32_t flow_idx); 4491 4492 int 4493 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4494 struct mlx5_list_entry *entry, void *cb_ctx) 4495 { 4496 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4497 struct mlx5_flow_mreg_copy_resource *mcp_res = 4498 container_of(entry, typeof(*mcp_res), hlist_ent); 4499 4500 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4501 } 4502 4503 struct mlx5_list_entry * 4504 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4505 { 4506 struct rte_eth_dev *dev = tool_ctx; 4507 struct mlx5_priv *priv = dev->data->dev_private; 4508 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4509 struct mlx5_flow_mreg_copy_resource *mcp_res; 4510 struct rte_flow_error *error = ctx->error; 4511 uint32_t idx = 0; 4512 int ret; 4513 uint32_t mark_id = *(uint32_t *)(ctx->data); 4514 struct rte_flow_attr attr = { 4515 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4516 .ingress = 1, 4517 }; 4518 struct mlx5_rte_flow_item_tag tag_spec = { 4519 .data = mark_id, 4520 }; 4521 struct rte_flow_item items[] = { 4522 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4523 }; 4524 struct rte_flow_action_mark ftag = { 4525 .id = mark_id, 4526 }; 4527 struct mlx5_flow_action_copy_mreg cp_mreg = { 4528 .dst = REG_B, 4529 .src = REG_NON, 4530 }; 4531 struct rte_flow_action_jump jump = { 4532 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4533 }; 4534 struct rte_flow_action actions[] = { 4535 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4536 }; 4537 4538 /* Fill the register fields in the flow. */ 4539 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4540 if (ret < 0) 4541 return NULL; 4542 tag_spec.id = ret; 4543 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4544 if (ret < 0) 4545 return NULL; 4546 cp_mreg.src = ret; 4547 /* Provide the full width of FLAG specific value. */ 4548 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4549 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4550 /* Build a new flow. */ 4551 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4552 items[0] = (struct rte_flow_item){ 4553 .type = (enum rte_flow_item_type) 4554 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4555 .spec = &tag_spec, 4556 }; 4557 items[1] = (struct rte_flow_item){ 4558 .type = RTE_FLOW_ITEM_TYPE_END, 4559 }; 4560 actions[0] = (struct rte_flow_action){ 4561 .type = (enum rte_flow_action_type) 4562 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4563 .conf = &ftag, 4564 }; 4565 actions[1] = (struct rte_flow_action){ 4566 .type = (enum rte_flow_action_type) 4567 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4568 .conf = &cp_mreg, 4569 }; 4570 actions[2] = (struct rte_flow_action){ 4571 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4572 .conf = &jump, 4573 }; 4574 actions[3] = (struct rte_flow_action){ 4575 .type = RTE_FLOW_ACTION_TYPE_END, 4576 }; 4577 } else { 4578 /* Default rule, wildcard match. */ 4579 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4580 items[0] = (struct rte_flow_item){ 4581 .type = RTE_FLOW_ITEM_TYPE_END, 4582 }; 4583 actions[0] = (struct rte_flow_action){ 4584 .type = (enum rte_flow_action_type) 4585 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4586 .conf = &cp_mreg, 4587 }; 4588 actions[1] = (struct rte_flow_action){ 4589 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4590 .conf = &jump, 4591 }; 4592 actions[2] = (struct rte_flow_action){ 4593 .type = RTE_FLOW_ACTION_TYPE_END, 4594 }; 4595 } 4596 /* Build a new entry. */ 4597 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4598 if (!mcp_res) { 4599 rte_errno = ENOMEM; 4600 return NULL; 4601 } 4602 mcp_res->idx = idx; 4603 mcp_res->mark_id = mark_id; 4604 /* 4605 * The copy Flows are not included in any list. There 4606 * ones are referenced from other Flows and can not 4607 * be applied, removed, deleted in arbitrary order 4608 * by list traversing. 4609 */ 4610 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4611 &attr, items, actions, false, error); 4612 if (!mcp_res->rix_flow) { 4613 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4614 return NULL; 4615 } 4616 return &mcp_res->hlist_ent; 4617 } 4618 4619 struct mlx5_list_entry * 4620 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4621 void *cb_ctx __rte_unused) 4622 { 4623 struct rte_eth_dev *dev = tool_ctx; 4624 struct mlx5_priv *priv = dev->data->dev_private; 4625 struct mlx5_flow_mreg_copy_resource *mcp_res; 4626 uint32_t idx = 0; 4627 4628 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4629 if (!mcp_res) { 4630 rte_errno = ENOMEM; 4631 return NULL; 4632 } 4633 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4634 mcp_res->idx = idx; 4635 return &mcp_res->hlist_ent; 4636 } 4637 4638 void 4639 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4640 { 4641 struct mlx5_flow_mreg_copy_resource *mcp_res = 4642 container_of(entry, typeof(*mcp_res), hlist_ent); 4643 struct rte_eth_dev *dev = tool_ctx; 4644 struct mlx5_priv *priv = dev->data->dev_private; 4645 4646 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4647 } 4648 4649 /** 4650 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4651 * 4652 * As mark_id is unique, if there's already a registered flow for the mark_id, 4653 * return by increasing the reference counter of the resource. Otherwise, create 4654 * the resource (mcp_res) and flow. 4655 * 4656 * Flow looks like, 4657 * - If ingress port is ANY and reg_c[1] is mark_id, 4658 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4659 * 4660 * For default flow (zero mark_id), flow is like, 4661 * - If ingress port is ANY, 4662 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4663 * 4664 * @param dev 4665 * Pointer to Ethernet device. 4666 * @param mark_id 4667 * ID of MARK action, zero means default flow for META. 4668 * @param[out] error 4669 * Perform verbose error reporting if not NULL. 4670 * 4671 * @return 4672 * Associated resource on success, NULL otherwise and rte_errno is set. 4673 */ 4674 static struct mlx5_flow_mreg_copy_resource * 4675 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4676 struct rte_flow_error *error) 4677 { 4678 struct mlx5_priv *priv = dev->data->dev_private; 4679 struct mlx5_list_entry *entry; 4680 struct mlx5_flow_cb_ctx ctx = { 4681 .dev = dev, 4682 .error = error, 4683 .data = &mark_id, 4684 }; 4685 4686 /* Check if already registered. */ 4687 MLX5_ASSERT(priv->mreg_cp_tbl); 4688 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4689 if (!entry) 4690 return NULL; 4691 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4692 hlist_ent); 4693 } 4694 4695 void 4696 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4697 { 4698 struct mlx5_flow_mreg_copy_resource *mcp_res = 4699 container_of(entry, typeof(*mcp_res), hlist_ent); 4700 struct rte_eth_dev *dev = tool_ctx; 4701 struct mlx5_priv *priv = dev->data->dev_private; 4702 4703 MLX5_ASSERT(mcp_res->rix_flow); 4704 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4705 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4706 } 4707 4708 /** 4709 * Release flow in RX_CP_TBL. 4710 * 4711 * @param dev 4712 * Pointer to Ethernet device. 4713 * @flow 4714 * Parent flow for wich copying is provided. 4715 */ 4716 static void 4717 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4718 struct rte_flow *flow) 4719 { 4720 struct mlx5_flow_mreg_copy_resource *mcp_res; 4721 struct mlx5_priv *priv = dev->data->dev_private; 4722 4723 if (!flow->rix_mreg_copy) 4724 return; 4725 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4726 flow->rix_mreg_copy); 4727 if (!mcp_res || !priv->mreg_cp_tbl) 4728 return; 4729 MLX5_ASSERT(mcp_res->rix_flow); 4730 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4731 flow->rix_mreg_copy = 0; 4732 } 4733 4734 /** 4735 * Remove the default copy action from RX_CP_TBL. 4736 * 4737 * This functions is called in the mlx5_dev_start(). No thread safe 4738 * is guaranteed. 4739 * 4740 * @param dev 4741 * Pointer to Ethernet device. 4742 */ 4743 static void 4744 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4745 { 4746 struct mlx5_list_entry *entry; 4747 struct mlx5_priv *priv = dev->data->dev_private; 4748 struct mlx5_flow_cb_ctx ctx; 4749 uint32_t mark_id; 4750 4751 /* Check if default flow is registered. */ 4752 if (!priv->mreg_cp_tbl) 4753 return; 4754 mark_id = MLX5_DEFAULT_COPY_ID; 4755 ctx.data = &mark_id; 4756 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4757 if (!entry) 4758 return; 4759 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4760 } 4761 4762 /** 4763 * Add the default copy action in in RX_CP_TBL. 4764 * 4765 * This functions is called in the mlx5_dev_start(). No thread safe 4766 * is guaranteed. 4767 * 4768 * @param dev 4769 * Pointer to Ethernet device. 4770 * @param[out] error 4771 * Perform verbose error reporting if not NULL. 4772 * 4773 * @return 4774 * 0 for success, negative value otherwise and rte_errno is set. 4775 */ 4776 static int 4777 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4778 struct rte_flow_error *error) 4779 { 4780 struct mlx5_priv *priv = dev->data->dev_private; 4781 struct mlx5_flow_mreg_copy_resource *mcp_res; 4782 struct mlx5_flow_cb_ctx ctx; 4783 uint32_t mark_id; 4784 4785 /* Check whether extensive metadata feature is engaged. */ 4786 if (!priv->sh->config.dv_flow_en || 4787 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4788 !mlx5_flow_ext_mreg_supported(dev) || 4789 !priv->sh->dv_regc0_mask) 4790 return 0; 4791 /* 4792 * Add default mreg copy flow may be called multiple time, but 4793 * only be called once in stop. Avoid register it twice. 4794 */ 4795 mark_id = MLX5_DEFAULT_COPY_ID; 4796 ctx.data = &mark_id; 4797 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4798 return 0; 4799 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4800 if (!mcp_res) 4801 return -rte_errno; 4802 return 0; 4803 } 4804 4805 /** 4806 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4807 * 4808 * All the flow having Q/RSS action should be split by 4809 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4810 * performs the following, 4811 * - CQE->flow_tag := reg_c[1] (MARK) 4812 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4813 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4814 * but there should be a flow per each MARK ID set by MARK action. 4815 * 4816 * For the aforementioned reason, if there's a MARK action in flow's action 4817 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4818 * the MARK ID to CQE's flow_tag like, 4819 * - If reg_c[1] is mark_id, 4820 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4821 * 4822 * For SET_META action which stores value in reg_c[0], as the destination is 4823 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4824 * MARK ID means the default flow. The default flow looks like, 4825 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4826 * 4827 * @param dev 4828 * Pointer to Ethernet device. 4829 * @param flow 4830 * Pointer to flow structure. 4831 * @param[in] actions 4832 * Pointer to the list of actions. 4833 * @param[out] error 4834 * Perform verbose error reporting if not NULL. 4835 * 4836 * @return 4837 * 0 on success, negative value otherwise and rte_errno is set. 4838 */ 4839 static int 4840 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4841 struct rte_flow *flow, 4842 const struct rte_flow_action *actions, 4843 struct rte_flow_error *error) 4844 { 4845 struct mlx5_priv *priv = dev->data->dev_private; 4846 struct mlx5_sh_config *config = &priv->sh->config; 4847 struct mlx5_flow_mreg_copy_resource *mcp_res; 4848 const struct rte_flow_action_mark *mark; 4849 4850 /* Check whether extensive metadata feature is engaged. */ 4851 if (!config->dv_flow_en || 4852 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4853 !mlx5_flow_ext_mreg_supported(dev) || 4854 !priv->sh->dv_regc0_mask) 4855 return 0; 4856 /* Find MARK action. */ 4857 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4858 switch (actions->type) { 4859 case RTE_FLOW_ACTION_TYPE_FLAG: 4860 mcp_res = flow_mreg_add_copy_action 4861 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4862 if (!mcp_res) 4863 return -rte_errno; 4864 flow->rix_mreg_copy = mcp_res->idx; 4865 return 0; 4866 case RTE_FLOW_ACTION_TYPE_MARK: 4867 mark = (const struct rte_flow_action_mark *) 4868 actions->conf; 4869 mcp_res = 4870 flow_mreg_add_copy_action(dev, mark->id, error); 4871 if (!mcp_res) 4872 return -rte_errno; 4873 flow->rix_mreg_copy = mcp_res->idx; 4874 return 0; 4875 default: 4876 break; 4877 } 4878 } 4879 return 0; 4880 } 4881 4882 #define MLX5_MAX_SPLIT_ACTIONS 24 4883 #define MLX5_MAX_SPLIT_ITEMS 24 4884 4885 /** 4886 * Split the hairpin flow. 4887 * Since HW can't support encap and push-vlan on Rx, we move these 4888 * actions to Tx. 4889 * If the count action is after the encap then we also 4890 * move the count action. in this case the count will also measure 4891 * the outer bytes. 4892 * 4893 * @param dev 4894 * Pointer to Ethernet device. 4895 * @param[in] actions 4896 * Associated actions (list terminated by the END action). 4897 * @param[out] actions_rx 4898 * Rx flow actions. 4899 * @param[out] actions_tx 4900 * Tx flow actions.. 4901 * @param[out] pattern_tx 4902 * The pattern items for the Tx flow. 4903 * @param[out] flow_id 4904 * The flow ID connected to this flow. 4905 * 4906 * @return 4907 * 0 on success. 4908 */ 4909 static int 4910 flow_hairpin_split(struct rte_eth_dev *dev, 4911 const struct rte_flow_action actions[], 4912 struct rte_flow_action actions_rx[], 4913 struct rte_flow_action actions_tx[], 4914 struct rte_flow_item pattern_tx[], 4915 uint32_t flow_id) 4916 { 4917 const struct rte_flow_action_raw_encap *raw_encap; 4918 const struct rte_flow_action_raw_decap *raw_decap; 4919 struct mlx5_rte_flow_action_set_tag *set_tag; 4920 struct rte_flow_action *tag_action; 4921 struct mlx5_rte_flow_item_tag *tag_item; 4922 struct rte_flow_item *item; 4923 char *addr; 4924 int encap = 0; 4925 4926 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4927 switch (actions->type) { 4928 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4929 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4930 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4931 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4932 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4933 rte_memcpy(actions_tx, actions, 4934 sizeof(struct rte_flow_action)); 4935 actions_tx++; 4936 break; 4937 case RTE_FLOW_ACTION_TYPE_COUNT: 4938 if (encap) { 4939 rte_memcpy(actions_tx, actions, 4940 sizeof(struct rte_flow_action)); 4941 actions_tx++; 4942 } else { 4943 rte_memcpy(actions_rx, actions, 4944 sizeof(struct rte_flow_action)); 4945 actions_rx++; 4946 } 4947 break; 4948 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4949 raw_encap = actions->conf; 4950 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4951 memcpy(actions_tx, actions, 4952 sizeof(struct rte_flow_action)); 4953 actions_tx++; 4954 encap = 1; 4955 } else { 4956 rte_memcpy(actions_rx, actions, 4957 sizeof(struct rte_flow_action)); 4958 actions_rx++; 4959 } 4960 break; 4961 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4962 raw_decap = actions->conf; 4963 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4964 memcpy(actions_tx, actions, 4965 sizeof(struct rte_flow_action)); 4966 actions_tx++; 4967 } else { 4968 rte_memcpy(actions_rx, actions, 4969 sizeof(struct rte_flow_action)); 4970 actions_rx++; 4971 } 4972 break; 4973 default: 4974 rte_memcpy(actions_rx, actions, 4975 sizeof(struct rte_flow_action)); 4976 actions_rx++; 4977 break; 4978 } 4979 } 4980 /* Add set meta action and end action for the Rx flow. */ 4981 tag_action = actions_rx; 4982 tag_action->type = (enum rte_flow_action_type) 4983 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4984 actions_rx++; 4985 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4986 actions_rx++; 4987 set_tag = (void *)actions_rx; 4988 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4989 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4990 .data = flow_id, 4991 }; 4992 MLX5_ASSERT(set_tag->id > REG_NON); 4993 tag_action->conf = set_tag; 4994 /* Create Tx item list. */ 4995 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4996 addr = (void *)&pattern_tx[2]; 4997 item = pattern_tx; 4998 item->type = (enum rte_flow_item_type) 4999 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5000 tag_item = (void *)addr; 5001 tag_item->data = flow_id; 5002 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5003 MLX5_ASSERT(set_tag->id > REG_NON); 5004 item->spec = tag_item; 5005 addr += sizeof(struct mlx5_rte_flow_item_tag); 5006 tag_item = (void *)addr; 5007 tag_item->data = UINT32_MAX; 5008 tag_item->id = UINT16_MAX; 5009 item->mask = tag_item; 5010 item->last = NULL; 5011 item++; 5012 item->type = RTE_FLOW_ITEM_TYPE_END; 5013 return 0; 5014 } 5015 5016 /** 5017 * The last stage of splitting chain, just creates the subflow 5018 * without any modification. 5019 * 5020 * @param[in] dev 5021 * Pointer to Ethernet device. 5022 * @param[in] flow 5023 * Parent flow structure pointer. 5024 * @param[in, out] sub_flow 5025 * Pointer to return the created subflow, may be NULL. 5026 * @param[in] attr 5027 * Flow rule attributes. 5028 * @param[in] items 5029 * Pattern specification (list terminated by the END pattern item). 5030 * @param[in] actions 5031 * Associated actions (list terminated by the END action). 5032 * @param[in] flow_split_info 5033 * Pointer to flow split info structure. 5034 * @param[out] error 5035 * Perform verbose error reporting if not NULL. 5036 * @return 5037 * 0 on success, negative value otherwise 5038 */ 5039 static int 5040 flow_create_split_inner(struct rte_eth_dev *dev, 5041 struct rte_flow *flow, 5042 struct mlx5_flow **sub_flow, 5043 const struct rte_flow_attr *attr, 5044 const struct rte_flow_item items[], 5045 const struct rte_flow_action actions[], 5046 struct mlx5_flow_split_info *flow_split_info, 5047 struct rte_flow_error *error) 5048 { 5049 struct mlx5_flow *dev_flow; 5050 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5051 5052 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5053 flow_split_info->flow_idx, error); 5054 if (!dev_flow) 5055 return -rte_errno; 5056 dev_flow->flow = flow; 5057 dev_flow->external = flow_split_info->external; 5058 dev_flow->skip_scale = flow_split_info->skip_scale; 5059 /* Subflow object was created, we must include one in the list. */ 5060 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5061 dev_flow->handle, next); 5062 /* 5063 * If dev_flow is as one of the suffix flow, some actions in suffix 5064 * flow may need some user defined item layer flags, and pass the 5065 * Metadata rxq mark flag to suffix flow as well. 5066 */ 5067 if (flow_split_info->prefix_layers) 5068 dev_flow->handle->layers = flow_split_info->prefix_layers; 5069 if (flow_split_info->prefix_mark) { 5070 MLX5_ASSERT(wks); 5071 wks->mark = 1; 5072 } 5073 if (sub_flow) 5074 *sub_flow = dev_flow; 5075 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5076 dev_flow->dv.table_id = flow_split_info->table_id; 5077 #endif 5078 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5079 } 5080 5081 /** 5082 * Get the sub policy of a meter. 5083 * 5084 * @param[in] dev 5085 * Pointer to Ethernet device. 5086 * @param[in] flow 5087 * Parent flow structure pointer. 5088 * @param wks 5089 * Pointer to thread flow work space. 5090 * @param[in] attr 5091 * Flow rule attributes. 5092 * @param[in] items 5093 * Pattern specification (list terminated by the END pattern item). 5094 * @param[out] error 5095 * Perform verbose error reporting if not NULL. 5096 * 5097 * @return 5098 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5099 */ 5100 static struct mlx5_flow_meter_sub_policy * 5101 get_meter_sub_policy(struct rte_eth_dev *dev, 5102 struct rte_flow *flow, 5103 struct mlx5_flow_workspace *wks, 5104 const struct rte_flow_attr *attr, 5105 const struct rte_flow_item items[], 5106 struct rte_flow_error *error) 5107 { 5108 struct mlx5_flow_meter_policy *policy; 5109 struct mlx5_flow_meter_policy *final_policy; 5110 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5111 5112 policy = wks->policy; 5113 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5114 if (final_policy->is_rss || final_policy->is_queue) { 5115 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5116 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5117 uint32_t i; 5118 5119 /* 5120 * This is a tmp dev_flow, 5121 * no need to register any matcher for it in translate. 5122 */ 5123 wks->skip_matcher_reg = 1; 5124 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5125 struct mlx5_flow dev_flow = {0}; 5126 struct mlx5_flow_handle dev_handle = { {0} }; 5127 uint8_t fate = final_policy->act_cnt[i].fate_action; 5128 5129 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5130 const struct rte_flow_action_rss *rss_act = 5131 final_policy->act_cnt[i].rss->conf; 5132 struct rte_flow_action rss_actions[2] = { 5133 [0] = { 5134 .type = RTE_FLOW_ACTION_TYPE_RSS, 5135 .conf = rss_act, 5136 }, 5137 [1] = { 5138 .type = RTE_FLOW_ACTION_TYPE_END, 5139 .conf = NULL, 5140 } 5141 }; 5142 5143 dev_flow.handle = &dev_handle; 5144 dev_flow.ingress = attr->ingress; 5145 dev_flow.flow = flow; 5146 dev_flow.external = 0; 5147 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5148 dev_flow.dv.transfer = attr->transfer; 5149 #endif 5150 /** 5151 * Translate RSS action to get rss hash fields. 5152 */ 5153 if (flow_drv_translate(dev, &dev_flow, attr, 5154 items, rss_actions, error)) 5155 goto exit; 5156 rss_desc_v[i] = wks->rss_desc; 5157 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5158 rss_desc_v[i].hash_fields = 5159 dev_flow.hash_fields; 5160 rss_desc_v[i].queue_num = 5161 rss_desc_v[i].hash_fields ? 5162 rss_desc_v[i].queue_num : 1; 5163 rss_desc_v[i].tunnel = 5164 !!(dev_flow.handle->layers & 5165 MLX5_FLOW_LAYER_TUNNEL); 5166 /* Use the RSS queues in the containers. */ 5167 rss_desc_v[i].queue = 5168 (uint16_t *)(uintptr_t)rss_act->queue; 5169 rss_desc[i] = &rss_desc_v[i]; 5170 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5171 /* This is queue action. */ 5172 rss_desc_v[i] = wks->rss_desc; 5173 rss_desc_v[i].key_len = 0; 5174 rss_desc_v[i].hash_fields = 0; 5175 rss_desc_v[i].queue = 5176 &final_policy->act_cnt[i].queue; 5177 rss_desc_v[i].queue_num = 1; 5178 rss_desc[i] = &rss_desc_v[i]; 5179 } else { 5180 rss_desc[i] = NULL; 5181 } 5182 } 5183 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5184 flow, policy, rss_desc); 5185 } else { 5186 enum mlx5_meter_domain mtr_domain = 5187 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5188 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5189 MLX5_MTR_DOMAIN_INGRESS); 5190 sub_policy = policy->sub_policys[mtr_domain][0]; 5191 } 5192 if (!sub_policy) 5193 rte_flow_error_set(error, EINVAL, 5194 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5195 "Failed to get meter sub-policy."); 5196 exit: 5197 return sub_policy; 5198 } 5199 5200 /** 5201 * Split the meter flow. 5202 * 5203 * As meter flow will split to three sub flow, other than meter 5204 * action, the other actions make sense to only meter accepts 5205 * the packet. If it need to be dropped, no other additional 5206 * actions should be take. 5207 * 5208 * One kind of special action which decapsulates the L3 tunnel 5209 * header will be in the prefix sub flow, as not to take the 5210 * L3 tunnel header into account. 5211 * 5212 * @param[in] dev 5213 * Pointer to Ethernet device. 5214 * @param[in] flow 5215 * Parent flow structure pointer. 5216 * @param wks 5217 * Pointer to thread flow work space. 5218 * @param[in] attr 5219 * Flow rule attributes. 5220 * @param[in] items 5221 * Pattern specification (list terminated by the END pattern item). 5222 * @param[out] sfx_items 5223 * Suffix flow match items (list terminated by the END pattern item). 5224 * @param[in] actions 5225 * Associated actions (list terminated by the END action). 5226 * @param[out] actions_sfx 5227 * Suffix flow actions. 5228 * @param[out] actions_pre 5229 * Prefix flow actions. 5230 * @param[out] mtr_flow_id 5231 * Pointer to meter flow id. 5232 * @param[out] error 5233 * Perform verbose error reporting if not NULL. 5234 * 5235 * @return 5236 * 0 on success, a negative errno value otherwise and rte_errno is set. 5237 */ 5238 static int 5239 flow_meter_split_prep(struct rte_eth_dev *dev, 5240 struct rte_flow *flow, 5241 struct mlx5_flow_workspace *wks, 5242 const struct rte_flow_attr *attr, 5243 const struct rte_flow_item items[], 5244 struct rte_flow_item sfx_items[], 5245 const struct rte_flow_action actions[], 5246 struct rte_flow_action actions_sfx[], 5247 struct rte_flow_action actions_pre[], 5248 uint32_t *mtr_flow_id, 5249 struct rte_flow_error *error) 5250 { 5251 struct mlx5_priv *priv = dev->data->dev_private; 5252 struct mlx5_flow_meter_info *fm = wks->fm; 5253 struct rte_flow_action *tag_action = NULL; 5254 struct rte_flow_item *tag_item; 5255 struct mlx5_rte_flow_action_set_tag *set_tag; 5256 const struct rte_flow_action_raw_encap *raw_encap; 5257 const struct rte_flow_action_raw_decap *raw_decap; 5258 struct mlx5_rte_flow_item_tag *tag_item_spec; 5259 struct mlx5_rte_flow_item_tag *tag_item_mask; 5260 uint32_t tag_id = 0; 5261 struct rte_flow_item *vlan_item_dst = NULL; 5262 const struct rte_flow_item *vlan_item_src = NULL; 5263 const struct rte_flow_item *orig_items = items; 5264 struct rte_flow_action *hw_mtr_action; 5265 struct rte_flow_action *action_pre_head = NULL; 5266 int32_t flow_src_port = priv->representor_id; 5267 bool mtr_first; 5268 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5269 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5270 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5271 uint32_t flow_id = 0; 5272 uint32_t flow_id_reversed = 0; 5273 uint8_t flow_id_bits = 0; 5274 bool after_meter = false; 5275 int shift; 5276 5277 /* Prepare the suffix subflow items. */ 5278 tag_item = sfx_items++; 5279 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5280 struct mlx5_priv *port_priv; 5281 const struct rte_flow_item_port_id *pid_v; 5282 int item_type = items->type; 5283 5284 switch (item_type) { 5285 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5286 pid_v = items->spec; 5287 MLX5_ASSERT(pid_v); 5288 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5289 if (!port_priv) 5290 return rte_flow_error_set(error, 5291 rte_errno, 5292 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5293 pid_v, 5294 "Failed to get port info."); 5295 flow_src_port = port_priv->representor_id; 5296 if (!fm->def_policy && wks->policy->is_hierarchy && 5297 flow_src_port != priv->representor_id) { 5298 if (flow_drv_mtr_hierarchy_rule_create(dev, 5299 flow, fm, 5300 flow_src_port, 5301 items, 5302 error)) 5303 return -rte_errno; 5304 } 5305 memcpy(sfx_items, items, sizeof(*sfx_items)); 5306 sfx_items++; 5307 break; 5308 case RTE_FLOW_ITEM_TYPE_VLAN: 5309 /* Determine if copy vlan item below. */ 5310 vlan_item_src = items; 5311 vlan_item_dst = sfx_items++; 5312 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5313 break; 5314 default: 5315 break; 5316 } 5317 } 5318 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5319 sfx_items++; 5320 mtr_first = priv->sh->meter_aso_en && 5321 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5322 /* For ASO meter, meter must be before tag in TX direction. */ 5323 if (mtr_first) { 5324 action_pre_head = actions_pre++; 5325 /* Leave space for tag action. */ 5326 tag_action = actions_pre++; 5327 } 5328 /* Prepare the actions for prefix and suffix flow. */ 5329 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5330 struct rte_flow_action *action_cur = NULL; 5331 5332 switch (actions->type) { 5333 case RTE_FLOW_ACTION_TYPE_METER: 5334 if (mtr_first) { 5335 action_cur = action_pre_head; 5336 } else { 5337 /* Leave space for tag action. */ 5338 tag_action = actions_pre++; 5339 action_cur = actions_pre++; 5340 } 5341 after_meter = true; 5342 break; 5343 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5344 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5345 action_cur = actions_pre++; 5346 break; 5347 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5348 raw_encap = actions->conf; 5349 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5350 action_cur = actions_pre++; 5351 break; 5352 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5353 raw_decap = actions->conf; 5354 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5355 action_cur = actions_pre++; 5356 break; 5357 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5358 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5359 if (vlan_item_dst && vlan_item_src) { 5360 memcpy(vlan_item_dst, vlan_item_src, 5361 sizeof(*vlan_item_dst)); 5362 /* 5363 * Convert to internal match item, it is used 5364 * for vlan push and set vid. 5365 */ 5366 vlan_item_dst->type = (enum rte_flow_item_type) 5367 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5368 } 5369 break; 5370 case RTE_FLOW_ACTION_TYPE_COUNT: 5371 if (fm->def_policy) 5372 action_cur = after_meter ? 5373 actions_sfx++ : actions_pre++; 5374 break; 5375 default: 5376 break; 5377 } 5378 if (!action_cur) 5379 action_cur = (fm->def_policy) ? 5380 actions_sfx++ : actions_pre++; 5381 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5382 } 5383 /* Add end action to the actions. */ 5384 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5385 if (priv->sh->meter_aso_en) { 5386 /** 5387 * For ASO meter, need to add an extra jump action explicitly, 5388 * to jump from meter to policer table. 5389 */ 5390 struct mlx5_flow_meter_sub_policy *sub_policy; 5391 struct mlx5_flow_tbl_data_entry *tbl_data; 5392 5393 if (!fm->def_policy) { 5394 sub_policy = get_meter_sub_policy(dev, flow, wks, 5395 attr, orig_items, 5396 error); 5397 if (!sub_policy) 5398 return -rte_errno; 5399 } else { 5400 enum mlx5_meter_domain mtr_domain = 5401 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5402 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5403 MLX5_MTR_DOMAIN_INGRESS); 5404 5405 sub_policy = 5406 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5407 } 5408 tbl_data = container_of(sub_policy->tbl_rsc, 5409 struct mlx5_flow_tbl_data_entry, tbl); 5410 hw_mtr_action = actions_pre++; 5411 hw_mtr_action->type = (enum rte_flow_action_type) 5412 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5413 hw_mtr_action->conf = tbl_data->jump.action; 5414 } 5415 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5416 actions_pre++; 5417 if (!tag_action) 5418 return rte_flow_error_set(error, ENOMEM, 5419 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5420 NULL, "No tag action space."); 5421 if (!mtr_flow_id) { 5422 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5423 goto exit; 5424 } 5425 /* Only default-policy Meter creates mtr flow id. */ 5426 if (fm->def_policy) { 5427 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5428 if (!tag_id) 5429 return rte_flow_error_set(error, ENOMEM, 5430 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5431 "Failed to allocate meter flow id."); 5432 flow_id = tag_id - 1; 5433 flow_id_bits = (!flow_id) ? 1 : 5434 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5435 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5436 mtr_reg_bits) { 5437 mlx5_ipool_free(fm->flow_ipool, tag_id); 5438 return rte_flow_error_set(error, EINVAL, 5439 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5440 "Meter flow id exceeds max limit."); 5441 } 5442 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5443 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5444 } 5445 /* Build tag actions and items for meter_id/meter flow_id. */ 5446 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5447 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5448 tag_item_mask = tag_item_spec + 1; 5449 /* Both flow_id and meter_id share the same register. */ 5450 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5451 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5452 0, error), 5453 .offset = mtr_id_offset, 5454 .length = mtr_reg_bits, 5455 .data = flow->meter, 5456 }; 5457 /* 5458 * The color Reg bits used by flow_id are growing from 5459 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5460 */ 5461 for (shift = 0; shift < flow_id_bits; shift++) 5462 flow_id_reversed = (flow_id_reversed << 1) | 5463 ((flow_id >> shift) & 0x1); 5464 set_tag->data |= 5465 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5466 tag_item_spec->id = set_tag->id; 5467 tag_item_spec->data = set_tag->data << mtr_id_offset; 5468 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5469 tag_action->type = (enum rte_flow_action_type) 5470 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5471 tag_action->conf = set_tag; 5472 tag_item->type = (enum rte_flow_item_type) 5473 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5474 tag_item->spec = tag_item_spec; 5475 tag_item->last = NULL; 5476 tag_item->mask = tag_item_mask; 5477 exit: 5478 if (mtr_flow_id) 5479 *mtr_flow_id = tag_id; 5480 return 0; 5481 } 5482 5483 /** 5484 * Split action list having QUEUE/RSS for metadata register copy. 5485 * 5486 * Once Q/RSS action is detected in user's action list, the flow action 5487 * should be split in order to copy metadata registers, which will happen in 5488 * RX_CP_TBL like, 5489 * - CQE->flow_tag := reg_c[1] (MARK) 5490 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5491 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5492 * This is because the last action of each flow must be a terminal action 5493 * (QUEUE, RSS or DROP). 5494 * 5495 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5496 * stored and kept in the mlx5_flow structure per each sub_flow. 5497 * 5498 * The Q/RSS action is replaced with, 5499 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5500 * And the following JUMP action is added at the end, 5501 * - JUMP, to RX_CP_TBL. 5502 * 5503 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5504 * flow_create_split_metadata() routine. The flow will look like, 5505 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5506 * 5507 * @param dev 5508 * Pointer to Ethernet device. 5509 * @param[out] split_actions 5510 * Pointer to store split actions to jump to CP_TBL. 5511 * @param[in] actions 5512 * Pointer to the list of original flow actions. 5513 * @param[in] qrss 5514 * Pointer to the Q/RSS action. 5515 * @param[in] actions_n 5516 * Number of original actions. 5517 * @param[in] mtr_sfx 5518 * Check if it is in meter suffix table. 5519 * @param[out] error 5520 * Perform verbose error reporting if not NULL. 5521 * 5522 * @return 5523 * non-zero unique flow_id on success, otherwise 0 and 5524 * error/rte_error are set. 5525 */ 5526 static uint32_t 5527 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5528 struct rte_flow_action *split_actions, 5529 const struct rte_flow_action *actions, 5530 const struct rte_flow_action *qrss, 5531 int actions_n, int mtr_sfx, 5532 struct rte_flow_error *error) 5533 { 5534 struct mlx5_priv *priv = dev->data->dev_private; 5535 struct mlx5_rte_flow_action_set_tag *set_tag; 5536 struct rte_flow_action_jump *jump; 5537 const int qrss_idx = qrss - actions; 5538 uint32_t flow_id = 0; 5539 int ret = 0; 5540 5541 /* 5542 * Given actions will be split 5543 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5544 * - Add jump to mreg CP_TBL. 5545 * As a result, there will be one more action. 5546 */ 5547 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5548 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5549 ++actions_n; 5550 set_tag = (void *)(split_actions + actions_n); 5551 /* 5552 * If we are not the meter suffix flow, add the tag action. 5553 * Since meter suffix flow already has the tag added. 5554 */ 5555 if (!mtr_sfx) { 5556 /* 5557 * Allocate the new subflow ID. This one is unique within 5558 * device and not shared with representors. Otherwise, 5559 * we would have to resolve multi-thread access synch 5560 * issue. Each flow on the shared device is appended 5561 * with source vport identifier, so the resulting 5562 * flows will be unique in the shared (by master and 5563 * representors) domain even if they have coinciding 5564 * IDs. 5565 */ 5566 mlx5_ipool_malloc(priv->sh->ipool 5567 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5568 if (!flow_id) 5569 return rte_flow_error_set(error, ENOMEM, 5570 RTE_FLOW_ERROR_TYPE_ACTION, 5571 NULL, "can't allocate id " 5572 "for split Q/RSS subflow"); 5573 /* Internal SET_TAG action to set flow ID. */ 5574 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5575 .data = flow_id, 5576 }; 5577 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5578 if (ret < 0) 5579 return ret; 5580 set_tag->id = ret; 5581 /* Construct new actions array. */ 5582 /* Replace QUEUE/RSS action. */ 5583 split_actions[qrss_idx] = (struct rte_flow_action){ 5584 .type = (enum rte_flow_action_type) 5585 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5586 .conf = set_tag, 5587 }; 5588 } else { 5589 /* 5590 * If we are the suffix flow of meter, tag already exist. 5591 * Set the QUEUE/RSS action to void. 5592 */ 5593 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5594 } 5595 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5596 jump = (void *)(set_tag + 1); 5597 *jump = (struct rte_flow_action_jump){ 5598 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5599 }; 5600 split_actions[actions_n - 2] = (struct rte_flow_action){ 5601 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5602 .conf = jump, 5603 }; 5604 split_actions[actions_n - 1] = (struct rte_flow_action){ 5605 .type = RTE_FLOW_ACTION_TYPE_END, 5606 }; 5607 return flow_id; 5608 } 5609 5610 /** 5611 * Extend the given action list for Tx metadata copy. 5612 * 5613 * Copy the given action list to the ext_actions and add flow metadata register 5614 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5615 * 5616 * @param[out] ext_actions 5617 * Pointer to the extended action list. 5618 * @param[in] actions 5619 * Pointer to the list of actions. 5620 * @param[in] actions_n 5621 * Number of actions in the list. 5622 * @param[out] error 5623 * Perform verbose error reporting if not NULL. 5624 * @param[in] encap_idx 5625 * The encap action index. 5626 * 5627 * @return 5628 * 0 on success, negative value otherwise 5629 */ 5630 static int 5631 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5632 struct rte_flow_action *ext_actions, 5633 const struct rte_flow_action *actions, 5634 int actions_n, struct rte_flow_error *error, 5635 int encap_idx) 5636 { 5637 struct mlx5_flow_action_copy_mreg *cp_mreg = 5638 (struct mlx5_flow_action_copy_mreg *) 5639 (ext_actions + actions_n + 1); 5640 int ret; 5641 5642 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5643 if (ret < 0) 5644 return ret; 5645 cp_mreg->dst = ret; 5646 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5647 if (ret < 0) 5648 return ret; 5649 cp_mreg->src = ret; 5650 if (encap_idx != 0) 5651 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5652 if (encap_idx == actions_n - 1) { 5653 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5654 .type = (enum rte_flow_action_type) 5655 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5656 .conf = cp_mreg, 5657 }; 5658 ext_actions[actions_n] = (struct rte_flow_action){ 5659 .type = RTE_FLOW_ACTION_TYPE_END, 5660 }; 5661 } else { 5662 ext_actions[encap_idx] = (struct rte_flow_action){ 5663 .type = (enum rte_flow_action_type) 5664 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5665 .conf = cp_mreg, 5666 }; 5667 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5668 sizeof(*ext_actions) * (actions_n - encap_idx)); 5669 } 5670 return 0; 5671 } 5672 5673 /** 5674 * Check the match action from the action list. 5675 * 5676 * @param[in] actions 5677 * Pointer to the list of actions. 5678 * @param[in] attr 5679 * Flow rule attributes. 5680 * @param[in] action 5681 * The action to be check if exist. 5682 * @param[out] match_action_pos 5683 * Pointer to the position of the matched action if exists, otherwise is -1. 5684 * @param[out] qrss_action_pos 5685 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5686 * @param[out] modify_after_mirror 5687 * Pointer to the flag of modify action after FDB mirroring. 5688 * 5689 * @return 5690 * > 0 the total number of actions. 5691 * 0 if not found match action in action list. 5692 */ 5693 static int 5694 flow_check_match_action(const struct rte_flow_action actions[], 5695 const struct rte_flow_attr *attr, 5696 enum rte_flow_action_type action, 5697 int *match_action_pos, int *qrss_action_pos, 5698 int *modify_after_mirror) 5699 { 5700 const struct rte_flow_action_sample *sample; 5701 const struct rte_flow_action_raw_decap *decap; 5702 int actions_n = 0; 5703 uint32_t ratio = 0; 5704 int sub_type = 0; 5705 int flag = 0; 5706 int fdb_mirror = 0; 5707 5708 *match_action_pos = -1; 5709 *qrss_action_pos = -1; 5710 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5711 if (actions->type == action) { 5712 flag = 1; 5713 *match_action_pos = actions_n; 5714 } 5715 switch (actions->type) { 5716 case RTE_FLOW_ACTION_TYPE_QUEUE: 5717 case RTE_FLOW_ACTION_TYPE_RSS: 5718 *qrss_action_pos = actions_n; 5719 break; 5720 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5721 sample = actions->conf; 5722 ratio = sample->ratio; 5723 sub_type = ((const struct rte_flow_action *) 5724 (sample->actions))->type; 5725 if (ratio == 1 && attr->transfer) 5726 fdb_mirror = 1; 5727 break; 5728 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5729 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5730 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5731 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5732 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5733 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5734 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5735 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5736 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5737 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5738 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5739 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5740 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5741 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5742 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5743 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5744 case RTE_FLOW_ACTION_TYPE_FLAG: 5745 case RTE_FLOW_ACTION_TYPE_MARK: 5746 case RTE_FLOW_ACTION_TYPE_SET_META: 5747 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5748 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5749 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5750 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5751 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5752 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5753 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5754 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5755 case RTE_FLOW_ACTION_TYPE_METER: 5756 if (fdb_mirror) 5757 *modify_after_mirror = 1; 5758 break; 5759 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5760 decap = actions->conf; 5761 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5762 ; 5763 actions_n++; 5764 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5765 const struct rte_flow_action_raw_encap *encap = 5766 actions->conf; 5767 if (decap->size <= 5768 MLX5_ENCAPSULATION_DECISION_SIZE && 5769 encap->size > 5770 MLX5_ENCAPSULATION_DECISION_SIZE) 5771 /* L3 encap. */ 5772 break; 5773 } 5774 if (fdb_mirror) 5775 *modify_after_mirror = 1; 5776 break; 5777 default: 5778 break; 5779 } 5780 actions_n++; 5781 } 5782 if (flag && fdb_mirror && !*modify_after_mirror) { 5783 /* FDB mirroring uses the destination array to implement 5784 * instead of FLOW_SAMPLER object. 5785 */ 5786 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5787 flag = 0; 5788 } 5789 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5790 return flag ? actions_n + 1 : 0; 5791 } 5792 5793 #define SAMPLE_SUFFIX_ITEM 3 5794 5795 /** 5796 * Split the sample flow. 5797 * 5798 * As sample flow will split to two sub flow, sample flow with 5799 * sample action, the other actions will move to new suffix flow. 5800 * 5801 * Also add unique tag id with tag action in the sample flow, 5802 * the same tag id will be as match in the suffix flow. 5803 * 5804 * @param dev 5805 * Pointer to Ethernet device. 5806 * @param[in] add_tag 5807 * Add extra tag action flag. 5808 * @param[out] sfx_items 5809 * Suffix flow match items (list terminated by the END pattern item). 5810 * @param[in] actions 5811 * Associated actions (list terminated by the END action). 5812 * @param[out] actions_sfx 5813 * Suffix flow actions. 5814 * @param[out] actions_pre 5815 * Prefix flow actions. 5816 * @param[in] actions_n 5817 * The total number of actions. 5818 * @param[in] sample_action_pos 5819 * The sample action position. 5820 * @param[in] qrss_action_pos 5821 * The Queue/RSS action position. 5822 * @param[in] jump_table 5823 * Add extra jump action flag. 5824 * @param[out] error 5825 * Perform verbose error reporting if not NULL. 5826 * 5827 * @return 5828 * 0 on success, or unique flow_id, a negative errno value 5829 * otherwise and rte_errno is set. 5830 */ 5831 static int 5832 flow_sample_split_prep(struct rte_eth_dev *dev, 5833 int add_tag, 5834 const struct rte_flow_item items[], 5835 struct rte_flow_item sfx_items[], 5836 const struct rte_flow_action actions[], 5837 struct rte_flow_action actions_sfx[], 5838 struct rte_flow_action actions_pre[], 5839 int actions_n, 5840 int sample_action_pos, 5841 int qrss_action_pos, 5842 int jump_table, 5843 struct rte_flow_error *error) 5844 { 5845 struct mlx5_priv *priv = dev->data->dev_private; 5846 struct mlx5_rte_flow_action_set_tag *set_tag; 5847 struct mlx5_rte_flow_item_tag *tag_spec; 5848 struct mlx5_rte_flow_item_tag *tag_mask; 5849 struct rte_flow_action_jump *jump_action; 5850 uint32_t tag_id = 0; 5851 int append_index = 0; 5852 int set_tag_idx = -1; 5853 int index; 5854 int ret; 5855 5856 if (sample_action_pos < 0) 5857 return rte_flow_error_set(error, EINVAL, 5858 RTE_FLOW_ERROR_TYPE_ACTION, 5859 NULL, "invalid position of sample " 5860 "action in list"); 5861 /* Prepare the actions for prefix and suffix flow. */ 5862 if (add_tag) { 5863 /* Update the new added tag action index preceding 5864 * the PUSH_VLAN or ENCAP action. 5865 */ 5866 const struct rte_flow_action_raw_encap *raw_encap; 5867 const struct rte_flow_action *action = actions; 5868 int encap_idx; 5869 int action_idx = 0; 5870 int raw_decap_idx = -1; 5871 int push_vlan_idx = -1; 5872 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 5873 switch (action->type) { 5874 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5875 raw_decap_idx = action_idx; 5876 break; 5877 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5878 raw_encap = action->conf; 5879 if (raw_encap->size > 5880 MLX5_ENCAPSULATION_DECISION_SIZE) { 5881 encap_idx = raw_decap_idx != -1 ? 5882 raw_decap_idx : action_idx; 5883 if (encap_idx < sample_action_pos && 5884 push_vlan_idx == -1) 5885 set_tag_idx = encap_idx; 5886 } 5887 break; 5888 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5889 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5890 encap_idx = action_idx; 5891 if (encap_idx < sample_action_pos && 5892 push_vlan_idx == -1) 5893 set_tag_idx = encap_idx; 5894 break; 5895 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5896 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5897 push_vlan_idx = action_idx; 5898 if (push_vlan_idx < sample_action_pos) 5899 set_tag_idx = action_idx; 5900 break; 5901 default: 5902 break; 5903 } 5904 action_idx++; 5905 } 5906 } 5907 /* Prepare the actions for prefix and suffix flow. */ 5908 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5909 index = qrss_action_pos; 5910 /* Put the preceding the Queue/RSS action into prefix flow. */ 5911 if (index != 0) 5912 memcpy(actions_pre, actions, 5913 sizeof(struct rte_flow_action) * index); 5914 /* Put others preceding the sample action into prefix flow. */ 5915 if (sample_action_pos > index + 1) 5916 memcpy(actions_pre + index, actions + index + 1, 5917 sizeof(struct rte_flow_action) * 5918 (sample_action_pos - index - 1)); 5919 index = sample_action_pos - 1; 5920 /* Put Queue/RSS action into Suffix flow. */ 5921 memcpy(actions_sfx, actions + qrss_action_pos, 5922 sizeof(struct rte_flow_action)); 5923 actions_sfx++; 5924 } else if (add_tag && set_tag_idx >= 0) { 5925 if (set_tag_idx > 0) 5926 memcpy(actions_pre, actions, 5927 sizeof(struct rte_flow_action) * set_tag_idx); 5928 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 5929 sizeof(struct rte_flow_action) * 5930 (sample_action_pos - set_tag_idx)); 5931 index = sample_action_pos; 5932 } else { 5933 index = sample_action_pos; 5934 if (index != 0) 5935 memcpy(actions_pre, actions, 5936 sizeof(struct rte_flow_action) * index); 5937 } 5938 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5939 * For CX6DX and above, metadata registers Cx preserve their value, 5940 * add an extra tag action for NIC-RX and E-Switch Domain. 5941 */ 5942 if (add_tag) { 5943 /* Prepare the prefix tag action. */ 5944 append_index++; 5945 set_tag = (void *)(actions_pre + actions_n + append_index); 5946 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5947 /* Trust VF/SF on CX5 not supported meter so that the reserved 5948 * metadata regC is REG_NON, back to use application tag 5949 * index 0. 5950 */ 5951 if (unlikely(ret == REG_NON)) 5952 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 5953 if (ret < 0) 5954 return ret; 5955 mlx5_ipool_malloc(priv->sh->ipool 5956 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5957 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5958 .id = ret, 5959 .data = tag_id, 5960 }; 5961 /* Prepare the suffix subflow items. */ 5962 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5963 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 5964 memcpy(sfx_items, items, sizeof(*sfx_items)); 5965 sfx_items++; 5966 } 5967 } 5968 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5969 tag_spec->data = tag_id; 5970 tag_spec->id = set_tag->id; 5971 tag_mask = tag_spec + 1; 5972 tag_mask->data = UINT32_MAX; 5973 sfx_items[0] = (struct rte_flow_item){ 5974 .type = (enum rte_flow_item_type) 5975 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5976 .spec = tag_spec, 5977 .last = NULL, 5978 .mask = tag_mask, 5979 }; 5980 sfx_items[1] = (struct rte_flow_item){ 5981 .type = (enum rte_flow_item_type) 5982 RTE_FLOW_ITEM_TYPE_END, 5983 }; 5984 /* Prepare the tag action in prefix subflow. */ 5985 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 5986 actions_pre[set_tag_idx] = 5987 (struct rte_flow_action){ 5988 .type = (enum rte_flow_action_type) 5989 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5990 .conf = set_tag, 5991 }; 5992 /* Update next sample position due to add one tag action */ 5993 index += 1; 5994 } 5995 /* Copy the sample action into prefix flow. */ 5996 memcpy(actions_pre + index, actions + sample_action_pos, 5997 sizeof(struct rte_flow_action)); 5998 index += 1; 5999 /* For the modify action after the sample action in E-Switch mirroring, 6000 * Add the extra jump action in prefix subflow and jump into the next 6001 * table, then do the modify action in the new table. 6002 */ 6003 if (jump_table) { 6004 /* Prepare the prefix jump action. */ 6005 append_index++; 6006 jump_action = (void *)(actions_pre + actions_n + append_index); 6007 jump_action->group = jump_table; 6008 actions_pre[index++] = 6009 (struct rte_flow_action){ 6010 .type = (enum rte_flow_action_type) 6011 RTE_FLOW_ACTION_TYPE_JUMP, 6012 .conf = jump_action, 6013 }; 6014 } 6015 actions_pre[index] = (struct rte_flow_action){ 6016 .type = (enum rte_flow_action_type) 6017 RTE_FLOW_ACTION_TYPE_END, 6018 }; 6019 /* Put the actions after sample into Suffix flow. */ 6020 memcpy(actions_sfx, actions + sample_action_pos + 1, 6021 sizeof(struct rte_flow_action) * 6022 (actions_n - sample_action_pos - 1)); 6023 return tag_id; 6024 } 6025 6026 /** 6027 * The splitting for metadata feature. 6028 * 6029 * - Q/RSS action on NIC Rx should be split in order to pass by 6030 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6031 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6032 * 6033 * - All the actions on NIC Tx should have a mreg copy action to 6034 * copy reg_a from WQE to reg_c[0]. 6035 * 6036 * @param dev 6037 * Pointer to Ethernet device. 6038 * @param[in] flow 6039 * Parent flow structure pointer. 6040 * @param[in] attr 6041 * Flow rule attributes. 6042 * @param[in] items 6043 * Pattern specification (list terminated by the END pattern item). 6044 * @param[in] actions 6045 * Associated actions (list terminated by the END action). 6046 * @param[in] flow_split_info 6047 * Pointer to flow split info structure. 6048 * @param[out] error 6049 * Perform verbose error reporting if not NULL. 6050 * @return 6051 * 0 on success, negative value otherwise 6052 */ 6053 static int 6054 flow_create_split_metadata(struct rte_eth_dev *dev, 6055 struct rte_flow *flow, 6056 const struct rte_flow_attr *attr, 6057 const struct rte_flow_item items[], 6058 const struct rte_flow_action actions[], 6059 struct mlx5_flow_split_info *flow_split_info, 6060 struct rte_flow_error *error) 6061 { 6062 struct mlx5_priv *priv = dev->data->dev_private; 6063 struct mlx5_sh_config *config = &priv->sh->config; 6064 const struct rte_flow_action *qrss = NULL; 6065 struct rte_flow_action *ext_actions = NULL; 6066 struct mlx5_flow *dev_flow = NULL; 6067 uint32_t qrss_id = 0; 6068 int mtr_sfx = 0; 6069 size_t act_size; 6070 int actions_n; 6071 int encap_idx; 6072 int ret; 6073 6074 /* Check whether extensive metadata feature is engaged. */ 6075 if (!config->dv_flow_en || 6076 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6077 !mlx5_flow_ext_mreg_supported(dev)) 6078 return flow_create_split_inner(dev, flow, NULL, attr, items, 6079 actions, flow_split_info, error); 6080 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6081 &encap_idx); 6082 if (qrss) { 6083 /* Exclude hairpin flows from splitting. */ 6084 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6085 const struct rte_flow_action_queue *queue; 6086 6087 queue = qrss->conf; 6088 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6089 qrss = NULL; 6090 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6091 const struct rte_flow_action_rss *rss; 6092 6093 rss = qrss->conf; 6094 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6095 qrss = NULL; 6096 } 6097 } 6098 if (qrss) { 6099 /* Check if it is in meter suffix table. */ 6100 mtr_sfx = attr->group == (attr->transfer ? 6101 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6102 MLX5_FLOW_TABLE_LEVEL_METER); 6103 /* 6104 * Q/RSS action on NIC Rx should be split in order to pass by 6105 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6106 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6107 */ 6108 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6109 sizeof(struct rte_flow_action_set_tag) + 6110 sizeof(struct rte_flow_action_jump); 6111 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6112 SOCKET_ID_ANY); 6113 if (!ext_actions) 6114 return rte_flow_error_set(error, ENOMEM, 6115 RTE_FLOW_ERROR_TYPE_ACTION, 6116 NULL, "no memory to split " 6117 "metadata flow"); 6118 /* 6119 * Create the new actions list with removed Q/RSS action 6120 * and appended set tag and jump to register copy table 6121 * (RX_CP_TBL). We should preallocate unique tag ID here 6122 * in advance, because it is needed for set tag action. 6123 */ 6124 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6125 qrss, actions_n, 6126 mtr_sfx, error); 6127 if (!mtr_sfx && !qrss_id) { 6128 ret = -rte_errno; 6129 goto exit; 6130 } 6131 } else if (attr->egress && !attr->transfer) { 6132 /* 6133 * All the actions on NIC Tx should have a metadata register 6134 * copy action to copy reg_a from WQE to reg_c[meta] 6135 */ 6136 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6137 sizeof(struct mlx5_flow_action_copy_mreg); 6138 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6139 SOCKET_ID_ANY); 6140 if (!ext_actions) 6141 return rte_flow_error_set(error, ENOMEM, 6142 RTE_FLOW_ERROR_TYPE_ACTION, 6143 NULL, "no memory to split " 6144 "metadata flow"); 6145 /* Create the action list appended with copy register. */ 6146 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6147 actions_n, error, encap_idx); 6148 if (ret < 0) 6149 goto exit; 6150 } 6151 /* Add the unmodified original or prefix subflow. */ 6152 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6153 items, ext_actions ? ext_actions : 6154 actions, flow_split_info, error); 6155 if (ret < 0) 6156 goto exit; 6157 MLX5_ASSERT(dev_flow); 6158 if (qrss) { 6159 const struct rte_flow_attr q_attr = { 6160 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6161 .ingress = 1, 6162 }; 6163 /* Internal PMD action to set register. */ 6164 struct mlx5_rte_flow_item_tag q_tag_spec = { 6165 .data = qrss_id, 6166 .id = REG_NON, 6167 }; 6168 struct rte_flow_item q_items[] = { 6169 { 6170 .type = (enum rte_flow_item_type) 6171 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6172 .spec = &q_tag_spec, 6173 .last = NULL, 6174 .mask = NULL, 6175 }, 6176 { 6177 .type = RTE_FLOW_ITEM_TYPE_END, 6178 }, 6179 }; 6180 struct rte_flow_action q_actions[] = { 6181 { 6182 .type = qrss->type, 6183 .conf = qrss->conf, 6184 }, 6185 { 6186 .type = RTE_FLOW_ACTION_TYPE_END, 6187 }, 6188 }; 6189 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6190 6191 /* 6192 * Configure the tag item only if there is no meter subflow. 6193 * Since tag is already marked in the meter suffix subflow 6194 * we can just use the meter suffix items as is. 6195 */ 6196 if (qrss_id) { 6197 /* Not meter subflow. */ 6198 MLX5_ASSERT(!mtr_sfx); 6199 /* 6200 * Put unique id in prefix flow due to it is destroyed 6201 * after suffix flow and id will be freed after there 6202 * is no actual flows with this id and identifier 6203 * reallocation becomes possible (for example, for 6204 * other flows in other threads). 6205 */ 6206 dev_flow->handle->split_flow_id = qrss_id; 6207 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6208 error); 6209 if (ret < 0) 6210 goto exit; 6211 q_tag_spec.id = ret; 6212 } 6213 dev_flow = NULL; 6214 /* Add suffix subflow to execute Q/RSS. */ 6215 flow_split_info->prefix_layers = layers; 6216 flow_split_info->prefix_mark = 0; 6217 flow_split_info->table_id = 0; 6218 ret = flow_create_split_inner(dev, flow, &dev_flow, 6219 &q_attr, mtr_sfx ? items : 6220 q_items, q_actions, 6221 flow_split_info, error); 6222 if (ret < 0) 6223 goto exit; 6224 /* qrss ID should be freed if failed. */ 6225 qrss_id = 0; 6226 MLX5_ASSERT(dev_flow); 6227 } 6228 6229 exit: 6230 /* 6231 * We do not destroy the partially created sub_flows in case of error. 6232 * These ones are included into parent flow list and will be destroyed 6233 * by flow_drv_destroy. 6234 */ 6235 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6236 qrss_id); 6237 mlx5_free(ext_actions); 6238 return ret; 6239 } 6240 6241 /** 6242 * Create meter internal drop flow with the original pattern. 6243 * 6244 * @param dev 6245 * Pointer to Ethernet device. 6246 * @param[in] flow 6247 * Parent flow structure pointer. 6248 * @param[in] attr 6249 * Flow rule attributes. 6250 * @param[in] items 6251 * Pattern specification (list terminated by the END pattern item). 6252 * @param[in] flow_split_info 6253 * Pointer to flow split info structure. 6254 * @param[in] fm 6255 * Pointer to flow meter structure. 6256 * @param[out] error 6257 * Perform verbose error reporting if not NULL. 6258 * @return 6259 * 0 on success, negative value otherwise 6260 */ 6261 static uint32_t 6262 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6263 struct rte_flow *flow, 6264 const struct rte_flow_attr *attr, 6265 const struct rte_flow_item items[], 6266 struct mlx5_flow_split_info *flow_split_info, 6267 struct mlx5_flow_meter_info *fm, 6268 struct rte_flow_error *error) 6269 { 6270 struct mlx5_flow *dev_flow = NULL; 6271 struct rte_flow_attr drop_attr = *attr; 6272 struct rte_flow_action drop_actions[3]; 6273 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6274 6275 MLX5_ASSERT(fm->drop_cnt); 6276 drop_actions[0].type = 6277 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6278 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6279 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6280 drop_actions[1].conf = NULL; 6281 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6282 drop_actions[2].conf = NULL; 6283 drop_split_info.external = false; 6284 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6285 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6286 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6287 return flow_create_split_inner(dev, flow, &dev_flow, 6288 &drop_attr, items, drop_actions, 6289 &drop_split_info, error); 6290 } 6291 6292 /** 6293 * The splitting for meter feature. 6294 * 6295 * - The meter flow will be split to two flows as prefix and 6296 * suffix flow. The packets make sense only it pass the prefix 6297 * meter action. 6298 * 6299 * - Reg_C_5 is used for the packet to match betweend prefix and 6300 * suffix flow. 6301 * 6302 * @param dev 6303 * Pointer to Ethernet device. 6304 * @param[in] flow 6305 * Parent flow structure pointer. 6306 * @param[in] attr 6307 * Flow rule attributes. 6308 * @param[in] items 6309 * Pattern specification (list terminated by the END pattern item). 6310 * @param[in] actions 6311 * Associated actions (list terminated by the END action). 6312 * @param[in] flow_split_info 6313 * Pointer to flow split info structure. 6314 * @param[out] error 6315 * Perform verbose error reporting if not NULL. 6316 * @return 6317 * 0 on success, negative value otherwise 6318 */ 6319 static int 6320 flow_create_split_meter(struct rte_eth_dev *dev, 6321 struct rte_flow *flow, 6322 const struct rte_flow_attr *attr, 6323 const struct rte_flow_item items[], 6324 const struct rte_flow_action actions[], 6325 struct mlx5_flow_split_info *flow_split_info, 6326 struct rte_flow_error *error) 6327 { 6328 struct mlx5_priv *priv = dev->data->dev_private; 6329 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6330 struct rte_flow_action *sfx_actions = NULL; 6331 struct rte_flow_action *pre_actions = NULL; 6332 struct rte_flow_item *sfx_items = NULL; 6333 struct mlx5_flow *dev_flow = NULL; 6334 struct rte_flow_attr sfx_attr = *attr; 6335 struct mlx5_flow_meter_info *fm = NULL; 6336 uint8_t skip_scale_restore; 6337 bool has_mtr = false; 6338 bool has_modify = false; 6339 bool set_mtr_reg = true; 6340 bool is_mtr_hierarchy = false; 6341 uint32_t meter_id = 0; 6342 uint32_t mtr_idx = 0; 6343 uint32_t mtr_flow_id = 0; 6344 size_t act_size; 6345 size_t item_size; 6346 int actions_n = 0; 6347 int ret = 0; 6348 6349 if (priv->mtr_en) 6350 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6351 &has_modify, &meter_id); 6352 if (has_mtr) { 6353 if (flow->meter) { 6354 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6355 if (!fm) 6356 return rte_flow_error_set(error, EINVAL, 6357 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6358 NULL, "Meter not found."); 6359 } else { 6360 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6361 if (!fm) 6362 return rte_flow_error_set(error, EINVAL, 6363 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6364 NULL, "Meter not found."); 6365 ret = mlx5_flow_meter_attach(priv, fm, 6366 &sfx_attr, error); 6367 if (ret) 6368 return -rte_errno; 6369 flow->meter = mtr_idx; 6370 } 6371 MLX5_ASSERT(wks); 6372 wks->fm = fm; 6373 if (!fm->def_policy) { 6374 wks->policy = mlx5_flow_meter_policy_find(dev, 6375 fm->policy_id, 6376 NULL); 6377 MLX5_ASSERT(wks->policy); 6378 if (wks->policy->mark) 6379 wks->mark = 1; 6380 if (wks->policy->is_hierarchy) { 6381 wks->final_policy = 6382 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6383 wks->policy); 6384 if (!wks->final_policy) 6385 return rte_flow_error_set(error, 6386 EINVAL, 6387 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6388 "Failed to find terminal policy of hierarchy."); 6389 is_mtr_hierarchy = true; 6390 } 6391 } 6392 /* 6393 * If it isn't default-policy Meter, and 6394 * 1. There's no action in flow to change 6395 * packet (modify/encap/decap etc.), OR 6396 * 2. No drop count needed for this meter. 6397 * 3. It's not meter hierarchy. 6398 * Then no need to use regC to save meter id anymore. 6399 */ 6400 if (!fm->def_policy && !is_mtr_hierarchy && 6401 (!has_modify || !fm->drop_cnt)) 6402 set_mtr_reg = false; 6403 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6404 #define METER_PREFIX_ACTION 7 6405 act_size = (sizeof(struct rte_flow_action) * 6406 (actions_n + METER_PREFIX_ACTION)) + 6407 sizeof(struct mlx5_rte_flow_action_set_tag); 6408 /* Suffix items: tag, vlan, port id, end. */ 6409 #define METER_SUFFIX_ITEM 4 6410 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6411 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6412 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6413 0, SOCKET_ID_ANY); 6414 if (!sfx_actions) 6415 return rte_flow_error_set(error, ENOMEM, 6416 RTE_FLOW_ERROR_TYPE_ACTION, 6417 NULL, "no memory to split " 6418 "meter flow"); 6419 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6420 act_size); 6421 /* There's no suffix flow for meter of non-default policy. */ 6422 if (!fm->def_policy) 6423 pre_actions = sfx_actions + 1; 6424 else 6425 pre_actions = sfx_actions + actions_n; 6426 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6427 items, sfx_items, actions, 6428 sfx_actions, pre_actions, 6429 (set_mtr_reg ? &mtr_flow_id : NULL), 6430 error); 6431 if (ret) { 6432 ret = -rte_errno; 6433 goto exit; 6434 } 6435 /* Add the prefix subflow. */ 6436 skip_scale_restore = flow_split_info->skip_scale; 6437 flow_split_info->skip_scale |= 6438 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6439 ret = flow_create_split_inner(dev, flow, &dev_flow, 6440 attr, items, pre_actions, 6441 flow_split_info, error); 6442 flow_split_info->skip_scale = skip_scale_restore; 6443 if (ret) { 6444 if (mtr_flow_id) 6445 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6446 ret = -rte_errno; 6447 goto exit; 6448 } 6449 if (mtr_flow_id) { 6450 dev_flow->handle->split_flow_id = mtr_flow_id; 6451 dev_flow->handle->is_meter_flow_id = 1; 6452 } 6453 if (!fm->def_policy) { 6454 if (!set_mtr_reg && fm->drop_cnt) 6455 ret = 6456 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6457 &sfx_attr, items, 6458 flow_split_info, 6459 fm, error); 6460 goto exit; 6461 } 6462 /* Setting the sfx group atrr. */ 6463 sfx_attr.group = sfx_attr.transfer ? 6464 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6465 MLX5_FLOW_TABLE_LEVEL_METER; 6466 flow_split_info->prefix_layers = 6467 flow_get_prefix_layer_flags(dev_flow); 6468 flow_split_info->prefix_mark |= wks->mark; 6469 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6470 } 6471 /* Add the prefix subflow. */ 6472 ret = flow_create_split_metadata(dev, flow, 6473 &sfx_attr, sfx_items ? 6474 sfx_items : items, 6475 sfx_actions ? sfx_actions : actions, 6476 flow_split_info, error); 6477 exit: 6478 if (sfx_actions) 6479 mlx5_free(sfx_actions); 6480 return ret; 6481 } 6482 6483 /** 6484 * The splitting for sample feature. 6485 * 6486 * Once Sample action is detected in the action list, the flow actions should 6487 * be split into prefix sub flow and suffix sub flow. 6488 * 6489 * The original items remain in the prefix sub flow, all actions preceding the 6490 * sample action and the sample action itself will be copied to the prefix 6491 * sub flow, the actions following the sample action will be copied to the 6492 * suffix sub flow, Queue action always be located in the suffix sub flow. 6493 * 6494 * In order to make the packet from prefix sub flow matches with suffix sub 6495 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6496 * flow uses tag item with the unique flow id. 6497 * 6498 * @param dev 6499 * Pointer to Ethernet device. 6500 * @param[in] flow 6501 * Parent flow structure pointer. 6502 * @param[in] attr 6503 * Flow rule attributes. 6504 * @param[in] items 6505 * Pattern specification (list terminated by the END pattern item). 6506 * @param[in] actions 6507 * Associated actions (list terminated by the END action). 6508 * @param[in] flow_split_info 6509 * Pointer to flow split info structure. 6510 * @param[out] error 6511 * Perform verbose error reporting if not NULL. 6512 * @return 6513 * 0 on success, negative value otherwise 6514 */ 6515 static int 6516 flow_create_split_sample(struct rte_eth_dev *dev, 6517 struct rte_flow *flow, 6518 const struct rte_flow_attr *attr, 6519 const struct rte_flow_item items[], 6520 const struct rte_flow_action actions[], 6521 struct mlx5_flow_split_info *flow_split_info, 6522 struct rte_flow_error *error) 6523 { 6524 struct mlx5_priv *priv = dev->data->dev_private; 6525 struct rte_flow_action *sfx_actions = NULL; 6526 struct rte_flow_action *pre_actions = NULL; 6527 struct rte_flow_item *sfx_items = NULL; 6528 struct mlx5_flow *dev_flow = NULL; 6529 struct rte_flow_attr sfx_attr = *attr; 6530 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6531 struct mlx5_flow_dv_sample_resource *sample_res; 6532 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6533 struct mlx5_flow_tbl_resource *sfx_tbl; 6534 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6535 #endif 6536 size_t act_size; 6537 size_t item_size; 6538 uint32_t fdb_tx = 0; 6539 int32_t tag_id = 0; 6540 int actions_n = 0; 6541 int sample_action_pos; 6542 int qrss_action_pos; 6543 int add_tag = 0; 6544 int modify_after_mirror = 0; 6545 uint16_t jump_table = 0; 6546 const uint32_t next_ft_step = 1; 6547 int ret = 0; 6548 6549 if (priv->sampler_en) 6550 actions_n = flow_check_match_action(actions, attr, 6551 RTE_FLOW_ACTION_TYPE_SAMPLE, 6552 &sample_action_pos, &qrss_action_pos, 6553 &modify_after_mirror); 6554 if (actions_n) { 6555 /* The prefix actions must includes sample, tag, end. */ 6556 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6557 + sizeof(struct mlx5_rte_flow_action_set_tag); 6558 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6559 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6560 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6561 item_size), 0, SOCKET_ID_ANY); 6562 if (!sfx_actions) 6563 return rte_flow_error_set(error, ENOMEM, 6564 RTE_FLOW_ERROR_TYPE_ACTION, 6565 NULL, "no memory to split " 6566 "sample flow"); 6567 /* The representor_id is UINT16_MAX for uplink. */ 6568 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6569 /* 6570 * When reg_c_preserve is set, metadata registers Cx preserve 6571 * their value even through packet duplication. 6572 */ 6573 add_tag = (!fdb_tx || 6574 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6575 if (add_tag) 6576 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6577 + act_size); 6578 if (modify_after_mirror) 6579 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6580 next_ft_step; 6581 pre_actions = sfx_actions + actions_n; 6582 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6583 actions, sfx_actions, 6584 pre_actions, actions_n, 6585 sample_action_pos, 6586 qrss_action_pos, jump_table, 6587 error); 6588 if (tag_id < 0 || (add_tag && !tag_id)) { 6589 ret = -rte_errno; 6590 goto exit; 6591 } 6592 if (modify_after_mirror) 6593 flow_split_info->skip_scale = 6594 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6595 /* Add the prefix subflow. */ 6596 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6597 items, pre_actions, 6598 flow_split_info, error); 6599 if (ret) { 6600 ret = -rte_errno; 6601 goto exit; 6602 } 6603 dev_flow->handle->split_flow_id = tag_id; 6604 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6605 if (!modify_after_mirror) { 6606 /* Set the sfx group attr. */ 6607 sample_res = (struct mlx5_flow_dv_sample_resource *) 6608 dev_flow->dv.sample_res; 6609 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6610 sample_res->normal_path_tbl; 6611 sfx_tbl_data = container_of(sfx_tbl, 6612 struct mlx5_flow_tbl_data_entry, 6613 tbl); 6614 sfx_attr.group = sfx_attr.transfer ? 6615 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6616 } else { 6617 MLX5_ASSERT(attr->transfer); 6618 sfx_attr.group = jump_table; 6619 } 6620 flow_split_info->prefix_layers = 6621 flow_get_prefix_layer_flags(dev_flow); 6622 MLX5_ASSERT(wks); 6623 flow_split_info->prefix_mark |= wks->mark; 6624 /* Suffix group level already be scaled with factor, set 6625 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6626 * again in translation. 6627 */ 6628 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6629 #endif 6630 } 6631 /* Add the suffix subflow. */ 6632 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6633 sfx_items ? sfx_items : items, 6634 sfx_actions ? sfx_actions : actions, 6635 flow_split_info, error); 6636 exit: 6637 if (sfx_actions) 6638 mlx5_free(sfx_actions); 6639 return ret; 6640 } 6641 6642 /** 6643 * Split the flow to subflow set. The splitters might be linked 6644 * in the chain, like this: 6645 * flow_create_split_outer() calls: 6646 * flow_create_split_meter() calls: 6647 * flow_create_split_metadata(meter_subflow_0) calls: 6648 * flow_create_split_inner(metadata_subflow_0) 6649 * flow_create_split_inner(metadata_subflow_1) 6650 * flow_create_split_inner(metadata_subflow_2) 6651 * flow_create_split_metadata(meter_subflow_1) calls: 6652 * flow_create_split_inner(metadata_subflow_0) 6653 * flow_create_split_inner(metadata_subflow_1) 6654 * flow_create_split_inner(metadata_subflow_2) 6655 * 6656 * This provide flexible way to add new levels of flow splitting. 6657 * The all of successfully created subflows are included to the 6658 * parent flow dev_flow list. 6659 * 6660 * @param dev 6661 * Pointer to Ethernet device. 6662 * @param[in] flow 6663 * Parent flow structure pointer. 6664 * @param[in] attr 6665 * Flow rule attributes. 6666 * @param[in] items 6667 * Pattern specification (list terminated by the END pattern item). 6668 * @param[in] actions 6669 * Associated actions (list terminated by the END action). 6670 * @param[in] flow_split_info 6671 * Pointer to flow split info structure. 6672 * @param[out] error 6673 * Perform verbose error reporting if not NULL. 6674 * @return 6675 * 0 on success, negative value otherwise 6676 */ 6677 static int 6678 flow_create_split_outer(struct rte_eth_dev *dev, 6679 struct rte_flow *flow, 6680 const struct rte_flow_attr *attr, 6681 const struct rte_flow_item items[], 6682 const struct rte_flow_action actions[], 6683 struct mlx5_flow_split_info *flow_split_info, 6684 struct rte_flow_error *error) 6685 { 6686 int ret; 6687 6688 ret = flow_create_split_sample(dev, flow, attr, items, 6689 actions, flow_split_info, error); 6690 MLX5_ASSERT(ret <= 0); 6691 return ret; 6692 } 6693 6694 static inline struct mlx5_flow_tunnel * 6695 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6696 { 6697 struct mlx5_flow_tunnel *tunnel; 6698 6699 #pragma GCC diagnostic push 6700 #pragma GCC diagnostic ignored "-Wcast-qual" 6701 tunnel = (typeof(tunnel))flow->tunnel; 6702 #pragma GCC diagnostic pop 6703 6704 return tunnel; 6705 } 6706 6707 /** 6708 * Adjust flow RSS workspace if needed. 6709 * 6710 * @param wks 6711 * Pointer to thread flow work space. 6712 * @param rss_desc 6713 * Pointer to RSS descriptor. 6714 * @param[in] nrssq_num 6715 * New RSS queue number. 6716 * 6717 * @return 6718 * 0 on success, -1 otherwise and rte_errno is set. 6719 */ 6720 static int 6721 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6722 struct mlx5_flow_rss_desc *rss_desc, 6723 uint32_t nrssq_num) 6724 { 6725 if (likely(nrssq_num <= wks->rssq_num)) 6726 return 0; 6727 rss_desc->queue = realloc(rss_desc->queue, 6728 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6729 if (!rss_desc->queue) { 6730 rte_errno = ENOMEM; 6731 return -1; 6732 } 6733 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6734 return 0; 6735 } 6736 6737 /** 6738 * Create a flow and add it to @p list. 6739 * 6740 * @param dev 6741 * Pointer to Ethernet device. 6742 * @param list 6743 * Pointer to a TAILQ flow list. If this parameter NULL, 6744 * no list insertion occurred, flow is just created, 6745 * this is caller's responsibility to track the 6746 * created flow. 6747 * @param[in] attr 6748 * Flow rule attributes. 6749 * @param[in] items 6750 * Pattern specification (list terminated by the END pattern item). 6751 * @param[in] actions 6752 * Associated actions (list terminated by the END action). 6753 * @param[in] external 6754 * This flow rule is created by request external to PMD. 6755 * @param[out] error 6756 * Perform verbose error reporting if not NULL. 6757 * 6758 * @return 6759 * A flow index on success, 0 otherwise and rte_errno is set. 6760 */ 6761 static uint32_t 6762 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6763 const struct rte_flow_attr *attr, 6764 const struct rte_flow_item items[], 6765 const struct rte_flow_action original_actions[], 6766 bool external, struct rte_flow_error *error) 6767 { 6768 struct mlx5_priv *priv = dev->data->dev_private; 6769 struct rte_flow *flow = NULL; 6770 struct mlx5_flow *dev_flow; 6771 const struct rte_flow_action_rss *rss = NULL; 6772 struct mlx5_translated_action_handle 6773 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6774 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6775 union { 6776 struct mlx5_flow_expand_rss buf; 6777 uint8_t buffer[4096]; 6778 } expand_buffer; 6779 union { 6780 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6781 uint8_t buffer[2048]; 6782 } actions_rx; 6783 union { 6784 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6785 uint8_t buffer[2048]; 6786 } actions_hairpin_tx; 6787 union { 6788 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6789 uint8_t buffer[2048]; 6790 } items_tx; 6791 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6792 struct mlx5_flow_rss_desc *rss_desc; 6793 const struct rte_flow_action *p_actions_rx; 6794 uint32_t i; 6795 uint32_t idx = 0; 6796 int hairpin_flow; 6797 struct rte_flow_attr attr_tx = { .priority = 0 }; 6798 const struct rte_flow_action *actions; 6799 struct rte_flow_action *translated_actions = NULL; 6800 struct mlx5_flow_tunnel *tunnel; 6801 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6802 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6803 struct mlx5_flow_split_info flow_split_info = { 6804 .external = !!external, 6805 .skip_scale = 0, 6806 .flow_idx = 0, 6807 .prefix_mark = 0, 6808 .prefix_layers = 0, 6809 .table_id = 0 6810 }; 6811 int ret; 6812 6813 MLX5_ASSERT(wks); 6814 rss_desc = &wks->rss_desc; 6815 ret = flow_action_handles_translate(dev, original_actions, 6816 indir_actions, 6817 &indir_actions_n, 6818 &translated_actions, error); 6819 if (ret < 0) { 6820 MLX5_ASSERT(translated_actions == NULL); 6821 return 0; 6822 } 6823 actions = translated_actions ? translated_actions : original_actions; 6824 p_actions_rx = actions; 6825 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6826 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6827 external, hairpin_flow, error); 6828 if (ret < 0) 6829 goto error_before_hairpin_split; 6830 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6831 if (!flow) { 6832 rte_errno = ENOMEM; 6833 goto error_before_hairpin_split; 6834 } 6835 if (hairpin_flow > 0) { 6836 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6837 rte_errno = EINVAL; 6838 goto error_before_hairpin_split; 6839 } 6840 flow_hairpin_split(dev, actions, actions_rx.actions, 6841 actions_hairpin_tx.actions, items_tx.items, 6842 idx); 6843 p_actions_rx = actions_rx.actions; 6844 } 6845 flow_split_info.flow_idx = idx; 6846 flow->drv_type = flow_get_drv_type(dev, attr); 6847 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6848 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6849 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6850 /* RSS Action only works on NIC RX domain */ 6851 if (attr->ingress && !attr->transfer) 6852 rss = flow_get_rss_action(dev, p_actions_rx); 6853 if (rss) { 6854 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6855 return 0; 6856 /* 6857 * The following information is required by 6858 * mlx5_flow_hashfields_adjust() in advance. 6859 */ 6860 rss_desc->level = rss->level; 6861 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6862 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6863 } 6864 flow->dev_handles = 0; 6865 if (rss && rss->types) { 6866 unsigned int graph_root; 6867 6868 graph_root = find_graph_root(rss->level); 6869 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6870 items, rss->types, 6871 mlx5_support_expansion, graph_root); 6872 MLX5_ASSERT(ret > 0 && 6873 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6874 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6875 for (i = 0; i < buf->entries; ++i) 6876 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6877 } 6878 } else { 6879 buf->entries = 1; 6880 buf->entry[0].pattern = (void *)(uintptr_t)items; 6881 } 6882 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6883 indir_actions_n); 6884 for (i = 0; i < buf->entries; ++i) { 6885 /* Initialize flow split data. */ 6886 flow_split_info.prefix_layers = 0; 6887 flow_split_info.prefix_mark = 0; 6888 flow_split_info.skip_scale = 0; 6889 /* 6890 * The splitter may create multiple dev_flows, 6891 * depending on configuration. In the simplest 6892 * case it just creates unmodified original flow. 6893 */ 6894 ret = flow_create_split_outer(dev, flow, attr, 6895 buf->entry[i].pattern, 6896 p_actions_rx, &flow_split_info, 6897 error); 6898 if (ret < 0) 6899 goto error; 6900 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6901 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6902 p_actions_rx, 6903 idx, 6904 wks->flows[0].tunnel, 6905 &default_miss_ctx, 6906 error); 6907 if (ret < 0) { 6908 mlx5_free(default_miss_ctx.queue); 6909 goto error; 6910 } 6911 } 6912 } 6913 /* Create the tx flow. */ 6914 if (hairpin_flow) { 6915 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6916 attr_tx.ingress = 0; 6917 attr_tx.egress = 1; 6918 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6919 actions_hairpin_tx.actions, 6920 idx, error); 6921 if (!dev_flow) 6922 goto error; 6923 dev_flow->flow = flow; 6924 dev_flow->external = 0; 6925 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6926 dev_flow->handle, next); 6927 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6928 items_tx.items, 6929 actions_hairpin_tx.actions, error); 6930 if (ret < 0) 6931 goto error; 6932 } 6933 /* 6934 * Update the metadata register copy table. If extensive 6935 * metadata feature is enabled and registers are supported 6936 * we might create the extra rte_flow for each unique 6937 * MARK/FLAG action ID. 6938 * 6939 * The table is updated for ingress Flows only, because 6940 * the egress Flows belong to the different device and 6941 * copy table should be updated in peer NIC Rx domain. 6942 */ 6943 if (attr->ingress && 6944 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6945 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6946 if (ret) 6947 goto error; 6948 } 6949 /* 6950 * If the flow is external (from application) OR device is started, 6951 * OR mreg discover, then apply immediately. 6952 */ 6953 if (external || dev->data->dev_started || 6954 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6955 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6956 ret = flow_drv_apply(dev, flow, error); 6957 if (ret < 0) 6958 goto error; 6959 } 6960 flow->type = type; 6961 flow_rxq_flags_set(dev, flow); 6962 rte_free(translated_actions); 6963 tunnel = flow_tunnel_from_rule(wks->flows); 6964 if (tunnel) { 6965 flow->tunnel = 1; 6966 flow->tunnel_id = tunnel->tunnel_id; 6967 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6968 mlx5_free(default_miss_ctx.queue); 6969 } 6970 mlx5_flow_pop_thread_workspace(); 6971 return idx; 6972 error: 6973 MLX5_ASSERT(flow); 6974 ret = rte_errno; /* Save rte_errno before cleanup. */ 6975 flow_mreg_del_copy_action(dev, flow); 6976 flow_drv_destroy(dev, flow); 6977 if (rss_desc->shared_rss) 6978 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6979 mlx5_ipool_get 6980 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6981 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6982 mlx5_ipool_free(priv->flows[type], idx); 6983 rte_errno = ret; /* Restore rte_errno. */ 6984 ret = rte_errno; 6985 rte_errno = ret; 6986 mlx5_flow_pop_thread_workspace(); 6987 error_before_hairpin_split: 6988 rte_free(translated_actions); 6989 return 0; 6990 } 6991 6992 /** 6993 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6994 * incoming packets to table 1. 6995 * 6996 * Other flow rules, requested for group n, will be created in 6997 * e-switch table n+1. 6998 * Jump action to e-switch group n will be created to group n+1. 6999 * 7000 * Used when working in switchdev mode, to utilise advantages of table 1 7001 * and above. 7002 * 7003 * @param dev 7004 * Pointer to Ethernet device. 7005 * 7006 * @return 7007 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7008 */ 7009 struct rte_flow * 7010 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7011 { 7012 const struct rte_flow_attr attr = { 7013 .group = 0, 7014 .priority = 0, 7015 .ingress = 1, 7016 .egress = 0, 7017 .transfer = 1, 7018 }; 7019 const struct rte_flow_item pattern = { 7020 .type = RTE_FLOW_ITEM_TYPE_END, 7021 }; 7022 struct rte_flow_action_jump jump = { 7023 .group = 1, 7024 }; 7025 const struct rte_flow_action actions[] = { 7026 { 7027 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7028 .conf = &jump, 7029 }, 7030 { 7031 .type = RTE_FLOW_ACTION_TYPE_END, 7032 }, 7033 }; 7034 struct rte_flow_error error; 7035 7036 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7037 &attr, &pattern, 7038 actions, false, &error); 7039 } 7040 7041 /** 7042 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7043 * and sq number, directs all packets to peer vport. 7044 * 7045 * @param dev 7046 * Pointer to Ethernet device. 7047 * @param txq 7048 * Txq index. 7049 * 7050 * @return 7051 * Flow ID on success, 0 otherwise and rte_errno is set. 7052 */ 7053 uint32_t 7054 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 7055 { 7056 struct rte_flow_attr attr = { 7057 .group = 0, 7058 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7059 .ingress = 1, 7060 .egress = 0, 7061 .transfer = 1, 7062 }; 7063 struct rte_flow_item_port_id port_spec = { 7064 .id = MLX5_PORT_ESW_MGR, 7065 }; 7066 struct mlx5_rte_flow_item_tx_queue txq_spec = { 7067 .queue = txq, 7068 }; 7069 struct rte_flow_item pattern[] = { 7070 { 7071 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7072 .spec = &port_spec, 7073 }, 7074 { 7075 .type = (enum rte_flow_item_type) 7076 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7077 .spec = &txq_spec, 7078 }, 7079 { 7080 .type = RTE_FLOW_ITEM_TYPE_END, 7081 }, 7082 }; 7083 struct rte_flow_action_jump jump = { 7084 .group = 1, 7085 }; 7086 struct rte_flow_action_port_id port = { 7087 .id = dev->data->port_id, 7088 }; 7089 struct rte_flow_action actions[] = { 7090 { 7091 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7092 .conf = &jump, 7093 }, 7094 { 7095 .type = RTE_FLOW_ACTION_TYPE_END, 7096 }, 7097 }; 7098 struct rte_flow_error error; 7099 7100 /* 7101 * Creates group 0, highest priority jump flow. 7102 * Matches txq to bypass kernel packets. 7103 */ 7104 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7105 false, &error) == 0) 7106 return 0; 7107 /* Create group 1, lowest priority redirect flow for txq. */ 7108 attr.group = 1; 7109 actions[0].conf = &port; 7110 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7111 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7112 actions, false, &error); 7113 } 7114 7115 /** 7116 * Validate a flow supported by the NIC. 7117 * 7118 * @see rte_flow_validate() 7119 * @see rte_flow_ops 7120 */ 7121 int 7122 mlx5_flow_validate(struct rte_eth_dev *dev, 7123 const struct rte_flow_attr *attr, 7124 const struct rte_flow_item items[], 7125 const struct rte_flow_action original_actions[], 7126 struct rte_flow_error *error) 7127 { 7128 int hairpin_flow; 7129 struct mlx5_translated_action_handle 7130 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7131 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7132 const struct rte_flow_action *actions; 7133 struct rte_flow_action *translated_actions = NULL; 7134 int ret = flow_action_handles_translate(dev, original_actions, 7135 indir_actions, 7136 &indir_actions_n, 7137 &translated_actions, error); 7138 7139 if (ret) 7140 return ret; 7141 actions = translated_actions ? translated_actions : original_actions; 7142 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7143 ret = flow_drv_validate(dev, attr, items, actions, 7144 true, hairpin_flow, error); 7145 rte_free(translated_actions); 7146 return ret; 7147 } 7148 7149 /** 7150 * Create a flow. 7151 * 7152 * @see rte_flow_create() 7153 * @see rte_flow_ops 7154 */ 7155 struct rte_flow * 7156 mlx5_flow_create(struct rte_eth_dev *dev, 7157 const struct rte_flow_attr *attr, 7158 const struct rte_flow_item items[], 7159 const struct rte_flow_action actions[], 7160 struct rte_flow_error *error) 7161 { 7162 struct mlx5_priv *priv = dev->data->dev_private; 7163 7164 if (priv->sh->config.dv_flow_en == 2) { 7165 rte_flow_error_set(error, ENOTSUP, 7166 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7167 NULL, 7168 "Flow non-Q creation not supported"); 7169 return NULL; 7170 } 7171 /* 7172 * If the device is not started yet, it is not allowed to created a 7173 * flow from application. PMD default flows and traffic control flows 7174 * are not affected. 7175 */ 7176 if (unlikely(!dev->data->dev_started)) { 7177 DRV_LOG(DEBUG, "port %u is not started when " 7178 "inserting a flow", dev->data->port_id); 7179 rte_flow_error_set(error, ENODEV, 7180 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7181 NULL, 7182 "port not started"); 7183 return NULL; 7184 } 7185 7186 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7187 attr, items, actions, 7188 true, error); 7189 } 7190 7191 /** 7192 * Destroy a flow in a list. 7193 * 7194 * @param dev 7195 * Pointer to Ethernet device. 7196 * @param[in] flow_idx 7197 * Index of flow to destroy. 7198 */ 7199 static void 7200 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7201 uint32_t flow_idx) 7202 { 7203 struct mlx5_priv *priv = dev->data->dev_private; 7204 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7205 7206 if (!flow) 7207 return; 7208 MLX5_ASSERT(flow->type == type); 7209 /* 7210 * Update RX queue flags only if port is started, otherwise it is 7211 * already clean. 7212 */ 7213 if (dev->data->dev_started) 7214 flow_rxq_flags_trim(dev, flow); 7215 flow_drv_destroy(dev, flow); 7216 if (flow->tunnel) { 7217 struct mlx5_flow_tunnel *tunnel; 7218 7219 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7220 RTE_VERIFY(tunnel); 7221 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7222 mlx5_flow_tunnel_free(dev, tunnel); 7223 } 7224 flow_mreg_del_copy_action(dev, flow); 7225 mlx5_ipool_free(priv->flows[type], flow_idx); 7226 } 7227 7228 /** 7229 * Destroy all flows. 7230 * 7231 * @param dev 7232 * Pointer to Ethernet device. 7233 * @param type 7234 * Flow type to be flushed. 7235 * @param active 7236 * If flushing is called actively. 7237 */ 7238 void 7239 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7240 bool active) 7241 { 7242 struct mlx5_priv *priv = dev->data->dev_private; 7243 uint32_t num_flushed = 0, fidx = 1; 7244 struct rte_flow *flow; 7245 7246 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7247 if (priv->sh->config.dv_flow_en == 2 && 7248 type == MLX5_FLOW_TYPE_GEN) { 7249 flow_hw_q_flow_flush(dev, NULL); 7250 return; 7251 } 7252 #endif 7253 7254 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7255 flow_list_destroy(dev, type, fidx); 7256 num_flushed++; 7257 } 7258 if (active) { 7259 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7260 dev->data->port_id, num_flushed); 7261 } 7262 } 7263 7264 /** 7265 * Stop all default actions for flows. 7266 * 7267 * @param dev 7268 * Pointer to Ethernet device. 7269 */ 7270 void 7271 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7272 { 7273 flow_mreg_del_default_copy_action(dev); 7274 flow_rxq_flags_clear(dev); 7275 } 7276 7277 /** 7278 * Start all default actions for flows. 7279 * 7280 * @param dev 7281 * Pointer to Ethernet device. 7282 * @return 7283 * 0 on success, a negative errno value otherwise and rte_errno is set. 7284 */ 7285 int 7286 mlx5_flow_start_default(struct rte_eth_dev *dev) 7287 { 7288 struct rte_flow_error error; 7289 7290 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7291 return flow_mreg_add_default_copy_action(dev, &error); 7292 } 7293 7294 /** 7295 * Release key of thread specific flow workspace data. 7296 */ 7297 void 7298 flow_release_workspace(void *data) 7299 { 7300 struct mlx5_flow_workspace *wks = data; 7301 struct mlx5_flow_workspace *next; 7302 7303 while (wks) { 7304 next = wks->next; 7305 free(wks->rss_desc.queue); 7306 free(wks); 7307 wks = next; 7308 } 7309 } 7310 7311 /** 7312 * Get thread specific current flow workspace. 7313 * 7314 * @return pointer to thread specific flow workspace data, NULL on error. 7315 */ 7316 struct mlx5_flow_workspace* 7317 mlx5_flow_get_thread_workspace(void) 7318 { 7319 struct mlx5_flow_workspace *data; 7320 7321 data = mlx5_flow_os_get_specific_workspace(); 7322 MLX5_ASSERT(data && data->inuse); 7323 if (!data || !data->inuse) 7324 DRV_LOG(ERR, "flow workspace not initialized."); 7325 return data; 7326 } 7327 7328 /** 7329 * Allocate and init new flow workspace. 7330 * 7331 * @return pointer to flow workspace data, NULL on error. 7332 */ 7333 static struct mlx5_flow_workspace* 7334 flow_alloc_thread_workspace(void) 7335 { 7336 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7337 7338 if (!data) { 7339 DRV_LOG(ERR, "Failed to allocate flow workspace " 7340 "memory."); 7341 return NULL; 7342 } 7343 data->rss_desc.queue = calloc(1, 7344 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7345 if (!data->rss_desc.queue) 7346 goto err; 7347 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7348 return data; 7349 err: 7350 free(data->rss_desc.queue); 7351 free(data); 7352 return NULL; 7353 } 7354 7355 /** 7356 * Get new thread specific flow workspace. 7357 * 7358 * If current workspace inuse, create new one and set as current. 7359 * 7360 * @return pointer to thread specific flow workspace data, NULL on error. 7361 */ 7362 static struct mlx5_flow_workspace* 7363 mlx5_flow_push_thread_workspace(void) 7364 { 7365 struct mlx5_flow_workspace *curr; 7366 struct mlx5_flow_workspace *data; 7367 7368 curr = mlx5_flow_os_get_specific_workspace(); 7369 if (!curr) { 7370 data = flow_alloc_thread_workspace(); 7371 if (!data) 7372 return NULL; 7373 } else if (!curr->inuse) { 7374 data = curr; 7375 } else if (curr->next) { 7376 data = curr->next; 7377 } else { 7378 data = flow_alloc_thread_workspace(); 7379 if (!data) 7380 return NULL; 7381 curr->next = data; 7382 data->prev = curr; 7383 } 7384 data->inuse = 1; 7385 data->flow_idx = 0; 7386 /* Set as current workspace */ 7387 if (mlx5_flow_os_set_specific_workspace(data)) 7388 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7389 return data; 7390 } 7391 7392 /** 7393 * Close current thread specific flow workspace. 7394 * 7395 * If previous workspace available, set it as current. 7396 * 7397 * @return pointer to thread specific flow workspace data, NULL on error. 7398 */ 7399 static void 7400 mlx5_flow_pop_thread_workspace(void) 7401 { 7402 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7403 7404 if (!data) 7405 return; 7406 if (!data->inuse) { 7407 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7408 return; 7409 } 7410 data->inuse = 0; 7411 if (!data->prev) 7412 return; 7413 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7414 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7415 } 7416 7417 /** 7418 * Verify the flow list is empty 7419 * 7420 * @param dev 7421 * Pointer to Ethernet device. 7422 * 7423 * @return the number of flows not released. 7424 */ 7425 int 7426 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7427 { 7428 struct mlx5_priv *priv = dev->data->dev_private; 7429 struct rte_flow *flow; 7430 uint32_t idx = 0; 7431 int ret = 0, i; 7432 7433 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7434 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7435 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7436 dev->data->port_id, (void *)flow); 7437 ret++; 7438 } 7439 } 7440 return ret; 7441 } 7442 7443 /** 7444 * Enable default hairpin egress flow. 7445 * 7446 * @param dev 7447 * Pointer to Ethernet device. 7448 * @param queue 7449 * The queue index. 7450 * 7451 * @return 7452 * 0 on success, a negative errno value otherwise and rte_errno is set. 7453 */ 7454 int 7455 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7456 uint32_t queue) 7457 { 7458 const struct rte_flow_attr attr = { 7459 .egress = 1, 7460 .priority = 0, 7461 }; 7462 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7463 .queue = queue, 7464 }; 7465 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7466 .queue = UINT32_MAX, 7467 }; 7468 struct rte_flow_item items[] = { 7469 { 7470 .type = (enum rte_flow_item_type) 7471 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7472 .spec = &queue_spec, 7473 .last = NULL, 7474 .mask = &queue_mask, 7475 }, 7476 { 7477 .type = RTE_FLOW_ITEM_TYPE_END, 7478 }, 7479 }; 7480 struct rte_flow_action_jump jump = { 7481 .group = MLX5_HAIRPIN_TX_TABLE, 7482 }; 7483 struct rte_flow_action actions[2]; 7484 uint32_t flow_idx; 7485 struct rte_flow_error error; 7486 7487 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7488 actions[0].conf = &jump; 7489 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7490 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7491 &attr, items, actions, false, &error); 7492 if (!flow_idx) { 7493 DRV_LOG(DEBUG, 7494 "Failed to create ctrl flow: rte_errno(%d)," 7495 " type(%d), message(%s)", 7496 rte_errno, error.type, 7497 error.message ? error.message : " (no stated reason)"); 7498 return -rte_errno; 7499 } 7500 return 0; 7501 } 7502 7503 /** 7504 * Enable a control flow configured from the control plane. 7505 * 7506 * @param dev 7507 * Pointer to Ethernet device. 7508 * @param eth_spec 7509 * An Ethernet flow spec to apply. 7510 * @param eth_mask 7511 * An Ethernet flow mask to apply. 7512 * @param vlan_spec 7513 * A VLAN flow spec to apply. 7514 * @param vlan_mask 7515 * A VLAN flow mask to apply. 7516 * 7517 * @return 7518 * 0 on success, a negative errno value otherwise and rte_errno is set. 7519 */ 7520 int 7521 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7522 struct rte_flow_item_eth *eth_spec, 7523 struct rte_flow_item_eth *eth_mask, 7524 struct rte_flow_item_vlan *vlan_spec, 7525 struct rte_flow_item_vlan *vlan_mask) 7526 { 7527 struct mlx5_priv *priv = dev->data->dev_private; 7528 const struct rte_flow_attr attr = { 7529 .ingress = 1, 7530 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7531 }; 7532 struct rte_flow_item items[] = { 7533 { 7534 .type = RTE_FLOW_ITEM_TYPE_ETH, 7535 .spec = eth_spec, 7536 .last = NULL, 7537 .mask = eth_mask, 7538 }, 7539 { 7540 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7541 RTE_FLOW_ITEM_TYPE_END, 7542 .spec = vlan_spec, 7543 .last = NULL, 7544 .mask = vlan_mask, 7545 }, 7546 { 7547 .type = RTE_FLOW_ITEM_TYPE_END, 7548 }, 7549 }; 7550 uint16_t queue[priv->reta_idx_n]; 7551 struct rte_flow_action_rss action_rss = { 7552 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7553 .level = 0, 7554 .types = priv->rss_conf.rss_hf, 7555 .key_len = priv->rss_conf.rss_key_len, 7556 .queue_num = priv->reta_idx_n, 7557 .key = priv->rss_conf.rss_key, 7558 .queue = queue, 7559 }; 7560 struct rte_flow_action actions[] = { 7561 { 7562 .type = RTE_FLOW_ACTION_TYPE_RSS, 7563 .conf = &action_rss, 7564 }, 7565 { 7566 .type = RTE_FLOW_ACTION_TYPE_END, 7567 }, 7568 }; 7569 uint32_t flow_idx; 7570 struct rte_flow_error error; 7571 unsigned int i; 7572 7573 if (!priv->reta_idx_n || !priv->rxqs_n) { 7574 return 0; 7575 } 7576 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7577 action_rss.types = 0; 7578 for (i = 0; i != priv->reta_idx_n; ++i) 7579 queue[i] = (*priv->reta_idx)[i]; 7580 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7581 &attr, items, actions, false, &error); 7582 if (!flow_idx) 7583 return -rte_errno; 7584 return 0; 7585 } 7586 7587 /** 7588 * Enable a flow control configured from the control plane. 7589 * 7590 * @param dev 7591 * Pointer to Ethernet device. 7592 * @param eth_spec 7593 * An Ethernet flow spec to apply. 7594 * @param eth_mask 7595 * An Ethernet flow mask to apply. 7596 * 7597 * @return 7598 * 0 on success, a negative errno value otherwise and rte_errno is set. 7599 */ 7600 int 7601 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7602 struct rte_flow_item_eth *eth_spec, 7603 struct rte_flow_item_eth *eth_mask) 7604 { 7605 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7606 } 7607 7608 /** 7609 * Create default miss flow rule matching lacp traffic 7610 * 7611 * @param dev 7612 * Pointer to Ethernet device. 7613 * @param eth_spec 7614 * An Ethernet flow spec to apply. 7615 * 7616 * @return 7617 * 0 on success, a negative errno value otherwise and rte_errno is set. 7618 */ 7619 int 7620 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7621 { 7622 /* 7623 * The LACP matching is done by only using ether type since using 7624 * a multicast dst mac causes kernel to give low priority to this flow. 7625 */ 7626 static const struct rte_flow_item_eth lacp_spec = { 7627 .type = RTE_BE16(0x8809), 7628 }; 7629 static const struct rte_flow_item_eth lacp_mask = { 7630 .type = 0xffff, 7631 }; 7632 const struct rte_flow_attr attr = { 7633 .ingress = 1, 7634 }; 7635 struct rte_flow_item items[] = { 7636 { 7637 .type = RTE_FLOW_ITEM_TYPE_ETH, 7638 .spec = &lacp_spec, 7639 .mask = &lacp_mask, 7640 }, 7641 { 7642 .type = RTE_FLOW_ITEM_TYPE_END, 7643 }, 7644 }; 7645 struct rte_flow_action actions[] = { 7646 { 7647 .type = (enum rte_flow_action_type) 7648 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7649 }, 7650 { 7651 .type = RTE_FLOW_ACTION_TYPE_END, 7652 }, 7653 }; 7654 struct rte_flow_error error; 7655 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7656 &attr, items, actions, 7657 false, &error); 7658 7659 if (!flow_idx) 7660 return -rte_errno; 7661 return 0; 7662 } 7663 7664 /** 7665 * Destroy a flow. 7666 * 7667 * @see rte_flow_destroy() 7668 * @see rte_flow_ops 7669 */ 7670 int 7671 mlx5_flow_destroy(struct rte_eth_dev *dev, 7672 struct rte_flow *flow, 7673 struct rte_flow_error *error __rte_unused) 7674 { 7675 struct mlx5_priv *priv = dev->data->dev_private; 7676 7677 if (priv->sh->config.dv_flow_en == 2) 7678 return rte_flow_error_set(error, ENOTSUP, 7679 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7680 NULL, 7681 "Flow non-Q destruction not supported"); 7682 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7683 (uintptr_t)(void *)flow); 7684 return 0; 7685 } 7686 7687 /** 7688 * Destroy all flows. 7689 * 7690 * @see rte_flow_flush() 7691 * @see rte_flow_ops 7692 */ 7693 int 7694 mlx5_flow_flush(struct rte_eth_dev *dev, 7695 struct rte_flow_error *error __rte_unused) 7696 { 7697 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7698 return 0; 7699 } 7700 7701 /** 7702 * Isolated mode. 7703 * 7704 * @see rte_flow_isolate() 7705 * @see rte_flow_ops 7706 */ 7707 int 7708 mlx5_flow_isolate(struct rte_eth_dev *dev, 7709 int enable, 7710 struct rte_flow_error *error) 7711 { 7712 struct mlx5_priv *priv = dev->data->dev_private; 7713 7714 if (dev->data->dev_started) { 7715 rte_flow_error_set(error, EBUSY, 7716 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7717 NULL, 7718 "port must be stopped first"); 7719 return -rte_errno; 7720 } 7721 priv->isolated = !!enable; 7722 if (enable) 7723 dev->dev_ops = &mlx5_dev_ops_isolate; 7724 else 7725 dev->dev_ops = &mlx5_dev_ops; 7726 7727 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7728 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7729 7730 return 0; 7731 } 7732 7733 /** 7734 * Query a flow. 7735 * 7736 * @see rte_flow_query() 7737 * @see rte_flow_ops 7738 */ 7739 static int 7740 flow_drv_query(struct rte_eth_dev *dev, 7741 uint32_t flow_idx, 7742 const struct rte_flow_action *actions, 7743 void *data, 7744 struct rte_flow_error *error) 7745 { 7746 struct mlx5_priv *priv = dev->data->dev_private; 7747 const struct mlx5_flow_driver_ops *fops; 7748 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7749 flow_idx); 7750 enum mlx5_flow_drv_type ftype; 7751 7752 if (!flow) { 7753 return rte_flow_error_set(error, ENOENT, 7754 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7755 NULL, 7756 "invalid flow handle"); 7757 } 7758 ftype = flow->drv_type; 7759 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7760 fops = flow_get_drv_ops(ftype); 7761 7762 return fops->query(dev, flow, actions, data, error); 7763 } 7764 7765 /** 7766 * Query a flow. 7767 * 7768 * @see rte_flow_query() 7769 * @see rte_flow_ops 7770 */ 7771 int 7772 mlx5_flow_query(struct rte_eth_dev *dev, 7773 struct rte_flow *flow, 7774 const struct rte_flow_action *actions, 7775 void *data, 7776 struct rte_flow_error *error) 7777 { 7778 int ret; 7779 struct mlx5_priv *priv = dev->data->dev_private; 7780 7781 if (priv->sh->config.dv_flow_en == 2) 7782 return rte_flow_error_set(error, ENOTSUP, 7783 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7784 NULL, 7785 "Flow non-Q query not supported"); 7786 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7787 error); 7788 if (ret < 0) 7789 return ret; 7790 return 0; 7791 } 7792 7793 /** 7794 * Get rte_flow callbacks. 7795 * 7796 * @param dev 7797 * Pointer to Ethernet device structure. 7798 * @param ops 7799 * Pointer to operation-specific structure. 7800 * 7801 * @return 0 7802 */ 7803 int 7804 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7805 const struct rte_flow_ops **ops) 7806 { 7807 *ops = &mlx5_flow_ops; 7808 return 0; 7809 } 7810 7811 /** 7812 * Validate meter policy actions. 7813 * Dispatcher for action type specific validation. 7814 * 7815 * @param[in] dev 7816 * Pointer to the Ethernet device structure. 7817 * @param[in] action 7818 * The meter policy action object to validate. 7819 * @param[in] attr 7820 * Attributes of flow to determine steering domain. 7821 * @param[out] is_rss 7822 * Is RSS or not. 7823 * @param[out] domain_bitmap 7824 * Domain bitmap. 7825 * @param[out] is_def_policy 7826 * Is default policy or not. 7827 * @param[out] error 7828 * Perform verbose error reporting if not NULL. Initialized in case of 7829 * error only. 7830 * 7831 * @return 7832 * 0 on success, otherwise negative errno value. 7833 */ 7834 int 7835 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7836 const struct rte_flow_action *actions[RTE_COLORS], 7837 struct rte_flow_attr *attr, 7838 bool *is_rss, 7839 uint8_t *domain_bitmap, 7840 uint8_t *policy_mode, 7841 struct rte_mtr_error *error) 7842 { 7843 const struct mlx5_flow_driver_ops *fops; 7844 7845 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7846 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7847 domain_bitmap, policy_mode, error); 7848 } 7849 7850 /** 7851 * Destroy the meter table set. 7852 * 7853 * @param[in] dev 7854 * Pointer to Ethernet device. 7855 * @param[in] mtr_policy 7856 * Meter policy struct. 7857 */ 7858 void 7859 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7860 struct mlx5_flow_meter_policy *mtr_policy) 7861 { 7862 const struct mlx5_flow_driver_ops *fops; 7863 7864 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7865 fops->destroy_mtr_acts(dev, mtr_policy); 7866 } 7867 7868 /** 7869 * Create policy action, lock free, 7870 * (mutex should be acquired by caller). 7871 * Dispatcher for action type specific call. 7872 * 7873 * @param[in] dev 7874 * Pointer to the Ethernet device structure. 7875 * @param[in] mtr_policy 7876 * Meter policy struct. 7877 * @param[in] action 7878 * Action specification used to create meter actions. 7879 * @param[out] error 7880 * Perform verbose error reporting if not NULL. Initialized in case of 7881 * error only. 7882 * 7883 * @return 7884 * 0 on success, otherwise negative errno value. 7885 */ 7886 int 7887 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7888 struct mlx5_flow_meter_policy *mtr_policy, 7889 const struct rte_flow_action *actions[RTE_COLORS], 7890 struct rte_mtr_error *error) 7891 { 7892 const struct mlx5_flow_driver_ops *fops; 7893 7894 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7895 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7896 } 7897 7898 /** 7899 * Create policy rules, lock free, 7900 * (mutex should be acquired by caller). 7901 * Dispatcher for action type specific call. 7902 * 7903 * @param[in] dev 7904 * Pointer to the Ethernet device structure. 7905 * @param[in] mtr_policy 7906 * Meter policy struct. 7907 * 7908 * @return 7909 * 0 on success, -1 otherwise. 7910 */ 7911 int 7912 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7913 struct mlx5_flow_meter_policy *mtr_policy) 7914 { 7915 const struct mlx5_flow_driver_ops *fops; 7916 7917 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7918 return fops->create_policy_rules(dev, mtr_policy); 7919 } 7920 7921 /** 7922 * Destroy policy rules, lock free, 7923 * (mutex should be acquired by caller). 7924 * Dispatcher for action type specific call. 7925 * 7926 * @param[in] dev 7927 * Pointer to the Ethernet device structure. 7928 * @param[in] mtr_policy 7929 * Meter policy struct. 7930 */ 7931 void 7932 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7933 struct mlx5_flow_meter_policy *mtr_policy) 7934 { 7935 const struct mlx5_flow_driver_ops *fops; 7936 7937 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7938 fops->destroy_policy_rules(dev, mtr_policy); 7939 } 7940 7941 /** 7942 * Destroy the default policy table set. 7943 * 7944 * @param[in] dev 7945 * Pointer to Ethernet device. 7946 */ 7947 void 7948 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7949 { 7950 const struct mlx5_flow_driver_ops *fops; 7951 7952 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7953 fops->destroy_def_policy(dev); 7954 } 7955 7956 /** 7957 * Destroy the default policy table set. 7958 * 7959 * @param[in] dev 7960 * Pointer to Ethernet device. 7961 * 7962 * @return 7963 * 0 on success, -1 otherwise. 7964 */ 7965 int 7966 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7967 { 7968 const struct mlx5_flow_driver_ops *fops; 7969 7970 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7971 return fops->create_def_policy(dev); 7972 } 7973 7974 /** 7975 * Create the needed meter and suffix tables. 7976 * 7977 * @param[in] dev 7978 * Pointer to Ethernet device. 7979 * 7980 * @return 7981 * 0 on success, -1 otherwise. 7982 */ 7983 int 7984 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7985 struct mlx5_flow_meter_info *fm, 7986 uint32_t mtr_idx, 7987 uint8_t domain_bitmap) 7988 { 7989 const struct mlx5_flow_driver_ops *fops; 7990 7991 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7992 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7993 } 7994 7995 /** 7996 * Destroy the meter table set. 7997 * 7998 * @param[in] dev 7999 * Pointer to Ethernet device. 8000 * @param[in] tbl 8001 * Pointer to the meter table set. 8002 */ 8003 void 8004 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8005 struct mlx5_flow_meter_info *fm) 8006 { 8007 const struct mlx5_flow_driver_ops *fops; 8008 8009 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8010 fops->destroy_mtr_tbls(dev, fm); 8011 } 8012 8013 /** 8014 * Destroy the global meter drop table. 8015 * 8016 * @param[in] dev 8017 * Pointer to Ethernet device. 8018 */ 8019 void 8020 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8021 { 8022 const struct mlx5_flow_driver_ops *fops; 8023 8024 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8025 fops->destroy_mtr_drop_tbls(dev); 8026 } 8027 8028 /** 8029 * Destroy the sub policy table with RX queue. 8030 * 8031 * @param[in] dev 8032 * Pointer to Ethernet device. 8033 * @param[in] mtr_policy 8034 * Pointer to meter policy table. 8035 */ 8036 void 8037 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8038 struct mlx5_flow_meter_policy *mtr_policy) 8039 { 8040 const struct mlx5_flow_driver_ops *fops; 8041 8042 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8043 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8044 } 8045 8046 /** 8047 * Allocate the needed aso flow meter id. 8048 * 8049 * @param[in] dev 8050 * Pointer to Ethernet device. 8051 * 8052 * @return 8053 * Index to aso flow meter on success, NULL otherwise. 8054 */ 8055 uint32_t 8056 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8057 { 8058 const struct mlx5_flow_driver_ops *fops; 8059 8060 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8061 return fops->create_meter(dev); 8062 } 8063 8064 /** 8065 * Free the aso flow meter id. 8066 * 8067 * @param[in] dev 8068 * Pointer to Ethernet device. 8069 * @param[in] mtr_idx 8070 * Index to aso flow meter to be free. 8071 * 8072 * @return 8073 * 0 on success. 8074 */ 8075 void 8076 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8077 { 8078 const struct mlx5_flow_driver_ops *fops; 8079 8080 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8081 fops->free_meter(dev, mtr_idx); 8082 } 8083 8084 /** 8085 * Allocate a counter. 8086 * 8087 * @param[in] dev 8088 * Pointer to Ethernet device structure. 8089 * 8090 * @return 8091 * Index to allocated counter on success, 0 otherwise. 8092 */ 8093 uint32_t 8094 mlx5_counter_alloc(struct rte_eth_dev *dev) 8095 { 8096 const struct mlx5_flow_driver_ops *fops; 8097 struct rte_flow_attr attr = { .transfer = 0 }; 8098 8099 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8100 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8101 return fops->counter_alloc(dev); 8102 } 8103 DRV_LOG(ERR, 8104 "port %u counter allocate is not supported.", 8105 dev->data->port_id); 8106 return 0; 8107 } 8108 8109 /** 8110 * Free a counter. 8111 * 8112 * @param[in] dev 8113 * Pointer to Ethernet device structure. 8114 * @param[in] cnt 8115 * Index to counter to be free. 8116 */ 8117 void 8118 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8119 { 8120 const struct mlx5_flow_driver_ops *fops; 8121 struct rte_flow_attr attr = { .transfer = 0 }; 8122 8123 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8124 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8125 fops->counter_free(dev, cnt); 8126 return; 8127 } 8128 DRV_LOG(ERR, 8129 "port %u counter free is not supported.", 8130 dev->data->port_id); 8131 } 8132 8133 /** 8134 * Query counter statistics. 8135 * 8136 * @param[in] dev 8137 * Pointer to Ethernet device structure. 8138 * @param[in] cnt 8139 * Index to counter to query. 8140 * @param[in] clear 8141 * Set to clear counter statistics. 8142 * @param[out] pkts 8143 * The counter hits packets number to save. 8144 * @param[out] bytes 8145 * The counter hits bytes number to save. 8146 * 8147 * @return 8148 * 0 on success, a negative errno value otherwise. 8149 */ 8150 int 8151 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8152 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8153 { 8154 const struct mlx5_flow_driver_ops *fops; 8155 struct rte_flow_attr attr = { .transfer = 0 }; 8156 8157 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8158 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8159 return fops->counter_query(dev, cnt, clear, pkts, 8160 bytes, action); 8161 } 8162 DRV_LOG(ERR, 8163 "port %u counter query is not supported.", 8164 dev->data->port_id); 8165 return -ENOTSUP; 8166 } 8167 8168 /** 8169 * Get information about HWS pre-configurable resources. 8170 * 8171 * @param[in] dev 8172 * Pointer to the rte_eth_dev structure. 8173 * @param[out] port_info 8174 * Pointer to port information. 8175 * @param[out] queue_info 8176 * Pointer to queue information. 8177 * @param[out] error 8178 * Pointer to error structure. 8179 * 8180 * @return 8181 * 0 on success, a negative errno value otherwise and rte_errno is set. 8182 */ 8183 static int 8184 mlx5_flow_info_get(struct rte_eth_dev *dev, 8185 struct rte_flow_port_info *port_info, 8186 struct rte_flow_queue_info *queue_info, 8187 struct rte_flow_error *error) 8188 { 8189 const struct mlx5_flow_driver_ops *fops; 8190 8191 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8192 return rte_flow_error_set(error, ENOTSUP, 8193 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8194 NULL, 8195 "info get with incorrect steering mode"); 8196 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8197 return fops->info_get(dev, port_info, queue_info, error); 8198 } 8199 8200 /** 8201 * Configure port HWS resources. 8202 * 8203 * @param[in] dev 8204 * Pointer to the rte_eth_dev structure. 8205 * @param[in] port_attr 8206 * Port configuration attributes. 8207 * @param[in] nb_queue 8208 * Number of queue. 8209 * @param[in] queue_attr 8210 * Array that holds attributes for each flow queue. 8211 * @param[out] error 8212 * Pointer to error structure. 8213 * 8214 * @return 8215 * 0 on success, a negative errno value otherwise and rte_errno is set. 8216 */ 8217 static int 8218 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8219 const struct rte_flow_port_attr *port_attr, 8220 uint16_t nb_queue, 8221 const struct rte_flow_queue_attr *queue_attr[], 8222 struct rte_flow_error *error) 8223 { 8224 const struct mlx5_flow_driver_ops *fops; 8225 8226 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8227 return rte_flow_error_set(error, ENOTSUP, 8228 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8229 NULL, 8230 "port configure with incorrect steering mode"); 8231 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8232 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8233 } 8234 8235 /** 8236 * Create flow item template. 8237 * 8238 * @param[in] dev 8239 * Pointer to the rte_eth_dev structure. 8240 * @param[in] attr 8241 * Pointer to the item template attributes. 8242 * @param[in] items 8243 * The template item pattern. 8244 * @param[out] error 8245 * Pointer to error structure. 8246 * 8247 * @return 8248 * 0 on success, a negative errno value otherwise and rte_errno is set. 8249 */ 8250 static struct rte_flow_pattern_template * 8251 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8252 const struct rte_flow_pattern_template_attr *attr, 8253 const struct rte_flow_item items[], 8254 struct rte_flow_error *error) 8255 { 8256 const struct mlx5_flow_driver_ops *fops; 8257 8258 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8259 rte_flow_error_set(error, ENOTSUP, 8260 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8261 NULL, 8262 "pattern create with incorrect steering mode"); 8263 return NULL; 8264 } 8265 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8266 return fops->pattern_template_create(dev, attr, items, error); 8267 } 8268 8269 /** 8270 * Destroy flow item template. 8271 * 8272 * @param[in] dev 8273 * Pointer to the rte_eth_dev structure. 8274 * @param[in] template 8275 * Pointer to the item template to be destroyed. 8276 * @param[out] error 8277 * Pointer to error structure. 8278 * 8279 * @return 8280 * 0 on success, a negative errno value otherwise and rte_errno is set. 8281 */ 8282 static int 8283 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8284 struct rte_flow_pattern_template *template, 8285 struct rte_flow_error *error) 8286 { 8287 const struct mlx5_flow_driver_ops *fops; 8288 8289 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8290 return rte_flow_error_set(error, ENOTSUP, 8291 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8292 NULL, 8293 "pattern destroy with incorrect steering mode"); 8294 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8295 return fops->pattern_template_destroy(dev, template, error); 8296 } 8297 8298 /** 8299 * Create flow item template. 8300 * 8301 * @param[in] dev 8302 * Pointer to the rte_eth_dev structure. 8303 * @param[in] attr 8304 * Pointer to the action template attributes. 8305 * @param[in] actions 8306 * Associated actions (list terminated by the END action). 8307 * @param[in] masks 8308 * List of actions that marks which of the action's member is constant. 8309 * @param[out] error 8310 * Pointer to error structure. 8311 * 8312 * @return 8313 * 0 on success, a negative errno value otherwise and rte_errno is set. 8314 */ 8315 static struct rte_flow_actions_template * 8316 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8317 const struct rte_flow_actions_template_attr *attr, 8318 const struct rte_flow_action actions[], 8319 const struct rte_flow_action masks[], 8320 struct rte_flow_error *error) 8321 { 8322 const struct mlx5_flow_driver_ops *fops; 8323 8324 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8325 rte_flow_error_set(error, ENOTSUP, 8326 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8327 NULL, 8328 "action create with incorrect steering mode"); 8329 return NULL; 8330 } 8331 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8332 return fops->actions_template_create(dev, attr, actions, masks, error); 8333 } 8334 8335 /** 8336 * Destroy flow action template. 8337 * 8338 * @param[in] dev 8339 * Pointer to the rte_eth_dev structure. 8340 * @param[in] template 8341 * Pointer to the action template to be destroyed. 8342 * @param[out] error 8343 * Pointer to error structure. 8344 * 8345 * @return 8346 * 0 on success, a negative errno value otherwise and rte_errno is set. 8347 */ 8348 static int 8349 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8350 struct rte_flow_actions_template *template, 8351 struct rte_flow_error *error) 8352 { 8353 const struct mlx5_flow_driver_ops *fops; 8354 8355 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8356 return rte_flow_error_set(error, ENOTSUP, 8357 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8358 NULL, 8359 "action destroy with incorrect steering mode"); 8360 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8361 return fops->actions_template_destroy(dev, template, error); 8362 } 8363 8364 /** 8365 * Create flow table. 8366 * 8367 * @param[in] dev 8368 * Pointer to the rte_eth_dev structure. 8369 * @param[in] attr 8370 * Pointer to the table attributes. 8371 * @param[in] item_templates 8372 * Item template array to be binded to the table. 8373 * @param[in] nb_item_templates 8374 * Number of item template. 8375 * @param[in] action_templates 8376 * Action template array to be binded to the table. 8377 * @param[in] nb_action_templates 8378 * Number of action template. 8379 * @param[out] error 8380 * Pointer to error structure. 8381 * 8382 * @return 8383 * Table on success, NULL otherwise and rte_errno is set. 8384 */ 8385 static struct rte_flow_template_table * 8386 mlx5_flow_table_create(struct rte_eth_dev *dev, 8387 const struct rte_flow_template_table_attr *attr, 8388 struct rte_flow_pattern_template *item_templates[], 8389 uint8_t nb_item_templates, 8390 struct rte_flow_actions_template *action_templates[], 8391 uint8_t nb_action_templates, 8392 struct rte_flow_error *error) 8393 { 8394 const struct mlx5_flow_driver_ops *fops; 8395 8396 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8397 rte_flow_error_set(error, ENOTSUP, 8398 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8399 NULL, 8400 "table create with incorrect steering mode"); 8401 return NULL; 8402 } 8403 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8404 return fops->template_table_create(dev, 8405 attr, 8406 item_templates, 8407 nb_item_templates, 8408 action_templates, 8409 nb_action_templates, 8410 error); 8411 } 8412 8413 /** 8414 * PMD destroy flow table. 8415 * 8416 * @param[in] dev 8417 * Pointer to the rte_eth_dev structure. 8418 * @param[in] table 8419 * Pointer to the table to be destroyed. 8420 * @param[out] error 8421 * Pointer to error structure. 8422 * 8423 * @return 8424 * 0 on success, a negative errno value otherwise and rte_errno is set. 8425 */ 8426 static int 8427 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8428 struct rte_flow_template_table *table, 8429 struct rte_flow_error *error) 8430 { 8431 const struct mlx5_flow_driver_ops *fops; 8432 8433 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8434 return rte_flow_error_set(error, ENOTSUP, 8435 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8436 NULL, 8437 "table destroy with incorrect steering mode"); 8438 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8439 return fops->template_table_destroy(dev, table, error); 8440 } 8441 8442 /** 8443 * Enqueue flow creation. 8444 * 8445 * @param[in] dev 8446 * Pointer to the rte_eth_dev structure. 8447 * @param[in] queue_id 8448 * The queue to create the flow. 8449 * @param[in] attr 8450 * Pointer to the flow operation attributes. 8451 * @param[in] items 8452 * Items with flow spec value. 8453 * @param[in] pattern_template_index 8454 * The item pattern flow follows from the table. 8455 * @param[in] actions 8456 * Action with flow spec value. 8457 * @param[in] action_template_index 8458 * The action pattern flow follows from the table. 8459 * @param[in] user_data 8460 * Pointer to the user_data. 8461 * @param[out] error 8462 * Pointer to error structure. 8463 * 8464 * @return 8465 * Flow pointer on success, NULL otherwise and rte_errno is set. 8466 */ 8467 static struct rte_flow * 8468 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8469 uint32_t queue_id, 8470 const struct rte_flow_op_attr *attr, 8471 struct rte_flow_template_table *table, 8472 const struct rte_flow_item items[], 8473 uint8_t pattern_template_index, 8474 const struct rte_flow_action actions[], 8475 uint8_t action_template_index, 8476 void *user_data, 8477 struct rte_flow_error *error) 8478 { 8479 const struct mlx5_flow_driver_ops *fops; 8480 8481 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8482 rte_flow_error_set(error, ENOTSUP, 8483 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8484 NULL, 8485 "flow_q create with incorrect steering mode"); 8486 return NULL; 8487 } 8488 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8489 return fops->async_flow_create(dev, queue_id, attr, table, 8490 items, pattern_template_index, 8491 actions, action_template_index, 8492 user_data, error); 8493 } 8494 8495 /** 8496 * Enqueue flow destruction. 8497 * 8498 * @param[in] dev 8499 * Pointer to the rte_eth_dev structure. 8500 * @param[in] queue 8501 * The queue to destroy the flow. 8502 * @param[in] attr 8503 * Pointer to the flow operation attributes. 8504 * @param[in] flow 8505 * Pointer to the flow to be destroyed. 8506 * @param[in] user_data 8507 * Pointer to the user_data. 8508 * @param[out] error 8509 * Pointer to error structure. 8510 * 8511 * @return 8512 * 0 on success, negative value otherwise and rte_errno is set. 8513 */ 8514 static int 8515 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8516 uint32_t queue, 8517 const struct rte_flow_op_attr *attr, 8518 struct rte_flow *flow, 8519 void *user_data, 8520 struct rte_flow_error *error) 8521 { 8522 const struct mlx5_flow_driver_ops *fops; 8523 8524 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8525 return rte_flow_error_set(error, ENOTSUP, 8526 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8527 NULL, 8528 "flow_q destroy with incorrect steering mode"); 8529 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8530 return fops->async_flow_destroy(dev, queue, attr, flow, 8531 user_data, error); 8532 } 8533 8534 /** 8535 * Pull the enqueued flows. 8536 * 8537 * @param[in] dev 8538 * Pointer to the rte_eth_dev structure. 8539 * @param[in] queue 8540 * The queue to pull the result. 8541 * @param[in/out] res 8542 * Array to save the results. 8543 * @param[in] n_res 8544 * Available result with the array. 8545 * @param[out] error 8546 * Pointer to error structure. 8547 * 8548 * @return 8549 * Result number on success, negative value otherwise and rte_errno is set. 8550 */ 8551 static int 8552 mlx5_flow_pull(struct rte_eth_dev *dev, 8553 uint32_t queue, 8554 struct rte_flow_op_result res[], 8555 uint16_t n_res, 8556 struct rte_flow_error *error) 8557 { 8558 const struct mlx5_flow_driver_ops *fops; 8559 8560 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8561 return rte_flow_error_set(error, ENOTSUP, 8562 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8563 NULL, 8564 "flow_q pull with incorrect steering mode"); 8565 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8566 return fops->pull(dev, queue, res, n_res, error); 8567 } 8568 8569 /** 8570 * Push the enqueued flows. 8571 * 8572 * @param[in] dev 8573 * Pointer to the rte_eth_dev structure. 8574 * @param[in] queue 8575 * The queue to push the flows. 8576 * @param[out] error 8577 * Pointer to error structure. 8578 * 8579 * @return 8580 * 0 on success, negative value otherwise and rte_errno is set. 8581 */ 8582 static int 8583 mlx5_flow_push(struct rte_eth_dev *dev, 8584 uint32_t queue, 8585 struct rte_flow_error *error) 8586 { 8587 const struct mlx5_flow_driver_ops *fops; 8588 8589 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8590 return rte_flow_error_set(error, ENOTSUP, 8591 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8592 NULL, 8593 "flow_q push with incorrect steering mode"); 8594 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8595 return fops->push(dev, queue, error); 8596 } 8597 8598 /** 8599 * Create shared action. 8600 * 8601 * @param[in] dev 8602 * Pointer to the rte_eth_dev structure. 8603 * @param[in] queue 8604 * Which queue to be used.. 8605 * @param[in] attr 8606 * Operation attribute. 8607 * @param[in] conf 8608 * Indirect action configuration. 8609 * @param[in] action 8610 * rte_flow action detail. 8611 * @param[in] user_data 8612 * Pointer to the user_data. 8613 * @param[out] error 8614 * Pointer to error structure. 8615 * 8616 * @return 8617 * Action handle on success, NULL otherwise and rte_errno is set. 8618 */ 8619 static struct rte_flow_action_handle * 8620 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8621 const struct rte_flow_op_attr *attr, 8622 const struct rte_flow_indir_action_conf *conf, 8623 const struct rte_flow_action *action, 8624 void *user_data, 8625 struct rte_flow_error *error) 8626 { 8627 const struct mlx5_flow_driver_ops *fops = 8628 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8629 8630 return fops->async_action_create(dev, queue, attr, conf, action, 8631 user_data, error); 8632 } 8633 8634 /** 8635 * Update shared action. 8636 * 8637 * @param[in] dev 8638 * Pointer to the rte_eth_dev structure. 8639 * @param[in] queue 8640 * Which queue to be used.. 8641 * @param[in] attr 8642 * Operation attribute. 8643 * @param[in] handle 8644 * Action handle to be updated. 8645 * @param[in] update 8646 * Update value. 8647 * @param[in] user_data 8648 * Pointer to the user_data. 8649 * @param[out] error 8650 * Pointer to error structure. 8651 * 8652 * @return 8653 * 0 on success, negative value otherwise and rte_errno is set. 8654 */ 8655 static int 8656 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 8657 const struct rte_flow_op_attr *attr, 8658 struct rte_flow_action_handle *handle, 8659 const void *update, 8660 void *user_data, 8661 struct rte_flow_error *error) 8662 { 8663 const struct mlx5_flow_driver_ops *fops = 8664 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8665 8666 return fops->async_action_update(dev, queue, attr, handle, 8667 update, user_data, error); 8668 } 8669 8670 /** 8671 * Destroy shared action. 8672 * 8673 * @param[in] dev 8674 * Pointer to the rte_eth_dev structure. 8675 * @param[in] queue 8676 * Which queue to be used.. 8677 * @param[in] attr 8678 * Operation attribute. 8679 * @param[in] handle 8680 * Action handle to be destroyed. 8681 * @param[in] user_data 8682 * Pointer to the user_data. 8683 * @param[out] error 8684 * Pointer to error structure. 8685 * 8686 * @return 8687 * 0 on success, negative value otherwise and rte_errno is set. 8688 */ 8689 static int 8690 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 8691 const struct rte_flow_op_attr *attr, 8692 struct rte_flow_action_handle *handle, 8693 void *user_data, 8694 struct rte_flow_error *error) 8695 { 8696 const struct mlx5_flow_driver_ops *fops = 8697 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8698 8699 return fops->async_action_destroy(dev, queue, attr, handle, 8700 user_data, error); 8701 } 8702 8703 /** 8704 * Allocate a new memory for the counter values wrapped by all the needed 8705 * management. 8706 * 8707 * @param[in] sh 8708 * Pointer to mlx5_dev_ctx_shared object. 8709 * 8710 * @return 8711 * 0 on success, a negative errno value otherwise. 8712 */ 8713 static int 8714 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 8715 { 8716 struct mlx5_counter_stats_mem_mng *mem_mng; 8717 volatile struct flow_counter_stats *raw_data; 8718 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 8719 int size = (sizeof(struct flow_counter_stats) * 8720 MLX5_COUNTERS_PER_POOL + 8721 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 8722 sizeof(struct mlx5_counter_stats_mem_mng); 8723 size_t pgsize = rte_mem_page_size(); 8724 uint8_t *mem; 8725 int ret; 8726 int i; 8727 8728 if (pgsize == (size_t)-1) { 8729 DRV_LOG(ERR, "Failed to get mem page size"); 8730 rte_errno = ENOMEM; 8731 return -ENOMEM; 8732 } 8733 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 8734 if (!mem) { 8735 rte_errno = ENOMEM; 8736 return -ENOMEM; 8737 } 8738 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 8739 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 8740 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 8741 sh->cdev->pdn, mem, size, 8742 &mem_mng->wm); 8743 if (ret) { 8744 rte_errno = errno; 8745 mlx5_free(mem); 8746 return -rte_errno; 8747 } 8748 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 8749 raw_data = (volatile struct flow_counter_stats *)mem; 8750 for (i = 0; i < raws_n; ++i) { 8751 mem_mng->raws[i].mem_mng = mem_mng; 8752 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 8753 } 8754 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 8755 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 8756 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 8757 next); 8758 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 8759 sh->cmng.mem_mng = mem_mng; 8760 return 0; 8761 } 8762 8763 /** 8764 * Set the statistic memory to the new counter pool. 8765 * 8766 * @param[in] sh 8767 * Pointer to mlx5_dev_ctx_shared object. 8768 * @param[in] pool 8769 * Pointer to the pool to set the statistic memory. 8770 * 8771 * @return 8772 * 0 on success, a negative errno value otherwise. 8773 */ 8774 static int 8775 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 8776 struct mlx5_flow_counter_pool *pool) 8777 { 8778 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8779 /* Resize statistic memory once used out. */ 8780 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 8781 mlx5_flow_create_counter_stat_mem_mng(sh)) { 8782 DRV_LOG(ERR, "Cannot resize counter stat mem."); 8783 return -1; 8784 } 8785 rte_spinlock_lock(&pool->sl); 8786 pool->raw = cmng->mem_mng->raws + pool->index % 8787 MLX5_CNT_CONTAINER_RESIZE; 8788 rte_spinlock_unlock(&pool->sl); 8789 pool->raw_hw = NULL; 8790 return 0; 8791 } 8792 8793 #define MLX5_POOL_QUERY_FREQ_US 1000000 8794 8795 /** 8796 * Set the periodic procedure for triggering asynchronous batch queries for all 8797 * the counter pools. 8798 * 8799 * @param[in] sh 8800 * Pointer to mlx5_dev_ctx_shared object. 8801 */ 8802 void 8803 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 8804 { 8805 uint32_t pools_n, us; 8806 8807 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 8808 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 8809 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 8810 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 8811 sh->cmng.query_thread_on = 0; 8812 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 8813 } else { 8814 sh->cmng.query_thread_on = 1; 8815 } 8816 } 8817 8818 /** 8819 * The periodic procedure for triggering asynchronous batch queries for all the 8820 * counter pools. This function is probably called by the host thread. 8821 * 8822 * @param[in] arg 8823 * The parameter for the alarm process. 8824 */ 8825 void 8826 mlx5_flow_query_alarm(void *arg) 8827 { 8828 struct mlx5_dev_ctx_shared *sh = arg; 8829 int ret; 8830 uint16_t pool_index = sh->cmng.pool_index; 8831 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8832 struct mlx5_flow_counter_pool *pool; 8833 uint16_t n_valid; 8834 8835 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 8836 goto set_alarm; 8837 rte_spinlock_lock(&cmng->pool_update_sl); 8838 pool = cmng->pools[pool_index]; 8839 n_valid = cmng->n_valid; 8840 rte_spinlock_unlock(&cmng->pool_update_sl); 8841 /* Set the statistic memory to the new created pool. */ 8842 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 8843 goto set_alarm; 8844 if (pool->raw_hw) 8845 /* There is a pool query in progress. */ 8846 goto set_alarm; 8847 pool->raw_hw = 8848 LIST_FIRST(&sh->cmng.free_stat_raws); 8849 if (!pool->raw_hw) 8850 /* No free counter statistics raw memory. */ 8851 goto set_alarm; 8852 /* 8853 * Identify the counters released between query trigger and query 8854 * handle more efficiently. The counter released in this gap period 8855 * should wait for a new round of query as the new arrived packets 8856 * will not be taken into account. 8857 */ 8858 pool->query_gen++; 8859 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 8860 MLX5_COUNTERS_PER_POOL, 8861 NULL, NULL, 8862 pool->raw_hw->mem_mng->wm.lkey, 8863 (void *)(uintptr_t) 8864 pool->raw_hw->data, 8865 sh->devx_comp, 8866 (uint64_t)(uintptr_t)pool); 8867 if (ret) { 8868 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 8869 " %d", pool->min_dcs->id); 8870 pool->raw_hw = NULL; 8871 goto set_alarm; 8872 } 8873 LIST_REMOVE(pool->raw_hw, next); 8874 sh->cmng.pending_queries++; 8875 pool_index++; 8876 if (pool_index >= n_valid) 8877 pool_index = 0; 8878 set_alarm: 8879 sh->cmng.pool_index = pool_index; 8880 mlx5_set_query_alarm(sh); 8881 } 8882 8883 /** 8884 * Check and callback event for new aged flow in the counter pool 8885 * 8886 * @param[in] sh 8887 * Pointer to mlx5_dev_ctx_shared object. 8888 * @param[in] pool 8889 * Pointer to Current counter pool. 8890 */ 8891 static void 8892 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8893 struct mlx5_flow_counter_pool *pool) 8894 { 8895 struct mlx5_priv *priv; 8896 struct mlx5_flow_counter *cnt; 8897 struct mlx5_age_info *age_info; 8898 struct mlx5_age_param *age_param; 8899 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8900 struct mlx5_counter_stats_raw *prev = pool->raw; 8901 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8902 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8903 uint16_t expected = AGE_CANDIDATE; 8904 uint32_t i; 8905 8906 pool->time_of_last_age_check = curr_time; 8907 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8908 cnt = MLX5_POOL_GET_CNT(pool, i); 8909 age_param = MLX5_CNT_TO_AGE(cnt); 8910 if (__atomic_load_n(&age_param->state, 8911 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8912 continue; 8913 if (cur->data[i].hits != prev->data[i].hits) { 8914 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8915 __ATOMIC_RELAXED); 8916 continue; 8917 } 8918 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8919 time_delta, 8920 __ATOMIC_RELAXED) <= age_param->timeout) 8921 continue; 8922 /** 8923 * Hold the lock first, or if between the 8924 * state AGE_TMOUT and tailq operation the 8925 * release happened, the release procedure 8926 * may delete a non-existent tailq node. 8927 */ 8928 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8929 age_info = GET_PORT_AGE_INFO(priv); 8930 rte_spinlock_lock(&age_info->aged_sl); 8931 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8932 AGE_TMOUT, false, 8933 __ATOMIC_RELAXED, 8934 __ATOMIC_RELAXED)) { 8935 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8936 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8937 } 8938 rte_spinlock_unlock(&age_info->aged_sl); 8939 } 8940 mlx5_age_event_prepare(sh); 8941 } 8942 8943 /** 8944 * Handler for the HW respond about ready values from an asynchronous batch 8945 * query. This function is probably called by the host thread. 8946 * 8947 * @param[in] sh 8948 * The pointer to the shared device context. 8949 * @param[in] async_id 8950 * The Devx async ID. 8951 * @param[in] status 8952 * The status of the completion. 8953 */ 8954 void 8955 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8956 uint64_t async_id, int status) 8957 { 8958 struct mlx5_flow_counter_pool *pool = 8959 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8960 struct mlx5_counter_stats_raw *raw_to_free; 8961 uint8_t query_gen = pool->query_gen ^ 1; 8962 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8963 enum mlx5_counter_type cnt_type = 8964 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8965 MLX5_COUNTER_TYPE_ORIGIN; 8966 8967 if (unlikely(status)) { 8968 raw_to_free = pool->raw_hw; 8969 } else { 8970 raw_to_free = pool->raw; 8971 if (pool->is_aged) 8972 mlx5_flow_aging_check(sh, pool); 8973 rte_spinlock_lock(&pool->sl); 8974 pool->raw = pool->raw_hw; 8975 rte_spinlock_unlock(&pool->sl); 8976 /* Be sure the new raw counters data is updated in memory. */ 8977 rte_io_wmb(); 8978 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8979 rte_spinlock_lock(&cmng->csl[cnt_type]); 8980 TAILQ_CONCAT(&cmng->counters[cnt_type], 8981 &pool->counters[query_gen], next); 8982 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8983 } 8984 } 8985 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8986 pool->raw_hw = NULL; 8987 sh->cmng.pending_queries--; 8988 } 8989 8990 static int 8991 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8992 const struct flow_grp_info *grp_info, 8993 struct rte_flow_error *error) 8994 { 8995 if (grp_info->transfer && grp_info->external && 8996 grp_info->fdb_def_rule) { 8997 if (group == UINT32_MAX) 8998 return rte_flow_error_set 8999 (error, EINVAL, 9000 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9001 NULL, 9002 "group index not supported"); 9003 *table = group + 1; 9004 } else { 9005 *table = group; 9006 } 9007 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9008 return 0; 9009 } 9010 9011 /** 9012 * Translate the rte_flow group index to HW table value. 9013 * 9014 * If tunnel offload is disabled, all group ids converted to flow table 9015 * id using the standard method. 9016 * If tunnel offload is enabled, group id can be converted using the 9017 * standard or tunnel conversion method. Group conversion method 9018 * selection depends on flags in `grp_info` parameter: 9019 * - Internal (grp_info.external == 0) groups conversion uses the 9020 * standard method. 9021 * - Group ids in JUMP action converted with the tunnel conversion. 9022 * - Group id in rule attribute conversion depends on a rule type and 9023 * group id value: 9024 * ** non zero group attributes converted with the tunnel method 9025 * ** zero group attribute in non-tunnel rule is converted using the 9026 * standard method - there's only one root table 9027 * ** zero group attribute in steer tunnel rule is converted with the 9028 * standard method - single root table 9029 * ** zero group attribute in match tunnel rule is a special OvS 9030 * case: that value is used for portability reasons. That group 9031 * id is converted with the tunnel conversion method. 9032 * 9033 * @param[in] dev 9034 * Port device 9035 * @param[in] tunnel 9036 * PMD tunnel offload object 9037 * @param[in] group 9038 * rte_flow group index value. 9039 * @param[out] table 9040 * HW table value. 9041 * @param[in] grp_info 9042 * flags used for conversion 9043 * @param[out] error 9044 * Pointer to error structure. 9045 * 9046 * @return 9047 * 0 on success, a negative errno value otherwise and rte_errno is set. 9048 */ 9049 int 9050 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9051 const struct mlx5_flow_tunnel *tunnel, 9052 uint32_t group, uint32_t *table, 9053 const struct flow_grp_info *grp_info, 9054 struct rte_flow_error *error) 9055 { 9056 int ret; 9057 bool standard_translation; 9058 9059 if (!grp_info->skip_scale && grp_info->external && 9060 group < MLX5_MAX_TABLES_EXTERNAL) 9061 group *= MLX5_FLOW_TABLE_FACTOR; 9062 if (is_tunnel_offload_active(dev)) { 9063 standard_translation = !grp_info->external || 9064 grp_info->std_tbl_fix; 9065 } else { 9066 standard_translation = true; 9067 } 9068 DRV_LOG(DEBUG, 9069 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9070 dev->data->port_id, group, grp_info->transfer, 9071 grp_info->external, grp_info->fdb_def_rule, 9072 standard_translation ? "STANDARD" : "TUNNEL"); 9073 if (standard_translation) 9074 ret = flow_group_to_table(dev->data->port_id, group, table, 9075 grp_info, error); 9076 else 9077 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9078 table, error); 9079 9080 return ret; 9081 } 9082 9083 /** 9084 * Discover availability of metadata reg_c's. 9085 * 9086 * Iteratively use test flows to check availability. 9087 * 9088 * @param[in] dev 9089 * Pointer to the Ethernet device structure. 9090 * 9091 * @return 9092 * 0 on success, a negative errno value otherwise and rte_errno is set. 9093 */ 9094 int 9095 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9096 { 9097 struct mlx5_priv *priv = dev->data->dev_private; 9098 enum modify_reg idx; 9099 int n = 0; 9100 9101 /* reg_c[0] and reg_c[1] are reserved. */ 9102 priv->sh->flow_mreg_c[n++] = REG_C_0; 9103 priv->sh->flow_mreg_c[n++] = REG_C_1; 9104 /* Discover availability of other reg_c's. */ 9105 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9106 struct rte_flow_attr attr = { 9107 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9108 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9109 .ingress = 1, 9110 }; 9111 struct rte_flow_item items[] = { 9112 [0] = { 9113 .type = RTE_FLOW_ITEM_TYPE_END, 9114 }, 9115 }; 9116 struct rte_flow_action actions[] = { 9117 [0] = { 9118 .type = (enum rte_flow_action_type) 9119 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9120 .conf = &(struct mlx5_flow_action_copy_mreg){ 9121 .src = REG_C_1, 9122 .dst = idx, 9123 }, 9124 }, 9125 [1] = { 9126 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9127 .conf = &(struct rte_flow_action_jump){ 9128 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9129 }, 9130 }, 9131 [2] = { 9132 .type = RTE_FLOW_ACTION_TYPE_END, 9133 }, 9134 }; 9135 uint32_t flow_idx; 9136 struct rte_flow *flow; 9137 struct rte_flow_error error; 9138 9139 if (!priv->sh->config.dv_flow_en) 9140 break; 9141 /* Create internal flow, validation skips copy action. */ 9142 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9143 items, actions, false, &error); 9144 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9145 flow_idx); 9146 if (!flow) 9147 continue; 9148 priv->sh->flow_mreg_c[n++] = idx; 9149 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9150 } 9151 for (; n < MLX5_MREG_C_NUM; ++n) 9152 priv->sh->flow_mreg_c[n] = REG_NON; 9153 priv->sh->metadata_regc_check_flag = 1; 9154 return 0; 9155 } 9156 9157 int 9158 save_dump_file(const uint8_t *data, uint32_t size, 9159 uint32_t type, uint64_t id, void *arg, FILE *file) 9160 { 9161 char line[BUF_SIZE]; 9162 uint32_t out = 0; 9163 uint32_t k; 9164 uint32_t actions_num; 9165 struct rte_flow_query_count *count; 9166 9167 memset(line, 0, BUF_SIZE); 9168 switch (type) { 9169 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9170 actions_num = *(uint32_t *)(arg); 9171 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9172 type, id, actions_num); 9173 break; 9174 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9175 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9176 type, id); 9177 break; 9178 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9179 count = (struct rte_flow_query_count *)arg; 9180 fprintf(file, 9181 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9182 type, id, count->hits, count->bytes); 9183 return 0; 9184 default: 9185 return -1; 9186 } 9187 9188 for (k = 0; k < size; k++) { 9189 /* Make sure we do not overrun the line buffer length. */ 9190 if (out >= BUF_SIZE - 4) { 9191 line[out] = '\0'; 9192 break; 9193 } 9194 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9195 (data[k]) & 0xff); 9196 } 9197 fprintf(file, "%s\n", line); 9198 return 0; 9199 } 9200 9201 int 9202 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9203 struct rte_flow_query_count *count, struct rte_flow_error *error) 9204 { 9205 struct rte_flow_action action[2]; 9206 enum mlx5_flow_drv_type ftype; 9207 const struct mlx5_flow_driver_ops *fops; 9208 9209 if (!flow) { 9210 return rte_flow_error_set(error, ENOENT, 9211 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9212 NULL, 9213 "invalid flow handle"); 9214 } 9215 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9216 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9217 if (flow->counter) { 9218 memset(count, 0, sizeof(struct rte_flow_query_count)); 9219 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9220 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9221 ftype < MLX5_FLOW_TYPE_MAX); 9222 fops = flow_get_drv_ops(ftype); 9223 return fops->query(dev, flow, action, count, error); 9224 } 9225 return -1; 9226 } 9227 9228 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9229 /** 9230 * Dump flow ipool data to file 9231 * 9232 * @param[in] dev 9233 * The pointer to Ethernet device. 9234 * @param[in] file 9235 * A pointer to a file for output. 9236 * @param[out] error 9237 * Perform verbose error reporting if not NULL. PMDs initialize this 9238 * structure in case of error only. 9239 * @return 9240 * 0 on success, a negative value otherwise. 9241 */ 9242 int 9243 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9244 struct rte_flow *flow, FILE *file, 9245 struct rte_flow_error *error) 9246 { 9247 struct mlx5_priv *priv = dev->data->dev_private; 9248 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9249 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9250 uint32_t handle_idx; 9251 struct mlx5_flow_handle *dh; 9252 struct rte_flow_query_count count; 9253 uint32_t actions_num; 9254 const uint8_t *data; 9255 size_t size; 9256 uint64_t id; 9257 uint32_t type; 9258 void *action = NULL; 9259 9260 if (!flow) { 9261 return rte_flow_error_set(error, ENOENT, 9262 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9263 NULL, 9264 "invalid flow handle"); 9265 } 9266 handle_idx = flow->dev_handles; 9267 /* query counter */ 9268 if (flow->counter && 9269 (!mlx5_counter_query(dev, flow->counter, false, 9270 &count.hits, &count.bytes, &action)) && action) { 9271 id = (uint64_t)(uintptr_t)action; 9272 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9273 save_dump_file(NULL, 0, type, 9274 id, (void *)&count, file); 9275 } 9276 9277 while (handle_idx) { 9278 dh = mlx5_ipool_get(priv->sh->ipool 9279 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9280 if (!dh) 9281 continue; 9282 handle_idx = dh->next.next; 9283 9284 /* Get modify_hdr and encap_decap buf from ipools. */ 9285 encap_decap = NULL; 9286 modify_hdr = dh->dvh.modify_hdr; 9287 9288 if (dh->dvh.rix_encap_decap) { 9289 encap_decap = mlx5_ipool_get(priv->sh->ipool 9290 [MLX5_IPOOL_DECAP_ENCAP], 9291 dh->dvh.rix_encap_decap); 9292 } 9293 if (modify_hdr) { 9294 data = (const uint8_t *)modify_hdr->actions; 9295 size = (size_t)(modify_hdr->actions_num) * 8; 9296 id = (uint64_t)(uintptr_t)modify_hdr->action; 9297 actions_num = modify_hdr->actions_num; 9298 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9299 save_dump_file(data, size, type, id, 9300 (void *)(&actions_num), file); 9301 } 9302 if (encap_decap) { 9303 data = encap_decap->buf; 9304 size = encap_decap->size; 9305 id = (uint64_t)(uintptr_t)encap_decap->action; 9306 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9307 save_dump_file(data, size, type, 9308 id, NULL, file); 9309 } 9310 } 9311 return 0; 9312 } 9313 9314 /** 9315 * Dump all flow's encap_decap/modify_hdr/counter data to file 9316 * 9317 * @param[in] dev 9318 * The pointer to Ethernet device. 9319 * @param[in] file 9320 * A pointer to a file for output. 9321 * @param[out] error 9322 * Perform verbose error reporting if not NULL. PMDs initialize this 9323 * structure in case of error only. 9324 * @return 9325 * 0 on success, a negative value otherwise. 9326 */ 9327 static int 9328 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9329 FILE *file, struct rte_flow_error *error __rte_unused) 9330 { 9331 struct mlx5_priv *priv = dev->data->dev_private; 9332 struct mlx5_dev_ctx_shared *sh = priv->sh; 9333 struct mlx5_hlist *h; 9334 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9335 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9336 struct rte_flow_query_count count; 9337 uint32_t actions_num; 9338 const uint8_t *data; 9339 size_t size; 9340 uint64_t id; 9341 uint32_t type; 9342 uint32_t i; 9343 uint32_t j; 9344 struct mlx5_list_inconst *l_inconst; 9345 struct mlx5_list_entry *e; 9346 int lcore_index; 9347 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 9348 uint32_t max; 9349 void *action; 9350 9351 /* encap_decap hlist is lcore_share, get global core cache. */ 9352 i = MLX5_LIST_GLOBAL; 9353 h = sh->encaps_decaps; 9354 if (h) { 9355 for (j = 0; j <= h->mask; j++) { 9356 l_inconst = &h->buckets[j].l; 9357 if (!l_inconst || !l_inconst->cache[i]) 9358 continue; 9359 9360 e = LIST_FIRST(&l_inconst->cache[i]->h); 9361 while (e) { 9362 encap_decap = 9363 (struct mlx5_flow_dv_encap_decap_resource *)e; 9364 data = encap_decap->buf; 9365 size = encap_decap->size; 9366 id = (uint64_t)(uintptr_t)encap_decap->action; 9367 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9368 save_dump_file(data, size, type, 9369 id, NULL, file); 9370 e = LIST_NEXT(e, next); 9371 } 9372 } 9373 } 9374 9375 /* get modify_hdr */ 9376 h = sh->modify_cmds; 9377 if (h) { 9378 lcore_index = rte_lcore_index(rte_lcore_id()); 9379 if (unlikely(lcore_index == -1)) { 9380 lcore_index = MLX5_LIST_NLCORE; 9381 rte_spinlock_lock(&h->l_const.lcore_lock); 9382 } 9383 i = lcore_index; 9384 9385 for (j = 0; j <= h->mask; j++) { 9386 l_inconst = &h->buckets[j].l; 9387 if (!l_inconst || !l_inconst->cache[i]) 9388 continue; 9389 9390 e = LIST_FIRST(&l_inconst->cache[i]->h); 9391 while (e) { 9392 modify_hdr = 9393 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9394 data = (const uint8_t *)modify_hdr->actions; 9395 size = (size_t)(modify_hdr->actions_num) * 8; 9396 actions_num = modify_hdr->actions_num; 9397 id = (uint64_t)(uintptr_t)modify_hdr->action; 9398 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9399 save_dump_file(data, size, type, id, 9400 (void *)(&actions_num), file); 9401 e = LIST_NEXT(e, next); 9402 } 9403 } 9404 9405 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9406 rte_spinlock_unlock(&h->l_const.lcore_lock); 9407 } 9408 9409 /* get counter */ 9410 MLX5_ASSERT(cmng->n_valid <= cmng->n); 9411 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9412 for (j = 1; j <= max; j++) { 9413 action = NULL; 9414 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9415 &count.bytes, &action)) && action) { 9416 id = (uint64_t)(uintptr_t)action; 9417 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9418 save_dump_file(NULL, 0, type, 9419 id, (void *)&count, file); 9420 } 9421 } 9422 return 0; 9423 } 9424 #endif 9425 9426 /** 9427 * Dump flow raw hw data to file 9428 * 9429 * @param[in] dev 9430 * The pointer to Ethernet device. 9431 * @param[in] file 9432 * A pointer to a file for output. 9433 * @param[out] error 9434 * Perform verbose error reporting if not NULL. PMDs initialize this 9435 * structure in case of error only. 9436 * @return 9437 * 0 on success, a negative value otherwise. 9438 */ 9439 int 9440 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9441 FILE *file, 9442 struct rte_flow_error *error __rte_unused) 9443 { 9444 struct mlx5_priv *priv = dev->data->dev_private; 9445 struct mlx5_dev_ctx_shared *sh = priv->sh; 9446 uint32_t handle_idx; 9447 int ret; 9448 struct mlx5_flow_handle *dh; 9449 struct rte_flow *flow; 9450 9451 if (!sh->config.dv_flow_en) { 9452 if (fputs("device dv flow disabled\n", file) <= 0) 9453 return -errno; 9454 return -ENOTSUP; 9455 } 9456 9457 /* dump all */ 9458 if (!flow_idx) { 9459 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9460 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9461 return -EINVAL; 9462 #endif 9463 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9464 sh->rx_domain, 9465 sh->tx_domain, file); 9466 } 9467 /* dump one */ 9468 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9469 (uintptr_t)(void *)flow_idx); 9470 if (!flow) 9471 return -EINVAL; 9472 9473 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9474 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9475 #endif 9476 handle_idx = flow->dev_handles; 9477 while (handle_idx) { 9478 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9479 handle_idx); 9480 if (!dh) 9481 return -ENOENT; 9482 if (dh->drv_flow) { 9483 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9484 file); 9485 if (ret) 9486 return -ENOENT; 9487 } 9488 handle_idx = dh->next.next; 9489 } 9490 return 0; 9491 } 9492 9493 /** 9494 * Get aged-out flows. 9495 * 9496 * @param[in] dev 9497 * Pointer to the Ethernet device structure. 9498 * @param[in] context 9499 * The address of an array of pointers to the aged-out flows contexts. 9500 * @param[in] nb_countexts 9501 * The length of context array pointers. 9502 * @param[out] error 9503 * Perform verbose error reporting if not NULL. Initialized in case of 9504 * error only. 9505 * 9506 * @return 9507 * how many contexts get in success, otherwise negative errno value. 9508 * if nb_contexts is 0, return the amount of all aged contexts. 9509 * if nb_contexts is not 0 , return the amount of aged flows reported 9510 * in the context array. 9511 */ 9512 int 9513 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9514 uint32_t nb_contexts, struct rte_flow_error *error) 9515 { 9516 const struct mlx5_flow_driver_ops *fops; 9517 struct rte_flow_attr attr = { .transfer = 0 }; 9518 9519 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 9520 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 9521 return fops->get_aged_flows(dev, contexts, nb_contexts, 9522 error); 9523 } 9524 DRV_LOG(ERR, 9525 "port %u get aged flows is not supported.", 9526 dev->data->port_id); 9527 return -ENOTSUP; 9528 } 9529 9530 /* Wrapper for driver action_validate op callback */ 9531 static int 9532 flow_drv_action_validate(struct rte_eth_dev *dev, 9533 const struct rte_flow_indir_action_conf *conf, 9534 const struct rte_flow_action *action, 9535 const struct mlx5_flow_driver_ops *fops, 9536 struct rte_flow_error *error) 9537 { 9538 static const char err_msg[] = "indirect action validation unsupported"; 9539 9540 if (!fops->action_validate) { 9541 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9542 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9543 NULL, err_msg); 9544 return -rte_errno; 9545 } 9546 return fops->action_validate(dev, conf, action, error); 9547 } 9548 9549 /** 9550 * Destroys the shared action by handle. 9551 * 9552 * @param dev 9553 * Pointer to Ethernet device structure. 9554 * @param[in] handle 9555 * Handle for the indirect action object to be destroyed. 9556 * @param[out] error 9557 * Perform verbose error reporting if not NULL. PMDs initialize this 9558 * structure in case of error only. 9559 * 9560 * @return 9561 * 0 on success, a negative errno value otherwise and rte_errno is set. 9562 * 9563 * @note: wrapper for driver action_create op callback. 9564 */ 9565 static int 9566 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9567 struct rte_flow_action_handle *handle, 9568 struct rte_flow_error *error) 9569 { 9570 static const char err_msg[] = "indirect action destruction unsupported"; 9571 struct rte_flow_attr attr = { .transfer = 0 }; 9572 const struct mlx5_flow_driver_ops *fops = 9573 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9574 9575 if (!fops->action_destroy) { 9576 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9577 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9578 NULL, err_msg); 9579 return -rte_errno; 9580 } 9581 return fops->action_destroy(dev, handle, error); 9582 } 9583 9584 /* Wrapper for driver action_destroy op callback */ 9585 static int 9586 flow_drv_action_update(struct rte_eth_dev *dev, 9587 struct rte_flow_action_handle *handle, 9588 const void *update, 9589 const struct mlx5_flow_driver_ops *fops, 9590 struct rte_flow_error *error) 9591 { 9592 static const char err_msg[] = "indirect action update unsupported"; 9593 9594 if (!fops->action_update) { 9595 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9596 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9597 NULL, err_msg); 9598 return -rte_errno; 9599 } 9600 return fops->action_update(dev, handle, update, error); 9601 } 9602 9603 /* Wrapper for driver action_destroy op callback */ 9604 static int 9605 flow_drv_action_query(struct rte_eth_dev *dev, 9606 const struct rte_flow_action_handle *handle, 9607 void *data, 9608 const struct mlx5_flow_driver_ops *fops, 9609 struct rte_flow_error *error) 9610 { 9611 static const char err_msg[] = "indirect action query unsupported"; 9612 9613 if (!fops->action_query) { 9614 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9615 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9616 NULL, err_msg); 9617 return -rte_errno; 9618 } 9619 return fops->action_query(dev, handle, data, error); 9620 } 9621 9622 /** 9623 * Create indirect action for reuse in multiple flow rules. 9624 * 9625 * @param dev 9626 * Pointer to Ethernet device structure. 9627 * @param conf 9628 * Pointer to indirect action object configuration. 9629 * @param[in] action 9630 * Action configuration for indirect action object creation. 9631 * @param[out] error 9632 * Perform verbose error reporting if not NULL. PMDs initialize this 9633 * structure in case of error only. 9634 * @return 9635 * A valid handle in case of success, NULL otherwise and rte_errno is set. 9636 */ 9637 static struct rte_flow_action_handle * 9638 mlx5_action_handle_create(struct rte_eth_dev *dev, 9639 const struct rte_flow_indir_action_conf *conf, 9640 const struct rte_flow_action *action, 9641 struct rte_flow_error *error) 9642 { 9643 static const char err_msg[] = "indirect action creation unsupported"; 9644 struct rte_flow_attr attr = { .transfer = 0 }; 9645 const struct mlx5_flow_driver_ops *fops = 9646 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9647 9648 if (flow_drv_action_validate(dev, conf, action, fops, error)) 9649 return NULL; 9650 if (!fops->action_create) { 9651 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9652 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9653 NULL, err_msg); 9654 return NULL; 9655 } 9656 return fops->action_create(dev, conf, action, error); 9657 } 9658 9659 /** 9660 * Updates inplace the indirect action configuration pointed by *handle* 9661 * with the configuration provided as *update* argument. 9662 * The update of the indirect action configuration effects all flow rules 9663 * reusing the action via handle. 9664 * 9665 * @param dev 9666 * Pointer to Ethernet device structure. 9667 * @param[in] handle 9668 * Handle for the indirect action to be updated. 9669 * @param[in] update 9670 * Action specification used to modify the action pointed by handle. 9671 * *update* could be of same type with the action pointed by the *handle* 9672 * handle argument, or some other structures like a wrapper, depending on 9673 * the indirect action type. 9674 * @param[out] error 9675 * Perform verbose error reporting if not NULL. PMDs initialize this 9676 * structure in case of error only. 9677 * 9678 * @return 9679 * 0 on success, a negative errno value otherwise and rte_errno is set. 9680 */ 9681 static int 9682 mlx5_action_handle_update(struct rte_eth_dev *dev, 9683 struct rte_flow_action_handle *handle, 9684 const void *update, 9685 struct rte_flow_error *error) 9686 { 9687 struct rte_flow_attr attr = { .transfer = 0 }; 9688 const struct mlx5_flow_driver_ops *fops = 9689 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9690 int ret; 9691 9692 ret = flow_drv_action_validate(dev, NULL, 9693 (const struct rte_flow_action *)update, fops, error); 9694 if (ret) 9695 return ret; 9696 return flow_drv_action_update(dev, handle, update, fops, 9697 error); 9698 } 9699 9700 /** 9701 * Query the indirect action by handle. 9702 * 9703 * This function allows retrieving action-specific data such as counters. 9704 * Data is gathered by special action which may be present/referenced in 9705 * more than one flow rule definition. 9706 * 9707 * see @RTE_FLOW_ACTION_TYPE_COUNT 9708 * 9709 * @param dev 9710 * Pointer to Ethernet device structure. 9711 * @param[in] handle 9712 * Handle for the indirect action to query. 9713 * @param[in, out] data 9714 * Pointer to storage for the associated query data type. 9715 * @param[out] error 9716 * Perform verbose error reporting if not NULL. PMDs initialize this 9717 * structure in case of error only. 9718 * 9719 * @return 9720 * 0 on success, a negative errno value otherwise and rte_errno is set. 9721 */ 9722 static int 9723 mlx5_action_handle_query(struct rte_eth_dev *dev, 9724 const struct rte_flow_action_handle *handle, 9725 void *data, 9726 struct rte_flow_error *error) 9727 { 9728 struct rte_flow_attr attr = { .transfer = 0 }; 9729 const struct mlx5_flow_driver_ops *fops = 9730 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9731 9732 return flow_drv_action_query(dev, handle, data, fops, error); 9733 } 9734 9735 /** 9736 * Destroy all indirect actions (shared RSS). 9737 * 9738 * @param dev 9739 * Pointer to Ethernet device. 9740 * 9741 * @return 9742 * 0 on success, a negative errno value otherwise and rte_errno is set. 9743 */ 9744 int 9745 mlx5_action_handle_flush(struct rte_eth_dev *dev) 9746 { 9747 struct rte_flow_error error; 9748 struct mlx5_priv *priv = dev->data->dev_private; 9749 struct mlx5_shared_action_rss *shared_rss; 9750 int ret = 0; 9751 uint32_t idx; 9752 9753 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 9754 priv->rss_shared_actions, idx, shared_rss, next) { 9755 ret |= mlx5_action_handle_destroy(dev, 9756 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 9757 } 9758 return ret; 9759 } 9760 9761 /** 9762 * Validate existing indirect actions against current device configuration 9763 * and attach them to device resources. 9764 * 9765 * @param dev 9766 * Pointer to Ethernet device. 9767 * 9768 * @return 9769 * 0 on success, a negative errno value otherwise and rte_errno is set. 9770 */ 9771 int 9772 mlx5_action_handle_attach(struct rte_eth_dev *dev) 9773 { 9774 struct mlx5_priv *priv = dev->data->dev_private; 9775 int ret = 0; 9776 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9777 9778 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9779 const char *message; 9780 uint32_t queue_idx; 9781 9782 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 9783 ind_tbl->queues_n, 9784 &message, &queue_idx); 9785 if (ret != 0) { 9786 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 9787 dev->data->port_id, ind_tbl->queues[queue_idx], 9788 message); 9789 break; 9790 } 9791 } 9792 if (ret != 0) 9793 return ret; 9794 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9795 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 9796 if (ret != 0) { 9797 DRV_LOG(ERR, "Port %u could not attach " 9798 "indirection table obj %p", 9799 dev->data->port_id, (void *)ind_tbl); 9800 goto error; 9801 } 9802 } 9803 9804 return 0; 9805 error: 9806 ind_tbl_last = ind_tbl; 9807 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9808 if (ind_tbl == ind_tbl_last) 9809 break; 9810 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 9811 DRV_LOG(CRIT, "Port %u could not detach " 9812 "indirection table obj %p on rollback", 9813 dev->data->port_id, (void *)ind_tbl); 9814 } 9815 return ret; 9816 } 9817 9818 /** 9819 * Detach indirect actions of the device from its resources. 9820 * 9821 * @param dev 9822 * Pointer to Ethernet device. 9823 * 9824 * @return 9825 * 0 on success, a negative errno value otherwise and rte_errno is set. 9826 */ 9827 int 9828 mlx5_action_handle_detach(struct rte_eth_dev *dev) 9829 { 9830 struct mlx5_priv *priv = dev->data->dev_private; 9831 int ret = 0; 9832 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9833 9834 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9835 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 9836 if (ret != 0) { 9837 DRV_LOG(ERR, "Port %u could not detach " 9838 "indirection table obj %p", 9839 dev->data->port_id, (void *)ind_tbl); 9840 goto error; 9841 } 9842 } 9843 return 0; 9844 error: 9845 ind_tbl_last = ind_tbl; 9846 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9847 if (ind_tbl == ind_tbl_last) 9848 break; 9849 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 9850 DRV_LOG(CRIT, "Port %u could not attach " 9851 "indirection table obj %p on rollback", 9852 dev->data->port_id, (void *)ind_tbl); 9853 } 9854 return ret; 9855 } 9856 9857 #ifndef HAVE_MLX5DV_DR 9858 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 9859 #else 9860 #define MLX5_DOMAIN_SYNC_FLOW \ 9861 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 9862 #endif 9863 9864 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 9865 { 9866 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 9867 const struct mlx5_flow_driver_ops *fops; 9868 int ret; 9869 struct rte_flow_attr attr = { .transfer = 0 }; 9870 9871 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9872 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9873 if (ret > 0) 9874 ret = -ret; 9875 return ret; 9876 } 9877 9878 const struct mlx5_flow_tunnel * 9879 mlx5_get_tof(const struct rte_flow_item *item, 9880 const struct rte_flow_action *action, 9881 enum mlx5_tof_rule_type *rule_type) 9882 { 9883 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9884 if (item->type == (typeof(item->type)) 9885 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9886 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9887 return flow_items_to_tunnel(item); 9888 } 9889 } 9890 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9891 if (action->type == (typeof(action->type)) 9892 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9893 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9894 return flow_actions_to_tunnel(action); 9895 } 9896 } 9897 return NULL; 9898 } 9899 9900 /** 9901 * tunnel offload functionality is defined for DV environment only 9902 */ 9903 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9904 __extension__ 9905 union tunnel_offload_mark { 9906 uint32_t val; 9907 struct { 9908 uint32_t app_reserve:8; 9909 uint32_t table_id:15; 9910 uint32_t transfer:1; 9911 uint32_t _unused_:8; 9912 }; 9913 }; 9914 9915 static bool 9916 mlx5_access_tunnel_offload_db 9917 (struct rte_eth_dev *dev, 9918 bool (*match)(struct rte_eth_dev *, 9919 struct mlx5_flow_tunnel *, const void *), 9920 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9921 void (*miss)(struct rte_eth_dev *, void *), 9922 void *ctx, bool lock_op); 9923 9924 static int 9925 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9926 struct rte_flow *flow, 9927 const struct rte_flow_attr *attr, 9928 const struct rte_flow_action *app_actions, 9929 uint32_t flow_idx, 9930 const struct mlx5_flow_tunnel *tunnel, 9931 struct tunnel_default_miss_ctx *ctx, 9932 struct rte_flow_error *error) 9933 { 9934 struct mlx5_priv *priv = dev->data->dev_private; 9935 struct mlx5_flow *dev_flow; 9936 struct rte_flow_attr miss_attr = *attr; 9937 const struct rte_flow_item miss_items[2] = { 9938 { 9939 .type = RTE_FLOW_ITEM_TYPE_ETH, 9940 .spec = NULL, 9941 .last = NULL, 9942 .mask = NULL 9943 }, 9944 { 9945 .type = RTE_FLOW_ITEM_TYPE_END, 9946 .spec = NULL, 9947 .last = NULL, 9948 .mask = NULL 9949 } 9950 }; 9951 union tunnel_offload_mark mark_id; 9952 struct rte_flow_action_mark miss_mark; 9953 struct rte_flow_action miss_actions[3] = { 9954 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9955 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9956 }; 9957 const struct rte_flow_action_jump *jump_data; 9958 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9959 struct flow_grp_info grp_info = { 9960 .external = 1, 9961 .transfer = attr->transfer, 9962 .fdb_def_rule = !!priv->fdb_def_rule, 9963 .std_tbl_fix = 0, 9964 }; 9965 int ret; 9966 9967 if (!attr->transfer) { 9968 uint32_t q_size; 9969 9970 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9971 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9972 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9973 0, SOCKET_ID_ANY); 9974 if (!ctx->queue) 9975 return rte_flow_error_set 9976 (error, ENOMEM, 9977 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9978 NULL, "invalid default miss RSS"); 9979 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9980 ctx->action_rss.level = 0, 9981 ctx->action_rss.types = priv->rss_conf.rss_hf, 9982 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9983 ctx->action_rss.queue_num = priv->reta_idx_n, 9984 ctx->action_rss.key = priv->rss_conf.rss_key, 9985 ctx->action_rss.queue = ctx->queue; 9986 if (!priv->reta_idx_n || !priv->rxqs_n) 9987 return rte_flow_error_set 9988 (error, EINVAL, 9989 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9990 NULL, "invalid port configuration"); 9991 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9992 ctx->action_rss.types = 0; 9993 for (i = 0; i != priv->reta_idx_n; ++i) 9994 ctx->queue[i] = (*priv->reta_idx)[i]; 9995 } else { 9996 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9997 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9998 } 9999 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10000 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10001 jump_data = app_actions->conf; 10002 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10003 miss_attr.group = jump_data->group; 10004 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10005 &flow_table, &grp_info, error); 10006 if (ret) 10007 return rte_flow_error_set(error, EINVAL, 10008 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10009 NULL, "invalid tunnel id"); 10010 mark_id.app_reserve = 0; 10011 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10012 mark_id.transfer = !!attr->transfer; 10013 mark_id._unused_ = 0; 10014 miss_mark.id = mark_id.val; 10015 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10016 miss_items, miss_actions, flow_idx, error); 10017 if (!dev_flow) 10018 return -rte_errno; 10019 dev_flow->flow = flow; 10020 dev_flow->external = true; 10021 dev_flow->tunnel = tunnel; 10022 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10023 /* Subflow object was created, we must include one in the list. */ 10024 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10025 dev_flow->handle, next); 10026 DRV_LOG(DEBUG, 10027 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10028 dev->data->port_id, tunnel->app_tunnel.type, 10029 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10030 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10031 miss_actions, error); 10032 if (!ret) 10033 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10034 error); 10035 10036 return ret; 10037 } 10038 10039 static const struct mlx5_flow_tbl_data_entry * 10040 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10041 { 10042 struct mlx5_priv *priv = dev->data->dev_private; 10043 struct mlx5_dev_ctx_shared *sh = priv->sh; 10044 struct mlx5_list_entry *he; 10045 union tunnel_offload_mark mbits = { .val = mark }; 10046 union mlx5_flow_tbl_key table_key = { 10047 { 10048 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10049 .id = 0, 10050 .reserved = 0, 10051 .dummy = 0, 10052 .is_fdb = !!mbits.transfer, 10053 .is_egress = 0, 10054 } 10055 }; 10056 struct mlx5_flow_cb_ctx ctx = { 10057 .data = &table_key.v64, 10058 }; 10059 10060 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10061 return he ? 10062 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10063 } 10064 10065 static void 10066 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10067 struct mlx5_list_entry *entry) 10068 { 10069 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10070 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10071 10072 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10073 tunnel_flow_tbl_to_id(tte->flow_table)); 10074 mlx5_free(tte); 10075 } 10076 10077 static int 10078 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10079 struct mlx5_list_entry *entry, void *cb_ctx) 10080 { 10081 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10082 union tunnel_tbl_key tbl = { 10083 .val = *(uint64_t *)(ctx->data), 10084 }; 10085 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10086 10087 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10088 } 10089 10090 static struct mlx5_list_entry * 10091 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10092 { 10093 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10094 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10095 struct tunnel_tbl_entry *tte; 10096 union tunnel_tbl_key tbl = { 10097 .val = *(uint64_t *)(ctx->data), 10098 }; 10099 10100 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10101 sizeof(*tte), 0, 10102 SOCKET_ID_ANY); 10103 if (!tte) 10104 goto err; 10105 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10106 &tte->flow_table); 10107 if (tte->flow_table >= MLX5_MAX_TABLES) { 10108 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10109 tte->flow_table); 10110 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10111 tte->flow_table); 10112 goto err; 10113 } else if (!tte->flow_table) { 10114 goto err; 10115 } 10116 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10117 tte->tunnel_id = tbl.tunnel_id; 10118 tte->group = tbl.group; 10119 return &tte->hash; 10120 err: 10121 if (tte) 10122 mlx5_free(tte); 10123 return NULL; 10124 } 10125 10126 static struct mlx5_list_entry * 10127 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10128 struct mlx5_list_entry *oentry, 10129 void *cb_ctx __rte_unused) 10130 { 10131 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10132 0, SOCKET_ID_ANY); 10133 10134 if (!tte) 10135 return NULL; 10136 memcpy(tte, oentry, sizeof(*tte)); 10137 return &tte->hash; 10138 } 10139 10140 static void 10141 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10142 struct mlx5_list_entry *entry) 10143 { 10144 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10145 10146 mlx5_free(tte); 10147 } 10148 10149 static uint32_t 10150 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10151 const struct mlx5_flow_tunnel *tunnel, 10152 uint32_t group, uint32_t *table, 10153 struct rte_flow_error *error) 10154 { 10155 struct mlx5_list_entry *he; 10156 struct tunnel_tbl_entry *tte; 10157 union tunnel_tbl_key key = { 10158 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10159 .group = group 10160 }; 10161 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10162 struct mlx5_hlist *group_hash; 10163 struct mlx5_flow_cb_ctx ctx = { 10164 .data = &key.val, 10165 }; 10166 10167 group_hash = tunnel ? tunnel->groups : thub->groups; 10168 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10169 if (!he) 10170 return rte_flow_error_set(error, EINVAL, 10171 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10172 NULL, 10173 "tunnel group index not supported"); 10174 tte = container_of(he, typeof(*tte), hash); 10175 *table = tte->flow_table; 10176 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10177 dev->data->port_id, key.tunnel_id, group, *table); 10178 return 0; 10179 } 10180 10181 static void 10182 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10183 struct mlx5_flow_tunnel *tunnel) 10184 { 10185 struct mlx5_priv *priv = dev->data->dev_private; 10186 struct mlx5_indexed_pool *ipool; 10187 10188 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10189 dev->data->port_id, tunnel->tunnel_id); 10190 LIST_REMOVE(tunnel, chain); 10191 mlx5_hlist_destroy(tunnel->groups); 10192 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10193 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10194 } 10195 10196 static bool 10197 mlx5_access_tunnel_offload_db 10198 (struct rte_eth_dev *dev, 10199 bool (*match)(struct rte_eth_dev *, 10200 struct mlx5_flow_tunnel *, const void *), 10201 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10202 void (*miss)(struct rte_eth_dev *, void *), 10203 void *ctx, bool lock_op) 10204 { 10205 bool verdict = false; 10206 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10207 struct mlx5_flow_tunnel *tunnel; 10208 10209 rte_spinlock_lock(&thub->sl); 10210 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10211 verdict = match(dev, tunnel, (const void *)ctx); 10212 if (verdict) 10213 break; 10214 } 10215 if (!lock_op) 10216 rte_spinlock_unlock(&thub->sl); 10217 if (verdict && hit) 10218 hit(dev, tunnel, ctx); 10219 if (!verdict && miss) 10220 miss(dev, ctx); 10221 if (lock_op) 10222 rte_spinlock_unlock(&thub->sl); 10223 10224 return verdict; 10225 } 10226 10227 struct tunnel_db_find_tunnel_id_ctx { 10228 uint32_t tunnel_id; 10229 struct mlx5_flow_tunnel *tunnel; 10230 }; 10231 10232 static bool 10233 find_tunnel_id_match(struct rte_eth_dev *dev, 10234 struct mlx5_flow_tunnel *tunnel, const void *x) 10235 { 10236 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10237 10238 RTE_SET_USED(dev); 10239 return tunnel->tunnel_id == ctx->tunnel_id; 10240 } 10241 10242 static void 10243 find_tunnel_id_hit(struct rte_eth_dev *dev, 10244 struct mlx5_flow_tunnel *tunnel, void *x) 10245 { 10246 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10247 RTE_SET_USED(dev); 10248 ctx->tunnel = tunnel; 10249 } 10250 10251 static struct mlx5_flow_tunnel * 10252 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10253 { 10254 struct tunnel_db_find_tunnel_id_ctx ctx = { 10255 .tunnel_id = id, 10256 }; 10257 10258 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10259 find_tunnel_id_hit, NULL, &ctx, true); 10260 10261 return ctx.tunnel; 10262 } 10263 10264 static struct mlx5_flow_tunnel * 10265 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10266 const struct rte_flow_tunnel *app_tunnel) 10267 { 10268 struct mlx5_priv *priv = dev->data->dev_private; 10269 struct mlx5_indexed_pool *ipool; 10270 struct mlx5_flow_tunnel *tunnel; 10271 uint32_t id; 10272 10273 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10274 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10275 if (!tunnel) 10276 return NULL; 10277 if (id >= MLX5_MAX_TUNNELS) { 10278 mlx5_ipool_free(ipool, id); 10279 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10280 return NULL; 10281 } 10282 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10283 priv->sh, 10284 mlx5_flow_tunnel_grp2tbl_create_cb, 10285 mlx5_flow_tunnel_grp2tbl_match_cb, 10286 mlx5_flow_tunnel_grp2tbl_remove_cb, 10287 mlx5_flow_tunnel_grp2tbl_clone_cb, 10288 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10289 if (!tunnel->groups) { 10290 mlx5_ipool_free(ipool, id); 10291 return NULL; 10292 } 10293 /* initiate new PMD tunnel */ 10294 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10295 tunnel->tunnel_id = id; 10296 tunnel->action.type = (typeof(tunnel->action.type)) 10297 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10298 tunnel->action.conf = tunnel; 10299 tunnel->item.type = (typeof(tunnel->item.type)) 10300 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10301 tunnel->item.spec = tunnel; 10302 tunnel->item.last = NULL; 10303 tunnel->item.mask = NULL; 10304 10305 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10306 dev->data->port_id, tunnel->tunnel_id); 10307 10308 return tunnel; 10309 } 10310 10311 struct tunnel_db_get_tunnel_ctx { 10312 const struct rte_flow_tunnel *app_tunnel; 10313 struct mlx5_flow_tunnel *tunnel; 10314 }; 10315 10316 static bool get_tunnel_match(struct rte_eth_dev *dev, 10317 struct mlx5_flow_tunnel *tunnel, const void *x) 10318 { 10319 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10320 10321 RTE_SET_USED(dev); 10322 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10323 sizeof(*ctx->app_tunnel)); 10324 } 10325 10326 static void get_tunnel_hit(struct rte_eth_dev *dev, 10327 struct mlx5_flow_tunnel *tunnel, void *x) 10328 { 10329 /* called under tunnel spinlock protection */ 10330 struct tunnel_db_get_tunnel_ctx *ctx = x; 10331 10332 RTE_SET_USED(dev); 10333 tunnel->refctn++; 10334 ctx->tunnel = tunnel; 10335 } 10336 10337 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10338 { 10339 /* called under tunnel spinlock protection */ 10340 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10341 struct tunnel_db_get_tunnel_ctx *ctx = x; 10342 10343 rte_spinlock_unlock(&thub->sl); 10344 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10345 rte_spinlock_lock(&thub->sl); 10346 if (ctx->tunnel) { 10347 ctx->tunnel->refctn = 1; 10348 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10349 } 10350 } 10351 10352 10353 static int 10354 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10355 const struct rte_flow_tunnel *app_tunnel, 10356 struct mlx5_flow_tunnel **tunnel) 10357 { 10358 struct tunnel_db_get_tunnel_ctx ctx = { 10359 .app_tunnel = app_tunnel, 10360 }; 10361 10362 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10363 get_tunnel_miss, &ctx, true); 10364 *tunnel = ctx.tunnel; 10365 return ctx.tunnel ? 0 : -ENOMEM; 10366 } 10367 10368 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10369 { 10370 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10371 10372 if (!thub) 10373 return; 10374 if (!LIST_EMPTY(&thub->tunnels)) 10375 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10376 mlx5_hlist_destroy(thub->groups); 10377 mlx5_free(thub); 10378 } 10379 10380 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10381 { 10382 int err; 10383 struct mlx5_flow_tunnel_hub *thub; 10384 10385 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10386 0, SOCKET_ID_ANY); 10387 if (!thub) 10388 return -ENOMEM; 10389 LIST_INIT(&thub->tunnels); 10390 rte_spinlock_init(&thub->sl); 10391 thub->groups = mlx5_hlist_create("flow groups", 64, 10392 false, true, sh, 10393 mlx5_flow_tunnel_grp2tbl_create_cb, 10394 mlx5_flow_tunnel_grp2tbl_match_cb, 10395 mlx5_flow_tunnel_grp2tbl_remove_cb, 10396 mlx5_flow_tunnel_grp2tbl_clone_cb, 10397 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10398 if (!thub->groups) { 10399 err = -rte_errno; 10400 goto err; 10401 } 10402 sh->tunnel_hub = thub; 10403 10404 return 0; 10405 10406 err: 10407 if (thub->groups) 10408 mlx5_hlist_destroy(thub->groups); 10409 if (thub) 10410 mlx5_free(thub); 10411 return err; 10412 } 10413 10414 static inline int 10415 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10416 struct rte_flow_tunnel *tunnel, 10417 struct rte_flow_error *error) 10418 { 10419 struct mlx5_priv *priv = dev->data->dev_private; 10420 10421 if (!priv->sh->config.dv_flow_en) 10422 return rte_flow_error_set(error, ENOTSUP, 10423 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10424 "flow DV interface is off"); 10425 if (!is_tunnel_offload_active(dev)) 10426 return rte_flow_error_set(error, ENOTSUP, 10427 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10428 "tunnel offload was not activated"); 10429 if (!tunnel) 10430 return rte_flow_error_set(error, EINVAL, 10431 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10432 "no application tunnel"); 10433 switch (tunnel->type) { 10434 default: 10435 return rte_flow_error_set(error, EINVAL, 10436 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10437 "unsupported tunnel type"); 10438 case RTE_FLOW_ITEM_TYPE_VXLAN: 10439 case RTE_FLOW_ITEM_TYPE_GRE: 10440 case RTE_FLOW_ITEM_TYPE_NVGRE: 10441 case RTE_FLOW_ITEM_TYPE_GENEVE: 10442 break; 10443 } 10444 return 0; 10445 } 10446 10447 static int 10448 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10449 struct rte_flow_tunnel *app_tunnel, 10450 struct rte_flow_action **actions, 10451 uint32_t *num_of_actions, 10452 struct rte_flow_error *error) 10453 { 10454 struct mlx5_flow_tunnel *tunnel; 10455 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10456 10457 if (ret) 10458 return ret; 10459 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10460 if (ret < 0) { 10461 return rte_flow_error_set(error, ret, 10462 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10463 "failed to initialize pmd tunnel"); 10464 } 10465 *actions = &tunnel->action; 10466 *num_of_actions = 1; 10467 return 0; 10468 } 10469 10470 static int 10471 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10472 struct rte_flow_tunnel *app_tunnel, 10473 struct rte_flow_item **items, 10474 uint32_t *num_of_items, 10475 struct rte_flow_error *error) 10476 { 10477 struct mlx5_flow_tunnel *tunnel; 10478 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10479 10480 if (ret) 10481 return ret; 10482 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10483 if (ret < 0) { 10484 return rte_flow_error_set(error, ret, 10485 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10486 "failed to initialize pmd tunnel"); 10487 } 10488 *items = &tunnel->item; 10489 *num_of_items = 1; 10490 return 0; 10491 } 10492 10493 struct tunnel_db_element_release_ctx { 10494 struct rte_flow_item *items; 10495 struct rte_flow_action *actions; 10496 uint32_t num_elements; 10497 struct rte_flow_error *error; 10498 int ret; 10499 }; 10500 10501 static bool 10502 tunnel_element_release_match(struct rte_eth_dev *dev, 10503 struct mlx5_flow_tunnel *tunnel, const void *x) 10504 { 10505 const struct tunnel_db_element_release_ctx *ctx = x; 10506 10507 RTE_SET_USED(dev); 10508 if (ctx->num_elements != 1) 10509 return false; 10510 else if (ctx->items) 10511 return ctx->items == &tunnel->item; 10512 else if (ctx->actions) 10513 return ctx->actions == &tunnel->action; 10514 10515 return false; 10516 } 10517 10518 static void 10519 tunnel_element_release_hit(struct rte_eth_dev *dev, 10520 struct mlx5_flow_tunnel *tunnel, void *x) 10521 { 10522 struct tunnel_db_element_release_ctx *ctx = x; 10523 ctx->ret = 0; 10524 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10525 mlx5_flow_tunnel_free(dev, tunnel); 10526 } 10527 10528 static void 10529 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10530 { 10531 struct tunnel_db_element_release_ctx *ctx = x; 10532 RTE_SET_USED(dev); 10533 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10534 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10535 "invalid argument"); 10536 } 10537 10538 static int 10539 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10540 struct rte_flow_item *pmd_items, 10541 uint32_t num_items, struct rte_flow_error *err) 10542 { 10543 struct tunnel_db_element_release_ctx ctx = { 10544 .items = pmd_items, 10545 .actions = NULL, 10546 .num_elements = num_items, 10547 .error = err, 10548 }; 10549 10550 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10551 tunnel_element_release_hit, 10552 tunnel_element_release_miss, &ctx, false); 10553 10554 return ctx.ret; 10555 } 10556 10557 static int 10558 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10559 struct rte_flow_action *pmd_actions, 10560 uint32_t num_actions, struct rte_flow_error *err) 10561 { 10562 struct tunnel_db_element_release_ctx ctx = { 10563 .items = NULL, 10564 .actions = pmd_actions, 10565 .num_elements = num_actions, 10566 .error = err, 10567 }; 10568 10569 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10570 tunnel_element_release_hit, 10571 tunnel_element_release_miss, &ctx, false); 10572 10573 return ctx.ret; 10574 } 10575 10576 static int 10577 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10578 struct rte_mbuf *m, 10579 struct rte_flow_restore_info *info, 10580 struct rte_flow_error *err) 10581 { 10582 uint64_t ol_flags = m->ol_flags; 10583 const struct mlx5_flow_tbl_data_entry *tble; 10584 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10585 10586 if (!is_tunnel_offload_active(dev)) { 10587 info->flags = 0; 10588 return 0; 10589 } 10590 10591 if ((ol_flags & mask) != mask) 10592 goto err; 10593 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10594 if (!tble) { 10595 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10596 dev->data->port_id, m->hash.fdir.hi); 10597 goto err; 10598 } 10599 MLX5_ASSERT(tble->tunnel); 10600 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10601 info->group_id = tble->group_id; 10602 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10603 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10604 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10605 10606 return 0; 10607 10608 err: 10609 return rte_flow_error_set(err, EINVAL, 10610 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10611 "failed to get restore info"); 10612 } 10613 10614 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 10615 static int 10616 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 10617 __rte_unused struct rte_flow_tunnel *app_tunnel, 10618 __rte_unused struct rte_flow_action **actions, 10619 __rte_unused uint32_t *num_of_actions, 10620 __rte_unused struct rte_flow_error *error) 10621 { 10622 return -ENOTSUP; 10623 } 10624 10625 static int 10626 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 10627 __rte_unused struct rte_flow_tunnel *app_tunnel, 10628 __rte_unused struct rte_flow_item **items, 10629 __rte_unused uint32_t *num_of_items, 10630 __rte_unused struct rte_flow_error *error) 10631 { 10632 return -ENOTSUP; 10633 } 10634 10635 static int 10636 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 10637 __rte_unused struct rte_flow_item *pmd_items, 10638 __rte_unused uint32_t num_items, 10639 __rte_unused struct rte_flow_error *err) 10640 { 10641 return -ENOTSUP; 10642 } 10643 10644 static int 10645 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 10646 __rte_unused struct rte_flow_action *pmd_action, 10647 __rte_unused uint32_t num_actions, 10648 __rte_unused struct rte_flow_error *err) 10649 { 10650 return -ENOTSUP; 10651 } 10652 10653 static int 10654 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 10655 __rte_unused struct rte_mbuf *m, 10656 __rte_unused struct rte_flow_restore_info *i, 10657 __rte_unused struct rte_flow_error *err) 10658 { 10659 return -ENOTSUP; 10660 } 10661 10662 static int 10663 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 10664 __rte_unused struct rte_flow *flow, 10665 __rte_unused const struct rte_flow_attr *attr, 10666 __rte_unused const struct rte_flow_action *actions, 10667 __rte_unused uint32_t flow_idx, 10668 __rte_unused const struct mlx5_flow_tunnel *tunnel, 10669 __rte_unused struct tunnel_default_miss_ctx *ctx, 10670 __rte_unused struct rte_flow_error *error) 10671 { 10672 return -ENOTSUP; 10673 } 10674 10675 static struct mlx5_flow_tunnel * 10676 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 10677 __rte_unused uint32_t id) 10678 { 10679 return NULL; 10680 } 10681 10682 static void 10683 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 10684 __rte_unused struct mlx5_flow_tunnel *tunnel) 10685 { 10686 } 10687 10688 static uint32_t 10689 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 10690 __rte_unused const struct mlx5_flow_tunnel *t, 10691 __rte_unused uint32_t group, 10692 __rte_unused uint32_t *table, 10693 struct rte_flow_error *error) 10694 { 10695 return rte_flow_error_set(error, ENOTSUP, 10696 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10697 "tunnel offload requires DV support"); 10698 } 10699 10700 void 10701 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 10702 __rte_unused uint16_t port_id) 10703 { 10704 } 10705 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10706 10707 /* Flex flow item API */ 10708 static struct rte_flow_item_flex_handle * 10709 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 10710 const struct rte_flow_item_flex_conf *conf, 10711 struct rte_flow_error *error) 10712 { 10713 static const char err_msg[] = "flex item creation unsupported"; 10714 struct mlx5_priv *priv = dev->data->dev_private; 10715 struct rte_flow_attr attr = { .transfer = 0 }; 10716 const struct mlx5_flow_driver_ops *fops = 10717 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10718 10719 if (!priv->pci_dev) { 10720 rte_flow_error_set(error, ENOTSUP, 10721 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10722 "create flex item on PF only"); 10723 return NULL; 10724 } 10725 switch (priv->pci_dev->id.device_id) { 10726 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 10727 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 10728 break; 10729 default: 10730 rte_flow_error_set(error, ENOTSUP, 10731 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10732 "flex item available on BlueField ports only"); 10733 return NULL; 10734 } 10735 if (!fops->item_create) { 10736 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10737 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10738 NULL, err_msg); 10739 return NULL; 10740 } 10741 return fops->item_create(dev, conf, error); 10742 } 10743 10744 static int 10745 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 10746 const struct rte_flow_item_flex_handle *handle, 10747 struct rte_flow_error *error) 10748 { 10749 static const char err_msg[] = "flex item release unsupported"; 10750 struct rte_flow_attr attr = { .transfer = 0 }; 10751 const struct mlx5_flow_driver_ops *fops = 10752 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10753 10754 if (!fops->item_release) { 10755 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10756 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10757 NULL, err_msg); 10758 return -rte_errno; 10759 } 10760 return fops->item_release(dev, handle, error); 10761 } 10762 10763 static void 10764 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 10765 { 10766 int ret; 10767 struct rte_flow_error error; 10768 10769 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10770 char *item_name; 10771 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 10772 sizeof(item_name), 10773 (void *)(uintptr_t)item->type, &error); 10774 if (ret > 0) 10775 printf("%s ", item_name); 10776 else 10777 printf("%d\n", (int)item->type); 10778 } 10779 printf("END\n"); 10780 } 10781 10782 static int 10783 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 10784 { 10785 const struct rte_flow_item_udp *spec = udp_item->spec; 10786 const struct rte_flow_item_udp *mask = udp_item->mask; 10787 uint16_t udp_dport = 0; 10788 10789 if (spec != NULL) { 10790 if (!mask) 10791 mask = &rte_flow_item_udp_mask; 10792 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 10793 mask->hdr.dst_port); 10794 } 10795 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 10796 } 10797 10798 static const struct mlx5_flow_expand_node * 10799 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 10800 unsigned int item_idx, 10801 const struct mlx5_flow_expand_node graph[], 10802 const struct mlx5_flow_expand_node *node) 10803 { 10804 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 10805 10806 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 10807 node != NULL && 10808 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 10809 /* 10810 * The expansion node is VXLAN and it is also the last 10811 * expandable item in the pattern, so need to continue 10812 * expansion of the inner tunnel. 10813 */ 10814 MLX5_ASSERT(item_idx > 0); 10815 prev_item = pattern + item_idx - 1; 10816 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 10817 if (mlx5_flow_is_std_vxlan_port(prev_item)) 10818 return &graph[MLX5_EXPANSION_STD_VXLAN]; 10819 return &graph[MLX5_EXPANSION_L3_VXLAN]; 10820 } 10821 return node; 10822 } 10823 10824 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 10825 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 10826 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 10827 }; 10828 10829 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 10830 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 10831 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 10832 { 9, 10, 11 }, { 12, 13, 14 }, 10833 }; 10834 10835 /** 10836 * Discover the number of available flow priorities. 10837 * 10838 * @param dev 10839 * Ethernet device. 10840 * 10841 * @return 10842 * On success, number of available flow priorities. 10843 * On failure, a negative errno-style code and rte_errno is set. 10844 */ 10845 int 10846 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 10847 { 10848 static const uint16_t vprio[] = {8, 16}; 10849 const struct mlx5_priv *priv = dev->data->dev_private; 10850 const struct mlx5_flow_driver_ops *fops; 10851 enum mlx5_flow_drv_type type; 10852 int ret; 10853 10854 type = mlx5_flow_os_get_type(); 10855 if (type == MLX5_FLOW_TYPE_MAX) { 10856 type = MLX5_FLOW_TYPE_VERBS; 10857 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 10858 type = MLX5_FLOW_TYPE_DV; 10859 } 10860 fops = flow_get_drv_ops(type); 10861 if (fops->discover_priorities == NULL) { 10862 DRV_LOG(ERR, "Priority discovery not supported"); 10863 rte_errno = ENOTSUP; 10864 return -rte_errno; 10865 } 10866 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 10867 if (ret < 0) 10868 return ret; 10869 switch (ret) { 10870 case 8: 10871 ret = RTE_DIM(priority_map_3); 10872 break; 10873 case 16: 10874 ret = RTE_DIM(priority_map_5); 10875 break; 10876 default: 10877 rte_errno = ENOTSUP; 10878 DRV_LOG(ERR, 10879 "port %u maximum priority: %d expected 8/16", 10880 dev->data->port_id, ret); 10881 return -rte_errno; 10882 } 10883 DRV_LOG(INFO, "port %u supported flow priorities:" 10884 " 0-%d for ingress or egress root table," 10885 " 0-%d for non-root table or transfer root table.", 10886 dev->data->port_id, ret - 2, 10887 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 10888 return ret; 10889 } 10890 10891 /** 10892 * Adjust flow priority based on the highest layer and the request priority. 10893 * 10894 * @param[in] dev 10895 * Pointer to the Ethernet device structure. 10896 * @param[in] priority 10897 * The rule base priority. 10898 * @param[in] subpriority 10899 * The priority based on the items. 10900 * 10901 * @return 10902 * The new priority. 10903 */ 10904 uint32_t 10905 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10906 uint32_t subpriority) 10907 { 10908 uint32_t res = 0; 10909 struct mlx5_priv *priv = dev->data->dev_private; 10910 10911 switch (priv->sh->flow_max_priority) { 10912 case RTE_DIM(priority_map_3): 10913 res = priority_map_3[priority][subpriority]; 10914 break; 10915 case RTE_DIM(priority_map_5): 10916 res = priority_map_5[priority][subpriority]; 10917 break; 10918 } 10919 return res; 10920 } 10921