1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 80 #endif 81 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 82 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 83 }; 84 85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 87 (const int []){ \ 88 __VA_ARGS__, 0, \ 89 } 90 91 /** Node object of input graph for mlx5_flow_expand_rss(). */ 92 struct mlx5_flow_expand_node { 93 const int *const next; 94 /**< 95 * List of next node indexes. Index 0 is interpreted as a terminator. 96 */ 97 const enum rte_flow_item_type type; 98 /**< Pattern item type of current node. */ 99 uint64_t rss_types; 100 /**< 101 * RSS types bit-field associated with this node 102 * (see RTE_ETH_RSS_* definitions). 103 */ 104 uint64_t node_flags; 105 /**< 106 * Bit-fields that define how the node is used in the expansion. 107 * (see MLX5_EXPANSION_NODE_* definitions). 108 */ 109 }; 110 111 /* Optional expand field. The expansion alg will not go deeper. */ 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 113 114 /* The node is not added implicitly as expansion to the flow pattern. 115 * If the node type does not match the flow pattern item type, the 116 * expansion alg will go deeper to its next items. 117 * In the current implementation, the list of next nodes indexes can 118 * have up to one node with this flag set and it has to be the last 119 * node index (before the list terminator). 120 */ 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 122 123 /** Object returned by mlx5_flow_expand_rss(). */ 124 struct mlx5_flow_expand_rss { 125 uint32_t entries; 126 /**< Number of entries @p patterns and @p priorities. */ 127 struct { 128 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 129 uint32_t priority; /**< Priority offset for each expansion. */ 130 } entry[]; 131 }; 132 133 static void 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 135 136 static const struct mlx5_flow_expand_node * 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 138 unsigned int item_idx, 139 const struct mlx5_flow_expand_node graph[], 140 const struct mlx5_flow_expand_node *node); 141 142 static bool 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 144 { 145 switch (item->type) { 146 case RTE_FLOW_ITEM_TYPE_ETH: 147 case RTE_FLOW_ITEM_TYPE_VLAN: 148 case RTE_FLOW_ITEM_TYPE_IPV4: 149 case RTE_FLOW_ITEM_TYPE_IPV6: 150 case RTE_FLOW_ITEM_TYPE_UDP: 151 case RTE_FLOW_ITEM_TYPE_TCP: 152 case RTE_FLOW_ITEM_TYPE_VXLAN: 153 case RTE_FLOW_ITEM_TYPE_NVGRE: 154 case RTE_FLOW_ITEM_TYPE_GRE: 155 case RTE_FLOW_ITEM_TYPE_GENEVE: 156 case RTE_FLOW_ITEM_TYPE_MPLS: 157 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 158 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 159 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 160 case RTE_FLOW_ITEM_TYPE_GTP: 161 return true; 162 default: 163 break; 164 } 165 return false; 166 } 167 168 /** 169 * Network Service Header (NSH) and its next protocol values 170 * are described in RFC-8393. 171 */ 172 static enum rte_flow_item_type 173 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 174 { 175 enum rte_flow_item_type type; 176 177 switch (proto_mask & proto_spec) { 178 case RTE_VXLAN_GPE_TYPE_IPV4: 179 type = RTE_FLOW_ITEM_TYPE_IPV4; 180 break; 181 case RTE_VXLAN_GPE_TYPE_IPV6: 182 type = RTE_VXLAN_GPE_TYPE_IPV6; 183 break; 184 case RTE_VXLAN_GPE_TYPE_ETH: 185 type = RTE_FLOW_ITEM_TYPE_ETH; 186 break; 187 default: 188 type = RTE_FLOW_ITEM_TYPE_END; 189 } 190 return type; 191 } 192 193 static enum rte_flow_item_type 194 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 195 { 196 enum rte_flow_item_type type; 197 198 switch (proto_mask & proto_spec) { 199 case IPPROTO_UDP: 200 type = RTE_FLOW_ITEM_TYPE_UDP; 201 break; 202 case IPPROTO_TCP: 203 type = RTE_FLOW_ITEM_TYPE_TCP; 204 break; 205 case IPPROTO_IP: 206 type = RTE_FLOW_ITEM_TYPE_IPV4; 207 break; 208 case IPPROTO_IPV6: 209 type = RTE_FLOW_ITEM_TYPE_IPV6; 210 break; 211 default: 212 type = RTE_FLOW_ITEM_TYPE_END; 213 } 214 return type; 215 } 216 217 static enum rte_flow_item_type 218 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 219 rte_be16_t type_mask, bool is_tunnel) 220 { 221 enum rte_flow_item_type type; 222 223 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 224 case RTE_ETHER_TYPE_TEB: 225 type = is_tunnel ? 226 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 227 break; 228 case RTE_ETHER_TYPE_VLAN: 229 type = !is_tunnel ? 230 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 231 break; 232 case RTE_ETHER_TYPE_IPV4: 233 type = RTE_FLOW_ITEM_TYPE_IPV4; 234 break; 235 case RTE_ETHER_TYPE_IPV6: 236 type = RTE_FLOW_ITEM_TYPE_IPV6; 237 break; 238 default: 239 type = RTE_FLOW_ITEM_TYPE_END; 240 } 241 return type; 242 } 243 244 static enum rte_flow_item_type 245 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 246 { 247 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 248 do { \ 249 const void *m = item->mask; \ 250 const void *s = item->spec; \ 251 mask = m ? \ 252 ((const struct rte_flow_item_##type *)m)->fld : \ 253 rte_flow_item_##type##_mask.fld; \ 254 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 255 } while (0) 256 257 enum rte_flow_item_type ret; 258 uint16_t spec, mask; 259 260 if (item == NULL || item->spec == NULL) 261 return RTE_FLOW_ITEM_TYPE_VOID; 262 switch (item->type) { 263 case RTE_FLOW_ITEM_TYPE_ETH: 264 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 265 if (!mask) 266 return RTE_FLOW_ITEM_TYPE_VOID; 267 ret = mlx5_ethertype_to_item_type(spec, mask, false); 268 break; 269 case RTE_FLOW_ITEM_TYPE_VLAN: 270 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 271 if (!mask) 272 return RTE_FLOW_ITEM_TYPE_VOID; 273 ret = mlx5_ethertype_to_item_type(spec, mask, false); 274 break; 275 case RTE_FLOW_ITEM_TYPE_IPV4: 276 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 277 if (!mask) 278 return RTE_FLOW_ITEM_TYPE_VOID; 279 ret = mlx5_inet_proto_to_item_type(spec, mask); 280 break; 281 case RTE_FLOW_ITEM_TYPE_IPV6: 282 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 283 if (!mask) 284 return RTE_FLOW_ITEM_TYPE_VOID; 285 ret = mlx5_inet_proto_to_item_type(spec, mask); 286 break; 287 case RTE_FLOW_ITEM_TYPE_GENEVE: 288 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 289 ret = mlx5_ethertype_to_item_type(spec, mask, true); 290 break; 291 case RTE_FLOW_ITEM_TYPE_GRE: 292 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 293 ret = mlx5_ethertype_to_item_type(spec, mask, true); 294 break; 295 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 296 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 297 ret = mlx5_nsh_proto_to_item_type(spec, mask); 298 break; 299 default: 300 ret = RTE_FLOW_ITEM_TYPE_VOID; 301 break; 302 } 303 return ret; 304 #undef MLX5_XSET_ITEM_MASK_SPEC 305 } 306 307 static const int * 308 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 309 const int *next_node) 310 { 311 const struct mlx5_flow_expand_node *node = NULL; 312 const int *next = next_node; 313 314 while (next && *next) { 315 /* 316 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 317 * flag set, because they were not found in the flow pattern. 318 */ 319 node = &graph[*next]; 320 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 321 break; 322 next = node->next; 323 } 324 return next; 325 } 326 327 #define MLX5_RSS_EXP_ELT_N 16 328 329 /** 330 * Expand RSS flows into several possible flows according to the RSS hash 331 * fields requested and the driver capabilities. 332 * 333 * @param[out] buf 334 * Buffer to store the result expansion. 335 * @param[in] size 336 * Buffer size in bytes. If 0, @p buf can be NULL. 337 * @param[in] pattern 338 * User flow pattern. 339 * @param[in] types 340 * RSS types to expand (see RTE_ETH_RSS_* definitions). 341 * @param[in] graph 342 * Input graph to expand @p pattern according to @p types. 343 * @param[in] graph_root_index 344 * Index of root node in @p graph, typically 0. 345 * 346 * @return 347 * A positive value representing the size of @p buf in bytes regardless of 348 * @p size on success, a negative errno value otherwise and rte_errno is 349 * set, the following errors are defined: 350 * 351 * -E2BIG: graph-depth @p graph is too deep. 352 * -EINVAL: @p size has not enough space for expanded pattern. 353 */ 354 static int 355 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 356 const struct rte_flow_item *pattern, uint64_t types, 357 const struct mlx5_flow_expand_node graph[], 358 int graph_root_index) 359 { 360 const struct rte_flow_item *item; 361 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 362 const int *next_node; 363 const int *stack[MLX5_RSS_EXP_ELT_N]; 364 int stack_pos = 0; 365 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 366 unsigned int i, item_idx, last_expand_item_idx = 0; 367 size_t lsize; 368 size_t user_pattern_size = 0; 369 void *addr = NULL; 370 const struct mlx5_flow_expand_node *next = NULL; 371 struct rte_flow_item missed_item; 372 int missed = 0; 373 int elt = 0; 374 const struct rte_flow_item *last_expand_item = NULL; 375 376 memset(&missed_item, 0, sizeof(missed_item)); 377 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 378 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 379 if (lsize > size) 380 return -EINVAL; 381 buf->entry[0].priority = 0; 382 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 383 buf->entries = 0; 384 addr = buf->entry[0].pattern; 385 for (item = pattern, item_idx = 0; 386 item->type != RTE_FLOW_ITEM_TYPE_END; 387 item++, item_idx++) { 388 if (!mlx5_flow_is_rss_expandable_item(item)) { 389 user_pattern_size += sizeof(*item); 390 continue; 391 } 392 last_expand_item = item; 393 last_expand_item_idx = item_idx; 394 i = 0; 395 while (node->next && node->next[i]) { 396 next = &graph[node->next[i]]; 397 if (next->type == item->type) 398 break; 399 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 400 node = next; 401 i = 0; 402 } else { 403 ++i; 404 } 405 } 406 if (next) 407 node = next; 408 user_pattern_size += sizeof(*item); 409 } 410 user_pattern_size += sizeof(*item); /* Handle END item. */ 411 lsize += user_pattern_size; 412 if (lsize > size) 413 return -EINVAL; 414 /* Copy the user pattern in the first entry of the buffer. */ 415 rte_memcpy(addr, pattern, user_pattern_size); 416 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 417 buf->entries = 1; 418 /* Start expanding. */ 419 memset(flow_items, 0, sizeof(flow_items)); 420 user_pattern_size -= sizeof(*item); 421 /* 422 * Check if the last valid item has spec set, need complete pattern, 423 * and the pattern can be used for expansion. 424 */ 425 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 426 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 427 /* Item type END indicates expansion is not required. */ 428 return lsize; 429 } 430 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 431 next = NULL; 432 missed = 1; 433 i = 0; 434 while (node->next && node->next[i]) { 435 next = &graph[node->next[i]]; 436 if (next->type == missed_item.type) { 437 flow_items[0].type = missed_item.type; 438 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 439 break; 440 } 441 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 442 node = next; 443 i = 0; 444 } else { 445 ++i; 446 } 447 next = NULL; 448 } 449 } 450 if (next && missed) { 451 elt = 2; /* missed item + item end. */ 452 node = next; 453 lsize += elt * sizeof(*item) + user_pattern_size; 454 if (lsize > size) 455 return -EINVAL; 456 if (node->rss_types & types) { 457 buf->entry[buf->entries].priority = 1; 458 buf->entry[buf->entries].pattern = addr; 459 buf->entries++; 460 rte_memcpy(addr, buf->entry[0].pattern, 461 user_pattern_size); 462 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 463 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 464 addr = (void *)(((uintptr_t)addr) + 465 elt * sizeof(*item)); 466 } 467 } else if (last_expand_item != NULL) { 468 node = mlx5_flow_expand_rss_adjust_node(pattern, 469 last_expand_item_idx, graph, node); 470 } 471 memset(flow_items, 0, sizeof(flow_items)); 472 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 473 node->next); 474 stack[stack_pos] = next_node; 475 node = next_node ? &graph[*next_node] : NULL; 476 while (node) { 477 flow_items[stack_pos].type = node->type; 478 if (node->rss_types & types) { 479 size_t n; 480 /* 481 * compute the number of items to copy from the 482 * expansion and copy it. 483 * When the stack_pos is 0, there are 1 element in it, 484 * plus the addition END item. 485 */ 486 elt = stack_pos + 2; 487 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 488 lsize += elt * sizeof(*item) + user_pattern_size; 489 if (lsize > size) 490 return -EINVAL; 491 n = elt * sizeof(*item); 492 buf->entry[buf->entries].priority = 493 stack_pos + 1 + missed; 494 buf->entry[buf->entries].pattern = addr; 495 buf->entries++; 496 rte_memcpy(addr, buf->entry[0].pattern, 497 user_pattern_size); 498 addr = (void *)(((uintptr_t)addr) + 499 user_pattern_size); 500 rte_memcpy(addr, &missed_item, 501 missed * sizeof(*item)); 502 addr = (void *)(((uintptr_t)addr) + 503 missed * sizeof(*item)); 504 rte_memcpy(addr, flow_items, n); 505 addr = (void *)(((uintptr_t)addr) + n); 506 } 507 /* Go deeper. */ 508 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 509 node->next) { 510 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 511 node->next); 512 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 513 rte_errno = E2BIG; 514 return -rte_errno; 515 } 516 stack[stack_pos] = next_node; 517 } else if (*(next_node + 1)) { 518 /* Follow up with the next possibility. */ 519 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 520 ++next_node); 521 } else if (!stack_pos) { 522 /* 523 * Completing the traverse over the different paths. 524 * The next_node is advanced to the terminator. 525 */ 526 ++next_node; 527 } else { 528 /* Move to the next path. */ 529 while (stack_pos) { 530 next_node = stack[--stack_pos]; 531 next_node++; 532 if (*next_node) 533 break; 534 } 535 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 536 next_node); 537 stack[stack_pos] = next_node; 538 } 539 node = next_node && *next_node ? &graph[*next_node] : NULL; 540 }; 541 return lsize; 542 } 543 544 enum mlx5_expansion { 545 MLX5_EXPANSION_ROOT, 546 MLX5_EXPANSION_ROOT_OUTER, 547 MLX5_EXPANSION_OUTER_ETH, 548 MLX5_EXPANSION_OUTER_VLAN, 549 MLX5_EXPANSION_OUTER_IPV4, 550 MLX5_EXPANSION_OUTER_IPV4_UDP, 551 MLX5_EXPANSION_OUTER_IPV4_TCP, 552 MLX5_EXPANSION_OUTER_IPV6, 553 MLX5_EXPANSION_OUTER_IPV6_UDP, 554 MLX5_EXPANSION_OUTER_IPV6_TCP, 555 MLX5_EXPANSION_VXLAN, 556 MLX5_EXPANSION_STD_VXLAN, 557 MLX5_EXPANSION_L3_VXLAN, 558 MLX5_EXPANSION_VXLAN_GPE, 559 MLX5_EXPANSION_GRE, 560 MLX5_EXPANSION_NVGRE, 561 MLX5_EXPANSION_GRE_KEY, 562 MLX5_EXPANSION_MPLS, 563 MLX5_EXPANSION_ETH, 564 MLX5_EXPANSION_VLAN, 565 MLX5_EXPANSION_IPV4, 566 MLX5_EXPANSION_IPV4_UDP, 567 MLX5_EXPANSION_IPV4_TCP, 568 MLX5_EXPANSION_IPV6, 569 MLX5_EXPANSION_IPV6_UDP, 570 MLX5_EXPANSION_IPV6_TCP, 571 MLX5_EXPANSION_IPV6_FRAG_EXT, 572 MLX5_EXPANSION_GTP, 573 MLX5_EXPANSION_GENEVE, 574 }; 575 576 /** Supported expansion of items. */ 577 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 578 [MLX5_EXPANSION_ROOT] = { 579 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 580 MLX5_EXPANSION_IPV4, 581 MLX5_EXPANSION_IPV6), 582 .type = RTE_FLOW_ITEM_TYPE_END, 583 }, 584 [MLX5_EXPANSION_ROOT_OUTER] = { 585 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 586 MLX5_EXPANSION_OUTER_IPV4, 587 MLX5_EXPANSION_OUTER_IPV6), 588 .type = RTE_FLOW_ITEM_TYPE_END, 589 }, 590 [MLX5_EXPANSION_OUTER_ETH] = { 591 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 592 .type = RTE_FLOW_ITEM_TYPE_ETH, 593 .rss_types = 0, 594 }, 595 [MLX5_EXPANSION_OUTER_VLAN] = { 596 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 597 MLX5_EXPANSION_OUTER_IPV6), 598 .type = RTE_FLOW_ITEM_TYPE_VLAN, 599 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 600 }, 601 [MLX5_EXPANSION_OUTER_IPV4] = { 602 .next = MLX5_FLOW_EXPAND_RSS_NEXT 603 (MLX5_EXPANSION_OUTER_IPV4_UDP, 604 MLX5_EXPANSION_OUTER_IPV4_TCP, 605 MLX5_EXPANSION_GRE, 606 MLX5_EXPANSION_NVGRE, 607 MLX5_EXPANSION_IPV4, 608 MLX5_EXPANSION_IPV6), 609 .type = RTE_FLOW_ITEM_TYPE_IPV4, 610 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 611 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 612 }, 613 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 614 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 615 MLX5_EXPANSION_VXLAN_GPE, 616 MLX5_EXPANSION_MPLS, 617 MLX5_EXPANSION_GENEVE, 618 MLX5_EXPANSION_GTP), 619 .type = RTE_FLOW_ITEM_TYPE_UDP, 620 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 621 }, 622 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 623 .type = RTE_FLOW_ITEM_TYPE_TCP, 624 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 625 }, 626 [MLX5_EXPANSION_OUTER_IPV6] = { 627 .next = MLX5_FLOW_EXPAND_RSS_NEXT 628 (MLX5_EXPANSION_OUTER_IPV6_UDP, 629 MLX5_EXPANSION_OUTER_IPV6_TCP, 630 MLX5_EXPANSION_IPV4, 631 MLX5_EXPANSION_IPV6, 632 MLX5_EXPANSION_GRE, 633 MLX5_EXPANSION_NVGRE), 634 .type = RTE_FLOW_ITEM_TYPE_IPV6, 635 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 636 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 637 }, 638 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 639 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 640 MLX5_EXPANSION_VXLAN_GPE, 641 MLX5_EXPANSION_MPLS, 642 MLX5_EXPANSION_GENEVE, 643 MLX5_EXPANSION_GTP), 644 .type = RTE_FLOW_ITEM_TYPE_UDP, 645 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 646 }, 647 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 648 .type = RTE_FLOW_ITEM_TYPE_TCP, 649 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 650 }, 651 [MLX5_EXPANSION_VXLAN] = { 652 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 653 MLX5_EXPANSION_IPV4, 654 MLX5_EXPANSION_IPV6), 655 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 656 }, 657 [MLX5_EXPANSION_STD_VXLAN] = { 658 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 659 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 660 }, 661 [MLX5_EXPANSION_L3_VXLAN] = { 662 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 663 MLX5_EXPANSION_IPV6), 664 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 665 }, 666 [MLX5_EXPANSION_VXLAN_GPE] = { 667 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 668 MLX5_EXPANSION_IPV4, 669 MLX5_EXPANSION_IPV6), 670 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 671 }, 672 [MLX5_EXPANSION_GRE] = { 673 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 674 MLX5_EXPANSION_IPV4, 675 MLX5_EXPANSION_IPV6, 676 MLX5_EXPANSION_GRE_KEY, 677 MLX5_EXPANSION_MPLS), 678 .type = RTE_FLOW_ITEM_TYPE_GRE, 679 }, 680 [MLX5_EXPANSION_GRE_KEY] = { 681 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 682 MLX5_EXPANSION_IPV6, 683 MLX5_EXPANSION_MPLS), 684 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 685 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 686 }, 687 [MLX5_EXPANSION_NVGRE] = { 688 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 689 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 690 }, 691 [MLX5_EXPANSION_MPLS] = { 692 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 693 MLX5_EXPANSION_IPV6, 694 MLX5_EXPANSION_ETH), 695 .type = RTE_FLOW_ITEM_TYPE_MPLS, 696 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 697 }, 698 [MLX5_EXPANSION_ETH] = { 699 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 700 .type = RTE_FLOW_ITEM_TYPE_ETH, 701 }, 702 [MLX5_EXPANSION_VLAN] = { 703 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 704 MLX5_EXPANSION_IPV6), 705 .type = RTE_FLOW_ITEM_TYPE_VLAN, 706 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 707 }, 708 [MLX5_EXPANSION_IPV4] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 710 MLX5_EXPANSION_IPV4_TCP), 711 .type = RTE_FLOW_ITEM_TYPE_IPV4, 712 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 713 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 714 }, 715 [MLX5_EXPANSION_IPV4_UDP] = { 716 .type = RTE_FLOW_ITEM_TYPE_UDP, 717 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 718 }, 719 [MLX5_EXPANSION_IPV4_TCP] = { 720 .type = RTE_FLOW_ITEM_TYPE_TCP, 721 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 722 }, 723 [MLX5_EXPANSION_IPV6] = { 724 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 725 MLX5_EXPANSION_IPV6_TCP, 726 MLX5_EXPANSION_IPV6_FRAG_EXT), 727 .type = RTE_FLOW_ITEM_TYPE_IPV6, 728 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 729 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 730 }, 731 [MLX5_EXPANSION_IPV6_UDP] = { 732 .type = RTE_FLOW_ITEM_TYPE_UDP, 733 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 734 }, 735 [MLX5_EXPANSION_IPV6_TCP] = { 736 .type = RTE_FLOW_ITEM_TYPE_TCP, 737 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 738 }, 739 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 740 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 741 }, 742 [MLX5_EXPANSION_GTP] = { 743 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_GTP, 746 }, 747 [MLX5_EXPANSION_GENEVE] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 749 MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 752 }, 753 }; 754 755 static struct rte_flow_action_handle * 756 mlx5_action_handle_create(struct rte_eth_dev *dev, 757 const struct rte_flow_indir_action_conf *conf, 758 const struct rte_flow_action *action, 759 struct rte_flow_error *error); 760 static int mlx5_action_handle_destroy 761 (struct rte_eth_dev *dev, 762 struct rte_flow_action_handle *handle, 763 struct rte_flow_error *error); 764 static int mlx5_action_handle_update 765 (struct rte_eth_dev *dev, 766 struct rte_flow_action_handle *handle, 767 const void *update, 768 struct rte_flow_error *error); 769 static int mlx5_action_handle_query 770 (struct rte_eth_dev *dev, 771 const struct rte_flow_action_handle *handle, 772 void *data, 773 struct rte_flow_error *error); 774 static int 775 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 776 struct rte_flow_tunnel *app_tunnel, 777 struct rte_flow_action **actions, 778 uint32_t *num_of_actions, 779 struct rte_flow_error *error); 780 static int 781 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 782 struct rte_flow_tunnel *app_tunnel, 783 struct rte_flow_item **items, 784 uint32_t *num_of_items, 785 struct rte_flow_error *error); 786 static int 787 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 788 struct rte_flow_item *pmd_items, 789 uint32_t num_items, struct rte_flow_error *err); 790 static int 791 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 792 struct rte_flow_action *pmd_actions, 793 uint32_t num_actions, 794 struct rte_flow_error *err); 795 static int 796 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 797 struct rte_mbuf *m, 798 struct rte_flow_restore_info *info, 799 struct rte_flow_error *err); 800 static struct rte_flow_item_flex_handle * 801 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 802 const struct rte_flow_item_flex_conf *conf, 803 struct rte_flow_error *error); 804 static int 805 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 806 const struct rte_flow_item_flex_handle *handle, 807 struct rte_flow_error *error); 808 static int 809 mlx5_flow_info_get(struct rte_eth_dev *dev, 810 struct rte_flow_port_info *port_info, 811 struct rte_flow_queue_info *queue_info, 812 struct rte_flow_error *error); 813 static int 814 mlx5_flow_port_configure(struct rte_eth_dev *dev, 815 const struct rte_flow_port_attr *port_attr, 816 uint16_t nb_queue, 817 const struct rte_flow_queue_attr *queue_attr[], 818 struct rte_flow_error *err); 819 820 static const struct rte_flow_ops mlx5_flow_ops = { 821 .validate = mlx5_flow_validate, 822 .create = mlx5_flow_create, 823 .destroy = mlx5_flow_destroy, 824 .flush = mlx5_flow_flush, 825 .isolate = mlx5_flow_isolate, 826 .query = mlx5_flow_query, 827 .dev_dump = mlx5_flow_dev_dump, 828 .get_aged_flows = mlx5_flow_get_aged_flows, 829 .action_handle_create = mlx5_action_handle_create, 830 .action_handle_destroy = mlx5_action_handle_destroy, 831 .action_handle_update = mlx5_action_handle_update, 832 .action_handle_query = mlx5_action_handle_query, 833 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 834 .tunnel_match = mlx5_flow_tunnel_match, 835 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 836 .tunnel_item_release = mlx5_flow_tunnel_item_release, 837 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 838 .flex_item_create = mlx5_flow_flex_item_create, 839 .flex_item_release = mlx5_flow_flex_item_release, 840 .info_get = mlx5_flow_info_get, 841 .configure = mlx5_flow_port_configure, 842 }; 843 844 /* Tunnel information. */ 845 struct mlx5_flow_tunnel_info { 846 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 847 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 848 }; 849 850 static struct mlx5_flow_tunnel_info tunnels_info[] = { 851 { 852 .tunnel = MLX5_FLOW_LAYER_VXLAN, 853 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 854 }, 855 { 856 .tunnel = MLX5_FLOW_LAYER_GENEVE, 857 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 858 }, 859 { 860 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 861 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 862 }, 863 { 864 .tunnel = MLX5_FLOW_LAYER_GRE, 865 .ptype = RTE_PTYPE_TUNNEL_GRE, 866 }, 867 { 868 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 869 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 870 }, 871 { 872 .tunnel = MLX5_FLOW_LAYER_MPLS, 873 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 874 }, 875 { 876 .tunnel = MLX5_FLOW_LAYER_NVGRE, 877 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 878 }, 879 { 880 .tunnel = MLX5_FLOW_LAYER_IPIP, 881 .ptype = RTE_PTYPE_TUNNEL_IP, 882 }, 883 { 884 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 885 .ptype = RTE_PTYPE_TUNNEL_IP, 886 }, 887 { 888 .tunnel = MLX5_FLOW_LAYER_GTP, 889 .ptype = RTE_PTYPE_TUNNEL_GTPU, 890 }, 891 }; 892 893 894 895 /** 896 * Translate tag ID to register. 897 * 898 * @param[in] dev 899 * Pointer to the Ethernet device structure. 900 * @param[in] feature 901 * The feature that request the register. 902 * @param[in] id 903 * The request register ID. 904 * @param[out] error 905 * Error description in case of any. 906 * 907 * @return 908 * The request register on success, a negative errno 909 * value otherwise and rte_errno is set. 910 */ 911 int 912 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 913 enum mlx5_feature_name feature, 914 uint32_t id, 915 struct rte_flow_error *error) 916 { 917 struct mlx5_priv *priv = dev->data->dev_private; 918 struct mlx5_sh_config *config = &priv->sh->config; 919 enum modify_reg start_reg; 920 bool skip_mtr_reg = false; 921 922 switch (feature) { 923 case MLX5_HAIRPIN_RX: 924 return REG_B; 925 case MLX5_HAIRPIN_TX: 926 return REG_A; 927 case MLX5_METADATA_RX: 928 switch (config->dv_xmeta_en) { 929 case MLX5_XMETA_MODE_LEGACY: 930 return REG_B; 931 case MLX5_XMETA_MODE_META16: 932 return REG_C_0; 933 case MLX5_XMETA_MODE_META32: 934 return REG_C_1; 935 } 936 break; 937 case MLX5_METADATA_TX: 938 return REG_A; 939 case MLX5_METADATA_FDB: 940 switch (config->dv_xmeta_en) { 941 case MLX5_XMETA_MODE_LEGACY: 942 return REG_NON; 943 case MLX5_XMETA_MODE_META16: 944 return REG_C_0; 945 case MLX5_XMETA_MODE_META32: 946 return REG_C_1; 947 } 948 break; 949 case MLX5_FLOW_MARK: 950 switch (config->dv_xmeta_en) { 951 case MLX5_XMETA_MODE_LEGACY: 952 return REG_NON; 953 case MLX5_XMETA_MODE_META16: 954 return REG_C_1; 955 case MLX5_XMETA_MODE_META32: 956 return REG_C_0; 957 } 958 break; 959 case MLX5_MTR_ID: 960 /* 961 * If meter color and meter id share one register, flow match 962 * should use the meter color register for match. 963 */ 964 if (priv->mtr_reg_share) 965 return priv->mtr_color_reg; 966 else 967 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 968 REG_C_3; 969 case MLX5_MTR_COLOR: 970 case MLX5_ASO_FLOW_HIT: 971 case MLX5_ASO_CONNTRACK: 972 case MLX5_SAMPLE_ID: 973 /* All features use the same REG_C. */ 974 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 975 return priv->mtr_color_reg; 976 case MLX5_COPY_MARK: 977 /* 978 * Metadata COPY_MARK register using is in meter suffix sub 979 * flow while with meter. It's safe to share the same register. 980 */ 981 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 982 case MLX5_APP_TAG: 983 /* 984 * If meter is enable, it will engage the register for color 985 * match and flow match. If meter color match is not using the 986 * REG_C_2, need to skip the REG_C_x be used by meter color 987 * match. 988 * If meter is disable, free to use all available registers. 989 */ 990 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 991 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 992 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 993 if (id > (uint32_t)(REG_C_7 - start_reg)) 994 return rte_flow_error_set(error, EINVAL, 995 RTE_FLOW_ERROR_TYPE_ITEM, 996 NULL, "invalid tag id"); 997 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 998 return rte_flow_error_set(error, ENOTSUP, 999 RTE_FLOW_ERROR_TYPE_ITEM, 1000 NULL, "unsupported tag id"); 1001 /* 1002 * This case means meter is using the REG_C_x great than 2. 1003 * Take care not to conflict with meter color REG_C_x. 1004 * If the available index REG_C_y >= REG_C_x, skip the 1005 * color register. 1006 */ 1007 if (skip_mtr_reg && priv->sh->flow_mreg_c 1008 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1009 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1010 return rte_flow_error_set(error, EINVAL, 1011 RTE_FLOW_ERROR_TYPE_ITEM, 1012 NULL, "invalid tag id"); 1013 if (priv->sh->flow_mreg_c 1014 [id + 1 + start_reg - REG_C_0] != REG_NON) 1015 return priv->sh->flow_mreg_c 1016 [id + 1 + start_reg - REG_C_0]; 1017 return rte_flow_error_set(error, ENOTSUP, 1018 RTE_FLOW_ERROR_TYPE_ITEM, 1019 NULL, "unsupported tag id"); 1020 } 1021 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1022 } 1023 MLX5_ASSERT(false); 1024 return rte_flow_error_set(error, EINVAL, 1025 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1026 NULL, "invalid feature name"); 1027 } 1028 1029 /** 1030 * Check extensive flow metadata register support. 1031 * 1032 * @param dev 1033 * Pointer to rte_eth_dev structure. 1034 * 1035 * @return 1036 * True if device supports extensive flow metadata register, otherwise false. 1037 */ 1038 bool 1039 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1040 { 1041 struct mlx5_priv *priv = dev->data->dev_private; 1042 1043 /* 1044 * Having available reg_c can be regarded inclusively as supporting 1045 * extensive flow metadata register, which could mean, 1046 * - metadata register copy action by modify header. 1047 * - 16 modify header actions is supported. 1048 * - reg_c's are preserved across different domain (FDB and NIC) on 1049 * packet loopback by flow lookup miss. 1050 */ 1051 return priv->sh->flow_mreg_c[2] != REG_NON; 1052 } 1053 1054 /** 1055 * Get the lowest priority. 1056 * 1057 * @param[in] dev 1058 * Pointer to the Ethernet device structure. 1059 * @param[in] attributes 1060 * Pointer to device flow rule attributes. 1061 * 1062 * @return 1063 * The value of lowest priority of flow. 1064 */ 1065 uint32_t 1066 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1067 const struct rte_flow_attr *attr) 1068 { 1069 struct mlx5_priv *priv = dev->data->dev_private; 1070 1071 if (!attr->group && !attr->transfer) 1072 return priv->sh->flow_max_priority - 2; 1073 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1074 } 1075 1076 /** 1077 * Calculate matcher priority of the flow. 1078 * 1079 * @param[in] dev 1080 * Pointer to the Ethernet device structure. 1081 * @param[in] attr 1082 * Pointer to device flow rule attributes. 1083 * @param[in] subpriority 1084 * The priority based on the items. 1085 * @param[in] external 1086 * Flow is user flow. 1087 * @return 1088 * The matcher priority of the flow. 1089 */ 1090 uint16_t 1091 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1092 const struct rte_flow_attr *attr, 1093 uint32_t subpriority, bool external) 1094 { 1095 uint16_t priority = (uint16_t)attr->priority; 1096 struct mlx5_priv *priv = dev->data->dev_private; 1097 1098 if (!attr->group && !attr->transfer) { 1099 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1100 priority = priv->sh->flow_max_priority - 1; 1101 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1102 } else if (!external && attr->transfer && attr->group == 0 && 1103 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1104 return (priv->sh->flow_max_priority - 1) * 3; 1105 } 1106 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1107 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1108 return priority * 3 + subpriority; 1109 } 1110 1111 /** 1112 * Verify the @p item specifications (spec, last, mask) are compatible with the 1113 * NIC capabilities. 1114 * 1115 * @param[in] item 1116 * Item specification. 1117 * @param[in] mask 1118 * @p item->mask or flow default bit-masks. 1119 * @param[in] nic_mask 1120 * Bit-masks covering supported fields by the NIC to compare with user mask. 1121 * @param[in] size 1122 * Bit-masks size in bytes. 1123 * @param[in] range_accepted 1124 * True if range of values is accepted for specific fields, false otherwise. 1125 * @param[out] error 1126 * Pointer to error structure. 1127 * 1128 * @return 1129 * 0 on success, a negative errno value otherwise and rte_errno is set. 1130 */ 1131 int 1132 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1133 const uint8_t *mask, 1134 const uint8_t *nic_mask, 1135 unsigned int size, 1136 bool range_accepted, 1137 struct rte_flow_error *error) 1138 { 1139 unsigned int i; 1140 1141 MLX5_ASSERT(nic_mask); 1142 for (i = 0; i < size; ++i) 1143 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1144 return rte_flow_error_set(error, ENOTSUP, 1145 RTE_FLOW_ERROR_TYPE_ITEM, 1146 item, 1147 "mask enables non supported" 1148 " bits"); 1149 if (!item->spec && (item->mask || item->last)) 1150 return rte_flow_error_set(error, EINVAL, 1151 RTE_FLOW_ERROR_TYPE_ITEM, item, 1152 "mask/last without a spec is not" 1153 " supported"); 1154 if (item->spec && item->last && !range_accepted) { 1155 uint8_t spec[size]; 1156 uint8_t last[size]; 1157 unsigned int i; 1158 int ret; 1159 1160 for (i = 0; i < size; ++i) { 1161 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1162 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1163 } 1164 ret = memcmp(spec, last, size); 1165 if (ret != 0) 1166 return rte_flow_error_set(error, EINVAL, 1167 RTE_FLOW_ERROR_TYPE_ITEM, 1168 item, 1169 "range is not valid"); 1170 } 1171 return 0; 1172 } 1173 1174 /** 1175 * Adjust the hash fields according to the @p flow information. 1176 * 1177 * @param[in] dev_flow. 1178 * Pointer to the mlx5_flow. 1179 * @param[in] tunnel 1180 * 1 when the hash field is for a tunnel item. 1181 * @param[in] layer_types 1182 * RTE_ETH_RSS_* types. 1183 * @param[in] hash_fields 1184 * Item hash fields. 1185 * 1186 * @return 1187 * The hash fields that should be used. 1188 */ 1189 uint64_t 1190 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1191 int tunnel __rte_unused, uint64_t layer_types, 1192 uint64_t hash_fields) 1193 { 1194 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1195 int rss_request_inner = rss_desc->level >= 2; 1196 1197 /* Check RSS hash level for tunnel. */ 1198 if (tunnel && rss_request_inner) 1199 hash_fields |= IBV_RX_HASH_INNER; 1200 else if (tunnel || rss_request_inner) 1201 return 0; 1202 #endif 1203 /* Check if requested layer matches RSS hash fields. */ 1204 if (!(rss_desc->types & layer_types)) 1205 return 0; 1206 return hash_fields; 1207 } 1208 1209 /** 1210 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1211 * if several tunnel rules are used on this queue, the tunnel ptype will be 1212 * cleared. 1213 * 1214 * @param rxq_ctrl 1215 * Rx queue to update. 1216 */ 1217 static void 1218 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1219 { 1220 unsigned int i; 1221 uint32_t tunnel_ptype = 0; 1222 1223 /* Look up for the ptype to use. */ 1224 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1225 if (!rxq_ctrl->flow_tunnels_n[i]) 1226 continue; 1227 if (!tunnel_ptype) { 1228 tunnel_ptype = tunnels_info[i].ptype; 1229 } else { 1230 tunnel_ptype = 0; 1231 break; 1232 } 1233 } 1234 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1235 } 1236 1237 /** 1238 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1239 * flow. 1240 * 1241 * @param[in] dev 1242 * Pointer to the Ethernet device structure. 1243 * @param[in] dev_handle 1244 * Pointer to device flow handle structure. 1245 */ 1246 void 1247 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1248 struct mlx5_flow_handle *dev_handle) 1249 { 1250 struct mlx5_priv *priv = dev->data->dev_private; 1251 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1252 struct mlx5_ind_table_obj *ind_tbl = NULL; 1253 unsigned int i; 1254 1255 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1256 struct mlx5_hrxq *hrxq; 1257 1258 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1259 dev_handle->rix_hrxq); 1260 if (hrxq) 1261 ind_tbl = hrxq->ind_table; 1262 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1263 struct mlx5_shared_action_rss *shared_rss; 1264 1265 shared_rss = mlx5_ipool_get 1266 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1267 dev_handle->rix_srss); 1268 if (shared_rss) 1269 ind_tbl = shared_rss->ind_tbl; 1270 } 1271 if (!ind_tbl) 1272 return; 1273 for (i = 0; i != ind_tbl->queues_n; ++i) { 1274 int idx = ind_tbl->queues[i]; 1275 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1276 1277 MLX5_ASSERT(rxq_ctrl != NULL); 1278 if (rxq_ctrl == NULL) 1279 continue; 1280 /* 1281 * To support metadata register copy on Tx loopback, 1282 * this must be always enabled (metadata may arive 1283 * from other port - not from local flows only. 1284 */ 1285 if (tunnel) { 1286 unsigned int j; 1287 1288 /* Increase the counter matching the flow. */ 1289 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1290 if ((tunnels_info[j].tunnel & 1291 dev_handle->layers) == 1292 tunnels_info[j].tunnel) { 1293 rxq_ctrl->flow_tunnels_n[j]++; 1294 break; 1295 } 1296 } 1297 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1298 } 1299 } 1300 } 1301 1302 static void 1303 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1304 { 1305 struct mlx5_priv *priv = dev->data->dev_private; 1306 struct mlx5_rxq_ctrl *rxq_ctrl; 1307 1308 if (priv->mark_enabled) 1309 return; 1310 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1311 rxq_ctrl->rxq.mark = 1; 1312 } 1313 priv->mark_enabled = 1; 1314 } 1315 1316 /** 1317 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1318 * 1319 * @param[in] dev 1320 * Pointer to the Ethernet device structure. 1321 * @param[in] flow 1322 * Pointer to flow structure. 1323 */ 1324 static void 1325 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1326 { 1327 struct mlx5_priv *priv = dev->data->dev_private; 1328 uint32_t handle_idx; 1329 struct mlx5_flow_handle *dev_handle; 1330 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1331 1332 MLX5_ASSERT(wks); 1333 if (wks->mark) 1334 flow_rxq_mark_flag_set(dev); 1335 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1336 handle_idx, dev_handle, next) 1337 flow_drv_rxq_flags_set(dev, dev_handle); 1338 } 1339 1340 /** 1341 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1342 * device flow if no other flow uses it with the same kind of request. 1343 * 1344 * @param dev 1345 * Pointer to Ethernet device. 1346 * @param[in] dev_handle 1347 * Pointer to the device flow handle structure. 1348 */ 1349 static void 1350 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1351 struct mlx5_flow_handle *dev_handle) 1352 { 1353 struct mlx5_priv *priv = dev->data->dev_private; 1354 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1355 struct mlx5_ind_table_obj *ind_tbl = NULL; 1356 unsigned int i; 1357 1358 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1359 struct mlx5_hrxq *hrxq; 1360 1361 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1362 dev_handle->rix_hrxq); 1363 if (hrxq) 1364 ind_tbl = hrxq->ind_table; 1365 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1366 struct mlx5_shared_action_rss *shared_rss; 1367 1368 shared_rss = mlx5_ipool_get 1369 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1370 dev_handle->rix_srss); 1371 if (shared_rss) 1372 ind_tbl = shared_rss->ind_tbl; 1373 } 1374 if (!ind_tbl) 1375 return; 1376 MLX5_ASSERT(dev->data->dev_started); 1377 for (i = 0; i != ind_tbl->queues_n; ++i) { 1378 int idx = ind_tbl->queues[i]; 1379 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1380 1381 MLX5_ASSERT(rxq_ctrl != NULL); 1382 if (rxq_ctrl == NULL) 1383 continue; 1384 if (tunnel) { 1385 unsigned int j; 1386 1387 /* Decrease the counter matching the flow. */ 1388 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1389 if ((tunnels_info[j].tunnel & 1390 dev_handle->layers) == 1391 tunnels_info[j].tunnel) { 1392 rxq_ctrl->flow_tunnels_n[j]--; 1393 break; 1394 } 1395 } 1396 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1397 } 1398 } 1399 } 1400 1401 /** 1402 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1403 * @p flow if no other flow uses it with the same kind of request. 1404 * 1405 * @param dev 1406 * Pointer to Ethernet device. 1407 * @param[in] flow 1408 * Pointer to the flow. 1409 */ 1410 static void 1411 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1412 { 1413 struct mlx5_priv *priv = dev->data->dev_private; 1414 uint32_t handle_idx; 1415 struct mlx5_flow_handle *dev_handle; 1416 1417 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1418 handle_idx, dev_handle, next) 1419 flow_drv_rxq_flags_trim(dev, dev_handle); 1420 } 1421 1422 /** 1423 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1424 * 1425 * @param dev 1426 * Pointer to Ethernet device. 1427 */ 1428 static void 1429 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1430 { 1431 struct mlx5_priv *priv = dev->data->dev_private; 1432 unsigned int i; 1433 1434 for (i = 0; i != priv->rxqs_n; ++i) { 1435 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1436 unsigned int j; 1437 1438 if (rxq == NULL || rxq->ctrl == NULL) 1439 continue; 1440 rxq->ctrl->rxq.mark = 0; 1441 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1442 rxq->ctrl->flow_tunnels_n[j] = 0; 1443 rxq->ctrl->rxq.tunnel = 0; 1444 } 1445 priv->mark_enabled = 0; 1446 } 1447 1448 /** 1449 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1450 * 1451 * @param[in] dev 1452 * Pointer to the Ethernet device structure. 1453 */ 1454 void 1455 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1456 { 1457 struct mlx5_priv *priv = dev->data->dev_private; 1458 unsigned int i; 1459 1460 for (i = 0; i != priv->rxqs_n; ++i) { 1461 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1462 struct mlx5_rxq_data *data; 1463 1464 if (rxq == NULL || rxq->ctrl == NULL) 1465 continue; 1466 data = &rxq->ctrl->rxq; 1467 if (!rte_flow_dynf_metadata_avail()) { 1468 data->dynf_meta = 0; 1469 data->flow_meta_mask = 0; 1470 data->flow_meta_offset = -1; 1471 data->flow_meta_port_mask = 0; 1472 } else { 1473 data->dynf_meta = 1; 1474 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1475 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1476 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1477 } 1478 } 1479 } 1480 1481 /* 1482 * return a pointer to the desired action in the list of actions. 1483 * 1484 * @param[in] actions 1485 * The list of actions to search the action in. 1486 * @param[in] action 1487 * The action to find. 1488 * 1489 * @return 1490 * Pointer to the action in the list, if found. NULL otherwise. 1491 */ 1492 const struct rte_flow_action * 1493 mlx5_flow_find_action(const struct rte_flow_action *actions, 1494 enum rte_flow_action_type action) 1495 { 1496 if (actions == NULL) 1497 return NULL; 1498 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1499 if (actions->type == action) 1500 return actions; 1501 return NULL; 1502 } 1503 1504 /* 1505 * Validate the flag action. 1506 * 1507 * @param[in] action_flags 1508 * Bit-fields that holds the actions detected until now. 1509 * @param[in] attr 1510 * Attributes of flow that includes this action. 1511 * @param[out] error 1512 * Pointer to error structure. 1513 * 1514 * @return 1515 * 0 on success, a negative errno value otherwise and rte_errno is set. 1516 */ 1517 int 1518 mlx5_flow_validate_action_flag(uint64_t action_flags, 1519 const struct rte_flow_attr *attr, 1520 struct rte_flow_error *error) 1521 { 1522 if (action_flags & MLX5_FLOW_ACTION_MARK) 1523 return rte_flow_error_set(error, EINVAL, 1524 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1525 "can't mark and flag in same flow"); 1526 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1527 return rte_flow_error_set(error, EINVAL, 1528 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1529 "can't have 2 flag" 1530 " actions in same flow"); 1531 if (attr->egress) 1532 return rte_flow_error_set(error, ENOTSUP, 1533 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1534 "flag action not supported for " 1535 "egress"); 1536 return 0; 1537 } 1538 1539 /* 1540 * Validate the mark action. 1541 * 1542 * @param[in] action 1543 * Pointer to the queue action. 1544 * @param[in] action_flags 1545 * Bit-fields that holds the actions detected until now. 1546 * @param[in] attr 1547 * Attributes of flow that includes this action. 1548 * @param[out] error 1549 * Pointer to error structure. 1550 * 1551 * @return 1552 * 0 on success, a negative errno value otherwise and rte_errno is set. 1553 */ 1554 int 1555 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1556 uint64_t action_flags, 1557 const struct rte_flow_attr *attr, 1558 struct rte_flow_error *error) 1559 { 1560 const struct rte_flow_action_mark *mark = action->conf; 1561 1562 if (!mark) 1563 return rte_flow_error_set(error, EINVAL, 1564 RTE_FLOW_ERROR_TYPE_ACTION, 1565 action, 1566 "configuration cannot be null"); 1567 if (mark->id >= MLX5_FLOW_MARK_MAX) 1568 return rte_flow_error_set(error, EINVAL, 1569 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1570 &mark->id, 1571 "mark id must in 0 <= id < " 1572 RTE_STR(MLX5_FLOW_MARK_MAX)); 1573 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1574 return rte_flow_error_set(error, EINVAL, 1575 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1576 "can't flag and mark in same flow"); 1577 if (action_flags & MLX5_FLOW_ACTION_MARK) 1578 return rte_flow_error_set(error, EINVAL, 1579 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1580 "can't have 2 mark actions in same" 1581 " flow"); 1582 if (attr->egress) 1583 return rte_flow_error_set(error, ENOTSUP, 1584 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1585 "mark action not supported for " 1586 "egress"); 1587 return 0; 1588 } 1589 1590 /* 1591 * Validate the drop action. 1592 * 1593 * @param[in] action_flags 1594 * Bit-fields that holds the actions detected until now. 1595 * @param[in] attr 1596 * Attributes of flow that includes this action. 1597 * @param[out] error 1598 * Pointer to error structure. 1599 * 1600 * @return 1601 * 0 on success, a negative errno value otherwise and rte_errno is set. 1602 */ 1603 int 1604 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1605 const struct rte_flow_attr *attr, 1606 struct rte_flow_error *error) 1607 { 1608 if (attr->egress) 1609 return rte_flow_error_set(error, ENOTSUP, 1610 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1611 "drop action not supported for " 1612 "egress"); 1613 return 0; 1614 } 1615 1616 /* 1617 * Validate the queue action. 1618 * 1619 * @param[in] action 1620 * Pointer to the queue action. 1621 * @param[in] action_flags 1622 * Bit-fields that holds the actions detected until now. 1623 * @param[in] dev 1624 * Pointer to the Ethernet device structure. 1625 * @param[in] attr 1626 * Attributes of flow that includes this action. 1627 * @param[out] error 1628 * Pointer to error structure. 1629 * 1630 * @return 1631 * 0 on success, a negative errno value otherwise and rte_errno is set. 1632 */ 1633 int 1634 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1635 uint64_t action_flags, 1636 struct rte_eth_dev *dev, 1637 const struct rte_flow_attr *attr, 1638 struct rte_flow_error *error) 1639 { 1640 struct mlx5_priv *priv = dev->data->dev_private; 1641 const struct rte_flow_action_queue *queue = action->conf; 1642 1643 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1644 return rte_flow_error_set(error, EINVAL, 1645 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1646 "can't have 2 fate actions in" 1647 " same flow"); 1648 if (!priv->rxqs_n) 1649 return rte_flow_error_set(error, EINVAL, 1650 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1651 NULL, "No Rx queues configured"); 1652 if (queue->index >= priv->rxqs_n) 1653 return rte_flow_error_set(error, EINVAL, 1654 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1655 &queue->index, 1656 "queue index out of range"); 1657 if (mlx5_rxq_get(dev, queue->index) == NULL) 1658 return rte_flow_error_set(error, EINVAL, 1659 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1660 &queue->index, 1661 "queue is not configured"); 1662 if (attr->egress) 1663 return rte_flow_error_set(error, ENOTSUP, 1664 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1665 "queue action not supported for " 1666 "egress"); 1667 return 0; 1668 } 1669 1670 /** 1671 * Validate queue numbers for device RSS. 1672 * 1673 * @param[in] dev 1674 * Configured device. 1675 * @param[in] queues 1676 * Array of queue numbers. 1677 * @param[in] queues_n 1678 * Size of the @p queues array. 1679 * @param[out] error 1680 * On error, filled with a textual error description. 1681 * @param[out] queue 1682 * On error, filled with an offending queue index in @p queues array. 1683 * 1684 * @return 1685 * 0 on success, a negative errno code on error. 1686 */ 1687 static int 1688 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1689 const uint16_t *queues, uint32_t queues_n, 1690 const char **error, uint32_t *queue_idx) 1691 { 1692 const struct mlx5_priv *priv = dev->data->dev_private; 1693 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1694 uint32_t i; 1695 1696 for (i = 0; i != queues_n; ++i) { 1697 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, 1698 queues[i]); 1699 1700 if (queues[i] >= priv->rxqs_n) { 1701 *error = "queue index out of range"; 1702 *queue_idx = i; 1703 return -EINVAL; 1704 } 1705 if (rxq_ctrl == NULL) { 1706 *error = "queue is not configured"; 1707 *queue_idx = i; 1708 return -EINVAL; 1709 } 1710 if (i == 0) 1711 rxq_type = rxq_ctrl->type; 1712 if (rxq_type != rxq_ctrl->type) { 1713 *error = "combining hairpin and regular RSS queues is not supported"; 1714 *queue_idx = i; 1715 return -ENOTSUP; 1716 } 1717 } 1718 return 0; 1719 } 1720 1721 /* 1722 * Validate the rss action. 1723 * 1724 * @param[in] dev 1725 * Pointer to the Ethernet device structure. 1726 * @param[in] action 1727 * Pointer to the queue action. 1728 * @param[out] error 1729 * Pointer to error structure. 1730 * 1731 * @return 1732 * 0 on success, a negative errno value otherwise and rte_errno is set. 1733 */ 1734 int 1735 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1736 const struct rte_flow_action *action, 1737 struct rte_flow_error *error) 1738 { 1739 struct mlx5_priv *priv = dev->data->dev_private; 1740 const struct rte_flow_action_rss *rss = action->conf; 1741 int ret; 1742 const char *message; 1743 uint32_t queue_idx; 1744 1745 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1746 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1747 return rte_flow_error_set(error, ENOTSUP, 1748 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1749 &rss->func, 1750 "RSS hash function not supported"); 1751 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1752 if (rss->level > 2) 1753 #else 1754 if (rss->level > 1) 1755 #endif 1756 return rte_flow_error_set(error, ENOTSUP, 1757 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1758 &rss->level, 1759 "tunnel RSS is not supported"); 1760 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1761 if (rss->key_len == 0 && rss->key != NULL) 1762 return rte_flow_error_set(error, ENOTSUP, 1763 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1764 &rss->key_len, 1765 "RSS hash key length 0"); 1766 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1767 return rte_flow_error_set(error, ENOTSUP, 1768 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1769 &rss->key_len, 1770 "RSS hash key too small"); 1771 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1772 return rte_flow_error_set(error, ENOTSUP, 1773 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1774 &rss->key_len, 1775 "RSS hash key too large"); 1776 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1777 return rte_flow_error_set(error, ENOTSUP, 1778 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1779 &rss->queue_num, 1780 "number of queues too large"); 1781 if (rss->types & MLX5_RSS_HF_MASK) 1782 return rte_flow_error_set(error, ENOTSUP, 1783 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1784 &rss->types, 1785 "some RSS protocols are not" 1786 " supported"); 1787 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1788 !(rss->types & RTE_ETH_RSS_IP)) 1789 return rte_flow_error_set(error, EINVAL, 1790 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1791 "L3 partial RSS requested but L3 RSS" 1792 " type not specified"); 1793 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1794 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1795 return rte_flow_error_set(error, EINVAL, 1796 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1797 "L4 partial RSS requested but L4 RSS" 1798 " type not specified"); 1799 if (!priv->rxqs_n) 1800 return rte_flow_error_set(error, EINVAL, 1801 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1802 NULL, "No Rx queues configured"); 1803 if (!rss->queue_num) 1804 return rte_flow_error_set(error, EINVAL, 1805 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1806 NULL, "No queues configured"); 1807 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1808 &message, &queue_idx); 1809 if (ret != 0) { 1810 return rte_flow_error_set(error, -ret, 1811 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1812 &rss->queue[queue_idx], message); 1813 } 1814 return 0; 1815 } 1816 1817 /* 1818 * Validate the rss action. 1819 * 1820 * @param[in] action 1821 * Pointer to the queue action. 1822 * @param[in] action_flags 1823 * Bit-fields that holds the actions detected until now. 1824 * @param[in] dev 1825 * Pointer to the Ethernet device structure. 1826 * @param[in] attr 1827 * Attributes of flow that includes this action. 1828 * @param[in] item_flags 1829 * Items that were detected. 1830 * @param[out] error 1831 * Pointer to error structure. 1832 * 1833 * @return 1834 * 0 on success, a negative errno value otherwise and rte_errno is set. 1835 */ 1836 int 1837 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1838 uint64_t action_flags, 1839 struct rte_eth_dev *dev, 1840 const struct rte_flow_attr *attr, 1841 uint64_t item_flags, 1842 struct rte_flow_error *error) 1843 { 1844 const struct rte_flow_action_rss *rss = action->conf; 1845 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1846 int ret; 1847 1848 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1849 return rte_flow_error_set(error, EINVAL, 1850 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1851 "can't have 2 fate actions" 1852 " in same flow"); 1853 ret = mlx5_validate_action_rss(dev, action, error); 1854 if (ret) 1855 return ret; 1856 if (attr->egress) 1857 return rte_flow_error_set(error, ENOTSUP, 1858 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1859 "rss action not supported for " 1860 "egress"); 1861 if (rss->level > 1 && !tunnel) 1862 return rte_flow_error_set(error, EINVAL, 1863 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1864 "inner RSS is not supported for " 1865 "non-tunnel flows"); 1866 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1867 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1868 return rte_flow_error_set(error, EINVAL, 1869 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1870 "RSS on eCPRI is not supported now"); 1871 } 1872 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1873 !(item_flags & 1874 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1875 rss->level > 1) 1876 return rte_flow_error_set(error, EINVAL, 1877 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1878 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1879 return 0; 1880 } 1881 1882 /* 1883 * Validate the default miss action. 1884 * 1885 * @param[in] action_flags 1886 * Bit-fields that holds the actions detected until now. 1887 * @param[out] error 1888 * Pointer to error structure. 1889 * 1890 * @return 1891 * 0 on success, a negative errno value otherwise and rte_errno is set. 1892 */ 1893 int 1894 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1895 const struct rte_flow_attr *attr, 1896 struct rte_flow_error *error) 1897 { 1898 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1899 return rte_flow_error_set(error, EINVAL, 1900 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1901 "can't have 2 fate actions in" 1902 " same flow"); 1903 if (attr->egress) 1904 return rte_flow_error_set(error, ENOTSUP, 1905 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1906 "default miss action not supported " 1907 "for egress"); 1908 if (attr->group) 1909 return rte_flow_error_set(error, ENOTSUP, 1910 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1911 "only group 0 is supported"); 1912 if (attr->transfer) 1913 return rte_flow_error_set(error, ENOTSUP, 1914 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1915 NULL, "transfer is not supported"); 1916 return 0; 1917 } 1918 1919 /* 1920 * Validate the count action. 1921 * 1922 * @param[in] dev 1923 * Pointer to the Ethernet device structure. 1924 * @param[in] attr 1925 * Attributes of flow that includes this action. 1926 * @param[out] error 1927 * Pointer to error structure. 1928 * 1929 * @return 1930 * 0 on success, a negative errno value otherwise and rte_errno is set. 1931 */ 1932 int 1933 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1934 const struct rte_flow_attr *attr, 1935 struct rte_flow_error *error) 1936 { 1937 if (attr->egress) 1938 return rte_flow_error_set(error, ENOTSUP, 1939 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1940 "count action not supported for " 1941 "egress"); 1942 return 0; 1943 } 1944 1945 /* 1946 * Validate the ASO CT action. 1947 * 1948 * @param[in] dev 1949 * Pointer to the Ethernet device structure. 1950 * @param[in] conntrack 1951 * Pointer to the CT action profile. 1952 * @param[out] error 1953 * Pointer to error structure. 1954 * 1955 * @return 1956 * 0 on success, a negative errno value otherwise and rte_errno is set. 1957 */ 1958 int 1959 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1960 const struct rte_flow_action_conntrack *conntrack, 1961 struct rte_flow_error *error) 1962 { 1963 RTE_SET_USED(dev); 1964 1965 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1966 return rte_flow_error_set(error, EINVAL, 1967 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1968 "Invalid CT state"); 1969 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1970 return rte_flow_error_set(error, EINVAL, 1971 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1972 "Invalid last TCP packet flag"); 1973 return 0; 1974 } 1975 1976 /** 1977 * Verify the @p attributes will be correctly understood by the NIC and store 1978 * them in the @p flow if everything is correct. 1979 * 1980 * @param[in] dev 1981 * Pointer to the Ethernet device structure. 1982 * @param[in] attributes 1983 * Pointer to flow attributes 1984 * @param[out] error 1985 * Pointer to error structure. 1986 * 1987 * @return 1988 * 0 on success, a negative errno value otherwise and rte_errno is set. 1989 */ 1990 int 1991 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1992 const struct rte_flow_attr *attributes, 1993 struct rte_flow_error *error) 1994 { 1995 struct mlx5_priv *priv = dev->data->dev_private; 1996 uint32_t priority_max = priv->sh->flow_max_priority - 1; 1997 1998 if (attributes->group) 1999 return rte_flow_error_set(error, ENOTSUP, 2000 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2001 NULL, "groups is not supported"); 2002 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2003 attributes->priority >= priority_max) 2004 return rte_flow_error_set(error, ENOTSUP, 2005 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2006 NULL, "priority out of range"); 2007 if (attributes->egress) 2008 return rte_flow_error_set(error, ENOTSUP, 2009 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2010 "egress is not supported"); 2011 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2012 return rte_flow_error_set(error, ENOTSUP, 2013 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2014 NULL, "transfer is not supported"); 2015 if (!attributes->ingress) 2016 return rte_flow_error_set(error, EINVAL, 2017 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2018 NULL, 2019 "ingress attribute is mandatory"); 2020 return 0; 2021 } 2022 2023 /** 2024 * Validate ICMP6 item. 2025 * 2026 * @param[in] item 2027 * Item specification. 2028 * @param[in] item_flags 2029 * Bit-fields that holds the items detected until now. 2030 * @param[in] ext_vlan_sup 2031 * Whether extended VLAN features are supported or not. 2032 * @param[out] error 2033 * Pointer to error structure. 2034 * 2035 * @return 2036 * 0 on success, a negative errno value otherwise and rte_errno is set. 2037 */ 2038 int 2039 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2040 uint64_t item_flags, 2041 uint8_t target_protocol, 2042 struct rte_flow_error *error) 2043 { 2044 const struct rte_flow_item_icmp6 *mask = item->mask; 2045 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2046 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2047 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2048 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2049 MLX5_FLOW_LAYER_OUTER_L4; 2050 int ret; 2051 2052 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2053 return rte_flow_error_set(error, EINVAL, 2054 RTE_FLOW_ERROR_TYPE_ITEM, item, 2055 "protocol filtering not compatible" 2056 " with ICMP6 layer"); 2057 if (!(item_flags & l3m)) 2058 return rte_flow_error_set(error, EINVAL, 2059 RTE_FLOW_ERROR_TYPE_ITEM, item, 2060 "IPv6 is mandatory to filter on" 2061 " ICMP6"); 2062 if (item_flags & l4m) 2063 return rte_flow_error_set(error, EINVAL, 2064 RTE_FLOW_ERROR_TYPE_ITEM, item, 2065 "multiple L4 layers not supported"); 2066 if (!mask) 2067 mask = &rte_flow_item_icmp6_mask; 2068 ret = mlx5_flow_item_acceptable 2069 (item, (const uint8_t *)mask, 2070 (const uint8_t *)&rte_flow_item_icmp6_mask, 2071 sizeof(struct rte_flow_item_icmp6), 2072 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2073 if (ret < 0) 2074 return ret; 2075 return 0; 2076 } 2077 2078 /** 2079 * Validate ICMP item. 2080 * 2081 * @param[in] item 2082 * Item specification. 2083 * @param[in] item_flags 2084 * Bit-fields that holds the items detected until now. 2085 * @param[out] error 2086 * Pointer to error structure. 2087 * 2088 * @return 2089 * 0 on success, a negative errno value otherwise and rte_errno is set. 2090 */ 2091 int 2092 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2093 uint64_t item_flags, 2094 uint8_t target_protocol, 2095 struct rte_flow_error *error) 2096 { 2097 const struct rte_flow_item_icmp *mask = item->mask; 2098 const struct rte_flow_item_icmp nic_mask = { 2099 .hdr.icmp_type = 0xff, 2100 .hdr.icmp_code = 0xff, 2101 .hdr.icmp_ident = RTE_BE16(0xffff), 2102 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2103 }; 2104 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2105 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2106 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2107 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2108 MLX5_FLOW_LAYER_OUTER_L4; 2109 int ret; 2110 2111 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2112 return rte_flow_error_set(error, EINVAL, 2113 RTE_FLOW_ERROR_TYPE_ITEM, item, 2114 "protocol filtering not compatible" 2115 " with ICMP layer"); 2116 if (!(item_flags & l3m)) 2117 return rte_flow_error_set(error, EINVAL, 2118 RTE_FLOW_ERROR_TYPE_ITEM, item, 2119 "IPv4 is mandatory to filter" 2120 " on ICMP"); 2121 if (item_flags & l4m) 2122 return rte_flow_error_set(error, EINVAL, 2123 RTE_FLOW_ERROR_TYPE_ITEM, item, 2124 "multiple L4 layers not supported"); 2125 if (!mask) 2126 mask = &nic_mask; 2127 ret = mlx5_flow_item_acceptable 2128 (item, (const uint8_t *)mask, 2129 (const uint8_t *)&nic_mask, 2130 sizeof(struct rte_flow_item_icmp), 2131 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2132 if (ret < 0) 2133 return ret; 2134 return 0; 2135 } 2136 2137 /** 2138 * Validate Ethernet item. 2139 * 2140 * @param[in] item 2141 * Item specification. 2142 * @param[in] item_flags 2143 * Bit-fields that holds the items detected until now. 2144 * @param[out] error 2145 * Pointer to error structure. 2146 * 2147 * @return 2148 * 0 on success, a negative errno value otherwise and rte_errno is set. 2149 */ 2150 int 2151 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2152 uint64_t item_flags, bool ext_vlan_sup, 2153 struct rte_flow_error *error) 2154 { 2155 const struct rte_flow_item_eth *mask = item->mask; 2156 const struct rte_flow_item_eth nic_mask = { 2157 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2158 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2159 .type = RTE_BE16(0xffff), 2160 .has_vlan = ext_vlan_sup ? 1 : 0, 2161 }; 2162 int ret; 2163 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2164 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2165 MLX5_FLOW_LAYER_OUTER_L2; 2166 2167 if (item_flags & ethm) 2168 return rte_flow_error_set(error, ENOTSUP, 2169 RTE_FLOW_ERROR_TYPE_ITEM, item, 2170 "multiple L2 layers not supported"); 2171 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2172 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2173 return rte_flow_error_set(error, EINVAL, 2174 RTE_FLOW_ERROR_TYPE_ITEM, item, 2175 "L2 layer should not follow " 2176 "L3 layers"); 2177 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2178 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2179 return rte_flow_error_set(error, EINVAL, 2180 RTE_FLOW_ERROR_TYPE_ITEM, item, 2181 "L2 layer should not follow VLAN"); 2182 if (item_flags & MLX5_FLOW_LAYER_GTP) 2183 return rte_flow_error_set(error, EINVAL, 2184 RTE_FLOW_ERROR_TYPE_ITEM, item, 2185 "L2 layer should not follow GTP"); 2186 if (!mask) 2187 mask = &rte_flow_item_eth_mask; 2188 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2189 (const uint8_t *)&nic_mask, 2190 sizeof(struct rte_flow_item_eth), 2191 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2192 return ret; 2193 } 2194 2195 /** 2196 * Validate VLAN item. 2197 * 2198 * @param[in] item 2199 * Item specification. 2200 * @param[in] item_flags 2201 * Bit-fields that holds the items detected until now. 2202 * @param[in] dev 2203 * Ethernet device flow is being created on. 2204 * @param[out] error 2205 * Pointer to error structure. 2206 * 2207 * @return 2208 * 0 on success, a negative errno value otherwise and rte_errno is set. 2209 */ 2210 int 2211 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2212 uint64_t item_flags, 2213 struct rte_eth_dev *dev, 2214 struct rte_flow_error *error) 2215 { 2216 const struct rte_flow_item_vlan *spec = item->spec; 2217 const struct rte_flow_item_vlan *mask = item->mask; 2218 const struct rte_flow_item_vlan nic_mask = { 2219 .tci = RTE_BE16(UINT16_MAX), 2220 .inner_type = RTE_BE16(UINT16_MAX), 2221 }; 2222 uint16_t vlan_tag = 0; 2223 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2224 int ret; 2225 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2226 MLX5_FLOW_LAYER_INNER_L4) : 2227 (MLX5_FLOW_LAYER_OUTER_L3 | 2228 MLX5_FLOW_LAYER_OUTER_L4); 2229 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2230 MLX5_FLOW_LAYER_OUTER_VLAN; 2231 2232 if (item_flags & vlanm) 2233 return rte_flow_error_set(error, EINVAL, 2234 RTE_FLOW_ERROR_TYPE_ITEM, item, 2235 "multiple VLAN layers not supported"); 2236 else if ((item_flags & l34m) != 0) 2237 return rte_flow_error_set(error, EINVAL, 2238 RTE_FLOW_ERROR_TYPE_ITEM, item, 2239 "VLAN cannot follow L3/L4 layer"); 2240 if (!mask) 2241 mask = &rte_flow_item_vlan_mask; 2242 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2243 (const uint8_t *)&nic_mask, 2244 sizeof(struct rte_flow_item_vlan), 2245 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2246 if (ret) 2247 return ret; 2248 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2249 struct mlx5_priv *priv = dev->data->dev_private; 2250 2251 if (priv->vmwa_context) { 2252 /* 2253 * Non-NULL context means we have a virtual machine 2254 * and SR-IOV enabled, we have to create VLAN interface 2255 * to make hypervisor to setup E-Switch vport 2256 * context correctly. We avoid creating the multiple 2257 * VLAN interfaces, so we cannot support VLAN tag mask. 2258 */ 2259 return rte_flow_error_set(error, EINVAL, 2260 RTE_FLOW_ERROR_TYPE_ITEM, 2261 item, 2262 "VLAN tag mask is not" 2263 " supported in virtual" 2264 " environment"); 2265 } 2266 } 2267 if (spec) { 2268 vlan_tag = spec->tci; 2269 vlan_tag &= mask->tci; 2270 } 2271 /* 2272 * From verbs perspective an empty VLAN is equivalent 2273 * to a packet without VLAN layer. 2274 */ 2275 if (!vlan_tag) 2276 return rte_flow_error_set(error, EINVAL, 2277 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2278 item->spec, 2279 "VLAN cannot be empty"); 2280 return 0; 2281 } 2282 2283 /** 2284 * Validate IPV4 item. 2285 * 2286 * @param[in] item 2287 * Item specification. 2288 * @param[in] item_flags 2289 * Bit-fields that holds the items detected until now. 2290 * @param[in] last_item 2291 * Previous validated item in the pattern items. 2292 * @param[in] ether_type 2293 * Type in the ethernet layer header (including dot1q). 2294 * @param[in] acc_mask 2295 * Acceptable mask, if NULL default internal default mask 2296 * will be used to check whether item fields are supported. 2297 * @param[in] range_accepted 2298 * True if range of values is accepted for specific fields, false otherwise. 2299 * @param[out] error 2300 * Pointer to error structure. 2301 * 2302 * @return 2303 * 0 on success, a negative errno value otherwise and rte_errno is set. 2304 */ 2305 int 2306 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2307 uint64_t item_flags, 2308 uint64_t last_item, 2309 uint16_t ether_type, 2310 const struct rte_flow_item_ipv4 *acc_mask, 2311 bool range_accepted, 2312 struct rte_flow_error *error) 2313 { 2314 const struct rte_flow_item_ipv4 *mask = item->mask; 2315 const struct rte_flow_item_ipv4 *spec = item->spec; 2316 const struct rte_flow_item_ipv4 nic_mask = { 2317 .hdr = { 2318 .src_addr = RTE_BE32(0xffffffff), 2319 .dst_addr = RTE_BE32(0xffffffff), 2320 .type_of_service = 0xff, 2321 .next_proto_id = 0xff, 2322 }, 2323 }; 2324 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2325 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2326 MLX5_FLOW_LAYER_OUTER_L3; 2327 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2328 MLX5_FLOW_LAYER_OUTER_L4; 2329 int ret; 2330 uint8_t next_proto = 0xFF; 2331 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2332 MLX5_FLOW_LAYER_OUTER_VLAN | 2333 MLX5_FLOW_LAYER_INNER_VLAN); 2334 2335 if ((last_item & l2_vlan) && ether_type && 2336 ether_type != RTE_ETHER_TYPE_IPV4) 2337 return rte_flow_error_set(error, EINVAL, 2338 RTE_FLOW_ERROR_TYPE_ITEM, item, 2339 "IPv4 cannot follow L2/VLAN layer " 2340 "which ether type is not IPv4"); 2341 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2342 if (mask && spec) 2343 next_proto = mask->hdr.next_proto_id & 2344 spec->hdr.next_proto_id; 2345 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2346 return rte_flow_error_set(error, EINVAL, 2347 RTE_FLOW_ERROR_TYPE_ITEM, 2348 item, 2349 "multiple tunnel " 2350 "not supported"); 2351 } 2352 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2353 return rte_flow_error_set(error, EINVAL, 2354 RTE_FLOW_ERROR_TYPE_ITEM, item, 2355 "wrong tunnel type - IPv6 specified " 2356 "but IPv4 item provided"); 2357 if (item_flags & l3m) 2358 return rte_flow_error_set(error, ENOTSUP, 2359 RTE_FLOW_ERROR_TYPE_ITEM, item, 2360 "multiple L3 layers not supported"); 2361 else if (item_flags & l4m) 2362 return rte_flow_error_set(error, EINVAL, 2363 RTE_FLOW_ERROR_TYPE_ITEM, item, 2364 "L3 cannot follow an L4 layer."); 2365 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2366 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2367 return rte_flow_error_set(error, EINVAL, 2368 RTE_FLOW_ERROR_TYPE_ITEM, item, 2369 "L3 cannot follow an NVGRE layer."); 2370 if (!mask) 2371 mask = &rte_flow_item_ipv4_mask; 2372 else if (mask->hdr.next_proto_id != 0 && 2373 mask->hdr.next_proto_id != 0xff) 2374 return rte_flow_error_set(error, EINVAL, 2375 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2376 "partial mask is not supported" 2377 " for protocol"); 2378 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2379 acc_mask ? (const uint8_t *)acc_mask 2380 : (const uint8_t *)&nic_mask, 2381 sizeof(struct rte_flow_item_ipv4), 2382 range_accepted, error); 2383 if (ret < 0) 2384 return ret; 2385 return 0; 2386 } 2387 2388 /** 2389 * Validate IPV6 item. 2390 * 2391 * @param[in] item 2392 * Item specification. 2393 * @param[in] item_flags 2394 * Bit-fields that holds the items detected until now. 2395 * @param[in] last_item 2396 * Previous validated item in the pattern items. 2397 * @param[in] ether_type 2398 * Type in the ethernet layer header (including dot1q). 2399 * @param[in] acc_mask 2400 * Acceptable mask, if NULL default internal default mask 2401 * will be used to check whether item fields are supported. 2402 * @param[out] error 2403 * Pointer to error structure. 2404 * 2405 * @return 2406 * 0 on success, a negative errno value otherwise and rte_errno is set. 2407 */ 2408 int 2409 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2410 uint64_t item_flags, 2411 uint64_t last_item, 2412 uint16_t ether_type, 2413 const struct rte_flow_item_ipv6 *acc_mask, 2414 struct rte_flow_error *error) 2415 { 2416 const struct rte_flow_item_ipv6 *mask = item->mask; 2417 const struct rte_flow_item_ipv6 *spec = item->spec; 2418 const struct rte_flow_item_ipv6 nic_mask = { 2419 .hdr = { 2420 .src_addr = 2421 "\xff\xff\xff\xff\xff\xff\xff\xff" 2422 "\xff\xff\xff\xff\xff\xff\xff\xff", 2423 .dst_addr = 2424 "\xff\xff\xff\xff\xff\xff\xff\xff" 2425 "\xff\xff\xff\xff\xff\xff\xff\xff", 2426 .vtc_flow = RTE_BE32(0xffffffff), 2427 .proto = 0xff, 2428 }, 2429 }; 2430 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2431 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2432 MLX5_FLOW_LAYER_OUTER_L3; 2433 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2434 MLX5_FLOW_LAYER_OUTER_L4; 2435 int ret; 2436 uint8_t next_proto = 0xFF; 2437 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2438 MLX5_FLOW_LAYER_OUTER_VLAN | 2439 MLX5_FLOW_LAYER_INNER_VLAN); 2440 2441 if ((last_item & l2_vlan) && ether_type && 2442 ether_type != RTE_ETHER_TYPE_IPV6) 2443 return rte_flow_error_set(error, EINVAL, 2444 RTE_FLOW_ERROR_TYPE_ITEM, item, 2445 "IPv6 cannot follow L2/VLAN layer " 2446 "which ether type is not IPv6"); 2447 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2448 next_proto = spec->hdr.proto; 2449 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2450 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2451 return rte_flow_error_set(error, EINVAL, 2452 RTE_FLOW_ERROR_TYPE_ITEM, 2453 item, 2454 "multiple tunnel " 2455 "not supported"); 2456 } 2457 if (next_proto == IPPROTO_HOPOPTS || 2458 next_proto == IPPROTO_ROUTING || 2459 next_proto == IPPROTO_FRAGMENT || 2460 next_proto == IPPROTO_ESP || 2461 next_proto == IPPROTO_AH || 2462 next_proto == IPPROTO_DSTOPTS) 2463 return rte_flow_error_set(error, EINVAL, 2464 RTE_FLOW_ERROR_TYPE_ITEM, item, 2465 "IPv6 proto (next header) should " 2466 "not be set as extension header"); 2467 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2468 return rte_flow_error_set(error, EINVAL, 2469 RTE_FLOW_ERROR_TYPE_ITEM, item, 2470 "wrong tunnel type - IPv4 specified " 2471 "but IPv6 item provided"); 2472 if (item_flags & l3m) 2473 return rte_flow_error_set(error, ENOTSUP, 2474 RTE_FLOW_ERROR_TYPE_ITEM, item, 2475 "multiple L3 layers not supported"); 2476 else if (item_flags & l4m) 2477 return rte_flow_error_set(error, EINVAL, 2478 RTE_FLOW_ERROR_TYPE_ITEM, item, 2479 "L3 cannot follow an L4 layer."); 2480 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2481 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2482 return rte_flow_error_set(error, EINVAL, 2483 RTE_FLOW_ERROR_TYPE_ITEM, item, 2484 "L3 cannot follow an NVGRE layer."); 2485 if (!mask) 2486 mask = &rte_flow_item_ipv6_mask; 2487 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2488 acc_mask ? (const uint8_t *)acc_mask 2489 : (const uint8_t *)&nic_mask, 2490 sizeof(struct rte_flow_item_ipv6), 2491 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2492 if (ret < 0) 2493 return ret; 2494 return 0; 2495 } 2496 2497 /** 2498 * Validate UDP item. 2499 * 2500 * @param[in] item 2501 * Item specification. 2502 * @param[in] item_flags 2503 * Bit-fields that holds the items detected until now. 2504 * @param[in] target_protocol 2505 * The next protocol in the previous item. 2506 * @param[in] flow_mask 2507 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2508 * @param[out] error 2509 * Pointer to error structure. 2510 * 2511 * @return 2512 * 0 on success, a negative errno value otherwise and rte_errno is set. 2513 */ 2514 int 2515 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2516 uint64_t item_flags, 2517 uint8_t target_protocol, 2518 struct rte_flow_error *error) 2519 { 2520 const struct rte_flow_item_udp *mask = item->mask; 2521 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2522 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2523 MLX5_FLOW_LAYER_OUTER_L3; 2524 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2525 MLX5_FLOW_LAYER_OUTER_L4; 2526 int ret; 2527 2528 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2529 return rte_flow_error_set(error, EINVAL, 2530 RTE_FLOW_ERROR_TYPE_ITEM, item, 2531 "protocol filtering not compatible" 2532 " with UDP layer"); 2533 if (!(item_flags & l3m)) 2534 return rte_flow_error_set(error, EINVAL, 2535 RTE_FLOW_ERROR_TYPE_ITEM, item, 2536 "L3 is mandatory to filter on L4"); 2537 if (item_flags & l4m) 2538 return rte_flow_error_set(error, EINVAL, 2539 RTE_FLOW_ERROR_TYPE_ITEM, item, 2540 "multiple L4 layers not supported"); 2541 if (!mask) 2542 mask = &rte_flow_item_udp_mask; 2543 ret = mlx5_flow_item_acceptable 2544 (item, (const uint8_t *)mask, 2545 (const uint8_t *)&rte_flow_item_udp_mask, 2546 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2547 error); 2548 if (ret < 0) 2549 return ret; 2550 return 0; 2551 } 2552 2553 /** 2554 * Validate TCP item. 2555 * 2556 * @param[in] item 2557 * Item specification. 2558 * @param[in] item_flags 2559 * Bit-fields that holds the items detected until now. 2560 * @param[in] target_protocol 2561 * The next protocol in the previous item. 2562 * @param[out] error 2563 * Pointer to error structure. 2564 * 2565 * @return 2566 * 0 on success, a negative errno value otherwise and rte_errno is set. 2567 */ 2568 int 2569 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2570 uint64_t item_flags, 2571 uint8_t target_protocol, 2572 const struct rte_flow_item_tcp *flow_mask, 2573 struct rte_flow_error *error) 2574 { 2575 const struct rte_flow_item_tcp *mask = item->mask; 2576 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2577 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2578 MLX5_FLOW_LAYER_OUTER_L3; 2579 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2580 MLX5_FLOW_LAYER_OUTER_L4; 2581 int ret; 2582 2583 MLX5_ASSERT(flow_mask); 2584 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2585 return rte_flow_error_set(error, EINVAL, 2586 RTE_FLOW_ERROR_TYPE_ITEM, item, 2587 "protocol filtering not compatible" 2588 " with TCP layer"); 2589 if (!(item_flags & l3m)) 2590 return rte_flow_error_set(error, EINVAL, 2591 RTE_FLOW_ERROR_TYPE_ITEM, item, 2592 "L3 is mandatory to filter on L4"); 2593 if (item_flags & l4m) 2594 return rte_flow_error_set(error, EINVAL, 2595 RTE_FLOW_ERROR_TYPE_ITEM, item, 2596 "multiple L4 layers not supported"); 2597 if (!mask) 2598 mask = &rte_flow_item_tcp_mask; 2599 ret = mlx5_flow_item_acceptable 2600 (item, (const uint8_t *)mask, 2601 (const uint8_t *)flow_mask, 2602 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2603 error); 2604 if (ret < 0) 2605 return ret; 2606 return 0; 2607 } 2608 2609 /** 2610 * Validate VXLAN item. 2611 * 2612 * @param[in] dev 2613 * Pointer to the Ethernet device structure. 2614 * @param[in] udp_dport 2615 * UDP destination port 2616 * @param[in] item 2617 * Item specification. 2618 * @param[in] item_flags 2619 * Bit-fields that holds the items detected until now. 2620 * @param[in] attr 2621 * Flow rule attributes. 2622 * @param[out] error 2623 * Pointer to error structure. 2624 * 2625 * @return 2626 * 0 on success, a negative errno value otherwise and rte_errno is set. 2627 */ 2628 int 2629 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2630 uint16_t udp_dport, 2631 const struct rte_flow_item *item, 2632 uint64_t item_flags, 2633 const struct rte_flow_attr *attr, 2634 struct rte_flow_error *error) 2635 { 2636 const struct rte_flow_item_vxlan *spec = item->spec; 2637 const struct rte_flow_item_vxlan *mask = item->mask; 2638 int ret; 2639 struct mlx5_priv *priv = dev->data->dev_private; 2640 union vni { 2641 uint32_t vlan_id; 2642 uint8_t vni[4]; 2643 } id = { .vlan_id = 0, }; 2644 const struct rte_flow_item_vxlan nic_mask = { 2645 .vni = "\xff\xff\xff", 2646 .rsvd1 = 0xff, 2647 }; 2648 const struct rte_flow_item_vxlan *valid_mask; 2649 2650 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2651 return rte_flow_error_set(error, ENOTSUP, 2652 RTE_FLOW_ERROR_TYPE_ITEM, item, 2653 "multiple tunnel layers not" 2654 " supported"); 2655 valid_mask = &rte_flow_item_vxlan_mask; 2656 /* 2657 * Verify only UDPv4 is present as defined in 2658 * https://tools.ietf.org/html/rfc7348 2659 */ 2660 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2661 return rte_flow_error_set(error, EINVAL, 2662 RTE_FLOW_ERROR_TYPE_ITEM, item, 2663 "no outer UDP layer found"); 2664 if (!mask) 2665 mask = &rte_flow_item_vxlan_mask; 2666 2667 if (priv->sh->steering_format_version != 2668 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2669 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2670 /* FDB domain & NIC domain non-zero group */ 2671 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2672 valid_mask = &nic_mask; 2673 /* Group zero in NIC domain */ 2674 if (!attr->group && !attr->transfer && 2675 priv->sh->tunnel_header_0_1) 2676 valid_mask = &nic_mask; 2677 } 2678 ret = mlx5_flow_item_acceptable 2679 (item, (const uint8_t *)mask, 2680 (const uint8_t *)valid_mask, 2681 sizeof(struct rte_flow_item_vxlan), 2682 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2683 if (ret < 0) 2684 return ret; 2685 if (spec) { 2686 memcpy(&id.vni[1], spec->vni, 3); 2687 memcpy(&id.vni[1], mask->vni, 3); 2688 } 2689 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2690 return rte_flow_error_set(error, ENOTSUP, 2691 RTE_FLOW_ERROR_TYPE_ITEM, item, 2692 "VXLAN tunnel must be fully defined"); 2693 return 0; 2694 } 2695 2696 /** 2697 * Validate VXLAN_GPE item. 2698 * 2699 * @param[in] item 2700 * Item specification. 2701 * @param[in] item_flags 2702 * Bit-fields that holds the items detected until now. 2703 * @param[in] priv 2704 * Pointer to the private data structure. 2705 * @param[in] target_protocol 2706 * The next protocol in the previous item. 2707 * @param[out] error 2708 * Pointer to error structure. 2709 * 2710 * @return 2711 * 0 on success, a negative errno value otherwise and rte_errno is set. 2712 */ 2713 int 2714 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2715 uint64_t item_flags, 2716 struct rte_eth_dev *dev, 2717 struct rte_flow_error *error) 2718 { 2719 struct mlx5_priv *priv = dev->data->dev_private; 2720 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2721 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2722 int ret; 2723 union vni { 2724 uint32_t vlan_id; 2725 uint8_t vni[4]; 2726 } id = { .vlan_id = 0, }; 2727 2728 if (!priv->sh->config.l3_vxlan_en) 2729 return rte_flow_error_set(error, ENOTSUP, 2730 RTE_FLOW_ERROR_TYPE_ITEM, item, 2731 "L3 VXLAN is not enabled by device" 2732 " parameter and/or not configured in" 2733 " firmware"); 2734 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2735 return rte_flow_error_set(error, ENOTSUP, 2736 RTE_FLOW_ERROR_TYPE_ITEM, item, 2737 "multiple tunnel layers not" 2738 " supported"); 2739 /* 2740 * Verify only UDPv4 is present as defined in 2741 * https://tools.ietf.org/html/rfc7348 2742 */ 2743 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2744 return rte_flow_error_set(error, EINVAL, 2745 RTE_FLOW_ERROR_TYPE_ITEM, item, 2746 "no outer UDP layer found"); 2747 if (!mask) 2748 mask = &rte_flow_item_vxlan_gpe_mask; 2749 ret = mlx5_flow_item_acceptable 2750 (item, (const uint8_t *)mask, 2751 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2752 sizeof(struct rte_flow_item_vxlan_gpe), 2753 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2754 if (ret < 0) 2755 return ret; 2756 if (spec) { 2757 if (spec->protocol) 2758 return rte_flow_error_set(error, ENOTSUP, 2759 RTE_FLOW_ERROR_TYPE_ITEM, 2760 item, 2761 "VxLAN-GPE protocol" 2762 " not supported"); 2763 memcpy(&id.vni[1], spec->vni, 3); 2764 memcpy(&id.vni[1], mask->vni, 3); 2765 } 2766 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2767 return rte_flow_error_set(error, ENOTSUP, 2768 RTE_FLOW_ERROR_TYPE_ITEM, item, 2769 "VXLAN-GPE tunnel must be fully" 2770 " defined"); 2771 return 0; 2772 } 2773 /** 2774 * Validate GRE Key item. 2775 * 2776 * @param[in] item 2777 * Item specification. 2778 * @param[in] item_flags 2779 * Bit flags to mark detected items. 2780 * @param[in] gre_item 2781 * Pointer to gre_item 2782 * @param[out] error 2783 * Pointer to error structure. 2784 * 2785 * @return 2786 * 0 on success, a negative errno value otherwise and rte_errno is set. 2787 */ 2788 int 2789 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2790 uint64_t item_flags, 2791 const struct rte_flow_item *gre_item, 2792 struct rte_flow_error *error) 2793 { 2794 const rte_be32_t *mask = item->mask; 2795 int ret = 0; 2796 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2797 const struct rte_flow_item_gre *gre_spec; 2798 const struct rte_flow_item_gre *gre_mask; 2799 2800 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2801 return rte_flow_error_set(error, ENOTSUP, 2802 RTE_FLOW_ERROR_TYPE_ITEM, item, 2803 "Multiple GRE key not support"); 2804 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2805 return rte_flow_error_set(error, ENOTSUP, 2806 RTE_FLOW_ERROR_TYPE_ITEM, item, 2807 "No preceding GRE header"); 2808 if (item_flags & MLX5_FLOW_LAYER_INNER) 2809 return rte_flow_error_set(error, ENOTSUP, 2810 RTE_FLOW_ERROR_TYPE_ITEM, item, 2811 "GRE key following a wrong item"); 2812 gre_mask = gre_item->mask; 2813 if (!gre_mask) 2814 gre_mask = &rte_flow_item_gre_mask; 2815 gre_spec = gre_item->spec; 2816 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2817 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2818 return rte_flow_error_set(error, EINVAL, 2819 RTE_FLOW_ERROR_TYPE_ITEM, item, 2820 "Key bit must be on"); 2821 2822 if (!mask) 2823 mask = &gre_key_default_mask; 2824 ret = mlx5_flow_item_acceptable 2825 (item, (const uint8_t *)mask, 2826 (const uint8_t *)&gre_key_default_mask, 2827 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2828 return ret; 2829 } 2830 2831 /** 2832 * Validate GRE item. 2833 * 2834 * @param[in] item 2835 * Item specification. 2836 * @param[in] item_flags 2837 * Bit flags to mark detected items. 2838 * @param[in] target_protocol 2839 * The next protocol in the previous item. 2840 * @param[out] error 2841 * Pointer to error structure. 2842 * 2843 * @return 2844 * 0 on success, a negative errno value otherwise and rte_errno is set. 2845 */ 2846 int 2847 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2848 uint64_t item_flags, 2849 uint8_t target_protocol, 2850 struct rte_flow_error *error) 2851 { 2852 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2853 const struct rte_flow_item_gre *mask = item->mask; 2854 int ret; 2855 const struct rte_flow_item_gre nic_mask = { 2856 .c_rsvd0_ver = RTE_BE16(0xB000), 2857 .protocol = RTE_BE16(UINT16_MAX), 2858 }; 2859 2860 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2861 return rte_flow_error_set(error, EINVAL, 2862 RTE_FLOW_ERROR_TYPE_ITEM, item, 2863 "protocol filtering not compatible" 2864 " with this GRE layer"); 2865 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2866 return rte_flow_error_set(error, ENOTSUP, 2867 RTE_FLOW_ERROR_TYPE_ITEM, item, 2868 "multiple tunnel layers not" 2869 " supported"); 2870 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2871 return rte_flow_error_set(error, ENOTSUP, 2872 RTE_FLOW_ERROR_TYPE_ITEM, item, 2873 "L3 Layer is missing"); 2874 if (!mask) 2875 mask = &rte_flow_item_gre_mask; 2876 ret = mlx5_flow_item_acceptable 2877 (item, (const uint8_t *)mask, 2878 (const uint8_t *)&nic_mask, 2879 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2880 error); 2881 if (ret < 0) 2882 return ret; 2883 #ifndef HAVE_MLX5DV_DR 2884 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2885 if (spec && (spec->protocol & mask->protocol)) 2886 return rte_flow_error_set(error, ENOTSUP, 2887 RTE_FLOW_ERROR_TYPE_ITEM, item, 2888 "without MPLS support the" 2889 " specification cannot be used for" 2890 " filtering"); 2891 #endif 2892 #endif 2893 return 0; 2894 } 2895 2896 /** 2897 * Validate Geneve item. 2898 * 2899 * @param[in] item 2900 * Item specification. 2901 * @param[in] itemFlags 2902 * Bit-fields that holds the items detected until now. 2903 * @param[in] enPriv 2904 * Pointer to the private data structure. 2905 * @param[out] error 2906 * Pointer to error structure. 2907 * 2908 * @return 2909 * 0 on success, a negative errno value otherwise and rte_errno is set. 2910 */ 2911 2912 int 2913 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2914 uint64_t item_flags, 2915 struct rte_eth_dev *dev, 2916 struct rte_flow_error *error) 2917 { 2918 struct mlx5_priv *priv = dev->data->dev_private; 2919 const struct rte_flow_item_geneve *spec = item->spec; 2920 const struct rte_flow_item_geneve *mask = item->mask; 2921 int ret; 2922 uint16_t gbhdr; 2923 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 2924 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2925 const struct rte_flow_item_geneve nic_mask = { 2926 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2927 .vni = "\xff\xff\xff", 2928 .protocol = RTE_BE16(UINT16_MAX), 2929 }; 2930 2931 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 2932 return rte_flow_error_set(error, ENOTSUP, 2933 RTE_FLOW_ERROR_TYPE_ITEM, item, 2934 "L3 Geneve is not enabled by device" 2935 " parameter and/or not configured in" 2936 " firmware"); 2937 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2938 return rte_flow_error_set(error, ENOTSUP, 2939 RTE_FLOW_ERROR_TYPE_ITEM, item, 2940 "multiple tunnel layers not" 2941 " supported"); 2942 /* 2943 * Verify only UDPv4 is present as defined in 2944 * https://tools.ietf.org/html/rfc7348 2945 */ 2946 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2947 return rte_flow_error_set(error, EINVAL, 2948 RTE_FLOW_ERROR_TYPE_ITEM, item, 2949 "no outer UDP layer found"); 2950 if (!mask) 2951 mask = &rte_flow_item_geneve_mask; 2952 ret = mlx5_flow_item_acceptable 2953 (item, (const uint8_t *)mask, 2954 (const uint8_t *)&nic_mask, 2955 sizeof(struct rte_flow_item_geneve), 2956 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2957 if (ret) 2958 return ret; 2959 if (spec) { 2960 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2961 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2962 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2963 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2964 return rte_flow_error_set(error, ENOTSUP, 2965 RTE_FLOW_ERROR_TYPE_ITEM, 2966 item, 2967 "Geneve protocol unsupported" 2968 " fields are being used"); 2969 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2970 return rte_flow_error_set 2971 (error, ENOTSUP, 2972 RTE_FLOW_ERROR_TYPE_ITEM, 2973 item, 2974 "Unsupported Geneve options length"); 2975 } 2976 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2977 return rte_flow_error_set 2978 (error, ENOTSUP, 2979 RTE_FLOW_ERROR_TYPE_ITEM, item, 2980 "Geneve tunnel must be fully defined"); 2981 return 0; 2982 } 2983 2984 /** 2985 * Validate Geneve TLV option item. 2986 * 2987 * @param[in] item 2988 * Item specification. 2989 * @param[in] last_item 2990 * Previous validated item in the pattern items. 2991 * @param[in] geneve_item 2992 * Previous GENEVE item specification. 2993 * @param[in] dev 2994 * Pointer to the rte_eth_dev structure. 2995 * @param[out] error 2996 * Pointer to error structure. 2997 * 2998 * @return 2999 * 0 on success, a negative errno value otherwise and rte_errno is set. 3000 */ 3001 int 3002 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3003 uint64_t last_item, 3004 const struct rte_flow_item *geneve_item, 3005 struct rte_eth_dev *dev, 3006 struct rte_flow_error *error) 3007 { 3008 struct mlx5_priv *priv = dev->data->dev_private; 3009 struct mlx5_dev_ctx_shared *sh = priv->sh; 3010 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3011 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3012 uint8_t data_max_supported = 3013 hca_attr->max_geneve_tlv_option_data_len * 4; 3014 const struct rte_flow_item_geneve *geneve_spec; 3015 const struct rte_flow_item_geneve *geneve_mask; 3016 const struct rte_flow_item_geneve_opt *spec = item->spec; 3017 const struct rte_flow_item_geneve_opt *mask = item->mask; 3018 unsigned int i; 3019 unsigned int data_len; 3020 uint8_t tlv_option_len; 3021 uint16_t optlen_m, optlen_v; 3022 const struct rte_flow_item_geneve_opt full_mask = { 3023 .option_class = RTE_BE16(0xffff), 3024 .option_type = 0xff, 3025 .option_len = 0x1f, 3026 }; 3027 3028 if (!mask) 3029 mask = &rte_flow_item_geneve_opt_mask; 3030 if (!spec) 3031 return rte_flow_error_set 3032 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3033 "Geneve TLV opt class/type/length must be specified"); 3034 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3035 return rte_flow_error_set 3036 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3037 "Geneve TLV opt length exceeds the limit (31)"); 3038 /* Check if class type and length masks are full. */ 3039 if (full_mask.option_class != mask->option_class || 3040 full_mask.option_type != mask->option_type || 3041 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3042 return rte_flow_error_set 3043 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3044 "Geneve TLV opt class/type/length masks must be full"); 3045 /* Check if length is supported */ 3046 if ((uint32_t)spec->option_len > 3047 hca_attr->max_geneve_tlv_option_data_len) 3048 return rte_flow_error_set 3049 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3050 "Geneve TLV opt length not supported"); 3051 if (hca_attr->max_geneve_tlv_options > 1) 3052 DRV_LOG(DEBUG, 3053 "max_geneve_tlv_options supports more than 1 option"); 3054 /* Check GENEVE item preceding. */ 3055 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3056 return rte_flow_error_set 3057 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3058 "Geneve opt item must be preceded with Geneve item"); 3059 geneve_spec = geneve_item->spec; 3060 geneve_mask = geneve_item->mask ? geneve_item->mask : 3061 &rte_flow_item_geneve_mask; 3062 /* Check if GENEVE TLV option size doesn't exceed option length */ 3063 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3064 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3065 tlv_option_len = spec->option_len & mask->option_len; 3066 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3067 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3068 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3069 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3070 if ((optlen_v & optlen_m) <= tlv_option_len) 3071 return rte_flow_error_set 3072 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3073 "GENEVE TLV option length exceeds optlen"); 3074 } 3075 /* Check if length is 0 or data is 0. */ 3076 if (spec->data == NULL || spec->option_len == 0) 3077 return rte_flow_error_set 3078 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3079 "Geneve TLV opt with zero data/length not supported"); 3080 /* Check not all data & mask are 0. */ 3081 data_len = spec->option_len * 4; 3082 if (mask->data == NULL) { 3083 for (i = 0; i < data_len; i++) 3084 if (spec->data[i]) 3085 break; 3086 if (i == data_len) 3087 return rte_flow_error_set(error, ENOTSUP, 3088 RTE_FLOW_ERROR_TYPE_ITEM, item, 3089 "Can't match on Geneve option data 0"); 3090 } else { 3091 for (i = 0; i < data_len; i++) 3092 if (spec->data[i] & mask->data[i]) 3093 break; 3094 if (i == data_len) 3095 return rte_flow_error_set(error, ENOTSUP, 3096 RTE_FLOW_ERROR_TYPE_ITEM, item, 3097 "Can't match on Geneve option data and mask 0"); 3098 /* Check data mask supported. */ 3099 for (i = data_max_supported; i < data_len ; i++) 3100 if (mask->data[i]) 3101 return rte_flow_error_set(error, ENOTSUP, 3102 RTE_FLOW_ERROR_TYPE_ITEM, item, 3103 "Data mask is of unsupported size"); 3104 } 3105 /* Check GENEVE option is supported in NIC. */ 3106 if (!hca_attr->geneve_tlv_opt) 3107 return rte_flow_error_set 3108 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3109 "Geneve TLV opt not supported"); 3110 /* Check if we already have geneve option with different type/class. */ 3111 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3112 geneve_opt_resource = sh->geneve_tlv_option_resource; 3113 if (geneve_opt_resource != NULL) 3114 if (geneve_opt_resource->option_class != spec->option_class || 3115 geneve_opt_resource->option_type != spec->option_type || 3116 geneve_opt_resource->length != spec->option_len) { 3117 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3118 return rte_flow_error_set(error, ENOTSUP, 3119 RTE_FLOW_ERROR_TYPE_ITEM, item, 3120 "Only one Geneve TLV option supported"); 3121 } 3122 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3123 return 0; 3124 } 3125 3126 /** 3127 * Validate MPLS item. 3128 * 3129 * @param[in] dev 3130 * Pointer to the rte_eth_dev structure. 3131 * @param[in] item 3132 * Item specification. 3133 * @param[in] item_flags 3134 * Bit-fields that holds the items detected until now. 3135 * @param[in] prev_layer 3136 * The protocol layer indicated in previous item. 3137 * @param[out] error 3138 * Pointer to error structure. 3139 * 3140 * @return 3141 * 0 on success, a negative errno value otherwise and rte_errno is set. 3142 */ 3143 int 3144 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3145 const struct rte_flow_item *item __rte_unused, 3146 uint64_t item_flags __rte_unused, 3147 uint64_t prev_layer __rte_unused, 3148 struct rte_flow_error *error) 3149 { 3150 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3151 const struct rte_flow_item_mpls *mask = item->mask; 3152 struct mlx5_priv *priv = dev->data->dev_private; 3153 int ret; 3154 3155 if (!priv->sh->dev_cap.mpls_en) 3156 return rte_flow_error_set(error, ENOTSUP, 3157 RTE_FLOW_ERROR_TYPE_ITEM, item, 3158 "MPLS not supported or" 3159 " disabled in firmware" 3160 " configuration."); 3161 /* MPLS over UDP, GRE is allowed */ 3162 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3163 MLX5_FLOW_LAYER_GRE | 3164 MLX5_FLOW_LAYER_GRE_KEY))) 3165 return rte_flow_error_set(error, EINVAL, 3166 RTE_FLOW_ERROR_TYPE_ITEM, item, 3167 "protocol filtering not compatible" 3168 " with MPLS layer"); 3169 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3170 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3171 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3172 return rte_flow_error_set(error, ENOTSUP, 3173 RTE_FLOW_ERROR_TYPE_ITEM, item, 3174 "multiple tunnel layers not" 3175 " supported"); 3176 if (!mask) 3177 mask = &rte_flow_item_mpls_mask; 3178 ret = mlx5_flow_item_acceptable 3179 (item, (const uint8_t *)mask, 3180 (const uint8_t *)&rte_flow_item_mpls_mask, 3181 sizeof(struct rte_flow_item_mpls), 3182 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3183 if (ret < 0) 3184 return ret; 3185 return 0; 3186 #else 3187 return rte_flow_error_set(error, ENOTSUP, 3188 RTE_FLOW_ERROR_TYPE_ITEM, item, 3189 "MPLS is not supported by Verbs, please" 3190 " update."); 3191 #endif 3192 } 3193 3194 /** 3195 * Validate NVGRE item. 3196 * 3197 * @param[in] item 3198 * Item specification. 3199 * @param[in] item_flags 3200 * Bit flags to mark detected items. 3201 * @param[in] target_protocol 3202 * The next protocol in the previous item. 3203 * @param[out] error 3204 * Pointer to error structure. 3205 * 3206 * @return 3207 * 0 on success, a negative errno value otherwise and rte_errno is set. 3208 */ 3209 int 3210 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3211 uint64_t item_flags, 3212 uint8_t target_protocol, 3213 struct rte_flow_error *error) 3214 { 3215 const struct rte_flow_item_nvgre *mask = item->mask; 3216 int ret; 3217 3218 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3219 return rte_flow_error_set(error, EINVAL, 3220 RTE_FLOW_ERROR_TYPE_ITEM, item, 3221 "protocol filtering not compatible" 3222 " with this GRE layer"); 3223 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3224 return rte_flow_error_set(error, ENOTSUP, 3225 RTE_FLOW_ERROR_TYPE_ITEM, item, 3226 "multiple tunnel layers not" 3227 " supported"); 3228 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3229 return rte_flow_error_set(error, ENOTSUP, 3230 RTE_FLOW_ERROR_TYPE_ITEM, item, 3231 "L3 Layer is missing"); 3232 if (!mask) 3233 mask = &rte_flow_item_nvgre_mask; 3234 ret = mlx5_flow_item_acceptable 3235 (item, (const uint8_t *)mask, 3236 (const uint8_t *)&rte_flow_item_nvgre_mask, 3237 sizeof(struct rte_flow_item_nvgre), 3238 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3239 if (ret < 0) 3240 return ret; 3241 return 0; 3242 } 3243 3244 /** 3245 * Validate eCPRI item. 3246 * 3247 * @param[in] item 3248 * Item specification. 3249 * @param[in] item_flags 3250 * Bit-fields that holds the items detected until now. 3251 * @param[in] last_item 3252 * Previous validated item in the pattern items. 3253 * @param[in] ether_type 3254 * Type in the ethernet layer header (including dot1q). 3255 * @param[in] acc_mask 3256 * Acceptable mask, if NULL default internal default mask 3257 * will be used to check whether item fields are supported. 3258 * @param[out] error 3259 * Pointer to error structure. 3260 * 3261 * @return 3262 * 0 on success, a negative errno value otherwise and rte_errno is set. 3263 */ 3264 int 3265 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3266 uint64_t item_flags, 3267 uint64_t last_item, 3268 uint16_t ether_type, 3269 const struct rte_flow_item_ecpri *acc_mask, 3270 struct rte_flow_error *error) 3271 { 3272 const struct rte_flow_item_ecpri *mask = item->mask; 3273 const struct rte_flow_item_ecpri nic_mask = { 3274 .hdr = { 3275 .common = { 3276 .u32 = 3277 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3278 .type = 0xFF, 3279 }).u32), 3280 }, 3281 .dummy[0] = 0xFFFFFFFF, 3282 }, 3283 }; 3284 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3285 MLX5_FLOW_LAYER_OUTER_VLAN); 3286 struct rte_flow_item_ecpri mask_lo; 3287 3288 if (!(last_item & outer_l2_vlan) && 3289 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3290 return rte_flow_error_set(error, EINVAL, 3291 RTE_FLOW_ERROR_TYPE_ITEM, item, 3292 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3293 if ((last_item & outer_l2_vlan) && ether_type && 3294 ether_type != RTE_ETHER_TYPE_ECPRI) 3295 return rte_flow_error_set(error, EINVAL, 3296 RTE_FLOW_ERROR_TYPE_ITEM, item, 3297 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3298 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3299 return rte_flow_error_set(error, EINVAL, 3300 RTE_FLOW_ERROR_TYPE_ITEM, item, 3301 "eCPRI with tunnel is not supported right now"); 3302 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3303 return rte_flow_error_set(error, ENOTSUP, 3304 RTE_FLOW_ERROR_TYPE_ITEM, item, 3305 "multiple L3 layers not supported"); 3306 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3307 return rte_flow_error_set(error, EINVAL, 3308 RTE_FLOW_ERROR_TYPE_ITEM, item, 3309 "eCPRI cannot coexist with a TCP layer"); 3310 /* In specification, eCPRI could be over UDP layer. */ 3311 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3312 return rte_flow_error_set(error, EINVAL, 3313 RTE_FLOW_ERROR_TYPE_ITEM, item, 3314 "eCPRI over UDP layer is not yet supported right now"); 3315 /* Mask for type field in common header could be zero. */ 3316 if (!mask) 3317 mask = &rte_flow_item_ecpri_mask; 3318 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3319 /* Input mask is in big-endian format. */ 3320 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3321 return rte_flow_error_set(error, EINVAL, 3322 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3323 "partial mask is not supported for protocol"); 3324 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3325 return rte_flow_error_set(error, EINVAL, 3326 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3327 "message header mask must be after a type mask"); 3328 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3329 acc_mask ? (const uint8_t *)acc_mask 3330 : (const uint8_t *)&nic_mask, 3331 sizeof(struct rte_flow_item_ecpri), 3332 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3333 } 3334 3335 static int 3336 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3337 const struct rte_flow_attr *attr __rte_unused, 3338 const struct rte_flow_item items[] __rte_unused, 3339 const struct rte_flow_action actions[] __rte_unused, 3340 bool external __rte_unused, 3341 int hairpin __rte_unused, 3342 struct rte_flow_error *error) 3343 { 3344 return rte_flow_error_set(error, ENOTSUP, 3345 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3346 } 3347 3348 static struct mlx5_flow * 3349 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3350 const struct rte_flow_attr *attr __rte_unused, 3351 const struct rte_flow_item items[] __rte_unused, 3352 const struct rte_flow_action actions[] __rte_unused, 3353 struct rte_flow_error *error) 3354 { 3355 rte_flow_error_set(error, ENOTSUP, 3356 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3357 return NULL; 3358 } 3359 3360 static int 3361 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3362 struct mlx5_flow *dev_flow __rte_unused, 3363 const struct rte_flow_attr *attr __rte_unused, 3364 const struct rte_flow_item items[] __rte_unused, 3365 const struct rte_flow_action actions[] __rte_unused, 3366 struct rte_flow_error *error) 3367 { 3368 return rte_flow_error_set(error, ENOTSUP, 3369 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3370 } 3371 3372 static int 3373 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3374 struct rte_flow *flow __rte_unused, 3375 struct rte_flow_error *error) 3376 { 3377 return rte_flow_error_set(error, ENOTSUP, 3378 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3379 } 3380 3381 static void 3382 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3383 struct rte_flow *flow __rte_unused) 3384 { 3385 } 3386 3387 static void 3388 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3389 struct rte_flow *flow __rte_unused) 3390 { 3391 } 3392 3393 static int 3394 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3395 struct rte_flow *flow __rte_unused, 3396 const struct rte_flow_action *actions __rte_unused, 3397 void *data __rte_unused, 3398 struct rte_flow_error *error) 3399 { 3400 return rte_flow_error_set(error, ENOTSUP, 3401 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3402 } 3403 3404 static int 3405 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3406 uint32_t domains __rte_unused, 3407 uint32_t flags __rte_unused) 3408 { 3409 return 0; 3410 } 3411 3412 /* Void driver to protect from null pointer reference. */ 3413 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3414 .validate = flow_null_validate, 3415 .prepare = flow_null_prepare, 3416 .translate = flow_null_translate, 3417 .apply = flow_null_apply, 3418 .remove = flow_null_remove, 3419 .destroy = flow_null_destroy, 3420 .query = flow_null_query, 3421 .sync_domain = flow_null_sync_domain, 3422 }; 3423 3424 /** 3425 * Select flow driver type according to flow attributes and device 3426 * configuration. 3427 * 3428 * @param[in] dev 3429 * Pointer to the dev structure. 3430 * @param[in] attr 3431 * Pointer to the flow attributes. 3432 * 3433 * @return 3434 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3435 */ 3436 static enum mlx5_flow_drv_type 3437 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3438 { 3439 struct mlx5_priv *priv = dev->data->dev_private; 3440 /* The OS can determine first a specific flow type (DV, VERBS) */ 3441 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3442 3443 if (type != MLX5_FLOW_TYPE_MAX) 3444 return type; 3445 /* 3446 * Currently when dv_flow_en == 2, only HW steering engine is 3447 * supported. New engines can also be chosen here if ready. 3448 */ 3449 if (priv->sh->config.dv_flow_en == 2) 3450 return MLX5_FLOW_TYPE_HW; 3451 /* If no OS specific type - continue with DV/VERBS selection */ 3452 if (attr->transfer && priv->sh->config.dv_esw_en) 3453 type = MLX5_FLOW_TYPE_DV; 3454 if (!attr->transfer) 3455 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3456 MLX5_FLOW_TYPE_VERBS; 3457 return type; 3458 } 3459 3460 #define flow_get_drv_ops(type) flow_drv_ops[type] 3461 3462 /** 3463 * Flow driver validation API. This abstracts calling driver specific functions. 3464 * The type of flow driver is determined according to flow attributes. 3465 * 3466 * @param[in] dev 3467 * Pointer to the dev structure. 3468 * @param[in] attr 3469 * Pointer to the flow attributes. 3470 * @param[in] items 3471 * Pointer to the list of items. 3472 * @param[in] actions 3473 * Pointer to the list of actions. 3474 * @param[in] external 3475 * This flow rule is created by request external to PMD. 3476 * @param[in] hairpin 3477 * Number of hairpin TX actions, 0 means classic flow. 3478 * @param[out] error 3479 * Pointer to the error structure. 3480 * 3481 * @return 3482 * 0 on success, a negative errno value otherwise and rte_errno is set. 3483 */ 3484 static inline int 3485 flow_drv_validate(struct rte_eth_dev *dev, 3486 const struct rte_flow_attr *attr, 3487 const struct rte_flow_item items[], 3488 const struct rte_flow_action actions[], 3489 bool external, int hairpin, struct rte_flow_error *error) 3490 { 3491 const struct mlx5_flow_driver_ops *fops; 3492 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3493 3494 fops = flow_get_drv_ops(type); 3495 return fops->validate(dev, attr, items, actions, external, 3496 hairpin, error); 3497 } 3498 3499 /** 3500 * Flow driver preparation API. This abstracts calling driver specific 3501 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3502 * calculates the size of memory required for device flow, allocates the memory, 3503 * initializes the device flow and returns the pointer. 3504 * 3505 * @note 3506 * This function initializes device flow structure such as dv or verbs in 3507 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3508 * rest. For example, adding returning device flow to flow->dev_flow list and 3509 * setting backward reference to the flow should be done out of this function. 3510 * layers field is not filled either. 3511 * 3512 * @param[in] dev 3513 * Pointer to the dev structure. 3514 * @param[in] attr 3515 * Pointer to the flow attributes. 3516 * @param[in] items 3517 * Pointer to the list of items. 3518 * @param[in] actions 3519 * Pointer to the list of actions. 3520 * @param[in] flow_idx 3521 * This memory pool index to the flow. 3522 * @param[out] error 3523 * Pointer to the error structure. 3524 * 3525 * @return 3526 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3527 */ 3528 static inline struct mlx5_flow * 3529 flow_drv_prepare(struct rte_eth_dev *dev, 3530 const struct rte_flow *flow, 3531 const struct rte_flow_attr *attr, 3532 const struct rte_flow_item items[], 3533 const struct rte_flow_action actions[], 3534 uint32_t flow_idx, 3535 struct rte_flow_error *error) 3536 { 3537 const struct mlx5_flow_driver_ops *fops; 3538 enum mlx5_flow_drv_type type = flow->drv_type; 3539 struct mlx5_flow *mlx5_flow = NULL; 3540 3541 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3542 fops = flow_get_drv_ops(type); 3543 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3544 if (mlx5_flow) 3545 mlx5_flow->flow_idx = flow_idx; 3546 return mlx5_flow; 3547 } 3548 3549 /** 3550 * Flow driver translation API. This abstracts calling driver specific 3551 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3552 * translates a generic flow into a driver flow. flow_drv_prepare() must 3553 * precede. 3554 * 3555 * @note 3556 * dev_flow->layers could be filled as a result of parsing during translation 3557 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3558 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3559 * flow->actions could be overwritten even though all the expanded dev_flows 3560 * have the same actions. 3561 * 3562 * @param[in] dev 3563 * Pointer to the rte dev structure. 3564 * @param[in, out] dev_flow 3565 * Pointer to the mlx5 flow. 3566 * @param[in] attr 3567 * Pointer to the flow attributes. 3568 * @param[in] items 3569 * Pointer to the list of items. 3570 * @param[in] actions 3571 * Pointer to the list of actions. 3572 * @param[out] error 3573 * Pointer to the error structure. 3574 * 3575 * @return 3576 * 0 on success, a negative errno value otherwise and rte_errno is set. 3577 */ 3578 static inline int 3579 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3580 const struct rte_flow_attr *attr, 3581 const struct rte_flow_item items[], 3582 const struct rte_flow_action actions[], 3583 struct rte_flow_error *error) 3584 { 3585 const struct mlx5_flow_driver_ops *fops; 3586 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3587 3588 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3589 fops = flow_get_drv_ops(type); 3590 return fops->translate(dev, dev_flow, attr, items, actions, error); 3591 } 3592 3593 /** 3594 * Flow driver apply API. This abstracts calling driver specific functions. 3595 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3596 * translated driver flows on to device. flow_drv_translate() must precede. 3597 * 3598 * @param[in] dev 3599 * Pointer to Ethernet device structure. 3600 * @param[in, out] flow 3601 * Pointer to flow structure. 3602 * @param[out] error 3603 * Pointer to error structure. 3604 * 3605 * @return 3606 * 0 on success, a negative errno value otherwise and rte_errno is set. 3607 */ 3608 static inline int 3609 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3610 struct rte_flow_error *error) 3611 { 3612 const struct mlx5_flow_driver_ops *fops; 3613 enum mlx5_flow_drv_type type = flow->drv_type; 3614 3615 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3616 fops = flow_get_drv_ops(type); 3617 return fops->apply(dev, flow, error); 3618 } 3619 3620 /** 3621 * Flow driver destroy API. This abstracts calling driver specific functions. 3622 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3623 * on device and releases resources of the flow. 3624 * 3625 * @param[in] dev 3626 * Pointer to Ethernet device. 3627 * @param[in, out] flow 3628 * Pointer to flow structure. 3629 */ 3630 static inline void 3631 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3632 { 3633 const struct mlx5_flow_driver_ops *fops; 3634 enum mlx5_flow_drv_type type = flow->drv_type; 3635 3636 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3637 fops = flow_get_drv_ops(type); 3638 fops->destroy(dev, flow); 3639 } 3640 3641 /** 3642 * Flow driver find RSS policy tbl API. This abstracts calling driver 3643 * specific functions. Parent flow (rte_flow) should have driver 3644 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3645 * 3646 * @param[in] dev 3647 * Pointer to Ethernet device. 3648 * @param[in, out] flow 3649 * Pointer to flow structure. 3650 * @param[in] policy 3651 * Pointer to meter policy table. 3652 * @param[in] rss_desc 3653 * Pointer to rss_desc 3654 */ 3655 static struct mlx5_flow_meter_sub_policy * 3656 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3657 struct rte_flow *flow, 3658 struct mlx5_flow_meter_policy *policy, 3659 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3660 { 3661 const struct mlx5_flow_driver_ops *fops; 3662 enum mlx5_flow_drv_type type = flow->drv_type; 3663 3664 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3665 fops = flow_get_drv_ops(type); 3666 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3667 } 3668 3669 /** 3670 * Flow driver color tag rule API. This abstracts calling driver 3671 * specific functions. Parent flow (rte_flow) should have driver 3672 * type (drv_type). It will create the color tag rules in hierarchy meter. 3673 * 3674 * @param[in] dev 3675 * Pointer to Ethernet device. 3676 * @param[in, out] flow 3677 * Pointer to flow structure. 3678 * @param[in] fm 3679 * Pointer to flow meter structure. 3680 * @param[in] src_port 3681 * The src port this extra rule should use. 3682 * @param[in] item 3683 * The src port id match item. 3684 * @param[out] error 3685 * Pointer to error structure. 3686 */ 3687 static int 3688 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3689 struct rte_flow *flow, 3690 struct mlx5_flow_meter_info *fm, 3691 int32_t src_port, 3692 const struct rte_flow_item *item, 3693 struct rte_flow_error *error) 3694 { 3695 const struct mlx5_flow_driver_ops *fops; 3696 enum mlx5_flow_drv_type type = flow->drv_type; 3697 3698 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3699 fops = flow_get_drv_ops(type); 3700 return fops->meter_hierarchy_rule_create(dev, fm, 3701 src_port, item, error); 3702 } 3703 3704 /** 3705 * Get RSS action from the action list. 3706 * 3707 * @param[in] dev 3708 * Pointer to Ethernet device. 3709 * @param[in] actions 3710 * Pointer to the list of actions. 3711 * @param[in] flow 3712 * Parent flow structure pointer. 3713 * 3714 * @return 3715 * Pointer to the RSS action if exist, else return NULL. 3716 */ 3717 static const struct rte_flow_action_rss* 3718 flow_get_rss_action(struct rte_eth_dev *dev, 3719 const struct rte_flow_action actions[]) 3720 { 3721 struct mlx5_priv *priv = dev->data->dev_private; 3722 const struct rte_flow_action_rss *rss = NULL; 3723 struct mlx5_meter_policy_action_container *acg; 3724 struct mlx5_meter_policy_action_container *acy; 3725 3726 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3727 switch (actions->type) { 3728 case RTE_FLOW_ACTION_TYPE_RSS: 3729 rss = actions->conf; 3730 break; 3731 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3732 { 3733 const struct rte_flow_action_sample *sample = 3734 actions->conf; 3735 const struct rte_flow_action *act = sample->actions; 3736 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3737 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3738 rss = act->conf; 3739 break; 3740 } 3741 case RTE_FLOW_ACTION_TYPE_METER: 3742 { 3743 uint32_t mtr_idx; 3744 struct mlx5_flow_meter_info *fm; 3745 struct mlx5_flow_meter_policy *policy; 3746 const struct rte_flow_action_meter *mtr = actions->conf; 3747 3748 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3749 if (fm && !fm->def_policy) { 3750 policy = mlx5_flow_meter_policy_find(dev, 3751 fm->policy_id, NULL); 3752 MLX5_ASSERT(policy); 3753 if (policy->is_hierarchy) { 3754 policy = 3755 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3756 policy); 3757 if (!policy) 3758 return NULL; 3759 } 3760 if (policy->is_rss) { 3761 acg = 3762 &policy->act_cnt[RTE_COLOR_GREEN]; 3763 acy = 3764 &policy->act_cnt[RTE_COLOR_YELLOW]; 3765 if (acg->fate_action == 3766 MLX5_FLOW_FATE_SHARED_RSS) 3767 rss = acg->rss->conf; 3768 else if (acy->fate_action == 3769 MLX5_FLOW_FATE_SHARED_RSS) 3770 rss = acy->rss->conf; 3771 } 3772 } 3773 break; 3774 } 3775 default: 3776 break; 3777 } 3778 } 3779 return rss; 3780 } 3781 3782 /** 3783 * Get ASO age action by index. 3784 * 3785 * @param[in] dev 3786 * Pointer to the Ethernet device structure. 3787 * @param[in] age_idx 3788 * Index to the ASO age action. 3789 * 3790 * @return 3791 * The specified ASO age action. 3792 */ 3793 struct mlx5_aso_age_action* 3794 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3795 { 3796 uint16_t pool_idx = age_idx & UINT16_MAX; 3797 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3798 struct mlx5_priv *priv = dev->data->dev_private; 3799 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3800 struct mlx5_aso_age_pool *pool; 3801 3802 rte_rwlock_read_lock(&mng->resize_rwl); 3803 pool = mng->pools[pool_idx]; 3804 rte_rwlock_read_unlock(&mng->resize_rwl); 3805 return &pool->actions[offset - 1]; 3806 } 3807 3808 /* maps indirect action to translated direct in some actions array */ 3809 struct mlx5_translated_action_handle { 3810 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3811 int index; /**< Index in related array of rte_flow_action. */ 3812 }; 3813 3814 /** 3815 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3816 * direct action if translation possible. 3817 * This functionality used to run same execution path for both direct and 3818 * indirect actions on flow create. All necessary preparations for indirect 3819 * action handling should be performed on *handle* actions list returned 3820 * from this call. 3821 * 3822 * @param[in] dev 3823 * Pointer to Ethernet device. 3824 * @param[in] actions 3825 * List of actions to translate. 3826 * @param[out] handle 3827 * List to store translated indirect action object handles. 3828 * @param[in, out] indir_n 3829 * Size of *handle* array. On return should be updated with number of 3830 * indirect actions retrieved from the *actions* list. 3831 * @param[out] translated_actions 3832 * List of actions where all indirect actions were translated to direct 3833 * if possible. NULL if no translation took place. 3834 * @param[out] error 3835 * Pointer to the error structure. 3836 * 3837 * @return 3838 * 0 on success, a negative errno value otherwise and rte_errno is set. 3839 */ 3840 static int 3841 flow_action_handles_translate(struct rte_eth_dev *dev, 3842 const struct rte_flow_action actions[], 3843 struct mlx5_translated_action_handle *handle, 3844 int *indir_n, 3845 struct rte_flow_action **translated_actions, 3846 struct rte_flow_error *error) 3847 { 3848 struct mlx5_priv *priv = dev->data->dev_private; 3849 struct rte_flow_action *translated = NULL; 3850 size_t actions_size; 3851 int n; 3852 int copied_n = 0; 3853 struct mlx5_translated_action_handle *handle_end = NULL; 3854 3855 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3856 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3857 continue; 3858 if (copied_n == *indir_n) { 3859 return rte_flow_error_set 3860 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3861 NULL, "too many shared actions"); 3862 } 3863 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3864 sizeof(actions[n].conf)); 3865 handle[copied_n].index = n; 3866 copied_n++; 3867 } 3868 n++; 3869 *indir_n = copied_n; 3870 if (!copied_n) 3871 return 0; 3872 actions_size = sizeof(struct rte_flow_action) * n; 3873 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3874 if (!translated) { 3875 rte_errno = ENOMEM; 3876 return -ENOMEM; 3877 } 3878 memcpy(translated, actions, actions_size); 3879 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3880 struct mlx5_shared_action_rss *shared_rss; 3881 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3882 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3883 uint32_t idx = act_idx & 3884 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3885 3886 switch (type) { 3887 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3888 shared_rss = mlx5_ipool_get 3889 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3890 translated[handle->index].type = 3891 RTE_FLOW_ACTION_TYPE_RSS; 3892 translated[handle->index].conf = 3893 &shared_rss->origin; 3894 break; 3895 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3896 translated[handle->index].type = 3897 (enum rte_flow_action_type) 3898 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3899 translated[handle->index].conf = (void *)(uintptr_t)idx; 3900 break; 3901 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3902 if (priv->sh->flow_hit_aso_en) { 3903 translated[handle->index].type = 3904 (enum rte_flow_action_type) 3905 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3906 translated[handle->index].conf = 3907 (void *)(uintptr_t)idx; 3908 break; 3909 } 3910 /* Fall-through */ 3911 case MLX5_INDIRECT_ACTION_TYPE_CT: 3912 if (priv->sh->ct_aso_en) { 3913 translated[handle->index].type = 3914 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3915 translated[handle->index].conf = 3916 (void *)(uintptr_t)idx; 3917 break; 3918 } 3919 /* Fall-through */ 3920 default: 3921 mlx5_free(translated); 3922 return rte_flow_error_set 3923 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3924 NULL, "invalid indirect action type"); 3925 } 3926 } 3927 *translated_actions = translated; 3928 return 0; 3929 } 3930 3931 /** 3932 * Get Shared RSS action from the action list. 3933 * 3934 * @param[in] dev 3935 * Pointer to Ethernet device. 3936 * @param[in] shared 3937 * Pointer to the list of actions. 3938 * @param[in] shared_n 3939 * Actions list length. 3940 * 3941 * @return 3942 * The MLX5 RSS action ID if exists, otherwise return 0. 3943 */ 3944 static uint32_t 3945 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3946 struct mlx5_translated_action_handle *handle, 3947 int shared_n) 3948 { 3949 struct mlx5_translated_action_handle *handle_end; 3950 struct mlx5_priv *priv = dev->data->dev_private; 3951 struct mlx5_shared_action_rss *shared_rss; 3952 3953 3954 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3955 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3956 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3957 uint32_t idx = act_idx & 3958 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3959 switch (type) { 3960 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3961 shared_rss = mlx5_ipool_get 3962 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3963 idx); 3964 __atomic_add_fetch(&shared_rss->refcnt, 1, 3965 __ATOMIC_RELAXED); 3966 return idx; 3967 default: 3968 break; 3969 } 3970 } 3971 return 0; 3972 } 3973 3974 static unsigned int 3975 find_graph_root(uint32_t rss_level) 3976 { 3977 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3978 MLX5_EXPANSION_ROOT_OUTER; 3979 } 3980 3981 /** 3982 * Get layer flags from the prefix flow. 3983 * 3984 * Some flows may be split to several subflows, the prefix subflow gets the 3985 * match items and the suffix sub flow gets the actions. 3986 * Some actions need the user defined match item flags to get the detail for 3987 * the action. 3988 * This function helps the suffix flow to get the item layer flags from prefix 3989 * subflow. 3990 * 3991 * @param[in] dev_flow 3992 * Pointer the created prefix subflow. 3993 * 3994 * @return 3995 * The layers get from prefix subflow. 3996 */ 3997 static inline uint64_t 3998 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3999 { 4000 uint64_t layers = 0; 4001 4002 /* 4003 * Layers bits could be localization, but usually the compiler will 4004 * help to do the optimization work for source code. 4005 * If no decap actions, use the layers directly. 4006 */ 4007 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4008 return dev_flow->handle->layers; 4009 /* Convert L3 layers with decap action. */ 4010 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4011 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4012 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4013 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4014 /* Convert L4 layers with decap action. */ 4015 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4016 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4017 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4018 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4019 return layers; 4020 } 4021 4022 /** 4023 * Get metadata split action information. 4024 * 4025 * @param[in] actions 4026 * Pointer to the list of actions. 4027 * @param[out] qrss 4028 * Pointer to the return pointer. 4029 * @param[out] qrss_type 4030 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4031 * if no QUEUE/RSS is found. 4032 * @param[out] encap_idx 4033 * Pointer to the index of the encap action if exists, otherwise the last 4034 * action index. 4035 * 4036 * @return 4037 * Total number of actions. 4038 */ 4039 static int 4040 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4041 const struct rte_flow_action **qrss, 4042 int *encap_idx) 4043 { 4044 const struct rte_flow_action_raw_encap *raw_encap; 4045 int actions_n = 0; 4046 int raw_decap_idx = -1; 4047 4048 *encap_idx = -1; 4049 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4050 switch (actions->type) { 4051 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4052 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4053 *encap_idx = actions_n; 4054 break; 4055 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4056 raw_decap_idx = actions_n; 4057 break; 4058 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4059 raw_encap = actions->conf; 4060 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4061 *encap_idx = raw_decap_idx != -1 ? 4062 raw_decap_idx : actions_n; 4063 break; 4064 case RTE_FLOW_ACTION_TYPE_QUEUE: 4065 case RTE_FLOW_ACTION_TYPE_RSS: 4066 *qrss = actions; 4067 break; 4068 default: 4069 break; 4070 } 4071 actions_n++; 4072 } 4073 if (*encap_idx == -1) 4074 *encap_idx = actions_n; 4075 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4076 return actions_n + 1; 4077 } 4078 4079 /** 4080 * Check if the action will change packet. 4081 * 4082 * @param dev 4083 * Pointer to Ethernet device. 4084 * @param[in] type 4085 * action type. 4086 * 4087 * @return 4088 * true if action will change packet, false otherwise. 4089 */ 4090 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4091 enum rte_flow_action_type type) 4092 { 4093 struct mlx5_priv *priv = dev->data->dev_private; 4094 4095 switch (type) { 4096 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4097 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4098 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4099 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4100 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4101 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4102 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4103 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4104 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4105 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4106 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4107 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4108 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4109 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4110 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4111 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4112 case RTE_FLOW_ACTION_TYPE_SET_META: 4113 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4114 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4115 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4116 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4117 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4118 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4119 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4120 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4121 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4122 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4123 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4124 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4125 return true; 4126 case RTE_FLOW_ACTION_TYPE_FLAG: 4127 case RTE_FLOW_ACTION_TYPE_MARK: 4128 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4129 return true; 4130 else 4131 return false; 4132 default: 4133 return false; 4134 } 4135 } 4136 4137 /** 4138 * Check meter action from the action list. 4139 * 4140 * @param dev 4141 * Pointer to Ethernet device. 4142 * @param[in] actions 4143 * Pointer to the list of actions. 4144 * @param[out] has_mtr 4145 * Pointer to the meter exist flag. 4146 * @param[out] has_modify 4147 * Pointer to the flag showing there's packet change action. 4148 * @param[out] meter_id 4149 * Pointer to the meter id. 4150 * 4151 * @return 4152 * Total number of actions. 4153 */ 4154 static int 4155 flow_check_meter_action(struct rte_eth_dev *dev, 4156 const struct rte_flow_action actions[], 4157 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4158 { 4159 const struct rte_flow_action_meter *mtr = NULL; 4160 int actions_n = 0; 4161 4162 MLX5_ASSERT(has_mtr); 4163 *has_mtr = false; 4164 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4165 switch (actions->type) { 4166 case RTE_FLOW_ACTION_TYPE_METER: 4167 mtr = actions->conf; 4168 *meter_id = mtr->mtr_id; 4169 *has_mtr = true; 4170 break; 4171 default: 4172 break; 4173 } 4174 if (!*has_mtr) 4175 *has_modify |= flow_check_modify_action_type(dev, 4176 actions->type); 4177 actions_n++; 4178 } 4179 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4180 return actions_n + 1; 4181 } 4182 4183 /** 4184 * Check if the flow should be split due to hairpin. 4185 * The reason for the split is that in current HW we can't 4186 * support encap and push-vlan on Rx, so if a flow contains 4187 * these actions we move it to Tx. 4188 * 4189 * @param dev 4190 * Pointer to Ethernet device. 4191 * @param[in] attr 4192 * Flow rule attributes. 4193 * @param[in] actions 4194 * Associated actions (list terminated by the END action). 4195 * 4196 * @return 4197 * > 0 the number of actions and the flow should be split, 4198 * 0 when no split required. 4199 */ 4200 static int 4201 flow_check_hairpin_split(struct rte_eth_dev *dev, 4202 const struct rte_flow_attr *attr, 4203 const struct rte_flow_action actions[]) 4204 { 4205 int queue_action = 0; 4206 int action_n = 0; 4207 int split = 0; 4208 const struct rte_flow_action_queue *queue; 4209 const struct rte_flow_action_rss *rss; 4210 const struct rte_flow_action_raw_encap *raw_encap; 4211 const struct rte_eth_hairpin_conf *conf; 4212 4213 if (!attr->ingress) 4214 return 0; 4215 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4216 switch (actions->type) { 4217 case RTE_FLOW_ACTION_TYPE_QUEUE: 4218 queue = actions->conf; 4219 if (queue == NULL) 4220 return 0; 4221 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4222 if (conf == NULL || conf->tx_explicit != 0) 4223 return 0; 4224 queue_action = 1; 4225 action_n++; 4226 break; 4227 case RTE_FLOW_ACTION_TYPE_RSS: 4228 rss = actions->conf; 4229 if (rss == NULL || rss->queue_num == 0) 4230 return 0; 4231 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4232 if (conf == NULL || conf->tx_explicit != 0) 4233 return 0; 4234 queue_action = 1; 4235 action_n++; 4236 break; 4237 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4238 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4239 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4240 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4241 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4242 split++; 4243 action_n++; 4244 break; 4245 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4246 raw_encap = actions->conf; 4247 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4248 split++; 4249 action_n++; 4250 break; 4251 default: 4252 action_n++; 4253 break; 4254 } 4255 } 4256 if (split && queue_action) 4257 return action_n; 4258 return 0; 4259 } 4260 4261 /* Declare flow create/destroy prototype in advance. */ 4262 static uint32_t 4263 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4264 const struct rte_flow_attr *attr, 4265 const struct rte_flow_item items[], 4266 const struct rte_flow_action actions[], 4267 bool external, struct rte_flow_error *error); 4268 4269 static void 4270 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4271 uint32_t flow_idx); 4272 4273 int 4274 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4275 struct mlx5_list_entry *entry, void *cb_ctx) 4276 { 4277 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4278 struct mlx5_flow_mreg_copy_resource *mcp_res = 4279 container_of(entry, typeof(*mcp_res), hlist_ent); 4280 4281 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4282 } 4283 4284 struct mlx5_list_entry * 4285 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4286 { 4287 struct rte_eth_dev *dev = tool_ctx; 4288 struct mlx5_priv *priv = dev->data->dev_private; 4289 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4290 struct mlx5_flow_mreg_copy_resource *mcp_res; 4291 struct rte_flow_error *error = ctx->error; 4292 uint32_t idx = 0; 4293 int ret; 4294 uint32_t mark_id = *(uint32_t *)(ctx->data); 4295 struct rte_flow_attr attr = { 4296 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4297 .ingress = 1, 4298 }; 4299 struct mlx5_rte_flow_item_tag tag_spec = { 4300 .data = mark_id, 4301 }; 4302 struct rte_flow_item items[] = { 4303 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4304 }; 4305 struct rte_flow_action_mark ftag = { 4306 .id = mark_id, 4307 }; 4308 struct mlx5_flow_action_copy_mreg cp_mreg = { 4309 .dst = REG_B, 4310 .src = REG_NON, 4311 }; 4312 struct rte_flow_action_jump jump = { 4313 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4314 }; 4315 struct rte_flow_action actions[] = { 4316 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4317 }; 4318 4319 /* Fill the register fields in the flow. */ 4320 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4321 if (ret < 0) 4322 return NULL; 4323 tag_spec.id = ret; 4324 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4325 if (ret < 0) 4326 return NULL; 4327 cp_mreg.src = ret; 4328 /* Provide the full width of FLAG specific value. */ 4329 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4330 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4331 /* Build a new flow. */ 4332 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4333 items[0] = (struct rte_flow_item){ 4334 .type = (enum rte_flow_item_type) 4335 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4336 .spec = &tag_spec, 4337 }; 4338 items[1] = (struct rte_flow_item){ 4339 .type = RTE_FLOW_ITEM_TYPE_END, 4340 }; 4341 actions[0] = (struct rte_flow_action){ 4342 .type = (enum rte_flow_action_type) 4343 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4344 .conf = &ftag, 4345 }; 4346 actions[1] = (struct rte_flow_action){ 4347 .type = (enum rte_flow_action_type) 4348 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4349 .conf = &cp_mreg, 4350 }; 4351 actions[2] = (struct rte_flow_action){ 4352 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4353 .conf = &jump, 4354 }; 4355 actions[3] = (struct rte_flow_action){ 4356 .type = RTE_FLOW_ACTION_TYPE_END, 4357 }; 4358 } else { 4359 /* Default rule, wildcard match. */ 4360 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4361 items[0] = (struct rte_flow_item){ 4362 .type = RTE_FLOW_ITEM_TYPE_END, 4363 }; 4364 actions[0] = (struct rte_flow_action){ 4365 .type = (enum rte_flow_action_type) 4366 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4367 .conf = &cp_mreg, 4368 }; 4369 actions[1] = (struct rte_flow_action){ 4370 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4371 .conf = &jump, 4372 }; 4373 actions[2] = (struct rte_flow_action){ 4374 .type = RTE_FLOW_ACTION_TYPE_END, 4375 }; 4376 } 4377 /* Build a new entry. */ 4378 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4379 if (!mcp_res) { 4380 rte_errno = ENOMEM; 4381 return NULL; 4382 } 4383 mcp_res->idx = idx; 4384 mcp_res->mark_id = mark_id; 4385 /* 4386 * The copy Flows are not included in any list. There 4387 * ones are referenced from other Flows and can not 4388 * be applied, removed, deleted in arbitrary order 4389 * by list traversing. 4390 */ 4391 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4392 &attr, items, actions, false, error); 4393 if (!mcp_res->rix_flow) { 4394 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4395 return NULL; 4396 } 4397 return &mcp_res->hlist_ent; 4398 } 4399 4400 struct mlx5_list_entry * 4401 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4402 void *cb_ctx __rte_unused) 4403 { 4404 struct rte_eth_dev *dev = tool_ctx; 4405 struct mlx5_priv *priv = dev->data->dev_private; 4406 struct mlx5_flow_mreg_copy_resource *mcp_res; 4407 uint32_t idx = 0; 4408 4409 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4410 if (!mcp_res) { 4411 rte_errno = ENOMEM; 4412 return NULL; 4413 } 4414 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4415 mcp_res->idx = idx; 4416 return &mcp_res->hlist_ent; 4417 } 4418 4419 void 4420 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4421 { 4422 struct mlx5_flow_mreg_copy_resource *mcp_res = 4423 container_of(entry, typeof(*mcp_res), hlist_ent); 4424 struct rte_eth_dev *dev = tool_ctx; 4425 struct mlx5_priv *priv = dev->data->dev_private; 4426 4427 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4428 } 4429 4430 /** 4431 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4432 * 4433 * As mark_id is unique, if there's already a registered flow for the mark_id, 4434 * return by increasing the reference counter of the resource. Otherwise, create 4435 * the resource (mcp_res) and flow. 4436 * 4437 * Flow looks like, 4438 * - If ingress port is ANY and reg_c[1] is mark_id, 4439 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4440 * 4441 * For default flow (zero mark_id), flow is like, 4442 * - If ingress port is ANY, 4443 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4444 * 4445 * @param dev 4446 * Pointer to Ethernet device. 4447 * @param mark_id 4448 * ID of MARK action, zero means default flow for META. 4449 * @param[out] error 4450 * Perform verbose error reporting if not NULL. 4451 * 4452 * @return 4453 * Associated resource on success, NULL otherwise and rte_errno is set. 4454 */ 4455 static struct mlx5_flow_mreg_copy_resource * 4456 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4457 struct rte_flow_error *error) 4458 { 4459 struct mlx5_priv *priv = dev->data->dev_private; 4460 struct mlx5_list_entry *entry; 4461 struct mlx5_flow_cb_ctx ctx = { 4462 .dev = dev, 4463 .error = error, 4464 .data = &mark_id, 4465 }; 4466 4467 /* Check if already registered. */ 4468 MLX5_ASSERT(priv->mreg_cp_tbl); 4469 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4470 if (!entry) 4471 return NULL; 4472 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4473 hlist_ent); 4474 } 4475 4476 void 4477 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4478 { 4479 struct mlx5_flow_mreg_copy_resource *mcp_res = 4480 container_of(entry, typeof(*mcp_res), hlist_ent); 4481 struct rte_eth_dev *dev = tool_ctx; 4482 struct mlx5_priv *priv = dev->data->dev_private; 4483 4484 MLX5_ASSERT(mcp_res->rix_flow); 4485 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4486 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4487 } 4488 4489 /** 4490 * Release flow in RX_CP_TBL. 4491 * 4492 * @param dev 4493 * Pointer to Ethernet device. 4494 * @flow 4495 * Parent flow for wich copying is provided. 4496 */ 4497 static void 4498 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4499 struct rte_flow *flow) 4500 { 4501 struct mlx5_flow_mreg_copy_resource *mcp_res; 4502 struct mlx5_priv *priv = dev->data->dev_private; 4503 4504 if (!flow->rix_mreg_copy) 4505 return; 4506 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4507 flow->rix_mreg_copy); 4508 if (!mcp_res || !priv->mreg_cp_tbl) 4509 return; 4510 MLX5_ASSERT(mcp_res->rix_flow); 4511 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4512 flow->rix_mreg_copy = 0; 4513 } 4514 4515 /** 4516 * Remove the default copy action from RX_CP_TBL. 4517 * 4518 * This functions is called in the mlx5_dev_start(). No thread safe 4519 * is guaranteed. 4520 * 4521 * @param dev 4522 * Pointer to Ethernet device. 4523 */ 4524 static void 4525 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4526 { 4527 struct mlx5_list_entry *entry; 4528 struct mlx5_priv *priv = dev->data->dev_private; 4529 struct mlx5_flow_cb_ctx ctx; 4530 uint32_t mark_id; 4531 4532 /* Check if default flow is registered. */ 4533 if (!priv->mreg_cp_tbl) 4534 return; 4535 mark_id = MLX5_DEFAULT_COPY_ID; 4536 ctx.data = &mark_id; 4537 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4538 if (!entry) 4539 return; 4540 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4541 } 4542 4543 /** 4544 * Add the default copy action in in RX_CP_TBL. 4545 * 4546 * This functions is called in the mlx5_dev_start(). No thread safe 4547 * is guaranteed. 4548 * 4549 * @param dev 4550 * Pointer to Ethernet device. 4551 * @param[out] error 4552 * Perform verbose error reporting if not NULL. 4553 * 4554 * @return 4555 * 0 for success, negative value otherwise and rte_errno is set. 4556 */ 4557 static int 4558 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4559 struct rte_flow_error *error) 4560 { 4561 struct mlx5_priv *priv = dev->data->dev_private; 4562 struct mlx5_flow_mreg_copy_resource *mcp_res; 4563 struct mlx5_flow_cb_ctx ctx; 4564 uint32_t mark_id; 4565 4566 /* Check whether extensive metadata feature is engaged. */ 4567 if (!priv->sh->config.dv_flow_en || 4568 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4569 !mlx5_flow_ext_mreg_supported(dev) || 4570 !priv->sh->dv_regc0_mask) 4571 return 0; 4572 /* 4573 * Add default mreg copy flow may be called multiple time, but 4574 * only be called once in stop. Avoid register it twice. 4575 */ 4576 mark_id = MLX5_DEFAULT_COPY_ID; 4577 ctx.data = &mark_id; 4578 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4579 return 0; 4580 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4581 if (!mcp_res) 4582 return -rte_errno; 4583 return 0; 4584 } 4585 4586 /** 4587 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4588 * 4589 * All the flow having Q/RSS action should be split by 4590 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4591 * performs the following, 4592 * - CQE->flow_tag := reg_c[1] (MARK) 4593 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4594 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4595 * but there should be a flow per each MARK ID set by MARK action. 4596 * 4597 * For the aforementioned reason, if there's a MARK action in flow's action 4598 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4599 * the MARK ID to CQE's flow_tag like, 4600 * - If reg_c[1] is mark_id, 4601 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4602 * 4603 * For SET_META action which stores value in reg_c[0], as the destination is 4604 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4605 * MARK ID means the default flow. The default flow looks like, 4606 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4607 * 4608 * @param dev 4609 * Pointer to Ethernet device. 4610 * @param flow 4611 * Pointer to flow structure. 4612 * @param[in] actions 4613 * Pointer to the list of actions. 4614 * @param[out] error 4615 * Perform verbose error reporting if not NULL. 4616 * 4617 * @return 4618 * 0 on success, negative value otherwise and rte_errno is set. 4619 */ 4620 static int 4621 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4622 struct rte_flow *flow, 4623 const struct rte_flow_action *actions, 4624 struct rte_flow_error *error) 4625 { 4626 struct mlx5_priv *priv = dev->data->dev_private; 4627 struct mlx5_sh_config *config = &priv->sh->config; 4628 struct mlx5_flow_mreg_copy_resource *mcp_res; 4629 const struct rte_flow_action_mark *mark; 4630 4631 /* Check whether extensive metadata feature is engaged. */ 4632 if (!config->dv_flow_en || 4633 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4634 !mlx5_flow_ext_mreg_supported(dev) || 4635 !priv->sh->dv_regc0_mask) 4636 return 0; 4637 /* Find MARK action. */ 4638 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4639 switch (actions->type) { 4640 case RTE_FLOW_ACTION_TYPE_FLAG: 4641 mcp_res = flow_mreg_add_copy_action 4642 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4643 if (!mcp_res) 4644 return -rte_errno; 4645 flow->rix_mreg_copy = mcp_res->idx; 4646 return 0; 4647 case RTE_FLOW_ACTION_TYPE_MARK: 4648 mark = (const struct rte_flow_action_mark *) 4649 actions->conf; 4650 mcp_res = 4651 flow_mreg_add_copy_action(dev, mark->id, error); 4652 if (!mcp_res) 4653 return -rte_errno; 4654 flow->rix_mreg_copy = mcp_res->idx; 4655 return 0; 4656 default: 4657 break; 4658 } 4659 } 4660 return 0; 4661 } 4662 4663 #define MLX5_MAX_SPLIT_ACTIONS 24 4664 #define MLX5_MAX_SPLIT_ITEMS 24 4665 4666 /** 4667 * Split the hairpin flow. 4668 * Since HW can't support encap and push-vlan on Rx, we move these 4669 * actions to Tx. 4670 * If the count action is after the encap then we also 4671 * move the count action. in this case the count will also measure 4672 * the outer bytes. 4673 * 4674 * @param dev 4675 * Pointer to Ethernet device. 4676 * @param[in] actions 4677 * Associated actions (list terminated by the END action). 4678 * @param[out] actions_rx 4679 * Rx flow actions. 4680 * @param[out] actions_tx 4681 * Tx flow actions.. 4682 * @param[out] pattern_tx 4683 * The pattern items for the Tx flow. 4684 * @param[out] flow_id 4685 * The flow ID connected to this flow. 4686 * 4687 * @return 4688 * 0 on success. 4689 */ 4690 static int 4691 flow_hairpin_split(struct rte_eth_dev *dev, 4692 const struct rte_flow_action actions[], 4693 struct rte_flow_action actions_rx[], 4694 struct rte_flow_action actions_tx[], 4695 struct rte_flow_item pattern_tx[], 4696 uint32_t flow_id) 4697 { 4698 const struct rte_flow_action_raw_encap *raw_encap; 4699 const struct rte_flow_action_raw_decap *raw_decap; 4700 struct mlx5_rte_flow_action_set_tag *set_tag; 4701 struct rte_flow_action *tag_action; 4702 struct mlx5_rte_flow_item_tag *tag_item; 4703 struct rte_flow_item *item; 4704 char *addr; 4705 int encap = 0; 4706 4707 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4708 switch (actions->type) { 4709 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4710 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4711 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4712 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4713 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4714 rte_memcpy(actions_tx, actions, 4715 sizeof(struct rte_flow_action)); 4716 actions_tx++; 4717 break; 4718 case RTE_FLOW_ACTION_TYPE_COUNT: 4719 if (encap) { 4720 rte_memcpy(actions_tx, actions, 4721 sizeof(struct rte_flow_action)); 4722 actions_tx++; 4723 } else { 4724 rte_memcpy(actions_rx, actions, 4725 sizeof(struct rte_flow_action)); 4726 actions_rx++; 4727 } 4728 break; 4729 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4730 raw_encap = actions->conf; 4731 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4732 memcpy(actions_tx, actions, 4733 sizeof(struct rte_flow_action)); 4734 actions_tx++; 4735 encap = 1; 4736 } else { 4737 rte_memcpy(actions_rx, actions, 4738 sizeof(struct rte_flow_action)); 4739 actions_rx++; 4740 } 4741 break; 4742 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4743 raw_decap = actions->conf; 4744 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4745 memcpy(actions_tx, actions, 4746 sizeof(struct rte_flow_action)); 4747 actions_tx++; 4748 } else { 4749 rte_memcpy(actions_rx, actions, 4750 sizeof(struct rte_flow_action)); 4751 actions_rx++; 4752 } 4753 break; 4754 default: 4755 rte_memcpy(actions_rx, actions, 4756 sizeof(struct rte_flow_action)); 4757 actions_rx++; 4758 break; 4759 } 4760 } 4761 /* Add set meta action and end action for the Rx flow. */ 4762 tag_action = actions_rx; 4763 tag_action->type = (enum rte_flow_action_type) 4764 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4765 actions_rx++; 4766 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4767 actions_rx++; 4768 set_tag = (void *)actions_rx; 4769 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4770 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4771 .data = flow_id, 4772 }; 4773 MLX5_ASSERT(set_tag->id > REG_NON); 4774 tag_action->conf = set_tag; 4775 /* Create Tx item list. */ 4776 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4777 addr = (void *)&pattern_tx[2]; 4778 item = pattern_tx; 4779 item->type = (enum rte_flow_item_type) 4780 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4781 tag_item = (void *)addr; 4782 tag_item->data = flow_id; 4783 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4784 MLX5_ASSERT(set_tag->id > REG_NON); 4785 item->spec = tag_item; 4786 addr += sizeof(struct mlx5_rte_flow_item_tag); 4787 tag_item = (void *)addr; 4788 tag_item->data = UINT32_MAX; 4789 tag_item->id = UINT16_MAX; 4790 item->mask = tag_item; 4791 item->last = NULL; 4792 item++; 4793 item->type = RTE_FLOW_ITEM_TYPE_END; 4794 return 0; 4795 } 4796 4797 /** 4798 * The last stage of splitting chain, just creates the subflow 4799 * without any modification. 4800 * 4801 * @param[in] dev 4802 * Pointer to Ethernet device. 4803 * @param[in] flow 4804 * Parent flow structure pointer. 4805 * @param[in, out] sub_flow 4806 * Pointer to return the created subflow, may be NULL. 4807 * @param[in] attr 4808 * Flow rule attributes. 4809 * @param[in] items 4810 * Pattern specification (list terminated by the END pattern item). 4811 * @param[in] actions 4812 * Associated actions (list terminated by the END action). 4813 * @param[in] flow_split_info 4814 * Pointer to flow split info structure. 4815 * @param[out] error 4816 * Perform verbose error reporting if not NULL. 4817 * @return 4818 * 0 on success, negative value otherwise 4819 */ 4820 static int 4821 flow_create_split_inner(struct rte_eth_dev *dev, 4822 struct rte_flow *flow, 4823 struct mlx5_flow **sub_flow, 4824 const struct rte_flow_attr *attr, 4825 const struct rte_flow_item items[], 4826 const struct rte_flow_action actions[], 4827 struct mlx5_flow_split_info *flow_split_info, 4828 struct rte_flow_error *error) 4829 { 4830 struct mlx5_flow *dev_flow; 4831 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 4832 4833 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4834 flow_split_info->flow_idx, error); 4835 if (!dev_flow) 4836 return -rte_errno; 4837 dev_flow->flow = flow; 4838 dev_flow->external = flow_split_info->external; 4839 dev_flow->skip_scale = flow_split_info->skip_scale; 4840 /* Subflow object was created, we must include one in the list. */ 4841 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4842 dev_flow->handle, next); 4843 /* 4844 * If dev_flow is as one of the suffix flow, some actions in suffix 4845 * flow may need some user defined item layer flags, and pass the 4846 * Metadata rxq mark flag to suffix flow as well. 4847 */ 4848 if (flow_split_info->prefix_layers) 4849 dev_flow->handle->layers = flow_split_info->prefix_layers; 4850 if (flow_split_info->prefix_mark) { 4851 MLX5_ASSERT(wks); 4852 wks->mark = 1; 4853 } 4854 if (sub_flow) 4855 *sub_flow = dev_flow; 4856 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4857 dev_flow->dv.table_id = flow_split_info->table_id; 4858 #endif 4859 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4860 } 4861 4862 /** 4863 * Get the sub policy of a meter. 4864 * 4865 * @param[in] dev 4866 * Pointer to Ethernet device. 4867 * @param[in] flow 4868 * Parent flow structure pointer. 4869 * @param wks 4870 * Pointer to thread flow work space. 4871 * @param[in] attr 4872 * Flow rule attributes. 4873 * @param[in] items 4874 * Pattern specification (list terminated by the END pattern item). 4875 * @param[out] error 4876 * Perform verbose error reporting if not NULL. 4877 * 4878 * @return 4879 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4880 */ 4881 static struct mlx5_flow_meter_sub_policy * 4882 get_meter_sub_policy(struct rte_eth_dev *dev, 4883 struct rte_flow *flow, 4884 struct mlx5_flow_workspace *wks, 4885 const struct rte_flow_attr *attr, 4886 const struct rte_flow_item items[], 4887 struct rte_flow_error *error) 4888 { 4889 struct mlx5_flow_meter_policy *policy; 4890 struct mlx5_flow_meter_policy *final_policy; 4891 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4892 4893 policy = wks->policy; 4894 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4895 if (final_policy->is_rss || final_policy->is_queue) { 4896 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4897 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4898 uint32_t i; 4899 4900 /* 4901 * This is a tmp dev_flow, 4902 * no need to register any matcher for it in translate. 4903 */ 4904 wks->skip_matcher_reg = 1; 4905 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4906 struct mlx5_flow dev_flow = {0}; 4907 struct mlx5_flow_handle dev_handle = { {0} }; 4908 uint8_t fate = final_policy->act_cnt[i].fate_action; 4909 4910 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4911 const struct rte_flow_action_rss *rss_act = 4912 final_policy->act_cnt[i].rss->conf; 4913 struct rte_flow_action rss_actions[2] = { 4914 [0] = { 4915 .type = RTE_FLOW_ACTION_TYPE_RSS, 4916 .conf = rss_act, 4917 }, 4918 [1] = { 4919 .type = RTE_FLOW_ACTION_TYPE_END, 4920 .conf = NULL, 4921 } 4922 }; 4923 4924 dev_flow.handle = &dev_handle; 4925 dev_flow.ingress = attr->ingress; 4926 dev_flow.flow = flow; 4927 dev_flow.external = 0; 4928 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4929 dev_flow.dv.transfer = attr->transfer; 4930 #endif 4931 /** 4932 * Translate RSS action to get rss hash fields. 4933 */ 4934 if (flow_drv_translate(dev, &dev_flow, attr, 4935 items, rss_actions, error)) 4936 goto exit; 4937 rss_desc_v[i] = wks->rss_desc; 4938 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4939 rss_desc_v[i].hash_fields = 4940 dev_flow.hash_fields; 4941 rss_desc_v[i].queue_num = 4942 rss_desc_v[i].hash_fields ? 4943 rss_desc_v[i].queue_num : 1; 4944 rss_desc_v[i].tunnel = 4945 !!(dev_flow.handle->layers & 4946 MLX5_FLOW_LAYER_TUNNEL); 4947 /* Use the RSS queues in the containers. */ 4948 rss_desc_v[i].queue = 4949 (uint16_t *)(uintptr_t)rss_act->queue; 4950 rss_desc[i] = &rss_desc_v[i]; 4951 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 4952 /* This is queue action. */ 4953 rss_desc_v[i] = wks->rss_desc; 4954 rss_desc_v[i].key_len = 0; 4955 rss_desc_v[i].hash_fields = 0; 4956 rss_desc_v[i].queue = 4957 &final_policy->act_cnt[i].queue; 4958 rss_desc_v[i].queue_num = 1; 4959 rss_desc[i] = &rss_desc_v[i]; 4960 } else { 4961 rss_desc[i] = NULL; 4962 } 4963 } 4964 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4965 flow, policy, rss_desc); 4966 } else { 4967 enum mlx5_meter_domain mtr_domain = 4968 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4969 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4970 MLX5_MTR_DOMAIN_INGRESS); 4971 sub_policy = policy->sub_policys[mtr_domain][0]; 4972 } 4973 if (!sub_policy) 4974 rte_flow_error_set(error, EINVAL, 4975 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4976 "Failed to get meter sub-policy."); 4977 exit: 4978 return sub_policy; 4979 } 4980 4981 /** 4982 * Split the meter flow. 4983 * 4984 * As meter flow will split to three sub flow, other than meter 4985 * action, the other actions make sense to only meter accepts 4986 * the packet. If it need to be dropped, no other additional 4987 * actions should be take. 4988 * 4989 * One kind of special action which decapsulates the L3 tunnel 4990 * header will be in the prefix sub flow, as not to take the 4991 * L3 tunnel header into account. 4992 * 4993 * @param[in] dev 4994 * Pointer to Ethernet device. 4995 * @param[in] flow 4996 * Parent flow structure pointer. 4997 * @param wks 4998 * Pointer to thread flow work space. 4999 * @param[in] attr 5000 * Flow rule attributes. 5001 * @param[in] items 5002 * Pattern specification (list terminated by the END pattern item). 5003 * @param[out] sfx_items 5004 * Suffix flow match items (list terminated by the END pattern item). 5005 * @param[in] actions 5006 * Associated actions (list terminated by the END action). 5007 * @param[out] actions_sfx 5008 * Suffix flow actions. 5009 * @param[out] actions_pre 5010 * Prefix flow actions. 5011 * @param[out] mtr_flow_id 5012 * Pointer to meter flow id. 5013 * @param[out] error 5014 * Perform verbose error reporting if not NULL. 5015 * 5016 * @return 5017 * 0 on success, a negative errno value otherwise and rte_errno is set. 5018 */ 5019 static int 5020 flow_meter_split_prep(struct rte_eth_dev *dev, 5021 struct rte_flow *flow, 5022 struct mlx5_flow_workspace *wks, 5023 const struct rte_flow_attr *attr, 5024 const struct rte_flow_item items[], 5025 struct rte_flow_item sfx_items[], 5026 const struct rte_flow_action actions[], 5027 struct rte_flow_action actions_sfx[], 5028 struct rte_flow_action actions_pre[], 5029 uint32_t *mtr_flow_id, 5030 struct rte_flow_error *error) 5031 { 5032 struct mlx5_priv *priv = dev->data->dev_private; 5033 struct mlx5_flow_meter_info *fm = wks->fm; 5034 struct rte_flow_action *tag_action = NULL; 5035 struct rte_flow_item *tag_item; 5036 struct mlx5_rte_flow_action_set_tag *set_tag; 5037 const struct rte_flow_action_raw_encap *raw_encap; 5038 const struct rte_flow_action_raw_decap *raw_decap; 5039 struct mlx5_rte_flow_item_tag *tag_item_spec; 5040 struct mlx5_rte_flow_item_tag *tag_item_mask; 5041 uint32_t tag_id = 0; 5042 struct rte_flow_item *vlan_item_dst = NULL; 5043 const struct rte_flow_item *vlan_item_src = NULL; 5044 const struct rte_flow_item *orig_items = items; 5045 struct rte_flow_action *hw_mtr_action; 5046 struct rte_flow_action *action_pre_head = NULL; 5047 int32_t flow_src_port = priv->representor_id; 5048 bool mtr_first; 5049 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5050 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5051 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5052 uint32_t flow_id = 0; 5053 uint32_t flow_id_reversed = 0; 5054 uint8_t flow_id_bits = 0; 5055 int shift; 5056 5057 /* Prepare the suffix subflow items. */ 5058 tag_item = sfx_items++; 5059 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5060 struct mlx5_priv *port_priv; 5061 const struct rte_flow_item_port_id *pid_v; 5062 int item_type = items->type; 5063 5064 switch (item_type) { 5065 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5066 pid_v = items->spec; 5067 MLX5_ASSERT(pid_v); 5068 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5069 if (!port_priv) 5070 return rte_flow_error_set(error, 5071 rte_errno, 5072 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5073 pid_v, 5074 "Failed to get port info."); 5075 flow_src_port = port_priv->representor_id; 5076 if (!fm->def_policy && wks->policy->is_hierarchy && 5077 flow_src_port != priv->representor_id) { 5078 if (flow_drv_mtr_hierarchy_rule_create(dev, 5079 flow, fm, 5080 flow_src_port, 5081 items, 5082 error)) 5083 return -rte_errno; 5084 } 5085 memcpy(sfx_items, items, sizeof(*sfx_items)); 5086 sfx_items++; 5087 break; 5088 case RTE_FLOW_ITEM_TYPE_VLAN: 5089 /* Determine if copy vlan item below. */ 5090 vlan_item_src = items; 5091 vlan_item_dst = sfx_items++; 5092 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5093 break; 5094 default: 5095 break; 5096 } 5097 } 5098 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5099 sfx_items++; 5100 mtr_first = priv->sh->meter_aso_en && 5101 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5102 /* For ASO meter, meter must be before tag in TX direction. */ 5103 if (mtr_first) { 5104 action_pre_head = actions_pre++; 5105 /* Leave space for tag action. */ 5106 tag_action = actions_pre++; 5107 } 5108 /* Prepare the actions for prefix and suffix flow. */ 5109 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5110 struct rte_flow_action *action_cur = NULL; 5111 5112 switch (actions->type) { 5113 case RTE_FLOW_ACTION_TYPE_METER: 5114 if (mtr_first) { 5115 action_cur = action_pre_head; 5116 } else { 5117 /* Leave space for tag action. */ 5118 tag_action = actions_pre++; 5119 action_cur = actions_pre++; 5120 } 5121 break; 5122 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5123 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5124 action_cur = actions_pre++; 5125 break; 5126 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5127 raw_encap = actions->conf; 5128 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5129 action_cur = actions_pre++; 5130 break; 5131 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5132 raw_decap = actions->conf; 5133 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5134 action_cur = actions_pre++; 5135 break; 5136 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5137 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5138 if (vlan_item_dst && vlan_item_src) { 5139 memcpy(vlan_item_dst, vlan_item_src, 5140 sizeof(*vlan_item_dst)); 5141 /* 5142 * Convert to internal match item, it is used 5143 * for vlan push and set vid. 5144 */ 5145 vlan_item_dst->type = (enum rte_flow_item_type) 5146 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5147 } 5148 break; 5149 default: 5150 break; 5151 } 5152 if (!action_cur) 5153 action_cur = (fm->def_policy) ? 5154 actions_sfx++ : actions_pre++; 5155 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5156 } 5157 /* Add end action to the actions. */ 5158 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5159 if (priv->sh->meter_aso_en) { 5160 /** 5161 * For ASO meter, need to add an extra jump action explicitly, 5162 * to jump from meter to policer table. 5163 */ 5164 struct mlx5_flow_meter_sub_policy *sub_policy; 5165 struct mlx5_flow_tbl_data_entry *tbl_data; 5166 5167 if (!fm->def_policy) { 5168 sub_policy = get_meter_sub_policy(dev, flow, wks, 5169 attr, orig_items, 5170 error); 5171 if (!sub_policy) 5172 return -rte_errno; 5173 } else { 5174 enum mlx5_meter_domain mtr_domain = 5175 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5176 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5177 MLX5_MTR_DOMAIN_INGRESS); 5178 5179 sub_policy = 5180 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5181 } 5182 tbl_data = container_of(sub_policy->tbl_rsc, 5183 struct mlx5_flow_tbl_data_entry, tbl); 5184 hw_mtr_action = actions_pre++; 5185 hw_mtr_action->type = (enum rte_flow_action_type) 5186 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5187 hw_mtr_action->conf = tbl_data->jump.action; 5188 } 5189 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5190 actions_pre++; 5191 if (!tag_action) 5192 return rte_flow_error_set(error, ENOMEM, 5193 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5194 NULL, "No tag action space."); 5195 if (!mtr_flow_id) { 5196 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5197 goto exit; 5198 } 5199 /* Only default-policy Meter creates mtr flow id. */ 5200 if (fm->def_policy) { 5201 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5202 if (!tag_id) 5203 return rte_flow_error_set(error, ENOMEM, 5204 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5205 "Failed to allocate meter flow id."); 5206 flow_id = tag_id - 1; 5207 flow_id_bits = (!flow_id) ? 1 : 5208 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5209 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5210 mtr_reg_bits) { 5211 mlx5_ipool_free(fm->flow_ipool, tag_id); 5212 return rte_flow_error_set(error, EINVAL, 5213 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5214 "Meter flow id exceeds max limit."); 5215 } 5216 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5217 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5218 } 5219 /* Build tag actions and items for meter_id/meter flow_id. */ 5220 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5221 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5222 tag_item_mask = tag_item_spec + 1; 5223 /* Both flow_id and meter_id share the same register. */ 5224 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5225 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5226 0, error), 5227 .offset = mtr_id_offset, 5228 .length = mtr_reg_bits, 5229 .data = flow->meter, 5230 }; 5231 /* 5232 * The color Reg bits used by flow_id are growing from 5233 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5234 */ 5235 for (shift = 0; shift < flow_id_bits; shift++) 5236 flow_id_reversed = (flow_id_reversed << 1) | 5237 ((flow_id >> shift) & 0x1); 5238 set_tag->data |= 5239 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5240 tag_item_spec->id = set_tag->id; 5241 tag_item_spec->data = set_tag->data << mtr_id_offset; 5242 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5243 tag_action->type = (enum rte_flow_action_type) 5244 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5245 tag_action->conf = set_tag; 5246 tag_item->type = (enum rte_flow_item_type) 5247 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5248 tag_item->spec = tag_item_spec; 5249 tag_item->last = NULL; 5250 tag_item->mask = tag_item_mask; 5251 exit: 5252 if (mtr_flow_id) 5253 *mtr_flow_id = tag_id; 5254 return 0; 5255 } 5256 5257 /** 5258 * Split action list having QUEUE/RSS for metadata register copy. 5259 * 5260 * Once Q/RSS action is detected in user's action list, the flow action 5261 * should be split in order to copy metadata registers, which will happen in 5262 * RX_CP_TBL like, 5263 * - CQE->flow_tag := reg_c[1] (MARK) 5264 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5265 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5266 * This is because the last action of each flow must be a terminal action 5267 * (QUEUE, RSS or DROP). 5268 * 5269 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5270 * stored and kept in the mlx5_flow structure per each sub_flow. 5271 * 5272 * The Q/RSS action is replaced with, 5273 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5274 * And the following JUMP action is added at the end, 5275 * - JUMP, to RX_CP_TBL. 5276 * 5277 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5278 * flow_create_split_metadata() routine. The flow will look like, 5279 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5280 * 5281 * @param dev 5282 * Pointer to Ethernet device. 5283 * @param[out] split_actions 5284 * Pointer to store split actions to jump to CP_TBL. 5285 * @param[in] actions 5286 * Pointer to the list of original flow actions. 5287 * @param[in] qrss 5288 * Pointer to the Q/RSS action. 5289 * @param[in] actions_n 5290 * Number of original actions. 5291 * @param[in] mtr_sfx 5292 * Check if it is in meter suffix table. 5293 * @param[out] error 5294 * Perform verbose error reporting if not NULL. 5295 * 5296 * @return 5297 * non-zero unique flow_id on success, otherwise 0 and 5298 * error/rte_error are set. 5299 */ 5300 static uint32_t 5301 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5302 struct rte_flow_action *split_actions, 5303 const struct rte_flow_action *actions, 5304 const struct rte_flow_action *qrss, 5305 int actions_n, int mtr_sfx, 5306 struct rte_flow_error *error) 5307 { 5308 struct mlx5_priv *priv = dev->data->dev_private; 5309 struct mlx5_rte_flow_action_set_tag *set_tag; 5310 struct rte_flow_action_jump *jump; 5311 const int qrss_idx = qrss - actions; 5312 uint32_t flow_id = 0; 5313 int ret = 0; 5314 5315 /* 5316 * Given actions will be split 5317 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5318 * - Add jump to mreg CP_TBL. 5319 * As a result, there will be one more action. 5320 */ 5321 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5322 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5323 ++actions_n; 5324 set_tag = (void *)(split_actions + actions_n); 5325 /* 5326 * If we are not the meter suffix flow, add the tag action. 5327 * Since meter suffix flow already has the tag added. 5328 */ 5329 if (!mtr_sfx) { 5330 /* 5331 * Allocate the new subflow ID. This one is unique within 5332 * device and not shared with representors. Otherwise, 5333 * we would have to resolve multi-thread access synch 5334 * issue. Each flow on the shared device is appended 5335 * with source vport identifier, so the resulting 5336 * flows will be unique in the shared (by master and 5337 * representors) domain even if they have coinciding 5338 * IDs. 5339 */ 5340 mlx5_ipool_malloc(priv->sh->ipool 5341 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5342 if (!flow_id) 5343 return rte_flow_error_set(error, ENOMEM, 5344 RTE_FLOW_ERROR_TYPE_ACTION, 5345 NULL, "can't allocate id " 5346 "for split Q/RSS subflow"); 5347 /* Internal SET_TAG action to set flow ID. */ 5348 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5349 .data = flow_id, 5350 }; 5351 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5352 if (ret < 0) 5353 return ret; 5354 set_tag->id = ret; 5355 /* Construct new actions array. */ 5356 /* Replace QUEUE/RSS action. */ 5357 split_actions[qrss_idx] = (struct rte_flow_action){ 5358 .type = (enum rte_flow_action_type) 5359 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5360 .conf = set_tag, 5361 }; 5362 } else { 5363 /* 5364 * If we are the suffix flow of meter, tag already exist. 5365 * Set the QUEUE/RSS action to void. 5366 */ 5367 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5368 } 5369 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5370 jump = (void *)(set_tag + 1); 5371 *jump = (struct rte_flow_action_jump){ 5372 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5373 }; 5374 split_actions[actions_n - 2] = (struct rte_flow_action){ 5375 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5376 .conf = jump, 5377 }; 5378 split_actions[actions_n - 1] = (struct rte_flow_action){ 5379 .type = RTE_FLOW_ACTION_TYPE_END, 5380 }; 5381 return flow_id; 5382 } 5383 5384 /** 5385 * Extend the given action list for Tx metadata copy. 5386 * 5387 * Copy the given action list to the ext_actions and add flow metadata register 5388 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5389 * 5390 * @param[out] ext_actions 5391 * Pointer to the extended action list. 5392 * @param[in] actions 5393 * Pointer to the list of actions. 5394 * @param[in] actions_n 5395 * Number of actions in the list. 5396 * @param[out] error 5397 * Perform verbose error reporting if not NULL. 5398 * @param[in] encap_idx 5399 * The encap action index. 5400 * 5401 * @return 5402 * 0 on success, negative value otherwise 5403 */ 5404 static int 5405 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5406 struct rte_flow_action *ext_actions, 5407 const struct rte_flow_action *actions, 5408 int actions_n, struct rte_flow_error *error, 5409 int encap_idx) 5410 { 5411 struct mlx5_flow_action_copy_mreg *cp_mreg = 5412 (struct mlx5_flow_action_copy_mreg *) 5413 (ext_actions + actions_n + 1); 5414 int ret; 5415 5416 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5417 if (ret < 0) 5418 return ret; 5419 cp_mreg->dst = ret; 5420 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5421 if (ret < 0) 5422 return ret; 5423 cp_mreg->src = ret; 5424 if (encap_idx != 0) 5425 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5426 if (encap_idx == actions_n - 1) { 5427 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5428 .type = (enum rte_flow_action_type) 5429 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5430 .conf = cp_mreg, 5431 }; 5432 ext_actions[actions_n] = (struct rte_flow_action){ 5433 .type = RTE_FLOW_ACTION_TYPE_END, 5434 }; 5435 } else { 5436 ext_actions[encap_idx] = (struct rte_flow_action){ 5437 .type = (enum rte_flow_action_type) 5438 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5439 .conf = cp_mreg, 5440 }; 5441 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5442 sizeof(*ext_actions) * (actions_n - encap_idx)); 5443 } 5444 return 0; 5445 } 5446 5447 /** 5448 * Check the match action from the action list. 5449 * 5450 * @param[in] actions 5451 * Pointer to the list of actions. 5452 * @param[in] attr 5453 * Flow rule attributes. 5454 * @param[in] action 5455 * The action to be check if exist. 5456 * @param[out] match_action_pos 5457 * Pointer to the position of the matched action if exists, otherwise is -1. 5458 * @param[out] qrss_action_pos 5459 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5460 * @param[out] modify_after_mirror 5461 * Pointer to the flag of modify action after FDB mirroring. 5462 * 5463 * @return 5464 * > 0 the total number of actions. 5465 * 0 if not found match action in action list. 5466 */ 5467 static int 5468 flow_check_match_action(const struct rte_flow_action actions[], 5469 const struct rte_flow_attr *attr, 5470 enum rte_flow_action_type action, 5471 int *match_action_pos, int *qrss_action_pos, 5472 int *modify_after_mirror) 5473 { 5474 const struct rte_flow_action_sample *sample; 5475 const struct rte_flow_action_raw_decap *decap; 5476 int actions_n = 0; 5477 uint32_t ratio = 0; 5478 int sub_type = 0; 5479 int flag = 0; 5480 int fdb_mirror = 0; 5481 5482 *match_action_pos = -1; 5483 *qrss_action_pos = -1; 5484 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5485 if (actions->type == action) { 5486 flag = 1; 5487 *match_action_pos = actions_n; 5488 } 5489 switch (actions->type) { 5490 case RTE_FLOW_ACTION_TYPE_QUEUE: 5491 case RTE_FLOW_ACTION_TYPE_RSS: 5492 *qrss_action_pos = actions_n; 5493 break; 5494 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5495 sample = actions->conf; 5496 ratio = sample->ratio; 5497 sub_type = ((const struct rte_flow_action *) 5498 (sample->actions))->type; 5499 if (ratio == 1 && attr->transfer) 5500 fdb_mirror = 1; 5501 break; 5502 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5503 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5504 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5505 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5506 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5507 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5508 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5509 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5510 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5511 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5512 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5513 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5514 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5515 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5516 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5517 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5518 case RTE_FLOW_ACTION_TYPE_FLAG: 5519 case RTE_FLOW_ACTION_TYPE_MARK: 5520 case RTE_FLOW_ACTION_TYPE_SET_META: 5521 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5522 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5523 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5524 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5525 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5526 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5527 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5528 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5529 case RTE_FLOW_ACTION_TYPE_METER: 5530 if (fdb_mirror) 5531 *modify_after_mirror = 1; 5532 break; 5533 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5534 decap = actions->conf; 5535 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5536 ; 5537 actions_n++; 5538 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5539 const struct rte_flow_action_raw_encap *encap = 5540 actions->conf; 5541 if (decap->size <= 5542 MLX5_ENCAPSULATION_DECISION_SIZE && 5543 encap->size > 5544 MLX5_ENCAPSULATION_DECISION_SIZE) 5545 /* L3 encap. */ 5546 break; 5547 } 5548 if (fdb_mirror) 5549 *modify_after_mirror = 1; 5550 break; 5551 default: 5552 break; 5553 } 5554 actions_n++; 5555 } 5556 if (flag && fdb_mirror && !*modify_after_mirror) { 5557 /* FDB mirroring uses the destination array to implement 5558 * instead of FLOW_SAMPLER object. 5559 */ 5560 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5561 flag = 0; 5562 } 5563 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5564 return flag ? actions_n + 1 : 0; 5565 } 5566 5567 #define SAMPLE_SUFFIX_ITEM 2 5568 5569 /** 5570 * Split the sample flow. 5571 * 5572 * As sample flow will split to two sub flow, sample flow with 5573 * sample action, the other actions will move to new suffix flow. 5574 * 5575 * Also add unique tag id with tag action in the sample flow, 5576 * the same tag id will be as match in the suffix flow. 5577 * 5578 * @param dev 5579 * Pointer to Ethernet device. 5580 * @param[in] add_tag 5581 * Add extra tag action flag. 5582 * @param[out] sfx_items 5583 * Suffix flow match items (list terminated by the END pattern item). 5584 * @param[in] actions 5585 * Associated actions (list terminated by the END action). 5586 * @param[out] actions_sfx 5587 * Suffix flow actions. 5588 * @param[out] actions_pre 5589 * Prefix flow actions. 5590 * @param[in] actions_n 5591 * The total number of actions. 5592 * @param[in] sample_action_pos 5593 * The sample action position. 5594 * @param[in] qrss_action_pos 5595 * The Queue/RSS action position. 5596 * @param[in] jump_table 5597 * Add extra jump action flag. 5598 * @param[out] error 5599 * Perform verbose error reporting if not NULL. 5600 * 5601 * @return 5602 * 0 on success, or unique flow_id, a negative errno value 5603 * otherwise and rte_errno is set. 5604 */ 5605 static int 5606 flow_sample_split_prep(struct rte_eth_dev *dev, 5607 int add_tag, 5608 struct rte_flow_item sfx_items[], 5609 const struct rte_flow_action actions[], 5610 struct rte_flow_action actions_sfx[], 5611 struct rte_flow_action actions_pre[], 5612 int actions_n, 5613 int sample_action_pos, 5614 int qrss_action_pos, 5615 int jump_table, 5616 struct rte_flow_error *error) 5617 { 5618 struct mlx5_priv *priv = dev->data->dev_private; 5619 struct mlx5_rte_flow_action_set_tag *set_tag; 5620 struct mlx5_rte_flow_item_tag *tag_spec; 5621 struct mlx5_rte_flow_item_tag *tag_mask; 5622 struct rte_flow_action_jump *jump_action; 5623 uint32_t tag_id = 0; 5624 int index; 5625 int append_index = 0; 5626 int ret; 5627 5628 if (sample_action_pos < 0) 5629 return rte_flow_error_set(error, EINVAL, 5630 RTE_FLOW_ERROR_TYPE_ACTION, 5631 NULL, "invalid position of sample " 5632 "action in list"); 5633 /* Prepare the actions for prefix and suffix flow. */ 5634 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5635 index = qrss_action_pos; 5636 /* Put the preceding the Queue/RSS action into prefix flow. */ 5637 if (index != 0) 5638 memcpy(actions_pre, actions, 5639 sizeof(struct rte_flow_action) * index); 5640 /* Put others preceding the sample action into prefix flow. */ 5641 if (sample_action_pos > index + 1) 5642 memcpy(actions_pre + index, actions + index + 1, 5643 sizeof(struct rte_flow_action) * 5644 (sample_action_pos - index - 1)); 5645 index = sample_action_pos - 1; 5646 /* Put Queue/RSS action into Suffix flow. */ 5647 memcpy(actions_sfx, actions + qrss_action_pos, 5648 sizeof(struct rte_flow_action)); 5649 actions_sfx++; 5650 } else { 5651 index = sample_action_pos; 5652 if (index != 0) 5653 memcpy(actions_pre, actions, 5654 sizeof(struct rte_flow_action) * index); 5655 } 5656 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5657 * For CX6DX and above, metadata registers Cx preserve their value, 5658 * add an extra tag action for NIC-RX and E-Switch Domain. 5659 */ 5660 if (add_tag) { 5661 /* Prepare the prefix tag action. */ 5662 append_index++; 5663 set_tag = (void *)(actions_pre + actions_n + append_index); 5664 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5665 if (ret < 0) 5666 return ret; 5667 mlx5_ipool_malloc(priv->sh->ipool 5668 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5669 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5670 .id = ret, 5671 .data = tag_id, 5672 }; 5673 /* Prepare the suffix subflow items. */ 5674 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5675 tag_spec->data = tag_id; 5676 tag_spec->id = set_tag->id; 5677 tag_mask = tag_spec + 1; 5678 tag_mask->data = UINT32_MAX; 5679 sfx_items[0] = (struct rte_flow_item){ 5680 .type = (enum rte_flow_item_type) 5681 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5682 .spec = tag_spec, 5683 .last = NULL, 5684 .mask = tag_mask, 5685 }; 5686 sfx_items[1] = (struct rte_flow_item){ 5687 .type = (enum rte_flow_item_type) 5688 RTE_FLOW_ITEM_TYPE_END, 5689 }; 5690 /* Prepare the tag action in prefix subflow. */ 5691 actions_pre[index++] = 5692 (struct rte_flow_action){ 5693 .type = (enum rte_flow_action_type) 5694 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5695 .conf = set_tag, 5696 }; 5697 } 5698 memcpy(actions_pre + index, actions + sample_action_pos, 5699 sizeof(struct rte_flow_action)); 5700 index += 1; 5701 /* For the modify action after the sample action in E-Switch mirroring, 5702 * Add the extra jump action in prefix subflow and jump into the next 5703 * table, then do the modify action in the new table. 5704 */ 5705 if (jump_table) { 5706 /* Prepare the prefix jump action. */ 5707 append_index++; 5708 jump_action = (void *)(actions_pre + actions_n + append_index); 5709 jump_action->group = jump_table; 5710 actions_pre[index++] = 5711 (struct rte_flow_action){ 5712 .type = (enum rte_flow_action_type) 5713 RTE_FLOW_ACTION_TYPE_JUMP, 5714 .conf = jump_action, 5715 }; 5716 } 5717 actions_pre[index] = (struct rte_flow_action){ 5718 .type = (enum rte_flow_action_type) 5719 RTE_FLOW_ACTION_TYPE_END, 5720 }; 5721 /* Put the actions after sample into Suffix flow. */ 5722 memcpy(actions_sfx, actions + sample_action_pos + 1, 5723 sizeof(struct rte_flow_action) * 5724 (actions_n - sample_action_pos - 1)); 5725 return tag_id; 5726 } 5727 5728 /** 5729 * The splitting for metadata feature. 5730 * 5731 * - Q/RSS action on NIC Rx should be split in order to pass by 5732 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5733 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5734 * 5735 * - All the actions on NIC Tx should have a mreg copy action to 5736 * copy reg_a from WQE to reg_c[0]. 5737 * 5738 * @param dev 5739 * Pointer to Ethernet device. 5740 * @param[in] flow 5741 * Parent flow structure pointer. 5742 * @param[in] attr 5743 * Flow rule attributes. 5744 * @param[in] items 5745 * Pattern specification (list terminated by the END pattern item). 5746 * @param[in] actions 5747 * Associated actions (list terminated by the END action). 5748 * @param[in] flow_split_info 5749 * Pointer to flow split info structure. 5750 * @param[out] error 5751 * Perform verbose error reporting if not NULL. 5752 * @return 5753 * 0 on success, negative value otherwise 5754 */ 5755 static int 5756 flow_create_split_metadata(struct rte_eth_dev *dev, 5757 struct rte_flow *flow, 5758 const struct rte_flow_attr *attr, 5759 const struct rte_flow_item items[], 5760 const struct rte_flow_action actions[], 5761 struct mlx5_flow_split_info *flow_split_info, 5762 struct rte_flow_error *error) 5763 { 5764 struct mlx5_priv *priv = dev->data->dev_private; 5765 struct mlx5_sh_config *config = &priv->sh->config; 5766 const struct rte_flow_action *qrss = NULL; 5767 struct rte_flow_action *ext_actions = NULL; 5768 struct mlx5_flow *dev_flow = NULL; 5769 uint32_t qrss_id = 0; 5770 int mtr_sfx = 0; 5771 size_t act_size; 5772 int actions_n; 5773 int encap_idx; 5774 int ret; 5775 5776 /* Check whether extensive metadata feature is engaged. */ 5777 if (!config->dv_flow_en || 5778 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5779 !mlx5_flow_ext_mreg_supported(dev)) 5780 return flow_create_split_inner(dev, flow, NULL, attr, items, 5781 actions, flow_split_info, error); 5782 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5783 &encap_idx); 5784 if (qrss) { 5785 /* Exclude hairpin flows from splitting. */ 5786 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5787 const struct rte_flow_action_queue *queue; 5788 5789 queue = qrss->conf; 5790 if (mlx5_rxq_get_type(dev, queue->index) == 5791 MLX5_RXQ_TYPE_HAIRPIN) 5792 qrss = NULL; 5793 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5794 const struct rte_flow_action_rss *rss; 5795 5796 rss = qrss->conf; 5797 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5798 MLX5_RXQ_TYPE_HAIRPIN) 5799 qrss = NULL; 5800 } 5801 } 5802 if (qrss) { 5803 /* Check if it is in meter suffix table. */ 5804 mtr_sfx = attr->group == (attr->transfer ? 5805 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5806 MLX5_FLOW_TABLE_LEVEL_METER); 5807 /* 5808 * Q/RSS action on NIC Rx should be split in order to pass by 5809 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5810 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5811 */ 5812 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5813 sizeof(struct rte_flow_action_set_tag) + 5814 sizeof(struct rte_flow_action_jump); 5815 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5816 SOCKET_ID_ANY); 5817 if (!ext_actions) 5818 return rte_flow_error_set(error, ENOMEM, 5819 RTE_FLOW_ERROR_TYPE_ACTION, 5820 NULL, "no memory to split " 5821 "metadata flow"); 5822 /* 5823 * Create the new actions list with removed Q/RSS action 5824 * and appended set tag and jump to register copy table 5825 * (RX_CP_TBL). We should preallocate unique tag ID here 5826 * in advance, because it is needed for set tag action. 5827 */ 5828 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5829 qrss, actions_n, 5830 mtr_sfx, error); 5831 if (!mtr_sfx && !qrss_id) { 5832 ret = -rte_errno; 5833 goto exit; 5834 } 5835 } else if (attr->egress && !attr->transfer) { 5836 /* 5837 * All the actions on NIC Tx should have a metadata register 5838 * copy action to copy reg_a from WQE to reg_c[meta] 5839 */ 5840 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5841 sizeof(struct mlx5_flow_action_copy_mreg); 5842 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5843 SOCKET_ID_ANY); 5844 if (!ext_actions) 5845 return rte_flow_error_set(error, ENOMEM, 5846 RTE_FLOW_ERROR_TYPE_ACTION, 5847 NULL, "no memory to split " 5848 "metadata flow"); 5849 /* Create the action list appended with copy register. */ 5850 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5851 actions_n, error, encap_idx); 5852 if (ret < 0) 5853 goto exit; 5854 } 5855 /* Add the unmodified original or prefix subflow. */ 5856 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5857 items, ext_actions ? ext_actions : 5858 actions, flow_split_info, error); 5859 if (ret < 0) 5860 goto exit; 5861 MLX5_ASSERT(dev_flow); 5862 if (qrss) { 5863 const struct rte_flow_attr q_attr = { 5864 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5865 .ingress = 1, 5866 }; 5867 /* Internal PMD action to set register. */ 5868 struct mlx5_rte_flow_item_tag q_tag_spec = { 5869 .data = qrss_id, 5870 .id = REG_NON, 5871 }; 5872 struct rte_flow_item q_items[] = { 5873 { 5874 .type = (enum rte_flow_item_type) 5875 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5876 .spec = &q_tag_spec, 5877 .last = NULL, 5878 .mask = NULL, 5879 }, 5880 { 5881 .type = RTE_FLOW_ITEM_TYPE_END, 5882 }, 5883 }; 5884 struct rte_flow_action q_actions[] = { 5885 { 5886 .type = qrss->type, 5887 .conf = qrss->conf, 5888 }, 5889 { 5890 .type = RTE_FLOW_ACTION_TYPE_END, 5891 }, 5892 }; 5893 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5894 5895 /* 5896 * Configure the tag item only if there is no meter subflow. 5897 * Since tag is already marked in the meter suffix subflow 5898 * we can just use the meter suffix items as is. 5899 */ 5900 if (qrss_id) { 5901 /* Not meter subflow. */ 5902 MLX5_ASSERT(!mtr_sfx); 5903 /* 5904 * Put unique id in prefix flow due to it is destroyed 5905 * after suffix flow and id will be freed after there 5906 * is no actual flows with this id and identifier 5907 * reallocation becomes possible (for example, for 5908 * other flows in other threads). 5909 */ 5910 dev_flow->handle->split_flow_id = qrss_id; 5911 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5912 error); 5913 if (ret < 0) 5914 goto exit; 5915 q_tag_spec.id = ret; 5916 } 5917 dev_flow = NULL; 5918 /* Add suffix subflow to execute Q/RSS. */ 5919 flow_split_info->prefix_layers = layers; 5920 flow_split_info->prefix_mark = 0; 5921 flow_split_info->table_id = 0; 5922 ret = flow_create_split_inner(dev, flow, &dev_flow, 5923 &q_attr, mtr_sfx ? items : 5924 q_items, q_actions, 5925 flow_split_info, error); 5926 if (ret < 0) 5927 goto exit; 5928 /* qrss ID should be freed if failed. */ 5929 qrss_id = 0; 5930 MLX5_ASSERT(dev_flow); 5931 } 5932 5933 exit: 5934 /* 5935 * We do not destroy the partially created sub_flows in case of error. 5936 * These ones are included into parent flow list and will be destroyed 5937 * by flow_drv_destroy. 5938 */ 5939 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5940 qrss_id); 5941 mlx5_free(ext_actions); 5942 return ret; 5943 } 5944 5945 /** 5946 * Create meter internal drop flow with the original pattern. 5947 * 5948 * @param dev 5949 * Pointer to Ethernet device. 5950 * @param[in] flow 5951 * Parent flow structure pointer. 5952 * @param[in] attr 5953 * Flow rule attributes. 5954 * @param[in] items 5955 * Pattern specification (list terminated by the END pattern item). 5956 * @param[in] flow_split_info 5957 * Pointer to flow split info structure. 5958 * @param[in] fm 5959 * Pointer to flow meter structure. 5960 * @param[out] error 5961 * Perform verbose error reporting if not NULL. 5962 * @return 5963 * 0 on success, negative value otherwise 5964 */ 5965 static uint32_t 5966 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5967 struct rte_flow *flow, 5968 const struct rte_flow_attr *attr, 5969 const struct rte_flow_item items[], 5970 struct mlx5_flow_split_info *flow_split_info, 5971 struct mlx5_flow_meter_info *fm, 5972 struct rte_flow_error *error) 5973 { 5974 struct mlx5_flow *dev_flow = NULL; 5975 struct rte_flow_attr drop_attr = *attr; 5976 struct rte_flow_action drop_actions[3]; 5977 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5978 5979 MLX5_ASSERT(fm->drop_cnt); 5980 drop_actions[0].type = 5981 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5982 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5983 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5984 drop_actions[1].conf = NULL; 5985 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5986 drop_actions[2].conf = NULL; 5987 drop_split_info.external = false; 5988 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5989 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5990 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5991 return flow_create_split_inner(dev, flow, &dev_flow, 5992 &drop_attr, items, drop_actions, 5993 &drop_split_info, error); 5994 } 5995 5996 /** 5997 * The splitting for meter feature. 5998 * 5999 * - The meter flow will be split to two flows as prefix and 6000 * suffix flow. The packets make sense only it pass the prefix 6001 * meter action. 6002 * 6003 * - Reg_C_5 is used for the packet to match betweend prefix and 6004 * suffix flow. 6005 * 6006 * @param dev 6007 * Pointer to Ethernet device. 6008 * @param[in] flow 6009 * Parent flow structure pointer. 6010 * @param[in] attr 6011 * Flow rule attributes. 6012 * @param[in] items 6013 * Pattern specification (list terminated by the END pattern item). 6014 * @param[in] actions 6015 * Associated actions (list terminated by the END action). 6016 * @param[in] flow_split_info 6017 * Pointer to flow split info structure. 6018 * @param[out] error 6019 * Perform verbose error reporting if not NULL. 6020 * @return 6021 * 0 on success, negative value otherwise 6022 */ 6023 static int 6024 flow_create_split_meter(struct rte_eth_dev *dev, 6025 struct rte_flow *flow, 6026 const struct rte_flow_attr *attr, 6027 const struct rte_flow_item items[], 6028 const struct rte_flow_action actions[], 6029 struct mlx5_flow_split_info *flow_split_info, 6030 struct rte_flow_error *error) 6031 { 6032 struct mlx5_priv *priv = dev->data->dev_private; 6033 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6034 struct rte_flow_action *sfx_actions = NULL; 6035 struct rte_flow_action *pre_actions = NULL; 6036 struct rte_flow_item *sfx_items = NULL; 6037 struct mlx5_flow *dev_flow = NULL; 6038 struct rte_flow_attr sfx_attr = *attr; 6039 struct mlx5_flow_meter_info *fm = NULL; 6040 uint8_t skip_scale_restore; 6041 bool has_mtr = false; 6042 bool has_modify = false; 6043 bool set_mtr_reg = true; 6044 bool is_mtr_hierarchy = false; 6045 uint32_t meter_id = 0; 6046 uint32_t mtr_idx = 0; 6047 uint32_t mtr_flow_id = 0; 6048 size_t act_size; 6049 size_t item_size; 6050 int actions_n = 0; 6051 int ret = 0; 6052 6053 if (priv->mtr_en) 6054 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6055 &has_modify, &meter_id); 6056 if (has_mtr) { 6057 if (flow->meter) { 6058 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6059 if (!fm) 6060 return rte_flow_error_set(error, EINVAL, 6061 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6062 NULL, "Meter not found."); 6063 } else { 6064 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6065 if (!fm) 6066 return rte_flow_error_set(error, EINVAL, 6067 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6068 NULL, "Meter not found."); 6069 ret = mlx5_flow_meter_attach(priv, fm, 6070 &sfx_attr, error); 6071 if (ret) 6072 return -rte_errno; 6073 flow->meter = mtr_idx; 6074 } 6075 MLX5_ASSERT(wks); 6076 wks->fm = fm; 6077 if (!fm->def_policy) { 6078 wks->policy = mlx5_flow_meter_policy_find(dev, 6079 fm->policy_id, 6080 NULL); 6081 MLX5_ASSERT(wks->policy); 6082 if (wks->policy->is_hierarchy) { 6083 wks->final_policy = 6084 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6085 wks->policy); 6086 if (!wks->final_policy) 6087 return rte_flow_error_set(error, 6088 EINVAL, 6089 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6090 "Failed to find terminal policy of hierarchy."); 6091 is_mtr_hierarchy = true; 6092 } 6093 } 6094 /* 6095 * If it isn't default-policy Meter, and 6096 * 1. There's no action in flow to change 6097 * packet (modify/encap/decap etc.), OR 6098 * 2. No drop count needed for this meter. 6099 * 3. It's not meter hierarchy. 6100 * Then no need to use regC to save meter id anymore. 6101 */ 6102 if (!fm->def_policy && !is_mtr_hierarchy && 6103 (!has_modify || !fm->drop_cnt)) 6104 set_mtr_reg = false; 6105 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6106 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6107 sizeof(struct mlx5_rte_flow_action_set_tag); 6108 /* Suffix items: tag, vlan, port id, end. */ 6109 #define METER_SUFFIX_ITEM 4 6110 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6111 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6112 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6113 0, SOCKET_ID_ANY); 6114 if (!sfx_actions) 6115 return rte_flow_error_set(error, ENOMEM, 6116 RTE_FLOW_ERROR_TYPE_ACTION, 6117 NULL, "no memory to split " 6118 "meter flow"); 6119 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6120 act_size); 6121 /* There's no suffix flow for meter of non-default policy. */ 6122 if (!fm->def_policy) 6123 pre_actions = sfx_actions + 1; 6124 else 6125 pre_actions = sfx_actions + actions_n; 6126 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6127 items, sfx_items, actions, 6128 sfx_actions, pre_actions, 6129 (set_mtr_reg ? &mtr_flow_id : NULL), 6130 error); 6131 if (ret) { 6132 ret = -rte_errno; 6133 goto exit; 6134 } 6135 /* Add the prefix subflow. */ 6136 skip_scale_restore = flow_split_info->skip_scale; 6137 flow_split_info->skip_scale |= 6138 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6139 ret = flow_create_split_inner(dev, flow, &dev_flow, 6140 attr, items, pre_actions, 6141 flow_split_info, error); 6142 flow_split_info->skip_scale = skip_scale_restore; 6143 if (ret) { 6144 if (mtr_flow_id) 6145 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6146 ret = -rte_errno; 6147 goto exit; 6148 } 6149 if (mtr_flow_id) { 6150 dev_flow->handle->split_flow_id = mtr_flow_id; 6151 dev_flow->handle->is_meter_flow_id = 1; 6152 } 6153 if (!fm->def_policy) { 6154 if (!set_mtr_reg && fm->drop_cnt) 6155 ret = 6156 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6157 &sfx_attr, items, 6158 flow_split_info, 6159 fm, error); 6160 goto exit; 6161 } 6162 /* Setting the sfx group atrr. */ 6163 sfx_attr.group = sfx_attr.transfer ? 6164 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6165 MLX5_FLOW_TABLE_LEVEL_METER; 6166 flow_split_info->prefix_layers = 6167 flow_get_prefix_layer_flags(dev_flow); 6168 flow_split_info->prefix_mark |= wks->mark; 6169 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6170 } 6171 /* Add the prefix subflow. */ 6172 ret = flow_create_split_metadata(dev, flow, 6173 &sfx_attr, sfx_items ? 6174 sfx_items : items, 6175 sfx_actions ? sfx_actions : actions, 6176 flow_split_info, error); 6177 exit: 6178 if (sfx_actions) 6179 mlx5_free(sfx_actions); 6180 return ret; 6181 } 6182 6183 /** 6184 * The splitting for sample feature. 6185 * 6186 * Once Sample action is detected in the action list, the flow actions should 6187 * be split into prefix sub flow and suffix sub flow. 6188 * 6189 * The original items remain in the prefix sub flow, all actions preceding the 6190 * sample action and the sample action itself will be copied to the prefix 6191 * sub flow, the actions following the sample action will be copied to the 6192 * suffix sub flow, Queue action always be located in the suffix sub flow. 6193 * 6194 * In order to make the packet from prefix sub flow matches with suffix sub 6195 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6196 * flow uses tag item with the unique flow id. 6197 * 6198 * @param dev 6199 * Pointer to Ethernet device. 6200 * @param[in] flow 6201 * Parent flow structure pointer. 6202 * @param[in] attr 6203 * Flow rule attributes. 6204 * @param[in] items 6205 * Pattern specification (list terminated by the END pattern item). 6206 * @param[in] actions 6207 * Associated actions (list terminated by the END action). 6208 * @param[in] flow_split_info 6209 * Pointer to flow split info structure. 6210 * @param[out] error 6211 * Perform verbose error reporting if not NULL. 6212 * @return 6213 * 0 on success, negative value otherwise 6214 */ 6215 static int 6216 flow_create_split_sample(struct rte_eth_dev *dev, 6217 struct rte_flow *flow, 6218 const struct rte_flow_attr *attr, 6219 const struct rte_flow_item items[], 6220 const struct rte_flow_action actions[], 6221 struct mlx5_flow_split_info *flow_split_info, 6222 struct rte_flow_error *error) 6223 { 6224 struct mlx5_priv *priv = dev->data->dev_private; 6225 struct rte_flow_action *sfx_actions = NULL; 6226 struct rte_flow_action *pre_actions = NULL; 6227 struct rte_flow_item *sfx_items = NULL; 6228 struct mlx5_flow *dev_flow = NULL; 6229 struct rte_flow_attr sfx_attr = *attr; 6230 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6231 struct mlx5_flow_dv_sample_resource *sample_res; 6232 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6233 struct mlx5_flow_tbl_resource *sfx_tbl; 6234 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6235 #endif 6236 size_t act_size; 6237 size_t item_size; 6238 uint32_t fdb_tx = 0; 6239 int32_t tag_id = 0; 6240 int actions_n = 0; 6241 int sample_action_pos; 6242 int qrss_action_pos; 6243 int add_tag = 0; 6244 int modify_after_mirror = 0; 6245 uint16_t jump_table = 0; 6246 const uint32_t next_ft_step = 1; 6247 int ret = 0; 6248 6249 if (priv->sampler_en) 6250 actions_n = flow_check_match_action(actions, attr, 6251 RTE_FLOW_ACTION_TYPE_SAMPLE, 6252 &sample_action_pos, &qrss_action_pos, 6253 &modify_after_mirror); 6254 if (actions_n) { 6255 /* The prefix actions must includes sample, tag, end. */ 6256 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6257 + sizeof(struct mlx5_rte_flow_action_set_tag); 6258 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6259 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6260 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6261 item_size), 0, SOCKET_ID_ANY); 6262 if (!sfx_actions) 6263 return rte_flow_error_set(error, ENOMEM, 6264 RTE_FLOW_ERROR_TYPE_ACTION, 6265 NULL, "no memory to split " 6266 "sample flow"); 6267 /* The representor_id is UINT16_MAX for uplink. */ 6268 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6269 /* 6270 * When reg_c_preserve is set, metadata registers Cx preserve 6271 * their value even through packet duplication. 6272 */ 6273 add_tag = (!fdb_tx || 6274 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6275 if (add_tag) 6276 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6277 + act_size); 6278 if (modify_after_mirror) 6279 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6280 next_ft_step; 6281 pre_actions = sfx_actions + actions_n; 6282 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6283 actions, sfx_actions, 6284 pre_actions, actions_n, 6285 sample_action_pos, 6286 qrss_action_pos, jump_table, 6287 error); 6288 if (tag_id < 0 || (add_tag && !tag_id)) { 6289 ret = -rte_errno; 6290 goto exit; 6291 } 6292 if (modify_after_mirror) 6293 flow_split_info->skip_scale = 6294 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6295 /* Add the prefix subflow. */ 6296 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6297 items, pre_actions, 6298 flow_split_info, error); 6299 if (ret) { 6300 ret = -rte_errno; 6301 goto exit; 6302 } 6303 dev_flow->handle->split_flow_id = tag_id; 6304 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6305 if (!modify_after_mirror) { 6306 /* Set the sfx group attr. */ 6307 sample_res = (struct mlx5_flow_dv_sample_resource *) 6308 dev_flow->dv.sample_res; 6309 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6310 sample_res->normal_path_tbl; 6311 sfx_tbl_data = container_of(sfx_tbl, 6312 struct mlx5_flow_tbl_data_entry, 6313 tbl); 6314 sfx_attr.group = sfx_attr.transfer ? 6315 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6316 } else { 6317 MLX5_ASSERT(attr->transfer); 6318 sfx_attr.group = jump_table; 6319 } 6320 flow_split_info->prefix_layers = 6321 flow_get_prefix_layer_flags(dev_flow); 6322 MLX5_ASSERT(wks); 6323 flow_split_info->prefix_mark |= wks->mark; 6324 /* Suffix group level already be scaled with factor, set 6325 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6326 * again in translation. 6327 */ 6328 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6329 #endif 6330 } 6331 /* Add the suffix subflow. */ 6332 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6333 sfx_items ? sfx_items : items, 6334 sfx_actions ? sfx_actions : actions, 6335 flow_split_info, error); 6336 exit: 6337 if (sfx_actions) 6338 mlx5_free(sfx_actions); 6339 return ret; 6340 } 6341 6342 /** 6343 * Split the flow to subflow set. The splitters might be linked 6344 * in the chain, like this: 6345 * flow_create_split_outer() calls: 6346 * flow_create_split_meter() calls: 6347 * flow_create_split_metadata(meter_subflow_0) calls: 6348 * flow_create_split_inner(metadata_subflow_0) 6349 * flow_create_split_inner(metadata_subflow_1) 6350 * flow_create_split_inner(metadata_subflow_2) 6351 * flow_create_split_metadata(meter_subflow_1) calls: 6352 * flow_create_split_inner(metadata_subflow_0) 6353 * flow_create_split_inner(metadata_subflow_1) 6354 * flow_create_split_inner(metadata_subflow_2) 6355 * 6356 * This provide flexible way to add new levels of flow splitting. 6357 * The all of successfully created subflows are included to the 6358 * parent flow dev_flow list. 6359 * 6360 * @param dev 6361 * Pointer to Ethernet device. 6362 * @param[in] flow 6363 * Parent flow structure pointer. 6364 * @param[in] attr 6365 * Flow rule attributes. 6366 * @param[in] items 6367 * Pattern specification (list terminated by the END pattern item). 6368 * @param[in] actions 6369 * Associated actions (list terminated by the END action). 6370 * @param[in] flow_split_info 6371 * Pointer to flow split info structure. 6372 * @param[out] error 6373 * Perform verbose error reporting if not NULL. 6374 * @return 6375 * 0 on success, negative value otherwise 6376 */ 6377 static int 6378 flow_create_split_outer(struct rte_eth_dev *dev, 6379 struct rte_flow *flow, 6380 const struct rte_flow_attr *attr, 6381 const struct rte_flow_item items[], 6382 const struct rte_flow_action actions[], 6383 struct mlx5_flow_split_info *flow_split_info, 6384 struct rte_flow_error *error) 6385 { 6386 int ret; 6387 6388 ret = flow_create_split_sample(dev, flow, attr, items, 6389 actions, flow_split_info, error); 6390 MLX5_ASSERT(ret <= 0); 6391 return ret; 6392 } 6393 6394 static inline struct mlx5_flow_tunnel * 6395 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6396 { 6397 struct mlx5_flow_tunnel *tunnel; 6398 6399 #pragma GCC diagnostic push 6400 #pragma GCC diagnostic ignored "-Wcast-qual" 6401 tunnel = (typeof(tunnel))flow->tunnel; 6402 #pragma GCC diagnostic pop 6403 6404 return tunnel; 6405 } 6406 6407 /** 6408 * Adjust flow RSS workspace if needed. 6409 * 6410 * @param wks 6411 * Pointer to thread flow work space. 6412 * @param rss_desc 6413 * Pointer to RSS descriptor. 6414 * @param[in] nrssq_num 6415 * New RSS queue number. 6416 * 6417 * @return 6418 * 0 on success, -1 otherwise and rte_errno is set. 6419 */ 6420 static int 6421 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6422 struct mlx5_flow_rss_desc *rss_desc, 6423 uint32_t nrssq_num) 6424 { 6425 if (likely(nrssq_num <= wks->rssq_num)) 6426 return 0; 6427 rss_desc->queue = realloc(rss_desc->queue, 6428 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6429 if (!rss_desc->queue) { 6430 rte_errno = ENOMEM; 6431 return -1; 6432 } 6433 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6434 return 0; 6435 } 6436 6437 /** 6438 * Create a flow and add it to @p list. 6439 * 6440 * @param dev 6441 * Pointer to Ethernet device. 6442 * @param list 6443 * Pointer to a TAILQ flow list. If this parameter NULL, 6444 * no list insertion occurred, flow is just created, 6445 * this is caller's responsibility to track the 6446 * created flow. 6447 * @param[in] attr 6448 * Flow rule attributes. 6449 * @param[in] items 6450 * Pattern specification (list terminated by the END pattern item). 6451 * @param[in] actions 6452 * Associated actions (list terminated by the END action). 6453 * @param[in] external 6454 * This flow rule is created by request external to PMD. 6455 * @param[out] error 6456 * Perform verbose error reporting if not NULL. 6457 * 6458 * @return 6459 * A flow index on success, 0 otherwise and rte_errno is set. 6460 */ 6461 static uint32_t 6462 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6463 const struct rte_flow_attr *attr, 6464 const struct rte_flow_item items[], 6465 const struct rte_flow_action original_actions[], 6466 bool external, struct rte_flow_error *error) 6467 { 6468 struct mlx5_priv *priv = dev->data->dev_private; 6469 struct rte_flow *flow = NULL; 6470 struct mlx5_flow *dev_flow; 6471 const struct rte_flow_action_rss *rss = NULL; 6472 struct mlx5_translated_action_handle 6473 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6474 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6475 union { 6476 struct mlx5_flow_expand_rss buf; 6477 uint8_t buffer[4096]; 6478 } expand_buffer; 6479 union { 6480 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6481 uint8_t buffer[2048]; 6482 } actions_rx; 6483 union { 6484 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6485 uint8_t buffer[2048]; 6486 } actions_hairpin_tx; 6487 union { 6488 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6489 uint8_t buffer[2048]; 6490 } items_tx; 6491 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6492 struct mlx5_flow_rss_desc *rss_desc; 6493 const struct rte_flow_action *p_actions_rx; 6494 uint32_t i; 6495 uint32_t idx = 0; 6496 int hairpin_flow; 6497 struct rte_flow_attr attr_tx = { .priority = 0 }; 6498 const struct rte_flow_action *actions; 6499 struct rte_flow_action *translated_actions = NULL; 6500 struct mlx5_flow_tunnel *tunnel; 6501 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6502 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6503 struct mlx5_flow_split_info flow_split_info = { 6504 .external = !!external, 6505 .skip_scale = 0, 6506 .flow_idx = 0, 6507 .prefix_mark = 0, 6508 .prefix_layers = 0, 6509 .table_id = 0 6510 }; 6511 int ret; 6512 6513 MLX5_ASSERT(wks); 6514 rss_desc = &wks->rss_desc; 6515 ret = flow_action_handles_translate(dev, original_actions, 6516 indir_actions, 6517 &indir_actions_n, 6518 &translated_actions, error); 6519 if (ret < 0) { 6520 MLX5_ASSERT(translated_actions == NULL); 6521 return 0; 6522 } 6523 actions = translated_actions ? translated_actions : original_actions; 6524 p_actions_rx = actions; 6525 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6526 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6527 external, hairpin_flow, error); 6528 if (ret < 0) 6529 goto error_before_hairpin_split; 6530 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6531 if (!flow) { 6532 rte_errno = ENOMEM; 6533 goto error_before_hairpin_split; 6534 } 6535 if (hairpin_flow > 0) { 6536 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6537 rte_errno = EINVAL; 6538 goto error_before_hairpin_split; 6539 } 6540 flow_hairpin_split(dev, actions, actions_rx.actions, 6541 actions_hairpin_tx.actions, items_tx.items, 6542 idx); 6543 p_actions_rx = actions_rx.actions; 6544 } 6545 flow_split_info.flow_idx = idx; 6546 flow->drv_type = flow_get_drv_type(dev, attr); 6547 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6548 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6549 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6550 /* RSS Action only works on NIC RX domain */ 6551 if (attr->ingress && !attr->transfer) 6552 rss = flow_get_rss_action(dev, p_actions_rx); 6553 if (rss) { 6554 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6555 return 0; 6556 /* 6557 * The following information is required by 6558 * mlx5_flow_hashfields_adjust() in advance. 6559 */ 6560 rss_desc->level = rss->level; 6561 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6562 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6563 } 6564 flow->dev_handles = 0; 6565 if (rss && rss->types) { 6566 unsigned int graph_root; 6567 6568 graph_root = find_graph_root(rss->level); 6569 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6570 items, rss->types, 6571 mlx5_support_expansion, graph_root); 6572 MLX5_ASSERT(ret > 0 && 6573 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6574 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6575 for (i = 0; i < buf->entries; ++i) 6576 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6577 } 6578 } else { 6579 buf->entries = 1; 6580 buf->entry[0].pattern = (void *)(uintptr_t)items; 6581 } 6582 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6583 indir_actions_n); 6584 for (i = 0; i < buf->entries; ++i) { 6585 /* Initialize flow split data. */ 6586 flow_split_info.prefix_layers = 0; 6587 flow_split_info.prefix_mark = 0; 6588 flow_split_info.skip_scale = 0; 6589 /* 6590 * The splitter may create multiple dev_flows, 6591 * depending on configuration. In the simplest 6592 * case it just creates unmodified original flow. 6593 */ 6594 ret = flow_create_split_outer(dev, flow, attr, 6595 buf->entry[i].pattern, 6596 p_actions_rx, &flow_split_info, 6597 error); 6598 if (ret < 0) 6599 goto error; 6600 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6601 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6602 p_actions_rx, 6603 idx, 6604 wks->flows[0].tunnel, 6605 &default_miss_ctx, 6606 error); 6607 if (ret < 0) { 6608 mlx5_free(default_miss_ctx.queue); 6609 goto error; 6610 } 6611 } 6612 } 6613 /* Create the tx flow. */ 6614 if (hairpin_flow) { 6615 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6616 attr_tx.ingress = 0; 6617 attr_tx.egress = 1; 6618 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6619 actions_hairpin_tx.actions, 6620 idx, error); 6621 if (!dev_flow) 6622 goto error; 6623 dev_flow->flow = flow; 6624 dev_flow->external = 0; 6625 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6626 dev_flow->handle, next); 6627 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6628 items_tx.items, 6629 actions_hairpin_tx.actions, error); 6630 if (ret < 0) 6631 goto error; 6632 } 6633 /* 6634 * Update the metadata register copy table. If extensive 6635 * metadata feature is enabled and registers are supported 6636 * we might create the extra rte_flow for each unique 6637 * MARK/FLAG action ID. 6638 * 6639 * The table is updated for ingress Flows only, because 6640 * the egress Flows belong to the different device and 6641 * copy table should be updated in peer NIC Rx domain. 6642 */ 6643 if (attr->ingress && 6644 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6645 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6646 if (ret) 6647 goto error; 6648 } 6649 /* 6650 * If the flow is external (from application) OR device is started, 6651 * OR mreg discover, then apply immediately. 6652 */ 6653 if (external || dev->data->dev_started || 6654 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6655 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6656 ret = flow_drv_apply(dev, flow, error); 6657 if (ret < 0) 6658 goto error; 6659 } 6660 flow->type = type; 6661 flow_rxq_flags_set(dev, flow); 6662 rte_free(translated_actions); 6663 tunnel = flow_tunnel_from_rule(wks->flows); 6664 if (tunnel) { 6665 flow->tunnel = 1; 6666 flow->tunnel_id = tunnel->tunnel_id; 6667 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6668 mlx5_free(default_miss_ctx.queue); 6669 } 6670 mlx5_flow_pop_thread_workspace(); 6671 return idx; 6672 error: 6673 MLX5_ASSERT(flow); 6674 ret = rte_errno; /* Save rte_errno before cleanup. */ 6675 flow_mreg_del_copy_action(dev, flow); 6676 flow_drv_destroy(dev, flow); 6677 if (rss_desc->shared_rss) 6678 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6679 mlx5_ipool_get 6680 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6681 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6682 mlx5_ipool_free(priv->flows[type], idx); 6683 rte_errno = ret; /* Restore rte_errno. */ 6684 ret = rte_errno; 6685 rte_errno = ret; 6686 mlx5_flow_pop_thread_workspace(); 6687 error_before_hairpin_split: 6688 rte_free(translated_actions); 6689 return 0; 6690 } 6691 6692 /** 6693 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6694 * incoming packets to table 1. 6695 * 6696 * Other flow rules, requested for group n, will be created in 6697 * e-switch table n+1. 6698 * Jump action to e-switch group n will be created to group n+1. 6699 * 6700 * Used when working in switchdev mode, to utilise advantages of table 1 6701 * and above. 6702 * 6703 * @param dev 6704 * Pointer to Ethernet device. 6705 * 6706 * @return 6707 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6708 */ 6709 struct rte_flow * 6710 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6711 { 6712 const struct rte_flow_attr attr = { 6713 .group = 0, 6714 .priority = 0, 6715 .ingress = 1, 6716 .egress = 0, 6717 .transfer = 1, 6718 }; 6719 const struct rte_flow_item pattern = { 6720 .type = RTE_FLOW_ITEM_TYPE_END, 6721 }; 6722 struct rte_flow_action_jump jump = { 6723 .group = 1, 6724 }; 6725 const struct rte_flow_action actions[] = { 6726 { 6727 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6728 .conf = &jump, 6729 }, 6730 { 6731 .type = RTE_FLOW_ACTION_TYPE_END, 6732 }, 6733 }; 6734 struct rte_flow_error error; 6735 6736 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6737 &attr, &pattern, 6738 actions, false, &error); 6739 } 6740 6741 /** 6742 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6743 * and sq number, directs all packets to peer vport. 6744 * 6745 * @param dev 6746 * Pointer to Ethernet device. 6747 * @param txq 6748 * Txq index. 6749 * 6750 * @return 6751 * Flow ID on success, 0 otherwise and rte_errno is set. 6752 */ 6753 uint32_t 6754 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6755 { 6756 struct rte_flow_attr attr = { 6757 .group = 0, 6758 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6759 .ingress = 1, 6760 .egress = 0, 6761 .transfer = 1, 6762 }; 6763 struct rte_flow_item_port_id port_spec = { 6764 .id = MLX5_PORT_ESW_MGR, 6765 }; 6766 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6767 .queue = txq, 6768 }; 6769 struct rte_flow_item pattern[] = { 6770 { 6771 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6772 .spec = &port_spec, 6773 }, 6774 { 6775 .type = (enum rte_flow_item_type) 6776 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6777 .spec = &txq_spec, 6778 }, 6779 { 6780 .type = RTE_FLOW_ITEM_TYPE_END, 6781 }, 6782 }; 6783 struct rte_flow_action_jump jump = { 6784 .group = 1, 6785 }; 6786 struct rte_flow_action_port_id port = { 6787 .id = dev->data->port_id, 6788 }; 6789 struct rte_flow_action actions[] = { 6790 { 6791 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6792 .conf = &jump, 6793 }, 6794 { 6795 .type = RTE_FLOW_ACTION_TYPE_END, 6796 }, 6797 }; 6798 struct rte_flow_error error; 6799 6800 /* 6801 * Creates group 0, highest priority jump flow. 6802 * Matches txq to bypass kernel packets. 6803 */ 6804 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 6805 false, &error) == 0) 6806 return 0; 6807 /* Create group 1, lowest priority redirect flow for txq. */ 6808 attr.group = 1; 6809 actions[0].conf = &port; 6810 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 6811 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 6812 actions, false, &error); 6813 } 6814 6815 /** 6816 * Validate a flow supported by the NIC. 6817 * 6818 * @see rte_flow_validate() 6819 * @see rte_flow_ops 6820 */ 6821 int 6822 mlx5_flow_validate(struct rte_eth_dev *dev, 6823 const struct rte_flow_attr *attr, 6824 const struct rte_flow_item items[], 6825 const struct rte_flow_action original_actions[], 6826 struct rte_flow_error *error) 6827 { 6828 int hairpin_flow; 6829 struct mlx5_translated_action_handle 6830 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6831 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6832 const struct rte_flow_action *actions; 6833 struct rte_flow_action *translated_actions = NULL; 6834 int ret = flow_action_handles_translate(dev, original_actions, 6835 indir_actions, 6836 &indir_actions_n, 6837 &translated_actions, error); 6838 6839 if (ret) 6840 return ret; 6841 actions = translated_actions ? translated_actions : original_actions; 6842 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6843 ret = flow_drv_validate(dev, attr, items, actions, 6844 true, hairpin_flow, error); 6845 rte_free(translated_actions); 6846 return ret; 6847 } 6848 6849 /** 6850 * Create a flow. 6851 * 6852 * @see rte_flow_create() 6853 * @see rte_flow_ops 6854 */ 6855 struct rte_flow * 6856 mlx5_flow_create(struct rte_eth_dev *dev, 6857 const struct rte_flow_attr *attr, 6858 const struct rte_flow_item items[], 6859 const struct rte_flow_action actions[], 6860 struct rte_flow_error *error) 6861 { 6862 struct mlx5_priv *priv = dev->data->dev_private; 6863 6864 if (priv->sh->config.dv_flow_en == 2) { 6865 rte_flow_error_set(error, ENOTSUP, 6866 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6867 NULL, 6868 "Flow non-Q creation not supported"); 6869 return NULL; 6870 } 6871 /* 6872 * If the device is not started yet, it is not allowed to created a 6873 * flow from application. PMD default flows and traffic control flows 6874 * are not affected. 6875 */ 6876 if (unlikely(!dev->data->dev_started)) { 6877 DRV_LOG(DEBUG, "port %u is not started when " 6878 "inserting a flow", dev->data->port_id); 6879 rte_flow_error_set(error, ENODEV, 6880 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6881 NULL, 6882 "port not started"); 6883 return NULL; 6884 } 6885 6886 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6887 attr, items, actions, 6888 true, error); 6889 } 6890 6891 /** 6892 * Destroy a flow in a list. 6893 * 6894 * @param dev 6895 * Pointer to Ethernet device. 6896 * @param[in] flow_idx 6897 * Index of flow to destroy. 6898 */ 6899 static void 6900 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6901 uint32_t flow_idx) 6902 { 6903 struct mlx5_priv *priv = dev->data->dev_private; 6904 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6905 6906 if (!flow) 6907 return; 6908 MLX5_ASSERT(flow->type == type); 6909 /* 6910 * Update RX queue flags only if port is started, otherwise it is 6911 * already clean. 6912 */ 6913 if (dev->data->dev_started) 6914 flow_rxq_flags_trim(dev, flow); 6915 flow_drv_destroy(dev, flow); 6916 if (flow->tunnel) { 6917 struct mlx5_flow_tunnel *tunnel; 6918 6919 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6920 RTE_VERIFY(tunnel); 6921 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6922 mlx5_flow_tunnel_free(dev, tunnel); 6923 } 6924 flow_mreg_del_copy_action(dev, flow); 6925 mlx5_ipool_free(priv->flows[type], flow_idx); 6926 } 6927 6928 /** 6929 * Destroy all flows. 6930 * 6931 * @param dev 6932 * Pointer to Ethernet device. 6933 * @param type 6934 * Flow type to be flushed. 6935 * @param active 6936 * If flushing is called actively. 6937 */ 6938 void 6939 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6940 bool active) 6941 { 6942 struct mlx5_priv *priv = dev->data->dev_private; 6943 uint32_t num_flushed = 0, fidx = 1; 6944 struct rte_flow *flow; 6945 6946 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6947 flow_list_destroy(dev, type, fidx); 6948 num_flushed++; 6949 } 6950 if (active) { 6951 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6952 dev->data->port_id, num_flushed); 6953 } 6954 } 6955 6956 /** 6957 * Stop all default actions for flows. 6958 * 6959 * @param dev 6960 * Pointer to Ethernet device. 6961 */ 6962 void 6963 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6964 { 6965 flow_mreg_del_default_copy_action(dev); 6966 flow_rxq_flags_clear(dev); 6967 } 6968 6969 /** 6970 * Start all default actions for flows. 6971 * 6972 * @param dev 6973 * Pointer to Ethernet device. 6974 * @return 6975 * 0 on success, a negative errno value otherwise and rte_errno is set. 6976 */ 6977 int 6978 mlx5_flow_start_default(struct rte_eth_dev *dev) 6979 { 6980 struct rte_flow_error error; 6981 6982 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6983 return flow_mreg_add_default_copy_action(dev, &error); 6984 } 6985 6986 /** 6987 * Release key of thread specific flow workspace data. 6988 */ 6989 void 6990 flow_release_workspace(void *data) 6991 { 6992 struct mlx5_flow_workspace *wks = data; 6993 struct mlx5_flow_workspace *next; 6994 6995 while (wks) { 6996 next = wks->next; 6997 free(wks->rss_desc.queue); 6998 free(wks); 6999 wks = next; 7000 } 7001 } 7002 7003 /** 7004 * Get thread specific current flow workspace. 7005 * 7006 * @return pointer to thread specific flow workspace data, NULL on error. 7007 */ 7008 struct mlx5_flow_workspace* 7009 mlx5_flow_get_thread_workspace(void) 7010 { 7011 struct mlx5_flow_workspace *data; 7012 7013 data = mlx5_flow_os_get_specific_workspace(); 7014 MLX5_ASSERT(data && data->inuse); 7015 if (!data || !data->inuse) 7016 DRV_LOG(ERR, "flow workspace not initialized."); 7017 return data; 7018 } 7019 7020 /** 7021 * Allocate and init new flow workspace. 7022 * 7023 * @return pointer to flow workspace data, NULL on error. 7024 */ 7025 static struct mlx5_flow_workspace* 7026 flow_alloc_thread_workspace(void) 7027 { 7028 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7029 7030 if (!data) { 7031 DRV_LOG(ERR, "Failed to allocate flow workspace " 7032 "memory."); 7033 return NULL; 7034 } 7035 data->rss_desc.queue = calloc(1, 7036 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7037 if (!data->rss_desc.queue) 7038 goto err; 7039 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7040 return data; 7041 err: 7042 free(data->rss_desc.queue); 7043 free(data); 7044 return NULL; 7045 } 7046 7047 /** 7048 * Get new thread specific flow workspace. 7049 * 7050 * If current workspace inuse, create new one and set as current. 7051 * 7052 * @return pointer to thread specific flow workspace data, NULL on error. 7053 */ 7054 static struct mlx5_flow_workspace* 7055 mlx5_flow_push_thread_workspace(void) 7056 { 7057 struct mlx5_flow_workspace *curr; 7058 struct mlx5_flow_workspace *data; 7059 7060 curr = mlx5_flow_os_get_specific_workspace(); 7061 if (!curr) { 7062 data = flow_alloc_thread_workspace(); 7063 if (!data) 7064 return NULL; 7065 } else if (!curr->inuse) { 7066 data = curr; 7067 } else if (curr->next) { 7068 data = curr->next; 7069 } else { 7070 data = flow_alloc_thread_workspace(); 7071 if (!data) 7072 return NULL; 7073 curr->next = data; 7074 data->prev = curr; 7075 } 7076 data->inuse = 1; 7077 data->flow_idx = 0; 7078 /* Set as current workspace */ 7079 if (mlx5_flow_os_set_specific_workspace(data)) 7080 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7081 return data; 7082 } 7083 7084 /** 7085 * Close current thread specific flow workspace. 7086 * 7087 * If previous workspace available, set it as current. 7088 * 7089 * @return pointer to thread specific flow workspace data, NULL on error. 7090 */ 7091 static void 7092 mlx5_flow_pop_thread_workspace(void) 7093 { 7094 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7095 7096 if (!data) 7097 return; 7098 if (!data->inuse) { 7099 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7100 return; 7101 } 7102 data->inuse = 0; 7103 if (!data->prev) 7104 return; 7105 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7106 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7107 } 7108 7109 /** 7110 * Verify the flow list is empty 7111 * 7112 * @param dev 7113 * Pointer to Ethernet device. 7114 * 7115 * @return the number of flows not released. 7116 */ 7117 int 7118 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7119 { 7120 struct mlx5_priv *priv = dev->data->dev_private; 7121 struct rte_flow *flow; 7122 uint32_t idx = 0; 7123 int ret = 0, i; 7124 7125 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7126 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7127 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7128 dev->data->port_id, (void *)flow); 7129 ret++; 7130 } 7131 } 7132 return ret; 7133 } 7134 7135 /** 7136 * Enable default hairpin egress flow. 7137 * 7138 * @param dev 7139 * Pointer to Ethernet device. 7140 * @param queue 7141 * The queue index. 7142 * 7143 * @return 7144 * 0 on success, a negative errno value otherwise and rte_errno is set. 7145 */ 7146 int 7147 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7148 uint32_t queue) 7149 { 7150 const struct rte_flow_attr attr = { 7151 .egress = 1, 7152 .priority = 0, 7153 }; 7154 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7155 .queue = queue, 7156 }; 7157 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7158 .queue = UINT32_MAX, 7159 }; 7160 struct rte_flow_item items[] = { 7161 { 7162 .type = (enum rte_flow_item_type) 7163 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7164 .spec = &queue_spec, 7165 .last = NULL, 7166 .mask = &queue_mask, 7167 }, 7168 { 7169 .type = RTE_FLOW_ITEM_TYPE_END, 7170 }, 7171 }; 7172 struct rte_flow_action_jump jump = { 7173 .group = MLX5_HAIRPIN_TX_TABLE, 7174 }; 7175 struct rte_flow_action actions[2]; 7176 uint32_t flow_idx; 7177 struct rte_flow_error error; 7178 7179 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7180 actions[0].conf = &jump; 7181 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7182 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7183 &attr, items, actions, false, &error); 7184 if (!flow_idx) { 7185 DRV_LOG(DEBUG, 7186 "Failed to create ctrl flow: rte_errno(%d)," 7187 " type(%d), message(%s)", 7188 rte_errno, error.type, 7189 error.message ? error.message : " (no stated reason)"); 7190 return -rte_errno; 7191 } 7192 return 0; 7193 } 7194 7195 /** 7196 * Enable a control flow configured from the control plane. 7197 * 7198 * @param dev 7199 * Pointer to Ethernet device. 7200 * @param eth_spec 7201 * An Ethernet flow spec to apply. 7202 * @param eth_mask 7203 * An Ethernet flow mask to apply. 7204 * @param vlan_spec 7205 * A VLAN flow spec to apply. 7206 * @param vlan_mask 7207 * A VLAN flow mask to apply. 7208 * 7209 * @return 7210 * 0 on success, a negative errno value otherwise and rte_errno is set. 7211 */ 7212 int 7213 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7214 struct rte_flow_item_eth *eth_spec, 7215 struct rte_flow_item_eth *eth_mask, 7216 struct rte_flow_item_vlan *vlan_spec, 7217 struct rte_flow_item_vlan *vlan_mask) 7218 { 7219 struct mlx5_priv *priv = dev->data->dev_private; 7220 const struct rte_flow_attr attr = { 7221 .ingress = 1, 7222 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7223 }; 7224 struct rte_flow_item items[] = { 7225 { 7226 .type = RTE_FLOW_ITEM_TYPE_ETH, 7227 .spec = eth_spec, 7228 .last = NULL, 7229 .mask = eth_mask, 7230 }, 7231 { 7232 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7233 RTE_FLOW_ITEM_TYPE_END, 7234 .spec = vlan_spec, 7235 .last = NULL, 7236 .mask = vlan_mask, 7237 }, 7238 { 7239 .type = RTE_FLOW_ITEM_TYPE_END, 7240 }, 7241 }; 7242 uint16_t queue[priv->reta_idx_n]; 7243 struct rte_flow_action_rss action_rss = { 7244 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7245 .level = 0, 7246 .types = priv->rss_conf.rss_hf, 7247 .key_len = priv->rss_conf.rss_key_len, 7248 .queue_num = priv->reta_idx_n, 7249 .key = priv->rss_conf.rss_key, 7250 .queue = queue, 7251 }; 7252 struct rte_flow_action actions[] = { 7253 { 7254 .type = RTE_FLOW_ACTION_TYPE_RSS, 7255 .conf = &action_rss, 7256 }, 7257 { 7258 .type = RTE_FLOW_ACTION_TYPE_END, 7259 }, 7260 }; 7261 uint32_t flow_idx; 7262 struct rte_flow_error error; 7263 unsigned int i; 7264 7265 if (!priv->reta_idx_n || !priv->rxqs_n) { 7266 return 0; 7267 } 7268 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7269 action_rss.types = 0; 7270 for (i = 0; i != priv->reta_idx_n; ++i) 7271 queue[i] = (*priv->reta_idx)[i]; 7272 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7273 &attr, items, actions, false, &error); 7274 if (!flow_idx) 7275 return -rte_errno; 7276 return 0; 7277 } 7278 7279 /** 7280 * Enable a flow control configured from the control plane. 7281 * 7282 * @param dev 7283 * Pointer to Ethernet device. 7284 * @param eth_spec 7285 * An Ethernet flow spec to apply. 7286 * @param eth_mask 7287 * An Ethernet flow mask to apply. 7288 * 7289 * @return 7290 * 0 on success, a negative errno value otherwise and rte_errno is set. 7291 */ 7292 int 7293 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7294 struct rte_flow_item_eth *eth_spec, 7295 struct rte_flow_item_eth *eth_mask) 7296 { 7297 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7298 } 7299 7300 /** 7301 * Create default miss flow rule matching lacp traffic 7302 * 7303 * @param dev 7304 * Pointer to Ethernet device. 7305 * @param eth_spec 7306 * An Ethernet flow spec to apply. 7307 * 7308 * @return 7309 * 0 on success, a negative errno value otherwise and rte_errno is set. 7310 */ 7311 int 7312 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7313 { 7314 /* 7315 * The LACP matching is done by only using ether type since using 7316 * a multicast dst mac causes kernel to give low priority to this flow. 7317 */ 7318 static const struct rte_flow_item_eth lacp_spec = { 7319 .type = RTE_BE16(0x8809), 7320 }; 7321 static const struct rte_flow_item_eth lacp_mask = { 7322 .type = 0xffff, 7323 }; 7324 const struct rte_flow_attr attr = { 7325 .ingress = 1, 7326 }; 7327 struct rte_flow_item items[] = { 7328 { 7329 .type = RTE_FLOW_ITEM_TYPE_ETH, 7330 .spec = &lacp_spec, 7331 .mask = &lacp_mask, 7332 }, 7333 { 7334 .type = RTE_FLOW_ITEM_TYPE_END, 7335 }, 7336 }; 7337 struct rte_flow_action actions[] = { 7338 { 7339 .type = (enum rte_flow_action_type) 7340 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7341 }, 7342 { 7343 .type = RTE_FLOW_ACTION_TYPE_END, 7344 }, 7345 }; 7346 struct rte_flow_error error; 7347 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7348 &attr, items, actions, 7349 false, &error); 7350 7351 if (!flow_idx) 7352 return -rte_errno; 7353 return 0; 7354 } 7355 7356 /** 7357 * Destroy a flow. 7358 * 7359 * @see rte_flow_destroy() 7360 * @see rte_flow_ops 7361 */ 7362 int 7363 mlx5_flow_destroy(struct rte_eth_dev *dev, 7364 struct rte_flow *flow, 7365 struct rte_flow_error *error __rte_unused) 7366 { 7367 struct mlx5_priv *priv = dev->data->dev_private; 7368 7369 if (priv->sh->config.dv_flow_en == 2) 7370 return rte_flow_error_set(error, ENOTSUP, 7371 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7372 NULL, 7373 "Flow non-Q destruction not supported"); 7374 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7375 (uintptr_t)(void *)flow); 7376 return 0; 7377 } 7378 7379 /** 7380 * Destroy all flows. 7381 * 7382 * @see rte_flow_flush() 7383 * @see rte_flow_ops 7384 */ 7385 int 7386 mlx5_flow_flush(struct rte_eth_dev *dev, 7387 struct rte_flow_error *error __rte_unused) 7388 { 7389 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7390 return 0; 7391 } 7392 7393 /** 7394 * Isolated mode. 7395 * 7396 * @see rte_flow_isolate() 7397 * @see rte_flow_ops 7398 */ 7399 int 7400 mlx5_flow_isolate(struct rte_eth_dev *dev, 7401 int enable, 7402 struct rte_flow_error *error) 7403 { 7404 struct mlx5_priv *priv = dev->data->dev_private; 7405 7406 if (dev->data->dev_started) { 7407 rte_flow_error_set(error, EBUSY, 7408 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7409 NULL, 7410 "port must be stopped first"); 7411 return -rte_errno; 7412 } 7413 priv->isolated = !!enable; 7414 if (enable) 7415 dev->dev_ops = &mlx5_dev_ops_isolate; 7416 else 7417 dev->dev_ops = &mlx5_dev_ops; 7418 7419 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7420 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7421 7422 return 0; 7423 } 7424 7425 /** 7426 * Query a flow. 7427 * 7428 * @see rte_flow_query() 7429 * @see rte_flow_ops 7430 */ 7431 static int 7432 flow_drv_query(struct rte_eth_dev *dev, 7433 uint32_t flow_idx, 7434 const struct rte_flow_action *actions, 7435 void *data, 7436 struct rte_flow_error *error) 7437 { 7438 struct mlx5_priv *priv = dev->data->dev_private; 7439 const struct mlx5_flow_driver_ops *fops; 7440 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7441 flow_idx); 7442 enum mlx5_flow_drv_type ftype; 7443 7444 if (!flow) { 7445 return rte_flow_error_set(error, ENOENT, 7446 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7447 NULL, 7448 "invalid flow handle"); 7449 } 7450 ftype = flow->drv_type; 7451 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7452 fops = flow_get_drv_ops(ftype); 7453 7454 return fops->query(dev, flow, actions, data, error); 7455 } 7456 7457 /** 7458 * Query a flow. 7459 * 7460 * @see rte_flow_query() 7461 * @see rte_flow_ops 7462 */ 7463 int 7464 mlx5_flow_query(struct rte_eth_dev *dev, 7465 struct rte_flow *flow, 7466 const struct rte_flow_action *actions, 7467 void *data, 7468 struct rte_flow_error *error) 7469 { 7470 int ret; 7471 struct mlx5_priv *priv = dev->data->dev_private; 7472 7473 if (priv->sh->config.dv_flow_en == 2) 7474 return rte_flow_error_set(error, ENOTSUP, 7475 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7476 NULL, 7477 "Flow non-Q query not supported"); 7478 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7479 error); 7480 if (ret < 0) 7481 return ret; 7482 return 0; 7483 } 7484 7485 /** 7486 * Get rte_flow callbacks. 7487 * 7488 * @param dev 7489 * Pointer to Ethernet device structure. 7490 * @param ops 7491 * Pointer to operation-specific structure. 7492 * 7493 * @return 0 7494 */ 7495 int 7496 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7497 const struct rte_flow_ops **ops) 7498 { 7499 *ops = &mlx5_flow_ops; 7500 return 0; 7501 } 7502 7503 /** 7504 * Validate meter policy actions. 7505 * Dispatcher for action type specific validation. 7506 * 7507 * @param[in] dev 7508 * Pointer to the Ethernet device structure. 7509 * @param[in] action 7510 * The meter policy action object to validate. 7511 * @param[in] attr 7512 * Attributes of flow to determine steering domain. 7513 * @param[out] is_rss 7514 * Is RSS or not. 7515 * @param[out] domain_bitmap 7516 * Domain bitmap. 7517 * @param[out] is_def_policy 7518 * Is default policy or not. 7519 * @param[out] error 7520 * Perform verbose error reporting if not NULL. Initialized in case of 7521 * error only. 7522 * 7523 * @return 7524 * 0 on success, otherwise negative errno value. 7525 */ 7526 int 7527 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7528 const struct rte_flow_action *actions[RTE_COLORS], 7529 struct rte_flow_attr *attr, 7530 bool *is_rss, 7531 uint8_t *domain_bitmap, 7532 uint8_t *policy_mode, 7533 struct rte_mtr_error *error) 7534 { 7535 const struct mlx5_flow_driver_ops *fops; 7536 7537 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7538 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7539 domain_bitmap, policy_mode, error); 7540 } 7541 7542 /** 7543 * Destroy the meter table set. 7544 * 7545 * @param[in] dev 7546 * Pointer to Ethernet device. 7547 * @param[in] mtr_policy 7548 * Meter policy struct. 7549 */ 7550 void 7551 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7552 struct mlx5_flow_meter_policy *mtr_policy) 7553 { 7554 const struct mlx5_flow_driver_ops *fops; 7555 7556 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7557 fops->destroy_mtr_acts(dev, mtr_policy); 7558 } 7559 7560 /** 7561 * Create policy action, lock free, 7562 * (mutex should be acquired by caller). 7563 * Dispatcher for action type specific call. 7564 * 7565 * @param[in] dev 7566 * Pointer to the Ethernet device structure. 7567 * @param[in] mtr_policy 7568 * Meter policy struct. 7569 * @param[in] action 7570 * Action specification used to create meter actions. 7571 * @param[out] error 7572 * Perform verbose error reporting if not NULL. Initialized in case of 7573 * error only. 7574 * 7575 * @return 7576 * 0 on success, otherwise negative errno value. 7577 */ 7578 int 7579 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7580 struct mlx5_flow_meter_policy *mtr_policy, 7581 const struct rte_flow_action *actions[RTE_COLORS], 7582 struct rte_mtr_error *error) 7583 { 7584 const struct mlx5_flow_driver_ops *fops; 7585 7586 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7587 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7588 } 7589 7590 /** 7591 * Create policy rules, lock free, 7592 * (mutex should be acquired by caller). 7593 * Dispatcher for action type specific call. 7594 * 7595 * @param[in] dev 7596 * Pointer to the Ethernet device structure. 7597 * @param[in] mtr_policy 7598 * Meter policy struct. 7599 * 7600 * @return 7601 * 0 on success, -1 otherwise. 7602 */ 7603 int 7604 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7605 struct mlx5_flow_meter_policy *mtr_policy) 7606 { 7607 const struct mlx5_flow_driver_ops *fops; 7608 7609 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7610 return fops->create_policy_rules(dev, mtr_policy); 7611 } 7612 7613 /** 7614 * Destroy policy rules, lock free, 7615 * (mutex should be acquired by caller). 7616 * Dispatcher for action type specific call. 7617 * 7618 * @param[in] dev 7619 * Pointer to the Ethernet device structure. 7620 * @param[in] mtr_policy 7621 * Meter policy struct. 7622 */ 7623 void 7624 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7625 struct mlx5_flow_meter_policy *mtr_policy) 7626 { 7627 const struct mlx5_flow_driver_ops *fops; 7628 7629 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7630 fops->destroy_policy_rules(dev, mtr_policy); 7631 } 7632 7633 /** 7634 * Destroy the default policy table set. 7635 * 7636 * @param[in] dev 7637 * Pointer to Ethernet device. 7638 */ 7639 void 7640 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7641 { 7642 const struct mlx5_flow_driver_ops *fops; 7643 7644 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7645 fops->destroy_def_policy(dev); 7646 } 7647 7648 /** 7649 * Destroy the default policy table set. 7650 * 7651 * @param[in] dev 7652 * Pointer to Ethernet device. 7653 * 7654 * @return 7655 * 0 on success, -1 otherwise. 7656 */ 7657 int 7658 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7659 { 7660 const struct mlx5_flow_driver_ops *fops; 7661 7662 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7663 return fops->create_def_policy(dev); 7664 } 7665 7666 /** 7667 * Create the needed meter and suffix tables. 7668 * 7669 * @param[in] dev 7670 * Pointer to Ethernet device. 7671 * 7672 * @return 7673 * 0 on success, -1 otherwise. 7674 */ 7675 int 7676 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7677 struct mlx5_flow_meter_info *fm, 7678 uint32_t mtr_idx, 7679 uint8_t domain_bitmap) 7680 { 7681 const struct mlx5_flow_driver_ops *fops; 7682 7683 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7684 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7685 } 7686 7687 /** 7688 * Destroy the meter table set. 7689 * 7690 * @param[in] dev 7691 * Pointer to Ethernet device. 7692 * @param[in] tbl 7693 * Pointer to the meter table set. 7694 */ 7695 void 7696 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7697 struct mlx5_flow_meter_info *fm) 7698 { 7699 const struct mlx5_flow_driver_ops *fops; 7700 7701 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7702 fops->destroy_mtr_tbls(dev, fm); 7703 } 7704 7705 /** 7706 * Destroy the global meter drop table. 7707 * 7708 * @param[in] dev 7709 * Pointer to Ethernet device. 7710 */ 7711 void 7712 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7713 { 7714 const struct mlx5_flow_driver_ops *fops; 7715 7716 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7717 fops->destroy_mtr_drop_tbls(dev); 7718 } 7719 7720 /** 7721 * Destroy the sub policy table with RX queue. 7722 * 7723 * @param[in] dev 7724 * Pointer to Ethernet device. 7725 * @param[in] mtr_policy 7726 * Pointer to meter policy table. 7727 */ 7728 void 7729 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7730 struct mlx5_flow_meter_policy *mtr_policy) 7731 { 7732 const struct mlx5_flow_driver_ops *fops; 7733 7734 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7735 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7736 } 7737 7738 /** 7739 * Allocate the needed aso flow meter id. 7740 * 7741 * @param[in] dev 7742 * Pointer to Ethernet device. 7743 * 7744 * @return 7745 * Index to aso flow meter on success, NULL otherwise. 7746 */ 7747 uint32_t 7748 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7749 { 7750 const struct mlx5_flow_driver_ops *fops; 7751 7752 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7753 return fops->create_meter(dev); 7754 } 7755 7756 /** 7757 * Free the aso flow meter id. 7758 * 7759 * @param[in] dev 7760 * Pointer to Ethernet device. 7761 * @param[in] mtr_idx 7762 * Index to aso flow meter to be free. 7763 * 7764 * @return 7765 * 0 on success. 7766 */ 7767 void 7768 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7769 { 7770 const struct mlx5_flow_driver_ops *fops; 7771 7772 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7773 fops->free_meter(dev, mtr_idx); 7774 } 7775 7776 /** 7777 * Allocate a counter. 7778 * 7779 * @param[in] dev 7780 * Pointer to Ethernet device structure. 7781 * 7782 * @return 7783 * Index to allocated counter on success, 0 otherwise. 7784 */ 7785 uint32_t 7786 mlx5_counter_alloc(struct rte_eth_dev *dev) 7787 { 7788 const struct mlx5_flow_driver_ops *fops; 7789 struct rte_flow_attr attr = { .transfer = 0 }; 7790 7791 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7792 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7793 return fops->counter_alloc(dev); 7794 } 7795 DRV_LOG(ERR, 7796 "port %u counter allocate is not supported.", 7797 dev->data->port_id); 7798 return 0; 7799 } 7800 7801 /** 7802 * Free a counter. 7803 * 7804 * @param[in] dev 7805 * Pointer to Ethernet device structure. 7806 * @param[in] cnt 7807 * Index to counter to be free. 7808 */ 7809 void 7810 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7811 { 7812 const struct mlx5_flow_driver_ops *fops; 7813 struct rte_flow_attr attr = { .transfer = 0 }; 7814 7815 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7816 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7817 fops->counter_free(dev, cnt); 7818 return; 7819 } 7820 DRV_LOG(ERR, 7821 "port %u counter free is not supported.", 7822 dev->data->port_id); 7823 } 7824 7825 /** 7826 * Query counter statistics. 7827 * 7828 * @param[in] dev 7829 * Pointer to Ethernet device structure. 7830 * @param[in] cnt 7831 * Index to counter to query. 7832 * @param[in] clear 7833 * Set to clear counter statistics. 7834 * @param[out] pkts 7835 * The counter hits packets number to save. 7836 * @param[out] bytes 7837 * The counter hits bytes number to save. 7838 * 7839 * @return 7840 * 0 on success, a negative errno value otherwise. 7841 */ 7842 int 7843 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7844 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 7845 { 7846 const struct mlx5_flow_driver_ops *fops; 7847 struct rte_flow_attr attr = { .transfer = 0 }; 7848 7849 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7850 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7851 return fops->counter_query(dev, cnt, clear, pkts, 7852 bytes, action); 7853 } 7854 DRV_LOG(ERR, 7855 "port %u counter query is not supported.", 7856 dev->data->port_id); 7857 return -ENOTSUP; 7858 } 7859 7860 /** 7861 * Get information about HWS pre-configurable resources. 7862 * 7863 * @param[in] dev 7864 * Pointer to the rte_eth_dev structure. 7865 * @param[out] port_info 7866 * Pointer to port information. 7867 * @param[out] queue_info 7868 * Pointer to queue information. 7869 * @param[out] error 7870 * Pointer to error structure. 7871 * 7872 * @return 7873 * 0 on success, a negative errno value otherwise and rte_errno is set. 7874 */ 7875 static int 7876 mlx5_flow_info_get(struct rte_eth_dev *dev, 7877 struct rte_flow_port_info *port_info, 7878 struct rte_flow_queue_info *queue_info, 7879 struct rte_flow_error *error) 7880 { 7881 const struct mlx5_flow_driver_ops *fops; 7882 7883 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 7884 return rte_flow_error_set(error, ENOTSUP, 7885 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7886 NULL, 7887 "info get with incorrect steering mode"); 7888 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 7889 return fops->info_get(dev, port_info, queue_info, error); 7890 } 7891 7892 /** 7893 * Configure port HWS resources. 7894 * 7895 * @param[in] dev 7896 * Pointer to the rte_eth_dev structure. 7897 * @param[in] port_attr 7898 * Port configuration attributes. 7899 * @param[in] nb_queue 7900 * Number of queue. 7901 * @param[in] queue_attr 7902 * Array that holds attributes for each flow queue. 7903 * @param[out] error 7904 * Pointer to error structure. 7905 * 7906 * @return 7907 * 0 on success, a negative errno value otherwise and rte_errno is set. 7908 */ 7909 static int 7910 mlx5_flow_port_configure(struct rte_eth_dev *dev, 7911 const struct rte_flow_port_attr *port_attr, 7912 uint16_t nb_queue, 7913 const struct rte_flow_queue_attr *queue_attr[], 7914 struct rte_flow_error *error) 7915 { 7916 const struct mlx5_flow_driver_ops *fops; 7917 7918 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 7919 return rte_flow_error_set(error, ENOTSUP, 7920 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7921 NULL, 7922 "port configure with incorrect steering mode"); 7923 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 7924 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 7925 } 7926 7927 /** 7928 * Allocate a new memory for the counter values wrapped by all the needed 7929 * management. 7930 * 7931 * @param[in] sh 7932 * Pointer to mlx5_dev_ctx_shared object. 7933 * 7934 * @return 7935 * 0 on success, a negative errno value otherwise. 7936 */ 7937 static int 7938 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7939 { 7940 struct mlx5_counter_stats_mem_mng *mem_mng; 7941 volatile struct flow_counter_stats *raw_data; 7942 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7943 int size = (sizeof(struct flow_counter_stats) * 7944 MLX5_COUNTERS_PER_POOL + 7945 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7946 sizeof(struct mlx5_counter_stats_mem_mng); 7947 size_t pgsize = rte_mem_page_size(); 7948 uint8_t *mem; 7949 int ret; 7950 int i; 7951 7952 if (pgsize == (size_t)-1) { 7953 DRV_LOG(ERR, "Failed to get mem page size"); 7954 rte_errno = ENOMEM; 7955 return -ENOMEM; 7956 } 7957 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7958 if (!mem) { 7959 rte_errno = ENOMEM; 7960 return -ENOMEM; 7961 } 7962 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7963 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7964 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 7965 sh->cdev->pdn, mem, size, 7966 &mem_mng->wm); 7967 if (ret) { 7968 rte_errno = errno; 7969 mlx5_free(mem); 7970 return -rte_errno; 7971 } 7972 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7973 raw_data = (volatile struct flow_counter_stats *)mem; 7974 for (i = 0; i < raws_n; ++i) { 7975 mem_mng->raws[i].mem_mng = mem_mng; 7976 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7977 } 7978 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7979 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7980 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7981 next); 7982 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7983 sh->cmng.mem_mng = mem_mng; 7984 return 0; 7985 } 7986 7987 /** 7988 * Set the statistic memory to the new counter pool. 7989 * 7990 * @param[in] sh 7991 * Pointer to mlx5_dev_ctx_shared object. 7992 * @param[in] pool 7993 * Pointer to the pool to set the statistic memory. 7994 * 7995 * @return 7996 * 0 on success, a negative errno value otherwise. 7997 */ 7998 static int 7999 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 8000 struct mlx5_flow_counter_pool *pool) 8001 { 8002 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8003 /* Resize statistic memory once used out. */ 8004 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 8005 mlx5_flow_create_counter_stat_mem_mng(sh)) { 8006 DRV_LOG(ERR, "Cannot resize counter stat mem."); 8007 return -1; 8008 } 8009 rte_spinlock_lock(&pool->sl); 8010 pool->raw = cmng->mem_mng->raws + pool->index % 8011 MLX5_CNT_CONTAINER_RESIZE; 8012 rte_spinlock_unlock(&pool->sl); 8013 pool->raw_hw = NULL; 8014 return 0; 8015 } 8016 8017 #define MLX5_POOL_QUERY_FREQ_US 1000000 8018 8019 /** 8020 * Set the periodic procedure for triggering asynchronous batch queries for all 8021 * the counter pools. 8022 * 8023 * @param[in] sh 8024 * Pointer to mlx5_dev_ctx_shared object. 8025 */ 8026 void 8027 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 8028 { 8029 uint32_t pools_n, us; 8030 8031 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 8032 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 8033 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 8034 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 8035 sh->cmng.query_thread_on = 0; 8036 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 8037 } else { 8038 sh->cmng.query_thread_on = 1; 8039 } 8040 } 8041 8042 /** 8043 * The periodic procedure for triggering asynchronous batch queries for all the 8044 * counter pools. This function is probably called by the host thread. 8045 * 8046 * @param[in] arg 8047 * The parameter for the alarm process. 8048 */ 8049 void 8050 mlx5_flow_query_alarm(void *arg) 8051 { 8052 struct mlx5_dev_ctx_shared *sh = arg; 8053 int ret; 8054 uint16_t pool_index = sh->cmng.pool_index; 8055 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8056 struct mlx5_flow_counter_pool *pool; 8057 uint16_t n_valid; 8058 8059 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 8060 goto set_alarm; 8061 rte_spinlock_lock(&cmng->pool_update_sl); 8062 pool = cmng->pools[pool_index]; 8063 n_valid = cmng->n_valid; 8064 rte_spinlock_unlock(&cmng->pool_update_sl); 8065 /* Set the statistic memory to the new created pool. */ 8066 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 8067 goto set_alarm; 8068 if (pool->raw_hw) 8069 /* There is a pool query in progress. */ 8070 goto set_alarm; 8071 pool->raw_hw = 8072 LIST_FIRST(&sh->cmng.free_stat_raws); 8073 if (!pool->raw_hw) 8074 /* No free counter statistics raw memory. */ 8075 goto set_alarm; 8076 /* 8077 * Identify the counters released between query trigger and query 8078 * handle more efficiently. The counter released in this gap period 8079 * should wait for a new round of query as the new arrived packets 8080 * will not be taken into account. 8081 */ 8082 pool->query_gen++; 8083 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 8084 MLX5_COUNTERS_PER_POOL, 8085 NULL, NULL, 8086 pool->raw_hw->mem_mng->wm.lkey, 8087 (void *)(uintptr_t) 8088 pool->raw_hw->data, 8089 sh->devx_comp, 8090 (uint64_t)(uintptr_t)pool); 8091 if (ret) { 8092 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 8093 " %d", pool->min_dcs->id); 8094 pool->raw_hw = NULL; 8095 goto set_alarm; 8096 } 8097 LIST_REMOVE(pool->raw_hw, next); 8098 sh->cmng.pending_queries++; 8099 pool_index++; 8100 if (pool_index >= n_valid) 8101 pool_index = 0; 8102 set_alarm: 8103 sh->cmng.pool_index = pool_index; 8104 mlx5_set_query_alarm(sh); 8105 } 8106 8107 /** 8108 * Check and callback event for new aged flow in the counter pool 8109 * 8110 * @param[in] sh 8111 * Pointer to mlx5_dev_ctx_shared object. 8112 * @param[in] pool 8113 * Pointer to Current counter pool. 8114 */ 8115 static void 8116 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8117 struct mlx5_flow_counter_pool *pool) 8118 { 8119 struct mlx5_priv *priv; 8120 struct mlx5_flow_counter *cnt; 8121 struct mlx5_age_info *age_info; 8122 struct mlx5_age_param *age_param; 8123 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8124 struct mlx5_counter_stats_raw *prev = pool->raw; 8125 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8126 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8127 uint16_t expected = AGE_CANDIDATE; 8128 uint32_t i; 8129 8130 pool->time_of_last_age_check = curr_time; 8131 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8132 cnt = MLX5_POOL_GET_CNT(pool, i); 8133 age_param = MLX5_CNT_TO_AGE(cnt); 8134 if (__atomic_load_n(&age_param->state, 8135 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8136 continue; 8137 if (cur->data[i].hits != prev->data[i].hits) { 8138 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8139 __ATOMIC_RELAXED); 8140 continue; 8141 } 8142 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8143 time_delta, 8144 __ATOMIC_RELAXED) <= age_param->timeout) 8145 continue; 8146 /** 8147 * Hold the lock first, or if between the 8148 * state AGE_TMOUT and tailq operation the 8149 * release happened, the release procedure 8150 * may delete a non-existent tailq node. 8151 */ 8152 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8153 age_info = GET_PORT_AGE_INFO(priv); 8154 rte_spinlock_lock(&age_info->aged_sl); 8155 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8156 AGE_TMOUT, false, 8157 __ATOMIC_RELAXED, 8158 __ATOMIC_RELAXED)) { 8159 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8160 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8161 } 8162 rte_spinlock_unlock(&age_info->aged_sl); 8163 } 8164 mlx5_age_event_prepare(sh); 8165 } 8166 8167 /** 8168 * Handler for the HW respond about ready values from an asynchronous batch 8169 * query. This function is probably called by the host thread. 8170 * 8171 * @param[in] sh 8172 * The pointer to the shared device context. 8173 * @param[in] async_id 8174 * The Devx async ID. 8175 * @param[in] status 8176 * The status of the completion. 8177 */ 8178 void 8179 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8180 uint64_t async_id, int status) 8181 { 8182 struct mlx5_flow_counter_pool *pool = 8183 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8184 struct mlx5_counter_stats_raw *raw_to_free; 8185 uint8_t query_gen = pool->query_gen ^ 1; 8186 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8187 enum mlx5_counter_type cnt_type = 8188 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8189 MLX5_COUNTER_TYPE_ORIGIN; 8190 8191 if (unlikely(status)) { 8192 raw_to_free = pool->raw_hw; 8193 } else { 8194 raw_to_free = pool->raw; 8195 if (pool->is_aged) 8196 mlx5_flow_aging_check(sh, pool); 8197 rte_spinlock_lock(&pool->sl); 8198 pool->raw = pool->raw_hw; 8199 rte_spinlock_unlock(&pool->sl); 8200 /* Be sure the new raw counters data is updated in memory. */ 8201 rte_io_wmb(); 8202 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8203 rte_spinlock_lock(&cmng->csl[cnt_type]); 8204 TAILQ_CONCAT(&cmng->counters[cnt_type], 8205 &pool->counters[query_gen], next); 8206 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8207 } 8208 } 8209 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8210 pool->raw_hw = NULL; 8211 sh->cmng.pending_queries--; 8212 } 8213 8214 static int 8215 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8216 const struct flow_grp_info *grp_info, 8217 struct rte_flow_error *error) 8218 { 8219 if (grp_info->transfer && grp_info->external && 8220 grp_info->fdb_def_rule) { 8221 if (group == UINT32_MAX) 8222 return rte_flow_error_set 8223 (error, EINVAL, 8224 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8225 NULL, 8226 "group index not supported"); 8227 *table = group + 1; 8228 } else { 8229 *table = group; 8230 } 8231 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8232 return 0; 8233 } 8234 8235 /** 8236 * Translate the rte_flow group index to HW table value. 8237 * 8238 * If tunnel offload is disabled, all group ids converted to flow table 8239 * id using the standard method. 8240 * If tunnel offload is enabled, group id can be converted using the 8241 * standard or tunnel conversion method. Group conversion method 8242 * selection depends on flags in `grp_info` parameter: 8243 * - Internal (grp_info.external == 0) groups conversion uses the 8244 * standard method. 8245 * - Group ids in JUMP action converted with the tunnel conversion. 8246 * - Group id in rule attribute conversion depends on a rule type and 8247 * group id value: 8248 * ** non zero group attributes converted with the tunnel method 8249 * ** zero group attribute in non-tunnel rule is converted using the 8250 * standard method - there's only one root table 8251 * ** zero group attribute in steer tunnel rule is converted with the 8252 * standard method - single root table 8253 * ** zero group attribute in match tunnel rule is a special OvS 8254 * case: that value is used for portability reasons. That group 8255 * id is converted with the tunnel conversion method. 8256 * 8257 * @param[in] dev 8258 * Port device 8259 * @param[in] tunnel 8260 * PMD tunnel offload object 8261 * @param[in] group 8262 * rte_flow group index value. 8263 * @param[out] table 8264 * HW table value. 8265 * @param[in] grp_info 8266 * flags used for conversion 8267 * @param[out] error 8268 * Pointer to error structure. 8269 * 8270 * @return 8271 * 0 on success, a negative errno value otherwise and rte_errno is set. 8272 */ 8273 int 8274 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8275 const struct mlx5_flow_tunnel *tunnel, 8276 uint32_t group, uint32_t *table, 8277 const struct flow_grp_info *grp_info, 8278 struct rte_flow_error *error) 8279 { 8280 int ret; 8281 bool standard_translation; 8282 8283 if (!grp_info->skip_scale && grp_info->external && 8284 group < MLX5_MAX_TABLES_EXTERNAL) 8285 group *= MLX5_FLOW_TABLE_FACTOR; 8286 if (is_tunnel_offload_active(dev)) { 8287 standard_translation = !grp_info->external || 8288 grp_info->std_tbl_fix; 8289 } else { 8290 standard_translation = true; 8291 } 8292 DRV_LOG(DEBUG, 8293 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8294 dev->data->port_id, group, grp_info->transfer, 8295 grp_info->external, grp_info->fdb_def_rule, 8296 standard_translation ? "STANDARD" : "TUNNEL"); 8297 if (standard_translation) 8298 ret = flow_group_to_table(dev->data->port_id, group, table, 8299 grp_info, error); 8300 else 8301 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8302 table, error); 8303 8304 return ret; 8305 } 8306 8307 /** 8308 * Discover availability of metadata reg_c's. 8309 * 8310 * Iteratively use test flows to check availability. 8311 * 8312 * @param[in] dev 8313 * Pointer to the Ethernet device structure. 8314 * 8315 * @return 8316 * 0 on success, a negative errno value otherwise and rte_errno is set. 8317 */ 8318 int 8319 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8320 { 8321 struct mlx5_priv *priv = dev->data->dev_private; 8322 enum modify_reg idx; 8323 int n = 0; 8324 8325 /* reg_c[0] and reg_c[1] are reserved. */ 8326 priv->sh->flow_mreg_c[n++] = REG_C_0; 8327 priv->sh->flow_mreg_c[n++] = REG_C_1; 8328 /* Discover availability of other reg_c's. */ 8329 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 8330 struct rte_flow_attr attr = { 8331 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 8332 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8333 .ingress = 1, 8334 }; 8335 struct rte_flow_item items[] = { 8336 [0] = { 8337 .type = RTE_FLOW_ITEM_TYPE_END, 8338 }, 8339 }; 8340 struct rte_flow_action actions[] = { 8341 [0] = { 8342 .type = (enum rte_flow_action_type) 8343 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8344 .conf = &(struct mlx5_flow_action_copy_mreg){ 8345 .src = REG_C_1, 8346 .dst = idx, 8347 }, 8348 }, 8349 [1] = { 8350 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8351 .conf = &(struct rte_flow_action_jump){ 8352 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8353 }, 8354 }, 8355 [2] = { 8356 .type = RTE_FLOW_ACTION_TYPE_END, 8357 }, 8358 }; 8359 uint32_t flow_idx; 8360 struct rte_flow *flow; 8361 struct rte_flow_error error; 8362 8363 if (!priv->sh->config.dv_flow_en) 8364 break; 8365 /* Create internal flow, validation skips copy action. */ 8366 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8367 items, actions, false, &error); 8368 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8369 flow_idx); 8370 if (!flow) 8371 continue; 8372 priv->sh->flow_mreg_c[n++] = idx; 8373 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8374 } 8375 for (; n < MLX5_MREG_C_NUM; ++n) 8376 priv->sh->flow_mreg_c[n] = REG_NON; 8377 priv->sh->metadata_regc_check_flag = 1; 8378 return 0; 8379 } 8380 8381 int 8382 save_dump_file(const uint8_t *data, uint32_t size, 8383 uint32_t type, uint64_t id, void *arg, FILE *file) 8384 { 8385 char line[BUF_SIZE]; 8386 uint32_t out = 0; 8387 uint32_t k; 8388 uint32_t actions_num; 8389 struct rte_flow_query_count *count; 8390 8391 memset(line, 0, BUF_SIZE); 8392 switch (type) { 8393 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8394 actions_num = *(uint32_t *)(arg); 8395 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 8396 type, id, actions_num); 8397 break; 8398 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8399 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 8400 type, id); 8401 break; 8402 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8403 count = (struct rte_flow_query_count *)arg; 8404 fprintf(file, 8405 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 8406 type, id, count->hits, count->bytes); 8407 return 0; 8408 default: 8409 return -1; 8410 } 8411 8412 for (k = 0; k < size; k++) { 8413 /* Make sure we do not overrun the line buffer length. */ 8414 if (out >= BUF_SIZE - 4) { 8415 line[out] = '\0'; 8416 break; 8417 } 8418 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8419 (data[k]) & 0xff); 8420 } 8421 fprintf(file, "%s\n", line); 8422 return 0; 8423 } 8424 8425 int 8426 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8427 struct rte_flow_query_count *count, struct rte_flow_error *error) 8428 { 8429 struct rte_flow_action action[2]; 8430 enum mlx5_flow_drv_type ftype; 8431 const struct mlx5_flow_driver_ops *fops; 8432 8433 if (!flow) { 8434 return rte_flow_error_set(error, ENOENT, 8435 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8436 NULL, 8437 "invalid flow handle"); 8438 } 8439 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8440 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8441 if (flow->counter) { 8442 memset(count, 0, sizeof(struct rte_flow_query_count)); 8443 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8444 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8445 ftype < MLX5_FLOW_TYPE_MAX); 8446 fops = flow_get_drv_ops(ftype); 8447 return fops->query(dev, flow, action, count, error); 8448 } 8449 return -1; 8450 } 8451 8452 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8453 /** 8454 * Dump flow ipool data to file 8455 * 8456 * @param[in] dev 8457 * The pointer to Ethernet device. 8458 * @param[in] file 8459 * A pointer to a file for output. 8460 * @param[out] error 8461 * Perform verbose error reporting if not NULL. PMDs initialize this 8462 * structure in case of error only. 8463 * @return 8464 * 0 on success, a negative value otherwise. 8465 */ 8466 int 8467 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8468 struct rte_flow *flow, FILE *file, 8469 struct rte_flow_error *error) 8470 { 8471 struct mlx5_priv *priv = dev->data->dev_private; 8472 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8473 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8474 uint32_t handle_idx; 8475 struct mlx5_flow_handle *dh; 8476 struct rte_flow_query_count count; 8477 uint32_t actions_num; 8478 const uint8_t *data; 8479 size_t size; 8480 uint64_t id; 8481 uint32_t type; 8482 void *action = NULL; 8483 8484 if (!flow) { 8485 return rte_flow_error_set(error, ENOENT, 8486 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8487 NULL, 8488 "invalid flow handle"); 8489 } 8490 handle_idx = flow->dev_handles; 8491 /* query counter */ 8492 if (flow->counter && 8493 (!mlx5_counter_query(dev, flow->counter, false, 8494 &count.hits, &count.bytes, &action)) && action) { 8495 id = (uint64_t)(uintptr_t)action; 8496 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8497 save_dump_file(NULL, 0, type, 8498 id, (void *)&count, file); 8499 } 8500 8501 while (handle_idx) { 8502 dh = mlx5_ipool_get(priv->sh->ipool 8503 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8504 if (!dh) 8505 continue; 8506 handle_idx = dh->next.next; 8507 8508 /* Get modify_hdr and encap_decap buf from ipools. */ 8509 encap_decap = NULL; 8510 modify_hdr = dh->dvh.modify_hdr; 8511 8512 if (dh->dvh.rix_encap_decap) { 8513 encap_decap = mlx5_ipool_get(priv->sh->ipool 8514 [MLX5_IPOOL_DECAP_ENCAP], 8515 dh->dvh.rix_encap_decap); 8516 } 8517 if (modify_hdr) { 8518 data = (const uint8_t *)modify_hdr->actions; 8519 size = (size_t)(modify_hdr->actions_num) * 8; 8520 id = (uint64_t)(uintptr_t)modify_hdr->action; 8521 actions_num = modify_hdr->actions_num; 8522 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8523 save_dump_file(data, size, type, id, 8524 (void *)(&actions_num), file); 8525 } 8526 if (encap_decap) { 8527 data = encap_decap->buf; 8528 size = encap_decap->size; 8529 id = (uint64_t)(uintptr_t)encap_decap->action; 8530 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8531 save_dump_file(data, size, type, 8532 id, NULL, file); 8533 } 8534 } 8535 return 0; 8536 } 8537 8538 /** 8539 * Dump all flow's encap_decap/modify_hdr/counter data to file 8540 * 8541 * @param[in] dev 8542 * The pointer to Ethernet device. 8543 * @param[in] file 8544 * A pointer to a file for output. 8545 * @param[out] error 8546 * Perform verbose error reporting if not NULL. PMDs initialize this 8547 * structure in case of error only. 8548 * @return 8549 * 0 on success, a negative value otherwise. 8550 */ 8551 static int 8552 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 8553 FILE *file, struct rte_flow_error *error __rte_unused) 8554 { 8555 struct mlx5_priv *priv = dev->data->dev_private; 8556 struct mlx5_dev_ctx_shared *sh = priv->sh; 8557 struct mlx5_hlist *h; 8558 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8559 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8560 struct rte_flow_query_count count; 8561 uint32_t actions_num; 8562 const uint8_t *data; 8563 size_t size; 8564 uint64_t id; 8565 uint32_t type; 8566 uint32_t i; 8567 uint32_t j; 8568 struct mlx5_list_inconst *l_inconst; 8569 struct mlx5_list_entry *e; 8570 int lcore_index; 8571 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 8572 uint32_t max; 8573 void *action; 8574 8575 /* encap_decap hlist is lcore_share, get global core cache. */ 8576 i = MLX5_LIST_GLOBAL; 8577 h = sh->encaps_decaps; 8578 if (h) { 8579 for (j = 0; j <= h->mask; j++) { 8580 l_inconst = &h->buckets[j].l; 8581 if (!l_inconst || !l_inconst->cache[i]) 8582 continue; 8583 8584 e = LIST_FIRST(&l_inconst->cache[i]->h); 8585 while (e) { 8586 encap_decap = 8587 (struct mlx5_flow_dv_encap_decap_resource *)e; 8588 data = encap_decap->buf; 8589 size = encap_decap->size; 8590 id = (uint64_t)(uintptr_t)encap_decap->action; 8591 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8592 save_dump_file(data, size, type, 8593 id, NULL, file); 8594 e = LIST_NEXT(e, next); 8595 } 8596 } 8597 } 8598 8599 /* get modify_hdr */ 8600 h = sh->modify_cmds; 8601 if (h) { 8602 lcore_index = rte_lcore_index(rte_lcore_id()); 8603 if (unlikely(lcore_index == -1)) { 8604 lcore_index = MLX5_LIST_NLCORE; 8605 rte_spinlock_lock(&h->l_const.lcore_lock); 8606 } 8607 i = lcore_index; 8608 8609 for (j = 0; j <= h->mask; j++) { 8610 l_inconst = &h->buckets[j].l; 8611 if (!l_inconst || !l_inconst->cache[i]) 8612 continue; 8613 8614 e = LIST_FIRST(&l_inconst->cache[i]->h); 8615 while (e) { 8616 modify_hdr = 8617 (struct mlx5_flow_dv_modify_hdr_resource *)e; 8618 data = (const uint8_t *)modify_hdr->actions; 8619 size = (size_t)(modify_hdr->actions_num) * 8; 8620 actions_num = modify_hdr->actions_num; 8621 id = (uint64_t)(uintptr_t)modify_hdr->action; 8622 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8623 save_dump_file(data, size, type, id, 8624 (void *)(&actions_num), file); 8625 e = LIST_NEXT(e, next); 8626 } 8627 } 8628 8629 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 8630 rte_spinlock_unlock(&h->l_const.lcore_lock); 8631 } 8632 8633 /* get counter */ 8634 MLX5_ASSERT(cmng->n_valid <= cmng->n); 8635 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 8636 for (j = 1; j <= max; j++) { 8637 action = NULL; 8638 if ((!mlx5_counter_query(dev, j, false, &count.hits, 8639 &count.bytes, &action)) && action) { 8640 id = (uint64_t)(uintptr_t)action; 8641 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8642 save_dump_file(NULL, 0, type, 8643 id, (void *)&count, file); 8644 } 8645 } 8646 return 0; 8647 } 8648 #endif 8649 8650 /** 8651 * Dump flow raw hw data to file 8652 * 8653 * @param[in] dev 8654 * The pointer to Ethernet device. 8655 * @param[in] file 8656 * A pointer to a file for output. 8657 * @param[out] error 8658 * Perform verbose error reporting if not NULL. PMDs initialize this 8659 * structure in case of error only. 8660 * @return 8661 * 0 on success, a negative value otherwise. 8662 */ 8663 int 8664 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8665 FILE *file, 8666 struct rte_flow_error *error __rte_unused) 8667 { 8668 struct mlx5_priv *priv = dev->data->dev_private; 8669 struct mlx5_dev_ctx_shared *sh = priv->sh; 8670 uint32_t handle_idx; 8671 int ret; 8672 struct mlx5_flow_handle *dh; 8673 struct rte_flow *flow; 8674 8675 if (!sh->config.dv_flow_en) { 8676 if (fputs("device dv flow disabled\n", file) <= 0) 8677 return -errno; 8678 return -ENOTSUP; 8679 } 8680 8681 /* dump all */ 8682 if (!flow_idx) { 8683 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8684 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 8685 return -EINVAL; 8686 #endif 8687 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8688 sh->rx_domain, 8689 sh->tx_domain, file); 8690 } 8691 /* dump one */ 8692 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8693 (uintptr_t)(void *)flow_idx); 8694 if (!flow) 8695 return -EINVAL; 8696 8697 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8698 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8699 #endif 8700 handle_idx = flow->dev_handles; 8701 while (handle_idx) { 8702 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8703 handle_idx); 8704 if (!dh) 8705 return -ENOENT; 8706 if (dh->drv_flow) { 8707 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8708 file); 8709 if (ret) 8710 return -ENOENT; 8711 } 8712 handle_idx = dh->next.next; 8713 } 8714 return 0; 8715 } 8716 8717 /** 8718 * Get aged-out flows. 8719 * 8720 * @param[in] dev 8721 * Pointer to the Ethernet device structure. 8722 * @param[in] context 8723 * The address of an array of pointers to the aged-out flows contexts. 8724 * @param[in] nb_countexts 8725 * The length of context array pointers. 8726 * @param[out] error 8727 * Perform verbose error reporting if not NULL. Initialized in case of 8728 * error only. 8729 * 8730 * @return 8731 * how many contexts get in success, otherwise negative errno value. 8732 * if nb_contexts is 0, return the amount of all aged contexts. 8733 * if nb_contexts is not 0 , return the amount of aged flows reported 8734 * in the context array. 8735 */ 8736 int 8737 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8738 uint32_t nb_contexts, struct rte_flow_error *error) 8739 { 8740 const struct mlx5_flow_driver_ops *fops; 8741 struct rte_flow_attr attr = { .transfer = 0 }; 8742 8743 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8744 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8745 return fops->get_aged_flows(dev, contexts, nb_contexts, 8746 error); 8747 } 8748 DRV_LOG(ERR, 8749 "port %u get aged flows is not supported.", 8750 dev->data->port_id); 8751 return -ENOTSUP; 8752 } 8753 8754 /* Wrapper for driver action_validate op callback */ 8755 static int 8756 flow_drv_action_validate(struct rte_eth_dev *dev, 8757 const struct rte_flow_indir_action_conf *conf, 8758 const struct rte_flow_action *action, 8759 const struct mlx5_flow_driver_ops *fops, 8760 struct rte_flow_error *error) 8761 { 8762 static const char err_msg[] = "indirect action validation unsupported"; 8763 8764 if (!fops->action_validate) { 8765 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8766 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8767 NULL, err_msg); 8768 return -rte_errno; 8769 } 8770 return fops->action_validate(dev, conf, action, error); 8771 } 8772 8773 /** 8774 * Destroys the shared action by handle. 8775 * 8776 * @param dev 8777 * Pointer to Ethernet device structure. 8778 * @param[in] handle 8779 * Handle for the indirect action object to be destroyed. 8780 * @param[out] error 8781 * Perform verbose error reporting if not NULL. PMDs initialize this 8782 * structure in case of error only. 8783 * 8784 * @return 8785 * 0 on success, a negative errno value otherwise and rte_errno is set. 8786 * 8787 * @note: wrapper for driver action_create op callback. 8788 */ 8789 static int 8790 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8791 struct rte_flow_action_handle *handle, 8792 struct rte_flow_error *error) 8793 { 8794 static const char err_msg[] = "indirect action destruction unsupported"; 8795 struct rte_flow_attr attr = { .transfer = 0 }; 8796 const struct mlx5_flow_driver_ops *fops = 8797 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8798 8799 if (!fops->action_destroy) { 8800 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8801 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8802 NULL, err_msg); 8803 return -rte_errno; 8804 } 8805 return fops->action_destroy(dev, handle, error); 8806 } 8807 8808 /* Wrapper for driver action_destroy op callback */ 8809 static int 8810 flow_drv_action_update(struct rte_eth_dev *dev, 8811 struct rte_flow_action_handle *handle, 8812 const void *update, 8813 const struct mlx5_flow_driver_ops *fops, 8814 struct rte_flow_error *error) 8815 { 8816 static const char err_msg[] = "indirect action update unsupported"; 8817 8818 if (!fops->action_update) { 8819 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8820 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8821 NULL, err_msg); 8822 return -rte_errno; 8823 } 8824 return fops->action_update(dev, handle, update, error); 8825 } 8826 8827 /* Wrapper for driver action_destroy op callback */ 8828 static int 8829 flow_drv_action_query(struct rte_eth_dev *dev, 8830 const struct rte_flow_action_handle *handle, 8831 void *data, 8832 const struct mlx5_flow_driver_ops *fops, 8833 struct rte_flow_error *error) 8834 { 8835 static const char err_msg[] = "indirect action query unsupported"; 8836 8837 if (!fops->action_query) { 8838 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8839 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8840 NULL, err_msg); 8841 return -rte_errno; 8842 } 8843 return fops->action_query(dev, handle, data, error); 8844 } 8845 8846 /** 8847 * Create indirect action for reuse in multiple flow rules. 8848 * 8849 * @param dev 8850 * Pointer to Ethernet device structure. 8851 * @param conf 8852 * Pointer to indirect action object configuration. 8853 * @param[in] action 8854 * Action configuration for indirect action object creation. 8855 * @param[out] error 8856 * Perform verbose error reporting if not NULL. PMDs initialize this 8857 * structure in case of error only. 8858 * @return 8859 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8860 */ 8861 static struct rte_flow_action_handle * 8862 mlx5_action_handle_create(struct rte_eth_dev *dev, 8863 const struct rte_flow_indir_action_conf *conf, 8864 const struct rte_flow_action *action, 8865 struct rte_flow_error *error) 8866 { 8867 static const char err_msg[] = "indirect action creation unsupported"; 8868 struct rte_flow_attr attr = { .transfer = 0 }; 8869 const struct mlx5_flow_driver_ops *fops = 8870 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8871 8872 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8873 return NULL; 8874 if (!fops->action_create) { 8875 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8876 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8877 NULL, err_msg); 8878 return NULL; 8879 } 8880 return fops->action_create(dev, conf, action, error); 8881 } 8882 8883 /** 8884 * Updates inplace the indirect action configuration pointed by *handle* 8885 * with the configuration provided as *update* argument. 8886 * The update of the indirect action configuration effects all flow rules 8887 * reusing the action via handle. 8888 * 8889 * @param dev 8890 * Pointer to Ethernet device structure. 8891 * @param[in] handle 8892 * Handle for the indirect action to be updated. 8893 * @param[in] update 8894 * Action specification used to modify the action pointed by handle. 8895 * *update* could be of same type with the action pointed by the *handle* 8896 * handle argument, or some other structures like a wrapper, depending on 8897 * the indirect action type. 8898 * @param[out] error 8899 * Perform verbose error reporting if not NULL. PMDs initialize this 8900 * structure in case of error only. 8901 * 8902 * @return 8903 * 0 on success, a negative errno value otherwise and rte_errno is set. 8904 */ 8905 static int 8906 mlx5_action_handle_update(struct rte_eth_dev *dev, 8907 struct rte_flow_action_handle *handle, 8908 const void *update, 8909 struct rte_flow_error *error) 8910 { 8911 struct rte_flow_attr attr = { .transfer = 0 }; 8912 const struct mlx5_flow_driver_ops *fops = 8913 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8914 int ret; 8915 8916 ret = flow_drv_action_validate(dev, NULL, 8917 (const struct rte_flow_action *)update, fops, error); 8918 if (ret) 8919 return ret; 8920 return flow_drv_action_update(dev, handle, update, fops, 8921 error); 8922 } 8923 8924 /** 8925 * Query the indirect action by handle. 8926 * 8927 * This function allows retrieving action-specific data such as counters. 8928 * Data is gathered by special action which may be present/referenced in 8929 * more than one flow rule definition. 8930 * 8931 * see @RTE_FLOW_ACTION_TYPE_COUNT 8932 * 8933 * @param dev 8934 * Pointer to Ethernet device structure. 8935 * @param[in] handle 8936 * Handle for the indirect action to query. 8937 * @param[in, out] data 8938 * Pointer to storage for the associated query data type. 8939 * @param[out] error 8940 * Perform verbose error reporting if not NULL. PMDs initialize this 8941 * structure in case of error only. 8942 * 8943 * @return 8944 * 0 on success, a negative errno value otherwise and rte_errno is set. 8945 */ 8946 static int 8947 mlx5_action_handle_query(struct rte_eth_dev *dev, 8948 const struct rte_flow_action_handle *handle, 8949 void *data, 8950 struct rte_flow_error *error) 8951 { 8952 struct rte_flow_attr attr = { .transfer = 0 }; 8953 const struct mlx5_flow_driver_ops *fops = 8954 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8955 8956 return flow_drv_action_query(dev, handle, data, fops, error); 8957 } 8958 8959 /** 8960 * Destroy all indirect actions (shared RSS). 8961 * 8962 * @param dev 8963 * Pointer to Ethernet device. 8964 * 8965 * @return 8966 * 0 on success, a negative errno value otherwise and rte_errno is set. 8967 */ 8968 int 8969 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8970 { 8971 struct rte_flow_error error; 8972 struct mlx5_priv *priv = dev->data->dev_private; 8973 struct mlx5_shared_action_rss *shared_rss; 8974 int ret = 0; 8975 uint32_t idx; 8976 8977 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8978 priv->rss_shared_actions, idx, shared_rss, next) { 8979 ret |= mlx5_action_handle_destroy(dev, 8980 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8981 } 8982 return ret; 8983 } 8984 8985 /** 8986 * Validate existing indirect actions against current device configuration 8987 * and attach them to device resources. 8988 * 8989 * @param dev 8990 * Pointer to Ethernet device. 8991 * 8992 * @return 8993 * 0 on success, a negative errno value otherwise and rte_errno is set. 8994 */ 8995 int 8996 mlx5_action_handle_attach(struct rte_eth_dev *dev) 8997 { 8998 struct mlx5_priv *priv = dev->data->dev_private; 8999 struct mlx5_indexed_pool *ipool = 9000 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 9001 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 9002 int ret = 0; 9003 uint32_t idx; 9004 9005 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9006 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9007 const char *message; 9008 uint32_t queue_idx; 9009 9010 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 9011 ind_tbl->queues_n, 9012 &message, &queue_idx); 9013 if (ret != 0) { 9014 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 9015 dev->data->port_id, ind_tbl->queues[queue_idx], 9016 message); 9017 break; 9018 } 9019 } 9020 if (ret != 0) 9021 return ret; 9022 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9023 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9024 9025 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 9026 if (ret != 0) { 9027 DRV_LOG(ERR, "Port %u could not attach " 9028 "indirection table obj %p", 9029 dev->data->port_id, (void *)ind_tbl); 9030 goto error; 9031 } 9032 } 9033 return 0; 9034 error: 9035 shared_rss_last = shared_rss; 9036 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9037 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9038 9039 if (shared_rss == shared_rss_last) 9040 break; 9041 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 9042 DRV_LOG(CRIT, "Port %u could not detach " 9043 "indirection table obj %p on rollback", 9044 dev->data->port_id, (void *)ind_tbl); 9045 } 9046 return ret; 9047 } 9048 9049 /** 9050 * Detach indirect actions of the device from its resources. 9051 * 9052 * @param dev 9053 * Pointer to Ethernet device. 9054 * 9055 * @return 9056 * 0 on success, a negative errno value otherwise and rte_errno is set. 9057 */ 9058 int 9059 mlx5_action_handle_detach(struct rte_eth_dev *dev) 9060 { 9061 struct mlx5_priv *priv = dev->data->dev_private; 9062 struct mlx5_indexed_pool *ipool = 9063 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 9064 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 9065 int ret = 0; 9066 uint32_t idx; 9067 9068 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9069 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9070 9071 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 9072 if (ret != 0) { 9073 DRV_LOG(ERR, "Port %u could not detach " 9074 "indirection table obj %p", 9075 dev->data->port_id, (void *)ind_tbl); 9076 goto error; 9077 } 9078 } 9079 return 0; 9080 error: 9081 shared_rss_last = shared_rss; 9082 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9083 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9084 9085 if (shared_rss == shared_rss_last) 9086 break; 9087 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 9088 DRV_LOG(CRIT, "Port %u could not attach " 9089 "indirection table obj %p on rollback", 9090 dev->data->port_id, (void *)ind_tbl); 9091 } 9092 return ret; 9093 } 9094 9095 #ifndef HAVE_MLX5DV_DR 9096 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 9097 #else 9098 #define MLX5_DOMAIN_SYNC_FLOW \ 9099 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 9100 #endif 9101 9102 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 9103 { 9104 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 9105 const struct mlx5_flow_driver_ops *fops; 9106 int ret; 9107 struct rte_flow_attr attr = { .transfer = 0 }; 9108 9109 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9110 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9111 if (ret > 0) 9112 ret = -ret; 9113 return ret; 9114 } 9115 9116 const struct mlx5_flow_tunnel * 9117 mlx5_get_tof(const struct rte_flow_item *item, 9118 const struct rte_flow_action *action, 9119 enum mlx5_tof_rule_type *rule_type) 9120 { 9121 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9122 if (item->type == (typeof(item->type)) 9123 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9124 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9125 return flow_items_to_tunnel(item); 9126 } 9127 } 9128 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9129 if (action->type == (typeof(action->type)) 9130 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9131 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9132 return flow_actions_to_tunnel(action); 9133 } 9134 } 9135 return NULL; 9136 } 9137 9138 /** 9139 * tunnel offload functionality is defined for DV environment only 9140 */ 9141 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9142 __extension__ 9143 union tunnel_offload_mark { 9144 uint32_t val; 9145 struct { 9146 uint32_t app_reserve:8; 9147 uint32_t table_id:15; 9148 uint32_t transfer:1; 9149 uint32_t _unused_:8; 9150 }; 9151 }; 9152 9153 static bool 9154 mlx5_access_tunnel_offload_db 9155 (struct rte_eth_dev *dev, 9156 bool (*match)(struct rte_eth_dev *, 9157 struct mlx5_flow_tunnel *, const void *), 9158 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9159 void (*miss)(struct rte_eth_dev *, void *), 9160 void *ctx, bool lock_op); 9161 9162 static int 9163 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9164 struct rte_flow *flow, 9165 const struct rte_flow_attr *attr, 9166 const struct rte_flow_action *app_actions, 9167 uint32_t flow_idx, 9168 const struct mlx5_flow_tunnel *tunnel, 9169 struct tunnel_default_miss_ctx *ctx, 9170 struct rte_flow_error *error) 9171 { 9172 struct mlx5_priv *priv = dev->data->dev_private; 9173 struct mlx5_flow *dev_flow; 9174 struct rte_flow_attr miss_attr = *attr; 9175 const struct rte_flow_item miss_items[2] = { 9176 { 9177 .type = RTE_FLOW_ITEM_TYPE_ETH, 9178 .spec = NULL, 9179 .last = NULL, 9180 .mask = NULL 9181 }, 9182 { 9183 .type = RTE_FLOW_ITEM_TYPE_END, 9184 .spec = NULL, 9185 .last = NULL, 9186 .mask = NULL 9187 } 9188 }; 9189 union tunnel_offload_mark mark_id; 9190 struct rte_flow_action_mark miss_mark; 9191 struct rte_flow_action miss_actions[3] = { 9192 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9193 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9194 }; 9195 const struct rte_flow_action_jump *jump_data; 9196 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9197 struct flow_grp_info grp_info = { 9198 .external = 1, 9199 .transfer = attr->transfer, 9200 .fdb_def_rule = !!priv->fdb_def_rule, 9201 .std_tbl_fix = 0, 9202 }; 9203 int ret; 9204 9205 if (!attr->transfer) { 9206 uint32_t q_size; 9207 9208 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9209 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9210 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9211 0, SOCKET_ID_ANY); 9212 if (!ctx->queue) 9213 return rte_flow_error_set 9214 (error, ENOMEM, 9215 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9216 NULL, "invalid default miss RSS"); 9217 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9218 ctx->action_rss.level = 0, 9219 ctx->action_rss.types = priv->rss_conf.rss_hf, 9220 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9221 ctx->action_rss.queue_num = priv->reta_idx_n, 9222 ctx->action_rss.key = priv->rss_conf.rss_key, 9223 ctx->action_rss.queue = ctx->queue; 9224 if (!priv->reta_idx_n || !priv->rxqs_n) 9225 return rte_flow_error_set 9226 (error, EINVAL, 9227 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9228 NULL, "invalid port configuration"); 9229 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9230 ctx->action_rss.types = 0; 9231 for (i = 0; i != priv->reta_idx_n; ++i) 9232 ctx->queue[i] = (*priv->reta_idx)[i]; 9233 } else { 9234 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9235 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9236 } 9237 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9238 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9239 jump_data = app_actions->conf; 9240 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9241 miss_attr.group = jump_data->group; 9242 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9243 &flow_table, &grp_info, error); 9244 if (ret) 9245 return rte_flow_error_set(error, EINVAL, 9246 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9247 NULL, "invalid tunnel id"); 9248 mark_id.app_reserve = 0; 9249 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9250 mark_id.transfer = !!attr->transfer; 9251 mark_id._unused_ = 0; 9252 miss_mark.id = mark_id.val; 9253 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9254 miss_items, miss_actions, flow_idx, error); 9255 if (!dev_flow) 9256 return -rte_errno; 9257 dev_flow->flow = flow; 9258 dev_flow->external = true; 9259 dev_flow->tunnel = tunnel; 9260 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9261 /* Subflow object was created, we must include one in the list. */ 9262 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9263 dev_flow->handle, next); 9264 DRV_LOG(DEBUG, 9265 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9266 dev->data->port_id, tunnel->app_tunnel.type, 9267 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9268 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9269 miss_actions, error); 9270 if (!ret) 9271 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9272 error); 9273 9274 return ret; 9275 } 9276 9277 static const struct mlx5_flow_tbl_data_entry * 9278 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9279 { 9280 struct mlx5_priv *priv = dev->data->dev_private; 9281 struct mlx5_dev_ctx_shared *sh = priv->sh; 9282 struct mlx5_list_entry *he; 9283 union tunnel_offload_mark mbits = { .val = mark }; 9284 union mlx5_flow_tbl_key table_key = { 9285 { 9286 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9287 .id = 0, 9288 .reserved = 0, 9289 .dummy = 0, 9290 .is_fdb = !!mbits.transfer, 9291 .is_egress = 0, 9292 } 9293 }; 9294 struct mlx5_flow_cb_ctx ctx = { 9295 .data = &table_key.v64, 9296 }; 9297 9298 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9299 return he ? 9300 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9301 } 9302 9303 static void 9304 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9305 struct mlx5_list_entry *entry) 9306 { 9307 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9308 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9309 9310 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9311 tunnel_flow_tbl_to_id(tte->flow_table)); 9312 mlx5_free(tte); 9313 } 9314 9315 static int 9316 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9317 struct mlx5_list_entry *entry, void *cb_ctx) 9318 { 9319 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9320 union tunnel_tbl_key tbl = { 9321 .val = *(uint64_t *)(ctx->data), 9322 }; 9323 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9324 9325 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9326 } 9327 9328 static struct mlx5_list_entry * 9329 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 9330 { 9331 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9332 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9333 struct tunnel_tbl_entry *tte; 9334 union tunnel_tbl_key tbl = { 9335 .val = *(uint64_t *)(ctx->data), 9336 }; 9337 9338 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 9339 sizeof(*tte), 0, 9340 SOCKET_ID_ANY); 9341 if (!tte) 9342 goto err; 9343 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9344 &tte->flow_table); 9345 if (tte->flow_table >= MLX5_MAX_TABLES) { 9346 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 9347 tte->flow_table); 9348 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9349 tte->flow_table); 9350 goto err; 9351 } else if (!tte->flow_table) { 9352 goto err; 9353 } 9354 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 9355 tte->tunnel_id = tbl.tunnel_id; 9356 tte->group = tbl.group; 9357 return &tte->hash; 9358 err: 9359 if (tte) 9360 mlx5_free(tte); 9361 return NULL; 9362 } 9363 9364 static struct mlx5_list_entry * 9365 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 9366 struct mlx5_list_entry *oentry, 9367 void *cb_ctx __rte_unused) 9368 { 9369 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 9370 0, SOCKET_ID_ANY); 9371 9372 if (!tte) 9373 return NULL; 9374 memcpy(tte, oentry, sizeof(*tte)); 9375 return &tte->hash; 9376 } 9377 9378 static void 9379 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 9380 struct mlx5_list_entry *entry) 9381 { 9382 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9383 9384 mlx5_free(tte); 9385 } 9386 9387 static uint32_t 9388 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 9389 const struct mlx5_flow_tunnel *tunnel, 9390 uint32_t group, uint32_t *table, 9391 struct rte_flow_error *error) 9392 { 9393 struct mlx5_list_entry *he; 9394 struct tunnel_tbl_entry *tte; 9395 union tunnel_tbl_key key = { 9396 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 9397 .group = group 9398 }; 9399 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9400 struct mlx5_hlist *group_hash; 9401 struct mlx5_flow_cb_ctx ctx = { 9402 .data = &key.val, 9403 }; 9404 9405 group_hash = tunnel ? tunnel->groups : thub->groups; 9406 he = mlx5_hlist_register(group_hash, key.val, &ctx); 9407 if (!he) 9408 return rte_flow_error_set(error, EINVAL, 9409 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9410 NULL, 9411 "tunnel group index not supported"); 9412 tte = container_of(he, typeof(*tte), hash); 9413 *table = tte->flow_table; 9414 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 9415 dev->data->port_id, key.tunnel_id, group, *table); 9416 return 0; 9417 } 9418 9419 static void 9420 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 9421 struct mlx5_flow_tunnel *tunnel) 9422 { 9423 struct mlx5_priv *priv = dev->data->dev_private; 9424 struct mlx5_indexed_pool *ipool; 9425 9426 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 9427 dev->data->port_id, tunnel->tunnel_id); 9428 LIST_REMOVE(tunnel, chain); 9429 mlx5_hlist_destroy(tunnel->groups); 9430 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9431 mlx5_ipool_free(ipool, tunnel->tunnel_id); 9432 } 9433 9434 static bool 9435 mlx5_access_tunnel_offload_db 9436 (struct rte_eth_dev *dev, 9437 bool (*match)(struct rte_eth_dev *, 9438 struct mlx5_flow_tunnel *, const void *), 9439 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9440 void (*miss)(struct rte_eth_dev *, void *), 9441 void *ctx, bool lock_op) 9442 { 9443 bool verdict = false; 9444 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9445 struct mlx5_flow_tunnel *tunnel; 9446 9447 rte_spinlock_lock(&thub->sl); 9448 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 9449 verdict = match(dev, tunnel, (const void *)ctx); 9450 if (verdict) 9451 break; 9452 } 9453 if (!lock_op) 9454 rte_spinlock_unlock(&thub->sl); 9455 if (verdict && hit) 9456 hit(dev, tunnel, ctx); 9457 if (!verdict && miss) 9458 miss(dev, ctx); 9459 if (lock_op) 9460 rte_spinlock_unlock(&thub->sl); 9461 9462 return verdict; 9463 } 9464 9465 struct tunnel_db_find_tunnel_id_ctx { 9466 uint32_t tunnel_id; 9467 struct mlx5_flow_tunnel *tunnel; 9468 }; 9469 9470 static bool 9471 find_tunnel_id_match(struct rte_eth_dev *dev, 9472 struct mlx5_flow_tunnel *tunnel, const void *x) 9473 { 9474 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9475 9476 RTE_SET_USED(dev); 9477 return tunnel->tunnel_id == ctx->tunnel_id; 9478 } 9479 9480 static void 9481 find_tunnel_id_hit(struct rte_eth_dev *dev, 9482 struct mlx5_flow_tunnel *tunnel, void *x) 9483 { 9484 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9485 RTE_SET_USED(dev); 9486 ctx->tunnel = tunnel; 9487 } 9488 9489 static struct mlx5_flow_tunnel * 9490 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 9491 { 9492 struct tunnel_db_find_tunnel_id_ctx ctx = { 9493 .tunnel_id = id, 9494 }; 9495 9496 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 9497 find_tunnel_id_hit, NULL, &ctx, true); 9498 9499 return ctx.tunnel; 9500 } 9501 9502 static struct mlx5_flow_tunnel * 9503 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 9504 const struct rte_flow_tunnel *app_tunnel) 9505 { 9506 struct mlx5_priv *priv = dev->data->dev_private; 9507 struct mlx5_indexed_pool *ipool; 9508 struct mlx5_flow_tunnel *tunnel; 9509 uint32_t id; 9510 9511 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9512 tunnel = mlx5_ipool_zmalloc(ipool, &id); 9513 if (!tunnel) 9514 return NULL; 9515 if (id >= MLX5_MAX_TUNNELS) { 9516 mlx5_ipool_free(ipool, id); 9517 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 9518 return NULL; 9519 } 9520 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 9521 priv->sh, 9522 mlx5_flow_tunnel_grp2tbl_create_cb, 9523 mlx5_flow_tunnel_grp2tbl_match_cb, 9524 mlx5_flow_tunnel_grp2tbl_remove_cb, 9525 mlx5_flow_tunnel_grp2tbl_clone_cb, 9526 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9527 if (!tunnel->groups) { 9528 mlx5_ipool_free(ipool, id); 9529 return NULL; 9530 } 9531 /* initiate new PMD tunnel */ 9532 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 9533 tunnel->tunnel_id = id; 9534 tunnel->action.type = (typeof(tunnel->action.type)) 9535 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 9536 tunnel->action.conf = tunnel; 9537 tunnel->item.type = (typeof(tunnel->item.type)) 9538 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 9539 tunnel->item.spec = tunnel; 9540 tunnel->item.last = NULL; 9541 tunnel->item.mask = NULL; 9542 9543 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 9544 dev->data->port_id, tunnel->tunnel_id); 9545 9546 return tunnel; 9547 } 9548 9549 struct tunnel_db_get_tunnel_ctx { 9550 const struct rte_flow_tunnel *app_tunnel; 9551 struct mlx5_flow_tunnel *tunnel; 9552 }; 9553 9554 static bool get_tunnel_match(struct rte_eth_dev *dev, 9555 struct mlx5_flow_tunnel *tunnel, const void *x) 9556 { 9557 const struct tunnel_db_get_tunnel_ctx *ctx = x; 9558 9559 RTE_SET_USED(dev); 9560 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 9561 sizeof(*ctx->app_tunnel)); 9562 } 9563 9564 static void get_tunnel_hit(struct rte_eth_dev *dev, 9565 struct mlx5_flow_tunnel *tunnel, void *x) 9566 { 9567 /* called under tunnel spinlock protection */ 9568 struct tunnel_db_get_tunnel_ctx *ctx = x; 9569 9570 RTE_SET_USED(dev); 9571 tunnel->refctn++; 9572 ctx->tunnel = tunnel; 9573 } 9574 9575 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 9576 { 9577 /* called under tunnel spinlock protection */ 9578 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9579 struct tunnel_db_get_tunnel_ctx *ctx = x; 9580 9581 rte_spinlock_unlock(&thub->sl); 9582 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 9583 rte_spinlock_lock(&thub->sl); 9584 if (ctx->tunnel) { 9585 ctx->tunnel->refctn = 1; 9586 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 9587 } 9588 } 9589 9590 9591 static int 9592 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 9593 const struct rte_flow_tunnel *app_tunnel, 9594 struct mlx5_flow_tunnel **tunnel) 9595 { 9596 struct tunnel_db_get_tunnel_ctx ctx = { 9597 .app_tunnel = app_tunnel, 9598 }; 9599 9600 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 9601 get_tunnel_miss, &ctx, true); 9602 *tunnel = ctx.tunnel; 9603 return ctx.tunnel ? 0 : -ENOMEM; 9604 } 9605 9606 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 9607 { 9608 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 9609 9610 if (!thub) 9611 return; 9612 if (!LIST_EMPTY(&thub->tunnels)) 9613 DRV_LOG(WARNING, "port %u tunnels present", port_id); 9614 mlx5_hlist_destroy(thub->groups); 9615 mlx5_free(thub); 9616 } 9617 9618 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 9619 { 9620 int err; 9621 struct mlx5_flow_tunnel_hub *thub; 9622 9623 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 9624 0, SOCKET_ID_ANY); 9625 if (!thub) 9626 return -ENOMEM; 9627 LIST_INIT(&thub->tunnels); 9628 rte_spinlock_init(&thub->sl); 9629 thub->groups = mlx5_hlist_create("flow groups", 64, 9630 false, true, sh, 9631 mlx5_flow_tunnel_grp2tbl_create_cb, 9632 mlx5_flow_tunnel_grp2tbl_match_cb, 9633 mlx5_flow_tunnel_grp2tbl_remove_cb, 9634 mlx5_flow_tunnel_grp2tbl_clone_cb, 9635 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9636 if (!thub->groups) { 9637 err = -rte_errno; 9638 goto err; 9639 } 9640 sh->tunnel_hub = thub; 9641 9642 return 0; 9643 9644 err: 9645 if (thub->groups) 9646 mlx5_hlist_destroy(thub->groups); 9647 if (thub) 9648 mlx5_free(thub); 9649 return err; 9650 } 9651 9652 static inline int 9653 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 9654 struct rte_flow_tunnel *tunnel, 9655 struct rte_flow_error *error) 9656 { 9657 struct mlx5_priv *priv = dev->data->dev_private; 9658 9659 if (!priv->sh->config.dv_flow_en) 9660 return rte_flow_error_set(error, ENOTSUP, 9661 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9662 "flow DV interface is off"); 9663 if (!is_tunnel_offload_active(dev)) 9664 return rte_flow_error_set(error, ENOTSUP, 9665 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9666 "tunnel offload was not activated"); 9667 if (!tunnel) 9668 return rte_flow_error_set(error, EINVAL, 9669 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9670 "no application tunnel"); 9671 switch (tunnel->type) { 9672 default: 9673 return rte_flow_error_set(error, EINVAL, 9674 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9675 "unsupported tunnel type"); 9676 case RTE_FLOW_ITEM_TYPE_VXLAN: 9677 case RTE_FLOW_ITEM_TYPE_GRE: 9678 case RTE_FLOW_ITEM_TYPE_NVGRE: 9679 case RTE_FLOW_ITEM_TYPE_GENEVE: 9680 break; 9681 } 9682 return 0; 9683 } 9684 9685 static int 9686 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 9687 struct rte_flow_tunnel *app_tunnel, 9688 struct rte_flow_action **actions, 9689 uint32_t *num_of_actions, 9690 struct rte_flow_error *error) 9691 { 9692 struct mlx5_flow_tunnel *tunnel; 9693 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9694 9695 if (ret) 9696 return ret; 9697 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9698 if (ret < 0) { 9699 return rte_flow_error_set(error, ret, 9700 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9701 "failed to initialize pmd tunnel"); 9702 } 9703 *actions = &tunnel->action; 9704 *num_of_actions = 1; 9705 return 0; 9706 } 9707 9708 static int 9709 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9710 struct rte_flow_tunnel *app_tunnel, 9711 struct rte_flow_item **items, 9712 uint32_t *num_of_items, 9713 struct rte_flow_error *error) 9714 { 9715 struct mlx5_flow_tunnel *tunnel; 9716 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9717 9718 if (ret) 9719 return ret; 9720 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9721 if (ret < 0) { 9722 return rte_flow_error_set(error, ret, 9723 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9724 "failed to initialize pmd tunnel"); 9725 } 9726 *items = &tunnel->item; 9727 *num_of_items = 1; 9728 return 0; 9729 } 9730 9731 struct tunnel_db_element_release_ctx { 9732 struct rte_flow_item *items; 9733 struct rte_flow_action *actions; 9734 uint32_t num_elements; 9735 struct rte_flow_error *error; 9736 int ret; 9737 }; 9738 9739 static bool 9740 tunnel_element_release_match(struct rte_eth_dev *dev, 9741 struct mlx5_flow_tunnel *tunnel, const void *x) 9742 { 9743 const struct tunnel_db_element_release_ctx *ctx = x; 9744 9745 RTE_SET_USED(dev); 9746 if (ctx->num_elements != 1) 9747 return false; 9748 else if (ctx->items) 9749 return ctx->items == &tunnel->item; 9750 else if (ctx->actions) 9751 return ctx->actions == &tunnel->action; 9752 9753 return false; 9754 } 9755 9756 static void 9757 tunnel_element_release_hit(struct rte_eth_dev *dev, 9758 struct mlx5_flow_tunnel *tunnel, void *x) 9759 { 9760 struct tunnel_db_element_release_ctx *ctx = x; 9761 ctx->ret = 0; 9762 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9763 mlx5_flow_tunnel_free(dev, tunnel); 9764 } 9765 9766 static void 9767 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9768 { 9769 struct tunnel_db_element_release_ctx *ctx = x; 9770 RTE_SET_USED(dev); 9771 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9772 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9773 "invalid argument"); 9774 } 9775 9776 static int 9777 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9778 struct rte_flow_item *pmd_items, 9779 uint32_t num_items, struct rte_flow_error *err) 9780 { 9781 struct tunnel_db_element_release_ctx ctx = { 9782 .items = pmd_items, 9783 .actions = NULL, 9784 .num_elements = num_items, 9785 .error = err, 9786 }; 9787 9788 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9789 tunnel_element_release_hit, 9790 tunnel_element_release_miss, &ctx, false); 9791 9792 return ctx.ret; 9793 } 9794 9795 static int 9796 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9797 struct rte_flow_action *pmd_actions, 9798 uint32_t num_actions, struct rte_flow_error *err) 9799 { 9800 struct tunnel_db_element_release_ctx ctx = { 9801 .items = NULL, 9802 .actions = pmd_actions, 9803 .num_elements = num_actions, 9804 .error = err, 9805 }; 9806 9807 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9808 tunnel_element_release_hit, 9809 tunnel_element_release_miss, &ctx, false); 9810 9811 return ctx.ret; 9812 } 9813 9814 static int 9815 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9816 struct rte_mbuf *m, 9817 struct rte_flow_restore_info *info, 9818 struct rte_flow_error *err) 9819 { 9820 uint64_t ol_flags = m->ol_flags; 9821 const struct mlx5_flow_tbl_data_entry *tble; 9822 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 9823 9824 if (!is_tunnel_offload_active(dev)) { 9825 info->flags = 0; 9826 return 0; 9827 } 9828 9829 if ((ol_flags & mask) != mask) 9830 goto err; 9831 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9832 if (!tble) { 9833 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9834 dev->data->port_id, m->hash.fdir.hi); 9835 goto err; 9836 } 9837 MLX5_ASSERT(tble->tunnel); 9838 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9839 info->group_id = tble->group_id; 9840 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9841 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9842 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9843 9844 return 0; 9845 9846 err: 9847 return rte_flow_error_set(err, EINVAL, 9848 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9849 "failed to get restore info"); 9850 } 9851 9852 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9853 static int 9854 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9855 __rte_unused struct rte_flow_tunnel *app_tunnel, 9856 __rte_unused struct rte_flow_action **actions, 9857 __rte_unused uint32_t *num_of_actions, 9858 __rte_unused struct rte_flow_error *error) 9859 { 9860 return -ENOTSUP; 9861 } 9862 9863 static int 9864 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9865 __rte_unused struct rte_flow_tunnel *app_tunnel, 9866 __rte_unused struct rte_flow_item **items, 9867 __rte_unused uint32_t *num_of_items, 9868 __rte_unused struct rte_flow_error *error) 9869 { 9870 return -ENOTSUP; 9871 } 9872 9873 static int 9874 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9875 __rte_unused struct rte_flow_item *pmd_items, 9876 __rte_unused uint32_t num_items, 9877 __rte_unused struct rte_flow_error *err) 9878 { 9879 return -ENOTSUP; 9880 } 9881 9882 static int 9883 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9884 __rte_unused struct rte_flow_action *pmd_action, 9885 __rte_unused uint32_t num_actions, 9886 __rte_unused struct rte_flow_error *err) 9887 { 9888 return -ENOTSUP; 9889 } 9890 9891 static int 9892 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9893 __rte_unused struct rte_mbuf *m, 9894 __rte_unused struct rte_flow_restore_info *i, 9895 __rte_unused struct rte_flow_error *err) 9896 { 9897 return -ENOTSUP; 9898 } 9899 9900 static int 9901 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9902 __rte_unused struct rte_flow *flow, 9903 __rte_unused const struct rte_flow_attr *attr, 9904 __rte_unused const struct rte_flow_action *actions, 9905 __rte_unused uint32_t flow_idx, 9906 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9907 __rte_unused struct tunnel_default_miss_ctx *ctx, 9908 __rte_unused struct rte_flow_error *error) 9909 { 9910 return -ENOTSUP; 9911 } 9912 9913 static struct mlx5_flow_tunnel * 9914 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9915 __rte_unused uint32_t id) 9916 { 9917 return NULL; 9918 } 9919 9920 static void 9921 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9922 __rte_unused struct mlx5_flow_tunnel *tunnel) 9923 { 9924 } 9925 9926 static uint32_t 9927 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9928 __rte_unused const struct mlx5_flow_tunnel *t, 9929 __rte_unused uint32_t group, 9930 __rte_unused uint32_t *table, 9931 struct rte_flow_error *error) 9932 { 9933 return rte_flow_error_set(error, ENOTSUP, 9934 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9935 "tunnel offload requires DV support"); 9936 } 9937 9938 void 9939 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9940 __rte_unused uint16_t port_id) 9941 { 9942 } 9943 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9944 9945 /* Flex flow item API */ 9946 static struct rte_flow_item_flex_handle * 9947 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 9948 const struct rte_flow_item_flex_conf *conf, 9949 struct rte_flow_error *error) 9950 { 9951 static const char err_msg[] = "flex item creation unsupported"; 9952 struct rte_flow_attr attr = { .transfer = 0 }; 9953 const struct mlx5_flow_driver_ops *fops = 9954 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9955 9956 if (!fops->item_create) { 9957 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9958 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9959 NULL, err_msg); 9960 return NULL; 9961 } 9962 return fops->item_create(dev, conf, error); 9963 } 9964 9965 static int 9966 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 9967 const struct rte_flow_item_flex_handle *handle, 9968 struct rte_flow_error *error) 9969 { 9970 static const char err_msg[] = "flex item release unsupported"; 9971 struct rte_flow_attr attr = { .transfer = 0 }; 9972 const struct mlx5_flow_driver_ops *fops = 9973 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9974 9975 if (!fops->item_release) { 9976 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9977 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9978 NULL, err_msg); 9979 return -rte_errno; 9980 } 9981 return fops->item_release(dev, handle, error); 9982 } 9983 9984 static void 9985 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9986 { 9987 int ret; 9988 struct rte_flow_error error; 9989 9990 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9991 char *item_name; 9992 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9993 sizeof(item_name), 9994 (void *)(uintptr_t)item->type, &error); 9995 if (ret > 0) 9996 printf("%s ", item_name); 9997 else 9998 printf("%d\n", (int)item->type); 9999 } 10000 printf("END\n"); 10001 } 10002 10003 static int 10004 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 10005 { 10006 const struct rte_flow_item_udp *spec = udp_item->spec; 10007 const struct rte_flow_item_udp *mask = udp_item->mask; 10008 uint16_t udp_dport = 0; 10009 10010 if (spec != NULL) { 10011 if (!mask) 10012 mask = &rte_flow_item_udp_mask; 10013 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 10014 mask->hdr.dst_port); 10015 } 10016 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 10017 } 10018 10019 static const struct mlx5_flow_expand_node * 10020 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 10021 unsigned int item_idx, 10022 const struct mlx5_flow_expand_node graph[], 10023 const struct mlx5_flow_expand_node *node) 10024 { 10025 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 10026 10027 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 10028 node != NULL && 10029 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 10030 /* 10031 * The expansion node is VXLAN and it is also the last 10032 * expandable item in the pattern, so need to continue 10033 * expansion of the inner tunnel. 10034 */ 10035 MLX5_ASSERT(item_idx > 0); 10036 prev_item = pattern + item_idx - 1; 10037 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 10038 if (mlx5_flow_is_std_vxlan_port(prev_item)) 10039 return &graph[MLX5_EXPANSION_STD_VXLAN]; 10040 return &graph[MLX5_EXPANSION_L3_VXLAN]; 10041 } 10042 return node; 10043 } 10044 10045 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 10046 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 10047 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 10048 }; 10049 10050 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 10051 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 10052 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 10053 { 9, 10, 11 }, { 12, 13, 14 }, 10054 }; 10055 10056 /** 10057 * Discover the number of available flow priorities. 10058 * 10059 * @param dev 10060 * Ethernet device. 10061 * 10062 * @return 10063 * On success, number of available flow priorities. 10064 * On failure, a negative errno-style code and rte_errno is set. 10065 */ 10066 int 10067 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 10068 { 10069 static const uint16_t vprio[] = {8, 16}; 10070 const struct mlx5_priv *priv = dev->data->dev_private; 10071 const struct mlx5_flow_driver_ops *fops; 10072 enum mlx5_flow_drv_type type; 10073 int ret; 10074 10075 type = mlx5_flow_os_get_type(); 10076 if (type == MLX5_FLOW_TYPE_MAX) { 10077 type = MLX5_FLOW_TYPE_VERBS; 10078 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 10079 type = MLX5_FLOW_TYPE_DV; 10080 } 10081 fops = flow_get_drv_ops(type); 10082 if (fops->discover_priorities == NULL) { 10083 DRV_LOG(ERR, "Priority discovery not supported"); 10084 rte_errno = ENOTSUP; 10085 return -rte_errno; 10086 } 10087 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 10088 if (ret < 0) 10089 return ret; 10090 switch (ret) { 10091 case 8: 10092 ret = RTE_DIM(priority_map_3); 10093 break; 10094 case 16: 10095 ret = RTE_DIM(priority_map_5); 10096 break; 10097 default: 10098 rte_errno = ENOTSUP; 10099 DRV_LOG(ERR, 10100 "port %u maximum priority: %d expected 8/16", 10101 dev->data->port_id, ret); 10102 return -rte_errno; 10103 } 10104 DRV_LOG(INFO, "port %u supported flow priorities:" 10105 " 0-%d for ingress or egress root table," 10106 " 0-%d for non-root table or transfer root table.", 10107 dev->data->port_id, ret - 2, 10108 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 10109 return ret; 10110 } 10111 10112 /** 10113 * Adjust flow priority based on the highest layer and the request priority. 10114 * 10115 * @param[in] dev 10116 * Pointer to the Ethernet device structure. 10117 * @param[in] priority 10118 * The rule base priority. 10119 * @param[in] subpriority 10120 * The priority based on the items. 10121 * 10122 * @return 10123 * The new priority. 10124 */ 10125 uint32_t 10126 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10127 uint32_t subpriority) 10128 { 10129 uint32_t res = 0; 10130 struct mlx5_priv *priv = dev->data->dev_private; 10131 10132 switch (priv->sh->flow_max_priority) { 10133 case RTE_DIM(priority_map_3): 10134 res = priority_map_3[priority][subpriority]; 10135 break; 10136 case RTE_DIM(priority_map_5): 10137 res = priority_map_5[priority][subpriority]; 10138 break; 10139 } 10140 return res; 10141 } 10142