1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 80 #endif 81 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 82 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 83 }; 84 85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 87 (const int []){ \ 88 __VA_ARGS__, 0, \ 89 } 90 91 /** Node object of input graph for mlx5_flow_expand_rss(). */ 92 struct mlx5_flow_expand_node { 93 const int *const next; 94 /**< 95 * List of next node indexes. Index 0 is interpreted as a terminator. 96 */ 97 const enum rte_flow_item_type type; 98 /**< Pattern item type of current node. */ 99 uint64_t rss_types; 100 /**< 101 * RSS types bit-field associated with this node 102 * (see RTE_ETH_RSS_* definitions). 103 */ 104 uint64_t node_flags; 105 /**< 106 * Bit-fields that define how the node is used in the expansion. 107 * (see MLX5_EXPANSION_NODE_* definitions). 108 */ 109 }; 110 111 /* Optional expand field. The expansion alg will not go deeper. */ 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 113 114 /* The node is not added implicitly as expansion to the flow pattern. 115 * If the node type does not match the flow pattern item type, the 116 * expansion alg will go deeper to its next items. 117 * In the current implementation, the list of next nodes indexes can 118 * have up to one node with this flag set and it has to be the last 119 * node index (before the list terminator). 120 */ 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 122 123 /** Object returned by mlx5_flow_expand_rss(). */ 124 struct mlx5_flow_expand_rss { 125 uint32_t entries; 126 /**< Number of entries @p patterns and @p priorities. */ 127 struct { 128 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 129 uint32_t priority; /**< Priority offset for each expansion. */ 130 } entry[]; 131 }; 132 133 static void 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 135 136 static const struct mlx5_flow_expand_node * 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 138 unsigned int item_idx, 139 const struct mlx5_flow_expand_node graph[], 140 const struct mlx5_flow_expand_node *node); 141 142 static bool 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 144 { 145 switch (item->type) { 146 case RTE_FLOW_ITEM_TYPE_ETH: 147 case RTE_FLOW_ITEM_TYPE_VLAN: 148 case RTE_FLOW_ITEM_TYPE_IPV4: 149 case RTE_FLOW_ITEM_TYPE_IPV6: 150 case RTE_FLOW_ITEM_TYPE_UDP: 151 case RTE_FLOW_ITEM_TYPE_TCP: 152 case RTE_FLOW_ITEM_TYPE_VXLAN: 153 case RTE_FLOW_ITEM_TYPE_NVGRE: 154 case RTE_FLOW_ITEM_TYPE_GRE: 155 case RTE_FLOW_ITEM_TYPE_GENEVE: 156 case RTE_FLOW_ITEM_TYPE_MPLS: 157 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 158 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 159 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 160 case RTE_FLOW_ITEM_TYPE_GTP: 161 return true; 162 default: 163 break; 164 } 165 return false; 166 } 167 168 /** 169 * Network Service Header (NSH) and its next protocol values 170 * are described in RFC-8393. 171 */ 172 static enum rte_flow_item_type 173 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 174 { 175 enum rte_flow_item_type type; 176 177 switch (proto_mask & proto_spec) { 178 case RTE_VXLAN_GPE_TYPE_IPV4: 179 type = RTE_FLOW_ITEM_TYPE_IPV4; 180 break; 181 case RTE_VXLAN_GPE_TYPE_IPV6: 182 type = RTE_VXLAN_GPE_TYPE_IPV6; 183 break; 184 case RTE_VXLAN_GPE_TYPE_ETH: 185 type = RTE_FLOW_ITEM_TYPE_ETH; 186 break; 187 default: 188 type = RTE_FLOW_ITEM_TYPE_END; 189 } 190 return type; 191 } 192 193 static enum rte_flow_item_type 194 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 195 { 196 enum rte_flow_item_type type; 197 198 switch (proto_mask & proto_spec) { 199 case IPPROTO_UDP: 200 type = RTE_FLOW_ITEM_TYPE_UDP; 201 break; 202 case IPPROTO_TCP: 203 type = RTE_FLOW_ITEM_TYPE_TCP; 204 break; 205 case IPPROTO_IP: 206 type = RTE_FLOW_ITEM_TYPE_IPV4; 207 break; 208 case IPPROTO_IPV6: 209 type = RTE_FLOW_ITEM_TYPE_IPV6; 210 break; 211 default: 212 type = RTE_FLOW_ITEM_TYPE_END; 213 } 214 return type; 215 } 216 217 static enum rte_flow_item_type 218 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 219 rte_be16_t type_mask, bool is_tunnel) 220 { 221 enum rte_flow_item_type type; 222 223 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 224 case RTE_ETHER_TYPE_TEB: 225 type = is_tunnel ? 226 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 227 break; 228 case RTE_ETHER_TYPE_VLAN: 229 type = !is_tunnel ? 230 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 231 break; 232 case RTE_ETHER_TYPE_IPV4: 233 type = RTE_FLOW_ITEM_TYPE_IPV4; 234 break; 235 case RTE_ETHER_TYPE_IPV6: 236 type = RTE_FLOW_ITEM_TYPE_IPV6; 237 break; 238 default: 239 type = RTE_FLOW_ITEM_TYPE_END; 240 } 241 return type; 242 } 243 244 static enum rte_flow_item_type 245 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 246 { 247 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 248 do { \ 249 const void *m = item->mask; \ 250 const void *s = item->spec; \ 251 mask = m ? \ 252 ((const struct rte_flow_item_##type *)m)->fld : \ 253 rte_flow_item_##type##_mask.fld; \ 254 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 255 } while (0) 256 257 enum rte_flow_item_type ret; 258 uint16_t spec, mask; 259 260 if (item == NULL || item->spec == NULL) 261 return RTE_FLOW_ITEM_TYPE_VOID; 262 switch (item->type) { 263 case RTE_FLOW_ITEM_TYPE_ETH: 264 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 265 if (!mask) 266 return RTE_FLOW_ITEM_TYPE_VOID; 267 ret = mlx5_ethertype_to_item_type(spec, mask, false); 268 break; 269 case RTE_FLOW_ITEM_TYPE_VLAN: 270 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 271 if (!mask) 272 return RTE_FLOW_ITEM_TYPE_VOID; 273 ret = mlx5_ethertype_to_item_type(spec, mask, false); 274 break; 275 case RTE_FLOW_ITEM_TYPE_IPV4: 276 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 277 if (!mask) 278 return RTE_FLOW_ITEM_TYPE_VOID; 279 ret = mlx5_inet_proto_to_item_type(spec, mask); 280 break; 281 case RTE_FLOW_ITEM_TYPE_IPV6: 282 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 283 if (!mask) 284 return RTE_FLOW_ITEM_TYPE_VOID; 285 ret = mlx5_inet_proto_to_item_type(spec, mask); 286 break; 287 case RTE_FLOW_ITEM_TYPE_GENEVE: 288 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 289 ret = mlx5_ethertype_to_item_type(spec, mask, true); 290 break; 291 case RTE_FLOW_ITEM_TYPE_GRE: 292 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 293 ret = mlx5_ethertype_to_item_type(spec, mask, true); 294 break; 295 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 296 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 297 ret = mlx5_nsh_proto_to_item_type(spec, mask); 298 break; 299 default: 300 ret = RTE_FLOW_ITEM_TYPE_VOID; 301 break; 302 } 303 return ret; 304 #undef MLX5_XSET_ITEM_MASK_SPEC 305 } 306 307 static const int * 308 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 309 const int *next_node) 310 { 311 const struct mlx5_flow_expand_node *node = NULL; 312 const int *next = next_node; 313 314 while (next && *next) { 315 /* 316 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 317 * flag set, because they were not found in the flow pattern. 318 */ 319 node = &graph[*next]; 320 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 321 break; 322 next = node->next; 323 } 324 return next; 325 } 326 327 #define MLX5_RSS_EXP_ELT_N 16 328 329 /** 330 * Expand RSS flows into several possible flows according to the RSS hash 331 * fields requested and the driver capabilities. 332 * 333 * @param[out] buf 334 * Buffer to store the result expansion. 335 * @param[in] size 336 * Buffer size in bytes. If 0, @p buf can be NULL. 337 * @param[in] pattern 338 * User flow pattern. 339 * @param[in] types 340 * RSS types to expand (see RTE_ETH_RSS_* definitions). 341 * @param[in] graph 342 * Input graph to expand @p pattern according to @p types. 343 * @param[in] graph_root_index 344 * Index of root node in @p graph, typically 0. 345 * 346 * @return 347 * A positive value representing the size of @p buf in bytes regardless of 348 * @p size on success, a negative errno value otherwise and rte_errno is 349 * set, the following errors are defined: 350 * 351 * -E2BIG: graph-depth @p graph is too deep. 352 * -EINVAL: @p size has not enough space for expanded pattern. 353 */ 354 static int 355 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 356 const struct rte_flow_item *pattern, uint64_t types, 357 const struct mlx5_flow_expand_node graph[], 358 int graph_root_index) 359 { 360 const struct rte_flow_item *item; 361 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 362 const int *next_node; 363 const int *stack[MLX5_RSS_EXP_ELT_N]; 364 int stack_pos = 0; 365 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 366 unsigned int i, item_idx, last_expand_item_idx = 0; 367 size_t lsize; 368 size_t user_pattern_size = 0; 369 void *addr = NULL; 370 const struct mlx5_flow_expand_node *next = NULL; 371 struct rte_flow_item missed_item; 372 int missed = 0; 373 int elt = 0; 374 const struct rte_flow_item *last_expand_item = NULL; 375 376 memset(&missed_item, 0, sizeof(missed_item)); 377 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 378 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 379 if (lsize > size) 380 return -EINVAL; 381 buf->entry[0].priority = 0; 382 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 383 buf->entries = 0; 384 addr = buf->entry[0].pattern; 385 for (item = pattern, item_idx = 0; 386 item->type != RTE_FLOW_ITEM_TYPE_END; 387 item++, item_idx++) { 388 if (!mlx5_flow_is_rss_expandable_item(item)) { 389 user_pattern_size += sizeof(*item); 390 continue; 391 } 392 last_expand_item = item; 393 last_expand_item_idx = item_idx; 394 i = 0; 395 while (node->next && node->next[i]) { 396 next = &graph[node->next[i]]; 397 if (next->type == item->type) 398 break; 399 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 400 node = next; 401 i = 0; 402 } else { 403 ++i; 404 } 405 } 406 if (next) 407 node = next; 408 user_pattern_size += sizeof(*item); 409 } 410 user_pattern_size += sizeof(*item); /* Handle END item. */ 411 lsize += user_pattern_size; 412 if (lsize > size) 413 return -EINVAL; 414 /* Copy the user pattern in the first entry of the buffer. */ 415 rte_memcpy(addr, pattern, user_pattern_size); 416 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 417 buf->entries = 1; 418 /* Start expanding. */ 419 memset(flow_items, 0, sizeof(flow_items)); 420 user_pattern_size -= sizeof(*item); 421 /* 422 * Check if the last valid item has spec set, need complete pattern, 423 * and the pattern can be used for expansion. 424 */ 425 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 426 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 427 /* Item type END indicates expansion is not required. */ 428 return lsize; 429 } 430 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 431 next = NULL; 432 missed = 1; 433 i = 0; 434 while (node->next && node->next[i]) { 435 next = &graph[node->next[i]]; 436 if (next->type == missed_item.type) { 437 flow_items[0].type = missed_item.type; 438 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 439 break; 440 } 441 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 442 node = next; 443 i = 0; 444 } else { 445 ++i; 446 } 447 next = NULL; 448 } 449 } 450 if (next && missed) { 451 elt = 2; /* missed item + item end. */ 452 node = next; 453 lsize += elt * sizeof(*item) + user_pattern_size; 454 if (lsize > size) 455 return -EINVAL; 456 if (node->rss_types & types) { 457 buf->entry[buf->entries].priority = 1; 458 buf->entry[buf->entries].pattern = addr; 459 buf->entries++; 460 rte_memcpy(addr, buf->entry[0].pattern, 461 user_pattern_size); 462 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 463 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 464 addr = (void *)(((uintptr_t)addr) + 465 elt * sizeof(*item)); 466 } 467 } else if (last_expand_item != NULL) { 468 node = mlx5_flow_expand_rss_adjust_node(pattern, 469 last_expand_item_idx, graph, node); 470 } 471 memset(flow_items, 0, sizeof(flow_items)); 472 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 473 node->next); 474 stack[stack_pos] = next_node; 475 node = next_node ? &graph[*next_node] : NULL; 476 while (node) { 477 flow_items[stack_pos].type = node->type; 478 if (node->rss_types & types) { 479 size_t n; 480 /* 481 * compute the number of items to copy from the 482 * expansion and copy it. 483 * When the stack_pos is 0, there are 1 element in it, 484 * plus the addition END item. 485 */ 486 elt = stack_pos + 2; 487 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 488 lsize += elt * sizeof(*item) + user_pattern_size; 489 if (lsize > size) 490 return -EINVAL; 491 n = elt * sizeof(*item); 492 buf->entry[buf->entries].priority = 493 stack_pos + 1 + missed; 494 buf->entry[buf->entries].pattern = addr; 495 buf->entries++; 496 rte_memcpy(addr, buf->entry[0].pattern, 497 user_pattern_size); 498 addr = (void *)(((uintptr_t)addr) + 499 user_pattern_size); 500 rte_memcpy(addr, &missed_item, 501 missed * sizeof(*item)); 502 addr = (void *)(((uintptr_t)addr) + 503 missed * sizeof(*item)); 504 rte_memcpy(addr, flow_items, n); 505 addr = (void *)(((uintptr_t)addr) + n); 506 } 507 /* Go deeper. */ 508 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 509 node->next) { 510 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 511 node->next); 512 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 513 rte_errno = E2BIG; 514 return -rte_errno; 515 } 516 stack[stack_pos] = next_node; 517 } else if (*(next_node + 1)) { 518 /* Follow up with the next possibility. */ 519 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 520 ++next_node); 521 } else if (!stack_pos) { 522 /* 523 * Completing the traverse over the different paths. 524 * The next_node is advanced to the terminator. 525 */ 526 ++next_node; 527 } else { 528 /* Move to the next path. */ 529 while (stack_pos) { 530 next_node = stack[--stack_pos]; 531 next_node++; 532 if (*next_node) 533 break; 534 } 535 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 536 next_node); 537 stack[stack_pos] = next_node; 538 } 539 node = next_node && *next_node ? &graph[*next_node] : NULL; 540 }; 541 return lsize; 542 } 543 544 enum mlx5_expansion { 545 MLX5_EXPANSION_ROOT, 546 MLX5_EXPANSION_ROOT_OUTER, 547 MLX5_EXPANSION_OUTER_ETH, 548 MLX5_EXPANSION_OUTER_VLAN, 549 MLX5_EXPANSION_OUTER_IPV4, 550 MLX5_EXPANSION_OUTER_IPV4_UDP, 551 MLX5_EXPANSION_OUTER_IPV4_TCP, 552 MLX5_EXPANSION_OUTER_IPV6, 553 MLX5_EXPANSION_OUTER_IPV6_UDP, 554 MLX5_EXPANSION_OUTER_IPV6_TCP, 555 MLX5_EXPANSION_VXLAN, 556 MLX5_EXPANSION_STD_VXLAN, 557 MLX5_EXPANSION_L3_VXLAN, 558 MLX5_EXPANSION_VXLAN_GPE, 559 MLX5_EXPANSION_GRE, 560 MLX5_EXPANSION_NVGRE, 561 MLX5_EXPANSION_GRE_KEY, 562 MLX5_EXPANSION_MPLS, 563 MLX5_EXPANSION_ETH, 564 MLX5_EXPANSION_VLAN, 565 MLX5_EXPANSION_IPV4, 566 MLX5_EXPANSION_IPV4_UDP, 567 MLX5_EXPANSION_IPV4_TCP, 568 MLX5_EXPANSION_IPV6, 569 MLX5_EXPANSION_IPV6_UDP, 570 MLX5_EXPANSION_IPV6_TCP, 571 MLX5_EXPANSION_IPV6_FRAG_EXT, 572 MLX5_EXPANSION_GTP, 573 MLX5_EXPANSION_GENEVE, 574 }; 575 576 /** Supported expansion of items. */ 577 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 578 [MLX5_EXPANSION_ROOT] = { 579 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 580 MLX5_EXPANSION_IPV4, 581 MLX5_EXPANSION_IPV6), 582 .type = RTE_FLOW_ITEM_TYPE_END, 583 }, 584 [MLX5_EXPANSION_ROOT_OUTER] = { 585 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 586 MLX5_EXPANSION_OUTER_IPV4, 587 MLX5_EXPANSION_OUTER_IPV6), 588 .type = RTE_FLOW_ITEM_TYPE_END, 589 }, 590 [MLX5_EXPANSION_OUTER_ETH] = { 591 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 592 .type = RTE_FLOW_ITEM_TYPE_ETH, 593 .rss_types = 0, 594 }, 595 [MLX5_EXPANSION_OUTER_VLAN] = { 596 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 597 MLX5_EXPANSION_OUTER_IPV6), 598 .type = RTE_FLOW_ITEM_TYPE_VLAN, 599 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 600 }, 601 [MLX5_EXPANSION_OUTER_IPV4] = { 602 .next = MLX5_FLOW_EXPAND_RSS_NEXT 603 (MLX5_EXPANSION_OUTER_IPV4_UDP, 604 MLX5_EXPANSION_OUTER_IPV4_TCP, 605 MLX5_EXPANSION_GRE, 606 MLX5_EXPANSION_NVGRE, 607 MLX5_EXPANSION_IPV4, 608 MLX5_EXPANSION_IPV6), 609 .type = RTE_FLOW_ITEM_TYPE_IPV4, 610 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 611 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 612 }, 613 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 614 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 615 MLX5_EXPANSION_VXLAN_GPE, 616 MLX5_EXPANSION_MPLS, 617 MLX5_EXPANSION_GENEVE, 618 MLX5_EXPANSION_GTP), 619 .type = RTE_FLOW_ITEM_TYPE_UDP, 620 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 621 }, 622 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 623 .type = RTE_FLOW_ITEM_TYPE_TCP, 624 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 625 }, 626 [MLX5_EXPANSION_OUTER_IPV6] = { 627 .next = MLX5_FLOW_EXPAND_RSS_NEXT 628 (MLX5_EXPANSION_OUTER_IPV6_UDP, 629 MLX5_EXPANSION_OUTER_IPV6_TCP, 630 MLX5_EXPANSION_IPV4, 631 MLX5_EXPANSION_IPV6, 632 MLX5_EXPANSION_GRE, 633 MLX5_EXPANSION_NVGRE), 634 .type = RTE_FLOW_ITEM_TYPE_IPV6, 635 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 636 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 637 }, 638 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 639 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 640 MLX5_EXPANSION_VXLAN_GPE, 641 MLX5_EXPANSION_MPLS, 642 MLX5_EXPANSION_GENEVE, 643 MLX5_EXPANSION_GTP), 644 .type = RTE_FLOW_ITEM_TYPE_UDP, 645 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 646 }, 647 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 648 .type = RTE_FLOW_ITEM_TYPE_TCP, 649 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 650 }, 651 [MLX5_EXPANSION_VXLAN] = { 652 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 653 MLX5_EXPANSION_IPV4, 654 MLX5_EXPANSION_IPV6), 655 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 656 }, 657 [MLX5_EXPANSION_STD_VXLAN] = { 658 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 659 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 660 }, 661 [MLX5_EXPANSION_L3_VXLAN] = { 662 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 663 MLX5_EXPANSION_IPV6), 664 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 665 }, 666 [MLX5_EXPANSION_VXLAN_GPE] = { 667 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 668 MLX5_EXPANSION_IPV4, 669 MLX5_EXPANSION_IPV6), 670 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 671 }, 672 [MLX5_EXPANSION_GRE] = { 673 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 674 MLX5_EXPANSION_IPV4, 675 MLX5_EXPANSION_IPV6, 676 MLX5_EXPANSION_GRE_KEY, 677 MLX5_EXPANSION_MPLS), 678 .type = RTE_FLOW_ITEM_TYPE_GRE, 679 }, 680 [MLX5_EXPANSION_GRE_KEY] = { 681 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 682 MLX5_EXPANSION_IPV6, 683 MLX5_EXPANSION_MPLS), 684 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 685 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 686 }, 687 [MLX5_EXPANSION_NVGRE] = { 688 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 689 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 690 }, 691 [MLX5_EXPANSION_MPLS] = { 692 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 693 MLX5_EXPANSION_IPV6, 694 MLX5_EXPANSION_ETH), 695 .type = RTE_FLOW_ITEM_TYPE_MPLS, 696 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 697 }, 698 [MLX5_EXPANSION_ETH] = { 699 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 700 .type = RTE_FLOW_ITEM_TYPE_ETH, 701 }, 702 [MLX5_EXPANSION_VLAN] = { 703 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 704 MLX5_EXPANSION_IPV6), 705 .type = RTE_FLOW_ITEM_TYPE_VLAN, 706 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 707 }, 708 [MLX5_EXPANSION_IPV4] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 710 MLX5_EXPANSION_IPV4_TCP), 711 .type = RTE_FLOW_ITEM_TYPE_IPV4, 712 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 713 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 714 }, 715 [MLX5_EXPANSION_IPV4_UDP] = { 716 .type = RTE_FLOW_ITEM_TYPE_UDP, 717 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 718 }, 719 [MLX5_EXPANSION_IPV4_TCP] = { 720 .type = RTE_FLOW_ITEM_TYPE_TCP, 721 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 722 }, 723 [MLX5_EXPANSION_IPV6] = { 724 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 725 MLX5_EXPANSION_IPV6_TCP, 726 MLX5_EXPANSION_IPV6_FRAG_EXT), 727 .type = RTE_FLOW_ITEM_TYPE_IPV6, 728 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 729 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 730 }, 731 [MLX5_EXPANSION_IPV6_UDP] = { 732 .type = RTE_FLOW_ITEM_TYPE_UDP, 733 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 734 }, 735 [MLX5_EXPANSION_IPV6_TCP] = { 736 .type = RTE_FLOW_ITEM_TYPE_TCP, 737 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 738 }, 739 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 740 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 741 }, 742 [MLX5_EXPANSION_GTP] = { 743 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_GTP, 746 }, 747 [MLX5_EXPANSION_GENEVE] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 749 MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 752 }, 753 }; 754 755 static struct rte_flow_action_handle * 756 mlx5_action_handle_create(struct rte_eth_dev *dev, 757 const struct rte_flow_indir_action_conf *conf, 758 const struct rte_flow_action *action, 759 struct rte_flow_error *error); 760 static int mlx5_action_handle_destroy 761 (struct rte_eth_dev *dev, 762 struct rte_flow_action_handle *handle, 763 struct rte_flow_error *error); 764 static int mlx5_action_handle_update 765 (struct rte_eth_dev *dev, 766 struct rte_flow_action_handle *handle, 767 const void *update, 768 struct rte_flow_error *error); 769 static int mlx5_action_handle_query 770 (struct rte_eth_dev *dev, 771 const struct rte_flow_action_handle *handle, 772 void *data, 773 struct rte_flow_error *error); 774 static int 775 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 776 struct rte_flow_tunnel *app_tunnel, 777 struct rte_flow_action **actions, 778 uint32_t *num_of_actions, 779 struct rte_flow_error *error); 780 static int 781 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 782 struct rte_flow_tunnel *app_tunnel, 783 struct rte_flow_item **items, 784 uint32_t *num_of_items, 785 struct rte_flow_error *error); 786 static int 787 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 788 struct rte_flow_item *pmd_items, 789 uint32_t num_items, struct rte_flow_error *err); 790 static int 791 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 792 struct rte_flow_action *pmd_actions, 793 uint32_t num_actions, 794 struct rte_flow_error *err); 795 static int 796 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 797 struct rte_mbuf *m, 798 struct rte_flow_restore_info *info, 799 struct rte_flow_error *err); 800 static struct rte_flow_item_flex_handle * 801 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 802 const struct rte_flow_item_flex_conf *conf, 803 struct rte_flow_error *error); 804 static int 805 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 806 const struct rte_flow_item_flex_handle *handle, 807 struct rte_flow_error *error); 808 static int 809 mlx5_flow_info_get(struct rte_eth_dev *dev, 810 struct rte_flow_port_info *port_info, 811 struct rte_flow_queue_info *queue_info, 812 struct rte_flow_error *error); 813 static int 814 mlx5_flow_port_configure(struct rte_eth_dev *dev, 815 const struct rte_flow_port_attr *port_attr, 816 uint16_t nb_queue, 817 const struct rte_flow_queue_attr *queue_attr[], 818 struct rte_flow_error *err); 819 820 static struct rte_flow_pattern_template * 821 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 822 const struct rte_flow_pattern_template_attr *attr, 823 const struct rte_flow_item items[], 824 struct rte_flow_error *error); 825 826 static int 827 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 828 struct rte_flow_pattern_template *template, 829 struct rte_flow_error *error); 830 static struct rte_flow_actions_template * 831 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 832 const struct rte_flow_actions_template_attr *attr, 833 const struct rte_flow_action actions[], 834 const struct rte_flow_action masks[], 835 struct rte_flow_error *error); 836 static int 837 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 838 struct rte_flow_actions_template *template, 839 struct rte_flow_error *error); 840 841 static struct rte_flow_template_table * 842 mlx5_flow_table_create(struct rte_eth_dev *dev, 843 const struct rte_flow_template_table_attr *attr, 844 struct rte_flow_pattern_template *item_templates[], 845 uint8_t nb_item_templates, 846 struct rte_flow_actions_template *action_templates[], 847 uint8_t nb_action_templates, 848 struct rte_flow_error *error); 849 static int 850 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 851 struct rte_flow_template_table *table, 852 struct rte_flow_error *error); 853 static struct rte_flow * 854 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 855 uint32_t queue, 856 const struct rte_flow_op_attr *attr, 857 struct rte_flow_template_table *table, 858 const struct rte_flow_item items[], 859 uint8_t pattern_template_index, 860 const struct rte_flow_action actions[], 861 uint8_t action_template_index, 862 void *user_data, 863 struct rte_flow_error *error); 864 static int 865 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 866 uint32_t queue, 867 const struct rte_flow_op_attr *attr, 868 struct rte_flow *flow, 869 void *user_data, 870 struct rte_flow_error *error); 871 static int 872 mlx5_flow_pull(struct rte_eth_dev *dev, 873 uint32_t queue, 874 struct rte_flow_op_result res[], 875 uint16_t n_res, 876 struct rte_flow_error *error); 877 static int 878 mlx5_flow_push(struct rte_eth_dev *dev, 879 uint32_t queue, 880 struct rte_flow_error *error); 881 882 static struct rte_flow_action_handle * 883 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 884 const struct rte_flow_op_attr *attr, 885 const struct rte_flow_indir_action_conf *conf, 886 const struct rte_flow_action *action, 887 void *user_data, 888 struct rte_flow_error *error); 889 890 static int 891 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 892 const struct rte_flow_op_attr *attr, 893 struct rte_flow_action_handle *handle, 894 const void *update, 895 void *user_data, 896 struct rte_flow_error *error); 897 898 static int 899 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 900 const struct rte_flow_op_attr *attr, 901 struct rte_flow_action_handle *handle, 902 void *user_data, 903 struct rte_flow_error *error); 904 905 static const struct rte_flow_ops mlx5_flow_ops = { 906 .validate = mlx5_flow_validate, 907 .create = mlx5_flow_create, 908 .destroy = mlx5_flow_destroy, 909 .flush = mlx5_flow_flush, 910 .isolate = mlx5_flow_isolate, 911 .query = mlx5_flow_query, 912 .dev_dump = mlx5_flow_dev_dump, 913 .get_aged_flows = mlx5_flow_get_aged_flows, 914 .action_handle_create = mlx5_action_handle_create, 915 .action_handle_destroy = mlx5_action_handle_destroy, 916 .action_handle_update = mlx5_action_handle_update, 917 .action_handle_query = mlx5_action_handle_query, 918 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 919 .tunnel_match = mlx5_flow_tunnel_match, 920 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 921 .tunnel_item_release = mlx5_flow_tunnel_item_release, 922 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 923 .flex_item_create = mlx5_flow_flex_item_create, 924 .flex_item_release = mlx5_flow_flex_item_release, 925 .info_get = mlx5_flow_info_get, 926 .configure = mlx5_flow_port_configure, 927 .pattern_template_create = mlx5_flow_pattern_template_create, 928 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 929 .actions_template_create = mlx5_flow_actions_template_create, 930 .actions_template_destroy = mlx5_flow_actions_template_destroy, 931 .template_table_create = mlx5_flow_table_create, 932 .template_table_destroy = mlx5_flow_table_destroy, 933 .async_create = mlx5_flow_async_flow_create, 934 .async_destroy = mlx5_flow_async_flow_destroy, 935 .pull = mlx5_flow_pull, 936 .push = mlx5_flow_push, 937 .async_action_handle_create = mlx5_flow_async_action_handle_create, 938 .async_action_handle_update = mlx5_flow_async_action_handle_update, 939 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 940 }; 941 942 /* Tunnel information. */ 943 struct mlx5_flow_tunnel_info { 944 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 945 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 946 }; 947 948 static struct mlx5_flow_tunnel_info tunnels_info[] = { 949 { 950 .tunnel = MLX5_FLOW_LAYER_VXLAN, 951 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 952 }, 953 { 954 .tunnel = MLX5_FLOW_LAYER_GENEVE, 955 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 956 }, 957 { 958 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 959 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 960 }, 961 { 962 .tunnel = MLX5_FLOW_LAYER_GRE, 963 .ptype = RTE_PTYPE_TUNNEL_GRE, 964 }, 965 { 966 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 967 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 968 }, 969 { 970 .tunnel = MLX5_FLOW_LAYER_MPLS, 971 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 972 }, 973 { 974 .tunnel = MLX5_FLOW_LAYER_NVGRE, 975 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 976 }, 977 { 978 .tunnel = MLX5_FLOW_LAYER_IPIP, 979 .ptype = RTE_PTYPE_TUNNEL_IP, 980 }, 981 { 982 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 983 .ptype = RTE_PTYPE_TUNNEL_IP, 984 }, 985 { 986 .tunnel = MLX5_FLOW_LAYER_GTP, 987 .ptype = RTE_PTYPE_TUNNEL_GTPU, 988 }, 989 }; 990 991 992 993 /** 994 * Translate tag ID to register. 995 * 996 * @param[in] dev 997 * Pointer to the Ethernet device structure. 998 * @param[in] feature 999 * The feature that request the register. 1000 * @param[in] id 1001 * The request register ID. 1002 * @param[out] error 1003 * Error description in case of any. 1004 * 1005 * @return 1006 * The request register on success, a negative errno 1007 * value otherwise and rte_errno is set. 1008 */ 1009 int 1010 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1011 enum mlx5_feature_name feature, 1012 uint32_t id, 1013 struct rte_flow_error *error) 1014 { 1015 struct mlx5_priv *priv = dev->data->dev_private; 1016 struct mlx5_sh_config *config = &priv->sh->config; 1017 enum modify_reg start_reg; 1018 bool skip_mtr_reg = false; 1019 1020 switch (feature) { 1021 case MLX5_HAIRPIN_RX: 1022 return REG_B; 1023 case MLX5_HAIRPIN_TX: 1024 return REG_A; 1025 case MLX5_METADATA_RX: 1026 switch (config->dv_xmeta_en) { 1027 case MLX5_XMETA_MODE_LEGACY: 1028 return REG_B; 1029 case MLX5_XMETA_MODE_META16: 1030 return REG_C_0; 1031 case MLX5_XMETA_MODE_META32: 1032 return REG_C_1; 1033 } 1034 break; 1035 case MLX5_METADATA_TX: 1036 return REG_A; 1037 case MLX5_METADATA_FDB: 1038 switch (config->dv_xmeta_en) { 1039 case MLX5_XMETA_MODE_LEGACY: 1040 return REG_NON; 1041 case MLX5_XMETA_MODE_META16: 1042 return REG_C_0; 1043 case MLX5_XMETA_MODE_META32: 1044 return REG_C_1; 1045 } 1046 break; 1047 case MLX5_FLOW_MARK: 1048 switch (config->dv_xmeta_en) { 1049 case MLX5_XMETA_MODE_LEGACY: 1050 return REG_NON; 1051 case MLX5_XMETA_MODE_META16: 1052 return REG_C_1; 1053 case MLX5_XMETA_MODE_META32: 1054 return REG_C_0; 1055 } 1056 break; 1057 case MLX5_MTR_ID: 1058 /* 1059 * If meter color and meter id share one register, flow match 1060 * should use the meter color register for match. 1061 */ 1062 if (priv->mtr_reg_share) 1063 return priv->mtr_color_reg; 1064 else 1065 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1066 REG_C_3; 1067 case MLX5_MTR_COLOR: 1068 case MLX5_ASO_FLOW_HIT: 1069 case MLX5_ASO_CONNTRACK: 1070 case MLX5_SAMPLE_ID: 1071 /* All features use the same REG_C. */ 1072 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1073 return priv->mtr_color_reg; 1074 case MLX5_COPY_MARK: 1075 /* 1076 * Metadata COPY_MARK register using is in meter suffix sub 1077 * flow while with meter. It's safe to share the same register. 1078 */ 1079 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1080 case MLX5_APP_TAG: 1081 /* 1082 * If meter is enable, it will engage the register for color 1083 * match and flow match. If meter color match is not using the 1084 * REG_C_2, need to skip the REG_C_x be used by meter color 1085 * match. 1086 * If meter is disable, free to use all available registers. 1087 */ 1088 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1089 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1090 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1091 if (id > (uint32_t)(REG_C_7 - start_reg)) 1092 return rte_flow_error_set(error, EINVAL, 1093 RTE_FLOW_ERROR_TYPE_ITEM, 1094 NULL, "invalid tag id"); 1095 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1096 return rte_flow_error_set(error, ENOTSUP, 1097 RTE_FLOW_ERROR_TYPE_ITEM, 1098 NULL, "unsupported tag id"); 1099 /* 1100 * This case means meter is using the REG_C_x great than 2. 1101 * Take care not to conflict with meter color REG_C_x. 1102 * If the available index REG_C_y >= REG_C_x, skip the 1103 * color register. 1104 */ 1105 if (skip_mtr_reg && priv->sh->flow_mreg_c 1106 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1107 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1108 return rte_flow_error_set(error, EINVAL, 1109 RTE_FLOW_ERROR_TYPE_ITEM, 1110 NULL, "invalid tag id"); 1111 if (priv->sh->flow_mreg_c 1112 [id + 1 + start_reg - REG_C_0] != REG_NON) 1113 return priv->sh->flow_mreg_c 1114 [id + 1 + start_reg - REG_C_0]; 1115 return rte_flow_error_set(error, ENOTSUP, 1116 RTE_FLOW_ERROR_TYPE_ITEM, 1117 NULL, "unsupported tag id"); 1118 } 1119 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1120 } 1121 MLX5_ASSERT(false); 1122 return rte_flow_error_set(error, EINVAL, 1123 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1124 NULL, "invalid feature name"); 1125 } 1126 1127 /** 1128 * Check extensive flow metadata register support. 1129 * 1130 * @param dev 1131 * Pointer to rte_eth_dev structure. 1132 * 1133 * @return 1134 * True if device supports extensive flow metadata register, otherwise false. 1135 */ 1136 bool 1137 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1138 { 1139 struct mlx5_priv *priv = dev->data->dev_private; 1140 1141 /* 1142 * Having available reg_c can be regarded inclusively as supporting 1143 * extensive flow metadata register, which could mean, 1144 * - metadata register copy action by modify header. 1145 * - 16 modify header actions is supported. 1146 * - reg_c's are preserved across different domain (FDB and NIC) on 1147 * packet loopback by flow lookup miss. 1148 */ 1149 return priv->sh->flow_mreg_c[2] != REG_NON; 1150 } 1151 1152 /** 1153 * Get the lowest priority. 1154 * 1155 * @param[in] dev 1156 * Pointer to the Ethernet device structure. 1157 * @param[in] attributes 1158 * Pointer to device flow rule attributes. 1159 * 1160 * @return 1161 * The value of lowest priority of flow. 1162 */ 1163 uint32_t 1164 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1165 const struct rte_flow_attr *attr) 1166 { 1167 struct mlx5_priv *priv = dev->data->dev_private; 1168 1169 if (!attr->group && !attr->transfer) 1170 return priv->sh->flow_max_priority - 2; 1171 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1172 } 1173 1174 /** 1175 * Calculate matcher priority of the flow. 1176 * 1177 * @param[in] dev 1178 * Pointer to the Ethernet device structure. 1179 * @param[in] attr 1180 * Pointer to device flow rule attributes. 1181 * @param[in] subpriority 1182 * The priority based on the items. 1183 * @param[in] external 1184 * Flow is user flow. 1185 * @return 1186 * The matcher priority of the flow. 1187 */ 1188 uint16_t 1189 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1190 const struct rte_flow_attr *attr, 1191 uint32_t subpriority, bool external) 1192 { 1193 uint16_t priority = (uint16_t)attr->priority; 1194 struct mlx5_priv *priv = dev->data->dev_private; 1195 1196 if (!attr->group && !attr->transfer) { 1197 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1198 priority = priv->sh->flow_max_priority - 1; 1199 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1200 } else if (!external && attr->transfer && attr->group == 0 && 1201 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1202 return (priv->sh->flow_max_priority - 1) * 3; 1203 } 1204 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1205 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1206 return priority * 3 + subpriority; 1207 } 1208 1209 /** 1210 * Verify the @p item specifications (spec, last, mask) are compatible with the 1211 * NIC capabilities. 1212 * 1213 * @param[in] item 1214 * Item specification. 1215 * @param[in] mask 1216 * @p item->mask or flow default bit-masks. 1217 * @param[in] nic_mask 1218 * Bit-masks covering supported fields by the NIC to compare with user mask. 1219 * @param[in] size 1220 * Bit-masks size in bytes. 1221 * @param[in] range_accepted 1222 * True if range of values is accepted for specific fields, false otherwise. 1223 * @param[out] error 1224 * Pointer to error structure. 1225 * 1226 * @return 1227 * 0 on success, a negative errno value otherwise and rte_errno is set. 1228 */ 1229 int 1230 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1231 const uint8_t *mask, 1232 const uint8_t *nic_mask, 1233 unsigned int size, 1234 bool range_accepted, 1235 struct rte_flow_error *error) 1236 { 1237 unsigned int i; 1238 1239 MLX5_ASSERT(nic_mask); 1240 for (i = 0; i < size; ++i) 1241 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1242 return rte_flow_error_set(error, ENOTSUP, 1243 RTE_FLOW_ERROR_TYPE_ITEM, 1244 item, 1245 "mask enables non supported" 1246 " bits"); 1247 if (!item->spec && (item->mask || item->last)) 1248 return rte_flow_error_set(error, EINVAL, 1249 RTE_FLOW_ERROR_TYPE_ITEM, item, 1250 "mask/last without a spec is not" 1251 " supported"); 1252 if (item->spec && item->last && !range_accepted) { 1253 uint8_t spec[size]; 1254 uint8_t last[size]; 1255 unsigned int i; 1256 int ret; 1257 1258 for (i = 0; i < size; ++i) { 1259 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1260 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1261 } 1262 ret = memcmp(spec, last, size); 1263 if (ret != 0) 1264 return rte_flow_error_set(error, EINVAL, 1265 RTE_FLOW_ERROR_TYPE_ITEM, 1266 item, 1267 "range is not valid"); 1268 } 1269 return 0; 1270 } 1271 1272 /** 1273 * Adjust the hash fields according to the @p flow information. 1274 * 1275 * @param[in] dev_flow. 1276 * Pointer to the mlx5_flow. 1277 * @param[in] tunnel 1278 * 1 when the hash field is for a tunnel item. 1279 * @param[in] layer_types 1280 * RTE_ETH_RSS_* types. 1281 * @param[in] hash_fields 1282 * Item hash fields. 1283 * 1284 * @return 1285 * The hash fields that should be used. 1286 */ 1287 uint64_t 1288 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1289 int tunnel __rte_unused, uint64_t layer_types, 1290 uint64_t hash_fields) 1291 { 1292 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1293 int rss_request_inner = rss_desc->level >= 2; 1294 1295 /* Check RSS hash level for tunnel. */ 1296 if (tunnel && rss_request_inner) 1297 hash_fields |= IBV_RX_HASH_INNER; 1298 else if (tunnel || rss_request_inner) 1299 return 0; 1300 #endif 1301 /* Check if requested layer matches RSS hash fields. */ 1302 if (!(rss_desc->types & layer_types)) 1303 return 0; 1304 return hash_fields; 1305 } 1306 1307 /** 1308 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1309 * if several tunnel rules are used on this queue, the tunnel ptype will be 1310 * cleared. 1311 * 1312 * @param rxq_ctrl 1313 * Rx queue to update. 1314 */ 1315 static void 1316 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1317 { 1318 unsigned int i; 1319 uint32_t tunnel_ptype = 0; 1320 1321 /* Look up for the ptype to use. */ 1322 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1323 if (!rxq_ctrl->flow_tunnels_n[i]) 1324 continue; 1325 if (!tunnel_ptype) { 1326 tunnel_ptype = tunnels_info[i].ptype; 1327 } else { 1328 tunnel_ptype = 0; 1329 break; 1330 } 1331 } 1332 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1333 } 1334 1335 /** 1336 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1337 * flow. 1338 * 1339 * @param[in] dev 1340 * Pointer to the Ethernet device structure. 1341 * @param[in] dev_handle 1342 * Pointer to device flow handle structure. 1343 */ 1344 void 1345 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1346 struct mlx5_flow_handle *dev_handle) 1347 { 1348 struct mlx5_priv *priv = dev->data->dev_private; 1349 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1350 struct mlx5_ind_table_obj *ind_tbl = NULL; 1351 unsigned int i; 1352 1353 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1354 struct mlx5_hrxq *hrxq; 1355 1356 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1357 dev_handle->rix_hrxq); 1358 if (hrxq) 1359 ind_tbl = hrxq->ind_table; 1360 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1361 struct mlx5_shared_action_rss *shared_rss; 1362 1363 shared_rss = mlx5_ipool_get 1364 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1365 dev_handle->rix_srss); 1366 if (shared_rss) 1367 ind_tbl = shared_rss->ind_tbl; 1368 } 1369 if (!ind_tbl) 1370 return; 1371 for (i = 0; i != ind_tbl->queues_n; ++i) { 1372 int idx = ind_tbl->queues[i]; 1373 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1374 1375 MLX5_ASSERT(rxq_ctrl != NULL); 1376 if (rxq_ctrl == NULL) 1377 continue; 1378 /* 1379 * To support metadata register copy on Tx loopback, 1380 * this must be always enabled (metadata may arive 1381 * from other port - not from local flows only. 1382 */ 1383 if (tunnel) { 1384 unsigned int j; 1385 1386 /* Increase the counter matching the flow. */ 1387 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1388 if ((tunnels_info[j].tunnel & 1389 dev_handle->layers) == 1390 tunnels_info[j].tunnel) { 1391 rxq_ctrl->flow_tunnels_n[j]++; 1392 break; 1393 } 1394 } 1395 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1396 } 1397 } 1398 } 1399 1400 static void 1401 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1402 { 1403 struct mlx5_priv *priv = dev->data->dev_private; 1404 struct mlx5_rxq_ctrl *rxq_ctrl; 1405 1406 if (priv->mark_enabled) 1407 return; 1408 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1409 rxq_ctrl->rxq.mark = 1; 1410 } 1411 priv->mark_enabled = 1; 1412 } 1413 1414 /** 1415 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1416 * 1417 * @param[in] dev 1418 * Pointer to the Ethernet device structure. 1419 * @param[in] flow 1420 * Pointer to flow structure. 1421 */ 1422 static void 1423 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1424 { 1425 struct mlx5_priv *priv = dev->data->dev_private; 1426 uint32_t handle_idx; 1427 struct mlx5_flow_handle *dev_handle; 1428 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1429 1430 MLX5_ASSERT(wks); 1431 if (wks->mark) 1432 flow_rxq_mark_flag_set(dev); 1433 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1434 handle_idx, dev_handle, next) 1435 flow_drv_rxq_flags_set(dev, dev_handle); 1436 } 1437 1438 /** 1439 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1440 * device flow if no other flow uses it with the same kind of request. 1441 * 1442 * @param dev 1443 * Pointer to Ethernet device. 1444 * @param[in] dev_handle 1445 * Pointer to the device flow handle structure. 1446 */ 1447 static void 1448 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1449 struct mlx5_flow_handle *dev_handle) 1450 { 1451 struct mlx5_priv *priv = dev->data->dev_private; 1452 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1453 struct mlx5_ind_table_obj *ind_tbl = NULL; 1454 unsigned int i; 1455 1456 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1457 struct mlx5_hrxq *hrxq; 1458 1459 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1460 dev_handle->rix_hrxq); 1461 if (hrxq) 1462 ind_tbl = hrxq->ind_table; 1463 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1464 struct mlx5_shared_action_rss *shared_rss; 1465 1466 shared_rss = mlx5_ipool_get 1467 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1468 dev_handle->rix_srss); 1469 if (shared_rss) 1470 ind_tbl = shared_rss->ind_tbl; 1471 } 1472 if (!ind_tbl) 1473 return; 1474 MLX5_ASSERT(dev->data->dev_started); 1475 for (i = 0; i != ind_tbl->queues_n; ++i) { 1476 int idx = ind_tbl->queues[i]; 1477 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1478 1479 MLX5_ASSERT(rxq_ctrl != NULL); 1480 if (rxq_ctrl == NULL) 1481 continue; 1482 if (tunnel) { 1483 unsigned int j; 1484 1485 /* Decrease the counter matching the flow. */ 1486 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1487 if ((tunnels_info[j].tunnel & 1488 dev_handle->layers) == 1489 tunnels_info[j].tunnel) { 1490 rxq_ctrl->flow_tunnels_n[j]--; 1491 break; 1492 } 1493 } 1494 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1495 } 1496 } 1497 } 1498 1499 /** 1500 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1501 * @p flow if no other flow uses it with the same kind of request. 1502 * 1503 * @param dev 1504 * Pointer to Ethernet device. 1505 * @param[in] flow 1506 * Pointer to the flow. 1507 */ 1508 static void 1509 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1510 { 1511 struct mlx5_priv *priv = dev->data->dev_private; 1512 uint32_t handle_idx; 1513 struct mlx5_flow_handle *dev_handle; 1514 1515 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1516 handle_idx, dev_handle, next) 1517 flow_drv_rxq_flags_trim(dev, dev_handle); 1518 } 1519 1520 /** 1521 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1522 * 1523 * @param dev 1524 * Pointer to Ethernet device. 1525 */ 1526 static void 1527 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1528 { 1529 struct mlx5_priv *priv = dev->data->dev_private; 1530 unsigned int i; 1531 1532 for (i = 0; i != priv->rxqs_n; ++i) { 1533 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1534 unsigned int j; 1535 1536 if (rxq == NULL || rxq->ctrl == NULL) 1537 continue; 1538 rxq->ctrl->rxq.mark = 0; 1539 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1540 rxq->ctrl->flow_tunnels_n[j] = 0; 1541 rxq->ctrl->rxq.tunnel = 0; 1542 } 1543 priv->mark_enabled = 0; 1544 } 1545 1546 /** 1547 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1548 * 1549 * @param[in] dev 1550 * Pointer to the Ethernet device structure. 1551 */ 1552 void 1553 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1554 { 1555 struct mlx5_priv *priv = dev->data->dev_private; 1556 unsigned int i; 1557 1558 for (i = 0; i != priv->rxqs_n; ++i) { 1559 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1560 struct mlx5_rxq_data *data; 1561 1562 if (rxq == NULL || rxq->ctrl == NULL) 1563 continue; 1564 data = &rxq->ctrl->rxq; 1565 if (!rte_flow_dynf_metadata_avail()) { 1566 data->dynf_meta = 0; 1567 data->flow_meta_mask = 0; 1568 data->flow_meta_offset = -1; 1569 data->flow_meta_port_mask = 0; 1570 } else { 1571 data->dynf_meta = 1; 1572 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1573 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1574 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1575 } 1576 } 1577 } 1578 1579 /* 1580 * return a pointer to the desired action in the list of actions. 1581 * 1582 * @param[in] actions 1583 * The list of actions to search the action in. 1584 * @param[in] action 1585 * The action to find. 1586 * 1587 * @return 1588 * Pointer to the action in the list, if found. NULL otherwise. 1589 */ 1590 const struct rte_flow_action * 1591 mlx5_flow_find_action(const struct rte_flow_action *actions, 1592 enum rte_flow_action_type action) 1593 { 1594 if (actions == NULL) 1595 return NULL; 1596 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1597 if (actions->type == action) 1598 return actions; 1599 return NULL; 1600 } 1601 1602 /* 1603 * Validate the flag action. 1604 * 1605 * @param[in] action_flags 1606 * Bit-fields that holds the actions detected until now. 1607 * @param[in] attr 1608 * Attributes of flow that includes this action. 1609 * @param[out] error 1610 * Pointer to error structure. 1611 * 1612 * @return 1613 * 0 on success, a negative errno value otherwise and rte_errno is set. 1614 */ 1615 int 1616 mlx5_flow_validate_action_flag(uint64_t action_flags, 1617 const struct rte_flow_attr *attr, 1618 struct rte_flow_error *error) 1619 { 1620 if (action_flags & MLX5_FLOW_ACTION_MARK) 1621 return rte_flow_error_set(error, EINVAL, 1622 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1623 "can't mark and flag in same flow"); 1624 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1625 return rte_flow_error_set(error, EINVAL, 1626 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1627 "can't have 2 flag" 1628 " actions in same flow"); 1629 if (attr->egress) 1630 return rte_flow_error_set(error, ENOTSUP, 1631 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1632 "flag action not supported for " 1633 "egress"); 1634 return 0; 1635 } 1636 1637 /* 1638 * Validate the mark action. 1639 * 1640 * @param[in] action 1641 * Pointer to the queue action. 1642 * @param[in] action_flags 1643 * Bit-fields that holds the actions detected until now. 1644 * @param[in] attr 1645 * Attributes of flow that includes this action. 1646 * @param[out] error 1647 * Pointer to error structure. 1648 * 1649 * @return 1650 * 0 on success, a negative errno value otherwise and rte_errno is set. 1651 */ 1652 int 1653 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1654 uint64_t action_flags, 1655 const struct rte_flow_attr *attr, 1656 struct rte_flow_error *error) 1657 { 1658 const struct rte_flow_action_mark *mark = action->conf; 1659 1660 if (!mark) 1661 return rte_flow_error_set(error, EINVAL, 1662 RTE_FLOW_ERROR_TYPE_ACTION, 1663 action, 1664 "configuration cannot be null"); 1665 if (mark->id >= MLX5_FLOW_MARK_MAX) 1666 return rte_flow_error_set(error, EINVAL, 1667 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1668 &mark->id, 1669 "mark id must in 0 <= id < " 1670 RTE_STR(MLX5_FLOW_MARK_MAX)); 1671 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1672 return rte_flow_error_set(error, EINVAL, 1673 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1674 "can't flag and mark in same flow"); 1675 if (action_flags & MLX5_FLOW_ACTION_MARK) 1676 return rte_flow_error_set(error, EINVAL, 1677 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1678 "can't have 2 mark actions in same" 1679 " flow"); 1680 if (attr->egress) 1681 return rte_flow_error_set(error, ENOTSUP, 1682 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1683 "mark action not supported for " 1684 "egress"); 1685 return 0; 1686 } 1687 1688 /* 1689 * Validate the drop action. 1690 * 1691 * @param[in] action_flags 1692 * Bit-fields that holds the actions detected until now. 1693 * @param[in] attr 1694 * Attributes of flow that includes this action. 1695 * @param[out] error 1696 * Pointer to error structure. 1697 * 1698 * @return 1699 * 0 on success, a negative errno value otherwise and rte_errno is set. 1700 */ 1701 int 1702 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1703 const struct rte_flow_attr *attr, 1704 struct rte_flow_error *error) 1705 { 1706 if (attr->egress) 1707 return rte_flow_error_set(error, ENOTSUP, 1708 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1709 "drop action not supported for " 1710 "egress"); 1711 return 0; 1712 } 1713 1714 /* 1715 * Validate the queue action. 1716 * 1717 * @param[in] action 1718 * Pointer to the queue action. 1719 * @param[in] action_flags 1720 * Bit-fields that holds the actions detected until now. 1721 * @param[in] dev 1722 * Pointer to the Ethernet device structure. 1723 * @param[in] attr 1724 * Attributes of flow that includes this action. 1725 * @param[out] error 1726 * Pointer to error structure. 1727 * 1728 * @return 1729 * 0 on success, a negative errno value otherwise and rte_errno is set. 1730 */ 1731 int 1732 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1733 uint64_t action_flags, 1734 struct rte_eth_dev *dev, 1735 const struct rte_flow_attr *attr, 1736 struct rte_flow_error *error) 1737 { 1738 struct mlx5_priv *priv = dev->data->dev_private; 1739 const struct rte_flow_action_queue *queue = action->conf; 1740 1741 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1742 return rte_flow_error_set(error, EINVAL, 1743 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1744 "can't have 2 fate actions in" 1745 " same flow"); 1746 if (!priv->rxqs_n) 1747 return rte_flow_error_set(error, EINVAL, 1748 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1749 NULL, "No Rx queues configured"); 1750 if (queue->index >= priv->rxqs_n) 1751 return rte_flow_error_set(error, EINVAL, 1752 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1753 &queue->index, 1754 "queue index out of range"); 1755 if (mlx5_rxq_get(dev, queue->index) == NULL) 1756 return rte_flow_error_set(error, EINVAL, 1757 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1758 &queue->index, 1759 "queue is not configured"); 1760 if (attr->egress) 1761 return rte_flow_error_set(error, ENOTSUP, 1762 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1763 "queue action not supported for " 1764 "egress"); 1765 return 0; 1766 } 1767 1768 /** 1769 * Validate queue numbers for device RSS. 1770 * 1771 * @param[in] dev 1772 * Configured device. 1773 * @param[in] queues 1774 * Array of queue numbers. 1775 * @param[in] queues_n 1776 * Size of the @p queues array. 1777 * @param[out] error 1778 * On error, filled with a textual error description. 1779 * @param[out] queue 1780 * On error, filled with an offending queue index in @p queues array. 1781 * 1782 * @return 1783 * 0 on success, a negative errno code on error. 1784 */ 1785 static int 1786 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1787 const uint16_t *queues, uint32_t queues_n, 1788 const char **error, uint32_t *queue_idx) 1789 { 1790 const struct mlx5_priv *priv = dev->data->dev_private; 1791 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1792 uint32_t i; 1793 1794 for (i = 0; i != queues_n; ++i) { 1795 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, 1796 queues[i]); 1797 1798 if (queues[i] >= priv->rxqs_n) { 1799 *error = "queue index out of range"; 1800 *queue_idx = i; 1801 return -EINVAL; 1802 } 1803 if (rxq_ctrl == NULL) { 1804 *error = "queue is not configured"; 1805 *queue_idx = i; 1806 return -EINVAL; 1807 } 1808 if (i == 0) 1809 rxq_type = rxq_ctrl->type; 1810 if (rxq_type != rxq_ctrl->type) { 1811 *error = "combining hairpin and regular RSS queues is not supported"; 1812 *queue_idx = i; 1813 return -ENOTSUP; 1814 } 1815 } 1816 return 0; 1817 } 1818 1819 /* 1820 * Validate the rss action. 1821 * 1822 * @param[in] dev 1823 * Pointer to the Ethernet device structure. 1824 * @param[in] action 1825 * Pointer to the queue action. 1826 * @param[out] error 1827 * Pointer to error structure. 1828 * 1829 * @return 1830 * 0 on success, a negative errno value otherwise and rte_errno is set. 1831 */ 1832 int 1833 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1834 const struct rte_flow_action *action, 1835 struct rte_flow_error *error) 1836 { 1837 struct mlx5_priv *priv = dev->data->dev_private; 1838 const struct rte_flow_action_rss *rss = action->conf; 1839 int ret; 1840 const char *message; 1841 uint32_t queue_idx; 1842 1843 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1844 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1845 return rte_flow_error_set(error, ENOTSUP, 1846 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1847 &rss->func, 1848 "RSS hash function not supported"); 1849 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1850 if (rss->level > 2) 1851 #else 1852 if (rss->level > 1) 1853 #endif 1854 return rte_flow_error_set(error, ENOTSUP, 1855 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1856 &rss->level, 1857 "tunnel RSS is not supported"); 1858 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1859 if (rss->key_len == 0 && rss->key != NULL) 1860 return rte_flow_error_set(error, ENOTSUP, 1861 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1862 &rss->key_len, 1863 "RSS hash key length 0"); 1864 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1865 return rte_flow_error_set(error, ENOTSUP, 1866 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1867 &rss->key_len, 1868 "RSS hash key too small"); 1869 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1870 return rte_flow_error_set(error, ENOTSUP, 1871 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1872 &rss->key_len, 1873 "RSS hash key too large"); 1874 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1875 return rte_flow_error_set(error, ENOTSUP, 1876 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1877 &rss->queue_num, 1878 "number of queues too large"); 1879 if (rss->types & MLX5_RSS_HF_MASK) 1880 return rte_flow_error_set(error, ENOTSUP, 1881 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1882 &rss->types, 1883 "some RSS protocols are not" 1884 " supported"); 1885 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1886 !(rss->types & RTE_ETH_RSS_IP)) 1887 return rte_flow_error_set(error, EINVAL, 1888 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1889 "L3 partial RSS requested but L3 RSS" 1890 " type not specified"); 1891 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1892 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1893 return rte_flow_error_set(error, EINVAL, 1894 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1895 "L4 partial RSS requested but L4 RSS" 1896 " type not specified"); 1897 if (!priv->rxqs_n) 1898 return rte_flow_error_set(error, EINVAL, 1899 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1900 NULL, "No Rx queues configured"); 1901 if (!rss->queue_num) 1902 return rte_flow_error_set(error, EINVAL, 1903 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1904 NULL, "No queues configured"); 1905 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1906 &message, &queue_idx); 1907 if (ret != 0) { 1908 return rte_flow_error_set(error, -ret, 1909 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1910 &rss->queue[queue_idx], message); 1911 } 1912 return 0; 1913 } 1914 1915 /* 1916 * Validate the rss action. 1917 * 1918 * @param[in] action 1919 * Pointer to the queue action. 1920 * @param[in] action_flags 1921 * Bit-fields that holds the actions detected until now. 1922 * @param[in] dev 1923 * Pointer to the Ethernet device structure. 1924 * @param[in] attr 1925 * Attributes of flow that includes this action. 1926 * @param[in] item_flags 1927 * Items that were detected. 1928 * @param[out] error 1929 * Pointer to error structure. 1930 * 1931 * @return 1932 * 0 on success, a negative errno value otherwise and rte_errno is set. 1933 */ 1934 int 1935 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1936 uint64_t action_flags, 1937 struct rte_eth_dev *dev, 1938 const struct rte_flow_attr *attr, 1939 uint64_t item_flags, 1940 struct rte_flow_error *error) 1941 { 1942 const struct rte_flow_action_rss *rss = action->conf; 1943 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1944 int ret; 1945 1946 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1947 return rte_flow_error_set(error, EINVAL, 1948 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1949 "can't have 2 fate actions" 1950 " in same flow"); 1951 ret = mlx5_validate_action_rss(dev, action, error); 1952 if (ret) 1953 return ret; 1954 if (attr->egress) 1955 return rte_flow_error_set(error, ENOTSUP, 1956 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1957 "rss action not supported for " 1958 "egress"); 1959 if (rss->level > 1 && !tunnel) 1960 return rte_flow_error_set(error, EINVAL, 1961 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1962 "inner RSS is not supported for " 1963 "non-tunnel flows"); 1964 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1965 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1966 return rte_flow_error_set(error, EINVAL, 1967 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1968 "RSS on eCPRI is not supported now"); 1969 } 1970 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1971 !(item_flags & 1972 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1973 rss->level > 1) 1974 return rte_flow_error_set(error, EINVAL, 1975 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1976 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1977 return 0; 1978 } 1979 1980 /* 1981 * Validate the default miss action. 1982 * 1983 * @param[in] action_flags 1984 * Bit-fields that holds the actions detected until now. 1985 * @param[out] error 1986 * Pointer to error structure. 1987 * 1988 * @return 1989 * 0 on success, a negative errno value otherwise and rte_errno is set. 1990 */ 1991 int 1992 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1993 const struct rte_flow_attr *attr, 1994 struct rte_flow_error *error) 1995 { 1996 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1997 return rte_flow_error_set(error, EINVAL, 1998 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1999 "can't have 2 fate actions in" 2000 " same flow"); 2001 if (attr->egress) 2002 return rte_flow_error_set(error, ENOTSUP, 2003 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2004 "default miss action not supported " 2005 "for egress"); 2006 if (attr->group) 2007 return rte_flow_error_set(error, ENOTSUP, 2008 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2009 "only group 0 is supported"); 2010 if (attr->transfer) 2011 return rte_flow_error_set(error, ENOTSUP, 2012 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2013 NULL, "transfer is not supported"); 2014 return 0; 2015 } 2016 2017 /* 2018 * Validate the count action. 2019 * 2020 * @param[in] dev 2021 * Pointer to the Ethernet device structure. 2022 * @param[in] attr 2023 * Attributes of flow that includes this action. 2024 * @param[out] error 2025 * Pointer to error structure. 2026 * 2027 * @return 2028 * 0 on success, a negative errno value otherwise and rte_errno is set. 2029 */ 2030 int 2031 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2032 const struct rte_flow_attr *attr, 2033 struct rte_flow_error *error) 2034 { 2035 if (attr->egress) 2036 return rte_flow_error_set(error, ENOTSUP, 2037 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2038 "count action not supported for " 2039 "egress"); 2040 return 0; 2041 } 2042 2043 /* 2044 * Validate the ASO CT action. 2045 * 2046 * @param[in] dev 2047 * Pointer to the Ethernet device structure. 2048 * @param[in] conntrack 2049 * Pointer to the CT action profile. 2050 * @param[out] error 2051 * Pointer to error structure. 2052 * 2053 * @return 2054 * 0 on success, a negative errno value otherwise and rte_errno is set. 2055 */ 2056 int 2057 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2058 const struct rte_flow_action_conntrack *conntrack, 2059 struct rte_flow_error *error) 2060 { 2061 RTE_SET_USED(dev); 2062 2063 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2064 return rte_flow_error_set(error, EINVAL, 2065 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2066 "Invalid CT state"); 2067 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2068 return rte_flow_error_set(error, EINVAL, 2069 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2070 "Invalid last TCP packet flag"); 2071 return 0; 2072 } 2073 2074 /** 2075 * Verify the @p attributes will be correctly understood by the NIC and store 2076 * them in the @p flow if everything is correct. 2077 * 2078 * @param[in] dev 2079 * Pointer to the Ethernet device structure. 2080 * @param[in] attributes 2081 * Pointer to flow attributes 2082 * @param[out] error 2083 * Pointer to error structure. 2084 * 2085 * @return 2086 * 0 on success, a negative errno value otherwise and rte_errno is set. 2087 */ 2088 int 2089 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2090 const struct rte_flow_attr *attributes, 2091 struct rte_flow_error *error) 2092 { 2093 struct mlx5_priv *priv = dev->data->dev_private; 2094 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2095 2096 if (attributes->group) 2097 return rte_flow_error_set(error, ENOTSUP, 2098 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2099 NULL, "groups is not supported"); 2100 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2101 attributes->priority >= priority_max) 2102 return rte_flow_error_set(error, ENOTSUP, 2103 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2104 NULL, "priority out of range"); 2105 if (attributes->egress) 2106 return rte_flow_error_set(error, ENOTSUP, 2107 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2108 "egress is not supported"); 2109 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2110 return rte_flow_error_set(error, ENOTSUP, 2111 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2112 NULL, "transfer is not supported"); 2113 if (!attributes->ingress) 2114 return rte_flow_error_set(error, EINVAL, 2115 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2116 NULL, 2117 "ingress attribute is mandatory"); 2118 return 0; 2119 } 2120 2121 /** 2122 * Validate ICMP6 item. 2123 * 2124 * @param[in] item 2125 * Item specification. 2126 * @param[in] item_flags 2127 * Bit-fields that holds the items detected until now. 2128 * @param[in] ext_vlan_sup 2129 * Whether extended VLAN features are supported or not. 2130 * @param[out] error 2131 * Pointer to error structure. 2132 * 2133 * @return 2134 * 0 on success, a negative errno value otherwise and rte_errno is set. 2135 */ 2136 int 2137 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2138 uint64_t item_flags, 2139 uint8_t target_protocol, 2140 struct rte_flow_error *error) 2141 { 2142 const struct rte_flow_item_icmp6 *mask = item->mask; 2143 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2144 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2145 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2146 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2147 MLX5_FLOW_LAYER_OUTER_L4; 2148 int ret; 2149 2150 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2151 return rte_flow_error_set(error, EINVAL, 2152 RTE_FLOW_ERROR_TYPE_ITEM, item, 2153 "protocol filtering not compatible" 2154 " with ICMP6 layer"); 2155 if (!(item_flags & l3m)) 2156 return rte_flow_error_set(error, EINVAL, 2157 RTE_FLOW_ERROR_TYPE_ITEM, item, 2158 "IPv6 is mandatory to filter on" 2159 " ICMP6"); 2160 if (item_flags & l4m) 2161 return rte_flow_error_set(error, EINVAL, 2162 RTE_FLOW_ERROR_TYPE_ITEM, item, 2163 "multiple L4 layers not supported"); 2164 if (!mask) 2165 mask = &rte_flow_item_icmp6_mask; 2166 ret = mlx5_flow_item_acceptable 2167 (item, (const uint8_t *)mask, 2168 (const uint8_t *)&rte_flow_item_icmp6_mask, 2169 sizeof(struct rte_flow_item_icmp6), 2170 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2171 if (ret < 0) 2172 return ret; 2173 return 0; 2174 } 2175 2176 /** 2177 * Validate ICMP item. 2178 * 2179 * @param[in] item 2180 * Item specification. 2181 * @param[in] item_flags 2182 * Bit-fields that holds the items detected until now. 2183 * @param[out] error 2184 * Pointer to error structure. 2185 * 2186 * @return 2187 * 0 on success, a negative errno value otherwise and rte_errno is set. 2188 */ 2189 int 2190 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2191 uint64_t item_flags, 2192 uint8_t target_protocol, 2193 struct rte_flow_error *error) 2194 { 2195 const struct rte_flow_item_icmp *mask = item->mask; 2196 const struct rte_flow_item_icmp nic_mask = { 2197 .hdr.icmp_type = 0xff, 2198 .hdr.icmp_code = 0xff, 2199 .hdr.icmp_ident = RTE_BE16(0xffff), 2200 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2201 }; 2202 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2203 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2204 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2205 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2206 MLX5_FLOW_LAYER_OUTER_L4; 2207 int ret; 2208 2209 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2210 return rte_flow_error_set(error, EINVAL, 2211 RTE_FLOW_ERROR_TYPE_ITEM, item, 2212 "protocol filtering not compatible" 2213 " with ICMP layer"); 2214 if (!(item_flags & l3m)) 2215 return rte_flow_error_set(error, EINVAL, 2216 RTE_FLOW_ERROR_TYPE_ITEM, item, 2217 "IPv4 is mandatory to filter" 2218 " on ICMP"); 2219 if (item_flags & l4m) 2220 return rte_flow_error_set(error, EINVAL, 2221 RTE_FLOW_ERROR_TYPE_ITEM, item, 2222 "multiple L4 layers not supported"); 2223 if (!mask) 2224 mask = &nic_mask; 2225 ret = mlx5_flow_item_acceptable 2226 (item, (const uint8_t *)mask, 2227 (const uint8_t *)&nic_mask, 2228 sizeof(struct rte_flow_item_icmp), 2229 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2230 if (ret < 0) 2231 return ret; 2232 return 0; 2233 } 2234 2235 /** 2236 * Validate Ethernet item. 2237 * 2238 * @param[in] item 2239 * Item specification. 2240 * @param[in] item_flags 2241 * Bit-fields that holds the items detected until now. 2242 * @param[out] error 2243 * Pointer to error structure. 2244 * 2245 * @return 2246 * 0 on success, a negative errno value otherwise and rte_errno is set. 2247 */ 2248 int 2249 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2250 uint64_t item_flags, bool ext_vlan_sup, 2251 struct rte_flow_error *error) 2252 { 2253 const struct rte_flow_item_eth *mask = item->mask; 2254 const struct rte_flow_item_eth nic_mask = { 2255 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2256 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2257 .type = RTE_BE16(0xffff), 2258 .has_vlan = ext_vlan_sup ? 1 : 0, 2259 }; 2260 int ret; 2261 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2262 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2263 MLX5_FLOW_LAYER_OUTER_L2; 2264 2265 if (item_flags & ethm) 2266 return rte_flow_error_set(error, ENOTSUP, 2267 RTE_FLOW_ERROR_TYPE_ITEM, item, 2268 "multiple L2 layers not supported"); 2269 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2270 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2271 return rte_flow_error_set(error, EINVAL, 2272 RTE_FLOW_ERROR_TYPE_ITEM, item, 2273 "L2 layer should not follow " 2274 "L3 layers"); 2275 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2276 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2277 return rte_flow_error_set(error, EINVAL, 2278 RTE_FLOW_ERROR_TYPE_ITEM, item, 2279 "L2 layer should not follow VLAN"); 2280 if (item_flags & MLX5_FLOW_LAYER_GTP) 2281 return rte_flow_error_set(error, EINVAL, 2282 RTE_FLOW_ERROR_TYPE_ITEM, item, 2283 "L2 layer should not follow GTP"); 2284 if (!mask) 2285 mask = &rte_flow_item_eth_mask; 2286 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2287 (const uint8_t *)&nic_mask, 2288 sizeof(struct rte_flow_item_eth), 2289 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2290 return ret; 2291 } 2292 2293 /** 2294 * Validate VLAN item. 2295 * 2296 * @param[in] item 2297 * Item specification. 2298 * @param[in] item_flags 2299 * Bit-fields that holds the items detected until now. 2300 * @param[in] dev 2301 * Ethernet device flow is being created on. 2302 * @param[out] error 2303 * Pointer to error structure. 2304 * 2305 * @return 2306 * 0 on success, a negative errno value otherwise and rte_errno is set. 2307 */ 2308 int 2309 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2310 uint64_t item_flags, 2311 struct rte_eth_dev *dev, 2312 struct rte_flow_error *error) 2313 { 2314 const struct rte_flow_item_vlan *spec = item->spec; 2315 const struct rte_flow_item_vlan *mask = item->mask; 2316 const struct rte_flow_item_vlan nic_mask = { 2317 .tci = RTE_BE16(UINT16_MAX), 2318 .inner_type = RTE_BE16(UINT16_MAX), 2319 }; 2320 uint16_t vlan_tag = 0; 2321 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2322 int ret; 2323 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2324 MLX5_FLOW_LAYER_INNER_L4) : 2325 (MLX5_FLOW_LAYER_OUTER_L3 | 2326 MLX5_FLOW_LAYER_OUTER_L4); 2327 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2328 MLX5_FLOW_LAYER_OUTER_VLAN; 2329 2330 if (item_flags & vlanm) 2331 return rte_flow_error_set(error, EINVAL, 2332 RTE_FLOW_ERROR_TYPE_ITEM, item, 2333 "multiple VLAN layers not supported"); 2334 else if ((item_flags & l34m) != 0) 2335 return rte_flow_error_set(error, EINVAL, 2336 RTE_FLOW_ERROR_TYPE_ITEM, item, 2337 "VLAN cannot follow L3/L4 layer"); 2338 if (!mask) 2339 mask = &rte_flow_item_vlan_mask; 2340 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2341 (const uint8_t *)&nic_mask, 2342 sizeof(struct rte_flow_item_vlan), 2343 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2344 if (ret) 2345 return ret; 2346 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2347 struct mlx5_priv *priv = dev->data->dev_private; 2348 2349 if (priv->vmwa_context) { 2350 /* 2351 * Non-NULL context means we have a virtual machine 2352 * and SR-IOV enabled, we have to create VLAN interface 2353 * to make hypervisor to setup E-Switch vport 2354 * context correctly. We avoid creating the multiple 2355 * VLAN interfaces, so we cannot support VLAN tag mask. 2356 */ 2357 return rte_flow_error_set(error, EINVAL, 2358 RTE_FLOW_ERROR_TYPE_ITEM, 2359 item, 2360 "VLAN tag mask is not" 2361 " supported in virtual" 2362 " environment"); 2363 } 2364 } 2365 if (spec) { 2366 vlan_tag = spec->tci; 2367 vlan_tag &= mask->tci; 2368 } 2369 /* 2370 * From verbs perspective an empty VLAN is equivalent 2371 * to a packet without VLAN layer. 2372 */ 2373 if (!vlan_tag) 2374 return rte_flow_error_set(error, EINVAL, 2375 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2376 item->spec, 2377 "VLAN cannot be empty"); 2378 return 0; 2379 } 2380 2381 /** 2382 * Validate IPV4 item. 2383 * 2384 * @param[in] item 2385 * Item specification. 2386 * @param[in] item_flags 2387 * Bit-fields that holds the items detected until now. 2388 * @param[in] last_item 2389 * Previous validated item in the pattern items. 2390 * @param[in] ether_type 2391 * Type in the ethernet layer header (including dot1q). 2392 * @param[in] acc_mask 2393 * Acceptable mask, if NULL default internal default mask 2394 * will be used to check whether item fields are supported. 2395 * @param[in] range_accepted 2396 * True if range of values is accepted for specific fields, false otherwise. 2397 * @param[out] error 2398 * Pointer to error structure. 2399 * 2400 * @return 2401 * 0 on success, a negative errno value otherwise and rte_errno is set. 2402 */ 2403 int 2404 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2405 uint64_t item_flags, 2406 uint64_t last_item, 2407 uint16_t ether_type, 2408 const struct rte_flow_item_ipv4 *acc_mask, 2409 bool range_accepted, 2410 struct rte_flow_error *error) 2411 { 2412 const struct rte_flow_item_ipv4 *mask = item->mask; 2413 const struct rte_flow_item_ipv4 *spec = item->spec; 2414 const struct rte_flow_item_ipv4 nic_mask = { 2415 .hdr = { 2416 .src_addr = RTE_BE32(0xffffffff), 2417 .dst_addr = RTE_BE32(0xffffffff), 2418 .type_of_service = 0xff, 2419 .next_proto_id = 0xff, 2420 }, 2421 }; 2422 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2423 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2424 MLX5_FLOW_LAYER_OUTER_L3; 2425 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2426 MLX5_FLOW_LAYER_OUTER_L4; 2427 int ret; 2428 uint8_t next_proto = 0xFF; 2429 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2430 MLX5_FLOW_LAYER_OUTER_VLAN | 2431 MLX5_FLOW_LAYER_INNER_VLAN); 2432 2433 if ((last_item & l2_vlan) && ether_type && 2434 ether_type != RTE_ETHER_TYPE_IPV4) 2435 return rte_flow_error_set(error, EINVAL, 2436 RTE_FLOW_ERROR_TYPE_ITEM, item, 2437 "IPv4 cannot follow L2/VLAN layer " 2438 "which ether type is not IPv4"); 2439 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2440 if (mask && spec) 2441 next_proto = mask->hdr.next_proto_id & 2442 spec->hdr.next_proto_id; 2443 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2444 return rte_flow_error_set(error, EINVAL, 2445 RTE_FLOW_ERROR_TYPE_ITEM, 2446 item, 2447 "multiple tunnel " 2448 "not supported"); 2449 } 2450 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2451 return rte_flow_error_set(error, EINVAL, 2452 RTE_FLOW_ERROR_TYPE_ITEM, item, 2453 "wrong tunnel type - IPv6 specified " 2454 "but IPv4 item provided"); 2455 if (item_flags & l3m) 2456 return rte_flow_error_set(error, ENOTSUP, 2457 RTE_FLOW_ERROR_TYPE_ITEM, item, 2458 "multiple L3 layers not supported"); 2459 else if (item_flags & l4m) 2460 return rte_flow_error_set(error, EINVAL, 2461 RTE_FLOW_ERROR_TYPE_ITEM, item, 2462 "L3 cannot follow an L4 layer."); 2463 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2464 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2465 return rte_flow_error_set(error, EINVAL, 2466 RTE_FLOW_ERROR_TYPE_ITEM, item, 2467 "L3 cannot follow an NVGRE layer."); 2468 if (!mask) 2469 mask = &rte_flow_item_ipv4_mask; 2470 else if (mask->hdr.next_proto_id != 0 && 2471 mask->hdr.next_proto_id != 0xff) 2472 return rte_flow_error_set(error, EINVAL, 2473 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2474 "partial mask is not supported" 2475 " for protocol"); 2476 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2477 acc_mask ? (const uint8_t *)acc_mask 2478 : (const uint8_t *)&nic_mask, 2479 sizeof(struct rte_flow_item_ipv4), 2480 range_accepted, error); 2481 if (ret < 0) 2482 return ret; 2483 return 0; 2484 } 2485 2486 /** 2487 * Validate IPV6 item. 2488 * 2489 * @param[in] item 2490 * Item specification. 2491 * @param[in] item_flags 2492 * Bit-fields that holds the items detected until now. 2493 * @param[in] last_item 2494 * Previous validated item in the pattern items. 2495 * @param[in] ether_type 2496 * Type in the ethernet layer header (including dot1q). 2497 * @param[in] acc_mask 2498 * Acceptable mask, if NULL default internal default mask 2499 * will be used to check whether item fields are supported. 2500 * @param[out] error 2501 * Pointer to error structure. 2502 * 2503 * @return 2504 * 0 on success, a negative errno value otherwise and rte_errno is set. 2505 */ 2506 int 2507 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2508 uint64_t item_flags, 2509 uint64_t last_item, 2510 uint16_t ether_type, 2511 const struct rte_flow_item_ipv6 *acc_mask, 2512 struct rte_flow_error *error) 2513 { 2514 const struct rte_flow_item_ipv6 *mask = item->mask; 2515 const struct rte_flow_item_ipv6 *spec = item->spec; 2516 const struct rte_flow_item_ipv6 nic_mask = { 2517 .hdr = { 2518 .src_addr = 2519 "\xff\xff\xff\xff\xff\xff\xff\xff" 2520 "\xff\xff\xff\xff\xff\xff\xff\xff", 2521 .dst_addr = 2522 "\xff\xff\xff\xff\xff\xff\xff\xff" 2523 "\xff\xff\xff\xff\xff\xff\xff\xff", 2524 .vtc_flow = RTE_BE32(0xffffffff), 2525 .proto = 0xff, 2526 }, 2527 }; 2528 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2529 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2530 MLX5_FLOW_LAYER_OUTER_L3; 2531 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2532 MLX5_FLOW_LAYER_OUTER_L4; 2533 int ret; 2534 uint8_t next_proto = 0xFF; 2535 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2536 MLX5_FLOW_LAYER_OUTER_VLAN | 2537 MLX5_FLOW_LAYER_INNER_VLAN); 2538 2539 if ((last_item & l2_vlan) && ether_type && 2540 ether_type != RTE_ETHER_TYPE_IPV6) 2541 return rte_flow_error_set(error, EINVAL, 2542 RTE_FLOW_ERROR_TYPE_ITEM, item, 2543 "IPv6 cannot follow L2/VLAN layer " 2544 "which ether type is not IPv6"); 2545 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2546 next_proto = spec->hdr.proto; 2547 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2548 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2549 return rte_flow_error_set(error, EINVAL, 2550 RTE_FLOW_ERROR_TYPE_ITEM, 2551 item, 2552 "multiple tunnel " 2553 "not supported"); 2554 } 2555 if (next_proto == IPPROTO_HOPOPTS || 2556 next_proto == IPPROTO_ROUTING || 2557 next_proto == IPPROTO_FRAGMENT || 2558 next_proto == IPPROTO_ESP || 2559 next_proto == IPPROTO_AH || 2560 next_proto == IPPROTO_DSTOPTS) 2561 return rte_flow_error_set(error, EINVAL, 2562 RTE_FLOW_ERROR_TYPE_ITEM, item, 2563 "IPv6 proto (next header) should " 2564 "not be set as extension header"); 2565 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2566 return rte_flow_error_set(error, EINVAL, 2567 RTE_FLOW_ERROR_TYPE_ITEM, item, 2568 "wrong tunnel type - IPv4 specified " 2569 "but IPv6 item provided"); 2570 if (item_flags & l3m) 2571 return rte_flow_error_set(error, ENOTSUP, 2572 RTE_FLOW_ERROR_TYPE_ITEM, item, 2573 "multiple L3 layers not supported"); 2574 else if (item_flags & l4m) 2575 return rte_flow_error_set(error, EINVAL, 2576 RTE_FLOW_ERROR_TYPE_ITEM, item, 2577 "L3 cannot follow an L4 layer."); 2578 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2579 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2580 return rte_flow_error_set(error, EINVAL, 2581 RTE_FLOW_ERROR_TYPE_ITEM, item, 2582 "L3 cannot follow an NVGRE layer."); 2583 if (!mask) 2584 mask = &rte_flow_item_ipv6_mask; 2585 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2586 acc_mask ? (const uint8_t *)acc_mask 2587 : (const uint8_t *)&nic_mask, 2588 sizeof(struct rte_flow_item_ipv6), 2589 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2590 if (ret < 0) 2591 return ret; 2592 return 0; 2593 } 2594 2595 /** 2596 * Validate UDP item. 2597 * 2598 * @param[in] item 2599 * Item specification. 2600 * @param[in] item_flags 2601 * Bit-fields that holds the items detected until now. 2602 * @param[in] target_protocol 2603 * The next protocol in the previous item. 2604 * @param[in] flow_mask 2605 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2606 * @param[out] error 2607 * Pointer to error structure. 2608 * 2609 * @return 2610 * 0 on success, a negative errno value otherwise and rte_errno is set. 2611 */ 2612 int 2613 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2614 uint64_t item_flags, 2615 uint8_t target_protocol, 2616 struct rte_flow_error *error) 2617 { 2618 const struct rte_flow_item_udp *mask = item->mask; 2619 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2620 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2621 MLX5_FLOW_LAYER_OUTER_L3; 2622 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2623 MLX5_FLOW_LAYER_OUTER_L4; 2624 int ret; 2625 2626 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2627 return rte_flow_error_set(error, EINVAL, 2628 RTE_FLOW_ERROR_TYPE_ITEM, item, 2629 "protocol filtering not compatible" 2630 " with UDP layer"); 2631 if (!(item_flags & l3m)) 2632 return rte_flow_error_set(error, EINVAL, 2633 RTE_FLOW_ERROR_TYPE_ITEM, item, 2634 "L3 is mandatory to filter on L4"); 2635 if (item_flags & l4m) 2636 return rte_flow_error_set(error, EINVAL, 2637 RTE_FLOW_ERROR_TYPE_ITEM, item, 2638 "multiple L4 layers not supported"); 2639 if (!mask) 2640 mask = &rte_flow_item_udp_mask; 2641 ret = mlx5_flow_item_acceptable 2642 (item, (const uint8_t *)mask, 2643 (const uint8_t *)&rte_flow_item_udp_mask, 2644 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2645 error); 2646 if (ret < 0) 2647 return ret; 2648 return 0; 2649 } 2650 2651 /** 2652 * Validate TCP item. 2653 * 2654 * @param[in] item 2655 * Item specification. 2656 * @param[in] item_flags 2657 * Bit-fields that holds the items detected until now. 2658 * @param[in] target_protocol 2659 * The next protocol in the previous item. 2660 * @param[out] error 2661 * Pointer to error structure. 2662 * 2663 * @return 2664 * 0 on success, a negative errno value otherwise and rte_errno is set. 2665 */ 2666 int 2667 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2668 uint64_t item_flags, 2669 uint8_t target_protocol, 2670 const struct rte_flow_item_tcp *flow_mask, 2671 struct rte_flow_error *error) 2672 { 2673 const struct rte_flow_item_tcp *mask = item->mask; 2674 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2675 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2676 MLX5_FLOW_LAYER_OUTER_L3; 2677 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2678 MLX5_FLOW_LAYER_OUTER_L4; 2679 int ret; 2680 2681 MLX5_ASSERT(flow_mask); 2682 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2683 return rte_flow_error_set(error, EINVAL, 2684 RTE_FLOW_ERROR_TYPE_ITEM, item, 2685 "protocol filtering not compatible" 2686 " with TCP layer"); 2687 if (!(item_flags & l3m)) 2688 return rte_flow_error_set(error, EINVAL, 2689 RTE_FLOW_ERROR_TYPE_ITEM, item, 2690 "L3 is mandatory to filter on L4"); 2691 if (item_flags & l4m) 2692 return rte_flow_error_set(error, EINVAL, 2693 RTE_FLOW_ERROR_TYPE_ITEM, item, 2694 "multiple L4 layers not supported"); 2695 if (!mask) 2696 mask = &rte_flow_item_tcp_mask; 2697 ret = mlx5_flow_item_acceptable 2698 (item, (const uint8_t *)mask, 2699 (const uint8_t *)flow_mask, 2700 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2701 error); 2702 if (ret < 0) 2703 return ret; 2704 return 0; 2705 } 2706 2707 /** 2708 * Validate VXLAN item. 2709 * 2710 * @param[in] dev 2711 * Pointer to the Ethernet device structure. 2712 * @param[in] udp_dport 2713 * UDP destination port 2714 * @param[in] item 2715 * Item specification. 2716 * @param[in] item_flags 2717 * Bit-fields that holds the items detected until now. 2718 * @param[in] attr 2719 * Flow rule attributes. 2720 * @param[out] error 2721 * Pointer to error structure. 2722 * 2723 * @return 2724 * 0 on success, a negative errno value otherwise and rte_errno is set. 2725 */ 2726 int 2727 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2728 uint16_t udp_dport, 2729 const struct rte_flow_item *item, 2730 uint64_t item_flags, 2731 const struct rte_flow_attr *attr, 2732 struct rte_flow_error *error) 2733 { 2734 const struct rte_flow_item_vxlan *spec = item->spec; 2735 const struct rte_flow_item_vxlan *mask = item->mask; 2736 int ret; 2737 struct mlx5_priv *priv = dev->data->dev_private; 2738 union vni { 2739 uint32_t vlan_id; 2740 uint8_t vni[4]; 2741 } id = { .vlan_id = 0, }; 2742 const struct rte_flow_item_vxlan nic_mask = { 2743 .vni = "\xff\xff\xff", 2744 .rsvd1 = 0xff, 2745 }; 2746 const struct rte_flow_item_vxlan *valid_mask; 2747 2748 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2749 return rte_flow_error_set(error, ENOTSUP, 2750 RTE_FLOW_ERROR_TYPE_ITEM, item, 2751 "multiple tunnel layers not" 2752 " supported"); 2753 valid_mask = &rte_flow_item_vxlan_mask; 2754 /* 2755 * Verify only UDPv4 is present as defined in 2756 * https://tools.ietf.org/html/rfc7348 2757 */ 2758 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2759 return rte_flow_error_set(error, EINVAL, 2760 RTE_FLOW_ERROR_TYPE_ITEM, item, 2761 "no outer UDP layer found"); 2762 if (!mask) 2763 mask = &rte_flow_item_vxlan_mask; 2764 2765 if (priv->sh->steering_format_version != 2766 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2767 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2768 /* FDB domain & NIC domain non-zero group */ 2769 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2770 valid_mask = &nic_mask; 2771 /* Group zero in NIC domain */ 2772 if (!attr->group && !attr->transfer && 2773 priv->sh->tunnel_header_0_1) 2774 valid_mask = &nic_mask; 2775 } 2776 ret = mlx5_flow_item_acceptable 2777 (item, (const uint8_t *)mask, 2778 (const uint8_t *)valid_mask, 2779 sizeof(struct rte_flow_item_vxlan), 2780 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2781 if (ret < 0) 2782 return ret; 2783 if (spec) { 2784 memcpy(&id.vni[1], spec->vni, 3); 2785 memcpy(&id.vni[1], mask->vni, 3); 2786 } 2787 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2788 return rte_flow_error_set(error, ENOTSUP, 2789 RTE_FLOW_ERROR_TYPE_ITEM, item, 2790 "VXLAN tunnel must be fully defined"); 2791 return 0; 2792 } 2793 2794 /** 2795 * Validate VXLAN_GPE item. 2796 * 2797 * @param[in] item 2798 * Item specification. 2799 * @param[in] item_flags 2800 * Bit-fields that holds the items detected until now. 2801 * @param[in] priv 2802 * Pointer to the private data structure. 2803 * @param[in] target_protocol 2804 * The next protocol in the previous item. 2805 * @param[out] error 2806 * Pointer to error structure. 2807 * 2808 * @return 2809 * 0 on success, a negative errno value otherwise and rte_errno is set. 2810 */ 2811 int 2812 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2813 uint64_t item_flags, 2814 struct rte_eth_dev *dev, 2815 struct rte_flow_error *error) 2816 { 2817 struct mlx5_priv *priv = dev->data->dev_private; 2818 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2819 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2820 int ret; 2821 union vni { 2822 uint32_t vlan_id; 2823 uint8_t vni[4]; 2824 } id = { .vlan_id = 0, }; 2825 2826 if (!priv->sh->config.l3_vxlan_en) 2827 return rte_flow_error_set(error, ENOTSUP, 2828 RTE_FLOW_ERROR_TYPE_ITEM, item, 2829 "L3 VXLAN is not enabled by device" 2830 " parameter and/or not configured in" 2831 " firmware"); 2832 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2833 return rte_flow_error_set(error, ENOTSUP, 2834 RTE_FLOW_ERROR_TYPE_ITEM, item, 2835 "multiple tunnel layers not" 2836 " supported"); 2837 /* 2838 * Verify only UDPv4 is present as defined in 2839 * https://tools.ietf.org/html/rfc7348 2840 */ 2841 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2842 return rte_flow_error_set(error, EINVAL, 2843 RTE_FLOW_ERROR_TYPE_ITEM, item, 2844 "no outer UDP layer found"); 2845 if (!mask) 2846 mask = &rte_flow_item_vxlan_gpe_mask; 2847 ret = mlx5_flow_item_acceptable 2848 (item, (const uint8_t *)mask, 2849 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2850 sizeof(struct rte_flow_item_vxlan_gpe), 2851 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2852 if (ret < 0) 2853 return ret; 2854 if (spec) { 2855 if (spec->protocol) 2856 return rte_flow_error_set(error, ENOTSUP, 2857 RTE_FLOW_ERROR_TYPE_ITEM, 2858 item, 2859 "VxLAN-GPE protocol" 2860 " not supported"); 2861 memcpy(&id.vni[1], spec->vni, 3); 2862 memcpy(&id.vni[1], mask->vni, 3); 2863 } 2864 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2865 return rte_flow_error_set(error, ENOTSUP, 2866 RTE_FLOW_ERROR_TYPE_ITEM, item, 2867 "VXLAN-GPE tunnel must be fully" 2868 " defined"); 2869 return 0; 2870 } 2871 /** 2872 * Validate GRE Key item. 2873 * 2874 * @param[in] item 2875 * Item specification. 2876 * @param[in] item_flags 2877 * Bit flags to mark detected items. 2878 * @param[in] gre_item 2879 * Pointer to gre_item 2880 * @param[out] error 2881 * Pointer to error structure. 2882 * 2883 * @return 2884 * 0 on success, a negative errno value otherwise and rte_errno is set. 2885 */ 2886 int 2887 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2888 uint64_t item_flags, 2889 const struct rte_flow_item *gre_item, 2890 struct rte_flow_error *error) 2891 { 2892 const rte_be32_t *mask = item->mask; 2893 int ret = 0; 2894 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2895 const struct rte_flow_item_gre *gre_spec; 2896 const struct rte_flow_item_gre *gre_mask; 2897 2898 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2899 return rte_flow_error_set(error, ENOTSUP, 2900 RTE_FLOW_ERROR_TYPE_ITEM, item, 2901 "Multiple GRE key not support"); 2902 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2903 return rte_flow_error_set(error, ENOTSUP, 2904 RTE_FLOW_ERROR_TYPE_ITEM, item, 2905 "No preceding GRE header"); 2906 if (item_flags & MLX5_FLOW_LAYER_INNER) 2907 return rte_flow_error_set(error, ENOTSUP, 2908 RTE_FLOW_ERROR_TYPE_ITEM, item, 2909 "GRE key following a wrong item"); 2910 gre_mask = gre_item->mask; 2911 if (!gre_mask) 2912 gre_mask = &rte_flow_item_gre_mask; 2913 gre_spec = gre_item->spec; 2914 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2915 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2916 return rte_flow_error_set(error, EINVAL, 2917 RTE_FLOW_ERROR_TYPE_ITEM, item, 2918 "Key bit must be on"); 2919 2920 if (!mask) 2921 mask = &gre_key_default_mask; 2922 ret = mlx5_flow_item_acceptable 2923 (item, (const uint8_t *)mask, 2924 (const uint8_t *)&gre_key_default_mask, 2925 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2926 return ret; 2927 } 2928 2929 /** 2930 * Validate GRE optional item. 2931 * 2932 * @param[in] dev 2933 * Pointer to the Ethernet device structure. 2934 * @param[in] item 2935 * Item specification. 2936 * @param[in] item_flags 2937 * Bit flags to mark detected items. 2938 * @param[in] attr 2939 * Flow rule attributes. 2940 * @param[in] gre_item 2941 * Pointer to gre_item 2942 * @param[out] error 2943 * Pointer to error structure. 2944 * 2945 * @return 2946 * 0 on success, a negative errno value otherwise and rte_errno is set. 2947 */ 2948 int 2949 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 2950 const struct rte_flow_item *item, 2951 uint64_t item_flags, 2952 const struct rte_flow_attr *attr, 2953 const struct rte_flow_item *gre_item, 2954 struct rte_flow_error *error) 2955 { 2956 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 2957 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 2958 const struct rte_flow_item_gre_opt *spec = item->spec; 2959 const struct rte_flow_item_gre_opt *mask = item->mask; 2960 struct mlx5_priv *priv = dev->data->dev_private; 2961 int ret = 0; 2962 struct rte_flow_item_gre_opt nic_mask = { 2963 .checksum_rsvd = { 2964 .checksum = RTE_BE16(UINT16_MAX), 2965 .reserved1 = 0x0, 2966 }, 2967 .key = { 2968 .key = RTE_BE32(UINT32_MAX), 2969 }, 2970 .sequence = { 2971 .sequence = RTE_BE32(UINT32_MAX), 2972 }, 2973 }; 2974 2975 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2976 return rte_flow_error_set(error, ENOTSUP, 2977 RTE_FLOW_ERROR_TYPE_ITEM, item, 2978 "No preceding GRE header"); 2979 if (item_flags & MLX5_FLOW_LAYER_INNER) 2980 return rte_flow_error_set(error, ENOTSUP, 2981 RTE_FLOW_ERROR_TYPE_ITEM, item, 2982 "GRE option following a wrong item"); 2983 if (!spec || !mask) 2984 return rte_flow_error_set(error, EINVAL, 2985 RTE_FLOW_ERROR_TYPE_ITEM, item, 2986 "At least one field gre_option(checksum/key/sequence) must be specified"); 2987 if (!gre_mask) 2988 gre_mask = &rte_flow_item_gre_mask; 2989 if (mask->checksum_rsvd.checksum) 2990 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 2991 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 2992 return rte_flow_error_set(error, EINVAL, 2993 RTE_FLOW_ERROR_TYPE_ITEM, 2994 item, 2995 "Checksum bit must be on"); 2996 if (mask->key.key) 2997 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2998 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2999 return rte_flow_error_set(error, EINVAL, 3000 RTE_FLOW_ERROR_TYPE_ITEM, 3001 item, "Key bit must be on"); 3002 if (mask->sequence.sequence) 3003 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3004 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3005 return rte_flow_error_set(error, EINVAL, 3006 RTE_FLOW_ERROR_TYPE_ITEM, 3007 item, 3008 "Sequence bit must be on"); 3009 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3010 if (priv->sh->steering_format_version == 3011 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3012 ((attr->group || attr->transfer) && 3013 !priv->sh->misc5_cap) || 3014 (!(priv->sh->tunnel_header_0_1 && 3015 priv->sh->tunnel_header_2_3) && 3016 !attr->group && !attr->transfer)) 3017 return rte_flow_error_set(error, EINVAL, 3018 RTE_FLOW_ERROR_TYPE_ITEM, 3019 item, 3020 "Checksum/Sequence not supported"); 3021 } 3022 ret = mlx5_flow_item_acceptable 3023 (item, (const uint8_t *)mask, 3024 (const uint8_t *)&nic_mask, 3025 sizeof(struct rte_flow_item_gre_opt), 3026 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3027 return ret; 3028 } 3029 3030 /** 3031 * Validate GRE item. 3032 * 3033 * @param[in] item 3034 * Item specification. 3035 * @param[in] item_flags 3036 * Bit flags to mark detected items. 3037 * @param[in] target_protocol 3038 * The next protocol in the previous item. 3039 * @param[out] error 3040 * Pointer to error structure. 3041 * 3042 * @return 3043 * 0 on success, a negative errno value otherwise and rte_errno is set. 3044 */ 3045 int 3046 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3047 uint64_t item_flags, 3048 uint8_t target_protocol, 3049 struct rte_flow_error *error) 3050 { 3051 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3052 const struct rte_flow_item_gre *mask = item->mask; 3053 int ret; 3054 const struct rte_flow_item_gre nic_mask = { 3055 .c_rsvd0_ver = RTE_BE16(0xB000), 3056 .protocol = RTE_BE16(UINT16_MAX), 3057 }; 3058 3059 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3060 return rte_flow_error_set(error, EINVAL, 3061 RTE_FLOW_ERROR_TYPE_ITEM, item, 3062 "protocol filtering not compatible" 3063 " with this GRE layer"); 3064 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3065 return rte_flow_error_set(error, ENOTSUP, 3066 RTE_FLOW_ERROR_TYPE_ITEM, item, 3067 "multiple tunnel layers not" 3068 " supported"); 3069 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3070 return rte_flow_error_set(error, ENOTSUP, 3071 RTE_FLOW_ERROR_TYPE_ITEM, item, 3072 "L3 Layer is missing"); 3073 if (!mask) 3074 mask = &rte_flow_item_gre_mask; 3075 ret = mlx5_flow_item_acceptable 3076 (item, (const uint8_t *)mask, 3077 (const uint8_t *)&nic_mask, 3078 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3079 error); 3080 if (ret < 0) 3081 return ret; 3082 #ifndef HAVE_MLX5DV_DR 3083 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3084 if (spec && (spec->protocol & mask->protocol)) 3085 return rte_flow_error_set(error, ENOTSUP, 3086 RTE_FLOW_ERROR_TYPE_ITEM, item, 3087 "without MPLS support the" 3088 " specification cannot be used for" 3089 " filtering"); 3090 #endif 3091 #endif 3092 return 0; 3093 } 3094 3095 /** 3096 * Validate Geneve item. 3097 * 3098 * @param[in] item 3099 * Item specification. 3100 * @param[in] itemFlags 3101 * Bit-fields that holds the items detected until now. 3102 * @param[in] enPriv 3103 * Pointer to the private data structure. 3104 * @param[out] error 3105 * Pointer to error structure. 3106 * 3107 * @return 3108 * 0 on success, a negative errno value otherwise and rte_errno is set. 3109 */ 3110 3111 int 3112 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3113 uint64_t item_flags, 3114 struct rte_eth_dev *dev, 3115 struct rte_flow_error *error) 3116 { 3117 struct mlx5_priv *priv = dev->data->dev_private; 3118 const struct rte_flow_item_geneve *spec = item->spec; 3119 const struct rte_flow_item_geneve *mask = item->mask; 3120 int ret; 3121 uint16_t gbhdr; 3122 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3123 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3124 const struct rte_flow_item_geneve nic_mask = { 3125 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3126 .vni = "\xff\xff\xff", 3127 .protocol = RTE_BE16(UINT16_MAX), 3128 }; 3129 3130 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3131 return rte_flow_error_set(error, ENOTSUP, 3132 RTE_FLOW_ERROR_TYPE_ITEM, item, 3133 "L3 Geneve is not enabled by device" 3134 " parameter and/or not configured in" 3135 " firmware"); 3136 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3137 return rte_flow_error_set(error, ENOTSUP, 3138 RTE_FLOW_ERROR_TYPE_ITEM, item, 3139 "multiple tunnel layers not" 3140 " supported"); 3141 /* 3142 * Verify only UDPv4 is present as defined in 3143 * https://tools.ietf.org/html/rfc7348 3144 */ 3145 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3146 return rte_flow_error_set(error, EINVAL, 3147 RTE_FLOW_ERROR_TYPE_ITEM, item, 3148 "no outer UDP layer found"); 3149 if (!mask) 3150 mask = &rte_flow_item_geneve_mask; 3151 ret = mlx5_flow_item_acceptable 3152 (item, (const uint8_t *)mask, 3153 (const uint8_t *)&nic_mask, 3154 sizeof(struct rte_flow_item_geneve), 3155 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3156 if (ret) 3157 return ret; 3158 if (spec) { 3159 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3160 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3161 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3162 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3163 return rte_flow_error_set(error, ENOTSUP, 3164 RTE_FLOW_ERROR_TYPE_ITEM, 3165 item, 3166 "Geneve protocol unsupported" 3167 " fields are being used"); 3168 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3169 return rte_flow_error_set 3170 (error, ENOTSUP, 3171 RTE_FLOW_ERROR_TYPE_ITEM, 3172 item, 3173 "Unsupported Geneve options length"); 3174 } 3175 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3176 return rte_flow_error_set 3177 (error, ENOTSUP, 3178 RTE_FLOW_ERROR_TYPE_ITEM, item, 3179 "Geneve tunnel must be fully defined"); 3180 return 0; 3181 } 3182 3183 /** 3184 * Validate Geneve TLV option item. 3185 * 3186 * @param[in] item 3187 * Item specification. 3188 * @param[in] last_item 3189 * Previous validated item in the pattern items. 3190 * @param[in] geneve_item 3191 * Previous GENEVE item specification. 3192 * @param[in] dev 3193 * Pointer to the rte_eth_dev structure. 3194 * @param[out] error 3195 * Pointer to error structure. 3196 * 3197 * @return 3198 * 0 on success, a negative errno value otherwise and rte_errno is set. 3199 */ 3200 int 3201 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3202 uint64_t last_item, 3203 const struct rte_flow_item *geneve_item, 3204 struct rte_eth_dev *dev, 3205 struct rte_flow_error *error) 3206 { 3207 struct mlx5_priv *priv = dev->data->dev_private; 3208 struct mlx5_dev_ctx_shared *sh = priv->sh; 3209 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3210 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3211 uint8_t data_max_supported = 3212 hca_attr->max_geneve_tlv_option_data_len * 4; 3213 const struct rte_flow_item_geneve *geneve_spec; 3214 const struct rte_flow_item_geneve *geneve_mask; 3215 const struct rte_flow_item_geneve_opt *spec = item->spec; 3216 const struct rte_flow_item_geneve_opt *mask = item->mask; 3217 unsigned int i; 3218 unsigned int data_len; 3219 uint8_t tlv_option_len; 3220 uint16_t optlen_m, optlen_v; 3221 const struct rte_flow_item_geneve_opt full_mask = { 3222 .option_class = RTE_BE16(0xffff), 3223 .option_type = 0xff, 3224 .option_len = 0x1f, 3225 }; 3226 3227 if (!mask) 3228 mask = &rte_flow_item_geneve_opt_mask; 3229 if (!spec) 3230 return rte_flow_error_set 3231 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3232 "Geneve TLV opt class/type/length must be specified"); 3233 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3234 return rte_flow_error_set 3235 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3236 "Geneve TLV opt length exceeds the limit (31)"); 3237 /* Check if class type and length masks are full. */ 3238 if (full_mask.option_class != mask->option_class || 3239 full_mask.option_type != mask->option_type || 3240 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3241 return rte_flow_error_set 3242 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3243 "Geneve TLV opt class/type/length masks must be full"); 3244 /* Check if length is supported */ 3245 if ((uint32_t)spec->option_len > 3246 hca_attr->max_geneve_tlv_option_data_len) 3247 return rte_flow_error_set 3248 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3249 "Geneve TLV opt length not supported"); 3250 if (hca_attr->max_geneve_tlv_options > 1) 3251 DRV_LOG(DEBUG, 3252 "max_geneve_tlv_options supports more than 1 option"); 3253 /* Check GENEVE item preceding. */ 3254 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3255 return rte_flow_error_set 3256 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3257 "Geneve opt item must be preceded with Geneve item"); 3258 geneve_spec = geneve_item->spec; 3259 geneve_mask = geneve_item->mask ? geneve_item->mask : 3260 &rte_flow_item_geneve_mask; 3261 /* Check if GENEVE TLV option size doesn't exceed option length */ 3262 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3263 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3264 tlv_option_len = spec->option_len & mask->option_len; 3265 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3266 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3267 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3268 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3269 if ((optlen_v & optlen_m) <= tlv_option_len) 3270 return rte_flow_error_set 3271 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3272 "GENEVE TLV option length exceeds optlen"); 3273 } 3274 /* Check if length is 0 or data is 0. */ 3275 if (spec->data == NULL || spec->option_len == 0) 3276 return rte_flow_error_set 3277 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3278 "Geneve TLV opt with zero data/length not supported"); 3279 /* Check not all data & mask are 0. */ 3280 data_len = spec->option_len * 4; 3281 if (mask->data == NULL) { 3282 for (i = 0; i < data_len; i++) 3283 if (spec->data[i]) 3284 break; 3285 if (i == data_len) 3286 return rte_flow_error_set(error, ENOTSUP, 3287 RTE_FLOW_ERROR_TYPE_ITEM, item, 3288 "Can't match on Geneve option data 0"); 3289 } else { 3290 for (i = 0; i < data_len; i++) 3291 if (spec->data[i] & mask->data[i]) 3292 break; 3293 if (i == data_len) 3294 return rte_flow_error_set(error, ENOTSUP, 3295 RTE_FLOW_ERROR_TYPE_ITEM, item, 3296 "Can't match on Geneve option data and mask 0"); 3297 /* Check data mask supported. */ 3298 for (i = data_max_supported; i < data_len ; i++) 3299 if (mask->data[i]) 3300 return rte_flow_error_set(error, ENOTSUP, 3301 RTE_FLOW_ERROR_TYPE_ITEM, item, 3302 "Data mask is of unsupported size"); 3303 } 3304 /* Check GENEVE option is supported in NIC. */ 3305 if (!hca_attr->geneve_tlv_opt) 3306 return rte_flow_error_set 3307 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3308 "Geneve TLV opt not supported"); 3309 /* Check if we already have geneve option with different type/class. */ 3310 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3311 geneve_opt_resource = sh->geneve_tlv_option_resource; 3312 if (geneve_opt_resource != NULL) 3313 if (geneve_opt_resource->option_class != spec->option_class || 3314 geneve_opt_resource->option_type != spec->option_type || 3315 geneve_opt_resource->length != spec->option_len) { 3316 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3317 return rte_flow_error_set(error, ENOTSUP, 3318 RTE_FLOW_ERROR_TYPE_ITEM, item, 3319 "Only one Geneve TLV option supported"); 3320 } 3321 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3322 return 0; 3323 } 3324 3325 /** 3326 * Validate MPLS item. 3327 * 3328 * @param[in] dev 3329 * Pointer to the rte_eth_dev structure. 3330 * @param[in] item 3331 * Item specification. 3332 * @param[in] item_flags 3333 * Bit-fields that holds the items detected until now. 3334 * @param[in] prev_layer 3335 * The protocol layer indicated in previous item. 3336 * @param[out] error 3337 * Pointer to error structure. 3338 * 3339 * @return 3340 * 0 on success, a negative errno value otherwise and rte_errno is set. 3341 */ 3342 int 3343 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3344 const struct rte_flow_item *item __rte_unused, 3345 uint64_t item_flags __rte_unused, 3346 uint64_t prev_layer __rte_unused, 3347 struct rte_flow_error *error) 3348 { 3349 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3350 const struct rte_flow_item_mpls *mask = item->mask; 3351 struct mlx5_priv *priv = dev->data->dev_private; 3352 int ret; 3353 3354 if (!priv->sh->dev_cap.mpls_en) 3355 return rte_flow_error_set(error, ENOTSUP, 3356 RTE_FLOW_ERROR_TYPE_ITEM, item, 3357 "MPLS not supported or" 3358 " disabled in firmware" 3359 " configuration."); 3360 /* MPLS over UDP, GRE is allowed */ 3361 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3362 MLX5_FLOW_LAYER_GRE | 3363 MLX5_FLOW_LAYER_GRE_KEY))) 3364 return rte_flow_error_set(error, EINVAL, 3365 RTE_FLOW_ERROR_TYPE_ITEM, item, 3366 "protocol filtering not compatible" 3367 " with MPLS layer"); 3368 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3369 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3370 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3371 return rte_flow_error_set(error, ENOTSUP, 3372 RTE_FLOW_ERROR_TYPE_ITEM, item, 3373 "multiple tunnel layers not" 3374 " supported"); 3375 if (!mask) 3376 mask = &rte_flow_item_mpls_mask; 3377 ret = mlx5_flow_item_acceptable 3378 (item, (const uint8_t *)mask, 3379 (const uint8_t *)&rte_flow_item_mpls_mask, 3380 sizeof(struct rte_flow_item_mpls), 3381 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3382 if (ret < 0) 3383 return ret; 3384 return 0; 3385 #else 3386 return rte_flow_error_set(error, ENOTSUP, 3387 RTE_FLOW_ERROR_TYPE_ITEM, item, 3388 "MPLS is not supported by Verbs, please" 3389 " update."); 3390 #endif 3391 } 3392 3393 /** 3394 * Validate NVGRE item. 3395 * 3396 * @param[in] item 3397 * Item specification. 3398 * @param[in] item_flags 3399 * Bit flags to mark detected items. 3400 * @param[in] target_protocol 3401 * The next protocol in the previous item. 3402 * @param[out] error 3403 * Pointer to error structure. 3404 * 3405 * @return 3406 * 0 on success, a negative errno value otherwise and rte_errno is set. 3407 */ 3408 int 3409 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3410 uint64_t item_flags, 3411 uint8_t target_protocol, 3412 struct rte_flow_error *error) 3413 { 3414 const struct rte_flow_item_nvgre *mask = item->mask; 3415 int ret; 3416 3417 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3418 return rte_flow_error_set(error, EINVAL, 3419 RTE_FLOW_ERROR_TYPE_ITEM, item, 3420 "protocol filtering not compatible" 3421 " with this GRE layer"); 3422 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3423 return rte_flow_error_set(error, ENOTSUP, 3424 RTE_FLOW_ERROR_TYPE_ITEM, item, 3425 "multiple tunnel layers not" 3426 " supported"); 3427 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3428 return rte_flow_error_set(error, ENOTSUP, 3429 RTE_FLOW_ERROR_TYPE_ITEM, item, 3430 "L3 Layer is missing"); 3431 if (!mask) 3432 mask = &rte_flow_item_nvgre_mask; 3433 ret = mlx5_flow_item_acceptable 3434 (item, (const uint8_t *)mask, 3435 (const uint8_t *)&rte_flow_item_nvgre_mask, 3436 sizeof(struct rte_flow_item_nvgre), 3437 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3438 if (ret < 0) 3439 return ret; 3440 return 0; 3441 } 3442 3443 /** 3444 * Validate eCPRI item. 3445 * 3446 * @param[in] item 3447 * Item specification. 3448 * @param[in] item_flags 3449 * Bit-fields that holds the items detected until now. 3450 * @param[in] last_item 3451 * Previous validated item in the pattern items. 3452 * @param[in] ether_type 3453 * Type in the ethernet layer header (including dot1q). 3454 * @param[in] acc_mask 3455 * Acceptable mask, if NULL default internal default mask 3456 * will be used to check whether item fields are supported. 3457 * @param[out] error 3458 * Pointer to error structure. 3459 * 3460 * @return 3461 * 0 on success, a negative errno value otherwise and rte_errno is set. 3462 */ 3463 int 3464 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3465 uint64_t item_flags, 3466 uint64_t last_item, 3467 uint16_t ether_type, 3468 const struct rte_flow_item_ecpri *acc_mask, 3469 struct rte_flow_error *error) 3470 { 3471 const struct rte_flow_item_ecpri *mask = item->mask; 3472 const struct rte_flow_item_ecpri nic_mask = { 3473 .hdr = { 3474 .common = { 3475 .u32 = 3476 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3477 .type = 0xFF, 3478 }).u32), 3479 }, 3480 .dummy[0] = 0xFFFFFFFF, 3481 }, 3482 }; 3483 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3484 MLX5_FLOW_LAYER_OUTER_VLAN); 3485 struct rte_flow_item_ecpri mask_lo; 3486 3487 if (!(last_item & outer_l2_vlan) && 3488 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3489 return rte_flow_error_set(error, EINVAL, 3490 RTE_FLOW_ERROR_TYPE_ITEM, item, 3491 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3492 if ((last_item & outer_l2_vlan) && ether_type && 3493 ether_type != RTE_ETHER_TYPE_ECPRI) 3494 return rte_flow_error_set(error, EINVAL, 3495 RTE_FLOW_ERROR_TYPE_ITEM, item, 3496 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3497 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3498 return rte_flow_error_set(error, EINVAL, 3499 RTE_FLOW_ERROR_TYPE_ITEM, item, 3500 "eCPRI with tunnel is not supported right now"); 3501 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3502 return rte_flow_error_set(error, ENOTSUP, 3503 RTE_FLOW_ERROR_TYPE_ITEM, item, 3504 "multiple L3 layers not supported"); 3505 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3506 return rte_flow_error_set(error, EINVAL, 3507 RTE_FLOW_ERROR_TYPE_ITEM, item, 3508 "eCPRI cannot coexist with a TCP layer"); 3509 /* In specification, eCPRI could be over UDP layer. */ 3510 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3511 return rte_flow_error_set(error, EINVAL, 3512 RTE_FLOW_ERROR_TYPE_ITEM, item, 3513 "eCPRI over UDP layer is not yet supported right now"); 3514 /* Mask for type field in common header could be zero. */ 3515 if (!mask) 3516 mask = &rte_flow_item_ecpri_mask; 3517 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3518 /* Input mask is in big-endian format. */ 3519 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3520 return rte_flow_error_set(error, EINVAL, 3521 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3522 "partial mask is not supported for protocol"); 3523 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3524 return rte_flow_error_set(error, EINVAL, 3525 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3526 "message header mask must be after a type mask"); 3527 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3528 acc_mask ? (const uint8_t *)acc_mask 3529 : (const uint8_t *)&nic_mask, 3530 sizeof(struct rte_flow_item_ecpri), 3531 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3532 } 3533 3534 static int 3535 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3536 const struct rte_flow_attr *attr __rte_unused, 3537 const struct rte_flow_item items[] __rte_unused, 3538 const struct rte_flow_action actions[] __rte_unused, 3539 bool external __rte_unused, 3540 int hairpin __rte_unused, 3541 struct rte_flow_error *error) 3542 { 3543 return rte_flow_error_set(error, ENOTSUP, 3544 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3545 } 3546 3547 static struct mlx5_flow * 3548 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3549 const struct rte_flow_attr *attr __rte_unused, 3550 const struct rte_flow_item items[] __rte_unused, 3551 const struct rte_flow_action actions[] __rte_unused, 3552 struct rte_flow_error *error) 3553 { 3554 rte_flow_error_set(error, ENOTSUP, 3555 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3556 return NULL; 3557 } 3558 3559 static int 3560 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3561 struct mlx5_flow *dev_flow __rte_unused, 3562 const struct rte_flow_attr *attr __rte_unused, 3563 const struct rte_flow_item items[] __rte_unused, 3564 const struct rte_flow_action actions[] __rte_unused, 3565 struct rte_flow_error *error) 3566 { 3567 return rte_flow_error_set(error, ENOTSUP, 3568 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3569 } 3570 3571 static int 3572 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3573 struct rte_flow *flow __rte_unused, 3574 struct rte_flow_error *error) 3575 { 3576 return rte_flow_error_set(error, ENOTSUP, 3577 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3578 } 3579 3580 static void 3581 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3582 struct rte_flow *flow __rte_unused) 3583 { 3584 } 3585 3586 static void 3587 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3588 struct rte_flow *flow __rte_unused) 3589 { 3590 } 3591 3592 static int 3593 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3594 struct rte_flow *flow __rte_unused, 3595 const struct rte_flow_action *actions __rte_unused, 3596 void *data __rte_unused, 3597 struct rte_flow_error *error) 3598 { 3599 return rte_flow_error_set(error, ENOTSUP, 3600 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3601 } 3602 3603 static int 3604 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3605 uint32_t domains __rte_unused, 3606 uint32_t flags __rte_unused) 3607 { 3608 return 0; 3609 } 3610 3611 /* Void driver to protect from null pointer reference. */ 3612 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3613 .validate = flow_null_validate, 3614 .prepare = flow_null_prepare, 3615 .translate = flow_null_translate, 3616 .apply = flow_null_apply, 3617 .remove = flow_null_remove, 3618 .destroy = flow_null_destroy, 3619 .query = flow_null_query, 3620 .sync_domain = flow_null_sync_domain, 3621 }; 3622 3623 /** 3624 * Select flow driver type according to flow attributes and device 3625 * configuration. 3626 * 3627 * @param[in] dev 3628 * Pointer to the dev structure. 3629 * @param[in] attr 3630 * Pointer to the flow attributes. 3631 * 3632 * @return 3633 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3634 */ 3635 static enum mlx5_flow_drv_type 3636 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3637 { 3638 struct mlx5_priv *priv = dev->data->dev_private; 3639 /* The OS can determine first a specific flow type (DV, VERBS) */ 3640 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3641 3642 if (type != MLX5_FLOW_TYPE_MAX) 3643 return type; 3644 /* 3645 * Currently when dv_flow_en == 2, only HW steering engine is 3646 * supported. New engines can also be chosen here if ready. 3647 */ 3648 if (priv->sh->config.dv_flow_en == 2) 3649 return MLX5_FLOW_TYPE_HW; 3650 /* If no OS specific type - continue with DV/VERBS selection */ 3651 if (attr->transfer && priv->sh->config.dv_esw_en) 3652 type = MLX5_FLOW_TYPE_DV; 3653 if (!attr->transfer) 3654 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3655 MLX5_FLOW_TYPE_VERBS; 3656 return type; 3657 } 3658 3659 #define flow_get_drv_ops(type) flow_drv_ops[type] 3660 3661 /** 3662 * Flow driver validation API. This abstracts calling driver specific functions. 3663 * The type of flow driver is determined according to flow attributes. 3664 * 3665 * @param[in] dev 3666 * Pointer to the dev structure. 3667 * @param[in] attr 3668 * Pointer to the flow attributes. 3669 * @param[in] items 3670 * Pointer to the list of items. 3671 * @param[in] actions 3672 * Pointer to the list of actions. 3673 * @param[in] external 3674 * This flow rule is created by request external to PMD. 3675 * @param[in] hairpin 3676 * Number of hairpin TX actions, 0 means classic flow. 3677 * @param[out] error 3678 * Pointer to the error structure. 3679 * 3680 * @return 3681 * 0 on success, a negative errno value otherwise and rte_errno is set. 3682 */ 3683 static inline int 3684 flow_drv_validate(struct rte_eth_dev *dev, 3685 const struct rte_flow_attr *attr, 3686 const struct rte_flow_item items[], 3687 const struct rte_flow_action actions[], 3688 bool external, int hairpin, struct rte_flow_error *error) 3689 { 3690 const struct mlx5_flow_driver_ops *fops; 3691 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3692 3693 fops = flow_get_drv_ops(type); 3694 return fops->validate(dev, attr, items, actions, external, 3695 hairpin, error); 3696 } 3697 3698 /** 3699 * Flow driver preparation API. This abstracts calling driver specific 3700 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3701 * calculates the size of memory required for device flow, allocates the memory, 3702 * initializes the device flow and returns the pointer. 3703 * 3704 * @note 3705 * This function initializes device flow structure such as dv or verbs in 3706 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3707 * rest. For example, adding returning device flow to flow->dev_flow list and 3708 * setting backward reference to the flow should be done out of this function. 3709 * layers field is not filled either. 3710 * 3711 * @param[in] dev 3712 * Pointer to the dev structure. 3713 * @param[in] attr 3714 * Pointer to the flow attributes. 3715 * @param[in] items 3716 * Pointer to the list of items. 3717 * @param[in] actions 3718 * Pointer to the list of actions. 3719 * @param[in] flow_idx 3720 * This memory pool index to the flow. 3721 * @param[out] error 3722 * Pointer to the error structure. 3723 * 3724 * @return 3725 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3726 */ 3727 static inline struct mlx5_flow * 3728 flow_drv_prepare(struct rte_eth_dev *dev, 3729 const struct rte_flow *flow, 3730 const struct rte_flow_attr *attr, 3731 const struct rte_flow_item items[], 3732 const struct rte_flow_action actions[], 3733 uint32_t flow_idx, 3734 struct rte_flow_error *error) 3735 { 3736 const struct mlx5_flow_driver_ops *fops; 3737 enum mlx5_flow_drv_type type = flow->drv_type; 3738 struct mlx5_flow *mlx5_flow = NULL; 3739 3740 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3741 fops = flow_get_drv_ops(type); 3742 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3743 if (mlx5_flow) 3744 mlx5_flow->flow_idx = flow_idx; 3745 return mlx5_flow; 3746 } 3747 3748 /** 3749 * Flow driver translation API. This abstracts calling driver specific 3750 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3751 * translates a generic flow into a driver flow. flow_drv_prepare() must 3752 * precede. 3753 * 3754 * @note 3755 * dev_flow->layers could be filled as a result of parsing during translation 3756 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3757 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3758 * flow->actions could be overwritten even though all the expanded dev_flows 3759 * have the same actions. 3760 * 3761 * @param[in] dev 3762 * Pointer to the rte dev structure. 3763 * @param[in, out] dev_flow 3764 * Pointer to the mlx5 flow. 3765 * @param[in] attr 3766 * Pointer to the flow attributes. 3767 * @param[in] items 3768 * Pointer to the list of items. 3769 * @param[in] actions 3770 * Pointer to the list of actions. 3771 * @param[out] error 3772 * Pointer to the error structure. 3773 * 3774 * @return 3775 * 0 on success, a negative errno value otherwise and rte_errno is set. 3776 */ 3777 static inline int 3778 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3779 const struct rte_flow_attr *attr, 3780 const struct rte_flow_item items[], 3781 const struct rte_flow_action actions[], 3782 struct rte_flow_error *error) 3783 { 3784 const struct mlx5_flow_driver_ops *fops; 3785 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3786 3787 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3788 fops = flow_get_drv_ops(type); 3789 return fops->translate(dev, dev_flow, attr, items, actions, error); 3790 } 3791 3792 /** 3793 * Flow driver apply API. This abstracts calling driver specific functions. 3794 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3795 * translated driver flows on to device. flow_drv_translate() must precede. 3796 * 3797 * @param[in] dev 3798 * Pointer to Ethernet device structure. 3799 * @param[in, out] flow 3800 * Pointer to flow structure. 3801 * @param[out] error 3802 * Pointer to error structure. 3803 * 3804 * @return 3805 * 0 on success, a negative errno value otherwise and rte_errno is set. 3806 */ 3807 static inline int 3808 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3809 struct rte_flow_error *error) 3810 { 3811 const struct mlx5_flow_driver_ops *fops; 3812 enum mlx5_flow_drv_type type = flow->drv_type; 3813 3814 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3815 fops = flow_get_drv_ops(type); 3816 return fops->apply(dev, flow, error); 3817 } 3818 3819 /** 3820 * Flow driver destroy API. This abstracts calling driver specific functions. 3821 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3822 * on device and releases resources of the flow. 3823 * 3824 * @param[in] dev 3825 * Pointer to Ethernet device. 3826 * @param[in, out] flow 3827 * Pointer to flow structure. 3828 */ 3829 static inline void 3830 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3831 { 3832 const struct mlx5_flow_driver_ops *fops; 3833 enum mlx5_flow_drv_type type = flow->drv_type; 3834 3835 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3836 fops = flow_get_drv_ops(type); 3837 fops->destroy(dev, flow); 3838 } 3839 3840 /** 3841 * Flow driver find RSS policy tbl API. This abstracts calling driver 3842 * specific functions. Parent flow (rte_flow) should have driver 3843 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3844 * 3845 * @param[in] dev 3846 * Pointer to Ethernet device. 3847 * @param[in, out] flow 3848 * Pointer to flow structure. 3849 * @param[in] policy 3850 * Pointer to meter policy table. 3851 * @param[in] rss_desc 3852 * Pointer to rss_desc 3853 */ 3854 static struct mlx5_flow_meter_sub_policy * 3855 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3856 struct rte_flow *flow, 3857 struct mlx5_flow_meter_policy *policy, 3858 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3859 { 3860 const struct mlx5_flow_driver_ops *fops; 3861 enum mlx5_flow_drv_type type = flow->drv_type; 3862 3863 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3864 fops = flow_get_drv_ops(type); 3865 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3866 } 3867 3868 /** 3869 * Flow driver color tag rule API. This abstracts calling driver 3870 * specific functions. Parent flow (rte_flow) should have driver 3871 * type (drv_type). It will create the color tag rules in hierarchy meter. 3872 * 3873 * @param[in] dev 3874 * Pointer to Ethernet device. 3875 * @param[in, out] flow 3876 * Pointer to flow structure. 3877 * @param[in] fm 3878 * Pointer to flow meter structure. 3879 * @param[in] src_port 3880 * The src port this extra rule should use. 3881 * @param[in] item 3882 * The src port id match item. 3883 * @param[out] error 3884 * Pointer to error structure. 3885 */ 3886 static int 3887 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3888 struct rte_flow *flow, 3889 struct mlx5_flow_meter_info *fm, 3890 int32_t src_port, 3891 const struct rte_flow_item *item, 3892 struct rte_flow_error *error) 3893 { 3894 const struct mlx5_flow_driver_ops *fops; 3895 enum mlx5_flow_drv_type type = flow->drv_type; 3896 3897 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3898 fops = flow_get_drv_ops(type); 3899 return fops->meter_hierarchy_rule_create(dev, fm, 3900 src_port, item, error); 3901 } 3902 3903 /** 3904 * Get RSS action from the action list. 3905 * 3906 * @param[in] dev 3907 * Pointer to Ethernet device. 3908 * @param[in] actions 3909 * Pointer to the list of actions. 3910 * @param[in] flow 3911 * Parent flow structure pointer. 3912 * 3913 * @return 3914 * Pointer to the RSS action if exist, else return NULL. 3915 */ 3916 static const struct rte_flow_action_rss* 3917 flow_get_rss_action(struct rte_eth_dev *dev, 3918 const struct rte_flow_action actions[]) 3919 { 3920 struct mlx5_priv *priv = dev->data->dev_private; 3921 const struct rte_flow_action_rss *rss = NULL; 3922 struct mlx5_meter_policy_action_container *acg; 3923 struct mlx5_meter_policy_action_container *acy; 3924 3925 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3926 switch (actions->type) { 3927 case RTE_FLOW_ACTION_TYPE_RSS: 3928 rss = actions->conf; 3929 break; 3930 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3931 { 3932 const struct rte_flow_action_sample *sample = 3933 actions->conf; 3934 const struct rte_flow_action *act = sample->actions; 3935 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3936 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3937 rss = act->conf; 3938 break; 3939 } 3940 case RTE_FLOW_ACTION_TYPE_METER: 3941 { 3942 uint32_t mtr_idx; 3943 struct mlx5_flow_meter_info *fm; 3944 struct mlx5_flow_meter_policy *policy; 3945 const struct rte_flow_action_meter *mtr = actions->conf; 3946 3947 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3948 if (fm && !fm->def_policy) { 3949 policy = mlx5_flow_meter_policy_find(dev, 3950 fm->policy_id, NULL); 3951 MLX5_ASSERT(policy); 3952 if (policy->is_hierarchy) { 3953 policy = 3954 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3955 policy); 3956 if (!policy) 3957 return NULL; 3958 } 3959 if (policy->is_rss) { 3960 acg = 3961 &policy->act_cnt[RTE_COLOR_GREEN]; 3962 acy = 3963 &policy->act_cnt[RTE_COLOR_YELLOW]; 3964 if (acg->fate_action == 3965 MLX5_FLOW_FATE_SHARED_RSS) 3966 rss = acg->rss->conf; 3967 else if (acy->fate_action == 3968 MLX5_FLOW_FATE_SHARED_RSS) 3969 rss = acy->rss->conf; 3970 } 3971 } 3972 break; 3973 } 3974 default: 3975 break; 3976 } 3977 } 3978 return rss; 3979 } 3980 3981 /** 3982 * Get ASO age action by index. 3983 * 3984 * @param[in] dev 3985 * Pointer to the Ethernet device structure. 3986 * @param[in] age_idx 3987 * Index to the ASO age action. 3988 * 3989 * @return 3990 * The specified ASO age action. 3991 */ 3992 struct mlx5_aso_age_action* 3993 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3994 { 3995 uint16_t pool_idx = age_idx & UINT16_MAX; 3996 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3997 struct mlx5_priv *priv = dev->data->dev_private; 3998 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3999 struct mlx5_aso_age_pool *pool; 4000 4001 rte_rwlock_read_lock(&mng->resize_rwl); 4002 pool = mng->pools[pool_idx]; 4003 rte_rwlock_read_unlock(&mng->resize_rwl); 4004 return &pool->actions[offset - 1]; 4005 } 4006 4007 /* maps indirect action to translated direct in some actions array */ 4008 struct mlx5_translated_action_handle { 4009 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4010 int index; /**< Index in related array of rte_flow_action. */ 4011 }; 4012 4013 /** 4014 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4015 * direct action if translation possible. 4016 * This functionality used to run same execution path for both direct and 4017 * indirect actions on flow create. All necessary preparations for indirect 4018 * action handling should be performed on *handle* actions list returned 4019 * from this call. 4020 * 4021 * @param[in] dev 4022 * Pointer to Ethernet device. 4023 * @param[in] actions 4024 * List of actions to translate. 4025 * @param[out] handle 4026 * List to store translated indirect action object handles. 4027 * @param[in, out] indir_n 4028 * Size of *handle* array. On return should be updated with number of 4029 * indirect actions retrieved from the *actions* list. 4030 * @param[out] translated_actions 4031 * List of actions where all indirect actions were translated to direct 4032 * if possible. NULL if no translation took place. 4033 * @param[out] error 4034 * Pointer to the error structure. 4035 * 4036 * @return 4037 * 0 on success, a negative errno value otherwise and rte_errno is set. 4038 */ 4039 static int 4040 flow_action_handles_translate(struct rte_eth_dev *dev, 4041 const struct rte_flow_action actions[], 4042 struct mlx5_translated_action_handle *handle, 4043 int *indir_n, 4044 struct rte_flow_action **translated_actions, 4045 struct rte_flow_error *error) 4046 { 4047 struct mlx5_priv *priv = dev->data->dev_private; 4048 struct rte_flow_action *translated = NULL; 4049 size_t actions_size; 4050 int n; 4051 int copied_n = 0; 4052 struct mlx5_translated_action_handle *handle_end = NULL; 4053 4054 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4055 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4056 continue; 4057 if (copied_n == *indir_n) { 4058 return rte_flow_error_set 4059 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4060 NULL, "too many shared actions"); 4061 } 4062 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4063 sizeof(actions[n].conf)); 4064 handle[copied_n].index = n; 4065 copied_n++; 4066 } 4067 n++; 4068 *indir_n = copied_n; 4069 if (!copied_n) 4070 return 0; 4071 actions_size = sizeof(struct rte_flow_action) * n; 4072 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4073 if (!translated) { 4074 rte_errno = ENOMEM; 4075 return -ENOMEM; 4076 } 4077 memcpy(translated, actions, actions_size); 4078 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4079 struct mlx5_shared_action_rss *shared_rss; 4080 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4081 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4082 uint32_t idx = act_idx & 4083 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4084 4085 switch (type) { 4086 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4087 shared_rss = mlx5_ipool_get 4088 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4089 translated[handle->index].type = 4090 RTE_FLOW_ACTION_TYPE_RSS; 4091 translated[handle->index].conf = 4092 &shared_rss->origin; 4093 break; 4094 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4095 translated[handle->index].type = 4096 (enum rte_flow_action_type) 4097 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4098 translated[handle->index].conf = (void *)(uintptr_t)idx; 4099 break; 4100 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4101 if (priv->sh->flow_hit_aso_en) { 4102 translated[handle->index].type = 4103 (enum rte_flow_action_type) 4104 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4105 translated[handle->index].conf = 4106 (void *)(uintptr_t)idx; 4107 break; 4108 } 4109 /* Fall-through */ 4110 case MLX5_INDIRECT_ACTION_TYPE_CT: 4111 if (priv->sh->ct_aso_en) { 4112 translated[handle->index].type = 4113 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4114 translated[handle->index].conf = 4115 (void *)(uintptr_t)idx; 4116 break; 4117 } 4118 /* Fall-through */ 4119 default: 4120 mlx5_free(translated); 4121 return rte_flow_error_set 4122 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4123 NULL, "invalid indirect action type"); 4124 } 4125 } 4126 *translated_actions = translated; 4127 return 0; 4128 } 4129 4130 /** 4131 * Get Shared RSS action from the action list. 4132 * 4133 * @param[in] dev 4134 * Pointer to Ethernet device. 4135 * @param[in] shared 4136 * Pointer to the list of actions. 4137 * @param[in] shared_n 4138 * Actions list length. 4139 * 4140 * @return 4141 * The MLX5 RSS action ID if exists, otherwise return 0. 4142 */ 4143 static uint32_t 4144 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4145 struct mlx5_translated_action_handle *handle, 4146 int shared_n) 4147 { 4148 struct mlx5_translated_action_handle *handle_end; 4149 struct mlx5_priv *priv = dev->data->dev_private; 4150 struct mlx5_shared_action_rss *shared_rss; 4151 4152 4153 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4154 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4155 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4156 uint32_t idx = act_idx & 4157 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4158 switch (type) { 4159 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4160 shared_rss = mlx5_ipool_get 4161 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4162 idx); 4163 __atomic_add_fetch(&shared_rss->refcnt, 1, 4164 __ATOMIC_RELAXED); 4165 return idx; 4166 default: 4167 break; 4168 } 4169 } 4170 return 0; 4171 } 4172 4173 static unsigned int 4174 find_graph_root(uint32_t rss_level) 4175 { 4176 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4177 MLX5_EXPANSION_ROOT_OUTER; 4178 } 4179 4180 /** 4181 * Get layer flags from the prefix flow. 4182 * 4183 * Some flows may be split to several subflows, the prefix subflow gets the 4184 * match items and the suffix sub flow gets the actions. 4185 * Some actions need the user defined match item flags to get the detail for 4186 * the action. 4187 * This function helps the suffix flow to get the item layer flags from prefix 4188 * subflow. 4189 * 4190 * @param[in] dev_flow 4191 * Pointer the created prefix subflow. 4192 * 4193 * @return 4194 * The layers get from prefix subflow. 4195 */ 4196 static inline uint64_t 4197 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4198 { 4199 uint64_t layers = 0; 4200 4201 /* 4202 * Layers bits could be localization, but usually the compiler will 4203 * help to do the optimization work for source code. 4204 * If no decap actions, use the layers directly. 4205 */ 4206 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4207 return dev_flow->handle->layers; 4208 /* Convert L3 layers with decap action. */ 4209 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4210 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4211 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4212 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4213 /* Convert L4 layers with decap action. */ 4214 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4215 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4216 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4217 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4218 return layers; 4219 } 4220 4221 /** 4222 * Get metadata split action information. 4223 * 4224 * @param[in] actions 4225 * Pointer to the list of actions. 4226 * @param[out] qrss 4227 * Pointer to the return pointer. 4228 * @param[out] qrss_type 4229 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4230 * if no QUEUE/RSS is found. 4231 * @param[out] encap_idx 4232 * Pointer to the index of the encap action if exists, otherwise the last 4233 * action index. 4234 * 4235 * @return 4236 * Total number of actions. 4237 */ 4238 static int 4239 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4240 const struct rte_flow_action **qrss, 4241 int *encap_idx) 4242 { 4243 const struct rte_flow_action_raw_encap *raw_encap; 4244 int actions_n = 0; 4245 int raw_decap_idx = -1; 4246 4247 *encap_idx = -1; 4248 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4249 switch (actions->type) { 4250 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4251 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4252 *encap_idx = actions_n; 4253 break; 4254 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4255 raw_decap_idx = actions_n; 4256 break; 4257 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4258 raw_encap = actions->conf; 4259 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4260 *encap_idx = raw_decap_idx != -1 ? 4261 raw_decap_idx : actions_n; 4262 break; 4263 case RTE_FLOW_ACTION_TYPE_QUEUE: 4264 case RTE_FLOW_ACTION_TYPE_RSS: 4265 *qrss = actions; 4266 break; 4267 default: 4268 break; 4269 } 4270 actions_n++; 4271 } 4272 if (*encap_idx == -1) 4273 *encap_idx = actions_n; 4274 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4275 return actions_n + 1; 4276 } 4277 4278 /** 4279 * Check if the action will change packet. 4280 * 4281 * @param dev 4282 * Pointer to Ethernet device. 4283 * @param[in] type 4284 * action type. 4285 * 4286 * @return 4287 * true if action will change packet, false otherwise. 4288 */ 4289 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4290 enum rte_flow_action_type type) 4291 { 4292 struct mlx5_priv *priv = dev->data->dev_private; 4293 4294 switch (type) { 4295 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4296 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4297 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4298 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4299 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4300 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4301 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4302 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4303 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4304 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4305 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4306 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4307 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4308 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4309 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4310 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4311 case RTE_FLOW_ACTION_TYPE_SET_META: 4312 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4313 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4314 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4315 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4316 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4317 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4318 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4319 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4320 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4321 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4322 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4323 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4324 return true; 4325 case RTE_FLOW_ACTION_TYPE_FLAG: 4326 case RTE_FLOW_ACTION_TYPE_MARK: 4327 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4328 return true; 4329 else 4330 return false; 4331 default: 4332 return false; 4333 } 4334 } 4335 4336 /** 4337 * Check meter action from the action list. 4338 * 4339 * @param dev 4340 * Pointer to Ethernet device. 4341 * @param[in] actions 4342 * Pointer to the list of actions. 4343 * @param[out] has_mtr 4344 * Pointer to the meter exist flag. 4345 * @param[out] has_modify 4346 * Pointer to the flag showing there's packet change action. 4347 * @param[out] meter_id 4348 * Pointer to the meter id. 4349 * 4350 * @return 4351 * Total number of actions. 4352 */ 4353 static int 4354 flow_check_meter_action(struct rte_eth_dev *dev, 4355 const struct rte_flow_action actions[], 4356 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4357 { 4358 const struct rte_flow_action_meter *mtr = NULL; 4359 int actions_n = 0; 4360 4361 MLX5_ASSERT(has_mtr); 4362 *has_mtr = false; 4363 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4364 switch (actions->type) { 4365 case RTE_FLOW_ACTION_TYPE_METER: 4366 mtr = actions->conf; 4367 *meter_id = mtr->mtr_id; 4368 *has_mtr = true; 4369 break; 4370 default: 4371 break; 4372 } 4373 if (!*has_mtr) 4374 *has_modify |= flow_check_modify_action_type(dev, 4375 actions->type); 4376 actions_n++; 4377 } 4378 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4379 return actions_n + 1; 4380 } 4381 4382 /** 4383 * Check if the flow should be split due to hairpin. 4384 * The reason for the split is that in current HW we can't 4385 * support encap and push-vlan on Rx, so if a flow contains 4386 * these actions we move it to Tx. 4387 * 4388 * @param dev 4389 * Pointer to Ethernet device. 4390 * @param[in] attr 4391 * Flow rule attributes. 4392 * @param[in] actions 4393 * Associated actions (list terminated by the END action). 4394 * 4395 * @return 4396 * > 0 the number of actions and the flow should be split, 4397 * 0 when no split required. 4398 */ 4399 static int 4400 flow_check_hairpin_split(struct rte_eth_dev *dev, 4401 const struct rte_flow_attr *attr, 4402 const struct rte_flow_action actions[]) 4403 { 4404 int queue_action = 0; 4405 int action_n = 0; 4406 int split = 0; 4407 const struct rte_flow_action_queue *queue; 4408 const struct rte_flow_action_rss *rss; 4409 const struct rte_flow_action_raw_encap *raw_encap; 4410 const struct rte_eth_hairpin_conf *conf; 4411 4412 if (!attr->ingress) 4413 return 0; 4414 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4415 switch (actions->type) { 4416 case RTE_FLOW_ACTION_TYPE_QUEUE: 4417 queue = actions->conf; 4418 if (queue == NULL) 4419 return 0; 4420 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4421 if (conf == NULL || conf->tx_explicit != 0) 4422 return 0; 4423 queue_action = 1; 4424 action_n++; 4425 break; 4426 case RTE_FLOW_ACTION_TYPE_RSS: 4427 rss = actions->conf; 4428 if (rss == NULL || rss->queue_num == 0) 4429 return 0; 4430 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4431 if (conf == NULL || conf->tx_explicit != 0) 4432 return 0; 4433 queue_action = 1; 4434 action_n++; 4435 break; 4436 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4437 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4438 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4439 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4440 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4441 split++; 4442 action_n++; 4443 break; 4444 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4445 raw_encap = actions->conf; 4446 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4447 split++; 4448 action_n++; 4449 break; 4450 default: 4451 action_n++; 4452 break; 4453 } 4454 } 4455 if (split && queue_action) 4456 return action_n; 4457 return 0; 4458 } 4459 4460 /* Declare flow create/destroy prototype in advance. */ 4461 static uint32_t 4462 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4463 const struct rte_flow_attr *attr, 4464 const struct rte_flow_item items[], 4465 const struct rte_flow_action actions[], 4466 bool external, struct rte_flow_error *error); 4467 4468 static void 4469 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4470 uint32_t flow_idx); 4471 4472 int 4473 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4474 struct mlx5_list_entry *entry, void *cb_ctx) 4475 { 4476 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4477 struct mlx5_flow_mreg_copy_resource *mcp_res = 4478 container_of(entry, typeof(*mcp_res), hlist_ent); 4479 4480 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4481 } 4482 4483 struct mlx5_list_entry * 4484 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4485 { 4486 struct rte_eth_dev *dev = tool_ctx; 4487 struct mlx5_priv *priv = dev->data->dev_private; 4488 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4489 struct mlx5_flow_mreg_copy_resource *mcp_res; 4490 struct rte_flow_error *error = ctx->error; 4491 uint32_t idx = 0; 4492 int ret; 4493 uint32_t mark_id = *(uint32_t *)(ctx->data); 4494 struct rte_flow_attr attr = { 4495 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4496 .ingress = 1, 4497 }; 4498 struct mlx5_rte_flow_item_tag tag_spec = { 4499 .data = mark_id, 4500 }; 4501 struct rte_flow_item items[] = { 4502 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4503 }; 4504 struct rte_flow_action_mark ftag = { 4505 .id = mark_id, 4506 }; 4507 struct mlx5_flow_action_copy_mreg cp_mreg = { 4508 .dst = REG_B, 4509 .src = REG_NON, 4510 }; 4511 struct rte_flow_action_jump jump = { 4512 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4513 }; 4514 struct rte_flow_action actions[] = { 4515 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4516 }; 4517 4518 /* Fill the register fields in the flow. */ 4519 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4520 if (ret < 0) 4521 return NULL; 4522 tag_spec.id = ret; 4523 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4524 if (ret < 0) 4525 return NULL; 4526 cp_mreg.src = ret; 4527 /* Provide the full width of FLAG specific value. */ 4528 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4529 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4530 /* Build a new flow. */ 4531 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4532 items[0] = (struct rte_flow_item){ 4533 .type = (enum rte_flow_item_type) 4534 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4535 .spec = &tag_spec, 4536 }; 4537 items[1] = (struct rte_flow_item){ 4538 .type = RTE_FLOW_ITEM_TYPE_END, 4539 }; 4540 actions[0] = (struct rte_flow_action){ 4541 .type = (enum rte_flow_action_type) 4542 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4543 .conf = &ftag, 4544 }; 4545 actions[1] = (struct rte_flow_action){ 4546 .type = (enum rte_flow_action_type) 4547 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4548 .conf = &cp_mreg, 4549 }; 4550 actions[2] = (struct rte_flow_action){ 4551 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4552 .conf = &jump, 4553 }; 4554 actions[3] = (struct rte_flow_action){ 4555 .type = RTE_FLOW_ACTION_TYPE_END, 4556 }; 4557 } else { 4558 /* Default rule, wildcard match. */ 4559 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4560 items[0] = (struct rte_flow_item){ 4561 .type = RTE_FLOW_ITEM_TYPE_END, 4562 }; 4563 actions[0] = (struct rte_flow_action){ 4564 .type = (enum rte_flow_action_type) 4565 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4566 .conf = &cp_mreg, 4567 }; 4568 actions[1] = (struct rte_flow_action){ 4569 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4570 .conf = &jump, 4571 }; 4572 actions[2] = (struct rte_flow_action){ 4573 .type = RTE_FLOW_ACTION_TYPE_END, 4574 }; 4575 } 4576 /* Build a new entry. */ 4577 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4578 if (!mcp_res) { 4579 rte_errno = ENOMEM; 4580 return NULL; 4581 } 4582 mcp_res->idx = idx; 4583 mcp_res->mark_id = mark_id; 4584 /* 4585 * The copy Flows are not included in any list. There 4586 * ones are referenced from other Flows and can not 4587 * be applied, removed, deleted in arbitrary order 4588 * by list traversing. 4589 */ 4590 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4591 &attr, items, actions, false, error); 4592 if (!mcp_res->rix_flow) { 4593 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4594 return NULL; 4595 } 4596 return &mcp_res->hlist_ent; 4597 } 4598 4599 struct mlx5_list_entry * 4600 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4601 void *cb_ctx __rte_unused) 4602 { 4603 struct rte_eth_dev *dev = tool_ctx; 4604 struct mlx5_priv *priv = dev->data->dev_private; 4605 struct mlx5_flow_mreg_copy_resource *mcp_res; 4606 uint32_t idx = 0; 4607 4608 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4609 if (!mcp_res) { 4610 rte_errno = ENOMEM; 4611 return NULL; 4612 } 4613 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4614 mcp_res->idx = idx; 4615 return &mcp_res->hlist_ent; 4616 } 4617 4618 void 4619 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4620 { 4621 struct mlx5_flow_mreg_copy_resource *mcp_res = 4622 container_of(entry, typeof(*mcp_res), hlist_ent); 4623 struct rte_eth_dev *dev = tool_ctx; 4624 struct mlx5_priv *priv = dev->data->dev_private; 4625 4626 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4627 } 4628 4629 /** 4630 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4631 * 4632 * As mark_id is unique, if there's already a registered flow for the mark_id, 4633 * return by increasing the reference counter of the resource. Otherwise, create 4634 * the resource (mcp_res) and flow. 4635 * 4636 * Flow looks like, 4637 * - If ingress port is ANY and reg_c[1] is mark_id, 4638 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4639 * 4640 * For default flow (zero mark_id), flow is like, 4641 * - If ingress port is ANY, 4642 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4643 * 4644 * @param dev 4645 * Pointer to Ethernet device. 4646 * @param mark_id 4647 * ID of MARK action, zero means default flow for META. 4648 * @param[out] error 4649 * Perform verbose error reporting if not NULL. 4650 * 4651 * @return 4652 * Associated resource on success, NULL otherwise and rte_errno is set. 4653 */ 4654 static struct mlx5_flow_mreg_copy_resource * 4655 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4656 struct rte_flow_error *error) 4657 { 4658 struct mlx5_priv *priv = dev->data->dev_private; 4659 struct mlx5_list_entry *entry; 4660 struct mlx5_flow_cb_ctx ctx = { 4661 .dev = dev, 4662 .error = error, 4663 .data = &mark_id, 4664 }; 4665 4666 /* Check if already registered. */ 4667 MLX5_ASSERT(priv->mreg_cp_tbl); 4668 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4669 if (!entry) 4670 return NULL; 4671 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4672 hlist_ent); 4673 } 4674 4675 void 4676 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4677 { 4678 struct mlx5_flow_mreg_copy_resource *mcp_res = 4679 container_of(entry, typeof(*mcp_res), hlist_ent); 4680 struct rte_eth_dev *dev = tool_ctx; 4681 struct mlx5_priv *priv = dev->data->dev_private; 4682 4683 MLX5_ASSERT(mcp_res->rix_flow); 4684 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4685 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4686 } 4687 4688 /** 4689 * Release flow in RX_CP_TBL. 4690 * 4691 * @param dev 4692 * Pointer to Ethernet device. 4693 * @flow 4694 * Parent flow for wich copying is provided. 4695 */ 4696 static void 4697 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4698 struct rte_flow *flow) 4699 { 4700 struct mlx5_flow_mreg_copy_resource *mcp_res; 4701 struct mlx5_priv *priv = dev->data->dev_private; 4702 4703 if (!flow->rix_mreg_copy) 4704 return; 4705 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4706 flow->rix_mreg_copy); 4707 if (!mcp_res || !priv->mreg_cp_tbl) 4708 return; 4709 MLX5_ASSERT(mcp_res->rix_flow); 4710 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4711 flow->rix_mreg_copy = 0; 4712 } 4713 4714 /** 4715 * Remove the default copy action from RX_CP_TBL. 4716 * 4717 * This functions is called in the mlx5_dev_start(). No thread safe 4718 * is guaranteed. 4719 * 4720 * @param dev 4721 * Pointer to Ethernet device. 4722 */ 4723 static void 4724 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4725 { 4726 struct mlx5_list_entry *entry; 4727 struct mlx5_priv *priv = dev->data->dev_private; 4728 struct mlx5_flow_cb_ctx ctx; 4729 uint32_t mark_id; 4730 4731 /* Check if default flow is registered. */ 4732 if (!priv->mreg_cp_tbl) 4733 return; 4734 mark_id = MLX5_DEFAULT_COPY_ID; 4735 ctx.data = &mark_id; 4736 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4737 if (!entry) 4738 return; 4739 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4740 } 4741 4742 /** 4743 * Add the default copy action in in RX_CP_TBL. 4744 * 4745 * This functions is called in the mlx5_dev_start(). No thread safe 4746 * is guaranteed. 4747 * 4748 * @param dev 4749 * Pointer to Ethernet device. 4750 * @param[out] error 4751 * Perform verbose error reporting if not NULL. 4752 * 4753 * @return 4754 * 0 for success, negative value otherwise and rte_errno is set. 4755 */ 4756 static int 4757 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4758 struct rte_flow_error *error) 4759 { 4760 struct mlx5_priv *priv = dev->data->dev_private; 4761 struct mlx5_flow_mreg_copy_resource *mcp_res; 4762 struct mlx5_flow_cb_ctx ctx; 4763 uint32_t mark_id; 4764 4765 /* Check whether extensive metadata feature is engaged. */ 4766 if (!priv->sh->config.dv_flow_en || 4767 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4768 !mlx5_flow_ext_mreg_supported(dev) || 4769 !priv->sh->dv_regc0_mask) 4770 return 0; 4771 /* 4772 * Add default mreg copy flow may be called multiple time, but 4773 * only be called once in stop. Avoid register it twice. 4774 */ 4775 mark_id = MLX5_DEFAULT_COPY_ID; 4776 ctx.data = &mark_id; 4777 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4778 return 0; 4779 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4780 if (!mcp_res) 4781 return -rte_errno; 4782 return 0; 4783 } 4784 4785 /** 4786 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4787 * 4788 * All the flow having Q/RSS action should be split by 4789 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4790 * performs the following, 4791 * - CQE->flow_tag := reg_c[1] (MARK) 4792 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4793 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4794 * but there should be a flow per each MARK ID set by MARK action. 4795 * 4796 * For the aforementioned reason, if there's a MARK action in flow's action 4797 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4798 * the MARK ID to CQE's flow_tag like, 4799 * - If reg_c[1] is mark_id, 4800 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4801 * 4802 * For SET_META action which stores value in reg_c[0], as the destination is 4803 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4804 * MARK ID means the default flow. The default flow looks like, 4805 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4806 * 4807 * @param dev 4808 * Pointer to Ethernet device. 4809 * @param flow 4810 * Pointer to flow structure. 4811 * @param[in] actions 4812 * Pointer to the list of actions. 4813 * @param[out] error 4814 * Perform verbose error reporting if not NULL. 4815 * 4816 * @return 4817 * 0 on success, negative value otherwise and rte_errno is set. 4818 */ 4819 static int 4820 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4821 struct rte_flow *flow, 4822 const struct rte_flow_action *actions, 4823 struct rte_flow_error *error) 4824 { 4825 struct mlx5_priv *priv = dev->data->dev_private; 4826 struct mlx5_sh_config *config = &priv->sh->config; 4827 struct mlx5_flow_mreg_copy_resource *mcp_res; 4828 const struct rte_flow_action_mark *mark; 4829 4830 /* Check whether extensive metadata feature is engaged. */ 4831 if (!config->dv_flow_en || 4832 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4833 !mlx5_flow_ext_mreg_supported(dev) || 4834 !priv->sh->dv_regc0_mask) 4835 return 0; 4836 /* Find MARK action. */ 4837 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4838 switch (actions->type) { 4839 case RTE_FLOW_ACTION_TYPE_FLAG: 4840 mcp_res = flow_mreg_add_copy_action 4841 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4842 if (!mcp_res) 4843 return -rte_errno; 4844 flow->rix_mreg_copy = mcp_res->idx; 4845 return 0; 4846 case RTE_FLOW_ACTION_TYPE_MARK: 4847 mark = (const struct rte_flow_action_mark *) 4848 actions->conf; 4849 mcp_res = 4850 flow_mreg_add_copy_action(dev, mark->id, error); 4851 if (!mcp_res) 4852 return -rte_errno; 4853 flow->rix_mreg_copy = mcp_res->idx; 4854 return 0; 4855 default: 4856 break; 4857 } 4858 } 4859 return 0; 4860 } 4861 4862 #define MLX5_MAX_SPLIT_ACTIONS 24 4863 #define MLX5_MAX_SPLIT_ITEMS 24 4864 4865 /** 4866 * Split the hairpin flow. 4867 * Since HW can't support encap and push-vlan on Rx, we move these 4868 * actions to Tx. 4869 * If the count action is after the encap then we also 4870 * move the count action. in this case the count will also measure 4871 * the outer bytes. 4872 * 4873 * @param dev 4874 * Pointer to Ethernet device. 4875 * @param[in] actions 4876 * Associated actions (list terminated by the END action). 4877 * @param[out] actions_rx 4878 * Rx flow actions. 4879 * @param[out] actions_tx 4880 * Tx flow actions.. 4881 * @param[out] pattern_tx 4882 * The pattern items for the Tx flow. 4883 * @param[out] flow_id 4884 * The flow ID connected to this flow. 4885 * 4886 * @return 4887 * 0 on success. 4888 */ 4889 static int 4890 flow_hairpin_split(struct rte_eth_dev *dev, 4891 const struct rte_flow_action actions[], 4892 struct rte_flow_action actions_rx[], 4893 struct rte_flow_action actions_tx[], 4894 struct rte_flow_item pattern_tx[], 4895 uint32_t flow_id) 4896 { 4897 const struct rte_flow_action_raw_encap *raw_encap; 4898 const struct rte_flow_action_raw_decap *raw_decap; 4899 struct mlx5_rte_flow_action_set_tag *set_tag; 4900 struct rte_flow_action *tag_action; 4901 struct mlx5_rte_flow_item_tag *tag_item; 4902 struct rte_flow_item *item; 4903 char *addr; 4904 int encap = 0; 4905 4906 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4907 switch (actions->type) { 4908 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4909 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4910 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4911 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4912 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4913 rte_memcpy(actions_tx, actions, 4914 sizeof(struct rte_flow_action)); 4915 actions_tx++; 4916 break; 4917 case RTE_FLOW_ACTION_TYPE_COUNT: 4918 if (encap) { 4919 rte_memcpy(actions_tx, actions, 4920 sizeof(struct rte_flow_action)); 4921 actions_tx++; 4922 } else { 4923 rte_memcpy(actions_rx, actions, 4924 sizeof(struct rte_flow_action)); 4925 actions_rx++; 4926 } 4927 break; 4928 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4929 raw_encap = actions->conf; 4930 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4931 memcpy(actions_tx, actions, 4932 sizeof(struct rte_flow_action)); 4933 actions_tx++; 4934 encap = 1; 4935 } else { 4936 rte_memcpy(actions_rx, actions, 4937 sizeof(struct rte_flow_action)); 4938 actions_rx++; 4939 } 4940 break; 4941 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4942 raw_decap = actions->conf; 4943 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4944 memcpy(actions_tx, actions, 4945 sizeof(struct rte_flow_action)); 4946 actions_tx++; 4947 } else { 4948 rte_memcpy(actions_rx, actions, 4949 sizeof(struct rte_flow_action)); 4950 actions_rx++; 4951 } 4952 break; 4953 default: 4954 rte_memcpy(actions_rx, actions, 4955 sizeof(struct rte_flow_action)); 4956 actions_rx++; 4957 break; 4958 } 4959 } 4960 /* Add set meta action and end action for the Rx flow. */ 4961 tag_action = actions_rx; 4962 tag_action->type = (enum rte_flow_action_type) 4963 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4964 actions_rx++; 4965 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4966 actions_rx++; 4967 set_tag = (void *)actions_rx; 4968 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4969 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4970 .data = flow_id, 4971 }; 4972 MLX5_ASSERT(set_tag->id > REG_NON); 4973 tag_action->conf = set_tag; 4974 /* Create Tx item list. */ 4975 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4976 addr = (void *)&pattern_tx[2]; 4977 item = pattern_tx; 4978 item->type = (enum rte_flow_item_type) 4979 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4980 tag_item = (void *)addr; 4981 tag_item->data = flow_id; 4982 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4983 MLX5_ASSERT(set_tag->id > REG_NON); 4984 item->spec = tag_item; 4985 addr += sizeof(struct mlx5_rte_flow_item_tag); 4986 tag_item = (void *)addr; 4987 tag_item->data = UINT32_MAX; 4988 tag_item->id = UINT16_MAX; 4989 item->mask = tag_item; 4990 item->last = NULL; 4991 item++; 4992 item->type = RTE_FLOW_ITEM_TYPE_END; 4993 return 0; 4994 } 4995 4996 /** 4997 * The last stage of splitting chain, just creates the subflow 4998 * without any modification. 4999 * 5000 * @param[in] dev 5001 * Pointer to Ethernet device. 5002 * @param[in] flow 5003 * Parent flow structure pointer. 5004 * @param[in, out] sub_flow 5005 * Pointer to return the created subflow, may be NULL. 5006 * @param[in] attr 5007 * Flow rule attributes. 5008 * @param[in] items 5009 * Pattern specification (list terminated by the END pattern item). 5010 * @param[in] actions 5011 * Associated actions (list terminated by the END action). 5012 * @param[in] flow_split_info 5013 * Pointer to flow split info structure. 5014 * @param[out] error 5015 * Perform verbose error reporting if not NULL. 5016 * @return 5017 * 0 on success, negative value otherwise 5018 */ 5019 static int 5020 flow_create_split_inner(struct rte_eth_dev *dev, 5021 struct rte_flow *flow, 5022 struct mlx5_flow **sub_flow, 5023 const struct rte_flow_attr *attr, 5024 const struct rte_flow_item items[], 5025 const struct rte_flow_action actions[], 5026 struct mlx5_flow_split_info *flow_split_info, 5027 struct rte_flow_error *error) 5028 { 5029 struct mlx5_flow *dev_flow; 5030 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5031 5032 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5033 flow_split_info->flow_idx, error); 5034 if (!dev_flow) 5035 return -rte_errno; 5036 dev_flow->flow = flow; 5037 dev_flow->external = flow_split_info->external; 5038 dev_flow->skip_scale = flow_split_info->skip_scale; 5039 /* Subflow object was created, we must include one in the list. */ 5040 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5041 dev_flow->handle, next); 5042 /* 5043 * If dev_flow is as one of the suffix flow, some actions in suffix 5044 * flow may need some user defined item layer flags, and pass the 5045 * Metadata rxq mark flag to suffix flow as well. 5046 */ 5047 if (flow_split_info->prefix_layers) 5048 dev_flow->handle->layers = flow_split_info->prefix_layers; 5049 if (flow_split_info->prefix_mark) { 5050 MLX5_ASSERT(wks); 5051 wks->mark = 1; 5052 } 5053 if (sub_flow) 5054 *sub_flow = dev_flow; 5055 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5056 dev_flow->dv.table_id = flow_split_info->table_id; 5057 #endif 5058 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5059 } 5060 5061 /** 5062 * Get the sub policy of a meter. 5063 * 5064 * @param[in] dev 5065 * Pointer to Ethernet device. 5066 * @param[in] flow 5067 * Parent flow structure pointer. 5068 * @param wks 5069 * Pointer to thread flow work space. 5070 * @param[in] attr 5071 * Flow rule attributes. 5072 * @param[in] items 5073 * Pattern specification (list terminated by the END pattern item). 5074 * @param[out] error 5075 * Perform verbose error reporting if not NULL. 5076 * 5077 * @return 5078 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5079 */ 5080 static struct mlx5_flow_meter_sub_policy * 5081 get_meter_sub_policy(struct rte_eth_dev *dev, 5082 struct rte_flow *flow, 5083 struct mlx5_flow_workspace *wks, 5084 const struct rte_flow_attr *attr, 5085 const struct rte_flow_item items[], 5086 struct rte_flow_error *error) 5087 { 5088 struct mlx5_flow_meter_policy *policy; 5089 struct mlx5_flow_meter_policy *final_policy; 5090 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5091 5092 policy = wks->policy; 5093 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5094 if (final_policy->is_rss || final_policy->is_queue) { 5095 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5096 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5097 uint32_t i; 5098 5099 /* 5100 * This is a tmp dev_flow, 5101 * no need to register any matcher for it in translate. 5102 */ 5103 wks->skip_matcher_reg = 1; 5104 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5105 struct mlx5_flow dev_flow = {0}; 5106 struct mlx5_flow_handle dev_handle = { {0} }; 5107 uint8_t fate = final_policy->act_cnt[i].fate_action; 5108 5109 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5110 const struct rte_flow_action_rss *rss_act = 5111 final_policy->act_cnt[i].rss->conf; 5112 struct rte_flow_action rss_actions[2] = { 5113 [0] = { 5114 .type = RTE_FLOW_ACTION_TYPE_RSS, 5115 .conf = rss_act, 5116 }, 5117 [1] = { 5118 .type = RTE_FLOW_ACTION_TYPE_END, 5119 .conf = NULL, 5120 } 5121 }; 5122 5123 dev_flow.handle = &dev_handle; 5124 dev_flow.ingress = attr->ingress; 5125 dev_flow.flow = flow; 5126 dev_flow.external = 0; 5127 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5128 dev_flow.dv.transfer = attr->transfer; 5129 #endif 5130 /** 5131 * Translate RSS action to get rss hash fields. 5132 */ 5133 if (flow_drv_translate(dev, &dev_flow, attr, 5134 items, rss_actions, error)) 5135 goto exit; 5136 rss_desc_v[i] = wks->rss_desc; 5137 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5138 rss_desc_v[i].hash_fields = 5139 dev_flow.hash_fields; 5140 rss_desc_v[i].queue_num = 5141 rss_desc_v[i].hash_fields ? 5142 rss_desc_v[i].queue_num : 1; 5143 rss_desc_v[i].tunnel = 5144 !!(dev_flow.handle->layers & 5145 MLX5_FLOW_LAYER_TUNNEL); 5146 /* Use the RSS queues in the containers. */ 5147 rss_desc_v[i].queue = 5148 (uint16_t *)(uintptr_t)rss_act->queue; 5149 rss_desc[i] = &rss_desc_v[i]; 5150 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5151 /* This is queue action. */ 5152 rss_desc_v[i] = wks->rss_desc; 5153 rss_desc_v[i].key_len = 0; 5154 rss_desc_v[i].hash_fields = 0; 5155 rss_desc_v[i].queue = 5156 &final_policy->act_cnt[i].queue; 5157 rss_desc_v[i].queue_num = 1; 5158 rss_desc[i] = &rss_desc_v[i]; 5159 } else { 5160 rss_desc[i] = NULL; 5161 } 5162 } 5163 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5164 flow, policy, rss_desc); 5165 } else { 5166 enum mlx5_meter_domain mtr_domain = 5167 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5168 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5169 MLX5_MTR_DOMAIN_INGRESS); 5170 sub_policy = policy->sub_policys[mtr_domain][0]; 5171 } 5172 if (!sub_policy) 5173 rte_flow_error_set(error, EINVAL, 5174 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5175 "Failed to get meter sub-policy."); 5176 exit: 5177 return sub_policy; 5178 } 5179 5180 /** 5181 * Split the meter flow. 5182 * 5183 * As meter flow will split to three sub flow, other than meter 5184 * action, the other actions make sense to only meter accepts 5185 * the packet. If it need to be dropped, no other additional 5186 * actions should be take. 5187 * 5188 * One kind of special action which decapsulates the L3 tunnel 5189 * header will be in the prefix sub flow, as not to take the 5190 * L3 tunnel header into account. 5191 * 5192 * @param[in] dev 5193 * Pointer to Ethernet device. 5194 * @param[in] flow 5195 * Parent flow structure pointer. 5196 * @param wks 5197 * Pointer to thread flow work space. 5198 * @param[in] attr 5199 * Flow rule attributes. 5200 * @param[in] items 5201 * Pattern specification (list terminated by the END pattern item). 5202 * @param[out] sfx_items 5203 * Suffix flow match items (list terminated by the END pattern item). 5204 * @param[in] actions 5205 * Associated actions (list terminated by the END action). 5206 * @param[out] actions_sfx 5207 * Suffix flow actions. 5208 * @param[out] actions_pre 5209 * Prefix flow actions. 5210 * @param[out] mtr_flow_id 5211 * Pointer to meter flow id. 5212 * @param[out] error 5213 * Perform verbose error reporting if not NULL. 5214 * 5215 * @return 5216 * 0 on success, a negative errno value otherwise and rte_errno is set. 5217 */ 5218 static int 5219 flow_meter_split_prep(struct rte_eth_dev *dev, 5220 struct rte_flow *flow, 5221 struct mlx5_flow_workspace *wks, 5222 const struct rte_flow_attr *attr, 5223 const struct rte_flow_item items[], 5224 struct rte_flow_item sfx_items[], 5225 const struct rte_flow_action actions[], 5226 struct rte_flow_action actions_sfx[], 5227 struct rte_flow_action actions_pre[], 5228 uint32_t *mtr_flow_id, 5229 struct rte_flow_error *error) 5230 { 5231 struct mlx5_priv *priv = dev->data->dev_private; 5232 struct mlx5_flow_meter_info *fm = wks->fm; 5233 struct rte_flow_action *tag_action = NULL; 5234 struct rte_flow_item *tag_item; 5235 struct mlx5_rte_flow_action_set_tag *set_tag; 5236 const struct rte_flow_action_raw_encap *raw_encap; 5237 const struct rte_flow_action_raw_decap *raw_decap; 5238 struct mlx5_rte_flow_item_tag *tag_item_spec; 5239 struct mlx5_rte_flow_item_tag *tag_item_mask; 5240 uint32_t tag_id = 0; 5241 struct rte_flow_item *vlan_item_dst = NULL; 5242 const struct rte_flow_item *vlan_item_src = NULL; 5243 const struct rte_flow_item *orig_items = items; 5244 struct rte_flow_action *hw_mtr_action; 5245 struct rte_flow_action *action_pre_head = NULL; 5246 int32_t flow_src_port = priv->representor_id; 5247 bool mtr_first; 5248 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5249 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5250 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5251 uint32_t flow_id = 0; 5252 uint32_t flow_id_reversed = 0; 5253 uint8_t flow_id_bits = 0; 5254 int shift; 5255 5256 /* Prepare the suffix subflow items. */ 5257 tag_item = sfx_items++; 5258 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5259 struct mlx5_priv *port_priv; 5260 const struct rte_flow_item_port_id *pid_v; 5261 int item_type = items->type; 5262 5263 switch (item_type) { 5264 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5265 pid_v = items->spec; 5266 MLX5_ASSERT(pid_v); 5267 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5268 if (!port_priv) 5269 return rte_flow_error_set(error, 5270 rte_errno, 5271 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5272 pid_v, 5273 "Failed to get port info."); 5274 flow_src_port = port_priv->representor_id; 5275 if (!fm->def_policy && wks->policy->is_hierarchy && 5276 flow_src_port != priv->representor_id) { 5277 if (flow_drv_mtr_hierarchy_rule_create(dev, 5278 flow, fm, 5279 flow_src_port, 5280 items, 5281 error)) 5282 return -rte_errno; 5283 } 5284 memcpy(sfx_items, items, sizeof(*sfx_items)); 5285 sfx_items++; 5286 break; 5287 case RTE_FLOW_ITEM_TYPE_VLAN: 5288 /* Determine if copy vlan item below. */ 5289 vlan_item_src = items; 5290 vlan_item_dst = sfx_items++; 5291 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5292 break; 5293 default: 5294 break; 5295 } 5296 } 5297 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5298 sfx_items++; 5299 mtr_first = priv->sh->meter_aso_en && 5300 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5301 /* For ASO meter, meter must be before tag in TX direction. */ 5302 if (mtr_first) { 5303 action_pre_head = actions_pre++; 5304 /* Leave space for tag action. */ 5305 tag_action = actions_pre++; 5306 } 5307 /* Prepare the actions for prefix and suffix flow. */ 5308 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5309 struct rte_flow_action *action_cur = NULL; 5310 5311 switch (actions->type) { 5312 case RTE_FLOW_ACTION_TYPE_METER: 5313 if (mtr_first) { 5314 action_cur = action_pre_head; 5315 } else { 5316 /* Leave space for tag action. */ 5317 tag_action = actions_pre++; 5318 action_cur = actions_pre++; 5319 } 5320 break; 5321 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5322 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5323 action_cur = actions_pre++; 5324 break; 5325 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5326 raw_encap = actions->conf; 5327 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5328 action_cur = actions_pre++; 5329 break; 5330 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5331 raw_decap = actions->conf; 5332 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5333 action_cur = actions_pre++; 5334 break; 5335 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5336 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5337 if (vlan_item_dst && vlan_item_src) { 5338 memcpy(vlan_item_dst, vlan_item_src, 5339 sizeof(*vlan_item_dst)); 5340 /* 5341 * Convert to internal match item, it is used 5342 * for vlan push and set vid. 5343 */ 5344 vlan_item_dst->type = (enum rte_flow_item_type) 5345 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5346 } 5347 break; 5348 default: 5349 break; 5350 } 5351 if (!action_cur) 5352 action_cur = (fm->def_policy) ? 5353 actions_sfx++ : actions_pre++; 5354 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5355 } 5356 /* Add end action to the actions. */ 5357 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5358 if (priv->sh->meter_aso_en) { 5359 /** 5360 * For ASO meter, need to add an extra jump action explicitly, 5361 * to jump from meter to policer table. 5362 */ 5363 struct mlx5_flow_meter_sub_policy *sub_policy; 5364 struct mlx5_flow_tbl_data_entry *tbl_data; 5365 5366 if (!fm->def_policy) { 5367 sub_policy = get_meter_sub_policy(dev, flow, wks, 5368 attr, orig_items, 5369 error); 5370 if (!sub_policy) 5371 return -rte_errno; 5372 } else { 5373 enum mlx5_meter_domain mtr_domain = 5374 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5375 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5376 MLX5_MTR_DOMAIN_INGRESS); 5377 5378 sub_policy = 5379 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5380 } 5381 tbl_data = container_of(sub_policy->tbl_rsc, 5382 struct mlx5_flow_tbl_data_entry, tbl); 5383 hw_mtr_action = actions_pre++; 5384 hw_mtr_action->type = (enum rte_flow_action_type) 5385 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5386 hw_mtr_action->conf = tbl_data->jump.action; 5387 } 5388 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5389 actions_pre++; 5390 if (!tag_action) 5391 return rte_flow_error_set(error, ENOMEM, 5392 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5393 NULL, "No tag action space."); 5394 if (!mtr_flow_id) { 5395 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5396 goto exit; 5397 } 5398 /* Only default-policy Meter creates mtr flow id. */ 5399 if (fm->def_policy) { 5400 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5401 if (!tag_id) 5402 return rte_flow_error_set(error, ENOMEM, 5403 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5404 "Failed to allocate meter flow id."); 5405 flow_id = tag_id - 1; 5406 flow_id_bits = (!flow_id) ? 1 : 5407 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5408 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5409 mtr_reg_bits) { 5410 mlx5_ipool_free(fm->flow_ipool, tag_id); 5411 return rte_flow_error_set(error, EINVAL, 5412 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5413 "Meter flow id exceeds max limit."); 5414 } 5415 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5416 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5417 } 5418 /* Build tag actions and items for meter_id/meter flow_id. */ 5419 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5420 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5421 tag_item_mask = tag_item_spec + 1; 5422 /* Both flow_id and meter_id share the same register. */ 5423 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5424 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5425 0, error), 5426 .offset = mtr_id_offset, 5427 .length = mtr_reg_bits, 5428 .data = flow->meter, 5429 }; 5430 /* 5431 * The color Reg bits used by flow_id are growing from 5432 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5433 */ 5434 for (shift = 0; shift < flow_id_bits; shift++) 5435 flow_id_reversed = (flow_id_reversed << 1) | 5436 ((flow_id >> shift) & 0x1); 5437 set_tag->data |= 5438 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5439 tag_item_spec->id = set_tag->id; 5440 tag_item_spec->data = set_tag->data << mtr_id_offset; 5441 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5442 tag_action->type = (enum rte_flow_action_type) 5443 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5444 tag_action->conf = set_tag; 5445 tag_item->type = (enum rte_flow_item_type) 5446 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5447 tag_item->spec = tag_item_spec; 5448 tag_item->last = NULL; 5449 tag_item->mask = tag_item_mask; 5450 exit: 5451 if (mtr_flow_id) 5452 *mtr_flow_id = tag_id; 5453 return 0; 5454 } 5455 5456 /** 5457 * Split action list having QUEUE/RSS for metadata register copy. 5458 * 5459 * Once Q/RSS action is detected in user's action list, the flow action 5460 * should be split in order to copy metadata registers, which will happen in 5461 * RX_CP_TBL like, 5462 * - CQE->flow_tag := reg_c[1] (MARK) 5463 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5464 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5465 * This is because the last action of each flow must be a terminal action 5466 * (QUEUE, RSS or DROP). 5467 * 5468 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5469 * stored and kept in the mlx5_flow structure per each sub_flow. 5470 * 5471 * The Q/RSS action is replaced with, 5472 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5473 * And the following JUMP action is added at the end, 5474 * - JUMP, to RX_CP_TBL. 5475 * 5476 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5477 * flow_create_split_metadata() routine. The flow will look like, 5478 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5479 * 5480 * @param dev 5481 * Pointer to Ethernet device. 5482 * @param[out] split_actions 5483 * Pointer to store split actions to jump to CP_TBL. 5484 * @param[in] actions 5485 * Pointer to the list of original flow actions. 5486 * @param[in] qrss 5487 * Pointer to the Q/RSS action. 5488 * @param[in] actions_n 5489 * Number of original actions. 5490 * @param[in] mtr_sfx 5491 * Check if it is in meter suffix table. 5492 * @param[out] error 5493 * Perform verbose error reporting if not NULL. 5494 * 5495 * @return 5496 * non-zero unique flow_id on success, otherwise 0 and 5497 * error/rte_error are set. 5498 */ 5499 static uint32_t 5500 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5501 struct rte_flow_action *split_actions, 5502 const struct rte_flow_action *actions, 5503 const struct rte_flow_action *qrss, 5504 int actions_n, int mtr_sfx, 5505 struct rte_flow_error *error) 5506 { 5507 struct mlx5_priv *priv = dev->data->dev_private; 5508 struct mlx5_rte_flow_action_set_tag *set_tag; 5509 struct rte_flow_action_jump *jump; 5510 const int qrss_idx = qrss - actions; 5511 uint32_t flow_id = 0; 5512 int ret = 0; 5513 5514 /* 5515 * Given actions will be split 5516 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5517 * - Add jump to mreg CP_TBL. 5518 * As a result, there will be one more action. 5519 */ 5520 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5521 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5522 ++actions_n; 5523 set_tag = (void *)(split_actions + actions_n); 5524 /* 5525 * If we are not the meter suffix flow, add the tag action. 5526 * Since meter suffix flow already has the tag added. 5527 */ 5528 if (!mtr_sfx) { 5529 /* 5530 * Allocate the new subflow ID. This one is unique within 5531 * device and not shared with representors. Otherwise, 5532 * we would have to resolve multi-thread access synch 5533 * issue. Each flow on the shared device is appended 5534 * with source vport identifier, so the resulting 5535 * flows will be unique in the shared (by master and 5536 * representors) domain even if they have coinciding 5537 * IDs. 5538 */ 5539 mlx5_ipool_malloc(priv->sh->ipool 5540 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5541 if (!flow_id) 5542 return rte_flow_error_set(error, ENOMEM, 5543 RTE_FLOW_ERROR_TYPE_ACTION, 5544 NULL, "can't allocate id " 5545 "for split Q/RSS subflow"); 5546 /* Internal SET_TAG action to set flow ID. */ 5547 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5548 .data = flow_id, 5549 }; 5550 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5551 if (ret < 0) 5552 return ret; 5553 set_tag->id = ret; 5554 /* Construct new actions array. */ 5555 /* Replace QUEUE/RSS action. */ 5556 split_actions[qrss_idx] = (struct rte_flow_action){ 5557 .type = (enum rte_flow_action_type) 5558 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5559 .conf = set_tag, 5560 }; 5561 } else { 5562 /* 5563 * If we are the suffix flow of meter, tag already exist. 5564 * Set the QUEUE/RSS action to void. 5565 */ 5566 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5567 } 5568 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5569 jump = (void *)(set_tag + 1); 5570 *jump = (struct rte_flow_action_jump){ 5571 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5572 }; 5573 split_actions[actions_n - 2] = (struct rte_flow_action){ 5574 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5575 .conf = jump, 5576 }; 5577 split_actions[actions_n - 1] = (struct rte_flow_action){ 5578 .type = RTE_FLOW_ACTION_TYPE_END, 5579 }; 5580 return flow_id; 5581 } 5582 5583 /** 5584 * Extend the given action list for Tx metadata copy. 5585 * 5586 * Copy the given action list to the ext_actions and add flow metadata register 5587 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5588 * 5589 * @param[out] ext_actions 5590 * Pointer to the extended action list. 5591 * @param[in] actions 5592 * Pointer to the list of actions. 5593 * @param[in] actions_n 5594 * Number of actions in the list. 5595 * @param[out] error 5596 * Perform verbose error reporting if not NULL. 5597 * @param[in] encap_idx 5598 * The encap action index. 5599 * 5600 * @return 5601 * 0 on success, negative value otherwise 5602 */ 5603 static int 5604 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5605 struct rte_flow_action *ext_actions, 5606 const struct rte_flow_action *actions, 5607 int actions_n, struct rte_flow_error *error, 5608 int encap_idx) 5609 { 5610 struct mlx5_flow_action_copy_mreg *cp_mreg = 5611 (struct mlx5_flow_action_copy_mreg *) 5612 (ext_actions + actions_n + 1); 5613 int ret; 5614 5615 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5616 if (ret < 0) 5617 return ret; 5618 cp_mreg->dst = ret; 5619 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5620 if (ret < 0) 5621 return ret; 5622 cp_mreg->src = ret; 5623 if (encap_idx != 0) 5624 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5625 if (encap_idx == actions_n - 1) { 5626 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5627 .type = (enum rte_flow_action_type) 5628 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5629 .conf = cp_mreg, 5630 }; 5631 ext_actions[actions_n] = (struct rte_flow_action){ 5632 .type = RTE_FLOW_ACTION_TYPE_END, 5633 }; 5634 } else { 5635 ext_actions[encap_idx] = (struct rte_flow_action){ 5636 .type = (enum rte_flow_action_type) 5637 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5638 .conf = cp_mreg, 5639 }; 5640 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5641 sizeof(*ext_actions) * (actions_n - encap_idx)); 5642 } 5643 return 0; 5644 } 5645 5646 /** 5647 * Check the match action from the action list. 5648 * 5649 * @param[in] actions 5650 * Pointer to the list of actions. 5651 * @param[in] attr 5652 * Flow rule attributes. 5653 * @param[in] action 5654 * The action to be check if exist. 5655 * @param[out] match_action_pos 5656 * Pointer to the position of the matched action if exists, otherwise is -1. 5657 * @param[out] qrss_action_pos 5658 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5659 * @param[out] modify_after_mirror 5660 * Pointer to the flag of modify action after FDB mirroring. 5661 * 5662 * @return 5663 * > 0 the total number of actions. 5664 * 0 if not found match action in action list. 5665 */ 5666 static int 5667 flow_check_match_action(const struct rte_flow_action actions[], 5668 const struct rte_flow_attr *attr, 5669 enum rte_flow_action_type action, 5670 int *match_action_pos, int *qrss_action_pos, 5671 int *modify_after_mirror) 5672 { 5673 const struct rte_flow_action_sample *sample; 5674 const struct rte_flow_action_raw_decap *decap; 5675 int actions_n = 0; 5676 uint32_t ratio = 0; 5677 int sub_type = 0; 5678 int flag = 0; 5679 int fdb_mirror = 0; 5680 5681 *match_action_pos = -1; 5682 *qrss_action_pos = -1; 5683 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5684 if (actions->type == action) { 5685 flag = 1; 5686 *match_action_pos = actions_n; 5687 } 5688 switch (actions->type) { 5689 case RTE_FLOW_ACTION_TYPE_QUEUE: 5690 case RTE_FLOW_ACTION_TYPE_RSS: 5691 *qrss_action_pos = actions_n; 5692 break; 5693 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5694 sample = actions->conf; 5695 ratio = sample->ratio; 5696 sub_type = ((const struct rte_flow_action *) 5697 (sample->actions))->type; 5698 if (ratio == 1 && attr->transfer) 5699 fdb_mirror = 1; 5700 break; 5701 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5702 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5703 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5704 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5705 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5706 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5707 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5708 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5709 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5710 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5711 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5712 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5713 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5714 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5715 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5716 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5717 case RTE_FLOW_ACTION_TYPE_FLAG: 5718 case RTE_FLOW_ACTION_TYPE_MARK: 5719 case RTE_FLOW_ACTION_TYPE_SET_META: 5720 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5721 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5722 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5723 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5724 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5725 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5726 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5727 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5728 case RTE_FLOW_ACTION_TYPE_METER: 5729 if (fdb_mirror) 5730 *modify_after_mirror = 1; 5731 break; 5732 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5733 decap = actions->conf; 5734 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5735 ; 5736 actions_n++; 5737 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5738 const struct rte_flow_action_raw_encap *encap = 5739 actions->conf; 5740 if (decap->size <= 5741 MLX5_ENCAPSULATION_DECISION_SIZE && 5742 encap->size > 5743 MLX5_ENCAPSULATION_DECISION_SIZE) 5744 /* L3 encap. */ 5745 break; 5746 } 5747 if (fdb_mirror) 5748 *modify_after_mirror = 1; 5749 break; 5750 default: 5751 break; 5752 } 5753 actions_n++; 5754 } 5755 if (flag && fdb_mirror && !*modify_after_mirror) { 5756 /* FDB mirroring uses the destination array to implement 5757 * instead of FLOW_SAMPLER object. 5758 */ 5759 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5760 flag = 0; 5761 } 5762 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5763 return flag ? actions_n + 1 : 0; 5764 } 5765 5766 #define SAMPLE_SUFFIX_ITEM 2 5767 5768 /** 5769 * Split the sample flow. 5770 * 5771 * As sample flow will split to two sub flow, sample flow with 5772 * sample action, the other actions will move to new suffix flow. 5773 * 5774 * Also add unique tag id with tag action in the sample flow, 5775 * the same tag id will be as match in the suffix flow. 5776 * 5777 * @param dev 5778 * Pointer to Ethernet device. 5779 * @param[in] add_tag 5780 * Add extra tag action flag. 5781 * @param[out] sfx_items 5782 * Suffix flow match items (list terminated by the END pattern item). 5783 * @param[in] actions 5784 * Associated actions (list terminated by the END action). 5785 * @param[out] actions_sfx 5786 * Suffix flow actions. 5787 * @param[out] actions_pre 5788 * Prefix flow actions. 5789 * @param[in] actions_n 5790 * The total number of actions. 5791 * @param[in] sample_action_pos 5792 * The sample action position. 5793 * @param[in] qrss_action_pos 5794 * The Queue/RSS action position. 5795 * @param[in] jump_table 5796 * Add extra jump action flag. 5797 * @param[out] error 5798 * Perform verbose error reporting if not NULL. 5799 * 5800 * @return 5801 * 0 on success, or unique flow_id, a negative errno value 5802 * otherwise and rte_errno is set. 5803 */ 5804 static int 5805 flow_sample_split_prep(struct rte_eth_dev *dev, 5806 int add_tag, 5807 struct rte_flow_item sfx_items[], 5808 const struct rte_flow_action actions[], 5809 struct rte_flow_action actions_sfx[], 5810 struct rte_flow_action actions_pre[], 5811 int actions_n, 5812 int sample_action_pos, 5813 int qrss_action_pos, 5814 int jump_table, 5815 struct rte_flow_error *error) 5816 { 5817 struct mlx5_priv *priv = dev->data->dev_private; 5818 struct mlx5_rte_flow_action_set_tag *set_tag; 5819 struct mlx5_rte_flow_item_tag *tag_spec; 5820 struct mlx5_rte_flow_item_tag *tag_mask; 5821 struct rte_flow_action_jump *jump_action; 5822 uint32_t tag_id = 0; 5823 int index; 5824 int append_index = 0; 5825 int ret; 5826 5827 if (sample_action_pos < 0) 5828 return rte_flow_error_set(error, EINVAL, 5829 RTE_FLOW_ERROR_TYPE_ACTION, 5830 NULL, "invalid position of sample " 5831 "action in list"); 5832 /* Prepare the actions for prefix and suffix flow. */ 5833 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5834 index = qrss_action_pos; 5835 /* Put the preceding the Queue/RSS action into prefix flow. */ 5836 if (index != 0) 5837 memcpy(actions_pre, actions, 5838 sizeof(struct rte_flow_action) * index); 5839 /* Put others preceding the sample action into prefix flow. */ 5840 if (sample_action_pos > index + 1) 5841 memcpy(actions_pre + index, actions + index + 1, 5842 sizeof(struct rte_flow_action) * 5843 (sample_action_pos - index - 1)); 5844 index = sample_action_pos - 1; 5845 /* Put Queue/RSS action into Suffix flow. */ 5846 memcpy(actions_sfx, actions + qrss_action_pos, 5847 sizeof(struct rte_flow_action)); 5848 actions_sfx++; 5849 } else { 5850 index = sample_action_pos; 5851 if (index != 0) 5852 memcpy(actions_pre, actions, 5853 sizeof(struct rte_flow_action) * index); 5854 } 5855 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5856 * For CX6DX and above, metadata registers Cx preserve their value, 5857 * add an extra tag action for NIC-RX and E-Switch Domain. 5858 */ 5859 if (add_tag) { 5860 /* Prepare the prefix tag action. */ 5861 append_index++; 5862 set_tag = (void *)(actions_pre + actions_n + append_index); 5863 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5864 if (ret < 0) 5865 return ret; 5866 mlx5_ipool_malloc(priv->sh->ipool 5867 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5868 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5869 .id = ret, 5870 .data = tag_id, 5871 }; 5872 /* Prepare the suffix subflow items. */ 5873 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5874 tag_spec->data = tag_id; 5875 tag_spec->id = set_tag->id; 5876 tag_mask = tag_spec + 1; 5877 tag_mask->data = UINT32_MAX; 5878 sfx_items[0] = (struct rte_flow_item){ 5879 .type = (enum rte_flow_item_type) 5880 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5881 .spec = tag_spec, 5882 .last = NULL, 5883 .mask = tag_mask, 5884 }; 5885 sfx_items[1] = (struct rte_flow_item){ 5886 .type = (enum rte_flow_item_type) 5887 RTE_FLOW_ITEM_TYPE_END, 5888 }; 5889 /* Prepare the tag action in prefix subflow. */ 5890 actions_pre[index++] = 5891 (struct rte_flow_action){ 5892 .type = (enum rte_flow_action_type) 5893 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5894 .conf = set_tag, 5895 }; 5896 } 5897 memcpy(actions_pre + index, actions + sample_action_pos, 5898 sizeof(struct rte_flow_action)); 5899 index += 1; 5900 /* For the modify action after the sample action in E-Switch mirroring, 5901 * Add the extra jump action in prefix subflow and jump into the next 5902 * table, then do the modify action in the new table. 5903 */ 5904 if (jump_table) { 5905 /* Prepare the prefix jump action. */ 5906 append_index++; 5907 jump_action = (void *)(actions_pre + actions_n + append_index); 5908 jump_action->group = jump_table; 5909 actions_pre[index++] = 5910 (struct rte_flow_action){ 5911 .type = (enum rte_flow_action_type) 5912 RTE_FLOW_ACTION_TYPE_JUMP, 5913 .conf = jump_action, 5914 }; 5915 } 5916 actions_pre[index] = (struct rte_flow_action){ 5917 .type = (enum rte_flow_action_type) 5918 RTE_FLOW_ACTION_TYPE_END, 5919 }; 5920 /* Put the actions after sample into Suffix flow. */ 5921 memcpy(actions_sfx, actions + sample_action_pos + 1, 5922 sizeof(struct rte_flow_action) * 5923 (actions_n - sample_action_pos - 1)); 5924 return tag_id; 5925 } 5926 5927 /** 5928 * The splitting for metadata feature. 5929 * 5930 * - Q/RSS action on NIC Rx should be split in order to pass by 5931 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5932 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5933 * 5934 * - All the actions on NIC Tx should have a mreg copy action to 5935 * copy reg_a from WQE to reg_c[0]. 5936 * 5937 * @param dev 5938 * Pointer to Ethernet device. 5939 * @param[in] flow 5940 * Parent flow structure pointer. 5941 * @param[in] attr 5942 * Flow rule attributes. 5943 * @param[in] items 5944 * Pattern specification (list terminated by the END pattern item). 5945 * @param[in] actions 5946 * Associated actions (list terminated by the END action). 5947 * @param[in] flow_split_info 5948 * Pointer to flow split info structure. 5949 * @param[out] error 5950 * Perform verbose error reporting if not NULL. 5951 * @return 5952 * 0 on success, negative value otherwise 5953 */ 5954 static int 5955 flow_create_split_metadata(struct rte_eth_dev *dev, 5956 struct rte_flow *flow, 5957 const struct rte_flow_attr *attr, 5958 const struct rte_flow_item items[], 5959 const struct rte_flow_action actions[], 5960 struct mlx5_flow_split_info *flow_split_info, 5961 struct rte_flow_error *error) 5962 { 5963 struct mlx5_priv *priv = dev->data->dev_private; 5964 struct mlx5_sh_config *config = &priv->sh->config; 5965 const struct rte_flow_action *qrss = NULL; 5966 struct rte_flow_action *ext_actions = NULL; 5967 struct mlx5_flow *dev_flow = NULL; 5968 uint32_t qrss_id = 0; 5969 int mtr_sfx = 0; 5970 size_t act_size; 5971 int actions_n; 5972 int encap_idx; 5973 int ret; 5974 5975 /* Check whether extensive metadata feature is engaged. */ 5976 if (!config->dv_flow_en || 5977 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5978 !mlx5_flow_ext_mreg_supported(dev)) 5979 return flow_create_split_inner(dev, flow, NULL, attr, items, 5980 actions, flow_split_info, error); 5981 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5982 &encap_idx); 5983 if (qrss) { 5984 /* Exclude hairpin flows from splitting. */ 5985 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5986 const struct rte_flow_action_queue *queue; 5987 5988 queue = qrss->conf; 5989 if (mlx5_rxq_get_type(dev, queue->index) == 5990 MLX5_RXQ_TYPE_HAIRPIN) 5991 qrss = NULL; 5992 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5993 const struct rte_flow_action_rss *rss; 5994 5995 rss = qrss->conf; 5996 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5997 MLX5_RXQ_TYPE_HAIRPIN) 5998 qrss = NULL; 5999 } 6000 } 6001 if (qrss) { 6002 /* Check if it is in meter suffix table. */ 6003 mtr_sfx = attr->group == (attr->transfer ? 6004 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6005 MLX5_FLOW_TABLE_LEVEL_METER); 6006 /* 6007 * Q/RSS action on NIC Rx should be split in order to pass by 6008 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6009 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6010 */ 6011 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6012 sizeof(struct rte_flow_action_set_tag) + 6013 sizeof(struct rte_flow_action_jump); 6014 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6015 SOCKET_ID_ANY); 6016 if (!ext_actions) 6017 return rte_flow_error_set(error, ENOMEM, 6018 RTE_FLOW_ERROR_TYPE_ACTION, 6019 NULL, "no memory to split " 6020 "metadata flow"); 6021 /* 6022 * Create the new actions list with removed Q/RSS action 6023 * and appended set tag and jump to register copy table 6024 * (RX_CP_TBL). We should preallocate unique tag ID here 6025 * in advance, because it is needed for set tag action. 6026 */ 6027 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6028 qrss, actions_n, 6029 mtr_sfx, error); 6030 if (!mtr_sfx && !qrss_id) { 6031 ret = -rte_errno; 6032 goto exit; 6033 } 6034 } else if (attr->egress && !attr->transfer) { 6035 /* 6036 * All the actions on NIC Tx should have a metadata register 6037 * copy action to copy reg_a from WQE to reg_c[meta] 6038 */ 6039 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6040 sizeof(struct mlx5_flow_action_copy_mreg); 6041 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6042 SOCKET_ID_ANY); 6043 if (!ext_actions) 6044 return rte_flow_error_set(error, ENOMEM, 6045 RTE_FLOW_ERROR_TYPE_ACTION, 6046 NULL, "no memory to split " 6047 "metadata flow"); 6048 /* Create the action list appended with copy register. */ 6049 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6050 actions_n, error, encap_idx); 6051 if (ret < 0) 6052 goto exit; 6053 } 6054 /* Add the unmodified original or prefix subflow. */ 6055 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6056 items, ext_actions ? ext_actions : 6057 actions, flow_split_info, error); 6058 if (ret < 0) 6059 goto exit; 6060 MLX5_ASSERT(dev_flow); 6061 if (qrss) { 6062 const struct rte_flow_attr q_attr = { 6063 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6064 .ingress = 1, 6065 }; 6066 /* Internal PMD action to set register. */ 6067 struct mlx5_rte_flow_item_tag q_tag_spec = { 6068 .data = qrss_id, 6069 .id = REG_NON, 6070 }; 6071 struct rte_flow_item q_items[] = { 6072 { 6073 .type = (enum rte_flow_item_type) 6074 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6075 .spec = &q_tag_spec, 6076 .last = NULL, 6077 .mask = NULL, 6078 }, 6079 { 6080 .type = RTE_FLOW_ITEM_TYPE_END, 6081 }, 6082 }; 6083 struct rte_flow_action q_actions[] = { 6084 { 6085 .type = qrss->type, 6086 .conf = qrss->conf, 6087 }, 6088 { 6089 .type = RTE_FLOW_ACTION_TYPE_END, 6090 }, 6091 }; 6092 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6093 6094 /* 6095 * Configure the tag item only if there is no meter subflow. 6096 * Since tag is already marked in the meter suffix subflow 6097 * we can just use the meter suffix items as is. 6098 */ 6099 if (qrss_id) { 6100 /* Not meter subflow. */ 6101 MLX5_ASSERT(!mtr_sfx); 6102 /* 6103 * Put unique id in prefix flow due to it is destroyed 6104 * after suffix flow and id will be freed after there 6105 * is no actual flows with this id and identifier 6106 * reallocation becomes possible (for example, for 6107 * other flows in other threads). 6108 */ 6109 dev_flow->handle->split_flow_id = qrss_id; 6110 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6111 error); 6112 if (ret < 0) 6113 goto exit; 6114 q_tag_spec.id = ret; 6115 } 6116 dev_flow = NULL; 6117 /* Add suffix subflow to execute Q/RSS. */ 6118 flow_split_info->prefix_layers = layers; 6119 flow_split_info->prefix_mark = 0; 6120 flow_split_info->table_id = 0; 6121 ret = flow_create_split_inner(dev, flow, &dev_flow, 6122 &q_attr, mtr_sfx ? items : 6123 q_items, q_actions, 6124 flow_split_info, error); 6125 if (ret < 0) 6126 goto exit; 6127 /* qrss ID should be freed if failed. */ 6128 qrss_id = 0; 6129 MLX5_ASSERT(dev_flow); 6130 } 6131 6132 exit: 6133 /* 6134 * We do not destroy the partially created sub_flows in case of error. 6135 * These ones are included into parent flow list and will be destroyed 6136 * by flow_drv_destroy. 6137 */ 6138 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6139 qrss_id); 6140 mlx5_free(ext_actions); 6141 return ret; 6142 } 6143 6144 /** 6145 * Create meter internal drop flow with the original pattern. 6146 * 6147 * @param dev 6148 * Pointer to Ethernet device. 6149 * @param[in] flow 6150 * Parent flow structure pointer. 6151 * @param[in] attr 6152 * Flow rule attributes. 6153 * @param[in] items 6154 * Pattern specification (list terminated by the END pattern item). 6155 * @param[in] flow_split_info 6156 * Pointer to flow split info structure. 6157 * @param[in] fm 6158 * Pointer to flow meter structure. 6159 * @param[out] error 6160 * Perform verbose error reporting if not NULL. 6161 * @return 6162 * 0 on success, negative value otherwise 6163 */ 6164 static uint32_t 6165 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6166 struct rte_flow *flow, 6167 const struct rte_flow_attr *attr, 6168 const struct rte_flow_item items[], 6169 struct mlx5_flow_split_info *flow_split_info, 6170 struct mlx5_flow_meter_info *fm, 6171 struct rte_flow_error *error) 6172 { 6173 struct mlx5_flow *dev_flow = NULL; 6174 struct rte_flow_attr drop_attr = *attr; 6175 struct rte_flow_action drop_actions[3]; 6176 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6177 6178 MLX5_ASSERT(fm->drop_cnt); 6179 drop_actions[0].type = 6180 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6181 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6182 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6183 drop_actions[1].conf = NULL; 6184 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6185 drop_actions[2].conf = NULL; 6186 drop_split_info.external = false; 6187 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6188 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6189 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6190 return flow_create_split_inner(dev, flow, &dev_flow, 6191 &drop_attr, items, drop_actions, 6192 &drop_split_info, error); 6193 } 6194 6195 /** 6196 * The splitting for meter feature. 6197 * 6198 * - The meter flow will be split to two flows as prefix and 6199 * suffix flow. The packets make sense only it pass the prefix 6200 * meter action. 6201 * 6202 * - Reg_C_5 is used for the packet to match betweend prefix and 6203 * suffix flow. 6204 * 6205 * @param dev 6206 * Pointer to Ethernet device. 6207 * @param[in] flow 6208 * Parent flow structure pointer. 6209 * @param[in] attr 6210 * Flow rule attributes. 6211 * @param[in] items 6212 * Pattern specification (list terminated by the END pattern item). 6213 * @param[in] actions 6214 * Associated actions (list terminated by the END action). 6215 * @param[in] flow_split_info 6216 * Pointer to flow split info structure. 6217 * @param[out] error 6218 * Perform verbose error reporting if not NULL. 6219 * @return 6220 * 0 on success, negative value otherwise 6221 */ 6222 static int 6223 flow_create_split_meter(struct rte_eth_dev *dev, 6224 struct rte_flow *flow, 6225 const struct rte_flow_attr *attr, 6226 const struct rte_flow_item items[], 6227 const struct rte_flow_action actions[], 6228 struct mlx5_flow_split_info *flow_split_info, 6229 struct rte_flow_error *error) 6230 { 6231 struct mlx5_priv *priv = dev->data->dev_private; 6232 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6233 struct rte_flow_action *sfx_actions = NULL; 6234 struct rte_flow_action *pre_actions = NULL; 6235 struct rte_flow_item *sfx_items = NULL; 6236 struct mlx5_flow *dev_flow = NULL; 6237 struct rte_flow_attr sfx_attr = *attr; 6238 struct mlx5_flow_meter_info *fm = NULL; 6239 uint8_t skip_scale_restore; 6240 bool has_mtr = false; 6241 bool has_modify = false; 6242 bool set_mtr_reg = true; 6243 bool is_mtr_hierarchy = false; 6244 uint32_t meter_id = 0; 6245 uint32_t mtr_idx = 0; 6246 uint32_t mtr_flow_id = 0; 6247 size_t act_size; 6248 size_t item_size; 6249 int actions_n = 0; 6250 int ret = 0; 6251 6252 if (priv->mtr_en) 6253 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6254 &has_modify, &meter_id); 6255 if (has_mtr) { 6256 if (flow->meter) { 6257 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6258 if (!fm) 6259 return rte_flow_error_set(error, EINVAL, 6260 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6261 NULL, "Meter not found."); 6262 } else { 6263 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6264 if (!fm) 6265 return rte_flow_error_set(error, EINVAL, 6266 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6267 NULL, "Meter not found."); 6268 ret = mlx5_flow_meter_attach(priv, fm, 6269 &sfx_attr, error); 6270 if (ret) 6271 return -rte_errno; 6272 flow->meter = mtr_idx; 6273 } 6274 MLX5_ASSERT(wks); 6275 wks->fm = fm; 6276 if (!fm->def_policy) { 6277 wks->policy = mlx5_flow_meter_policy_find(dev, 6278 fm->policy_id, 6279 NULL); 6280 MLX5_ASSERT(wks->policy); 6281 if (wks->policy->is_hierarchy) { 6282 wks->final_policy = 6283 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6284 wks->policy); 6285 if (!wks->final_policy) 6286 return rte_flow_error_set(error, 6287 EINVAL, 6288 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6289 "Failed to find terminal policy of hierarchy."); 6290 is_mtr_hierarchy = true; 6291 } 6292 } 6293 /* 6294 * If it isn't default-policy Meter, and 6295 * 1. There's no action in flow to change 6296 * packet (modify/encap/decap etc.), OR 6297 * 2. No drop count needed for this meter. 6298 * 3. It's not meter hierarchy. 6299 * Then no need to use regC to save meter id anymore. 6300 */ 6301 if (!fm->def_policy && !is_mtr_hierarchy && 6302 (!has_modify || !fm->drop_cnt)) 6303 set_mtr_reg = false; 6304 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6305 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6306 sizeof(struct mlx5_rte_flow_action_set_tag); 6307 /* Suffix items: tag, vlan, port id, end. */ 6308 #define METER_SUFFIX_ITEM 4 6309 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6310 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6311 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6312 0, SOCKET_ID_ANY); 6313 if (!sfx_actions) 6314 return rte_flow_error_set(error, ENOMEM, 6315 RTE_FLOW_ERROR_TYPE_ACTION, 6316 NULL, "no memory to split " 6317 "meter flow"); 6318 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6319 act_size); 6320 /* There's no suffix flow for meter of non-default policy. */ 6321 if (!fm->def_policy) 6322 pre_actions = sfx_actions + 1; 6323 else 6324 pre_actions = sfx_actions + actions_n; 6325 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6326 items, sfx_items, actions, 6327 sfx_actions, pre_actions, 6328 (set_mtr_reg ? &mtr_flow_id : NULL), 6329 error); 6330 if (ret) { 6331 ret = -rte_errno; 6332 goto exit; 6333 } 6334 /* Add the prefix subflow. */ 6335 skip_scale_restore = flow_split_info->skip_scale; 6336 flow_split_info->skip_scale |= 6337 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6338 ret = flow_create_split_inner(dev, flow, &dev_flow, 6339 attr, items, pre_actions, 6340 flow_split_info, error); 6341 flow_split_info->skip_scale = skip_scale_restore; 6342 if (ret) { 6343 if (mtr_flow_id) 6344 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6345 ret = -rte_errno; 6346 goto exit; 6347 } 6348 if (mtr_flow_id) { 6349 dev_flow->handle->split_flow_id = mtr_flow_id; 6350 dev_flow->handle->is_meter_flow_id = 1; 6351 } 6352 if (!fm->def_policy) { 6353 if (!set_mtr_reg && fm->drop_cnt) 6354 ret = 6355 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6356 &sfx_attr, items, 6357 flow_split_info, 6358 fm, error); 6359 goto exit; 6360 } 6361 /* Setting the sfx group atrr. */ 6362 sfx_attr.group = sfx_attr.transfer ? 6363 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6364 MLX5_FLOW_TABLE_LEVEL_METER; 6365 flow_split_info->prefix_layers = 6366 flow_get_prefix_layer_flags(dev_flow); 6367 flow_split_info->prefix_mark |= wks->mark; 6368 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6369 } 6370 /* Add the prefix subflow. */ 6371 ret = flow_create_split_metadata(dev, flow, 6372 &sfx_attr, sfx_items ? 6373 sfx_items : items, 6374 sfx_actions ? sfx_actions : actions, 6375 flow_split_info, error); 6376 exit: 6377 if (sfx_actions) 6378 mlx5_free(sfx_actions); 6379 return ret; 6380 } 6381 6382 /** 6383 * The splitting for sample feature. 6384 * 6385 * Once Sample action is detected in the action list, the flow actions should 6386 * be split into prefix sub flow and suffix sub flow. 6387 * 6388 * The original items remain in the prefix sub flow, all actions preceding the 6389 * sample action and the sample action itself will be copied to the prefix 6390 * sub flow, the actions following the sample action will be copied to the 6391 * suffix sub flow, Queue action always be located in the suffix sub flow. 6392 * 6393 * In order to make the packet from prefix sub flow matches with suffix sub 6394 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6395 * flow uses tag item with the unique flow id. 6396 * 6397 * @param dev 6398 * Pointer to Ethernet device. 6399 * @param[in] flow 6400 * Parent flow structure pointer. 6401 * @param[in] attr 6402 * Flow rule attributes. 6403 * @param[in] items 6404 * Pattern specification (list terminated by the END pattern item). 6405 * @param[in] actions 6406 * Associated actions (list terminated by the END action). 6407 * @param[in] flow_split_info 6408 * Pointer to flow split info structure. 6409 * @param[out] error 6410 * Perform verbose error reporting if not NULL. 6411 * @return 6412 * 0 on success, negative value otherwise 6413 */ 6414 static int 6415 flow_create_split_sample(struct rte_eth_dev *dev, 6416 struct rte_flow *flow, 6417 const struct rte_flow_attr *attr, 6418 const struct rte_flow_item items[], 6419 const struct rte_flow_action actions[], 6420 struct mlx5_flow_split_info *flow_split_info, 6421 struct rte_flow_error *error) 6422 { 6423 struct mlx5_priv *priv = dev->data->dev_private; 6424 struct rte_flow_action *sfx_actions = NULL; 6425 struct rte_flow_action *pre_actions = NULL; 6426 struct rte_flow_item *sfx_items = NULL; 6427 struct mlx5_flow *dev_flow = NULL; 6428 struct rte_flow_attr sfx_attr = *attr; 6429 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6430 struct mlx5_flow_dv_sample_resource *sample_res; 6431 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6432 struct mlx5_flow_tbl_resource *sfx_tbl; 6433 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6434 #endif 6435 size_t act_size; 6436 size_t item_size; 6437 uint32_t fdb_tx = 0; 6438 int32_t tag_id = 0; 6439 int actions_n = 0; 6440 int sample_action_pos; 6441 int qrss_action_pos; 6442 int add_tag = 0; 6443 int modify_after_mirror = 0; 6444 uint16_t jump_table = 0; 6445 const uint32_t next_ft_step = 1; 6446 int ret = 0; 6447 6448 if (priv->sampler_en) 6449 actions_n = flow_check_match_action(actions, attr, 6450 RTE_FLOW_ACTION_TYPE_SAMPLE, 6451 &sample_action_pos, &qrss_action_pos, 6452 &modify_after_mirror); 6453 if (actions_n) { 6454 /* The prefix actions must includes sample, tag, end. */ 6455 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6456 + sizeof(struct mlx5_rte_flow_action_set_tag); 6457 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6458 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6459 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6460 item_size), 0, SOCKET_ID_ANY); 6461 if (!sfx_actions) 6462 return rte_flow_error_set(error, ENOMEM, 6463 RTE_FLOW_ERROR_TYPE_ACTION, 6464 NULL, "no memory to split " 6465 "sample flow"); 6466 /* The representor_id is UINT16_MAX for uplink. */ 6467 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6468 /* 6469 * When reg_c_preserve is set, metadata registers Cx preserve 6470 * their value even through packet duplication. 6471 */ 6472 add_tag = (!fdb_tx || 6473 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6474 if (add_tag) 6475 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6476 + act_size); 6477 if (modify_after_mirror) 6478 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6479 next_ft_step; 6480 pre_actions = sfx_actions + actions_n; 6481 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6482 actions, sfx_actions, 6483 pre_actions, actions_n, 6484 sample_action_pos, 6485 qrss_action_pos, jump_table, 6486 error); 6487 if (tag_id < 0 || (add_tag && !tag_id)) { 6488 ret = -rte_errno; 6489 goto exit; 6490 } 6491 if (modify_after_mirror) 6492 flow_split_info->skip_scale = 6493 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6494 /* Add the prefix subflow. */ 6495 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6496 items, pre_actions, 6497 flow_split_info, error); 6498 if (ret) { 6499 ret = -rte_errno; 6500 goto exit; 6501 } 6502 dev_flow->handle->split_flow_id = tag_id; 6503 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6504 if (!modify_after_mirror) { 6505 /* Set the sfx group attr. */ 6506 sample_res = (struct mlx5_flow_dv_sample_resource *) 6507 dev_flow->dv.sample_res; 6508 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6509 sample_res->normal_path_tbl; 6510 sfx_tbl_data = container_of(sfx_tbl, 6511 struct mlx5_flow_tbl_data_entry, 6512 tbl); 6513 sfx_attr.group = sfx_attr.transfer ? 6514 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6515 } else { 6516 MLX5_ASSERT(attr->transfer); 6517 sfx_attr.group = jump_table; 6518 } 6519 flow_split_info->prefix_layers = 6520 flow_get_prefix_layer_flags(dev_flow); 6521 MLX5_ASSERT(wks); 6522 flow_split_info->prefix_mark |= wks->mark; 6523 /* Suffix group level already be scaled with factor, set 6524 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6525 * again in translation. 6526 */ 6527 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6528 #endif 6529 } 6530 /* Add the suffix subflow. */ 6531 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6532 sfx_items ? sfx_items : items, 6533 sfx_actions ? sfx_actions : actions, 6534 flow_split_info, error); 6535 exit: 6536 if (sfx_actions) 6537 mlx5_free(sfx_actions); 6538 return ret; 6539 } 6540 6541 /** 6542 * Split the flow to subflow set. The splitters might be linked 6543 * in the chain, like this: 6544 * flow_create_split_outer() calls: 6545 * flow_create_split_meter() calls: 6546 * flow_create_split_metadata(meter_subflow_0) calls: 6547 * flow_create_split_inner(metadata_subflow_0) 6548 * flow_create_split_inner(metadata_subflow_1) 6549 * flow_create_split_inner(metadata_subflow_2) 6550 * flow_create_split_metadata(meter_subflow_1) calls: 6551 * flow_create_split_inner(metadata_subflow_0) 6552 * flow_create_split_inner(metadata_subflow_1) 6553 * flow_create_split_inner(metadata_subflow_2) 6554 * 6555 * This provide flexible way to add new levels of flow splitting. 6556 * The all of successfully created subflows are included to the 6557 * parent flow dev_flow list. 6558 * 6559 * @param dev 6560 * Pointer to Ethernet device. 6561 * @param[in] flow 6562 * Parent flow structure pointer. 6563 * @param[in] attr 6564 * Flow rule attributes. 6565 * @param[in] items 6566 * Pattern specification (list terminated by the END pattern item). 6567 * @param[in] actions 6568 * Associated actions (list terminated by the END action). 6569 * @param[in] flow_split_info 6570 * Pointer to flow split info structure. 6571 * @param[out] error 6572 * Perform verbose error reporting if not NULL. 6573 * @return 6574 * 0 on success, negative value otherwise 6575 */ 6576 static int 6577 flow_create_split_outer(struct rte_eth_dev *dev, 6578 struct rte_flow *flow, 6579 const struct rte_flow_attr *attr, 6580 const struct rte_flow_item items[], 6581 const struct rte_flow_action actions[], 6582 struct mlx5_flow_split_info *flow_split_info, 6583 struct rte_flow_error *error) 6584 { 6585 int ret; 6586 6587 ret = flow_create_split_sample(dev, flow, attr, items, 6588 actions, flow_split_info, error); 6589 MLX5_ASSERT(ret <= 0); 6590 return ret; 6591 } 6592 6593 static inline struct mlx5_flow_tunnel * 6594 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6595 { 6596 struct mlx5_flow_tunnel *tunnel; 6597 6598 #pragma GCC diagnostic push 6599 #pragma GCC diagnostic ignored "-Wcast-qual" 6600 tunnel = (typeof(tunnel))flow->tunnel; 6601 #pragma GCC diagnostic pop 6602 6603 return tunnel; 6604 } 6605 6606 /** 6607 * Adjust flow RSS workspace if needed. 6608 * 6609 * @param wks 6610 * Pointer to thread flow work space. 6611 * @param rss_desc 6612 * Pointer to RSS descriptor. 6613 * @param[in] nrssq_num 6614 * New RSS queue number. 6615 * 6616 * @return 6617 * 0 on success, -1 otherwise and rte_errno is set. 6618 */ 6619 static int 6620 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6621 struct mlx5_flow_rss_desc *rss_desc, 6622 uint32_t nrssq_num) 6623 { 6624 if (likely(nrssq_num <= wks->rssq_num)) 6625 return 0; 6626 rss_desc->queue = realloc(rss_desc->queue, 6627 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6628 if (!rss_desc->queue) { 6629 rte_errno = ENOMEM; 6630 return -1; 6631 } 6632 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6633 return 0; 6634 } 6635 6636 /** 6637 * Create a flow and add it to @p list. 6638 * 6639 * @param dev 6640 * Pointer to Ethernet device. 6641 * @param list 6642 * Pointer to a TAILQ flow list. If this parameter NULL, 6643 * no list insertion occurred, flow is just created, 6644 * this is caller's responsibility to track the 6645 * created flow. 6646 * @param[in] attr 6647 * Flow rule attributes. 6648 * @param[in] items 6649 * Pattern specification (list terminated by the END pattern item). 6650 * @param[in] actions 6651 * Associated actions (list terminated by the END action). 6652 * @param[in] external 6653 * This flow rule is created by request external to PMD. 6654 * @param[out] error 6655 * Perform verbose error reporting if not NULL. 6656 * 6657 * @return 6658 * A flow index on success, 0 otherwise and rte_errno is set. 6659 */ 6660 static uint32_t 6661 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6662 const struct rte_flow_attr *attr, 6663 const struct rte_flow_item items[], 6664 const struct rte_flow_action original_actions[], 6665 bool external, struct rte_flow_error *error) 6666 { 6667 struct mlx5_priv *priv = dev->data->dev_private; 6668 struct rte_flow *flow = NULL; 6669 struct mlx5_flow *dev_flow; 6670 const struct rte_flow_action_rss *rss = NULL; 6671 struct mlx5_translated_action_handle 6672 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6673 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6674 union { 6675 struct mlx5_flow_expand_rss buf; 6676 uint8_t buffer[4096]; 6677 } expand_buffer; 6678 union { 6679 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6680 uint8_t buffer[2048]; 6681 } actions_rx; 6682 union { 6683 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6684 uint8_t buffer[2048]; 6685 } actions_hairpin_tx; 6686 union { 6687 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6688 uint8_t buffer[2048]; 6689 } items_tx; 6690 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6691 struct mlx5_flow_rss_desc *rss_desc; 6692 const struct rte_flow_action *p_actions_rx; 6693 uint32_t i; 6694 uint32_t idx = 0; 6695 int hairpin_flow; 6696 struct rte_flow_attr attr_tx = { .priority = 0 }; 6697 const struct rte_flow_action *actions; 6698 struct rte_flow_action *translated_actions = NULL; 6699 struct mlx5_flow_tunnel *tunnel; 6700 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6701 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6702 struct mlx5_flow_split_info flow_split_info = { 6703 .external = !!external, 6704 .skip_scale = 0, 6705 .flow_idx = 0, 6706 .prefix_mark = 0, 6707 .prefix_layers = 0, 6708 .table_id = 0 6709 }; 6710 int ret; 6711 6712 MLX5_ASSERT(wks); 6713 rss_desc = &wks->rss_desc; 6714 ret = flow_action_handles_translate(dev, original_actions, 6715 indir_actions, 6716 &indir_actions_n, 6717 &translated_actions, error); 6718 if (ret < 0) { 6719 MLX5_ASSERT(translated_actions == NULL); 6720 return 0; 6721 } 6722 actions = translated_actions ? translated_actions : original_actions; 6723 p_actions_rx = actions; 6724 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6725 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6726 external, hairpin_flow, error); 6727 if (ret < 0) 6728 goto error_before_hairpin_split; 6729 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6730 if (!flow) { 6731 rte_errno = ENOMEM; 6732 goto error_before_hairpin_split; 6733 } 6734 if (hairpin_flow > 0) { 6735 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6736 rte_errno = EINVAL; 6737 goto error_before_hairpin_split; 6738 } 6739 flow_hairpin_split(dev, actions, actions_rx.actions, 6740 actions_hairpin_tx.actions, items_tx.items, 6741 idx); 6742 p_actions_rx = actions_rx.actions; 6743 } 6744 flow_split_info.flow_idx = idx; 6745 flow->drv_type = flow_get_drv_type(dev, attr); 6746 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6747 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6748 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6749 /* RSS Action only works on NIC RX domain */ 6750 if (attr->ingress && !attr->transfer) 6751 rss = flow_get_rss_action(dev, p_actions_rx); 6752 if (rss) { 6753 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6754 return 0; 6755 /* 6756 * The following information is required by 6757 * mlx5_flow_hashfields_adjust() in advance. 6758 */ 6759 rss_desc->level = rss->level; 6760 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6761 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6762 } 6763 flow->dev_handles = 0; 6764 if (rss && rss->types) { 6765 unsigned int graph_root; 6766 6767 graph_root = find_graph_root(rss->level); 6768 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6769 items, rss->types, 6770 mlx5_support_expansion, graph_root); 6771 MLX5_ASSERT(ret > 0 && 6772 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6773 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6774 for (i = 0; i < buf->entries; ++i) 6775 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6776 } 6777 } else { 6778 buf->entries = 1; 6779 buf->entry[0].pattern = (void *)(uintptr_t)items; 6780 } 6781 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6782 indir_actions_n); 6783 for (i = 0; i < buf->entries; ++i) { 6784 /* Initialize flow split data. */ 6785 flow_split_info.prefix_layers = 0; 6786 flow_split_info.prefix_mark = 0; 6787 flow_split_info.skip_scale = 0; 6788 /* 6789 * The splitter may create multiple dev_flows, 6790 * depending on configuration. In the simplest 6791 * case it just creates unmodified original flow. 6792 */ 6793 ret = flow_create_split_outer(dev, flow, attr, 6794 buf->entry[i].pattern, 6795 p_actions_rx, &flow_split_info, 6796 error); 6797 if (ret < 0) 6798 goto error; 6799 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6800 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6801 p_actions_rx, 6802 idx, 6803 wks->flows[0].tunnel, 6804 &default_miss_ctx, 6805 error); 6806 if (ret < 0) { 6807 mlx5_free(default_miss_ctx.queue); 6808 goto error; 6809 } 6810 } 6811 } 6812 /* Create the tx flow. */ 6813 if (hairpin_flow) { 6814 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6815 attr_tx.ingress = 0; 6816 attr_tx.egress = 1; 6817 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6818 actions_hairpin_tx.actions, 6819 idx, error); 6820 if (!dev_flow) 6821 goto error; 6822 dev_flow->flow = flow; 6823 dev_flow->external = 0; 6824 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6825 dev_flow->handle, next); 6826 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6827 items_tx.items, 6828 actions_hairpin_tx.actions, error); 6829 if (ret < 0) 6830 goto error; 6831 } 6832 /* 6833 * Update the metadata register copy table. If extensive 6834 * metadata feature is enabled and registers are supported 6835 * we might create the extra rte_flow for each unique 6836 * MARK/FLAG action ID. 6837 * 6838 * The table is updated for ingress Flows only, because 6839 * the egress Flows belong to the different device and 6840 * copy table should be updated in peer NIC Rx domain. 6841 */ 6842 if (attr->ingress && 6843 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6844 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6845 if (ret) 6846 goto error; 6847 } 6848 /* 6849 * If the flow is external (from application) OR device is started, 6850 * OR mreg discover, then apply immediately. 6851 */ 6852 if (external || dev->data->dev_started || 6853 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6854 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6855 ret = flow_drv_apply(dev, flow, error); 6856 if (ret < 0) 6857 goto error; 6858 } 6859 flow->type = type; 6860 flow_rxq_flags_set(dev, flow); 6861 rte_free(translated_actions); 6862 tunnel = flow_tunnel_from_rule(wks->flows); 6863 if (tunnel) { 6864 flow->tunnel = 1; 6865 flow->tunnel_id = tunnel->tunnel_id; 6866 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6867 mlx5_free(default_miss_ctx.queue); 6868 } 6869 mlx5_flow_pop_thread_workspace(); 6870 return idx; 6871 error: 6872 MLX5_ASSERT(flow); 6873 ret = rte_errno; /* Save rte_errno before cleanup. */ 6874 flow_mreg_del_copy_action(dev, flow); 6875 flow_drv_destroy(dev, flow); 6876 if (rss_desc->shared_rss) 6877 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6878 mlx5_ipool_get 6879 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6880 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6881 mlx5_ipool_free(priv->flows[type], idx); 6882 rte_errno = ret; /* Restore rte_errno. */ 6883 ret = rte_errno; 6884 rte_errno = ret; 6885 mlx5_flow_pop_thread_workspace(); 6886 error_before_hairpin_split: 6887 rte_free(translated_actions); 6888 return 0; 6889 } 6890 6891 /** 6892 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6893 * incoming packets to table 1. 6894 * 6895 * Other flow rules, requested for group n, will be created in 6896 * e-switch table n+1. 6897 * Jump action to e-switch group n will be created to group n+1. 6898 * 6899 * Used when working in switchdev mode, to utilise advantages of table 1 6900 * and above. 6901 * 6902 * @param dev 6903 * Pointer to Ethernet device. 6904 * 6905 * @return 6906 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6907 */ 6908 struct rte_flow * 6909 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6910 { 6911 const struct rte_flow_attr attr = { 6912 .group = 0, 6913 .priority = 0, 6914 .ingress = 1, 6915 .egress = 0, 6916 .transfer = 1, 6917 }; 6918 const struct rte_flow_item pattern = { 6919 .type = RTE_FLOW_ITEM_TYPE_END, 6920 }; 6921 struct rte_flow_action_jump jump = { 6922 .group = 1, 6923 }; 6924 const struct rte_flow_action actions[] = { 6925 { 6926 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6927 .conf = &jump, 6928 }, 6929 { 6930 .type = RTE_FLOW_ACTION_TYPE_END, 6931 }, 6932 }; 6933 struct rte_flow_error error; 6934 6935 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6936 &attr, &pattern, 6937 actions, false, &error); 6938 } 6939 6940 /** 6941 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6942 * and sq number, directs all packets to peer vport. 6943 * 6944 * @param dev 6945 * Pointer to Ethernet device. 6946 * @param txq 6947 * Txq index. 6948 * 6949 * @return 6950 * Flow ID on success, 0 otherwise and rte_errno is set. 6951 */ 6952 uint32_t 6953 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6954 { 6955 struct rte_flow_attr attr = { 6956 .group = 0, 6957 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6958 .ingress = 1, 6959 .egress = 0, 6960 .transfer = 1, 6961 }; 6962 struct rte_flow_item_port_id port_spec = { 6963 .id = MLX5_PORT_ESW_MGR, 6964 }; 6965 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6966 .queue = txq, 6967 }; 6968 struct rte_flow_item pattern[] = { 6969 { 6970 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6971 .spec = &port_spec, 6972 }, 6973 { 6974 .type = (enum rte_flow_item_type) 6975 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6976 .spec = &txq_spec, 6977 }, 6978 { 6979 .type = RTE_FLOW_ITEM_TYPE_END, 6980 }, 6981 }; 6982 struct rte_flow_action_jump jump = { 6983 .group = 1, 6984 }; 6985 struct rte_flow_action_port_id port = { 6986 .id = dev->data->port_id, 6987 }; 6988 struct rte_flow_action actions[] = { 6989 { 6990 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6991 .conf = &jump, 6992 }, 6993 { 6994 .type = RTE_FLOW_ACTION_TYPE_END, 6995 }, 6996 }; 6997 struct rte_flow_error error; 6998 6999 /* 7000 * Creates group 0, highest priority jump flow. 7001 * Matches txq to bypass kernel packets. 7002 */ 7003 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7004 false, &error) == 0) 7005 return 0; 7006 /* Create group 1, lowest priority redirect flow for txq. */ 7007 attr.group = 1; 7008 actions[0].conf = &port; 7009 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7010 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7011 actions, false, &error); 7012 } 7013 7014 /** 7015 * Validate a flow supported by the NIC. 7016 * 7017 * @see rte_flow_validate() 7018 * @see rte_flow_ops 7019 */ 7020 int 7021 mlx5_flow_validate(struct rte_eth_dev *dev, 7022 const struct rte_flow_attr *attr, 7023 const struct rte_flow_item items[], 7024 const struct rte_flow_action original_actions[], 7025 struct rte_flow_error *error) 7026 { 7027 int hairpin_flow; 7028 struct mlx5_translated_action_handle 7029 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7030 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7031 const struct rte_flow_action *actions; 7032 struct rte_flow_action *translated_actions = NULL; 7033 int ret = flow_action_handles_translate(dev, original_actions, 7034 indir_actions, 7035 &indir_actions_n, 7036 &translated_actions, error); 7037 7038 if (ret) 7039 return ret; 7040 actions = translated_actions ? translated_actions : original_actions; 7041 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7042 ret = flow_drv_validate(dev, attr, items, actions, 7043 true, hairpin_flow, error); 7044 rte_free(translated_actions); 7045 return ret; 7046 } 7047 7048 /** 7049 * Create a flow. 7050 * 7051 * @see rte_flow_create() 7052 * @see rte_flow_ops 7053 */ 7054 struct rte_flow * 7055 mlx5_flow_create(struct rte_eth_dev *dev, 7056 const struct rte_flow_attr *attr, 7057 const struct rte_flow_item items[], 7058 const struct rte_flow_action actions[], 7059 struct rte_flow_error *error) 7060 { 7061 struct mlx5_priv *priv = dev->data->dev_private; 7062 7063 if (priv->sh->config.dv_flow_en == 2) { 7064 rte_flow_error_set(error, ENOTSUP, 7065 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7066 NULL, 7067 "Flow non-Q creation not supported"); 7068 return NULL; 7069 } 7070 /* 7071 * If the device is not started yet, it is not allowed to created a 7072 * flow from application. PMD default flows and traffic control flows 7073 * are not affected. 7074 */ 7075 if (unlikely(!dev->data->dev_started)) { 7076 DRV_LOG(DEBUG, "port %u is not started when " 7077 "inserting a flow", dev->data->port_id); 7078 rte_flow_error_set(error, ENODEV, 7079 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7080 NULL, 7081 "port not started"); 7082 return NULL; 7083 } 7084 7085 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7086 attr, items, actions, 7087 true, error); 7088 } 7089 7090 /** 7091 * Destroy a flow in a list. 7092 * 7093 * @param dev 7094 * Pointer to Ethernet device. 7095 * @param[in] flow_idx 7096 * Index of flow to destroy. 7097 */ 7098 static void 7099 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7100 uint32_t flow_idx) 7101 { 7102 struct mlx5_priv *priv = dev->data->dev_private; 7103 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7104 7105 if (!flow) 7106 return; 7107 MLX5_ASSERT(flow->type == type); 7108 /* 7109 * Update RX queue flags only if port is started, otherwise it is 7110 * already clean. 7111 */ 7112 if (dev->data->dev_started) 7113 flow_rxq_flags_trim(dev, flow); 7114 flow_drv_destroy(dev, flow); 7115 if (flow->tunnel) { 7116 struct mlx5_flow_tunnel *tunnel; 7117 7118 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7119 RTE_VERIFY(tunnel); 7120 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7121 mlx5_flow_tunnel_free(dev, tunnel); 7122 } 7123 flow_mreg_del_copy_action(dev, flow); 7124 mlx5_ipool_free(priv->flows[type], flow_idx); 7125 } 7126 7127 /** 7128 * Destroy all flows. 7129 * 7130 * @param dev 7131 * Pointer to Ethernet device. 7132 * @param type 7133 * Flow type to be flushed. 7134 * @param active 7135 * If flushing is called actively. 7136 */ 7137 void 7138 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7139 bool active) 7140 { 7141 struct mlx5_priv *priv = dev->data->dev_private; 7142 uint32_t num_flushed = 0, fidx = 1; 7143 struct rte_flow *flow; 7144 7145 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7146 if (priv->sh->config.dv_flow_en == 2 && 7147 type == MLX5_FLOW_TYPE_GEN) { 7148 flow_hw_q_flow_flush(dev, NULL); 7149 return; 7150 } 7151 #endif 7152 7153 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7154 flow_list_destroy(dev, type, fidx); 7155 num_flushed++; 7156 } 7157 if (active) { 7158 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7159 dev->data->port_id, num_flushed); 7160 } 7161 } 7162 7163 /** 7164 * Stop all default actions for flows. 7165 * 7166 * @param dev 7167 * Pointer to Ethernet device. 7168 */ 7169 void 7170 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7171 { 7172 flow_mreg_del_default_copy_action(dev); 7173 flow_rxq_flags_clear(dev); 7174 } 7175 7176 /** 7177 * Start all default actions for flows. 7178 * 7179 * @param dev 7180 * Pointer to Ethernet device. 7181 * @return 7182 * 0 on success, a negative errno value otherwise and rte_errno is set. 7183 */ 7184 int 7185 mlx5_flow_start_default(struct rte_eth_dev *dev) 7186 { 7187 struct rte_flow_error error; 7188 7189 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7190 return flow_mreg_add_default_copy_action(dev, &error); 7191 } 7192 7193 /** 7194 * Release key of thread specific flow workspace data. 7195 */ 7196 void 7197 flow_release_workspace(void *data) 7198 { 7199 struct mlx5_flow_workspace *wks = data; 7200 struct mlx5_flow_workspace *next; 7201 7202 while (wks) { 7203 next = wks->next; 7204 free(wks->rss_desc.queue); 7205 free(wks); 7206 wks = next; 7207 } 7208 } 7209 7210 /** 7211 * Get thread specific current flow workspace. 7212 * 7213 * @return pointer to thread specific flow workspace data, NULL on error. 7214 */ 7215 struct mlx5_flow_workspace* 7216 mlx5_flow_get_thread_workspace(void) 7217 { 7218 struct mlx5_flow_workspace *data; 7219 7220 data = mlx5_flow_os_get_specific_workspace(); 7221 MLX5_ASSERT(data && data->inuse); 7222 if (!data || !data->inuse) 7223 DRV_LOG(ERR, "flow workspace not initialized."); 7224 return data; 7225 } 7226 7227 /** 7228 * Allocate and init new flow workspace. 7229 * 7230 * @return pointer to flow workspace data, NULL on error. 7231 */ 7232 static struct mlx5_flow_workspace* 7233 flow_alloc_thread_workspace(void) 7234 { 7235 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7236 7237 if (!data) { 7238 DRV_LOG(ERR, "Failed to allocate flow workspace " 7239 "memory."); 7240 return NULL; 7241 } 7242 data->rss_desc.queue = calloc(1, 7243 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7244 if (!data->rss_desc.queue) 7245 goto err; 7246 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7247 return data; 7248 err: 7249 free(data->rss_desc.queue); 7250 free(data); 7251 return NULL; 7252 } 7253 7254 /** 7255 * Get new thread specific flow workspace. 7256 * 7257 * If current workspace inuse, create new one and set as current. 7258 * 7259 * @return pointer to thread specific flow workspace data, NULL on error. 7260 */ 7261 static struct mlx5_flow_workspace* 7262 mlx5_flow_push_thread_workspace(void) 7263 { 7264 struct mlx5_flow_workspace *curr; 7265 struct mlx5_flow_workspace *data; 7266 7267 curr = mlx5_flow_os_get_specific_workspace(); 7268 if (!curr) { 7269 data = flow_alloc_thread_workspace(); 7270 if (!data) 7271 return NULL; 7272 } else if (!curr->inuse) { 7273 data = curr; 7274 } else if (curr->next) { 7275 data = curr->next; 7276 } else { 7277 data = flow_alloc_thread_workspace(); 7278 if (!data) 7279 return NULL; 7280 curr->next = data; 7281 data->prev = curr; 7282 } 7283 data->inuse = 1; 7284 data->flow_idx = 0; 7285 /* Set as current workspace */ 7286 if (mlx5_flow_os_set_specific_workspace(data)) 7287 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7288 return data; 7289 } 7290 7291 /** 7292 * Close current thread specific flow workspace. 7293 * 7294 * If previous workspace available, set it as current. 7295 * 7296 * @return pointer to thread specific flow workspace data, NULL on error. 7297 */ 7298 static void 7299 mlx5_flow_pop_thread_workspace(void) 7300 { 7301 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7302 7303 if (!data) 7304 return; 7305 if (!data->inuse) { 7306 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7307 return; 7308 } 7309 data->inuse = 0; 7310 if (!data->prev) 7311 return; 7312 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7313 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7314 } 7315 7316 /** 7317 * Verify the flow list is empty 7318 * 7319 * @param dev 7320 * Pointer to Ethernet device. 7321 * 7322 * @return the number of flows not released. 7323 */ 7324 int 7325 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7326 { 7327 struct mlx5_priv *priv = dev->data->dev_private; 7328 struct rte_flow *flow; 7329 uint32_t idx = 0; 7330 int ret = 0, i; 7331 7332 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7333 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7334 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7335 dev->data->port_id, (void *)flow); 7336 ret++; 7337 } 7338 } 7339 return ret; 7340 } 7341 7342 /** 7343 * Enable default hairpin egress flow. 7344 * 7345 * @param dev 7346 * Pointer to Ethernet device. 7347 * @param queue 7348 * The queue index. 7349 * 7350 * @return 7351 * 0 on success, a negative errno value otherwise and rte_errno is set. 7352 */ 7353 int 7354 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7355 uint32_t queue) 7356 { 7357 const struct rte_flow_attr attr = { 7358 .egress = 1, 7359 .priority = 0, 7360 }; 7361 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7362 .queue = queue, 7363 }; 7364 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7365 .queue = UINT32_MAX, 7366 }; 7367 struct rte_flow_item items[] = { 7368 { 7369 .type = (enum rte_flow_item_type) 7370 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7371 .spec = &queue_spec, 7372 .last = NULL, 7373 .mask = &queue_mask, 7374 }, 7375 { 7376 .type = RTE_FLOW_ITEM_TYPE_END, 7377 }, 7378 }; 7379 struct rte_flow_action_jump jump = { 7380 .group = MLX5_HAIRPIN_TX_TABLE, 7381 }; 7382 struct rte_flow_action actions[2]; 7383 uint32_t flow_idx; 7384 struct rte_flow_error error; 7385 7386 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7387 actions[0].conf = &jump; 7388 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7389 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7390 &attr, items, actions, false, &error); 7391 if (!flow_idx) { 7392 DRV_LOG(DEBUG, 7393 "Failed to create ctrl flow: rte_errno(%d)," 7394 " type(%d), message(%s)", 7395 rte_errno, error.type, 7396 error.message ? error.message : " (no stated reason)"); 7397 return -rte_errno; 7398 } 7399 return 0; 7400 } 7401 7402 /** 7403 * Enable a control flow configured from the control plane. 7404 * 7405 * @param dev 7406 * Pointer to Ethernet device. 7407 * @param eth_spec 7408 * An Ethernet flow spec to apply. 7409 * @param eth_mask 7410 * An Ethernet flow mask to apply. 7411 * @param vlan_spec 7412 * A VLAN flow spec to apply. 7413 * @param vlan_mask 7414 * A VLAN flow mask to apply. 7415 * 7416 * @return 7417 * 0 on success, a negative errno value otherwise and rte_errno is set. 7418 */ 7419 int 7420 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7421 struct rte_flow_item_eth *eth_spec, 7422 struct rte_flow_item_eth *eth_mask, 7423 struct rte_flow_item_vlan *vlan_spec, 7424 struct rte_flow_item_vlan *vlan_mask) 7425 { 7426 struct mlx5_priv *priv = dev->data->dev_private; 7427 const struct rte_flow_attr attr = { 7428 .ingress = 1, 7429 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7430 }; 7431 struct rte_flow_item items[] = { 7432 { 7433 .type = RTE_FLOW_ITEM_TYPE_ETH, 7434 .spec = eth_spec, 7435 .last = NULL, 7436 .mask = eth_mask, 7437 }, 7438 { 7439 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7440 RTE_FLOW_ITEM_TYPE_END, 7441 .spec = vlan_spec, 7442 .last = NULL, 7443 .mask = vlan_mask, 7444 }, 7445 { 7446 .type = RTE_FLOW_ITEM_TYPE_END, 7447 }, 7448 }; 7449 uint16_t queue[priv->reta_idx_n]; 7450 struct rte_flow_action_rss action_rss = { 7451 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7452 .level = 0, 7453 .types = priv->rss_conf.rss_hf, 7454 .key_len = priv->rss_conf.rss_key_len, 7455 .queue_num = priv->reta_idx_n, 7456 .key = priv->rss_conf.rss_key, 7457 .queue = queue, 7458 }; 7459 struct rte_flow_action actions[] = { 7460 { 7461 .type = RTE_FLOW_ACTION_TYPE_RSS, 7462 .conf = &action_rss, 7463 }, 7464 { 7465 .type = RTE_FLOW_ACTION_TYPE_END, 7466 }, 7467 }; 7468 uint32_t flow_idx; 7469 struct rte_flow_error error; 7470 unsigned int i; 7471 7472 if (!priv->reta_idx_n || !priv->rxqs_n) { 7473 return 0; 7474 } 7475 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7476 action_rss.types = 0; 7477 for (i = 0; i != priv->reta_idx_n; ++i) 7478 queue[i] = (*priv->reta_idx)[i]; 7479 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7480 &attr, items, actions, false, &error); 7481 if (!flow_idx) 7482 return -rte_errno; 7483 return 0; 7484 } 7485 7486 /** 7487 * Enable a flow control configured from the control plane. 7488 * 7489 * @param dev 7490 * Pointer to Ethernet device. 7491 * @param eth_spec 7492 * An Ethernet flow spec to apply. 7493 * @param eth_mask 7494 * An Ethernet flow mask to apply. 7495 * 7496 * @return 7497 * 0 on success, a negative errno value otherwise and rte_errno is set. 7498 */ 7499 int 7500 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7501 struct rte_flow_item_eth *eth_spec, 7502 struct rte_flow_item_eth *eth_mask) 7503 { 7504 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7505 } 7506 7507 /** 7508 * Create default miss flow rule matching lacp traffic 7509 * 7510 * @param dev 7511 * Pointer to Ethernet device. 7512 * @param eth_spec 7513 * An Ethernet flow spec to apply. 7514 * 7515 * @return 7516 * 0 on success, a negative errno value otherwise and rte_errno is set. 7517 */ 7518 int 7519 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7520 { 7521 /* 7522 * The LACP matching is done by only using ether type since using 7523 * a multicast dst mac causes kernel to give low priority to this flow. 7524 */ 7525 static const struct rte_flow_item_eth lacp_spec = { 7526 .type = RTE_BE16(0x8809), 7527 }; 7528 static const struct rte_flow_item_eth lacp_mask = { 7529 .type = 0xffff, 7530 }; 7531 const struct rte_flow_attr attr = { 7532 .ingress = 1, 7533 }; 7534 struct rte_flow_item items[] = { 7535 { 7536 .type = RTE_FLOW_ITEM_TYPE_ETH, 7537 .spec = &lacp_spec, 7538 .mask = &lacp_mask, 7539 }, 7540 { 7541 .type = RTE_FLOW_ITEM_TYPE_END, 7542 }, 7543 }; 7544 struct rte_flow_action actions[] = { 7545 { 7546 .type = (enum rte_flow_action_type) 7547 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7548 }, 7549 { 7550 .type = RTE_FLOW_ACTION_TYPE_END, 7551 }, 7552 }; 7553 struct rte_flow_error error; 7554 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7555 &attr, items, actions, 7556 false, &error); 7557 7558 if (!flow_idx) 7559 return -rte_errno; 7560 return 0; 7561 } 7562 7563 /** 7564 * Destroy a flow. 7565 * 7566 * @see rte_flow_destroy() 7567 * @see rte_flow_ops 7568 */ 7569 int 7570 mlx5_flow_destroy(struct rte_eth_dev *dev, 7571 struct rte_flow *flow, 7572 struct rte_flow_error *error __rte_unused) 7573 { 7574 struct mlx5_priv *priv = dev->data->dev_private; 7575 7576 if (priv->sh->config.dv_flow_en == 2) 7577 return rte_flow_error_set(error, ENOTSUP, 7578 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7579 NULL, 7580 "Flow non-Q destruction not supported"); 7581 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7582 (uintptr_t)(void *)flow); 7583 return 0; 7584 } 7585 7586 /** 7587 * Destroy all flows. 7588 * 7589 * @see rte_flow_flush() 7590 * @see rte_flow_ops 7591 */ 7592 int 7593 mlx5_flow_flush(struct rte_eth_dev *dev, 7594 struct rte_flow_error *error __rte_unused) 7595 { 7596 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7597 return 0; 7598 } 7599 7600 /** 7601 * Isolated mode. 7602 * 7603 * @see rte_flow_isolate() 7604 * @see rte_flow_ops 7605 */ 7606 int 7607 mlx5_flow_isolate(struct rte_eth_dev *dev, 7608 int enable, 7609 struct rte_flow_error *error) 7610 { 7611 struct mlx5_priv *priv = dev->data->dev_private; 7612 7613 if (dev->data->dev_started) { 7614 rte_flow_error_set(error, EBUSY, 7615 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7616 NULL, 7617 "port must be stopped first"); 7618 return -rte_errno; 7619 } 7620 priv->isolated = !!enable; 7621 if (enable) 7622 dev->dev_ops = &mlx5_dev_ops_isolate; 7623 else 7624 dev->dev_ops = &mlx5_dev_ops; 7625 7626 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7627 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7628 7629 return 0; 7630 } 7631 7632 /** 7633 * Query a flow. 7634 * 7635 * @see rte_flow_query() 7636 * @see rte_flow_ops 7637 */ 7638 static int 7639 flow_drv_query(struct rte_eth_dev *dev, 7640 uint32_t flow_idx, 7641 const struct rte_flow_action *actions, 7642 void *data, 7643 struct rte_flow_error *error) 7644 { 7645 struct mlx5_priv *priv = dev->data->dev_private; 7646 const struct mlx5_flow_driver_ops *fops; 7647 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7648 flow_idx); 7649 enum mlx5_flow_drv_type ftype; 7650 7651 if (!flow) { 7652 return rte_flow_error_set(error, ENOENT, 7653 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7654 NULL, 7655 "invalid flow handle"); 7656 } 7657 ftype = flow->drv_type; 7658 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7659 fops = flow_get_drv_ops(ftype); 7660 7661 return fops->query(dev, flow, actions, data, error); 7662 } 7663 7664 /** 7665 * Query a flow. 7666 * 7667 * @see rte_flow_query() 7668 * @see rte_flow_ops 7669 */ 7670 int 7671 mlx5_flow_query(struct rte_eth_dev *dev, 7672 struct rte_flow *flow, 7673 const struct rte_flow_action *actions, 7674 void *data, 7675 struct rte_flow_error *error) 7676 { 7677 int ret; 7678 struct mlx5_priv *priv = dev->data->dev_private; 7679 7680 if (priv->sh->config.dv_flow_en == 2) 7681 return rte_flow_error_set(error, ENOTSUP, 7682 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7683 NULL, 7684 "Flow non-Q query not supported"); 7685 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7686 error); 7687 if (ret < 0) 7688 return ret; 7689 return 0; 7690 } 7691 7692 /** 7693 * Get rte_flow callbacks. 7694 * 7695 * @param dev 7696 * Pointer to Ethernet device structure. 7697 * @param ops 7698 * Pointer to operation-specific structure. 7699 * 7700 * @return 0 7701 */ 7702 int 7703 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7704 const struct rte_flow_ops **ops) 7705 { 7706 *ops = &mlx5_flow_ops; 7707 return 0; 7708 } 7709 7710 /** 7711 * Validate meter policy actions. 7712 * Dispatcher for action type specific validation. 7713 * 7714 * @param[in] dev 7715 * Pointer to the Ethernet device structure. 7716 * @param[in] action 7717 * The meter policy action object to validate. 7718 * @param[in] attr 7719 * Attributes of flow to determine steering domain. 7720 * @param[out] is_rss 7721 * Is RSS or not. 7722 * @param[out] domain_bitmap 7723 * Domain bitmap. 7724 * @param[out] is_def_policy 7725 * Is default policy or not. 7726 * @param[out] error 7727 * Perform verbose error reporting if not NULL. Initialized in case of 7728 * error only. 7729 * 7730 * @return 7731 * 0 on success, otherwise negative errno value. 7732 */ 7733 int 7734 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7735 const struct rte_flow_action *actions[RTE_COLORS], 7736 struct rte_flow_attr *attr, 7737 bool *is_rss, 7738 uint8_t *domain_bitmap, 7739 uint8_t *policy_mode, 7740 struct rte_mtr_error *error) 7741 { 7742 const struct mlx5_flow_driver_ops *fops; 7743 7744 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7745 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7746 domain_bitmap, policy_mode, error); 7747 } 7748 7749 /** 7750 * Destroy the meter table set. 7751 * 7752 * @param[in] dev 7753 * Pointer to Ethernet device. 7754 * @param[in] mtr_policy 7755 * Meter policy struct. 7756 */ 7757 void 7758 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7759 struct mlx5_flow_meter_policy *mtr_policy) 7760 { 7761 const struct mlx5_flow_driver_ops *fops; 7762 7763 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7764 fops->destroy_mtr_acts(dev, mtr_policy); 7765 } 7766 7767 /** 7768 * Create policy action, lock free, 7769 * (mutex should be acquired by caller). 7770 * Dispatcher for action type specific call. 7771 * 7772 * @param[in] dev 7773 * Pointer to the Ethernet device structure. 7774 * @param[in] mtr_policy 7775 * Meter policy struct. 7776 * @param[in] action 7777 * Action specification used to create meter actions. 7778 * @param[out] error 7779 * Perform verbose error reporting if not NULL. Initialized in case of 7780 * error only. 7781 * 7782 * @return 7783 * 0 on success, otherwise negative errno value. 7784 */ 7785 int 7786 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7787 struct mlx5_flow_meter_policy *mtr_policy, 7788 const struct rte_flow_action *actions[RTE_COLORS], 7789 struct rte_mtr_error *error) 7790 { 7791 const struct mlx5_flow_driver_ops *fops; 7792 7793 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7794 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7795 } 7796 7797 /** 7798 * Create policy rules, lock free, 7799 * (mutex should be acquired by caller). 7800 * Dispatcher for action type specific call. 7801 * 7802 * @param[in] dev 7803 * Pointer to the Ethernet device structure. 7804 * @param[in] mtr_policy 7805 * Meter policy struct. 7806 * 7807 * @return 7808 * 0 on success, -1 otherwise. 7809 */ 7810 int 7811 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7812 struct mlx5_flow_meter_policy *mtr_policy) 7813 { 7814 const struct mlx5_flow_driver_ops *fops; 7815 7816 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7817 return fops->create_policy_rules(dev, mtr_policy); 7818 } 7819 7820 /** 7821 * Destroy policy rules, lock free, 7822 * (mutex should be acquired by caller). 7823 * Dispatcher for action type specific call. 7824 * 7825 * @param[in] dev 7826 * Pointer to the Ethernet device structure. 7827 * @param[in] mtr_policy 7828 * Meter policy struct. 7829 */ 7830 void 7831 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7832 struct mlx5_flow_meter_policy *mtr_policy) 7833 { 7834 const struct mlx5_flow_driver_ops *fops; 7835 7836 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7837 fops->destroy_policy_rules(dev, mtr_policy); 7838 } 7839 7840 /** 7841 * Destroy the default policy table set. 7842 * 7843 * @param[in] dev 7844 * Pointer to Ethernet device. 7845 */ 7846 void 7847 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7848 { 7849 const struct mlx5_flow_driver_ops *fops; 7850 7851 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7852 fops->destroy_def_policy(dev); 7853 } 7854 7855 /** 7856 * Destroy the default policy table set. 7857 * 7858 * @param[in] dev 7859 * Pointer to Ethernet device. 7860 * 7861 * @return 7862 * 0 on success, -1 otherwise. 7863 */ 7864 int 7865 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7866 { 7867 const struct mlx5_flow_driver_ops *fops; 7868 7869 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7870 return fops->create_def_policy(dev); 7871 } 7872 7873 /** 7874 * Create the needed meter and suffix tables. 7875 * 7876 * @param[in] dev 7877 * Pointer to Ethernet device. 7878 * 7879 * @return 7880 * 0 on success, -1 otherwise. 7881 */ 7882 int 7883 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7884 struct mlx5_flow_meter_info *fm, 7885 uint32_t mtr_idx, 7886 uint8_t domain_bitmap) 7887 { 7888 const struct mlx5_flow_driver_ops *fops; 7889 7890 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7891 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7892 } 7893 7894 /** 7895 * Destroy the meter table set. 7896 * 7897 * @param[in] dev 7898 * Pointer to Ethernet device. 7899 * @param[in] tbl 7900 * Pointer to the meter table set. 7901 */ 7902 void 7903 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7904 struct mlx5_flow_meter_info *fm) 7905 { 7906 const struct mlx5_flow_driver_ops *fops; 7907 7908 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7909 fops->destroy_mtr_tbls(dev, fm); 7910 } 7911 7912 /** 7913 * Destroy the global meter drop table. 7914 * 7915 * @param[in] dev 7916 * Pointer to Ethernet device. 7917 */ 7918 void 7919 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7920 { 7921 const struct mlx5_flow_driver_ops *fops; 7922 7923 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7924 fops->destroy_mtr_drop_tbls(dev); 7925 } 7926 7927 /** 7928 * Destroy the sub policy table with RX queue. 7929 * 7930 * @param[in] dev 7931 * Pointer to Ethernet device. 7932 * @param[in] mtr_policy 7933 * Pointer to meter policy table. 7934 */ 7935 void 7936 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7937 struct mlx5_flow_meter_policy *mtr_policy) 7938 { 7939 const struct mlx5_flow_driver_ops *fops; 7940 7941 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7942 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7943 } 7944 7945 /** 7946 * Allocate the needed aso flow meter id. 7947 * 7948 * @param[in] dev 7949 * Pointer to Ethernet device. 7950 * 7951 * @return 7952 * Index to aso flow meter on success, NULL otherwise. 7953 */ 7954 uint32_t 7955 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7956 { 7957 const struct mlx5_flow_driver_ops *fops; 7958 7959 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7960 return fops->create_meter(dev); 7961 } 7962 7963 /** 7964 * Free the aso flow meter id. 7965 * 7966 * @param[in] dev 7967 * Pointer to Ethernet device. 7968 * @param[in] mtr_idx 7969 * Index to aso flow meter to be free. 7970 * 7971 * @return 7972 * 0 on success. 7973 */ 7974 void 7975 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7976 { 7977 const struct mlx5_flow_driver_ops *fops; 7978 7979 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7980 fops->free_meter(dev, mtr_idx); 7981 } 7982 7983 /** 7984 * Allocate a counter. 7985 * 7986 * @param[in] dev 7987 * Pointer to Ethernet device structure. 7988 * 7989 * @return 7990 * Index to allocated counter on success, 0 otherwise. 7991 */ 7992 uint32_t 7993 mlx5_counter_alloc(struct rte_eth_dev *dev) 7994 { 7995 const struct mlx5_flow_driver_ops *fops; 7996 struct rte_flow_attr attr = { .transfer = 0 }; 7997 7998 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7999 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8000 return fops->counter_alloc(dev); 8001 } 8002 DRV_LOG(ERR, 8003 "port %u counter allocate is not supported.", 8004 dev->data->port_id); 8005 return 0; 8006 } 8007 8008 /** 8009 * Free a counter. 8010 * 8011 * @param[in] dev 8012 * Pointer to Ethernet device structure. 8013 * @param[in] cnt 8014 * Index to counter to be free. 8015 */ 8016 void 8017 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8018 { 8019 const struct mlx5_flow_driver_ops *fops; 8020 struct rte_flow_attr attr = { .transfer = 0 }; 8021 8022 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8023 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8024 fops->counter_free(dev, cnt); 8025 return; 8026 } 8027 DRV_LOG(ERR, 8028 "port %u counter free is not supported.", 8029 dev->data->port_id); 8030 } 8031 8032 /** 8033 * Query counter statistics. 8034 * 8035 * @param[in] dev 8036 * Pointer to Ethernet device structure. 8037 * @param[in] cnt 8038 * Index to counter to query. 8039 * @param[in] clear 8040 * Set to clear counter statistics. 8041 * @param[out] pkts 8042 * The counter hits packets number to save. 8043 * @param[out] bytes 8044 * The counter hits bytes number to save. 8045 * 8046 * @return 8047 * 0 on success, a negative errno value otherwise. 8048 */ 8049 int 8050 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8051 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8052 { 8053 const struct mlx5_flow_driver_ops *fops; 8054 struct rte_flow_attr attr = { .transfer = 0 }; 8055 8056 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8057 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8058 return fops->counter_query(dev, cnt, clear, pkts, 8059 bytes, action); 8060 } 8061 DRV_LOG(ERR, 8062 "port %u counter query is not supported.", 8063 dev->data->port_id); 8064 return -ENOTSUP; 8065 } 8066 8067 /** 8068 * Get information about HWS pre-configurable resources. 8069 * 8070 * @param[in] dev 8071 * Pointer to the rte_eth_dev structure. 8072 * @param[out] port_info 8073 * Pointer to port information. 8074 * @param[out] queue_info 8075 * Pointer to queue information. 8076 * @param[out] error 8077 * Pointer to error structure. 8078 * 8079 * @return 8080 * 0 on success, a negative errno value otherwise and rte_errno is set. 8081 */ 8082 static int 8083 mlx5_flow_info_get(struct rte_eth_dev *dev, 8084 struct rte_flow_port_info *port_info, 8085 struct rte_flow_queue_info *queue_info, 8086 struct rte_flow_error *error) 8087 { 8088 const struct mlx5_flow_driver_ops *fops; 8089 8090 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8091 return rte_flow_error_set(error, ENOTSUP, 8092 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8093 NULL, 8094 "info get with incorrect steering mode"); 8095 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8096 return fops->info_get(dev, port_info, queue_info, error); 8097 } 8098 8099 /** 8100 * Configure port HWS resources. 8101 * 8102 * @param[in] dev 8103 * Pointer to the rte_eth_dev structure. 8104 * @param[in] port_attr 8105 * Port configuration attributes. 8106 * @param[in] nb_queue 8107 * Number of queue. 8108 * @param[in] queue_attr 8109 * Array that holds attributes for each flow queue. 8110 * @param[out] error 8111 * Pointer to error structure. 8112 * 8113 * @return 8114 * 0 on success, a negative errno value otherwise and rte_errno is set. 8115 */ 8116 static int 8117 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8118 const struct rte_flow_port_attr *port_attr, 8119 uint16_t nb_queue, 8120 const struct rte_flow_queue_attr *queue_attr[], 8121 struct rte_flow_error *error) 8122 { 8123 const struct mlx5_flow_driver_ops *fops; 8124 8125 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8126 return rte_flow_error_set(error, ENOTSUP, 8127 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8128 NULL, 8129 "port configure with incorrect steering mode"); 8130 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8131 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8132 } 8133 8134 /** 8135 * Create flow item template. 8136 * 8137 * @param[in] dev 8138 * Pointer to the rte_eth_dev structure. 8139 * @param[in] attr 8140 * Pointer to the item template attributes. 8141 * @param[in] items 8142 * The template item pattern. 8143 * @param[out] error 8144 * Pointer to error structure. 8145 * 8146 * @return 8147 * 0 on success, a negative errno value otherwise and rte_errno is set. 8148 */ 8149 static struct rte_flow_pattern_template * 8150 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8151 const struct rte_flow_pattern_template_attr *attr, 8152 const struct rte_flow_item items[], 8153 struct rte_flow_error *error) 8154 { 8155 const struct mlx5_flow_driver_ops *fops; 8156 8157 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8158 rte_flow_error_set(error, ENOTSUP, 8159 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8160 NULL, 8161 "pattern create with incorrect steering mode"); 8162 return NULL; 8163 } 8164 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8165 return fops->pattern_template_create(dev, attr, items, error); 8166 } 8167 8168 /** 8169 * Destroy flow item template. 8170 * 8171 * @param[in] dev 8172 * Pointer to the rte_eth_dev structure. 8173 * @param[in] template 8174 * Pointer to the item template to be destroyed. 8175 * @param[out] error 8176 * Pointer to error structure. 8177 * 8178 * @return 8179 * 0 on success, a negative errno value otherwise and rte_errno is set. 8180 */ 8181 static int 8182 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8183 struct rte_flow_pattern_template *template, 8184 struct rte_flow_error *error) 8185 { 8186 const struct mlx5_flow_driver_ops *fops; 8187 8188 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8189 return rte_flow_error_set(error, ENOTSUP, 8190 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8191 NULL, 8192 "pattern destroy with incorrect steering mode"); 8193 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8194 return fops->pattern_template_destroy(dev, template, error); 8195 } 8196 8197 /** 8198 * Create flow item template. 8199 * 8200 * @param[in] dev 8201 * Pointer to the rte_eth_dev structure. 8202 * @param[in] attr 8203 * Pointer to the action template attributes. 8204 * @param[in] actions 8205 * Associated actions (list terminated by the END action). 8206 * @param[in] masks 8207 * List of actions that marks which of the action's member is constant. 8208 * @param[out] error 8209 * Pointer to error structure. 8210 * 8211 * @return 8212 * 0 on success, a negative errno value otherwise and rte_errno is set. 8213 */ 8214 static struct rte_flow_actions_template * 8215 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8216 const struct rte_flow_actions_template_attr *attr, 8217 const struct rte_flow_action actions[], 8218 const struct rte_flow_action masks[], 8219 struct rte_flow_error *error) 8220 { 8221 const struct mlx5_flow_driver_ops *fops; 8222 8223 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8224 rte_flow_error_set(error, ENOTSUP, 8225 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8226 NULL, 8227 "action create with incorrect steering mode"); 8228 return NULL; 8229 } 8230 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8231 return fops->actions_template_create(dev, attr, actions, masks, error); 8232 } 8233 8234 /** 8235 * Destroy flow action template. 8236 * 8237 * @param[in] dev 8238 * Pointer to the rte_eth_dev structure. 8239 * @param[in] template 8240 * Pointer to the action template to be destroyed. 8241 * @param[out] error 8242 * Pointer to error structure. 8243 * 8244 * @return 8245 * 0 on success, a negative errno value otherwise and rte_errno is set. 8246 */ 8247 static int 8248 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8249 struct rte_flow_actions_template *template, 8250 struct rte_flow_error *error) 8251 { 8252 const struct mlx5_flow_driver_ops *fops; 8253 8254 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8255 return rte_flow_error_set(error, ENOTSUP, 8256 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8257 NULL, 8258 "action destroy with incorrect steering mode"); 8259 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8260 return fops->actions_template_destroy(dev, template, error); 8261 } 8262 8263 /** 8264 * Create flow table. 8265 * 8266 * @param[in] dev 8267 * Pointer to the rte_eth_dev structure. 8268 * @param[in] attr 8269 * Pointer to the table attributes. 8270 * @param[in] item_templates 8271 * Item template array to be binded to the table. 8272 * @param[in] nb_item_templates 8273 * Number of item template. 8274 * @param[in] action_templates 8275 * Action template array to be binded to the table. 8276 * @param[in] nb_action_templates 8277 * Number of action template. 8278 * @param[out] error 8279 * Pointer to error structure. 8280 * 8281 * @return 8282 * Table on success, NULL otherwise and rte_errno is set. 8283 */ 8284 static struct rte_flow_template_table * 8285 mlx5_flow_table_create(struct rte_eth_dev *dev, 8286 const struct rte_flow_template_table_attr *attr, 8287 struct rte_flow_pattern_template *item_templates[], 8288 uint8_t nb_item_templates, 8289 struct rte_flow_actions_template *action_templates[], 8290 uint8_t nb_action_templates, 8291 struct rte_flow_error *error) 8292 { 8293 const struct mlx5_flow_driver_ops *fops; 8294 8295 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8296 rte_flow_error_set(error, ENOTSUP, 8297 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8298 NULL, 8299 "table create with incorrect steering mode"); 8300 return NULL; 8301 } 8302 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8303 return fops->template_table_create(dev, 8304 attr, 8305 item_templates, 8306 nb_item_templates, 8307 action_templates, 8308 nb_action_templates, 8309 error); 8310 } 8311 8312 /** 8313 * PMD destroy flow table. 8314 * 8315 * @param[in] dev 8316 * Pointer to the rte_eth_dev structure. 8317 * @param[in] table 8318 * Pointer to the table to be destroyed. 8319 * @param[out] error 8320 * Pointer to error structure. 8321 * 8322 * @return 8323 * 0 on success, a negative errno value otherwise and rte_errno is set. 8324 */ 8325 static int 8326 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8327 struct rte_flow_template_table *table, 8328 struct rte_flow_error *error) 8329 { 8330 const struct mlx5_flow_driver_ops *fops; 8331 8332 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8333 return rte_flow_error_set(error, ENOTSUP, 8334 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8335 NULL, 8336 "table destroy with incorrect steering mode"); 8337 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8338 return fops->template_table_destroy(dev, table, error); 8339 } 8340 8341 /** 8342 * Enqueue flow creation. 8343 * 8344 * @param[in] dev 8345 * Pointer to the rte_eth_dev structure. 8346 * @param[in] queue_id 8347 * The queue to create the flow. 8348 * @param[in] attr 8349 * Pointer to the flow operation attributes. 8350 * @param[in] items 8351 * Items with flow spec value. 8352 * @param[in] pattern_template_index 8353 * The item pattern flow follows from the table. 8354 * @param[in] actions 8355 * Action with flow spec value. 8356 * @param[in] action_template_index 8357 * The action pattern flow follows from the table. 8358 * @param[in] user_data 8359 * Pointer to the user_data. 8360 * @param[out] error 8361 * Pointer to error structure. 8362 * 8363 * @return 8364 * Flow pointer on success, NULL otherwise and rte_errno is set. 8365 */ 8366 static struct rte_flow * 8367 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8368 uint32_t queue_id, 8369 const struct rte_flow_op_attr *attr, 8370 struct rte_flow_template_table *table, 8371 const struct rte_flow_item items[], 8372 uint8_t pattern_template_index, 8373 const struct rte_flow_action actions[], 8374 uint8_t action_template_index, 8375 void *user_data, 8376 struct rte_flow_error *error) 8377 { 8378 const struct mlx5_flow_driver_ops *fops; 8379 8380 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8381 rte_flow_error_set(error, ENOTSUP, 8382 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8383 NULL, 8384 "flow_q create with incorrect steering mode"); 8385 return NULL; 8386 } 8387 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8388 return fops->async_flow_create(dev, queue_id, attr, table, 8389 items, pattern_template_index, 8390 actions, action_template_index, 8391 user_data, error); 8392 } 8393 8394 /** 8395 * Enqueue flow destruction. 8396 * 8397 * @param[in] dev 8398 * Pointer to the rte_eth_dev structure. 8399 * @param[in] queue 8400 * The queue to destroy the flow. 8401 * @param[in] attr 8402 * Pointer to the flow operation attributes. 8403 * @param[in] flow 8404 * Pointer to the flow to be destroyed. 8405 * @param[in] user_data 8406 * Pointer to the user_data. 8407 * @param[out] error 8408 * Pointer to error structure. 8409 * 8410 * @return 8411 * 0 on success, negative value otherwise and rte_errno is set. 8412 */ 8413 static int 8414 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8415 uint32_t queue, 8416 const struct rte_flow_op_attr *attr, 8417 struct rte_flow *flow, 8418 void *user_data, 8419 struct rte_flow_error *error) 8420 { 8421 const struct mlx5_flow_driver_ops *fops; 8422 8423 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8424 return rte_flow_error_set(error, ENOTSUP, 8425 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8426 NULL, 8427 "flow_q destroy with incorrect steering mode"); 8428 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8429 return fops->async_flow_destroy(dev, queue, attr, flow, 8430 user_data, error); 8431 } 8432 8433 /** 8434 * Pull the enqueued flows. 8435 * 8436 * @param[in] dev 8437 * Pointer to the rte_eth_dev structure. 8438 * @param[in] queue 8439 * The queue to pull the result. 8440 * @param[in/out] res 8441 * Array to save the results. 8442 * @param[in] n_res 8443 * Available result with the array. 8444 * @param[out] error 8445 * Pointer to error structure. 8446 * 8447 * @return 8448 * Result number on success, negative value otherwise and rte_errno is set. 8449 */ 8450 static int 8451 mlx5_flow_pull(struct rte_eth_dev *dev, 8452 uint32_t queue, 8453 struct rte_flow_op_result res[], 8454 uint16_t n_res, 8455 struct rte_flow_error *error) 8456 { 8457 const struct mlx5_flow_driver_ops *fops; 8458 8459 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8460 return rte_flow_error_set(error, ENOTSUP, 8461 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8462 NULL, 8463 "flow_q pull with incorrect steering mode"); 8464 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8465 return fops->pull(dev, queue, res, n_res, error); 8466 } 8467 8468 /** 8469 * Push the enqueued flows. 8470 * 8471 * @param[in] dev 8472 * Pointer to the rte_eth_dev structure. 8473 * @param[in] queue 8474 * The queue to push the flows. 8475 * @param[out] error 8476 * Pointer to error structure. 8477 * 8478 * @return 8479 * 0 on success, negative value otherwise and rte_errno is set. 8480 */ 8481 static int 8482 mlx5_flow_push(struct rte_eth_dev *dev, 8483 uint32_t queue, 8484 struct rte_flow_error *error) 8485 { 8486 const struct mlx5_flow_driver_ops *fops; 8487 8488 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8489 return rte_flow_error_set(error, ENOTSUP, 8490 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8491 NULL, 8492 "flow_q push with incorrect steering mode"); 8493 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8494 return fops->push(dev, queue, error); 8495 } 8496 8497 /** 8498 * Create shared action. 8499 * 8500 * @param[in] dev 8501 * Pointer to the rte_eth_dev structure. 8502 * @param[in] queue 8503 * Which queue to be used.. 8504 * @param[in] attr 8505 * Operation attribute. 8506 * @param[in] conf 8507 * Indirect action configuration. 8508 * @param[in] action 8509 * rte_flow action detail. 8510 * @param[in] user_data 8511 * Pointer to the user_data. 8512 * @param[out] error 8513 * Pointer to error structure. 8514 * 8515 * @return 8516 * Action handle on success, NULL otherwise and rte_errno is set. 8517 */ 8518 static struct rte_flow_action_handle * 8519 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8520 const struct rte_flow_op_attr *attr, 8521 const struct rte_flow_indir_action_conf *conf, 8522 const struct rte_flow_action *action, 8523 void *user_data, 8524 struct rte_flow_error *error) 8525 { 8526 const struct mlx5_flow_driver_ops *fops = 8527 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8528 8529 return fops->async_action_create(dev, queue, attr, conf, action, 8530 user_data, error); 8531 } 8532 8533 /** 8534 * Update shared action. 8535 * 8536 * @param[in] dev 8537 * Pointer to the rte_eth_dev structure. 8538 * @param[in] queue 8539 * Which queue to be used.. 8540 * @param[in] attr 8541 * Operation attribute. 8542 * @param[in] handle 8543 * Action handle to be updated. 8544 * @param[in] update 8545 * Update value. 8546 * @param[in] user_data 8547 * Pointer to the user_data. 8548 * @param[out] error 8549 * Pointer to error structure. 8550 * 8551 * @return 8552 * 0 on success, negative value otherwise and rte_errno is set. 8553 */ 8554 static int 8555 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 8556 const struct rte_flow_op_attr *attr, 8557 struct rte_flow_action_handle *handle, 8558 const void *update, 8559 void *user_data, 8560 struct rte_flow_error *error) 8561 { 8562 const struct mlx5_flow_driver_ops *fops = 8563 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8564 8565 return fops->async_action_update(dev, queue, attr, handle, 8566 update, user_data, error); 8567 } 8568 8569 /** 8570 * Destroy shared action. 8571 * 8572 * @param[in] dev 8573 * Pointer to the rte_eth_dev structure. 8574 * @param[in] queue 8575 * Which queue to be used.. 8576 * @param[in] attr 8577 * Operation attribute. 8578 * @param[in] handle 8579 * Action handle to be destroyed. 8580 * @param[in] user_data 8581 * Pointer to the user_data. 8582 * @param[out] error 8583 * Pointer to error structure. 8584 * 8585 * @return 8586 * 0 on success, negative value otherwise and rte_errno is set. 8587 */ 8588 static int 8589 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 8590 const struct rte_flow_op_attr *attr, 8591 struct rte_flow_action_handle *handle, 8592 void *user_data, 8593 struct rte_flow_error *error) 8594 { 8595 const struct mlx5_flow_driver_ops *fops = 8596 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8597 8598 return fops->async_action_destroy(dev, queue, attr, handle, 8599 user_data, error); 8600 } 8601 8602 /** 8603 * Allocate a new memory for the counter values wrapped by all the needed 8604 * management. 8605 * 8606 * @param[in] sh 8607 * Pointer to mlx5_dev_ctx_shared object. 8608 * 8609 * @return 8610 * 0 on success, a negative errno value otherwise. 8611 */ 8612 static int 8613 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 8614 { 8615 struct mlx5_counter_stats_mem_mng *mem_mng; 8616 volatile struct flow_counter_stats *raw_data; 8617 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 8618 int size = (sizeof(struct flow_counter_stats) * 8619 MLX5_COUNTERS_PER_POOL + 8620 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 8621 sizeof(struct mlx5_counter_stats_mem_mng); 8622 size_t pgsize = rte_mem_page_size(); 8623 uint8_t *mem; 8624 int ret; 8625 int i; 8626 8627 if (pgsize == (size_t)-1) { 8628 DRV_LOG(ERR, "Failed to get mem page size"); 8629 rte_errno = ENOMEM; 8630 return -ENOMEM; 8631 } 8632 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 8633 if (!mem) { 8634 rte_errno = ENOMEM; 8635 return -ENOMEM; 8636 } 8637 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 8638 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 8639 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 8640 sh->cdev->pdn, mem, size, 8641 &mem_mng->wm); 8642 if (ret) { 8643 rte_errno = errno; 8644 mlx5_free(mem); 8645 return -rte_errno; 8646 } 8647 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 8648 raw_data = (volatile struct flow_counter_stats *)mem; 8649 for (i = 0; i < raws_n; ++i) { 8650 mem_mng->raws[i].mem_mng = mem_mng; 8651 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 8652 } 8653 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 8654 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 8655 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 8656 next); 8657 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 8658 sh->cmng.mem_mng = mem_mng; 8659 return 0; 8660 } 8661 8662 /** 8663 * Set the statistic memory to the new counter pool. 8664 * 8665 * @param[in] sh 8666 * Pointer to mlx5_dev_ctx_shared object. 8667 * @param[in] pool 8668 * Pointer to the pool to set the statistic memory. 8669 * 8670 * @return 8671 * 0 on success, a negative errno value otherwise. 8672 */ 8673 static int 8674 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 8675 struct mlx5_flow_counter_pool *pool) 8676 { 8677 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8678 /* Resize statistic memory once used out. */ 8679 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 8680 mlx5_flow_create_counter_stat_mem_mng(sh)) { 8681 DRV_LOG(ERR, "Cannot resize counter stat mem."); 8682 return -1; 8683 } 8684 rte_spinlock_lock(&pool->sl); 8685 pool->raw = cmng->mem_mng->raws + pool->index % 8686 MLX5_CNT_CONTAINER_RESIZE; 8687 rte_spinlock_unlock(&pool->sl); 8688 pool->raw_hw = NULL; 8689 return 0; 8690 } 8691 8692 #define MLX5_POOL_QUERY_FREQ_US 1000000 8693 8694 /** 8695 * Set the periodic procedure for triggering asynchronous batch queries for all 8696 * the counter pools. 8697 * 8698 * @param[in] sh 8699 * Pointer to mlx5_dev_ctx_shared object. 8700 */ 8701 void 8702 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 8703 { 8704 uint32_t pools_n, us; 8705 8706 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 8707 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 8708 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 8709 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 8710 sh->cmng.query_thread_on = 0; 8711 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 8712 } else { 8713 sh->cmng.query_thread_on = 1; 8714 } 8715 } 8716 8717 /** 8718 * The periodic procedure for triggering asynchronous batch queries for all the 8719 * counter pools. This function is probably called by the host thread. 8720 * 8721 * @param[in] arg 8722 * The parameter for the alarm process. 8723 */ 8724 void 8725 mlx5_flow_query_alarm(void *arg) 8726 { 8727 struct mlx5_dev_ctx_shared *sh = arg; 8728 int ret; 8729 uint16_t pool_index = sh->cmng.pool_index; 8730 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8731 struct mlx5_flow_counter_pool *pool; 8732 uint16_t n_valid; 8733 8734 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 8735 goto set_alarm; 8736 rte_spinlock_lock(&cmng->pool_update_sl); 8737 pool = cmng->pools[pool_index]; 8738 n_valid = cmng->n_valid; 8739 rte_spinlock_unlock(&cmng->pool_update_sl); 8740 /* Set the statistic memory to the new created pool. */ 8741 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 8742 goto set_alarm; 8743 if (pool->raw_hw) 8744 /* There is a pool query in progress. */ 8745 goto set_alarm; 8746 pool->raw_hw = 8747 LIST_FIRST(&sh->cmng.free_stat_raws); 8748 if (!pool->raw_hw) 8749 /* No free counter statistics raw memory. */ 8750 goto set_alarm; 8751 /* 8752 * Identify the counters released between query trigger and query 8753 * handle more efficiently. The counter released in this gap period 8754 * should wait for a new round of query as the new arrived packets 8755 * will not be taken into account. 8756 */ 8757 pool->query_gen++; 8758 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 8759 MLX5_COUNTERS_PER_POOL, 8760 NULL, NULL, 8761 pool->raw_hw->mem_mng->wm.lkey, 8762 (void *)(uintptr_t) 8763 pool->raw_hw->data, 8764 sh->devx_comp, 8765 (uint64_t)(uintptr_t)pool); 8766 if (ret) { 8767 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 8768 " %d", pool->min_dcs->id); 8769 pool->raw_hw = NULL; 8770 goto set_alarm; 8771 } 8772 LIST_REMOVE(pool->raw_hw, next); 8773 sh->cmng.pending_queries++; 8774 pool_index++; 8775 if (pool_index >= n_valid) 8776 pool_index = 0; 8777 set_alarm: 8778 sh->cmng.pool_index = pool_index; 8779 mlx5_set_query_alarm(sh); 8780 } 8781 8782 /** 8783 * Check and callback event for new aged flow in the counter pool 8784 * 8785 * @param[in] sh 8786 * Pointer to mlx5_dev_ctx_shared object. 8787 * @param[in] pool 8788 * Pointer to Current counter pool. 8789 */ 8790 static void 8791 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8792 struct mlx5_flow_counter_pool *pool) 8793 { 8794 struct mlx5_priv *priv; 8795 struct mlx5_flow_counter *cnt; 8796 struct mlx5_age_info *age_info; 8797 struct mlx5_age_param *age_param; 8798 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8799 struct mlx5_counter_stats_raw *prev = pool->raw; 8800 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8801 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8802 uint16_t expected = AGE_CANDIDATE; 8803 uint32_t i; 8804 8805 pool->time_of_last_age_check = curr_time; 8806 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8807 cnt = MLX5_POOL_GET_CNT(pool, i); 8808 age_param = MLX5_CNT_TO_AGE(cnt); 8809 if (__atomic_load_n(&age_param->state, 8810 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8811 continue; 8812 if (cur->data[i].hits != prev->data[i].hits) { 8813 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8814 __ATOMIC_RELAXED); 8815 continue; 8816 } 8817 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8818 time_delta, 8819 __ATOMIC_RELAXED) <= age_param->timeout) 8820 continue; 8821 /** 8822 * Hold the lock first, or if between the 8823 * state AGE_TMOUT and tailq operation the 8824 * release happened, the release procedure 8825 * may delete a non-existent tailq node. 8826 */ 8827 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8828 age_info = GET_PORT_AGE_INFO(priv); 8829 rte_spinlock_lock(&age_info->aged_sl); 8830 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8831 AGE_TMOUT, false, 8832 __ATOMIC_RELAXED, 8833 __ATOMIC_RELAXED)) { 8834 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8835 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8836 } 8837 rte_spinlock_unlock(&age_info->aged_sl); 8838 } 8839 mlx5_age_event_prepare(sh); 8840 } 8841 8842 /** 8843 * Handler for the HW respond about ready values from an asynchronous batch 8844 * query. This function is probably called by the host thread. 8845 * 8846 * @param[in] sh 8847 * The pointer to the shared device context. 8848 * @param[in] async_id 8849 * The Devx async ID. 8850 * @param[in] status 8851 * The status of the completion. 8852 */ 8853 void 8854 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8855 uint64_t async_id, int status) 8856 { 8857 struct mlx5_flow_counter_pool *pool = 8858 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8859 struct mlx5_counter_stats_raw *raw_to_free; 8860 uint8_t query_gen = pool->query_gen ^ 1; 8861 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8862 enum mlx5_counter_type cnt_type = 8863 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8864 MLX5_COUNTER_TYPE_ORIGIN; 8865 8866 if (unlikely(status)) { 8867 raw_to_free = pool->raw_hw; 8868 } else { 8869 raw_to_free = pool->raw; 8870 if (pool->is_aged) 8871 mlx5_flow_aging_check(sh, pool); 8872 rte_spinlock_lock(&pool->sl); 8873 pool->raw = pool->raw_hw; 8874 rte_spinlock_unlock(&pool->sl); 8875 /* Be sure the new raw counters data is updated in memory. */ 8876 rte_io_wmb(); 8877 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8878 rte_spinlock_lock(&cmng->csl[cnt_type]); 8879 TAILQ_CONCAT(&cmng->counters[cnt_type], 8880 &pool->counters[query_gen], next); 8881 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8882 } 8883 } 8884 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8885 pool->raw_hw = NULL; 8886 sh->cmng.pending_queries--; 8887 } 8888 8889 static int 8890 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8891 const struct flow_grp_info *grp_info, 8892 struct rte_flow_error *error) 8893 { 8894 if (grp_info->transfer && grp_info->external && 8895 grp_info->fdb_def_rule) { 8896 if (group == UINT32_MAX) 8897 return rte_flow_error_set 8898 (error, EINVAL, 8899 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8900 NULL, 8901 "group index not supported"); 8902 *table = group + 1; 8903 } else { 8904 *table = group; 8905 } 8906 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8907 return 0; 8908 } 8909 8910 /** 8911 * Translate the rte_flow group index to HW table value. 8912 * 8913 * If tunnel offload is disabled, all group ids converted to flow table 8914 * id using the standard method. 8915 * If tunnel offload is enabled, group id can be converted using the 8916 * standard or tunnel conversion method. Group conversion method 8917 * selection depends on flags in `grp_info` parameter: 8918 * - Internal (grp_info.external == 0) groups conversion uses the 8919 * standard method. 8920 * - Group ids in JUMP action converted with the tunnel conversion. 8921 * - Group id in rule attribute conversion depends on a rule type and 8922 * group id value: 8923 * ** non zero group attributes converted with the tunnel method 8924 * ** zero group attribute in non-tunnel rule is converted using the 8925 * standard method - there's only one root table 8926 * ** zero group attribute in steer tunnel rule is converted with the 8927 * standard method - single root table 8928 * ** zero group attribute in match tunnel rule is a special OvS 8929 * case: that value is used for portability reasons. That group 8930 * id is converted with the tunnel conversion method. 8931 * 8932 * @param[in] dev 8933 * Port device 8934 * @param[in] tunnel 8935 * PMD tunnel offload object 8936 * @param[in] group 8937 * rte_flow group index value. 8938 * @param[out] table 8939 * HW table value. 8940 * @param[in] grp_info 8941 * flags used for conversion 8942 * @param[out] error 8943 * Pointer to error structure. 8944 * 8945 * @return 8946 * 0 on success, a negative errno value otherwise and rte_errno is set. 8947 */ 8948 int 8949 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8950 const struct mlx5_flow_tunnel *tunnel, 8951 uint32_t group, uint32_t *table, 8952 const struct flow_grp_info *grp_info, 8953 struct rte_flow_error *error) 8954 { 8955 int ret; 8956 bool standard_translation; 8957 8958 if (!grp_info->skip_scale && grp_info->external && 8959 group < MLX5_MAX_TABLES_EXTERNAL) 8960 group *= MLX5_FLOW_TABLE_FACTOR; 8961 if (is_tunnel_offload_active(dev)) { 8962 standard_translation = !grp_info->external || 8963 grp_info->std_tbl_fix; 8964 } else { 8965 standard_translation = true; 8966 } 8967 DRV_LOG(DEBUG, 8968 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8969 dev->data->port_id, group, grp_info->transfer, 8970 grp_info->external, grp_info->fdb_def_rule, 8971 standard_translation ? "STANDARD" : "TUNNEL"); 8972 if (standard_translation) 8973 ret = flow_group_to_table(dev->data->port_id, group, table, 8974 grp_info, error); 8975 else 8976 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8977 table, error); 8978 8979 return ret; 8980 } 8981 8982 /** 8983 * Discover availability of metadata reg_c's. 8984 * 8985 * Iteratively use test flows to check availability. 8986 * 8987 * @param[in] dev 8988 * Pointer to the Ethernet device structure. 8989 * 8990 * @return 8991 * 0 on success, a negative errno value otherwise and rte_errno is set. 8992 */ 8993 int 8994 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8995 { 8996 struct mlx5_priv *priv = dev->data->dev_private; 8997 enum modify_reg idx; 8998 int n = 0; 8999 9000 /* reg_c[0] and reg_c[1] are reserved. */ 9001 priv->sh->flow_mreg_c[n++] = REG_C_0; 9002 priv->sh->flow_mreg_c[n++] = REG_C_1; 9003 /* Discover availability of other reg_c's. */ 9004 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9005 struct rte_flow_attr attr = { 9006 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9007 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9008 .ingress = 1, 9009 }; 9010 struct rte_flow_item items[] = { 9011 [0] = { 9012 .type = RTE_FLOW_ITEM_TYPE_END, 9013 }, 9014 }; 9015 struct rte_flow_action actions[] = { 9016 [0] = { 9017 .type = (enum rte_flow_action_type) 9018 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9019 .conf = &(struct mlx5_flow_action_copy_mreg){ 9020 .src = REG_C_1, 9021 .dst = idx, 9022 }, 9023 }, 9024 [1] = { 9025 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9026 .conf = &(struct rte_flow_action_jump){ 9027 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9028 }, 9029 }, 9030 [2] = { 9031 .type = RTE_FLOW_ACTION_TYPE_END, 9032 }, 9033 }; 9034 uint32_t flow_idx; 9035 struct rte_flow *flow; 9036 struct rte_flow_error error; 9037 9038 if (!priv->sh->config.dv_flow_en) 9039 break; 9040 /* Create internal flow, validation skips copy action. */ 9041 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9042 items, actions, false, &error); 9043 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9044 flow_idx); 9045 if (!flow) 9046 continue; 9047 priv->sh->flow_mreg_c[n++] = idx; 9048 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9049 } 9050 for (; n < MLX5_MREG_C_NUM; ++n) 9051 priv->sh->flow_mreg_c[n] = REG_NON; 9052 priv->sh->metadata_regc_check_flag = 1; 9053 return 0; 9054 } 9055 9056 int 9057 save_dump_file(const uint8_t *data, uint32_t size, 9058 uint32_t type, uint64_t id, void *arg, FILE *file) 9059 { 9060 char line[BUF_SIZE]; 9061 uint32_t out = 0; 9062 uint32_t k; 9063 uint32_t actions_num; 9064 struct rte_flow_query_count *count; 9065 9066 memset(line, 0, BUF_SIZE); 9067 switch (type) { 9068 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9069 actions_num = *(uint32_t *)(arg); 9070 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9071 type, id, actions_num); 9072 break; 9073 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9074 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9075 type, id); 9076 break; 9077 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9078 count = (struct rte_flow_query_count *)arg; 9079 fprintf(file, 9080 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9081 type, id, count->hits, count->bytes); 9082 return 0; 9083 default: 9084 return -1; 9085 } 9086 9087 for (k = 0; k < size; k++) { 9088 /* Make sure we do not overrun the line buffer length. */ 9089 if (out >= BUF_SIZE - 4) { 9090 line[out] = '\0'; 9091 break; 9092 } 9093 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9094 (data[k]) & 0xff); 9095 } 9096 fprintf(file, "%s\n", line); 9097 return 0; 9098 } 9099 9100 int 9101 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9102 struct rte_flow_query_count *count, struct rte_flow_error *error) 9103 { 9104 struct rte_flow_action action[2]; 9105 enum mlx5_flow_drv_type ftype; 9106 const struct mlx5_flow_driver_ops *fops; 9107 9108 if (!flow) { 9109 return rte_flow_error_set(error, ENOENT, 9110 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9111 NULL, 9112 "invalid flow handle"); 9113 } 9114 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9115 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9116 if (flow->counter) { 9117 memset(count, 0, sizeof(struct rte_flow_query_count)); 9118 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9119 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9120 ftype < MLX5_FLOW_TYPE_MAX); 9121 fops = flow_get_drv_ops(ftype); 9122 return fops->query(dev, flow, action, count, error); 9123 } 9124 return -1; 9125 } 9126 9127 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9128 /** 9129 * Dump flow ipool data to file 9130 * 9131 * @param[in] dev 9132 * The pointer to Ethernet device. 9133 * @param[in] file 9134 * A pointer to a file for output. 9135 * @param[out] error 9136 * Perform verbose error reporting if not NULL. PMDs initialize this 9137 * structure in case of error only. 9138 * @return 9139 * 0 on success, a negative value otherwise. 9140 */ 9141 int 9142 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9143 struct rte_flow *flow, FILE *file, 9144 struct rte_flow_error *error) 9145 { 9146 struct mlx5_priv *priv = dev->data->dev_private; 9147 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9148 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9149 uint32_t handle_idx; 9150 struct mlx5_flow_handle *dh; 9151 struct rte_flow_query_count count; 9152 uint32_t actions_num; 9153 const uint8_t *data; 9154 size_t size; 9155 uint64_t id; 9156 uint32_t type; 9157 void *action = NULL; 9158 9159 if (!flow) { 9160 return rte_flow_error_set(error, ENOENT, 9161 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9162 NULL, 9163 "invalid flow handle"); 9164 } 9165 handle_idx = flow->dev_handles; 9166 /* query counter */ 9167 if (flow->counter && 9168 (!mlx5_counter_query(dev, flow->counter, false, 9169 &count.hits, &count.bytes, &action)) && action) { 9170 id = (uint64_t)(uintptr_t)action; 9171 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9172 save_dump_file(NULL, 0, type, 9173 id, (void *)&count, file); 9174 } 9175 9176 while (handle_idx) { 9177 dh = mlx5_ipool_get(priv->sh->ipool 9178 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9179 if (!dh) 9180 continue; 9181 handle_idx = dh->next.next; 9182 9183 /* Get modify_hdr and encap_decap buf from ipools. */ 9184 encap_decap = NULL; 9185 modify_hdr = dh->dvh.modify_hdr; 9186 9187 if (dh->dvh.rix_encap_decap) { 9188 encap_decap = mlx5_ipool_get(priv->sh->ipool 9189 [MLX5_IPOOL_DECAP_ENCAP], 9190 dh->dvh.rix_encap_decap); 9191 } 9192 if (modify_hdr) { 9193 data = (const uint8_t *)modify_hdr->actions; 9194 size = (size_t)(modify_hdr->actions_num) * 8; 9195 id = (uint64_t)(uintptr_t)modify_hdr->action; 9196 actions_num = modify_hdr->actions_num; 9197 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9198 save_dump_file(data, size, type, id, 9199 (void *)(&actions_num), file); 9200 } 9201 if (encap_decap) { 9202 data = encap_decap->buf; 9203 size = encap_decap->size; 9204 id = (uint64_t)(uintptr_t)encap_decap->action; 9205 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9206 save_dump_file(data, size, type, 9207 id, NULL, file); 9208 } 9209 } 9210 return 0; 9211 } 9212 9213 /** 9214 * Dump all flow's encap_decap/modify_hdr/counter data to file 9215 * 9216 * @param[in] dev 9217 * The pointer to Ethernet device. 9218 * @param[in] file 9219 * A pointer to a file for output. 9220 * @param[out] error 9221 * Perform verbose error reporting if not NULL. PMDs initialize this 9222 * structure in case of error only. 9223 * @return 9224 * 0 on success, a negative value otherwise. 9225 */ 9226 static int 9227 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9228 FILE *file, struct rte_flow_error *error __rte_unused) 9229 { 9230 struct mlx5_priv *priv = dev->data->dev_private; 9231 struct mlx5_dev_ctx_shared *sh = priv->sh; 9232 struct mlx5_hlist *h; 9233 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9234 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9235 struct rte_flow_query_count count; 9236 uint32_t actions_num; 9237 const uint8_t *data; 9238 size_t size; 9239 uint64_t id; 9240 uint32_t type; 9241 uint32_t i; 9242 uint32_t j; 9243 struct mlx5_list_inconst *l_inconst; 9244 struct mlx5_list_entry *e; 9245 int lcore_index; 9246 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 9247 uint32_t max; 9248 void *action; 9249 9250 /* encap_decap hlist is lcore_share, get global core cache. */ 9251 i = MLX5_LIST_GLOBAL; 9252 h = sh->encaps_decaps; 9253 if (h) { 9254 for (j = 0; j <= h->mask; j++) { 9255 l_inconst = &h->buckets[j].l; 9256 if (!l_inconst || !l_inconst->cache[i]) 9257 continue; 9258 9259 e = LIST_FIRST(&l_inconst->cache[i]->h); 9260 while (e) { 9261 encap_decap = 9262 (struct mlx5_flow_dv_encap_decap_resource *)e; 9263 data = encap_decap->buf; 9264 size = encap_decap->size; 9265 id = (uint64_t)(uintptr_t)encap_decap->action; 9266 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9267 save_dump_file(data, size, type, 9268 id, NULL, file); 9269 e = LIST_NEXT(e, next); 9270 } 9271 } 9272 } 9273 9274 /* get modify_hdr */ 9275 h = sh->modify_cmds; 9276 if (h) { 9277 lcore_index = rte_lcore_index(rte_lcore_id()); 9278 if (unlikely(lcore_index == -1)) { 9279 lcore_index = MLX5_LIST_NLCORE; 9280 rte_spinlock_lock(&h->l_const.lcore_lock); 9281 } 9282 i = lcore_index; 9283 9284 for (j = 0; j <= h->mask; j++) { 9285 l_inconst = &h->buckets[j].l; 9286 if (!l_inconst || !l_inconst->cache[i]) 9287 continue; 9288 9289 e = LIST_FIRST(&l_inconst->cache[i]->h); 9290 while (e) { 9291 modify_hdr = 9292 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9293 data = (const uint8_t *)modify_hdr->actions; 9294 size = (size_t)(modify_hdr->actions_num) * 8; 9295 actions_num = modify_hdr->actions_num; 9296 id = (uint64_t)(uintptr_t)modify_hdr->action; 9297 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9298 save_dump_file(data, size, type, id, 9299 (void *)(&actions_num), file); 9300 e = LIST_NEXT(e, next); 9301 } 9302 } 9303 9304 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9305 rte_spinlock_unlock(&h->l_const.lcore_lock); 9306 } 9307 9308 /* get counter */ 9309 MLX5_ASSERT(cmng->n_valid <= cmng->n); 9310 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9311 for (j = 1; j <= max; j++) { 9312 action = NULL; 9313 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9314 &count.bytes, &action)) && action) { 9315 id = (uint64_t)(uintptr_t)action; 9316 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9317 save_dump_file(NULL, 0, type, 9318 id, (void *)&count, file); 9319 } 9320 } 9321 return 0; 9322 } 9323 #endif 9324 9325 /** 9326 * Dump flow raw hw data to file 9327 * 9328 * @param[in] dev 9329 * The pointer to Ethernet device. 9330 * @param[in] file 9331 * A pointer to a file for output. 9332 * @param[out] error 9333 * Perform verbose error reporting if not NULL. PMDs initialize this 9334 * structure in case of error only. 9335 * @return 9336 * 0 on success, a negative value otherwise. 9337 */ 9338 int 9339 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9340 FILE *file, 9341 struct rte_flow_error *error __rte_unused) 9342 { 9343 struct mlx5_priv *priv = dev->data->dev_private; 9344 struct mlx5_dev_ctx_shared *sh = priv->sh; 9345 uint32_t handle_idx; 9346 int ret; 9347 struct mlx5_flow_handle *dh; 9348 struct rte_flow *flow; 9349 9350 if (!sh->config.dv_flow_en) { 9351 if (fputs("device dv flow disabled\n", file) <= 0) 9352 return -errno; 9353 return -ENOTSUP; 9354 } 9355 9356 /* dump all */ 9357 if (!flow_idx) { 9358 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9359 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9360 return -EINVAL; 9361 #endif 9362 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9363 sh->rx_domain, 9364 sh->tx_domain, file); 9365 } 9366 /* dump one */ 9367 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9368 (uintptr_t)(void *)flow_idx); 9369 if (!flow) 9370 return -EINVAL; 9371 9372 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9373 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9374 #endif 9375 handle_idx = flow->dev_handles; 9376 while (handle_idx) { 9377 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9378 handle_idx); 9379 if (!dh) 9380 return -ENOENT; 9381 if (dh->drv_flow) { 9382 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9383 file); 9384 if (ret) 9385 return -ENOENT; 9386 } 9387 handle_idx = dh->next.next; 9388 } 9389 return 0; 9390 } 9391 9392 /** 9393 * Get aged-out flows. 9394 * 9395 * @param[in] dev 9396 * Pointer to the Ethernet device structure. 9397 * @param[in] context 9398 * The address of an array of pointers to the aged-out flows contexts. 9399 * @param[in] nb_countexts 9400 * The length of context array pointers. 9401 * @param[out] error 9402 * Perform verbose error reporting if not NULL. Initialized in case of 9403 * error only. 9404 * 9405 * @return 9406 * how many contexts get in success, otherwise negative errno value. 9407 * if nb_contexts is 0, return the amount of all aged contexts. 9408 * if nb_contexts is not 0 , return the amount of aged flows reported 9409 * in the context array. 9410 */ 9411 int 9412 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9413 uint32_t nb_contexts, struct rte_flow_error *error) 9414 { 9415 const struct mlx5_flow_driver_ops *fops; 9416 struct rte_flow_attr attr = { .transfer = 0 }; 9417 9418 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 9419 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 9420 return fops->get_aged_flows(dev, contexts, nb_contexts, 9421 error); 9422 } 9423 DRV_LOG(ERR, 9424 "port %u get aged flows is not supported.", 9425 dev->data->port_id); 9426 return -ENOTSUP; 9427 } 9428 9429 /* Wrapper for driver action_validate op callback */ 9430 static int 9431 flow_drv_action_validate(struct rte_eth_dev *dev, 9432 const struct rte_flow_indir_action_conf *conf, 9433 const struct rte_flow_action *action, 9434 const struct mlx5_flow_driver_ops *fops, 9435 struct rte_flow_error *error) 9436 { 9437 static const char err_msg[] = "indirect action validation unsupported"; 9438 9439 if (!fops->action_validate) { 9440 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9441 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9442 NULL, err_msg); 9443 return -rte_errno; 9444 } 9445 return fops->action_validate(dev, conf, action, error); 9446 } 9447 9448 /** 9449 * Destroys the shared action by handle. 9450 * 9451 * @param dev 9452 * Pointer to Ethernet device structure. 9453 * @param[in] handle 9454 * Handle for the indirect action object to be destroyed. 9455 * @param[out] error 9456 * Perform verbose error reporting if not NULL. PMDs initialize this 9457 * structure in case of error only. 9458 * 9459 * @return 9460 * 0 on success, a negative errno value otherwise and rte_errno is set. 9461 * 9462 * @note: wrapper for driver action_create op callback. 9463 */ 9464 static int 9465 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9466 struct rte_flow_action_handle *handle, 9467 struct rte_flow_error *error) 9468 { 9469 static const char err_msg[] = "indirect action destruction unsupported"; 9470 struct rte_flow_attr attr = { .transfer = 0 }; 9471 const struct mlx5_flow_driver_ops *fops = 9472 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9473 9474 if (!fops->action_destroy) { 9475 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9476 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9477 NULL, err_msg); 9478 return -rte_errno; 9479 } 9480 return fops->action_destroy(dev, handle, error); 9481 } 9482 9483 /* Wrapper for driver action_destroy op callback */ 9484 static int 9485 flow_drv_action_update(struct rte_eth_dev *dev, 9486 struct rte_flow_action_handle *handle, 9487 const void *update, 9488 const struct mlx5_flow_driver_ops *fops, 9489 struct rte_flow_error *error) 9490 { 9491 static const char err_msg[] = "indirect action update unsupported"; 9492 9493 if (!fops->action_update) { 9494 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9495 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9496 NULL, err_msg); 9497 return -rte_errno; 9498 } 9499 return fops->action_update(dev, handle, update, error); 9500 } 9501 9502 /* Wrapper for driver action_destroy op callback */ 9503 static int 9504 flow_drv_action_query(struct rte_eth_dev *dev, 9505 const struct rte_flow_action_handle *handle, 9506 void *data, 9507 const struct mlx5_flow_driver_ops *fops, 9508 struct rte_flow_error *error) 9509 { 9510 static const char err_msg[] = "indirect action query unsupported"; 9511 9512 if (!fops->action_query) { 9513 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9514 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9515 NULL, err_msg); 9516 return -rte_errno; 9517 } 9518 return fops->action_query(dev, handle, data, error); 9519 } 9520 9521 /** 9522 * Create indirect action for reuse in multiple flow rules. 9523 * 9524 * @param dev 9525 * Pointer to Ethernet device structure. 9526 * @param conf 9527 * Pointer to indirect action object configuration. 9528 * @param[in] action 9529 * Action configuration for indirect action object creation. 9530 * @param[out] error 9531 * Perform verbose error reporting if not NULL. PMDs initialize this 9532 * structure in case of error only. 9533 * @return 9534 * A valid handle in case of success, NULL otherwise and rte_errno is set. 9535 */ 9536 static struct rte_flow_action_handle * 9537 mlx5_action_handle_create(struct rte_eth_dev *dev, 9538 const struct rte_flow_indir_action_conf *conf, 9539 const struct rte_flow_action *action, 9540 struct rte_flow_error *error) 9541 { 9542 static const char err_msg[] = "indirect action creation unsupported"; 9543 struct rte_flow_attr attr = { .transfer = 0 }; 9544 const struct mlx5_flow_driver_ops *fops = 9545 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9546 9547 if (flow_drv_action_validate(dev, conf, action, fops, error)) 9548 return NULL; 9549 if (!fops->action_create) { 9550 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9551 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9552 NULL, err_msg); 9553 return NULL; 9554 } 9555 return fops->action_create(dev, conf, action, error); 9556 } 9557 9558 /** 9559 * Updates inplace the indirect action configuration pointed by *handle* 9560 * with the configuration provided as *update* argument. 9561 * The update of the indirect action configuration effects all flow rules 9562 * reusing the action via handle. 9563 * 9564 * @param dev 9565 * Pointer to Ethernet device structure. 9566 * @param[in] handle 9567 * Handle for the indirect action to be updated. 9568 * @param[in] update 9569 * Action specification used to modify the action pointed by handle. 9570 * *update* could be of same type with the action pointed by the *handle* 9571 * handle argument, or some other structures like a wrapper, depending on 9572 * the indirect action type. 9573 * @param[out] error 9574 * Perform verbose error reporting if not NULL. PMDs initialize this 9575 * structure in case of error only. 9576 * 9577 * @return 9578 * 0 on success, a negative errno value otherwise and rte_errno is set. 9579 */ 9580 static int 9581 mlx5_action_handle_update(struct rte_eth_dev *dev, 9582 struct rte_flow_action_handle *handle, 9583 const void *update, 9584 struct rte_flow_error *error) 9585 { 9586 struct rte_flow_attr attr = { .transfer = 0 }; 9587 const struct mlx5_flow_driver_ops *fops = 9588 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9589 int ret; 9590 9591 ret = flow_drv_action_validate(dev, NULL, 9592 (const struct rte_flow_action *)update, fops, error); 9593 if (ret) 9594 return ret; 9595 return flow_drv_action_update(dev, handle, update, fops, 9596 error); 9597 } 9598 9599 /** 9600 * Query the indirect action by handle. 9601 * 9602 * This function allows retrieving action-specific data such as counters. 9603 * Data is gathered by special action which may be present/referenced in 9604 * more than one flow rule definition. 9605 * 9606 * see @RTE_FLOW_ACTION_TYPE_COUNT 9607 * 9608 * @param dev 9609 * Pointer to Ethernet device structure. 9610 * @param[in] handle 9611 * Handle for the indirect action to query. 9612 * @param[in, out] data 9613 * Pointer to storage for the associated query data type. 9614 * @param[out] error 9615 * Perform verbose error reporting if not NULL. PMDs initialize this 9616 * structure in case of error only. 9617 * 9618 * @return 9619 * 0 on success, a negative errno value otherwise and rte_errno is set. 9620 */ 9621 static int 9622 mlx5_action_handle_query(struct rte_eth_dev *dev, 9623 const struct rte_flow_action_handle *handle, 9624 void *data, 9625 struct rte_flow_error *error) 9626 { 9627 struct rte_flow_attr attr = { .transfer = 0 }; 9628 const struct mlx5_flow_driver_ops *fops = 9629 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9630 9631 return flow_drv_action_query(dev, handle, data, fops, error); 9632 } 9633 9634 /** 9635 * Destroy all indirect actions (shared RSS). 9636 * 9637 * @param dev 9638 * Pointer to Ethernet device. 9639 * 9640 * @return 9641 * 0 on success, a negative errno value otherwise and rte_errno is set. 9642 */ 9643 int 9644 mlx5_action_handle_flush(struct rte_eth_dev *dev) 9645 { 9646 struct rte_flow_error error; 9647 struct mlx5_priv *priv = dev->data->dev_private; 9648 struct mlx5_shared_action_rss *shared_rss; 9649 int ret = 0; 9650 uint32_t idx; 9651 9652 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 9653 priv->rss_shared_actions, idx, shared_rss, next) { 9654 ret |= mlx5_action_handle_destroy(dev, 9655 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 9656 } 9657 return ret; 9658 } 9659 9660 /** 9661 * Validate existing indirect actions against current device configuration 9662 * and attach them to device resources. 9663 * 9664 * @param dev 9665 * Pointer to Ethernet device. 9666 * 9667 * @return 9668 * 0 on success, a negative errno value otherwise and rte_errno is set. 9669 */ 9670 int 9671 mlx5_action_handle_attach(struct rte_eth_dev *dev) 9672 { 9673 struct mlx5_priv *priv = dev->data->dev_private; 9674 int ret = 0; 9675 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9676 9677 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9678 const char *message; 9679 uint32_t queue_idx; 9680 9681 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 9682 ind_tbl->queues_n, 9683 &message, &queue_idx); 9684 if (ret != 0) { 9685 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 9686 dev->data->port_id, ind_tbl->queues[queue_idx], 9687 message); 9688 break; 9689 } 9690 } 9691 if (ret != 0) 9692 return ret; 9693 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9694 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 9695 if (ret != 0) { 9696 DRV_LOG(ERR, "Port %u could not attach " 9697 "indirection table obj %p", 9698 dev->data->port_id, (void *)ind_tbl); 9699 goto error; 9700 } 9701 } 9702 9703 return 0; 9704 error: 9705 ind_tbl_last = ind_tbl; 9706 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9707 if (ind_tbl == ind_tbl_last) 9708 break; 9709 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 9710 DRV_LOG(CRIT, "Port %u could not detach " 9711 "indirection table obj %p on rollback", 9712 dev->data->port_id, (void *)ind_tbl); 9713 } 9714 return ret; 9715 } 9716 9717 /** 9718 * Detach indirect actions of the device from its resources. 9719 * 9720 * @param dev 9721 * Pointer to Ethernet device. 9722 * 9723 * @return 9724 * 0 on success, a negative errno value otherwise and rte_errno is set. 9725 */ 9726 int 9727 mlx5_action_handle_detach(struct rte_eth_dev *dev) 9728 { 9729 struct mlx5_priv *priv = dev->data->dev_private; 9730 int ret = 0; 9731 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9732 9733 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9734 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 9735 if (ret != 0) { 9736 DRV_LOG(ERR, "Port %u could not detach " 9737 "indirection table obj %p", 9738 dev->data->port_id, (void *)ind_tbl); 9739 goto error; 9740 } 9741 } 9742 return 0; 9743 error: 9744 ind_tbl_last = ind_tbl; 9745 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9746 if (ind_tbl == ind_tbl_last) 9747 break; 9748 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 9749 DRV_LOG(CRIT, "Port %u could not attach " 9750 "indirection table obj %p on rollback", 9751 dev->data->port_id, (void *)ind_tbl); 9752 } 9753 return ret; 9754 } 9755 9756 #ifndef HAVE_MLX5DV_DR 9757 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 9758 #else 9759 #define MLX5_DOMAIN_SYNC_FLOW \ 9760 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 9761 #endif 9762 9763 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 9764 { 9765 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 9766 const struct mlx5_flow_driver_ops *fops; 9767 int ret; 9768 struct rte_flow_attr attr = { .transfer = 0 }; 9769 9770 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9771 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9772 if (ret > 0) 9773 ret = -ret; 9774 return ret; 9775 } 9776 9777 const struct mlx5_flow_tunnel * 9778 mlx5_get_tof(const struct rte_flow_item *item, 9779 const struct rte_flow_action *action, 9780 enum mlx5_tof_rule_type *rule_type) 9781 { 9782 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9783 if (item->type == (typeof(item->type)) 9784 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9785 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9786 return flow_items_to_tunnel(item); 9787 } 9788 } 9789 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9790 if (action->type == (typeof(action->type)) 9791 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9792 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9793 return flow_actions_to_tunnel(action); 9794 } 9795 } 9796 return NULL; 9797 } 9798 9799 /** 9800 * tunnel offload functionality is defined for DV environment only 9801 */ 9802 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9803 __extension__ 9804 union tunnel_offload_mark { 9805 uint32_t val; 9806 struct { 9807 uint32_t app_reserve:8; 9808 uint32_t table_id:15; 9809 uint32_t transfer:1; 9810 uint32_t _unused_:8; 9811 }; 9812 }; 9813 9814 static bool 9815 mlx5_access_tunnel_offload_db 9816 (struct rte_eth_dev *dev, 9817 bool (*match)(struct rte_eth_dev *, 9818 struct mlx5_flow_tunnel *, const void *), 9819 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9820 void (*miss)(struct rte_eth_dev *, void *), 9821 void *ctx, bool lock_op); 9822 9823 static int 9824 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9825 struct rte_flow *flow, 9826 const struct rte_flow_attr *attr, 9827 const struct rte_flow_action *app_actions, 9828 uint32_t flow_idx, 9829 const struct mlx5_flow_tunnel *tunnel, 9830 struct tunnel_default_miss_ctx *ctx, 9831 struct rte_flow_error *error) 9832 { 9833 struct mlx5_priv *priv = dev->data->dev_private; 9834 struct mlx5_flow *dev_flow; 9835 struct rte_flow_attr miss_attr = *attr; 9836 const struct rte_flow_item miss_items[2] = { 9837 { 9838 .type = RTE_FLOW_ITEM_TYPE_ETH, 9839 .spec = NULL, 9840 .last = NULL, 9841 .mask = NULL 9842 }, 9843 { 9844 .type = RTE_FLOW_ITEM_TYPE_END, 9845 .spec = NULL, 9846 .last = NULL, 9847 .mask = NULL 9848 } 9849 }; 9850 union tunnel_offload_mark mark_id; 9851 struct rte_flow_action_mark miss_mark; 9852 struct rte_flow_action miss_actions[3] = { 9853 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9854 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9855 }; 9856 const struct rte_flow_action_jump *jump_data; 9857 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9858 struct flow_grp_info grp_info = { 9859 .external = 1, 9860 .transfer = attr->transfer, 9861 .fdb_def_rule = !!priv->fdb_def_rule, 9862 .std_tbl_fix = 0, 9863 }; 9864 int ret; 9865 9866 if (!attr->transfer) { 9867 uint32_t q_size; 9868 9869 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9870 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9871 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9872 0, SOCKET_ID_ANY); 9873 if (!ctx->queue) 9874 return rte_flow_error_set 9875 (error, ENOMEM, 9876 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9877 NULL, "invalid default miss RSS"); 9878 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9879 ctx->action_rss.level = 0, 9880 ctx->action_rss.types = priv->rss_conf.rss_hf, 9881 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9882 ctx->action_rss.queue_num = priv->reta_idx_n, 9883 ctx->action_rss.key = priv->rss_conf.rss_key, 9884 ctx->action_rss.queue = ctx->queue; 9885 if (!priv->reta_idx_n || !priv->rxqs_n) 9886 return rte_flow_error_set 9887 (error, EINVAL, 9888 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9889 NULL, "invalid port configuration"); 9890 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9891 ctx->action_rss.types = 0; 9892 for (i = 0; i != priv->reta_idx_n; ++i) 9893 ctx->queue[i] = (*priv->reta_idx)[i]; 9894 } else { 9895 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9896 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9897 } 9898 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9899 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9900 jump_data = app_actions->conf; 9901 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9902 miss_attr.group = jump_data->group; 9903 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9904 &flow_table, &grp_info, error); 9905 if (ret) 9906 return rte_flow_error_set(error, EINVAL, 9907 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9908 NULL, "invalid tunnel id"); 9909 mark_id.app_reserve = 0; 9910 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9911 mark_id.transfer = !!attr->transfer; 9912 mark_id._unused_ = 0; 9913 miss_mark.id = mark_id.val; 9914 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9915 miss_items, miss_actions, flow_idx, error); 9916 if (!dev_flow) 9917 return -rte_errno; 9918 dev_flow->flow = flow; 9919 dev_flow->external = true; 9920 dev_flow->tunnel = tunnel; 9921 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9922 /* Subflow object was created, we must include one in the list. */ 9923 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9924 dev_flow->handle, next); 9925 DRV_LOG(DEBUG, 9926 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9927 dev->data->port_id, tunnel->app_tunnel.type, 9928 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9929 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9930 miss_actions, error); 9931 if (!ret) 9932 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9933 error); 9934 9935 return ret; 9936 } 9937 9938 static const struct mlx5_flow_tbl_data_entry * 9939 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9940 { 9941 struct mlx5_priv *priv = dev->data->dev_private; 9942 struct mlx5_dev_ctx_shared *sh = priv->sh; 9943 struct mlx5_list_entry *he; 9944 union tunnel_offload_mark mbits = { .val = mark }; 9945 union mlx5_flow_tbl_key table_key = { 9946 { 9947 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9948 .id = 0, 9949 .reserved = 0, 9950 .dummy = 0, 9951 .is_fdb = !!mbits.transfer, 9952 .is_egress = 0, 9953 } 9954 }; 9955 struct mlx5_flow_cb_ctx ctx = { 9956 .data = &table_key.v64, 9957 }; 9958 9959 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9960 return he ? 9961 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9962 } 9963 9964 static void 9965 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9966 struct mlx5_list_entry *entry) 9967 { 9968 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9969 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9970 9971 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9972 tunnel_flow_tbl_to_id(tte->flow_table)); 9973 mlx5_free(tte); 9974 } 9975 9976 static int 9977 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9978 struct mlx5_list_entry *entry, void *cb_ctx) 9979 { 9980 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9981 union tunnel_tbl_key tbl = { 9982 .val = *(uint64_t *)(ctx->data), 9983 }; 9984 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9985 9986 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9987 } 9988 9989 static struct mlx5_list_entry * 9990 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 9991 { 9992 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9993 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9994 struct tunnel_tbl_entry *tte; 9995 union tunnel_tbl_key tbl = { 9996 .val = *(uint64_t *)(ctx->data), 9997 }; 9998 9999 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10000 sizeof(*tte), 0, 10001 SOCKET_ID_ANY); 10002 if (!tte) 10003 goto err; 10004 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10005 &tte->flow_table); 10006 if (tte->flow_table >= MLX5_MAX_TABLES) { 10007 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10008 tte->flow_table); 10009 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10010 tte->flow_table); 10011 goto err; 10012 } else if (!tte->flow_table) { 10013 goto err; 10014 } 10015 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10016 tte->tunnel_id = tbl.tunnel_id; 10017 tte->group = tbl.group; 10018 return &tte->hash; 10019 err: 10020 if (tte) 10021 mlx5_free(tte); 10022 return NULL; 10023 } 10024 10025 static struct mlx5_list_entry * 10026 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10027 struct mlx5_list_entry *oentry, 10028 void *cb_ctx __rte_unused) 10029 { 10030 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10031 0, SOCKET_ID_ANY); 10032 10033 if (!tte) 10034 return NULL; 10035 memcpy(tte, oentry, sizeof(*tte)); 10036 return &tte->hash; 10037 } 10038 10039 static void 10040 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10041 struct mlx5_list_entry *entry) 10042 { 10043 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10044 10045 mlx5_free(tte); 10046 } 10047 10048 static uint32_t 10049 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10050 const struct mlx5_flow_tunnel *tunnel, 10051 uint32_t group, uint32_t *table, 10052 struct rte_flow_error *error) 10053 { 10054 struct mlx5_list_entry *he; 10055 struct tunnel_tbl_entry *tte; 10056 union tunnel_tbl_key key = { 10057 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10058 .group = group 10059 }; 10060 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10061 struct mlx5_hlist *group_hash; 10062 struct mlx5_flow_cb_ctx ctx = { 10063 .data = &key.val, 10064 }; 10065 10066 group_hash = tunnel ? tunnel->groups : thub->groups; 10067 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10068 if (!he) 10069 return rte_flow_error_set(error, EINVAL, 10070 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10071 NULL, 10072 "tunnel group index not supported"); 10073 tte = container_of(he, typeof(*tte), hash); 10074 *table = tte->flow_table; 10075 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10076 dev->data->port_id, key.tunnel_id, group, *table); 10077 return 0; 10078 } 10079 10080 static void 10081 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10082 struct mlx5_flow_tunnel *tunnel) 10083 { 10084 struct mlx5_priv *priv = dev->data->dev_private; 10085 struct mlx5_indexed_pool *ipool; 10086 10087 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10088 dev->data->port_id, tunnel->tunnel_id); 10089 LIST_REMOVE(tunnel, chain); 10090 mlx5_hlist_destroy(tunnel->groups); 10091 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10092 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10093 } 10094 10095 static bool 10096 mlx5_access_tunnel_offload_db 10097 (struct rte_eth_dev *dev, 10098 bool (*match)(struct rte_eth_dev *, 10099 struct mlx5_flow_tunnel *, const void *), 10100 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10101 void (*miss)(struct rte_eth_dev *, void *), 10102 void *ctx, bool lock_op) 10103 { 10104 bool verdict = false; 10105 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10106 struct mlx5_flow_tunnel *tunnel; 10107 10108 rte_spinlock_lock(&thub->sl); 10109 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10110 verdict = match(dev, tunnel, (const void *)ctx); 10111 if (verdict) 10112 break; 10113 } 10114 if (!lock_op) 10115 rte_spinlock_unlock(&thub->sl); 10116 if (verdict && hit) 10117 hit(dev, tunnel, ctx); 10118 if (!verdict && miss) 10119 miss(dev, ctx); 10120 if (lock_op) 10121 rte_spinlock_unlock(&thub->sl); 10122 10123 return verdict; 10124 } 10125 10126 struct tunnel_db_find_tunnel_id_ctx { 10127 uint32_t tunnel_id; 10128 struct mlx5_flow_tunnel *tunnel; 10129 }; 10130 10131 static bool 10132 find_tunnel_id_match(struct rte_eth_dev *dev, 10133 struct mlx5_flow_tunnel *tunnel, const void *x) 10134 { 10135 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10136 10137 RTE_SET_USED(dev); 10138 return tunnel->tunnel_id == ctx->tunnel_id; 10139 } 10140 10141 static void 10142 find_tunnel_id_hit(struct rte_eth_dev *dev, 10143 struct mlx5_flow_tunnel *tunnel, void *x) 10144 { 10145 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10146 RTE_SET_USED(dev); 10147 ctx->tunnel = tunnel; 10148 } 10149 10150 static struct mlx5_flow_tunnel * 10151 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10152 { 10153 struct tunnel_db_find_tunnel_id_ctx ctx = { 10154 .tunnel_id = id, 10155 }; 10156 10157 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10158 find_tunnel_id_hit, NULL, &ctx, true); 10159 10160 return ctx.tunnel; 10161 } 10162 10163 static struct mlx5_flow_tunnel * 10164 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10165 const struct rte_flow_tunnel *app_tunnel) 10166 { 10167 struct mlx5_priv *priv = dev->data->dev_private; 10168 struct mlx5_indexed_pool *ipool; 10169 struct mlx5_flow_tunnel *tunnel; 10170 uint32_t id; 10171 10172 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10173 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10174 if (!tunnel) 10175 return NULL; 10176 if (id >= MLX5_MAX_TUNNELS) { 10177 mlx5_ipool_free(ipool, id); 10178 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10179 return NULL; 10180 } 10181 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10182 priv->sh, 10183 mlx5_flow_tunnel_grp2tbl_create_cb, 10184 mlx5_flow_tunnel_grp2tbl_match_cb, 10185 mlx5_flow_tunnel_grp2tbl_remove_cb, 10186 mlx5_flow_tunnel_grp2tbl_clone_cb, 10187 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10188 if (!tunnel->groups) { 10189 mlx5_ipool_free(ipool, id); 10190 return NULL; 10191 } 10192 /* initiate new PMD tunnel */ 10193 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10194 tunnel->tunnel_id = id; 10195 tunnel->action.type = (typeof(tunnel->action.type)) 10196 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10197 tunnel->action.conf = tunnel; 10198 tunnel->item.type = (typeof(tunnel->item.type)) 10199 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10200 tunnel->item.spec = tunnel; 10201 tunnel->item.last = NULL; 10202 tunnel->item.mask = NULL; 10203 10204 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10205 dev->data->port_id, tunnel->tunnel_id); 10206 10207 return tunnel; 10208 } 10209 10210 struct tunnel_db_get_tunnel_ctx { 10211 const struct rte_flow_tunnel *app_tunnel; 10212 struct mlx5_flow_tunnel *tunnel; 10213 }; 10214 10215 static bool get_tunnel_match(struct rte_eth_dev *dev, 10216 struct mlx5_flow_tunnel *tunnel, const void *x) 10217 { 10218 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10219 10220 RTE_SET_USED(dev); 10221 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10222 sizeof(*ctx->app_tunnel)); 10223 } 10224 10225 static void get_tunnel_hit(struct rte_eth_dev *dev, 10226 struct mlx5_flow_tunnel *tunnel, void *x) 10227 { 10228 /* called under tunnel spinlock protection */ 10229 struct tunnel_db_get_tunnel_ctx *ctx = x; 10230 10231 RTE_SET_USED(dev); 10232 tunnel->refctn++; 10233 ctx->tunnel = tunnel; 10234 } 10235 10236 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10237 { 10238 /* called under tunnel spinlock protection */ 10239 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10240 struct tunnel_db_get_tunnel_ctx *ctx = x; 10241 10242 rte_spinlock_unlock(&thub->sl); 10243 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10244 rte_spinlock_lock(&thub->sl); 10245 if (ctx->tunnel) { 10246 ctx->tunnel->refctn = 1; 10247 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10248 } 10249 } 10250 10251 10252 static int 10253 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10254 const struct rte_flow_tunnel *app_tunnel, 10255 struct mlx5_flow_tunnel **tunnel) 10256 { 10257 struct tunnel_db_get_tunnel_ctx ctx = { 10258 .app_tunnel = app_tunnel, 10259 }; 10260 10261 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10262 get_tunnel_miss, &ctx, true); 10263 *tunnel = ctx.tunnel; 10264 return ctx.tunnel ? 0 : -ENOMEM; 10265 } 10266 10267 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10268 { 10269 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10270 10271 if (!thub) 10272 return; 10273 if (!LIST_EMPTY(&thub->tunnels)) 10274 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10275 mlx5_hlist_destroy(thub->groups); 10276 mlx5_free(thub); 10277 } 10278 10279 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10280 { 10281 int err; 10282 struct mlx5_flow_tunnel_hub *thub; 10283 10284 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10285 0, SOCKET_ID_ANY); 10286 if (!thub) 10287 return -ENOMEM; 10288 LIST_INIT(&thub->tunnels); 10289 rte_spinlock_init(&thub->sl); 10290 thub->groups = mlx5_hlist_create("flow groups", 64, 10291 false, true, sh, 10292 mlx5_flow_tunnel_grp2tbl_create_cb, 10293 mlx5_flow_tunnel_grp2tbl_match_cb, 10294 mlx5_flow_tunnel_grp2tbl_remove_cb, 10295 mlx5_flow_tunnel_grp2tbl_clone_cb, 10296 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10297 if (!thub->groups) { 10298 err = -rte_errno; 10299 goto err; 10300 } 10301 sh->tunnel_hub = thub; 10302 10303 return 0; 10304 10305 err: 10306 if (thub->groups) 10307 mlx5_hlist_destroy(thub->groups); 10308 if (thub) 10309 mlx5_free(thub); 10310 return err; 10311 } 10312 10313 static inline int 10314 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10315 struct rte_flow_tunnel *tunnel, 10316 struct rte_flow_error *error) 10317 { 10318 struct mlx5_priv *priv = dev->data->dev_private; 10319 10320 if (!priv->sh->config.dv_flow_en) 10321 return rte_flow_error_set(error, ENOTSUP, 10322 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10323 "flow DV interface is off"); 10324 if (!is_tunnel_offload_active(dev)) 10325 return rte_flow_error_set(error, ENOTSUP, 10326 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10327 "tunnel offload was not activated"); 10328 if (!tunnel) 10329 return rte_flow_error_set(error, EINVAL, 10330 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10331 "no application tunnel"); 10332 switch (tunnel->type) { 10333 default: 10334 return rte_flow_error_set(error, EINVAL, 10335 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10336 "unsupported tunnel type"); 10337 case RTE_FLOW_ITEM_TYPE_VXLAN: 10338 case RTE_FLOW_ITEM_TYPE_GRE: 10339 case RTE_FLOW_ITEM_TYPE_NVGRE: 10340 case RTE_FLOW_ITEM_TYPE_GENEVE: 10341 break; 10342 } 10343 return 0; 10344 } 10345 10346 static int 10347 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10348 struct rte_flow_tunnel *app_tunnel, 10349 struct rte_flow_action **actions, 10350 uint32_t *num_of_actions, 10351 struct rte_flow_error *error) 10352 { 10353 struct mlx5_flow_tunnel *tunnel; 10354 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10355 10356 if (ret) 10357 return ret; 10358 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10359 if (ret < 0) { 10360 return rte_flow_error_set(error, ret, 10361 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10362 "failed to initialize pmd tunnel"); 10363 } 10364 *actions = &tunnel->action; 10365 *num_of_actions = 1; 10366 return 0; 10367 } 10368 10369 static int 10370 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10371 struct rte_flow_tunnel *app_tunnel, 10372 struct rte_flow_item **items, 10373 uint32_t *num_of_items, 10374 struct rte_flow_error *error) 10375 { 10376 struct mlx5_flow_tunnel *tunnel; 10377 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10378 10379 if (ret) 10380 return ret; 10381 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10382 if (ret < 0) { 10383 return rte_flow_error_set(error, ret, 10384 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10385 "failed to initialize pmd tunnel"); 10386 } 10387 *items = &tunnel->item; 10388 *num_of_items = 1; 10389 return 0; 10390 } 10391 10392 struct tunnel_db_element_release_ctx { 10393 struct rte_flow_item *items; 10394 struct rte_flow_action *actions; 10395 uint32_t num_elements; 10396 struct rte_flow_error *error; 10397 int ret; 10398 }; 10399 10400 static bool 10401 tunnel_element_release_match(struct rte_eth_dev *dev, 10402 struct mlx5_flow_tunnel *tunnel, const void *x) 10403 { 10404 const struct tunnel_db_element_release_ctx *ctx = x; 10405 10406 RTE_SET_USED(dev); 10407 if (ctx->num_elements != 1) 10408 return false; 10409 else if (ctx->items) 10410 return ctx->items == &tunnel->item; 10411 else if (ctx->actions) 10412 return ctx->actions == &tunnel->action; 10413 10414 return false; 10415 } 10416 10417 static void 10418 tunnel_element_release_hit(struct rte_eth_dev *dev, 10419 struct mlx5_flow_tunnel *tunnel, void *x) 10420 { 10421 struct tunnel_db_element_release_ctx *ctx = x; 10422 ctx->ret = 0; 10423 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10424 mlx5_flow_tunnel_free(dev, tunnel); 10425 } 10426 10427 static void 10428 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10429 { 10430 struct tunnel_db_element_release_ctx *ctx = x; 10431 RTE_SET_USED(dev); 10432 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10433 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10434 "invalid argument"); 10435 } 10436 10437 static int 10438 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10439 struct rte_flow_item *pmd_items, 10440 uint32_t num_items, struct rte_flow_error *err) 10441 { 10442 struct tunnel_db_element_release_ctx ctx = { 10443 .items = pmd_items, 10444 .actions = NULL, 10445 .num_elements = num_items, 10446 .error = err, 10447 }; 10448 10449 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10450 tunnel_element_release_hit, 10451 tunnel_element_release_miss, &ctx, false); 10452 10453 return ctx.ret; 10454 } 10455 10456 static int 10457 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10458 struct rte_flow_action *pmd_actions, 10459 uint32_t num_actions, struct rte_flow_error *err) 10460 { 10461 struct tunnel_db_element_release_ctx ctx = { 10462 .items = NULL, 10463 .actions = pmd_actions, 10464 .num_elements = num_actions, 10465 .error = err, 10466 }; 10467 10468 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10469 tunnel_element_release_hit, 10470 tunnel_element_release_miss, &ctx, false); 10471 10472 return ctx.ret; 10473 } 10474 10475 static int 10476 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10477 struct rte_mbuf *m, 10478 struct rte_flow_restore_info *info, 10479 struct rte_flow_error *err) 10480 { 10481 uint64_t ol_flags = m->ol_flags; 10482 const struct mlx5_flow_tbl_data_entry *tble; 10483 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10484 10485 if (!is_tunnel_offload_active(dev)) { 10486 info->flags = 0; 10487 return 0; 10488 } 10489 10490 if ((ol_flags & mask) != mask) 10491 goto err; 10492 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10493 if (!tble) { 10494 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10495 dev->data->port_id, m->hash.fdir.hi); 10496 goto err; 10497 } 10498 MLX5_ASSERT(tble->tunnel); 10499 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10500 info->group_id = tble->group_id; 10501 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10502 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10503 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10504 10505 return 0; 10506 10507 err: 10508 return rte_flow_error_set(err, EINVAL, 10509 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10510 "failed to get restore info"); 10511 } 10512 10513 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 10514 static int 10515 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 10516 __rte_unused struct rte_flow_tunnel *app_tunnel, 10517 __rte_unused struct rte_flow_action **actions, 10518 __rte_unused uint32_t *num_of_actions, 10519 __rte_unused struct rte_flow_error *error) 10520 { 10521 return -ENOTSUP; 10522 } 10523 10524 static int 10525 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 10526 __rte_unused struct rte_flow_tunnel *app_tunnel, 10527 __rte_unused struct rte_flow_item **items, 10528 __rte_unused uint32_t *num_of_items, 10529 __rte_unused struct rte_flow_error *error) 10530 { 10531 return -ENOTSUP; 10532 } 10533 10534 static int 10535 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 10536 __rte_unused struct rte_flow_item *pmd_items, 10537 __rte_unused uint32_t num_items, 10538 __rte_unused struct rte_flow_error *err) 10539 { 10540 return -ENOTSUP; 10541 } 10542 10543 static int 10544 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 10545 __rte_unused struct rte_flow_action *pmd_action, 10546 __rte_unused uint32_t num_actions, 10547 __rte_unused struct rte_flow_error *err) 10548 { 10549 return -ENOTSUP; 10550 } 10551 10552 static int 10553 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 10554 __rte_unused struct rte_mbuf *m, 10555 __rte_unused struct rte_flow_restore_info *i, 10556 __rte_unused struct rte_flow_error *err) 10557 { 10558 return -ENOTSUP; 10559 } 10560 10561 static int 10562 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 10563 __rte_unused struct rte_flow *flow, 10564 __rte_unused const struct rte_flow_attr *attr, 10565 __rte_unused const struct rte_flow_action *actions, 10566 __rte_unused uint32_t flow_idx, 10567 __rte_unused const struct mlx5_flow_tunnel *tunnel, 10568 __rte_unused struct tunnel_default_miss_ctx *ctx, 10569 __rte_unused struct rte_flow_error *error) 10570 { 10571 return -ENOTSUP; 10572 } 10573 10574 static struct mlx5_flow_tunnel * 10575 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 10576 __rte_unused uint32_t id) 10577 { 10578 return NULL; 10579 } 10580 10581 static void 10582 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 10583 __rte_unused struct mlx5_flow_tunnel *tunnel) 10584 { 10585 } 10586 10587 static uint32_t 10588 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 10589 __rte_unused const struct mlx5_flow_tunnel *t, 10590 __rte_unused uint32_t group, 10591 __rte_unused uint32_t *table, 10592 struct rte_flow_error *error) 10593 { 10594 return rte_flow_error_set(error, ENOTSUP, 10595 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10596 "tunnel offload requires DV support"); 10597 } 10598 10599 void 10600 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 10601 __rte_unused uint16_t port_id) 10602 { 10603 } 10604 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10605 10606 /* Flex flow item API */ 10607 static struct rte_flow_item_flex_handle * 10608 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 10609 const struct rte_flow_item_flex_conf *conf, 10610 struct rte_flow_error *error) 10611 { 10612 static const char err_msg[] = "flex item creation unsupported"; 10613 struct rte_flow_attr attr = { .transfer = 0 }; 10614 const struct mlx5_flow_driver_ops *fops = 10615 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10616 10617 if (!fops->item_create) { 10618 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10619 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10620 NULL, err_msg); 10621 return NULL; 10622 } 10623 return fops->item_create(dev, conf, error); 10624 } 10625 10626 static int 10627 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 10628 const struct rte_flow_item_flex_handle *handle, 10629 struct rte_flow_error *error) 10630 { 10631 static const char err_msg[] = "flex item release unsupported"; 10632 struct rte_flow_attr attr = { .transfer = 0 }; 10633 const struct mlx5_flow_driver_ops *fops = 10634 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10635 10636 if (!fops->item_release) { 10637 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10638 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10639 NULL, err_msg); 10640 return -rte_errno; 10641 } 10642 return fops->item_release(dev, handle, error); 10643 } 10644 10645 static void 10646 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 10647 { 10648 int ret; 10649 struct rte_flow_error error; 10650 10651 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10652 char *item_name; 10653 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 10654 sizeof(item_name), 10655 (void *)(uintptr_t)item->type, &error); 10656 if (ret > 0) 10657 printf("%s ", item_name); 10658 else 10659 printf("%d\n", (int)item->type); 10660 } 10661 printf("END\n"); 10662 } 10663 10664 static int 10665 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 10666 { 10667 const struct rte_flow_item_udp *spec = udp_item->spec; 10668 const struct rte_flow_item_udp *mask = udp_item->mask; 10669 uint16_t udp_dport = 0; 10670 10671 if (spec != NULL) { 10672 if (!mask) 10673 mask = &rte_flow_item_udp_mask; 10674 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 10675 mask->hdr.dst_port); 10676 } 10677 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 10678 } 10679 10680 static const struct mlx5_flow_expand_node * 10681 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 10682 unsigned int item_idx, 10683 const struct mlx5_flow_expand_node graph[], 10684 const struct mlx5_flow_expand_node *node) 10685 { 10686 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 10687 10688 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 10689 node != NULL && 10690 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 10691 /* 10692 * The expansion node is VXLAN and it is also the last 10693 * expandable item in the pattern, so need to continue 10694 * expansion of the inner tunnel. 10695 */ 10696 MLX5_ASSERT(item_idx > 0); 10697 prev_item = pattern + item_idx - 1; 10698 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 10699 if (mlx5_flow_is_std_vxlan_port(prev_item)) 10700 return &graph[MLX5_EXPANSION_STD_VXLAN]; 10701 return &graph[MLX5_EXPANSION_L3_VXLAN]; 10702 } 10703 return node; 10704 } 10705 10706 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 10707 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 10708 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 10709 }; 10710 10711 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 10712 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 10713 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 10714 { 9, 10, 11 }, { 12, 13, 14 }, 10715 }; 10716 10717 /** 10718 * Discover the number of available flow priorities. 10719 * 10720 * @param dev 10721 * Ethernet device. 10722 * 10723 * @return 10724 * On success, number of available flow priorities. 10725 * On failure, a negative errno-style code and rte_errno is set. 10726 */ 10727 int 10728 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 10729 { 10730 static const uint16_t vprio[] = {8, 16}; 10731 const struct mlx5_priv *priv = dev->data->dev_private; 10732 const struct mlx5_flow_driver_ops *fops; 10733 enum mlx5_flow_drv_type type; 10734 int ret; 10735 10736 type = mlx5_flow_os_get_type(); 10737 if (type == MLX5_FLOW_TYPE_MAX) { 10738 type = MLX5_FLOW_TYPE_VERBS; 10739 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 10740 type = MLX5_FLOW_TYPE_DV; 10741 } 10742 fops = flow_get_drv_ops(type); 10743 if (fops->discover_priorities == NULL) { 10744 DRV_LOG(ERR, "Priority discovery not supported"); 10745 rte_errno = ENOTSUP; 10746 return -rte_errno; 10747 } 10748 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 10749 if (ret < 0) 10750 return ret; 10751 switch (ret) { 10752 case 8: 10753 ret = RTE_DIM(priority_map_3); 10754 break; 10755 case 16: 10756 ret = RTE_DIM(priority_map_5); 10757 break; 10758 default: 10759 rte_errno = ENOTSUP; 10760 DRV_LOG(ERR, 10761 "port %u maximum priority: %d expected 8/16", 10762 dev->data->port_id, ret); 10763 return -rte_errno; 10764 } 10765 DRV_LOG(INFO, "port %u supported flow priorities:" 10766 " 0-%d for ingress or egress root table," 10767 " 0-%d for non-root table or transfer root table.", 10768 dev->data->port_id, ret - 2, 10769 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 10770 return ret; 10771 } 10772 10773 /** 10774 * Adjust flow priority based on the highest layer and the request priority. 10775 * 10776 * @param[in] dev 10777 * Pointer to the Ethernet device structure. 10778 * @param[in] priority 10779 * The rule base priority. 10780 * @param[in] subpriority 10781 * The priority based on the items. 10782 * 10783 * @return 10784 * The new priority. 10785 */ 10786 uint32_t 10787 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10788 uint32_t subpriority) 10789 { 10790 uint32_t res = 0; 10791 struct mlx5_priv *priv = dev->data->dev_private; 10792 10793 switch (priv->sh->flow_max_priority) { 10794 case RTE_DIM(priority_map_3): 10795 res = priority_map_3[priority][subpriority]; 10796 break; 10797 case RTE_DIM(priority_map_5): 10798 res = priority_map_5[priority][subpriority]; 10799 break; 10800 } 10801 return res; 10802 } 10803